Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambda): retarget provisioned alias allocations (#6335)

Andre Kurait7c8daff08f09
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index ace1d2726f..52a1826686 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -483,14 +483,14 @@ for Get does not authorize Put, Delete, or another qualifier.
Provisioned environments are separate from ordinary idle eviction. Desired configurations
are persisted; restarting Floci initializes new processes before reporting `READY`.
-Deleting the configuration, alias or function removes the owned environments. If an
-alias changes its version, this stage marks its old allocation `FAILED` and retires its
-environments. Invocations use on-demand capacity until a new Put initializes the new
-version. Automatic alias retargeting is a separate follow-up. The requested quota is
-retained until configuration replacement or deletion. An invalid replacement Put preserves
-the existing healthy allocation. An alias with a configuration cannot move to `$LATEST`
-or to a version targeted by another configuration; remove the conflicting configuration
-first. Get and Delete remain available if the alias gains
+Deleting the configuration, alias or function removes the owned environments. Changing an
+alias's version reinitializes its existing allocation for the new published version and
+releases the old version's configuration conflict. The alias retains its requested quota;
+retired environments retain their local container capacity until cleanup succeeds. An identical
+Put preserves the existing healthy allocation. Failed background alias refreshes retain their
+previous allocation and retry on later reconciliation without blocking other configurations.
+An alias with a configuration cannot move to
+`$LATEST`; remove the configuration first. Get and Delete remain available if the alias gains
unsupported weighted routing.
This implementation supports the non-ephemeral Docker executor with immutable published
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.java
index 3ce750e6b4..556bbcb6d4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPool.java
@@ -163,9 +163,7 @@ public class ProvisionedConcurrencyPool implements Resettable {
throw invalid("Provisioned concurrency requires the non-ephemeral Docker Lambda executor");
}
LambdaFunction fn = currentFunction(target);
- if (fn.isHotReload() || fn.isDurable()) {
- throw invalid("Provisioned concurrency is not supported for hot-reload or durable functions");
- }
+ validateSupportedTarget(fn);
List<ContainerHandle> retired = List.of();
Status result;
synchronized (lock) {
@@ -378,7 +376,15 @@ public class ProvisionedConcurrencyPool implements Resettable {
if (stopped) {
return;
}
- for (Allocation allocation : new ArrayList<>(allocations.values())) {
+ for (Allocation previous : new ArrayList<>(allocations.values())) {
+ Allocation allocation;
+ try {
+ allocation = refreshAlias(previous, retired);
+ } catch (RuntimeException failure) {
+ LOG.warnv(failure, "Could not refresh provisioned Lambda alias allocation {0}",
+ previous.configuration.key());
+ continue;
+ }
if ("FAILED".equals(allocation.status)) {
continue;
}
@@ -434,6 +440,37 @@ public class ProvisionedConcurrencyPool implements Resettable {
.map(LambdaAlias::getFunctionVersion).orElse(null);
}
+ private Allocation refreshAlias(Allocation previous, List<ContainerHandle> retired) {
+ Configuration value = previous.configuration;
+ if (value.qualifier().matches("[0-9]+")) {
+ return previous;
+ }
+ LambdaAlias alias = aliases.getForAccount(value.account(), value.region(), value.name(), value.qualifier())
+ .orElse(null);
+ if (alias == null || Objects.equals(value.version(), alias.getFunctionVersion())
+ || (alias.getRoutingConfig() != null && !alias.getRoutingConfig().isEmpty())
+ || alias.getFunctionVersion() == null || !alias.getFunctionVersion().matches("[0-9]+")) {
+ return previous;
+ }
+ if (functions.getForAccount(value.account(), value.region(), value.name()).isEmpty()) {
+ return previous;
+ }
+ LambdaFunction fn = functions.getForAccount(value.account(), value.region(), value.name(),
+ alias.getFunctionVersion()).orElse(null);
+ if (fn == null) {
+ return previous;
+ }
+ validateSupportedTarget(fn);
+ Configuration target = new Configuration(value.account(), value.region(), value.name(),
+ value.functionArn(), value.qualifier(), fn.getVersion(), fn.getRevisionId(), value.requested(),
+ LAST_MODIFIED.format(Instant.ofEpochMilli(clock.millis())));
+ write(target);
+ Allocation next = new Allocation(target);
+ allocations.put(target.key(), next);
+ retired.addAll(retire(previous));
+ return next;
+ }
+
void validateAliasUpdate(LambdaFunction fn, String qualifier, String version) {
synchronized (lock) {
String functionArn = baseArn(fn);
@@ -443,6 +480,11 @@ public class ProvisionedConcurrencyPool implements Resettable {
if ("$LATEST".equals(version)) {
throw invalid("Provisioned Concurrency Configs cannot be applied to unpublished function versions.");
}
+ if (version != null && version.matches("[0-9]+")) {
+ AwsArnUtils.Arn arn = AwsArnUtils.parse(functionArn);
+ functions.getForAccount(arn.accountId(), arn.region(), fn.getFunctionName(), version)
+ .ifPresent(ProvisionedConcurrencyPool::validateSupportedTarget);
+ }
if (version != null && version.matches("[0-9]+") && allocations.values().stream()
.anyMatch(entry -> functionArn.equals(entry.configuration.functionArn())
&& !qualifier.equals(entry.configuration.qualifier())
@@ -453,6 +495,12 @@ public class ProvisionedConcurrencyPool implements Resettable {
}
}
+ private static void validateSupportedTarget(LambdaFunction fn) {
+ if (fn.isHotReload() || fn.isDurable()) {
+ throw invalid("Provisioned concurrency is not supported for hot-reload or durable functions");
+ }
+ }
+
private int availableCapacity(Configuration configuration) {
long functionOwned = owned.values().stream()
.filter(value -> value.functionArn().equals(configuration.functionArn())).count();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.java
index f726bc7a50..97b2ac2e70 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/ProvisionedConcurrencyPoolTest.java
@@ -594,8 +594,13 @@ class ProvisionedConcurrencyPoolTest {
default -> throw new AssertionError(lookup);
}
assertSame(failure, assertThrows(IllegalStateException.class, () -> pool.acquire(version, "live")));
- assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcile));
- assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcileSafely));
+ if ("alias".equals(lookup)) {
+ assertDoesNotThrow(pool::reconcile);
+ assertDoesNotThrow(pool::reconcileSafely);
+ } else {
+ assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcile));
+ assertSame(failure, assertThrows(IllegalStateException.class, pool::reconcileSafely));
+ }
assertTrue(first.alive);
assertEquals(0, first.stops.get());
reset(functionLookup, aliasLookup);
@@ -688,6 +693,56 @@ class ProvisionedConcurrencyPoolTest {
assertEquals(2, launcher.starts.get());
}
+ @Test
+ void targetChangeRefusesOldInitializedEnvironment() throws Exception {
+ put(1);
+ Environment first = launched();
+ first.initialized.complete(null);
+ ready();
+ function(ACCOUNT, REGION, "2");
+ alias(ACCOUNT, REGION, "live", "2", Map.of());
+ assertNull(pool.acquire(version, "live"));
+ assertEquals("IN_PROGRESS", get().status());
+ assertEquals(1, first.stops.get());
+ pool.reconcile();
+ Environment second = launched();
+ second.initialized.complete(null);
+ ready();
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void movedAliasFreesItsOldVersionButRetainsItsReservedBudget(boolean previouslyFailed) throws Exception {
+ LambdaFunction latest = functions.getForAccount(ACCOUNT, REGION, "example").orElseThrow();
+ latest.setReservedConcurrentExecutions(2);
+ put(1);
+ Environment first = launched();
+ first.initialized.complete(null);
+ ready();
+ if (previouslyFailed) {
+ alias(ACCOUNT, REGION, "live", "1", Map.of("2", 0.1));
+ assertEquals("FAILED", get().status());
+ }
+ function(ACCOUNT, REGION, "2");
+ alias(ACCOUNT, REGION, "live", "2", Map.of());
+
+ pool.put(REGION, "example", "1", 1, ACCOUNT);
+ assertEquals("2", storage.scanAllAccounts().stream()
+ .filter(value -> "live".equals(value.qualifier())).findFirst().orElseThrow().version());
+ assertEquals(1, first.stops.get());
+ assertEquals(2, storage.scanAllAccounts().size());
+ AwsException budget = assertThrows(AwsException.class,
+ () -> pool.put(REGION, "example", "1", 2, ACCOUNT));
+ assertEquals("InvalidParameterValueException", budget.getErrorCode());
+ AwsException bothInvalid = assertThrows(AwsException.class,
+ () -> pool.put(REGION, "example", "2", 1, ACCOUNT));
+ assertEquals("InvalidParameterValueException", bothInvalid.getErrorCode());
+ latest.setReservedConcurrentExecutions(3);
+ AwsException conflict = assertThrows(AwsException.class,
+ () -> pool.put(REGION, "example", "2", 1, ACCOUNT));
+ assertEquals("Version is pointed by a Provisioned Concurrency alias", conflict.getMessage());
+ }
+
@ParameterizedTest
@ValueSource(strings = {"weighted", "latest"})
void changedAliasRemainsReadableAndDeletable(String change) throws Exception {
@@ -716,6 +771,148 @@ class ProvisionedConcurrencyPoolTest {
assertDoesNotThrow(() -> pool.validateAliasUpdate(version, "live", "$LATEST"));
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void configuredAliasRejectsUnsupportedTargetBeforeMutation(boolean durable) throws Exception {
+ put(1);
+ Environment first = launched();
+ first.initialized.complete(null);
+ ready();
+ LambdaFunction target = function(ACCOUNT, REGION, "2");
+ if (durable) {
+ target.setDurableExecutionTimeout(60);
+ } else {
+ target.setHotReloadHostPath("/tmp/unsupported-target");
+ }
+ functions.saveForAccount(ACCOUNT, REGION, target);
+
+ AwsException error = assertThrows(AwsException.class,
+ () -> pool.validateAliasUpdate(version, "live", "2"));
+
+ assertEquals("InvalidParameterValueException", error.getErrorCode());
+ assertEquals("1", aliases.getForAccount(ACCOUNT, REGION, "example", "live")
+ .orElseThrow().getFunctionVersion());
+ assertEquals("1", storage.scanAllAccounts().getFirst().version());
+ assertEquals(0, first.stops.get());
+ assertEquals(1, launcher.starts.get());
+ pool.delete(REGION, "example", "live", ACCOUNT);
+ assertDoesNotThrow(() -> pool.validateAliasUpdate(version, "live", "2"));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void unsupportedRetargetPreservesPriorAllocationUntilTargetIsEligible(boolean durable) throws Exception {
+ put(1);
+ Environment first = launched();
+ first.initialized.complete(null);
+ ready();
+ LambdaFunction target = function(ACCOUNT, REGION, "2");
+ if (durable) {
+ target.setDurableExecutionTimeout(60);
+ } else {
+ target.setHotReloadHostPath("/tmp/unsupported-target");
+ }
+ functions.saveForAccount(ACCOUNT, REGION, target);
+ alias(ACCOUNT, REGION, "live", "2", Map.of());
+
+ assertDoesNotThrow(pool::reconcile);
+ assertEquals("1", storage.scanAllAccounts().getFirst().version());
+ assertEquals(0, first.stops.get());
+ assertEquals(1, launcher.starts.get());
+ target.setDurableExecutionTimeout(null);
+ target.setHotReloadHostPath(null);
+ functions.saveForAccount(ACCOUNT, REGION, target);
+ pool.reconcile();
+ pool.reconcile();
+ Environment replacement = launched();
+ replacement.initialized.complete(null);
+ ready();
+ assertEquals("2", storage.scanAllAccounts().getFirst().version());
+ assertEquals(1, first.stops.get());
+ assertEquals(2, launcher.starts.get());
+ }
+
+ @Test
+ void aliasRetargetStorageFailurePreservesThePriorAllocation() throws Exception {
+ AccountAwareStorageBackend<ProvisionedConcurrencyPool.Configuration> backend = spy(storage);
+ pool.closeExecutors();
+ pool = new ProvisionedConcurrencyPool(launcher, functions, aliases, backend,
+ Clock.systemUTC(), 4, 8, false, 15);
+ put(1);
+ Environment first = launched();
+ first.initialized.complete(null);
+ ready();
+ function(ACCOUNT, REGION, "2");
+ alias(ACCOUNT, REGION, "live", "2", Map.of());
+ IllegalStateException failure = new IllegalStateException("controlled alias write failure");
+ doThrow(failure).when(backend).putForAccount(eq(ACCOUNT), any(), any());
+ assertEquals("READY", get().status());
+ assertEquals("1", storage.scanAllAccounts().getFirst().version());
+ assertEquals(0, first.stops.get());
+ doCallRealMethod().when(backend).putForAccount(eq(ACCOUNT), any(), any());
+ assertEquals("IN_PROGRESS", get().status());
+ assertEquals("2", storage.scanAllAccounts().getFirst().version());
+ assertEquals(1, first.stops.get());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"storage", "hotReload", "durable"})
+ void failedAliasRefreshDoesNotBlockOtherAccountsAndRecovers(String failureKind) throws Exception {
+ AccountAwareStorageBackend<ProvisionedConcurrencyPool.Configuration> backend = spy(storage);
+ pool.closeExecutors();
+ pool = new ProvisionedConcurrencyPool(launcher, functions, aliases, backend,
+ Clock.systemUTC(), 4, 8, false, 15);
+ put(1);
+ Environment original = launched();
+ original.initialized.complete(null);
+ ready();
+ LambdaFunction target = function(ACCOUNT, REGION, "2");
+ if ("hotReload".equals(failureKind)) {
+ target.setHotReloadHostPath("/tmp/unsupported-target");
+ } else if ("durable".equals(failureKind)) {
+ target.setDurableExecutionTimeout(60);
+ } else {
+ doThrow(new IllegalStateException("controlled alias write failure"))
+ .when(backend).putForAccount(eq(ACCOUNT), any(), any());
+ }
+ functions.saveForAccount(ACCOUNT, REGION, target);
+ alias(ACCOUNT, REGION, "live", "2", Map.of());
+ String other = "222222222222";
+ function(other, REGION, "$LATEST");
+ function(other, REGION, "1");
+ assertDoesNotThrow(() -> pool.put(REGION, "example", "1", 1, other));
+ Environment peer = launched();
+ peer.initialized.complete(null);
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ assertEquals("READY", pool.get(REGION, "example", "1", other).status()));
+ peer.faulted.set(true);
+ assertDoesNotThrow(pool::reconcile);
+ assertEquals("IN_PROGRESS", pool.get(REGION, "example", "1", other).status());
+ Environment peerReplacement = launched();
+ peerReplacement.initialized.complete(null);
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ assertEquals("READY", pool.get(REGION, "example", "1", other).status()));
+ assertEquals(1, peer.stops.get());
+ assertEquals(0, original.stops.get());
+ assertEquals("1", storage.scanAllAccounts().stream()
+ .filter(value -> ACCOUNT.equals(value.account())).findFirst().orElseThrow().version());
+ if ("storage".equals(failureKind)) {
+ doCallRealMethod().when(backend).putForAccount(eq(ACCOUNT), any(), any());
+ }
+ target.setDurableExecutionTimeout(null);
+ target.setHotReloadHostPath(null);
+ functions.saveForAccount(ACCOUNT, REGION, target);
+ pool.reconcile();
+ pool.reconcile();
+ Environment replacement = launched();
+ replacement.initialized.complete(null);
+ ready();
+ assertEquals(1, original.stops.get());
+ assertEquals(0, peerReplacement.stops.get());
+ assertEquals("2", storage.scanAllAccounts().stream()
+ .filter(value -> ACCOUNT.equals(value.account())).findFirst().orElseThrow().version());
+ }
+
@Test
void restoredDesiredConfigurationMustInitializeNewProcesses() throws Exception {
put(1);
@@ -1119,26 +1316,6 @@ class ProvisionedConcurrencyPoolTest {
}
}
- @Test
- void changedAliasRequiresExplicitPutBeforeUsingNewVersion() throws Exception {
- put(1);
- Environment first = launched();
- first.initialized.complete(null);
- ready();
- function(ACCOUNT, REGION, "2");
- alias(ACCOUNT, REGION, "live", "2", Map.of());
- assertNull(pool.acquire(version, "live"));
- assertEquals("FAILED", get().status());
- assertEquals(1, get().requested());
- assertEquals(1, first.stops.get());
- assertEquals("1", storage.scanAllAccounts().getFirst().version());
- put(1);
- Environment second = launched();
- second.initialized.complete(null);
- ready();
- assertEquals("2", storage.scanAllAccounts().getFirst().version());
- }
-
@ParameterizedTest
@ValueSource(booleans = {false, true})
void configuredAliasCannotRetargetToCapacityOwnedByAnotherQualifier(boolean otherAlias) throws Exception {