Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambda): enforce reserved and per-region concurrency (#424)

Kenshin Okinakafb2673113149
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java
index 32a4831ce3..d8ea6abf1a 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java
@@ -135,4 +135,98 @@ class LambdaConcurrencyTest {
.build()))
.isInstanceOf(ResourceNotFoundException.class);
}
+
+ @Test
+ @Order(9)
+ void putFunctionConcurrency_exceedsAccountUnreservedMin_throwsLimitExceeded() {
+ // Floci default: regionLimit=1000, unreservedMin=100 → max single Put = 900
+ // The Lambda SDK v2 model does not declare LimitExceededException as a
+ // dedicated subclass, so the SDK surfaces it as the generic
+ // LambdaException. We therefore assert the wire-level identity
+ // (status code + __type error code) rather than a Java type, which is
+ // what AWS clients actually discriminate on.
+ assertThatThrownBy(() -> lambda.putFunctionConcurrency(
+ PutFunctionConcurrencyRequest.builder()
+ .functionName(FUNCTION_NAME)
+ .reservedConcurrentExecutions(901)
+ .build()))
+ .isInstanceOfSatisfying(LambdaException.class, ex -> {
+ assertThat(ex.statusCode()).isEqualTo(400);
+ assertThat(ex.awsErrorDetails().errorCode()).isEqualTo("LimitExceededException");
+ assertThat(ex.getMessage()).contains("UnreservedConcurrentExecution");
+ });
+ }
+
+ @Test
+ @Order(10)
+ void invoke_whenReservedZero_throwsTooManyRequests() {
+ lambda.putFunctionConcurrency(PutFunctionConcurrencyRequest.builder()
+ .functionName(FUNCTION_NAME)
+ .reservedConcurrentExecutions(0)
+ .build());
+
+ // Event-type invoke still goes through the concurrency gate; reserved=0
+ // should throttle every request regardless of invocation type.
+ assertThatThrownBy(() -> lambda.invoke(InvokeRequest.builder()
+ .functionName(FUNCTION_NAME)
+ .invocationType(InvocationType.EVENT)
+ .payload(SdkBytes.fromUtf8String("{}"))
+ .build()))
+ .isInstanceOf(TooManyRequestsException.class);
+
+ // Clear so teardown and other tests are not affected
+ lambda.deleteFunctionConcurrency(DeleteFunctionConcurrencyRequest.builder()
+ .functionName(FUNCTION_NAME).build());
+ }
+
+ @Test
+ @Order(11)
+ void invoke_dryRunBypassesConcurrencyGate() {
+ lambda.putFunctionConcurrency(PutFunctionConcurrencyRequest.builder()
+ .functionName(FUNCTION_NAME)
+ .reservedConcurrentExecutions(0)
+ .build());
+
+ // DryRun validates inputs without dispatching; it must not be throttled.
+ InvokeResponse response = lambda.invoke(InvokeRequest.builder()
+ .functionName(FUNCTION_NAME)
+ .invocationType(InvocationType.DRY_RUN)
+ .payload(SdkBytes.fromUtf8String("{}"))
+ .build());
+
+ assertThat(response.statusCode()).isEqualTo(204);
+
+ lambda.deleteFunctionConcurrency(DeleteFunctionConcurrencyRequest.builder()
+ .functionName(FUNCTION_NAME).build());
+ }
+
+ @Test
+ @Order(12)
+ void invoke_withVersionQualifier_stillHonorsReservedOnLatest() {
+ // Regression guard: if a future change adds Qualifier routing that
+ // resolves to a published version snapshot, the snapshot currently
+ // has reservedConcurrentExecutions=null and would silently bypass a
+ // reserved=0 on $LATEST. Today Floci ignores the qualifier and
+ // routes the invoke to $LATEST, so reserved=0 must still throttle.
+ // Keeping this test green after a qualifier-routing change will
+ // require copying the reservation onto the snapshot (or keying the
+ // limiter off the base ARN).
+ lambda.publishVersion(PublishVersionRequest.builder()
+ .functionName(FUNCTION_NAME).build());
+ lambda.putFunctionConcurrency(PutFunctionConcurrencyRequest.builder()
+ .functionName(FUNCTION_NAME)
+ .reservedConcurrentExecutions(0)
+ .build());
+
+ assertThatThrownBy(() -> lambda.invoke(InvokeRequest.builder()
+ .functionName(FUNCTION_NAME)
+ .qualifier("1")
+ .invocationType(InvocationType.EVENT)
+ .payload(SdkBytes.fromUtf8String("{}"))
+ .build()))
+ .isInstanceOf(TooManyRequestsException.class);
+
+ lambda.deleteFunctionConcurrency(DeleteFunctionConcurrencyRequest.builder()
+ .functionName(FUNCTION_NAME).build());
+ }
}
diff --git a/docs/configuration/application-yml.md b/docs/configuration/application-yml.md
index a67087c63c..13167b7496 100644
--- a/docs/configuration/application-yml.md
+++ b/docs/configuration/application-yml.md
@@ -99,6 +99,8 @@ floci:
code-path: ./data/lambda-code # Where ZIP archives are stored
poll-interval-ms: 1000
container-idle-timeout-seconds: 300 # Remove idle containers after this
+ region-concurrency-limit: 1000 # Concurrent executions ceiling per region
+ unreserved-concurrency-min: 100 # Minimum unreserved capacity PutFunctionConcurrency must leave
apigateway:
enabled: true
diff --git a/docs/services/lambda.md b/docs/services/lambda.md
index 8ff55d5493..a1e0557019 100644
--- a/docs/services/lambda.md
+++ b/docs/services/lambda.md
@@ -39,14 +39,28 @@ Lambda runs your function code inside real Docker containers — the same way re
| `ListTags` | List tags on a function |
| `TagResource` | Tag a function |
| `UntagResource` | Untag a function |
-| `PutFunctionConcurrency` | Set reserved concurrent executions (stub — value stored but not enforced) |
+| `PutFunctionConcurrency` | Set reserved concurrent executions |
| `GetFunctionConcurrency` | Get reserved concurrent executions |
| `DeleteFunctionConcurrency` | Clear reserved concurrent executions |
-!!! note "Concurrency is a stub"
- `PutFunctionConcurrency` persists `ReservedConcurrentExecutions` on the function
- and echoes it back, but Floci does not enforce the limit at invocation time and
- does not validate against the account-level concurrent execution limit.
+!!! note "Concurrency enforcement"
+ Reserved concurrency is enforced: invocations beyond the reserved value
+ return `TooManyRequestsException` (HTTP 429). Functions without a reserved
+ value share a **per-region** pool — AWS Lambda's "account-level" limit is
+ in fact a per-account-per-region quota, and Floci mirrors that by
+ partitioning counters on the ARN's region segment. The pool size (default
+ 1000) is configurable via `floci.services.lambda.region-concurrency-limit`
+ and applies independently to each region. `PutFunctionConcurrency`
+ validates that the requested value leaves at least
+ `floci.services.lambda.unreserved-concurrency-min` (default 100) available
+ for unreserved functions in that region. `PutProvisionedConcurrencyConfig`
+ and related provisioned-concurrency operations remain unimplemented.
+
+ Reducing or clearing a function's reserved value does not kill
+ invocations that are already in flight — this matches AWS, which
+ applies changes only to new invocations. As a consequence, during the
+ drain window `Σreserved-inflight + unreserved-inflight` can briefly
+ exceed `region-concurrency-limit`.
Function URLs are also reachable directly on `/{proxy:.*}` under the Lambda URL controller, which routes the request into the normal `Invoke` path.
@@ -80,6 +94,8 @@ floci:
code-path: ./data/lambda-code # ZIP storage location
poll-interval-ms: 1000
container-idle-timeout-seconds: 300 # Idle container cleanup
+ region-concurrency-limit: 1000 # Concurrent executions ceiling per region
+ unreserved-concurrency-min: 100 # Min unreserved capacity PutFunctionConcurrency must leave
```
### Docker socket requirement
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index ee8e080f6a..c8c381c3dc 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -509,6 +509,22 @@ public interface EmulatorConfig {
/** Docker network to attach Lambda containers to. Empty = default bridge. */
Optional<String> dockerNetwork();
+
+ /**
+ * Concurrent executions ceiling applied per region. AWS Lambda's
+ * "account-level" concurrency is in fact a per-region quota (default 1000);
+ * Floci mirrors that semantics and partitions counters by the region
+ * segment of each function ARN.
+ */
+ @WithDefault("1000")
+ int regionConcurrencyLimit();
+
+ /**
+ * Minimum unreserved concurrency that must remain after PutFunctionConcurrency,
+ * matching AWS (100). Puts that would leave less than this are rejected.
+ */
+ @WithDefault("100")
+ int unreservedConcurrencyMin();
}
interface Ec2ServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
index 60c0898150..73cfba498d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java
@@ -1,10 +1,12 @@
package io.github.hectorvent.floci.services.lambda;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;
import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;
import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -129,7 +131,17 @@ public class DynamoDbStreamsEventSourcePoller {
esm.getUuid(), records.size(), esm.getFunctionName());
String eventJson = buildDynamoDbEvent(records, esm);
- var invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);
+ InvokeResult invokeResult;
+ try {
+ invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);
+ } catch (AwsException e) {
+ if ("TooManyRequestsException".equals(e.getErrorCode())) {
+ LOG.infov("DynamoDB Streams ESM {0}: function {1} throttled, shard iterator not advanced",
+ esm.getUuid(), fn.getFunctionName());
+ return;
+ }
+ throw e;
+ }
if (invokeResult.getFunctionError() == null) {
String newestSeq = records.get(records.size() - 1).getSequenceNumber();
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.java
index 7dcb4fb271..3d6c29573b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.lambda;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.kinesis.KinesisService;
import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
@@ -118,8 +119,18 @@ public class KinesisEventSourcePoller {
if (!records.isEmpty()) {
String eventJson = buildKinesisEvent(records, esm, shard.getShardId());
- InvokeResult invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);
-
+ InvokeResult invokeResult;
+ try {
+ invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);
+ } catch (AwsException e) {
+ if ("TooManyRequestsException".equals(e.getErrorCode())) {
+ LOG.infov("Kinesis ESM {0}: function {1} throttled, shard iterator not advanced",
+ esm.getUuid(), fn.getFunctionName());
+ continue;
+ }
+ throw e;
+ }
+
if (invokeResult.getFunctionError() == null) {
String newestSeq = records.get(records.size() - 1).getSequenceNumber();
esm.getShardSequenceNumbers().put(shard.getShardId(), newestSeq);
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.java
index 76d3964d5c..72313ecd61 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.java
@@ -30,8 +30,9 @@ import java.util.Map;
* The class-level {@code @Path("/")} lets each method declare its own absolute
* version prefix rather than inheriting a single one.
*
- * The stored value is not enforced at invocation time — this is a stub so that
- * tools and SDKs expecting the endpoint to exist can proceed.
+ * The stored value is enforced at invocation time by
+ * {@link LambdaConcurrencyLimiter}; Put also validates against the per-region
+ * unreserved-minimum.
*/
@Path("/")
@Produces(MediaType.APPLICATION_JSON)
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiter.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiter.java
new file mode 100644
index 0000000000..63d9809c7b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiter.java
@@ -0,0 +1,330 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Inject;
+import org.jboss.logging.Logger;
+
+import java.util.Collections;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Enforces Lambda concurrency limits at invocation time.
+ *
+ * <p>AWS Lambda concurrency is scoped to an account <b>per region</b>; a
+ * function's reserved value does not compete with functions in other regions.
+ * Accordingly this limiter partitions its state by region (extracted from the
+ * function ARN), and the configured {@code regionLimit}/{@code unreservedMin}
+ * apply independently to each region.
+ *
+ * <p>Two layers of enforcement:
+ * <ul>
+ * <li><b>Reserved (per-function)</b>: when a function has a reserved value,
+ * inflight invocations are counted against that value and do not consume
+ * the region's unreserved pool.</li>
+ * <li><b>Unreserved (region-shared)</b>: functions without a reserved value
+ * share {@code regionLimit - Σreserved} permits within their region.</li>
+ * </ul>
+ */
+@ApplicationScoped
+public class LambdaConcurrencyLimiter {
+
+ private static final Logger LOG = Logger.getLogger(LambdaConcurrencyLimiter.class);
+ /** Tracks malformed ARNs already logged so the warn fires once per unique input. */
+ private static final Set<String> LOGGED_MALFORMED_ARNS =
+ Collections.newSetFromMap(new ConcurrentHashMap<>());
+
+ /**
+ * Inflight counts per function ARN (globally unique). Entries are
+ * retained even when the count drops to zero — see {@link #reset} for
+ * the race this avoids. The map therefore grows by one entry per
+ * distinct ARN the limiter has ever seen, including entries left over
+ * from functions that have been deleted and recreated under a new
+ * name. For a local emulator the resulting footprint is negligible.
+ */
+ private final ConcurrentHashMap<String, AtomicInteger> inflight = new ConcurrentHashMap<>();
+ /** Reserved values partitioned by region. */
+ private final ConcurrentHashMap<String, ConcurrentHashMap<String, Integer>> reservedByRegion
+ = new ConcurrentHashMap<>();
+ /**
+ * Running sum of {@link #reservedByRegion} values per region. Maintained
+ * under {@link #reservedLock} on every mutation so that unreserved
+ * acquisition can read a consistent cap in O(1).
+ */
+ private final ConcurrentHashMap<String, AtomicInteger> regionReservedTotal = new ConcurrentHashMap<>();
+ /** Unreserved inflight counters partitioned by region. */
+ private final ConcurrentHashMap<String, AtomicInteger> unreservedByRegion = new ConcurrentHashMap<>();
+ /** Guards atomic validate-then-set and rollback operations on the reserved state. */
+ private final Object reservedLock = new Object();
+ private final int regionLimit;
+ private final int unreservedMin;
+
+ @Inject
+ public LambdaConcurrencyLimiter(EmulatorConfig config) {
+ this(config.services().lambda().regionConcurrencyLimit(),
+ config.services().lambda().unreservedConcurrencyMin());
+ }
+
+ /** Test-only constructor with explicit limits. */
+ LambdaConcurrencyLimiter(int regionLimit, int unreservedMin) {
+ this.regionLimit = regionLimit;
+ this.unreservedMin = unreservedMin;
+ }
+
+ /** Test-only no-arg constructor using AWS defaults (1000 / 100). */
+ LambdaConcurrencyLimiter() {
+ this(1000, 100);
+ }
+
+ public Permit acquire(LambdaFunction fn) {
+ Integer r = fn.getReservedConcurrentExecutions();
+ String region = regionOf(fn.getFunctionArn());
+ if (r == null) {
+ return acquireUnreserved(region);
+ }
+ return acquireReserved(fn.getFunctionArn(), r);
+ }
+
+ private Permit acquireReserved(String key, int limit) {
+ AtomicInteger counter = inflight.computeIfAbsent(key, k -> new AtomicInteger());
+ while (true) {
+ int current = counter.get();
+ if (current >= limit) {
+ throw throttle();
+ }
+ if (counter.compareAndSet(current, current + 1)) {
+ return idempotentPermit(counter);
+ }
+ }
+ }
+
+ private Permit acquireUnreserved(String region) {
+ AtomicInteger counter = unreservedByRegion.computeIfAbsent(region, k -> new AtomicInteger());
+ AtomicInteger reservedTotal = regionTotal(region);
+ while (true) {
+ int current = counter.get();
+ // reservedTotal is an AtomicInteger updated under reservedLock on
+ // each reserved change, so the cap is consistent and computed in
+ // O(1) regardless of the number of reserved functions.
+ int cap = Math.max(0, regionLimit - reservedTotal.get());
+ if (current >= cap) {
+ throw throttle();
+ }
+ if (counter.compareAndSet(current, current + 1)) {
+ return idempotentPermit(counter);
+ }
+ }
+ }
+
+ /**
+ * Wraps {@code counter.decrementAndGet()} in a close-once guard so a
+ * future caller that accidentally double-closes a permit cannot drive
+ * the inflight counter negative.
+ */
+ private static Permit idempotentPermit(AtomicInteger counter) {
+ AtomicBoolean closed = new AtomicBoolean(false);
+ return () -> {
+ if (closed.compareAndSet(false, true)) {
+ counter.decrementAndGet();
+ }
+ };
+ }
+
+ /**
+ * Register (or update) a function's reserved value without validation.
+ * Intended for startup rehydration from persisted state.
+ *
+ * @return the previous value, or {@code null} if none was set.
+ */
+ public Integer setReserved(String functionArn, int value) {
+ synchronized (reservedLock) {
+ return writeReserved(functionArn, value);
+ }
+ }
+
+ /**
+ * @return the cleared value, or {@code null} if no reservation was set.
+ */
+ public Integer clearReserved(String functionArn) {
+ synchronized (reservedLock) {
+ return writeReserved(functionArn, null);
+ }
+ }
+
+ /**
+ * Atomically validates and applies a reserved value within the function's
+ * region. Two concurrent Puts for different functions cannot each pass
+ * validation against stale totals and then collectively push the region's
+ * unreserved capacity below the minimum.
+ *
+ * @return the previous reserved value for this ARN, or {@code null} if none;
+ * callers may use it with {@link #rollbackReservedIfExpected} on a
+ * subsequent persistence failure.
+ * @throws AwsException {@code LimitExceededException} if the value would
+ * drop unreserved below the minimum.
+ */
+ public Integer validateAndSetReserved(String functionArn, int target) {
+ String region = regionOf(functionArn);
+ synchronized (reservedLock) {
+ ConcurrentHashMap<String, Integer> regionReserved = reservedOf(region);
+ int currentForThis = regionReserved.getOrDefault(functionArn, 0);
+ // A reduction (or no-op) cannot decrease unreserved capacity any
+ // further than it already is, so always allow it. This lets
+ // operators recover from an over-committed state — e.g. after
+ // lowering region-concurrency-limit at runtime — without first
+ // having to delete every reservation.
+ if (target > currentForThis) {
+ int otherReserved = regionTotal(region).get() - currentForThis;
+ int maxAllowed = regionLimit - unreservedMin - otherReserved;
+ if (target > maxAllowed) {
+ throw new AwsException("LimitExceededException",
+ "Specified ReservedConcurrentExecutions for function decreases account's "
+ + "UnreservedConcurrentExecution below its minimum value of ["
+ + unreservedMin + "].", 400);
+ }
+ }
+ return writeReserved(functionArn, target);
+ }
+ }
+
+ /**
+ * Conditionally restores a prior reserved value if the current value still
+ * matches {@code expectedCurrent}. Used by callers that updated the limiter
+ * before a persistence step and need to undo the change on failure without
+ * clobbering a concurrent successful write.
+ */
+ public void rollbackReservedIfExpected(String functionArn,
+ Integer expectedCurrent,
+ Integer rollbackTo) {
+ synchronized (reservedLock) {
+ Integer actual = reservedOf(regionOf(functionArn)).get(functionArn);
+ if (!Objects.equals(actual, expectedCurrent)) {
+ // A newer update has superseded ours; leave it alone.
+ return;
+ }
+ writeReserved(functionArn, rollbackTo);
+ }
+ }
+
+ /**
+ * Clears the reserved entry for a deleted function. The inflight counter
+ * is intentionally retained — permits held by still-running invocations
+ * decrement into it on close, and an ARN recreated later reuses the same
+ * counter so new invocations correctly see any remaining inflight.
+ *
+ * <p>The counter is also retained even when it momentarily reads zero:
+ * conditionally removing it would race with a concurrent
+ * {@code acquireReserved} that has already obtained the {@code AtomicInteger}
+ * reference and is about to increment it. After such a race the next
+ * acquire would allocate a fresh counter and undercount the inflight
+ * permit, allowing reserved over-subscription. The trade-off is one
+ * {@code AtomicInteger} per historical function, which is bounded for an
+ * emulator workload.
+ */
+ public void reset(String functionArn) {
+ synchronized (reservedLock) {
+ writeReserved(functionArn, null);
+ }
+ }
+
+ public int totalReserved(String region) {
+ return regionTotal(region).get();
+ }
+
+ public int availableUnreserved(String region) {
+ AtomicInteger counter = unreservedByRegion.get(region);
+ int inflightNow = counter == null ? 0 : counter.get();
+ return Math.max(0, regionLimit - totalReserved(region) - inflightNow);
+ }
+
+ int inflightCount(String functionArn) {
+ AtomicInteger counter = inflight.get(functionArn);
+ return counter == null ? 0 : counter.get();
+ }
+
+ int unreservedInflightCount(String region) {
+ AtomicInteger counter = unreservedByRegion.get(region);
+ return counter == null ? 0 : counter.get();
+ }
+
+ /**
+ * Must be called with {@link #reservedLock} held. Updates the per-function
+ * reserved entry and the region's running total in one atomic step from the
+ * perspective of any code that also holds the lock.
+ */
+ private Integer writeReserved(String functionArn, Integer newValue) {
+ String region = regionOf(functionArn);
+ ConcurrentHashMap<String, Integer> regionReserved = reservedOf(region);
+ Integer previous = newValue == null
+ ? regionReserved.remove(functionArn)
+ : regionReserved.put(functionArn, newValue);
+ int delta = (newValue == null ? 0 : newValue) - (previous == null ? 0 : previous);
+ if (delta != 0) {
+ regionTotal(region).addAndGet(delta);
+ }
+ return previous;
+ }
+
+ private ConcurrentHashMap<String, Integer> reservedOf(String region) {
+ return reservedByRegion.computeIfAbsent(region, k -> new ConcurrentHashMap<>());
+ }
+
+ private AtomicInteger regionTotal(String region) {
+ return regionReservedTotal.computeIfAbsent(region, k -> new AtomicInteger());
+ }
+
+ /**
+ * Extracts the region segment from a Lambda function ARN. Falls back to
+ * {@code "unknown"} if the ARN is malformed so state still partitions
+ * cleanly rather than mixing with another region's data.
+ *
+ * <p>This is called on every acquire/release, so the parse avoids the
+ * regex machinery and array allocation of {@code String.split(":")} by
+ * scanning for the fourth ':' segment (index 3 in an ARN:
+ * {@code arn:aws:lambda:REGION:account:function:name}).
+ */
+ private static String regionOf(String arn) {
+ if (arn == null) {
+ warnMalformed("<null>");
+ return "unknown";
+ }
+ int segmentStart = 0;
+ int delimiters = 0;
+ for (int i = 0; i < arn.length(); i++) {
+ if (arn.charAt(i) == ':') {
+ if (delimiters == 2) {
+ segmentStart = i + 1;
+ } else if (delimiters == 3) {
+ return arn.substring(segmentStart, i);
+ }
+ delimiters++;
+ }
+ }
+ warnMalformed(arn);
+ return "unknown";
+ }
+
+ private static void warnMalformed(String arn) {
+ if (LOGGED_MALFORMED_ARNS.add(arn)) {
+ LOG.warnv("Concurrency limiter received non-ARN function identifier "
+ + "\"{0}\"; bucketing under region=\"unknown\". This likely "
+ + "indicates a bare function name reached the limiter.", arn);
+ }
+ }
+
+ private static AwsException throttle() {
+ return new AwsException("TooManyRequestsException", "Rate Exceeded.", 429);
+ }
+
+ @FunctionalInterface
+ public interface Permit extends AutoCloseable {
+ @Override
+ void close();
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
index 919e19fd4d..20d49634cf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java
@@ -32,6 +32,7 @@ public class LambdaExecutorService {
private final WarmPool warmPool;
private final ObjectMapper objectMapper;
+ private final LambdaConcurrencyLimiter concurrencyLimiter;
private final ExecutorService asyncExecutor = new ThreadPoolExecutor(
Math.max(4, Runtime.getRuntime().availableProcessors() * 2),
Math.max(8, Runtime.getRuntime().availableProcessors() * 4),
@@ -40,25 +41,43 @@ public class LambdaExecutorService {
new ThreadPoolExecutor.CallerRunsPolicy());
@Inject
- public LambdaExecutorService(WarmPool warmPool, ObjectMapper objectMapper) {
+ public LambdaExecutorService(WarmPool warmPool,
+ ObjectMapper objectMapper,
+ LambdaConcurrencyLimiter concurrencyLimiter) {
this.warmPool = warmPool;
this.objectMapper = objectMapper;
+ this.concurrencyLimiter = concurrencyLimiter;
}
public InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type) {
String requestId = UUID.randomUUID().toString();
- switch (type) {
- case DryRun -> {
- return new InvokeResult(204, null, new byte[0], null, requestId);
- }
- case Event -> {
- asyncExecutor.submit(() -> executeSync(fn, payload, requestId));
- return new InvokeResult(202, null, new byte[0], null, requestId);
- }
- default -> {
- return executeSync(fn, payload, requestId);
+ if (type == InvocationType.DryRun) {
+ return new InvokeResult(204, null, new byte[0], null, requestId);
+ }
+
+ LambdaConcurrencyLimiter.Permit permit = concurrencyLimiter.acquire(fn);
+
+ if (type == InvocationType.Event) {
+ try {
+ asyncExecutor.submit(() -> {
+ try {
+ executeSync(fn, payload, requestId);
+ } finally {
+ permit.close();
+ }
+ });
+ } catch (RuntimeException e) {
+ permit.close();
+ throw e;
}
+ return new InvokeResult(202, null, new byte[0], null, requestId);
+ }
+
+ try {
+ return executeSync(fn, payload, requestId);
+ } finally {
+ permit.close();
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
index 495950824c..7a13282e6b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java
@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.lambda.zip.ZipExtractor;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.github.hectorvent.floci.services.s3.model.S3Object;
import io.github.hectorvent.floci.services.sqs.SqsService;
+import jakarta.annotation.PostConstruct;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
@@ -43,6 +44,7 @@ public class LambdaService {
private final LambdaFunctionStore functionStore;
private final LambdaExecutorService executorService;
+ private final LambdaConcurrencyLimiter concurrencyLimiter;
private final WarmPool warmPool;
private final CodeStore codeStore;
private final ZipExtractor zipExtractor;
@@ -56,8 +58,28 @@ public class LambdaService {
private final KinesisEventSourcePoller kinesisPoller;
private final DynamoDbStreamsEventSourcePoller dynamodbStreamsPoller;
private final ConcurrentHashMap<String, Integer> versionCounters = new ConcurrentHashMap<>();
-
- /** Package-private constructor for testing without CDI. Config defaults (timeout=3, memory=128) apply. */
+ /**
+ * Per-function locks covering PutFunctionConcurrency,
+ * DeleteFunctionConcurrency, and deleteFunction itself. Serializing the
+ * limiter update + persistence pair against itself for a given function
+ * prevents the limiter and store from diverging on interleaved concurrent
+ * requests.
+ *
+ * <p>Entries are intentionally never removed — see {@code deleteFunction}
+ * for the race this avoids. The map therefore grows by one {@code Object}
+ * per distinct function ARN the emulator has ever seen (create/delete
+ * cycles with fresh names included). Acceptable footprint for a local
+ * emulator workload.
+ */
+ private final ConcurrentHashMap<String, Object> concurrencyOpLocks = new ConcurrentHashMap<>();
+
+ /**
+ * Package-private constructor for testing without CDI. Config defaults
+ * (timeout=3, memory=128) apply. A real {@link LambdaConcurrencyLimiter}
+ * with AWS-default limits is wired so concurrency operations exercise
+ * the same validation and bookkeeping as production rather than
+ * silently no-op'ing past null checks.
+ */
LambdaService(LambdaFunctionStore functionStore,
WarmPool warmPool,
CodeStore codeStore,
@@ -65,6 +87,7 @@ public class LambdaService {
RegionResolver regionResolver) {
this.functionStore = functionStore;
this.executorService = null;
+ this.concurrencyLimiter = new LambdaConcurrencyLimiter();
this.warmPool = warmPool;
this.codeStore = codeStore;
this.zipExtractor = zipExtractor;
@@ -82,6 +105,7 @@ public class LambdaService {
@Inject
public LambdaService(LambdaFunctionStore functionStore,
LambdaExecutorService executorService,
+ LambdaConcurrencyLimiter concurrencyLimiter,
WarmPool warmPool,
CodeStore codeStore,
ZipExtractor zipExtractor,
@@ -96,6 +120,7 @@ public class LambdaService {
DynamoDbStreamsEventSourcePoller dynamodbStreamsPoller) {
this.functionStore = functionStore;
this.executorService = executorService;
+ this.concurrencyLimiter = concurrencyLimiter;
this.warmPool = warmPool;
this.codeStore = codeStore;
this.zipExtractor = zipExtractor;
@@ -110,6 +135,40 @@ public class LambdaService {
this.dynamodbStreamsPoller = dynamodbStreamsPoller;
}
+ /** Package-private accessor for tests that want to assert limiter state directly. */
+ LambdaConcurrencyLimiter concurrencyLimiter() {
+ return concurrencyLimiter;
+ }
+
+ /**
+ * Rehydrates reserved concurrency into the limiter from persisted function state.
+ * Without this, restarts leave {@code totalReserved()=0} and allow validatePut /
+ * unreserved-pool sizing to drift until each function is re-Put.
+ */
+ @PostConstruct
+ void rehydrateConcurrency() {
+ if (concurrencyLimiter == null) {
+ return;
+ }
+ int count = 0;
+ for (LambdaFunction fn : functionStore.listAll()) {
+ // Reserved concurrency is a function-level property; published
+ // versions share the $LATEST record's value. Skip non-$LATEST
+ // entries to avoid double-counting into totalReserved().
+ if (!"$LATEST".equals(fn.getVersion())) {
+ continue;
+ }
+ Integer reserved = fn.getReservedConcurrentExecutions();
+ if (reserved != null) {
+ concurrencyLimiter.setReserved(fn.getFunctionArn(), reserved);
+ count++;
+ }
+ }
+ if (count > 0) {
+ LOG.infov("Restored reserved concurrency for {0} function(s)", count);
+ }
+ }
+
public LambdaFunction createFunction(String region, Map<String, Object> request) {
String functionName = (String) request.get("FunctionName");
String role = (String) request.get("Role");
@@ -230,10 +289,23 @@ public class LambdaService {
}
public void deleteFunction(String region, String functionName) {
- getFunction(region, functionName); // throws 404 if not found
+ LambdaFunction fn = getFunction(region, functionName); // throws 404 if not found
+ String arn = fn.getFunctionArn();
warmPool.drainFunction(functionName);
- codeStore.delete(functionName);
- functionStore.delete(region, functionName);
+ // Take the same per-function lock used by Put/DeleteFunctionConcurrency
+ // so a concurrent concurrency mutation cannot interleave with the
+ // limiter reset and store delete and leave the two views out of sync.
+ // The lock entry itself stays in the map after the delete: removing it
+ // could race with another thread already synchronized on the same
+ // object, letting a follow-up request allocate a fresh lock and run
+ // in parallel — the very serialization this map exists to prevent.
+ synchronized (lockForConcurrencyOp(arn)) {
+ if (concurrencyLimiter != null) {
+ concurrencyLimiter.reset(arn);
+ }
+ codeStore.delete(functionName);
+ functionStore.delete(region, functionName);
+ }
LOG.infov("Deleted Lambda function: {0}", functionName);
}
@@ -597,8 +669,29 @@ public class LambdaService {
"ReservedConcurrentExecutions must be a non-negative integer", 400);
}
LambdaFunction fn = getFunction(region, functionName);
- fn.setReservedConcurrentExecutions(reservedConcurrentExecutions);
- functionStore.save(region, fn);
+ String arn = fn.getFunctionArn();
+ // Serialize limiter update + store save for this function so that two
+ // concurrent Puts cannot leave the limiter and persisted state out of
+ // sync, regardless of which call acquires the reservedLock first.
+ synchronized (lockForConcurrencyOp(arn)) {
+ Integer previousReserved = null;
+ boolean limiterUpdated = false;
+ if (concurrencyLimiter != null) {
+ previousReserved = concurrencyLimiter.validateAndSetReserved(
+ arn, reservedConcurrentExecutions);
+ limiterUpdated = true;
+ }
+ fn.setReservedConcurrentExecutions(reservedConcurrentExecutions);
+ try {
+ functionStore.save(region, fn);
+ } catch (RuntimeException e) {
+ if (limiterUpdated) {
+ concurrencyLimiter.rollbackReservedIfExpected(
+ arn, reservedConcurrentExecutions, previousReserved);
+ }
+ throw e;
+ }
+ }
return fn;
}
@@ -609,8 +702,29 @@ public class LambdaService {
public void deleteFunctionConcurrency(String region, String functionName) {
LambdaFunction fn = getFunction(region, functionName);
- fn.setReservedConcurrentExecutions(null);
- functionStore.save(region, fn);
+ String arn = fn.getFunctionArn();
+ synchronized (lockForConcurrencyOp(arn)) {
+ Integer previousReserved = null;
+ boolean limiterCleared = false;
+ if (concurrencyLimiter != null) {
+ previousReserved = concurrencyLimiter.clearReserved(arn);
+ limiterCleared = true;
+ }
+ fn.setReservedConcurrentExecutions(null);
+ try {
+ functionStore.save(region, fn);
+ } catch (RuntimeException e) {
+ if (limiterCleared && previousReserved != null) {
+ concurrencyLimiter.rollbackReservedIfExpected(
+ arn, null, previousReserved);
+ }
+ throw e;
+ }
+ }
+ }
+
+ private Object lockForConcurrencyOp(String functionArn) {
+ return concurrencyOpLocks.computeIfAbsent(functionArn, k -> new Object());
}
public LambdaFunction getFunctionByUrlId(String urlId) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
index 35480db0de..44e7863a07 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java
@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;
import io.github.hectorvent.floci.services.lambda.model.InvocationType;
import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
@@ -146,8 +147,18 @@ public class SqsEventSourcePoller {
String eventJson = buildSqsEvent(messages, esm);
LOG.infov("ESM {0}: invoking function {1}", esm.getUuid(), fn.getFunctionName());
- InvokeResult result = executorService.invoke(
- fn, eventJson.getBytes(), InvocationType.RequestResponse);
+ InvokeResult result;
+ try {
+ result = executorService.invoke(
+ fn, eventJson.getBytes(), InvocationType.RequestResponse);
+ } catch (AwsException e) {
+ if ("TooManyRequestsException".equals(e.getErrorCode())) {
+ LOG.infov("ESM {0}: function {1} throttled, messages will return to queue after visibility timeout",
+ esm.getUuid(), fn.getFunctionName());
+ return;
+ }
+ throw e;
+ }
if (result.getFunctionError() == null) {
Set<String> failedIds = extractBatchItemFailures(esm, result);
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 0875ed7fbe..8b2eb57ac5 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -80,6 +80,8 @@ floci:
code-path: ./data/lambda-code
poll-interval-ms: 1000
container-idle-timeout-seconds: 300
+ region-concurrency-limit: 1000
+ unreserved-concurrency-min: 100
apigateway:
enabled: true
apigatewayv2:
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiterTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiterTest.java
new file mode 100644
index 0000000000..1bade6233e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiterTest.java
@@ -0,0 +1,261 @@
+package io.github.hectorvent.floci.services.lambda;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class LambdaConcurrencyLimiterTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String OTHER_REGION = "ap-northeast-1";
+ private static final String ARN = "arn:aws:lambda:us-east-1:000000000000:function:fn";
+ private static final String ARN2 = "arn:aws:lambda:us-east-1:000000000000:function:other";
+ private static final String ARN_OTHER_REGION = "arn:aws:lambda:ap-northeast-1:000000000000:function:fn-apne1";
+
+ private LambdaFunction fn(String arn, Integer reserved) {
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("fn");
+ fn.setFunctionArn(arn);
+ fn.setReservedConcurrentExecutions(reserved);
+ return fn;
+ }
+
+ private LambdaFunction fn(Integer reserved) {
+ return fn(ARN, reserved);
+ }
+
+ @Test
+ void unsetReserved_countsAgainstAccountPool() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(2, 0);
+ LambdaConcurrencyLimiter.Permit p1 = limiter.acquire(fn(null));
+ LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(fn(null));
+ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(fn(null)));
+ assertEquals(429, ex.getHttpStatus());
+ p1.close();
+ p2.close();
+ assertEquals(0, limiter.unreservedInflightCount(REGION));
+ }
+
+ @Test
+ void reservedN_allowsUpToN_thenThrows() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ LambdaFunction f = fn(2);
+ LambdaConcurrencyLimiter.Permit p1 = limiter.acquire(f);
+ LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(f);
+
+ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(f));
+ assertEquals("TooManyRequestsException", ex.getErrorCode());
+ assertEquals(429, ex.getHttpStatus());
+
+ p1.close();
+ LambdaConcurrencyLimiter.Permit p3 = limiter.acquire(f);
+ p2.close();
+ p3.close();
+ assertEquals(0, limiter.inflightCount(ARN));
+ }
+
+ @Test
+ void reservedZero_throwsImmediately() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(fn(0)));
+ assertEquals(429, ex.getHttpStatus());
+ }
+
+ @Test
+ void reservedPool_doesNotConsumeUnreserved() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(3, 0);
+ limiter.setReserved(ARN, 2);
+ // Reserved function consumes its own pool, not the region pool
+ try (LambdaConcurrencyLimiter.Permit p1 = limiter.acquire(fn(ARN, 2));
+ LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(fn(ARN, 2));
+ // Unreserved function can still use the full regionLimit - totalReserved = 1
+ LambdaConcurrencyLimiter.Permit p3 = limiter.acquire(fn(ARN2, null))) {
+ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(fn(ARN2, null)));
+ assertEquals(429, ex.getHttpStatus());
+ }
+ }
+
+ @Test
+ void reset_preservesInflightCounterWhenBusy() {
+ // If a function is deleted while invocations are still running, and the
+ // same ARN is recreated, new invocations must see the remaining inflight
+ // so we don't transiently over-subscribe.
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ LambdaFunction before = fn(3);
+ LambdaConcurrencyLimiter.Permit held = limiter.acquire(before);
+ assertEquals(1, limiter.inflightCount(ARN));
+
+ limiter.reset(ARN);
+ assertEquals(1, limiter.inflightCount(ARN), "inflight retained while permit is open");
+
+ LambdaFunction recreated = fn(3);
+ try (LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(recreated)) {
+ assertEquals(2, limiter.inflightCount(ARN));
+ }
+ held.close();
+ assertEquals(0, limiter.inflightCount(ARN));
+ }
+
+ @Test
+ void reset_keepsIdleInflightCounterToAvoidUndercount() {
+ // The counter is intentionally retained across reset to close the
+ // window where a concurrent acquire could otherwise allocate a fresh
+ // counter and undercount inflight permits already in flight.
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ try (LambdaConcurrencyLimiter.Permit p = limiter.acquire(fn(1))) {
+ assertEquals(1, limiter.inflightCount(ARN));
+ }
+ assertEquals(0, limiter.inflightCount(ARN));
+ limiter.reset(ARN);
+ // Counter still present at zero; new acquires share it.
+ assertEquals(0, limiter.inflightCount(ARN));
+ try (LambdaConcurrencyLimiter.Permit p = limiter.acquire(fn(1))) {
+ assertEquals(1, limiter.inflightCount(ARN));
+ }
+ }
+
+ @Test
+ void validateAndSetReserved_allowsReductionEvenWhenOverCommitted() {
+ // Simulate an over-committed state: total reserved exceeds the
+ // unreserved minimum's ceiling. (e.g. region-concurrency-limit was
+ // lowered at runtime, or state was migrated from earlier behavior.)
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);
+ // Bypass validation by using setReserved directly so we can engineer
+ // the broken state that validateAndSetReserved should still let us recover from.
+ limiter.setReserved(ARN, 950);
+ // Now totalReserved=950, unreserved capacity = 50 < min(100). Any
+ // increase should still be blocked.
+ assertThrows(AwsException.class, () -> limiter.validateAndSetReserved(ARN, 951));
+ // But a reduction must succeed so the operator can recover.
+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 500));
+ assertEquals(500, limiter.totalReserved(REGION));
+ }
+
+ @Test
+ void validatePut_rejectsWhenUnreservedMinViolated() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);
+ // totalReserved=0, max allowed for new function = 1000 - 100 = 900
+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 900));
+ AwsException ex = assertThrows(AwsException.class, () -> limiter.validateAndSetReserved(ARN, 901));
+ assertEquals("LimitExceededException", ex.getErrorCode());
+ assertEquals(400, ex.getHttpStatus());
+ }
+
+ @Test
+ void validatePut_excludesSelfWhenUpdating() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);
+ limiter.setReserved(ARN, 500);
+ // Updating the same ARN to 900 should succeed (self is excluded from "other")
+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 900));
+ }
+
+ @Test
+ void validatePut_considersOtherFunctions() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);
+ limiter.setReserved(ARN2, 500);
+ // otherReserved=500, max for ARN = 1000 - 100 - 500 = 400
+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 400));
+ assertThrows(AwsException.class, () -> limiter.validateAndSetReserved(ARN, 401));
+ }
+
+ @Test
+ void reset_clearsReservedEntry() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ limiter.setReserved(ARN, 1);
+ limiter.reset(ARN);
+ assertEquals(0, limiter.totalReserved(REGION));
+ }
+
+ @Test
+ void setReserved_returnsPreviousValue() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ assertNull(limiter.setReserved(ARN, 5));
+ assertEquals(5, limiter.setReserved(ARN, 10));
+ }
+
+ @Test
+ void clearReserved_returnsClearedValue() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ limiter.setReserved(ARN, 7);
+ assertEquals(7, limiter.clearReserved(ARN));
+ assertNull(limiter.clearReserved(ARN));
+ }
+
+ @Test
+ void rollbackReservedIfExpected_restoresWhenUnchanged() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ limiter.setReserved(ARN, 5);
+ limiter.setReserved(ARN, 10); // now at 10
+ limiter.rollbackReservedIfExpected(ARN, 10, 5);
+ assertEquals(5, limiter.totalReserved(REGION));
+ }
+
+ @Test
+ void rollbackReservedIfExpected_skipsWhenConcurrentlyChanged() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ // Request A wrote 10 (previous null), then another request superseded to 20
+ limiter.setReserved(ARN, 10);
+ limiter.setReserved(ARN, 20);
+ // A's rollback expects 10 still present — must not clobber 20
+ limiter.rollbackReservedIfExpected(ARN, 10, null);
+ assertEquals(20, limiter.totalReserved(REGION));
+ }
+
+ @Test
+ void totalReserved_tracksOverlappingUpdates() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ limiter.setReserved(ARN, 50);
+ limiter.setReserved(ARN2, 30);
+ assertEquals(80, limiter.totalReserved(REGION));
+ limiter.setReserved(ARN, 100); // +50
+ assertEquals(130, limiter.totalReserved(REGION));
+ limiter.clearReserved(ARN2); // -30
+ assertEquals(100, limiter.totalReserved(REGION));
+ }
+
+ @Test
+ void permit_closeIsIdempotent() {
+ // Future callers must not be able to drive the inflight counter
+ // negative by double-closing a permit (try-with-resources +
+ // explicit close, retry logic, etc.).
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();
+ LambdaConcurrencyLimiter.Permit p = limiter.acquire(fn(3));
+ assertEquals(1, limiter.inflightCount(ARN));
+ p.close();
+ assertEquals(0, limiter.inflightCount(ARN));
+ p.close(); // second close must be a no-op
+ assertEquals(0, limiter.inflightCount(ARN));
+ p.close(); // ...and so must the third
+ assertEquals(0, limiter.inflightCount(ARN));
+ }
+
+ @Test
+ void regions_areIndependent() {
+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);
+ // Fill one region's reserved pool near the limit
+ limiter.validateAndSetReserved(ARN, 900);
+ // Another region starts fresh — Put up to 900 still allowed
+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN_OTHER_REGION, 900));
+ assertEquals(900, limiter.totalReserved(REGION));
+ assertEquals(900, limiter.totalReserved(OTHER_REGION));
+
+ // Unreserved pool is also per-region
+ LambdaConcurrencyLimiter small = new LambdaConcurrencyLimiter(1, 0);
+ try (LambdaConcurrencyLimiter.Permit usEast = small.acquire(fn(ARN, null));
+ LambdaConcurrencyLimiter.Permit apne1 = small.acquire(fn(ARN_OTHER_REGION, null))) {
+ // Same exhaustion in us-east-1, but ap-northeast-1 has its own slot
+ // (already consumed by apne1 above, so second acquire there also throws)
+ assertThrows(AwsException.class, () -> small.acquire(fn(ARN2, null)));
+ assertThrows(AwsException.class, () -> small.acquire(fn(ARN_OTHER_REGION, null)));
+ }
+ // After close, both regions have capacity again
+ try (LambdaConcurrencyLimiter.Permit reacquire = small.acquire(fn(ARN, null))) {
+ assertEquals(1, small.unreservedInflightCount(REGION));
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java
index 821a2eecde..9e83f4ebdc 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java
@@ -237,4 +237,195 @@ class LambdaServiceTest {
LambdaFunction updated = service.updateFunctionCode(REGION, "update-fn", Map.of());
assertNotEquals(originalRevision, updated.getRevisionId());
}
+
+ @Test
+ void rehydrateConcurrency_restoresReservedFromStore() {
+ // Simulate a persisted state: functions already live in the store
+ // with reserved values before the limiter is populated.
+ service.createFunction(REGION, baseRequest("persisted-a"));
+ service.createFunction(REGION, baseRequest("persisted-b"));
+ service.putFunctionConcurrency(REGION, "persisted-a", 300);
+ service.putFunctionConcurrency(REGION, "persisted-b", 200);
+
+ // Build a second service over the same store with a fresh limiter.
+ LambdaFunctionStore store = new LambdaFunctionStore(new InMemoryStorage<>());
+ // Copy the two persisted functions into the new store to emulate a
+ // restart with the same disk state.
+ for (LambdaFunction fn : new java.util.ArrayList<>(
+ service.listFunctions(REGION))) {
+ store.save(REGION, fn);
+ }
+ LambdaService rebooted = new LambdaService(store, new WarmPool(),
+ new CodeStore(Path.of("target/test-data/lambda-code")),
+ new ZipExtractor(), new RegionResolver(REGION, "000000000000"));
+
+ // Starts empty…
+ assertEquals(0, rebooted.concurrencyLimiter().totalReserved(REGION));
+ // …until rehydrate walks the store and re-registers the reserved values.
+ rebooted.rehydrateConcurrency();
+ assertEquals(500, rebooted.concurrencyLimiter().totalReserved(REGION));
+ }
+
+ @Test
+ void multiArnPutFunctionConcurrency_respectsRegionTotalUnderContention() throws Exception {
+ // Two different functions racing a Put near the unreserved floor.
+ // Both try to reserve an amount that — summed — would push the
+ // region below unreserved-min. reservedLock must serialize so that
+ // only one wins.
+ service.createFunction(REGION, baseRequest("multi-a"));
+ service.createFunction(REGION, baseRequest("multi-b"));
+
+ // Defaults: regionLimit=1000, unreservedMin=100. Each Put asks for
+ // 500; together they would leave 0 unreserved.
+ java.util.concurrent.ExecutorService pool = java.util.concurrent.Executors.newFixedThreadPool(2);
+ java.util.concurrent.CountDownLatch start = new java.util.concurrent.CountDownLatch(1);
+ try {
+ java.util.concurrent.Future<Throwable> fA = pool.submit(() -> {
+ start.await();
+ try {
+ service.putFunctionConcurrency(REGION, "multi-a", 500);
+ return null;
+ } catch (Throwable t) { return t; }
+ });
+ java.util.concurrent.Future<Throwable> fB = pool.submit(() -> {
+ start.await();
+ try {
+ service.putFunctionConcurrency(REGION, "multi-b", 500);
+ return null;
+ } catch (Throwable t) { return t; }
+ });
+ start.countDown();
+
+ Throwable rA = fA.get();
+ Throwable rB = fB.get();
+
+ // Exactly one must have been rejected with LimitExceededException.
+ int successes = (rA == null ? 1 : 0) + (rB == null ? 1 : 0);
+ assertEquals(1, successes, "exactly one Put must win");
+ Throwable rejected = rA != null ? rA : rB;
+ assertTrue(rejected instanceof AwsException
+ && "LimitExceededException".equals(((AwsException) rejected).getErrorCode()),
+ "other Put must be rejected, got " + rejected);
+ assertEquals(500,
+ service.concurrencyLimiter().totalReserved(REGION),
+ "limiter total must reflect only the winning Put");
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ @Test
+ void putFunctionConcurrency_rollsBackLimiterIfSaveFails() {
+ // If functionStore.save throws after the limiter has been updated,
+ // the limiter must be restored so Σreserved stays consistent with
+ // what the store actually persisted.
+ FailingStore failing = new FailingStore();
+ LambdaService svc = new LambdaService(failing, new WarmPool(),
+ new CodeStore(Path.of("target/test-data/lambda-code")),
+ new ZipExtractor(), new RegionResolver(REGION, "000000000000"));
+ svc.createFunction(REGION, baseRequest("rb-fn"));
+ // Baseline: Put of 300 succeeds
+ svc.putFunctionConcurrency(REGION, "rb-fn", 300);
+ assertEquals(300, svc.concurrencyLimiter().totalReserved(REGION));
+
+ // Now make the next save() explode and try to Put 500
+ failing.shouldFail = true;
+ assertThrows(RuntimeException.class,
+ () -> svc.putFunctionConcurrency(REGION, "rb-fn", 500));
+
+ // Limiter must have rolled back to the previous 300
+ assertEquals(300, svc.concurrencyLimiter().totalReserved(REGION),
+ "limiter must unwind on save failure");
+ }
+
+ @Test
+ void deleteFunction_preservesInflightPermitUntilItCloses() {
+ // A deleteFunction that lands while an invocation is still holding a
+ // permit must not drop the counter — the permit close() at the end
+ // of the running invocation still has to decrement something valid.
+ service.createFunction(REGION, baseRequest("del-inflight"));
+ service.putFunctionConcurrency(REGION, "del-inflight", 2);
+
+ LambdaFunction fn = service.getFunction(REGION, "del-inflight");
+ String arn = fn.getFunctionArn();
+ LambdaConcurrencyLimiter.Permit held =
+ service.concurrencyLimiter().acquire(fn);
+ assertEquals(1, service.concurrencyLimiter().inflightCount(arn));
+
+ service.deleteFunction(REGION, "del-inflight");
+
+ // Reserved is cleared from the limiter, but the inflight counter
+ // must still be live for the held permit to decrement into.
+ assertEquals(0, service.concurrencyLimiter().totalReserved(REGION));
+ assertEquals(1, service.concurrencyLimiter().inflightCount(arn),
+ "inflight must survive delete until the permit closes");
+
+ held.close();
+ assertEquals(0, service.concurrencyLimiter().inflightCount(arn));
+ }
+
+ /**
+ * Test helper: a LambdaFunctionStore whose save() throws on demand so
+ * tests can exercise the LambdaService rollback path.
+ */
+ private static final class FailingStore extends LambdaFunctionStore {
+ boolean shouldFail = false;
+ FailingStore() {
+ super(new InMemoryStorage<String, LambdaFunction>());
+ }
+ @Override
+ public void save(String region, LambdaFunction fn) {
+ if (shouldFail) {
+ throw new RuntimeException("injected save failure");
+ }
+ super.save(region, fn);
+ }
+ }
+
+ @Test
+ void concurrentPutFunctionConcurrency_endsInConsistentState() throws Exception {
+ // Exercise the per-function serialization in concurrencyOpLocks:
+ // two threads racing Put on the same function must leave the
+ // limiter and the persisted reserved value in agreement with
+ // whichever write landed last.
+ service.createFunction(REGION, baseRequest("race-fn"));
+
+ int iterations = 50;
+ java.util.concurrent.ExecutorService pool = java.util.concurrent.Executors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < iterations; i++) {
+ int a = 100 + i;
+ int b = 200 + i;
+ java.util.concurrent.CountDownLatch start = new java.util.concurrent.CountDownLatch(1);
+ java.util.concurrent.Future<Integer> fA = pool.submit(() -> {
+ start.await();
+ return service.putFunctionConcurrency(REGION, "race-fn", a)
+ .getReservedConcurrentExecutions();
+ });
+ java.util.concurrent.Future<Integer> fB = pool.submit(() -> {
+ start.await();
+ return service.putFunctionConcurrency(REGION, "race-fn", b)
+ .getReservedConcurrentExecutions();
+ });
+ start.countDown();
+ fA.get();
+ fB.get();
+
+ LambdaFunction fn = service.getFunction(REGION, "race-fn");
+ Integer stored = fn.getReservedConcurrentExecutions();
+ assertTrue(stored.equals(a) || stored.equals(b),
+ "store should reflect one of the two writes, got " + stored);
+ // The real invariant: the limiter's Σreserved for this
+ // region must agree with what was persisted. Comparing
+ // getFunctionConcurrency() to stored would be a tautology —
+ // both read the same LambdaFunction field — so assert
+ // against the limiter's independently-maintained total.
+ assertEquals(stored.intValue(),
+ service.concurrencyLimiter().totalReserved(REGION),
+ "limiter totalReserved must match persisted reserved value");
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
}