Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambda): enforce reserved and per-region concurrency (#424)
Kenshin Okinakafb2673113149
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.javaindex 32a4831ce3..d8ea6abf1a 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/LambdaConcurrencyTest.java@@ -135,4 +135,98 @@ class LambdaConcurrencyTest {.build())).isInstanceOf(ResourceNotFoundException.class);}++ @Test+ @Order(9)+ void putFunctionConcurrency_exceedsAccountUnreservedMin_throwsLimitExceeded() {+ // Floci default: regionLimit=1000, unreservedMin=100 → max single Put = 900+ // The Lambda SDK v2 model does not declare LimitExceededException as a+ // dedicated subclass, so the SDK surfaces it as the generic+ // LambdaException. We therefore assert the wire-level identity+ // (status code + __type error code) rather than a Java type, which is+ // what AWS clients actually discriminate on.+ assertThatThrownBy(() -> lambda.putFunctionConcurrency(+ PutFunctionConcurrencyRequest.builder()+ .functionName(FUNCTION_NAME)+ .reservedConcurrentExecutions(901)+ .build()))+ .isInstanceOfSatisfying(LambdaException.class, ex -> {+ assertThat(ex.statusCode()).isEqualTo(400);+ assertThat(ex.awsErrorDetails().errorCode()).isEqualTo("LimitExceededException");+ assertThat(ex.getMessage()).contains("UnreservedConcurrentExecution");+ });+ }++ @Test+ @Order(10)+ void invoke_whenReservedZero_throwsTooManyRequests() {+ lambda.putFunctionConcurrency(PutFunctionConcurrencyRequest.builder()+ .functionName(FUNCTION_NAME)+ .reservedConcurrentExecutions(0)+ .build());++ // Event-type invoke still goes through the concurrency gate; reserved=0+ // should throttle every request regardless of invocation type.+ assertThatThrownBy(() -> lambda.invoke(InvokeRequest.builder()+ .functionName(FUNCTION_NAME)+ .invocationType(InvocationType.EVENT)+ .payload(SdkBytes.fromUtf8String("{}"))+ .build()))+ .isInstanceOf(TooManyRequestsException.class);++ // Clear so teardown and other tests are not affected+ lambda.deleteFunctionConcurrency(DeleteFunctionConcurrencyRequest.builder()+ .functionName(FUNCTION_NAME).build());+ }++ @Test+ @Order(11)+ void invoke_dryRunBypassesConcurrencyGate() {+ lambda.putFunctionConcurrency(PutFunctionConcurrencyRequest.builder()+ .functionName(FUNCTION_NAME)+ .reservedConcurrentExecutions(0)+ .build());++ // DryRun validates inputs without dispatching; it must not be throttled.+ InvokeResponse response = lambda.invoke(InvokeRequest.builder()+ .functionName(FUNCTION_NAME)+ .invocationType(InvocationType.DRY_RUN)+ .payload(SdkBytes.fromUtf8String("{}"))+ .build());++ assertThat(response.statusCode()).isEqualTo(204);++ lambda.deleteFunctionConcurrency(DeleteFunctionConcurrencyRequest.builder()+ .functionName(FUNCTION_NAME).build());+ }++ @Test+ @Order(12)+ void invoke_withVersionQualifier_stillHonorsReservedOnLatest() {+ // Regression guard: if a future change adds Qualifier routing that+ // resolves to a published version snapshot, the snapshot currently+ // has reservedConcurrentExecutions=null and would silently bypass a+ // reserved=0 on $LATEST. Today Floci ignores the qualifier and+ // routes the invoke to $LATEST, so reserved=0 must still throttle.+ // Keeping this test green after a qualifier-routing change will+ // require copying the reservation onto the snapshot (or keying the+ // limiter off the base ARN).+ lambda.publishVersion(PublishVersionRequest.builder()+ .functionName(FUNCTION_NAME).build());+ lambda.putFunctionConcurrency(PutFunctionConcurrencyRequest.builder()+ .functionName(FUNCTION_NAME)+ .reservedConcurrentExecutions(0)+ .build());++ assertThatThrownBy(() -> lambda.invoke(InvokeRequest.builder()+ .functionName(FUNCTION_NAME)+ .qualifier("1")+ .invocationType(InvocationType.EVENT)+ .payload(SdkBytes.fromUtf8String("{}"))+ .build()))+ .isInstanceOf(TooManyRequestsException.class);++ lambda.deleteFunctionConcurrency(DeleteFunctionConcurrencyRequest.builder()+ .functionName(FUNCTION_NAME).build());+ }}diff --git a/docs/configuration/application-yml.md b/docs/configuration/application-yml.mdindex a67087c63c..13167b7496 100644--- a/docs/configuration/application-yml.md+++ b/docs/configuration/application-yml.md@@ -99,6 +99,8 @@ floci:code-path: ./data/lambda-code # Where ZIP archives are storedpoll-interval-ms: 1000container-idle-timeout-seconds: 300 # Remove idle containers after this+ region-concurrency-limit: 1000 # Concurrent executions ceiling per region+ unreserved-concurrency-min: 100 # Minimum unreserved capacity PutFunctionConcurrency must leaveapigateway:enabled: truediff --git a/docs/services/lambda.md b/docs/services/lambda.mdindex 8ff55d5493..a1e0557019 100644--- a/docs/services/lambda.md+++ b/docs/services/lambda.md@@ -39,14 +39,28 @@ Lambda runs your function code inside real Docker containers — the same way re| `ListTags` | List tags on a function || `TagResource` | Tag a function || `UntagResource` | Untag a function |-| `PutFunctionConcurrency` | Set reserved concurrent executions (stub — value stored but not enforced) |+| `PutFunctionConcurrency` | Set reserved concurrent executions || `GetFunctionConcurrency` | Get reserved concurrent executions || `DeleteFunctionConcurrency` | Clear reserved concurrent executions |-!!! note "Concurrency is a stub"- `PutFunctionConcurrency` persists `ReservedConcurrentExecutions` on the function- and echoes it back, but Floci does not enforce the limit at invocation time and- does not validate against the account-level concurrent execution limit.+!!! note "Concurrency enforcement"+ Reserved concurrency is enforced: invocations beyond the reserved value+ return `TooManyRequestsException` (HTTP 429). Functions without a reserved+ value share a **per-region** pool — AWS Lambda's "account-level" limit is+ in fact a per-account-per-region quota, and Floci mirrors that by+ partitioning counters on the ARN's region segment. The pool size (default+ 1000) is configurable via `floci.services.lambda.region-concurrency-limit`+ and applies independently to each region. `PutFunctionConcurrency`+ validates that the requested value leaves at least+ `floci.services.lambda.unreserved-concurrency-min` (default 100) available+ for unreserved functions in that region. `PutProvisionedConcurrencyConfig`+ and related provisioned-concurrency operations remain unimplemented.++ Reducing or clearing a function's reserved value does not kill+ invocations that are already in flight — this matches AWS, which+ applies changes only to new invocations. As a consequence, during the+ drain window `Σreserved-inflight + unreserved-inflight` can briefly+ exceed `region-concurrency-limit`.Function URLs are also reachable directly on `/{proxy:.*}` under the Lambda URL controller, which routes the request into the normal `Invoke` path.@@ -80,6 +94,8 @@ floci:code-path: ./data/lambda-code # ZIP storage locationpoll-interval-ms: 1000container-idle-timeout-seconds: 300 # Idle container cleanup+ region-concurrency-limit: 1000 # Concurrent executions ceiling per region+ unreserved-concurrency-min: 100 # Min unreserved capacity PutFunctionConcurrency must leave```### Docker socket requirementdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex ee8e080f6a..c8c381c3dc 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -509,6 +509,22 @@ public interface EmulatorConfig {/** Docker network to attach Lambda containers to. Empty = default bridge. */Optional<String> dockerNetwork();++ /**+ * Concurrent executions ceiling applied per region. AWS Lambda's+ * "account-level" concurrency is in fact a per-region quota (default 1000);+ * Floci mirrors that semantics and partitions counters by the region+ * segment of each function ARN.+ */+ @WithDefault("1000")+ int regionConcurrencyLimit();++ /**+ * Minimum unreserved concurrency that must remain after PutFunctionConcurrency,+ * matching AWS (100). Puts that would leave less than this are rejected.+ */+ @WithDefault("100")+ int unreservedConcurrencyMin();}interface Ec2ServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.javaindex 60c0898150..73cfba498d 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePoller.java@@ -1,10 +1,12 @@package io.github.hectorvent.floci.services.lambda;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.dynamodb.DynamoDbStreamService;import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;@@ -129,7 +131,17 @@ public class DynamoDbStreamsEventSourcePoller {esm.getUuid(), records.size(), esm.getFunctionName());String eventJson = buildDynamoDbEvent(records, esm);- var invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);+ InvokeResult invokeResult;+ try {+ invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);+ } catch (AwsException e) {+ if ("TooManyRequestsException".equals(e.getErrorCode())) {+ LOG.infov("DynamoDB Streams ESM {0}: function {1} throttled, shard iterator not advanced",+ esm.getUuid(), fn.getFunctionName());+ return;+ }+ throw e;+ }if (invokeResult.getFunctionError() == null) {String newestSeq = records.get(records.size() - 1).getSequenceNumber();diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.javaindex 7dcb4fb271..3d6c29573b 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/KinesisEventSourcePoller.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.lambda;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.kinesis.KinesisService;import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;@@ -118,8 +119,18 @@ public class KinesisEventSourcePoller {if (!records.isEmpty()) {String eventJson = buildKinesisEvent(records, esm, shard.getShardId());- InvokeResult invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);-+ InvokeResult invokeResult;+ try {+ invokeResult = executorService.invoke(fn, eventJson.getBytes(), InvocationType.RequestResponse);+ } catch (AwsException e) {+ if ("TooManyRequestsException".equals(e.getErrorCode())) {+ LOG.infov("Kinesis ESM {0}: function {1} throttled, shard iterator not advanced",+ esm.getUuid(), fn.getFunctionName());+ continue;+ }+ throw e;+ }+if (invokeResult.getFunctionError() == null) {String newestSeq = records.get(records.size() - 1).getSequenceNumber();esm.getShardSequenceNumbers().put(shard.getShardId(), newestSeq);diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.javaindex 76d3964d5c..72313ecd61 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyController.java@@ -30,8 +30,9 @@ import java.util.Map;* The class-level {@code @Path("/")} lets each method declare its own absolute* version prefix rather than inheriting a single one.*- * The stored value is not enforced at invocation time — this is a stub so that- * tools and SDKs expecting the endpoint to exist can proceed.+ * The stored value is enforced at invocation time by+ * {@link LambdaConcurrencyLimiter}; Put also validates against the per-region+ * unreserved-minimum.*/@Path("/")@Produces(MediaType.APPLICATION_JSON)diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiter.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiter.javanew file mode 100644index 0000000000..63d9809c7b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiter.java@@ -0,0 +1,330 @@+package io.github.hectorvent.floci.services.lambda;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.util.Collections;+import java.util.Objects;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicInteger;++/**+ * Enforces Lambda concurrency limits at invocation time.+ *+ * <p>AWS Lambda concurrency is scoped to an account <b>per region</b>; a+ * function's reserved value does not compete with functions in other regions.+ * Accordingly this limiter partitions its state by region (extracted from the+ * function ARN), and the configured {@code regionLimit}/{@code unreservedMin}+ * apply independently to each region.+ *+ * <p>Two layers of enforcement:+ * <ul>+ * <li><b>Reserved (per-function)</b>: when a function has a reserved value,+ * inflight invocations are counted against that value and do not consume+ * the region's unreserved pool.</li>+ * <li><b>Unreserved (region-shared)</b>: functions without a reserved value+ * share {@code regionLimit - Σreserved} permits within their region.</li>+ * </ul>+ */+@ApplicationScoped+public class LambdaConcurrencyLimiter {++ private static final Logger LOG = Logger.getLogger(LambdaConcurrencyLimiter.class);+ /** Tracks malformed ARNs already logged so the warn fires once per unique input. */+ private static final Set<String> LOGGED_MALFORMED_ARNS =+ Collections.newSetFromMap(new ConcurrentHashMap<>());++ /**+ * Inflight counts per function ARN (globally unique). Entries are+ * retained even when the count drops to zero — see {@link #reset} for+ * the race this avoids. The map therefore grows by one entry per+ * distinct ARN the limiter has ever seen, including entries left over+ * from functions that have been deleted and recreated under a new+ * name. For a local emulator the resulting footprint is negligible.+ */+ private final ConcurrentHashMap<String, AtomicInteger> inflight = new ConcurrentHashMap<>();+ /** Reserved values partitioned by region. */+ private final ConcurrentHashMap<String, ConcurrentHashMap<String, Integer>> reservedByRegion+ = new ConcurrentHashMap<>();+ /**+ * Running sum of {@link #reservedByRegion} values per region. Maintained+ * under {@link #reservedLock} on every mutation so that unreserved+ * acquisition can read a consistent cap in O(1).+ */+ private final ConcurrentHashMap<String, AtomicInteger> regionReservedTotal = new ConcurrentHashMap<>();+ /** Unreserved inflight counters partitioned by region. */+ private final ConcurrentHashMap<String, AtomicInteger> unreservedByRegion = new ConcurrentHashMap<>();+ /** Guards atomic validate-then-set and rollback operations on the reserved state. */+ private final Object reservedLock = new Object();+ private final int regionLimit;+ private final int unreservedMin;++ @Inject+ public LambdaConcurrencyLimiter(EmulatorConfig config) {+ this(config.services().lambda().regionConcurrencyLimit(),+ config.services().lambda().unreservedConcurrencyMin());+ }++ /** Test-only constructor with explicit limits. */+ LambdaConcurrencyLimiter(int regionLimit, int unreservedMin) {+ this.regionLimit = regionLimit;+ this.unreservedMin = unreservedMin;+ }++ /** Test-only no-arg constructor using AWS defaults (1000 / 100). */+ LambdaConcurrencyLimiter() {+ this(1000, 100);+ }++ public Permit acquire(LambdaFunction fn) {+ Integer r = fn.getReservedConcurrentExecutions();+ String region = regionOf(fn.getFunctionArn());+ if (r == null) {+ return acquireUnreserved(region);+ }+ return acquireReserved(fn.getFunctionArn(), r);+ }++ private Permit acquireReserved(String key, int limit) {+ AtomicInteger counter = inflight.computeIfAbsent(key, k -> new AtomicInteger());+ while (true) {+ int current = counter.get();+ if (current >= limit) {+ throw throttle();+ }+ if (counter.compareAndSet(current, current + 1)) {+ return idempotentPermit(counter);+ }+ }+ }++ private Permit acquireUnreserved(String region) {+ AtomicInteger counter = unreservedByRegion.computeIfAbsent(region, k -> new AtomicInteger());+ AtomicInteger reservedTotal = regionTotal(region);+ while (true) {+ int current = counter.get();+ // reservedTotal is an AtomicInteger updated under reservedLock on+ // each reserved change, so the cap is consistent and computed in+ // O(1) regardless of the number of reserved functions.+ int cap = Math.max(0, regionLimit - reservedTotal.get());+ if (current >= cap) {+ throw throttle();+ }+ if (counter.compareAndSet(current, current + 1)) {+ return idempotentPermit(counter);+ }+ }+ }++ /**+ * Wraps {@code counter.decrementAndGet()} in a close-once guard so a+ * future caller that accidentally double-closes a permit cannot drive+ * the inflight counter negative.+ */+ private static Permit idempotentPermit(AtomicInteger counter) {+ AtomicBoolean closed = new AtomicBoolean(false);+ return () -> {+ if (closed.compareAndSet(false, true)) {+ counter.decrementAndGet();+ }+ };+ }++ /**+ * Register (or update) a function's reserved value without validation.+ * Intended for startup rehydration from persisted state.+ *+ * @return the previous value, or {@code null} if none was set.+ */+ public Integer setReserved(String functionArn, int value) {+ synchronized (reservedLock) {+ return writeReserved(functionArn, value);+ }+ }++ /**+ * @return the cleared value, or {@code null} if no reservation was set.+ */+ public Integer clearReserved(String functionArn) {+ synchronized (reservedLock) {+ return writeReserved(functionArn, null);+ }+ }++ /**+ * Atomically validates and applies a reserved value within the function's+ * region. Two concurrent Puts for different functions cannot each pass+ * validation against stale totals and then collectively push the region's+ * unreserved capacity below the minimum.+ *+ * @return the previous reserved value for this ARN, or {@code null} if none;+ * callers may use it with {@link #rollbackReservedIfExpected} on a+ * subsequent persistence failure.+ * @throws AwsException {@code LimitExceededException} if the value would+ * drop unreserved below the minimum.+ */+ public Integer validateAndSetReserved(String functionArn, int target) {+ String region = regionOf(functionArn);+ synchronized (reservedLock) {+ ConcurrentHashMap<String, Integer> regionReserved = reservedOf(region);+ int currentForThis = regionReserved.getOrDefault(functionArn, 0);+ // A reduction (or no-op) cannot decrease unreserved capacity any+ // further than it already is, so always allow it. This lets+ // operators recover from an over-committed state — e.g. after+ // lowering region-concurrency-limit at runtime — without first+ // having to delete every reservation.+ if (target > currentForThis) {+ int otherReserved = regionTotal(region).get() - currentForThis;+ int maxAllowed = regionLimit - unreservedMin - otherReserved;+ if (target > maxAllowed) {+ throw new AwsException("LimitExceededException",+ "Specified ReservedConcurrentExecutions for function decreases account's "+ + "UnreservedConcurrentExecution below its minimum value of ["+ + unreservedMin + "].", 400);+ }+ }+ return writeReserved(functionArn, target);+ }+ }++ /**+ * Conditionally restores a prior reserved value if the current value still+ * matches {@code expectedCurrent}. Used by callers that updated the limiter+ * before a persistence step and need to undo the change on failure without+ * clobbering a concurrent successful write.+ */+ public void rollbackReservedIfExpected(String functionArn,+ Integer expectedCurrent,+ Integer rollbackTo) {+ synchronized (reservedLock) {+ Integer actual = reservedOf(regionOf(functionArn)).get(functionArn);+ if (!Objects.equals(actual, expectedCurrent)) {+ // A newer update has superseded ours; leave it alone.+ return;+ }+ writeReserved(functionArn, rollbackTo);+ }+ }++ /**+ * Clears the reserved entry for a deleted function. The inflight counter+ * is intentionally retained — permits held by still-running invocations+ * decrement into it on close, and an ARN recreated later reuses the same+ * counter so new invocations correctly see any remaining inflight.+ *+ * <p>The counter is also retained even when it momentarily reads zero:+ * conditionally removing it would race with a concurrent+ * {@code acquireReserved} that has already obtained the {@code AtomicInteger}+ * reference and is about to increment it. After such a race the next+ * acquire would allocate a fresh counter and undercount the inflight+ * permit, allowing reserved over-subscription. The trade-off is one+ * {@code AtomicInteger} per historical function, which is bounded for an+ * emulator workload.+ */+ public void reset(String functionArn) {+ synchronized (reservedLock) {+ writeReserved(functionArn, null);+ }+ }++ public int totalReserved(String region) {+ return regionTotal(region).get();+ }++ public int availableUnreserved(String region) {+ AtomicInteger counter = unreservedByRegion.get(region);+ int inflightNow = counter == null ? 0 : counter.get();+ return Math.max(0, regionLimit - totalReserved(region) - inflightNow);+ }++ int inflightCount(String functionArn) {+ AtomicInteger counter = inflight.get(functionArn);+ return counter == null ? 0 : counter.get();+ }++ int unreservedInflightCount(String region) {+ AtomicInteger counter = unreservedByRegion.get(region);+ return counter == null ? 0 : counter.get();+ }++ /**+ * Must be called with {@link #reservedLock} held. Updates the per-function+ * reserved entry and the region's running total in one atomic step from the+ * perspective of any code that also holds the lock.+ */+ private Integer writeReserved(String functionArn, Integer newValue) {+ String region = regionOf(functionArn);+ ConcurrentHashMap<String, Integer> regionReserved = reservedOf(region);+ Integer previous = newValue == null+ ? regionReserved.remove(functionArn)+ : regionReserved.put(functionArn, newValue);+ int delta = (newValue == null ? 0 : newValue) - (previous == null ? 0 : previous);+ if (delta != 0) {+ regionTotal(region).addAndGet(delta);+ }+ return previous;+ }++ private ConcurrentHashMap<String, Integer> reservedOf(String region) {+ return reservedByRegion.computeIfAbsent(region, k -> new ConcurrentHashMap<>());+ }++ private AtomicInteger regionTotal(String region) {+ return regionReservedTotal.computeIfAbsent(region, k -> new AtomicInteger());+ }++ /**+ * Extracts the region segment from a Lambda function ARN. Falls back to+ * {@code "unknown"} if the ARN is malformed so state still partitions+ * cleanly rather than mixing with another region's data.+ *+ * <p>This is called on every acquire/release, so the parse avoids the+ * regex machinery and array allocation of {@code String.split(":")} by+ * scanning for the fourth ':' segment (index 3 in an ARN:+ * {@code arn:aws:lambda:REGION:account:function:name}).+ */+ private static String regionOf(String arn) {+ if (arn == null) {+ warnMalformed("<null>");+ return "unknown";+ }+ int segmentStart = 0;+ int delimiters = 0;+ for (int i = 0; i < arn.length(); i++) {+ if (arn.charAt(i) == ':') {+ if (delimiters == 2) {+ segmentStart = i + 1;+ } else if (delimiters == 3) {+ return arn.substring(segmentStart, i);+ }+ delimiters++;+ }+ }+ warnMalformed(arn);+ return "unknown";+ }++ private static void warnMalformed(String arn) {+ if (LOGGED_MALFORMED_ARNS.add(arn)) {+ LOG.warnv("Concurrency limiter received non-ARN function identifier "+ + "\"{0}\"; bucketing under region=\"unknown\". This likely "+ + "indicates a bare function name reached the limiter.", arn);+ }+ }++ private static AwsException throttle() {+ return new AwsException("TooManyRequestsException", "Rate Exceeded.", 429);+ }++ @FunctionalInterface+ public interface Permit extends AutoCloseable {+ @Override+ void close();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.javaindex 919e19fd4d..20d49634cf 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaExecutorService.java@@ -32,6 +32,7 @@ public class LambdaExecutorService {private final WarmPool warmPool;private final ObjectMapper objectMapper;+ private final LambdaConcurrencyLimiter concurrencyLimiter;private final ExecutorService asyncExecutor = new ThreadPoolExecutor(Math.max(4, Runtime.getRuntime().availableProcessors() * 2),Math.max(8, Runtime.getRuntime().availableProcessors() * 4),@@ -40,25 +41,43 @@ public class LambdaExecutorService {new ThreadPoolExecutor.CallerRunsPolicy());@Inject- public LambdaExecutorService(WarmPool warmPool, ObjectMapper objectMapper) {+ public LambdaExecutorService(WarmPool warmPool,+ ObjectMapper objectMapper,+ LambdaConcurrencyLimiter concurrencyLimiter) {this.warmPool = warmPool;this.objectMapper = objectMapper;+ this.concurrencyLimiter = concurrencyLimiter;}public InvokeResult invoke(LambdaFunction fn, byte[] payload, InvocationType type) {String requestId = UUID.randomUUID().toString();- switch (type) {- case DryRun -> {- return new InvokeResult(204, null, new byte[0], null, requestId);- }- case Event -> {- asyncExecutor.submit(() -> executeSync(fn, payload, requestId));- return new InvokeResult(202, null, new byte[0], null, requestId);- }- default -> {- return executeSync(fn, payload, requestId);+ if (type == InvocationType.DryRun) {+ return new InvokeResult(204, null, new byte[0], null, requestId);+ }++ LambdaConcurrencyLimiter.Permit permit = concurrencyLimiter.acquire(fn);++ if (type == InvocationType.Event) {+ try {+ asyncExecutor.submit(() -> {+ try {+ executeSync(fn, payload, requestId);+ } finally {+ permit.close();+ }+ });+ } catch (RuntimeException e) {+ permit.close();+ throw e;}+ return new InvokeResult(202, null, new byte[0], null, requestId);+ }++ try {+ return executeSync(fn, payload, requestId);+ } finally {+ permit.close();}}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.javaindex 495950824c..7a13282e6b 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/LambdaService.java@@ -15,6 +15,7 @@ import io.github.hectorvent.floci.services.lambda.zip.ZipExtractor;import io.github.hectorvent.floci.services.s3.S3Service;import io.github.hectorvent.floci.services.s3.model.S3Object;import io.github.hectorvent.floci.services.sqs.SqsService;+import jakarta.annotation.PostConstruct;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -43,6 +44,7 @@ public class LambdaService {private final LambdaFunctionStore functionStore;private final LambdaExecutorService executorService;+ private final LambdaConcurrencyLimiter concurrencyLimiter;private final WarmPool warmPool;private final CodeStore codeStore;private final ZipExtractor zipExtractor;@@ -56,8 +58,28 @@ public class LambdaService {private final KinesisEventSourcePoller kinesisPoller;private final DynamoDbStreamsEventSourcePoller dynamodbStreamsPoller;private final ConcurrentHashMap<String, Integer> versionCounters = new ConcurrentHashMap<>();-- /** Package-private constructor for testing without CDI. Config defaults (timeout=3, memory=128) apply. */+ /**+ * Per-function locks covering PutFunctionConcurrency,+ * DeleteFunctionConcurrency, and deleteFunction itself. Serializing the+ * limiter update + persistence pair against itself for a given function+ * prevents the limiter and store from diverging on interleaved concurrent+ * requests.+ *+ * <p>Entries are intentionally never removed — see {@code deleteFunction}+ * for the race this avoids. The map therefore grows by one {@code Object}+ * per distinct function ARN the emulator has ever seen (create/delete+ * cycles with fresh names included). Acceptable footprint for a local+ * emulator workload.+ */+ private final ConcurrentHashMap<String, Object> concurrencyOpLocks = new ConcurrentHashMap<>();++ /**+ * Package-private constructor for testing without CDI. Config defaults+ * (timeout=3, memory=128) apply. A real {@link LambdaConcurrencyLimiter}+ * with AWS-default limits is wired so concurrency operations exercise+ * the same validation and bookkeeping as production rather than+ * silently no-op'ing past null checks.+ */LambdaService(LambdaFunctionStore functionStore,WarmPool warmPool,CodeStore codeStore,@@ -65,6 +87,7 @@ public class LambdaService {RegionResolver regionResolver) {this.functionStore = functionStore;this.executorService = null;+ this.concurrencyLimiter = new LambdaConcurrencyLimiter();this.warmPool = warmPool;this.codeStore = codeStore;this.zipExtractor = zipExtractor;@@ -82,6 +105,7 @@ public class LambdaService {@Injectpublic LambdaService(LambdaFunctionStore functionStore,LambdaExecutorService executorService,+ LambdaConcurrencyLimiter concurrencyLimiter,WarmPool warmPool,CodeStore codeStore,ZipExtractor zipExtractor,@@ -96,6 +120,7 @@ public class LambdaService {DynamoDbStreamsEventSourcePoller dynamodbStreamsPoller) {this.functionStore = functionStore;this.executorService = executorService;+ this.concurrencyLimiter = concurrencyLimiter;this.warmPool = warmPool;this.codeStore = codeStore;this.zipExtractor = zipExtractor;@@ -110,6 +135,40 @@ public class LambdaService {this.dynamodbStreamsPoller = dynamodbStreamsPoller;}+ /** Package-private accessor for tests that want to assert limiter state directly. */+ LambdaConcurrencyLimiter concurrencyLimiter() {+ return concurrencyLimiter;+ }++ /**+ * Rehydrates reserved concurrency into the limiter from persisted function state.+ * Without this, restarts leave {@code totalReserved()=0} and allow validatePut /+ * unreserved-pool sizing to drift until each function is re-Put.+ */+ @PostConstruct+ void rehydrateConcurrency() {+ if (concurrencyLimiter == null) {+ return;+ }+ int count = 0;+ for (LambdaFunction fn : functionStore.listAll()) {+ // Reserved concurrency is a function-level property; published+ // versions share the $LATEST record's value. Skip non-$LATEST+ // entries to avoid double-counting into totalReserved().+ if (!"$LATEST".equals(fn.getVersion())) {+ continue;+ }+ Integer reserved = fn.getReservedConcurrentExecutions();+ if (reserved != null) {+ concurrencyLimiter.setReserved(fn.getFunctionArn(), reserved);+ count++;+ }+ }+ if (count > 0) {+ LOG.infov("Restored reserved concurrency for {0} function(s)", count);+ }+ }+public LambdaFunction createFunction(String region, Map<String, Object> request) {String functionName = (String) request.get("FunctionName");String role = (String) request.get("Role");@@ -230,10 +289,23 @@ public class LambdaService {}public void deleteFunction(String region, String functionName) {- getFunction(region, functionName); // throws 404 if not found+ LambdaFunction fn = getFunction(region, functionName); // throws 404 if not found+ String arn = fn.getFunctionArn();warmPool.drainFunction(functionName);- codeStore.delete(functionName);- functionStore.delete(region, functionName);+ // Take the same per-function lock used by Put/DeleteFunctionConcurrency+ // so a concurrent concurrency mutation cannot interleave with the+ // limiter reset and store delete and leave the two views out of sync.+ // The lock entry itself stays in the map after the delete: removing it+ // could race with another thread already synchronized on the same+ // object, letting a follow-up request allocate a fresh lock and run+ // in parallel — the very serialization this map exists to prevent.+ synchronized (lockForConcurrencyOp(arn)) {+ if (concurrencyLimiter != null) {+ concurrencyLimiter.reset(arn);+ }+ codeStore.delete(functionName);+ functionStore.delete(region, functionName);+ }LOG.infov("Deleted Lambda function: {0}", functionName);}@@ -597,8 +669,29 @@ public class LambdaService {"ReservedConcurrentExecutions must be a non-negative integer", 400);}LambdaFunction fn = getFunction(region, functionName);- fn.setReservedConcurrentExecutions(reservedConcurrentExecutions);- functionStore.save(region, fn);+ String arn = fn.getFunctionArn();+ // Serialize limiter update + store save for this function so that two+ // concurrent Puts cannot leave the limiter and persisted state out of+ // sync, regardless of which call acquires the reservedLock first.+ synchronized (lockForConcurrencyOp(arn)) {+ Integer previousReserved = null;+ boolean limiterUpdated = false;+ if (concurrencyLimiter != null) {+ previousReserved = concurrencyLimiter.validateAndSetReserved(+ arn, reservedConcurrentExecutions);+ limiterUpdated = true;+ }+ fn.setReservedConcurrentExecutions(reservedConcurrentExecutions);+ try {+ functionStore.save(region, fn);+ } catch (RuntimeException e) {+ if (limiterUpdated) {+ concurrencyLimiter.rollbackReservedIfExpected(+ arn, reservedConcurrentExecutions, previousReserved);+ }+ throw e;+ }+ }return fn;}@@ -609,8 +702,29 @@ public class LambdaService {public void deleteFunctionConcurrency(String region, String functionName) {LambdaFunction fn = getFunction(region, functionName);- fn.setReservedConcurrentExecutions(null);- functionStore.save(region, fn);+ String arn = fn.getFunctionArn();+ synchronized (lockForConcurrencyOp(arn)) {+ Integer previousReserved = null;+ boolean limiterCleared = false;+ if (concurrencyLimiter != null) {+ previousReserved = concurrencyLimiter.clearReserved(arn);+ limiterCleared = true;+ }+ fn.setReservedConcurrentExecutions(null);+ try {+ functionStore.save(region, fn);+ } catch (RuntimeException e) {+ if (limiterCleared && previousReserved != null) {+ concurrencyLimiter.rollbackReservedIfExpected(+ arn, null, previousReserved);+ }+ throw e;+ }+ }+ }++ private Object lockForConcurrencyOp(String functionArn) {+ return concurrencyOpLocks.computeIfAbsent(functionArn, k -> new Object());}public LambdaFunction getFunctionByUrlId(String urlId) {diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.javaindex 35480db0de..44e7863a07 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/SqsEventSourcePoller.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.services.lambda.model.EventSourceMapping;import io.github.hectorvent.floci.services.lambda.model.InvocationType;import io.github.hectorvent.floci.services.lambda.model.InvokeResult;@@ -146,8 +147,18 @@ public class SqsEventSourcePoller {String eventJson = buildSqsEvent(messages, esm);LOG.infov("ESM {0}: invoking function {1}", esm.getUuid(), fn.getFunctionName());- InvokeResult result = executorService.invoke(- fn, eventJson.getBytes(), InvocationType.RequestResponse);+ InvokeResult result;+ try {+ result = executorService.invoke(+ fn, eventJson.getBytes(), InvocationType.RequestResponse);+ } catch (AwsException e) {+ if ("TooManyRequestsException".equals(e.getErrorCode())) {+ LOG.infov("ESM {0}: function {1} throttled, messages will return to queue after visibility timeout",+ esm.getUuid(), fn.getFunctionName());+ return;+ }+ throw e;+ }if (result.getFunctionError() == null) {Set<String> failedIds = extractBatchItemFailures(esm, result);diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 0875ed7fbe..8b2eb57ac5 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -80,6 +80,8 @@ floci:code-path: ./data/lambda-codepoll-interval-ms: 1000container-idle-timeout-seconds: 300+ region-concurrency-limit: 1000+ unreserved-concurrency-min: 100apigateway:enabled: trueapigatewayv2:diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiterTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiterTest.javanew file mode 100644index 0000000000..1bade6233e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaConcurrencyLimiterTest.java@@ -0,0 +1,261 @@+package io.github.hectorvent.floci.services.lambda;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;+import org.junit.jupiter.api.Test;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;++class LambdaConcurrencyLimiterTest {++ private static final String REGION = "us-east-1";+ private static final String OTHER_REGION = "ap-northeast-1";+ private static final String ARN = "arn:aws:lambda:us-east-1:000000000000:function:fn";+ private static final String ARN2 = "arn:aws:lambda:us-east-1:000000000000:function:other";+ private static final String ARN_OTHER_REGION = "arn:aws:lambda:ap-northeast-1:000000000000:function:fn-apne1";++ private LambdaFunction fn(String arn, Integer reserved) {+ LambdaFunction fn = new LambdaFunction();+ fn.setFunctionName("fn");+ fn.setFunctionArn(arn);+ fn.setReservedConcurrentExecutions(reserved);+ return fn;+ }++ private LambdaFunction fn(Integer reserved) {+ return fn(ARN, reserved);+ }++ @Test+ void unsetReserved_countsAgainstAccountPool() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(2, 0);+ LambdaConcurrencyLimiter.Permit p1 = limiter.acquire(fn(null));+ LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(fn(null));+ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(fn(null)));+ assertEquals(429, ex.getHttpStatus());+ p1.close();+ p2.close();+ assertEquals(0, limiter.unreservedInflightCount(REGION));+ }++ @Test+ void reservedN_allowsUpToN_thenThrows() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ LambdaFunction f = fn(2);+ LambdaConcurrencyLimiter.Permit p1 = limiter.acquire(f);+ LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(f);++ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(f));+ assertEquals("TooManyRequestsException", ex.getErrorCode());+ assertEquals(429, ex.getHttpStatus());++ p1.close();+ LambdaConcurrencyLimiter.Permit p3 = limiter.acquire(f);+ p2.close();+ p3.close();+ assertEquals(0, limiter.inflightCount(ARN));+ }++ @Test+ void reservedZero_throwsImmediately() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(fn(0)));+ assertEquals(429, ex.getHttpStatus());+ }++ @Test+ void reservedPool_doesNotConsumeUnreserved() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(3, 0);+ limiter.setReserved(ARN, 2);+ // Reserved function consumes its own pool, not the region pool+ try (LambdaConcurrencyLimiter.Permit p1 = limiter.acquire(fn(ARN, 2));+ LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(fn(ARN, 2));+ // Unreserved function can still use the full regionLimit - totalReserved = 1+ LambdaConcurrencyLimiter.Permit p3 = limiter.acquire(fn(ARN2, null))) {+ AwsException ex = assertThrows(AwsException.class, () -> limiter.acquire(fn(ARN2, null)));+ assertEquals(429, ex.getHttpStatus());+ }+ }++ @Test+ void reset_preservesInflightCounterWhenBusy() {+ // If a function is deleted while invocations are still running, and the+ // same ARN is recreated, new invocations must see the remaining inflight+ // so we don't transiently over-subscribe.+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ LambdaFunction before = fn(3);+ LambdaConcurrencyLimiter.Permit held = limiter.acquire(before);+ assertEquals(1, limiter.inflightCount(ARN));++ limiter.reset(ARN);+ assertEquals(1, limiter.inflightCount(ARN), "inflight retained while permit is open");++ LambdaFunction recreated = fn(3);+ try (LambdaConcurrencyLimiter.Permit p2 = limiter.acquire(recreated)) {+ assertEquals(2, limiter.inflightCount(ARN));+ }+ held.close();+ assertEquals(0, limiter.inflightCount(ARN));+ }++ @Test+ void reset_keepsIdleInflightCounterToAvoidUndercount() {+ // The counter is intentionally retained across reset to close the+ // window where a concurrent acquire could otherwise allocate a fresh+ // counter and undercount inflight permits already in flight.+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ try (LambdaConcurrencyLimiter.Permit p = limiter.acquire(fn(1))) {+ assertEquals(1, limiter.inflightCount(ARN));+ }+ assertEquals(0, limiter.inflightCount(ARN));+ limiter.reset(ARN);+ // Counter still present at zero; new acquires share it.+ assertEquals(0, limiter.inflightCount(ARN));+ try (LambdaConcurrencyLimiter.Permit p = limiter.acquire(fn(1))) {+ assertEquals(1, limiter.inflightCount(ARN));+ }+ }++ @Test+ void validateAndSetReserved_allowsReductionEvenWhenOverCommitted() {+ // Simulate an over-committed state: total reserved exceeds the+ // unreserved minimum's ceiling. (e.g. region-concurrency-limit was+ // lowered at runtime, or state was migrated from earlier behavior.)+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);+ // Bypass validation by using setReserved directly so we can engineer+ // the broken state that validateAndSetReserved should still let us recover from.+ limiter.setReserved(ARN, 950);+ // Now totalReserved=950, unreserved capacity = 50 < min(100). Any+ // increase should still be blocked.+ assertThrows(AwsException.class, () -> limiter.validateAndSetReserved(ARN, 951));+ // But a reduction must succeed so the operator can recover.+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 500));+ assertEquals(500, limiter.totalReserved(REGION));+ }++ @Test+ void validatePut_rejectsWhenUnreservedMinViolated() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);+ // totalReserved=0, max allowed for new function = 1000 - 100 = 900+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 900));+ AwsException ex = assertThrows(AwsException.class, () -> limiter.validateAndSetReserved(ARN, 901));+ assertEquals("LimitExceededException", ex.getErrorCode());+ assertEquals(400, ex.getHttpStatus());+ }++ @Test+ void validatePut_excludesSelfWhenUpdating() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);+ limiter.setReserved(ARN, 500);+ // Updating the same ARN to 900 should succeed (self is excluded from "other")+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 900));+ }++ @Test+ void validatePut_considersOtherFunctions() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);+ limiter.setReserved(ARN2, 500);+ // otherReserved=500, max for ARN = 1000 - 100 - 500 = 400+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN, 400));+ assertThrows(AwsException.class, () -> limiter.validateAndSetReserved(ARN, 401));+ }++ @Test+ void reset_clearsReservedEntry() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ limiter.setReserved(ARN, 1);+ limiter.reset(ARN);+ assertEquals(0, limiter.totalReserved(REGION));+ }++ @Test+ void setReserved_returnsPreviousValue() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ assertNull(limiter.setReserved(ARN, 5));+ assertEquals(5, limiter.setReserved(ARN, 10));+ }++ @Test+ void clearReserved_returnsClearedValue() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ limiter.setReserved(ARN, 7);+ assertEquals(7, limiter.clearReserved(ARN));+ assertNull(limiter.clearReserved(ARN));+ }++ @Test+ void rollbackReservedIfExpected_restoresWhenUnchanged() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ limiter.setReserved(ARN, 5);+ limiter.setReserved(ARN, 10); // now at 10+ limiter.rollbackReservedIfExpected(ARN, 10, 5);+ assertEquals(5, limiter.totalReserved(REGION));+ }++ @Test+ void rollbackReservedIfExpected_skipsWhenConcurrentlyChanged() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ // Request A wrote 10 (previous null), then another request superseded to 20+ limiter.setReserved(ARN, 10);+ limiter.setReserved(ARN, 20);+ // A's rollback expects 10 still present — must not clobber 20+ limiter.rollbackReservedIfExpected(ARN, 10, null);+ assertEquals(20, limiter.totalReserved(REGION));+ }++ @Test+ void totalReserved_tracksOverlappingUpdates() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ limiter.setReserved(ARN, 50);+ limiter.setReserved(ARN2, 30);+ assertEquals(80, limiter.totalReserved(REGION));+ limiter.setReserved(ARN, 100); // +50+ assertEquals(130, limiter.totalReserved(REGION));+ limiter.clearReserved(ARN2); // -30+ assertEquals(100, limiter.totalReserved(REGION));+ }++ @Test+ void permit_closeIsIdempotent() {+ // Future callers must not be able to drive the inflight counter+ // negative by double-closing a permit (try-with-resources ++ // explicit close, retry logic, etc.).+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter();+ LambdaConcurrencyLimiter.Permit p = limiter.acquire(fn(3));+ assertEquals(1, limiter.inflightCount(ARN));+ p.close();+ assertEquals(0, limiter.inflightCount(ARN));+ p.close(); // second close must be a no-op+ assertEquals(0, limiter.inflightCount(ARN));+ p.close(); // ...and so must the third+ assertEquals(0, limiter.inflightCount(ARN));+ }++ @Test+ void regions_areIndependent() {+ LambdaConcurrencyLimiter limiter = new LambdaConcurrencyLimiter(1000, 100);+ // Fill one region's reserved pool near the limit+ limiter.validateAndSetReserved(ARN, 900);+ // Another region starts fresh — Put up to 900 still allowed+ assertDoesNotThrow(() -> limiter.validateAndSetReserved(ARN_OTHER_REGION, 900));+ assertEquals(900, limiter.totalReserved(REGION));+ assertEquals(900, limiter.totalReserved(OTHER_REGION));++ // Unreserved pool is also per-region+ LambdaConcurrencyLimiter small = new LambdaConcurrencyLimiter(1, 0);+ try (LambdaConcurrencyLimiter.Permit usEast = small.acquire(fn(ARN, null));+ LambdaConcurrencyLimiter.Permit apne1 = small.acquire(fn(ARN_OTHER_REGION, null))) {+ // Same exhaustion in us-east-1, but ap-northeast-1 has its own slot+ // (already consumed by apne1 above, so second acquire there also throws)+ assertThrows(AwsException.class, () -> small.acquire(fn(ARN2, null)));+ assertThrows(AwsException.class, () -> small.acquire(fn(ARN_OTHER_REGION, null)));+ }+ // After close, both regions have capacity again+ try (LambdaConcurrencyLimiter.Permit reacquire = small.acquire(fn(ARN, null))) {+ assertEquals(1, small.unreservedInflightCount(REGION));+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.javaindex 821a2eecde..9e83f4ebdc 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/LambdaServiceTest.java@@ -237,4 +237,195 @@ class LambdaServiceTest {LambdaFunction updated = service.updateFunctionCode(REGION, "update-fn", Map.of());assertNotEquals(originalRevision, updated.getRevisionId());}++ @Test+ void rehydrateConcurrency_restoresReservedFromStore() {+ // Simulate a persisted state: functions already live in the store+ // with reserved values before the limiter is populated.+ service.createFunction(REGION, baseRequest("persisted-a"));+ service.createFunction(REGION, baseRequest("persisted-b"));+ service.putFunctionConcurrency(REGION, "persisted-a", 300);+ service.putFunctionConcurrency(REGION, "persisted-b", 200);++ // Build a second service over the same store with a fresh limiter.+ LambdaFunctionStore store = new LambdaFunctionStore(new InMemoryStorage<>());+ // Copy the two persisted functions into the new store to emulate a+ // restart with the same disk state.+ for (LambdaFunction fn : new java.util.ArrayList<>(+ service.listFunctions(REGION))) {+ store.save(REGION, fn);+ }+ LambdaService rebooted = new LambdaService(store, new WarmPool(),+ new CodeStore(Path.of("target/test-data/lambda-code")),+ new ZipExtractor(), new RegionResolver(REGION, "000000000000"));++ // Starts empty…+ assertEquals(0, rebooted.concurrencyLimiter().totalReserved(REGION));+ // …until rehydrate walks the store and re-registers the reserved values.+ rebooted.rehydrateConcurrency();+ assertEquals(500, rebooted.concurrencyLimiter().totalReserved(REGION));+ }++ @Test+ void multiArnPutFunctionConcurrency_respectsRegionTotalUnderContention() throws Exception {+ // Two different functions racing a Put near the unreserved floor.+ // Both try to reserve an amount that — summed — would push the+ // region below unreserved-min. reservedLock must serialize so that+ // only one wins.+ service.createFunction(REGION, baseRequest("multi-a"));+ service.createFunction(REGION, baseRequest("multi-b"));++ // Defaults: regionLimit=1000, unreservedMin=100. Each Put asks for+ // 500; together they would leave 0 unreserved.+ java.util.concurrent.ExecutorService pool = java.util.concurrent.Executors.newFixedThreadPool(2);+ java.util.concurrent.CountDownLatch start = new java.util.concurrent.CountDownLatch(1);+ try {+ java.util.concurrent.Future<Throwable> fA = pool.submit(() -> {+ start.await();+ try {+ service.putFunctionConcurrency(REGION, "multi-a", 500);+ return null;+ } catch (Throwable t) { return t; }+ });+ java.util.concurrent.Future<Throwable> fB = pool.submit(() -> {+ start.await();+ try {+ service.putFunctionConcurrency(REGION, "multi-b", 500);+ return null;+ } catch (Throwable t) { return t; }+ });+ start.countDown();++ Throwable rA = fA.get();+ Throwable rB = fB.get();++ // Exactly one must have been rejected with LimitExceededException.+ int successes = (rA == null ? 1 : 0) + (rB == null ? 1 : 0);+ assertEquals(1, successes, "exactly one Put must win");+ Throwable rejected = rA != null ? rA : rB;+ assertTrue(rejected instanceof AwsException+ && "LimitExceededException".equals(((AwsException) rejected).getErrorCode()),+ "other Put must be rejected, got " + rejected);+ assertEquals(500,+ service.concurrencyLimiter().totalReserved(REGION),+ "limiter total must reflect only the winning Put");+ } finally {+ pool.shutdownNow();+ }+ }++ @Test+ void putFunctionConcurrency_rollsBackLimiterIfSaveFails() {+ // If functionStore.save throws after the limiter has been updated,+ // the limiter must be restored so Σreserved stays consistent with+ // what the store actually persisted.+ FailingStore failing = new FailingStore();+ LambdaService svc = new LambdaService(failing, new WarmPool(),+ new CodeStore(Path.of("target/test-data/lambda-code")),+ new ZipExtractor(), new RegionResolver(REGION, "000000000000"));+ svc.createFunction(REGION, baseRequest("rb-fn"));+ // Baseline: Put of 300 succeeds+ svc.putFunctionConcurrency(REGION, "rb-fn", 300);+ assertEquals(300, svc.concurrencyLimiter().totalReserved(REGION));++ // Now make the next save() explode and try to Put 500+ failing.shouldFail = true;+ assertThrows(RuntimeException.class,+ () -> svc.putFunctionConcurrency(REGION, "rb-fn", 500));++ // Limiter must have rolled back to the previous 300+ assertEquals(300, svc.concurrencyLimiter().totalReserved(REGION),+ "limiter must unwind on save failure");+ }++ @Test+ void deleteFunction_preservesInflightPermitUntilItCloses() {+ // A deleteFunction that lands while an invocation is still holding a+ // permit must not drop the counter — the permit close() at the end+ // of the running invocation still has to decrement something valid.+ service.createFunction(REGION, baseRequest("del-inflight"));+ service.putFunctionConcurrency(REGION, "del-inflight", 2);++ LambdaFunction fn = service.getFunction(REGION, "del-inflight");+ String arn = fn.getFunctionArn();+ LambdaConcurrencyLimiter.Permit held =+ service.concurrencyLimiter().acquire(fn);+ assertEquals(1, service.concurrencyLimiter().inflightCount(arn));++ service.deleteFunction(REGION, "del-inflight");++ // Reserved is cleared from the limiter, but the inflight counter+ // must still be live for the held permit to decrement into.+ assertEquals(0, service.concurrencyLimiter().totalReserved(REGION));+ assertEquals(1, service.concurrencyLimiter().inflightCount(arn),+ "inflight must survive delete until the permit closes");++ held.close();+ assertEquals(0, service.concurrencyLimiter().inflightCount(arn));+ }++ /**+ * Test helper: a LambdaFunctionStore whose save() throws on demand so+ * tests can exercise the LambdaService rollback path.+ */+ private static final class FailingStore extends LambdaFunctionStore {+ boolean shouldFail = false;+ FailingStore() {+ super(new InMemoryStorage<String, LambdaFunction>());+ }+ @Override+ public void save(String region, LambdaFunction fn) {+ if (shouldFail) {+ throw new RuntimeException("injected save failure");+ }+ super.save(region, fn);+ }+ }++ @Test+ void concurrentPutFunctionConcurrency_endsInConsistentState() throws Exception {+ // Exercise the per-function serialization in concurrencyOpLocks:+ // two threads racing Put on the same function must leave the+ // limiter and the persisted reserved value in agreement with+ // whichever write landed last.+ service.createFunction(REGION, baseRequest("race-fn"));++ int iterations = 50;+ java.util.concurrent.ExecutorService pool = java.util.concurrent.Executors.newFixedThreadPool(2);+ try {+ for (int i = 0; i < iterations; i++) {+ int a = 100 + i;+ int b = 200 + i;+ java.util.concurrent.CountDownLatch start = new java.util.concurrent.CountDownLatch(1);+ java.util.concurrent.Future<Integer> fA = pool.submit(() -> {+ start.await();+ return service.putFunctionConcurrency(REGION, "race-fn", a)+ .getReservedConcurrentExecutions();+ });+ java.util.concurrent.Future<Integer> fB = pool.submit(() -> {+ start.await();+ return service.putFunctionConcurrency(REGION, "race-fn", b)+ .getReservedConcurrentExecutions();+ });+ start.countDown();+ fA.get();+ fB.get();++ LambdaFunction fn = service.getFunction(REGION, "race-fn");+ Integer stored = fn.getReservedConcurrentExecutions();+ assertTrue(stored.equals(a) || stored.equals(b),+ "store should reflect one of the two writes, got " + stored);+ // The real invariant: the limiter's Σreserved for this+ // region must agree with what was persisted. Comparing+ // getFunctionConcurrency() to stored would be a tautology —+ // both read the same LambdaFunction field — so assert+ // against the limiter's independently-maintained total.+ assertEquals(stored.intValue(),+ service.concurrencyLimiter().totalReserved(REGION),+ "limiter totalReserved must match persisted reserved value");+ }+ } finally {+ pool.shutdownNow();+ }+ }}