Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudcontrol): bound the worker queue and reject overflow (#4479)
Daniël van Ziel517409a7aab1
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex 508f18ae4b..3d29354f04 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -254,6 +254,11 @@ floci:cloudformation:enabled: true+ cloudcontrol:+ enabled: true+ create-worker-threads: 4 # Threads that run CreateResource provisioning+ create-queue-capacity: 64 # Creates that may wait for a thread before ThrottlingException+acm:enabled: truevalidation-wait-seconds: 0 # Seconds before transitioning PENDING_VALIDATION → ISSUEDdiff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex 92ef333f3f..5cdc9cc858 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -358,6 +358,14 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s| `FLOCI_SERVICES_CLOUDFORMATION_ALLOW_STUB_LAMBDA_CODE` | `false` | Fall back to the built-in stub handler for an `AWS::Lambda::Function` whose S3 code cannot be read. Off matches real CloudFormation, which fails the resource and rolls the stack back || `FLOCI_SERVICES_CLOUDFORMATION_ALLOW_STUB_UNSUPPORTED_RESOURCE_TYPES` | `true` | Stub a resource whose type has no provisioner (synthetic physical ID, `arn:aws:stub:::` ARN attribute, `CREATE_COMPLETE`), logged at `WARN` with a resource status reason. Set `false` to fail the resource instead, which rolls the stack back |+### Cloud Control++| Variable | Default | Description |+|---|---|---|+| `FLOCI_SERVICES_CLOUDCONTROL_ENABLED` | `true` | Enable the Cloud Control API service |+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_WORKER_THREADS` | `4` | Threads that run `CreateResource` provisioning |+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_QUEUE_CAPACITY` | `64` | Creates that may wait for a free thread; past that a new `CreateResource` fails with `ThrottlingException` |+### ACM (Certificate Manager)| Variable | Default | Description |diff --git a/docs/services/cloudcontrol.md b/docs/services/cloudcontrol.mdindex 020cdecc1a..ab4ae18ad3 100644--- a/docs/services/cloudcontrol.md+++ b/docs/services/cloudcontrol.md@@ -26,6 +26,12 @@ Cloud Control.`GetResourceRequestStatus` with the token until the status is `SUCCESS` or `FAILED`.`DeleteResource` is synchronous (deletes are fast enough not to need polling) butstill returns a `ProgressEvent` for API-shape consistency.+- **Create backpressure**: creates run on 4 workers with room for 64 more queued, both+ set in [Configuration](#configuration). This is a Floci limit, not an AWS quota. A+ `CreateResource` beyond that fails with `ThrottlingException` (HTTP 400) and no+ request token; the AWS SDKs retry it automatically. Outside `memory` storage mode,+ creates still `IN_PROGRESS` when Floci stopped are replayed on the next start, even+ past that limit.- **`ListResources` / `GetResource` type coverage**: the read side lists`AWS::S3::Bucket`, `AWS::EC2::VPC`, `AWS::EC2::Subnet`, `AWS::EC2::SecurityGroup`,`AWS::EC2::Instance`, `AWS::EC2::LaunchTemplate`, `AWS::IAM::Role`, `AWS::IAM::User`,@@ -46,3 +52,11 @@ Cloud Control.that's still there.- **No account context**: Cloud Control requests don't carry an account id; Floci usesits default test account (`000000000000`) for all resources.++## Configuration++| Variable | Default | Description |+|---|---|---|+| `FLOCI_SERVICES_CLOUDCONTROL_ENABLED` | `true` | Enable or disable the service |+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_WORKER_THREADS` | `4` | Threads that run `CreateResource` provisioning |+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_QUEUE_CAPACITY` | `64` | Creates that may wait for a free thread; past that a new `CreateResource` fails with `ThrottlingException` |diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 10fcf65c6b..ba4553bdad 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1121,6 +1121,20 @@ public interface EmulatorConfig {interface CloudControlServiceConfig {@WithDefault("true")boolean enabled();++ /**+ * Worker threads that run CreateResource provisioning.+ * Env: FLOCI_SERVICES_CLOUDCONTROL_CREATE_WORKER_THREADS+ */+ @WithDefault("4")+ int createWorkerThreads();++ /**+ * CreateResource requests that may wait for a worker; past that a new create is refused with+ * ThrottlingException. Env: FLOCI_SERVICES_CLOUDCONTROL_CREATE_QUEUE_CAPACITY+ */+ @WithDefault("64")+ int createQueueCapacity();}interface S3VectorsServiceConfig {@WithDefault("true")diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.java b/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.javaindex fc59a63975..76106a4968 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.java@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;@@ -38,10 +39,15 @@ import java.util.List;import java.util.Map;import java.util.Set;import java.util.UUID;+import java.util.concurrent.ArrayBlockingQueue;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ConcurrentLinkedQueue;-import java.util.concurrent.ExecutorService;-import java.util.concurrent.Executors;+import java.util.concurrent.RejectedExecutionException;+import java.util.concurrent.ThreadPoolExecutor;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicLong;@ApplicationScopedpublic class CloudControlService {@@ -77,12 +83,38 @@ public class CloudControlService {* Entries are dropped when the resource is deleted.*/private final Map<String, CreatedResource> created = new ConcurrentHashMap<>();- private final ExecutorService executor =- Executors.newFixedThreadPool(4);+ /** For the constructors that take no {@link EmulatorConfig}; match its cloudcontrol defaults. */+ private static final int DEFAULT_WORKER_COUNT = 4;+ private static final int DEFAULT_QUEUE_CAPACITY = 64;++ /**+ * Bounded so a burst of CreateResource calls cannot grow the backlog without limit; a new create+ * that finds the queue full is refused with {@code ThrottlingException}, the async contract's+ * backpressure signal. Startup replay is exempt, see {@link #deferredReplay}.+ */+ private final ThreadPoolExecutor executor;++ /**+ * Replayed creates that did not fit {@link #executor}'s bounded queue during startup recovery.+ * Drained by {@link #drainDeferredReplay()} on a daemon thread: the request that constructs+ * Cloud Control must not wait on a provisioning task to free a slot, since provisioning can take+ * far longer than that request is willing to wait.+ */+ private final ConcurrentLinkedQueue<Runnable> deferredReplay = new ConcurrentLinkedQueue<>();+ /** Ensures only one drainer thread is handing {@link #deferredReplay} to the executor at a time. */+ private final AtomicBoolean replayDrainerRunning = new AtomicBoolean(false);+ /** The drainer thread, so {@link #shutdown()} can interrupt a blocked {@code put}. */+ private volatile Thread replayDrainer;+ /** How many replayed creates were parked because the worker queue was full at recovery time. */+ private final AtomicLong replayOverflowCount = new AtomicLong();@PreDestroyvoid shutdown() {executor.shutdownNow();+ Thread drainer = replayDrainer;+ if (drainer != null) {+ drainer.interrupt();+ }}private void restorePersistedState() {@@ -130,13 +162,15 @@ public class CloudControlService {if ("IN_PROGRESS".equals(normalized.operationStatus())&& "CREATE".equals(normalized.operation())&& persisted.desiredStateJson() != null) {+ JsonNode props;try {- JsonNode props = mapper.readTree(persisted.desiredStateJson());- submitCreate(persisted.region(), accountId, normalized.typeName(),- persisted.desiredStateJson(), normalized.requestToken(), normalized, props);- } catch (Exception e) {+ props = mapper.readTree(persisted.desiredStateJson());+ } catch (JsonProcessingException e) {record(normalized.failed("Persisted DesiredState is not valid JSON."));+ continue;}+ submitCreate(persisted.region(), accountId, normalized.typeName(),+ persisted.desiredStateJson(), normalized.requestToken(), normalized, props, true);}}trimPersistedRequests();@@ -182,12 +216,14 @@ public class CloudControlService {@Injectpublic CloudControlService(S3Service s3Service, Ec2Service ec2Service,IamService iamService, CfnResourceDispatcher provisioner,- ObjectMapper mapper, StorageFactory storageFactory) {+ ObjectMapper mapper, StorageFactory storageFactory, EmulatorConfig config) {this(s3Service, ec2Service, iamService, provisioner, mapper,storageFactory.create("cloudcontrol", "cloudcontrol-requests.json",new TypeReference<Map<String, PersistedRequest>>() {}),storageFactory.create("cloudcontrol", "cloudcontrol-created.json",- new TypeReference<Map<String, PersistedCreatedResource>>() {}));+ new TypeReference<Map<String, PersistedCreatedResource>>() {}),+ config.services().cloudcontrol().createWorkerThreads(),+ config.services().cloudcontrol().createQueueCapacity());}public CloudControlService(S3Service s3Service, Ec2Service ec2Service,@@ -203,6 +239,16 @@ public class CloudControlService {ObjectMapper mapper,AccountAwareStorageBackend<PersistedRequest> requestStore,AccountAwareStorageBackend<PersistedCreatedResource> createdStore) {+ this(s3Service, ec2Service, iamService, provisioner, mapper, requestStore, createdStore,+ DEFAULT_WORKER_COUNT, DEFAULT_QUEUE_CAPACITY);+ }++ CloudControlService(S3Service s3Service, Ec2Service ec2Service,+ IamService iamService, CfnResourceDispatcher provisioner,+ ObjectMapper mapper,+ AccountAwareStorageBackend<PersistedRequest> requestStore,+ AccountAwareStorageBackend<PersistedCreatedResource> createdStore,+ int workerCount, int queueCapacity) {this.s3Service = s3Service;this.ec2Service = ec2Service;this.iamService = iamService;@@ -210,6 +256,15 @@ public class CloudControlService {this.mapper = mapper;this.requestStore = requestStore;this.createdStore = createdStore;+ AtomicInteger workerNumber = new AtomicInteger();+ this.executor = new ThreadPoolExecutor(workerCount, workerCount, 0L, TimeUnit.MILLISECONDS,+ new ArrayBlockingQueue<>(queueCapacity),+ r -> {+ Thread t = new Thread(r, "cloudcontrol-worker-" + workerNumber.incrementAndGet());+ t.setDaemon(true);+ return t;+ },+ new ThreadPoolExecutor.AbortPolicy());restorePersistedState();}@@ -254,12 +309,17 @@ public class CloudControlService {private void submitCreate(String region, String accountId, String typeName, String desiredStateJson,String token, ProgressEvent pending, JsonNode props) {+ submitCreate(region, accountId, typeName, desiredStateJson, token, pending, props, false);+ }++ private void submitCreate(String region, String accountId, String typeName, String desiredStateJson,+ String token, ProgressEvent pending, JsonNode props, boolean replay) {persistRequest(new PersistedRequest(pending, region, desiredStateJson, System.currentTimeMillis()));- executor.submit(() -> RequestScopes.runAs(accountId, () -> {+ Runnable task = () -> RequestScopes.runAs(accountId, () -> {try {StackResource resource = provisioner.provisionStandalone(typeName, props, region, accountId);if (resource == null || resource.getPhysicalId() == null) {- record(pending.failed("CreateResource is not supported for " + typeName + "."));+ recordCreateFailure(pending, "CreateResource is not supported for " + typeName + ".");} else {String model = resourceModel(region, typeName, resource.getPhysicalId(), props);CreatedResource createdResource = new CreatedResource(token, accountId,@@ -270,9 +330,103 @@ public class CloudControlService {token, "CREATE", "SUCCESS", null, model, accountId));}} catch (Exception e) {- record(pending.failed(e.getMessage() == null ? e.toString() : e.getMessage()));+ recordCreateFailure(pending, e.getMessage() == null ? e.toString() : e.getMessage());+ }+ });+ try {+ executor.submit(task);+ } catch (RejectedExecutionException e) {+ if (replay) {+ // Startup replay is already-accepted, already-persisted work, so it must never be+ // dropped or failed for being unable to reach the queue. Park it and let the drainer+ // hand it to the executor once a slot frees; a blocking put here would block the+ // request that constructs Cloud Control for as long as provisioning takes.+ replayOverflowCount.incrementAndGet();+ deferredReplay.add(task);+ startReplayDrainer();+ return;+ }+ // The queue is full. Drop the pending token so it is not left unfulfillable and report+ // the async API's backpressure error rather than leaking a raw 500.+ requests.remove(token);+ requestOrder.remove(token);+ requestStore.deleteForAccount(accountId == null ? DEFAULT_ACCOUNT : accountId, token);+ throw new AwsException("ThrottlingException",+ "Cloud Control is at capacity; retry the request.", 400);+ }+ }++ /**+ * Records a create as FAILED, unless Cloud Control is shutting down. Shutdown interrupts in-flight+ * provisioning, so a failure then is the shutdown's, not the request's: the request stays+ * persisted as IN_PROGRESS and replays on the next start, like a create still in the queue.+ */+ private void recordCreateFailure(ProgressEvent pending, String message) {+ if (executor.isShutdown()) {+ return;+ }+ record(pending.failed(message));+ }++ /**+ * Starts the single replay drainer if one is not already running. Concurrent parks are safe:+ * only the thread that flips {@link #replayDrainerRunning} from false to true spawns a drainer.+ */+ private void startReplayDrainer() {+ if (replayDrainerRunning.compareAndSet(false, true)) {+ Thread drainer = new Thread(this::drainDeferredReplay, "cloudcontrol-replay-drainer");+ drainer.setDaemon(true);+ replayDrainer = drainer;+ drainer.start();+ }+ }++ /**+ * Hands parked replay tasks to {@link #executor} as slots free. {@code put} blocks only this+ * daemon thread, so the constructing thread has already returned. The drainer exits once the+ * backlog is empty, but re-checks after clearing the running flag so a task parked in that window+ * is not stranded. It also exits on shutdown: an interrupted {@code put} re-parks its task, and+ * every parked task stays persisted and replays on the next start.+ *+ * <p>Putting straight into the queue skips {@code execute}'s worker start, which is safe here: a+ * task is only parked after a rejection, and with core size equal to max size a rejection means+ * every worker thread is already running.+ */+ private void drainDeferredReplay() {+ while (!executor.isShutdown()) {+ Runnable next = deferredReplay.poll();+ if (next == null) {+ replayDrainerRunning.set(false);+ if (deferredReplay.isEmpty() || !replayDrainerRunning.compareAndSet(false, true)) {+ return;+ }+ continue;+ }+ try {+ executor.getQueue().put(next);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ deferredReplay.add(next);+ replayDrainerRunning.set(false);+ return;}- }));+ }+ replayDrainerRunning.set(false);+ }++ /** Number of replayed creates parked because the worker queue was full at recovery time. */+ long replayOverflowCount() {+ return replayOverflowCount.get();+ }++ /** The replay drainer thread, or null if recovery never overflowed the worker queue. */+ Thread replayDrainer() {+ return replayDrainer;+ }++ /** Waits for the workers to finish after {@link #shutdown()}; true once they all have. */+ boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {+ return executor.awaitTermination(timeout, unit);}/**diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 9b8f36de8b..e7eca24d74 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -811,6 +811,8 @@ floci:enabled: truecloudcontrol:enabled: true+ create-worker-threads: 4 # Threads that run CreateResource provisioning+ create-queue-capacity: 64 # Creates that may wait for a thread before ThrottlingExceptionresourceexplorer2:enabled: truecloudfront:diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.javaindex 325c5fb158..fe88c92790 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.java@@ -11,26 +11,47 @@ import io.github.hectorvent.floci.services.ec2.model.Tag;import io.github.hectorvent.floci.services.ec2.model.Vpc;import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.s3.S3Service;+import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.Test;+import java.util.ArrayList;import java.util.List;import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class CloudControlServiceTest {+ private final List<CloudControlService> services = new ArrayList<>();++ @AfterEach+ void shutDownServices() {+ services.forEach(CloudControlService::shutdown);+ }++ /** Registers a service so {@link #shutDownServices()} stops its worker and drainer threads. */+ private CloudControlService track(CloudControlService service) {+ services.add(service);+ return service;+ }+@Testvoid accountScopesCreateStatusLookupAndDelete() throws Exception {CfnResourceDispatcher provisioner = mock(CfnResourceDispatcher.class);@@ -40,18 +61,13 @@ class CloudControlServiceTest {when(provisioner.provisionStandalone(eq("AWS::EC2::VPC"),any(), eq("us-east-1"),eq("111111111111"))).thenReturn(resource);- CloudControlService service = new CloudControlService(+ CloudControlService service = track(new CloudControlService(mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,- new ObjectMapper());+ new ObjectMapper()));CloudControlService.ProgressEvent pending = service.createResource("us-east-1", "111111111111", "AWS::EC2::VPC", "{\"CidrBlock\":\"10.0.0.0/16\"}");- CloudControlService.ProgressEvent completed = pending;- for (int i = 0; i < 20 && !"SUCCESS".equals(completed.operationStatus()); i++) {- Thread.sleep(10);- completed = service.requestStatus("111111111111", pending.requestToken());- }- assertEquals("SUCCESS", completed.operationStatus());+ awaitSuccess(service, "111111111111", pending.requestToken());assertEquals("vpc-account-a", service.getResource("us-east-1", "111111111111","AWS::EC2::VPC", "vpc-account-a").identifier());assertThrows(AwsException.class, () -> service.requestStatus("222222222222", pending.requestToken()));@@ -82,19 +98,15 @@ class CloudControlServiceTest {AccountAwareStorageBackend.inMemory("000000000000");AccountAwareStorageBackend<CloudControlService.PersistedCreatedResource> created =AccountAwareStorageBackend.inMemory("000000000000");- CloudControlService first = new CloudControlService(+ CloudControlService first = track(new CloudControlService(mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,- new ObjectMapper(), requests, created);+ new ObjectMapper(), requests, created));CloudControlService.ProgressEvent pending = first.createResource("us-east-1", "111111111111","AWS::EC2::InternetGateway", "{}");- CloudControlService.ProgressEvent completed = first.requestStatus("111111111111", pending.requestToken());- for (int i = 0; i < 20 && !"SUCCESS".equals(completed.operationStatus()); i++) {- Thread.sleep(10);- completed = first.requestStatus("111111111111", pending.requestToken());- }- CloudControlService restarted = new CloudControlService(+ awaitSuccess(first, "111111111111", pending.requestToken());+ CloudControlService restarted = track(new CloudControlService(mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,- new ObjectMapper(), requests, created);+ new ObjectMapper(), requests, created));assertEquals("SUCCESS", restarted.requestStatus("111111111111", pending.requestToken()).operationStatus());assertEquals("igw-persisted", restarted.getResource("us-east-1", "111111111111",@@ -104,6 +116,72 @@ class CloudControlServiceTest {"AWS::EC2::InternetGateway", "igw-persisted"));}+ @Test+ void replayAdmitsPersistedCreatesWhenTheQueueIsFull() throws Exception {+ String accountId = "111111111111";+ String region = "us-east-1";+ String typeName = "AWS::EC2::VPC";+ CountDownLatch started = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ AtomicInteger completed = new AtomicInteger();+ CfnResourceDispatcher provisioner = blockingProvisioner(started, release, completed);+ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requestStore =+ AccountAwareStorageBackend.inMemory("000000000000");+ List<String> tokens = List.of("replay-1", "replay-2", "replay-3");+ seedInProgressCreates(requestStore, accountId, region, typeName, tokens);++ // The single worker stays blocked on `release` for the whole of recovery, so the first replay+ // runs, the second fills the queue, and the third overflows.+ CloudControlService service = track(new CloudControlService(+ mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,+ new ObjectMapper(), requestStore, AccountAwareStorageBackend.inMemory("000000000000"), 1, 1));++ assertTrue(started.await(10, TimeUnit.SECONDS), "the worker never started");+ assertEquals(0, completed.get(),+ "recovery waited for a provisioning task to free a slot before returning");+ assertEquals(1, service.replayOverflowCount(),+ "the third replayed create should have overflowed the full worker queue");++ release.countDown();+ for (String token : tokens) {+ awaitSuccess(service, accountId, token);+ }+ verify(provisioner, times(3)).provisionStandalone(eq(typeName), any(), eq(region), eq(accountId));+ assertEquals(3, requestStore.scanAllAccountEntries(k -> true).size());+ }++ @Test+ void shutdownStopsTheReplayDrainerAndKeepsEveryCreateForTheNextStart() throws Exception {+ String accountId = "111111111111";+ CountDownLatch started = new CountDownLatch(1);+ CfnResourceDispatcher provisioner = blockingProvisioner(started, new CountDownLatch(1), new AtomicInteger());+ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requestStore =+ AccountAwareStorageBackend.inMemory("000000000000");+ List<String> tokens = List.of("replay-1", "replay-2", "replay-3", "replay-4");+ seedInProgressCreates(requestStore, accountId, "us-east-1", "AWS::EC2::VPC", tokens);++ // One replay runs, one fills the queue, and two are parked. Shutdown empties the queue, so a+ // drainer that kept going would refill it with replay-3 and then block on replay-4 forever.+ CloudControlService service = track(new CloudControlService(+ mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,+ new ObjectMapper(), requestStore, AccountAwareStorageBackend.inMemory("000000000000"), 1, 1));+ assertTrue(started.await(10, TimeUnit.SECONDS), "the worker never started");+ Thread drainer = service.replayDrainer();+ assertNotNull(drainer, "the overflowing replays should have started the drainer");++ service.shutdown();+ drainer.join(10_000);+ assertTrue(service.awaitTermination(10, TimeUnit.SECONDS), "the worker outlived shutdown");++ assertFalse(drainer.isAlive(), "the replay drainer outlived shutdown");+ // replay-1 was mid-provisioning and saw the shutdown's interrupt, replay-2 was queued, and+ // replay-3 and replay-4 were parked. None of them failed, so all four replay on the next start.+ for (String token : tokens) {+ assertEquals("IN_PROGRESS", requestStore.getForAccount(accountId, token).orElseThrow()+ .event().operationStatus(), token + " should stay persisted for the next start");+ }+ }+@Testvoid emitsOnlyAwsShapedTagsForMalformedPersistedData() throws Exception {Ec2Service ec2Service = mock(Ec2Service.class);@@ -116,9 +194,9 @@ class CloudControlServiceTest {new Tag("Name", null)));when(ec2Service.describeVpcs("us-east-1", List.of(), Map.of())).thenReturn(List.of(vpc));ObjectMapper mapper = new ObjectMapper();- CloudControlService service = new CloudControlService(+ CloudControlService service = track(new CloudControlService(mock(S3Service.class), ec2Service, mock(IamService.class),- mock(CfnResourceDispatcher.class), mapper);+ mock(CfnResourceDispatcher.class), mapper));String properties = service.listResources("us-east-1", "AWS::EC2::VPC").getFirst().properties();JsonNode tags = mapper.readTree(properties).path("Tags");@@ -136,9 +214,9 @@ class CloudControlServiceTest {@Testvoid listResourcesRejectsARealButUnbackedTypeInsteadOfReturningEmpty() {- CloudControlService service = new CloudControlService(+ CloudControlService service = track(new CloudControlService(mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class),- mock(CfnResourceDispatcher.class), new ObjectMapper());+ mock(CfnResourceDispatcher.class), new ObjectMapper()));AwsException e = assertThrows(AwsException.class,() -> service.listResources("us-east-1", "AWS::SQS::Queue"));@@ -149,9 +227,9 @@ class CloudControlServiceTest {@Testvoid listResourcesRejectsATypeThatDoesNotExistInAwsAtAll() {- CloudControlService service = new CloudControlService(+ CloudControlService service = track(new CloudControlService(mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class),- mock(CfnResourceDispatcher.class), new ObjectMapper());+ mock(CfnResourceDispatcher.class), new ObjectMapper()));AwsException e = assertThrows(AwsException.class,() -> service.listResources("us-east-1", "AWS::NoSuch::Type"));@@ -164,10 +242,88 @@ class CloudControlServiceTest {void listResourcesStillReturnsASupportedType() {S3Service s3Service = mock(S3Service.class);when(s3Service.listBuckets()).thenReturn(List.of());- CloudControlService service = new CloudControlService(+ CloudControlService service = track(new CloudControlService(s3Service, mock(Ec2Service.class), mock(IamService.class),- mock(CfnResourceDispatcher.class), new ObjectMapper());+ mock(CfnResourceDispatcher.class), new ObjectMapper()));assertTrue(service.listResources("us-east-1", "AWS::S3::Bucket").isEmpty());}++ @Test+ void rejectsCreateWhenTheWorkerQueueIsFull() throws Exception {+ CountDownLatch started = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ CfnResourceDispatcher provisioner = blockingProvisioner(started, release, new AtomicInteger());++ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requests =+ AccountAwareStorageBackend.inMemory("000000000000");+ AccountAwareStorageBackend<CloudControlService.PersistedCreatedResource> created =+ AccountAwareStorageBackend.inMemory("000000000000");+ CloudControlService service = track(new CloudControlService(+ mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,+ new ObjectMapper(), requests, created, 1, 1));++ // The single worker takes the first task, the second queues, and the third overflows.+ service.createResource("us-east-1", "111111111111", "AWS::EC2::VPC", "{}");+ assertTrue(started.await(10, TimeUnit.SECONDS), "the worker never started");+ CloudControlService.ProgressEvent queued = service.createResource(+ "us-east-1", "111111111111", "AWS::EC2::VPC", "{}");++ AwsException e = assertThrows(AwsException.class,+ () -> service.createResource("us-east-1", "111111111111", "AWS::EC2::VPC", "{}"));+ assertEquals("ThrottlingException", e.getErrorCode());+ assertEquals(400, e.getHttpStatus());++ // The rejected token was dropped, so only the two accepted requests remain persisted.+ assertEquals(2, requests.scanAllAccountEntries(k -> true).size());++ // Once the worker drains, submission recovers.+ release.countDown();+ awaitSuccess(service, "111111111111", queued.requestToken());+ assertNotNull(service.createResource("us-east-1", "111111111111", "AWS::EC2::VPC", "{}"));+ }++ /**+ * A provisioner whose every call counts down {@code started}, then blocks on {@code release}+ * before returning a VPC numbered by {@code completed}. A release that never comes fails the+ * create, so a test cannot pass on a provisioner that was never let go.+ */+ private static CfnResourceDispatcher blockingProvisioner(CountDownLatch started, CountDownLatch release,+ AtomicInteger completed) {+ CfnResourceDispatcher provisioner = mock(CfnResourceDispatcher.class);+ doAnswer(invocation -> {+ started.countDown();+ if (!release.await(10, TimeUnit.SECONDS)) {+ throw new IllegalStateException("the test never released the provisioner");+ }+ StackResource resource = new StackResource();+ resource.setPhysicalId("vpc-" + completed.incrementAndGet());+ return resource;+ }).when(provisioner).provisionStandalone(anyString(), any(), anyString(), anyString());+ return provisioner;+ }++ /** Persists IN_PROGRESS creates as a previous run would have left them, oldest first. */+ private static void seedInProgressCreates(+ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requestStore,+ String accountId, String region, String typeName, List<String> tokens) {+ for (int i = 0; i < tokens.size(); i++) {+ String token = tokens.get(i);+ CloudControlService.ProgressEvent event = new CloudControlService.ProgressEvent(+ typeName, null, token, "CREATE", "IN_PROGRESS", null, null, accountId);+ requestStore.putForAccount(accountId, token,+ new CloudControlService.PersistedRequest(event, region, "{}", i + 1L));+ }+ }++ private static void awaitSuccess(CloudControlService service, String accountId, String requestToken)+ throws InterruptedException {+ CloudControlService.ProgressEvent status = service.requestStatus(accountId, requestToken);+ for (int i = 0; i < 50 && !"SUCCESS".equals(status.operationStatus()); i++) {+ Thread.sleep(10);+ status = service.requestStatus(accountId, requestToken);+ }+ assertEquals("SUCCESS", status.operationStatus(),+ requestToken + " ended " + status.operationStatus() + ": " + status.statusMessage());+ }}