Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudcontrol): bound the worker queue and reject overflow (#4479)

Daniël van Ziel517409a7aab1
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.md
index 508f18ae4b..3d29354f04 100644
--- a/docs/configuration/advanced/application-yml.md
+++ b/docs/configuration/advanced/application-yml.md
@@ -254,6 +254,11 @@ floci:
cloudformation:
enabled: true
+ cloudcontrol:
+ enabled: true
+ create-worker-threads: 4 # Threads that run CreateResource provisioning
+ create-queue-capacity: 64 # Creates that may wait for a thread before ThrottlingException
+
acm:
enabled: true
validation-wait-seconds: 0 # Seconds before transitioning PENDING_VALIDATION → ISSUED
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index 92ef333f3f..5cdc9cc858 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -358,6 +358,14 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s
| `FLOCI_SERVICES_CLOUDFORMATION_ALLOW_STUB_LAMBDA_CODE` | `false` | Fall back to the built-in stub handler for an `AWS::Lambda::Function` whose S3 code cannot be read. Off matches real CloudFormation, which fails the resource and rolls the stack back |
| `FLOCI_SERVICES_CLOUDFORMATION_ALLOW_STUB_UNSUPPORTED_RESOURCE_TYPES` | `true` | Stub a resource whose type has no provisioner (synthetic physical ID, `arn:aws:stub:::` ARN attribute, `CREATE_COMPLETE`), logged at `WARN` with a resource status reason. Set `false` to fail the resource instead, which rolls the stack back |
+### Cloud Control
+
+| Variable | Default | Description |
+|---|---|---|
+| `FLOCI_SERVICES_CLOUDCONTROL_ENABLED` | `true` | Enable the Cloud Control API service |
+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_WORKER_THREADS` | `4` | Threads that run `CreateResource` provisioning |
+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_QUEUE_CAPACITY` | `64` | Creates that may wait for a free thread; past that a new `CreateResource` fails with `ThrottlingException` |
+
### ACM (Certificate Manager)
| Variable | Default | Description |
diff --git a/docs/services/cloudcontrol.md b/docs/services/cloudcontrol.md
index 020cdecc1a..ab4ae18ad3 100644
--- a/docs/services/cloudcontrol.md
+++ b/docs/services/cloudcontrol.md
@@ -26,6 +26,12 @@ Cloud Control.
`GetResourceRequestStatus` with the token until the status is `SUCCESS` or `FAILED`.
`DeleteResource` is synchronous (deletes are fast enough not to need polling) but
still returns a `ProgressEvent` for API-shape consistency.
+- **Create backpressure**: creates run on 4 workers with room for 64 more queued, both
+ set in [Configuration](#configuration). This is a Floci limit, not an AWS quota. A
+ `CreateResource` beyond that fails with `ThrottlingException` (HTTP 400) and no
+ request token; the AWS SDKs retry it automatically. Outside `memory` storage mode,
+ creates still `IN_PROGRESS` when Floci stopped are replayed on the next start, even
+ past that limit.
- **`ListResources` / `GetResource` type coverage**: the read side lists
`AWS::S3::Bucket`, `AWS::EC2::VPC`, `AWS::EC2::Subnet`, `AWS::EC2::SecurityGroup`,
`AWS::EC2::Instance`, `AWS::EC2::LaunchTemplate`, `AWS::IAM::Role`, `AWS::IAM::User`,
@@ -46,3 +52,11 @@ Cloud Control.
that's still there.
- **No account context**: Cloud Control requests don't carry an account id; Floci uses
its default test account (`000000000000`) for all resources.
+
+## Configuration
+
+| Variable | Default | Description |
+|---|---|---|
+| `FLOCI_SERVICES_CLOUDCONTROL_ENABLED` | `true` | Enable or disable the service |
+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_WORKER_THREADS` | `4` | Threads that run `CreateResource` provisioning |
+| `FLOCI_SERVICES_CLOUDCONTROL_CREATE_QUEUE_CAPACITY` | `64` | Creates that may wait for a free thread; past that a new `CreateResource` fails with `ThrottlingException` |
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 10fcf65c6b..ba4553bdad 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1121,6 +1121,20 @@ public interface EmulatorConfig {
interface CloudControlServiceConfig {
@WithDefault("true")
boolean enabled();
+
+ /**
+ * Worker threads that run CreateResource provisioning.
+ * Env: FLOCI_SERVICES_CLOUDCONTROL_CREATE_WORKER_THREADS
+ */
+ @WithDefault("4")
+ int createWorkerThreads();
+
+ /**
+ * CreateResource requests that may wait for a worker; past that a new create is refused with
+ * ThrottlingException. Env: FLOCI_SERVICES_CLOUDCONTROL_CREATE_QUEUE_CAPACITY
+ */
+ @WithDefault("64")
+ int createQueueCapacity();
}
interface S3VectorsServiceConfig {
@WithDefault("true")
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.java b/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.java
index fc59a63975..76106a4968 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlService.java
@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RequestScopes;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
@@ -38,10 +39,15 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
@ApplicationScoped
public class CloudControlService {
@@ -77,12 +83,38 @@ public class CloudControlService {
* Entries are dropped when the resource is deleted.
*/
private final Map<String, CreatedResource> created = new ConcurrentHashMap<>();
- private final ExecutorService executor =
- Executors.newFixedThreadPool(4);
+ /** For the constructors that take no {@link EmulatorConfig}; match its cloudcontrol defaults. */
+ private static final int DEFAULT_WORKER_COUNT = 4;
+ private static final int DEFAULT_QUEUE_CAPACITY = 64;
+
+ /**
+ * Bounded so a burst of CreateResource calls cannot grow the backlog without limit; a new create
+ * that finds the queue full is refused with {@code ThrottlingException}, the async contract's
+ * backpressure signal. Startup replay is exempt, see {@link #deferredReplay}.
+ */
+ private final ThreadPoolExecutor executor;
+
+ /**
+ * Replayed creates that did not fit {@link #executor}'s bounded queue during startup recovery.
+ * Drained by {@link #drainDeferredReplay()} on a daemon thread: the request that constructs
+ * Cloud Control must not wait on a provisioning task to free a slot, since provisioning can take
+ * far longer than that request is willing to wait.
+ */
+ private final ConcurrentLinkedQueue<Runnable> deferredReplay = new ConcurrentLinkedQueue<>();
+ /** Ensures only one drainer thread is handing {@link #deferredReplay} to the executor at a time. */
+ private final AtomicBoolean replayDrainerRunning = new AtomicBoolean(false);
+ /** The drainer thread, so {@link #shutdown()} can interrupt a blocked {@code put}. */
+ private volatile Thread replayDrainer;
+ /** How many replayed creates were parked because the worker queue was full at recovery time. */
+ private final AtomicLong replayOverflowCount = new AtomicLong();
@PreDestroy
void shutdown() {
executor.shutdownNow();
+ Thread drainer = replayDrainer;
+ if (drainer != null) {
+ drainer.interrupt();
+ }
}
private void restorePersistedState() {
@@ -130,13 +162,15 @@ public class CloudControlService {
if ("IN_PROGRESS".equals(normalized.operationStatus())
&& "CREATE".equals(normalized.operation())
&& persisted.desiredStateJson() != null) {
+ JsonNode props;
try {
- JsonNode props = mapper.readTree(persisted.desiredStateJson());
- submitCreate(persisted.region(), accountId, normalized.typeName(),
- persisted.desiredStateJson(), normalized.requestToken(), normalized, props);
- } catch (Exception e) {
+ props = mapper.readTree(persisted.desiredStateJson());
+ } catch (JsonProcessingException e) {
record(normalized.failed("Persisted DesiredState is not valid JSON."));
+ continue;
}
+ submitCreate(persisted.region(), accountId, normalized.typeName(),
+ persisted.desiredStateJson(), normalized.requestToken(), normalized, props, true);
}
}
trimPersistedRequests();
@@ -182,12 +216,14 @@ public class CloudControlService {
@Inject
public CloudControlService(S3Service s3Service, Ec2Service ec2Service,
IamService iamService, CfnResourceDispatcher provisioner,
- ObjectMapper mapper, StorageFactory storageFactory) {
+ ObjectMapper mapper, StorageFactory storageFactory, EmulatorConfig config) {
this(s3Service, ec2Service, iamService, provisioner, mapper,
storageFactory.create("cloudcontrol", "cloudcontrol-requests.json",
new TypeReference<Map<String, PersistedRequest>>() {}),
storageFactory.create("cloudcontrol", "cloudcontrol-created.json",
- new TypeReference<Map<String, PersistedCreatedResource>>() {}));
+ new TypeReference<Map<String, PersistedCreatedResource>>() {}),
+ config.services().cloudcontrol().createWorkerThreads(),
+ config.services().cloudcontrol().createQueueCapacity());
}
public CloudControlService(S3Service s3Service, Ec2Service ec2Service,
@@ -203,6 +239,16 @@ public class CloudControlService {
ObjectMapper mapper,
AccountAwareStorageBackend<PersistedRequest> requestStore,
AccountAwareStorageBackend<PersistedCreatedResource> createdStore) {
+ this(s3Service, ec2Service, iamService, provisioner, mapper, requestStore, createdStore,
+ DEFAULT_WORKER_COUNT, DEFAULT_QUEUE_CAPACITY);
+ }
+
+ CloudControlService(S3Service s3Service, Ec2Service ec2Service,
+ IamService iamService, CfnResourceDispatcher provisioner,
+ ObjectMapper mapper,
+ AccountAwareStorageBackend<PersistedRequest> requestStore,
+ AccountAwareStorageBackend<PersistedCreatedResource> createdStore,
+ int workerCount, int queueCapacity) {
this.s3Service = s3Service;
this.ec2Service = ec2Service;
this.iamService = iamService;
@@ -210,6 +256,15 @@ public class CloudControlService {
this.mapper = mapper;
this.requestStore = requestStore;
this.createdStore = createdStore;
+ AtomicInteger workerNumber = new AtomicInteger();
+ this.executor = new ThreadPoolExecutor(workerCount, workerCount, 0L, TimeUnit.MILLISECONDS,
+ new ArrayBlockingQueue<>(queueCapacity),
+ r -> {
+ Thread t = new Thread(r, "cloudcontrol-worker-" + workerNumber.incrementAndGet());
+ t.setDaemon(true);
+ return t;
+ },
+ new ThreadPoolExecutor.AbortPolicy());
restorePersistedState();
}
@@ -254,12 +309,17 @@ public class CloudControlService {
private void submitCreate(String region, String accountId, String typeName, String desiredStateJson,
String token, ProgressEvent pending, JsonNode props) {
+ submitCreate(region, accountId, typeName, desiredStateJson, token, pending, props, false);
+ }
+
+ private void submitCreate(String region, String accountId, String typeName, String desiredStateJson,
+ String token, ProgressEvent pending, JsonNode props, boolean replay) {
persistRequest(new PersistedRequest(pending, region, desiredStateJson, System.currentTimeMillis()));
- executor.submit(() -> RequestScopes.runAs(accountId, () -> {
+ Runnable task = () -> RequestScopes.runAs(accountId, () -> {
try {
StackResource resource = provisioner.provisionStandalone(typeName, props, region, accountId);
if (resource == null || resource.getPhysicalId() == null) {
- record(pending.failed("CreateResource is not supported for " + typeName + "."));
+ recordCreateFailure(pending, "CreateResource is not supported for " + typeName + ".");
} else {
String model = resourceModel(region, typeName, resource.getPhysicalId(), props);
CreatedResource createdResource = new CreatedResource(token, accountId,
@@ -270,9 +330,103 @@ public class CloudControlService {
token, "CREATE", "SUCCESS", null, model, accountId));
}
} catch (Exception e) {
- record(pending.failed(e.getMessage() == null ? e.toString() : e.getMessage()));
+ recordCreateFailure(pending, e.getMessage() == null ? e.toString() : e.getMessage());
+ }
+ });
+ try {
+ executor.submit(task);
+ } catch (RejectedExecutionException e) {
+ if (replay) {
+ // Startup replay is already-accepted, already-persisted work, so it must never be
+ // dropped or failed for being unable to reach the queue. Park it and let the drainer
+ // hand it to the executor once a slot frees; a blocking put here would block the
+ // request that constructs Cloud Control for as long as provisioning takes.
+ replayOverflowCount.incrementAndGet();
+ deferredReplay.add(task);
+ startReplayDrainer();
+ return;
+ }
+ // The queue is full. Drop the pending token so it is not left unfulfillable and report
+ // the async API's backpressure error rather than leaking a raw 500.
+ requests.remove(token);
+ requestOrder.remove(token);
+ requestStore.deleteForAccount(accountId == null ? DEFAULT_ACCOUNT : accountId, token);
+ throw new AwsException("ThrottlingException",
+ "Cloud Control is at capacity; retry the request.", 400);
+ }
+ }
+
+ /**
+ * Records a create as FAILED, unless Cloud Control is shutting down. Shutdown interrupts in-flight
+ * provisioning, so a failure then is the shutdown's, not the request's: the request stays
+ * persisted as IN_PROGRESS and replays on the next start, like a create still in the queue.
+ */
+ private void recordCreateFailure(ProgressEvent pending, String message) {
+ if (executor.isShutdown()) {
+ return;
+ }
+ record(pending.failed(message));
+ }
+
+ /**
+ * Starts the single replay drainer if one is not already running. Concurrent parks are safe:
+ * only the thread that flips {@link #replayDrainerRunning} from false to true spawns a drainer.
+ */
+ private void startReplayDrainer() {
+ if (replayDrainerRunning.compareAndSet(false, true)) {
+ Thread drainer = new Thread(this::drainDeferredReplay, "cloudcontrol-replay-drainer");
+ drainer.setDaemon(true);
+ replayDrainer = drainer;
+ drainer.start();
+ }
+ }
+
+ /**
+ * Hands parked replay tasks to {@link #executor} as slots free. {@code put} blocks only this
+ * daemon thread, so the constructing thread has already returned. The drainer exits once the
+ * backlog is empty, but re-checks after clearing the running flag so a task parked in that window
+ * is not stranded. It also exits on shutdown: an interrupted {@code put} re-parks its task, and
+ * every parked task stays persisted and replays on the next start.
+ *
+ * <p>Putting straight into the queue skips {@code execute}'s worker start, which is safe here: a
+ * task is only parked after a rejection, and with core size equal to max size a rejection means
+ * every worker thread is already running.
+ */
+ private void drainDeferredReplay() {
+ while (!executor.isShutdown()) {
+ Runnable next = deferredReplay.poll();
+ if (next == null) {
+ replayDrainerRunning.set(false);
+ if (deferredReplay.isEmpty() || !replayDrainerRunning.compareAndSet(false, true)) {
+ return;
+ }
+ continue;
+ }
+ try {
+ executor.getQueue().put(next);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ deferredReplay.add(next);
+ replayDrainerRunning.set(false);
+ return;
}
- }));
+ }
+ replayDrainerRunning.set(false);
+ }
+
+ /** Number of replayed creates parked because the worker queue was full at recovery time. */
+ long replayOverflowCount() {
+ return replayOverflowCount.get();
+ }
+
+ /** The replay drainer thread, or null if recovery never overflowed the worker queue. */
+ Thread replayDrainer() {
+ return replayDrainer;
+ }
+
+ /** Waits for the workers to finish after {@link #shutdown()}; true once they all have. */
+ boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException {
+ return executor.awaitTermination(timeout, unit);
}
/**
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 9b8f36de8b..e7eca24d74 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -811,6 +811,8 @@ floci:
enabled: true
cloudcontrol:
enabled: true
+ create-worker-threads: 4 # Threads that run CreateResource provisioning
+ create-queue-capacity: 64 # Creates that may wait for a thread before ThrottlingException
resourceexplorer2:
enabled: true
cloudfront:
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.java
index 325c5fb158..fe88c92790 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudcontrol/CloudControlServiceTest.java
@@ -11,26 +11,47 @@ import io.github.hectorvent.floci.services.ec2.model.Tag;
import io.github.hectorvent.floci.services.ec2.model.Vpc;
import io.github.hectorvent.floci.services.iam.IamService;
import io.github.hectorvent.floci.services.s3.S3Service;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class CloudControlServiceTest {
+ private final List<CloudControlService> services = new ArrayList<>();
+
+ @AfterEach
+ void shutDownServices() {
+ services.forEach(CloudControlService::shutdown);
+ }
+
+ /** Registers a service so {@link #shutDownServices()} stops its worker and drainer threads. */
+ private CloudControlService track(CloudControlService service) {
+ services.add(service);
+ return service;
+ }
+
@Test
void accountScopesCreateStatusLookupAndDelete() throws Exception {
CfnResourceDispatcher provisioner = mock(CfnResourceDispatcher.class);
@@ -40,18 +61,13 @@ class CloudControlServiceTest {
when(provisioner.provisionStandalone(eq("AWS::EC2::VPC"),
any(), eq("us-east-1"),
eq("111111111111"))).thenReturn(resource);
- CloudControlService service = new CloudControlService(
+ CloudControlService service = track(new CloudControlService(
mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,
- new ObjectMapper());
+ new ObjectMapper()));
CloudControlService.ProgressEvent pending = service.createResource(
"us-east-1", "111111111111", "AWS::EC2::VPC", "{\"CidrBlock\":\"10.0.0.0/16\"}");
- CloudControlService.ProgressEvent completed = pending;
- for (int i = 0; i < 20 && !"SUCCESS".equals(completed.operationStatus()); i++) {
- Thread.sleep(10);
- completed = service.requestStatus("111111111111", pending.requestToken());
- }
- assertEquals("SUCCESS", completed.operationStatus());
+ awaitSuccess(service, "111111111111", pending.requestToken());
assertEquals("vpc-account-a", service.getResource("us-east-1", "111111111111",
"AWS::EC2::VPC", "vpc-account-a").identifier());
assertThrows(AwsException.class, () -> service.requestStatus("222222222222", pending.requestToken()));
@@ -82,19 +98,15 @@ class CloudControlServiceTest {
AccountAwareStorageBackend.inMemory("000000000000");
AccountAwareStorageBackend<CloudControlService.PersistedCreatedResource> created =
AccountAwareStorageBackend.inMemory("000000000000");
- CloudControlService first = new CloudControlService(
+ CloudControlService first = track(new CloudControlService(
mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,
- new ObjectMapper(), requests, created);
+ new ObjectMapper(), requests, created));
CloudControlService.ProgressEvent pending = first.createResource("us-east-1", "111111111111",
"AWS::EC2::InternetGateway", "{}");
- CloudControlService.ProgressEvent completed = first.requestStatus("111111111111", pending.requestToken());
- for (int i = 0; i < 20 && !"SUCCESS".equals(completed.operationStatus()); i++) {
- Thread.sleep(10);
- completed = first.requestStatus("111111111111", pending.requestToken());
- }
- CloudControlService restarted = new CloudControlService(
+ awaitSuccess(first, "111111111111", pending.requestToken());
+ CloudControlService restarted = track(new CloudControlService(
mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,
- new ObjectMapper(), requests, created);
+ new ObjectMapper(), requests, created));
assertEquals("SUCCESS", restarted.requestStatus("111111111111", pending.requestToken()).operationStatus());
assertEquals("igw-persisted", restarted.getResource("us-east-1", "111111111111",
@@ -104,6 +116,72 @@ class CloudControlServiceTest {
"AWS::EC2::InternetGateway", "igw-persisted"));
}
+ @Test
+ void replayAdmitsPersistedCreatesWhenTheQueueIsFull() throws Exception {
+ String accountId = "111111111111";
+ String region = "us-east-1";
+ String typeName = "AWS::EC2::VPC";
+ CountDownLatch started = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicInteger completed = new AtomicInteger();
+ CfnResourceDispatcher provisioner = blockingProvisioner(started, release, completed);
+ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requestStore =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ List<String> tokens = List.of("replay-1", "replay-2", "replay-3");
+ seedInProgressCreates(requestStore, accountId, region, typeName, tokens);
+
+ // The single worker stays blocked on `release` for the whole of recovery, so the first replay
+ // runs, the second fills the queue, and the third overflows.
+ CloudControlService service = track(new CloudControlService(
+ mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,
+ new ObjectMapper(), requestStore, AccountAwareStorageBackend.inMemory("000000000000"), 1, 1));
+
+ assertTrue(started.await(10, TimeUnit.SECONDS), "the worker never started");
+ assertEquals(0, completed.get(),
+ "recovery waited for a provisioning task to free a slot before returning");
+ assertEquals(1, service.replayOverflowCount(),
+ "the third replayed create should have overflowed the full worker queue");
+
+ release.countDown();
+ for (String token : tokens) {
+ awaitSuccess(service, accountId, token);
+ }
+ verify(provisioner, times(3)).provisionStandalone(eq(typeName), any(), eq(region), eq(accountId));
+ assertEquals(3, requestStore.scanAllAccountEntries(k -> true).size());
+ }
+
+ @Test
+ void shutdownStopsTheReplayDrainerAndKeepsEveryCreateForTheNextStart() throws Exception {
+ String accountId = "111111111111";
+ CountDownLatch started = new CountDownLatch(1);
+ CfnResourceDispatcher provisioner = blockingProvisioner(started, new CountDownLatch(1), new AtomicInteger());
+ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requestStore =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ List<String> tokens = List.of("replay-1", "replay-2", "replay-3", "replay-4");
+ seedInProgressCreates(requestStore, accountId, "us-east-1", "AWS::EC2::VPC", tokens);
+
+ // One replay runs, one fills the queue, and two are parked. Shutdown empties the queue, so a
+ // drainer that kept going would refill it with replay-3 and then block on replay-4 forever.
+ CloudControlService service = track(new CloudControlService(
+ mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,
+ new ObjectMapper(), requestStore, AccountAwareStorageBackend.inMemory("000000000000"), 1, 1));
+ assertTrue(started.await(10, TimeUnit.SECONDS), "the worker never started");
+ Thread drainer = service.replayDrainer();
+ assertNotNull(drainer, "the overflowing replays should have started the drainer");
+
+ service.shutdown();
+ drainer.join(10_000);
+ assertTrue(service.awaitTermination(10, TimeUnit.SECONDS), "the worker outlived shutdown");
+
+ assertFalse(drainer.isAlive(), "the replay drainer outlived shutdown");
+ // replay-1 was mid-provisioning and saw the shutdown's interrupt, replay-2 was queued, and
+ // replay-3 and replay-4 were parked. None of them failed, so all four replay on the next start.
+ for (String token : tokens) {
+ assertEquals("IN_PROGRESS", requestStore.getForAccount(accountId, token).orElseThrow()
+ .event().operationStatus(), token + " should stay persisted for the next start");
+ }
+ }
+
@Test
void emitsOnlyAwsShapedTagsForMalformedPersistedData() throws Exception {
Ec2Service ec2Service = mock(Ec2Service.class);
@@ -116,9 +194,9 @@ class CloudControlServiceTest {
new Tag("Name", null)));
when(ec2Service.describeVpcs("us-east-1", List.of(), Map.of())).thenReturn(List.of(vpc));
ObjectMapper mapper = new ObjectMapper();
- CloudControlService service = new CloudControlService(
+ CloudControlService service = track(new CloudControlService(
mock(S3Service.class), ec2Service, mock(IamService.class),
- mock(CfnResourceDispatcher.class), mapper);
+ mock(CfnResourceDispatcher.class), mapper));
String properties = service.listResources("us-east-1", "AWS::EC2::VPC").getFirst().properties();
JsonNode tags = mapper.readTree(properties).path("Tags");
@@ -136,9 +214,9 @@ class CloudControlServiceTest {
@Test
void listResourcesRejectsARealButUnbackedTypeInsteadOfReturningEmpty() {
- CloudControlService service = new CloudControlService(
+ CloudControlService service = track(new CloudControlService(
mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class),
- mock(CfnResourceDispatcher.class), new ObjectMapper());
+ mock(CfnResourceDispatcher.class), new ObjectMapper()));
AwsException e = assertThrows(AwsException.class,
() -> service.listResources("us-east-1", "AWS::SQS::Queue"));
@@ -149,9 +227,9 @@ class CloudControlServiceTest {
@Test
void listResourcesRejectsATypeThatDoesNotExistInAwsAtAll() {
- CloudControlService service = new CloudControlService(
+ CloudControlService service = track(new CloudControlService(
mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class),
- mock(CfnResourceDispatcher.class), new ObjectMapper());
+ mock(CfnResourceDispatcher.class), new ObjectMapper()));
AwsException e = assertThrows(AwsException.class,
() -> service.listResources("us-east-1", "AWS::NoSuch::Type"));
@@ -164,10 +242,88 @@ class CloudControlServiceTest {
void listResourcesStillReturnsASupportedType() {
S3Service s3Service = mock(S3Service.class);
when(s3Service.listBuckets()).thenReturn(List.of());
- CloudControlService service = new CloudControlService(
+ CloudControlService service = track(new CloudControlService(
s3Service, mock(Ec2Service.class), mock(IamService.class),
- mock(CfnResourceDispatcher.class), new ObjectMapper());
+ mock(CfnResourceDispatcher.class), new ObjectMapper()));
assertTrue(service.listResources("us-east-1", "AWS::S3::Bucket").isEmpty());
}
+
+ @Test
+ void rejectsCreateWhenTheWorkerQueueIsFull() throws Exception {
+ CountDownLatch started = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ CfnResourceDispatcher provisioner = blockingProvisioner(started, release, new AtomicInteger());
+
+ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requests =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ AccountAwareStorageBackend<CloudControlService.PersistedCreatedResource> created =
+ AccountAwareStorageBackend.inMemory("000000000000");
+ CloudControlService service = track(new CloudControlService(
+ mock(S3Service.class), mock(Ec2Service.class), mock(IamService.class), provisioner,
+ new ObjectMapper(), requests, created, 1, 1));
+
+ // The single worker takes the first task, the second queues, and the third overflows.
+ service.createResource("us-east-1", "111111111111", "AWS::EC2::VPC", "{}");
+ assertTrue(started.await(10, TimeUnit.SECONDS), "the worker never started");
+ CloudControlService.ProgressEvent queued = service.createResource(
+ "us-east-1", "111111111111", "AWS::EC2::VPC", "{}");
+
+ AwsException e = assertThrows(AwsException.class,
+ () -> service.createResource("us-east-1", "111111111111", "AWS::EC2::VPC", "{}"));
+ assertEquals("ThrottlingException", e.getErrorCode());
+ assertEquals(400, e.getHttpStatus());
+
+ // The rejected token was dropped, so only the two accepted requests remain persisted.
+ assertEquals(2, requests.scanAllAccountEntries(k -> true).size());
+
+ // Once the worker drains, submission recovers.
+ release.countDown();
+ awaitSuccess(service, "111111111111", queued.requestToken());
+ assertNotNull(service.createResource("us-east-1", "111111111111", "AWS::EC2::VPC", "{}"));
+ }
+
+ /**
+ * A provisioner whose every call counts down {@code started}, then blocks on {@code release}
+ * before returning a VPC numbered by {@code completed}. A release that never comes fails the
+ * create, so a test cannot pass on a provisioner that was never let go.
+ */
+ private static CfnResourceDispatcher blockingProvisioner(CountDownLatch started, CountDownLatch release,
+ AtomicInteger completed) {
+ CfnResourceDispatcher provisioner = mock(CfnResourceDispatcher.class);
+ doAnswer(invocation -> {
+ started.countDown();
+ if (!release.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("the test never released the provisioner");
+ }
+ StackResource resource = new StackResource();
+ resource.setPhysicalId("vpc-" + completed.incrementAndGet());
+ return resource;
+ }).when(provisioner).provisionStandalone(anyString(), any(), anyString(), anyString());
+ return provisioner;
+ }
+
+ /** Persists IN_PROGRESS creates as a previous run would have left them, oldest first. */
+ private static void seedInProgressCreates(
+ AccountAwareStorageBackend<CloudControlService.PersistedRequest> requestStore,
+ String accountId, String region, String typeName, List<String> tokens) {
+ for (int i = 0; i < tokens.size(); i++) {
+ String token = tokens.get(i);
+ CloudControlService.ProgressEvent event = new CloudControlService.ProgressEvent(
+ typeName, null, token, "CREATE", "IN_PROGRESS", null, null, accountId);
+ requestStore.putForAccount(accountId, token,
+ new CloudControlService.PersistedRequest(event, region, "{}", i + 1L));
+ }
+ }
+
+ private static void awaitSuccess(CloudControlService service, String accountId, String requestToken)
+ throws InterruptedException {
+ CloudControlService.ProgressEvent status = service.requestStatus(accountId, requestToken);
+ for (int i = 0; i < 50 && !"SUCCESS".equals(status.operationStatus()); i++) {
+ Thread.sleep(10);
+ status = service.requestStatus(accountId, requestToken);
+ }
+ assertEquals("SUCCESS", status.operationStatus(),
+ requestToken + " ended " + status.operationStatus() + ": " + status.statusMessage());
+ }
}