Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codepipeline): cap QUEUED and PARALLEL pipelines at 50 active executions (#3503)

Niels Kootstra2104b33ea74d
diff --git a/docs/services/codepipeline.md b/docs/services/codepipeline.md
index bc19c773c6..98a7a27412 100644
--- a/docs/services/codepipeline.md
+++ b/docs/services/codepipeline.md
@@ -22,6 +22,8 @@ state use Floci's configured storage backend.
Stages execute in declaration order. Actions with the same `runOrder` execute in parallel.
`SUPERSEDED`, `QUEUED`, and `PARALLEL` execution modes are recognized, with `QUEUED` and
`PARALLEL` restricted to V2 pipelines.
+A `QUEUED` or `PARALLEL` pipeline holds at most 50 active executions, as on AWS;
+`StartPipelineExecution` beyond that returns `ConcurrentPipelineExecutionsLimitExceededException`.
The following providers execute against local Floci services:
diff --git a/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java b/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java
index 0eb6524896..bbd5e2980e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java
@@ -45,6 +45,7 @@ import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
@ApplicationScoped
@@ -52,6 +53,7 @@ public class CodePipelineService {
private static final Logger LOG = Logger.getLogger(CodePipelineService.class);
private static final String DEFAULT_EXECUTION_MODE = "SUPERSEDED";
private static final String DEFAULT_PIPELINE_TYPE = "V1";
+ private static final int MAX_ACTIVE_EXECUTIONS = 50;
private static final long POLL_INTERVAL_MS = 100L;
private final AccountAwareStorageBackend<CodePipelinePipeline> pipelineStore;
@@ -64,6 +66,10 @@ public class CodePipelineService {
private final S3Service s3Service;
private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();
private final Map<String, Object> pipelineLocks = new ConcurrentHashMap<>();
+ // Admission is serialized per pipeline on its own lock. A QUEUED worker holds the pipelineLocks
+ // monitor for its whole run, so counting under that monitor would block StartPipelineExecution
+ // until the running execution finished.
+ private final Map<String, Object> startLocks = new ConcurrentHashMap<>();
private final Map<String, byte[]> runtimeArtifacts = new ConcurrentHashMap<>();
@Inject
@@ -282,12 +288,44 @@ public class CodePipelineService {
trigger.put("clientRequestToken", clientToken);
}
execution.setTrigger(trigger);
- putExecution(execution);
+ if (!persistExecutionIfSlotAvailable(execution)) {
+ throw new AwsException("ConcurrentPipelineExecutionsLimitExceededException",
+ "The pipeline has reached the limit for concurrent pipeline executions", 400);
+ }
applyExecutionMode(execution);
- executor.submit(() -> runExecution(pipeline, execution));
+ try {
+ executor.submit(() -> runExecution(pipeline, execution));
+ } catch (RejectedExecutionException exception) {
+ execution.setStatus("Failed");
+ execution.setStatusSummary("Pipeline execution could not be scheduled.");
+ execution.setLastUpdateTime(now());
+ putExecution(execution);
+ throw new AwsException("ConflictException",
+ "Your request cannot be handled because the pipeline is busy handling ongoing activities. "
+ + "Try again later.", 400);
+ }
return mapper.createObjectNode().put("pipelineExecutionId", execution.getPipelineExecutionId());
}
+ private boolean persistExecutionIfSlotAvailable(CodePipelineExecution execution) {
+ if (!List.of("QUEUED", "PARALLEL").contains(execution.getExecutionMode())) {
+ putExecution(execution);
+ return true;
+ }
+ synchronized (startLocks.computeIfAbsent(lockKey(execution), ignored -> new Object())) {
+ long active = executions(execution.getAccountId(), execution.getRegion(), execution.getPipelineName())
+ .stream()
+ .filter(candidate -> "InProgress".equals(candidate.getStatus())
+ || "Stopping".equals(candidate.getStatus()))
+ .count();
+ if (active >= MAX_ACTIVE_EXECUTIONS) {
+ return false;
+ }
+ putExecution(execution);
+ return true;
+ }
+ }
+
private ObjectNode stopPipelineExecution(JsonNode request, String region, String account) {
CodePipelineExecution execution = requireExecution(
account, region, text(request, "pipelineName"), text(request, "pipelineExecutionId"));
@@ -1509,6 +1547,14 @@ public class CodePipelineService {
@PreDestroy
void shutdown() {
executor.shutdownNow();
+ try {
+ if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
+ LOG.warn("CodePipeline execution workers did not stop within the shutdown timeout");
+ }
+ } catch (InterruptedException exception) {
+ Thread.currentThread().interrupt();
+ LOG.warn("Interrupted while stopping CodePipeline execution workers", exception);
+ }
}
private record Page(int start, int end) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.java
index a00c60ffe1..98e530e3f3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.java
@@ -8,8 +8,11 @@ import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
@@ -1165,6 +1168,50 @@ class CodePipelineIntegrationTest {
.body("__type", containsString("InvalidStructureException"));
}
+ @Test
+ void parallelPipelineRejectsExecutionAfterAwsActiveLimit() throws Exception {
+ String pipelineName = "parallel-limit-pipeline";
+ post("CreatePipeline", approvalPipeline(pipelineName, "PARALLEL")).then().statusCode(200);
+ List<String> executionIds = new ArrayList<>();
+ try {
+ while (executionIds.size() < 50) {
+ executionIds.add(startExecution(pipelineName));
+ }
+ waitForActiveExecutions(pipelineName, 50);
+
+ post("StartPipelineExecution", "{\"name\": \"%s\"}".formatted(pipelineName))
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ConcurrentPipelineExecutionsLimitExceededException"));
+ } finally {
+ stopExecutions(pipelineName, executionIds);
+ }
+ }
+
+ @Test
+ void queuedPipelineAcceptsStartsWhileAnExecutionRunsAndRejectsTheFiftyFirst() throws Exception {
+ String pipelineName = "queued-limit-pipeline";
+ post("CreatePipeline", approvalPipeline(pipelineName, "QUEUED")).then().statusCode(200);
+ List<String> executionIds = new ArrayList<>();
+ try {
+ executionIds.add(startExecution(pipelineName));
+ waitForApprovalToken(pipelineName);
+
+ assertTimeoutPreemptively(Duration.ofSeconds(10), () -> {
+ while (executionIds.size() < 50) {
+ executionIds.add(startExecution(pipelineName));
+ }
+ }, "StartPipelineExecution waited for the running execution to finish");
+
+ post("StartPipelineExecution", "{\"name\": \"%s\"}".formatted(pipelineName))
+ .then()
+ .statusCode(400)
+ .body("__type", containsString("ConcurrentPipelineExecutionsLimitExceededException"));
+ } finally {
+ stopExecutions(pipelineName, executionIds);
+ }
+ }
+
private Response waitForJob() throws Exception {
Instant deadline = Instant.now().plus(Duration.ofSeconds(5));
Response response;
@@ -1244,6 +1291,74 @@ class CodePipelineIntegrationTest {
throw new AssertionError("Approval token was not issued for pipeline " + pipelineName);
}
+ private void waitForActiveExecutions(String pipelineName, int expected) throws Exception {
+ Instant deadline = Instant.now().plus(Duration.ofSeconds(5));
+ do {
+ int active = post("ListPipelineExecutions", "{\"pipelineName\": \"%s\"}".formatted(pipelineName))
+ .jsonPath().getList("pipelineExecutionSummaries.status", String.class).stream()
+ .filter("InProgress"::equals)
+ .toList()
+ .size();
+ if (active >= expected) {
+ return;
+ }
+ Thread.sleep(50);
+ } while (Instant.now().isBefore(deadline));
+ throw new AssertionError("Pipeline did not reach " + expected + " active executions");
+ }
+
+ private static String approvalPipeline(String name, String executionMode) {
+ return """
+ {
+ "pipeline": {
+ "name": "%s",
+ "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+ "pipelineType": "V2",
+ "executionMode": "%s",
+ "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+ "stages": [{
+ "name": "Approve",
+ "actions": [{
+ "name": "ManualApproval",
+ "actionTypeId": {
+ "category": "Approval",
+ "owner": "AWS",
+ "provider": "Manual",
+ "version": "1"
+ }
+ }]
+ }, {
+ "name": "Complete",
+ "actions": [{
+ "name": "ManualApprovalComplete",
+ "actionTypeId": {
+ "category": "Approval",
+ "owner": "AWS",
+ "provider": "Manual",
+ "version": "1"
+ }
+ }]
+ }]
+ }
+ }
+ """.formatted(name, executionMode);
+ }
+
+ private static String startExecution(String pipelineName) {
+ return post("StartPipelineExecution", "{\"name\": \"%s\"}".formatted(pipelineName))
+ .then()
+ .statusCode(200)
+ .extract().jsonPath().getString("pipelineExecutionId");
+ }
+
+ private static void stopExecutions(String pipelineName, List<String> executionIds) {
+ for (String executionId : executionIds) {
+ post("StopPipelineExecution", """
+ {"pipelineName": "%s", "pipelineExecutionId": "%s", "abandon": true}
+ """.formatted(pipelineName, executionId)).then().statusCode(200);
+ }
+ }
+
private static Response post(String action, String body) {
return given()
.header("X-Amz-Target", TARGET + action)
diff --git a/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java
new file mode 100644
index 0000000000..3672537852
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java
@@ -0,0 +1,97 @@
+package io.github.hectorvent.floci.services.codepipeline;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.codebuild.CodeBuildService;
+import io.github.hectorvent.floci.services.codedeploy.CodeDeployService;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.s3.S3Service;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+
+class CodePipelineServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT = "000000000000";
+
+ private final ObjectMapper mapper = new ObjectMapper();
+ private final CodePipelineService service = new CodePipelineService(
+ new InMemoryStorageFactory(),
+ mapper,
+ mock(CodeBuildService.class),
+ mock(CodeDeployService.class),
+ mock(LambdaService.class),
+ mock(S3Service.class));
+
+ @Test
+ void aStartAfterShutdownIsRefusedAndTheExecutionIsPersistedAsFailed() throws Exception {
+ handle("CreatePipeline", """
+ {
+ "pipeline": {
+ "name": "pipeline",
+ "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",
+ "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},
+ "stages": [{
+ "name": "Approve",
+ "actions": [{
+ "name": "ManualApproval",
+ "actionTypeId": {
+ "category": "Approval",
+ "owner": "AWS",
+ "provider": "Manual",
+ "version": "1"
+ }
+ }]
+ }, {
+ "name": "Complete",
+ "actions": [{
+ "name": "ManualApprovalComplete",
+ "actionTypeId": {
+ "category": "Approval",
+ "owner": "AWS",
+ "provider": "Manual",
+ "version": "1"
+ }
+ }]
+ }]
+ }
+ }
+ """);
+ service.shutdown();
+
+ AwsException refused = assertThrows(AwsException.class,
+ () -> handle("StartPipelineExecution", "{\"name\": \"pipeline\"}"));
+
+ assertEquals("ConflictException", refused.getErrorCode());
+ assertEquals(400, refused.getHttpStatus());
+ JsonNode summaries = handle("ListPipelineExecutions", "{\"pipelineName\": \"pipeline\"}")
+ .path("pipelineExecutionSummaries");
+ assertEquals(1, summaries.size());
+ assertEquals("Failed", summaries.get(0).path("status").asText());
+ }
+
+ private JsonNode handle(String action, String body) throws Exception {
+ return service.handle(action, mapper.readTree(body), REGION, ACCOUNT);
+ }
+
+ private static final class InMemoryStorageFactory extends StorageFactory {
+ private InMemoryStorageFactory() {
+ super(null, null);
+ }
+
+ @Override
+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,
+ TypeReference<Map<String, V>> typeReference) {
+ return AccountAwareStorageBackend.inMemory(ACCOUNT);
+ }
+ }
+}