Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codepipeline): cap QUEUED and PARALLEL pipelines at 50 active executions (#3503)
Niels Kootstra2104b33ea74d
diff --git a/docs/services/codepipeline.md b/docs/services/codepipeline.mdindex bc19c773c6..98a7a27412 100644--- a/docs/services/codepipeline.md+++ b/docs/services/codepipeline.md@@ -22,6 +22,8 @@ state use Floci's configured storage backend.Stages execute in declaration order. Actions with the same `runOrder` execute in parallel.`SUPERSEDED`, `QUEUED`, and `PARALLEL` execution modes are recognized, with `QUEUED` and`PARALLEL` restricted to V2 pipelines.+A `QUEUED` or `PARALLEL` pipeline holds at most 50 active executions, as on AWS;+`StartPipelineExecution` beyond that returns `ConcurrentPipelineExecutionsLimitExceededException`.The following providers execute against local Floci services:diff --git a/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java b/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.javaindex 0eb6524896..bbd5e2980e 100644--- a/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineService.java@@ -45,6 +45,7 @@ import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;+import java.util.concurrent.RejectedExecutionException;import java.util.concurrent.TimeUnit;@ApplicationScoped@@ -52,6 +53,7 @@ public class CodePipelineService {private static final Logger LOG = Logger.getLogger(CodePipelineService.class);private static final String DEFAULT_EXECUTION_MODE = "SUPERSEDED";private static final String DEFAULT_PIPELINE_TYPE = "V1";+ private static final int MAX_ACTIVE_EXECUTIONS = 50;private static final long POLL_INTERVAL_MS = 100L;private final AccountAwareStorageBackend<CodePipelinePipeline> pipelineStore;@@ -64,6 +66,10 @@ public class CodePipelineService {private final S3Service s3Service;private final ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();private final Map<String, Object> pipelineLocks = new ConcurrentHashMap<>();+ // Admission is serialized per pipeline on its own lock. A QUEUED worker holds the pipelineLocks+ // monitor for its whole run, so counting under that monitor would block StartPipelineExecution+ // until the running execution finished.+ private final Map<String, Object> startLocks = new ConcurrentHashMap<>();private final Map<String, byte[]> runtimeArtifacts = new ConcurrentHashMap<>();@Inject@@ -282,12 +288,44 @@ public class CodePipelineService {trigger.put("clientRequestToken", clientToken);}execution.setTrigger(trigger);- putExecution(execution);+ if (!persistExecutionIfSlotAvailable(execution)) {+ throw new AwsException("ConcurrentPipelineExecutionsLimitExceededException",+ "The pipeline has reached the limit for concurrent pipeline executions", 400);+ }applyExecutionMode(execution);- executor.submit(() -> runExecution(pipeline, execution));+ try {+ executor.submit(() -> runExecution(pipeline, execution));+ } catch (RejectedExecutionException exception) {+ execution.setStatus("Failed");+ execution.setStatusSummary("Pipeline execution could not be scheduled.");+ execution.setLastUpdateTime(now());+ putExecution(execution);+ throw new AwsException("ConflictException",+ "Your request cannot be handled because the pipeline is busy handling ongoing activities. "+ + "Try again later.", 400);+ }return mapper.createObjectNode().put("pipelineExecutionId", execution.getPipelineExecutionId());}+ private boolean persistExecutionIfSlotAvailable(CodePipelineExecution execution) {+ if (!List.of("QUEUED", "PARALLEL").contains(execution.getExecutionMode())) {+ putExecution(execution);+ return true;+ }+ synchronized (startLocks.computeIfAbsent(lockKey(execution), ignored -> new Object())) {+ long active = executions(execution.getAccountId(), execution.getRegion(), execution.getPipelineName())+ .stream()+ .filter(candidate -> "InProgress".equals(candidate.getStatus())+ || "Stopping".equals(candidate.getStatus()))+ .count();+ if (active >= MAX_ACTIVE_EXECUTIONS) {+ return false;+ }+ putExecution(execution);+ return true;+ }+ }+private ObjectNode stopPipelineExecution(JsonNode request, String region, String account) {CodePipelineExecution execution = requireExecution(account, region, text(request, "pipelineName"), text(request, "pipelineExecutionId"));@@ -1509,6 +1547,14 @@ public class CodePipelineService {@PreDestroyvoid shutdown() {executor.shutdownNow();+ try {+ if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {+ LOG.warn("CodePipeline execution workers did not stop within the shutdown timeout");+ }+ } catch (InterruptedException exception) {+ Thread.currentThread().interrupt();+ LOG.warn("Interrupted while stopping CodePipeline execution workers", exception);+ }}private record Page(int start, int end) {diff --git a/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.javaindex a00c60ffe1..98e530e3f3 100644--- a/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineIntegrationTest.java@@ -8,8 +8,11 @@ import org.junit.jupiter.api.Test;import java.time.Duration;import java.time.Instant;+import java.util.ArrayList;+import java.util.List;import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.hasItem;@@ -1165,6 +1168,50 @@ class CodePipelineIntegrationTest {.body("__type", containsString("InvalidStructureException"));}+ @Test+ void parallelPipelineRejectsExecutionAfterAwsActiveLimit() throws Exception {+ String pipelineName = "parallel-limit-pipeline";+ post("CreatePipeline", approvalPipeline(pipelineName, "PARALLEL")).then().statusCode(200);+ List<String> executionIds = new ArrayList<>();+ try {+ while (executionIds.size() < 50) {+ executionIds.add(startExecution(pipelineName));+ }+ waitForActiveExecutions(pipelineName, 50);++ post("StartPipelineExecution", "{\"name\": \"%s\"}".formatted(pipelineName))+ .then()+ .statusCode(400)+ .body("__type", containsString("ConcurrentPipelineExecutionsLimitExceededException"));+ } finally {+ stopExecutions(pipelineName, executionIds);+ }+ }++ @Test+ void queuedPipelineAcceptsStartsWhileAnExecutionRunsAndRejectsTheFiftyFirst() throws Exception {+ String pipelineName = "queued-limit-pipeline";+ post("CreatePipeline", approvalPipeline(pipelineName, "QUEUED")).then().statusCode(200);+ List<String> executionIds = new ArrayList<>();+ try {+ executionIds.add(startExecution(pipelineName));+ waitForApprovalToken(pipelineName);++ assertTimeoutPreemptively(Duration.ofSeconds(10), () -> {+ while (executionIds.size() < 50) {+ executionIds.add(startExecution(pipelineName));+ }+ }, "StartPipelineExecution waited for the running execution to finish");++ post("StartPipelineExecution", "{\"name\": \"%s\"}".formatted(pipelineName))+ .then()+ .statusCode(400)+ .body("__type", containsString("ConcurrentPipelineExecutionsLimitExceededException"));+ } finally {+ stopExecutions(pipelineName, executionIds);+ }+ }+private Response waitForJob() throws Exception {Instant deadline = Instant.now().plus(Duration.ofSeconds(5));Response response;@@ -1244,6 +1291,74 @@ class CodePipelineIntegrationTest {throw new AssertionError("Approval token was not issued for pipeline " + pipelineName);}+ private void waitForActiveExecutions(String pipelineName, int expected) throws Exception {+ Instant deadline = Instant.now().plus(Duration.ofSeconds(5));+ do {+ int active = post("ListPipelineExecutions", "{\"pipelineName\": \"%s\"}".formatted(pipelineName))+ .jsonPath().getList("pipelineExecutionSummaries.status", String.class).stream()+ .filter("InProgress"::equals)+ .toList()+ .size();+ if (active >= expected) {+ return;+ }+ Thread.sleep(50);+ } while (Instant.now().isBefore(deadline));+ throw new AssertionError("Pipeline did not reach " + expected + " active executions");+ }++ private static String approvalPipeline(String name, String executionMode) {+ return """+ {+ "pipeline": {+ "name": "%s",+ "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",+ "pipelineType": "V2",+ "executionMode": "%s",+ "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},+ "stages": [{+ "name": "Approve",+ "actions": [{+ "name": "ManualApproval",+ "actionTypeId": {+ "category": "Approval",+ "owner": "AWS",+ "provider": "Manual",+ "version": "1"+ }+ }]+ }, {+ "name": "Complete",+ "actions": [{+ "name": "ManualApprovalComplete",+ "actionTypeId": {+ "category": "Approval",+ "owner": "AWS",+ "provider": "Manual",+ "version": "1"+ }+ }]+ }]+ }+ }+ """.formatted(name, executionMode);+ }++ private static String startExecution(String pipelineName) {+ return post("StartPipelineExecution", "{\"name\": \"%s\"}".formatted(pipelineName))+ .then()+ .statusCode(200)+ .extract().jsonPath().getString("pipelineExecutionId");+ }++ private static void stopExecutions(String pipelineName, List<String> executionIds) {+ for (String executionId : executionIds) {+ post("StopPipelineExecution", """+ {"pipelineName": "%s", "pipelineExecutionId": "%s", "abandon": true}+ """.formatted(pipelineName, executionId)).then().statusCode(200);+ }+ }+private static Response post(String action, String body) {return given().header("X-Amz-Target", TARGET + action)diff --git a/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.javanew file mode 100644index 0000000000..3672537852--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codepipeline/CodePipelineServiceTest.java@@ -0,0 +1,97 @@+package io.github.hectorvent.floci.services.codepipeline;++import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.codebuild.CodeBuildService;+import io.github.hectorvent.floci.services.codedeploy.CodeDeployService;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.s3.S3Service;+import org.junit.jupiter.api.Test;++import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.Mockito.mock;++class CodePipelineServiceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT = "000000000000";++ private final ObjectMapper mapper = new ObjectMapper();+ private final CodePipelineService service = new CodePipelineService(+ new InMemoryStorageFactory(),+ mapper,+ mock(CodeBuildService.class),+ mock(CodeDeployService.class),+ mock(LambdaService.class),+ mock(S3Service.class));++ @Test+ void aStartAfterShutdownIsRefusedAndTheExecutionIsPersistedAsFailed() throws Exception {+ handle("CreatePipeline", """+ {+ "pipeline": {+ "name": "pipeline",+ "roleArn": "arn:aws:iam::000000000000:role/codepipeline-role",+ "artifactStore": {"type": "S3", "location": "codepipeline-artifacts"},+ "stages": [{+ "name": "Approve",+ "actions": [{+ "name": "ManualApproval",+ "actionTypeId": {+ "category": "Approval",+ "owner": "AWS",+ "provider": "Manual",+ "version": "1"+ }+ }]+ }, {+ "name": "Complete",+ "actions": [{+ "name": "ManualApprovalComplete",+ "actionTypeId": {+ "category": "Approval",+ "owner": "AWS",+ "provider": "Manual",+ "version": "1"+ }+ }]+ }]+ }+ }+ """);+ service.shutdown();++ AwsException refused = assertThrows(AwsException.class,+ () -> handle("StartPipelineExecution", "{\"name\": \"pipeline\"}"));++ assertEquals("ConflictException", refused.getErrorCode());+ assertEquals(400, refused.getHttpStatus());+ JsonNode summaries = handle("ListPipelineExecutions", "{\"pipelineName\": \"pipeline\"}")+ .path("pipelineExecutionSummaries");+ assertEquals(1, summaries.size());+ assertEquals("Failed", summaries.get(0).path("status").asText());+ }++ private JsonNode handle(String action, String body) throws Exception {+ return service.handle(action, mapper.readTree(body), REGION, ACCOUNT);+ }++ private static final class InMemoryStorageFactory extends StorageFactory {+ private InMemoryStorageFactory() {+ super(null, null);+ }++ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return AccountAwareStorageBackend.inMemory(ACCOUNT);+ }+ }+}