Light, fluffy, and always free - The AWS Local Emulator alternative
fix(codedeploy): fail closed on unreachable, erroring, or timed-out lifecycle hooks (#3563)
Niels Kootstrad167cc5bf378
diff --git a/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java b/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.javaindex 26c5e04211..fab0f25353 100644--- a/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java+++ b/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java@@ -28,6 +28,8 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.time.Clock;+import java.time.Duration;import java.time.Instant;import java.util.ArrayList;import java.util.List;@@ -36,7 +38,9 @@ import java.util.UUID;import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CopyOnWriteArrayList;+import java.util.concurrent.ExecutionException;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;import java.util.concurrent.atomic.AtomicBoolean;import java.util.stream.Collectors;@@ -45,6 +49,10 @@ public class CodeDeployService {private static final Logger LOG = Logger.getLogger(CodeDeployService.class);+ // Lambda/ECS validation hooks must call PutLifecycleEventHookExecutionStatus within one hour.+ private static final Duration DEFAULT_HOOK_CALLBACK_TIMEOUT = Duration.ofHours(1);+ private static final Duration STOP_POLL_INTERVAL = Duration.ofMillis(200);+private final LambdaService lambdaService;private final EcsService ecsService;private final ElbV2Service elbV2Service;@@ -54,12 +62,22 @@ public class CodeDeployService {private final ObjectMapper yamlMapper;private final RegionResolver regionResolver;private final StorageFactory storageFactory;+ private final Duration hookCallbackTimeout;+ private final Clock clock;@Injectpublic CodeDeployService(LambdaService lambdaService, EcsService ecsService,ElbV2Service elbV2Service, SsmCommandService ssmCommandService,Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,StorageFactory storageFactory) {+ this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,+ regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC());+ }++ CodeDeployService(LambdaService lambdaService, EcsService ecsService,+ ElbV2Service elbV2Service, SsmCommandService ssmCommandService,+ Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,+ StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock) {this.lambdaService = lambdaService;this.ecsService = ecsService;this.elbV2Service = elbV2Service;@@ -69,6 +87,8 @@ public class CodeDeployService {this.yamlMapper = new ObjectMapper(new YAMLFactory());this.regionResolver = regionResolver;this.storageFactory = storageFactory;+ this.hookCallbackTimeout = hookCallbackTimeout;+ this.clock = clock;}// ---- Durable (persisted) ----@@ -845,8 +865,10 @@ public class CodeDeployService {if (anyFailed) {deployment.setStatus("Failed");- deployment.setErrorInformation(Map.of("code", "DeploymentFailed",- "message", "One or more instances failed deployment"));+ deployment.setErrorInformation(Map.of("code", "HEALTH_CONSTRAINTS",+ "message", "The overall deployment failed because too many individual instances failed "+ + "deployment, too few healthy instances are available for deployment, or some "+ + "instances in your deployment group are experiencing problems."));} else {deployment.setStatus("Succeeded");}@@ -925,8 +947,12 @@ public class CodeDeployService {// Check if instance is registered with SSMboolean hasSsm = ssmCommandService.isInstanceRegistered(instanceId, region);if (!hasSsm) {- LOG.debugv("Instance {0} not in SSM, marking hook {1} as Succeeded", instanceId, location);- continue;+ LOG.warnv("Instance {0} is not registered with SSM; cannot run hook script {1}",+ instanceId, location);+ finishLifecycleEvent(event, "Failed", "UnknownError",+ "CodeDeploy agent was not able to receive the lifecycle event. Check the CodeDeploy agent "+ + "logs on your host and make sure the agent is running and can connect to the CodeDeploy server.");+ return false;}try {@@ -938,21 +964,31 @@ public class CodeDeployService {instanceId, "AWS-RunShellScript", Map.of("commands", List.of(script)),timeout, region);- // Poll until done (max timeout seconds, capped at 30s for emulator)- long deadline = System.currentTimeMillis() + Math.min(timeout * 1000L, 30_000L);+ long deadline = clock.millis() + timeout * 1000L;String invocationStatus = "InProgress";- while (System.currentTimeMillis() < deadline && "InProgress".equals(invocationStatus)) {+ while (clock.millis() < deadline && "InProgress".equals(invocationStatus)) {Thread.sleep(500);invocationStatus = ssmCommandService.getCommandInvocationStatus(commandId, instanceId, region);}- if (!"Success".equals(invocationStatus) && !"InProgress".equals(invocationStatus)) {- finishLifecycleEvent(event, "Failed");+ if ("InProgress".equals(invocationStatus)) {+ LOG.warnv("Hook script {0} on instance {1} did not finish within {2}s",+ location, instanceId, timeout);+ finishLifecycleEvent(event, "Failed", "ScriptTimedOut",+ "Script at specified location: " + location + " failed to complete in " + timeout + " seconds");+ return false;+ }+ if (!"Success".equals(invocationStatus)) {+ finishLifecycleEvent(event, "Failed", "ScriptFailed",+ "Script at specified location: " + location + " failed with SSM command status "+ + invocationStatus);return false;}} catch (Exception e) {- LOG.debugv("SSM execution failed for {0} on {1}: {2}", location, instanceId, e.getMessage());- // Graceful degradation: if SSM fails, treat as succeeded+ LOG.warnv(e, "SSM execution failed for {0} on {1}: {2}", location, instanceId, e.getMessage());+ finishLifecycleEvent(event, "Failed", "UnknownError",+ "Failed to execute script " + location + ": " + e.getMessage());+ return false;}}finishLifecycleEvent(event, "Succeeded");@@ -1308,8 +1344,9 @@ public class CodeDeployService {if (appSpec.beforeInstall != null) {boolean ok = invokeHook(region, deployment, appSpec.beforeInstall,"BeforeInstall", ecsTargetMap, stopFlag);+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }if (!ok) {- finishEcsFailed(deployment, ecsTargetMap, "BeforeInstallHookFailed",+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE","Hook function reported failure: BeforeInstall");return;}@@ -1335,8 +1372,9 @@ public class CodeDeployService {if (appSpec.afterInstall != null) {boolean ok = invokeHook(region, deployment, appSpec.afterInstall,"AfterInstall", ecsTargetMap, stopFlag);+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }if (!ok) {- finishEcsFailed(deployment, ecsTargetMap, "AfterInstallHookFailed",+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE","Hook function reported failure: AfterInstall");return;}@@ -1347,8 +1385,9 @@ public class CodeDeployService {if (appSpec.beforeAllowTraffic != null) {boolean ok = invokeHook(region, deployment, appSpec.beforeAllowTraffic,"BeforeAllowTraffic", ecsTargetMap, stopFlag);+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }if (!ok) {- finishEcsFailed(deployment, ecsTargetMap, "BeforeAllowTrafficHookFailed",+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE","Hook function reported failure: BeforeAllowTraffic");return;}@@ -1370,8 +1409,9 @@ public class CodeDeployService {if (appSpec.afterAllowTraffic != null) {boolean ok = invokeHook(region, deployment, appSpec.afterAllowTraffic,"AfterAllowTraffic", ecsTargetMap, stopFlag);+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }if (!ok) {- finishEcsFailed(deployment, ecsTargetMap, "AfterAllowTrafficHookFailed",+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE","Hook function reported failure: AfterAllowTraffic");return;}@@ -1549,7 +1589,8 @@ public class CodeDeployService {if (appSpec.beforeAllowTraffic != null) {boolean ok = invokeHook(region, deployment, appSpec.beforeAllowTraffic,"BeforeAllowTraffic", lambdaTargetMap, stopFlag);- if (!ok) { finishFailed(deployment, lambdaTargetMap, "BeforeAllowTrafficHookFailed",+ if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }+ if (!ok) { finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE","Hook function reported failure: BeforeAllowTraffic"); return; }}@@ -1562,7 +1603,8 @@ public class CodeDeployService {if (appSpec.afterAllowTraffic != null) {boolean ok = invokeHook(region, deployment, appSpec.afterAllowTraffic,"AfterAllowTraffic", lambdaTargetMap, stopFlag);- if (!ok) { finishFailed(deployment, lambdaTargetMap, "AfterAllowTrafficHookFailed",+ if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }+ if (!ok) { finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE","Hook function reported failure: AfterAllowTraffic"); return; }}@@ -1651,26 +1693,59 @@ public class CodeDeployService {String payload = "{\"DeploymentId\":\"" + deployment.getDeploymentId()+ "\",\"LifecycleEventHookExecutionId\":\"" + executionId + "\"}";+ String failureMessage = null;try {InvokeResult result = lambdaService.invoke(region, hookFunctionName,payload.getBytes(), InvocationType.RequestResponse);- if (!future.isDone()) {- // Lambda didn't call PutLifecycleEventHookExecutionStatus; decide from invocation result- future.complete(result.getFunctionError() == null ? "Succeeded" : "Failed");+ if (result.getFunctionError() != null && !future.isDone()) {+ failureMessage = "Lambda function " + hookFunctionName+ + " returned a function error: " + result.getFunctionError();+ future.complete("Failed");}} catch (Exception e) {- LOG.debugv("Hook Lambda {0} not invokable: {1}", hookFunctionName, e.getMessage());- future.complete("Succeeded");+ LOG.warnv(e, "Hook Lambda {0} for lifecycle event {1} could not be invoked: {2}",+ hookFunctionName, lifecycleEventName, e.getMessage());+ failureMessage = "Lambda function " + hookFunctionName + " could not be invoked: " + e.getMessage();+ if (!future.isDone()) {+ future.complete("Failed");+ }}- String status = "Succeeded";- try {- status = future.get(30, TimeUnit.SECONDS);- } catch (Exception e) {- status = "Succeeded";+ String status = null;+ long deadlineNanos = System.nanoTime() + hookCallbackTimeout.toNanos();+ while (status == null) {+ if (stopFlag.get()) {+ finishLifecycleEvent(event, "Skipped");+ return false;+ }+ long remainingNanos = deadlineNanos - System.nanoTime();+ if (remainingNanos <= 0) {+ LOG.warnv("Hook {0} for lifecycle event {1} did not call PutLifecycleEventHookExecutionStatus within {2}",+ hookFunctionName, lifecycleEventName, hookCallbackTimeout);+ status = "Failed";+ failureMessage = "Lambda function " + hookFunctionName+ + " did not call PutLifecycleEventHookExecutionStatus within "+ + hookCallbackTimeout.toSeconds() + " seconds";+ break;+ }+ long sliceNanos = Math.min(remainingNanos, STOP_POLL_INTERVAL.toNanos());+ try {+ status = future.get(sliceNanos, TimeUnit.NANOSECONDS);+ } catch (TimeoutException e) {+ continue;+ } catch (ExecutionException e) {+ throw new IllegalStateException(e);+ }}- finishLifecycleEvent(event, status);+ if ("Failed".equals(status)) {+ finishLifecycleEvent(event, "Failed", "UnknownError",+ failureMessage != null ? failureMessage+ : "Lambda function " + hookFunctionName + " reported that lifecycle event "+ + lifecycleEventName + " failed");+ } else {+ finishLifecycleEvent(event, status);+ }return "Succeeded".equals(status);} finally {hookFutures.remove(executionId);@@ -1696,6 +1771,12 @@ public class CodeDeployService {event.put("status", status);}+ private void finishLifecycleEvent(Map<String, Object> event, String status,+ String diagnosticsErrorCode, String diagnosticsMessage) {+ finishLifecycleEvent(event, status);+ event.put("diagnostics", Map.of("errorCode", diagnosticsErrorCode, "message", diagnosticsMessage));+ }+private void updateTargetStatus(Map<String, Object> lambdaTargetMap, String status) {lambdaTargetMap.put("status", status);lambdaTargetMap.put("lastUpdatedAt", Instant.now().toEpochMilli() / 1000.0);diff --git a/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.javaindex 5113270ae7..44f3f7cfb0 100644--- a/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.java@@ -147,12 +147,9 @@ class CodeDeployServerIntegrationTest {@Test@Order(9)void createServerDeployment() {+ // on-prem-test-1 is not SSM-registered, so this AppSpec declares no hooks to run.String appSpec = """os: linux- hooks:- ApplicationStart:- - location: scripts/start_server.sh- timeout: 30""";String escapedAppSpec = appSpec.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n");@@ -231,6 +228,105 @@ class CodeDeployServerIntegrationTest {@Test@Order(13)+ void serverDeploymentWithUnreachableHookScriptFailsClosed() {+ given()+ .contentType(CT)+ .header("X-Amz-Target", "CodeDeploy_20141006.CreateApplication")+ .body("{\"applicationName\":\"server-test-app-hook\",\"computePlatform\":\"Server\"}")+ .when().post("/")+ .then().statusCode(200);++ given()+ .contentType(CT)+ .header("X-Amz-Target", "CodeDeploy_20141006.CreateDeploymentGroup")+ .body("""+ {+ "applicationName": "server-test-app-hook",+ "deploymentGroupName": "server-test-group-hook",+ "deploymentConfigName": "CodeDeployDefault.AllAtOnce",+ "serviceRoleArn": "arn:aws:iam::000000000000:role/CodeDeployRole",+ "onPremisesInstanceTagFilters": [{"Key": "Env", "Value": "test", "Type": "KEY_AND_VALUE"}]+ }+ """)+ .when().post("/")+ .then().statusCode(200);++ String appSpec = """+ os: linux+ hooks:+ ApplicationStart:+ - location: scripts/start_server.sh+ timeout: 30+ """;+ String escapedAppSpec = appSpec.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n");++ String hookDeploymentId = given()+ .contentType(CT)+ .header("X-Amz-Target", "CodeDeploy_20141006.CreateDeployment")+ .body("""+ {+ "applicationName": "server-test-app-hook",+ "deploymentGroupName": "server-test-group-hook",+ "description": "Test server deployment with unreachable hook",+ "revision": {+ "revisionType": "AppSpecContent",+ "appSpecContent": {+ "content": "%s"+ }+ }+ }+ """.formatted(escapedAppSpec))+ .when().post("/")+ .then().statusCode(200)+ .body("deploymentId", notNullValue())+ .extract().jsonPath().getString("deploymentId");++ String status = "Queued";+ for (int i = 0; i < 50 && !"Succeeded".equals(status) && !"Failed".equals(status); i++) {+ try {+ Thread.sleep(100);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ status = given()+ .contentType(CT)+ .header("X-Amz-Target", "CodeDeploy_20141006.GetDeployment")+ .body("{\"deploymentId\":\"" + hookDeploymentId + "\"}")+ .when().post("/")+ .then().statusCode(200)+ .extract().jsonPath().getString("deploymentInfo.status");+ }+ Assertions.assertEquals("Failed", status, "Deployment with an unreachable hook script must fail closed");++ given()+ .contentType(CT)+ .header("X-Amz-Target", "CodeDeploy_20141006.GetDeployment")+ .body("{\"deploymentId\":\"" + hookDeploymentId + "\"}")+ .when().post("/")+ .then().statusCode(200)+ .body("deploymentInfo.errorInformation.code", equalTo("HEALTH_CONSTRAINTS"));++ given()+ .contentType(CT)+ .header("X-Amz-Target", "CodeDeploy_20141006.BatchGetDeploymentTargets")+ .body("""+ {+ "deploymentId": "%s",+ "targetIds": ["on-prem-test-1"]+ }+ """.formatted(hookDeploymentId))+ .when().post("/")+ .then().statusCode(200)+ .body("deploymentTargets[0].instanceTarget.status", equalTo("Failed"))+ .body("deploymentTargets[0].instanceTarget.lifecycleEvents.find { it.lifecycleEventName == 'ApplicationStart' }.status",+ equalTo("Failed"))+ .body("deploymentTargets[0].instanceTarget.lifecycleEvents.find { it.lifecycleEventName == 'ApplicationStart' }.diagnostics.errorCode",+ equalTo("UnknownError"));+ }++ @Test+ @Order(14)void deregisterOnPremisesInstance() {given().contentType(CT)@@ -249,7 +345,7 @@ class CodeDeployServerIntegrationTest {}@Test- @Order(14)+ @Order(15)void listOnPremisesInstancesDeregistered() {given().contentType(CT)diff --git a/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.javanew file mode 100644index 0000000000..218662aa17--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java@@ -0,0 +1,439 @@+package io.github.hectorvent.floci.services.codedeploy;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.codedeploy.model.Deployment;+import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.services.ecs.EcsService;+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.lambda.model.InvocationType;+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;+import io.github.hectorvent.floci.services.ssm.SsmCommandService;+import io.github.hectorvent.floci.testing.MutableClock;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.time.Clock;+import java.time.Duration;+import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;++import static org.awaitility.Awaitility.await;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class CodeDeployServiceTest {++ private static final String REGION = "us-east-1";+ private static final String ACCOUNT_ID = "000000000000";+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/CodeDeployRole";+ private static final ObjectMapper JSON = new ObjectMapper();++ private LambdaService lambdaService;+ private SsmCommandService ssmCommandService;+ private CodeDeployService service;++ @BeforeEach+ void setUp() {+ lambdaService = mock(LambdaService.class);+ ssmCommandService = mock(SsmCommandService.class);+ service = newService(Duration.ofHours(1));+ }++ private CodeDeployService newService(Duration hookCallbackTimeout) {+ return newService(hookCallbackTimeout, Clock.systemUTC());+ }++ private CodeDeployService newService(Duration hookCallbackTimeout, Clock clock) {+ CodeDeployService svc = new CodeDeployService(lambdaService, mock(EcsService.class),+ mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),+ new RegionResolver(REGION, ACCOUNT_ID), null, hookCallbackTimeout, clock);+ svc.initializeStorage();+ return svc;+ }++ // ---- Server platform (SSM) hooks ---------------------------------------------------++ @Test+ void missingSsmInstanceFailsHook() {+ service.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",+ "arn:aws:iam::000000000000:user/u");+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(false);++ String deploymentId = createServerDeployment("app-missing-ssm", "group-missing-ssm", 30);++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Failed", deployment.getStatus());+ assertEquals("HEALTH_CONSTRAINTS", deployment.getErrorInformation().get("code"));+ Map<String, Object> event = instanceTargetEvent(deploymentId, "instance-1", "ApplicationStart");+ assertEquals("Failed", event.get("status"));+ Map<String, String> diagnostics = diagnosticsOf(event);+ assertEquals("UnknownError", diagnostics.get("errorCode"));+ assertEquals("CodeDeploy agent was not able to receive the lifecycle event. Check the CodeDeploy agent "+ + "logs on your host and make sure the agent is running and can connect to the CodeDeploy server.",+ diagnostics.get("message"));+ }++ @Test+ void ssmCommandFailureFailsHook() {+ service.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",+ "arn:aws:iam::000000000000:user/u");+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(true);+ when(ssmCommandService.sendCommandToInstance(eq("instance-1"), anyString(), any(), eq(30), eq(REGION)))+ .thenReturn("cmd-1");+ when(ssmCommandService.getCommandInvocationStatus("cmd-1", "instance-1", REGION)).thenReturn("Failed");++ String deploymentId = createServerDeployment("app-ssm-failure", "group-ssm-failure", 30);++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Failed", deployment.getStatus());+ assertEquals("HEALTH_CONSTRAINTS", deployment.getErrorInformation().get("code"));+ Map<String, Object> event = instanceTargetEvent(deploymentId, "instance-1", "ApplicationStart");+ assertEquals("Failed", event.get("status"));+ Map<String, String> diagnostics = diagnosticsOf(event);+ assertEquals("ScriptFailed", diagnostics.get("errorCode"));+ assertEquals("Script at specified location: scripts/start_server.sh failed with SSM command status Failed",+ diagnostics.get("message"));+ }++ @Test+ void ssmScriptTimeoutFailsHook() {+ service.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",+ "arn:aws:iam::000000000000:user/u");+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(true);+ when(ssmCommandService.sendCommandToInstance(eq("instance-1"), anyString(), any(), eq(1), eq(REGION)))+ .thenReturn("cmd-timeout");+ when(ssmCommandService.getCommandInvocationStatus("cmd-timeout", "instance-1", REGION))+ .thenReturn("InProgress");++ String deploymentId = createServerDeployment("app-ssm-timeout", "group-ssm-timeout", 1);++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Failed", deployment.getStatus());+ Map<String, Object> event = instanceTargetEvent(deploymentId, "instance-1", "ApplicationStart");+ Map<String, String> diagnostics = diagnosticsOf(event);+ assertEquals("ScriptTimedOut", diagnostics.get("errorCode"));+ assertEquals("Script at specified location: scripts/start_server.sh failed to complete in 1 seconds",+ diagnostics.get("message"));+ }++ @Test+ void ssmScriptRunningPastThirtySecondsWithinDeclaredTimeoutSucceeds() {+ MutableClock clock = new MutableClock();+ CodeDeployService clockedService = newService(Duration.ofHours(1), clock);+ clockedService.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",+ "arn:aws:iam::000000000000:user/u");+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(true);+ when(ssmCommandService.sendCommandToInstance(eq("instance-1"), anyString(), any(), eq(300), eq(REGION)))+ .thenReturn("cmd-long");+ AtomicInteger polls = new AtomicInteger();+ when(ssmCommandService.getCommandInvocationStatus("cmd-long", "instance-1", REGION))+ .thenAnswer(invocation -> {+ clock.advance(Duration.ofSeconds(20));+ return polls.incrementAndGet() < 3 ? "InProgress" : "Success";+ });++ String deploymentId = createServerDeployment(clockedService, "app-ssm-long", "group-ssm-long", 300);++ Deployment deployment = awaitTerminal(clockedService, deploymentId, Duration.ofSeconds(5));++ assertEquals("Succeeded", deployment.getStatus());+ }++ // ---- Lambda platform hooks ----------------------------------------------------------++ @Test+ void lambdaHookInvocationFailureMarksHookFailed() {+ createLambdaAppAndGroup("app-invoke-fail", "group-invoke-fail");+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenThrow(new RuntimeException("function not found"));++ String deploymentId = createLambdaDeployment("app-invoke-fail", "group-invoke-fail", "beforeHook", null);++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Failed", deployment.getStatus());+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));+ Map<String, Object> event = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");+ assertEquals("Failed", event.get("status"));+ Map<String, String> diagnostics = diagnosticsOf(event);+ assertEquals("UnknownError", diagnostics.get("errorCode"));+ assertEquals("Lambda function beforeHook could not be invoked: function not found",+ diagnostics.get("message"));+ }++ @Test+ void lambdaHookFunctionErrorMarksHookFailed() {+ createLambdaAppAndGroup("app-function-error", "group-function-error");+ InvokeResult erroring = new InvokeResult(200, "Unhandled", new byte[0], null, "req-1");+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(erroring);++ String deploymentId = createLambdaDeployment("app-function-error", "group-function-error", "beforeHook", null);++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Failed", deployment.getStatus());+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));+ Map<String, Object> event = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");+ assertEquals("Failed", event.get("status"));+ Map<String, String> diagnostics = diagnosticsOf(event);+ assertEquals("UnknownError", diagnostics.get("errorCode"));+ assertEquals("Lambda function beforeHook returned a function error: Unhandled",+ diagnostics.get("message"));+ }++ @Test+ void explicitSuccessfulCallbackMarksHookSucceeded() {+ createLambdaAppAndGroup("app-explicit-success", "group-explicit-success");+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-2");+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenAnswer(invocation -> {+ byte[] payload = invocation.getArgument(2);+ service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Succeeded");+ return cleanInvoke;+ });++ String deploymentId = createLambdaDeployment("app-explicit-success", "group-explicit-success", "beforeHook", null);++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Succeeded", deployment.getStatus());+ }++ @Test+ void lambdaHookCallbackTimeoutFailsHook() {+ CodeDeployService shortTimeoutService = newService(Duration.ofSeconds(1));+ shortTimeoutService.createApplication(REGION, "app-hook-timeout", "Lambda", null);+ shortTimeoutService.createDeploymentGroup(REGION, "app-hook-timeout", "group-hook-timeout",+ "CodeDeployDefault.LambdaAllAtOnce", ROLE_ARN, null);+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-3");+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(cleanInvoke);++ String deploymentId = shortTimeoutService.createDeployment(REGION, "app-hook-timeout",+ "group-hook-timeout", null, lambdaRevision("beforeHook", null), "no callback ever arrives");++ Deployment deployment = awaitTerminal(shortTimeoutService, deploymentId, Duration.ofSeconds(5));++ assertEquals("Failed", deployment.getStatus());+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));+ Map<String, Object> event = lambdaTargetEvent(shortTimeoutService, deploymentId, "BeforeAllowTraffic");+ assertEquals("Failed", event.get("status"));+ Map<String, String> diagnostics = diagnosticsOf(event);+ assertEquals("UnknownError", diagnostics.get("errorCode"));+ assertEquals("Lambda function beforeHook did not call PutLifecycleEventHookExecutionStatus within 1 seconds",+ diagnostics.get("message"));+ }++ @Test+ void stopDeploymentDuringHookWaitEndsAsStopped() {+ createLambdaAppAndGroup("app-stop-during-wait", "group-stop-during-wait");+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-6");+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenReturn(cleanInvoke);++ String deploymentId = createLambdaDeployment("app-stop-during-wait", "group-stop-during-wait",+ "beforeHook", null);++ // Wait until the hook has actually been invoked so the stop lands during the callback wait.+ await().atMost(Duration.ofSeconds(2)).untilAsserted(() ->+ verify(lambdaService).invoke(eq(REGION), eq("beforeHook"), any(byte[].class), any()));+ service.stopDeployment(REGION, deploymentId);++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Stopped", deployment.getStatus());+ Map<String, Object> event = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");+ assertEquals("Skipped", event.get("status"));+ }++ @Test+ void failedBeforeAllowTrafficHookPreventsTrafficShift() {+ createLambdaAppAndGroup("app-no-progress", "group-no-progress");+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-4");+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenAnswer(invocation -> {+ byte[] payload = invocation.getArgument(2);+ service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Failed");+ return cleanInvoke;+ });++ String deploymentId = createLambdaDeployment("app-no-progress", "group-no-progress", "beforeHook", "afterHook");++ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));++ assertEquals("Failed", deployment.getStatus());+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));+ Map<String, Object> beforeAllowTrafficEvent = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");+ assertEquals("Failed", beforeAllowTrafficEvent.get("status"));+ assertEquals("Lambda function beforeHook reported that lifecycle event BeforeAllowTraffic failed",+ diagnosticsOf(beforeAllowTrafficEvent).get("message"));+ verify(lambdaService, never()).invoke(eq(REGION), eq("afterHook"), any(byte[].class), any());+ verify(lambdaService, never()).updateAlias(anyString(), anyString(), anyString(), anyString(), any(), any());+ List<String> recordedEvents = lambdaTargetEvents(deploymentId).stream()+ .map(e -> (String) e.get("lifecycleEventName"))+ .toList();+ assertTrue(recordedEvents.contains("BeforeAllowTraffic"));+ assertFalse(recordedEvents.contains("AllowTraffic"), "AllowTraffic must not run after a failed hook");+ assertFalse(recordedEvents.contains("AfterAllowTraffic"), "AfterAllowTraffic must not run after a failed hook");+ }++ @Test+ void lateLifecycleCallbackAfterDeploymentTerminalIsNoOp() {+ createLambdaAppAndGroup("app-late-callback", "group-late-callback");+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-5");+ AtomicReference<String> capturedExecutionId = new AtomicReference<>();+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))+ .thenAnswer(invocation -> {+ byte[] payload = invocation.getArgument(2);+ String executionId = executionIdFrom(payload);+ capturedExecutionId.set(executionId);+ service.putLifecycleEventHookExecutionStatus("ignored", executionId, "Succeeded");+ return cleanInvoke;+ });++ String deploymentId = createLambdaDeployment("app-late-callback", "group-late-callback", "beforeHook", null);+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));+ assertEquals("Succeeded", deployment.getStatus());++ service.putLifecycleEventHookExecutionStatus("ignored", capturedExecutionId.get(), "Failed");++ assertEquals("Succeeded", service.getDeployment(REGION, deploymentId).getStatus());+ }++ // ---- Helpers --------------------------------------------------------------------------++ private String createServerDeployment(String appName, String groupName, int timeoutSeconds) {+ return createServerDeployment(service, appName, groupName, timeoutSeconds);+ }++ private String createServerDeployment(CodeDeployService target, String appName, String groupName,+ int timeoutSeconds) {+ target.createApplication(REGION, appName, "Server", null);+ target.createDeploymentGroup(REGION, appName, groupName, "CodeDeployDefault.AllAtOnce", ROLE_ARN, null);+ String appSpec = """+ os: linux+ hooks:+ ApplicationStart:+ - location: scripts/start_server.sh+ timeout: %d+ """.formatted(timeoutSeconds);+ Map<String, Object> revision = Map.of("revisionType", "AppSpecContent",+ "appSpecContent", Map.of("content", appSpec));+ return target.createDeployment(REGION, appName, groupName, null, revision, "server hook test");+ }++ private void createLambdaAppAndGroup(String appName, String groupName) {+ service.createApplication(REGION, appName, "Lambda", null);+ service.createDeploymentGroup(REGION, appName, groupName,+ "CodeDeployDefault.LambdaAllAtOnce", ROLE_ARN, null);+ }++ private String createLambdaDeployment(String appName, String groupName, String beforeHook, String afterHook) {+ return service.createDeployment(REGION, appName, groupName, null,+ lambdaRevision(beforeHook, afterHook), "lambda hook test");+ }++ private Map<String, Object> lambdaRevision(String beforeHook, String afterHook) {+ StringBuilder hooks = new StringBuilder("[");+ if (beforeHook != null) {+ hooks.append("{\"BeforeAllowTraffic\":\"").append(beforeHook).append("\"}");+ }+ if (afterHook != null) {+ if (hooks.length() > 1) {+ hooks.append(",");+ }+ hooks.append("{\"AfterAllowTraffic\":\"").append(afterHook).append("\"}");+ }+ hooks.append("]");+ String content = """+ {+ "version": "0.0",+ "Resources": [+ {"myFunc": {"Type": "AWS::Lambda::Function", "Properties": {+ "Name": "myFunc", "Alias": "live", "CurrentVersion": "1", "TargetVersion": "2"+ }}}+ ],+ "Hooks": %s+ }+ """.formatted(hooks);+ return Map.of("revisionType", "AppSpecContent", "appSpecContent", Map.of("content", content));+ }++ private static String executionIdFrom(byte[] payload) {+ try {+ JsonNode node = JSON.readTree(payload);+ return node.get("LifecycleEventHookExecutionId").asText();+ } catch (Exception e) {+ throw new IllegalStateException("Could not parse hook payload", e);+ }+ }++ private Deployment awaitTerminal(String deploymentId, Duration atMost) {+ return awaitTerminal(service, deploymentId, atMost);+ }++ private Deployment awaitTerminal(CodeDeployService target, String deploymentId, Duration atMost) {+ await().atMost(atMost).pollInterval(Duration.ofMillis(25)).until(() ->+ isTerminal(target.getDeployment(REGION, deploymentId).getStatus()));+ return target.getDeployment(REGION, deploymentId);+ }++ private boolean isTerminal(String status) {+ return "Succeeded".equals(status) || "Failed".equals(status) || "Stopped".equals(status);+ }++ @SuppressWarnings("unchecked")+ private Map<String, Object> instanceTargetEvent(String deploymentId, String targetId, String eventName) {+ Map<String, Object> target = service.batchGetDeploymentTargets(REGION, deploymentId, List.of(targetId)).get(0);+ Map<String, Object> instanceTarget = (Map<String, Object>) target.get("instanceTarget");+ List<Map<String, Object>> events = (List<Map<String, Object>>) instanceTarget.get("lifecycleEvents");+ return events.stream()+ .filter(e -> eventName.equals(e.get("lifecycleEventName")))+ .findFirst()+ .orElseThrow(() -> new AssertionError("No lifecycle event named " + eventName));+ }++ private Map<String, Object> lambdaTargetEvent(String deploymentId, String eventName) {+ return lambdaTargetEvent(service, deploymentId, eventName);+ }++ private Map<String, Object> lambdaTargetEvent(CodeDeployService target, String deploymentId, String eventName) {+ return lambdaTargetEvents(target, deploymentId).stream()+ .filter(e -> eventName.equals(e.get("lifecycleEventName")))+ .findFirst()+ .orElseThrow(() -> new AssertionError("No lifecycle event named " + eventName));+ }++ private List<Map<String, Object>> lambdaTargetEvents(String deploymentId) {+ return lambdaTargetEvents(service, deploymentId);+ }++ @SuppressWarnings("unchecked")+ private List<Map<String, Object>> lambdaTargetEvents(CodeDeployService target, String deploymentId) {+ Map<String, Object> targetMap = target.batchGetDeploymentTargets(REGION, deploymentId, List.of()).get(0);+ Map<String, Object> lambdaTarget = (Map<String, Object>) targetMap.get("lambdaTarget");+ return (List<Map<String, Object>>) lambdaTarget.get("lifecycleEvents");+ }++ @SuppressWarnings("unchecked")+ private Map<String, String> diagnosticsOf(Map<String, Object> event) {+ return (Map<String, String>) event.get("diagnostics");+ }+}