Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codedeploy): fail closed on unreachable, erroring, or timed-out lifecycle hooks (#3563)

Niels Kootstrad167cc5bf378
diff --git a/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java b/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java
index 26c5e04211..fab0f25353 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployService.java
@@ -28,6 +28,8 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.time.Clock;
+import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
@@ -36,7 +38,9 @@ import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@@ -45,6 +49,10 @@ public class CodeDeployService {
private static final Logger LOG = Logger.getLogger(CodeDeployService.class);
+ // Lambda/ECS validation hooks must call PutLifecycleEventHookExecutionStatus within one hour.
+ private static final Duration DEFAULT_HOOK_CALLBACK_TIMEOUT = Duration.ofHours(1);
+ private static final Duration STOP_POLL_INTERVAL = Duration.ofMillis(200);
+
private final LambdaService lambdaService;
private final EcsService ecsService;
private final ElbV2Service elbV2Service;
@@ -54,12 +62,22 @@ public class CodeDeployService {
private final ObjectMapper yamlMapper;
private final RegionResolver regionResolver;
private final StorageFactory storageFactory;
+ private final Duration hookCallbackTimeout;
+ private final Clock clock;
@Inject
public CodeDeployService(LambdaService lambdaService, EcsService ecsService,
ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
StorageFactory storageFactory) {
+ this(lambdaService, ecsService, elbV2Service, ssmCommandService, ec2Service, mapper,
+ regionResolver, storageFactory, DEFAULT_HOOK_CALLBACK_TIMEOUT, Clock.systemUTC());
+ }
+
+ CodeDeployService(LambdaService lambdaService, EcsService ecsService,
+ ElbV2Service elbV2Service, SsmCommandService ssmCommandService,
+ Ec2Service ec2Service, ObjectMapper mapper, RegionResolver regionResolver,
+ StorageFactory storageFactory, Duration hookCallbackTimeout, Clock clock) {
this.lambdaService = lambdaService;
this.ecsService = ecsService;
this.elbV2Service = elbV2Service;
@@ -69,6 +87,8 @@ public class CodeDeployService {
this.yamlMapper = new ObjectMapper(new YAMLFactory());
this.regionResolver = regionResolver;
this.storageFactory = storageFactory;
+ this.hookCallbackTimeout = hookCallbackTimeout;
+ this.clock = clock;
}
// ---- Durable (persisted) ----
@@ -845,8 +865,10 @@ public class CodeDeployService {
if (anyFailed) {
deployment.setStatus("Failed");
- deployment.setErrorInformation(Map.of("code", "DeploymentFailed",
- "message", "One or more instances failed deployment"));
+ deployment.setErrorInformation(Map.of("code", "HEALTH_CONSTRAINTS",
+ "message", "The overall deployment failed because too many individual instances failed "
+ + "deployment, too few healthy instances are available for deployment, or some "
+ + "instances in your deployment group are experiencing problems."));
} else {
deployment.setStatus("Succeeded");
}
@@ -925,8 +947,12 @@ public class CodeDeployService {
// Check if instance is registered with SSM
boolean hasSsm = ssmCommandService.isInstanceRegistered(instanceId, region);
if (!hasSsm) {
- LOG.debugv("Instance {0} not in SSM, marking hook {1} as Succeeded", instanceId, location);
- continue;
+ LOG.warnv("Instance {0} is not registered with SSM; cannot run hook script {1}",
+ instanceId, location);
+ finishLifecycleEvent(event, "Failed", "UnknownError",
+ "CodeDeploy agent was not able to receive the lifecycle event. Check the CodeDeploy agent "
+ + "logs on your host and make sure the agent is running and can connect to the CodeDeploy server.");
+ return false;
}
try {
@@ -938,21 +964,31 @@ public class CodeDeployService {
instanceId, "AWS-RunShellScript", Map.of("commands", List.of(script)),
timeout, region);
- // Poll until done (max timeout seconds, capped at 30s for emulator)
- long deadline = System.currentTimeMillis() + Math.min(timeout * 1000L, 30_000L);
+ long deadline = clock.millis() + timeout * 1000L;
String invocationStatus = "InProgress";
- while (System.currentTimeMillis() < deadline && "InProgress".equals(invocationStatus)) {
+ while (clock.millis() < deadline && "InProgress".equals(invocationStatus)) {
Thread.sleep(500);
invocationStatus = ssmCommandService.getCommandInvocationStatus(commandId, instanceId, region);
}
- if (!"Success".equals(invocationStatus) && !"InProgress".equals(invocationStatus)) {
- finishLifecycleEvent(event, "Failed");
+ if ("InProgress".equals(invocationStatus)) {
+ LOG.warnv("Hook script {0} on instance {1} did not finish within {2}s",
+ location, instanceId, timeout);
+ finishLifecycleEvent(event, "Failed", "ScriptTimedOut",
+ "Script at specified location: " + location + " failed to complete in " + timeout + " seconds");
+ return false;
+ }
+ if (!"Success".equals(invocationStatus)) {
+ finishLifecycleEvent(event, "Failed", "ScriptFailed",
+ "Script at specified location: " + location + " failed with SSM command status "
+ + invocationStatus);
return false;
}
} catch (Exception e) {
- LOG.debugv("SSM execution failed for {0} on {1}: {2}", location, instanceId, e.getMessage());
- // Graceful degradation: if SSM fails, treat as succeeded
+ LOG.warnv(e, "SSM execution failed for {0} on {1}: {2}", location, instanceId, e.getMessage());
+ finishLifecycleEvent(event, "Failed", "UnknownError",
+ "Failed to execute script " + location + ": " + e.getMessage());
+ return false;
}
}
finishLifecycleEvent(event, "Succeeded");
@@ -1308,8 +1344,9 @@ public class CodeDeployService {
if (appSpec.beforeInstall != null) {
boolean ok = invokeHook(region, deployment, appSpec.beforeInstall,
"BeforeInstall", ecsTargetMap, stopFlag);
+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }
if (!ok) {
- finishEcsFailed(deployment, ecsTargetMap, "BeforeInstallHookFailed",
+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE",
"Hook function reported failure: BeforeInstall");
return;
}
@@ -1335,8 +1372,9 @@ public class CodeDeployService {
if (appSpec.afterInstall != null) {
boolean ok = invokeHook(region, deployment, appSpec.afterInstall,
"AfterInstall", ecsTargetMap, stopFlag);
+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }
if (!ok) {
- finishEcsFailed(deployment, ecsTargetMap, "AfterInstallHookFailed",
+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE",
"Hook function reported failure: AfterInstall");
return;
}
@@ -1347,8 +1385,9 @@ public class CodeDeployService {
if (appSpec.beforeAllowTraffic != null) {
boolean ok = invokeHook(region, deployment, appSpec.beforeAllowTraffic,
"BeforeAllowTraffic", ecsTargetMap, stopFlag);
+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }
if (!ok) {
- finishEcsFailed(deployment, ecsTargetMap, "BeforeAllowTrafficHookFailed",
+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE",
"Hook function reported failure: BeforeAllowTraffic");
return;
}
@@ -1370,8 +1409,9 @@ public class CodeDeployService {
if (appSpec.afterAllowTraffic != null) {
boolean ok = invokeHook(region, deployment, appSpec.afterAllowTraffic,
"AfterAllowTraffic", ecsTargetMap, stopFlag);
+ if (stopFlag.get()) { finishEcsStopped(deployment, ecsTargetMap); return; }
if (!ok) {
- finishEcsFailed(deployment, ecsTargetMap, "AfterAllowTrafficHookFailed",
+ finishEcsFailed(deployment, ecsTargetMap, "HOOK_EXECUTION_FAILURE",
"Hook function reported failure: AfterAllowTraffic");
return;
}
@@ -1549,7 +1589,8 @@ public class CodeDeployService {
if (appSpec.beforeAllowTraffic != null) {
boolean ok = invokeHook(region, deployment, appSpec.beforeAllowTraffic,
"BeforeAllowTraffic", lambdaTargetMap, stopFlag);
- if (!ok) { finishFailed(deployment, lambdaTargetMap, "BeforeAllowTrafficHookFailed",
+ if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
+ if (!ok) { finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE",
"Hook function reported failure: BeforeAllowTraffic"); return; }
}
@@ -1562,7 +1603,8 @@ public class CodeDeployService {
if (appSpec.afterAllowTraffic != null) {
boolean ok = invokeHook(region, deployment, appSpec.afterAllowTraffic,
"AfterAllowTraffic", lambdaTargetMap, stopFlag);
- if (!ok) { finishFailed(deployment, lambdaTargetMap, "AfterAllowTrafficHookFailed",
+ if (stopFlag.get()) { finishStopped(deployment, lambdaTargetMap); return; }
+ if (!ok) { finishFailed(deployment, lambdaTargetMap, "HOOK_EXECUTION_FAILURE",
"Hook function reported failure: AfterAllowTraffic"); return; }
}
@@ -1651,26 +1693,59 @@ public class CodeDeployService {
String payload = "{\"DeploymentId\":\"" + deployment.getDeploymentId()
+ "\",\"LifecycleEventHookExecutionId\":\"" + executionId + "\"}";
+ String failureMessage = null;
try {
InvokeResult result = lambdaService.invoke(region, hookFunctionName,
payload.getBytes(), InvocationType.RequestResponse);
- if (!future.isDone()) {
- // Lambda didn't call PutLifecycleEventHookExecutionStatus; decide from invocation result
- future.complete(result.getFunctionError() == null ? "Succeeded" : "Failed");
+ if (result.getFunctionError() != null && !future.isDone()) {
+ failureMessage = "Lambda function " + hookFunctionName
+ + " returned a function error: " + result.getFunctionError();
+ future.complete("Failed");
}
} catch (Exception e) {
- LOG.debugv("Hook Lambda {0} not invokable: {1}", hookFunctionName, e.getMessage());
- future.complete("Succeeded");
+ LOG.warnv(e, "Hook Lambda {0} for lifecycle event {1} could not be invoked: {2}",
+ hookFunctionName, lifecycleEventName, e.getMessage());
+ failureMessage = "Lambda function " + hookFunctionName + " could not be invoked: " + e.getMessage();
+ if (!future.isDone()) {
+ future.complete("Failed");
+ }
}
- String status = "Succeeded";
- try {
- status = future.get(30, TimeUnit.SECONDS);
- } catch (Exception e) {
- status = "Succeeded";
+ String status = null;
+ long deadlineNanos = System.nanoTime() + hookCallbackTimeout.toNanos();
+ while (status == null) {
+ if (stopFlag.get()) {
+ finishLifecycleEvent(event, "Skipped");
+ return false;
+ }
+ long remainingNanos = deadlineNanos - System.nanoTime();
+ if (remainingNanos <= 0) {
+ LOG.warnv("Hook {0} for lifecycle event {1} did not call PutLifecycleEventHookExecutionStatus within {2}",
+ hookFunctionName, lifecycleEventName, hookCallbackTimeout);
+ status = "Failed";
+ failureMessage = "Lambda function " + hookFunctionName
+ + " did not call PutLifecycleEventHookExecutionStatus within "
+ + hookCallbackTimeout.toSeconds() + " seconds";
+ break;
+ }
+ long sliceNanos = Math.min(remainingNanos, STOP_POLL_INTERVAL.toNanos());
+ try {
+ status = future.get(sliceNanos, TimeUnit.NANOSECONDS);
+ } catch (TimeoutException e) {
+ continue;
+ } catch (ExecutionException e) {
+ throw new IllegalStateException(e);
+ }
}
- finishLifecycleEvent(event, status);
+ if ("Failed".equals(status)) {
+ finishLifecycleEvent(event, "Failed", "UnknownError",
+ failureMessage != null ? failureMessage
+ : "Lambda function " + hookFunctionName + " reported that lifecycle event "
+ + lifecycleEventName + " failed");
+ } else {
+ finishLifecycleEvent(event, status);
+ }
return "Succeeded".equals(status);
} finally {
hookFutures.remove(executionId);
@@ -1696,6 +1771,12 @@ public class CodeDeployService {
event.put("status", status);
}
+ private void finishLifecycleEvent(Map<String, Object> event, String status,
+ String diagnosticsErrorCode, String diagnosticsMessage) {
+ finishLifecycleEvent(event, status);
+ event.put("diagnostics", Map.of("errorCode", diagnosticsErrorCode, "message", diagnosticsMessage));
+ }
+
private void updateTargetStatus(Map<String, Object> lambdaTargetMap, String status) {
lambdaTargetMap.put("status", status);
lambdaTargetMap.put("lastUpdatedAt", Instant.now().toEpochMilli() / 1000.0);
diff --git a/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.java
index 5113270ae7..44f3f7cfb0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServerIntegrationTest.java
@@ -147,12 +147,9 @@ class CodeDeployServerIntegrationTest {
@Test
@Order(9)
void createServerDeployment() {
+ // on-prem-test-1 is not SSM-registered, so this AppSpec declares no hooks to run.
String appSpec = """
os: linux
- hooks:
- ApplicationStart:
- - location: scripts/start_server.sh
- timeout: 30
""";
String escapedAppSpec = appSpec.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n");
@@ -231,6 +228,105 @@ class CodeDeployServerIntegrationTest {
@Test
@Order(13)
+ void serverDeploymentWithUnreachableHookScriptFailsClosed() {
+ given()
+ .contentType(CT)
+ .header("X-Amz-Target", "CodeDeploy_20141006.CreateApplication")
+ .body("{\"applicationName\":\"server-test-app-hook\",\"computePlatform\":\"Server\"}")
+ .when().post("/")
+ .then().statusCode(200);
+
+ given()
+ .contentType(CT)
+ .header("X-Amz-Target", "CodeDeploy_20141006.CreateDeploymentGroup")
+ .body("""
+ {
+ "applicationName": "server-test-app-hook",
+ "deploymentGroupName": "server-test-group-hook",
+ "deploymentConfigName": "CodeDeployDefault.AllAtOnce",
+ "serviceRoleArn": "arn:aws:iam::000000000000:role/CodeDeployRole",
+ "onPremisesInstanceTagFilters": [{"Key": "Env", "Value": "test", "Type": "KEY_AND_VALUE"}]
+ }
+ """)
+ .when().post("/")
+ .then().statusCode(200);
+
+ String appSpec = """
+ os: linux
+ hooks:
+ ApplicationStart:
+ - location: scripts/start_server.sh
+ timeout: 30
+ """;
+ String escapedAppSpec = appSpec.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n");
+
+ String hookDeploymentId = given()
+ .contentType(CT)
+ .header("X-Amz-Target", "CodeDeploy_20141006.CreateDeployment")
+ .body("""
+ {
+ "applicationName": "server-test-app-hook",
+ "deploymentGroupName": "server-test-group-hook",
+ "description": "Test server deployment with unreachable hook",
+ "revision": {
+ "revisionType": "AppSpecContent",
+ "appSpecContent": {
+ "content": "%s"
+ }
+ }
+ }
+ """.formatted(escapedAppSpec))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("deploymentId", notNullValue())
+ .extract().jsonPath().getString("deploymentId");
+
+ String status = "Queued";
+ for (int i = 0; i < 50 && !"Succeeded".equals(status) && !"Failed".equals(status); i++) {
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ status = given()
+ .contentType(CT)
+ .header("X-Amz-Target", "CodeDeploy_20141006.GetDeployment")
+ .body("{\"deploymentId\":\"" + hookDeploymentId + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .extract().jsonPath().getString("deploymentInfo.status");
+ }
+ Assertions.assertEquals("Failed", status, "Deployment with an unreachable hook script must fail closed");
+
+ given()
+ .contentType(CT)
+ .header("X-Amz-Target", "CodeDeploy_20141006.GetDeployment")
+ .body("{\"deploymentId\":\"" + hookDeploymentId + "\"}")
+ .when().post("/")
+ .then().statusCode(200)
+ .body("deploymentInfo.errorInformation.code", equalTo("HEALTH_CONSTRAINTS"));
+
+ given()
+ .contentType(CT)
+ .header("X-Amz-Target", "CodeDeploy_20141006.BatchGetDeploymentTargets")
+ .body("""
+ {
+ "deploymentId": "%s",
+ "targetIds": ["on-prem-test-1"]
+ }
+ """.formatted(hookDeploymentId))
+ .when().post("/")
+ .then().statusCode(200)
+ .body("deploymentTargets[0].instanceTarget.status", equalTo("Failed"))
+ .body("deploymentTargets[0].instanceTarget.lifecycleEvents.find { it.lifecycleEventName == 'ApplicationStart' }.status",
+ equalTo("Failed"))
+ .body("deploymentTargets[0].instanceTarget.lifecycleEvents.find { it.lifecycleEventName == 'ApplicationStart' }.diagnostics.errorCode",
+ equalTo("UnknownError"));
+ }
+
+ @Test
+ @Order(14)
void deregisterOnPremisesInstance() {
given()
.contentType(CT)
@@ -249,7 +345,7 @@ class CodeDeployServerIntegrationTest {
}
@Test
- @Order(14)
+ @Order(15)
void listOnPremisesInstancesDeregistered() {
given()
.contentType(CT)
diff --git a/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java
new file mode 100644
index 0000000000..218662aa17
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codedeploy/CodeDeployServiceTest.java
@@ -0,0 +1,439 @@
+package io.github.hectorvent.floci.services.codedeploy;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.codedeploy.model.Deployment;
+import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.services.ecs.EcsService;
+import io.github.hectorvent.floci.services.elbv2.ElbV2Service;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.lambda.model.InvocationType;
+import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
+import io.github.hectorvent.floci.services.ssm.SsmCommandService;
+import io.github.hectorvent.floci.testing.MutableClock;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.awaitility.Awaitility.await;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class CodeDeployServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private static final String ACCOUNT_ID = "000000000000";
+ private static final String ROLE_ARN = "arn:aws:iam::000000000000:role/CodeDeployRole";
+ private static final ObjectMapper JSON = new ObjectMapper();
+
+ private LambdaService lambdaService;
+ private SsmCommandService ssmCommandService;
+ private CodeDeployService service;
+
+ @BeforeEach
+ void setUp() {
+ lambdaService = mock(LambdaService.class);
+ ssmCommandService = mock(SsmCommandService.class);
+ service = newService(Duration.ofHours(1));
+ }
+
+ private CodeDeployService newService(Duration hookCallbackTimeout) {
+ return newService(hookCallbackTimeout, Clock.systemUTC());
+ }
+
+ private CodeDeployService newService(Duration hookCallbackTimeout, Clock clock) {
+ CodeDeployService svc = new CodeDeployService(lambdaService, mock(EcsService.class),
+ mock(ElbV2Service.class), ssmCommandService, mock(Ec2Service.class), new ObjectMapper(),
+ new RegionResolver(REGION, ACCOUNT_ID), null, hookCallbackTimeout, clock);
+ svc.initializeStorage();
+ return svc;
+ }
+
+ // ---- Server platform (SSM) hooks ---------------------------------------------------
+
+ @Test
+ void missingSsmInstanceFailsHook() {
+ service.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",
+ "arn:aws:iam::000000000000:user/u");
+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(false);
+
+ String deploymentId = createServerDeployment("app-missing-ssm", "group-missing-ssm", 30);
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Failed", deployment.getStatus());
+ assertEquals("HEALTH_CONSTRAINTS", deployment.getErrorInformation().get("code"));
+ Map<String, Object> event = instanceTargetEvent(deploymentId, "instance-1", "ApplicationStart");
+ assertEquals("Failed", event.get("status"));
+ Map<String, String> diagnostics = diagnosticsOf(event);
+ assertEquals("UnknownError", diagnostics.get("errorCode"));
+ assertEquals("CodeDeploy agent was not able to receive the lifecycle event. Check the CodeDeploy agent "
+ + "logs on your host and make sure the agent is running and can connect to the CodeDeploy server.",
+ diagnostics.get("message"));
+ }
+
+ @Test
+ void ssmCommandFailureFailsHook() {
+ service.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",
+ "arn:aws:iam::000000000000:user/u");
+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(true);
+ when(ssmCommandService.sendCommandToInstance(eq("instance-1"), anyString(), any(), eq(30), eq(REGION)))
+ .thenReturn("cmd-1");
+ when(ssmCommandService.getCommandInvocationStatus("cmd-1", "instance-1", REGION)).thenReturn("Failed");
+
+ String deploymentId = createServerDeployment("app-ssm-failure", "group-ssm-failure", 30);
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Failed", deployment.getStatus());
+ assertEquals("HEALTH_CONSTRAINTS", deployment.getErrorInformation().get("code"));
+ Map<String, Object> event = instanceTargetEvent(deploymentId, "instance-1", "ApplicationStart");
+ assertEquals("Failed", event.get("status"));
+ Map<String, String> diagnostics = diagnosticsOf(event);
+ assertEquals("ScriptFailed", diagnostics.get("errorCode"));
+ assertEquals("Script at specified location: scripts/start_server.sh failed with SSM command status Failed",
+ diagnostics.get("message"));
+ }
+
+ @Test
+ void ssmScriptTimeoutFailsHook() {
+ service.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",
+ "arn:aws:iam::000000000000:user/u");
+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(true);
+ when(ssmCommandService.sendCommandToInstance(eq("instance-1"), anyString(), any(), eq(1), eq(REGION)))
+ .thenReturn("cmd-timeout");
+ when(ssmCommandService.getCommandInvocationStatus("cmd-timeout", "instance-1", REGION))
+ .thenReturn("InProgress");
+
+ String deploymentId = createServerDeployment("app-ssm-timeout", "group-ssm-timeout", 1);
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Failed", deployment.getStatus());
+ Map<String, Object> event = instanceTargetEvent(deploymentId, "instance-1", "ApplicationStart");
+ Map<String, String> diagnostics = diagnosticsOf(event);
+ assertEquals("ScriptTimedOut", diagnostics.get("errorCode"));
+ assertEquals("Script at specified location: scripts/start_server.sh failed to complete in 1 seconds",
+ diagnostics.get("message"));
+ }
+
+ @Test
+ void ssmScriptRunningPastThirtySecondsWithinDeclaredTimeoutSucceeds() {
+ MutableClock clock = new MutableClock();
+ CodeDeployService clockedService = newService(Duration.ofHours(1), clock);
+ clockedService.registerOnPremisesInstance(REGION, "instance-1", "arn:aws:sts::000000000000:session/s",
+ "arn:aws:iam::000000000000:user/u");
+ when(ssmCommandService.isInstanceRegistered("instance-1", REGION)).thenReturn(true);
+ when(ssmCommandService.sendCommandToInstance(eq("instance-1"), anyString(), any(), eq(300), eq(REGION)))
+ .thenReturn("cmd-long");
+ AtomicInteger polls = new AtomicInteger();
+ when(ssmCommandService.getCommandInvocationStatus("cmd-long", "instance-1", REGION))
+ .thenAnswer(invocation -> {
+ clock.advance(Duration.ofSeconds(20));
+ return polls.incrementAndGet() < 3 ? "InProgress" : "Success";
+ });
+
+ String deploymentId = createServerDeployment(clockedService, "app-ssm-long", "group-ssm-long", 300);
+
+ Deployment deployment = awaitTerminal(clockedService, deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Succeeded", deployment.getStatus());
+ }
+
+ // ---- Lambda platform hooks ----------------------------------------------------------
+
+ @Test
+ void lambdaHookInvocationFailureMarksHookFailed() {
+ createLambdaAppAndGroup("app-invoke-fail", "group-invoke-fail");
+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenThrow(new RuntimeException("function not found"));
+
+ String deploymentId = createLambdaDeployment("app-invoke-fail", "group-invoke-fail", "beforeHook", null);
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Failed", deployment.getStatus());
+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));
+ Map<String, Object> event = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");
+ assertEquals("Failed", event.get("status"));
+ Map<String, String> diagnostics = diagnosticsOf(event);
+ assertEquals("UnknownError", diagnostics.get("errorCode"));
+ assertEquals("Lambda function beforeHook could not be invoked: function not found",
+ diagnostics.get("message"));
+ }
+
+ @Test
+ void lambdaHookFunctionErrorMarksHookFailed() {
+ createLambdaAppAndGroup("app-function-error", "group-function-error");
+ InvokeResult erroring = new InvokeResult(200, "Unhandled", new byte[0], null, "req-1");
+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(erroring);
+
+ String deploymentId = createLambdaDeployment("app-function-error", "group-function-error", "beforeHook", null);
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Failed", deployment.getStatus());
+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));
+ Map<String, Object> event = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");
+ assertEquals("Failed", event.get("status"));
+ Map<String, String> diagnostics = diagnosticsOf(event);
+ assertEquals("UnknownError", diagnostics.get("errorCode"));
+ assertEquals("Lambda function beforeHook returned a function error: Unhandled",
+ diagnostics.get("message"));
+ }
+
+ @Test
+ void explicitSuccessfulCallbackMarksHookSucceeded() {
+ createLambdaAppAndGroup("app-explicit-success", "group-explicit-success");
+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-2");
+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenAnswer(invocation -> {
+ byte[] payload = invocation.getArgument(2);
+ service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Succeeded");
+ return cleanInvoke;
+ });
+
+ String deploymentId = createLambdaDeployment("app-explicit-success", "group-explicit-success", "beforeHook", null);
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Succeeded", deployment.getStatus());
+ }
+
+ @Test
+ void lambdaHookCallbackTimeoutFailsHook() {
+ CodeDeployService shortTimeoutService = newService(Duration.ofSeconds(1));
+ shortTimeoutService.createApplication(REGION, "app-hook-timeout", "Lambda", null);
+ shortTimeoutService.createDeploymentGroup(REGION, "app-hook-timeout", "group-hook-timeout",
+ "CodeDeployDefault.LambdaAllAtOnce", ROLE_ARN, null);
+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-3");
+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(cleanInvoke);
+
+ String deploymentId = shortTimeoutService.createDeployment(REGION, "app-hook-timeout",
+ "group-hook-timeout", null, lambdaRevision("beforeHook", null), "no callback ever arrives");
+
+ Deployment deployment = awaitTerminal(shortTimeoutService, deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Failed", deployment.getStatus());
+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));
+ Map<String, Object> event = lambdaTargetEvent(shortTimeoutService, deploymentId, "BeforeAllowTraffic");
+ assertEquals("Failed", event.get("status"));
+ Map<String, String> diagnostics = diagnosticsOf(event);
+ assertEquals("UnknownError", diagnostics.get("errorCode"));
+ assertEquals("Lambda function beforeHook did not call PutLifecycleEventHookExecutionStatus within 1 seconds",
+ diagnostics.get("message"));
+ }
+
+ @Test
+ void stopDeploymentDuringHookWaitEndsAsStopped() {
+ createLambdaAppAndGroup("app-stop-during-wait", "group-stop-during-wait");
+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-6");
+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenReturn(cleanInvoke);
+
+ String deploymentId = createLambdaDeployment("app-stop-during-wait", "group-stop-during-wait",
+ "beforeHook", null);
+
+ // Wait until the hook has actually been invoked so the stop lands during the callback wait.
+ await().atMost(Duration.ofSeconds(2)).untilAsserted(() ->
+ verify(lambdaService).invoke(eq(REGION), eq("beforeHook"), any(byte[].class), any()));
+ service.stopDeployment(REGION, deploymentId);
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Stopped", deployment.getStatus());
+ Map<String, Object> event = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");
+ assertEquals("Skipped", event.get("status"));
+ }
+
+ @Test
+ void failedBeforeAllowTrafficHookPreventsTrafficShift() {
+ createLambdaAppAndGroup("app-no-progress", "group-no-progress");
+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-4");
+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenAnswer(invocation -> {
+ byte[] payload = invocation.getArgument(2);
+ service.putLifecycleEventHookExecutionStatus("ignored", executionIdFrom(payload), "Failed");
+ return cleanInvoke;
+ });
+
+ String deploymentId = createLambdaDeployment("app-no-progress", "group-no-progress", "beforeHook", "afterHook");
+
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+
+ assertEquals("Failed", deployment.getStatus());
+ assertEquals("HOOK_EXECUTION_FAILURE", deployment.getErrorInformation().get("code"));
+ Map<String, Object> beforeAllowTrafficEvent = lambdaTargetEvent(deploymentId, "BeforeAllowTraffic");
+ assertEquals("Failed", beforeAllowTrafficEvent.get("status"));
+ assertEquals("Lambda function beforeHook reported that lifecycle event BeforeAllowTraffic failed",
+ diagnosticsOf(beforeAllowTrafficEvent).get("message"));
+ verify(lambdaService, never()).invoke(eq(REGION), eq("afterHook"), any(byte[].class), any());
+ verify(lambdaService, never()).updateAlias(anyString(), anyString(), anyString(), anyString(), any(), any());
+ List<String> recordedEvents = lambdaTargetEvents(deploymentId).stream()
+ .map(e -> (String) e.get("lifecycleEventName"))
+ .toList();
+ assertTrue(recordedEvents.contains("BeforeAllowTraffic"));
+ assertFalse(recordedEvents.contains("AllowTraffic"), "AllowTraffic must not run after a failed hook");
+ assertFalse(recordedEvents.contains("AfterAllowTraffic"), "AfterAllowTraffic must not run after a failed hook");
+ }
+
+ @Test
+ void lateLifecycleCallbackAfterDeploymentTerminalIsNoOp() {
+ createLambdaAppAndGroup("app-late-callback", "group-late-callback");
+ InvokeResult cleanInvoke = new InvokeResult(200, null, new byte[0], null, "req-5");
+ AtomicReference<String> capturedExecutionId = new AtomicReference<>();
+ when(lambdaService.invoke(eq(REGION), eq("beforeHook"), any(byte[].class), eq(InvocationType.RequestResponse)))
+ .thenAnswer(invocation -> {
+ byte[] payload = invocation.getArgument(2);
+ String executionId = executionIdFrom(payload);
+ capturedExecutionId.set(executionId);
+ service.putLifecycleEventHookExecutionStatus("ignored", executionId, "Succeeded");
+ return cleanInvoke;
+ });
+
+ String deploymentId = createLambdaDeployment("app-late-callback", "group-late-callback", "beforeHook", null);
+ Deployment deployment = awaitTerminal(deploymentId, Duration.ofSeconds(5));
+ assertEquals("Succeeded", deployment.getStatus());
+
+ service.putLifecycleEventHookExecutionStatus("ignored", capturedExecutionId.get(), "Failed");
+
+ assertEquals("Succeeded", service.getDeployment(REGION, deploymentId).getStatus());
+ }
+
+ // ---- Helpers --------------------------------------------------------------------------
+
+ private String createServerDeployment(String appName, String groupName, int timeoutSeconds) {
+ return createServerDeployment(service, appName, groupName, timeoutSeconds);
+ }
+
+ private String createServerDeployment(CodeDeployService target, String appName, String groupName,
+ int timeoutSeconds) {
+ target.createApplication(REGION, appName, "Server", null);
+ target.createDeploymentGroup(REGION, appName, groupName, "CodeDeployDefault.AllAtOnce", ROLE_ARN, null);
+ String appSpec = """
+ os: linux
+ hooks:
+ ApplicationStart:
+ - location: scripts/start_server.sh
+ timeout: %d
+ """.formatted(timeoutSeconds);
+ Map<String, Object> revision = Map.of("revisionType", "AppSpecContent",
+ "appSpecContent", Map.of("content", appSpec));
+ return target.createDeployment(REGION, appName, groupName, null, revision, "server hook test");
+ }
+
+ private void createLambdaAppAndGroup(String appName, String groupName) {
+ service.createApplication(REGION, appName, "Lambda", null);
+ service.createDeploymentGroup(REGION, appName, groupName,
+ "CodeDeployDefault.LambdaAllAtOnce", ROLE_ARN, null);
+ }
+
+ private String createLambdaDeployment(String appName, String groupName, String beforeHook, String afterHook) {
+ return service.createDeployment(REGION, appName, groupName, null,
+ lambdaRevision(beforeHook, afterHook), "lambda hook test");
+ }
+
+ private Map<String, Object> lambdaRevision(String beforeHook, String afterHook) {
+ StringBuilder hooks = new StringBuilder("[");
+ if (beforeHook != null) {
+ hooks.append("{\"BeforeAllowTraffic\":\"").append(beforeHook).append("\"}");
+ }
+ if (afterHook != null) {
+ if (hooks.length() > 1) {
+ hooks.append(",");
+ }
+ hooks.append("{\"AfterAllowTraffic\":\"").append(afterHook).append("\"}");
+ }
+ hooks.append("]");
+ String content = """
+ {
+ "version": "0.0",
+ "Resources": [
+ {"myFunc": {"Type": "AWS::Lambda::Function", "Properties": {
+ "Name": "myFunc", "Alias": "live", "CurrentVersion": "1", "TargetVersion": "2"
+ }}}
+ ],
+ "Hooks": %s
+ }
+ """.formatted(hooks);
+ return Map.of("revisionType", "AppSpecContent", "appSpecContent", Map.of("content", content));
+ }
+
+ private static String executionIdFrom(byte[] payload) {
+ try {
+ JsonNode node = JSON.readTree(payload);
+ return node.get("LifecycleEventHookExecutionId").asText();
+ } catch (Exception e) {
+ throw new IllegalStateException("Could not parse hook payload", e);
+ }
+ }
+
+ private Deployment awaitTerminal(String deploymentId, Duration atMost) {
+ return awaitTerminal(service, deploymentId, atMost);
+ }
+
+ private Deployment awaitTerminal(CodeDeployService target, String deploymentId, Duration atMost) {
+ await().atMost(atMost).pollInterval(Duration.ofMillis(25)).until(() ->
+ isTerminal(target.getDeployment(REGION, deploymentId).getStatus()));
+ return target.getDeployment(REGION, deploymentId);
+ }
+
+ private boolean isTerminal(String status) {
+ return "Succeeded".equals(status) || "Failed".equals(status) || "Stopped".equals(status);
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> instanceTargetEvent(String deploymentId, String targetId, String eventName) {
+ Map<String, Object> target = service.batchGetDeploymentTargets(REGION, deploymentId, List.of(targetId)).get(0);
+ Map<String, Object> instanceTarget = (Map<String, Object>) target.get("instanceTarget");
+ List<Map<String, Object>> events = (List<Map<String, Object>>) instanceTarget.get("lifecycleEvents");
+ return events.stream()
+ .filter(e -> eventName.equals(e.get("lifecycleEventName")))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("No lifecycle event named " + eventName));
+ }
+
+ private Map<String, Object> lambdaTargetEvent(String deploymentId, String eventName) {
+ return lambdaTargetEvent(service, deploymentId, eventName);
+ }
+
+ private Map<String, Object> lambdaTargetEvent(CodeDeployService target, String deploymentId, String eventName) {
+ return lambdaTargetEvents(target, deploymentId).stream()
+ .filter(e -> eventName.equals(e.get("lifecycleEventName")))
+ .findFirst()
+ .orElseThrow(() -> new AssertionError("No lifecycle event named " + eventName));
+ }
+
+ private List<Map<String, Object>> lambdaTargetEvents(String deploymentId) {
+ return lambdaTargetEvents(service, deploymentId);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<Map<String, Object>> lambdaTargetEvents(CodeDeployService target, String deploymentId) {
+ Map<String, Object> targetMap = target.batchGetDeploymentTargets(REGION, deploymentId, List.of()).get(0);
+ Map<String, Object> lambdaTarget = (Map<String, Object>) targetMap.get("lambdaTarget");
+ return (List<Map<String, Object>>) lambdaTarget.get("lifecycleEvents");
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, String> diagnosticsOf(Map<String, Object> event) {
+ return (Map<String, String>) event.get("diagnostics");
+ }
+}