Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ssm): retain local service diagnostics on command failure (#1529)
Jason van Zyl6a0efb7803ce
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.javaindex 0cb3f7c9e4..b11b51f001 100644--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java@@ -140,22 +140,106 @@ public class SsmDirectCommandExecutor {String standardOutput = stdout.toString(StandardCharsets.UTF_8);String standardError = stderr.toString(StandardCharsets.UTF_8);if (isTimeoutExitCode(responseCode)) {+ logFailureDiagnostics(containerId);return ExecutionResult.timedOut(standardOutput, standardError, start);}if (responseCode == 0) {return ExecutionResult.success(standardOutput, standardError, responseCode, start);}+ logFailureDiagnostics(containerId);return ExecutionResult.failed(standardOutput, standardError, responseCode, start);}- private static String timeoutWrappedScript(String script, int timeoutSeconds) {+ static String timeoutWrappedScript(String script, int timeoutSeconds) {+ return userScriptCommand(script, timeoutSeconds);+ }++ private static String userScriptCommand(String script, int timeoutSeconds) {+ String command = "sh -c " + shellSingleQuote(script);if (timeoutSeconds < 1) {- return script;+ return command;}String timeout = timeoutSeconds + "s";return "if command -v timeout >/dev/null 2>&1; then "- + "timeout --kill-after=1s " + shellSingleQuote(timeout) + " sh -c " + shellSingleQuote(script) + "; "- + "else sh -c " + shellSingleQuote(script) + "; fi";+ + "timeout --kill-after=1s " + shellSingleQuote(timeout) + " " + command + "; "+ + "else " + command + "; fi";+ }++ static String diagnosticWrappedScript(String script) {+ return "sh -c " + shellSingleQuote(script);+ }++ private void logFailureDiagnostics(String containerId) {+ try {+ String diagnostics = collectFailureDiagnostics(containerId);+ if (!diagnostics.isBlank()) {+ LOG.warnf("SSM direct command failed; compact service diagnostics:%n%s", diagnostics);+ }+ }+ catch (Exception e) {+ LOG.debugv(e, "Unable to collect SSM direct command diagnostics for container {0}", containerId);+ }+ }++ private String collectFailureDiagnostics(String containerId) throws InterruptedException {+ var create = dockerClient.execCreateCmd(containerId)+ .withCmd("sh", "-c", failureDiagnosticsScript())+ .withAttachStdout(true)+ .withAttachStderr(true);+ String execId = create.exec().getId();+ CountDownLatch latch = new CountDownLatch(1);+ ByteArrayOutputStream output = new ByteArrayOutputStream();+ ResultCallback<Frame> callback = dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {+ @Override+ public void onNext(Frame frame) {+ byte[] payload = frame.getPayload();+ if (payload == null) {+ return;+ }+ try {+ output.write(payload);+ }+ catch (IOException ignored) {+ }+ }++ @Override+ public void onComplete() {+ latch.countDown();+ }++ @Override+ public void onError(Throwable throwable) {+ latch.countDown();+ }+ });+ boolean completed = latch.await(2, TimeUnit.SECONDS);+ if (!completed) {+ closeQuietly(callback);+ }+ return output.toString(StandardCharsets.UTF_8);+ }++ static String failureDiagnosticsScript() {+ return """+ floci_ssm_redact() {+ sed -E \\+ -e 's/(Authorization:[[:space:]]*Bearer[[:space:]]*)[^[:space:]]+/\\1[REDACTED]/Ig' \\+ -e 's/((password|passwd|secret|token|client[-_ ]?secret)[^=:\\r\\n]{0,80}[=:][[:space:]]*)[^[:space:]]+/\\1[REDACTED]/Ig'+ }+ echo "[floci:ssm] listening ports"+ (ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null || true) | head -40 | floci_ssm_redact+ echo "[floci:ssm] processes"+ (ps -eo pid,ppid,comm,args 2>/dev/null || ps aux 2>/dev/null || true) | head -40 | floci_ssm_redact || true+ log_count=0+ for log in /var/log/*.log /var/log/*/*.log; do+ [ -f "$log" ] || continue+ log_count=$((log_count + 1))+ [ "$log_count" -le 5 ] || break+ echo "[floci:ssm] log tail: $log"+ tail -40 "$log" 2>/dev/null | floci_ssm_redact || true+ done+ """;}private static long hostTimeoutSeconds(int timeoutSeconds) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.javaindex f3ffe3148e..b2fe0e4f78 100644--- a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java@@ -20,6 +20,7 @@ import java.util.Optional;import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.argThat;@@ -83,7 +84,40 @@ class SsmDirectCommandExecutorTest {eq("sh"),eq("-c"),argThat(command -> command.contains("timeout --kill-after=1s '30s' sh -c")- && command.contains("'echo stdout'")));+ && !command.contains("floci_ssm_redact")+ && command.contains("echo stdout")));+ }++ @Test+ void timeoutWrapperLeavesCommandOutputAwsFaithful() {+ String command = SsmDirectCommandExecutor.timeoutWrappedScript("curl http://127.0.0.1:9999/health", 30);++ assertTrue(command.contains("curl http://127.0.0.1:9999/health"));+ assertFalse(command.contains("floci_ssm_redact"));+ assertFalse(command.contains("listening ports"));+ assertFalse(command.contains("/var/log/*.log"));+ }++ @Test+ void failureDiagnosticsAreGenericAndRedacted() {+ String command = SsmDirectCommandExecutor.failureDiagnosticsScript();++ assertTrue(command.contains("listening ports"));+ assertTrue(command.contains("processes"));+ assertTrue(command.contains("/var/log/*.log"));+ assertTrue(command.contains("REDACTED"));+ assertFalse(command.contains("airship"));+ assertFalse(command.contains("iceguard"));+ assertFalse(command.contains("trino"));+ assertFalse(command.contains("coordinator"));+ }++ @Test+ void timeoutAppliesOnlyToUserScriptBeforeDiagnosticsRun() {+ String command = SsmDirectCommandExecutor.timeoutWrappedScript("sleep 100", 30);++ assertTrue(command.contains("timeout --kill-after=1s '30s' sh -c 'sleep 100'"));+ assertFalse(command.contains("floci_ssm_redact"));}@Test