Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ssm): retain local service diagnostics on command failure (#1529)

Jason van Zyl6a0efb7803ce
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java
index 0cb3f7c9e4..b11b51f001 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java
@@ -140,22 +140,106 @@ public class SsmDirectCommandExecutor {
String standardOutput = stdout.toString(StandardCharsets.UTF_8);
String standardError = stderr.toString(StandardCharsets.UTF_8);
if (isTimeoutExitCode(responseCode)) {
+ logFailureDiagnostics(containerId);
return ExecutionResult.timedOut(standardOutput, standardError, start);
}
if (responseCode == 0) {
return ExecutionResult.success(standardOutput, standardError, responseCode, start);
}
+ logFailureDiagnostics(containerId);
return ExecutionResult.failed(standardOutput, standardError, responseCode, start);
}
- private static String timeoutWrappedScript(String script, int timeoutSeconds) {
+ static String timeoutWrappedScript(String script, int timeoutSeconds) {
+ return userScriptCommand(script, timeoutSeconds);
+ }
+
+ private static String userScriptCommand(String script, int timeoutSeconds) {
+ String command = "sh -c " + shellSingleQuote(script);
if (timeoutSeconds < 1) {
- return script;
+ return command;
}
String timeout = timeoutSeconds + "s";
return "if command -v timeout >/dev/null 2>&1; then "
- + "timeout --kill-after=1s " + shellSingleQuote(timeout) + " sh -c " + shellSingleQuote(script) + "; "
- + "else sh -c " + shellSingleQuote(script) + "; fi";
+ + "timeout --kill-after=1s " + shellSingleQuote(timeout) + " " + command + "; "
+ + "else " + command + "; fi";
+ }
+
+ static String diagnosticWrappedScript(String script) {
+ return "sh -c " + shellSingleQuote(script);
+ }
+
+ private void logFailureDiagnostics(String containerId) {
+ try {
+ String diagnostics = collectFailureDiagnostics(containerId);
+ if (!diagnostics.isBlank()) {
+ LOG.warnf("SSM direct command failed; compact service diagnostics:%n%s", diagnostics);
+ }
+ }
+ catch (Exception e) {
+ LOG.debugv(e, "Unable to collect SSM direct command diagnostics for container {0}", containerId);
+ }
+ }
+
+ private String collectFailureDiagnostics(String containerId) throws InterruptedException {
+ var create = dockerClient.execCreateCmd(containerId)
+ .withCmd("sh", "-c", failureDiagnosticsScript())
+ .withAttachStdout(true)
+ .withAttachStderr(true);
+ String execId = create.exec().getId();
+ CountDownLatch latch = new CountDownLatch(1);
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ ResultCallback<Frame> callback = dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
+ @Override
+ public void onNext(Frame frame) {
+ byte[] payload = frame.getPayload();
+ if (payload == null) {
+ return;
+ }
+ try {
+ output.write(payload);
+ }
+ catch (IOException ignored) {
+ }
+ }
+
+ @Override
+ public void onComplete() {
+ latch.countDown();
+ }
+
+ @Override
+ public void onError(Throwable throwable) {
+ latch.countDown();
+ }
+ });
+ boolean completed = latch.await(2, TimeUnit.SECONDS);
+ if (!completed) {
+ closeQuietly(callback);
+ }
+ return output.toString(StandardCharsets.UTF_8);
+ }
+
+ static String failureDiagnosticsScript() {
+ return """
+ floci_ssm_redact() {
+ sed -E \\
+ -e 's/(Authorization:[[:space:]]*Bearer[[:space:]]*)[^[:space:]]+/\\1[REDACTED]/Ig' \\
+ -e 's/((password|passwd|secret|token|client[-_ ]?secret)[^=:\\r\\n]{0,80}[=:][[:space:]]*)[^[:space:]]+/\\1[REDACTED]/Ig'
+ }
+ echo "[floci:ssm] listening ports"
+ (ss -ltnp 2>/dev/null || netstat -ltnp 2>/dev/null || true) | head -40 | floci_ssm_redact
+ echo "[floci:ssm] processes"
+ (ps -eo pid,ppid,comm,args 2>/dev/null || ps aux 2>/dev/null || true) | head -40 | floci_ssm_redact || true
+ log_count=0
+ for log in /var/log/*.log /var/log/*/*.log; do
+ [ -f "$log" ] || continue
+ log_count=$((log_count + 1))
+ [ "$log_count" -le 5 ] || break
+ echo "[floci:ssm] log tail: $log"
+ tail -40 "$log" 2>/dev/null | floci_ssm_redact || true
+ done
+ """;
}
private static long hostTimeoutSeconds(int timeoutSeconds) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java
index f3ffe3148e..b2fe0e4f78 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java
@@ -20,6 +20,7 @@ import java.util.Optional;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.argThat;
@@ -83,7 +84,40 @@ class SsmDirectCommandExecutorTest {
eq("sh"),
eq("-c"),
argThat(command -> command.contains("timeout --kill-after=1s '30s' sh -c")
- && command.contains("'echo stdout'")));
+ && !command.contains("floci_ssm_redact")
+ && command.contains("echo stdout")));
+ }
+
+ @Test
+ void timeoutWrapperLeavesCommandOutputAwsFaithful() {
+ String command = SsmDirectCommandExecutor.timeoutWrappedScript("curl http://127.0.0.1:9999/health", 30);
+
+ assertTrue(command.contains("curl http://127.0.0.1:9999/health"));
+ assertFalse(command.contains("floci_ssm_redact"));
+ assertFalse(command.contains("listening ports"));
+ assertFalse(command.contains("/var/log/*.log"));
+ }
+
+ @Test
+ void failureDiagnosticsAreGenericAndRedacted() {
+ String command = SsmDirectCommandExecutor.failureDiagnosticsScript();
+
+ assertTrue(command.contains("listening ports"));
+ assertTrue(command.contains("processes"));
+ assertTrue(command.contains("/var/log/*.log"));
+ assertTrue(command.contains("REDACTED"));
+ assertFalse(command.contains("airship"));
+ assertFalse(command.contains("iceguard"));
+ assertFalse(command.contains("trino"));
+ assertFalse(command.contains("coordinator"));
+ }
+
+ @Test
+ void timeoutAppliesOnlyToUserScriptBeforeDiagnosticsRun() {
+ String command = SsmDirectCommandExecutor.timeoutWrappedScript("sleep 100", 30);
+
+ assertTrue(command.contains("timeout --kill-after=1s '30s' sh -c 'sleep 100'"));
+ assertFalse(command.contains("floci_ssm_redact"));
}
@Test