Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ssm): bound direct-execution output to the GetCommandInvocation limits (#4257)

Niels Kootstra1027f0a60234
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/BoundedOutputCollector.java b/src/main/java/io/github/hectorvent/floci/services/ssm/BoundedOutputCollector.java
new file mode 100644
index 0000000000..8ba4f6b559
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/BoundedOutputCollector.java
@@ -0,0 +1,82 @@
+package io.github.hectorvent.floci.services.ssm;
+
+import java.nio.ByteBuffer;
+import java.nio.CharBuffer;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Accumulates UTF-8 bytes into a character buffer bounded to a maximum number of characters.
+ * Bytes received after the cap is reached are decoded and discarded rather than retained, so
+ * memory stays proportional to {@code maxChars} regardless of how much input is fed in.
+ *
+ * <p>Handles a multi-byte UTF-8 character split across separate {@link #write(byte[])} calls
+ * (e.g. Docker exec stream frames) by carrying any undecoded trailing bytes over to the next
+ * call instead of corrupting or dropping the character.
+ *
+ * <p>Thread-safe: Docker callback threads write while the executor thread may read after a timeout.
+ */
+final class BoundedOutputCollector {
+
+ private static final byte[] NO_BYTES = new byte[0];
+
+ private final int maxChars;
+ private final StringBuilder captured;
+ private final CharsetDecoder decoder;
+ private byte[] pending = NO_BYTES;
+ private boolean capped;
+
+ BoundedOutputCollector(int maxChars) {
+ if (maxChars < 0) {
+ throw new IllegalArgumentException("maxChars must be >= 0");
+ }
+ this.maxChars = maxChars;
+ this.captured = new StringBuilder(Math.min(maxChars, 4096));
+ this.decoder = StandardCharsets.UTF_8.newDecoder()
+ .onMalformedInput(CodingErrorAction.REPLACE)
+ .onUnmappableCharacter(CodingErrorAction.REPLACE);
+ this.capped = maxChars == 0;
+ }
+
+ synchronized void write(byte[] payload) {
+ if (payload == null || payload.length == 0 || capped) {
+ return;
+ }
+
+ byte[] combined = pending.length == 0 ? payload : concat(pending, payload);
+ ByteBuffer in = ByteBuffer.wrap(combined);
+ CharBuffer out = CharBuffer.allocate(combined.length + 1);
+ decoder.decode(in, out, false);
+ out.flip();
+
+ int remaining = maxChars - captured.length();
+ if (out.length() >= remaining) {
+ if (remaining > 0 && Character.isHighSurrogate(out.charAt(remaining - 1))) {
+ remaining--;
+ }
+ captured.append(out, 0, remaining);
+ capped = true;
+ pending = NO_BYTES;
+ return;
+ }
+
+ captured.append(out);
+ pending = new byte[in.remaining()];
+ in.get(pending);
+ }
+
+ synchronized String content() {
+ if (pending.length > 0 && captured.length() < maxChars) {
+ return captured + "\uFFFD";
+ }
+ return captured.toString();
+ }
+
+ private static byte[] concat(byte[] a, byte[] b) {
+ byte[] result = new byte[a.length + b.length];
+ System.arraycopy(a, 0, result, 0, a.length);
+ System.arraycopy(b, 0, result, a.length, b.length);
+ return result;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
index 2075500eb6..0dbc31ed1e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmCommandService.java
@@ -45,8 +45,8 @@ public class SsmCommandService implements Resettable {
private static final Logger LOG = Logger.getLogger(SsmCommandService.class);
private static final int MIN_TIMEOUT_SECONDS = 30;
- private static final int MAX_STDOUT_CHARS = 24000;
- private static final int MAX_STDERR_CHARS = 8000;
+ static final int MAX_STDOUT_CHARS = 24000;
+ static final int MAX_STDERR_CHARS = 8000;
private final StorageBackend<String, InstanceInformation> instanceStore;
private final StorageBackend<String, Command> commandStore;
diff --git a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java
index b3db5b5d57..309922bc52 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutor.java
@@ -10,7 +10,6 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
-import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -98,8 +97,8 @@ public class SsmDirectCommandExecutor {
String execId = create.exec().getId();
CountDownLatch latch = new CountDownLatch(1);
- ByteArrayOutputStream stdout = new ByteArrayOutputStream();
- ByteArrayOutputStream stderr = new ByteArrayOutputStream();
+ BoundedOutputCollector stdout = new BoundedOutputCollector(SsmCommandService.MAX_STDOUT_CHARS);
+ BoundedOutputCollector stderr = new BoundedOutputCollector(SsmCommandService.MAX_STDERR_CHARS);
Instant start = Instant.now();
ResultCallback<Frame> callback = dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
@@ -109,12 +108,8 @@ public class SsmDirectCommandExecutor {
if (payload == null) {
return;
}
- ByteArrayOutputStream target = frame.getStreamType() == StreamType.STDERR ? stderr : stdout;
- try {
- target.write(payload);
- }
- catch (IOException ignored) {
- }
+ BoundedOutputCollector target = frame.getStreamType() == StreamType.STDERR ? stderr : stdout;
+ target.write(payload);
}
@Override
@@ -124,12 +119,8 @@ public class SsmDirectCommandExecutor {
@Override
public void onError(Throwable throwable) {
- try {
- stderr.write((throwable.getMessage() != null ? throwable.getMessage() : throwable.toString())
- .getBytes(StandardCharsets.UTF_8));
- }
- catch (IOException ignored) {
- }
+ stderr.write((throwable.getMessage() != null ? throwable.getMessage() : throwable.toString())
+ .getBytes(StandardCharsets.UTF_8));
latch.countDown();
}
});
@@ -138,15 +129,15 @@ public class SsmDirectCommandExecutor {
if (!completed) {
closeQuietly(callback);
return ExecutionResult.timedOut(
- stdout.toString(StandardCharsets.UTF_8),
+ stdout.content(),
"Timed out after " + timeoutSeconds + "s",
start);
}
Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong();
int responseCode = exitCode != null ? exitCode.intValue() : 1;
- String standardOutput = stdout.toString(StandardCharsets.UTF_8);
- String standardError = stderr.toString(StandardCharsets.UTF_8);
+ String standardOutput = stdout.content();
+ String standardError = stderr.content();
if (isTimeoutExitCode(responseCode)) {
logFailureDiagnostics(containerId);
return ExecutionResult.timedOut(standardOutput, standardError, start);
@@ -196,7 +187,7 @@ public class SsmDirectCommandExecutor {
.withAttachStderr(true);
String execId = create.exec().getId();
CountDownLatch latch = new CountDownLatch(1);
- ByteArrayOutputStream output = new ByteArrayOutputStream();
+ BoundedOutputCollector output = new BoundedOutputCollector(SsmCommandService.MAX_STDOUT_CHARS);
ResultCallback<Frame> callback = dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
@Override
public void onNext(Frame frame) {
@@ -204,11 +195,7 @@ public class SsmDirectCommandExecutor {
if (payload == null) {
return;
}
- try {
- output.write(payload);
- }
- catch (IOException ignored) {
- }
+ output.write(payload);
}
@Override
@@ -225,7 +212,7 @@ public class SsmDirectCommandExecutor {
if (!completed) {
closeQuietly(callback);
}
- return output.toString(StandardCharsets.UTF_8);
+ return output.content();
}
static String failureDiagnosticsScript() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/BoundedOutputCollectorTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/BoundedOutputCollectorTest.java
new file mode 100644
index 0000000000..d835c873b6
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/BoundedOutputCollectorTest.java
@@ -0,0 +1,75 @@
+package io.github.hectorvent.floci.services.ssm;
+
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+class BoundedOutputCollectorTest {
+
+ @Test
+ void feedingMoreThanCapYieldsExactlyFirstNChars() {
+ BoundedOutputCollector collector = new BoundedOutputCollector(10);
+
+ collector.write("abcde".getBytes(StandardCharsets.UTF_8));
+ collector.write("fghijKLMNOP".getBytes(StandardCharsets.UTF_8));
+
+ assertEquals("abcdefghij", collector.content());
+ }
+
+ @Test
+ void additionalWritesAfterCapAreDiscardedWithoutGrowingContent() {
+ BoundedOutputCollector collector = new BoundedOutputCollector(5);
+
+ collector.write("abcde".getBytes(StandardCharsets.UTF_8));
+ collector.write("this is a lot more text that should be fully discarded".getBytes(StandardCharsets.UTF_8));
+ collector.write("even more after that".getBytes(StandardCharsets.UTF_8));
+
+ assertEquals("abcde", collector.content());
+ }
+
+ @Test
+ void multiByteUtf8CharacterSplitAcrossWritesDecodesCorrectly() {
+ BoundedOutputCollector collector = new BoundedOutputCollector(10);
+ byte[] euroSign = "€".getBytes(StandardCharsets.UTF_8); // 0xE2 0x82 0xAC
+
+ collector.write(new byte[] {euroSign[0], euroSign[1]});
+ collector.write(new byte[] {euroSign[2]});
+ collector.write("X".getBytes(StandardCharsets.UTF_8));
+
+ assertEquals("€X", collector.content());
+ }
+
+ @Test
+ void multiByteCharacterCompletingExactlyAtCapIsIncludedWithoutCorruption() {
+ BoundedOutputCollector collector = new BoundedOutputCollector(3);
+ byte[] euroSign = "€".getBytes(StandardCharsets.UTF_8);
+
+ collector.write("ab".getBytes(StandardCharsets.UTF_8));
+ collector.write(new byte[] {euroSign[0], euroSign[1]});
+ collector.write(new byte[] {euroSign[2]});
+
+ assertEquals("ab€", collector.content());
+ }
+
+ @Test
+ void capFallingInsideSurrogatePairKeepsOnlyWholeCharacters() {
+ BoundedOutputCollector collector = new BoundedOutputCollector(3);
+
+ collector.write("ab\uD83D\uDE00".getBytes(StandardCharsets.UTF_8));
+
+ assertEquals("ab", collector.content());
+ }
+
+ @Test
+ void outputEndingMidCharacterShowsReplacementCharacter() {
+ BoundedOutputCollector collector = new BoundedOutputCollector(10);
+ byte[] euroSign = "€".getBytes(StandardCharsets.UTF_8);
+
+ collector.write("X".getBytes(StandardCharsets.UTF_8));
+ collector.write(new byte[] {euroSign[0], euroSign[1]});
+
+ assertEquals("X\uFFFD", collector.content());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java
index ee0c26c0ca..b22172dcf0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ssm/SsmDirectCommandExecutorTest.java
@@ -312,6 +312,98 @@ class SsmDirectCommandExecutorTest {
assertEquals("terminated\n", result.get().standardError());
}
+ @Test
+ void stdoutExceedingAwsLimitIsTruncatedToFirst24000CharsWithoutBufferingItAll() {
+ DockerClient dockerClient = mock(DockerClient.class);
+ Ec2Service ec2Service = mock(Ec2Service.class);
+ Instance instance = instance("i-container", "container-1");
+ when(ec2Service.findInstanceById("i-container")).thenReturn(instance);
+ when(ec2Service.isInstanceContainerRunning("i-container")).thenReturn(true);
+
+ ExecCreateCmd execCreate = mock(ExecCreateCmd.class, withSettings().defaultAnswer(RETURNS_SELF));
+ ExecCreateCmdResponse execCreateResponse = mock(ExecCreateCmdResponse.class);
+ when(execCreateResponse.getId()).thenReturn("exec-1");
+ when(execCreate.exec()).thenReturn(execCreateResponse);
+ when(dockerClient.execCreateCmd("container-1")).thenReturn(execCreate);
+
+ ExecStartCmd execStart = mock(ExecStartCmd.class);
+ when(execStart.exec(any())).thenAnswer(invocation -> {
+ @SuppressWarnings("unchecked")
+ ResultCallback<Frame> resultCallback = invocation.getArgument(0);
+ // Far more than the AWS 24,000 char stdout limit, delivered as many frames.
+ byte[] chunk = "A".repeat(1000).getBytes();
+ for (int i = 0; i < 100; i++) {
+ resultCallback.onNext(new Frame(StreamType.STDOUT, chunk));
+ }
+ resultCallback.onComplete();
+ return resultCallback;
+ });
+ when(dockerClient.execStartCmd("exec-1")).thenReturn(execStart);
+
+ InspectExecCmd inspectExec = mock(InspectExecCmd.class);
+ InspectExecResponse inspectExecResponse = mock(InspectExecResponse.class);
+ when(inspectExecResponse.getExitCodeLong()).thenReturn(0L);
+ when(inspectExec.exec()).thenReturn(inspectExecResponse);
+ when(dockerClient.inspectExecCmd("exec-1")).thenReturn(inspectExec);
+
+ SsmDirectCommandExecutor executor = new SsmDirectCommandExecutor(dockerClient, ec2Service);
+ Optional<SsmDirectCommandExecutor.ExecutionResult> result = executor.executeIfSupported(
+ "i-container",
+ "AWS-RunShellScript",
+ Map.of("commands", List.of("yes A | head -c 100000")),
+ 30);
+
+ assertTrue(result.isPresent());
+ assertEquals(24_000, result.get().standardOutput().length());
+ assertEquals("A".repeat(24_000), result.get().standardOutput());
+ }
+
+ @Test
+ void stderrExceedingAwsLimitIsTruncatedToFirst8000CharsWithoutBufferingItAll() {
+ DockerClient dockerClient = mock(DockerClient.class);
+ Ec2Service ec2Service = mock(Ec2Service.class);
+ Instance instance = instance("i-container", "container-1");
+ when(ec2Service.findInstanceById("i-container")).thenReturn(instance);
+ when(ec2Service.isInstanceContainerRunning("i-container")).thenReturn(true);
+
+ ExecCreateCmd execCreate = mock(ExecCreateCmd.class, withSettings().defaultAnswer(RETURNS_SELF));
+ ExecCreateCmdResponse execCreateResponse = mock(ExecCreateCmdResponse.class);
+ when(execCreateResponse.getId()).thenReturn("exec-1");
+ when(execCreate.exec()).thenReturn(execCreateResponse);
+ when(dockerClient.execCreateCmd("container-1")).thenReturn(execCreate);
+
+ ExecStartCmd execStart = mock(ExecStartCmd.class);
+ when(execStart.exec(any())).thenAnswer(invocation -> {
+ @SuppressWarnings("unchecked")
+ ResultCallback<Frame> resultCallback = invocation.getArgument(0);
+ // Far more than the AWS 8,000 char stderr limit, delivered as many frames.
+ byte[] chunk = "E".repeat(1000).getBytes();
+ for (int i = 0; i < 100; i++) {
+ resultCallback.onNext(new Frame(StreamType.STDERR, chunk));
+ }
+ resultCallback.onComplete();
+ return resultCallback;
+ });
+ when(dockerClient.execStartCmd("exec-1")).thenReturn(execStart);
+
+ InspectExecCmd inspectExec = mock(InspectExecCmd.class);
+ InspectExecResponse inspectExecResponse = mock(InspectExecResponse.class);
+ when(inspectExecResponse.getExitCodeLong()).thenReturn(1L);
+ when(inspectExec.exec()).thenReturn(inspectExecResponse);
+ when(dockerClient.inspectExecCmd("exec-1")).thenReturn(inspectExec);
+
+ SsmDirectCommandExecutor executor = new SsmDirectCommandExecutor(dockerClient, ec2Service);
+ Optional<SsmDirectCommandExecutor.ExecutionResult> result = executor.executeIfSupported(
+ "i-container",
+ "AWS-RunShellScript",
+ Map.of("commands", List.of("yes E 1>&2 | head -c 100000")),
+ 30);
+
+ assertTrue(result.isPresent());
+ assertEquals(8_000, result.get().standardError().length());
+ assertEquals("E".repeat(8_000), result.get().standardError());
+ }
+
private static Instance instance(String instanceId, String containerId) {
Instance instance = new Instance();
instance.setInstanceId(instanceId);