Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(codebuild): keep only the tail of phase output for failure messages (#4690)

Niels Kootstra70efa35fe321
diff --git a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
index e8be73d55b..d4e53c44d2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
+++ b/src/main/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunner.java
@@ -59,6 +59,7 @@ import java.util.zip.ZipOutputStream;
public class CodeBuildRunner implements ContainerTeardown {
private static final Logger LOG = Logger.getLogger(CodeBuildRunner.class);
+ private static final int FAILURE_OUTPUT_CHARS = 512;
private final DockerClient dockerClient;
private final ContainerBuilder containerBuilder;
@@ -588,7 +589,7 @@ public class CodeBuildRunner implements ContainerTeardown {
return tar;
}
- private PhaseResult runPhase(String containerId, String workDir, List<String> env,
+ PhaseResult runPhase(String containerId, String workDir, List<String> env,
List<String> commands, int timeoutMinutes, AtomicBoolean stopFlag) {
if (commands.isEmpty()) {
return PhaseResult.ofSuccess();
@@ -611,13 +612,13 @@ public class CodeBuildRunner implements ContainerTeardown {
.getId();
CountDownLatch latch = new CountDownLatch(1);
- ByteArrayOutputStream outputCapture = new ByteArrayOutputStream();
+ OutputTail outputCapture = new OutputTail();
dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
@Override
public void onNext(Frame frame) {
if (frame.getPayload() != null) {
- try { outputCapture.write(frame.getPayload()); } catch (IOException ignored) {}
+ outputCapture.write(frame.getPayload());
}
}
@Override
@@ -636,11 +637,10 @@ public class CodeBuildRunner implements ContainerTeardown {
Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong();
if (exitCode != null && exitCode != 0) {
- String output = outputCapture.toString(StandardCharsets.UTF_8);
+ String output = outputCapture.asString().stripTrailing();
String msg = "Exit code " + exitCode;
if (!output.isBlank()) {
- int start = Math.max(0, output.length() - 512);
- msg += ": " + output.stripTrailing().substring(start);
+ msg += ": " + output.substring(Math.max(0, output.length() - FAILURE_OUTPUT_CHARS));
}
return PhaseResult.ofFailure(msg);
}
@@ -867,9 +867,63 @@ public class CodeBuildRunner implements ContainerTeardown {
return "text/plain";
}
+ /**
+ * The last bytes a phase printed. Only the tail reaches the failure message, so a phase that
+ * prints gigabytes of logs does not hold them all in the heap. Trailing whitespace is held
+ * apart until more output follows it, so a long run of blank lines cannot push the last
+ * error out of the tail before the message strips it. The tail keeps eight bytes for every
+ * character of the message, so it always decodes to more than {@link #FAILURE_OUTPUT_CHARS}
+ * characters and a character cut in half at its start never reaches the message.
+ */
+ private static final class OutputTail {
+ private final ByteTail content = new ByteTail(FAILURE_OUTPUT_CHARS * 8);
+ private final ByteTail trailingWhitespace = new ByteTail(FAILURE_OUTPUT_CHARS * 8);
+
+ synchronized void write(byte[] bytes) {
+ int contentEnd = bytes.length;
+ while (contentEnd > 0 && Character.isWhitespace(bytes[contentEnd - 1])) {
+ contentEnd--;
+ }
+ if (contentEnd > 0) {
+ content.append(trailingWhitespace.buffer, 0, trailingWhitespace.length);
+ trailingWhitespace.length = 0;
+ content.append(bytes, 0, contentEnd);
+ }
+ trailingWhitespace.append(bytes, contentEnd, bytes.length - contentEnd);
+ }
+
+ synchronized String asString() {
+ return new String(content.buffer, 0, content.length, StandardCharsets.UTF_8);
+ }
+ }
+
+ private static final class ByteTail {
+ private final byte[] buffer;
+ private int length;
+
+ ByteTail(int capacity) {
+ buffer = new byte[capacity];
+ }
+
+ void append(byte[] bytes, int offset, int count) {
+ if (count >= buffer.length) {
+ System.arraycopy(bytes, offset + count - buffer.length, buffer, 0, buffer.length);
+ length = buffer.length;
+ return;
+ }
+ int overflow = length + count - buffer.length;
+ if (overflow > 0) {
+ System.arraycopy(buffer, overflow, buffer, 0, length - overflow);
+ length -= overflow;
+ }
+ System.arraycopy(bytes, offset, buffer, length, count);
+ length += count;
+ }
+ }
+
private enum PhaseStatus { SUCCEEDED, FAILED, STOPPED }
- private record PhaseResult(PhaseStatus status, String errorMessage) {
+ record PhaseResult(PhaseStatus status, String errorMessage) {
boolean succeeded() { return status == PhaseStatus.SUCCEEDED; }
boolean failed() { return status == PhaseStatus.FAILED; }
boolean stopped() { return status == PhaseStatus.STOPPED; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerTest.java b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerTest.java
new file mode 100644
index 0000000000..68762a7074
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/codebuild/CodeBuildRunnerTest.java
@@ -0,0 +1,85 @@
+package io.github.hectorvent.floci.services.codebuild;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.async.ResultCallback;
+import com.github.dockerjava.api.command.ExecCreateCmd;
+import com.github.dockerjava.api.command.ExecCreateCmdResponse;
+import com.github.dockerjava.api.command.ExecStartCmd;
+import com.github.dockerjava.api.command.InspectExecCmd;
+import com.github.dockerjava.api.command.InspectExecResponse;
+import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.api.model.StreamType;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.RETURNS_SELF;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class CodeBuildRunnerTest {
+
+ @Test
+ void phaseWithMoreOutputThanOneBufferCanHoldStillReportsItsLastOutput() {
+ byte[] chunk = new byte[8 * 1024 * 1024];
+ Arrays.fill(chunk, (byte) 'x');
+ // 2.5 GiB in total: more than a single Java byte array can hold.
+ int chunkCount = 320;
+ String lastLine = "error: build step failed\n";
+ DockerClient dockerClient = dockerClientStreaming(chunk, chunkCount, lastLine, 2L);
+ CodeBuildRunner runner = new CodeBuildRunner(dockerClient, null, null, null, null, null, null, null, null, null);
+
+ CodeBuildRunner.PhaseResult result = runner.runPhase("container-1", "/src", List.of(),
+ List.of("./build.sh"), 60, new AtomicBoolean(false));
+
+ assertTrue(result.failed());
+ String expectedTail = "x".repeat(512 - "error: build step failed".length()) + "error: build step failed";
+ assertEquals("Exit code 2: " + expectedTail, result.errorMessage());
+ }
+
+ @Test
+ void failureMessageIgnoresLongTrailingWhitespace() {
+ DockerClient dockerClient = dockerClientStreaming(new byte[0], 0, "tests failed" + "\n".repeat(10_000), 1L);
+ CodeBuildRunner runner = new CodeBuildRunner(dockerClient, null, null, null, null, null, null, null, null, null);
+
+ CodeBuildRunner.PhaseResult result = runner.runPhase("container-1", "/src", List.of(),
+ List.of("./test.sh"), 60, new AtomicBoolean(false));
+
+ assertEquals("Exit code 1: tests failed", result.errorMessage());
+ }
+
+ private static DockerClient dockerClientStreaming(byte[] chunk, int chunkCount, String lastLine, long exitCode) {
+ DockerClient dockerClient = mock(DockerClient.class);
+
+ ExecCreateCmd createCmd = mock(ExecCreateCmd.class, RETURNS_SELF);
+ ExecCreateCmdResponse createResponse = mock(ExecCreateCmdResponse.class);
+ when(createResponse.getId()).thenReturn("exec-1");
+ when(createCmd.exec()).thenReturn(createResponse);
+ when(dockerClient.execCreateCmd("container-1")).thenReturn(createCmd);
+
+ ExecStartCmd startCmd = mock(ExecStartCmd.class);
+ when(startCmd.exec(any())).thenAnswer(invocation -> {
+ ResultCallback.Adapter<Frame> callback = invocation.getArgument(0);
+ for (int i = 0; i < chunkCount; i++) {
+ callback.onNext(new Frame(StreamType.STDOUT, chunk));
+ }
+ callback.onNext(new Frame(StreamType.STDERR, lastLine.getBytes(StandardCharsets.UTF_8)));
+ callback.onComplete();
+ return callback;
+ });
+ when(dockerClient.execStartCmd("exec-1")).thenReturn(startCmd);
+
+ InspectExecCmd inspectCmd = mock(InspectExecCmd.class);
+ InspectExecResponse inspectResponse = mock(InspectExecResponse.class);
+ when(inspectResponse.getExitCodeLong()).thenReturn(exitCode);
+ when(inspectCmd.exec()).thenReturn(inspectResponse);
+ when(dockerClient.inspectExecCmd("exec-1")).thenReturn(inspectCmd);
+ return dockerClient;
+ }
+}