Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

perf(docker): decode completed log frames with JDK fast paths (#6525)

Andre Kurait57889bbb45b8
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
index a4e651ad97..bc74678392 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
@@ -20,6 +20,7 @@ import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.Deque;
import java.util.EnumMap;
@@ -517,10 +518,29 @@ public class ContainerLogStreamer {
/** Appends a frame payload and returns the complete lines it produced (possibly none). */
List<String> append(byte[] payload) {
+ int size = buffer.size();
+ boolean fitsWithinLimit = payload.length <= MAX_LINE_BYTES - 3 - size;
+ // A complete frame safely below the cap needs no byte-level reassembly or UTF-8 tracking.
+ // Keep empty lines, including the one before a final newline, for the emitter to handle.
+ if (size == 0 && payload.length > 0 && fitsWithinLimit && payload[payload.length - 1] == '\n') {
+ return Arrays.asList(new String(payload, 0, payload.length - 1, StandardCharsets.UTF_8)
+ .split("\n", -1));
+ }
List<String> lines = new ArrayList<>();
- for (byte b : payload) {
+ int start = 0;
+ for (int i = 0; i < payload.length; i++) {
+ byte b = payload[i];
if (b == '\n') {
- lines.add(takeLine());
+ lines.add(takeLine(payload, start, i));
+ start = i + 1;
+ size = 0;
+ continue;
+ }
+ // Away from the cap, only the final four bytes can affect the next frame's UTF-8 state.
+ if (fitsWithinLimit) {
+ if (i >= payload.length - 4) {
+ trackUtf8(b);
+ }
continue;
}
// Force-split BEFORE a character that would push the buffer past the cap, so the emitted
@@ -528,20 +548,27 @@ public class ContainerLogStreamer {
// the next event whole instead of inflating this one. Only at a character boundary, and
// only for a real character start (not a continuation byte).
if (pendingContinuation == 0 && (b & 0xC0) != 0x80
- && buffer.size() > 0
- && buffer.size() + utf8CharLength(b) > MAX_LINE_BYTES) {
- lines.add(takeLine());
+ && size > 0
+ && size + utf8CharLength(b) > MAX_LINE_BYTES) {
+ lines.add(takeLine(payload, start, i));
+ start = i;
+ size = 0;
}
- buffer.write(b);
+ size++;
trackUtf8(b);
- if ((buffer.size() >= MAX_LINE_BYTES && pendingContinuation == 0)
- || buffer.size() >= MAX_LINE_BYTES + 3) {
+ if ((size >= MAX_LINE_BYTES && pendingContinuation == 0)
+ || size >= MAX_LINE_BYTES + 3) {
// Do not split a valid character whose continuation bytes arrive in a later frame.
// Malformed input may never reach a boundary, so allow at most the three continuation
// bytes a valid UTF-8 character could need before forcing the buffer to drain.
- lines.add(takeLine());
+ lines.add(takeLine(payload, start, i + 1));
+ start = i + 1;
+ size = 0;
}
}
+ if (start < payload.length) {
+ buffer.write(payload, start, payload.length - start);
+ }
return lines;
}
@@ -585,6 +612,15 @@ public class ContainerLogStreamer {
}
}
+ private String takeLine(byte[] payload, int start, int end) {
+ if (!hasPendingBytes()) {
+ pendingContinuation = 0;
+ return new String(payload, start, end - start, StandardCharsets.UTF_8);
+ }
+ buffer.write(payload, start, end - start);
+ return takeLine();
+ }
+
private String takeLine() {
String line = buffer.toString(StandardCharsets.UTF_8);
buffer.reset();
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
index 1951bb17fc..9a6fda50ec 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java
@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService
import io.github.hectorvent.floci.testutil.LogCapture;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
@@ -16,11 +17,13 @@ import java.io.Closeable;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.logging.LogRecord;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -200,6 +203,91 @@ class ContainerLogStreamerTest {
assertEquals(List.of("a", "", "b"), buffer.append(utf8("a\n\nb\n")));
}
+ static Stream<byte[]> completeFrames() {
+ return Stream.of(
+ utf8("\n"),
+ utf8("\n\n"),
+ utf8("first\n\nlast\n\n"),
+ utf8("café\n東京\n🚀\n"),
+ new byte[]{(byte) 0xe2, '\n', (byte) 0x80, '\n'},
+ new byte[]{(byte) 0xf0, (byte) 0x9f, '\n'},
+ utf8("a".repeat(256 * 1024 - 4) + "\n"));
+ }
+
+ @ParameterizedTest
+ @MethodSource("completeFrames")
+ void completeFrameMatchesBytewiseReassembly(byte[] payload) {
+ ContainerLogStreamer.LogLineBuffer whole = new ContainerLogStreamer.LogLineBuffer();
+ ContainerLogStreamer.LogLineBuffer fragmented = new ContainerLogStreamer.LogLineBuffer();
+ List<String> expected = new ArrayList<>();
+ for (byte value : payload) {
+ expected.addAll(fragmented.append(new byte[]{value}));
+ }
+
+ assertEquals(expected, whole.append(payload));
+ assertFalse(whole.hasPendingBytes());
+ assertTrue(whole.flush().isEmpty());
+ assertEquals(List.of("next"), whole.append(utf8("next\n")));
+ }
+
+ @Test
+ void nearCapFrameKeepsEarlySplitBeforeMalformedLead() {
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ String prefix = "a".repeat(256 * 1024 - 2);
+ byte[] payload = Arrays.copyOf(utf8(prefix), 256 * 1024);
+ payload[payload.length - 2] = (byte) 0xf0;
+ payload[payload.length - 1] = '\n';
+
+ assertEquals(List.of(prefix, "\uFFFD"), buffer.append(payload));
+ assertFalse(buffer.hasPendingBytes());
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 2, 3})
+ void fastFramePreservesContinuationStateForNextFrameAtCap(int characterBytesInFirstFrame) {
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ String prefix = "a".repeat(256 * 1024 - 3 - characterBytesInFirstFrame);
+ byte[] payload = utf8(prefix + "🚀€\n");
+ int boundary = 256 * 1024 - 3;
+
+ assertEquals(List.of(), buffer.append(Arrays.copyOfRange(payload, 0, boundary)));
+ assertEquals(List.of(prefix + "🚀", "€"),
+ buffer.append(Arrays.copyOfRange(payload, boundary, payload.length)));
+ assertFalse(buffer.hasPendingBytes());
+ }
+
+ @ParameterizedTest
+ @ValueSource(ints = {1, 3, 1024, 262145})
+ void frameBoundariesPreserveCappedUnicodeLinesAndTrailingContent(int frameSize) {
+ int cap = 256 * 1024;
+ String first = "a".repeat(cap - 3) + "€";
+ String rockets = "🚀".repeat(cap / 4);
+ byte[] payload = utf8(first + "\nshort\n" + rockets + "🚀\ntail");
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ List<String> lines = new ArrayList<>();
+
+ for (int offset = 0; offset < payload.length; offset += frameSize) {
+ lines.addAll(buffer.append(Arrays.copyOfRange(payload, offset,
+ Math.min(offset + frameSize, payload.length))));
+ }
+
+ assertEquals(List.of(first, "", "short", rockets, "🚀"), lines);
+ assertEquals("tail", buffer.flush().orElseThrow());
+ assertFalse(buffer.hasPendingBytes());
+ }
+
+ @Test
+ void emittedLinesAndBufferedFragmentsSurvivePayloadReuse() {
+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();
+ byte[] payload = utf8("first\npartial");
+
+ List<String> lines = buffer.append(payload);
+ Arrays.fill(payload, (byte) 'x');
+
+ assertEquals(List.of("first"), lines);
+ assertEquals(List.of("partial"), buffer.append(utf8("\n")));
+ }
+
@Test
void boundsBufferBySplittingOverlongUnterminatedRuns() {
// A 256 KB run with no newline must be force-emitted (bounded), not buffered until the