Light, fluffy, and always free - The AWS Local Emulator alternative
perf(docker): decode completed log frames with JDK fast paths (#6525)
Andre Kurait57889bbb45b8
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.javaindex a4e651ad97..bc74678392 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java@@ -20,6 +20,7 @@ import java.time.LocalDate;import java.time.format.DateTimeFormatter;import java.util.ArrayDeque;import java.util.ArrayList;+import java.util.Arrays;import java.util.Comparator;import java.util.Deque;import java.util.EnumMap;@@ -517,10 +518,29 @@ public class ContainerLogStreamer {/** Appends a frame payload and returns the complete lines it produced (possibly none). */List<String> append(byte[] payload) {+ int size = buffer.size();+ boolean fitsWithinLimit = payload.length <= MAX_LINE_BYTES - 3 - size;+ // A complete frame safely below the cap needs no byte-level reassembly or UTF-8 tracking.+ // Keep empty lines, including the one before a final newline, for the emitter to handle.+ if (size == 0 && payload.length > 0 && fitsWithinLimit && payload[payload.length - 1] == '\n') {+ return Arrays.asList(new String(payload, 0, payload.length - 1, StandardCharsets.UTF_8)+ .split("\n", -1));+ }List<String> lines = new ArrayList<>();- for (byte b : payload) {+ int start = 0;+ for (int i = 0; i < payload.length; i++) {+ byte b = payload[i];if (b == '\n') {- lines.add(takeLine());+ lines.add(takeLine(payload, start, i));+ start = i + 1;+ size = 0;+ continue;+ }+ // Away from the cap, only the final four bytes can affect the next frame's UTF-8 state.+ if (fitsWithinLimit) {+ if (i >= payload.length - 4) {+ trackUtf8(b);+ }continue;}// Force-split BEFORE a character that would push the buffer past the cap, so the emitted@@ -528,20 +548,27 @@ public class ContainerLogStreamer {// the next event whole instead of inflating this one. Only at a character boundary, and// only for a real character start (not a continuation byte).if (pendingContinuation == 0 && (b & 0xC0) != 0x80- && buffer.size() > 0- && buffer.size() + utf8CharLength(b) > MAX_LINE_BYTES) {- lines.add(takeLine());+ && size > 0+ && size + utf8CharLength(b) > MAX_LINE_BYTES) {+ lines.add(takeLine(payload, start, i));+ start = i;+ size = 0;}- buffer.write(b);+ size++;trackUtf8(b);- if ((buffer.size() >= MAX_LINE_BYTES && pendingContinuation == 0)- || buffer.size() >= MAX_LINE_BYTES + 3) {+ if ((size >= MAX_LINE_BYTES && pendingContinuation == 0)+ || size >= MAX_LINE_BYTES + 3) {// Do not split a valid character whose continuation bytes arrive in a later frame.// Malformed input may never reach a boundary, so allow at most the three continuation// bytes a valid UTF-8 character could need before forcing the buffer to drain.- lines.add(takeLine());+ lines.add(takeLine(payload, start, i + 1));+ start = i + 1;+ size = 0;}}+ if (start < payload.length) {+ buffer.write(payload, start, payload.length - start);+ }return lines;}@@ -585,6 +612,15 @@ public class ContainerLogStreamer {}}+ private String takeLine(byte[] payload, int start, int end) {+ if (!hasPendingBytes()) {+ pendingContinuation = 0;+ return new String(payload, start, end - start, StandardCharsets.UTF_8);+ }+ buffer.write(payload, start, end - start);+ return takeLine();+ }+private String takeLine() {String line = buffer.toString(StandardCharsets.UTF_8);buffer.reset();diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.javaindex 1951bb17fc..9a6fda50ec 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamerTest.java@@ -9,6 +9,7 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsServiceimport io.github.hectorvent.floci.testutil.LogCapture;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.MethodSource;import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;@@ -16,11 +17,13 @@ import java.io.Closeable;import java.nio.charset.StandardCharsets;import java.time.Instant;import java.util.ArrayList;+import java.util.Arrays;import java.util.Collections;import java.util.List;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import java.util.logging.LogRecord;+import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;@@ -200,6 +203,91 @@ class ContainerLogStreamerTest {assertEquals(List.of("a", "", "b"), buffer.append(utf8("a\n\nb\n")));}+ static Stream<byte[]> completeFrames() {+ return Stream.of(+ utf8("\n"),+ utf8("\n\n"),+ utf8("first\n\nlast\n\n"),+ utf8("café\n東京\n🚀\n"),+ new byte[]{(byte) 0xe2, '\n', (byte) 0x80, '\n'},+ new byte[]{(byte) 0xf0, (byte) 0x9f, '\n'},+ utf8("a".repeat(256 * 1024 - 4) + "\n"));+ }++ @ParameterizedTest+ @MethodSource("completeFrames")+ void completeFrameMatchesBytewiseReassembly(byte[] payload) {+ ContainerLogStreamer.LogLineBuffer whole = new ContainerLogStreamer.LogLineBuffer();+ ContainerLogStreamer.LogLineBuffer fragmented = new ContainerLogStreamer.LogLineBuffer();+ List<String> expected = new ArrayList<>();+ for (byte value : payload) {+ expected.addAll(fragmented.append(new byte[]{value}));+ }++ assertEquals(expected, whole.append(payload));+ assertFalse(whole.hasPendingBytes());+ assertTrue(whole.flush().isEmpty());+ assertEquals(List.of("next"), whole.append(utf8("next\n")));+ }++ @Test+ void nearCapFrameKeepsEarlySplitBeforeMalformedLead() {+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ String prefix = "a".repeat(256 * 1024 - 2);+ byte[] payload = Arrays.copyOf(utf8(prefix), 256 * 1024);+ payload[payload.length - 2] = (byte) 0xf0;+ payload[payload.length - 1] = '\n';++ assertEquals(List.of(prefix, "\uFFFD"), buffer.append(payload));+ assertFalse(buffer.hasPendingBytes());+ }++ @ParameterizedTest+ @ValueSource(ints = {1, 2, 3})+ void fastFramePreservesContinuationStateForNextFrameAtCap(int characterBytesInFirstFrame) {+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ String prefix = "a".repeat(256 * 1024 - 3 - characterBytesInFirstFrame);+ byte[] payload = utf8(prefix + "🚀€\n");+ int boundary = 256 * 1024 - 3;++ assertEquals(List.of(), buffer.append(Arrays.copyOfRange(payload, 0, boundary)));+ assertEquals(List.of(prefix + "🚀", "€"),+ buffer.append(Arrays.copyOfRange(payload, boundary, payload.length)));+ assertFalse(buffer.hasPendingBytes());+ }++ @ParameterizedTest+ @ValueSource(ints = {1, 3, 1024, 262145})+ void frameBoundariesPreserveCappedUnicodeLinesAndTrailingContent(int frameSize) {+ int cap = 256 * 1024;+ String first = "a".repeat(cap - 3) + "€";+ String rockets = "🚀".repeat(cap / 4);+ byte[] payload = utf8(first + "\nshort\n" + rockets + "🚀\ntail");+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ List<String> lines = new ArrayList<>();++ for (int offset = 0; offset < payload.length; offset += frameSize) {+ lines.addAll(buffer.append(Arrays.copyOfRange(payload, offset,+ Math.min(offset + frameSize, payload.length))));+ }++ assertEquals(List.of(first, "", "short", rockets, "🚀"), lines);+ assertEquals("tail", buffer.flush().orElseThrow());+ assertFalse(buffer.hasPendingBytes());+ }++ @Test+ void emittedLinesAndBufferedFragmentsSurvivePayloadReuse() {+ ContainerLogStreamer.LogLineBuffer buffer = new ContainerLogStreamer.LogLineBuffer();+ byte[] payload = utf8("first\npartial");++ List<String> lines = buffer.append(payload);+ Arrays.fill(payload, (byte) 'x');++ assertEquals(List.of("first"), lines);+ assertEquals(List.of("partial"), buffer.append(utf8("\n")));+ }+@Testvoid boundsBufferBySplittingOverlongUnterminatedRuns() {// A 256 KB run with no newline must be force-emitted (bounded), not buffered until the