Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: preserve stderr stream and Docker timestamps in project logs (#3770)
Kyle Mendellcac8a9090bae
diff --git a/backend/pkg/libarcane/ws/parse.go b/backend/pkg/libarcane/ws/parse.goindex f750b9fb7..450f85495 100644--- a/backend/pkg/libarcane/ws/parse.go+++ b/backend/pkg/libarcane/ws/parse.go@@ -7,12 +7,41 @@ import ()// Docker's RFC3339 timestamp when timestamps=true-var dockerTimestamp = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z\s+`)+var dockerTimestamp = regexp.MustCompile(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z)\s+`)-// NormalizeContainerLine parses a raw container log line into level + cleaned message.-// It extracts Docker's timestamp if present (when timestamps=true in Docker API).-func NormalizeContainerLine(raw string) (level string, msg string, timestamp string) {- // Fast trim right for common cases to avoid scanning the whole string+// stripStreamMetadataInternal removes one leading Arcane stderr marker and one+// leading Docker RFC3339 timestamp (in either order) from the front of line.+// It reports whether the marker was removed and, when allowTimestamp is true+// and a valid Docker timestamp was found, its normalized value. The transport+// adds at most one marker and one timestamp per line, so anything past the+// first of each is application content and stays; timestamps that fail to+// parse are left in place.+func stripStreamMetadataInternal(line string, allowTimestamp bool) (rest string, sawStderr bool, timestamp string) {+ for {+ line = strings.TrimLeft(line, " \t")++ if !sawStderr && strings.HasPrefix(line, "[STDERR] ") {+ sawStderr = true+ line = line[len("[STDERR] "):]+ continue+ }++ if allowTimestamp && timestamp == "" && len(line) > 20 && line[0] >= '0' && line[0] <= '9' {+ if loc := dockerTimestamp.FindStringSubmatchIndex(line); loc != nil {+ if parsed, err := time.Parse(time.RFC3339Nano, line[loc[2]:loc[3]]); err == nil {+ timestamp = parsed.UTC().Format(time.RFC3339Nano)+ line = line[loc[1]:]+ continue+ }+ }+ }+ break+ }++ return line, sawStderr, timestamp+}++func trimTrailingNewlinesInternal(raw string) string {end := len(raw)for end > 0 {c := raw[end-1]@@ -21,57 +50,55 @@ func NormalizeContainerLine(raw string) (level string, msg string, timestamp str}end--}- line := raw[:end]+ return raw[:end]+}+// NormalizeContainerLine parses a raw container log line into level + cleaned message.+// It extracts Docker's timestamp if present (when timestamps=true in Docker API).+func NormalizeContainerLine(raw string) (level string, msg string, timestamp string) {level = "stdout"- // Check prefixes using slicing for performance- switch {- case strings.HasPrefix(line, "[STDERR] "):- level = "stderr"- line = line[9:]- case strings.HasPrefix(line, "stderr:"):++ line, sawStderr, ts := stripStreamMetadataInternal(trimTrailingNewlinesInternal(raw), true)+ if sawStderr {level = "stderr"- line = line[7:]- case strings.HasPrefix(line, "stdout:"):- level = "stdout"- line = line[7:]}-- // Extract and strip Docker's RFC3339 timestamp (when timestamps=true)- // Optimization: Check if it looks like a timestamp before running regex- if len(line) > 20 && line[0] >= '0' && line[0] <= '9' {- if loc := dockerTimestamp.FindStringIndex(line); loc != nil {- // loc[0] is start, loc[1] is end- matchStr := line[loc[0]:loc[1]]- trimmed := strings.TrimSpace(matchStr)-- if parsed, err := time.Parse(time.RFC3339Nano, trimmed); err == nil {- timestamp = parsed.UTC().Format(time.RFC3339Nano)- } else if parsed, err := time.Parse(time.RFC3339, trimmed); err == nil {- timestamp = parsed.UTC().Format(time.RFC3339Nano)- }-- // Strip the timestamp from the line- line = line[loc[1]:]- }+ if ts != "" {+ timestamp = ts}- // Return the message as-is (including any application-level timestamps)return level, strings.TrimSpace(line), timestamp}// NormalizeProjectLine additionally extracts service (pattern: service | message).-// Returns level, service, message, timestamp (RFC3339Nano) — timestamp may be empty.+// It normalizes markers and Docker timestamps both before and after splitting+// the service prefix so either ordering resolves; stderr classification wins+// if either pass detects it, and the first valid Docker timestamp found is+// used and removed from the message. Application-generated timestamps deeper+// inside the message are preserved.func NormalizeProjectLine(raw string) (level, service, msg, timestamp string) {- level, base, ts := NormalizeContainerLine(raw)- timestamp = ts-+ level = "stdout"service = ""++ head, sawHeadStderr, headTS := stripStreamMetadataInternal(trimTrailingNewlinesInternal(raw), true)+ timestamp = headTS++ base := headif parts := strings.SplitN(base, " | ", 2); len(parts) == 2 {service = strings.TrimSpace(parts[0])- base = parts[1]++ message, sawMessageStderr, messageTS := stripStreamMetadataInternal(parts[1], headTS == "")+ if sawMessageStderr || sawHeadStderr {+ level = "stderr"+ }+ if messageTS != "" {+ timestamp = messageTS+ }+ base = message+ } else if sawHeadStderr {+ level = "stderr"}- return level, service, base, timestamp++ return level, service, strings.TrimSpace(base), timestamp}func NowRFC3339() string {diff --git a/backend/pkg/libarcane/ws/parse_test.go b/backend/pkg/libarcane/ws/parse_test.goindex 8a3208e79..7fbacfe8b 100644--- a/backend/pkg/libarcane/ws/parse_test.go+++ b/backend/pkg/libarcane/ws/parse_test.go@@ -31,17 +31,17 @@ func TestNormalizeContainerLine(t *testing.T) {expectedTS: "",},{- name: "stderr colon prefix",+ name: "stderr colon prefix is application content",raw: "stderr:error message",- expectedLevel: "stderr",- expectedMsg: "error message",+ expectedLevel: "stdout",+ expectedMsg: "stderr:error message",expectedTS: "",},{- name: "stdout colon prefix",- raw: "stdout:normal message",- expectedLevel: "stdout",- expectedMsg: "normal message",+ name: "repeated stderr marker keeps content",+ raw: "[STDERR] [STDERR] error occurred",+ expectedLevel: "stderr",+ expectedMsg: "[STDERR] error occurred",expectedTS: "",},{@@ -86,6 +86,13 @@ func TestNormalizeContainerLine(t *testing.T) {expectedMsg: "critical error",expectedTS: "2024-06-01T12:00:00Z",},+ {+ name: "Docker timestamp before stderr marker",+ raw: "2024-06-01T12:00:00.000Z [STDERR] late failure",+ expectedLevel: "stderr",+ expectedMsg: "late failure",+ expectedTS: "2024-06-01T12:00:00Z",+ },{name: "empty line",raw: "",@@ -183,6 +190,30 @@ func TestNormalizeProjectLine(t *testing.T) {expectedMsg: "PING received",expectedTS: "",},+ {+ name: "Docker timestamp before service prefix",+ raw: "2024-06-01T12:00:00.500Z web | booted",+ expectedLevel: "stdout",+ expectedService: "web",+ expectedMsg: "booted",+ expectedTS: "2024-06-01T12:00:00.5Z",+ },+ {+ name: "stderr marker and Docker timestamp after service prefix",+ raw: "web | [STDERR] 2024-06-01T12:00:00.250Z bind failed",+ expectedLevel: "stderr",+ expectedService: "web",+ expectedMsg: "bind failed",+ expectedTS: "2024-06-01T12:00:00.25Z",+ },+ {+ name: "application timestamp inside message is preserved",+ raw: "2024-06-01T09:00:00Z web | retried at 09:05:00.000 again",+ expectedLevel: "stdout",+ expectedService: "web",+ expectedMsg: "retried at 09:05:00.000 again",+ expectedTS: "2024-06-01T09:00:00Z",+ },}for _, tt := range tests {diff --git a/backend/pkg/projects/cmds.go b/backend/pkg/projects/cmds.goindex 35a818e32..84e19acd5 100644--- a/backend/pkg/projects/cmds.go+++ b/backend/pkg/projects/cmds.go@@ -163,14 +163,18 @@ func ComposeDown(ctx context.Context, proj *types.Project, removeVolumes bool) ereturn c.svc.Down(downCtx, proj.Name, api.DownOptions{RemoveOrphans: true, Volumes: removeVolumes})}+// ComposeLogs streams a project's logs through the compose service. The+// one-shot client for this call is built with the log-parity wrapper so+// non-TTY containers keep their stderr metadata; deploy, attach, and other+// compose operations use plain clients.func ComposeLogs(ctx context.Context, projectName string, out io.Writer, follow bool, tail, since string, timestamps bool) error {- c, err := NewClient(ctx, "", nil, nil)+ c, err := NewClient(ctx, "", nil, nil, streamDemuxedLogsInternal)if err != nil {return err}defer func() { _ = c.Close() }()- return c.svc.Logs(ctx, projectName, writerConsumer{out: out}, api.LogOptions{Follow: follow, Tail: tail, Since: since, Timestamps: timestamps})+ return c.svc.Logs(ctx, projectName, &writerConsumer{out: out}, api.LogOptions{Follow: follow, Tail: tail, Since: since, Timestamps: timestamps})}// ListGlobalComposeContainers lists every container carrying a composediff --git a/backend/pkg/projects/compose.go b/backend/pkg/projects/compose.goindex 01524448a..1fd4bc6bb 100644--- a/backend/pkg/projects/compose.go+++ b/backend/pkg/projects/compose.go@@ -5,7 +5,9 @@ import ("io""log/slog""maps"+ "slices""strings"+ "sync""sync/atomic""github.com/docker/cli/cli/command"@@ -27,10 +29,18 @@ type Client struct {logWriter io.WriteCloser}+// composeClientOptionInternal selects internal wiring variants for one-shot+// compose clients.+type composeClientOptionInternal uint8++// streamDemuxedLogsInternal wires the log-parity wrapper so non-TTY container+// log responses keep stderr metadata as Arcane's [STDERR] marker.+const streamDemuxedLogsInternal composeClientOptionInternal = 1+// NewClient builds a compose client. dockerHost, when non-empty, pins the// docker CLI to that daemon endpoint instead of letting it resolve one from// the environment.-func NewClient(ctx context.Context, dockerHost string, authConfigs map[string]registry.AuthConfig, prompt compose.Prompt) (*Client, error) {+func NewClient(ctx context.Context, dockerHost string, authConfigs map[string]registry.AuthConfig, prompt compose.Prompt, options ...composeClientOptionInternal) (*Client, error) {cli, err := command.NewDockerCli()if err != nil {return nil, err@@ -51,6 +61,9 @@ func NewClient(ctx context.Context, dockerHost string, authConfigs map[string]re}composeCLI := wrapDockerCLIWithInspectCompatibilityInternal(cli)+ if slices.Contains(options, streamDemuxedLogsInternal) {+ composeCLI = wrapDockerCLIWithLogsDemuxInternal(cli)+ }// When the caller streams operation output, render compose's own progress// events exactly as `docker compose --progress=plain` prints them.@@ -192,21 +205,35 @@ func (c *Client) Close() error {return nil}-type writerConsumer struct{ out io.Writer }+// writerConsumer serializes compose log events: compose may stream several+// containers concurrently into one consumer.+type writerConsumer struct {+ out io.Writer+ mu sync.Mutex+ writeErrLogged bool+}++// stderrLogLinePrefixInternal marks stderr content so the project-log parser+// (pkg/libarcane/ws) can classify lines the same way container logs do.+const stderrLogLinePrefixInternal = "[STDERR] "-func (w writerConsumer) Register(container string) {}-func (w writerConsumer) Start(container string) {}-func (w writerConsumer) Stop(container string) {}-func (w writerConsumer) Status(container, msg string) {}-func (w writerConsumer) Log(container, msg string) {+func (w *writerConsumer) Register(container string) {}+func (w *writerConsumer) Start(container string) {}+func (w *writerConsumer) Stop(container string) {}+func (w *writerConsumer) Status(container, msg string) {}+func (w *writerConsumer) Log(container, msg string) {w.write(container, msg)}-func (w writerConsumer) Err(container, msg string) {- w.write(container, msg)+func (w *writerConsumer) Err(container, msg string) {+ if msg == "" || strings.HasPrefix(msg, stderrLogLinePrefixInternal) {+ w.write(container, msg)+ return+ }+ w.write(container, stderrLogLinePrefixInternal+msg)}-func (w writerConsumer) write(container, msg string) {+func (w *writerConsumer) write(container, msg string) {if w.out == nil {return}@@ -217,5 +244,10 @@ func (w writerConsumer) write(container, msg string) {if !strings.HasSuffix(output, "\n") {output += "\n"}- _, _ = io.WriteString(w.out, output)+ w.mu.Lock()+ defer w.mu.Unlock()+ if _, err := io.WriteString(w.out, output); err != nil && !w.writeErrLogged {+ w.writeErrLogged = true+ slog.Debug("project log output write failed; subsequent output may be truncated", "error", err)+ }}diff --git a/backend/pkg/projects/compose_logs_test.go b/backend/pkg/projects/compose_logs_test.gonew file mode 100644index 000000000..bbd212883--- /dev/null+++ b/backend/pkg/projects/compose_logs_test.go@@ -0,0 +1,122 @@+package projects++import (+ "bytes"+ "context"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "regexp"+ "strings"+ "testing"++ "github.com/moby/moby/api/pkg/stdcopy"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++const (+ composeLogsTestProjectInternal = "projlogdemo"+ composeLogsTestNonTTYContainerInternal = "11112222333344445555666677778888"+ composeLogsTestTTYContainerInternal = "9999aaaabbbbccccddddeeeeffff0000"+ composeLogsTestTimestampInternal = "2024-08-26T10:00:00.000000000Z "+)++func TestComposeLogsMarksStderrWithoutChangingComposeOptions(t *testing.T) {+ logsQueries := make(chan string, 4)++ summary := func(id, service, image string) map[string]any {+ return map[string]any{+ "Id": id,+ "Names": []string{"/" + composeLogsTestProjectInternal + "-" + service + "-1"},+ "Image": image,+ "State": "running",+ "Status": "Up",+ "Labels": map[string]string{+ "com.docker.compose.project": composeLogsTestProjectInternal,+ "com.docker.compose.service": service,+ "com.docker.compose.container-number": "1",+ "com.docker.compose.oneoff": "False",+ },+ }+ }++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ path := regexp.MustCompile(`^/v[0-9.]+`).ReplaceAllString(r.URL.Path, "")++ switch path {+ case "/_ping":+ w.Header().Set("Api-Version", "1.47")+ w.WriteHeader(http.StatusOK)+ case "/containers/json":+ w.Header().Set("Content-Type", "application/json")+ require.NoError(t, json.NewEncoder(w).Encode([]map[string]any{+ summary(composeLogsTestNonTTYContainerInternal, "web", "nginx"),+ summary(composeLogsTestTTYContainerInternal, "ttyd", "ttyd"),+ }))+ case "/containers/" + composeLogsTestNonTTYContainerInternal + "/json":+ writeComposeInspectJSONInternal(t, w, composeLogsTestNonTTYContainerInternal, "web", false)+ case "/containers/" + composeLogsTestTTYContainerInternal + "/json":+ writeComposeInspectJSONInternal(t, w, composeLogsTestTTYContainerInternal, "ttyd", true)+ case "/containers/" + composeLogsTestNonTTYContainerInternal + "/logs":+ logsQueries <- r.URL.RawQuery+ w.Header().Set("Content-Type", "application/vnd.docker.multiplexed-stream")+ _, err := w.Write(bytes.Join([][]byte{+ frameLogEntryInternal(stdcopy.Stdout, []byte(composeLogsTestTimestampInternal+"hello stdout\n")),+ frameLogEntryInternal(stdcopy.Stderr, []byte(composeLogsTestTimestampInternal+"hello stderr\n")),+ }, nil))+ require.NoError(t, err)+ case "/containers/" + composeLogsTestTTYContainerInternal + "/logs":+ w.Header().Set("Content-Type", "application/vnd.docker.raw-stream")+ _, err := w.Write([]byte(composeLogsTestTimestampInternal + "raw tty output\n"))+ require.NoError(t, err)+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ t.Setenv("DOCKER_HOST", "tcp://"+strings.TrimPrefix(server.URL, "http://"))++ out := &bytes.Buffer{}+ err := ComposeLogs(context.Background(), composeLogsTestProjectInternal, out, false, "all", "", true)+ require.NoError(t, err)++ body := out.String()+ require.Contains(t, body, "hello stdout")+ require.Contains(t, body, "| [STDERR] 2024-08-26T10:00:00.000000000Z hello stderr\n")+ require.Contains(t, body, "raw tty output")+ assert.NotContains(t, body, "[STDERR] raw tty")+ assert.NotContains(t, body, "[STDERR] hello stdout")++ query := <-logsQueries+ assert.Contains(t, query, "stdout=1")+ assert.Contains(t, query, "stderr=1")+ assert.Contains(t, query, "timestamps=1")+ assert.NotContains(t, query, "follow=")+ assert.NotContains(t, query, "tail=")+}++func writeComposeInspectJSONInternal(t *testing.T, w http.ResponseWriter, containerID, serviceName string, tty bool) {+ t.Helper()++ w.Header().Set("Content-Type", "application/json")+ require.NoError(t, json.NewEncoder(w).Encode(map[string]any{+ "Id": containerID,+ "Name": "/" + composeLogsTestProjectInternal + "-" + serviceName + "-1",+ "Created": "2024-08-26T09:00:00Z",+ "Path": "/bin/sh",+ "State": map[string]any{+ "Running": true,+ "StartedAt": "2024-08-26T09:00:01Z",+ },+ "Config": map[string]any{+ "Tty": tty,+ "Image": "example-image",+ "Labels": map[string]string{},+ },+ "HostConfig": map[string]any{},+ "NetworkSettings": map[string]any{},+ "Mounts": []any{},+ }))+}diff --git a/backend/pkg/projects/logs_demux.go b/backend/pkg/projects/logs_demux.gonew file mode 100644index 000000000..85a5877ab--- /dev/null+++ b/backend/pkg/projects/logs_demux.go@@ -0,0 +1,186 @@+package projects++import (+ "bufio"+ "bytes"+ "context"+ "encoding/binary"+ "io"+ "strings"+ "sync"++ "github.com/docker/cli/cli/command"+ "github.com/moby/moby/api/pkg/stdcopy"+ "github.com/moby/moby/client"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"+)++// logsDemuxCompatibleDockerCliInternal exposes an API client that keeps+// stderr metadata alive in non-TTY container log streams. Used only by+// ComposeLogs; other compose operations keep the unmodified client.+type logsDemuxCompatibleDockerCliInternal struct {+ command.Cli++ apiClient client.APIClient+}++func (c *logsDemuxCompatibleDockerCliInternal) Client() client.APIClient {+ return c.apiClient+}++func wrapDockerCLIWithLogsDemuxInternal(cli command.Cli) command.Cli {+ if cli == nil {+ return nil+ }+ return &logsDemuxCompatibleDockerCliInternal{+ Cli: cli,+ apiClient: &logsDemuxDockerClientInternal{APIClient: libarcane.WrapDockerAPIClientForInspectCompatibility(cli.Client())},+ }+}++// logsDemuxDockerClientInternal records each container's TTY state from the+// inspect calls Compose already performs, and reworks non-TTY multiplexed log+// responses so stderr entries stay identifiable: stderr lines are prefixed+// with Arcane's stderr marker and every entry is re-framed as a stdout+// stdcopy frame, which Compose merges through its stdout consumer path. TTY+// containers pass through untouched because their stream carries no stderr+// metadata.+type logsDemuxDockerClientInternal struct {+ client.APIClient++ ttyMu sync.RWMutex+ containerTty map[string]bool+}++func (c *logsDemuxDockerClientInternal) ContainerInspect(ctx context.Context, containerID string, options client.ContainerInspectOptions) (client.ContainerInspectResult, error) {+ result, err := c.APIClient.ContainerInspect(ctx, containerID, options)+ if err == nil && result.Container.Config != nil {+ c.recordContainerTTYInternal(containerID, result.Container.Config.Tty)+ }+ return result, err+}++func (c *logsDemuxDockerClientInternal) ContainerLogs(ctx context.Context, containerID string, options client.ContainerLogsOptions) (client.ContainerLogsResult, error) {+ resp, err := c.APIClient.ContainerLogs(ctx, containerID, options)+ if err != nil {+ return resp, err+ }++ tty, known := c.containerTTYInternal(containerID)+ if tty || !known || !options.ShowStdout || !options.ShowStderr {+ return resp, nil+ }++ return &stderrMarkedDemuxedStreamInternal{upstream: resp, src: bufio.NewReader(resp)}, nil+}++func (c *logsDemuxDockerClientInternal) recordContainerTTYInternal(containerID string, tty bool) {+ key := strings.ToLower(strings.TrimSpace(containerID))+ c.ttyMu.Lock()+ defer c.ttyMu.Unlock()+ if c.containerTty == nil {+ c.containerTty = make(map[string]bool, 8)+ }+ c.containerTty[key] = tty+}++func (c *logsDemuxDockerClientInternal) containerTTYInternal(containerID string) (tty bool, known bool) {+ key := strings.ToLower(strings.TrimSpace(containerID))+ c.ttyMu.RLock()+ defer c.ttyMu.RUnlock()+ tty, known = c.containerTty[key]+ return tty, known+}++// maxLogEntryBytesInternal caps re-framed entries: a newline-less run is+// split into cap-sized entries so per-line buffering stays bounded end to end.+const maxLogEntryBytesInternal = 64 * 1024++// stderrMarkedDemuxedStreamInternal transforms a multiplexed Docker log+// response, pulled frame by frame on demand, into stdout stdcopy frames of+// one log line each, with stderr lines prefixed by Arcane's stderr marker.+// Lines are assembled per source stream before merging so frames interleaved+// mid-line cannot corrupt each other. Closing it closes the upstream+// response, which also unblocks any pending read.+type stderrMarkedDemuxedStreamInternal struct {+ upstream io.ReadCloser+ src *bufio.Reader+ out bytes.Buffer+ pending [2][]byte+ err error+}++func (s *stderrMarkedDemuxedStreamInternal) Read(p []byte) (int, error) {+ for s.out.Len() == 0 {+ if s.err != nil {+ return 0, s.err+ }+ s.ingestFrameInternal()+ }+ return s.out.Read(p)+}++func (s *stderrMarkedDemuxedStreamInternal) Close() error {+ return s.upstream.Close()+}++func (s *stderrMarkedDemuxedStreamInternal) ingestFrameInternal() {+ var header [8]byte+ if _, err := io.ReadFull(s.src, header[:]); err != nil {+ s.finishInternal(err)+ return+ }+ payload := make([]byte, binary.BigEndian.Uint32(header[4:]))+ if _, err := io.ReadFull(s.src, payload); err != nil {+ s.finishInternal(err)+ return+ }++ stream := 0+ if header[0] == byte(stdcopy.Stderr) {+ stream = 1+ }+ s.pending[stream] = append(s.pending[stream], payload...)+ buf := s.pending[stream]+ for len(buf) > 0 {+ nl := bytes.IndexByte(buf, '\n')+ if nl < 0 || nl > maxLogEntryBytesInternal {+ if len(buf) < maxLogEntryBytesInternal {+ break+ }+ s.emitLineInternal(stream, buf[:maxLogEntryBytesInternal])+ buf = buf[maxLogEntryBytesInternal:]+ continue+ }+ s.emitLineInternal(stream, buf[:nl])+ buf = buf[nl+1:]+ }+ s.pending[stream] = buf+}++func (s *stderrMarkedDemuxedStreamInternal) finishInternal(err error) {+ for stream, rest := range s.pending {+ if len(rest) > 0 {+ s.emitLineInternal(stream, rest)+ s.pending[stream] = nil+ }+ }+ s.err = err+}++func (s *stderrMarkedDemuxedStreamInternal) emitLineInternal(stream int, line []byte) {+ size := len(line) + 1+ if stream == 1 {+ size += len(stderrLogLinePrefixInternal)+ }+ var header [8]byte+ header[0] = byte(stdcopy.Stdout)+ binary.BigEndian.PutUint32(header[4:], uint32(size)) //nolint:gosec // entries are capped at maxLogEntryBytesInternal+ s.out.Write(header[:])+ if stream == 1 {+ s.out.WriteString(stderrLogLinePrefixInternal)+ }+ s.out.Write(line)+ s.out.WriteByte('\n')+}diff --git a/backend/pkg/projects/logs_demux_test.go b/backend/pkg/projects/logs_demux_test.gonew file mode 100644index 000000000..189388fbc--- /dev/null+++ b/backend/pkg/projects/logs_demux_test.go@@ -0,0 +1,118 @@+package projects++import (+ "bufio"+ "bytes"+ "context"+ "encoding/binary"+ "io"+ "strings"+ "testing"++ "github.com/moby/moby/api/pkg/stdcopy"+ "github.com/moby/moby/client"+ "github.com/stretchr/testify/require"+)++func frameLogEntryInternal(fd stdcopy.StdType, payload []byte) []byte {+ header := make([]byte, 8)+ header[0] = byte(fd)+ binary.BigEndian.PutUint32(header[4:], uint32(len(payload)))+ return append(header, payload...)+}++func decodeDemuxedStreamInternal(t *testing.T, stream io.Reader) string {+ t.Helper()+ decoded := &bytes.Buffer{}+ sinkErr := &bytes.Buffer{}+ _, err := stdcopy.StdCopy(decoded, sinkErr, stream)+ require.NoError(t, err)+ require.Empty(t, sinkErr.String(), "every entry must be re-framed as stdout")+ return decoded.String()+}++func TestDemultiplexedLogStreamWithStderrMarking(t *testing.T) {+ longRun := strings.Repeat("x", maxLogEntryBytesInternal+100)+ source := bytes.Join([][]byte{+ frameLogEntryInternal(stdcopy.Stdout, []byte("line-one\npar")),+ frameLogEntryInternal(stdcopy.Stderr, []byte("boom\n")),+ frameLogEntryInternal(stdcopy.Stdout, []byte("tial\n")),+ frameLogEntryInternal(stdcopy.Stderr, []byte(longRun)),+ frameLogEntryInternal(stdcopy.Stderr, []byte("\nfinal")),+ }, nil)++ resp := io.NopCloser(bytes.NewReader(source))+ stream := &stderrMarkedDemuxedStreamInternal{upstream: resp, src: bufio.NewReader(resp)}++ require.Equal(t,+ "line-one\n"++ "[STDERR] boom\n"++ "partial\n"++ "[STDERR] "+longRun[:maxLogEntryBytesInternal]+"\n"++ "[STDERR] "+longRun[maxLogEntryBytesInternal:]+"\n"++ "[STDERR] final\n",+ decodeDemuxedStreamInternal(t, stream))+}++type fakeResponseBodyInternal struct {+ io.Reader+ closed bool+}++func (b *fakeResponseBodyInternal) Close() error {+ b.closed = true+ return nil+}++type stubLogAPIClientInternal struct {+ client.APIClient++ resp io.ReadCloser+}++func (s *stubLogAPIClientInternal) ContainerLogs(ctx context.Context, containerID string, options client.ContainerLogsOptions) (client.ContainerLogsResult, error) {+ return s.resp, nil+}++func TestLogsDemuxClientContainerLogsDecisionInternal(t *testing.T) {+ muxed := bytes.Join([][]byte{+ frameLogEntryInternal(stdcopy.Stdout, []byte("out\n")),+ frameLogEntryInternal(stdcopy.Stderr, []byte("err\n")),+ }, nil)++ tests := []struct {+ name string+ ttyRecorded bool+ containerKnown bool+ showStdout bool+ showStderr bool+ wantWrapped bool+ }{+ {name: "non-tty streams are transformed", containerKnown: true, showStdout: true, showStderr: true, wantWrapped: true},+ {name: "tty passthrough", ttyRecorded: true, containerKnown: true, showStdout: true, showStderr: true, wantWrapped: false},+ {name: "unknown container passthrough", containerKnown: false, showStdout: true, showStderr: true, wantWrapped: false},+ {name: "single-stream request passthrough", containerKnown: true, showStdout: true, showStderr: false, wantWrapped: false},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ body := &fakeResponseBodyInternal{Reader: bytes.NewReader(muxed)}+ inner := &logsDemuxDockerClientInternal{APIClient: &stubLogAPIClientInternal{resp: body}}+ if tt.containerKnown {+ inner.recordContainerTTYInternal(" CTR-1 ", tt.ttyRecorded)+ }++ got, err := inner.ContainerLogs(context.Background(), "ctr-1", client.ContainerLogsOptions{ShowStdout: tt.showStdout, ShowStderr: tt.showStderr})+ require.NoError(t, err)++ if !tt.wantWrapped {+ require.Same(t, body, got)+ return+ }++ require.Equal(t, "out\n[STDERR] err\n", decodeDemuxedStreamInternal(t, got))+ require.NoError(t, got.Close())+ require.True(t, body.closed, "closing the wrapped stream must close the upstream response")+ })+ }+}