Skip to content

Modern Docker Management, Designed for Everyone

fix: certain tty:true containers logs never stream correctly (#2119)

Kyle Mendell0f146e8243df
diff --git a/backend/internal/api/ws_handler.go b/backend/internal/api/ws_handler.go
index 5f2ec3fb2..0df4b0c45 100644
--- a/backend/internal/api/ws_handler.go
+++ b/backend/internal/api/ws_handler.go
@@ -455,7 +455,14 @@ func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batc
lines := make(chan string, 256)
go func(ctx context.Context) {
defer close(lines)
- _ = h.containerService.StreamLogs(ctx, containerID, lines, follow, tail, since, timestamps)
+ if err := h.containerService.StreamLogs(ctx, containerID, lines, follow, tail, since, timestamps); err != nil {
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ return
+ }
+
+ slog.Warn("container log stream failed", "containerID", containerID, "error", err)
+ broadcastContainerLogStreamErrorInternal(ls, err)
+ }
}(ctx)
if format == "json" {
@@ -489,6 +496,28 @@ func (h *WebSocketHandler) startContainerLogHub(containerID, format string, batc
return ls.hub
}
+func broadcastContainerLogStreamErrorInternal(stream *wsLogStream, err error) {
+ if stream == nil || stream.hub == nil || err == nil {
+ return
+ }
+
+ message := "Failed to stream container logs: " + err.Error()
+ if stream.format == "json" {
+ msg := ws.LogMessage{
+ Seq: stream.seq.Add(1),
+ Level: "error",
+ Message: message,
+ Timestamp: ws.NowRFC3339(),
+ }
+ if b, marshalErr := json.Marshal(msg); marshalErr == nil {
+ stream.hub.Broadcast(b)
+ }
+ return
+ }
+
+ stream.hub.Broadcast([]byte(message))
+}
+
// ContainerStats streams container stats over WebSocket.
//
// @Summary Get container stats via WebSocket
diff --git a/backend/internal/api/ws_handler_test.go b/backend/internal/api/ws_handler_test.go
new file mode 100644
index 000000000..a7822800f
--- /dev/null
+++ b/backend/internal/api/ws_handler_test.go
@@ -0,0 +1,96 @@
+package api
+
+import (
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+
+ ws "github.com/getarcaneapp/arcane/backend/pkg/libarcane/ws"
+ "github.com/gorilla/websocket"
+ "github.com/stretchr/testify/require"
+)
+
+func TestBroadcastContainerLogStreamErrorInternal_JSON(t *testing.T) {
+ hub := ws.NewHub(10)
+ ctx := t.Context()
+ go hub.Run(ctx)
+
+ clientConn, serverConn, cleanup := newTestWSPairInternal(t)
+ t.Cleanup(cleanup)
+
+ ws.ServeClient(ctx, hub, serverConn)
+
+ stream := &wsLogStream{
+ hub: hub,
+ format: "json",
+ }
+
+ broadcastContainerLogStreamErrorInternal(stream, errors.New("boom"))
+
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, payload, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+
+ var message ws.LogMessage
+ require.NoError(t, json.Unmarshal(payload, &message))
+ require.Equal(t, "error", message.Level)
+ require.Equal(t, "Failed to stream container logs: boom", message.Message)
+ require.Empty(t, message.ContainerID)
+ require.NotEmpty(t, message.Timestamp)
+}
+
+func TestBroadcastContainerLogStreamErrorInternal_Text(t *testing.T) {
+ hub := ws.NewHub(10)
+ ctx := t.Context()
+ go hub.Run(ctx)
+
+ clientConn, serverConn, cleanup := newTestWSPairInternal(t)
+ t.Cleanup(cleanup)
+
+ ws.ServeClient(ctx, hub, serverConn)
+
+ stream := &wsLogStream{
+ hub: hub,
+ format: "text",
+ }
+
+ broadcastContainerLogStreamErrorInternal(stream, errors.New("boom"))
+
+ _ = clientConn.SetReadDeadline(time.Now().Add(2 * time.Second))
+ _, payload, err := clientConn.ReadMessage()
+ require.NoError(t, err)
+ require.Equal(t, "Failed to stream container logs: boom", string(payload))
+}
+
+func newTestWSPairInternal(t *testing.T) (clientConn *websocket.Conn, serverConn *websocket.Conn, cleanup func()) {
+ t.Helper()
+ serverReady := make(chan *websocket.Conn, 1)
+
+ upgrader := websocket.Upgrader{CheckOrigin: func(_ *http.Request) bool { return true }}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ conn, err := upgrader.Upgrade(w, r, nil)
+ if err != nil {
+ return
+ }
+ serverReady <- conn
+ }))
+
+ url := "ws" + strings.TrimPrefix(server.URL, "http")
+ clientConn, resp, err := websocket.DefaultDialer.Dial(url, nil)
+ require.NoError(t, err)
+ if resp != nil {
+ _ = resp.Body.Close()
+ }
+
+ serverConn = <-serverReady
+
+ return clientConn, serverConn, func() {
+ _ = clientConn.Close()
+ _ = serverConn.Close()
+ server.Close()
+ }
+}
diff --git a/backend/internal/services/container_service.go b/backend/internal/services/container_service.go
index 3cb11be4b..03310aefc 100644
--- a/backend/internal/services/container_service.go
+++ b/backend/internal/services/container_service.go
@@ -609,6 +609,11 @@ func (s *ContainerService) StreamLogs(ctx context.Context, containerID string, l
return fmt.Errorf("failed to connect to Docker: %w", err)
}
+ containerInspect, err := libarcane.ContainerInspectWithCompatibility(ctx, dockerClient, containerID, client.ContainerInspectOptions{})
+ if err != nil {
+ return fmt.Errorf("failed to inspect container for logs: %w", err)
+ }
+
options := client.ContainerLogsOptions{
ShowStdout: true,
ShowStderr: true,
@@ -624,114 +629,147 @@ func (s *ContainerService) StreamLogs(ctx context.Context, containerID string, l
}
defer func() { _ = logs.Close() }()
+ isTTY := containerInspect.Container.Config != nil && containerInspect.Container.Config.Tty
+ return s.streamContainerLogsInternal(ctx, logs, logsChan, follow, isTTY)
+}
+
+func (s *ContainerService) streamContainerLogsInternal(ctx context.Context, logs io.ReadCloser, logsChan chan<- string, follow bool, isTTY bool) error {
+ if isTTY {
+ return s.streamRawLogsInternal(ctx, logs, logsChan)
+ }
if follow {
return s.streamMultiplexedLogs(ctx, logs, logsChan)
}
+ return s.readAllLogs(ctx, logs, logsChan)
+}
- return s.readAllLogs(logs, logsChan)
+func (s *ContainerService) streamRawLogsInternal(ctx context.Context, logs io.Reader, logsChan chan<- string) error {
+ return s.readLogsFromReader(ctx, logs, logsChan, "")
}
-func (s *ContainerService) streamMultiplexedLogs(ctx context.Context, logs io.ReadCloser, logsChan chan<- string) error {
+func (s *ContainerService) streamMultiplexedLogs(ctx context.Context, logs io.Reader, logsChan chan<- string) error {
// Use stdcopy to demultiplex Docker's stream format
// Docker multiplexes stdout and stderr in a special format
stdoutReader, stdoutWriter := io.Pipe()
stderrReader, stderrWriter := io.Pipe()
+ errCh := make(chan error, 3)
// Start demultiplexing in a goroutine
go func() {
- defer func() { _ = stdoutWriter.Close() }()
- defer func() { _ = stderrWriter.Close() }()
_, err := stdcopy.StdCopy(stdoutWriter, stderrWriter, logs)
if err != nil && !errors.Is(err, io.EOF) {
- fmt.Printf("Error demultiplexing logs: %v\n", err)
+ _ = stdoutWriter.CloseWithError(err)
+ _ = stderrWriter.CloseWithError(err)
+ errCh <- fmt.Errorf("failed to demultiplex logs: %w", err)
+ return
}
+ _ = stdoutWriter.Close()
+ _ = stderrWriter.Close()
+ errCh <- nil
}()
// Read from both stdout and stderr concurrently
- done := make(chan error, 2)
-
- // Read stdout
go func() {
- done <- s.readLogsFromReader(ctx, stdoutReader, logsChan, "stdout")
+ defer func() { _ = stdoutReader.Close() }()
+ errCh <- s.readLogsFromReader(ctx, stdoutReader, logsChan, "")
}()
- // Read stderr
go func() {
- done <- s.readLogsFromReader(ctx, stderrReader, logsChan, "stderr")
+ defer func() { _ = stderrReader.Close() }()
+ errCh <- s.readLogsFromReader(ctx, stderrReader, logsChan, "[STDERR] ")
}()
- // Wait for context cancellation or error
- select {
- case <-ctx.Done():
- return ctx.Err()
- case err := <-done:
- if err != nil && !errors.Is(err, io.EOF) {
- return err
- }
- // Wait for the other goroutine or context cancellation
- select {
- case <-ctx.Done():
- return ctx.Err()
- case <-done:
- return nil
+ var firstErr error
+ for range 3 {
+ err := <-errCh
+ switch {
+ case err == nil:
+ case errors.Is(err, io.EOF):
+ case errors.Is(err, context.Canceled), errors.Is(err, context.DeadlineExceeded):
+ default:
+ if firstErr == nil {
+ firstErr = err
+ }
}
}
+
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+
+ return firstErr
}
// readLogsFromReader reads logs line by line from a reader
-func (s *ContainerService) readLogsFromReader(ctx context.Context, reader io.Reader, logsChan chan<- string, source string) error {
- scanner := bufio.NewScanner(reader)
+func (s *ContainerService) readLogsFromReader(ctx context.Context, reader io.Reader, logsChan chan<- string, prefix string) error {
+ bufferedReader := bufio.NewReader(reader)
- for scanner.Scan() {
- select {
- case <-ctx.Done():
- return ctx.Err()
- default:
- line := scanner.Text()
- if line != "" {
- // Add source prefix for stderr logs
- if source == "stderr" {
- line = "[STDERR] " + line
+ for {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+
+ line, err := bufferedReader.ReadString('\n')
+ if len(line) > 0 {
+ trimmed := strings.TrimRight(line, "\r\n")
+ if trimmed != "" {
+ if prefix != "" {
+ trimmed = prefix + trimmed
}
select {
- case logsChan <- line:
+ case logsChan <- trimmed:
case <-ctx.Done():
return ctx.Err()
}
}
}
- }
- return scanner.Err()
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ return nil
+ }
+ return err
+ }
+ }
}
-func (s *ContainerService) readAllLogs(logs io.ReadCloser, logsChan chan<- string) error {
+func (s *ContainerService) readAllLogs(ctx context.Context, logs io.ReadCloser, logsChan chan<- string) error {
stdoutBuf := &strings.Builder{}
stderrBuf := &strings.Builder{}
+ stdCopyDone := make(chan struct{})
+ defer close(stdCopyDone)
+
+ go func() {
+ select {
+ case <-ctx.Done():
+ _ = logs.Close()
+ case <-stdCopyDone:
+ }
+ }()
_, err := stdcopy.StdCopy(stdoutBuf, stderrBuf, logs)
if err != nil && !errors.Is(err, io.EOF) {
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return ctxErr
+ }
return fmt.Errorf("failed to demultiplex logs: %w", err)
}
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return ctxErr
+ }
// Send stdout lines
if stdoutBuf.Len() > 0 {
- lines := strings.SplitSeq(strings.TrimRight(stdoutBuf.String(), "\n"), "\n")
- for line := range lines {
- if line != "" {
- logsChan <- line
- }
+ if err := s.readLogsFromReader(ctx, strings.NewReader(stdoutBuf.String()), logsChan, ""); err != nil {
+ return err
}
}
// Send stderr lines with prefix
if stderrBuf.Len() > 0 {
- lines := strings.SplitSeq(strings.TrimRight(stderrBuf.String(), "\n"), "\n")
- for line := range lines {
- if line != "" {
- logsChan <- "[STDERR] " + line
- }
+ if err := s.readLogsFromReader(ctx, strings.NewReader(stderrBuf.String()), logsChan, "[STDERR] "); err != nil {
+ return err
}
}
diff --git a/backend/internal/services/container_service_test.go b/backend/internal/services/container_service_test.go
index e4128f795..4ec73acad 100644
--- a/backend/internal/services/container_service_test.go
+++ b/backend/internal/services/container_service_test.go
@@ -1,8 +1,14 @@
package services
import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "io"
"net/netip"
+ "strings"
"testing"
+ "time"
"github.com/getarcaneapp/arcane/backend/pkg/pagination"
containertypes "github.com/getarcaneapp/arcane/types/container"
@@ -95,6 +101,150 @@ func TestBuildCleanNetworkingConfigInternalPreservesEndpointSettings(t *testing.
require.Nil(t, out.EndpointsConfig["bridge"].IPAMConfig)
}
+func TestStreamContainerLogs_NonTTYFollowDemultiplexesStdoutAndStderr(t *testing.T) {
+ var stream bytes.Buffer
+ writeDockerLogFrameInternal(t, &stream, 1, "stdout line\n")
+ writeDockerLogFrameInternal(t, &stream, 2, "stderr line\n")
+
+ service := &ContainerService{}
+ logsChan := make(chan string, 4)
+
+ err := service.streamContainerLogsInternal(t.Context(), io.NopCloser(bytes.NewReader(stream.Bytes())), logsChan, true, false)
+ require.NoError(t, err)
+
+ require.ElementsMatch(t, []string{"stdout line", "[STDERR] stderr line"}, drainLogLinesInternal(logsChan))
+}
+
+func TestStreamContainerLogs_TTYFollowStreamsRawOutput(t *testing.T) {
+ service := &ContainerService{}
+ logsChan := make(chan string, 4)
+
+ err := service.streamContainerLogsInternal(t.Context(), io.NopCloser(strings.NewReader("first line\nsecond line")), logsChan, true, true)
+ require.NoError(t, err)
+
+ require.Equal(t, []string{"first line", "second line"}, drainLogLinesInternal(logsChan))
+}
+
+func TestStreamContainerLogs_NonTTYSnapshotDemultiplexesStdoutAndStderr(t *testing.T) {
+ var stream bytes.Buffer
+ writeDockerLogFrameInternal(t, &stream, 1, "stdout snapshot\n")
+ writeDockerLogFrameInternal(t, &stream, 2, "stderr snapshot\n")
+
+ service := &ContainerService{}
+ logsChan := make(chan string, 4)
+
+ err := service.streamContainerLogsInternal(t.Context(), io.NopCloser(bytes.NewReader(stream.Bytes())), logsChan, false, false)
+ require.NoError(t, err)
+
+ require.Equal(t, []string{"stdout snapshot", "[STDERR] stderr snapshot"}, drainLogLinesInternal(logsChan))
+}
+
+func TestStreamContainerLogs_TTYSnapshotStreamsRawOutput(t *testing.T) {
+ service := &ContainerService{}
+ logsChan := make(chan string, 4)
+
+ err := service.streamContainerLogsInternal(t.Context(), io.NopCloser(strings.NewReader("snapshot line\ntrailing line")), logsChan, false, true)
+ require.NoError(t, err)
+
+ require.Equal(t, []string{"snapshot line", "trailing line"}, drainLogLinesInternal(logsChan))
+}
+
+func TestReadLogsFromReader_HandlesLongLinesAndPartialEOF(t *testing.T) {
+ longLine := strings.Repeat("a", 70*1024)
+ service := &ContainerService{}
+ logsChan := make(chan string, 4)
+
+ err := service.readLogsFromReader(t.Context(), strings.NewReader(longLine+"\npartial tail"), logsChan, "")
+ require.NoError(t, err)
+
+ require.Equal(t, []string{longLine, "partial tail"}, drainLogLinesInternal(logsChan))
+}
+
+func TestStreamContainerLogs_TTYPythonLikeFollowDoesNotReturnEmptyLogs(t *testing.T) {
+ service := &ContainerService{}
+ logsChan := make(chan string, 4)
+
+ err := service.streamContainerLogsInternal(
+ t.Context(),
+ io.NopCloser(strings.NewReader("2026-03-22 10:15:00 - INFO - Starting miner\n2026-03-22 10:15:01 - INFO - Connected")),
+ logsChan,
+ true,
+ true,
+ )
+ require.NoError(t, err)
+
+ lines := drainLogLinesInternal(logsChan)
+ require.NotEmpty(t, lines)
+ require.Equal(t, []string{
+ "2026-03-22 10:15:00 - INFO - Starting miner",
+ "2026-03-22 10:15:01 - INFO - Connected",
+ }, lines)
+}
+
+func TestStreamMultiplexedLogs_ContextCancelDoesNotDeadlock(t *testing.T) {
+ var stream bytes.Buffer
+ writeDockerLogFrameInternal(t, &stream, 1, "line 1\n")
+ writeDockerLogFrameInternal(t, &stream, 1, "line 2\n")
+ writeDockerLogFrameInternal(t, &stream, 1, "line 3\n")
+
+ service := &ContainerService{}
+ logsChan := make(chan string, 1)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ done := make(chan error, 1)
+ go func() {
+ done <- service.streamMultiplexedLogs(ctx, io.NopCloser(bytes.NewReader(stream.Bytes())), logsChan)
+ }()
+
+ require.Eventually(t, func() bool {
+ return len(logsChan) == 1
+ }, time.Second, 10*time.Millisecond)
+
+ cancel()
+
+ select {
+ case err := <-done:
+ require.ErrorIs(t, err, context.Canceled)
+ case <-time.After(time.Second):
+ t.Fatal("streamMultiplexedLogs did not exit after cancellation")
+ }
+}
+
+func TestReadAllLogs_ContextCancelClosesReader(t *testing.T) {
+ service := &ContainerService{}
+ logsChan := make(chan string, 1)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ reader := &blockingReadCloser{readStarted: make(chan struct{}), closeCalled: make(chan struct{})}
+ done := make(chan error, 1)
+ go func() {
+ done <- service.readAllLogs(ctx, reader, logsChan)
+ }()
+
+ select {
+ case <-reader.readStarted:
+ case <-time.After(time.Second):
+ t.Fatal("readAllLogs did not start reading")
+ }
+
+ cancel()
+
+ select {
+ case err := <-done:
+ require.ErrorIs(t, err, context.Canceled)
+ case <-time.After(time.Second):
+ t.Fatal("readAllLogs did not exit after cancellation")
+ }
+
+ select {
+ case <-reader.closeCalled:
+ case <-time.After(time.Second):
+ t.Fatal("readAllLogs did not close the reader on cancellation")
+ }
+}
+
func newGroupedContainerSummary(name string, project string) containertypes.Summary {
labels := map[string]string{}
if project != "" {
@@ -108,3 +258,52 @@ func newGroupedContainerSummary(name string, project string) containertypes.Summ
State: "running",
}
}
+
+func drainLogLinesInternal(logsChan chan string) []string {
+ close(logsChan)
+
+ lines := make([]string, 0, len(logsChan))
+ for line := range logsChan {
+ lines = append(lines, line)
+ }
+
+ return lines
+}
+
+func writeDockerLogFrameInternal(t *testing.T, buffer *bytes.Buffer, streamType byte, payload string) {
+ t.Helper()
+
+ header := make([]byte, 8)
+ header[0] = streamType
+ binary.BigEndian.PutUint32(header[4:], uint32(len(payload)))
+
+ _, err := buffer.Write(header)
+ require.NoError(t, err)
+ _, err = buffer.WriteString(payload)
+ require.NoError(t, err)
+}
+
+type blockingReadCloser struct {
+ readStarted chan struct{}
+ closeCalled chan struct{}
+}
+
+func (r *blockingReadCloser) Read(_ []byte) (int, error) {
+ select {
+ case <-r.readStarted:
+ default:
+ close(r.readStarted)
+ }
+
+ <-r.closeCalled
+ return 0, io.EOF
+}
+
+func (r *blockingReadCloser) Close() error {
+ select {
+ case <-r.closeCalled:
+ default:
+ close(r.closeCalled)
+ }
+ return nil
+}
diff --git a/backend/internal/services/project_service_test.go b/backend/internal/services/project_service_test.go
index a98343970..fa86cd36e 100644
--- a/backend/internal/services/project_service_test.go
+++ b/backend/internal/services/project_service_test.go
@@ -1695,7 +1695,7 @@ func TestProjectService_UpdateProject_WritesThroughSymlinkedProjectPath(t *testi
project := &models.Project{
BaseModel: models.BaseModel{ID: "proj-symlink-update"},
Name: "demo",
- DirName: ptr("demo"),
+ DirName: new("demo"),
Path: linkPath,
Status: models.ProjectStatusStopped,
}
@@ -1704,7 +1704,7 @@ func TestProjectService_UpdateProject_WritesThroughSymlinkedProjectPath(t *testi
updatedCompose := "services:\n app:\n image: nginx:1.27-alpine\n"
updatedEnv := "FOO=updated\n"
- updated, err := svc.UpdateProject(ctx, project.ID, nil, ptr(updatedCompose), ptr(updatedEnv), models.User{
+ updated, err := svc.UpdateProject(ctx, project.ID, nil, new(updatedCompose), new(updatedEnv), models.User{
BaseModel: models.BaseModel{ID: "u1"},
Username: "tester",
})
diff --git a/backend/pkg/fswatch/watcher_test.go b/backend/pkg/fswatch/watcher_test.go
index dd2f8bcfb..35ac501da 100644
--- a/backend/pkg/fswatch/watcher_test.go
+++ b/backend/pkg/fswatch/watcher_test.go
@@ -20,8 +20,7 @@ func TestWatcher_StartWatchesExistingSymlinkDirectoriesWhenEnabled(t *testing.T)
require.NoError(t, os.Symlink(targetPath, linkPath))
changeCh := make(chan struct{}, 1)
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
+ ctx := t.Context()
watcher, err := NewWatcher(root, WatcherOptions{
Debounce: 25 * time.Millisecond,
@@ -59,8 +58,7 @@ func TestWatcher_StartSkipsExistingSymlinkDirectoriesWhenDisabled(t *testing.T)
require.NoError(t, os.Symlink(targetPath, linkPath))
changeCh := make(chan struct{}, 1)
- ctx, cancel := context.WithCancel(context.Background())
- defer cancel()
+ ctx := t.Context()
watcher, err := NewWatcher(root, WatcherOptions{
Debounce: 25 * time.Millisecond,