Skip to content

Modern Docker Management, Designed for Everyone

perf(activity): batch message appends and drop the per-line re-SELECT (#3611)

Kyle Mendella9d3df1fe2d9
diff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.go
index e017f0d9e..19a7bc910 100644
--- a/backend/internal/activity/service.go
+++ b/backend/internal/activity/service.go
@@ -3,6 +3,7 @@ package activity
import (
"context"
stderrors "errors"
+ "hash/fnv"
"log/slog"
"maps"
"slices"
@@ -55,8 +56,30 @@ type ActivityService struct {
limiter *activitySlotLimiter
slotMu sync.Mutex
slotReleases map[string]func()
+
+ // terminalPublished records when each activity's terminal snapshot was
+ // published, so later non-terminal snapshots from slower goroutines can
+ // be dropped (see admitActivityPublishInternal). Entries are pruned on
+ // terminal publishes once older than terminalPublishRetention.
+ terminalPublishedMu sync.Mutex
+ terminalPublished map[string]time.Time
+
+ // publishLocks serializes the commit→publish window of the non-terminal
+ // snapshot writers (AppendMessages, UpdateActivity) per activity, so their
+ // events reach subscribers in commit order — without it, a batch that
+ // commits first but publishes second would revert progress/step/status
+ // fields a concurrent update already streamed. Terminal snapshots are
+ // ordered separately by admitActivityPublishInternal. Striped by activity
+ // ID hash so no per-activity lifecycle management is needed.
+ publishLocks [64]sync.Mutex
}
+// terminalPublishRetention bounds how long a terminal publish suppresses
+// stale non-terminal snapshots for the same activity. Stale publishers are
+// goroutines already past their commit, so they publish within moments; the
+// retention only has to outlive that gap while keeping the latch map small.
+const terminalPublishRetention = 10 * time.Minute
+
// ErrActivityNotCancelable indicates the activity has already reached a terminal
// state and can no longer be cancelled.
const ErrActivityNotCancelable = errors.Sentinel("activity is not cancelable")
@@ -187,11 +210,12 @@ type AppendActivityMessageRequest = activitylib.AppendMessageRequest
func NewActivityService(db *database.DB, settingsService *settings.SettingsService) *ActivityService {
return &ActivityService{
- db: db,
- subscribers: map[int]*activitySubscriber{},
- running: map[string]context.CancelCauseFunc{},
- limiter: newActivitySlotLimiterInternal(settingsService),
- slotReleases: map[string]func(){},
+ db: db,
+ subscribers: map[int]*activitySubscriber{},
+ running: map[string]context.CancelCauseFunc{},
+ limiter: newActivitySlotLimiterInternal(settingsService),
+ slotReleases: map[string]func(){},
+ terminalPublished: map[string]time.Time{},
}
}
@@ -452,6 +476,13 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,
updates["metadata"] = cloneJSONInternal(req.Metadata)
}
+ // Hold the per-activity publish lock across commit and publish so this
+ // snapshot cannot be overtaken on the stream by a concurrent append batch
+ // that committed earlier but would publish later.
+ lock := s.publishLockInternal(activityID)
+ lock.Lock()
+ defer lock.Unlock()
+
var model models.Activity
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
result := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates)
@@ -475,6 +506,23 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,
}
func (s *ActivityService) AppendMessage(ctx context.Context, activityID string, req AppendActivityMessageRequest) (*activitytypes.Message, error) {
+ messages, err := s.AppendMessages(ctx, activityID, []AppendActivityMessageRequest{req})
+ if err != nil || len(messages) == 0 {
+ return nil, err
+ }
+ return &messages[0], nil
+}
+
+// AppendMessages persists a batch of output lines in one transaction — a
+// single multi-row message INSERT plus one coalesced Activity update —
+// instead of an INSERT+UPDATE+re-SELECT transaction per line, which turned
+// an image pull into hundreds of fsync'd SQLite transactions. The activity
+// publish DTO is re-SELECTed inside the transaction after the update (once
+// per batch, not per line) so it reflects lifecycle fields a concurrent
+// CompleteActivity/UpdateActivity committed first; a terminal write that
+// commits after this transaction but publishes before this snapshot is
+// handled by admitActivityPublishInternal dropping the stale event.
+func (s *ActivityService) AppendMessages(ctx context.Context, activityID string, reqs []AppendActivityMessageRequest) ([]activitytypes.Message, error) {
if err := s.checkInitInternal(); err != nil {
return nil, err
}
@@ -483,66 +531,121 @@ func (s *ActivityService) AppendMessage(ctx context.Context, activityID string,
return nil, errors.New("activity id is required")
}
- messageText := strings.TrimSpace(req.Message)
- if messageText == "" {
- return nil, nil
- }
- if len(messageText) > 8192 {
- messageText = messageText[:8192]
- }
-
- level := req.Level
- if level == "" {
- level = models.ActivityMessageLevelInfo
- }
-
now := time.Now()
- message := &models.ActivityMessage{
- ActivityID: activityID,
- Level: level,
- Message: messageText,
- Payload: cloneJSONInternal(req.Payload),
- BaseModel: models.BaseModel{
- CreatedAt: now,
- },
- }
-
- var updated models.Activity
- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
- if err := tx.Create(message).Error; err != nil {
- return errors.WrapIf(err, "failed to append activity message")
+ messages := make([]*models.ActivityMessage, 0, len(reqs))
+ latestMessage := ""
+ var latestProgress *int
+ latestStep := ""
+ for _, req := range reqs {
+ messageText := strings.TrimSpace(req.Message)
+ if messageText == "" {
+ continue
+ }
+ if len(messageText) > 8192 {
+ messageText = messageText[:8192]
}
- updates := map[string]any{
- "latest_message": messageText,
- "updated_at": now,
+ level := req.Level
+ if level == "" {
+ level = models.ActivityMessageLevelInfo
}
+
+ messages = append(messages, &models.ActivityMessage{
+ ActivityID: activityID,
+ Level: level,
+ Message: messageText,
+ Payload: cloneJSONInternal(req.Payload),
+ BaseModel: models.BaseModel{
+ // Spread inside the shared timestamp so the created_at sort
+ // used for retrieval keeps the original line order.
+ CreatedAt: now.Add(time.Duration(len(messages)) * time.Microsecond),
+ },
+ })
+
+ latestMessage = messageText
if req.Progress != nil {
- updates["progress"] = *clampProgressPtrInternal(req.Progress)
+ latestProgress = req.Progress
}
- if strings.TrimSpace(req.Step) != "" {
- updates["step"] = strings.TrimSpace(req.Step)
+ if step := strings.TrimSpace(req.Step); step != "" {
+ latestStep = step
}
+ }
+ if len(messages) == 0 {
+ return nil, nil
+ }
- result := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates)
- if result.Error != nil {
- return errors.WrapIf(result.Error, "failed to update activity latest message")
- }
- if result.RowsAffected == 0 {
- return errors.New("activity not found")
+ updates := map[string]any{
+ "latest_message": latestMessage,
+ "updated_at": now,
+ }
+ if latestProgress != nil {
+ updates["progress"] = *clampProgressPtrInternal(latestProgress)
+ }
+ if latestStep != "" {
+ updates["step"] = latestStep
+ }
+
+ // Hold the per-activity publish lock across commit and publish so this
+ // batch's snapshot cannot be overtaken on the stream by a concurrent
+ // UpdateActivity that committed later — subscribers must see the two
+ // snapshots in commit order.
+ lock := s.publishLockInternal(activityID)
+ lock.Lock()
+ defer lock.Unlock()
+
+ // Bulk appends contend with terminal-status writes for the SQLite write
+ // lock, and a batch lost to transient SQLITE_BUSY silently drops up to 32
+ // output lines — the Writer drain has no error path to replay them — so
+ // retry like CompleteActivity does.
+ var current models.Activity
+ const appendWriteAttempts = 3
+ var writeErr error
+ for attempt := 1; attempt <= appendWriteAttempts; attempt++ {
+ writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.First(&current, "id = ?", activityID).Error; err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.New("activity not found")
+ }
+ return errors.WrapIf(err, "failed to load activity")
+ }
+
+ if err := tx.Create(&messages).Error; err != nil {
+ return errors.WrapIf(err, "failed to append activity message")
+ }
+
+ result := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates)
+ if result.Error != nil {
+ return errors.WrapIf(result.Error, "failed to update activity latest message")
+ }
+ if result.RowsAffected == 0 {
+ return errors.New("activity not found")
+ }
+ if err := tx.First(&current, "id = ?", activityID).Error; err != nil {
+ return errors.WrapIf(err, "failed to load updated activity")
+ }
+ return nil
+ })
+ if writeErr == nil {
+ break
}
- if err := tx.First(&updated, "id = ?", activityID).Error; err != nil {
- return errors.WrapIf(err, "failed to load updated activity")
+ if attempt == appendWriteAttempts || !dbutil.IsRetryableWriteError(writeErr) {
+ return nil, writeErr
}
- return nil
- }); err != nil {
- return nil, err
+ slog.WarnContext(ctx, "retrying activity message batch write", "activityId", activityID, "attempt", attempt, "error", writeErr)
+ time.Sleep(250 * time.Millisecond * time.Duration(attempt))
+ }
+ if writeErr != nil {
+ return nil, writeErr
}
- dto := activityMessageToDTOInternal(message)
- s.publishMessageInternal(updated.EnvironmentID, dto)
- s.publishActivityInternal(activityToDTOInternal(&updated))
- return &dto, nil
+ out := make([]activitytypes.Message, 0, len(messages))
+ for _, message := range messages {
+ dto := activityMessageToDTOInternal(message)
+ s.publishMessageInternal(current.EnvironmentID, dto)
+ out = append(out, dto)
+ }
+ s.publishActivityInternal(activityToDTOInternal(&current))
+ return out, nil
}
func (s *ActivityService) CompleteActivity(ctx context.Context, activityID string, status models.ActivityStatus, finalMessage string, errMessage *string, finalStep ...string) (*activitytypes.Activity, error) {
@@ -621,16 +724,37 @@ func (s *ActivityService) CompleteActivity(ctx context.Context, activityID strin
}); err != nil {
slog.DebugContext(ctx, "failed to append final activity message", "activityId", activityID, "error", err)
}
- if err := s.db.WithContext(activityCtx).First(&model, "id = ?", activityID).Error; err != nil {
- slog.DebugContext(ctx, "failed to reload activity after appending message", "activityId", activityID, "error", err)
- }
}
- dto := activityToDTOInternal(&model)
- s.publishActivityInternal(dto)
+ dto := s.publishTerminalSnapshotInternal(ctx, &model)
return &dto, nil
}
+// publishTerminalSnapshotInternal publishes an activity's terminal event from
+// a snapshot re-read under the per-activity publish lock. Terminal writers
+// cannot hold that lock across their own transactions (CompleteActivity
+// appends its final message through the locked AppendMessages path), so a
+// snapshot captured in-transaction can be overtaken by an append or update
+// that commits later but publishes first; re-reading under the lock
+// guarantees the terminal event carries every field already streamed. The
+// passed model is published as-is if the re-read fails, and is updated in
+// place otherwise so callers return the published state.
+func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, model *models.Activity) activitytypes.Activity {
+ lock := s.publishLockInternal(model.ID)
+ lock.Lock()
+ defer lock.Unlock()
+
+ var fresh models.Activity
+ if err := s.db.WithContext(ctx).First(&fresh, "id = ?", model.ID).Error; err != nil {
+ slog.DebugContext(ctx, "failed to reload activity for terminal publish", "activityId", model.ID, "error", err)
+ } else {
+ *model = fresh
+ }
+ dto := activityToDTOInternal(model)
+ s.publishActivityInternal(dto)
+ return dto
+}
+
// CancelActivity requests cancellation of a running or queued activity. When the
// activity's work is running in this process it interrupts it (the work finalizes
// its own terminal status); otherwise it marks the activity cancelled directly,
@@ -707,8 +831,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, act
return nil, err
}
- dto := activityToDTOInternal(&finalized)
- s.publishActivityInternal(dto)
+ dto := s.publishTerminalSnapshotInternal(writeCtx, &finalized)
return &dto, nil
}
@@ -818,7 +941,7 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e
}
s.releaseSlotInternal(activityID)
- s.publishActivityInternal(activityToDTOInternal(&finalized))
+ s.publishTerminalSnapshotInternal(ctx, &finalized)
swept++
}
@@ -1161,6 +1284,9 @@ func (s *ActivityService) Subscribe(environmentID string) (<-chan activitytypes.
}
func (s *ActivityService) publishActivityInternal(activity activitytypes.Activity) {
+ if !s.admitActivityPublishInternal(activity) {
+ return
+ }
s.publishInternal(activity.EnvironmentID, activitytypes.StreamEvent{
Type: "activity",
ActivityID: activity.ID,
@@ -1169,6 +1295,50 @@ func (s *ActivityService) publishActivityInternal(activity activitytypes.Activit
})
}
+func (s *ActivityService) publishLockInternal(activityID string) *sync.Mutex {
+ h := fnv.New32a()
+ _, _ = h.Write([]byte(activityID))
+ return &s.publishLocks[h.Sum32()%uint32(len(s.publishLocks))]
+}
+
+func isTerminalActivityStatusInternal(status activitytypes.Status) bool {
+ return status == activitytypes.StatusSuccess ||
+ status == activitytypes.StatusFailed ||
+ status == activitytypes.StatusCancelled
+}
+
+// admitActivityPublishInternal orders activity events across the commit →
+// publish gap. Writes publish their snapshot after committing, so a goroutine
+// that committed before a terminal write can publish after it; since a
+// terminal activity emits no further events to correct the stream (and
+// subscriber coalescing would even replace an undelivered terminal event in
+// place), such a stale non-terminal snapshot must be dropped, not delivered.
+// Activities never leave a terminal status, so once a terminal snapshot is
+// published, any later non-terminal snapshot for that ID is stale by
+// construction.
+func (s *ActivityService) admitActivityPublishInternal(activity activitytypes.Activity) bool {
+ if s == nil {
+ return true
+ }
+ s.terminalPublishedMu.Lock()
+ defer s.terminalPublishedMu.Unlock()
+ if !isTerminalActivityStatusInternal(activity.Status) {
+ _, sealed := s.terminalPublished[activity.ID]
+ return !sealed
+ }
+ now := time.Now()
+ for id, publishedAt := range s.terminalPublished {
+ if now.Sub(publishedAt) > terminalPublishRetention {
+ delete(s.terminalPublished, id)
+ }
+ }
+ if s.terminalPublished == nil {
+ s.terminalPublished = map[string]time.Time{}
+ }
+ s.terminalPublished[activity.ID] = now
+ return true
+}
+
func (s *ActivityService) publishMessageInternal(environmentID string, message activitytypes.Message) {
s.publishInternal(environmentID, activitytypes.StreamEvent{
Type: "message",
diff --git a/backend/internal/activity/service_test.go b/backend/internal/activity/service_test.go
index 823f02cb0..1ba8cf964 100644
--- a/backend/internal/activity/service_test.go
+++ b/backend/internal/activity/service_test.go
@@ -732,3 +732,90 @@ func receiveActivityEventInternal(t *testing.T, events <-chan activitytypes.Stre
return activitytypes.StreamEvent{}
}
}
+
+// TestActivityServiceAppendMessagesBatchInternal verifies a batch lands all
+// messages in order and coalesces the activity update to the batch's last
+// message, last progress, and last step.
+func TestActivityServiceAppendMessagesBatchInternal(t *testing.T) {
+ ctx := context.Background()
+ db := setupActivityServiceTestDBInternal(t)
+ service := NewActivityService(db, nil)
+
+ created, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImagePull,
+ Step: "queued",
+ LatestMessage: "Pull queued",
+ })
+ require.NoError(t, err)
+
+ progress := 60
+ messages, err := service.AppendMessages(ctx, created.ID, []AppendActivityMessageRequest{
+ {Message: "layer 1/3", Step: "download"},
+ {Message: " "}, // blank lines are dropped, not persisted
+ {Message: "layer 2/3", Progress: &progress},
+ {Message: "layer 3/3"},
+ })
+ require.NoError(t, err)
+ require.Len(t, messages, 3)
+ require.Equal(t, "layer 1/3", messages[0].Message)
+ require.Equal(t, "layer 3/3", messages[2].Message)
+
+ detail, err := service.GetActivityDetail(ctx, "0", created.ID, 10)
+ require.NoError(t, err)
+ require.Len(t, detail.Messages, 3)
+ require.Equal(t, "layer 1/3", detail.Messages[0].Message)
+ require.Equal(t, "layer 2/3", detail.Messages[1].Message)
+ require.Equal(t, "layer 3/3", detail.Messages[2].Message)
+
+ // Coalesced activity update: last message wins, last non-nil progress
+ // and last non-empty step stick.
+ require.Equal(t, "layer 3/3", detail.Activity.LatestMessage)
+ require.NotNil(t, detail.Activity.Progress)
+ require.Equal(t, 60, *detail.Activity.Progress)
+ require.Equal(t, "download", detail.Activity.Step)
+
+ // Unknown activity IDs are still rejected.
+ _, err = service.AppendMessages(ctx, "missing", []AppendActivityMessageRequest{{Message: "x"}})
+ require.ErrorContains(t, err, "activity not found")
+}
+
+// TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal verifies
+// that a non-terminal activity snapshot published after the activity's
+// terminal event — a goroutine that committed before CompleteActivity but
+// publishes after it — is dropped instead of reverting subscribers to a
+// running state no later event would correct.
+func TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal(t *testing.T) {
+ ctx := context.Background()
+ db := setupActivityServiceTestDBInternal(t)
+ service := NewActivityService(db, nil)
+
+ created, err := service.StartActivity(ctx, StartActivityRequest{
+ EnvironmentID: "0",
+ Type: models.ActivityTypeImagePull,
+ LatestMessage: "Pull queued",
+ })
+ require.NoError(t, err)
+ stale := *created // snapshot taken while the activity is still active
+
+ events, _, unsubscribe := service.Subscribe("0")
+ defer unsubscribe()
+
+ completed, err := service.CompleteActivity(ctx, created.ID, models.ActivityStatusSuccess, "", nil)
+ require.NoError(t, err)
+ require.Equal(t, activitytypes.StatusSuccess, completed.Status)
+
+ event := receiveActivityEventInternal(t, events)
+ require.Equal(t, "activity", event.Type)
+ require.Equal(t, activitytypes.StatusSuccess, event.Activity.Status)
+
+ require.False(t, service.admitActivityPublishInternal(stale))
+ require.True(t, service.admitActivityPublishInternal(*completed))
+
+ service.publishActivityInternal(stale)
+ select {
+ case event := <-events:
+ t.Fatalf("stale non-terminal snapshot reached subscriber: %+v", event)
+ case <-time.After(100 * time.Millisecond):
+ }
+}
diff --git a/backend/pkg/libarcane/activity/requests.go b/backend/pkg/libarcane/activity/requests.go
index 69a8b05cb..1c0d6d60f 100644
--- a/backend/pkg/libarcane/activity/requests.go
+++ b/backend/pkg/libarcane/activity/requests.go
@@ -14,6 +14,10 @@ type Service interface {
type MessageAppender interface {
AppendMessage(ctx context.Context, activityID string, req AppendMessageRequest) (*activitytypes.Message, error)
+ // AppendMessages persists a batch of messages in one transaction with a
+ // single coalesced activity update; the Writer drains its queue through
+ // this so bulk docker output does not fsync per line.
+ AppendMessages(ctx context.Context, activityID string, reqs []AppendMessageRequest) ([]activitytypes.Message, error)
}
// Tracker is an optional interface a Service may implement to make activities
diff --git a/backend/pkg/libarcane/activity/writer.go b/backend/pkg/libarcane/activity/writer.go
index 0b7602502..794f7c51a 100644
--- a/backend/pkg/libarcane/activity/writer.go
+++ b/backend/pkg/libarcane/activity/writer.go
@@ -6,6 +6,7 @@ import (
"encoding/json/v2"
"fmt"
"io"
+ "log/slog"
"net/http"
"strings"
"sync"
@@ -15,7 +16,13 @@ import (
)
type Writer struct {
- ctx context.Context
+ ctx context.Context
+ // persistCtx is ctx detached from cancellation. Cancellation usually
+ // races the last — most diagnostic — output lines (the error frames
+ // explaining it), so batches persist on persistCtx while ctx only bounds
+ // the drain goroutine's intake, mirroring CompleteActivity's detached
+ // terminal write.
+ persistCtx context.Context
activityService MessageAppender
activityID string
writer io.Writer
@@ -26,7 +33,12 @@ type Writer struct {
buffer []byte
}
-const writerAppendQueueSize = 128
+const (
+ writerAppendQueueSize = 128
+ // writerAppendBatchSize caps how many queued lines are drained into one
+ // AppendMessages transaction.
+ writerAppendBatchSize = 32
+)
type writerAppendMessage struct {
level models.ActivityMessageLevel
@@ -50,8 +62,13 @@ func NewWriter(ctx context.Context, activityService MessageAppender, activityID
if existing, ok := writer.(*Writer); ok {
return existing
}
+ var persistCtx context.Context
+ if ctx != nil {
+ persistCtx = context.WithoutCancel(ctx)
+ }
out := &Writer{
ctx: ctx,
+ persistCtx: persistCtx,
activityService: activityService,
activityID: strings.TrimSpace(activityID),
writer: writer,
@@ -170,38 +187,82 @@ func (w *Writer) drainMessagesInternal(ctx context.Context) {
for {
select {
case item := <-w.queueCh:
- if item.flush != nil {
- close(item.flush)
- continue
- }
- if item.message != nil {
- w.appendMessageInternal(ctx, *item.message)
- }
+ w.drainQueueBatchInternal(item)
case <-doneInternal(ctx):
+ // Cancellation stops intake, but the lines already accepted into
+ // the queue — typically the ones explaining the cancellation —
+ // still get persisted before the goroutine exits.
+ w.drainRemainingInternal()
return
}
}
}
-func doneInternal(ctx context.Context) <-chan struct{} {
- if ctx == nil {
- return nil
+func (w *Writer) drainRemainingInternal() {
+ for {
+ select {
+ case item := <-w.queueCh:
+ w.drainQueueBatchInternal(item)
+ default:
+ return
+ }
}
- return ctx.Done()
}
-func (w *Writer) appendMessageInternal(ctx context.Context, message writerAppendMessage) {
- if ctx == nil {
- return
+// drainQueueBatchInternal drains whatever is already queued behind item (up
+// to writerAppendBatchSize lines) into a single AppendMessages call, so bulk
+// docker output costs one transaction per batch instead of one per line.
+// Flush signals encountered while draining close only after the write, since
+// the messages queued before them are only then persisted.
+func (w *Writer) drainQueueBatchInternal(item writerQueueItem) {
+ var batch []AppendMessageRequest
+ var flushes []chan struct{}
+
+ collect := func(queued writerQueueItem) {
+ if queued.flush != nil {
+ flushes = append(flushes, queued.flush)
+ return
+ }
+ if queued.message == nil {
+ return
+ }
+ batch = append(batch, AppendMessageRequest{
+ Level: queued.message.level,
+ Message: queued.message.message,
+ Payload: queued.message.payload,
+ Step: queued.message.step,
+ })
}
- if _, err := w.activityService.AppendMessage(ctx, w.activityID, AppendMessageRequest{
- Level: message.level,
- Message: message.message,
- Payload: message.payload,
- Step: message.step,
- }); err != nil {
- return
+
+ collect(item)
+drain:
+ for len(batch) < writerAppendBatchSize {
+ select {
+ case queued := <-w.queueCh:
+ collect(queued)
+ default:
+ break drain
+ }
+ }
+
+ if len(batch) > 0 && w.persistCtx != nil {
+ if _, err := w.activityService.AppendMessages(w.persistCtx, w.activityID, batch); err != nil {
+ slog.WarnContext(w.persistCtx, "failed to persist activity output batch", "activityId", w.activityID, "lines", len(batch), "error", err)
+ }
}
+ // Flushes release even when the append failed: Flush carries no error
+ // path, and holding them would stall activity completion on the very
+ // cancellation that typically caused the failure.
+ for _, flushDone := range flushes {
+ close(flushDone)
+ }
+}
+
+func doneInternal(ctx context.Context) <-chan struct{} {
+ if ctx == nil {
+ return nil
+ }
+ return ctx.Done()
}
func valueToStringInternal(value any) string {
diff --git a/backend/pkg/libarcane/activity/writer_test.go b/backend/pkg/libarcane/activity/writer_test.go
index 9e5fe9bf5..56bdbfdd0 100644
--- a/backend/pkg/libarcane/activity/writer_test.go
+++ b/backend/pkg/libarcane/activity/writer_test.go
@@ -3,7 +3,9 @@ package activity
import (
"context"
"errors"
+ "fmt"
"io"
+ "sync"
"testing"
"time"
@@ -13,14 +15,32 @@ import (
)
type recordingAppender struct {
- messages []AppendMessageRequest
+ mu sync.Mutex
+ messages []AppendMessageRequest
+ batchSizes []int
}
func (a *recordingAppender) AppendMessage(_ context.Context, _ string, req AppendMessageRequest) (*activitytypes.Message, error) {
+ a.mu.Lock()
+ defer a.mu.Unlock()
a.messages = append(a.messages, req)
return &activitytypes.Message{}, nil
}
+func (a *recordingAppender) AppendMessages(_ context.Context, _ string, reqs []AppendMessageRequest) ([]activitytypes.Message, error) {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ a.messages = append(a.messages, reqs...)
+ a.batchSizes = append(a.batchSizes, len(reqs))
+ return make([]activitytypes.Message, len(reqs)), nil
+}
+
+func (a *recordingAppender) recorded() ([]AppendMessageRequest, []int) {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return append([]AppendMessageRequest(nil), a.messages...), append([]int(nil), a.batchSizes...)
+}
+
type failingWriter struct{}
func (f failingWriter) Write(_ []byte) (int, error) {
@@ -37,10 +57,12 @@ func TestWriterContinuesActivityCaptureWhenResponseWriterFailsInternal(t *testin
FlushWriter(writer)
require.Eventually(t, func() bool {
- return len(appender.messages) == 1
+ messages, _ := appender.recorded()
+ return len(messages) == 1
}, time.Second, 10*time.Millisecond)
- require.Equal(t, "Downloading layer", appender.messages[0].Message)
- require.Equal(t, models.ActivityMessageLevelInfo, appender.messages[0].Level)
+ messages, _ := appender.recorded()
+ require.Equal(t, "Downloading layer", messages[0].Message)
+ require.Equal(t, models.ActivityMessageLevelInfo, messages[0].Level)
}
func TestWriterRecordsLogAndErrorFramesVerbatimInternal(t *testing.T) {
@@ -52,12 +74,64 @@ func TestWriterRecordsLogAndErrorFramesVerbatimInternal(t *testing.T) {
FlushWriter(writer)
require.Eventually(t, func() bool {
- return len(appender.messages) == 2
+ messages, _ := appender.recorded()
+ return len(messages) == 2
+ }, time.Second, 10*time.Millisecond)
+ messages, _ := appender.recorded()
+ require.Equal(t, "Container web-1 Created", messages[0].Message)
+ require.Equal(t, models.ActivityMessageLevelInfo, messages[0].Level)
+ require.Equal(t, "Error response from daemon: conflict", messages[1].Message)
+ require.Equal(t, models.ActivityMessageLevelError, messages[1].Level)
+}
+
+// gatedAppender blocks its first AppendMessages call until release is
+// closed, letting a test queue lines behind a known-stalled drain goroutine.
+type gatedAppender struct {
+ recordingAppender
+ started chan struct{}
+ release chan struct{}
+ startedOnce sync.Once
+}
+
+func (g *gatedAppender) AppendMessages(ctx context.Context, activityID string, reqs []AppendMessageRequest) ([]activitytypes.Message, error) {
+ g.startedOnce.Do(func() { close(g.started) })
+ <-g.release
+ return g.recordingAppender.AppendMessages(ctx, activityID, reqs)
+}
+
+// TestWriterBatchesQueuedLinesIntoOneAppendInternal verifies lines already
+// queued when the drain goroutine wakes are persisted through a single
+// AppendMessages call, in order, rather than one call per line.
+func TestWriterBatchesQueuedLinesIntoOneAppendInternal(t *testing.T) {
+ appender := &gatedAppender{started: make(chan struct{}), release: make(chan struct{})}
+ writer := NewWriter(context.Background(), appender, "activity-1", io.Discard, "Pulling image")
+
+ _, err := writer.Write([]byte("layer 0\n"))
+ require.NoError(t, err)
+ select {
+ case <-appender.started:
+ case <-time.After(time.Second):
+ t.Fatal("first append never started")
+ }
+
+ // The drain goroutine is stalled inside the first append; everything
+ // written now queues up behind it.
+ for i := 1; i < 10; i++ {
+ _, err := writer.Write(fmt.Appendf(nil, "layer %d\n", i))
+ require.NoError(t, err)
+ }
+ close(appender.release)
+
+ require.Eventually(t, func() bool {
+ messages, _ := appender.recorded()
+ return len(messages) == 10
}, time.Second, 10*time.Millisecond)
- require.Equal(t, "Container web-1 Created", appender.messages[0].Message)
- require.Equal(t, models.ActivityMessageLevelInfo, appender.messages[0].Level)
- require.Equal(t, "Error response from daemon: conflict", appender.messages[1].Message)
- require.Equal(t, models.ActivityMessageLevelError, appender.messages[1].Level)
+
+ messages, batchSizes := appender.recorded()
+ for i, message := range messages {
+ require.Equal(t, fmt.Sprintf("layer %d", i), message.Message)
+ }
+ require.Equal(t, []int{1, 9}, batchSizes)
}
func TestWriterReturnsWrappedWriteErrorWithoutActivityInternal(t *testing.T) {
@@ -69,3 +143,41 @@ func TestWriterReturnsWrappedWriteErrorWithoutActivityInternal(t *testing.T) {
var _ MessageAppender = (*recordingAppender)(nil)
var _ io.Writer = failingWriter{}
+
+// TestWriterPersistsQueuedLinesAfterCancelInternal verifies that cancelling
+// the work context does not abandon lines the writer already accepted:
+// batches persist on a detached context and the drain goroutine empties the
+// queue before exiting, since the lines queued at cancellation are usually
+// the ones explaining it.
+func TestWriterPersistsQueuedLinesAfterCancelInternal(t *testing.T) {
+ ctx, cancel := context.WithCancel(context.Background())
+ appender := &gatedAppender{started: make(chan struct{}), release: make(chan struct{})}
+ writer := NewWriter(ctx, appender, "activity-1", io.Discard, "Pulling image")
+
+ _, err := writer.Write([]byte("layer 0\n"))
+ require.NoError(t, err)
+ select {
+ case <-appender.started:
+ case <-time.After(time.Second):
+ t.Fatal("first append never started")
+ }
+
+ // The drain goroutine is stalled inside the first append; these queue up
+ // behind it and are still pending when the context is cancelled.
+ for i := 1; i < 10; i++ {
+ _, err := writer.Write(fmt.Appendf(nil, "layer %d\n", i))
+ require.NoError(t, err)
+ }
+ cancel()
+ close(appender.release)
+
+ require.Eventually(t, func() bool {
+ messages, _ := appender.recorded()
+ return len(messages) == 10
+ }, time.Second, 10*time.Millisecond)
+
+ messages, _ := appender.recorded()
+ for i, message := range messages {
+ require.Equal(t, fmt.Sprintf("layer %d", i), message.Message)
+ }
+}