Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
perf(activity): batch message appends and drop the per-line re-SELECT (#3611)
Kyle Mendella9d3df1fe2d9
diff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.goindex e017f0d9e..19a7bc910 100644--- a/backend/internal/activity/service.go+++ b/backend/internal/activity/service.go@@ -3,6 +3,7 @@ package activityimport ("context"stderrors "errors"+ "hash/fnv""log/slog""maps""slices"@@ -55,8 +56,30 @@ type ActivityService struct {limiter *activitySlotLimiterslotMu sync.MutexslotReleases map[string]func()++ // terminalPublished records when each activity's terminal snapshot was+ // published, so later non-terminal snapshots from slower goroutines can+ // be dropped (see admitActivityPublishInternal). Entries are pruned on+ // terminal publishes once older than terminalPublishRetention.+ terminalPublishedMu sync.Mutex+ terminalPublished map[string]time.Time++ // publishLocks serializes the commit→publish window of the non-terminal+ // snapshot writers (AppendMessages, UpdateActivity) per activity, so their+ // events reach subscribers in commit order — without it, a batch that+ // commits first but publishes second would revert progress/step/status+ // fields a concurrent update already streamed. Terminal snapshots are+ // ordered separately by admitActivityPublishInternal. Striped by activity+ // ID hash so no per-activity lifecycle management is needed.+ publishLocks [64]sync.Mutex}+// terminalPublishRetention bounds how long a terminal publish suppresses+// stale non-terminal snapshots for the same activity. Stale publishers are+// goroutines already past their commit, so they publish within moments; the+// retention only has to outlive that gap while keeping the latch map small.+const terminalPublishRetention = 10 * time.Minute+// ErrActivityNotCancelable indicates the activity has already reached a terminal// state and can no longer be cancelled.const ErrActivityNotCancelable = errors.Sentinel("activity is not cancelable")@@ -187,11 +210,12 @@ type AppendActivityMessageRequest = activitylib.AppendMessageRequestfunc NewActivityService(db *database.DB, settingsService *settings.SettingsService) *ActivityService {return &ActivityService{- db: db,- subscribers: map[int]*activitySubscriber{},- running: map[string]context.CancelCauseFunc{},- limiter: newActivitySlotLimiterInternal(settingsService),- slotReleases: map[string]func(){},+ db: db,+ subscribers: map[int]*activitySubscriber{},+ running: map[string]context.CancelCauseFunc{},+ limiter: newActivitySlotLimiterInternal(settingsService),+ slotReleases: map[string]func(){},+ terminalPublished: map[string]time.Time{},}}@@ -452,6 +476,13 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,updates["metadata"] = cloneJSONInternal(req.Metadata)}+ // Hold the per-activity publish lock across commit and publish so this+ // snapshot cannot be overtaken on the stream by a concurrent append batch+ // that committed earlier but would publish later.+ lock := s.publishLockInternal(activityID)+ lock.Lock()+ defer lock.Unlock()+var model models.Activityif err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {result := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates)@@ -475,6 +506,23 @@ func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string,}func (s *ActivityService) AppendMessage(ctx context.Context, activityID string, req AppendActivityMessageRequest) (*activitytypes.Message, error) {+ messages, err := s.AppendMessages(ctx, activityID, []AppendActivityMessageRequest{req})+ if err != nil || len(messages) == 0 {+ return nil, err+ }+ return &messages[0], nil+}++// AppendMessages persists a batch of output lines in one transaction — a+// single multi-row message INSERT plus one coalesced Activity update —+// instead of an INSERT+UPDATE+re-SELECT transaction per line, which turned+// an image pull into hundreds of fsync'd SQLite transactions. The activity+// publish DTO is re-SELECTed inside the transaction after the update (once+// per batch, not per line) so it reflects lifecycle fields a concurrent+// CompleteActivity/UpdateActivity committed first; a terminal write that+// commits after this transaction but publishes before this snapshot is+// handled by admitActivityPublishInternal dropping the stale event.+func (s *ActivityService) AppendMessages(ctx context.Context, activityID string, reqs []AppendActivityMessageRequest) ([]activitytypes.Message, error) {if err := s.checkInitInternal(); err != nil {return nil, err}@@ -483,66 +531,121 @@ func (s *ActivityService) AppendMessage(ctx context.Context, activityID string,return nil, errors.New("activity id is required")}- messageText := strings.TrimSpace(req.Message)- if messageText == "" {- return nil, nil- }- if len(messageText) > 8192 {- messageText = messageText[:8192]- }-- level := req.Level- if level == "" {- level = models.ActivityMessageLevelInfo- }-now := time.Now()- message := &models.ActivityMessage{- ActivityID: activityID,- Level: level,- Message: messageText,- Payload: cloneJSONInternal(req.Payload),- BaseModel: models.BaseModel{- CreatedAt: now,- },- }-- var updated models.Activity- if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- if err := tx.Create(message).Error; err != nil {- return errors.WrapIf(err, "failed to append activity message")+ messages := make([]*models.ActivityMessage, 0, len(reqs))+ latestMessage := ""+ var latestProgress *int+ latestStep := ""+ for _, req := range reqs {+ messageText := strings.TrimSpace(req.Message)+ if messageText == "" {+ continue+ }+ if len(messageText) > 8192 {+ messageText = messageText[:8192]}- updates := map[string]any{- "latest_message": messageText,- "updated_at": now,+ level := req.Level+ if level == "" {+ level = models.ActivityMessageLevelInfo}++ messages = append(messages, &models.ActivityMessage{+ ActivityID: activityID,+ Level: level,+ Message: messageText,+ Payload: cloneJSONInternal(req.Payload),+ BaseModel: models.BaseModel{+ // Spread inside the shared timestamp so the created_at sort+ // used for retrieval keeps the original line order.+ CreatedAt: now.Add(time.Duration(len(messages)) * time.Microsecond),+ },+ })++ latestMessage = messageTextif req.Progress != nil {- updates["progress"] = *clampProgressPtrInternal(req.Progress)+ latestProgress = req.Progress}- if strings.TrimSpace(req.Step) != "" {- updates["step"] = strings.TrimSpace(req.Step)+ if step := strings.TrimSpace(req.Step); step != "" {+ latestStep = step}+ }+ if len(messages) == 0 {+ return nil, nil+ }- result := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates)- if result.Error != nil {- return errors.WrapIf(result.Error, "failed to update activity latest message")- }- if result.RowsAffected == 0 {- return errors.New("activity not found")+ updates := map[string]any{+ "latest_message": latestMessage,+ "updated_at": now,+ }+ if latestProgress != nil {+ updates["progress"] = *clampProgressPtrInternal(latestProgress)+ }+ if latestStep != "" {+ updates["step"] = latestStep+ }++ // Hold the per-activity publish lock across commit and publish so this+ // batch's snapshot cannot be overtaken on the stream by a concurrent+ // UpdateActivity that committed later — subscribers must see the two+ // snapshots in commit order.+ lock := s.publishLockInternal(activityID)+ lock.Lock()+ defer lock.Unlock()++ // Bulk appends contend with terminal-status writes for the SQLite write+ // lock, and a batch lost to transient SQLITE_BUSY silently drops up to 32+ // output lines — the Writer drain has no error path to replay them — so+ // retry like CompleteActivity does.+ var current models.Activity+ const appendWriteAttempts = 3+ var writeErr error+ for attempt := 1; attempt <= appendWriteAttempts; attempt++ {+ writeErr = s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return errors.New("activity not found")+ }+ return errors.WrapIf(err, "failed to load activity")+ }++ if err := tx.Create(&messages).Error; err != nil {+ return errors.WrapIf(err, "failed to append activity message")+ }++ result := tx.Model(&models.Activity{}).Where("id = ?", activityID).Updates(updates)+ if result.Error != nil {+ return errors.WrapIf(result.Error, "failed to update activity latest message")+ }+ if result.RowsAffected == 0 {+ return errors.New("activity not found")+ }+ if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {+ return errors.WrapIf(err, "failed to load updated activity")+ }+ return nil+ })+ if writeErr == nil {+ break}- if err := tx.First(&updated, "id = ?", activityID).Error; err != nil {- return errors.WrapIf(err, "failed to load updated activity")+ if attempt == appendWriteAttempts || !dbutil.IsRetryableWriteError(writeErr) {+ return nil, writeErr}- return nil- }); err != nil {- return nil, err+ slog.WarnContext(ctx, "retrying activity message batch write", "activityId", activityID, "attempt", attempt, "error", writeErr)+ time.Sleep(250 * time.Millisecond * time.Duration(attempt))+ }+ if writeErr != nil {+ return nil, writeErr}- dto := activityMessageToDTOInternal(message)- s.publishMessageInternal(updated.EnvironmentID, dto)- s.publishActivityInternal(activityToDTOInternal(&updated))- return &dto, nil+ out := make([]activitytypes.Message, 0, len(messages))+ for _, message := range messages {+ dto := activityMessageToDTOInternal(message)+ s.publishMessageInternal(current.EnvironmentID, dto)+ out = append(out, dto)+ }+ s.publishActivityInternal(activityToDTOInternal(¤t))+ return out, nil}func (s *ActivityService) CompleteActivity(ctx context.Context, activityID string, status models.ActivityStatus, finalMessage string, errMessage *string, finalStep ...string) (*activitytypes.Activity, error) {@@ -621,16 +724,37 @@ func (s *ActivityService) CompleteActivity(ctx context.Context, activityID strin}); err != nil {slog.DebugContext(ctx, "failed to append final activity message", "activityId", activityID, "error", err)}- if err := s.db.WithContext(activityCtx).First(&model, "id = ?", activityID).Error; err != nil {- slog.DebugContext(ctx, "failed to reload activity after appending message", "activityId", activityID, "error", err)- }}- dto := activityToDTOInternal(&model)- s.publishActivityInternal(dto)+ dto := s.publishTerminalSnapshotInternal(ctx, &model)return &dto, nil}+// publishTerminalSnapshotInternal publishes an activity's terminal event from+// a snapshot re-read under the per-activity publish lock. Terminal writers+// cannot hold that lock across their own transactions (CompleteActivity+// appends its final message through the locked AppendMessages path), so a+// snapshot captured in-transaction can be overtaken by an append or update+// that commits later but publishes first; re-reading under the lock+// guarantees the terminal event carries every field already streamed. The+// passed model is published as-is if the re-read fails, and is updated in+// place otherwise so callers return the published state.+func (s *ActivityService) publishTerminalSnapshotInternal(ctx context.Context, model *models.Activity) activitytypes.Activity {+ lock := s.publishLockInternal(model.ID)+ lock.Lock()+ defer lock.Unlock()++ var fresh models.Activity+ if err := s.db.WithContext(ctx).First(&fresh, "id = ?", model.ID).Error; err != nil {+ slog.DebugContext(ctx, "failed to reload activity for terminal publish", "activityId", model.ID, "error", err)+ } else {+ *model = fresh+ }+ dto := activityToDTOInternal(model)+ s.publishActivityInternal(dto)+ return dto+}+// CancelActivity requests cancellation of a running or queued activity. When the// activity's work is running in this process it interrupts it (the work finalizes// its own terminal status); otherwise it marks the activity cancelled directly,@@ -707,8 +831,7 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, actreturn nil, err}- dto := activityToDTOInternal(&finalized)- s.publishActivityInternal(dto)+ dto := s.publishTerminalSnapshotInternal(writeCtx, &finalized)return &dto, nil}@@ -818,7 +941,7 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, e}s.releaseSlotInternal(activityID)- s.publishActivityInternal(activityToDTOInternal(&finalized))+ s.publishTerminalSnapshotInternal(ctx, &finalized)swept++}@@ -1161,6 +1284,9 @@ func (s *ActivityService) Subscribe(environmentID string) (<-chan activitytypes.}func (s *ActivityService) publishActivityInternal(activity activitytypes.Activity) {+ if !s.admitActivityPublishInternal(activity) {+ return+ }s.publishInternal(activity.EnvironmentID, activitytypes.StreamEvent{Type: "activity",ActivityID: activity.ID,@@ -1169,6 +1295,50 @@ func (s *ActivityService) publishActivityInternal(activity activitytypes.Activit})}+func (s *ActivityService) publishLockInternal(activityID string) *sync.Mutex {+ h := fnv.New32a()+ _, _ = h.Write([]byte(activityID))+ return &s.publishLocks[h.Sum32()%uint32(len(s.publishLocks))]+}++func isTerminalActivityStatusInternal(status activitytypes.Status) bool {+ return status == activitytypes.StatusSuccess ||+ status == activitytypes.StatusFailed ||+ status == activitytypes.StatusCancelled+}++// admitActivityPublishInternal orders activity events across the commit →+// publish gap. Writes publish their snapshot after committing, so a goroutine+// that committed before a terminal write can publish after it; since a+// terminal activity emits no further events to correct the stream (and+// subscriber coalescing would even replace an undelivered terminal event in+// place), such a stale non-terminal snapshot must be dropped, not delivered.+// Activities never leave a terminal status, so once a terminal snapshot is+// published, any later non-terminal snapshot for that ID is stale by+// construction.+func (s *ActivityService) admitActivityPublishInternal(activity activitytypes.Activity) bool {+ if s == nil {+ return true+ }+ s.terminalPublishedMu.Lock()+ defer s.terminalPublishedMu.Unlock()+ if !isTerminalActivityStatusInternal(activity.Status) {+ _, sealed := s.terminalPublished[activity.ID]+ return !sealed+ }+ now := time.Now()+ for id, publishedAt := range s.terminalPublished {+ if now.Sub(publishedAt) > terminalPublishRetention {+ delete(s.terminalPublished, id)+ }+ }+ if s.terminalPublished == nil {+ s.terminalPublished = map[string]time.Time{}+ }+ s.terminalPublished[activity.ID] = now+ return true+}+func (s *ActivityService) publishMessageInternal(environmentID string, message activitytypes.Message) {s.publishInternal(environmentID, activitytypes.StreamEvent{Type: "message",diff --git a/backend/internal/activity/service_test.go b/backend/internal/activity/service_test.goindex 823f02cb0..1ba8cf964 100644--- a/backend/internal/activity/service_test.go+++ b/backend/internal/activity/service_test.go@@ -732,3 +732,90 @@ func receiveActivityEventInternal(t *testing.T, events <-chan activitytypes.Strereturn activitytypes.StreamEvent{}}}++// TestActivityServiceAppendMessagesBatchInternal verifies a batch lands all+// messages in order and coalesces the activity update to the batch's last+// message, last progress, and last step.+func TestActivityServiceAppendMessagesBatchInternal(t *testing.T) {+ ctx := context.Background()+ db := setupActivityServiceTestDBInternal(t)+ service := NewActivityService(db, nil)++ created, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeImagePull,+ Step: "queued",+ LatestMessage: "Pull queued",+ })+ require.NoError(t, err)++ progress := 60+ messages, err := service.AppendMessages(ctx, created.ID, []AppendActivityMessageRequest{+ {Message: "layer 1/3", Step: "download"},+ {Message: " "}, // blank lines are dropped, not persisted+ {Message: "layer 2/3", Progress: &progress},+ {Message: "layer 3/3"},+ })+ require.NoError(t, err)+ require.Len(t, messages, 3)+ require.Equal(t, "layer 1/3", messages[0].Message)+ require.Equal(t, "layer 3/3", messages[2].Message)++ detail, err := service.GetActivityDetail(ctx, "0", created.ID, 10)+ require.NoError(t, err)+ require.Len(t, detail.Messages, 3)+ require.Equal(t, "layer 1/3", detail.Messages[0].Message)+ require.Equal(t, "layer 2/3", detail.Messages[1].Message)+ require.Equal(t, "layer 3/3", detail.Messages[2].Message)++ // Coalesced activity update: last message wins, last non-nil progress+ // and last non-empty step stick.+ require.Equal(t, "layer 3/3", detail.Activity.LatestMessage)+ require.NotNil(t, detail.Activity.Progress)+ require.Equal(t, 60, *detail.Activity.Progress)+ require.Equal(t, "download", detail.Activity.Step)++ // Unknown activity IDs are still rejected.+ _, err = service.AppendMessages(ctx, "missing", []AppendActivityMessageRequest{{Message: "x"}})+ require.ErrorContains(t, err, "activity not found")+}++// TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal verifies+// that a non-terminal activity snapshot published after the activity's+// terminal event — a goroutine that committed before CompleteActivity but+// publishes after it — is dropped instead of reverting subscribers to a+// running state no later event would correct.+func TestActivityServiceDropsStaleSnapshotAfterTerminalPublishInternal(t *testing.T) {+ ctx := context.Background()+ db := setupActivityServiceTestDBInternal(t)+ service := NewActivityService(db, nil)++ created, err := service.StartActivity(ctx, StartActivityRequest{+ EnvironmentID: "0",+ Type: models.ActivityTypeImagePull,+ LatestMessage: "Pull queued",+ })+ require.NoError(t, err)+ stale := *created // snapshot taken while the activity is still active++ events, _, unsubscribe := service.Subscribe("0")+ defer unsubscribe()++ completed, err := service.CompleteActivity(ctx, created.ID, models.ActivityStatusSuccess, "", nil)+ require.NoError(t, err)+ require.Equal(t, activitytypes.StatusSuccess, completed.Status)++ event := receiveActivityEventInternal(t, events)+ require.Equal(t, "activity", event.Type)+ require.Equal(t, activitytypes.StatusSuccess, event.Activity.Status)++ require.False(t, service.admitActivityPublishInternal(stale))+ require.True(t, service.admitActivityPublishInternal(*completed))++ service.publishActivityInternal(stale)+ select {+ case event := <-events:+ t.Fatalf("stale non-terminal snapshot reached subscriber: %+v", event)+ case <-time.After(100 * time.Millisecond):+ }+}diff --git a/backend/pkg/libarcane/activity/requests.go b/backend/pkg/libarcane/activity/requests.goindex 69a8b05cb..1c0d6d60f 100644--- a/backend/pkg/libarcane/activity/requests.go+++ b/backend/pkg/libarcane/activity/requests.go@@ -14,6 +14,10 @@ type Service interface {type MessageAppender interface {AppendMessage(ctx context.Context, activityID string, req AppendMessageRequest) (*activitytypes.Message, error)+ // AppendMessages persists a batch of messages in one transaction with a+ // single coalesced activity update; the Writer drains its queue through+ // this so bulk docker output does not fsync per line.+ AppendMessages(ctx context.Context, activityID string, reqs []AppendMessageRequest) ([]activitytypes.Message, error)}// Tracker is an optional interface a Service may implement to make activitiesdiff --git a/backend/pkg/libarcane/activity/writer.go b/backend/pkg/libarcane/activity/writer.goindex 0b7602502..794f7c51a 100644--- a/backend/pkg/libarcane/activity/writer.go+++ b/backend/pkg/libarcane/activity/writer.go@@ -6,6 +6,7 @@ import ("encoding/json/v2""fmt""io"+ "log/slog""net/http""strings""sync"@@ -15,7 +16,13 @@ import ()type Writer struct {- ctx context.Context+ ctx context.Context+ // persistCtx is ctx detached from cancellation. Cancellation usually+ // races the last — most diagnostic — output lines (the error frames+ // explaining it), so batches persist on persistCtx while ctx only bounds+ // the drain goroutine's intake, mirroring CompleteActivity's detached+ // terminal write.+ persistCtx context.ContextactivityService MessageAppenderactivityID stringwriter io.Writer@@ -26,7 +33,12 @@ type Writer struct {buffer []byte}-const writerAppendQueueSize = 128+const (+ writerAppendQueueSize = 128+ // writerAppendBatchSize caps how many queued lines are drained into one+ // AppendMessages transaction.+ writerAppendBatchSize = 32+)type writerAppendMessage struct {level models.ActivityMessageLevel@@ -50,8 +62,13 @@ func NewWriter(ctx context.Context, activityService MessageAppender, activityIDif existing, ok := writer.(*Writer); ok {return existing}+ var persistCtx context.Context+ if ctx != nil {+ persistCtx = context.WithoutCancel(ctx)+ }out := &Writer{ctx: ctx,+ persistCtx: persistCtx,activityService: activityService,activityID: strings.TrimSpace(activityID),writer: writer,@@ -170,38 +187,82 @@ func (w *Writer) drainMessagesInternal(ctx context.Context) {for {select {case item := <-w.queueCh:- if item.flush != nil {- close(item.flush)- continue- }- if item.message != nil {- w.appendMessageInternal(ctx, *item.message)- }+ w.drainQueueBatchInternal(item)case <-doneInternal(ctx):+ // Cancellation stops intake, but the lines already accepted into+ // the queue — typically the ones explaining the cancellation —+ // still get persisted before the goroutine exits.+ w.drainRemainingInternal()return}}}-func doneInternal(ctx context.Context) <-chan struct{} {- if ctx == nil {- return nil+func (w *Writer) drainRemainingInternal() {+ for {+ select {+ case item := <-w.queueCh:+ w.drainQueueBatchInternal(item)+ default:+ return+ }}- return ctx.Done()}-func (w *Writer) appendMessageInternal(ctx context.Context, message writerAppendMessage) {- if ctx == nil {- return+// drainQueueBatchInternal drains whatever is already queued behind item (up+// to writerAppendBatchSize lines) into a single AppendMessages call, so bulk+// docker output costs one transaction per batch instead of one per line.+// Flush signals encountered while draining close only after the write, since+// the messages queued before them are only then persisted.+func (w *Writer) drainQueueBatchInternal(item writerQueueItem) {+ var batch []AppendMessageRequest+ var flushes []chan struct{}++ collect := func(queued writerQueueItem) {+ if queued.flush != nil {+ flushes = append(flushes, queued.flush)+ return+ }+ if queued.message == nil {+ return+ }+ batch = append(batch, AppendMessageRequest{+ Level: queued.message.level,+ Message: queued.message.message,+ Payload: queued.message.payload,+ Step: queued.message.step,+ })}- if _, err := w.activityService.AppendMessage(ctx, w.activityID, AppendMessageRequest{- Level: message.level,- Message: message.message,- Payload: message.payload,- Step: message.step,- }); err != nil {- return++ collect(item)+drain:+ for len(batch) < writerAppendBatchSize {+ select {+ case queued := <-w.queueCh:+ collect(queued)+ default:+ break drain+ }+ }++ if len(batch) > 0 && w.persistCtx != nil {+ if _, err := w.activityService.AppendMessages(w.persistCtx, w.activityID, batch); err != nil {+ slog.WarnContext(w.persistCtx, "failed to persist activity output batch", "activityId", w.activityID, "lines", len(batch), "error", err)+ }}+ // Flushes release even when the append failed: Flush carries no error+ // path, and holding them would stall activity completion on the very+ // cancellation that typically caused the failure.+ for _, flushDone := range flushes {+ close(flushDone)+ }+}++func doneInternal(ctx context.Context) <-chan struct{} {+ if ctx == nil {+ return nil+ }+ return ctx.Done()}func valueToStringInternal(value any) string {diff --git a/backend/pkg/libarcane/activity/writer_test.go b/backend/pkg/libarcane/activity/writer_test.goindex 9e5fe9bf5..56bdbfdd0 100644--- a/backend/pkg/libarcane/activity/writer_test.go+++ b/backend/pkg/libarcane/activity/writer_test.go@@ -3,7 +3,9 @@ package activityimport ("context""errors"+ "fmt""io"+ "sync""testing""time"@@ -13,14 +15,32 @@ import ()type recordingAppender struct {- messages []AppendMessageRequest+ mu sync.Mutex+ messages []AppendMessageRequest+ batchSizes []int}func (a *recordingAppender) AppendMessage(_ context.Context, _ string, req AppendMessageRequest) (*activitytypes.Message, error) {+ a.mu.Lock()+ defer a.mu.Unlock()a.messages = append(a.messages, req)return &activitytypes.Message{}, nil}+func (a *recordingAppender) AppendMessages(_ context.Context, _ string, reqs []AppendMessageRequest) ([]activitytypes.Message, error) {+ a.mu.Lock()+ defer a.mu.Unlock()+ a.messages = append(a.messages, reqs...)+ a.batchSizes = append(a.batchSizes, len(reqs))+ return make([]activitytypes.Message, len(reqs)), nil+}++func (a *recordingAppender) recorded() ([]AppendMessageRequest, []int) {+ a.mu.Lock()+ defer a.mu.Unlock()+ return append([]AppendMessageRequest(nil), a.messages...), append([]int(nil), a.batchSizes...)+}+type failingWriter struct{}func (f failingWriter) Write(_ []byte) (int, error) {@@ -37,10 +57,12 @@ func TestWriterContinuesActivityCaptureWhenResponseWriterFailsInternal(t *testinFlushWriter(writer)require.Eventually(t, func() bool {- return len(appender.messages) == 1+ messages, _ := appender.recorded()+ return len(messages) == 1}, time.Second, 10*time.Millisecond)- require.Equal(t, "Downloading layer", appender.messages[0].Message)- require.Equal(t, models.ActivityMessageLevelInfo, appender.messages[0].Level)+ messages, _ := appender.recorded()+ require.Equal(t, "Downloading layer", messages[0].Message)+ require.Equal(t, models.ActivityMessageLevelInfo, messages[0].Level)}func TestWriterRecordsLogAndErrorFramesVerbatimInternal(t *testing.T) {@@ -52,12 +74,64 @@ func TestWriterRecordsLogAndErrorFramesVerbatimInternal(t *testing.T) {FlushWriter(writer)require.Eventually(t, func() bool {- return len(appender.messages) == 2+ messages, _ := appender.recorded()+ return len(messages) == 2+ }, time.Second, 10*time.Millisecond)+ messages, _ := appender.recorded()+ require.Equal(t, "Container web-1 Created", messages[0].Message)+ require.Equal(t, models.ActivityMessageLevelInfo, messages[0].Level)+ require.Equal(t, "Error response from daemon: conflict", messages[1].Message)+ require.Equal(t, models.ActivityMessageLevelError, messages[1].Level)+}++// gatedAppender blocks its first AppendMessages call until release is+// closed, letting a test queue lines behind a known-stalled drain goroutine.+type gatedAppender struct {+ recordingAppender+ started chan struct{}+ release chan struct{}+ startedOnce sync.Once+}++func (g *gatedAppender) AppendMessages(ctx context.Context, activityID string, reqs []AppendMessageRequest) ([]activitytypes.Message, error) {+ g.startedOnce.Do(func() { close(g.started) })+ <-g.release+ return g.recordingAppender.AppendMessages(ctx, activityID, reqs)+}++// TestWriterBatchesQueuedLinesIntoOneAppendInternal verifies lines already+// queued when the drain goroutine wakes are persisted through a single+// AppendMessages call, in order, rather than one call per line.+func TestWriterBatchesQueuedLinesIntoOneAppendInternal(t *testing.T) {+ appender := &gatedAppender{started: make(chan struct{}), release: make(chan struct{})}+ writer := NewWriter(context.Background(), appender, "activity-1", io.Discard, "Pulling image")++ _, err := writer.Write([]byte("layer 0\n"))+ require.NoError(t, err)+ select {+ case <-appender.started:+ case <-time.After(time.Second):+ t.Fatal("first append never started")+ }++ // The drain goroutine is stalled inside the first append; everything+ // written now queues up behind it.+ for i := 1; i < 10; i++ {+ _, err := writer.Write(fmt.Appendf(nil, "layer %d\n", i))+ require.NoError(t, err)+ }+ close(appender.release)++ require.Eventually(t, func() bool {+ messages, _ := appender.recorded()+ return len(messages) == 10}, time.Second, 10*time.Millisecond)- require.Equal(t, "Container web-1 Created", appender.messages[0].Message)- require.Equal(t, models.ActivityMessageLevelInfo, appender.messages[0].Level)- require.Equal(t, "Error response from daemon: conflict", appender.messages[1].Message)- require.Equal(t, models.ActivityMessageLevelError, appender.messages[1].Level)++ messages, batchSizes := appender.recorded()+ for i, message := range messages {+ require.Equal(t, fmt.Sprintf("layer %d", i), message.Message)+ }+ require.Equal(t, []int{1, 9}, batchSizes)}func TestWriterReturnsWrappedWriteErrorWithoutActivityInternal(t *testing.T) {@@ -69,3 +143,41 @@ func TestWriterReturnsWrappedWriteErrorWithoutActivityInternal(t *testing.T) {var _ MessageAppender = (*recordingAppender)(nil)var _ io.Writer = failingWriter{}++// TestWriterPersistsQueuedLinesAfterCancelInternal verifies that cancelling+// the work context does not abandon lines the writer already accepted:+// batches persist on a detached context and the drain goroutine empties the+// queue before exiting, since the lines queued at cancellation are usually+// the ones explaining it.+func TestWriterPersistsQueuedLinesAfterCancelInternal(t *testing.T) {+ ctx, cancel := context.WithCancel(context.Background())+ appender := &gatedAppender{started: make(chan struct{}), release: make(chan struct{})}+ writer := NewWriter(ctx, appender, "activity-1", io.Discard, "Pulling image")++ _, err := writer.Write([]byte("layer 0\n"))+ require.NoError(t, err)+ select {+ case <-appender.started:+ case <-time.After(time.Second):+ t.Fatal("first append never started")+ }++ // The drain goroutine is stalled inside the first append; these queue up+ // behind it and are still pending when the context is cancelled.+ for i := 1; i < 10; i++ {+ _, err := writer.Write(fmt.Appendf(nil, "layer %d\n", i))+ require.NoError(t, err)+ }+ cancel()+ close(appender.release)++ require.Eventually(t, func() bool {+ messages, _ := appender.recorded()+ return len(messages) == 10+ }, time.Second, 10*time.Millisecond)++ messages, _ := appender.recorded()+ for i, message := range messages {+ require.Equal(t, fmt.Sprintf("layer %d", i), message.Message)+ }+}