Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
package activity
import (
"cmp"
"context"
"errors"
"fmt"
"hash/fnv"
"log/slog"
"maps"
"slices"
"strings"
"sync"
"time"
"github.com/getarcaneapp/arcane/types/v2/activity"
"github.com/getarcaneapp/arcane/types/v2/scheduler"
"github.com/getarcaneapp/arcane/types/v2/user"
"github.com/samber/mo"
"go.getarcane.app/kit/pkg"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/pubsub"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/slots"
"github.com/getarcaneapp/arcane/backend/v2/internal/activity/children/sweeps"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/dbutil"
)
const (
defaultActivityRetentionDays = 30
defaultActivityHistoryLimit = 1000
defaultActivityMessages = 500
cancelledMessage = "Cancelled by user"
deleteActivitiesBatchSize = 500
)
var ErrActivityNotCancelable = errors.New("activity is not cancelable")
type ActivityService struct {
db *database.DB
pubsub *pubsub.Service
slots *slots.Service
sweeps *sweeps.Service
// running maps an active activity ID to the cancel function of its work
// context, so cancellation requests can interrupt in-flight work. Entries
// are added by Track and removed when the activity is completed.
runningMu sync.Mutex
running map[string]context.CancelCauseFunc
// publishLocks serializes the commit→publish window of the non-terminal
// snapshot writers (AppendMessages, UpdateActivity) per activity, so their
// events reach subscribers in commit order — without it, a batch that
// commits first but publishes second would revert progress/step/status
// fields a concurrent update already streamed. Terminal snapshots are
// ordered separately by the pubsub terminal latch. Striped by activity
// ID hash so no per-activity lifecycle management is needed.
publishLocks [64]sync.Mutex
}
type (
StartActivityRequest = activitylib.StartRequest
UpdateActivityRequest = activitylib.UpdateRequest
AppendActivityMessageRequest = activitylib.AppendMessageRequest
)
func NewActivityService(db *database.DB, settingsService *settings.SettingsService) *ActivityService {
return &ActivityService{
db: db,
pubsub: pubsub.NewService(),
slots: slots.NewService(settingsService),
sweeps: sweeps.NewService(db),
running: map[string]context.CancelCauseFunc{},
}
}
// Track derives a cancelable work context bound to activityID and registers its
// cancel function so RequestCancel can interrupt the work. The registration is
// released when the activity is completed (see CompleteActivity) or when the
// returned context is otherwise no longer needed. Implements activitylib.Tracker.
func (s *ActivityService) Track(ctx context.Context, activityID string) context.Context {
activityID = strings.TrimSpace(activityID)
if s == nil || activityID == "" {
return ctx
}
workCtx, cancel := context.WithCancelCause(ctx)
s.runningMu.Lock()
if s.running == nil {
s.running = map[string]context.CancelCauseFunc{}
}
if existing, ok := s.running[activityID]; ok {
// Replace any stale registration to avoid leaking the prior context.
existing(nil)
}
s.running[activityID] = cancel
s.runningMu.Unlock()
return workCtx
}
// RequestCancel cancels the work context registered for activityID, signalling
// activitylib.ErrCanceled as the cause. It returns whether a running activity
// was found in this process.
func (s *ActivityService) RequestCancel(activityID string) bool {
activityID = strings.TrimSpace(activityID)
if s == nil || activityID == "" {
return false
}
s.runningMu.Lock()
cancel, ok := s.running[activityID]
s.runningMu.Unlock()
if !ok {
return false
}
cancel(activitylib.ErrCanceled)
return true
}
func (s *ActivityService) checkInit() error {
if s == nil || s.db == nil {
return errors.New("activity service not initialized")
}
return nil
}
func (s *ActivityService) StartActivity(ctx context.Context, req StartActivityRequest) (*activity.Activity, error) {
if err := s.checkInit(); err != nil {
return nil, err
}
if req.ID != "" {
var existing Activity
err := s.db.WithContext(ctx).Where("id = ?", req.ID).Limit(1).Find(&existing).Error
if err != nil {
return nil, fmt.Errorf("failed to load activity: %w", err)
}
if existing.ID != "" {
dto := activityToDTO(&existing)
return &dto, nil
}
}
now := time.Now()
environmentID := cmp.Or(strings.TrimSpace(req.EnvironmentID), "0")
// Queue-opted activities take a concurrency slot up front when one is
// free; otherwise they are created as queued and AwaitActivitySlot blocks
// until a slot opens.
status := activity.StatusRunning
var slotRelease func()
if req.Queue {
if req.DeferSlot {
status = activity.StatusQueued
} else if release, ok := s.slots.TryAcquire(ctx, environmentID).Get(); ok {
slotRelease = release
} else {
status = activity.StatusQueued
}
}
startedBy := kit.FromPtr(req.StartedBy)
model := &Activity{
ID: req.ID,
EnvironmentID: environmentID,
BatchID: copyPtr(kit.Ternary(req.BatchID != nil, req.BatchID, mo.EmptyableToOption(utils.ActivityBatchIDFromContext(ctx)).ToPointer())),
Type: cmp.Or(req.Type, activity.TypeAutoUpdate),
Status: status,
ResourceType: copyPtr(req.ResourceType),
ResourceID: copyPtr(req.ResourceID),
ResourceName: copyPtr(req.ResourceName),
StartedByUserID: mo.EmptyableToOption(strings.TrimSpace(startedBy.ID)).ToPointer(),
StartedByUsername: mo.EmptyableToOption(strings.TrimSpace(startedBy.Username)).ToPointer(),
StartedByDisplayName: mo.EmptyableToOption(strings.TrimSpace(kit.FromPtr(startedBy.DisplayName))).ToPointer(),
Progress: clampProgressPtr(req.Progress),
Step: strings.TrimSpace(req.Step),
LatestMessage: strings.TrimSpace(req.LatestMessage),
StartedAt: now,
Metadata: cloneJSON(req.Metadata),
CreatedAt: now,
}
create := s.db.WithContext(ctx)
if req.ID != "" {
create = create.Clauses(clause.OnConflict{DoNothing: true})
}
created := create.Create(model)
if created.Error != nil || created.RowsAffected == 0 {
if slotRelease != nil {
slotRelease()
}
if created.Error != nil {
return nil, fmt.Errorf("failed to create activity: %w", created.Error)
}
// A concurrent start with the same ID created it first.
var existing Activity
if err := s.db.WithContext(ctx).First(&existing, "id = ?", req.ID).Error; err != nil {
return nil, fmt.Errorf("failed to load activity: %w", err)
}
dto := activityToDTO(&existing)
return &dto, nil
}
if slotRelease != nil {
s.slots.Register(model.ID, slotRelease)
}
dto := activityToDTO(model)
s.pubsub.PublishActivity(dto)
return &dto, nil
}
// StartTrackedActivity holds the cancellation lock across persistence and
// registration, so cancellation cannot treat the new row as untracked.
func (s *ActivityService) StartTrackedActivity(ctx context.Context, req StartActivityRequest) (*activity.Activity, context.Context, error) {
if err := s.checkInit(); err != nil {
return nil, nil, err
}
s.runningMu.Lock()
defer s.runningMu.Unlock()
started, err := s.StartActivity(ctx, req)
if err != nil {
return nil, nil, err
}
workCtx, cancel := context.WithCancelCause(ctx)
if s.running == nil {
s.running = map[string]context.CancelCauseFunc{}
}
s.running[started.ID] = cancel
return started, workCtx, nil
}
// AwaitActivitySlot blocks until the queued activity holds a concurrency slot,
// then flips its status to running. It returns immediately when the activity
// already took a slot at creation. On cancellation the context cause is
// returned and the activity stays queued for its caller to finalize.
// Implements activitylib.SlotWaiter.
func (s *ActivityService) AwaitActivitySlot(ctx context.Context, activityID, environmentID string) error {
if err := s.checkInit(); err != nil {
return err
}
activityID = strings.TrimSpace(activityID)
if activityID == "" {
return errors.New("activity id is required")
}
environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")
acquired, err := s.slots.Await(ctx, activityID, environmentID)
if err != nil || !acquired {
return err
}
if _, updateErr := s.UpdateActivity(ctx, activityID, UpdateActivityRequest{Status: activity.StatusRunning}); updateErr != nil {
slog.WarnContext(ctx, "failed to mark queued activity running", "activityId", activityID, "error", updateErr)
}
return nil
}
// AwaitActivitySlotBounded waits for a concurrency slot like AwaitActivitySlot
// but gives up after timeouts.DefaultActivitySlotWait, returning
// ActivitySlotWaitTimeoutError so the caller fails the queued activity loudly
// instead of parking forever behind long-running slot holders.
func (s *ActivityService) AwaitActivitySlotBounded(ctx context.Context, activityID, environmentID string) error {
slotCtx, cancel := context.WithTimeout(ctx, timeouts.DefaultActivitySlotWait)
defer cancel()
if err := s.AwaitActivitySlot(slotCtx, activityID, environmentID); err != nil {
if ctx.Err() == nil && errors.Is(err, context.DeadlineExceeded) {
return common.Classify(common.ErrTimeout, errors.New("timed out waiting for a free activity slot; other long-running activities are holding all slots"))
}
return err
}
return nil
}
func (s *ActivityService) UpdateActivity(ctx context.Context, activityID string, req UpdateActivityRequest) (*activity.Activity, error) {
if err := s.checkInit(); err != nil {
return nil, err
}
activityID = strings.TrimSpace(activityID)
if activityID == "" {
return nil, errors.New("activity id is required")
}
updates := map[string]any{
"updated_at": time.Now(),
}
if req.Status != "" {
updates["status"] = req.Status
}
if req.ResourceName != nil {
updates["resource_name"] = strings.TrimSpace(*req.ResourceName)
}
if req.Progress != nil {
updates["progress"] = *clampProgressPtr(req.Progress)
}
if req.Step != nil {
updates["step"] = strings.TrimSpace(*req.Step)
}
if req.LatestMessage != nil {
updates["latest_message"] = strings.TrimSpace(*req.LatestMessage)
}
if req.Error != nil {
updates["error"] = strings.TrimSpace(*req.Error)
}
if req.Metadata != nil {
updates["metadata"] = cloneJSON(req.Metadata)
}
// Hold the per-activity publish lock across commit and publish so this
// snapshot cannot be overtaken on the stream by a concurrent append batch
// that committed earlier but would publish later.
lock := s.publishLock(activityID)
lock.Lock()
defer lock.Unlock()
var model Activity
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
result := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates)
if result.Error != nil {
return fmt.Errorf("failed to update activity: %w", result.Error)
}
if result.RowsAffected == 0 {
return errors.New("activity not found")
}
if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
return fmt.Errorf("failed to load updated activity: %w", err)
}
return nil
}); err != nil {
return nil, err
}
dto := activityToDTO(&model)
s.pubsub.PublishActivity(dto)
return &dto, nil
}
func (s *ActivityService) AppendMessage(ctx context.Context, activityID string, req AppendActivityMessageRequest) (*activity.Message, error) {
messages, err := s.AppendMessages(ctx, activityID, []AppendActivityMessageRequest{req})
if err != nil || len(messages) == 0 {
return nil, err
}
return &messages[0], nil
}
// AppendMessages persists a batch of output lines in one transaction — a
// single multi-row message INSERT plus one coalesced Activity update —
// instead of an INSERT+UPDATE+re-SELECT transaction per line, which turned
// an image pull into hundreds of fsync'd SQLite transactions. The activity
// publish DTO is re-SELECTed inside the transaction after the update (once
// per batch, not per line) so it reflects lifecycle fields a concurrent
// CompleteActivity/UpdateActivity committed first; a terminal write that
// commits after this transaction but publishes before this snapshot is
// handled by the pubsub terminal latch dropping the stale event.
func (s *ActivityService) AppendMessages(ctx context.Context, activityID string, reqs []AppendActivityMessageRequest) ([]activity.Message, error) {
if err := s.checkInit(); err != nil {
return nil, err
}
activityID = strings.TrimSpace(activityID)
if activityID == "" {
return nil, errors.New("activity id is required")
}
messages, updates := buildAppendBatch(activityID, reqs)
if len(messages) == 0 {
return nil, nil
}
// Hold the per-activity publish lock across commit and publish so this
// batch's snapshot cannot be overtaken on the stream by a concurrent
// UpdateActivity that committed later — subscribers must see the two
// snapshots in commit order.
lock := s.publishLock(activityID)
lock.Lock()
defer lock.Unlock()
// The Writer drain cannot replay a lost batch, so retry like CompleteActivity.
var current Activity
if err := s.retryWrite(ctx, activityID, "message batch", func(tx *gorm.DB) error {
if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return errors.New("activity not found")
}
return fmt.Errorf("failed to load activity: %w", err)
}
if err := tx.Create(&messages).Error; err != nil {
return fmt.Errorf("failed to append activity message: %w", err)
}
result := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates)
if result.Error != nil {
return fmt.Errorf("failed to update activity latest message: %w", result.Error)
}
if result.RowsAffected == 0 {
return errors.New("activity not found")
}
if err := tx.First(¤t, "id = ?", activityID).Error; err != nil {
return fmt.Errorf("failed to load updated activity: %w", err)
}
return nil
}); err != nil {
return nil, err
}
out := make([]activity.Message, 0, len(messages))
for _, message := range messages {
dto := activity.Message{
ID: message.ID,
ActivityID: message.ActivityID,
Level: message.Level,
Message: message.Message,
Payload: cloneJSON(message.Payload),
CreatedAt: message.CreatedAt,
}
s.pubsub.PublishMessage(current.EnvironmentID, current.Type, dto)
out = append(out, dto)
}
s.pubsub.PublishActivity(activityToDTO(¤t))
return out, nil
}
// buildAppendBatch turns the request batch into insertable message rows
// plus the coalesced Activity column updates (latest message/progress/step from
// the last request carrying each).
func buildAppendBatch(activityID string, reqs []AppendActivityMessageRequest) ([]*ActivityMessage, map[string]any) {
now := time.Now()
messages := make([]*ActivityMessage, 0, len(reqs))
latestMessage := ""
var latestProgress *int
latestStep := ""
for _, req := range reqs {
messageText := strings.TrimSpace(req.Message)
if messageText == "" {
continue
}
if len(messageText) > 8192 {
messageText = messageText[:8192]
}
level := cmp.Or(req.Level, activity.MessageLevelInfo)
messages = append(messages, &ActivityMessage{
ActivityID: activityID,
Level: level,
Message: messageText,
Payload: cloneJSON(req.Payload),
// Spread inside the shared timestamp so the created_at sort
// used for retrieval keeps the original line order.
CreatedAt: now.Add(time.Duration(len(messages)) * time.Microsecond),
})
latestMessage = messageText
if req.Progress != nil {
latestProgress = req.Progress
}
if step := strings.TrimSpace(req.Step); step != "" {
latestStep = step
}
}
if len(messages) == 0 {
return nil, nil
}
updates := map[string]any{
"latest_message": latestMessage,
"updated_at": now,
}
if latestProgress != nil {
updates["progress"] = *clampProgressPtr(latestProgress)
}
if latestStep != "" {
updates["step"] = latestStep
}
return messages, updates
}
// retryWrite runs a write transaction, retrying transient SQLite contention
// (SQLITE_BUSY under bulk activity writes) up to three times.
func (s *ActivityService) retryWrite(ctx context.Context, activityID, what string, write func(tx *gorm.DB) error) error {
const attempts = 3
for attempt := 1; ; attempt++ {
err := s.db.WithContext(ctx).Transaction(write)
if err == nil || attempt == attempts || !dbutil.IsRetryableWriteError(err) {
return err
}
slog.WarnContext(ctx, "retrying activity write", "write", what, "activityId", activityID, "attempt", attempt, "error", err)
select {
case <-ctx.Done():
return errors.Join(err, context.Cause(ctx))
case <-time.After(250 * time.Millisecond * time.Duration(attempt)):
}
}
}
func (s *ActivityService) CompleteActivity(ctx context.Context, activityID string, status activity.Status, finalMessage string, errMessage *string, finalStep ...string) (*activity.Activity, error) {
if err := s.checkInit(); err != nil {
return nil, err
}
if !slices.Contains(terminalActivityStatuses, status) {
status = activity.StatusSuccess
}
activityID = strings.TrimSpace(activityID)
if activityID == "" {
return nil, errors.New("activity id is required")
}
// The activity is reaching a terminal state; release any cancel
// registration (a nil cause keeps an earlier ErrCanceled) and free its slot.
s.runningMu.Lock()
cancel, tracked := s.running[activityID]
delete(s.running, activityID)
s.runningMu.Unlock()
if tracked {
cancel(nil)
}
s.slots.Release(activityID)
// Detach from cancellation so the terminal write always lands — completion is
// often triggered precisely because the work context was cancelled.
ctx = context.WithoutCancel(ctx)
now := time.Now()
var model Activity
// A lost terminal write leaves the activity stuck in running forever.
if err := s.retryWrite(ctx, activityID, "terminal status", func(tx *gorm.DB) error {
if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
return fmt.Errorf("failed to load activity: %w", err)
}
updates := completeActivityUpdates(model.StartedAt, status, finalMessage, errMessage, finalStep, now)
if err := tx.Model(&Activity{}).Where("id = ?", activityID).Updates(updates).Error; err != nil {
return fmt.Errorf("failed to complete activity: %w", err)
}
if err := tx.First(&model, "id = ?", activityID).Error; err != nil {
return fmt.Errorf("failed to load completed activity: %w", err)
}
return nil
}); err != nil {
return nil, err
}
if strings.TrimSpace(finalMessage) != "" {
level := activity.MessageLevelSuccess
switch status {
case activity.StatusFailed:
level = activity.MessageLevelError
case activity.StatusCancelled:
level = activity.MessageLevelWarning
case activity.StatusQueued, activity.StatusRunning, activity.StatusSuccess:
}
activityCtx := utils.ActivityRuntimeContext(ctx, nil)
if _, err := s.AppendMessage(activityCtx, activityID, AppendActivityMessageRequest{
Level: level,
Message: finalMessage,
}); err != nil {
slog.DebugContext(ctx, "failed to append final activity message", "activityId", activityID, "error", err)
}
}
dto := s.publishTerminalSnapshot(ctx, &model)
return &dto, nil
}
// publishTerminalSnapshot publishes an activity's terminal event from
// a snapshot re-read under the per-activity publish lock. Terminal writers
// cannot hold that lock across their own transactions (CompleteActivity
// appends its final message through the locked AppendMessages path), so a
// snapshot captured in-transaction can be overtaken by an append or update
// that commits later but publishes first; re-reading under the lock
// guarantees the terminal event carries every field already streamed. The
// passed model is published as-is if the re-read fails, and is updated in
// place otherwise so callers return the published state.
func (s *ActivityService) publishTerminalSnapshot(ctx context.Context, model *Activity) activity.Activity {
lock := s.publishLock(model.ID)
lock.Lock()
defer lock.Unlock()
var fresh Activity
if err := s.db.WithContext(ctx).First(&fresh, "id = ?", model.ID).Error; err != nil {
slog.DebugContext(ctx, "failed to reload activity for terminal publish", "activityId", model.ID, "error", err)
} else {
*model = fresh
}
dto := activityToDTO(model)
s.pubsub.PublishActivity(dto)
return dto
}
// CancelActivity requests cancellation of a running or queued activity. When the
// activity's work is running in this process it interrupts it (the work finalizes
// its own terminal status); otherwise it marks the activity cancelled directly,
// but only if it is still active. Returns ErrActivityNotCancelable if the activity
// has already reached a terminal state, or gorm.ErrRecordNotFound if it is unknown.
func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, activityID, requestedBy string) (*activity.Activity, error) {
if err := s.checkInit(); err != nil {
return nil, err
}
activityID = strings.TrimSpace(activityID)
if activityID == "" {
return nil, errors.New("activity id is required")
}
environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")
var model Activity
if err := s.db.WithContext(ctx).Where("id = ? AND environment_id = ?", activityID, environmentID).First(&model).Error; err != nil {
return nil, err
}
if model.Type == activity.TypeJobRun {
return nil, ErrActivityNotCancelable
}
switch model.Status {
case activity.StatusSuccess, activity.StatusFailed, activity.StatusCancelled:
return nil, ErrActivityNotCancelable
case activity.StatusQueued, activity.StatusRunning:
// Active states — cancellation can proceed.
}
requestedBy = cmp.Or(strings.TrimSpace(requestedBy), "a user")
writeCtx := utils.ActivityRuntimeContext(ctx, nil)
if _, err := s.AppendMessage(writeCtx, activityID, AppendActivityMessageRequest{
Level: activity.MessageLevelWarning,
Message: "Cancellation requested by " + requestedBy,
}); err != nil {
slog.DebugContext(ctx, "failed to append cancellation message", "activityId", activityID, "error", err)
}
if s.RequestCancel(activityID) {
// The running work observes the cancelled context and writes its own
// terminal status, which reaches clients via the activity stream. Return
// the pre-cancel snapshot rather than reloading here: the worker has not
// finished unwinding yet, so a reload would still report "running".
return new(activityToDTO(&model)), nil
}
// Untracked work (e.g. after a process restart, or a queued activity with no
// runner): finalize directly, but only if it is still active to avoid
// clobbering a concurrently-completing activity.
now := time.Now()
var finalized Activity
if err := s.db.WithContext(writeCtx).Transaction(func(tx *gorm.DB) error {
if err := tx.First(&finalized, "id = ? AND environment_id = ?", activityID, environmentID).Error; err != nil {
return err
}
updates := completeActivityUpdates(finalized.StartedAt, activity.StatusCancelled, cancelledMessage, nil, nil, now)
result := tx.Model(&Activity{}).
Where("id = ? AND status IN ?", activityID, []activity.Status{activity.StatusQueued, activity.StatusRunning}).
Updates(updates)
if result.Error != nil {
return fmt.Errorf("failed to cancel activity: %w", result.Error)
}
if result.RowsAffected == 0 {
return ErrActivityNotCancelable
}
if err := tx.First(&finalized, "id = ? AND environment_id = ?", activityID, environmentID).Error; err != nil {
return fmt.Errorf("failed to load cancelled activity: %w", err)
}
return nil
}); err != nil {
return nil, err
}
dto := s.publishTerminalSnapshot(writeCtx, &finalized)
return &dto, nil
}
// FailStaleImageUpdateChecks marks image update checks that were left running
// across a prior process lifetime as failed. It intentionally scopes cleanup to
// old image-update-check activities so startup repair cannot affect other work.
func (s *ActivityService) FailStaleImageUpdateChecks(ctx context.Context) (int64, error) {
return s.finalizeSwept(ctx, "stale image update check",
func() ([]sweeps.Candidate, error) { return s.sweeps.StaleImageUpdateChecks(ctx) },
func(sweeps.Candidate) (activity.Status, string) {
return activity.StatusFailed, "Image update check failed because it was stale after Arcane restarted"
},
"Image update check failed",
)
}
// finalizeSwept completes each swept candidate with the status and message
// resolve picks; failed ones also record the message as their error.
func (s *ActivityService) finalizeSwept(
ctx context.Context,
label string,
find func() ([]sweeps.Candidate, error),
resolve func(sweeps.Candidate) (activity.Status, string),
finalStep ...string,
) (int64, error) {
candidates, err := find()
if err != nil {
return 0, fmt.Errorf("find %s activities: %w", label, err)
}
var finalized int64
var errs []error
for _, candidate := range candidates {
status, message := resolve(candidate)
errMessage := kit.Ternary(status == activity.StatusFailed, new(message), nil)
if _, completeErr := s.CompleteActivity(ctx, candidate.ID, status, message, errMessage, finalStep...); completeErr != nil {
errs = append(errs, fmt.Errorf("finalize %s activity %s: %w", label, candidate.ID, completeErr))
continue
}
finalized++
}
return finalized, errors.Join(errs...)
}
// FailAbandonedActivities marks queued/running activities whose worker is no
// longer alive in this process as failed, releasing any concurrency slot they
// still hold. Liveness comes from the running map, not age: every creation
// path Tracks its activity right after StartActivity, and the short grace
// period covers that create→Track window. This assumes exactly one Arcane
// process owns the database (managers and agents each own theirs); running
// multiple replicas against one database would make each replica sweep the
// other's live work.
//
// The terminal write is status-guarded like CancelActivity's untracked
// fallback: a worker completing concurrently wins the race and the row is
// skipped here.
func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, error) {
activeStatuses := []activity.Status{activity.StatusQueued, activity.StatusRunning}
s.runningMu.Lock()
trackedIDs := slices.Collect(maps.Keys(s.running))
s.runningMu.Unlock()
candidates, findErr := s.sweeps.Abandoned(ctx, trackedIDs)
if findErr != nil {
return 0, fmt.Errorf("find abandoned activities: %w", findErr)
}
// A worker may have Tracked its activity since the snapshot above.
s.runningMu.Lock()
candidates = slices.DeleteFunc(candidates, func(candidate sweeps.Candidate) bool {
_, tracked := s.running[candidate.ID]
return tracked
})
s.runningMu.Unlock()
if len(candidates) == 0 {
return 0, nil
}
const message = "Activity was marked failed because its worker is no longer running"
now := time.Now()
ids := make([]string, 0, len(candidates))
var durationExpr strings.Builder
durationArgs := make([]any, 0, 2*len(candidates))
durationExpr.WriteString("CASE id")
for _, candidate := range candidates {
ids = append(ids, candidate.ID)
durationExpr.WriteString(" WHEN ? THEN ?")
durationArgs = append(durationArgs, candidate.ID, now.Sub(candidate.StartedAt).Milliseconds())
}
durationExpr.WriteString(" ELSE duration_ms END")
updates := completeActivityUpdates(now, activity.StatusFailed, message, new(message), nil, now)
updates["duration_ms"] = gorm.Expr(durationExpr.String(), durationArgs...)
// Status-guarded so a worker completing concurrently wins; only rows this
// sweep failed carry its message, so the reload skips the race losers.
var finalized []Activity
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if err := tx.Model(&Activity{}).
Where("id IN ? AND status IN ?", ids, activeStatuses).
Updates(updates).Error; err != nil {
return fmt.Errorf("fail abandoned activities: %w", err)
}
if err := tx.Where("id IN ? AND status = ? AND error = ?", ids, activity.StatusFailed, message).Find(&finalized).Error; err != nil {
return fmt.Errorf("load failed abandoned activities: %w", err)
}
return nil
}); err != nil {
return 0, fmt.Errorf("sweep abandoned activities: %w", err)
}
for i := range finalized {
s.slots.Release(finalized[i].ID)
s.publishTerminalSnapshot(ctx, &finalized[i])
}
return int64(len(finalized)), nil
}
// ResolveOrphanedQueuedActivities fails any activity still queued at startup.
// Queued state is owned by a live goroutine blocked on AwaitActivitySlot, so a
// queued row after a restart can never start running.
func (s *ActivityService) ResolveOrphanedQueuedActivities(ctx context.Context, protectedIDs ...string) (int64, error) {
return s.finalizeSwept(ctx, "orphaned queued",
func() ([]sweeps.Candidate, error) { return s.sweeps.OrphanedQueued(ctx, protectedIDs) },
func(sweeps.Candidate) (activity.Status, string) {
return activity.StatusFailed, "Queued activity was interrupted by an Arcane restart"
},
)
}
// PatchActivityMetadata merges patch into the activity's existing metadata,
// unlike UpdateActivity which replaces the metadata wholesale.
func (s *ActivityService) PatchActivityMetadata(ctx context.Context, activityID string, patch database.JSON) error {
if err := s.checkInit(); err != nil {
return err
}
activityID = strings.TrimSpace(activityID)
if activityID == "" {
return errors.New("activity id is required")
}
if len(patch) == 0 {
return nil
}
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var record Activity
if err := tx.First(&record, "id = ?", activityID).Error; err != nil {
return fmt.Errorf("failed to load activity: %w", err)
}
merged := cloneJSON(record.Metadata)
if merged == nil {
merged = database.JSON{}
}
maps.Copy(merged, patch)
if err := tx.Model(&Activity{}).Where("id = ?", activityID).
Updates(map[string]any{"metadata": merged, "updated_at": time.Now()}).Error; err != nil {
return fmt.Errorf("failed to patch activity metadata: %w", err)
}
return nil
})
}
// ResolveStaleAutoUpdateActivities finalizes auto-update activities left
// running by a prior process lifetime. A run whose metadata marks a triggered
// self-update completed by restarting Arcane, so it is recorded as success;
// anything else still running at startup was interrupted and is failed.
func (s *ActivityService) ResolveStaleAutoUpdateActivities(ctx context.Context, protectedIDs ...string) (int64, error) {
return s.finalizeSwept(ctx, "stale auto-update",
func() ([]sweeps.Candidate, error) { return s.sweeps.StaleAutoUpdates(ctx, protectedIDs) },
func(candidate sweeps.Candidate) (activity.Status, string) {
if selfUpdate, _ := candidate.Metadata["selfUpdateTriggered"].(bool); selfUpdate {
return activity.StatusSuccess, "Auto-update completed — Arcane restarted with the updated image"
}
return activity.StatusFailed, "Auto-update interrupted by Arcane restart"
},
)
}
func completeActivityUpdates(startedAt time.Time, status activity.Status, finalMessage string, errMessage *string, finalStep []string, now time.Time) map[string]any {
updates := map[string]any{
"status": status,
"ended_at": now,
"duration_ms": now.Sub(startedAt).Milliseconds(),
"updated_at": now,
}
if trimmed := strings.TrimSpace(finalMessage); trimmed != "" {
updates["latest_message"] = trimmed
}
if len(finalStep) > 0 {
if step := strings.TrimSpace(finalStep[0]); step != "" {
updates["step"] = step
}
}
if errMessage != nil && strings.TrimSpace(*errMessage) != "" {
updates["error"] = strings.TrimSpace(*errMessage)
}
if status == activity.StatusSuccess {
updates["progress"] = 100
}
return updates
}
func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environmentID string, params pagination.QueryParams) ([]activity.Activity, pagination.Response, error) {
if err := s.checkInit(); err != nil {
return nil, pagination.Response{}, err
}
environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")
var activities []Activity
q := s.db.WithContext(ctx).Model(&Activity{}).Where("environment_id = ?", environmentID)
q = scopeJobActivityVisibility(ctx, q, environmentID, authz.PermActivitiesRead)
if term := strings.TrimSpace(params.Search); term != "" {
escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(term)
searchPattern := "%" + escaped + "%"
// Matches the Activity Center's own search fields.
q = q.Where(
"type LIKE ? ESCAPE '\\' OR COALESCE(resource_name, '') LIKE ? ESCAPE '\\' OR "+
"COALESCE(latest_message, '') LIKE ? ESCAPE '\\' OR COALESCE(error, '') LIKE ? ESCAPE '\\'",
searchPattern, searchPattern, searchPattern, searchPattern,
)
}
q = pagination.ApplyFilter(q, "status", params.Filters["status"])
q = pagination.ApplyFilter(q, "type", params.Filters["type"])
q = pagination.ApplyFilter(q, "resource_type", params.Filters["resourceType"])
if params.Sort == "" {
// Active rows sort by created_at (immutable) and terminal rows by ended_at
// (set once), so a row's position only changes on the active->terminal
// transition instead of on every progress update.
q = q.Order("CASE WHEN status IN ('queued', 'running') THEN 0 ELSE 1 END ASC").
Order("COALESCE(ended_at, created_at) DESC").
Order("id DESC")
}
paginationResp, err := pagination.PaginateAndSortDB(params, q, &activities)
if err != nil {
return nil, pagination.Response{}, fmt.Errorf("failed to paginate activities: %w", err)
}
out := make([]activity.Activity, 0, len(activities))
for i := range activities {
out = append(out, activityToDTO(&activities[i]))
}
return out, paginationResp, nil
}
func (s *ActivityService) GetActivityDetail(ctx context.Context, environmentID, activityID string, limit int) (*activity.Detail, error) {
if err := s.checkInit(); err != nil {
return nil, err
}
if limit <= 0 || limit > defaultActivityMessages {
limit = defaultActivityMessages
}
var model Activity
if err := s.db.WithContext(ctx).
Where("id = ? AND environment_id = ?", activityID, environmentID).
First(&model).Error; err != nil {
return nil, fmt.Errorf("failed to load activity: %w", err)
}
var messages []ActivityMessage
if err := s.db.WithContext(ctx).
Where("activity_id = ?", activityID).
Order("created_at DESC").
Limit(limit).
Find(&messages).Error; err != nil {
return nil, fmt.Errorf("failed to load activity messages: %w", err)
}
outMessages := make([]activity.Message, 0, len(messages))
for _, message := range slices.Backward(messages) {
outMessages = append(outMessages, activity.Message{
ID: message.ID,
ActivityID: message.ActivityID,
Level: message.Level,
Message: message.Message,
Payload: cloneJSON(message.Payload),
CreatedAt: message.CreatedAt,
})
}
return &activity.Detail{
Activity: activityToDTO(&model),
Messages: outMessages,
}, nil
}
func (s *ActivityService) PruneHistory(ctx context.Context, retentionDays, maxEntries int) (int64, error) {
if retentionDays < 0 {
retentionDays = defaultActivityRetentionDays
}
if maxEntries < 0 {
maxEntries = defaultActivityHistoryLimit
}
var deleted int64
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
if retentionDays > 0 {
cutoff := time.Now().Add(-time.Duration(retentionDays) * 24 * time.Hour)
var ids []string
if err := tx.Model(&Activity{}).
Where("status IN ? AND COALESCE(ended_at, updated_at, created_at) < ?", terminalActivityStatuses, cutoff).
Pluck("id", &ids).Error; err != nil {
return fmt.Errorf("failed to find activities older than retention window: %w", err)
}
count, err := deleteActivitiesByID(tx, ids)
if err != nil {
return err
}
deleted += count
}
if maxEntries > 0 {
var ids []string
if err := tx.Raw(`
SELECT ranked.id
FROM (
SELECT id,
ROW_NUMBER() OVER (
PARTITION BY environment_id
ORDER BY COALESCE(ended_at, updated_at, created_at) DESC, started_at DESC
) AS activity_rank
FROM activities
WHERE status IN ?
) ranked
WHERE ranked.activity_rank > ?
`, terminalActivityStatuses, maxEntries).Scan(&ids).Error; err != nil {
return fmt.Errorf("failed to find excess activities: %w", err)
}
count, err := deleteActivitiesByID(tx, ids)
if err != nil {
return err
}
deleted += count
}
return nil
}); err != nil {
return 0, err
}
return deleted, nil
}
func (s *ActivityService) DeleteHistory(ctx context.Context, environmentID string) (int64, error) {
environmentID = cmp.Or(strings.TrimSpace(environmentID), "0")
var deleted int64
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var ids []string
query := tx.Model(&Activity{}).Where("environment_id = ? AND status IN ?", environmentID, terminalActivityStatuses)
if err := scopeJobActivityVisibility(ctx, query, environmentID, authz.PermActivitiesDelete).Pluck("id", &ids).Error; err != nil {
return fmt.Errorf("failed to find activity history: %w", err)
}
count, err := deleteActivitiesByID(tx, ids)
if err != nil {
return err
}
deleted = count
return nil
}); err != nil {
return 0, err
}
return deleted, nil
}
func (s *ActivityService) publishLock(activityID string) *sync.Mutex {
h := fnv.New32a()
_, _ = h.Write([]byte(activityID))
return &s.publishLocks[h.Sum32()%uint32(len(s.publishLocks))]
}
func activityToDTO(model *Activity) activity.Activity {
if model == nil {
return activity.Activity{}
}
return activity.Activity{
ID: model.ID,
EnvironmentID: model.EnvironmentID,
SourceEnvironmentID: model.EnvironmentID,
BatchID: copyPtr(model.BatchID),
Type: model.Type,
Status: model.Status,
ResourceType: copyPtr(model.ResourceType),
ResourceID: copyPtr(model.ResourceID),
ResourceName: copyPtr(model.ResourceName),
Progress: clampProgressPtr(model.Progress),
Step: model.Step,
LatestMessage: model.LatestMessage,
StartedBy: func() *activity.StartedBy {
username := strings.TrimSpace(kit.FromPtr(model.StartedByUsername))
if username == "" {
return &activity.StartedBy{Username: "System"}
}
return &activity.StartedBy{
Username: username,
UserID: strings.TrimSpace(kit.FromPtr(model.StartedByUserID)),
DisplayName: strings.TrimSpace(kit.FromPtr(model.StartedByDisplayName)),
}
}(),
StartedAt: model.StartedAt,
EndedAt: copyPtr(model.EndedAt),
DurationMs: copyPtr(model.DurationMs),
Error: copyPtr(model.Error),
Metadata: cloneJSON(model.Metadata),
CreatedAt: model.CreatedAt,
UpdatedAt: copyPtr(model.UpdatedAt),
}
}
func copyPtr[T any](value *T) *T {
if value == nil {
return nil
}
return new(*value)
}
func clampProgressPtr(value *int) *int {
if value == nil {
return nil
}
return new(min(max(*value, 0), 100))
}
func cloneJSON(input database.JSON) database.JSON {
if len(input) == 0 {
return nil
}
out := make(database.JSON, len(input))
maps.Copy(out, input)
return out
}
var terminalActivityStatuses = []activity.Status{
activity.StatusSuccess,
activity.StatusFailed,
activity.StatusCancelled,
}
func deleteActivitiesByID(tx *gorm.DB, activityIDs []string) (int64, error) {
if len(activityIDs) == 0 {
return 0, nil
}
var totalDeleted int64
for i := 0; i < len(activityIDs); i += deleteActivitiesBatchSize {
end := min(i+deleteActivitiesBatchSize, len(activityIDs))
batch := activityIDs[i:end]
if err := tx.Where("activity_id IN ?", batch).Delete(&ActivityMessage{}).Error; err != nil {
return totalDeleted, fmt.Errorf("failed to delete activity messages: %w", err)
}
result := tx.Where("id IN ?", batch).Delete(&Activity{})
if result.Error != nil {
return totalDeleted, fmt.Errorf("failed to delete activities: %w", result.Error)
}
totalDeleted += result.RowsAffected
}
return totalDeleted, nil
}
// FailInterruptedBackups finalizes backup activities before startup admits work.
func (s *ActivityService) FailInterruptedBackups(ctx context.Context, protectedIDs ...string) error {
_, err := s.finalizeSwept(ctx, "interrupted backup",
func() ([]sweeps.Candidate, error) {
pending, err := s.sweeps.InterruptedBackups(ctx, protectedIDs)
s.runningMu.Lock()
defer s.runningMu.Unlock()
return slices.DeleteFunc(pending, func(candidate sweeps.Candidate) bool {
_, tracked := s.running[candidate.ID]
return tracked
}), err
},
func(sweeps.Candidate) (activity.Status, string) {
return activity.StatusFailed, "Backup interrupted by Arcane restart"
},
)
return err
}
// SyncJobRun updates a durable job summary without taking an execution slot.
// Only the queue owns its lifecycle, including reopening it for an explicit retry.
func (s *ActivityService) SyncJobRun(ctx context.Context, run scheduler.Run, name string) error {
if err := s.checkInit(); err != nil {
return err
}
if run.ActivityID == "" {
return errors.New("job activity ID is required")
}
lock := s.publishLock(run.ActivityID)
lock.Lock()
defer lock.Unlock()
model := jobActivity(run, name)
changed := false
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var existing Activity
err := tx.First(&existing, "id = ?", run.ActivityID).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
changed = true
return tx.Create(&model).Error
}
if err != nil {
return err
}
if existing.Type != activity.TypeJobRun {
return errors.New("job activity identity belongs to another operation")
}
if stamp, ok := existing.Metadata["runUpdatedAt"].(string); ok {
previous, parseErr := time.Parse(time.RFC3339Nano, stamp)
if parseErr == nil && !previous.Before(run.UpdatedAt) {
if existing.EnvironmentID == model.EnvironmentID {
return nil
}
existing.EnvironmentID = model.EnvironmentID
model = existing
}
}
changed = true
return tx.Table("activities").Where("id = ?", run.ActivityID).Updates(map[string]any{
"environment_id": model.EnvironmentID,
"status": model.Status, "step": model.Step, "latest_message": model.LatestMessage,
"ended_at": model.EndedAt, "duration_ms": model.DurationMs, "error": model.Error,
"metadata": model.Metadata, "updated_at": model.UpdatedAt,
}).Error
}); err != nil {
return fmt.Errorf("synchronize job activity: %w", err)
}
if changed {
// Ordinary activities cannot reopen. Job summaries can, after an explicit
// retry; the publish lock and durable timestamp reject stale snapshots.
s.pubsub.Reopen(model.ID)
s.pubsub.PublishActivity(activityToDTO(&model))
}
return nil
}
func (s *ActivityService) ResolveOrphanedJobRuns(ctx context.Context, finished map[string]scheduler.Run, protectedIDs ...string) (int64, error) {
if err := s.checkInit(); err != nil {
return 0, err
}
candidates, err := s.sweeps.OrphanedJobRuns(ctx, protectedIDs)
if err != nil {
return 0, fmt.Errorf("find orphaned job run activities: %w", err)
}
var resolved int64
var errs []error
for _, candidate := range candidates {
finalized, resolveErr := s.resolveOrphanedJobRun(ctx, candidate.ID, finished)
if resolveErr != nil {
errs = append(errs, fmt.Errorf("finalize orphaned job run activity %s: %w", candidate.ID, resolveErr))
continue
}
if finalized {
resolved++
}
}
return resolved, errors.Join(errs...)
}
func (s *ActivityService) resolveOrphanedJobRun(ctx context.Context, activityID string, finished map[string]scheduler.Run) (bool, error) {
lock := s.publishLock(activityID)
lock.Lock()
defer lock.Unlock()
var model Activity
var affected int64
if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var existing Activity
if err := tx.First(&existing, "id = ?", activityID).Error; err != nil {
return err
}
run, found := finished[activityID]
if !found {
trigger, _ := existing.Metadata["trigger"].(string)
domainActivityID, _ := existing.Metadata["domainActivityId"].(string)
run = scheduler.Run{
ActivityID: existing.ID,
ID: kit.FromPtr(existing.BatchID),
JobID: kit.FromPtr(existing.ResourceID),
EnvironmentID: existing.EnvironmentID,
Trigger: trigger,
RequestedBy: kit.FromPtr(existing.StartedByUserID),
Status: scheduler.Failed,
CreatedAt: existing.StartedAt,
UpdatedAt: time.Now().UTC(),
Outcome: scheduler.Outcome{
Status: scheduler.Failed,
Message: "Job run no longer exists; its result was not recorded",
ActivityID: domainActivityID,
},
}
}
model = jobActivity(run, kit.FromPtr(existing.ResourceName))
result := tx.Table("activities").
Where("id = ? AND status IN ?", activityID, []activity.Status{activity.StatusQueued, activity.StatusRunning}).
Updates(map[string]any{
"environment_id": model.EnvironmentID,
"status": model.Status,
"step": model.Step,
"latest_message": model.LatestMessage,
"ended_at": model.EndedAt,
"duration_ms": model.DurationMs,
"error": model.Error,
"metadata": model.Metadata,
"updated_at": model.UpdatedAt,
})
affected = result.RowsAffected
return result.Error
}); err != nil {
return false, err
}
if affected == 0 {
return false, nil
}
s.pubsub.PublishActivity(activityToDTO(&model))
return true, nil
}
func jobActivity(run scheduler.Run, name string) Activity {
status := activity.StatusQueued
switch run.Status {
case scheduler.Running:
status = activity.StatusRunning
case scheduler.Succeeded, scheduler.Skipped:
status = activity.StatusSuccess
case scheduler.Failed, scheduler.Partial, scheduler.NeedsAttention:
status = activity.StatusFailed
case scheduler.Canceled:
status = activity.StatusCancelled
case scheduler.Queued, scheduler.Waiting, scheduler.Retrying:
}
message := cmp.Or(run.Outcome.Message, name+": "+string(run.Status))
terminal := slices.Contains(terminalActivityStatuses, status)
resolvedByUser := run.Resolution != nil && run.Resolution.ResolvedBy != user.SystemUser.Username
return Activity{
ID: run.ActivityID,
CreatedAt: run.CreatedAt,
UpdatedAt: new(run.UpdatedAt),
EnvironmentID: run.EnvironmentID,
BatchID: new(run.ID),
Type: activity.TypeJobRun,
Status: status,
ResourceType: new("job"),
ResourceID: new(run.JobID),
ResourceName: new(name),
StartedByUserID: mo.EmptyableToOption(run.RequestedBy).ToPointer(),
StartedByUsername: mo.EmptyableToOption(run.RequestedBy).ToPointer(),
StartedAt: run.CreatedAt,
Step: string(run.Status),
LatestMessage: kit.Ternary(resolvedByUser, "Resolved after review", message),
EndedAt: kit.Ternary(terminal, new(run.UpdatedAt), nil),
DurationMs: kit.Ternary(terminal, new(max(int64(0), run.UpdatedAt.Sub(run.CreatedAt).Milliseconds())), nil),
Error: kit.Ternary(status == activity.StatusFailed, new(message), nil),
Metadata: func() database.JSON {
metadata := database.JSON{
"jobId": run.JobID,
"runId": run.ID,
"environmentId": run.EnvironmentID,
"jobStatus": string(run.Status),
"trigger": run.Trigger,
"attemptCount": run.AttemptCount,
"runUpdatedAt": run.UpdatedAt.Format(time.RFC3339Nano),
"domainActivityId": run.Outcome.ActivityID,
}
if run.Resolution != nil {
metadata["resolution"] = run.Resolution
}
return metadata
}(),
}
}
// remoteActivityPageCeiling mirrors the agent's page clamp in PaginateAndSortDB.
const remoteActivityPageCeiling = 100
// normalizeRemoteActivityParams applies the agent's paging bounds so both
// sources are cut the same way; -1 stays the explicit "show all" sentinel.
func normalizeRemoteActivityParams(params pagination.QueryParams) pagination.QueryParams {
params.Start = max(params.Start, 0)
switch {
case params.Limit == -1:
case params.Limit <= 0:
params.Limit = 20
case params.Limit > remoteActivityPageCeiling:
params.Limit = remoteActivityPageCeiling
}
return params
}
// collectActivityWindow reads window rows from a source in pages the
// agent accepts, stopping early when the source runs out. -1 reads everything.
func collectActivityWindow(window int, fetchPage func(start, limit int) ([]activity.Activity, int64, error)) ([]activity.Activity, int64, error) {
if window == -1 {
return fetchPage(0, -1)
}
var rows []activity.Activity
var total int64
for len(rows) < window {
limit := min(remoteActivityPageCeiling, window-len(rows))
page, pageTotal, err := fetchPage(len(rows), limit)
if err != nil {
return nil, 0, err
}
rows = append(rows, page...)
total = pageTotal
if len(page) < limit {
break
}
}
return rows, total, nil
}
// ListRemoteActivities combines manager-owned summaries with agent-owned work
// before slicing the page. The remote records must already match the filters
// and be the agent's newest window; remoteTotal is the agent's filtered count.
func (s *ActivityService) ListRemoteActivities(
ctx context.Context,
environmentID string,
remote []activity.Activity,
remoteTotal int64,
params pagination.QueryParams,
) ([]activity.Activity, pagination.Response, error) {
params = normalizeRemoteActivityParams(params)
window := kit.Ternary(params.Limit == -1, -1, params.Start+params.Limit)
activities, localTotal, err := collectActivityWindow(window, func(start, limit int) ([]activity.Activity, int64, error) {
local := params
local.Start, local.Limit = start, limit
rows, page, err := s.ListActivitiesPaginated(ctx, environmentID, local)
if err != nil {
return nil, 0, err
}
return rows, page.TotalItems, nil
})
if err != nil {
return nil, pagination.Response{}, err
}
for _, item := range remote {
item.EnvironmentID = environmentID
activities = append(activities, item)
}
total := remoteTotal + localTotal
config := pagination.Config[activity.Activity]{SortBindings: []pagination.SortBinding[activity.Activity]{{
Fn: func(a, b activity.Activity) int { return compareActivities(a, b, params) },
}}}
pageParams := params
pageParams.Order = pagination.SortAsc
activities = config.OrderAndPaginate(activities, pageParams)
return activities, pagination.BuildResponse(total, 0, params), nil
}
func compareActivities(a, b activity.Activity, params pagination.QueryParams) int {
if params.Sort == "" {
aActive := a.Status == activity.StatusQueued || a.Status == activity.StatusRunning
bActive := b.Status == activity.StatusQueued || b.Status == activity.StatusRunning
if aActive != bActive {
return kit.Ternary(aActive, -1, 1)
}
aTime, bTime := a.CreatedAt, b.CreatedAt
if a.EndedAt != nil {
aTime = *a.EndedAt
}
if b.EndedAt != nil {
bTime = *b.EndedAt
}
return cmp.Or(bTime.Compare(aTime), strings.Compare(b.ID, a.ID))
}
var result int
switch params.Sort {
case "environmentId":
result = strings.Compare(a.EnvironmentID, b.EnvironmentID)
case "type":
result = cmp.Compare(a.Type, b.Type)
case "status":
result = cmp.Compare(a.Status, b.Status)
case "resourceType":
result = strings.Compare(kit.FromPtr(a.ResourceType), kit.FromPtr(b.ResourceType))
case "resourceName":
result = strings.Compare(kit.FromPtr(a.ResourceName), kit.FromPtr(b.ResourceName))
case "startedAt":
result = a.StartedAt.Compare(b.StartedAt)
case "createdAt":
result = a.CreatedAt.Compare(b.CreatedAt)
case "updatedAt":
result = kit.FromPtr(a.UpdatedAt).Compare(kit.FromPtr(b.UpdatedAt))
case "endedAt":
result = kit.FromPtr(a.EndedAt).Compare(kit.FromPtr(b.EndedAt))
case "durationMs":
result = cmp.Compare(kit.FromPtr(a.DurationMs), kit.FromPtr(b.DurationMs))
}
if params.Order == pagination.SortDesc {
result = -result
}
return cmp.Or(result, strings.Compare(a.ID, b.ID))
}