Skip to content

Modern Docker Management, Designed for Everyone

38.7 KB
Raw Blame
package activity

import (
	"context"
	"net/http"
	"net/http/httptest"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/getarcaneapp/arcane/types/v2/activity"
	"github.com/getarcaneapp/arcane/types/v2/scheduler"
	"github.com/getarcaneapp/arcane/types/v2/user"
	"github.com/libtnb/sqlite"
	"github.com/stretchr/testify/require"
	"go.getarcane.app/sys/crypto"
	"gorm.io/gorm"

	"github.com/getarcaneapp/arcane/backend/v2/internal/config"
	"github.com/getarcaneapp/arcane/backend/v2/internal/database"
	"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
	"github.com/getarcaneapp/arcane/backend/v2/internal/gitrepo"
	"github.com/getarcaneapp/arcane/backend/v2/internal/middleware"
	"github.com/getarcaneapp/arcane/backend/v2/internal/registry"
	"github.com/getarcaneapp/arcane/backend/v2/internal/s3"
	"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/authz"
	activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
)

func setupActivityServiceTestDB(t *testing.T) *database.DB {
	t.Helper()

	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
	require.NoError(t, err)
	require.NoError(t, db.AutoMigrate(&Activity{}, &ActivityMessage{}))
	return &database.DB{DB: db}
}

func TestActivityServiceLifecycle(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	progress := 5
	startedBy := &user.Actor{
		ID:          "user-1",
		Username:    "arcane",
		DisplayName: new("Arcane Admin"),
	}
	created, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImagePull,
		ResourceType:  new("image"),
		ResourceID:    new("img-123"),
		ResourceName:  new("nginx:latest"),
		StartedBy:     startedBy,
		Progress:      &progress,
		Step:          "queued",
		LatestMessage: "Pull queued",
	})
	require.NoError(t, err)
	require.NotEmpty(t, created.ID)
	require.Equal(t, "0", created.EnvironmentID)
	require.Equal(t, "running", string(created.Status))
	require.Equal(t, 5, *created.Progress)
	require.NotNil(t, created.StartedBy)
	require.Equal(t, "user-1", created.StartedBy.UserID)
	require.Equal(t, "arcane", created.StartedBy.Username)
	require.Equal(t, "Arcane Admin", created.StartedBy.DisplayName)

	progress = 42
	message, err := service.AppendMessage(ctx, created.ID, AppendActivityMessageRequest{
		Level:    activity.MessageLevelInfo,
		Message:  "Downloading layers",
		Progress: &progress,
		Step:     "download",
	})
	require.NoError(t, err)
	require.NotNil(t, message)
	require.Equal(t, created.ID, message.ActivityID)

	completed, err := service.CompleteActivity(ctx, created.ID, activity.StatusSuccess, "Pull complete", nil)
	require.NoError(t, err)
	require.Equal(t, "success", string(completed.Status))
	require.NotNil(t, completed.EndedAt)
	require.NotNil(t, completed.DurationMs)
	require.Equal(t, 100, *completed.Progress)

	list, paginationResp, err := service.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{
		Limit: 10,
	})
	require.NoError(t, err)
	require.Len(t, list, 1)
	require.Equal(t, int64(1), paginationResp.TotalItems)
	require.Equal(t, created.ID, list[0].ID)

	detail, err := service.GetActivityDetail(ctx, "0", created.ID, 10)
	require.NoError(t, err)
	require.Equal(t, created.ID, detail.Activity.ID)
	require.Len(t, detail.Messages, 2)
	require.Equal(t, "Downloading layers", detail.Messages[0].Message)
	require.Equal(t, "Pull complete", detail.Messages[1].Message)
}

func TestActivityServiceStreamFanout(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	events, _, unsubscribe := service.pubsub.Subscribe("0")
	defer unsubscribe()
	otherEvents, _, unsubscribeOther := service.pubsub.Subscribe("0")
	defer unsubscribeOther()

	created, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeProjectDeploy,
		LatestMessage: "Deploy queued",
	})
	require.NoError(t, err)

	first := receiveActivityEvent(t, events)
	require.Equal(t, "activity", first.Type)
	require.Equal(t, created.ID, first.ActivityID)
	require.NotNil(t, first.Activity)
	other := receiveActivityEvent(t, otherEvents)
	require.Equal(t, first, other)
	require.NotSame(t, first.Activity, other.Activity)

	_, err = service.AppendMessage(ctx, created.ID, AppendActivityMessageRequest{
		Level:   activity.MessageLevelInfo,
		Message: "Deploying services",
		Step:    "deploy",
	})
	require.NoError(t, err)

	messageEvent := receiveActivityEvent(t, events)
	require.Equal(t, "message", messageEvent.Type)
	require.Equal(t, created.ID, messageEvent.ActivityID)
	require.Equal(t, activity.TypeProjectDeploy, messageEvent.ActivityType)
	require.NotNil(t, messageEvent.Message)
	require.Equal(t, "Deploying services", messageEvent.Message.Message)
	require.Nil(t, messageEvent.Activity)
	require.Nil(t, messageEvent.Activities)
	require.Equal(t, messageEvent, receiveActivityEvent(t, otherEvents))
}

func TestActivityServiceRetentionCleanup(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	created, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeSystemPrune,
		LatestMessage: "Prune started",
	})
	require.NoError(t, err)
	_, err = service.AppendMessage(ctx, created.ID, AppendActivityMessageRequest{
		Message: "Removing unused resources",
	})
	require.NoError(t, err)
	_, err = service.CompleteActivity(ctx, created.ID, activity.StatusSuccess, "Prune complete", nil)
	require.NoError(t, err)

	oldEndedAt := time.Now().Add(-((time.Duration(defaultActivityRetentionDays) * 24 * time.Hour) + time.Hour))
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", created.ID).Update("ended_at", oldEndedAt).Error)

	deleted, err := service.PruneHistory(ctx, defaultActivityRetentionDays, 0)
	require.NoError(t, err)
	require.EqualValues(t, 1, deleted)

	var activityCount int64
	require.NoError(t, db.Model(&Activity{}).Count(&activityCount).Error)
	require.Zero(t, activityCount)

	var messageCount int64
	require.NoError(t, db.Model(&ActivityMessage{}).Count(&messageCount).Error)
	require.Zero(t, messageCount)
}

func TestActivityServicePruneHistoryZeroRetentionDisablesAgeCleanup(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	created, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeSystemPrune,
		LatestMessage: "Prune started",
	})
	require.NoError(t, err)
	_, err = service.CompleteActivity(ctx, created.ID, activity.StatusSuccess, "Prune complete", nil)
	require.NoError(t, err)

	oldEndedAt := time.Now().Add(-((time.Duration(defaultActivityRetentionDays) * 24 * time.Hour) + time.Hour))
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", created.ID).Update("ended_at", oldEndedAt).Error)

	deleted, err := service.PruneHistory(ctx, 0, 0)
	require.NoError(t, err)
	require.Zero(t, deleted)

	var activityCount int64
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", created.ID).Count(&activityCount).Error)
	require.EqualValues(t, 1, activityCount)
}

func TestActivityServiceDeleteHistoryPreservesActiveActivities(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	completed, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeResourceAction})
	require.NoError(t, err)
	_, err = service.AppendMessage(ctx, completed.ID, AppendActivityMessageRequest{Message: "done"})
	require.NoError(t, err)
	_, err = service.CompleteActivity(ctx, completed.ID, activity.StatusSuccess, "complete", nil)
	require.NoError(t, err)

	running, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeResourceAction})
	require.NoError(t, err)

	remoteCompleted, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "remote-1", Type: activity.TypeResourceAction})
	require.NoError(t, err)
	_, err = service.CompleteActivity(ctx, remoteCompleted.ID, activity.StatusFailed, "failed", nil)
	require.NoError(t, err)

	deleted, err := service.DeleteHistory(ctx, "0")
	require.NoError(t, err)
	require.EqualValues(t, 1, deleted)

	var remaining []Activity
	require.NoError(t, db.Order("id").Find(&remaining).Error)
	require.Len(t, remaining, 2)
	require.ElementsMatch(t, []string{running.ID, remoteCompleted.ID}, []string{remaining[0].ID, remaining[1].ID})
}

func TestActivityServicePruneHistoryByAgeAndCount(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	oldActivity, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeResourceAction})
	require.NoError(t, err)
	_, err = service.CompleteActivity(ctx, oldActivity.ID, activity.StatusSuccess, "old", nil)
	require.NoError(t, err)
	oldTime := time.Now().Add(-48 * time.Hour)
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", oldActivity.ID).Updates(map[string]any{
		"ended_at":   oldTime,
		"updated_at": oldTime,
	}).Error)

	for i := range 3 {
		item, startErr := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "remote-1", Type: activity.TypeResourceAction})
		require.NoError(t, startErr)
		_, completeErr := service.CompleteActivity(ctx, item.ID, activity.StatusSuccess, "done", nil)
		require.NoError(t, completeErr)
		stamp := time.Now().Add(time.Duration(i) * time.Minute)
		require.NoError(t, db.Model(&Activity{}).Where("id = ?", item.ID).Updates(map[string]any{
			"ended_at":   stamp,
			"updated_at": stamp,
		}).Error)
	}

	running, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "remote-1", Type: activity.TypeResourceAction})
	require.NoError(t, err)

	deleted, err := service.PruneHistory(ctx, 1, 2)
	require.NoError(t, err)
	require.EqualValues(t, 2, deleted)

	var terminalRemoteCount int64
	require.NoError(t, db.Model(&Activity{}).
		Where("environment_id = ? AND status IN ?", "remote-1", terminalActivityStatuses).
		Count(&terminalRemoteCount).Error)
	require.EqualValues(t, 2, terminalRemoteCount)

	var runningCount int64
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", running.ID).Count(&runningCount).Error)
	require.EqualValues(t, 1, runningCount)

	var oldCount int64
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", oldActivity.ID).Count(&oldCount).Error)
	require.Zero(t, oldCount)
}

func TestActivityServiceListOrderStableUnderProgressUpdates(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	older, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull})
	require.NoError(t, err)
	newer, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull})
	require.NoError(t, err)
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", older.ID).
		Update("created_at", time.Now().Add(-time.Minute)).Error)

	terminal, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull})
	require.NoError(t, err)
	_, err = service.CompleteActivity(ctx, terminal.ID, activity.StatusSuccess, "done", nil)
	require.NoError(t, err)

	listIDs := func() []string {
		list, _, listErr := service.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{
			Limit: 10,
		})
		require.NoError(t, listErr)
		ids := make([]string, 0, len(list))
		for _, item := range list {
			ids = append(ids, item.ID)
		}
		return ids
	}

	expected := []string{newer.ID, older.ID, terminal.ID}
	require.Equal(t, expected, listIDs())

	progress := 50
	_, err = service.UpdateActivity(ctx, older.ID, UpdateActivityRequest{Progress: &progress, LatestMessage: new("halfway")})
	require.NoError(t, err)
	require.Equal(t, expected, listIDs())
}

func setupQueuedActivityService(t *testing.T) (*ActivityService, *settings.SettingsService, context.Context) {
	t.Helper()
	ctx := t.Context()

	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
	require.NoError(t, err)
	require.NoError(t, db.AutoMigrate(&Activity{}, &ActivityMessage{}, &settings.SettingVariable{}))
	wrapped := &database.DB{DB: db}

	// Every extra pooled connection to a :memory: SQLite database is a fresh
	// empty database; the await goroutine must see the same data.
	sqlDB, err := db.DB()
	require.NoError(t, err)
	sqlDB.SetMaxOpenConns(1)

	localSettings, err := newSettingsServiceForTest(t, ctx, wrapped)
	require.NoError(t, err)
	require.NoError(t, localSettings.SetIntSetting(ctx, "maxConcurrentActivities", 1))
	return NewActivityService(wrapped, localSettings), localSettings, ctx
}

func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {
	t.Helper()
	svc, err := settings.NewSettingsService(ctx, db, &config.Config{})
	if err == nil {
		t.Cleanup(func() { require.NoError(t, svc.Stop(context.WithoutCancel(t.Context()))) })
	}
	return svc, err
}

func TestActivityServiceQueuedActivityFlipsToRunningWhenSlotFrees(t *testing.T) {
	service, _, ctx := setupQueuedActivityService(t)

	first, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
	require.NoError(t, err)
	require.Equal(t, "running", string(first.Status))

	second, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
	require.NoError(t, err)
	require.Equal(t, "queued", string(second.Status))

	awaitDone := make(chan error, 1)
	go func() { awaitDone <- service.AwaitActivitySlot(ctx, second.ID, "0") }()

	select {
	case awaitErr := <-awaitDone:
		require.FailNowf(t, "unexpected failure", "await returned before the slot freed: %v", awaitErr)
	case <-time.After(100 * time.Millisecond):
	}

	_, err = service.CompleteActivity(ctx, first.ID, activity.StatusSuccess, "done", nil)
	require.NoError(t, err)

	select {
	case awaitErr := <-awaitDone:
		require.NoError(t, awaitErr)
	case <-time.After(5 * time.Second):
		require.FailNow(t, "await did not acquire the freed slot")
	}

	var model Activity
	require.NoError(t, service.db.First(&model, "id = ?", second.ID).Error)
	require.Equal(t, activity.StatusRunning, model.Status)

	_, err = service.CompleteActivity(ctx, second.ID, activity.StatusSuccess, "done", nil)
	require.NoError(t, err)
	firstDeferred, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true, DeferSlot: true})
	require.NoError(t, err)
	secondDeferred, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true, DeferSlot: true})
	require.NoError(t, err)
	require.Equal(t, activity.StatusQueued, firstDeferred.Status)
	require.Equal(t, activity.StatusQueued, secondDeferred.Status)
	require.NoError(t, service.AwaitActivitySlot(ctx, firstDeferred.ID, "0"))
	_, err = service.CompleteActivity(ctx, firstDeferred.ID, activity.StatusSuccess, "done", nil)
	require.NoError(t, err)
	require.NoError(t, service.AwaitActivitySlot(ctx, secondDeferred.ID, "0"))
}

func TestActivityServiceLimitIncreaseKeepsCountingActiveSlots(t *testing.T) {
	service, localSettings, ctx := setupQueuedActivityService(t)

	running, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
	require.NoError(t, err)
	require.Equal(t, "running", string(running.Status))
	waiting, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
	require.NoError(t, err)
	require.Equal(t, "queued", string(waiting.Status))

	awaitDone := make(chan error, 1)
	go func() { awaitDone <- service.AwaitActivitySlot(ctx, waiting.ID, "0") }()

	// Raising the limit to 2 must admit the waiter while still counting the
	// original holder, leaving no capacity for a third activity.
	require.NoError(t, localSettings.SetIntSetting(ctx, "maxConcurrentActivities", 2))

	select {
	case awaitErr := <-awaitDone:
		require.NoError(t, awaitErr)
	case <-time.After(4 * time.Second):
		require.FailNow(t, "waiter was not admitted after the limit increase")
	}

	third, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
	require.NoError(t, err)
	require.Equal(t, "queued", string(third.Status))
}

func TestActivityServiceCancelWhileQueuedUnblocksAwait(t *testing.T) {
	service, _, ctx := setupQueuedActivityService(t)

	_, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
	require.NoError(t, err)
	queued, err := service.StartActivity(ctx, StartActivityRequest{EnvironmentID: "0", Type: activity.TypeImagePull, Queue: true})
	require.NoError(t, err)
	require.Equal(t, "queued", string(queued.Status))

	workCtx := service.Track(ctx, queued.ID)
	awaitDone := make(chan error, 1)
	go func() { awaitDone <- service.AwaitActivitySlot(workCtx, queued.ID, "0") }()

	require.True(t, service.RequestCancel(queued.ID))

	select {
	case awaitErr := <-awaitDone:
		require.ErrorIs(t, awaitErr, activitylib.ErrCanceled)
	case <-time.After(5 * time.Second):
		require.FailNow(t, "cancel did not unblock the queued slot wait")
	}
}

func TestActivityServiceCompleteActivityRejectsUninitializedService(t *testing.T) {
	service := NewActivityService(nil, nil)
	_, err := service.CompleteActivity(t.Context(), "any-id", activity.StatusSuccess, "done", nil)
	require.Error(t, err)
}

func TestActivityServiceTrackAndRequestCancel(t *testing.T) {
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	// Mirror the handler flow: work runs under an app-lifecycle runtime context.
	appCtx := utils.WithAppLifecycleContext(t.Context())
	runtimeCtx := utils.ActivityRuntimeContext(t.Context(), appCtx)

	created, err := service.StartActivity(runtimeCtx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImagePull,
		LatestMessage: "running",
	})
	require.NoError(t, err)

	workCtx := service.Track(runtimeCtx, created.ID)
	require.NoError(t, workCtx.Err())

	// A tracked activity is found and cancelled with the ErrCanceled cause.
	require.True(t, service.RequestCancel(created.ID))
	require.ErrorIs(t, workCtx.Err(), context.Canceled)
	require.ErrorIs(t, context.Cause(workCtx), activitylib.ErrCanceled)
	require.True(t, activitylib.CancelledByContext(workCtx))

	// Completion must land even though the work context is cancelled (this is the
	// path CompleteHandlerActivity takes after re-wrapping the work context).
	completed, err := service.CompleteActivity(utils.ActivityRuntimeContext(workCtx, nil), created.ID, activity.StatusCancelled, "Cancelled by user", nil)
	require.NoError(t, err)
	require.Equal(t, "cancelled", string(completed.Status))
	require.NotNil(t, completed.EndedAt)

	// Completing the activity releases the registration.
	require.False(t, service.RequestCancel(created.ID))
}

func TestActivityServiceCancelActivity(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	// An untracked running activity (e.g. after a restart) is finalized directly.
	created, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeSystemPrune,
		LatestMessage: "running",
	})
	require.NoError(t, err)

	cancelled, err := service.CancelActivity(ctx, "0", created.ID, "Tester")
	require.NoError(t, err)
	require.Equal(t, "cancelled", string(cancelled.Status))
	require.NotNil(t, cancelled.EndedAt)

	// Cancelling an already-terminal activity is rejected.
	_, err = service.CancelActivity(ctx, "0", created.ID, "Tester")
	require.ErrorIs(t, err, ErrActivityNotCancelable)

	// Unknown activity reports not found.
	_, err = service.CancelActivity(ctx, "0", "missing", "Tester")
	require.ErrorIs(t, err, gorm.ErrRecordNotFound)
}

func TestActivityServiceFailStaleImageUpdateChecks(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	staleCheck, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImageUpdateCheck,
		LatestMessage: "checking",
	})
	require.NoError(t, err)
	freshCheck, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImageUpdateCheck,
		LatestMessage: "checking",
	})
	require.NoError(t, err)
	staleOtherType, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImagePull,
		LatestMessage: "pulling",
	})
	require.NoError(t, err)
	completedCheck, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImageUpdateCheck,
		LatestMessage: "checking",
	})
	require.NoError(t, err)
	_, err = service.CompleteActivity(ctx, completedCheck.ID, activity.StatusSuccess, "complete", nil)
	require.NoError(t, err)

	oldStartedAt := time.Now().Add(-7 * time.Hour)
	for _, id := range []string{staleCheck.ID, staleOtherType.ID, completedCheck.ID} {
		require.NoError(t, db.Model(&Activity{}).Where("id = ?", id).Updates(map[string]any{
			"started_at": oldStartedAt,
			"updated_at": oldStartedAt,
		}).Error)
	}

	failed, err := service.FailStaleImageUpdateChecks(ctx)
	require.NoError(t, err)
	require.EqualValues(t, 1, failed)

	var stale Activity
	require.NoError(t, db.First(&stale, "id = ?", staleCheck.ID).Error)
	require.Equal(t, activity.StatusFailed, stale.Status)
	require.NotNil(t, stale.EndedAt)
	require.NotNil(t, stale.DurationMs)
	require.Contains(t, stale.LatestMessage, "stale")
	require.NotNil(t, stale.Error)
	require.Contains(t, *stale.Error, "stale")

	var fresh Activity
	require.NoError(t, db.First(&fresh, "id = ?", freshCheck.ID).Error)
	require.Equal(t, activity.StatusRunning, fresh.Status)
	require.Nil(t, fresh.EndedAt)

	var other Activity
	require.NoError(t, db.First(&other, "id = ?", staleOtherType.ID).Error)
	require.Equal(t, activity.StatusRunning, other.Status)
	require.Nil(t, other.EndedAt)

	var completed Activity
	require.NoError(t, db.First(&completed, "id = ?", completedCheck.ID).Error)
	require.Equal(t, activity.StatusSuccess, completed.Status)
}

func TestActivityServiceFailAbandonedActivities(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	abandoned, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImageUpdateCheck,
		LatestMessage: "checking",
	})
	require.NoError(t, err)
	tracked, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeAutoUpdate,
		LatestMessage: "updating",
	})
	require.NoError(t, err)
	fresh, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImageUpdateCheck,
		LatestMessage: "checking",
	})
	require.NoError(t, err)

	backdated := time.Now().Add(-10 * time.Minute)
	require.NoError(t, db.Model(&Activity{}).
		Where("id IN ?", []string{abandoned.ID, tracked.ID}).
		Update("started_at", backdated).Error)
	_ = service.Track(ctx, tracked.ID)

	swept, err := service.FailAbandonedActivities(ctx)
	require.NoError(t, err)
	require.EqualValues(t, 1, swept)

	var abandonedRow Activity
	require.NoError(t, db.First(&abandonedRow, "id = ?", abandoned.ID).Error)
	require.Equal(t, activity.StatusFailed, abandonedRow.Status)
	require.NotNil(t, abandonedRow.EndedAt)
	require.Contains(t, abandonedRow.LatestMessage, "worker is no longer running")

	var trackedRow Activity
	require.NoError(t, db.First(&trackedRow, "id = ?", tracked.ID).Error)
	require.Equal(t, activity.StatusRunning, trackedRow.Status)

	var freshRow Activity
	require.NoError(t, db.First(&freshRow, "id = ?", fresh.ID).Error)
	require.Equal(t, activity.StatusRunning, freshRow.Status)
}

func TestActivityServiceResolveStaleAutoUpdateActivities(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	selfUpdateRun, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeAutoUpdate,
		LatestMessage: "updating",
		Metadata:      database.JSON{"dryRun": false},
	})
	require.NoError(t, err)
	require.NoError(t, service.PatchActivityMetadata(ctx, selfUpdateRun.ID, database.JSON{"selfUpdateTriggered": true}))

	interruptedRun, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeAutoUpdate,
		LatestMessage: "updating",
	})
	require.NoError(t, err)
	otherType, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImagePull,
		LatestMessage: "pulling",
	})
	require.NoError(t, err)

	resolved, err := service.ResolveStaleAutoUpdateActivities(ctx)
	require.NoError(t, err)
	require.EqualValues(t, 2, resolved)

	var selfUpdated Activity
	require.NoError(t, db.First(&selfUpdated, "id = ?", selfUpdateRun.ID).Error)
	require.Equal(t, activity.StatusSuccess, selfUpdated.Status)
	require.NotNil(t, selfUpdated.EndedAt)
	require.Contains(t, selfUpdated.LatestMessage, "restarted with the updated image")
	require.Equal(t, false, selfUpdated.Metadata["dryRun"])

	var interrupted Activity
	require.NoError(t, db.First(&interrupted, "id = ?", interruptedRun.ID).Error)
	require.Equal(t, activity.StatusFailed, interrupted.Status)
	require.Contains(t, interrupted.LatestMessage, "interrupted")

	var other Activity
	require.NoError(t, db.First(&other, "id = ?", otherType.ID).Error)
	require.Equal(t, activity.StatusRunning, other.Status)
}

func receiveActivityEvent(t *testing.T, events <-chan activity.StreamEvent) activity.StreamEvent {
	t.Helper()

	select {
	case event := <-events:
		return event
	case <-time.After(time.Second):
		require.FailNow(t, "timed out waiting for activity event")
		return activity.StreamEvent{}
	}
}

// TestActivityServiceAppendMessagesBatch verifies a batch lands all
// messages in order and coalesces the activity update to the batch's last
// message, last progress, and last step.
func TestActivityServiceAppendMessagesBatch(t *testing.T) {
	ctx := t.Context()
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	created, err := service.StartActivity(ctx, StartActivityRequest{
		EnvironmentID: "0",
		Type:          activity.TypeImagePull,
		Step:          "queued",
		LatestMessage: "Pull queued",
	})
	require.NoError(t, err)

	progress := 60
	messages, err := service.AppendMessages(ctx, created.ID, []AppendActivityMessageRequest{
		{Message: "layer 1/3", Step: "download"},
		{Message: "   "}, // blank lines are dropped, not persisted
		{Message: "layer 2/3", Progress: &progress},
		{Message: "layer 3/3"},
	})
	require.NoError(t, err)
	require.Len(t, messages, 3)
	require.Equal(t, "layer 1/3", messages[0].Message)
	require.Equal(t, "layer 3/3", messages[2].Message)

	detail, err := service.GetActivityDetail(ctx, "0", created.ID, 10)
	require.NoError(t, err)
	require.Len(t, detail.Messages, 3)
	require.Equal(t, "layer 1/3", detail.Messages[0].Message)
	require.Equal(t, "layer 2/3", detail.Messages[1].Message)
	require.Equal(t, "layer 3/3", detail.Messages[2].Message)

	// Coalesced activity update: last message wins, last non-nil progress
	// and last non-empty step stick.
	require.Equal(t, "layer 3/3", detail.Activity.LatestMessage)
	require.NotNil(t, detail.Activity.Progress)
	require.Equal(t, 60, *detail.Activity.Progress)
	require.Equal(t, "download", detail.Activity.Step)

	// Unknown activity IDs are still rejected.
	_, err = service.AppendMessages(ctx, "missing", []AppendActivityMessageRequest{{Message: "x"}})
	require.ErrorContains(t, err, "activity not found")
}

func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)
	now := time.Now().UTC()
	run := scheduler.Run{ID: "run", ActivityID: "summary", JobID: "auto-update", EnvironmentID: "0", Status: scheduler.NeedsAttention, CreatedAt: now.Add(-time.Hour), UpdatedAt: now}
	require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))
	stale := run
	run.Status = scheduler.Queued
	run.UpdatedAt = now.Add(time.Second)
	require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))
	require.NoError(t, service.SyncJobRun(t.Context(), stale, "Auto update"))
	detail, err := service.GetActivityDetail(t.Context(), "0", run.ActivityID, 10)
	require.NoError(t, err)
	require.Equal(t, activity.StatusQueued, detail.Activity.Status)
	require.Nil(t, detail.Activity.EndedAt)
	require.Nil(t, detail.Activity.Error)
	require.Empty(t, service.running)
	count, err := service.ResolveOrphanedQueuedActivities(t.Context())
	require.NoError(t, err)
	require.Zero(t, count)
	// Repair a legacy summary even when its durable timestamp has not changed.
	run.EnvironmentID = "remote"
	var legacy Activity
	require.NoError(t, db.First(&legacy, "id = ?", run.ActivityID).Error)
	legacy.Metadata["environmentId"] = "remote"
	require.NoError(t, db.Model(&legacy).Update("metadata", legacy.Metadata).Error)
	migration, err := os.ReadFile("../../resources/migrations/sqlite/084_route_job_activities_to_environment.sql")
	require.NoError(t, err)
	require.NoError(t, db.Exec(strings.Split(string(migration), "-- +goose Down")[0]).Error)
	require.NoError(t, db.First(&legacy, "id = ?", run.ActivityID).Error)
	require.Equal(t, "remote", legacy.EnvironmentID)
	// Startup reconciliation also repairs legacy rows without replaying the job.
	require.NoError(t, db.Model(&legacy).Update("environment_id", "0").Error)
	events, _, unsubscribe := service.pubsub.Subscribe("remote")
	defer unsubscribe()
	require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))
	select {
	case event := <-events:
		require.NotNil(t, event.Activity)
		require.Equal(t, "remote", event.Activity.EnvironmentID)
	case <-time.After(time.Second):
		t.Fatal("summary update was not published to the remote environment")
	}
	_, err = service.GetActivityDetail(t.Context(), "0", run.ActivityID, 10)
	require.Error(t, err)
	permissions := authz.NewPermissionSet()
	permissions.AddEnv("remote", authz.PermActivitiesRead, authz.PermActivitiesDelete)
	ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, permissions)
	rows, page, err := service.ListActivitiesPaginated(ctx, "remote", pagination.QueryParams{Limit: 10})
	require.NoError(t, err)
	require.EqualValues(t, 1, page.TotalItems)
	require.Equal(t, "remote", rows[0].EnvironmentID)
	remoteItems := []activity.Activity{{ID: "agent-work", EnvironmentID: "0", Type: activity.TypeImagePull, Status: activity.StatusRunning, CreatedAt: now}}
	rows, page, err = service.ListRemoteActivities(ctx, "remote", remoteItems, int64(len(remoteItems)), pagination.QueryParams{Start: 1, Limit: 1})
	require.NoError(t, err)
	require.EqualValues(t, 2, page.TotalItems)
	require.Len(t, rows, 1)
	require.Equal(t, run.ActivityID, rows[0].ID)
	rows, _, err = service.ListRemoteActivities(ctx, "remote", remoteItems, int64(len(remoteItems)), pagination.QueryParams{Limit: 1})
	require.NoError(t, err)
	require.Equal(t, "agent-work", rows[0].ID)
	require.Equal(t, "remote", rows[0].EnvironmentID)
	run.Status = scheduler.Succeeded
	run.UpdatedAt = run.UpdatedAt.Add(time.Second)
	require.NoError(t, service.SyncJobRun(ctx, run, "Auto update"))
	deleted, err := service.DeleteHistory(ctx, "remote")
	require.NoError(t, err)
	require.EqualValues(t, 1, deleted)
	count, err = service.FailAbandonedActivities(t.Context())
	require.NoError(t, err)
	require.Zero(t, count)
}

func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)
	for _, environmentID := range []string{"0", "private"} {
		require.NoError(
			t,
			db.Create(&Activity{
				ID:            environmentID + "-job",
				EnvironmentID: environmentID,
				Type:          activity.TypeJobRun,
				Status:        activity.StatusSuccess,
				StartedAt:     time.Now(),
				Metadata:      database.JSON{"environmentId": environmentID},
			}).Error,
		)
		require.NoError(
			t,
			db.Create(&Activity{
				ID:            environmentID + "-pull",
				EnvironmentID: environmentID,
				Type:          activity.TypeImagePull,
				Status:        activity.StatusSuccess,
				StartedAt:     time.Now(),
			}).Error,
		)
	}
	permissions := authz.NewPermissionSet()
	permissions.PerEnv["0"] = map[string]struct{}{authz.PermActivitiesRead: {}}
	permissions.PerEnv["allowed"] = map[string]struct{}{authz.PermActivitiesRead: {}}
	ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, permissions)
	seen := make(map[string]bool)
	for page := range 2 {
		activities, response, err := service.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{Start: page, Limit: 1})
		require.NoError(t, err)
		require.EqualValues(t, 2, response.TotalItems)
		require.Len(t, activities, 1)
		require.True(t, canReadJobActivity(ctx, activities[0]))
		seen[activities[0].ID] = true
	}
	require.Len(t, seen, 2)
	activities, response, err := service.ListActivitiesPaginated(ctx, "private", pagination.QueryParams{Limit: 10})
	require.NoError(t, err)
	require.EqualValues(t, 1, response.TotalItems)
	require.Len(t, activities, 1)
	require.Equal(t, "private-pull", activities[0].ID)
	require.False(
		t,
		canReadJobActivity(
			ctx,
			activity.Activity{
				Type:          activity.TypeJobRun,
				EnvironmentID: "private",
				Metadata:      map[string]any{"environmentId": "allowed"},
			},
		),
	)
	require.True(t, canReadJobActivity(ctx, activity.Activity{Type: activity.TypeJobRun, Metadata: map[string]any{"environmentId": "allowed"}}))
	privileged := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.SudoPermissionSet())
	activities, response, err = service.ListActivitiesPaginated(privileged, "private", pagination.QueryParams{Limit: 10})
	require.NoError(t, err)
	require.Len(t, activities, 2)
	require.EqualValues(t, 2, response.TotalItems)
	require.False(t, canReadJobActivity(privileged, activity.Activity{Type: activity.TypeJobRun}))
	permissions.PerEnv["0"][authz.PermActivitiesDelete] = struct{}{}
	deleted, err := service.DeleteHistory(ctx, "private")
	require.NoError(t, err)
	require.EqualValues(t, 1, deleted)
	require.NoError(t, db.First(&Activity{}, "id = ?", "private-job").Error)
	deleted, err = service.DeleteHistory(ctx, "0")
	require.NoError(t, err)
	require.EqualValues(t, 2, deleted)
}

func TestSyncResourcesToEnvironmentOutcomes(t *testing.T) {
	paths := []string{"/api/container-registries/sync", "/api/backups/s3/sync", "/api/git-repositories/sync"}
	cases := []struct {
		name     string
		failures map[string]string
		groups   string
	}{
		{name: "success"},
		{name: "registry failure", failures: map[string]string{paths[0]: "status"}, groups: "container registries"},
		{name: "S3 failure", failures: map[string]string{paths[1]: "status"}, groups: "S3 destinations"},
		{name: "repository failure", failures: map[string]string{paths[2]: "status"}, groups: "git repositories"},
		{name: "multiple failures", failures: map[string]string{paths[0]: "status", paths[2]: "status"}, groups: "container registries, git repositories"},
		{name: "all fail", failures: map[string]string{paths[0]: "status", paths[1]: "status", paths[2]: "status"}, groups: "container registries, S3 destinations, git repositories"},
		{name: "malformed response", failures: map[string]string{paths[0]: "malformed"}, groups: "container registries"},
		{name: "agent reports failure", failures: map[string]string{paths[0]: "false"}, groups: "container registries"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			db := setupActivityServiceTestDB(t)
			require.NoError(t, db.AutoMigrate(&environment.Environment{}, &registry.ContainerRegistry{}, &gitrepo.GitRepository{}, &s3.S3Destination{}))
			sqlDB, err := db.DB.DB()
			require.NoError(t, err)
			sqlDB.SetMaxOpenConns(1)
			t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })
			crypto.InitEncryption(&crypto.Config{EncryptionKey: "test-encryption-key-for-testing-32bytes-min", Environment: "test"})
			var mu sync.Mutex
			var calls []string
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				mu.Lock()
				calls = append(calls, r.URL.Path)
				mu.Unlock()
				switch tc.failures[r.URL.Path] {
				case "status":
					http.Error(w, "private-agent-response", http.StatusUnauthorized)
				case "malformed":
					_, _ = w.Write([]byte("private-agent-response"))
				case "false":
					_, _ = w.Write([]byte(`{"success":false,"data":{"message":"private-agent-response"}}`))
				default:
					_, _ = w.Write([]byte(`{"success":true}`))
				}
			}))
			defer server.Close()
			require.NoError(t, db.Create(&environment.Environment{ID: "remote", Name: "Remote", ApiUrl: server.URL, AccessToken: new("agent-token"), Enabled: true}).Error)
			service := environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)
			activityService := NewActivityService(db, nil)
			id, err := service.SyncResourcesToEnvironment(t.Context(), "remote", &user.Actor{ID: "operator", Username: "operator"}, activityService)
			require.NotEmpty(t, id)
			var recorded Activity
			require.NoError(t, db.First(&recorded, "id = ?", id).Error)
			require.NotNil(t, recorded.EndedAt)
			require.NotNil(t, recorded.DurationMs)
			require.Equal(t, "remote", recorded.EnvironmentID)
			if tc.groups == "" {
				require.NoError(t, err)
				require.Equal(t, activity.StatusSuccess, recorded.Status)
				require.Equal(t, "Environment synced successfully", recorded.LatestMessage)
				require.Nil(t, recorded.Error)
			} else {
				require.EqualError(t, err, "Failed to sync "+tc.groups+". Other resource groups may have synced successfully. Check the manager logs, correct the failed sync, and retry.")
				require.Equal(t, activity.StatusFailed, recorded.Status)
				require.NotNil(t, recorded.Error)
				require.Equal(t, err.Error(), *recorded.Error)
				require.Equal(t, err.Error(), recorded.LatestMessage)
				require.NotContains(t, err.Error(), "private-agent-response")
				require.NotContains(t, err.Error(), "agent-token")
			}
			mu.Lock()
			defer mu.Unlock()
			require.Equal(t, paths, calls)
		})
	}
}

func TestActivityService_StartActivityWithIDIsIdempotent(t *testing.T) {
	db := setupActivityServiceTestDB(t)
	service := NewActivityService(db, nil)

	first, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "first"})
	require.NoError(t, err)
	require.Equal(t, "workflow-activity", first.ID)

	again, err := service.StartActivity(t.Context(), StartActivityRequest{ID: "workflow-activity", Type: activity.TypeAutoUpdate, LatestMessage: "redelivered"})
	require.NoError(t, err)
	require.Equal(t, first.ID, again.ID)
	require.Equal(t, "first", again.LatestMessage)

	var count int64
	require.NoError(t, db.Model(&Activity{}).Where("id = ?", "workflow-activity").Count(&count).Error)
	require.Equal(t, int64(1), count)
}