Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: repair auto-update recovery and integrate jobs with activities (#3867)
Kyle Mendell5d055f66d796
diff --git a/backend/internal/activity/handler.go b/backend/internal/activity/handler.goindex a0d842b1b..ce1464f3b 100644--- a/backend/internal/activity/handler.go+++ b/backend/internal/activity/handler.go@@ -6,6 +6,7 @@ import ("context""fmt""hash/fnv"+ "maps""net/http""net/url""strconv"@@ -19,6 +20,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/remenv""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/base"@@ -138,16 +140,7 @@ func (h *ActivityHandler) ListActivities(ctx context.Context, input *ListActivitreturn h.proxyListActivitiesInternal(ctx, input)}- params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)- if input.Status != "" {- params.Filters["status"] = input.Status- }- if input.Type != "" {- params.Filters["type"] = input.Type- }- if input.ResourceType != "" {- params.Filters["resourceType"] = input.ResourceType- }+ params := activityListParamsInternal(input)activities, paginationResp, err := h.activityService.ListActivitiesPaginated(ctx, input.EnvironmentID, params)if err != nil {@@ -165,9 +158,6 @@ func (h *ActivityHandler) ListActivities(ctx context.Context, input *ListActivit}func (h *ActivityHandler) GetActivity(ctx context.Context, input *GetActivityInput) (*handlerutil.Out[activitytypes.Detail], error) {- if input.EnvironmentID != "0" {- return h.proxyGetActivityInternal(ctx, input)- }if input.ActivityID == "" {return nil, huma.Error400BadRequest("activity id is required")}@@ -175,10 +165,16 @@ func (h *ActivityHandler) GetActivity(ctx context.Context, input *GetActivityInpdetail, err := h.activityService.GetActivityDetail(ctx, input.EnvironmentID, input.ActivityID, input.Limit)if err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {+ if input.EnvironmentID != "0" {+ return h.proxyGetActivityInternal(ctx, input)+ }return nil, huma.Error404NotFound("activity not found")}return nil, huma.Error500InternalServerError(err.Error())}+ if !canReadJobActivityInternal(ctx, detail.Activity) {+ return nil, huma.Error404NotFound("activity not found")+ }h.applyActivitySourceLabelInternal(ctx, input.EnvironmentID, &detail.Activity)return &handlerutil.Out[activitytypes.Detail]{@@ -190,15 +186,20 @@ func (h *ActivityHandler) GetActivity(ctx context.Context, input *GetActivityInp}func (h *ActivityHandler) ClearHistory(ctx context.Context, input *ClearActivityHistoryInput) (*handlerutil.Out[activitytypes.ClearHistoryResult], error) {- if input.EnvironmentID != "0" {- return h.proxyClearHistoryInternal(ctx, input)- }-deleted, err := h.activityService.DeleteHistory(ctx, input.EnvironmentID)if err != nil {return nil, huma.Error500InternalServerError(err.Error())}+ if input.EnvironmentID != "0" {+ out, err := h.proxyClearHistoryInternal(ctx, input)+ if err != nil {+ return nil, err+ }+ out.Body.Data.Deleted += deleted+ return out, nil+ }+return &handlerutil.Out[activitytypes.ClearHistoryResult]{Body: base.ApiResponse[activitytypes.ClearHistoryResult]{Success: true,@@ -208,9 +209,6 @@ func (h *ActivityHandler) ClearHistory(ctx context.Context, input *ClearActivity}func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActivityInput) (*handlerutil.Out[activitytypes.Activity], error) {- if input.EnvironmentID != "0" {- return h.proxyCancelActivityInternal(ctx, input)- }if input.ActivityID == "" {return nil, huma.Error400BadRequest("activity id is required")}@@ -220,6 +218,9 @@ func (h *ActivityHandler) CancelActivity(ctx context.Context, input *CancelActivif err != nil {switch {case errors.Is(err, gorm.ErrRecordNotFound):+ if input.EnvironmentID != "0" {+ return h.proxyCancelActivityInternal(ctx, input)+ }return nil, huma.Error404NotFound("activity not found")case errors.Is(err, ErrActivityNotCancelable):return nil, huma.Error409Conflict("activity is not running")@@ -306,6 +307,9 @@ func (h *ActivityHandler) RunLocalStreamProducer(ctx context.Context, limit int,if !ok {return}+ if !h.canReadActivityStreamEventInternal(ctx, event) {+ continue+ }event.EnvironmentID = "0"h.applyActivityStreamEventSourceLabelInternal(ctx, "0", &event)if !agg.Send(ctx, events, event) {@@ -399,9 +403,6 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.ContlastFingerprint := ""poll := func() {- pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)- defer cancelPoll()-currentEnvironment := environmentif h.environment.GetActiveRemoteEnvironment != nil {var ok bool@@ -411,7 +412,7 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont}}- output, err := h.proxyListActivitiesForEnvironmentInternal(pollCtx, currentEnvironment, &ListActivitiesInput{+ output, err := h.proxyListActivitiesForEnvironmentInternal(ctx, currentEnvironment, &ListActivitiesInput{EnvironmentID: environmentID,Limit: resolveActivityStreamLimitInternal(limit),Order: "desc",@@ -420,6 +421,19 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Contif ctx.Err() != nil {return}+ if output != nil {+ fingerprint := activitySnapshotFingerprintInternal(output.Body.Data)+ if fingerprint != lastFingerprint {+ publish(activitytypes.StreamEvent{+ Type: "snapshot",+ EnvironmentID: environmentID,+ Activities: output.Body.Data,+ Timestamp: time.Now(),+ })+ lastFingerprint = fingerprint+ lastError = ""+ }+ }// A failing environment must not end the stream; surface the error// once per distinct message and keep polling.if msg := err.Error(); msg != lastError {@@ -431,10 +445,11 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.ContTimestamp: time.Now(),})}- // Force a resync once the environment recovers.- lastFingerprint = ""return}+ if lastError != "" {+ lastFingerprint = ""+ }lastError = ""fingerprint := activitySnapshotFingerprintInternal(output.Body.Data)if fingerprint == lastFingerprint {@@ -464,26 +479,6 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont}}-func (h *ActivityHandler) proxyListActivitiesInternal(ctx context.Context, input *ListActivitiesInput) (*handlerutil.Page[activitytypes.Activity], error) {- path := "/api/environments/0/activities?" + activityListQueryInternal(input).Encode()- out, err := h.environment.ProxyJSONRequest.JSON[base.Paginated[activitytypes.Activity]](ctx, input.EnvironmentID, http.MethodGet, path, nil)- if err != nil {- return nil, err- }- h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, out.Data)- return &handlerutil.Page[activitytypes.Activity]{Body: *out}, nil-}--func (h *ActivityHandler) proxyListActivitiesForEnvironmentInternal(ctx context.Context, environment environment.Environment, input *ListActivitiesInput) (*handlerutil.Page[activitytypes.Activity], error) {- path := "/api/environments/0/activities?" + activityListQueryInternal(input).Encode()- var out base.Paginated[activitytypes.Activity]- if err := h.environment.ProxyJSONRequestForEnvironment(ctx, environment, http.MethodGet, path, nil, &out); err != nil {- return nil, handlerutil.TranslateRemoteProxyError(err)- }- applyActivitySourceLabelsForEnvironmentInternal(environment, out.Data)- return &handlerutil.Page[activitytypes.Activity]{Body: out}, nil-}-func (h *ActivityHandler) proxyGetActivityInternal(ctx context.Context, input *GetActivityInput) (*handlerutil.Out[activitytypes.Detail], error) {path := fmt.Sprintf("/api/environments/0/activities/%s?limit=%d", url.PathEscape(input.ActivityID), input.Limit)out, err := h.environment.ProxyJSONRequest.JSON[base.ApiResponse[activitytypes.Detail]](ctx, input.EnvironmentID, http.MethodGet, path, nil)@@ -558,6 +553,11 @@ func applyActivitySourceInternal(item *activitytypes.Activity, sourceID, sourceN}item.SourceEnvironmentID = sourceIDitem.SourceEnvironmentName = sourceName+ item.EnvironmentID = sourceID+ if item.Type == activitytypes.TypeJobRun && item.Metadata != nil {+ item.Metadata = maps.Clone(item.Metadata)+ item.Metadata["environmentId"] = sourceID+ }}func activityListQueryInternal(input *ListActivitiesInput) url.Values {@@ -594,3 +594,75 @@ func resolveActivityStreamLimitInternal(limit int) int {}return limit}++func activityListParamsInternal(input *ListActivitiesInput) pagination.QueryParams {+ params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)+ params.Filters["status"] = input.Status+ params.Filters["type"] = input.Type+ params.Filters["resourceType"] = input.ResourceType+ return params+}++func (h *ActivityHandler) proxyListActivitiesInternal(ctx context.Context, input *ListActivitiesInput) (*handlerutil.Page[activitytypes.Activity], error) {+ unavailable := false+ out, err := h.listRemoteActivitiesInternal(ctx, input, func(path string) (*base.Paginated[activitytypes.Activity], error) {+ var remote base.Paginated[activitytypes.Activity]+ if proxyErr := h.environment.ProxyJSONRequest(ctx, input.EnvironmentID, http.MethodGet, path, nil, &remote); proxyErr != nil {+ var transportErr *remenv.TransportError+ unavailable = errors.As(proxyErr, &transportErr) && transportErr.Unavailable()+ return nil, handlerutil.TranslateRemoteProxyError(proxyErr)+ }+ return &remote, nil+ })+ if out == nil {+ return nil, err+ }+ if ctxErr := ctx.Err(); ctxErr != nil {+ return nil, ctxErr+ }+ if err != nil && !unavailable {+ return nil, err+ }+ // HTTP responses must not discard stored summaries when the agent is offline.+ // Stream pollers retain the proxy error from the shared listing path.+ out.Body.Success = true+ return out, nil+}++func (h *ActivityHandler) proxyListActivitiesForEnvironmentInternal(ctx context.Context, environment environment.Environment, input *ListActivitiesInput) (*handlerutil.Page[activitytypes.Activity], error) {+ out, err := h.listRemoteActivitiesInternal(ctx, input, func(path string) (*base.Paginated[activitytypes.Activity], error) {+ pollCtx, cancelPoll := context.WithTimeout(ctx, activityStreamRemotePollTimeout)+ defer cancelPoll()+ var out base.Paginated[activitytypes.Activity]+ if err := h.environment.ProxyJSONRequestForEnvironment(pollCtx, environment, http.MethodGet, path, nil, &out); err != nil {+ return nil, handlerutil.TranslateRemoteProxyError(err)+ }+ return &out, nil+ })+ if out != nil {+ applyActivitySourceLabelsForEnvironmentInternal(environment, out.Body.Data)+ }+ return out, err+}++func (h *ActivityHandler) listRemoteActivitiesInternal(ctx context.Context, input *ListActivitiesInput, fetch func(string) (*base.Paginated[activitytypes.Activity], error)) (*handlerutil.Page[activitytypes.Activity], error) {+ all := *input+ all.Start, all.Limit = 0, -1+ remote, remoteErr := fetch("/api/environments/0/activities?" + activityListQueryInternal(&all).Encode())+ var remoteActivities []activitytypes.Activity+ if remoteErr == nil {+ remoteActivities = remote.Data+ }+ // The proxy may have exhausted its timeout; manager summaries still need+ // to be read so an offline environment's queued work remains visible.+ activities, page, err := h.activityService.ListRemoteActivities(ctx, input.EnvironmentID, remoteActivities, activityListParamsInternal(input))+ if err != nil {+ return nil, huma.Error500InternalServerError(err.Error())+ }+ h.applyActivitySourceLabelsInternal(ctx, input.EnvironmentID, activities)+ return &handlerutil.Page[activitytypes.Activity]{Body: base.Paginated[activitytypes.Activity]{+ Success: remoteErr == nil,+ Data: activities,+ Pagination: handlerutil.PaginationResponse(page),+ }}, remoteErr+}diff --git a/backend/internal/activity/job.go b/backend/internal/activity/job.gonew file mode 100644index 000000000..489ffdf85--- /dev/null+++ b/backend/internal/activity/job.go@@ -0,0 +1,116 @@+package activity++import (+ "context"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "gorm.io/gorm"+)++// SyncJobRun updates a durable job summary without taking an execution slot.+// Only the queue owns its lifecycle, including reopening it for an explicit retry.+func (s *ActivityService) SyncJobRun(ctx context.Context, run st.Run, name string) error {+ if err := s.checkInitInternal(); err != nil {+ return err+ }+ if run.ActivityID == "" {+ return errors.New("job activity ID is required")+ }+ lock := s.publishLockInternal(run.ActivityID)+ lock.Lock()+ defer lock.Unlock()+ model := jobActivityInternal(run, name)+ changed := false+ if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ var existing Activity+ err := tx.First(&existing, "id = ?", run.ActivityID).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ changed = true+ return tx.Create(&model).Error+ }+ if err != nil {+ return err+ }+ if existing.Type != activitytypes.TypeJobRun {+ return errors.New("job activity identity belongs to another operation")+ }+ if stamp, ok := existing.Metadata["runUpdatedAt"].(string); ok {+ previous, parseErr := time.Parse(time.RFC3339Nano, stamp)+ if parseErr == nil && !previous.Before(run.UpdatedAt) {+ if existing.EnvironmentID == model.EnvironmentID {+ return nil+ }+ existing.EnvironmentID = model.EnvironmentID+ model = existing+ }+ }+ changed = true+ return tx.Table("activities").Where("id = ?", run.ActivityID).Updates(map[string]any{+ "environment_id": model.EnvironmentID,+ "status": model.Status, "step": model.Step, "latest_message": model.LatestMessage,+ "ended_at": model.EndedAt, "duration_ms": model.DurationMs, "error": model.Error,+ "metadata": model.Metadata, "updated_at": model.UpdatedAt,+ }).Error+ }); err != nil {+ return errors.WrapIf(err, "synchronize job activity")+ }+ if changed {+ // Ordinary activities cannot reopen. Job summaries can, after an explicit+ // retry; the publish lock and durable timestamp reject stale snapshots.+ s.terminalPublishedMu.Lock()+ delete(s.terminalPublished, model.ID)+ s.terminalPublishedMu.Unlock()+ s.publishActivityInternal(activityToDTOInternal(&model))+ }+ return nil+}++func jobActivityInternal(run st.Run, name string) Activity {+ status := activitytypes.StatusQueued+ switch run.Status {+ case st.Running:+ status = activitytypes.StatusRunning+ case st.Succeeded, st.Skipped:+ status = activitytypes.StatusSuccess+ case st.Failed, st.Partial, st.NeedsAttention:+ status = activitytypes.StatusFailed+ case st.Canceled:+ status = activitytypes.StatusCancelled+ case st.Queued, st.Waiting, st.Retrying:+ }+ message := run.Outcome.Message+ if message == "" {+ message = name + ": " + string(run.Status)+ }+ model := Activity{+ ID: run.ActivityID, CreatedAt: run.CreatedAt, UpdatedAt: new(run.UpdatedAt),+ EnvironmentID: run.EnvironmentID, BatchID: new(run.ID), Type: activitytypes.TypeJobRun, Status: status,+ ResourceType: new("job"), ResourceID: new(run.JobID), ResourceName: new(name),+ StartedAt: run.CreatedAt, Step: string(run.Status), LatestMessage: message,+ Metadata: database.JSON{+ "jobId": run.JobID, "runId": run.ID, "environmentId": run.EnvironmentID,+ "jobStatus": string(run.Status), "trigger": run.Trigger, "attemptCount": run.AttemptCount,+ "runUpdatedAt": run.UpdatedAt.Format(time.RFC3339Nano), "domainActivityId": run.Outcome.ActivityID,+ },+ }+ if run.RequestedBy != "" {+ model.StartedByUserID = new(run.RequestedBy)+ model.StartedByUsername = new(run.RequestedBy)+ }+ if run.Resolution != nil {+ model.Metadata["resolution"] = run.Resolution+ model.LatestMessage = "Resolved after review; future scheduled runs may proceed"+ }+ if isTerminalActivityStatusInternal(status) {+ model.EndedAt = new(run.UpdatedAt)+ model.DurationMs = new(max(int64(0), run.UpdatedAt.Sub(run.CreatedAt).Milliseconds()))+ }+ if status == activitytypes.StatusFailed {+ model.Error = new(message)+ }+ return model+}diff --git a/backend/internal/activity/job_test.go b/backend/internal/activity/job_test.gonew file mode 100644index 000000000..231042d13--- /dev/null+++ b/backend/internal/activity/job_test.go@@ -0,0 +1,91 @@+package activity++import (+ "context"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ "os"+ "strings"+ "testing"+ "time"++ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/stretchr/testify/require"+)++func TestJobSummaryReopensAndRejectsStaleSnapshots(t *testing.T) {+ db := setupActivityServiceTestDBInternal(t)+ service := NewActivityService(db, nil)+ now := time.Now().UTC()+ run := st.Run{ID: "run", ActivityID: "summary", JobID: "auto-update", EnvironmentID: "0", Status: st.NeedsAttention, CreatedAt: now.Add(-time.Hour), UpdatedAt: now}+ require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))+ stale := run+ run.Status = st.Queued+ run.UpdatedAt = now.Add(time.Second)+ require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))+ require.NoError(t, service.SyncJobRun(t.Context(), stale, "Auto update"))+ detail, err := service.GetActivityDetail(t.Context(), "0", run.ActivityID, 10)+ require.NoError(t, err)+ require.Equal(t, activitytypes.StatusQueued, detail.Activity.Status)+ require.Nil(t, detail.Activity.EndedAt)+ require.Nil(t, detail.Activity.Error)+ require.Empty(t, service.slotReleases)+ require.Empty(t, service.running)+ require.NotContains(t, service.terminalPublished, run.ActivityID)+ count, err := service.ResolveOrphanedQueuedActivities(t.Context())+ require.NoError(t, err)+ require.Zero(t, count)+ // Repair a legacy summary even when its durable timestamp has not changed.+ run.EnvironmentID = "remote"+ var legacy Activity+ require.NoError(t, db.First(&legacy, "id = ?", run.ActivityID).Error)+ legacy.Metadata["environmentId"] = "remote"+ require.NoError(t, db.Model(&legacy).Update("metadata", legacy.Metadata).Error)+ migration, err := os.ReadFile("../../resources/migrations/sqlite/084_route_job_activities_to_environment.sql")+ require.NoError(t, err)+ require.NoError(t, db.Exec(strings.Split(string(migration), "-- +goose Down")[0]).Error)+ require.NoError(t, db.First(&legacy, "id = ?", run.ActivityID).Error)+ require.Equal(t, "remote", legacy.EnvironmentID)+ // Startup reconciliation also repairs legacy rows without replaying the job.+ require.NoError(t, db.Model(&legacy).Update("environment_id", "0").Error)+ events, _, unsubscribe := service.Subscribe("remote")+ defer unsubscribe()+ require.NoError(t, service.SyncJobRun(t.Context(), run, "Auto update"))+ select {+ case event := <-events:+ require.NotNil(t, event.Activity)+ require.Equal(t, "remote", event.Activity.EnvironmentID)+ case <-time.After(time.Second):+ t.Fatal("summary update was not published to the remote environment")+ }+ _, err = service.GetActivityDetail(t.Context(), "0", run.ActivityID, 10)+ require.Error(t, err)+ permissions := authz.NewPermissionSet()+ permissions.AddEnv("remote", authz.PermActivitiesRead, authz.PermActivitiesDelete)+ ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, permissions)+ rows, page, err := service.ListActivitiesPaginated(ctx, "remote", pagination.QueryParams{Limit: 10})+ require.NoError(t, err)+ require.EqualValues(t, 1, page.TotalItems)+ require.Equal(t, "remote", rows[0].EnvironmentID)+ remoteItems := []activitytypes.Activity{{ID: "agent-work", EnvironmentID: "0", Type: activitytypes.TypeImagePull, Status: activitytypes.StatusRunning, CreatedAt: now}}+ rows, page, err = service.ListRemoteActivities(ctx, "remote", remoteItems, pagination.QueryParams{Start: 1, Limit: 1})+ require.NoError(t, err)+ require.EqualValues(t, 2, page.TotalItems)+ require.Len(t, rows, 1)+ require.Equal(t, run.ActivityID, rows[0].ID)+ rows, _, err = service.ListRemoteActivities(ctx, "remote", remoteItems, pagination.QueryParams{Limit: 1})+ require.NoError(t, err)+ require.Equal(t, "agent-work", rows[0].ID)+ require.Equal(t, "remote", rows[0].EnvironmentID)+ run.Status = st.Succeeded+ run.UpdatedAt = run.UpdatedAt.Add(time.Second)+ require.NoError(t, service.SyncJobRun(ctx, run, "Auto update"))+ deleted, err := service.DeleteHistory(ctx, "remote")+ require.NoError(t, err)+ require.EqualValues(t, 1, deleted)+ count, err = service.FailAbandonedActivities(t.Context())+ require.NoError(t, err)+ require.Zero(t, count)+}diff --git a/backend/internal/activity/remote.go b/backend/internal/activity/remote.gonew file mode 100644index 000000000..fb64bb8de--- /dev/null+++ b/backend/internal/activity/remote.go@@ -0,0 +1,90 @@+package activity++import (+ "cmp"+ "context"+ "strings"++ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/samber/lo"+)++// ListRemoteActivities combines manager-owned summaries with agent-owned work+// before slicing the page. The remote records must already match the filters.+func (s *ActivityService) ListRemoteActivities(ctx context.Context, environmentID string, remote []activitytypes.Activity, params pagination.QueryParams) ([]activitytypes.Activity, pagination.Response, error) {+ all := params+ all.Start, all.Limit = 0, -1+ activities, _, err := s.ListActivitiesPaginated(ctx, environmentID, all)+ if err != nil {+ return nil, pagination.Response{}, err+ }+ for _, item := range remote {+ item.EnvironmentID = environmentID+ activities = append(activities, item)+ }+ params.Start = max(params.Start, 0)+ if params.Limit != -1 {+ if params.Limit <= 0 {+ params.Limit = 20+ }+ params.Limit = min(params.Limit, 100)+ }+ total := int64(len(activities))+ config := pagination.Config[activitytypes.Activity]{SortBindings: []pagination.SortBinding[activitytypes.Activity]{{+ Fn: func(a, b activitytypes.Activity) int { return compareActivitiesInternal(a, b, params) },+ }}}+ pageParams := params+ pageParams.Order = pagination.SortAsc+ activities = config.OrderAndPaginate(activities, pageParams)+ return activities, pagination.BuildResponse(total, 0, params), nil+}++func compareActivitiesInternal(a, b activitytypes.Activity, params pagination.QueryParams) int {+ if params.Sort == "" {+ aActive := a.Status == activitytypes.StatusQueued || a.Status == activitytypes.StatusRunning+ bActive := b.Status == activitytypes.StatusQueued || b.Status == activitytypes.StatusRunning+ if aActive != bActive {+ if aActive {+ return -1+ }+ return 1+ }+ aTime, bTime := a.CreatedAt, b.CreatedAt+ if a.EndedAt != nil {+ aTime = *a.EndedAt+ }+ if b.EndedAt != nil {+ bTime = *b.EndedAt+ }+ return cmp.Or(bTime.Compare(aTime), strings.Compare(b.ID, a.ID))+ }++ var result int+ switch params.Sort {+ case "environmentId":+ result = strings.Compare(a.EnvironmentID, b.EnvironmentID)+ case "type":+ result = cmp.Compare(a.Type, b.Type)+ case "status":+ result = cmp.Compare(a.Status, b.Status)+ case "resourceType":+ result = strings.Compare(lo.FromPtr(a.ResourceType), lo.FromPtr(b.ResourceType))+ case "resourceName":+ result = strings.Compare(lo.FromPtr(a.ResourceName), lo.FromPtr(b.ResourceName))+ case "startedAt":+ result = a.StartedAt.Compare(b.StartedAt)+ case "createdAt":+ result = a.CreatedAt.Compare(b.CreatedAt)+ case "updatedAt":+ result = lo.FromPtr(a.UpdatedAt).Compare(lo.FromPtr(b.UpdatedAt))+ case "endedAt":+ result = lo.FromPtr(a.EndedAt).Compare(lo.FromPtr(b.EndedAt))+ case "durationMs":+ result = cmp.Compare(lo.FromPtr(a.DurationMs), lo.FromPtr(b.DurationMs))+ }+ if params.Order == pagination.SortDesc {+ result = -result+ }+ return cmp.Or(result, strings.Compare(a.ID, b.ID))+}diff --git a/backend/internal/activity/service.go b/backend/internal/activity/service.goindex cd0940951..0e8fe6683 100644--- a/backend/internal/activity/service.go+++ b/backend/internal/activity/service.go@@ -15,6 +15,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"@@ -789,6 +790,9 @@ func (s *ActivityService) CancelActivity(ctx context.Context, environmentID, actif err := s.db.WithContext(ctx).Where("id = ? AND environment_id = ?", activityID, environmentID).First(&model).Error; err != nil {return nil, err}+ if model.Type == activitytypes.TypeJobRun {+ return nil, ErrActivityNotCancelable+ }switch model.Status {case activitytypes.StatusSuccess, activitytypes.StatusFailed, activitytypes.StatusCancelled:return nil, ErrActivityNotCancelable@@ -911,6 +915,7 @@ func (s *ActivityService) FailAbandonedActivities(ctx context.Context) (int64, evar candidates []Activityif err := s.db.WithContext(ctx).Where("status IN ? AND started_at < ?", activeStatuses, cutoff).+ Where("type <> ?", activitytypes.TypeJobRun).Find(&candidates).Error; err != nil {return 0, errors.WrapIf(err, "find abandoned activities")}@@ -971,6 +976,7 @@ func (s *ActivityService) ResolveOrphanedQueuedActivities(ctx context.Context) (var queued []Activityif err := s.db.WithContext(ctx).Where("status = ?", activitytypes.StatusQueued).+ Where("type <> ?", activitytypes.TypeJobRun).Find(&queued).Error; err != nil {return 0, errors.WrapIf(err, "find orphaned queued activities")}@@ -1096,6 +1102,7 @@ func (s *ActivityService) ListActivitiesPaginated(ctx context.Context, environmevar activities []Activityq := s.db.WithContext(ctx).Model(&Activity{}).Where("environment_id = ?", environmentID)+ q = scopeJobActivityVisibilityInternal(ctx, q, authz.PermActivitiesRead)if term := strings.TrimSpace(params.Search); term != "" {escaped := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(term)@@ -1225,7 +1232,7 @@ func (s *ActivityService) DeleteHistory(ctx context.Context, environmentID strinvar deleted int64if err := s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {- ids, err := findTerminalActivityIDsInternal(tx.Where("environment_id = ?", environmentID))+ ids, err := findTerminalActivityIDsInternal(scopeJobActivityVisibilityInternal(ctx, tx.Where("environment_id = ?", environmentID), authz.PermActivitiesDelete))if err != nil {return errors.WrapIf(err, "failed to find activity history")}diff --git a/backend/internal/activity/visibility.go b/backend/internal/activity/visibility.gonew file mode 100644index 000000000..4b168aca2--- /dev/null+++ b/backend/internal/activity/visibility.go@@ -0,0 +1,67 @@+package activity++import (+ "context"++ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "gorm.io/gorm"+)++// Job summaries are stored on the manager but belong to their execution environment.+func scopeJobActivityVisibilityInternal(ctx context.Context, query *gorm.DB, permission string) *gorm.DB {+ permissions, present := middleware.PermissionsFromContext(ctx)+ if !present {+ return query+ }+ target := "json_extract(metadata, '$.environmentId')"+ valid := "json_type(metadata, '$.environmentId') = 'text' AND " + target + " <> ''"+ if query.Name() == "postgres" {+ target = "CAST(metadata AS jsonb)->>'environmentId'"+ valid = "jsonb_typeof(CAST(metadata AS jsonb)->'environmentId') = 'string' AND " + target + " <> ''"+ }+ if permissions.Allows(permission, "") {+ return query.Where("(type <> ? OR ("+valid+"))", activitytypes.TypeJobRun)+ }+ if permissions == nil {+ return query.Where("type <> ?", activitytypes.TypeJobRun)+ }+ allowed := make([]string, 0, len(permissions.PerEnv))+ for environmentID := range permissions.PerEnv {+ if permissions.Allows(permission, environmentID) {+ allowed = append(allowed, environmentID)+ }+ }+ if len(allowed) == 0 {+ return query.Where("type <> ?", activitytypes.TypeJobRun)+ }+ return query.Where("(type <> ? OR ("+valid+" AND "+target+" IN ?))", activitytypes.TypeJobRun, allowed)+}++func canReadJobActivityInternal(ctx context.Context, item activitytypes.Activity) bool {+ if item.Type != activitytypes.TypeJobRun {+ return true+ }+ environmentID, _ := item.Metadata["environmentId"].(string)+ if environmentID == "" {+ return false+ }+ permissions, _ := middleware.PermissionsFromContext(ctx)+ return permissions.Allows(authz.PermActivitiesRead, environmentID)+}++func (h *ActivityHandler) canReadActivityStreamEventInternal(ctx context.Context, event activitytypes.StreamEvent) bool {+ if event.Activity != nil {+ return canReadJobActivityInternal(ctx, *event.Activity)+ }+ if event.ActivityID == "" {+ return true+ }+ var model Activity+ // Message events carry no activity metadata. Check the owning row before output.+ if err := h.activityService.db.WithContext(ctx).Select("type", "environment_id", "metadata").Where("id = ? AND environment_id = ?", event.ActivityID, "0").First(&model).Error; err != nil {+ return false+ }+ return canReadJobActivityInternal(ctx, activitytypes.Activity{Type: model.Type, EnvironmentID: model.EnvironmentID, Metadata: model.Metadata})+}diff --git a/backend/internal/activity/visibility_test.go b/backend/internal/activity/visibility_test.gonew file mode 100644index 000000000..30b62fd47--- /dev/null+++ b/backend/internal/activity/visibility_test.go@@ -0,0 +1,56 @@+package activity++import (+ "context"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ "github.com/stretchr/testify/require"+)++func TestJobActivityVisibilityFiltersBeforePagination(t *testing.T) {+ db := setupActivityServiceTestDBInternal(t)+ service := NewActivityService(db, nil)+ for _, target := range []string{"0", "allowed", "private"} {+ require.NoError(t, db.Create(&Activity{BaseModel: database.BaseModel{ID: target}, EnvironmentID: "0", Type: activitytypes.TypeJobRun, Status: activitytypes.StatusQueued, StartedAt: time.Now(), Metadata: database.JSON{"environmentId": target}}).Error)+ }+ for _, metadata := range []database.JSON{nil, {"environmentId": 0}} {+ require.NoError(t, db.Create(&Activity{EnvironmentID: "0", Type: activitytypes.TypeJobRun, Status: activitytypes.StatusQueued, StartedAt: time.Now(), Metadata: metadata}).Error)+ }+ permissions := authz.NewPermissionSet()+ permissions.PerEnv["0"] = map[string]struct{}{authz.PermActivitiesRead: {}}+ permissions.PerEnv["allowed"] = map[string]struct{}{authz.PermActivitiesRead: {}}+ ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, permissions)+ seen := make(map[string]bool)+ for page := range 2 {+ activities, response, err := service.ListActivitiesPaginated(ctx, "0", pagination.QueryParams{Start: page, Limit: 1})+ require.NoError(t, err)+ require.EqualValues(t, 2, response.TotalItems)+ require.Len(t, activities, 1)+ require.NotEqual(t, "private", activities[0].ID)+ require.True(t, canReadJobActivityInternal(ctx, activities[0]))+ seen[activities[0].ID] = true+ }+ require.Len(t, seen, 2)+ require.False(t, canReadJobActivityInternal(ctx, activitytypes.Activity{Type: activitytypes.TypeJobRun, EnvironmentID: "0", Metadata: map[string]any{"environmentId": "private"}}))+ privileged := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.SudoPermissionSet())+ activities, response, err := service.ListActivitiesPaginated(privileged, "0", pagination.QueryParams{Limit: 10})+ require.NoError(t, err)+ require.Len(t, activities, 3)+ require.EqualValues(t, 3, response.TotalItems)+ require.False(t, canReadJobActivityInternal(privileged, activitytypes.Activity{Type: activitytypes.TypeJobRun, EnvironmentID: "0"}))+ require.NoError(t, db.Model(&Activity{}).Where("type = ?", activitytypes.TypeJobRun).Update("status", activitytypes.StatusSuccess).Error)+ permissions.PerEnv["0"][authz.PermActivitiesDelete] = struct{}{}+ deleted, err := service.DeleteHistory(ctx, "0")+ require.NoError(t, err)+ require.EqualValues(t, 1, deleted)+ var retained Activity+ require.NoError(t, db.First(&retained, "id = ?", "private").Error)+ require.NoError(t, db.First(&Activity{}, "id = ?", "allowed").Error)++}diff --git a/backend/internal/di/providers.go b/backend/internal/di/providers.goindex e9b5e8d51..f034f8dab 100644--- a/backend/internal/di/providers.go+++ b/backend/internal/di/providers.go@@ -362,8 +362,8 @@ func provideUserModuleInternal(service *user.UserService, auth *auth.AuthServicereturn user.New(user.Dependencies{Service: service, InvalidateUserTokenCache: auth.InvalidateUserTokenCache, Settings: settingsService})}-func provideJobModuleInternal(db *database.DB, settingsService *settings.SettingsService, cfg *config.Config, environment *environment.EnvironmentService, roles *role.RoleService) *job.Module {- return job.New(job.Dependencies{DB: db, Settings: settingsService, Config: cfg, Environment: environment, Roles: roles})+func provideJobModuleInternal(db *database.DB, settingsService *settings.SettingsService, cfg *config.Config, environment *environment.EnvironmentService, roles *role.RoleService, activityService *activity.ActivityService) *job.Module {+ return job.New(job.Dependencies{DB: db, Settings: settingsService, Config: cfg, Environment: environment, Roles: roles, Activity: activityService})}func provideJobServiceInternal(module *job.Module) *job.JobService {diff --git a/backend/internal/environment/environment_proxy.go b/backend/internal/environment/environment_proxy.goindex b8fa38ed9..43d2b5402 100644--- a/backend/internal/environment/environment_proxy.go+++ b/backend/internal/environment/environment_proxy.go@@ -303,7 +303,7 @@ func ensureRemoteEnvironmentTunnelAvailableInternal(ctx context.Context, envID sreturn nil}- return errors.New("edge agent is not connected")+ return remenv.ErrEnvironmentUnavailable}func doRemoteEnvironmentTunnelRequestInternal(@@ -316,14 +316,17 @@ func doRemoteEnvironmentTunnelRequestInternal() (*remenv.Response, error) {tunnel, ok := edge.GetRegistry().Get(envID).Get()if !ok {- return nil, errors.Errorf("no active tunnel for environment %s", envID)+ return nil, errors.WrapIff(remenv.ErrEnvironmentUnavailable, "no active tunnel for environment %s", envID)}if tunnel.Conn.IsClosed() {- return nil, errors.Errorf("tunnel for environment %s is closed", envID)+ return nil, errors.WrapIff(remenv.ErrEnvironmentUnavailable, "tunnel for environment %s is closed", envID)}statusCode, respHeaders, respBody, err := edge.ProxyRequest(ctx, tunnel, method, path, "", headers, body)if err != nil {+ if errors.Is(err, edge.ErrTunnelConnectionClosed) {+ err = errors.Combine(remenv.ErrEnvironmentUnavailable, err)+ }return nil, errors.WrapIf(err, "tunnel request failed")}diff --git a/backend/internal/job/activity.go b/backend/internal/job/activity.gonew file mode 100644index 000000000..176d79e5d--- /dev/null+++ b/backend/internal/job/activity.go@@ -0,0 +1,55 @@+package job++import (+ "context"+ "strings"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"+ "github.com/getarcaneapp/arcane/types/v2/meta"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/google/uuid"+)++// ActivityID gives visible runs a stable summary identity before execution.+func (s *JobService) ActivityID(run st.Run) string {+ // A remotely delivered run already has a summary on its accepting manager.+ if run.Trigger == "remote" {+ return ""+ }+ visible := run.Trigger == "manual"+ switch run.JobID {+ case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan":+ visible = true+ default:+ visible = visible || strings.HasPrefix(run.JobID, "gitops-sync:") || strings.HasPrefix(run.JobID, "volume-backup:") || strings.HasPrefix(run.JobID, "system-backup:")+ }+ if !visible {+ return ""+ }+ return uuid.NewSHA1(uuid.NameSpaceURL, []byte("arcane-job/"+run.EnvironmentID+"/"+run.JobID+"/"+run.ID)).String()+}++// SyncRunActivity projects the latest committed run without changing execution state.+func (s *JobService) SyncRunActivity(ctx context.Context, run st.Run) error {+ if s.activity == nil {+ return nil+ }+ s.activityMu.Lock()+ defer s.activityMu.Unlock()+ current, err := s.Queue.Get(ctx, run.EnvironmentID, run.JobID, run.ID)+ if errors.Is(err, queue.ErrRunNotFound) {+ return nil+ }+ if err != nil {+ return err+ }+ if current.ActivityID == "" {+ return nil+ }+ name := current.JobID+ if metadata, found := meta.GetJobMetadata(current.JobID); found {+ name = metadata.Name+ }+ return s.activity.SyncJobRun(ctx, current, name)+}diff --git a/backend/internal/job/activity_test.go b/backend/internal/job/activity_test.gonew file mode 100644index 000000000..90ea1b79a--- /dev/null+++ b/backend/internal/job/activity_test.go@@ -0,0 +1,139 @@+package job++import (+ "context"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/activity"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config"+ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ activitytypes "github.com/getarcaneapp/arcane/types/v2/activity"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/stretchr/testify/require"+)++func newJobActivityTestServiceInternal(t *testing.T) (*JobService, *activity.ActivityService, *database.DB) {+ t.Helper()+ db := setupSettingsTestDBInternal(t)+ sqlDB, err := db.DB.DB()+ require.NoError(t, err)+ sqlDB.SetMaxOpenConns(1)+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })+ require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))+ activities := activity.NewActivityService(db, nil)+ svc := New(Dependencies{DB: db, Config: &config.Config{}, Activity: activities}).Service()+ return svc, activities, db+}++func TestJobActivityLifecycle(t *testing.T) {+ svc, activities, db := newJobActivityTestServiceInternal(t)+ ctx := t.Context()+ run, err := svc.Queue.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "scheduled"})+ require.NoError(t, err)+ require.NotEmpty(t, run.ActivityID)+ require.Equal(t, "0", run.ActivityEnvironmentID)+ check := func(status activitytypes.Status, precise st.RunStatus) {+ t.Helper()+ detail, detailErr := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)+ require.NoError(t, detailErr)+ require.Equal(t, status, detail.Activity.Status)+ require.Equal(t, string(precise), detail.Activity.Metadata["jobStatus"])+ require.Equal(t, run.ID, *detail.Activity.BatchID)+ }+ check(activitytypes.StatusQueued, st.Queued)+ duplicate, err := svc.Queue.Submit(ctx, st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "scheduled"})+ require.NoError(t, err)+ require.Equal(t, run.ActivityID, duplicate.ActivityID)+ for _, status := range []st.RunStatus{st.Running, st.Waiting, st.NeedsAttention} {+ require.NoError(t, svc.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ current.Status = status+ current.UpdatedAt = time.Now().UTC()+ current.Outcome = st.Outcome{Status: status, Message: "operation detail"}+ return nil+ }))+ }+ check(activitytypes.StatusFailed, st.NeedsAttention)+ _, err = svc.Queue.Retry(ctx, "0", run.JobID, run.ID)+ require.NoError(t, err)+ check(activitytypes.StatusQueued, st.Queued)+ _, err = activities.CancelActivity(ctx, "0", run.ActivityID, "operator")+ require.ErrorIs(t, err, activity.ErrActivityNotCancelable)+ check(activitytypes.StatusQueued, st.Queued)+ require.NoError(t, svc.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ current.Status = st.NeedsAttention+ current.UpdatedAt = time.Now().UTC()+ return nil+ }))+ _, err = svc.Queue.Resolve(ctx, "0", run.JobID, run.ID, "operator")+ require.NoError(t, err)+ check(activitytypes.StatusCancelled, st.Canceled)+ var count int64+ require.NoError(t, db.Model(&activity.Activity{}).Count(&count).Error)+ require.Equal(t, int64(1), count)+}++func TestJobActivityVisibilityAndGrouping(t *testing.T) {+ svc, activities, _ := newJobActivityTestServiceInternal(t)+ for _, test := range []struct {+ job, trigger, environment string+ visible bool+ }{+ {"auto-update", "scheduled", "0", true},+ {"gitops-sync:project", "scheduled", "0", true},+ {"volume-backup:volume", "recovery", "0", true},+ {"system-backup:system", "scheduled", "0", true},+ {"image-polling", "manual", "0", true},+ {"auto-update", "manual", "remote", true},+ {"auto-update", "remote", "0", false},+ {"image-polling", "scheduled", "0", false},+ {"environment-health:0", "scheduled", "0", false},+ {"activity-sweep", "scheduled", "0", false},+ } {+ t.Run(test.job+"/"+test.trigger+"/"+test.environment, func(t *testing.T) {+ run, err := svc.Queue.Submit(t.Context(), st.Request{JobID: test.job, Trigger: test.trigger, EnvironmentID: test.environment})+ require.NoError(t, err)+ require.Equal(t, test.visible, run.ActivityID != "")+ if test.visible {+ require.Equal(t, test.environment, run.ActivityEnvironmentID)+ detail, err := activities.GetActivityDetail(t.Context(), test.environment, run.ActivityID, 10)+ require.NoError(t, err)+ require.Equal(t, test.environment, detail.Activity.Metadata["environmentId"])+ }+ child, err := activities.StartActivity(svc.runContextInternal(t.Context(), run), activity.StartActivityRequest{Type: activitytypes.TypeImagePull})+ require.NoError(t, err)+ require.Equal(t, run.ID, *child.BatchID)+ })+ }+}++func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {+ svc, activities, db := newJobActivityTestServiceInternal(t)+ ctx := t.Context()+ require.NoError(t, db.Migrator().DropTable(&activity.ActivityMessage{}, &activity.Activity{}))+ run, err := svc.Queue.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "scheduled"})+ require.NoError(t, err, "activity failure must not reject accepted work")+ require.NotEmpty(t, run.ActivityID)+ require.NoError(t, svc.Queue.UpdateRun(ctx, run, func(current *st.Run) error {+ current.Status = st.NeedsAttention+ current.Outcome = st.Outcome{Message: "interrupted before applying updates"}+ current.UpdatedAt = time.Now().UTC()+ return nil+ }))+ require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))+ restarted := New(Dependencies{DB: db, Config: &config.Config{}, Activity: activities}).Service()+ require.NoError(t, restarted.Queue.Start(ctx))+ t.Cleanup(func() {+ stopCtx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, restarted.Queue.Stop(stopCtx))+ })+ detail, err := activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)+ require.NoError(t, err)+ require.Equal(t, activitytypes.StatusFailed, detail.Activity.Status)+ require.Contains(t, detail.Activity.LatestMessage, "interrupted")+ persisted, err := restarted.Queue.Get(ctx, "0", run.JobID, run.ID)+ require.NoError(t, err)+ require.Zero(t, persisted.AttemptCount, "repair must not execute blocked work")+ require.Equal(t, run.ActivityID, persisted.ActivityID)+}diff --git a/backend/internal/job/execution.go b/backend/internal/job/execution.goindex 7cc734e8f..e108aa84d 100644--- a/backend/internal/job/execution.go+++ b/backend/internal/job/execution.go@@ -16,6 +16,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/pkg/authz""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils"activitytypes "github.com/getarcaneapp/arcane/types/v2/activity""github.com/getarcaneapp/arcane/types/v2/meta"st "github.com/getarcaneapp/arcane/types/v2/scheduler"@@ -135,12 +136,16 @@ func (s *JobService) executeRunInternal(ctx context.Context, run st.Run) (st.Outerr := s.scheduler.RunBusWatcherNow(ctx, run.JobID)return classifyOutcomeInternal(run.JobID, st.Outcome{}, err)}+ unavailableStatus := st.Canceled+ if run.JobID == "auto-update" && run.AttemptCount > 1 {+ unavailableStatus = st.NeedsAttention+ }job, ok := s.scheduler.GetJob(run.JobID)if !ok {- return st.Outcome{Status: st.Canceled, Message: "Job or target no longer exists"}, nil+ return st.Outcome{Status: unavailableStatus, Message: "Job or target no longer exists"}, nil}if conditional, ok := job.(st.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {- return st.Outcome{Status: st.Canceled, Message: "Job is disabled"}, nil+ return st.Outcome{Status: unavailableStatus, Message: "Job is disabled"}, nil}outcome, err := job.Run(ctx)return classifyOutcomeInternal(run.JobID, outcome, err)@@ -196,6 +201,9 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.O}// A successful activity proves only that target, never the entire batch.for index, target := range run.Outcome.Targets {+ if run.JobID == "auto-update" {+ break+ }if target.Status == st.Succeeded || target.Status == st.Skipped {continue}@@ -224,6 +232,7 @@ func (s *JobService) reconcileRunInternal(ctx context.Context, run st.Run) (st.O}func (s *JobService) runContextInternal(ctx context.Context, run st.Run) context.Context {+ ctx = utils.WithActivityBatchID(ctx, run.ID)ctx = jobcontext.WithExecution(ctx, run, func(target st.TargetOutcome) error {return s.Queue.UpdateRun(ctx, run, func(current *st.Run) error {if current.Status != st.Running || current.Owner != run.Owner {diff --git a/backend/internal/job/module.go b/backend/internal/job/module.goindex b296a7f66..e3021e4be 100644--- a/backend/internal/job/module.go+++ b/backend/internal/job/module.go@@ -3,6 +3,7 @@ package jobimport ("github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -12,6 +13,7 @@ import ()type Dependencies struct {+ Activity *activity.ActivityServiceRoles *role.RoleServiceDB *database.DBSettings *settings.SettingsService@@ -28,6 +30,10 @@ func New(deps Dependencies) *Module {service := NewJobService(deps.DB, deps.Settings, deps.Config)service.environment = deps.Environmentservice.roles = deps.Roles+ service.activity = deps.Activity+ if deps.Activity != nil {+ service.Queue.SetObserver(service)+ }return &Module{service: service,environment: deps.Environment,diff --git a/backend/internal/job/remote_history.go b/backend/internal/job/remote_history.goindex f6338e12b..806679bc3 100644--- a/backend/internal/job/remote_history.go+++ b/backend/internal/job/remote_history.go@@ -71,6 +71,9 @@ func (s *JobService) remoteHistoryInternal(ctx context.Context, environmentID, jprevious := len(runs)for _, run := range response.Runs {run.EnvironmentID = environmentID+ if run.ActivityID != "" {+ run.ActivityEnvironmentID = environmentID+ }runs[run.ID] = run}if len(runs) >= response.Total || len(runs) == previous {@@ -93,6 +96,9 @@ func (s *JobService) GetRun(ctx context.Context, environmentID, jobID, runID strreturn st.Run{}, errors.New("agent returned an inconsistent run identity")}run.EnvironmentID = environmentID+ if run.ActivityID != "" {+ run.ActivityEnvironmentID = environmentID+ }return run, nil}@@ -126,5 +132,8 @@ func (s *JobService) mutateAgentRunInternal(ctx context.Context, environmentID,return st.Run{}, errors.New("agent returned an inconsistent run identity")}run.EnvironmentID = environmentID+ if run.ActivityID != "" {+ run.ActivityEnvironmentID = environmentID+ }return run, nil}diff --git a/backend/internal/job/remote_resolution.go b/backend/internal/job/remote_resolution.gonew file mode 100644index 000000000..e07647a7a--- /dev/null+++ b/backend/internal/job/remote_resolution.go@@ -0,0 +1,101 @@+package job++import (+ "context"+ "encoding/json/v2"+ "net/http"+ "net/url"+ "time"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++func (s *JobService) resolveRemoteRunInternal(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {+ local, localErr := s.Queue.Get(ctx, environmentID, jobID, runID)+ if localErr != nil && !errors.Is(localErr, queue.ErrRunNotFound) {+ return st.Run{}, localErr+ }+ if localErr == nil {+ if local.Status == st.Canceled && local.Resolution != nil {+ return local, nil+ }+ if local.Status != st.NeedsAttention {+ return local, errors.New("only runs needing attention can be resolved")+ }+ if !local.RemoteDeliveryAttempted && !local.RemoteAccepted {+ return s.Queue.Resolve(ctx, environmentID, jobID, runID, actor)+ }+ }+ acknowledged, err := s.resolveAgentReviewInternal(ctx, environmentID, jobID, runID, actor)+ if err != nil {+ return st.Run{}, err+ }+ if localErr != nil {+ acknowledged.EnvironmentID = environmentID+ if acknowledged.ActivityID != "" {+ acknowledged.ActivityEnvironmentID = environmentID+ }+ return acknowledged, nil+ }+ now := time.Now().UTC()+ if err := s.Queue.UpdateRun(ctx, local, func(current *st.Run) error {+ if current.Status != st.NeedsAttention {+ return queue.ErrRunConflict+ }+ outcome := acknowledged.Outcome+ outcome.Status = acknowledged.Status+ current.RemoteOutcome = &outcome+ current.RemoteAccepted = true+ current.RemoteSettled = true+ current.LastConfirmedAt = &now+ return nil+ }); err != nil {+ return st.Run{}, err+ }+ return s.Queue.Resolve(ctx, environmentID, jobID, runID, acknowledged.Resolution.ResolvedBy)+}++// resolveAgentReviewInternal confirms ownership, records review, and settles delivery.+func (s *JobService) resolveAgentReviewInternal(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {+ env, err := s.environment.GetEnvironmentByID(ctx, environmentID)+ if err != nil {+ return st.Run{}, err+ }+ if !env.Enabled {+ return st.Run{}, errors.New("environment is disabled")+ }+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)+ var remote st.Run+ // Read the owner on every request, including retries after a lost response.+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &remote); err != nil {+ return st.Run{}, err+ }+ if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" {+ return st.Run{}, errors.New("agent returned an inconsistent run identity")+ }+ if remote.Status != st.Canceled || remote.Resolution == nil {+ if remote.Status != st.NeedsAttention {+ return st.Run{}, errors.New("agent run must need attention before review resolution")+ }+ body, err := json.Marshal(map[string]string{"resolvedBy": actor})+ if err != nil {+ return st.Run{}, err+ }+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path+"/resolve", body, &remote); err != nil {+ return st.Run{}, err+ }+ }+ if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || remote.Status != st.Canceled || remote.Resolution == nil {+ return st.Run{}, errors.New("agent resolution was not confirmed")+ }+ var acknowledged st.Run+ if err := s.environment.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path+"/ack", nil, &acknowledged); err != nil {+ return st.Run{}, err+ }+ if acknowledged.ID != runID || acknowledged.JobID != jobID || acknowledged.EnvironmentID != "0" || acknowledged.Status != st.Canceled || acknowledged.Resolution == nil || !acknowledged.RemoteSettled {+ return st.Run{}, errors.New("agent resolution acknowledgement was not confirmed")+ }+ return acknowledged, nil+}diff --git a/backend/internal/job/remote_resolution_test.go b/backend/internal/job/remote_resolution_test.gonew file mode 100644index 000000000..e69bbfeff--- /dev/null+++ b/backend/internal/job/remote_resolution_test.go@@ -0,0 +1,78 @@+package job++import (+ "context"+ "encoding/json/v2"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {+ db := setupSettingsTestDBInternal(t)+ require.NoError(t, db.AutoMigrate(&environment.Environment{}))+ q := queue.New(kv.NewKVService(db), nil, nil)+ local, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: "remote"})+ require.NoError(t, err)+ require.NoError(t, q.UpdateRun(t.Context(), local, func(run *st.Run) error {+ run.Status = st.NeedsAttention+ run.RemoteAccepted = true+ run.RemoteDeliveryAttempted = true+ run.Outcome = st.Outcome{Status: st.NeedsAttention, Message: "original failure"}+ return nil+ }))+ remote := st.Run{ID: local.ID, JobID: local.JobID, EnvironmentID: "0", Status: st.Running}+ mutations := 0+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodGet:+ case strings.HasSuffix(r.URL.Path, "/resolve"):+ mutations+++ remote.Status = st.Canceled+ remote.Resolution = &st.RunResolution{ResolvedBy: "operator", ResolvedAt: time.Now().UTC(), Reason: "resolved_after_review"}+ http.Error(w, "lost response", http.StatusServiceUnavailable)+ return+ case strings.HasSuffix(r.URL.Path, "/ack"):+ remote.RemoteSettled = true+ default:+ http.Error(w, "unexpected request", http.StatusBadRequest)+ return+ }+ raw, marshalErr := json.Marshal(remote)+ assert.NoError(t, marshalErr)+ w.Header().Set("Content-Type", "application/json")+ _, writeErr := w.Write(raw)+ assert.NoError(t, writeErr)+ }))+ defer server.Close()+ require.NoError(t, db.Create(&environment.Environment{BaseModel: database.BaseModel{ID: "remote"}, Name: "agent", ApiUrl: server.URL, Enabled: true}).Error)+ svc := &JobService{Queue: q, environment: environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)}+ ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("remote"))+ _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")+ require.ErrorContains(t, err, "agent run must need attention")+ require.Zero(t, mutations)+ remote.Status = st.NeedsAttention+ _, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")+ require.Error(t, err)+ unchanged, err := q.Get(t.Context(), "remote", local.JobID, local.ID)+ require.NoError(t, err)+ require.Equal(t, st.NeedsAttention, unchanged.Status)+ resolved, err := svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")+ require.NoError(t, err)+ require.Equal(t, st.Canceled, resolved.Status)+ require.True(t, resolved.RemoteSettled)+ require.Equal(t, "original failure", resolved.Outcome.Message)+ require.Equal(t, 1, mutations)+}diff --git a/backend/internal/job/resolution.go b/backend/internal/job/resolution.gonew file mode 100644index 000000000..948554503--- /dev/null+++ b/backend/internal/job/resolution.go@@ -0,0 +1,44 @@+package job++import (+ "context"++ "emperror.dev/errors"+ "github.com/danielgtaylor/huma/v2"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/types/v2/jobschedule"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++// ResolveRun records operator review and releases the schedule's blocked run.+func (h *JobSchedulesHandler) ResolveRun(ctx context.Context, input *jobschedule.ResolveRunInput) (*jobschedule.RunOutput, error) {+ actor, _ := middleware.GetUserIDFromContext(ctx)+ permissions, _ := middleware.PermissionsFromContext(ctx)+ if input.Body.ResolvedBy != "" {+ if permissions == nil || !permissions.Sudo || !h.jobService.cfg.AgentMode {+ return nil, huma.Error403Forbidden("only authenticated manager transport may forward an operator identity")+ }+ actor = input.Body.ResolvedBy+ }+ run, err := h.jobService.ResolveRun(ctx, input.ID, input.JobID, input.RunID, actor)+ if err != nil {+ return nil, jobHTTPErrorInternal(err)+ }+ return &jobschedule.RunOutput{Body: run}, nil+}++// ResolveRun authorizes the current operator independently of the original requester.+func (s *JobService) ResolveRun(ctx context.Context, environmentID, jobID, runID, actor string) (st.Run, error) {+ permissions, _ := middleware.PermissionsFromContext(ctx)+ if !permissions.Allows(authz.PermJobsManage, environmentID) {+ return st.Run{}, huma.Error403Forbidden("permission denied: " + authz.PermJobsManage)+ }+ if actor == "" {+ return st.Run{}, errors.New("resolving operator identity is required")+ }+ if environmentID != "0" {+ return s.resolveRemoteRunInternal(ctx, environmentID, jobID, runID, actor)+ }+ return s.Queue.Resolve(ctx, environmentID, jobID, runID, actor)+}diff --git a/backend/internal/job/resolution_test.go b/backend/internal/job/resolution_test.gonew file mode 100644index 000000000..0ff87c00a--- /dev/null+++ b/backend/internal/job/resolution_test.go@@ -0,0 +1,38 @@+package job++import (+ "context"+ "testing"++ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/internal/middleware"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/authz"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/queue"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/stretchr/testify/require"+)++func TestResolveRunChecksCurrentOperatorPermission(t *testing.T) {+ db := setupSettingsTestDBInternal(t)+ svc := &JobService{Queue: queue.New(kv.NewKVService(db), nil, nil)}+ run, err := svc.Queue.Submit(t.Context(), st.Request{JobID: "auto-update", Trigger: "scheduled"})+ require.NoError(t, err)+ require.NoError(t, svc.Queue.UpdateRun(t.Context(), run, func(current *st.Run) error {+ current.Status = st.NeedsAttention+ current.Outcome = st.Outcome{Status: st.NeedsAttention, Message: "interrupted"}+ return nil+ }))+ _, err = svc.ResolveRun(t.Context(), "0", "auto-update", run.ID, "operator")+ require.ErrorContains(t, err, "permission denied")+ otherEnvironment := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("different"))+ _, err = svc.ResolveRun(otherEnvironment, "0", "auto-update", run.ID, "operator")+ require.ErrorContains(t, err, "permission denied")+ authorized := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("0"))+ _, err = svc.ResolveRun(authorized, "0", "auto-update", run.ID, "")+ require.ErrorContains(t, err, "identity")+ resolved, err := svc.ResolveRun(authorized, "0", "auto-update", run.ID, "operator")+ require.NoError(t, err)+ require.Equal(t, st.Canceled, resolved.Status)+ require.Equal(t, "interrupted", resolved.Outcome.Message)+ require.Equal(t, "operator", resolved.Resolution.ResolvedBy)+}diff --git a/backend/internal/job/run_routes.go b/backend/internal/job/run_routes.goindex cc7053f2b..2c58ad8c4 100644--- a/backend/internal/job/run_routes.go+++ b/backend/internal/job/run_routes.go@@ -22,6 +22,7 @@ func (h *JobSchedulesHandler) registerRunRoutesInternal(api huma.API) {middleware.RegisterWithPermission(api, huma.Operation{OperationID: "get-job-run", Method: http.MethodGet, Path: base + "/runs/{runId}", Summary: "Get job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.GetRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "retry-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/retry", Summary: "Retry job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RetryRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "cancel-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/cancel", Summary: "Cancel pending job run", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.CancelRun)+ middleware.RegisterWithPermission(api, huma.Operation{OperationID: "resolve-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/resolve", Summary: "Resolve job run after review", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.ResolveRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "ack-job-run", Method: http.MethodPost, Path: base + "/runs/{runId}/ack", Summary: "Acknowledge remote run completion", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.AcknowledgeRun)middleware.RegisterWithPermission(api, huma.Operation{OperationID: "restart-job-worker", Method: http.MethodPost, Path: base + "/restart", Summary: "Restart continuous job worker", Security: handlerutil.DefaultOperationSecurity()}, authz.PermJobsManage, h.RestartWorker)}@@ -109,6 +110,10 @@ func (h *JobSchedulesHandler) RestartWorker(ctx context.Context, input *jobsched}func jobHTTPErrorInternal(err error) error {+ var status huma.StatusError+ if errors.As(err, &status) {+ return err+ }if errors.Is(err, queue.ErrRunNotFound) {return huma.Error404NotFound("Job run not found")}@@ -129,5 +134,18 @@ func (s *JobService) RetryRun(ctx context.Context, environmentID, jobID, runID sreturn st.Run{}, err}}+ run, err := s.Queue.Get(ctx, environmentID, jobID, runID)+ if err != nil {+ return st.Run{}, err+ }+ if s.scheduler != nil {+ if job, ok := s.scheduler.GetJob(jobID); ok {+ if validator, ok := job.(st.RetryValidator); ok {+ if err := validator.ValidateRetry(ctx, run); err != nil {+ return run, err+ }+ }+ }+ }return s.Queue.Retry(ctx, environmentID, jobID, runID)}diff --git a/backend/internal/job/service.go b/backend/internal/job/service.goindex 3689ef7ad..0cae5a3f4 100644--- a/backend/internal/job/service.go+++ b/backend/internal/job/service.go@@ -4,9 +4,11 @@ import ("context""log/slog""sort"+ "sync""time""emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/environment"@@ -31,6 +33,8 @@ import (// NOTE: This is intentionally separate from settings.SettingsService to keep the API// surface job-focused and to centralize schedule validation/rescheduling.type JobService struct {+ activity *activity.ActivityService+ activityMu sync.MutexQueue *queue.Queuestore *kv.KVServiceenvironment *environment.EnvironmentServicediff --git a/backend/internal/updater/recovery.go b/backend/internal/updater/recovery.gonew file mode 100644index 000000000..105eeee82--- /dev/null+++ b/backend/internal/updater/recovery.go@@ -0,0 +1,104 @@+package updater++import (+ "context"+ "sync"++ "emperror.dev/errors"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"+ arcaneupdater "github.com/getarcaneapp/arcane/types/v2/updater"+ "go.getarcane.app/updater"+)++type updateProgressKeyInternal struct{}+type updateProgressInternal struct {+ mu sync.Mutex+ err error+ selfTriggered bool+ selfID string+}++// RecordUpdateRun persists one updater resource result into Arcane history.+func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.ResourceResult) error {+ var recordErr error+ if s != nil && s.deps.DB != nil {+ recordErr = s.recordRunInternal(ctx, resourceResultFromModuleInternal(result))+ }+ status := schedulertypes.NeedsAttention+ switch result.Status {+ case updater.StatusUpdated, updater.StatusRestarted, updater.StatusUpToDate:+ status = schedulertypes.Succeeded+ case updater.StatusSkipped:+ status = schedulertypes.Skipped+ case updater.StatusChecked, updater.StatusUpdateAvailable:+ status = schedulertypes.NeedsAttention+ case updater.StatusFailed:+ status = schedulertypes.Failed+ }+ progressErr := jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: result.ResourceID, ResourceType: string(result.ResourceType), Status: status, Message: result.Error, ActivityID: activityIDFromContextInternal(ctx)})+ err := errors.Combine(recordErr, progressErr)+ if progress, ok := ctx.Value(updateProgressKeyInternal{}).(*updateProgressInternal); ok && err != nil {+ progress.mu.Lock()+ progress.err = errors.Combine(progress.err, err)+ progress.mu.Unlock()+ }+ return err+}++func (p *updateProgressInternal) completeBatchInternal(ctx context.Context, options arcaneupdater.Options, out *arcaneupdater.Result, batchCompleted bool, err error) error {+ activityID := activityIDFromContextInternal(ctx)+ p.mu.Lock()+ err = errors.Combine(err, p.err)+ recordingFailed := p.err != nil+ _, durableRun := jobcontext.Run(ctx)+ selfTriggered := p.selfTriggered && durableRun+ selfID := p.selfID+ p.mu.Unlock()+ status := schedulertypes.Succeeded+ if !batchCompleted && err == nil {+ err = errors.New("update batch ended without a confirmed result")+ recordingFailed = true+ }+ if err != nil || out.Failed > 0 {+ status = schedulertypes.Partial+ }+ if recordingFailed {+ status = schedulertypes.NeedsAttention+ }+ if selfTriggered {+ err = errors.Combine(err, jobcontext.Progress(ctx, schedulertypes.TargetOutcome{ID: selfID, ResourceType: "container", Status: schedulertypes.NeedsAttention, ActivityID: activityID, Message: "Self-update completion requires review"}))+ status = schedulertypes.NeedsAttention+ err = errors.Combine(err, errors.New("self-update was triggered but completion requires review"))+ }+ batchType := updateBatchTypeInternal(ctx, options, batchCompleted)+ checkpoint := schedulertypes.TargetOutcome{ID: "auto-update", ResourceType: batchType, Status: status, ActivityID: activityID}+ if err != nil {+ checkpoint.Message = err.Error()+ }+ checkpointErr := jobcontext.Progress(ctx, checkpoint)+ err = errors.Combine(err, checkpointErr)+ if err != nil {+ out.Success = false+ }+ if recordingFailed || selfTriggered || checkpointErr != nil {+ err = &schedulertypes.OutcomeError{Outcome: schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: err.Error(), ActivityID: activityID}, Cause: err}+ }+ return err+}++func updateBatchTypeInternal(ctx context.Context, options arcaneupdater.Options, batchCompleted bool) string {+ batchType := "update-batch"+ if previous, ok := jobcontext.Run(ctx); ok && len(options.ResourceIds) > 0 {+ batchType = "update-retry"+ for _, target := range previous.Outcome.Targets {+ if target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Partial) {+ batchType = "update-batch"+ }+ }+ }+ if !batchCompleted {+ batchType = "update-interrupted"+ }+ return batchType+}diff --git a/backend/internal/updater/service.go b/backend/internal/updater/service.goindex d085bf018..7d69628eb 100644--- a/backend/internal/updater/service.go+++ b/backend/internal/updater/service.go@@ -174,10 +174,13 @@ func (s *UpdaterService) registryDigestResolverInternal() updater.RegistryDigest// instead — same activity, events, and cleanup either way.func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater.Options) (out *arcaneupdater.Result, err error) {start := time.Now()+ batchCompleted := falseactivityID := s.startAutoUpdateActivityInternal(ctx, options.DryRun)out = &arcaneupdater.Result{Items: []arcaneupdater.ResourceResult{}, ActivityID: mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()}ctx = s.trackActivityInternal(ctx, activityID)ctx = contextWithActivityIDInternal(ctx, activityID)+ progress := &updateProgressInternal{}+ ctx = context.WithValue(ctx, updateProgressKeyInternal{}, progress)notifyBatch := &containerUpdateBatchInternal{}ctx = context.WithValue(ctx, containerUpdateBatchContextKeyInternal{}, notifyBatch)@@ -190,6 +193,7 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdaterout.Duration = time.Since(start).String()}out.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()+ err = progress.completeBatchInternal(ctx, options, out, batchCompleted, err)s.completeAutoUpdateActivityInternal(ctx, activityID, out, err)}()@@ -233,6 +237,9 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdaterout.ActivityID = mo.EmptyableToOption(strings.TrimSpace(activityID)).ToPointer()s.logResultItemsInternal(ctx, out)}+ if moduleResult == nil && engineErr == nil {+ engineErr = errors.New("updater returned no batch result")+ }if engineErr != nil {err = engineErrreturn out, err@@ -256,6 +263,7 @@ func (s *UpdaterService) ApplyPending(ctx context.Context, options arcaneupdater"duration": out.Duration,"time": time.Now().UTC().Format(time.RFC3339),})+ batchCompleted = truereturn out, nil}@@ -661,14 +669,6 @@ func (s *UpdaterService) ClearImageUpdateRecord(ctx context.Context, record updareturn s.clearImageUpdateRecordForModuleInternal(ctx, record)}-// RecordUpdateRun persists one updater resource result into Arcane history.-func (s *UpdaterService) RecordUpdateRun(ctx context.Context, result updater.ResourceResult) error {- if s == nil || s.deps.DB == nil {- return nil- }- return s.recordRunInternal(ctx, resourceResultFromModuleInternal(result))-}-// ExcludedContainers returns auto-update exclusions from Arcane settings.func (s *UpdaterService) ExcludedContainers(ctx context.Context) ([]string, error) {if s == nil {@@ -723,6 +723,12 @@ func (s *UpdaterService) TriggerSelfUpdate(ctx context.Context, target updater.Sif _, err := s.deps.SelfUpgrade.TriggerUpgradeViaCLI(ctx, s.deps.SystemUser, target); err != nil {return errors.WrapIf(err, "CLI upgrade failed")}+ if progress, ok := ctx.Value(updateProgressKeyInternal{}).(*updateProgressInternal); ok {+ progress.mu.Lock()+ progress.selfTriggered = true+ progress.selfID = target.ContainerID+ progress.mu.Unlock()+ }return nil}diff --git a/backend/internal/updater/service_test.go b/backend/internal/updater/service_test.goindex 9c3f951ff..983cb3972 100644--- a/backend/internal/updater/service_test.go+++ b/backend/internal/updater/service_test.go@@ -5,8 +5,10 @@ import (composetypes "github.com/compose-spec/compose-go/v2/types"projectspkg "github.com/getarcaneapp/arcane/backend/v2/pkg/projects"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/notifications"projecttypes "github.com/getarcaneapp/arcane/types/v2/project"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/backend/v2/internal/settings"@@ -239,7 +241,12 @@ func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {svc, svcErr := NewUpdaterService(db, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil)require.NoError(t, svcErr)+ var checkpoints []schedulertypes.TargetOutcome+ ctx = jobcontext.WithExecution(ctx, schedulertypes.Run{AttemptCount: 1}, func(target schedulertypes.TargetOutcome) error { checkpoints = append(checkpoints, target); return nil })result, err := svc.ApplyPending(ctx, arcaneupdater.Options{DryRun: true})+ require.Len(t, checkpoints, 2)+ assert.Equal(t, schedulertypes.Running, checkpoints[0].Status)+ assert.Equal(t, schedulertypes.Succeeded, checkpoints[1].Status)require.NoError(t, err)require.NotNil(t, result)@@ -248,6 +255,18 @@ func TestUpdaterService_ApplyPendingNoRecordsInternal(t *testing.T) {assert.Zero(t, result.Skipped)assert.Zero(t, result.Failed)assert.Empty(t, result.Items)+ ctx = jobcontext.WithExecution(context.Background(), schedulertypes.Run{AttemptCount: 1}, func(schedulertypes.TargetOutcome) error { return errors.New("checkpoint unavailable") })+ _, err = svc.ApplyPending(ctx, arcaneupdater.Options{DryRun: true})+ require.ErrorContains(t, err, "checkpoint unavailable")+ var outcomeErr *schedulertypes.OutcomeError+ require.ErrorAs(t, err, &outcomeErr)+ assert.Equal(t, schedulertypes.NeedsAttention, outcomeErr.Outcome.Status)+ checkpoints = nil+ ctx = jobcontext.WithExecution(context.Background(), schedulertypes.Run{AttemptCount: 1}, func(target schedulertypes.TargetOutcome) error { checkpoints = append(checkpoints, target); return nil })+ svc.engine = nil+ require.Panics(t, func() { _, _ = svc.ApplyPending(ctx, arcaneupdater.Options{}) })+ require.NotEmpty(t, checkpoints)+ assert.Equal(t, schedulertypes.NeedsAttention, checkpoints[len(checkpoints)-1].Status)}// Type and ResourceIds are Arcane-side scoping the engine never acted on;@@ -645,6 +664,15 @@ func TestUpdaterService_RecordUpdateRunAdapterInternal(t *testing.T) {assert.Equal(t, "nginx:1.2.3", record.OldImageVersions["main"])assert.Equal(t, "nginx:1.2.4", record.NewImageVersions["main"])assert.Equal(t, "test", record.Details["source"])+ progress := &updateProgressInternal{}+ ctx = context.WithValue(ctx, updateProgressKeyInternal{}, progress)+ ctx = jobcontext.WithExecution(ctx, schedulertypes.Run{AttemptCount: 1}, func(target schedulertypes.TargetOutcome) error {+ assert.Equal(t, schedulertypes.Failed, target.Status)+ return errors.New("progress unavailable")+ })+ err = svc.RecordUpdateRun(ctx, updater.ResourceResult{ResourceID: "failed-container", ResourceType: updater.ResourceTypeContainer, Status: updater.StatusFailed})+ require.ErrorContains(t, err, "progress unavailable")+ require.ErrorContains(t, progress.err, "progress unavailable")}func TestUpdaterService_ApplyPending_ProjectFailureDoesNotBlockOtherProjectsInternal(t *testing.T) {@@ -905,8 +933,9 @@ func TestUpdaterService_ApplyPending_RoutesLegacyArcaneServerThroughSelfUpgradeI})svc.engine = engine+ ctx = jobcontext.WithExecution(ctx, schedulertypes.Run{AttemptCount: 1}, nil)result, err := svc.ApplyPending(ctx, arcaneupdater.Options{})- require.NoError(t, err)+ require.ErrorContains(t, err, "self-update was triggered")require.NotNil(t, result)assert.True(t, mockUpgrade.triggerCalled, "scheduled auto-update should use CLI self-upgrade for legacy Arcane server labels")assert.Empty(t, projectUpdater.calls, "legacy Arcane server should not be updated through project services")diff --git a/backend/pkg/libarcane/edge/proxy.go b/backend/pkg/libarcane/edge/proxy.goindex 913d68ef7..5952218c4 100644--- a/backend/pkg/libarcane/edge/proxy.go+++ b/backend/pkg/libarcane/edge/proxy.go@@ -70,7 +70,7 @@ func collectCommandResponseInternal(ctx context.Context, tunnel *AgentTunnel, peif done, status, headers, body, err := state.drainTerminalResponseInternal(pending.ResponseCh, method); done {return status, headers, body, err}- return 0, nil, nil, errors.New("edge tunnel closed while waiting for response")+ return 0, nil, nil, errors.WrapIf(ErrTunnelConnectionClosed, "edge tunnel closed while waiting for response")case err := <-pending.failureCh:if done, status, headers, body, err := state.drainTerminalResponseInternal(pending.ResponseCh, method); done {return status, headers, body, err@@ -78,7 +78,7 @@ func collectCommandResponseInternal(ctx context.Context, tunnel *AgentTunnel, pereturn 0, nil, nil, errcase incoming, ok := <-pending.ResponseCh:if !ok {- return 0, nil, nil, errors.New("edge tunnel response channel closed before a response was received")+ return 0, nil, nil, errors.WrapIf(ErrTunnelConnectionClosed, "edge tunnel response channel closed before a response was received")}if done, status, headers, body, err := state.handleTunnelMessageInternal(method, incoming); done {return status, headers, body, err@@ -127,7 +127,7 @@ func (s *grpcResponseState) drainTerminalResponseInternal(respCh <-chan *TunnelMselect {case incoming, ok := <-respCh:if !ok {- return true, 0, nil, nil, errors.New("edge tunnel response channel closed before a response was received")+ return true, 0, nil, nil, errors.WrapIf(ErrTunnelConnectionClosed, "edge tunnel response channel closed before a response was received")}if done, status, headers, body, err := s.handleTunnelMessageInternal(method, incoming); done {return true, status, headers, body, errdiff --git a/backend/pkg/remenv/client.go b/backend/pkg/remenv/client.goindex 44f4af4b0..a706a59de 100644--- a/backend/pkg/remenv/client.go+++ b/backend/pkg/remenv/client.go@@ -7,6 +7,7 @@ import ("fmt""io""maps"+ "net""net/http""strings""time"@@ -15,10 +16,11 @@ import ()const (- HeaderAPIKey = "X-Api-Key" // #nosec G101: header name, not a credential- HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential- HeaderAuthorization = "Authorization"- bearerScheme = "Bearer "+ ErrEnvironmentUnavailable = errors.Sentinel("edge agent is not connected")+ HeaderAPIKey = "X-Api-Key" // #nosec G101: header name, not a credential+ HeaderAgentToken = "X-Arcane-Agent-Token" // #nosec G101: header name, not a credential+ HeaderAuthorization = "Authorization"+ bearerScheme = "Bearer ")// ExtractBearerToken returns the token portion of an "Authorization: Bearer <token>"@@ -204,6 +206,27 @@ type TransportError struct {Err error}+// Unavailable reports connection failures that permit using locally stored data.+// Invalid requests and failures reading an established response are not outages.+func (e *TransportError) Unavailable() bool {+ if e == nil || errors.Is(e.Err, context.Canceled) {+ return false+ }+ if errors.Is(e.Err, context.DeadlineExceeded) || errors.Is(e.Err, ErrEnvironmentUnavailable) {+ return true+ }+ var networkErr net.Error+ if errors.As(e.Err, &networkErr) && networkErr.Timeout() {+ return true+ }+ var operationErr *net.OpError+ if !errors.As(e.Err, &operationErr) || operationErr.Op != "dial" {+ return false+ }+ var addressErr *net.AddrError+ return !errors.As(operationErr, &addressErr)+}+func (e *TransportError) Error() string {if e == nil || e.Err == nil {return "transport error"diff --git a/backend/pkg/remenv/client_test.go b/backend/pkg/remenv/client_test.goindex 99429d70b..77321f859 100644--- a/backend/pkg/remenv/client_test.go+++ b/backend/pkg/remenv/client_test.go@@ -3,8 +3,11 @@ package remenvimport ("context""errors"+ "io"+ "net""net/http""net/http/httptest"+ "syscall""testing""github.com/stretchr/testify/assert"@@ -122,6 +125,8 @@ func TestClientDoJSON_ClassifiesStatusAndDecodeErrors(t *testing.T) {var statusErr *StatusErrorrequire.ErrorAs(t, err, &statusErr)require.Equal(t, http.StatusBadGateway, statusErr.StatusCode)+ var transportErr *TransportError+ require.False(t, errors.As(err, &transportErr))_, err = client.DoJSON[map[string]any](context.Background(), Request{Method: http.MethodGet,@@ -130,12 +135,13 @@ func TestClientDoJSON_ClassifiesStatusAndDecodeErrors(t *testing.T) {})var decodeErr *DecodeErrorrequire.ErrorAs(t, err, &decodeErr)+ require.False(t, errors.As(err, &transportErr))}func TestClientDo_WrapsTransportErrors(t *testing.T) {client := NewClient(nil, TunnelTransportFuncs{EnsureAvailableFunc: func(ctx context.Context, envID string) error {- return errors.New("not connected")+ return ErrEnvironmentUnavailable},})@@ -148,4 +154,22 @@ func TestClientDo_WrapsTransportErrors(t *testing.T) {var transportErr *TransportErrorrequire.ErrorAs(t, err, &transportErr)require.Contains(t, transportErr.Error(), "not connected")+ require.True(t, transportErr.Unavailable())+ for _, test := range []struct {+ name string+ cause error+ unavailable bool+ }{+ {"deadline", context.DeadlineExceeded, true},+ {"connection refused", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}, true},+ {"response truncated", io.ErrUnexpectedEOF, false},+ {"invalid request", errors.New("invalid URL"), false},+ {"invalid address", &net.OpError{Op: "dial", Err: &net.AddrError{Err: "invalid port", Addr: "agent:invalid"}}, false},+ {"canceled", context.Canceled, false},+ {"canceled dial", &net.OpError{Op: "dial", Err: context.Canceled}, false},+ } {+ t.Run(test.name, func(t *testing.T) {+ require.Equal(t, test.unavailable, (&TransportError{Err: test.cause}).Unavailable())+ })+ }}diff --git a/backend/pkg/scheduler/auto_update_job.go b/backend/pkg/scheduler/auto_update_job.goindex 65bf96989..bbcd35460 100644--- a/backend/pkg/scheduler/auto_update_job.go+++ b/backend/pkg/scheduler/auto_update_job.go@@ -3,6 +3,7 @@ package schedulerimport ("context""log/slog"+ "strings""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler"@@ -61,10 +62,16 @@ func (j *AutoUpdateJob) Schedule(ctx context.Context) string {func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error) {options := updatertypes.Options{}unresolvedTargets := false- if previous, ok := jobcontext.Run(ctx); ok {+ if previous, ok := jobcontext.Run(ctx); ok && (previous.AttemptCount > 1 || len(previous.Outcome.Targets) > 0) {var priorOutcome schedulertypes.Outcomeoptions, priorOutcome, unresolvedTargets = autoUpdateRetryOptionsInternal(previous)if priorOutcome.Status != "" {+ if priorOutcome.ActivityID == "" {+ priorOutcome.ActivityID = previous.Outcome.ActivityID+ }+ if priorOutcome.Status == schedulertypes.NeedsAttention && previous.Outcome.Message != "" {+ priorOutcome.Message += ": " + previous.Outcome.Message+ }return priorOutcome, nil}}@@ -72,6 +79,9 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)enabled := j.settingsService.GetBoolSetting(ctx, "autoUpdate", false)pollingEnabled := j.settingsService.GetBoolSetting(ctx, "pollingEnabled", true)if !enabled || !pollingEnabled {+ if len(options.ResourceIds) > 0 {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Auto-update is disabled; review this run before resolving it or retrying"}, nil+ }slog.DebugContext(ctx, "auto-update disabled or polling disabled; skipping run","autoUpdate", enabled, "pollingEnabled", pollingEnabled)return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil@@ -83,6 +93,9 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)return schedulertypes.Outcome{}, err}if !admitted {+ if len(options.ResourceIds) > 0 {+ return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Another update is active; review this run before retrying"}, nil+ }slog.WarnContext(ctx, "auto-update run still in progress; skipping overlapping run")return schedulertypes.Outcome{Status: schedulertypes.Skipped}, nil}@@ -101,32 +114,7 @@ func (j *AutoUpdateJob) Run(ctx context.Context) (schedulertypes.Outcome, error)return schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Updater returned no operation result"}, nil}- slog.InfoContext(ctx, "auto-update run completed",- "checked", result.Checked,- "updated", result.Updated,- "restarted", result.Restarted,- "skipped", result.Skipped,- "failed", result.Failed,- )- outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}- if result.ActivityID != nil {- outcome.ActivityID = *result.ActivityID- }- for _, item := range result.Items {- status := schedulertypes.Succeeded- if item.Status == "failed" {- status = schedulertypes.Failed- }- if item.Status == "skipped" {- status = schedulertypes.Skipped- }- outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: item.ResourceID, ResourceType: item.ResourceType, Status: status, Message: item.Error})- }- if result.Failed > 0 || err != nil || unresolvedTargets {- outcome.Status = schedulertypes.Partial- outcome.Message = "Some updates failed; review target outcomes before retrying"- }- return outcome, err+ return autoUpdateOutcomeInternal(ctx, result, unresolvedTargets, err)}func (j *AutoUpdateJob) Reschedule(ctx context.Context) error {@@ -135,13 +123,25 @@ func (j *AutoUpdateJob) Reschedule(ctx context.Context) error {}func (j *AutoUpdateJob) Reconcile(_ context.Context, previous schedulertypes.Run) (schedulertypes.Outcome, error) {- return jobcontext.ConfirmedTarget(previous, "auto-update"), nil+ confirmed := confirmedAutoUpdateInternal(previous)+ if confirmed.Status != schedulertypes.Succeeded {+ confirmed.Message = "Interrupted auto-update has no confirmed full-batch completion; review and resolve this run to resume the schedule"+ if previous.Outcome.Message != "" {+ confirmed.Message += ": " + previous.Outcome.Message+ }+ }+ return confirmed, nil}func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Options, schedulertypes.Outcome, bool) {options := updatertypes.Options{}- unresolvedTargets := false- confirmed := jobcontext.ConfirmedTarget(previous, "auto-update")+ unresolvedTargets := true+ for _, target := range previous.Outcome.Targets {+ if target.ID == "auto-update" && target.ResourceType == "update-batch" && (target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Partial) {+ unresolvedTargets = false+ }+ }+ confirmed := confirmedAutoUpdateInternal(previous)if confirmed.Status == schedulertypes.Succeeded {return options, confirmed, false}@@ -154,7 +154,7 @@ func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Oif target.Status == schedulertypes.Succeeded || target.Status == schedulertypes.Skipped {continue}- if target.ResourceType != "container" || target.Status != schedulertypes.Failed {+ if target.ResourceType != "container" || target.Status != schedulertypes.Failed || strings.TrimSpace(target.ID) == "" {unresolvedTargets = truecontinue}@@ -164,10 +164,82 @@ func autoUpdateRetryOptionsInternal(previous schedulertypes.Run) (updatertypes.Oif unresolvedTargets {return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Remaining image or unconfirmed targets need review before retrying", Targets: previous.Outcome.Targets}, true}- return options, schedulertypes.Outcome{Status: schedulertypes.Succeeded, Targets: previous.Outcome.Targets}, false+ return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "Target results do not confirm full-batch completion; review and resolve this run", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true+ }+ if !hasResourceTargets {+ return options, schedulertypes.Outcome{Status: schedulertypes.NeedsAttention, Message: "The previous update has no confirmed retry targets; review and resolve this run to resume the schedule", ActivityID: previous.Outcome.ActivityID, Targets: previous.Outcome.Targets}, true}if len(options.ResourceIds) > 0 {options.Type = "container"}return options, schedulertypes.Outcome{}, unresolvedTargets}++// ValidateRetry restricts replay to explicitly failed container updates.+func (j *AutoUpdateJob) ValidateRetry(_ context.Context, previous schedulertypes.Run) error {+ options, outcome, _ := autoUpdateRetryOptionsInternal(previous)+ if outcome.Status != "" || len(options.ResourceIds) == 0 {+ return errors.New("auto-update has no safely retryable failed containers; review and resolve the run")+ }+ return nil+}++func autoUpdateOutcomeInternal(ctx context.Context, result *updatertypes.Result, unresolvedTargets bool, err error) (schedulertypes.Outcome, error) {+ slog.InfoContext(ctx, "auto-update run completed",+ "checked", result.Checked,+ "updated", result.Updated,+ "restarted", result.Restarted,+ "skipped", result.Skipped,+ "failed", result.Failed,+ )+ outcome := schedulertypes.Outcome{Status: schedulertypes.Succeeded}+ if result.ActivityID != nil {+ outcome.ActivityID = *result.ActivityID+ }+ for _, item := range result.Items {+ status := schedulertypes.Succeeded+ if item.Status == "failed" {+ status = schedulertypes.Failed+ }+ if item.Status == "skipped" {+ status = schedulertypes.Skipped+ }+ outcome.Targets = append(outcome.Targets, schedulertypes.TargetOutcome{ID: item.ResourceID, ResourceType: item.ResourceType, Status: status, Message: item.Error, ActivityID: outcome.ActivityID})+ }+ if result.Failed > 0 || err != nil || unresolvedTargets {+ outcome.Status = schedulertypes.Partial+ outcome.Message = "Some updates failed; review target outcomes before retrying"+ if err != nil {+ outcome.Message += ": " + err.Error()+ }+ }+ if unresolvedTargets {+ outcome.Status = schedulertypes.NeedsAttention+ outcome.Message = "Unconfirmed targets remain; review and resolve this run to resume the schedule"+ }+ var outcomeErr *schedulertypes.OutcomeError+ if errors.As(err, &outcomeErr) {+ outcome.Status = outcomeErr.Outcome.Status+ outcome.Message = outcomeErr.Outcome.Message+ outcome.Targets = nil+ }+ return outcome, err+}++func confirmedAutoUpdateInternal(previous schedulertypes.Run) schedulertypes.Outcome {+ confirmed := jobcontext.ConfirmedTarget(previous, "auto-update")+ for _, target := range previous.Outcome.Targets {+ if target.ID == "auto-update" && target.ResourceType != "update-batch" {+ confirmed.Status = schedulertypes.NeedsAttention+ }+ }+ if confirmed.Status == schedulertypes.Succeeded {+ for _, target := range previous.Outcome.Targets {+ if target.Status != schedulertypes.Succeeded && target.Status != schedulertypes.Skipped {+ confirmed.Status = schedulertypes.NeedsAttention+ break+ }+ }+ }+ return confirmed+}diff --git a/backend/pkg/scheduler/auto_update_job_test.go b/backend/pkg/scheduler/auto_update_job_test.goindex 2c57b2180..7585db15e 100644--- a/backend/pkg/scheduler/auto_update_job_test.go+++ b/backend/pkg/scheduler/auto_update_job_test.go@@ -6,6 +6,8 @@ import ("testing""time"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/jobcontext"+ schedulertypes "github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/updater""github.com/stretchr/testify/require")@@ -76,4 +78,46 @@ func TestAutoUpdateJob_OverlappingRunIsSkippedInternal(t *testing.T) {// The guard resets once the run finishes.job.Run(ctx)require.Equal(t, int32(2), applier.calls.Load())+ for _, tc := range []struct {+ name string+ targets []schedulertypes.TargetOutcome+ retryIDs []string+ status schedulertypes.RunStatus+ unresolved bool+ }{+ {name: "missing progress", unresolved: true, status: schedulertypes.NeedsAttention},+ {name: "image pull alone", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "image", ResourceType: "image", Status: schedulertypes.Succeeded}}, status: schedulertypes.NeedsAttention},+ {name: "failed containers only", unresolved: true, targets: []schedulertypes.TargetOutcome{{ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}, {ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}, {ID: "image", ResourceType: "image", Status: schedulertypes.Failed}}, retryIDs: []string{"failed"}},+ {name: "completed failed batch retry", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-batch", Status: schedulertypes.Partial}, {ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}}, retryIDs: []string{"failed"}},+ {name: "scoped retry cannot confirm original batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-retry", Status: schedulertypes.Succeeded}, {ID: "done", ResourceType: "container", Status: schedulertypes.Succeeded}}, status: schedulertypes.NeedsAttention, unresolved: true},+ {name: "completed batch", targets: []schedulertypes.TargetOutcome{{ID: "auto-update", ResourceType: "update-batch", Status: schedulertypes.Succeeded}}, status: schedulertypes.Succeeded},+ } {+ t.Run(tc.name, func(t *testing.T) {+ previous := schedulertypes.Run{AttemptCount: 2, Outcome: schedulertypes.Outcome{Targets: tc.targets}}+ options, outcome, unresolved := autoUpdateRetryOptionsInternal(previous)+ require.Equal(t, tc.unresolved, unresolved)+ require.Equal(t, tc.retryIDs, options.ResourceIds)+ require.Equal(t, tc.status, outcome.Status)+ if len(tc.retryIDs) == 0 {+ require.Error(t, job.ValidateRetry(ctx, previous))+ } else {+ require.NoError(t, job.ValidateRetry(ctx, previous))+ }+ reconciled, err := job.Reconcile(ctx, previous)+ require.NoError(t, err)+ if tc.status == schedulertypes.Succeeded {+ require.Equal(t, schedulertypes.Succeeded, reconciled.Status)+ } else {+ require.Equal(t, schedulertypes.NeedsAttention, reconciled.Status)+ }+ })+ }++ require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoUpdate", false))+ retryCtx := jobcontext.WithExecution(ctx, schedulertypes.Run{AttemptCount: 2, Outcome: schedulertypes.Outcome{Targets: []schedulertypes.TargetOutcome{{ID: "failed", ResourceType: "container", Status: schedulertypes.Failed}}}}, nil)+ outcome, err := job.Run(retryCtx)+ require.NoError(t, err)+ require.Equal(t, schedulertypes.NeedsAttention, outcome.Status)+ require.Equal(t, int32(2), applier.calls.Load())+}diff --git a/backend/pkg/scheduler/queue/activity.go b/backend/pkg/scheduler/queue/activity.gonew file mode 100644index 000000000..c171456e7--- /dev/null+++ b/backend/pkg/scheduler/queue/activity.go@@ -0,0 +1,69 @@+package queue++import (+ "context"+ "log/slog"+ "time"++ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++// SetObserver connects activity projection before the queue starts accepting work.+func (q *Queue) SetObserver(observer st.RunObserver) {+ q.observer = observer+}++func (q *Queue) associateActivityInternal(run *st.Run) {+ if q.observer != nil && run.ActivityID == "" {+ run.ActivityID = q.observer.ActivityID(*run)+ }+ if run.ActivityID != "" {+ run.ActivityEnvironmentID = run.EnvironmentID+ }+}++func (q *Queue) observeRunInternal(ctx context.Context, run st.Run) {+ if q.observer == nil {+ return+ }+ // Serialize repair bookkeeping with notifications. The observer rereads the+ // durable run, so delayed notifications cannot restore an older activity state.+ q.observerMu.Lock()+ defer q.observerMu.Unlock()+ q.syncActivityInternal(ctx, run)+ if len(q.pendingSync) > 0 {+ q.signalInternal()+ }+}++func (q *Queue) syncActivityInternal(ctx context.Context, run st.Run) {+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)+ defer cancel()+ key := queueKeyInternal(run.EnvironmentID, run.JobID) + "/" + run.ID+ if err := q.observer.SyncRunActivity(ctx, run); err != nil {+ if q.pendingSync == nil {+ q.pendingSync = make(map[string]st.Run)+ }+ q.pendingSync[key] = run+ slog.ErrorContext(ctx, "Job activity synchronization failed; will retry", "runId", run.ID, "error", err)+ return+ }+ delete(q.pendingSync, key)+}++func (q *Queue) retryActivitySyncInternal(ctx context.Context) {+ q.observerMu.Lock()+ defer q.observerMu.Unlock()+ for _, run := range q.pendingSync {+ if ctx.Err() != nil {+ return+ }+ q.syncActivityInternal(ctx, run)+ }+}++func (q *Queue) hasPendingActivitySyncInternal() bool {+ q.observerMu.Lock()+ defer q.observerMu.Unlock()+ return len(q.pendingSync) > 0+}diff --git a/backend/pkg/scheduler/queue/activity_test.go b/backend/pkg/scheduler/queue/activity_test.gonew file mode 100644index 000000000..e9e94b641--- /dev/null+++ b/backend/pkg/scheduler/queue/activity_test.go@@ -0,0 +1,61 @@+package queue++import (+ "context"+ "sync/atomic"+ "testing"+ "time"++ "emperror.dev/errors"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/stretchr/testify/require"+)++type activityObserverFakeInternal struct {+ unavailable atomic.Bool+ observed atomic.Int32+}++func (o *activityObserverFakeInternal) ActivityID(run st.Run) string { return run.ID }++func (o *activityObserverFakeInternal) SyncRunActivity(context.Context, st.Run) error {+ if o.unavailable.Load() {+ return errors.New("activity storage unavailable")+ }+ o.observed.Add(1)+ return nil+}++func TestActivityFailureNeverReplaysAcceptedExecution(t *testing.T) {+ q := newResolutionQueueInternal(t)+ observer := &activityObserverFakeInternal{}+ observer.unavailable.Store(true)+ q.SetObserver(observer)+ var executions atomic.Int32+ q.execute = func(context.Context, st.Run) (st.Outcome, error) {+ executions.Add(1)+ return st.Outcome{Status: st.Succeeded}, nil+ }+ run, err := q.Submit(t.Context(), st.Request{JobID: "test-job", Trigger: "manual"})+ require.NoError(t, err)+ require.Equal(t, run.ID, run.ActivityID)+ require.NoError(t, q.Start(t.Context()))+ t.Cleanup(func() {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second)+ defer cancel()+ require.NoError(t, q.Stop(ctx))+ })+ require.Eventually(t, func() bool {+ persisted, getErr := q.Get(t.Context(), "0", run.JobID, run.ID)+ return getErr == nil && persisted.Status == st.Succeeded+ }, time.Second, time.Millisecond)+ require.True(t, q.hasPendingActivitySyncInternal())+ observer.unavailable.Store(false)+ q.retryActivitySyncInternal(t.Context())+ require.False(t, q.hasPendingActivitySyncInternal())+ require.Positive(t, observer.observed.Load())+ duplicate, err := q.Submit(t.Context(), st.Request{JobID: run.JobID, RunID: run.ID, Trigger: "manual"})+ require.NoError(t, err)+ require.Equal(t, st.Succeeded, duplicate.Status)+ require.Equal(t, int32(1), executions.Load())+}diff --git a/backend/pkg/scheduler/queue/queue.go b/backend/pkg/scheduler/queue/queue.goindex c6a8da88a..95ce64742 100644--- a/backend/pkg/scheduler/queue/queue.go+++ b/backend/pkg/scheduler/queue/queue.go@@ -33,6 +33,9 @@ type Queue struct {workers sync.WaitGroupexecute func(context.Context, st.Run) (st.Outcome, error)reconcile func(context.Context, st.Run) (st.Outcome, error)+ observer st.RunObserver+ observerMu sync.Mutex+ pendingSync map[string]st.Run}// Submit persists acceptance before waking the dispatcher. Run IDs deduplicate@@ -62,6 +65,7 @@ func (q *Queue) Submit(ctx context.Context, request st.Request) (st.Run, error)}now := time.Now().UTC()result := st.Run{ID: request.RunID, JobID: request.JobID, EnvironmentID: request.EnvironmentID, Trigger: request.Trigger, RequestedBy: request.RequestedBy, RequestedWithKey: request.RequestedWithKey, RemoteAccepted: request.Trigger == "remote", Status: st.Queued, CreatedAt: now, UpdatedAt: now}+ q.associateActivityInternal(&result)err := q.mutateInternal(ctx, request.EnvironmentID, request.JobID, func(record *st.QueueRecord) error {if request.Trigger == "scheduled" || request.Trigger == "recovery" {record.LastEnqueuedAt = now@@ -85,6 +89,7 @@ func (q *Queue) Submit(ctx context.Context, request st.Request) (st.Run, error)return nil})if err == nil {+ q.observeRunInternal(ctx, result)q.signalInternal()}return result, err@@ -103,6 +108,7 @@ func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next timdefer q.mu.Unlock()first := !q.checkpointed[jobID]recovered := false+ var recoveredRun st.Runerr := q.mutateInternal(ctx, "0", jobID, func(record *st.QueueRecord) error {now := time.Now().UTC()if first && record.Schedule == schedule && !record.NextRun.IsZero() && !record.NextRun.After(now) && record.LastEnqueuedAt.Before(record.NextRun) {@@ -116,7 +122,8 @@ func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next timif !pending {recovered = truerecord.LastEnqueuedAt = now- record.Runs = append(record.Runs, st.Run{ID: uuid.NewString(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now})+ recoveredRun = st.Run{ID: uuid.NewString(), JobID: jobID, EnvironmentID: "0", Trigger: "recovery", Status: st.Queued, CreatedAt: now, UpdatedAt: now}+ record.Runs = append(record.Runs, recoveredRun)}}record.Schedule = schedule@@ -126,6 +133,7 @@ func (q *Queue) Checkpoint(ctx context.Context, jobID, schedule string, next timif err == nil {q.checkpointed[jobID] = trueif recovered {+ q.observeRunInternal(ctx, recoveredRun)q.signalInternal()}}@@ -144,9 +152,19 @@ func (q *Queue) Start(ctx context.Context) error {return errors.New("job executor unavailable")}// Fail startup if persisted state cannot be read. Never replace it with empty state.- if _, err := q.Records(ctx); err != nil {+ records, err := q.Records(ctx)+ if err != nil {return err}+ for _, record := range records {+ for _, run := range record.Runs {+ if q.observer != nil {+ if err := q.UpdateRun(ctx, run, func(*st.Run) error { return nil }); err != nil {+ return err+ }+ }+ }+ }runCtx, cancel := context.WithCancel(ctx)q.cancel = cancelq.started = true@@ -180,6 +198,7 @@ func (q *Queue) dispatchInternal(ctx context.Context) {defer q.workers.Wait()nextRetention := time.Now().Add(time.Hour)for ctx.Err() == nil {+ q.retryActivitySyncInternal(ctx)if !time.Now().Before(nextRetention) {if err := q.pruneInternal(ctx); err != nil && ctx.Err() == nil {slog.ErrorContext(ctx, "Job retention failed", "error", err)@@ -192,6 +211,9 @@ func (q *Queue) dispatchInternal(ctx context.Context) {nextRetry = time.Now().Add(5 * time.Second)}nextWake := nextRetention+ if q.hasPendingActivitySyncInternal() {+ nextWake = time.Now().Add(5 * time.Second)+ }if !nextRetry.IsZero() && nextRetry.Before(nextWake) {nextWake = nextRetry}diff --git a/backend/pkg/scheduler/queue/queue_store.go b/backend/pkg/scheduler/queue/queue_store.goindex 92df79e87..4dceaa00f 100644--- a/backend/pkg/scheduler/queue/queue_store.go+++ b/backend/pkg/scheduler/queue/queue_store.go@@ -38,6 +38,9 @@ func (q *Queue) mutateInternal(ctx context.Context, environmentID, jobID string,if err := change(&record); err != nil {return err}+ for index := range record.Runs {+ q.associateActivityInternal(&record.Runs[index])+ }next, err := json.Marshal(record)if err != nil {return err@@ -74,6 +77,9 @@ func (q *Queue) Records(ctx context.Context) ([]st.QueueRecord, error) {if err := json.Unmarshal([]byte(entry.Value), &record); err != nil {return nil, errors.WrapIff(err, "decode job queue %s", entry.Key)}+ for index := range record.Runs {+ q.associateActivityInternal(&record.Runs[index])+ }records = append(records, record)}return records, nil@@ -95,10 +101,12 @@ func (q *Queue) Get(ctx context.Context, environmentID, jobID, runID string) (st}for _, run := range record.Runs {if run.ID == runID {+ q.associateActivityInternal(&run)return run, nil}}if run, ok := record.Receipts[runID]; ok {+ q.associateActivityInternal(&run)return run, nil}return st.Run{}, ErrRunNotFound@@ -158,6 +166,9 @@ func (q *Queue) UpdateRun(ctx context.Context, run st.Run, change func(*st.Run)if err == nil && wake {q.signalInternal()}+ if err == nil {+ q.observeRunInternal(ctx, run)+ }return err}diff --git a/backend/pkg/scheduler/queue/resolution.go b/backend/pkg/scheduler/queue/resolution.gonew file mode 100644index 000000000..f1cec1cee--- /dev/null+++ b/backend/pkg/scheduler/queue/resolution.go@@ -0,0 +1,47 @@+package queue++import (+ "context"+ "time"++ "emperror.dev/errors"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+)++// Resolve releases reviewed, inactive work without discarding its execution evidence.+func (q *Queue) Resolve(ctx context.Context, environmentID, jobID, runID, resolvedBy string) (st.Run, error) {+ q.mu.Lock()+ defer q.mu.Unlock()+ if q.active[queueKeyInternal(environmentID, jobID)] {+ return st.Run{}, errors.New("job is still active")+ }+ run, err := q.Get(ctx, environmentID, jobID, runID)+ if err != nil {+ return run, err+ }+ err = q.UpdateRun(ctx, run, func(current *st.Run) error {+ if current.Status == st.Canceled && current.Resolution != nil {+ run = *current+ return nil+ }+ if current.Status != st.NeedsAttention {+ return errors.New("only runs needing attention can be resolved")+ }+ if current.EnvironmentID != "0" && (current.RemoteDeliveryAttempted || current.RemoteAccepted) && !current.RemoteSettled {+ return errors.New("remote execution must be confirmed and acknowledged before resolution")+ }+ now := time.Now().UTC()+ current.Status = st.Canceled+ current.Resolution = &st.RunResolution{ResolvedBy: resolvedBy, ResolvedAt: now, Reason: "resolved_after_review"}+ current.UpdatedAt = now+ current.FinishedAt = &now+ current.NextAttempt = nil+ current.Owner = ""+ run = *current+ return nil+ })+ if err == nil {+ q.signalInternal()+ }+ return run, err+}diff --git a/backend/pkg/scheduler/queue/resolution_test.go b/backend/pkg/scheduler/queue/resolution_test.gonew file mode 100644index 000000000..43f9db08f--- /dev/null+++ b/backend/pkg/scheduler/queue/resolution_test.go@@ -0,0 +1,103 @@+package queue++import (+ "context"+ "testing"++ "github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ st "github.com/getarcaneapp/arcane/types/v2/scheduler"+ "github.com/libtnb/sqlite"+ "github.com/stretchr/testify/require"+ "gorm.io/gorm"+)++func newResolutionQueueInternal(t *testing.T) *Queue {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}))+ sqlDB, err := db.DB()+ require.NoError(t, err)+ sqlDB.SetMaxOpenConns(1)+ t.Cleanup(func() { require.NoError(t, sqlDB.Close()) })+ return New(kv.NewKVService(&database.DB{DB: db}), nil, nil)+}++func TestResolvePreservesEvidenceAndUnblocksSchedule(t *testing.T) {+ q := newResolutionQueueInternal(t)+ ctx := t.Context()+ run, err := q.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "manual"})+ require.NoError(t, err)+ evidence := st.Outcome{Status: st.NeedsAttention, Message: "original failure", Targets: []st.TargetOutcome{{ID: "container", Status: st.Succeeded}}}+ require.NoError(t, q.UpdateRun(ctx, run, func(current *st.Run) error {+ current.Status = st.NeedsAttention+ current.Outcome = evidence+ current.AttemptCount = 1+ current.Attempts = []st.Attempt{{Number: 1, Outcome: evidence}}+ return nil+ }))+ pending, err := q.Submit(ctx, st.Request{JobID: "auto-update", Trigger: "manual"})+ require.NoError(t, err)+ history, err := q.List(ctx, "0", "auto-update", 1, 20)+ require.NoError(t, err)+ selected, _ := q.nextRunInternal(history.Runs)+ require.Nil(t, selected)+ for len(q.wake) > 0 {+ <-q.wake+ }+ resolved, err := q.Resolve(ctx, "0", "auto-update", run.ID, "operator")+ require.NoError(t, err)+ require.Equal(t, st.Canceled, resolved.Status)+ require.Equal(t, evidence, resolved.Outcome)+ require.Len(t, resolved.Attempts, 1)+ require.Equal(t, "operator", resolved.Resolution.ResolvedBy)+ require.Equal(t, "resolved_after_review", resolved.Resolution.Reason)+ require.NotZero(t, resolved.Resolution.ResolvedAt)+ select {+ case <-q.wake:+ default:+ t.Fatal("resolution did not wake pending work")+ }+ history, err = q.List(ctx, "0", "auto-update", 1, 20)+ require.NoError(t, err)+ selected, _ = q.nextRunInternal(history.Runs)+ require.NotNil(t, selected)+ require.Equal(t, pending.ID, selected.ID)+ duplicate, err := q.Resolve(ctx, "0", "auto-update", run.ID, "different operator")+ require.NoError(t, err)+ require.Equal(t, resolved, duplicate)+}++func TestResolveRejectsUnsafeStates(t *testing.T) {+ for _, test := range []struct {+ name string+ status st.RunStatus+ active bool+ environment string+ delivered bool+ }{+ {name: "queued", status: st.Queued, environment: "0"},+ {name: "running", status: st.Running, environment: "0"},+ {name: "active worker", status: st.NeedsAttention, active: true, environment: "0"},+ {name: "uncertain remote", status: st.NeedsAttention, environment: "remote", delivered: true},+ } {+ t.Run(test.name, func(t *testing.T) {+ q := newResolutionQueueInternal(t)+ run, err := q.Submit(t.Context(), st.Request{JobID: "auto-update", EnvironmentID: test.environment})+ require.NoError(t, err)+ require.NoError(t, q.UpdateRun(t.Context(), run, func(current *st.Run) error {+ current.Status = test.status+ current.RemoteDeliveryAttempted = test.delivered+ return nil+ }))+ q.active[queueKeyInternal(test.environment, "auto-update")] = test.active+ _, err = q.Resolve(context.Background(), test.environment, "auto-update", run.ID, "operator")+ require.Error(t, err)+ stored, err := q.Get(t.Context(), test.environment, "auto-update", run.ID)+ require.NoError(t, err)+ require.Equal(t, test.status, stored.Status)+ require.Nil(t, stored.Resolution)+ })+ }+}diff --git a/backend/resources/migrations/postgres/084_route_job_activities_to_environment.sql b/backend/resources/migrations/postgres/084_route_job_activities_to_environment.sqlnew file mode 100644index 000000000..70aaec72d--- /dev/null+++ b/backend/resources/migrations/postgres/084_route_job_activities_to_environment.sql@@ -0,0 +1,9 @@+-- +goose Up+UPDATE activities+SET environment_id = CAST(metadata AS jsonb)->>'environmentId'+WHERE type = 'job_run'+ AND jsonb_typeof(CAST(metadata AS jsonb)->'environmentId') = 'string'+ AND CAST(metadata AS jsonb)->>'environmentId' <> '';++-- +goose Down+UPDATE activities SET environment_id = '0' WHERE type = 'job_run';diff --git a/backend/resources/migrations/sqlite/084_route_job_activities_to_environment.sql b/backend/resources/migrations/sqlite/084_route_job_activities_to_environment.sqlnew file mode 100644index 000000000..27be8946c--- /dev/null+++ b/backend/resources/migrations/sqlite/084_route_job_activities_to_environment.sql@@ -0,0 +1,9 @@+-- +goose Up+UPDATE activities+SET environment_id = json_extract(metadata, '$.environmentId')+WHERE type = 'job_run'+ AND json_type(metadata, '$.environmentId') = 'text'+ AND json_extract(metadata, '$.environmentId') <> '';++-- +goose Down+UPDATE activities SET environment_id = '0' WHERE type = 'job_run';diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 878c40e56..31d6caf9a 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -3511,5 +3511,13 @@"editor_compose_updater_constraint_detail": "Version constraint","editor_compose_updater_constraint_info": "Semantic version constraint for tag updates, such as 3.x or >=3.1.0 <4.0.0. Without a constraint, updates stay within the current major, or the current minor for 0.x.","editor_compose_updater_tag_pattern_detail": "Tag pattern",- "editor_compose_updater_tag_pattern_info": "Regular expression matched against the whole tag. Use a named version capture to select a variant, such as (?P<version>\\d+\\.\\d+\\.\\d+)-alpine."+ "editor_compose_updater_tag_pattern_info": "Regular expression matched against the whole tag. Use a named version capture to select a variant, such as (?P<version>\\d+\\.\\d+\\.\\d+)-alpine.",+ "activity_type_job_run": "Automation run",+ "jobs_resolve_run": "Resolve after review",+ "jobs_resolve_description": "Confirm you have reviewed this interrupted run and checked its results. Resolving it preserves its history and allows later scheduled work to proceed. It does not run updates or mark this run successful.",+ "jobs_resolution_details": "Resolved by {user} on {date}",+ "jobs_activity_summary": "Run activity",+ "jobs_activity_output": "Operation output",+ "jobs_attention_schedule_blocked": "This run needs review and blocks later runs of this job.",+ "jobs_resolution_reason": "Resolution reason"}diff --git a/frontend/src/lib/components/activity/activity-batch-item.svelte b/frontend/src/lib/components/activity/activity-batch-item.svelteindex 929103c93..e56a7e03f 100644--- a/frontend/src/lib/components/activity/activity-batch-item.svelte+++ b/frontend/src/lib/components/activity/activity-batch-item.svelte@@ -8,6 +8,7 @@import {activityStatusAccentClass,activityStatusLabel,+ activityRunStatusLabel,activityStatusVariant,activityTypeIcon,activityTypeLabel@@ -22,7 +23,8 @@} = $props();// Batch groups are constructed with at least one member.- const leadActivity = $derived(group.items[0]!);+ const jobActivity = $derived(group.items.find((item) => item.type === 'job_run'));+ const leadActivity = $derived(jobActivity ?? group.items[0]!);const IconComponent = $derived(activityTypeIcon(leadActivity.type));const isActive = $derived(group.status === 'running' || group.status === 'queued');</script>@@ -57,6 +59,7 @@<span class="truncate text-sm font-semibold text-foreground">{activityTypeLabel(leadActivity.type)}</span><span class="shrink-0 text-[11px] text-muted-foreground/70">· {m.activity_batch_items({ count: group.total })}</span></div>+ {#if jobActivity?.resourceName}<div class="truncate text-xs text-muted-foreground">{jobActivity.resourceName}</div>{/if}<div class="flex min-w-0 flex-wrap items-center gap-x-1.5 text-xs text-muted-foreground"><span>{m.activity_batch_done_of_total({ done: group.done, total: group.total })}</span>{#if group.failed > 0}@@ -65,7 +68,9 @@{/if}</div></div>- <Badge variant={activityStatusVariant(group.status)} size="sm">{activityStatusLabel(group.status)}</Badge>+ <Badge variant={activityStatusVariant(group.status)} size="sm"+ >{jobActivity ? activityRunStatusLabel(jobActivity) : activityStatusLabel(group.status)}</Badge+ ></div>{#if isActive}diff --git a/frontend/src/lib/components/activity/activity-center.svelte b/frontend/src/lib/components/activity/activity-center.svelteindex de584d38c..d2fde845d 100644--- a/frontend/src/lib/components/activity/activity-center.svelte+++ b/frontend/src/lib/components/activity/activity-center.svelte@@ -86,7 +86,7 @@{#snippet activityRow(activity: Activity, child: boolean)}{@const expanded = activityStore.isExpanded(activity.id)}- {@const cancelable = activity.status === 'running' || activity.status === 'queued'}+ {@const cancelable = activity.type !== 'job_run' && (activity.status === 'running' || activity.status === 'queued')}<div class="group/activity relative"><Collapsible.Root open={expanded} onOpenChange={(open) => activityStore.setActivityExpanded(activity.id, open)}><div class="relative">@@ -125,6 +125,7 @@{@render activityRow(group.activity, false)}{:else}{@const expanded = activityStore.isBatchExpanded(group.batchId)}+ {@const summary = group.items.find((item) => item.type === 'job_run')}<Collapsible.Root open={expanded} onOpenChange={(open) => activityStore.setBatchExpanded(group.batchId, open)}><Collapsible.Triggerclass="block w-full cursor-pointer appearance-none border-0 bg-transparent p-0 text-left focus-visible:ring-2 focus-visible:ring-ring focus-visible:outline-hidden focus-visible:ring-inset"@@ -135,8 +136,9 @@<Collapsible.Contentclass="overflow-hidden data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:slide-out-to-top-1 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:slide-in-from-top-1">+ {#if summary}<ActivityDetailPanel activity={summary} />{/if}<div class="ml-6 border-l border-border/40">- {#each group.items as activity (activity.id)}+ {#each group.items.filter((item) => item.type !== 'job_run') as activity (activity.id)}{@render activityRow(activity, true)}{/each}</div>diff --git a/frontend/src/lib/components/activity/activity-detail-panel.svelte b/frontend/src/lib/components/activity/activity-detail-panel.svelteindex dc286fd43..80e5a6c6f 100644--- a/frontend/src/lib/components/activity/activity-detail-panel.svelte+++ b/frontend/src/lib/components/activity/activity-detail-panel.svelte@@ -1,4 +1,6 @@<script lang="ts">+ import JobRunHistory from '#lib/components/job-card/job-run-history.svelte';+ import IfPermitted from '#lib/components/if-permitted.svelte';import { CopyButton } from '#lib/components/ui/copy-button/index.js';import { activityStore } from '#lib/stores/activity.store.svelte.js';import type { Activity } from '#lib/types/activity.type.js';@@ -15,49 +17,66 @@const outputText = $derived(messages.map((message) => message.message).join('\n'));const isLoading = $derived(activityStore.isDetailLoading(activity.id));const isDetailError = $derived(activityStore.isDetailError(activity.id));+ const jobId = $derived(typeof liveActivity.metadata?.['jobId'] === 'string' ? liveActivity.metadata['jobId'] : undefined);+ const runId = $derived(typeof liveActivity.metadata?.['runId'] === 'string' ? liveActivity.metadata['runId'] : undefined);+ const environmentId = $derived(+ typeof liveActivity.metadata?.['environmentId'] === 'string'+ ? liveActivity.metadata['environmentId']+ : liveActivity.sourceEnvironmentId || liveActivity.environmentId+ );</script><div class="border-b border-border/50 bg-muted/25">+ {#if liveActivity.type === 'job_run' && jobId && runId}+ <IfPermitted perm="jobs:manage" envId={environmentId}>+ <div class="px-4 py-2"><JobRunHistory {jobId} {environmentId} initialRunId={runId} /></div>+ </IfPermitted>+ {/if}+ {#if liveActivity.type === 'job_run' && liveActivity.latestMessage && liveActivity.latestMessage !== liveActivity.error}+ <p class="px-4 py-2 text-sm">{liveActivity.latestMessage}</p>+ {/if}{#if liveActivity.error}<div class="border-b border-destructive/30 bg-destructive/10 px-4 py-3 text-sm text-destructive">{liveActivity.error}</div>{/if}- <div>- <div class="flex items-center justify-between px-4 py-2.5">- <div class="flex items-center gap-2">- <TerminalIcon class="size-4 text-muted-foreground" aria-hidden="true" />- <span class="text-sm font-semibold">{m.activity_output_title()}</span>+ {#if liveActivity.type !== 'job_run'}+ <div>+ <div class="flex items-center justify-between px-4 py-2.5">+ <div class="flex items-center gap-2">+ <TerminalIcon class="size-4 text-muted-foreground" aria-hidden="true" />+ <span class="text-sm font-semibold">{m.activity_output_title()}</span>+ </div>+ <CopyButton text={outputText} variant="ghost" size="default" class="h-8 px-2 text-xs" tabindex={0}>+ <span class="text-xs">{m.activity_copy_output()}</span>+ </CopyButton></div>- <CopyButton text={outputText} variant="ghost" size="default" class="h-8 px-2 text-xs" tabindex={0}>- <span class="text-xs">{m.activity_copy_output()}</span>- </CopyButton>- </div>- <div class="bg-zinc-950 font-mono text-[12px] leading-relaxed text-zinc-100">- {#if isDetailError && messages.length === 0}- <div class="flex min-h-32 flex-col items-center justify-center gap-2 text-zinc-500">- <span>{m.activity_output_load_failed()}</span>- <button- type="button"- onclick={() => activityStore.retryLoadDetail(activity.id)}- class="text-xs text-primary underline hover:text-primary/80"- >- {m.common_retry()}- </button>- </div>- {:else if isLoading && messages.length === 0}- <div class="flex min-h-32 items-center justify-center text-zinc-500">- <ActivityIcon class="mr-2 size-4 animate-pulse" aria-hidden="true" />- {m.activity_output_loading()}- </div>- {:else if messages.length === 0}- <div class="flex min-h-32 items-center justify-center text-center text-zinc-500">- {m.activity_output_empty()}- </div>- {:else}- <ActivityOutput {messages} />- {/if}+ <div class="bg-zinc-950 font-mono text-[12px] leading-relaxed text-zinc-100">+ {#if isDetailError && messages.length === 0}+ <div class="flex min-h-32 flex-col items-center justify-center gap-2 text-zinc-500">+ <span>{m.activity_output_load_failed()}</span>+ <button+ type="button"+ onclick={() => activityStore.retryLoadDetail(activity.id)}+ class="text-xs text-primary underline hover:text-primary/80"+ >+ {m.common_retry()}+ </button>+ </div>+ {:else if isLoading && messages.length === 0}+ <div class="flex min-h-32 items-center justify-center text-zinc-500">+ <ActivityIcon class="mr-2 size-4 animate-pulse" aria-hidden="true" />+ {m.activity_output_loading()}+ </div>+ {:else if messages.length === 0}+ <div class="flex min-h-32 items-center justify-center text-center text-zinc-500">+ {m.activity_output_empty()}+ </div>+ {:else}+ <ActivityOutput {messages} />+ {/if}+ </div></div>- </div>+ {/if}</div>diff --git a/frontend/src/lib/components/activity/activity-filter-popover.svelte b/frontend/src/lib/components/activity/activity-filter-popover.svelteindex 6ee3d6eb2..5d2471fc9 100644--- a/frontend/src/lib/components/activity/activity-filter-popover.svelte+++ b/frontend/src/lib/components/activity/activity-filter-popover.svelte@@ -9,6 +9,7 @@const statuses: ActivityStatus[] = ['queued', 'running', 'success', 'failed', 'cancelled'];const types: ActivityType[] = [+ 'job_run','image_pull','image_build','image_update_check',diff --git a/frontend/src/lib/components/activity/activity-labels.ts b/frontend/src/lib/components/activity/activity-labels.tsindex bfe0842f2..ef2838615 100644--- a/frontend/src/lib/components/activity/activity-labels.ts+++ b/frontend/src/lib/components/activity/activity-labels.ts@@ -1,3 +1,5 @@+import { jobStatusLabel } from '#lib/components/job-card/job-status.js';+import type { Activity } from '#lib/types/activity.type.js';import { m } from '#lib/paraglide/messages.js';import type { ActivityStatus, ActivityType } from '#lib/types/activity.type.js';import type { IconType } from '#lib/icons/index.js';@@ -29,6 +31,7 @@ const statusDisplay = new Map(const typeDisplay = new Map(Object.entries({+ job_run: { label: m.activity_type_job_run, icon: ActivityIcon },image_pull: { label: m.activity_type_image_pull, icon: DownloadIcon },image_build: { label: m.activity_type_image_build, icon: HammerIcon },image_update_check: { label: m.activity_type_image_update_check, icon: RefreshIcon },@@ -71,3 +74,9 @@ export function activityTypeIcon(type: ActivityType): IconType {export function activityStatusAccentClass(status: ActivityStatus): string {return statusDisplay.get(status)?.accentClass as string;}++export function activityRunStatusLabel(activity: Activity): string {+ return activity.type === 'job_run' && typeof activity.metadata?.['jobStatus'] === 'string'+ ? jobStatusLabel(activity.metadata['jobStatus'])+ : activityStatusLabel(activity.status);+}diff --git a/frontend/src/lib/components/activity/activity-list-item.svelte b/frontend/src/lib/components/activity/activity-list-item.svelteindex 938432f79..2cf896628 100644--- a/frontend/src/lib/components/activity/activity-list-item.svelte+++ b/frontend/src/lib/components/activity/activity-list-item.svelte@@ -9,7 +9,7 @@import type { Activity } from '#lib/types/activity.type.js';import {activityStatusAccentClass,- activityStatusLabel,+ activityRunStatusLabel,activityStatusVariant,activityTypeIcon,activityTypeLabel@@ -119,7 +119,7 @@</div>{/if}</div>- <Badge variant={activityStatusVariant(activity.status)} size="sm">{activityStatusLabel(activity.status)}</Badge>+ <Badge variant={activityStatusVariant(activity.status)} size="sm">{activityRunStatusLabel(activity)}</Badge></div>{#if expanded}diff --git a/frontend/src/lib/components/job-card/job-run-history.svelte b/frontend/src/lib/components/job-card/job-run-history.svelteindex 8a3269272..9d7738821 100644--- a/frontend/src/lib/components/job-card/job-run-history.svelte+++ b/frontend/src/lib/components/job-card/job-run-history.svelte@@ -1,4 +1,6 @@<script lang="ts">+ import { openConfirmDialog } from '#lib/components/confirm-dialog/index.js';+ import IfPermitted from '#lib/components/if-permitted.svelte';import { queryKeys } from '#lib/query/query-keys.js';import { createQuery, createMutation } from '@tanstack/svelte-query';import { m } from '#lib/paraglide/messages.js';@@ -8,7 +10,12 @@import { activityStore } from '#lib/stores/activity.store.svelte.js';import { formatDateTimeShort } from '#lib/utils/formatting.js';import { jobStatusLabel } from './job-status';- let { jobId, environmentId, onUpdate }: { jobId: string; environmentId: string; onUpdate?: () => void } = $props();+ let {+ jobId,+ environmentId,+ initialRunId,+ onUpdate+ }: { jobId: string; environmentId: string; initialRunId?: string; onUpdate?: () => void } = $props();let open = $state(false);let page = $state(1);let selected = $state<string | undefined>();@@ -25,11 +32,12 @@refetchInterval: 5000}));const action = createMutation(() => ({- mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' }) =>+ mutationFn: ({ runId, action }: { runId: string; action: 'retry' | 'cancel' | 'resolve' }) =>jobScheduleService.updateRun(jobId, runId, action, environmentId),onSuccess: () => {void runs.refetch();void detail.refetch();+ void activityStore.refresh();onUpdate?.();}}));@@ -79,14 +87,30 @@}].filter((item) => item.visible));+ function resolveRun(runId: string) {+ openConfirmDialog({+ title: m.jobs_resolve_run(),+ message: m.jobs_resolve_description(),+ confirm: {+ label: m.jobs_resolve_run(),+ action: () => action.mutateAsync({ runId, action: 'resolve' }).then(() => undefined)+ }+ });+ }</script>{#snippet outcomeMessage(message: string | undefined)}{#if message}<p class="break-words whitespace-pre-wrap">{message}</p>{/if}{/snippet}-{#snippet outcomeActivity(activityId: string | undefined)}- {#if activityId}<Button variant="link" onclick={() => activityStore.openCenter(activityId)}>{m.activity()}</Button>{/if}+{#snippet outcomeActivity(+ activityId: string | undefined,+ label = m.jobs_activity_output(),+ activityEnvironmentId = environmentId+)}+ {#if activityId}<Button variant="link" onclick={() => activityStore.openCenter(activityId, undefined, activityEnvironmentId)}+ >{label}</Button+ >{/if}{/snippet}<Button@@ -95,7 +119,7 @@onclick={() => {open = true;page = 1;- selected = undefined;+ selected = initialRunId;}}>{m.jobs_run_history()}</Button><Dialog.Root bind:open>@@ -144,7 +168,15 @@<p class="break-all"><strong>{m.jobs_run_id()}:</strong> {run.id}</p><p>{jobStatusLabel(run.status)}</p>{#each metadata as item (item.id)}<p class={item.class}>{item.label ? `${item.label}: ` : ''}{item.value}</p>{/each}+ {@render outcomeActivity(run.activityId, m.jobs_activity_summary(), run.activityEnvironmentId ?? environmentId)}{@render outcomeActivity(run.outcome.activityId)}+ {#if run.status === 'needs_attention'}<p>{m.jobs_attention_schedule_blocked()}</p>{/if}+ {#if run.resolution}+ <p>+ {m.jobs_resolution_details({ user: run.resolution.resolvedBy, date: formatDateTimeShort(run.resolution.resolvedAt) })}+ </p>+ <p>{m.jobs_resolution_reason()}: {run.resolution.reason}</p>+ {/if}<h4 class="font-medium">{m.jobs_run_attempts()}: {run.attemptCount}</h4>{#each run.attempts ?? [] as attempt (attempt.number)}<div class="space-y-1 rounded border p-2">@@ -163,13 +195,20 @@{/each}{/if}{#if action.error}<p class="text-destructive">{action.error.message}</p>{/if}- {#each availableActions as item (item.id)}- <Button- variant={item.variant}- disabled={action.isPending}- onclick={() => action.mutate({ runId: run.id, action: item.id })}>{item.label}</Button- >- {/each}+ <IfPermitted perm="jobs:manage" envId={environmentId}>+ {#if run.status === 'needs_attention'}+ <Button variant="outline" disabled={action.isPending} onclick={() => resolveRun(run.id)}+ >{m.jobs_resolve_run()}</Button+ >+ {/if}+ {#each availableActions as item (item.id)}+ <Button+ variant={item.variant}+ disabled={action.isPending}+ onclick={() => action.mutate({ runId: run.id, action: item.id })}>{item.label}</Button+ >+ {/each}+ </IfPermitted></div>{/if}</Dialog.Content>diff --git a/frontend/src/lib/services/job-schedule-service.ts b/frontend/src/lib/services/job-schedule-service.tsindex 0a4c8df15..18663602e 100644--- a/frontend/src/lib/services/job-schedule-service.ts+++ b/frontend/src/lib/services/job-schedule-service.ts@@ -32,7 +32,7 @@ class JobScheduleService extends BaseAPIService {this.api.get(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}`));}- async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel', environmentId: string): Promise<JobRun> {+ async updateRun(jobId: string, runId: string, action: 'retry' | 'cancel' | 'resolve', environmentId: string): Promise<JobRun> {return this.handleResponse(this.api.post(`/environments/${environmentId}/jobs/${encodeURIComponent(jobId)}/runs/${encodeURIComponent(runId)}/${action}`diff --git a/frontend/src/lib/stores/activity.store.svelte.ts b/frontend/src/lib/stores/activity.store.svelte.tsindex 7ba7a2196..8696f7ad5 100644--- a/frontend/src/lib/stores/activity.store.svelte.ts+++ b/frontend/src/lib/stores/activity.store.svelte.ts@@ -27,7 +27,7 @@ import {discardPendingActivityToasts,queueActivityCompletionToast} from '#lib/components/activity/activity-completion-toasts.js';-import { instantEpochMilliseconds } from '#lib/utils/formatting.js';+import { instantEpochMilliseconds, parseInstant } from '#lib/utils/formatting.js';const ACTIVITY_LIST_LIMIT = 50;const ACTIVITY_DETAIL_LIMIT = 500;@@ -118,9 +118,15 @@ function groupActivitiesInternal(items: Activity[]): ActivityGroup[] {}function finalizeBatchGroupInternal(group: ActivityBatchGroup) {- group.total = group.items.length;- group.done = group.items.filter((item) => !isActiveStatusInternal(item.status)).length;- group.failed = group.items.filter((item) => item.status === 'failed').length;+ const jobActivity = group.items.find((item) => item.type === 'job_run');+ const operations = jobActivity ? group.items.filter((item) => item.type !== 'job_run') : group.items;+ group.total = operations.length;+ group.done = operations.filter((item) => !isActiveStatusInternal(item.status)).length;+ group.failed = operations.filter((item) => item.status === 'failed').length;+ if (jobActivity) {+ group.status = jobActivity.status;+ return;+ }if (group.items.some((item) => isActiveStatusInternal(item.status))) {group.status = group.items.some((item) => item.status === 'running') ? 'running' : 'queued';} else if (group.failed > 0) {@@ -155,6 +161,7 @@ function createActivityStore() {let _open = $state(false);let _currentEnvironmentId = $state(LOCAL_DOCKER_ENVIRONMENT_ID);let sessionGeneration = 0;+ let requestedEnvironments: Record<string, string> = {};// Last observed status per activity, for completion-toast transition// detection. Intentionally non-reactive: only stream handling reads it.const observedStatusById = new Map<string, ActivityStatus>();@@ -283,6 +290,13 @@ function createActivityStore() {});function normalizeActivityInternal(activity: Activity, environmentId: string): Activity {+ if (activity.type === 'job_run') {+ const current = _details[activity.id]?.activity ?? _activities.find((item) => item.id === activity.id);+ const currentTime = parseInstant(current?.updatedAt);+ const incomingTime = parseInstant(activity.updatedAt);+ if (current && currentTime && (!incomingTime || incomingTime.epochNanoseconds <= currentTime.epochNanoseconds))+ return current;+ }const state = core.environmentState(environmentId);return {...activity,@@ -304,6 +318,8 @@ function createActivityStore() {// for completion toasts has to happen here as well as in merges.for (const activity of normalizedActivities) {noteActivityStatusInternal(activity);+ const detail = _details[activity.id];+ if (activity.type === 'job_run' && detail) _details = { ..._details, [activity.id]: { ...detail, activity } };}_environmentActivities = {..._environmentActivities,@@ -394,7 +410,9 @@ function createActivityStore() {function activityEnvironmentIdInternal(activityId: string): string {const activity = _details[activityId]?.activity ?? _activities.find((item) => item.id === activityId) ?? null;- return sourceEnvironmentIdInternal(activity) || _currentEnvironmentId || LOCAL_DOCKER_ENVIRONMENT_ID;+ return activity+ ? sourceEnvironmentIdInternal(activity)+ : requestedEnvironments[activityId] || _currentEnvironmentId || LOCAL_DOCKER_ENVIRONMENT_ID;}async function loadDetailInternal(activityId: string) {@@ -415,12 +433,14 @@ function createActivityStore() {if (generation !== sessionGeneration) {return;}- const environmentId = sourceEnvironmentIdInternal(detail.activity);+ const environmentId = activityEnvironmentIdInternal(activityId);const normalized = {...detail,activity: normalizeActivityInternal(detail.activity, environmentId)};_details = { ..._details, [activityId]: normalized };+ mergeActivityInternal(normalized.activity);+ if (normalized.activity.batchId) setBatchExpandedInternal(normalized.activity.batchId, true);const nextErrors = { ..._detailErrorIds };delete nextErrors[activityId];_detailErrorIds = nextErrors;@@ -483,7 +503,8 @@ function createActivityStore() {}}- function openCenterInternal(activityId?: string, batchId?: string) {+ function openCenterInternal(activityId?: string, batchId?: string, environmentId?: string) {+ if (activityId && environmentId) requestedEnvironments[activityId] = environmentId;_open = true;discardPendingActivityToasts();if (activityId) {@@ -601,6 +622,7 @@ function createActivityStore() {const wasStarted = core.stop(options);if (options?.resetState) {sessionGeneration += 1;+ requestedEnvironments = {};observedStatusById.clear();discardPendingActivityToasts();_activities = [];diff --git a/frontend/src/lib/types/activity.type.ts b/frontend/src/lib/types/activity.type.tsindex 6c3fde44c..0201ca5c9 100644--- a/frontend/src/lib/types/activity.type.ts+++ b/frontend/src/lib/types/activity.type.ts@@ -1,6 +1,7 @@export type ActivityStatus = 'queued' | 'running' | 'success' | 'failed' | 'cancelled';export type ActivityType =+ | 'job_run'| 'image_pull'| 'image_build'| 'image_update_check'diff --git a/frontend/src/lib/types/job.ts b/frontend/src/lib/types/job.tsindex 486757bf1..8016367f0 100644--- a/frontend/src/lib/types/job.ts+++ b/frontend/src/lib/types/job.ts@@ -25,6 +25,9 @@ export type JobRun = {finishedAt?: string;nextAttempt?: string;attemptCount: number;+ activityId?: string;+ activityEnvironmentId?: string;+ resolution?: { resolvedBy: string; resolvedAt: string; reason: string };outcome: JobOutcome;attempts?: JobAttempt[];remoteAccepted: boolean;diff --git a/types/activity/activity.go b/types/activity/activity.goindex d06f84920..495d812d1 100644--- a/types/activity/activity.go+++ b/types/activity/activity.go@@ -5,6 +5,7 @@ import "time"type Status stringconst (+ TypeJobRun Type = "job_run"StatusQueued Status = "queued"StatusRunning Status = "running"StatusSuccess Status = "success"diff --git a/types/jobschedule/runs.go b/types/jobschedule/runs.goindex 72d6c4c74..81196ae42 100644--- a/types/jobschedule/runs.go+++ b/types/jobschedule/runs.go@@ -19,3 +19,12 @@ type RunInput struct {}type RunOutput struct{ Body st.Run }type ListRunsOutput struct{ Body st.RunList }++// ResolveRunInput accepts the original operator identity only from trusted agent transport.+type ResolveRunInput struct {+ RunInput++ Body struct {+ ResolvedBy string `json:"resolvedBy,omitempty"`+ }+}diff --git a/types/scheduler/queue.go b/types/scheduler/queue.goindex fdd27e015..f496eaeac 100644--- a/types/scheduler/queue.go+++ b/types/scheduler/queue.go@@ -1,6 +1,16 @@package scheduler-import "time"+import (+ "context"+ "time"+)++// RunObserver projects persisted runs into operator-facing activity records.+// ActivityID must be deterministic and empty for runs that should stay quiet.+type RunObserver interface {+ ActivityID(run Run) string+ SyncRunActivity(ctx context.Context, run Run) error+}// QueueRecord holds atomic admission, claims, and checkpoints for one job and target.type QueueRecord struct {diff --git a/types/scheduler/run.go b/types/scheduler/run.goindex 99de46ec4..46fbbd3e1 100644--- a/types/scheduler/run.go+++ b/types/scheduler/run.go@@ -56,31 +56,40 @@ type Attempt struct {Outcome Outcome `json:"outcome"`}+type RunResolution struct {+ ResolvedBy string `json:"resolvedBy"`+ ResolvedAt time.Time `json:"resolvedAt"`+ Reason string `json:"reason"`+}+type Run struct {- RequestedWithKey string `json:"requestedWithKey,omitempty"`- ID string `json:"id"`- JobID string `json:"jobId"`- EnvironmentID string `json:"environmentId"`- Trigger string `json:"trigger"`- RequestedBy string `json:"requestedBy,omitempty"`- Status RunStatus `json:"status"`- CreatedAt time.Time `json:"createdAt"`- UpdatedAt time.Time `json:"updatedAt"`- StartedAt *time.Time `json:"startedAt,omitempty"`- FinishedAt *time.Time `json:"finishedAt,omitempty"`- NextAttempt *time.Time `json:"nextAttempt,omitempty"`- AttemptCount int `json:"attemptCount"`- Owner string `json:"owner,omitempty"`- Outcome Outcome `json:"outcome"`- Attempts []Attempt `json:"attempts,omitempty"`- RemoteDeliveryAttempted bool `json:"remoteDeliveryAttempted"`- RemoteOutcome *Outcome `json:"remoteOutcome,omitempty"`- RemoteRetryRequested bool `json:"remoteRetryRequested,omitempty"`- RemoteRetryAttempted bool `json:"remoteRetryAttempted,omitempty"`- RemoteAttemptCount int `json:"remoteAttemptCount,omitempty"`- RemoteAccepted bool `json:"remoteAccepted"`- RemoteSettled bool `json:"remoteSettled"`- LastConfirmedAt *time.Time `json:"lastConfirmedAt,omitempty"`+ ActivityEnvironmentID string `json:"activityEnvironmentId,omitempty"`+ ActivityID string `json:"activityId,omitempty"`+ Resolution *RunResolution `json:"resolution,omitempty"`+ RequestedWithKey string `json:"requestedWithKey,omitempty"`+ ID string `json:"id"`+ JobID string `json:"jobId"`+ EnvironmentID string `json:"environmentId"`+ Trigger string `json:"trigger"`+ RequestedBy string `json:"requestedBy,omitempty"`+ Status RunStatus `json:"status"`+ CreatedAt time.Time `json:"createdAt"`+ UpdatedAt time.Time `json:"updatedAt"`+ StartedAt *time.Time `json:"startedAt,omitempty"`+ FinishedAt *time.Time `json:"finishedAt,omitempty"`+ NextAttempt *time.Time `json:"nextAttempt,omitempty"`+ AttemptCount int `json:"attemptCount"`+ Owner string `json:"owner,omitempty"`+ Outcome Outcome `json:"outcome"`+ Attempts []Attempt `json:"attempts,omitempty"`+ RemoteDeliveryAttempted bool `json:"remoteDeliveryAttempted"`+ RemoteOutcome *Outcome `json:"remoteOutcome,omitempty"`+ RemoteRetryRequested bool `json:"remoteRetryRequested,omitempty"`+ RemoteRetryAttempted bool `json:"remoteRetryAttempted,omitempty"`+ RemoteAttemptCount int `json:"remoteAttemptCount,omitempty"`+ RemoteAccepted bool `json:"remoteAccepted"`+ RemoteSettled bool `json:"remoteSettled"`+ LastConfirmedAt *time.Time `json:"lastConfirmedAt,omitempty"`}type RunList struct {@@ -125,3 +134,8 @@ func (e *OutcomeError) Error() string {}func (e *OutcomeError) Unwrap() error { return e.Cause }++// RetryValidator rejects retries without safe persisted target evidence.+type RetryValidator interface {+ ValidateRetry(ctx context.Context, run Run) error+}