Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
refactor: vulnerability and jobs migration to composite literals, drop DI nil checks
Kyle Mendell9d10c2a54355
diff --git a/backend/internal/bootstrap/bootstrap_test.go b/backend/internal/bootstrap/bootstrap_test.goindex 995db3cd6..48e3257d9 100644--- a/backend/internal/bootstrap/bootstrap_test.go+++ b/backend/internal/bootstrap/bootstrap_test.go@@ -21,6 +21,7 @@ import ("gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/api"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/auth""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/container"@@ -30,6 +31,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/project"+ "github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/internal/swarm""github.com/getarcaneapp/arcane/backend/v2/internal/system""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"@@ -385,7 +387,7 @@ func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {runtime := francistest.New(t)db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&kv.KVEntry{}, &environment.Environment{}, &project.GitOpsSync{}))+ require.NoError(t, db.AutoMigrate(&kv.KVEntry{}, &environment.Environment{}, &project.GitOpsSync{}, &activity.Activity{}, &activity.ActivityMessage{}))sqlDB, err := db.DB()require.NoError(t, err)sqlDB.SetMaxOpenConns(1)@@ -401,7 +403,15 @@ func TestJobSchedulerStopCancelsItsPrivateContext(t *testing.T) {OnStop: runtime.Stop,})t.Cleanup(func() { lifecycle.RequireStop() })- jobService := job.NewJobService(&database.DB{DB: db}, nil, &config.Config{}, coordinator, nil, nil, nil)+ jobService := job.NewJobService(+ &database.DB{DB: db},+ nil,+ &config.Config{},+ coordinator,+ role.NewRoleService(&database.DB{DB: db}),+ environment.NewEnvironmentService(&database.DB{DB: db}, nil, nil, nil, nil, nil),+ activity.NewActivityService(&database.DB{DB: db}, nil),+ )gitopsSync := gitops.NewGitOpsSyncService(&database.DB{DB: db}, nil, nil, nil, nil, nil)require.NoError(t, db.Create(&environment.Environment{ID: "0", Name: "Local", Enabled: true}).Error)require.NoError(t, db.Create(&project.GitOpsSync{ID: "overdue", EnvironmentID: "0", AutoSync: true, SyncInterval: 1}).Error)diff --git a/backend/internal/job/children/remote/service.go b/backend/internal/job/children/remote/service.goindex ebd33753e..7a93b4f8d 100644--- a/backend/internal/job/children/remote/service.go+++ b/backend/internal/job/children/remote/service.go@@ -34,7 +34,11 @@ type Service struct {}func New(coordinator *runs.Coordinator, store *kv.KVService, environments *environment.EnvironmentService) *Service {- return &Service{runs: coordinator, store: store, environments: environments}+ return &Service{+ runs: coordinator,+ store: store,+ environments: environments,+ }}// Deliver drives one manager-owned run through the remote delivery protocol.@@ -49,53 +53,138 @@ func (s *Service) Deliver(ctx context.Context, run scheduler.Run) (scheduler.OutbasePath := "/api/environments/0/jobs/" + url.PathEscape(run.JobID)runPath := basePath + "/runs/" + url.PathEscape(run.ID)// A previously confirmed terminal result needs only acknowledgement, never execution.- if !run.RemoteRetryRequested && run.RemoteAccepted && run.RemoteOutcome != nil && run.RemoteOutcome.Status.Terminal() {- return s.acknowledgeRemoteInternal(ctx, run, runPath, *run.RemoteOutcome)+ if !run.RemoteRetryRequested && run.RemoteAccepted && kit.FromPtr(run.RemoteOutcome).Status.Terminal() {+ return s.acknowledgeRemote(ctx, run, runPath, *run.RemoteOutcome)}var catalog jobschedule.JobListResponseif proxyJSONRequestErr := s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, "/api/environments/0/jobs", nil, &catalog); proxyJSONRequestErr != nil {- return remoteFailureInternal(proxyJSONRequestErr)+ return remoteFailure(proxyJSONRequestErr)}if !catalog.DurableRuns {return scheduler.Outcome{Status: scheduler.Failed, Message: "Upgrade required: agent does not support durable job runs"}, nil}var remoteRun scheduler.Run- err = s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, runPath, nil, &remoteRun)- if err != nil {+ if err = s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, runPath, nil, &remoteRun); err != nil {var status *remenv.StatusErrorif !errors.As(err, &status) || status.StatusCode != http.StatusNotFound {- return remoteFailureInternal(err)- }- remoteRun, err = s.admitRemoteInternal(ctx, run, catalog)- if err != nil {- if errors.Is(err, errRemoteIneligibleInternal) {- return scheduler.Outcome{Status: scheduler.Canceled, Message: err.Error()}, nil- }- if errors.Is(err, errRemoteReceiptMissingInternal) {- return scheduler.Outcome{Status: scheduler.Failed, Message: err.Error()}, nil+ return remoteFailure(err)+ }+ if run.RemoteAccepted {+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Agent no longer has the delivery receipt; verify the operation before retrying"}, nil+ }+ if !remoteJobEligible(catalog.Jobs, run.JobID) {+ return scheduler.Outcome{Status: scheduler.Canceled, Message: "Remote job is disabled or no longer eligible"}, nil+ }+ // A confirmed 404 permits another delivery of the same deduplicated ID.+ if updateRunErr := s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: current.Status,+ CreatedAt: current.CreatedAt,+ UpdatedAt: current.UpdatedAt,+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: true,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,}- return remoteFailureInternal(err)+ return nil+ }); updateRunErr != nil {+ return remoteFailure(updateRunErr)+ }+ body, marshalErr := json.Marshal(map[string]string{"runId": run.ID})+ if marshalErr != nil {+ return remoteFailure(marshalErr)+ }+ var accepted jobschedule.JobRunResponse+ if proxyJSONRequestErr := s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, basePath+"/run", body, &accepted); proxyJSONRequestErr != nil {+ return remoteFailure(proxyJSONRequestErr)+ }+ if accepted.RunID != run.ID {+ return remoteFailure(errors.New("Agent returned an inconsistent delivery receipt")) //nolint:staticcheck // Preserve the existing error message.+ }+ remoteRun = scheduler.Run{+ ID: run.ID,+ JobID: run.JobID,+ Status: accepted.Status,+ Outcome: scheduler.Outcome{Status: accepted.Status},}}if remoteRun.ID != run.ID || remoteRun.JobID != run.JobID {return scheduler.Outcome{Status: scheduler.Failed, Message: "Agent returned an inconsistent run identity"}, nil}if run.RemoteRetryRequested && (remoteRun.Status.Terminal() || remoteRun.Status == scheduler.NeedsAttention) && remoteRun.AttemptCount <= run.RemoteAttemptCount {- remoteRun, err = s.retryDeliveryInternal(ctx, run, runPath)- if err != nil {- if errors.Is(err, errRemoteRetryUncertainInternal) {- return scheduler.Outcome{Status: scheduler.Failed, Message: err.Error()}, nil+ if run.RemoteRetryAttempted {+ return scheduler.Outcome{Status: scheduler.Failed, Message: "Remote retry acknowledgement was lost; inspect the agent before retrying again"}, nil+ }+ if updateRunErr := s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: current.Status,+ CreatedAt: current.CreatedAt,+ UpdatedAt: current.UpdatedAt,+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: true,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,}- return remoteFailureInternal(err)+ return nil+ }); updateRunErr != nil {+ return remoteFailure(updateRunErr)+ }+ var retried scheduler.Run+ if proxyJSONRequestErr := s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, runPath+"/retry", nil, &retried); proxyJSONRequestErr != nil {+ return remoteFailure(proxyJSONRequestErr)}+ if retried.ID != run.ID || retried.JobID != run.JobID {+ return remoteFailure(errors.New("Agent returned an inconsistent retry receipt")) //nolint:staticcheck // Preserve the existing error message.+ }+ remoteRun = retried}return s.Confirm(ctx, run, remoteRun, runPath)}// Confirm records the agent-reported state of a delivered run.func (s *Service) Confirm(ctx context.Context, run, remoteRun scheduler.Run, runPath string) (scheduler.Outcome, error) {- if remoteRun.Status == scheduler.NeedsAttention || (remoteRun.Status == scheduler.Canceled && remoteRun.Resolution != nil && remoteRun.Resolution.ResolvedBy == user.SystemUser.Username) {- settled, err := s.resolveAgentReviewInternal(ctx, run.EnvironmentID, run.JobID, run.ID, user.SystemUser.Username)+ if remoteRun.Status == scheduler.NeedsAttention || (remoteRun.Status == scheduler.Canceled && kit.FromPtr(remoteRun.Resolution).ResolvedBy == user.SystemUser.Username) {+ settled, err := s.resolveAgentReview(ctx, run.EnvironmentID, run.JobID, run.ID, user.SystemUser.Username)if err != nil {return scheduler.Outcome{Status: scheduler.Waiting,@@ -105,25 +194,44 @@ func (s *Service) Confirm(ctx context.Context, run, remoteRun scheduler.Run, run}, err}remoteRun = settled- if remoteRun.Status == scheduler.Canceled && remoteRun.Resolution != nil && remoteRun.Resolution.ResolvedBy == user.SystemUser.Username {- remoteRun.Status = scheduler.Failed- }}- now := time.Now().UTC()- outcome := remoteRun.Outcome- outcome.Status = remoteRun.Status+ outcome := scheduler.Outcome{+ Status: kit.Ternary(remoteRun.Status == scheduler.Canceled && kit.FromPtr(remoteRun.Resolution).ResolvedBy == user.SystemUser.Username, scheduler.Failed, remoteRun.Status),+ Message: remoteRun.Outcome.Message,+ ActivityID: remoteRun.Outcome.ActivityID,+ Targets: remoteRun.Outcome.Targets,+ Steps: remoteRun.Outcome.Steps,+ }if err := s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- current.RemoteAccepted = true- current.RemoteDeliveryAttempted = true- current.LastConfirmedAt = &now- current.RemoteOutcome = &outcome- current.RemoteAttemptCount = remoteRun.AttemptCount- current.RemoteRetryRequested = false- current.RemoteSettled = remoteRun.RemoteSettled- current.Resolution = remoteRun.Resolution- current.StartedAt = remoteRun.StartedAt- current.FinishedAt = remoteRun.FinishedAt- current.Outcome = outcome+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: remoteRun.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: current.Status,+ CreatedAt: current.CreatedAt,+ UpdatedAt: current.UpdatedAt,+ StartedAt: remoteRun.StartedAt,+ FinishedAt: remoteRun.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: true,+ RemoteOutcome: new(outcome),+ RemoteRetryRequested: false,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: remoteRun.AttemptCount,+ RemoteAccepted: true,+ RemoteSettled: remoteRun.RemoteSettled,+ LastConfirmedAt: new(time.Now().UTC()),+ }return nil}); err != nil {return scheduler.Outcome{Status: scheduler.Retrying}, err@@ -132,27 +240,60 @@ func (s *Service) Confirm(ctx context.Context, run, remoteRun scheduler.Run, runif remoteRun.RemoteSettled {return outcome, nil}- return s.acknowledgeRemoteInternal(ctx, run, runPath, outcome)+ return s.acknowledgeRemote(ctx, run, runPath, outcome)}- return scheduler.Outcome{Status: scheduler.Waiting, Message: "Waiting for the agent to complete this run", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, nil+ return scheduler.Outcome{+ Status: scheduler.Waiting,+ Message: "Waiting for the agent to complete this run",+ Targets: outcome.Targets,+ ActivityID: outcome.ActivityID,+ }, nil}-func (s *Service) acknowledgeRemoteInternal(ctx context.Context, run scheduler.Run, path string, outcome scheduler.Outcome) (scheduler.Outcome, error) {+func (s *Service) acknowledgeRemote(ctx context.Context, run scheduler.Run, path string, outcome scheduler.Outcome) (scheduler.Outcome, error) {var response scheduler.Runif err := s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/ack", nil, &response); err != nil {- return scheduler.Outcome{Status: scheduler.Waiting, Message: "Remote operation completed; waiting for delivery acknowledgement", Targets: outcome.Targets, ActivityID: outcome.ActivityID}, err- }- status := response.Status- if status == scheduler.Canceled && response.Resolution != nil && response.Resolution.ResolvedBy == user.SystemUser.Username {- status = scheduler.Failed- }+ return scheduler.Outcome{+ Status: scheduler.Waiting,+ Message: "Remote operation completed; waiting for delivery acknowledgement",+ Targets: outcome.Targets,+ ActivityID: outcome.ActivityID,+ }, err+ }+ status := kit.Ternary(response.Status == scheduler.Canceled && kit.FromPtr(response.Resolution).ResolvedBy == user.SystemUser.Username, scheduler.Failed, response.Status)if response.ID != run.ID || response.JobID != run.JobID || response.EnvironmentID != "0" || status != outcome.Status || !response.RemoteSettled {return scheduler.Outcome{Status: scheduler.Waiting, Message: "Remote operation completed; acknowledgement not confirmed"}, nil}if err := s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- current.RemoteSettled = true- current.StartedAt = response.StartedAt- current.FinishedAt = response.FinishedAt+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: current.Status,+ CreatedAt: current.CreatedAt,+ UpdatedAt: current.UpdatedAt,+ StartedAt: response.StartedAt,+ FinishedAt: response.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: true,+ LastConfirmedAt: current.LastConfirmedAt,+ }return nil}); err != nil {return scheduler.Outcome{Status: scheduler.Retrying}, err@@ -169,57 +310,65 @@ func (s *Service) Catalog(ctx context.Context, environmentID string) (*jobscheduvar catalog jobschedule.JobListResponseremoteErr := s.environments.ProxyJSONRequest(ctx, environmentID, http.MethodGet, "/api/environments/0/jobs", nil, &catalog)if remoteErr == nil {- catalog.ObservedAt = time.Now().UTC()- catalog.Offline = false- if catalog.DurableRuns {- if err := s.ReconcileLegacyCatalog(ctx, environmentID, catalog.Jobs); err != nil {+ observed := jobschedule.JobListResponse{ //exhaustruct:enforce+ DurableRuns: catalog.DurableRuns,+ ObservedAt: time.Now().UTC(),+ Offline: false,+ Jobs: catalog.Jobs,+ IsAgent: catalog.IsAgent,+ }+ if observed.DurableRuns {+ if err := s.ReconcileLegacyCatalog(ctx, environmentID, observed.Jobs); err != nil {return nil, err}}- raw, err := json.Marshal(catalog)+ raw, err := json.Marshal(observed)if err != nil {return nil, err}if setErr := s.store.Set(ctx, key, string(raw)); setErr != nil {return nil, setErr}- } else {- var status *remenv.StatusError- if errors.As(remoteErr, &status) && (status.StatusCode == http.StatusUnauthorized || status.StatusCode == http.StatusForbidden) {- return nil, remoteErr- }- raw, found, err := s.store.Get(ctx, key)- if err != nil {- return nil, err+ return &observed, nil+ }+ var status *remenv.StatusError+ if errors.As(remoteErr, &status) && (status.StatusCode == http.StatusUnauthorized || status.StatusCode == http.StatusForbidden) {+ return nil, remoteErr+ }+ raw, found, err := s.store.Get(ctx, key)+ if err != nil {+ return nil, err+ }+ if found {+ if unmarshalErr := json.Unmarshal([]byte(raw), &catalog); unmarshalErr != nil {+ return nil, unmarshalErr}- if found {- if unmarshalErr := json.Unmarshal([]byte(raw), &catalog); unmarshalErr != nil {- return nil, unmarshalErr- }- } else {- catalog.IsAgent = true- for _, metadata := range meta.GetAllJobMetadata() {- if !metadata.ManagerOnly {- catalog.Jobs = append(catalog.Jobs, metadata.ToJobStatus("", nil, metadata.CanRunManually, []jobschedule.JobPrerequisite{}))- }- }+ }+ jobs := catalog.Jobs+ for _, metadata := range meta.GetAllJobMetadata() {+ if !found && !metadata.ManagerOnly {+ jobs = append(jobs, metadata.ToJobStatus("", nil, metadata.CanRunManually, []jobschedule.JobPrerequisite{}))}- catalog.Offline = true}- return &catalog, nil+ return &jobschedule.JobListResponse{ //exhaustruct:enforce+ DurableRuns: catalog.DurableRuns,+ ObservedAt: catalog.ObservedAt,+ Offline: true,+ Jobs: jobs,+ IsAgent: catalog.IsAgent || !found,+ }, nil}// RetryRun queues one explicit retry of an authorized manager-owned run while// retaining the agent's target progress.func (s *Service) RetryRun(ctx context.Context, run scheduler.Run) (scheduler.Run, error) {- environmentID, jobID, runID := run.EnvironmentID, run.JobID, run.ID- if run.Resolution != nil && run.Resolution.ResolvedBy == user.SystemUser.Username {+ if kit.FromPtr(run.Resolution).ResolvedBy == user.SystemUser.Username {var remote scheduler.Run- path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs/" + url.PathEscape(runID)- if proxyJSONRequestErr := s.environments.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &remote); proxyJSONRequestErr != nil {+ path := "/api/environments/0/jobs/" + url.PathEscape(run.JobID) + "/runs/" + url.PathEscape(run.ID)+ if proxyJSONRequestErr := s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodGet, path, nil, &remote); proxyJSONRequestErr != nil {return run, proxyJSONRequestErr}- if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || remote.Status != scheduler.Failed {+ if remote.ID != run.ID || remote.JobID != run.JobID || remote.EnvironmentID != "0" || remote.Status != scheduler.Failed {return run, errors.New("upgrade the agent before retrying a legacy settled run")}}@@ -227,84 +376,48 @@ func (s *Service) RetryRun(ctx context.Context, run scheduler.Run) (scheduler.Ruif current.Status != scheduler.Failed && current.Status != scheduler.Partial && current.Status != scheduler.NeedsAttention {return errors.New("run is not eligible for retry")}- if !current.RemoteAccepted {- if current.RemoteDeliveryAttempted {- return errors.New("remote delivery is uncertain; inspect the agent before submitting new work")- }- } else {- if current.RemoteOutcome == nil || (!current.RemoteOutcome.Status.Terminal() && current.RemoteOutcome.Status != scheduler.NeedsAttention) {- return errors.New("remote outcome must be confirmed before retry")- }- current.RemoteRetryRequested = true- current.RemoteRetryAttempted = false- current.RemoteSettled = false- }- current.Status = scheduler.Queued- current.Owner = ""- current.NextAttempt = nil- current.FinishedAt = nil- current.UpdatedAt = time.Now().UTC()+ if !current.RemoteAccepted && current.RemoteDeliveryAttempted {+ return errors.New("remote delivery is uncertain; inspect the agent before submitting new work")+ }+ remoteStatus := kit.FromPtr(current.RemoteOutcome).Status+ if current.RemoteAccepted && !remoteStatus.Terminal() && remoteStatus != scheduler.NeedsAttention {+ return errors.New("remote outcome must be confirmed before retry")+ }+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: scheduler.Queued,+ CreatedAt: current.CreatedAt,+ UpdatedAt: time.Now().UTC(),+ StartedAt: current.StartedAt,+ FinishedAt: nil,+ NextAttempt: nil,+ AttemptCount: current.AttemptCount,+ Owner: "",+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteAccepted || current.RemoteRetryRequested,+ RemoteRetryAttempted: !current.RemoteAccepted && current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: !current.RemoteAccepted && current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,+ }run = *currentreturn nil})return run, err}-var (- errRemoteIneligibleInternal = errors.New("Remote job is disabled or no longer eligible") //nolint:staticcheck // Preserve the existing error message.- errRemoteReceiptMissingInternal = errors.New("Agent no longer has the delivery receipt; verify the operation before retrying") //nolint:staticcheck // Preserve the existing error message.- errRemoteRetryUncertainInternal = errors.New("Remote retry acknowledgement was lost; inspect the agent before retrying again") //nolint:staticcheck // Preserve the existing error message.-)--func (s *Service) admitRemoteInternal(ctx context.Context, run scheduler.Run, catalog jobschedule.JobListResponse) (scheduler.Run, error) {- if run.RemoteAccepted {- return scheduler.Run{}, errRemoteReceiptMissingInternal- }- if !remoteJobEligibleInternal(catalog.Jobs, run.JobID) {- return scheduler.Run{}, errRemoteIneligibleInternal- }- // A confirmed 404 permits another delivery of the same deduplicated ID.- if err := s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- current.RemoteDeliveryAttempted = true- return nil- }); err != nil {- return scheduler.Run{}, err- }- body, err := json.Marshal(map[string]string{"runId": run.ID})- if err != nil {- return scheduler.Run{}, err- }- path := "/api/environments/0/jobs/" + url.PathEscape(run.JobID) + "/run"- var accepted jobschedule.JobRunResponse- if proxyJSONRequestErr := s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path, body, &accepted); proxyJSONRequestErr != nil {- return scheduler.Run{}, proxyJSONRequestErr- }- if accepted.RunID != run.ID {- return scheduler.Run{}, errors.New("Agent returned an inconsistent delivery receipt") //nolint:staticcheck // Preserve the existing error message.- }- return scheduler.Run{ID: run.ID, JobID: run.JobID, Status: accepted.Status, Outcome: scheduler.Outcome{Status: accepted.Status}}, nil-}--func (s *Service) retryDeliveryInternal(ctx context.Context, run scheduler.Run, path string) (scheduler.Run, error) {- if run.RemoteRetryAttempted {- return scheduler.Run{}, errRemoteRetryUncertainInternal- }- if err := s.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- current.RemoteRetryAttempted = true- return nil- }); err != nil {- return scheduler.Run{}, err- }- var retried scheduler.Run- if err := s.environments.ProxyJSONRequest(ctx, run.EnvironmentID, http.MethodPost, path+"/retry", nil, &retried); err != nil {- return scheduler.Run{}, err- }- if retried.ID != run.ID || retried.JobID != run.JobID {- return scheduler.Run{}, errors.New("Agent returned an inconsistent retry receipt") //nolint:staticcheck // Preserve the existing error message.- }- return retried, nil-}-// Runs merges agent history with requests accepted by this manager.func (s *Service) Runs(ctx context.Context, environmentID, jobID string) ([]scheduler.Run, error) {if _, err := s.environments.GetEnvironmentByID(ctx, environmentID); err != nil {@@ -314,13 +427,53 @@ func (s *Service) Runs(ctx context.Context, environmentID, jobID string) ([]scheif err != nil {return nil, err}- merged, err := s.remoteHistoryInternal(ctx, environmentID, jobID)- if err != nil {- var status *remenv.StatusError- if errors.As(err, &status) && (status.StatusCode == http.StatusForbidden || status.StatusCode == http.StatusUnauthorized) {- return nil, err+ merged := make(map[string]scheduler.Run)+ for page := 1; ; page++ {+ path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs?page=" + strconv.Itoa(page) + "&limit=100"+ var response scheduler.RunList+ if proxyJSONRequestErr := s.environments.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &response); proxyJSONRequestErr != nil {+ var status *remenv.StatusError+ if errors.As(proxyJSONRequestErr, &status) && (status.StatusCode == http.StatusForbidden || status.StatusCode == http.StatusUnauthorized) {+ return nil, proxyJSONRequestErr+ }+ clear(merged)+ break+ }+ previous := len(merged)+ for _, run := range response.Runs {+ merged[run.ID] = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: kit.Ternary(run.ActivityID != "", environmentID, run.ActivityEnvironmentID),+ ActivityID: run.ActivityID,+ Resolution: run.Resolution,+ RequestedWithKey: run.RequestedWithKey,+ ID: run.ID,+ JobID: run.JobID,+ EnvironmentID: environmentID,+ Trigger: run.Trigger,+ RequestedBy: run.RequestedBy,+ Status: run.Status,+ CreatedAt: run.CreatedAt,+ UpdatedAt: run.UpdatedAt,+ StartedAt: run.StartedAt,+ FinishedAt: run.FinishedAt,+ NextAttempt: run.NextAttempt,+ AttemptCount: run.AttemptCount,+ Owner: run.Owner,+ Outcome: run.Outcome,+ Attempts: run.Attempts,+ RemoteDeliveryAttempted: run.RemoteDeliveryAttempted,+ RemoteOutcome: run.RemoteOutcome,+ RemoteRetryRequested: run.RemoteRetryRequested,+ RemoteRetryAttempted: run.RemoteRetryAttempted,+ RemoteAttemptCount: run.RemoteAttemptCount,+ RemoteAccepted: run.RemoteAccepted,+ RemoteSettled: run.RemoteSettled,+ LastConfirmedAt: run.LastConfirmedAt,+ }+ }+ if len(merged) >= response.Total || len(merged) == previous {+ break}- merged = make(map[string]scheduler.Run)}for _, record := range records {if record.EnvironmentID != environmentID || record.JobID != jobID {@@ -333,28 +486,6 @@ func (s *Service) Runs(ctx context.Context, environmentID, jobID string) ([]schereturn slices.Collect(maps.Values(merged)), nil}-func (s *Service) remoteHistoryInternal(ctx context.Context, environmentID, jobID string) (map[string]scheduler.Run, error) {- localRuns := make(map[string]scheduler.Run)- for page := 1; ; page++ {- path := "/api/environments/0/jobs/" + url.PathEscape(jobID) + "/runs?page=" + strconv.Itoa(page) + "&limit=100"- var response scheduler.RunList- if err := s.environments.ProxyJSONRequest(ctx, environmentID, http.MethodGet, path, nil, &response); err != nil {- return nil, err- }- previous := len(localRuns)- for _, run := range response.Runs {- run.EnvironmentID = environmentID- if run.ActivityID != "" {- run.ActivityEnvironmentID = environmentID- }- localRuns[run.ID] = run- }- if len(localRuns) >= response.Total || len(localRuns) == previous {- return localRuns, nil- }- }-}-// AgentRun queries agent-owned history for a run the manager never accepted.func (s *Service) AgentRun(ctx context.Context, environmentID, jobID, runID string) (scheduler.Run, error) {var run scheduler.Run@@ -365,11 +496,35 @@ func (s *Service) AgentRun(ctx context.Context, environmentID, jobID, runID striif run.ID != runID || run.JobID != jobID {return scheduler.Run{}, errors.New("agent returned an inconsistent run identity")}- run.EnvironmentID = environmentID- if run.ActivityID != "" {- run.ActivityEnvironmentID = environmentID- }- return run, nil+ return scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: kit.Ternary(run.ActivityID != "", environmentID, run.ActivityEnvironmentID),+ ActivityID: run.ActivityID,+ Resolution: run.Resolution,+ RequestedWithKey: run.RequestedWithKey,+ ID: run.ID,+ JobID: run.JobID,+ EnvironmentID: environmentID,+ Trigger: run.Trigger,+ RequestedBy: run.RequestedBy,+ Status: run.Status,+ CreatedAt: run.CreatedAt,+ UpdatedAt: run.UpdatedAt,+ StartedAt: run.StartedAt,+ FinishedAt: run.FinishedAt,+ NextAttempt: run.NextAttempt,+ AttemptCount: run.AttemptCount,+ Owner: run.Owner,+ Outcome: run.Outcome,+ Attempts: run.Attempts,+ RemoteDeliveryAttempted: run.RemoteDeliveryAttempted,+ RemoteOutcome: run.RemoteOutcome,+ RemoteRetryRequested: run.RemoteRetryRequested,+ RemoteRetryAttempted: run.RemoteRetryAttempted,+ RemoteAttemptCount: run.RemoteAttemptCount,+ RemoteAccepted: run.RemoteAccepted,+ RemoteSettled: run.RemoteSettled,+ LastConfirmedAt: run.LastConfirmedAt,+ }, nil}// MutateAgentRun forwards an explicit operator action to an agent-owned run.@@ -390,11 +545,35 @@ func (s *Service) MutateAgentRun(ctx context.Context, environmentID, jobID, runIif run.ID != runID || run.JobID != jobID {return scheduler.Run{}, errors.New("agent returned an inconsistent run identity")}- run.EnvironmentID = environmentID- if run.ActivityID != "" {- run.ActivityEnvironmentID = environmentID- }- return run, nil+ return scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: kit.Ternary(run.ActivityID != "", environmentID, run.ActivityEnvironmentID),+ ActivityID: run.ActivityID,+ Resolution: run.Resolution,+ RequestedWithKey: run.RequestedWithKey,+ ID: run.ID,+ JobID: run.JobID,+ EnvironmentID: environmentID,+ Trigger: run.Trigger,+ RequestedBy: run.RequestedBy,+ Status: run.Status,+ CreatedAt: run.CreatedAt,+ UpdatedAt: run.UpdatedAt,+ StartedAt: run.StartedAt,+ FinishedAt: run.FinishedAt,+ NextAttempt: run.NextAttempt,+ AttemptCount: run.AttemptCount,+ Owner: run.Owner,+ Outcome: run.Outcome,+ Attempts: run.Attempts,+ RemoteDeliveryAttempted: run.RemoteDeliveryAttempted,+ RemoteOutcome: run.RemoteOutcome,+ RemoteRetryRequested: run.RemoteRetryRequested,+ RemoteRetryAttempted: run.RemoteRetryAttempted,+ RemoteAttemptCount: run.RemoteAttemptCount,+ RemoteAccepted: run.RemoteAccepted,+ RemoteSettled: run.RemoteSettled,+ LastConfirmedAt: run.LastConfirmedAt,+ }, nil}// ReconcileLegacyCatalog adopts legacy agent-owned runs needing attention.@@ -402,7 +581,7 @@ func (s *Service) MutateAgentRun(ctx context.Context, environmentID, jobID, runIfunc (s *Service) ReconcileLegacyCatalog(ctx context.Context, environmentID string, jobs []jobschedule.JobStatus) error {for _, job := range jobs {for _, remote := range []*scheduler.Run{job.CurrentRun, job.LastRun} {- if remote == nil || remote.Status != scheduler.NeedsAttention {+ if kit.FromPtr(remote).Status != scheduler.NeedsAttention {continue}if remote.JobID != job.ID || remote.EnvironmentID != "0" {@@ -415,7 +594,13 @@ func (s *Service) ReconcileLegacyCatalog(ctx context.Context, environmentID striif !errors.Is(err, runs.ErrRunNotFound) {return err}- _, err = s.runs.Submit(ctx, scheduler.Request{RunID: remote.ID, JobID: job.ID, EnvironmentID: environmentID, Trigger: "recovery", ObservedAgentRun: remote})+ _, err = s.runs.Submit(ctx, scheduler.Request{+ RunID: remote.ID,+ JobID: job.ID,+ EnvironmentID: environmentID,+ Trigger: "recovery",+ ObservedAgentRun: remote,+ })if err != nil {return err}@@ -444,28 +629,80 @@ func (s *Service) ResolveRun(ctx context.Context, environmentID, jobID, runID, areturn s.runs.Resolve(ctx, environmentID, jobID, runID, actor)}}- acknowledged, err := s.resolveAgentReviewInternal(ctx, environmentID, jobID, runID, actor)+ acknowledged, err := s.resolveAgentReview(ctx, environmentID, jobID, runID, actor)if err != nil {return scheduler.Run{}, err}if localErr != nil {- acknowledged.EnvironmentID = environmentID- if acknowledged.ActivityID != "" {- acknowledged.ActivityEnvironmentID = environmentID- }- return acknowledged, nil+ return scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: kit.Ternary(acknowledged.ActivityID != "", environmentID, acknowledged.ActivityEnvironmentID),+ ActivityID: acknowledged.ActivityID,+ Resolution: acknowledged.Resolution,+ RequestedWithKey: acknowledged.RequestedWithKey,+ ID: acknowledged.ID,+ JobID: acknowledged.JobID,+ EnvironmentID: environmentID,+ Trigger: acknowledged.Trigger,+ RequestedBy: acknowledged.RequestedBy,+ Status: acknowledged.Status,+ CreatedAt: acknowledged.CreatedAt,+ UpdatedAt: acknowledged.UpdatedAt,+ StartedAt: acknowledged.StartedAt,+ FinishedAt: acknowledged.FinishedAt,+ NextAttempt: acknowledged.NextAttempt,+ AttemptCount: acknowledged.AttemptCount,+ Owner: acknowledged.Owner,+ Outcome: acknowledged.Outcome,+ Attempts: acknowledged.Attempts,+ RemoteDeliveryAttempted: acknowledged.RemoteDeliveryAttempted,+ RemoteOutcome: acknowledged.RemoteOutcome,+ RemoteRetryRequested: acknowledged.RemoteRetryRequested,+ RemoteRetryAttempted: acknowledged.RemoteRetryAttempted,+ RemoteAttemptCount: acknowledged.RemoteAttemptCount,+ RemoteAccepted: acknowledged.RemoteAccepted,+ RemoteSettled: acknowledged.RemoteSettled,+ LastConfirmedAt: acknowledged.LastConfirmedAt,+ }, nil}- now := time.Now().UTC()if updateRunErr := s.runs.UpdateRun(ctx, local, func(current *scheduler.Run) error {if current.Status != scheduler.NeedsAttention {return runs.ErrRunConflict}- outcome := acknowledged.Outcome- outcome.Status = acknowledged.Status- current.RemoteOutcome = &outcome- current.RemoteAccepted = true- current.RemoteSettled = true- current.LastConfirmedAt = &now+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: current.Status,+ CreatedAt: current.CreatedAt,+ UpdatedAt: current.UpdatedAt,+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: &scheduler.Outcome{+ Status: acknowledged.Status,+ Message: acknowledged.Outcome.Message,+ ActivityID: acknowledged.Outcome.ActivityID,+ Targets: acknowledged.Outcome.Targets,+ Steps: acknowledged.Outcome.Steps,+ },+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: true,+ RemoteSettled: true,+ LastConfirmedAt: new(time.Now().UTC()),+ }return nil}); updateRunErr != nil {return scheduler.Run{}, updateRunErr@@ -473,9 +710,9 @@ func (s *Service) ResolveRun(ctx context.Context, environmentID, jobID, runID, areturn s.runs.Resolve(ctx, environmentID, jobID, runID, acknowledged.Resolution.ResolvedBy)}-// resolveAgentReviewInternal settles legacy inactive runs on their owning agent.+// resolveAgentReview settles legacy inactive runs on their owning agent.// TODO(v3): remove the legacy resolution protocol.-func (s *Service) resolveAgentReviewInternal(ctx context.Context, environmentID, jobID, runID, actor string) (scheduler.Run, error) {+func (s *Service) resolveAgentReview(ctx context.Context, environmentID, jobID, runID, actor string) (scheduler.Run, error) {env, err := s.environments.GetEnvironmentByID(ctx, environmentID)if err != nil {return scheduler.Run{}, err@@ -500,10 +737,8 @@ func (s *Service) resolveAgentReviewInternal(ctx context.Context, environmentID,if resolveRemoteJobErr := s.environments.ProxyJSONRequest(ctx, environmentID, http.MethodPost, path+"/resolve", body, &remote); resolveRemoteJobErr != nil {return scheduler.Run{}, resolveRemoteJobErr}- } else if actor != user.SystemUser.Username {- if remote.Status != scheduler.Canceled || remote.Resolution == nil {- return scheduler.Run{}, errors.New("agent run must need attention before review resolution")- }+ } else if actor != user.SystemUser.Username && (remote.Status != scheduler.Canceled || remote.Resolution == nil) {+ return scheduler.Run{}, errors.New("agent run must need attention before review resolution")}if remote.ID != runID || remote.JobID != jobID || remote.EnvironmentID != "0" || !remote.Status.Terminal() ||(actor != user.SystemUser.Username && (remote.Status != scheduler.Canceled || remote.Resolution == nil)) {@@ -520,28 +755,21 @@ func (s *Service) resolveAgentReviewInternal(ctx context.Context, environmentID,return acknowledged, nil}-func remoteJobEligibleInternal(jobs []jobschedule.JobStatus, id string) bool {+func remoteJobEligible(jobs []jobschedule.JobStatus, id string) bool {for _, job := range jobs {if job.ID != id {- if remoteJobEligibleInternal(job.Children, id) {+ if remoteJobEligible(job.Children, id) {return true}continue}- if !job.Enabled || !job.CanRunManually || job.ManagerOnly {- return false- }- for _, prerequisite := range job.Prerequisites {- if !prerequisite.IsMet {- return false- }- }- return true+ return job.Enabled && job.CanRunManually && !job.ManagerOnly &&+ !slices.ContainsFunc(job.Prerequisites, func(prerequisite jobschedule.JobPrerequisite) bool { return !prerequisite.IsMet })}return false}-func remoteFailureInternal(err error) (scheduler.Outcome, error) {+func remoteFailure(err error) (scheduler.Outcome, error) {var status *remenv.StatusErrorrejected := errors.As(err, &status) && status.StatusCode >= 400 && status.StatusCode < 500 &&status.StatusCode != http.StatusRequestTimeout && status.StatusCode != http.StatusTooManyRequestsdiff --git a/backend/internal/job/children/runtime/service.go b/backend/internal/job/children/runtime/service.goindex 59104c5a4..458bfcd59 100644--- a/backend/internal/job/children/runtime/service.go+++ b/backend/internal/job/children/runtime/service.go@@ -12,113 +12,79 @@ import ("github.com/getarcaneapp/arcane/types/v2/jobschedule""github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/getarcaneapp/arcane/types/v2/user"+ "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs")// ApplyStatuses merges durable runs for one environment into jobs. The local// environment also gains dynamic children and scheduler worker health.-func ApplyStatuses(ctx context.Context, coordinator *runs.Coordinator, controller scheduler.JobController, environmentID string, jobs *[]jobschedule.JobStatus) error {+func ApplyStatuses(ctx context.Context, coordinator *runs.Coordinator, controller scheduler.JobController, environmentID string, jobs []jobschedule.JobStatus) ([]jobschedule.JobStatus, error) {+ records, err := coordinator.Records(ctx)+ if err != nil {+ return nil, fmt.Errorf("load durable job status: %w", err)+ }runsByJob := make(map[string][]scheduler.Run)- if coordinator != nil {- records, err := coordinator.Records(ctx)- if err != nil {- return fmt.Errorf("load durable job status: %w", err)- }- for _, record := range records {- if record.EnvironmentID == environmentID {- runsByJob[record.JobID] = append(runsByJob[record.JobID], record.Runs...)- }+ for _, record := range records {+ if record.EnvironmentID == environmentID {+ runsByJob[record.JobID] = append(runsByJob[record.JobID], record.Runs...)}}jobScheduler, hasScheduler := controller.(scheduler.JobScheduler)- if environmentID == "0" && hasScheduler {- appendDynamicJobStatuses(ctx, jobScheduler, jobs)- }+ jobs, dynamicChildren := appendDynamicJobStatuses(ctx, controller, environmentID, jobs)- for index := range *jobs {- status := &(*jobs)[index]- ApplyRunStatus(status, runsByJob[status.ID])- if environmentID == "0" && hasScheduler {- setRuntimeHealth(ctx, jobScheduler, status)- }- for childIndex := range status.Children {- child := &status.Children[childIndex]- ApplyRunStatus(child, runsByJob[child.ID])+ statuses := make([]jobschedule.JobStatus, 0, len(jobs))+ for _, job := range jobs {+ status := ApplyRunStatus(job, runsByJob[job.ID])+ workerHealth, nextRun := func() (*scheduler.WorkerHealth, *time.Time) {+ if environmentID != "0" || !hasScheduler {+ return status.WorkerHealth, status.NextRun+ }+ if health, found := jobScheduler.WatcherHealth(status.ID); found {+ return &health, status.NextRun+ }+ registered, found := jobScheduler.GetJob(status.ID)+ if !found {+ return status.WorkerHealth, status.NextRun+ }+ if conditional, ok := registered.(scheduler.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {+ return status.WorkerHealth, status.NextRun+ }+ if state, stateFound := jobScheduler.GetJobRuntimeState(status.ID); !stateFound || !state.Scheduled {+ return &scheduler.WorkerHealth{+ Status: "needs_attention",+ LastError: "Job has no installed schedule",+ UpdatedAt: time.Now().UTC(),+ }, nil+ }+ return status.WorkerHealth, status.NextRun+ }()+ var children []jobschedule.JobStatus+ for _, child := range slices.Concat(status.Children, dynamicChildren[status.ID]) {+ children = append(children, ApplyRunStatus(child, runsByJob[child.ID]))}+ statuses = append(statuses, jobschedule.JobStatus{ //exhaustruct:enforce+ CurrentRun: status.CurrentRun,+ LastRun: status.LastRun,+ LastSuccess: status.LastSuccess,+ LastError: status.LastError,+ WorkerHealth: workerHealth,+ Children: children,+ ID: status.ID,+ Name: status.Name,+ Description: status.Description,+ Category: status.Category,+ Schedule: status.Schedule,+ NextRun: nextRun,+ Enabled: status.Enabled,+ ManagerOnly: status.ManagerOnly,+ IsContinuous: status.IsContinuous,+ CanRunManually: status.CanRunManually,+ Prerequisites: status.Prerequisites,+ SettingsKey: status.SettingsKey,+ })}- return nil-}--func setRuntimeHealth(ctx context.Context, jobScheduler scheduler.JobScheduler, status *jobschedule.JobStatus) {- if health, found := jobScheduler.WatcherHealth(status.ID); found {- status.WorkerHealth = &health- return- }- registered, found := jobScheduler.GetJob(status.ID)- if !found {- return- }- if conditional, ok := registered.(scheduler.ConditionalJob); ok && !conditional.ShouldSchedule(ctx) {- return- }- state, found := jobScheduler.GetJobRuntimeState(status.ID)- if !found || !state.Scheduled {- status.NextRun = nil- status.WorkerHealth = &scheduler.WorkerHealth{Status: "needs_attention", LastError: "Job has no installed schedule", UpdatedAt: time.Now().UTC()}- }-}--func currentRunPriority(status scheduler.RunStatus) int {- switch status {- case scheduler.Running:- return 0- case scheduler.Queued, scheduler.Waiting, scheduler.Retrying:- return 1- case scheduler.NeedsAttention, scheduler.Succeeded, scheduler.Partial, scheduler.Skipped, scheduler.Failed, scheduler.Canceled:- return 2- default:- return 2- }-}--func preferCurrentRun(candidate, current scheduler.Run) bool {- if candidate.ID == current.ID {- return candidate.UpdatedAt.After(current.UpdatedAt)- }- candidatePriority := currentRunPriority(candidate.Status)- currentPriority := currentRunPriority(current.Status)- if candidatePriority != currentPriority {- return candidatePriority < currentPriority- }- if candidate.CreatedAt.Equal(current.CreatedAt) {- return candidate.ID > current.ID- }- return candidate.CreatedAt.After(current.CreatedAt)-}--func runHasError(status scheduler.RunStatus) bool {- switch status {- case scheduler.Failed, scheduler.Partial, scheduler.NeedsAttention, scheduler.Retrying:- return true- case scheduler.Queued, scheduler.Running, scheduler.Waiting, scheduler.Succeeded, scheduler.Skipped, scheduler.Canceled:- return false- default:- return false- }-}--func updateLastSuccess(status *jobschedule.JobStatus, run scheduler.Run) {- if run.Status != scheduler.Succeeded {- return- }- succeededAt := runExecutionTime(run)- if run.FinishedAt != nil {- succeededAt = *run.FinishedAt- }- if status.LastSuccess == nil || succeededAt.After(*status.LastSuccess) {- status.LastSuccess = new(succeededAt)- }+ return statuses, nil}func runExecutionTime(run scheduler.Run) time.Time {@@ -137,36 +103,50 @@ func runExecutionTime(run scheduler.Run) time.Time {return startedAt}-func updateLatestRun(status *jobschedule.JobStatus, run scheduler.Run) {- if !run.Status.Terminal() && run.Status != scheduler.NeedsAttention {- if status.CurrentRun == nil || preferCurrentRun(run, *status.CurrentRun) {- status.CurrentRun = new(run)- }- return- }- if status.LastRun == nil || runExecutionTime(run).After(runExecutionTime(*status.LastRun)) ||- (runExecutionTime(run).Equal(runExecutionTime(*status.LastRun)) && run.ID > status.LastRun.ID) {- status.LastRun = new(run)- }-}-func ProjectRunOutcome(run scheduler.Run) scheduler.Run {// Delivery reconciliation must not hide a failure already reported by the agent.- if run.RemoteOutcome != nil && run.RemoteOutcome.Status == scheduler.NeedsAttention && !run.RemoteRetryRequested {- run.Status = scheduler.Failed- run.Outcome = *run.RemoteOutcome+ agentFailure := kit.FromPtr(run.RemoteOutcome).Status == scheduler.NeedsAttention && !run.RemoteRetryRequested+ failed := agentFailure || run.Status == scheduler.Failed || run.Status == scheduler.NeedsAttention ||+ (run.Status == scheduler.Canceled && kit.FromPtr(run.Resolution).ResolvedBy == user.SystemUser.Username)+ outcome := kit.Ternary(agentFailure, kit.FromPtr(run.RemoteOutcome), run.Outcome)+ return scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: run.ActivityEnvironmentID,+ ActivityID: run.ActivityID,+ Resolution: run.Resolution,+ RequestedWithKey: run.RequestedWithKey,+ ID: run.ID,+ JobID: run.JobID,+ EnvironmentID: run.EnvironmentID,+ Trigger: run.Trigger,+ RequestedBy: run.RequestedBy,+ Status: kit.Ternary(failed, scheduler.Failed, run.Status),+ CreatedAt: run.CreatedAt,+ UpdatedAt: run.UpdatedAt,+ StartedAt: run.StartedAt,+ FinishedAt: run.FinishedAt,+ NextAttempt: run.NextAttempt,+ AttemptCount: run.AttemptCount,+ Owner: run.Owner,+ Outcome: scheduler.Outcome{ //exhaustruct:enforce+ Status: kit.Ternary(failed, scheduler.Failed, outcome.Status),+ Message: outcome.Message,+ ActivityID: outcome.ActivityID,+ Targets: outcome.Targets,+ Steps: outcome.Steps,+ },+ Attempts: run.Attempts,+ RemoteDeliveryAttempted: run.RemoteDeliveryAttempted,+ RemoteOutcome: run.RemoteOutcome,+ RemoteRetryRequested: run.RemoteRetryRequested,+ RemoteRetryAttempted: run.RemoteRetryAttempted,+ RemoteAttemptCount: run.RemoteAttemptCount,+ RemoteAccepted: run.RemoteAccepted,+ RemoteSettled: run.RemoteSettled,+ LastConfirmedAt: run.LastConfirmedAt,}- automaticResolution := run.Status == scheduler.Canceled && run.Resolution != nil && run.Resolution.ResolvedBy == user.SystemUser.Username- if run.Status == scheduler.NeedsAttention || automaticResolution {- run.Status = scheduler.Failed- }- if run.Status == scheduler.Failed {- run.Outcome.Status = scheduler.Failed- }- return run}-func ApplyRunStatus(status *jobschedule.JobStatus, jobRuns []scheduler.Run) {+func ApplyRunStatus(status jobschedule.JobStatus, jobRuns []scheduler.Run) jobschedule.JobStatus {latest := make(map[string]scheduler.Run, len(jobRuns)+2)for _, run := range []*scheduler.Run{status.CurrentRun, status.LastRun} {if run != nil {@@ -180,94 +160,119 @@ func ApplyRunStatus(status *jobschedule.JobStatus, jobRuns []scheduler.Run) {latest[run.ID] = run}}- status.CurrentRun = nil- status.LastRun = nil- status.LastError = ""- for _, run := range latest {- run = ProjectRunOutcome(run)- updateLatestRun(status, run)- updateLastSuccess(status, run)- }- run := status.CurrentRun- if last := status.LastRun; last != nil {- if run == nil || runExecutionTime(*last).After(runExecutionTime(*run)) {- run = last+ priority := func(runStatus scheduler.RunStatus) int {+ switch runStatus {+ case scheduler.Running:+ return 0+ case scheduler.Queued, scheduler.Waiting, scheduler.Retrying:+ return 1+ case scheduler.NeedsAttention, scheduler.Succeeded, scheduler.Partial, scheduler.Skipped, scheduler.Failed, scheduler.Canceled:+ return 2+ default:+ return 2}}- if run != nil && runHasError(run.Status) {- status.LastError = run.Outcome.Message- }-}--func dynamicJobStatus(ctx context.Context, jobScheduler scheduler.JobScheduler, job scheduler.Job, category string, managerOnly bool) jobschedule.JobStatus {- child := jobschedule.JobStatus{- ID: job.Name(),- Name: job.Name(),- Category: category,- Schedule: job.Schedule(ctx),- Enabled: true,- ManagerOnly: managerOnly,- CanRunManually: true,- Prerequisites: []jobschedule.JobPrerequisite{},- }- if conditional, ok := job.(scheduler.ConditionalJob); ok {- child.Enabled = conditional.ShouldSchedule(ctx)+ preferCurrent := func(candidate, current scheduler.Run) bool {+ switch {+ case candidate.ID == current.ID:+ return candidate.UpdatedAt.After(current.UpdatedAt)+ case priority(candidate.Status) != priority(current.Status):+ return priority(candidate.Status) < priority(current.Status)+ case candidate.CreatedAt.Equal(current.CreatedAt):+ return candidate.ID > current.ID+ default:+ return candidate.CreatedAt.After(current.CreatedAt)+ }}- if state, ok := jobScheduler.GetJobRuntimeState(child.ID); ok {- child.Schedule = state.Schedule- child.NextRun = state.NextRun+ var currentRun, lastRun *scheduler.Run+ lastSuccess := status.LastSuccess+ for _, latestRun := range latest {+ run := ProjectRunOutcome(latestRun)+ switch {+ case !run.Status.Terminal() && run.Status != scheduler.NeedsAttention:+ if currentRun == nil || preferCurrent(run, *currentRun) {+ currentRun = new(run)+ }+ case lastRun == nil || runExecutionTime(run).After(runExecutionTime(*lastRun)) ||+ (runExecutionTime(run).Equal(runExecutionTime(*lastRun)) && run.ID > lastRun.ID):+ lastRun = new(run)+ }+ if run.Status == scheduler.Succeeded {+ succeededAt := kit.Ternary(run.FinishedAt != nil, kit.FromPtr(run.FinishedAt), runExecutionTime(run))+ if lastSuccess == nil || succeededAt.After(*lastSuccess) {+ lastSuccess = new(succeededAt)+ }+ }}- return child-}--func dynamicJobCategory(parentID string) (category string, managerOnly, supported bool) {- switch parentID {- case "environment-health":- return "monitoring", true, true- case "gitops-sync":- return "sync", false, true- case "system-backup":- return "maintenance", true, true- case "volume-backup":- return "maintenance", false, true- default:- return "", false, false+ errorRun := kit.Ternary(lastRun != nil && (currentRun == nil || runExecutionTime(*lastRun).After(runExecutionTime(*currentRun))), lastRun, currentRun)+ return jobschedule.JobStatus{ //exhaustruct:enforce+ CurrentRun: currentRun,+ LastRun: lastRun,+ LastSuccess: lastSuccess,+ LastError: kit.Ternary(+ slices.Contains([]scheduler.RunStatus{scheduler.Failed, scheduler.Partial, scheduler.NeedsAttention, scheduler.Retrying}, kit.FromPtr(errorRun).Status),+ kit.FromPtr(errorRun).Outcome.Message,+ "",+ ),+ WorkerHealth: status.WorkerHealth,+ Children: status.Children,+ ID: status.ID,+ Name: status.Name,+ Description: status.Description,+ Category: status.Category,+ Schedule: status.Schedule,+ NextRun: status.NextRun,+ Enabled: status.Enabled,+ ManagerOnly: status.ManagerOnly,+ IsContinuous: status.IsContinuous,+ CanRunManually: status.CanRunManually,+ Prerequisites: status.Prerequisites,+ SettingsKey: status.SettingsKey,}}-func appendDynamicJobStatuses(ctx context.Context, jobScheduler scheduler.JobScheduler, jobs *[]jobschedule.JobStatus) {- parentIndices := make(map[string]int, len(*jobs))- for index, status := range *jobs {- parentIndices[status.ID] = index+func appendDynamicJobStatuses(+ ctx context.Context,+ controller scheduler.JobController,+ environmentID string,+ jobs []jobschedule.JobStatus,+) ([]jobschedule.JobStatus, map[string][]jobschedule.JobStatus) {+ jobScheduler, hasScheduler := controller.(scheduler.JobScheduler)+ if environmentID != "0" || !hasScheduler {+ return jobs, nil}registered := jobScheduler.ListRegisteredJobs()slices.SortFunc(registered, func(left, right scheduler.Job) int {return strings.Compare(left.Name(), right.Name())})+ dynamicChildren := make(map[string][]jobschedule.JobStatus)for _, registeredJob := range registered {parentID, _, dynamic := strings.Cut(registeredJob.Name(), ":")- if !dynamic {- continue- }- if parentID == "environment-health" {- if registeredJob.Name() == "environment-health:0" {- if index, exists := parentIndices[parentID]; exists {- health := dynamicJobStatus(ctx, jobScheduler, registeredJob, "monitoring", true)- health.Name = (*jobs)[index].Name- (*jobs)[index] = health- }+ parentIndex := slices.IndexFunc(jobs, func(job jobschedule.JobStatus) bool { return job.ID == parentID })+ if !dynamic || parentID == "environment-health" {+ if registeredJob.Name() == "environment-health:0" && parentIndex >= 0 {+ jobs[parentIndex] = dynamicJobStatus(ctx, jobScheduler, registeredJob, jobs[parentIndex].Name, "monitoring", true)}continue}- category, managerOnly, supported := dynamicJobCategory(parentID)+ category, managerOnly, supported := func() (string, bool, bool) {+ switch parentID {+ case "gitops-sync":+ return "sync", false, true+ case "system-backup":+ return "maintenance", true, true+ case "volume-backup":+ return "maintenance", false, true+ default:+ return "", false, false+ }+ }()if !supported {continue}- parentIndex, exists := parentIndices[parentID]- if !exists {- parentIndex = len(*jobs)- parentIndices[parentID] = parentIndex- *jobs = append(*jobs, jobschedule.JobStatus{+ if parentIndex < 0 {+ parentIndex = len(jobs)+ jobs = append(jobs, jobschedule.JobStatus{ID: parentID,Name: parentID,Category: category,@@ -276,13 +281,30 @@ func appendDynamicJobStatuses(ctx context.Context, jobScheduler scheduler.JobSchPrerequisites: []jobschedule.JobPrerequisite{},})}- parent := &(*jobs)[parentIndex]- duplicate := slices.ContainsFunc(parent.Children, func(child jobschedule.JobStatus) bool {+ duplicate := slices.ContainsFunc(jobs[parentIndex].Children, func(child jobschedule.JobStatus) bool {return child.ID == registeredJob.Name()})- if duplicate {- continue+ if !duplicate {+ dynamicChildren[parentID] = append(dynamicChildren[parentID], dynamicJobStatus(ctx, jobScheduler, registeredJob, registeredJob.Name(), category, managerOnly))}- parent.Children = append(parent.Children, dynamicJobStatus(ctx, jobScheduler, registeredJob, category, managerOnly))+ }+ return jobs, dynamicChildren+}++func dynamicJobStatus(ctx context.Context, jobScheduler scheduler.JobScheduler, job scheduler.Job, name, category string, managerOnly bool) jobschedule.JobStatus {+ state, scheduled := jobScheduler.GetJobRuntimeState(job.Name())+ return jobschedule.JobStatus{+ ID: job.Name(),+ Name: name,+ Category: category,+ Schedule: kit.Ternary(scheduled, state.Schedule, job.Schedule(ctx)),+ NextRun: state.NextRun,+ Enabled: func() bool {+ conditional, ok := job.(scheduler.ConditionalJob)+ return !ok || conditional.ShouldSchedule(ctx)+ }(),+ ManagerOnly: managerOnly,+ CanRunManually: true,+ Prerequisites: []jobschedule.JobPrerequisite{},}}diff --git a/backend/internal/job/handler.go b/backend/internal/job/handler.goindex 56bfd4353..8a7bb8474 100644--- a/backend/internal/job/handler.go+++ b/backend/internal/job/handler.go@@ -1,6 +1,7 @@package jobimport (+ "cmp""context""errors""net/http"@@ -10,6 +11,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/jobschedule""github.com/getarcaneapp/arcane/types/v2/scheduler"+ "go.getarcane.app/kit/pkg""github.com/getarcaneapp/arcane/backend/v2/internal/middleware""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"@@ -54,39 +56,43 @@ type JobSchedulesHandler struct {}func (h *JobSchedulesHandler) ListJobs(ctx context.Context, input *ListJobsInput) (*GetJobsOutput, error) {- var jobs *jobschedule.JobListResponse- var err error- if input.ID != "0" {- jobs, err = h.jobService.ListRemoteJobs(ctx, input.ID)- } else {- jobs, err = h.jobService.ListJobs(ctx)- }+ jobs, err := func() (*jobschedule.JobListResponse, error) {+ if input.ID != "0" {+ return h.jobService.ListRemoteJobs(ctx, input.ID)+ }+ return h.jobService.ListJobs(ctx)+ }()if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}return &GetJobsOutput{Body: *jobs}, nil}func (h *JobSchedulesHandler) RunJob(ctx context.Context, input *RunJobInput) (*RunJobOutput, error) {- runID := input.RequestID- if input.Body != nil && input.Body.RunID != "" {- if runID != "" && runID != input.Body.RunID {- return nil, huma.Error400BadRequest("Conflicting run IDs")- }- runID = input.Body.RunID+ if kit.FromPtr(input.Body).RunID != "" && input.RequestID != "" && input.RequestID != kit.FromPtr(input.Body).RunID {+ return nil, huma.Error400BadRequest("Conflicting run IDs")}userID, _ := middleware.GetUserIDFromContext(ctx)keyID, _ := ctx.Value(middleware.ContextKeyApiKeyID).(string)- trigger := "manual"- if h.jobService.cfg.AgentMode && userID == "agent" {- userID = ""- trigger = "remote"- }- run, err := h.jobService.Submit(ctx, scheduler.Request{RunID: runID, JobID: input.JobID, EnvironmentID: input.ID, Trigger: trigger, RequestedBy: userID, RequestedWithKey: keyID})+ run, err := h.jobService.Submit(ctx, scheduler.Request{+ RunID: cmp.Or(kit.FromPtr(input.Body).RunID, input.RequestID),+ JobID: input.JobID,+ EnvironmentID: input.ID,+ Trigger: kit.Ternary(h.jobService.cfg.AgentMode && userID == "agent", "remote", "manual"),+ RequestedWithKey: keyID,+ RequestedBy: kit.Ternary(h.jobService.cfg.AgentMode && userID == "agent", "", userID),+ })if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}- return &RunJobOutput{Body: jobschedule.JobRunResponse{Success: true, Message: "Job accepted", RunID: run.ID, Status: run.Status}}, nil+ return &RunJobOutput{+ Body: jobschedule.JobRunResponse{+ Success: true,+ Message: "Job accepted",+ RunID: run.ID,+ Status: run.Status,+ },+ }, nil}func (h *JobSchedulesHandler) Get(ctx context.Context, input *GetJobSchedulesInput) (*GetJobSchedulesOutput, error) {@@ -130,7 +136,7 @@ func (h *JobSchedulesHandler) Update(ctx context.Context, input *UpdateJobSchedufunc (h *JobSchedulesHandler) ListRuns(ctx context.Context, input *jobschedule.ListRunsInput) (*jobschedule.ListRunsOutput, error) {result, err := h.jobService.ListRuns(ctx, input.ID, input.JobID, input.Page, input.Limit)if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}return &jobschedule.ListRunsOutput{Body: result}, nil}@@ -140,7 +146,7 @@ func (h *JobSchedulesHandler) ListRuns(ctx context.Context, input *jobschedule.Lfunc (h *JobSchedulesHandler) GetRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {result, err := h.jobService.GetRun(ctx, input.ID, input.JobID, input.RunID)if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}return &jobschedule.RunOutput{Body: result}, nil}@@ -150,7 +156,7 @@ func (h *JobSchedulesHandler) GetRun(ctx context.Context, input *jobschedule.Runfunc (h *JobSchedulesHandler) RetryRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {result, err := h.jobService.RetryRun(ctx, input.ID, input.JobID, input.RunID)if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}return &jobschedule.RunOutput{Body: result}, nil}@@ -160,7 +166,7 @@ func (h *JobSchedulesHandler) RetryRun(ctx context.Context, input *jobschedule.Rfunc (h *JobSchedulesHandler) CancelRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {result, err := h.jobService.CancelRun(ctx, input.ID, input.JobID, input.RunID)if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}return &jobschedule.RunOutput{Body: result}, nil}@@ -170,19 +176,46 @@ func (h *JobSchedulesHandler) CancelRun(ctx context.Context, input *jobschedule.func (h *JobSchedulesHandler) AcknowledgeRun(ctx context.Context, input *jobschedule.RunInput) (*jobschedule.RunOutput, error) {result, err := h.jobService.GetRun(ctx, input.ID, input.JobID, input.RunID)if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}err = h.jobService.runs.UpdateRun(ctx, result, func(current *scheduler.Run) error {if !current.Status.Terminal() {return errors.New("run is not terminal")}- current.RemoteSettled = true- current.UpdatedAt = time.Now().UTC()+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: current.Status,+ CreatedAt: current.CreatedAt,+ UpdatedAt: time.Now().UTC(),+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: true,+ LastConfirmedAt: current.LastConfirmedAt,+ }result = *currentreturn nil})if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}return &jobschedule.RunOutput{Body: result}, nil}@@ -202,12 +235,17 @@ func (h *JobSchedulesHandler) RestartWorker(ctx context.Context, input *jobschedreturn nil, huma.Error503ServiceUnavailable("Scheduler unavailable")}if err := workers.RestartWatcher(ctx, input.JobID); err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}- return &RunJobOutput{Body: jobschedule.JobRunResponse{Success: true, Message: "Worker restart requested"}}, nil+ return &RunJobOutput{+ Body: jobschedule.JobRunResponse{+ Success: true,+ Message: "Worker restart requested",+ },+ }, nil}-func jobHTTPErrorInternal(err error) error {+func jobHTTPError(err error) error {if _, ok := errors.AsType[huma.StatusError](err); ok {return err}@@ -233,7 +271,7 @@ func (h *JobSchedulesHandler) ResolveRun(ctx context.Context, input *jobschedule}run, err := h.jobService.ResolveRun(ctx, input.ID, input.JobID, input.RunID, actor)if err != nil {- return nil, jobHTTPErrorInternal(err)+ return nil, jobHTTPError(err)}return &jobschedule.RunOutput{Body: run}, nil}diff --git a/backend/internal/job/module.go b/backend/internal/job/module.goindex 514f70019..e0ca2feea 100644--- a/backend/internal/job/module.go+++ b/backend/internal/job/module.go@@ -82,10 +82,7 @@ func RegisterJobSchedules(api huma.API, jobSvc *JobService, envSvc *environment.Tags: []string{"JobSchedules"},Security: handlerutil.DefaultOperationSecurity(),}, authz.PermJobsManage, h.RunJob)- h.registerRunRoutesInternal(api)-}-func (h *JobSchedulesHandler) registerRunRoutesInternal(api huma.API) {basePath := "/environments/{id}/jobs/{jobId}"middleware.RegisterWithPermission(api,diff --git a/backend/internal/job/service.go b/backend/internal/job/service.goindex 7f80a2634..58c81a5e7 100644--- a/backend/internal/job/service.go+++ b/backend/internal/job/service.go@@ -7,7 +7,7 @@ import ("fmt""log/slog""net"- "sort"+ "slices""strings""sync""time"@@ -19,6 +19,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/meta""github.com/getarcaneapp/arcane/types/v2/scheduler""github.com/robfig/cron/v3"+ "github.com/samber/mo""go.getarcane.app/kit/pkg""go.opentelemetry.io/otel""go.opentelemetry.io/otel/attribute"@@ -84,37 +85,34 @@ func NewJobService(localEnvironment *environment.EnvironmentService,localActivity *activity.ActivityService,) *JobService {- service := &JobService{- roles: roles,- environment: localEnvironment,- activity: localActivity,- remote: remote.New(coordinator, kv.NewKVService(db), localEnvironment),- runs: coordinator,- db: db,- settings: localSettings,- cfg: cfg,- location: cfg.GetLocation(),- }-meter := otel.Meter(tracing.InstrumentationName)- var errs [3]error- service.runCount, errs[0] = meter.Int64Counter("arcane.job.runs",+ runCount, runCountErr := meter.Int64Counter("arcane.job.runs",metric.WithDescription("Executed job runs by outcome"), metric.WithUnit("{run}"))- service.runDuration, errs[1] = meter.Float64Histogram("arcane.job.duration",+ runDuration, runDurationErr := meter.Float64Histogram("arcane.job.duration",metric.WithDescription("Job run duration by outcome"), metric.WithUnit("s"),metric.WithExplicitBucketBoundaries(0.01, 0.05, 0.1, 0.5, 1, 5, 10, 30, 60, 120, 300, 600, 1800, 3600))- service.activeRuns, errs[2] = meter.Int64UpDownCounter("arcane.job.active",+ activeRuns, activeRunsErr := meter.Int64UpDownCounter("arcane.job.active",metric.WithDescription("Job runs currently executing"), metric.WithUnit("{run}"))- if err := errors.Join(errs[:]...); err != nil {+ if err := errors.Join(runCountErr, runDurationErr, activeRunsErr); err != nil {otel.Handle(err)}- if coordinator != nil {- coordinator.SetExecutor(service.executeRunInternal, service.reconcileRunInternal)- if localActivity != nil {- coordinator.SetObserver(service)- }+ service := &JobService{+ activity: localActivity,+ runs: coordinator,+ remote: remote.New(coordinator, kv.NewKVService(db), localEnvironment),+ environment: localEnvironment,+ roles: roles,+ db: db,+ settings: localSettings,+ cfg: cfg,+ location: cfg.GetLocation(),+ runCount: runCount,+ runDuration: runDuration,+ activeRuns: activeRuns,}+ coordinator.SetExecutor(service.executeRun, service.reconcileRun)+ coordinator.SetObserver(service)return service}@@ -145,10 +143,7 @@ func (s *JobService) GetJobSchedules(ctx context.Context) jobschedule.Config {}func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule.Update) (jobschedule.Config, error) {- if s == nil || s.db == nil || s.settings == nil {- return jobschedule.Config{}, errors.New("job service not initialized")- }- if s.cfg != nil && s.cfg.UIConfigurationDisabled {+ if s.cfg.UIConfigurationDisabled {return jobschedule.Config{}, errors.New("job schedule updates are disabled")}@@ -159,22 +154,62 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedulecurrent stringupdate *string}{- {key: "environmentHealthInterval", current: current.EnvironmentHealthInterval, update: updates.EnvironmentHealthInterval},- {key: "eventCleanupInterval", current: current.EventCleanupInterval, update: updates.EventCleanupInterval},- {key: "expiredSessionsCleanupInterval", current: current.ExpiredSessionsCleanupInterval, update: updates.ExpiredSessionsCleanupInterval},- {key: "autoUpdateInterval", current: current.AutoUpdateInterval, update: updates.AutoUpdateInterval},- {key: "dockerClientRefreshInterval", current: current.DockerClientRefreshInterval, update: updates.DockerClientRefreshInterval},- {key: "pollingInterval", current: current.PollingInterval, update: updates.PollingInterval},- {key: "scheduledPruneInterval", current: current.ScheduledPruneInterval, update: updates.ScheduledPruneInterval},- {key: "vulnerabilityScanInterval", current: current.VulnerabilityScanInterval, update: updates.VulnerabilityScanInterval},- {key: "imageAutoPatchInterval", current: current.ImageAutoPatchInterval, update: updates.ImageAutoPatchInterval},- {key: "autoHealInterval", current: current.AutoHealInterval, update: updates.AutoHealInterval},+ {+ key: "environmentHealthInterval",+ current: current.EnvironmentHealthInterval,+ update: updates.EnvironmentHealthInterval,+ },+ {+ key: "eventCleanupInterval",+ current: current.EventCleanupInterval,+ update: updates.EventCleanupInterval,+ },+ {+ key: "expiredSessionsCleanupInterval",+ current: current.ExpiredSessionsCleanupInterval,+ update: updates.ExpiredSessionsCleanupInterval,+ },+ {+ key: "autoUpdateInterval",+ current: current.AutoUpdateInterval,+ update: updates.AutoUpdateInterval,+ },+ {+ key: "dockerClientRefreshInterval",+ current: current.DockerClientRefreshInterval,+ update: updates.DockerClientRefreshInterval,+ },+ {+ key: "pollingInterval",+ current: current.PollingInterval,+ update: updates.PollingInterval,+ },+ {+ key: "scheduledPruneInterval",+ current: current.ScheduledPruneInterval,+ update: updates.ScheduledPruneInterval,+ },+ {+ key: "vulnerabilityScanInterval",+ current: current.VulnerabilityScanInterval,+ update: updates.VulnerabilityScanInterval,+ },+ {+ key: "imageAutoPatchInterval",+ current: current.ImageAutoPatchInterval,+ update: updates.ImageAutoPatchInterval,+ },+ {+ key: "autoHealInterval",+ current: current.AutoHealInterval,+ update: updates.AutoHealInterval,+ },}// Validate inputs (cron expressions)parser := scheduleutil.Parser()for _, field := range fields {- if field.update == nil || *field.update == "" {+ if kit.FromPtr(field.update) == "" {continue}if _, err := parser.Parse(*field.update); err != nil {@@ -183,18 +218,15 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule}changedKeys := make([]string, 0, len(fields))- previousValues := make(map[string]string, len(fields))+ previousValues := make([]libarcane.SettingUpdate, 0, len(fields))valuesToUpdate := make([]libarcane.SettingUpdate, 0, len(fields))for _, field := range fields {- key := field.key- value := field.update- currentValue := field.current- if value == nil || *value == currentValue {+ if mo.PointerToOption(field.update).OrElse(field.current) == field.current {continue}- changedKeys = append(changedKeys, key)- previousValues[key] = currentValue- valuesToUpdate = append(valuesToUpdate, libarcane.SettingUpdate{Key: key, Value: *value})+ changedKeys = append(changedKeys, field.key)+ previousValues = append(previousValues, libarcane.SettingUpdate{Key: field.key, Value: field.current})+ valuesToUpdate = append(valuesToUpdate, libarcane.SettingUpdate{Key: field.key, Value: *field.update})}if len(valuesToUpdate) == 0 {@@ -205,222 +237,157 @@ func (s *JobService) UpdateJobSchedules(ctx context.Context, updates jobschedule}if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {- restoreErr := s.restoreJobSchedulesInternal(ctx, previousValues, changedKeys)- return jobschedule.Config{}, errors.Join(fmt.Errorf("failed to apply job schedule update: %w", err), restoreErr)+ return jobschedule.Config{}, errors.Join(fmt.Errorf("failed to apply job schedule update: %w", err), func() error {+ if restoreErr := s.settings.UpdateSettingValues(ctx, previousValues); restoreErr != nil {+ return fmt.Errorf("failed to restore previous job schedules: %w", restoreErr)+ }+ if rescheduleErr := s.RescheduleJobsForSettingKeys(ctx, changedKeys); rescheduleErr != nil {+ return fmt.Errorf("failed to restore runtime job schedules: %w", rescheduleErr)+ }+ return nil+ }())}return s.GetJobSchedules(ctx), nil}func (s *JobService) RescheduleJobsForSettingKeys(ctx context.Context, changedKeys []string) error {- if len(changedKeys) == 0 {- return nil- }- if s == nil || s.scheduler == nil {- return errors.New("job scheduler not initialized")- }-- changed := make(map[string]struct{}, len(changedKeys))- for _, key := range changedKeys {- changed[key] = struct{}{}- }-var rescheduleErrors []errorfor jobID, jobMeta := range meta.GetAllJobMetadata() {- if !jobMetadataAffectedBySettingInternal(jobMeta, changed) {+ affected := slices.ContainsFunc([]string{jobMeta.SettingsKey, jobMeta.EnabledKey}, func(key string) bool {+ return key != "" && slices.Contains(changedKeys, key)+ })+ if !affected {continue}- if s.cfg != nil && s.cfg.AgentMode && jobMeta.ManagerOnly {+ if s.cfg.AgentMode && jobMeta.ManagerOnly {slog.DebugContext(ctx, "Skipping manager-only job reschedule in agent mode", "job", jobID)continue}- if err := s.rescheduleAffectedJobInternal(ctx, jobID, jobMeta); err != nil {- rescheduleErrors = append(rescheduleErrors, err)- }- }-- return errors.Join(rescheduleErrors...)-}--// jobRescheduleContextInternal prefers the app lifecycle context so cron jobs-// and watchers outlive the HTTP request that triggered the reschedule.-func (s *JobService) jobRescheduleContextInternal(ctx context.Context) context.Context {- if s.lifecycleCtx != nil {- return s.lifecycleCtx- }- return ctx-}--// rescheduleAffectedJobInternal applies a settings change to one job:-// delegated fan-out jobs and continuous bus watchers are notified, cron jobs-// are rescheduled and their runtime schedule verified.-func (s *JobService) rescheduleAffectedJobInternal(ctx context.Context, jobID string, jobMeta meta.JobMetadata) error {- // environment-health fans out to per-environment dynamic jobs; delegate the- // reschedule to environment.EnvironmentService instead of looking up a single job.- if jobID == "environment-health" {- if s.OnEnvironmentHealthReschedule == nil {- return errors.New("environment-health rescheduler not initialized")- }- s.OnEnvironmentHealthReschedule(s.jobRescheduleContextInternal(ctx))- return nil- }- // Continuous bus watchers have no cron entry to reschedule; notify the- // watcher so it re-reads its poll schedule instead.- if jobMeta.IsContinuous {- if jobID == "image-polling" && s.settings != nil {- return s.settings.NotifySettingsChanges(s.jobRescheduleContextInternal(ctx), "pollingInterval")+ // environment-health fans out to per-environment dynamic jobs; delegate the+ // reschedule to environment.EnvironmentService instead of looking up a single job.+ if jobID == "environment-health" {+ if s.OnEnvironmentHealthReschedule == nil {+ rescheduleErrors = append(rescheduleErrors, errors.New("environment-health rescheduler not initialized"))+ continue+ }+ s.OnEnvironmentHealthReschedule(s.lifecycleCtx)+ continue}- return nil- }-- job, ok := s.scheduler.GetJob(jobID)- if !ok {- return fmt.Errorf("job %s not found in scheduler", jobID)- }-- slog.DebugContext(ctx, "Processing job setting change", "job", jobID, "settingsKey", jobMeta.SettingsKey, "enabledKey", jobMeta.EnabledKey)- rescheduleCtx := s.jobRescheduleContextInternal(ctx)- if err := s.scheduler.RescheduleJob(rescheduleCtx, job); err != nil {- return fmt.Errorf("reschedule job %s: %w", jobID, err)- }-- runtimeState, ok := s.scheduler.GetJobRuntimeState(jobID)- if !ok {- return fmt.Errorf("job %s has no runtime scheduler state", jobID)- }-- expectedSchedule := job.Schedule(rescheduleCtx)- if conditional, isConditional := job.(scheduler.ConditionalJob); isConditional && !conditional.ShouldSchedule(rescheduleCtx) {- expectedSchedule = ""- }- if runtimeState.Schedule != expectedSchedule {- return fmt.Errorf("job %s runtime schedule %q does not match requested schedule %q", jobID, runtimeState.Schedule, expectedSchedule)- }- return nil-}--func (s *JobService) restoreJobSchedulesInternal(ctx context.Context, previousValues map[string]string, changedKeys []string) error {- if len(previousValues) == 0 {- return nil- }- updates := make([]libarcane.SettingUpdate, 0, len(previousValues))- for key, value := range previousValues {- updates = append(updates, libarcane.SettingUpdate{Key: key, Value: value})- }- if err := s.settings.UpdateSettingValues(ctx, updates); err != nil {- return fmt.Errorf("failed to restore previous job schedules: %w", err)- }+ // Continuous bus watchers have no cron entry to reschedule; notify the+ // watcher so it re-reads its poll schedule instead.+ if jobMeta.IsContinuous {+ if jobID == "image-polling" {+ if err := s.settings.NotifySettingsChanges(s.lifecycleCtx, "pollingInterval"); err != nil {+ rescheduleErrors = append(rescheduleErrors, err)+ }+ }+ continue+ }- if err := s.RescheduleJobsForSettingKeys(ctx, changedKeys); err != nil {- return fmt.Errorf("failed to restore runtime job schedules: %w", err)- }+ job, ok := s.scheduler.GetJob(jobID)+ if !ok {+ rescheduleErrors = append(rescheduleErrors, fmt.Errorf("job %s not found in scheduler", jobID))+ continue+ }- return nil-}+ slog.DebugContext(ctx, "Processing job setting change", "job", jobID, "settingsKey", jobMeta.SettingsKey, "enabledKey", jobMeta.EnabledKey)+ if err := s.scheduler.RescheduleJob(s.lifecycleCtx, job); err != nil {+ rescheduleErrors = append(rescheduleErrors, fmt.Errorf("reschedule job %s: %w", jobID, err))+ continue+ }-func jobMetadataAffectedBySettingInternal(jobMeta meta.JobMetadata, changed map[string]struct{}) bool {- if jobMeta.SettingsKey != "" {- if _, ok := changed[jobMeta.SettingsKey]; ok {- return true+ runtimeState, ok := s.scheduler.GetJobRuntimeState(jobID)+ if !ok {+ rescheduleErrors = append(rescheduleErrors, fmt.Errorf("job %s has no runtime scheduler state", jobID))+ continue}- }- if jobMeta.EnabledKey != "" {- if _, ok := changed[jobMeta.EnabledKey]; ok {- return true++ conditional, isConditional := job.(scheduler.ConditionalJob)+ expectedSchedule := kit.Ternary(isConditional && !conditional.ShouldSchedule(s.lifecycleCtx), "", job.Schedule(s.lifecycleCtx))+ if runtimeState.Schedule != expectedSchedule {+ rescheduleErrors = append(rescheduleErrors, fmt.Errorf("job %s runtime schedule %q does not match requested schedule %q", jobID, runtimeState.Schedule, expectedSchedule))}}- return false++ return errors.Join(rescheduleErrors...)}func (s *JobService) ListJobs(ctx context.Context) (*jobschedule.JobListResponse, error) {- if s == nil || s.settings == nil {- return nil, errors.New("job service not initialized")- }-allMetadata := meta.GetAllJobMetadata()jobs := make([]jobschedule.JobStatus, 0, len(allMetadata))for _, jobMeta := range allMetadata {- schedule := s.getJobScheduleInternal(ctx, jobMeta)- nextRun := s.calculateNextRunInternal(schedule)- enabled := s.isJobEnabledInternal(ctx, jobMeta)- prerequisites := s.evaluatePrerequisitesInternal(ctx, jobMeta)-- if s.scheduler != nil && (jobMeta.SettingsKey != "" || jobMeta.ID == "upgrade-log-cleanup") && jobMeta.ID != "environment-health" {- if runtimeState, ok := s.scheduler.GetJobRuntimeState(jobMeta.ID); ok {- schedule = cmp.Or(runtimeState.Schedule, schedule)- nextRun = runtimeState.NextRun+ schedule := func() string {+ switch {+ // Continuous jobs with a settings key (image-polling) are event-driven but+ // also poll on that cron schedule, so surface the real expression.+ case jobMeta.IsContinuous && jobMeta.SettingsKey == "":+ return "continuous"+ case jobMeta.ID == "analytics-heartbeat":+ return "automatic (checked hourly; sent once per 24h)"+ case jobMeta.SettingsKey == "":+ return ""}- }-- jobStatus := jobMeta.ToJobStatus(schedule, nextRun, enabled, prerequisites)- jobs = append(jobs, jobStatus)+ defaultSchedule, _, _, err := settings.DefaultSettingsConfig().FieldByKey(jobMeta.SettingsKey)+ return s.settings.GetStringSetting(ctx, jobMeta.SettingsKey, kit.Ternary(err != nil || defaultSchedule == "", "0 0 0 * * *", defaultSchedule))+ }()+ runtimeState, hasRuntimeState := s.scheduler.GetJobRuntimeState(jobMeta.ID)+ useRuntimeState := hasRuntimeState && (jobMeta.SettingsKey != "" || jobMeta.ID == "upgrade-log-cleanup") && jobMeta.ID != "environment-health"+ jobs = append(jobs, jobMeta.ToJobStatus(+ cmp.Or(kit.Ternary(useRuntimeState, runtimeState.Schedule, ""), schedule),+ kit.Ternary(useRuntimeState, runtimeState.NextRun, func() *time.Time {+ if schedule == "" || schedule == "continuous" {+ return nil+ }+ // Parse schedule and force it to use the same timezone as the scheduler.+ sched, err := scheduleutil.Parser().Parse(schedule)+ if err != nil {+ return nil+ }+ if specSchedule, ok := sched.(*cron.SpecSchedule); ok {+ specSchedule.Location = s.location // keep: cron's parser returns the schedule without a location option+ }+ // Calculate next run using the configured timezone.+ return new(sched.Next(time.Now().In(s.location)))+ }()),+ s.isJobEnabled(ctx, jobMeta),+ s.evaluatePrerequisites(ctx, jobMeta),+ ))}- if err := runtime.ApplyStatuses(ctx, s.runs, s.scheduler, "0", &jobs); err != nil {+ statuses, err := runtime.ApplyStatuses(ctx, s.runs, s.scheduler, "0", jobs)+ if err != nil {return nil, err}// Sort jobs by ID to ensure stable UI order- sort.Slice(jobs, func(i, j int) bool {- return jobs[i].ID < jobs[j].ID+ slices.SortFunc(statuses, func(left, right jobschedule.JobStatus) int {+ return strings.Compare(left.ID, right.ID)})- isAgent := s.cfg != nil && s.cfg.AgentMode-return &jobschedule.JobListResponse{- Jobs: jobs,- IsAgent: isAgent,- DurableRuns: true, ObservedAt: time.Now().UTC(),+ Jobs: statuses,+ IsAgent: s.cfg.AgentMode,+ DurableRuns: true,+ ObservedAt: time.Now().UTC(),}, nil}-func (s *JobService) getJobScheduleInternal(ctx context.Context, localMeta meta.JobMetadata) string {- // Continuous jobs with a settings key (image-polling) are event-driven but- // also poll on that cron schedule, so surface the real expression.- if localMeta.IsContinuous && localMeta.SettingsKey == "" {- return "continuous"- }-- if localMeta.ID == "analytics-heartbeat" {- return "automatic (checked hourly; sent once per 24h)"- }-- if localMeta.SettingsKey == "" {- return ""- }-- defaultSchedule, _, _, err := settings.DefaultSettingsConfig().FieldByKey(localMeta.SettingsKey)- if err != nil || defaultSchedule == "" {- defaultSchedule = "0 0 0 * * *"- }-- return s.settings.GetStringSetting(ctx, localMeta.SettingsKey, defaultSchedule)-}--func (s *JobService) isJobEnabledInternal(ctx context.Context, localMeta meta.JobMetadata) bool {- if localMeta.EnabledKey != "" {- return s.settings.GetBoolSetting(ctx, localMeta.EnabledKey, false)- }- if localMeta.IsContinuous {- return true- }-- return true+func (s *JobService) isJobEnabled(ctx context.Context, localMeta meta.JobMetadata) bool {+ return localMeta.EnabledKey == "" || s.settings.GetBoolSetting(ctx, localMeta.EnabledKey, false)}-func (s *JobService) evaluatePrerequisitesInternal(ctx context.Context, localMeta meta.JobMetadata) []jobschedule.JobPrerequisite {+func (s *JobService) evaluatePrerequisites(ctx context.Context, localMeta meta.JobMetadata) []jobschedule.JobPrerequisite {prerequisites := make([]jobschedule.JobPrerequisite, 0, len(localMeta.Prerequisites))for _, prereq := range localMeta.Prerequisites {- isMet := s.settings.GetBoolSetting(ctx, prereq.SettingKey, false)-prerequisites = append(prerequisites, jobschedule.JobPrerequisite{SettingKey: prereq.SettingKey,Label: prereq.Label,- IsMet: isMet,+ IsMet: s.settings.GetBoolSetting(ctx, prereq.SettingKey, false),SettingsURL: prereq.SettingsURL,})}@@ -428,32 +395,6 @@ func (s *JobService) evaluatePrerequisitesInternal(ctx context.Context, localMetreturn prerequisites}-func (s *JobService) calculateNextRunInternal(schedule string) *time.Time {- if schedule == "" || schedule == "continuous" {- return nil- }-- // Parse schedule and force it to use the same timezone as the scheduler.- parser := scheduleutil.Parser()- sched, err := parser.Parse(schedule)- if err != nil {- return nil- }-- location := time.UTC- if s != nil && s.location != nil {- location = s.location- }-- if specSchedule, ok := sched.(*cron.SpecSchedule); ok {- specSchedule.Location = location- }-- // Calculate next run using the configured timezone.- now := time.Now().In(location)- return new(sched.Next(now))-}-func (s *JobService) Coordinator() *runs.Coordinator { return s.runs }// ActivityID assigns a summary identity before a visible run is persisted.@@ -462,14 +403,9 @@ func (s *JobService) ActivityID(run scheduler.Run) string {if run.Trigger == "remote" {return ""}- visible := run.Trigger == "manual"- switch run.JobID {- case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan":- visible = true- default:- visible = visible || strings.HasPrefix(run.JobID, "gitops-sync:") || strings.HasPrefix(run.JobID, "volume-backup:") || strings.HasPrefix(run.JobID, "system-backup:")- }- if !visible {+ if run.Trigger != "manual" &&+ !slices.Contains([]string{"auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan"}, run.JobID) &&+ !strings.HasPrefix(run.JobID, "gitops-sync:") && !strings.HasPrefix(run.JobID, "volume-backup:") && !strings.HasPrefix(run.JobID, "system-backup:") {return ""}return uuid.New().String()@@ -477,9 +413,6 @@ func (s *JobService) ActivityID(run scheduler.Run) string {// SyncRunActivity projects the latest committed run without changing execution state.func (s *JobService) SyncRunActivity(ctx context.Context, run scheduler.Run) error {- if s.activity == nil {- return nil- }s.activityMu.Lock()defer s.activityMu.Unlock()current, err := s.runs.Get(ctx, run.EnvironmentID, run.JobID, run.ID)@@ -492,17 +425,11 @@ func (s *JobService) SyncRunActivity(ctx context.Context, run scheduler.Run) errif current.ActivityID == "" {return nil}- name := current.JobID- if metadata, found := meta.GetJobMetadata(current.JobID); found {- name = metadata.Name- }- return s.activity.SyncJobRun(ctx, runtime.ProjectRunOutcome(current), name)+ metadata, found := meta.GetJobMetadata(current.JobID)+ return s.activity.SyncJobRun(ctx, runtime.ProjectRunOutcome(current), kit.Ternary(found, metadata.Name, current.JobID))}func (s *JobService) ReconcileStartupActivities(ctx context.Context, extraProtectedIDs ...string) error {- if s.activity == nil {- return nil- }records, err := s.runs.Records(ctx)if err != nil {return err@@ -543,7 +470,15 @@ func (s *JobService) ReconcileStartupActivities(ctx context.Context, extraProtec// Managers may accept requests for offline environments. Reusing a run ID// deduplicates admission within the same job and environment.func (s *JobService) Submit(ctx context.Context, request scheduler.Request) (scheduler.Run, error) {- request.EnvironmentID = cmp.Or(request.EnvironmentID, "0")+ request = scheduler.Request{ //exhaustruct:enforce+ ObservedAgentRun: request.ObservedAgentRun,+ RunID: request.RunID,+ JobID: request.JobID,+ EnvironmentID: cmp.Or(request.EnvironmentID, "0"),+ Trigger: request.Trigger,+ RequestedWithKey: request.RequestedWithKey,+ RequestedBy: request.RequestedBy,+ }if request.EnvironmentID != "0" {if s.cfg.AgentMode {return scheduler.Run{}, errors.New("agents cannot queue work for another environment")@@ -556,14 +491,13 @@ func (s *JobService) Submit(ctx context.Context, request scheduler.Request) (schreturn scheduler.Run{}, errors.New("environment is disabled")}metadata, ok := meta.GetJobMetadata(request.JobID)- dynamicRemote := strings.HasPrefix(request.JobID, "gitops-sync:") || strings.HasPrefix(request.JobID, "volume-backup:")- if !dynamicRemote && (!ok || metadata.ManagerOnly || !metadata.CanRunManually) {+ if !strings.HasPrefix(request.JobID, "gitops-sync:") && !strings.HasPrefix(request.JobID, "volume-backup:") && (!ok || metadata.ManagerOnly || !metadata.CanRunManually) {return scheduler.Run{}, errors.New("job is not remotely runnable")}- } else if err := s.validateLocalJobInternal(ctx, request.JobID); err != nil {+ } else if err := s.validateLocalJob(ctx, request.JobID); err != nil {return scheduler.Run{}, err}- if err := s.authorizeRunInternal(+ if err := s.authorizeRun(ctx,scheduler.Run{Trigger: request.Trigger,@@ -577,7 +511,7 @@ func (s *JobService) Submit(ctx context.Context, request scheduler.Request) (schreturn s.runs.Submit(ctx, request)}-func (s *JobService) validateLocalJobInternal(ctx context.Context, jobID string) error {+func (s *JobService) validateLocalJob(ctx context.Context, jobID string) error {if metadata, ok := meta.GetJobMetadata(jobID); ok {if !metadata.CanRunManually {return errors.New("job cannot be run manually")@@ -585,21 +519,16 @@ func (s *JobService) validateLocalJobInternal(ctx context.Context, jobID string)if s.cfg.AgentMode && metadata.ManagerOnly {return errors.New("job is manager-only")}- if !s.isJobEnabledInternal(ctx, metadata) {+ if !s.isJobEnabled(ctx, metadata) {return errors.New("job is disabled")}- for _, prereq := range s.evaluatePrerequisitesInternal(ctx, metadata) {- if !prereq.IsMet {- return errors.New("job prerequisites are not met")- }+ if slices.ContainsFunc(s.evaluatePrerequisites(ctx, metadata), func(prereq jobschedule.JobPrerequisite) bool { return !prereq.IsMet }) {+ return errors.New("job prerequisites are not met")}if metadata.IsContinuous || jobID == "environment-health" {return nil}}- if s.scheduler == nil {- return errors.New("scheduler unavailable")- }job, ok := s.scheduler.GetJob(jobID)if !ok {return errors.New("job is not registered")@@ -610,16 +539,13 @@ func (s *JobService) validateLocalJobInternal(ctx context.Context, jobID string)return nil}-func (s *JobService) authorizeRunInternal(ctx context.Context, run scheduler.Run) error {+func (s *JobService) authorizeRun(ctx context.Context, run scheduler.Run) error {if run.RequestedBy == "" {if run.Trigger == "manual" && !s.cfg.AgentMode {return errors.New("requesting user unavailable")}return nil}- if s.roles == nil {- return errors.New("permission resolver unavailable")- }permissions, err := s.roles.ResolveExecutionPermissions(ctx, run.RequestedBy, run.RequestedWithKey)if err != nil {return err@@ -630,7 +556,7 @@ func (s *JobService) authorizeRunInternal(ctx context.Context, run scheduler.Runreturn nil}-func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run) (result scheduler.Outcome, runErr error) {+func (s *JobService) executeRun(ctx context.Context, run scheduler.Run) (result scheduler.Outcome, runErr error) {// Entity-scoped jobs ("gitops-sync:<id>") share one span name and metric series per job type.jobType, _, _ := strings.Cut(run.JobID, ":")ctx, span := otel.Tracer(tracing.InstrumentationName).Start(ctx, "job "+jobType, trace.WithAttributes(@@ -640,30 +566,27 @@ func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run)))jobAttr := attribute.String("arcane.job.id", jobType)started := time.Now()- if s.activeRuns != nil {- s.activeRuns.Add(ctx, 1, metric.WithAttributes(jobAttr))- }+ s.activeRuns.Add(ctx, 1, metric.WithAttributes(jobAttr))defer func() {outcomeAttr := attribute.String("arcane.job.outcome", string(result.Status))span.SetAttributes(outcomeAttr)tracing.End(span, runErr)- if s.activeRuns != nil {- // Record against a non-canceled context so shutdown-interrupted runs still count.- metricCtx := context.WithoutCancel(ctx)- attrs := metric.WithAttributes(jobAttr, outcomeAttr)- s.activeRuns.Add(metricCtx, -1, metric.WithAttributes(jobAttr))- s.runCount.Add(metricCtx, 1, attrs)- s.runDuration.Record(metricCtx, time.Since(started).Seconds(), attrs)- }+ // Record against a non-canceled context so shutdown-interrupted runs still count.+ metricCtx := context.WithoutCancel(ctx)+ attrs := metric.WithAttributes(jobAttr, outcomeAttr)+ s.activeRuns.Add(metricCtx, -1, metric.WithAttributes(jobAttr))+ s.runCount.Add(metricCtx, 1, attrs)+ s.runDuration.Record(metricCtx, time.Since(started).Seconds(), attrs)}()- if err := s.authorizeRunInternal(ctx, run); err != nil {+ if err := s.authorizeRun(ctx, run); err != nil {return scheduler.Outcome{Status: scheduler.Failed}, err}if run.EnvironmentID != "0" {return s.remote.Deliver(ctx, run)}- if requiresDockerInternal(run.JobID) && s.environment != nil {+ if slices.Contains([]string{"auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan", "image-polling"}, run.JobID) ||+ strings.HasPrefix(run.JobID, "gitops-sync:") || strings.HasPrefix(run.JobID, "volume-backup:") || strings.HasPrefix(run.JobID, "system-backup:") {status, err := s.environment.TestConnection(ctx, "0", nil)if err != nil || status != "online" {return scheduler.Outcome{Status: scheduler.Waiting, Message: "Waiting for Docker"}, err@@ -671,56 +594,64 @@ func (s *JobService) executeRunInternal(ctx context.Context, run scheduler.Run)}ctx = s.runs.ExecutionContext(ctx, run, "")if run.JobID == "environment-health" && s.RunEnvironmentHealthNow != nil {- err := s.RunEnvironmentHealthNow(ctx)- return classifyOutcomeInternal(run.JobID, scheduler.Outcome{}, err)+ return classifyOutcome(run.JobID, scheduler.Outcome{}, s.RunEnvironmentHealthNow(ctx))}if metadata, ok := meta.GetJobMetadata(run.JobID); ok && metadata.IsContinuous {- if err := s.validateLocalJobInternal(ctx, run.JobID); err != nil {+ if err := s.validateLocalJob(ctx, run.JobID); err != nil {return scheduler.Outcome{Status: scheduler.Canceled}, err}- err := s.scheduler.RunBusWatcherNow(ctx, run.JobID)- return classifyOutcomeInternal(run.JobID, scheduler.Outcome{}, err)+ return classifyOutcome(run.JobID, scheduler.Outcome{}, s.scheduler.RunBusWatcherNow(ctx, run.JobID))}- unavailableStatus := kit.Ternary(run.JobID == "auto-update" && run.AttemptCount > 1, scheduler.Failed, scheduler.Canceled)job, ok := s.scheduler.GetJob(run.JobID)if !ok {- return scheduler.Outcome{Status: unavailableStatus, Message: "Job or target no longer exists"}, nil+ return scheduler.Outcome{+ Status: kit.Ternary(run.JobID == "auto-update" && run.AttemptCount > 1, scheduler.Failed, scheduler.Canceled),+ Message: "Job or target no longer exists",+ }, nil}if conditional, localOk := job.(scheduler.ConditionalJob); localOk && !conditional.ShouldSchedule(ctx) {- return scheduler.Outcome{Status: unavailableStatus, Message: "Job is disabled"}, nil+ return scheduler.Outcome{+ Status: kit.Ternary(run.JobID == "auto-update" && run.AttemptCount > 1, scheduler.Failed, scheduler.Canceled),+ Message: "Job is disabled",+ }, nil}outcome, err := job.Run(ctx)- return classifyOutcomeInternal(run.JobID, outcome, err)+ return classifyOutcome(run.JobID, outcome, err)}-func classifyOutcomeInternal(jobID string, outcome scheduler.Outcome, err error) (scheduler.Outcome, error) {+func classifyOutcome(jobID string, outcome scheduler.Outcome, err error) (scheduler.Outcome, error) {if resultErr, ok := errors.AsType[*scheduler.OutcomeError](err); ok {outcome = resultErr.Outcome}- switch outcome.Status {- case scheduler.NeedsAttention:- outcome.Status = scheduler.Failed- return outcome, err- case scheduler.Waiting, scheduler.Retrying, scheduler.Failed, scheduler.Partial, scheduler.Canceled, scheduler.Skipped:- return outcome, err- case scheduler.Queued, scheduler.Running, scheduler.Succeeded:- // Classify these and unspecified outcomes below.- }-- var networkErr net.Error- switch {- case err == nil:- outcome.Status = cmp.Or(outcome.Status, scheduler.Succeeded)- case safeJobInternal(jobID) && (errors.As(err, &networkErr) || errors.Is(err, context.DeadlineExceeded)):- outcome.Status = scheduler.Retrying- case outcome.Status == "" || outcome.Status == scheduler.Succeeded:- outcome.Status = scheduler.Failed- }-- return outcome, err+ return scheduler.Outcome{ //exhaustruct:enforce+ Status: func() scheduler.RunStatus {+ switch outcome.Status {+ case scheduler.NeedsAttention:+ return scheduler.Failed+ case scheduler.Waiting, scheduler.Retrying, scheduler.Failed, scheduler.Partial, scheduler.Canceled, scheduler.Skipped:+ return outcome.Status+ case scheduler.Queued, scheduler.Running, scheduler.Succeeded:+ // Classify these and unspecified outcomes below.+ }+ var networkErr net.Error+ switch {+ case err == nil:+ return cmp.Or(outcome.Status, scheduler.Succeeded)+ case safeJob(jobID) && (errors.As(err, &networkErr) || errors.Is(err, context.DeadlineExceeded)):+ return scheduler.Retrying+ case outcome.Status == "" || outcome.Status == scheduler.Succeeded:+ return scheduler.Failed+ }+ return outcome.Status+ }(),+ Message: outcome.Message,+ ActivityID: outcome.ActivityID,+ Targets: outcome.Targets,+ Steps: outcome.Steps,+ }, err}-func safeJobInternal(jobID string) bool {+func safeJob(jobID string) bool {switch jobID {case "image-polling", "environment-health", "docker-client-refresh", "event-cleanup", "expired-sessions-cleanup", "activity-sweep", "upload-sessions-cleanup","git-clone-scratch-cleanup", "analytics-heartbeat", "apns-outbox", "vulnerability-scan":@@ -729,61 +660,57 @@ func safeJobInternal(jobID string) bool {return strings.HasPrefix(jobID, "environment-health:")}-func (s *JobService) reconcileRunInternal(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {- if err := s.authorizeRunInternal(ctx, run); err != nil {+func (s *JobService) reconcileRun(ctx context.Context, run scheduler.Run) (scheduler.Outcome, error) {+ if err := s.authorizeRun(ctx, run); err != nil {return scheduler.Outcome{Status: scheduler.Failed}, err}- if run.EnvironmentID != "0" {- return s.executeRunInternal(ctx, run)- }- if safeJobInternal(run.JobID) {- return s.executeRunInternal(ctx, run)+ if run.EnvironmentID != "0" || safeJob(run.JobID) {+ return s.executeRun(ctx, run)}- if s.scheduler != nil {- if job, ok := s.scheduler.GetJob(run.JobID); ok {- if reconciler, localOk := job.(scheduler.Reconciler); localOk {- return reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)- }+ if job, ok := s.scheduler.GetJob(run.JobID); ok {+ if reconciler, localOk := job.(scheduler.Reconciler); localOk {+ return reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)}}// A successful activity proves only that target, never the entire batch.+ targets := make([]scheduler.TargetOutcome, 0, len(run.Outcome.Targets))for index, target := range run.Outcome.Targets {- if run.JobID == "auto-update" {- break- }- if target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped {+ if run.JobID == "auto-update" || target.Status == scheduler.Succeeded || target.Status == scheduler.Skipped {+ targets = append(targets, target)continue}if target.ActivityID == "" {- return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted operation has no confirmed outcome", Targets: run.Outcome.Targets}, nil+ return scheduler.Outcome{+ Status: scheduler.Failed,+ Message: "Interrupted operation has no confirmed outcome",+ Targets: slices.Concat(targets, run.Outcome.Targets[index:]),+ }, nil}var record activity.Activityif err := s.db.WithContext(ctx).First(&record, "id = ?", target.ActivityID).Error; err != nil {return scheduler.Outcome{Status: scheduler.Failed}, err}if record.Status != activitytypes.StatusSuccess {- return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil- }- run.Outcome.Targets[index].Status = scheduler.Succeeded- }- if job, ok := s.scheduler.GetJob(run.JobID); ok {- if reconciler, localOk2 := job.(scheduler.Reconciler); localOk2 {- outcome, err := reconciler.Reconcile(s.runs.ExecutionContext(ctx, run, ""), run)- if err != nil {- slog.WarnContext(ctx, "Job reconciliation requires attention", "runId", run.ID, "error", err)- }- return outcome, err- }- }- return scheduler.Outcome{Status: scheduler.Failed, Message: "Interrupted operation has no confirmed completion", Targets: run.Outcome.Targets}, nil-}--func requiresDockerInternal(jobID string) bool {- switch jobID {- case "auto-update", "auto-patch", "auto-heal", "scheduled-prune", "vulnerability-scan", "image-polling":- return true+ return scheduler.Outcome{+ Status: scheduler.Failed,+ Message: "Interrupted operation has no confirmed completion",+ Targets: slices.Concat(targets, run.Outcome.Targets[index:]),+ }, nil+ }+ targets = append(targets, scheduler.TargetOutcome{ //exhaustruct:enforce+ RecoveryData: target.RecoveryData,+ ResourceType: target.ResourceType,+ ID: target.ID,+ Status: scheduler.Succeeded,+ Message: target.Message,+ ActivityID: target.ActivityID,+ })}- return strings.HasPrefix(jobID, "gitops-sync:") || strings.HasPrefix(jobID, "volume-backup:") || strings.HasPrefix(jobID, "system-backup:")+ return scheduler.Outcome{+ Status: scheduler.Failed,+ Message: "Interrupted operation has no confirmed completion",+ Targets: targets,+ }, nil}// ResolveRun authorizes the current operator independently of the original requester.@@ -806,23 +733,30 @@ func (s *JobService) ResolveRun(ctx context.Context, environmentID, jobID, runID// Remote retries follow the delivery protocol and retain confirmed target progress.func (s *JobService) RetryRun(ctx context.Context, environmentID, jobID, runID string) (scheduler.Run, error) {if environmentID != "0" {- return s.retryRemoteRunInternal(ctx, environmentID, jobID, runID)- }- if environmentID == "0" {- if err := s.validateLocalJobInternal(ctx, jobID); err != nil {- return scheduler.Run{}, err+ // Queue one explicit retry while retaining the agent's target progress.+ run, err := s.runs.Get(ctx, environmentID, jobID, runID)+ if errors.Is(err, runs.ErrRunNotFound) {+ return s.remote.MutateAgentRun(ctx, environmentID, jobID, runID, "retry")}+ if err != nil {+ return run, err+ }+ if authorizeRunErr := s.authorizeRun(ctx, run); authorizeRunErr != nil {+ return run, authorizeRunErr+ }+ return s.remote.RetryRun(ctx, run)+ }+ if err := s.validateLocalJob(ctx, jobID); err != nil {+ return scheduler.Run{}, err}run, err := s.runs.Get(ctx, environmentID, jobID, runID)if err != nil {return scheduler.Run{}, err}- if s.scheduler != nil {- if job, ok := s.scheduler.GetJob(jobID); ok {- if validator, localOk := job.(scheduler.RetryValidator); localOk {- if validateRetryErr := validator.ValidateRetry(ctx, run); validateRetryErr != nil {- return run, validateRetryErr- }+ if job, ok := s.scheduler.GetJob(jobID); ok {+ if validator, localOk := job.(scheduler.RetryValidator); localOk {+ if validateRetryErr := validator.ValidateRetry(ctx, run); validateRetryErr != nil {+ return run, validateRetryErr}}}@@ -835,26 +769,18 @@ func (s *JobService) ListRemoteJobs(ctx context.Context, environmentID string) (if err != nil {return nil, err}- if statusErr := runtime.ApplyStatuses(ctx, s.runs, s.scheduler, environmentID, &catalog.Jobs); statusErr != nil {- return nil, statusErr- }- sort.Slice(catalog.Jobs, func(i, j int) bool { return catalog.Jobs[i].ID < catalog.Jobs[j].ID })- return catalog, nil-}--// retryRemoteRunInternal queues one explicit retry while retaining the agent's target progress.-func (s *JobService) retryRemoteRunInternal(ctx context.Context, environmentID, jobID, runID string) (scheduler.Run, error) {- run, err := s.runs.Get(ctx, environmentID, jobID, runID)- if errors.Is(err, runs.ErrRunNotFound) {- return s.remote.MutateAgentRun(ctx, environmentID, jobID, runID, "retry")- }+ jobs, err := runtime.ApplyStatuses(ctx, s.runs, s.scheduler, environmentID, catalog.Jobs)if err != nil {- return run, err- }- if authorizeRunErr := s.authorizeRunInternal(ctx, run); authorizeRunErr != nil {- return run, authorizeRunErr+ return nil, err}- return s.remote.RetryRun(ctx, run)+ slices.SortFunc(jobs, func(left, right jobschedule.JobStatus) int { return strings.Compare(left.ID, right.ID) })+ return &jobschedule.JobListResponse{ //exhaustruct:enforce+ DurableRuns: catalog.DurableRuns,+ ObservedAt: catalog.ObservedAt,+ Offline: catalog.Offline,+ Jobs: jobs,+ IsAgent: catalog.IsAgent,+ }, nil}// ListRuns merges agent history with requests accepted by this manager.@@ -870,19 +796,21 @@ func (s *JobService) ListRuns(ctx context.Context, environmentID, jobID string,for _, run := range merged {localRuns = append(localRuns, runtime.ProjectRunOutcome(run))}- sort.Slice(localRuns, func(i, j int) bool {- if localRuns[i].CreatedAt.Equal(localRuns[j].CreatedAt) {- return localRuns[i].ID < localRuns[j].ID+ slices.SortFunc(localRuns, func(left, right scheduler.Run) int {+ if left.CreatedAt.Equal(right.CreatedAt) {+ return strings.Compare(left.ID, right.ID)}- return localRuns[i].CreatedAt.After(localRuns[j].CreatedAt)+ return right.CreatedAt.Compare(left.CreatedAt)})page = max(page, 1)limit = min(max(limit, 1), 100)- start := len(localRuns)- if page-1 <= len(localRuns)/limit {- start = min((page-1)*limit, len(localRuns))- }- return scheduler.RunList{Runs: localRuns[start:min(start+limit, len(localRuns))], Total: len(localRuns), Page: page, Limit: limit}, nil+ start := kit.Ternary(page-1 <= len(localRuns)/limit, min((page-1)*limit, len(localRuns)), len(localRuns))+ return scheduler.RunList{+ Runs: localRuns[start:min(start+limit, len(localRuns))],+ Total: len(localRuns),+ Page: page,+ Limit: limit,+ }, nil}// GetRun prefers the manager's durable delivery record, then queries agent-owned history.diff --git a/backend/internal/job/service_test.go b/backend/internal/job/service_test.goindex 3f042d277..6647bc734 100644--- a/backend/internal/job/service_test.go+++ b/backend/internal/job/service_test.go@@ -18,13 +18,13 @@ import ("github.com/libtnb/sqlite""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"+ "go.getarcane.app/kit/pkg""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/environment"- "github.com/getarcaneapp/arcane/backend/v2/internal/job/children/remote""github.com/getarcaneapp/arcane/backend/v2/internal/job/children/runtime""github.com/getarcaneapp/arcane/backend/v2/internal/kv""github.com/getarcaneapp/arcane/backend/v2/internal/middleware"@@ -36,11 +36,11 @@ import (francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")-func setupSettingsTestDBInternal(t *testing.T) *database.DB {+func setupSettingsTestDB(t *testing.T) *database.DB {t.Helper()db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}))+ require.NoError(t, db.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}, &activity.Activity{}, &activity.ActivityMessage{}))sqlDB, err := db.DB()require.NoError(t, err)sqlDB.SetMaxOpenConns(1)@@ -48,7 +48,7 @@ func setupSettingsTestDBInternal(t *testing.T) *database.DB {return &database.DB{DB: db}}-func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {+func newSettingsServiceForTest(t testing.TB, ctx context.Context, db *database.DB) (*settings.SettingsService, error) {t.Helper()service, err := settings.NewSettingsService(ctx, db)if err == nil {@@ -61,7 +61,7 @@ func newSettingsServiceForTestInternal(t testing.TB, ctx context.Context, db *dareturn service, err}-func newJobCoordinatorForTestInternal(t *testing.T, db *database.DB) *runs.Coordinator {+func newJobCoordinatorForTest(t *testing.T, db *database.DB) *runs.Coordinator {t.Helper()francisRuntime := francistest.New(t)coordinator := runs.New(kv.NewKVService(db), francisRuntime.Service(), time.UTC)@@ -70,19 +70,29 @@ func newJobCoordinatorForTestInternal(t *testing.T, db *database.DB) *runs.Coordreturn coordinator}-func newJobServiceForTestInternal(t *testing.T, db *database.DB, settingsService *settings.SettingsService, cfg *config.Config) *JobService {+func newJobServiceForTest(t *testing.T, db *database.DB, settingsService *settings.SettingsService, cfg *config.Config) *JobService {t.Helper()- return NewJobService(db, settingsService, cfg, newJobCoordinatorForTestInternal(t, db), nil, nil, nil)+ service := NewJobService(+ db,+ settingsService,+ cfg,+ newJobCoordinatorForTest(t, db),+ role.NewRoleService(db),+ environment.NewEnvironmentService(db, nil, nil, nil, nil, nil),+ activity.NewActivityService(db, settingsService),+ )+ service.SetScheduler(t.Context(), newFakeJobScheduler())+ return service}func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})cfg := jobSvc.GetJobSchedules(ctx)require.Equal(t, "0 */5 * * * *", cfg.DockerClientRefreshInterval)@@ -91,16 +101,16 @@ func TestJobService_GetJobSchedules_DefaultDockerClientRefreshInterval(t *testinfunc TestJobService_ListJobs_AnalyticsHeartbeatIsManagedInternally(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})jobs, err := jobSvc.ListJobs(ctx)require.NoError(t, err)- analyticsJob := findJobStatusByIDInternal(t, jobs.Jobs, "analytics-heartbeat")+ analyticsJob := findJobStatusByID(t, jobs.Jobs, "analytics-heartbeat")require.Equal(t, "automatic (checked hourly; sent once per 24h)", analyticsJob.Schedule)require.Empty(t, analyticsJob.SettingsKey)require.Nil(t, analyticsJob.NextRun)@@ -110,33 +120,33 @@ func TestJobService_ListJobs_AnalyticsHeartbeatIsManagedInternally(t *testing.T)func TestJobService_ListJobs_IncludesDisabledAutoHealJob(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetBoolSetting(ctx, "autoHealEnabled", false))- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})jobs, err := jobSvc.ListJobs(ctx)require.NoError(t, err)- autoHealJob := findJobStatusByIDInternal(t, jobs.Jobs, "auto-heal")+ autoHealJob := findJobStatusByID(t, jobs.Jobs, "auto-heal")require.False(t, autoHealJob.Enabled)require.Equal(t, "autoHealInterval", autoHealJob.SettingsKey)}func TestJobService_ListJobs_IncludesDockerClientRefreshJob(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})jobs, err := jobSvc.ListJobs(ctx)require.NoError(t, err)- refreshJob := findJobStatusByIDInternal(t, jobs.Jobs, "docker-client-refresh")+ refreshJob := findJobStatusByID(t, jobs.Jobs, "docker-client-refresh")require.True(t, refreshJob.Enabled)require.True(t, refreshJob.CanRunManually)require.Equal(t, "monitoring", refreshJob.Category)@@ -146,32 +156,36 @@ func TestJobService_ListJobs_IncludesDockerClientRefreshJob(t *testing.T) {func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)nextRun := time.Date(2026, time.July, 10, 8, 0, 0, 0, time.UTC)- fakeScheduler := newFakeJobSchedulerInternal("auto-update")+ fakeScheduler := newFakeJobScheduler("auto-update")fakeScheduler.runtimeStates["auto-update"] = scheduler.JobRuntimeState{Schedule: "0 0 8 * * *",NextRun: &nextRun,Scheduled: true,}- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})jobSvc.SetScheduler(ctx, fakeScheduler)jobs, err := jobSvc.ListJobs(ctx)require.NoError(t, err)- autoUpdateJob := findJobStatusByIDInternal(t, jobs.Jobs, "auto-update")+ autoUpdateJob := findJobStatusByID(t, jobs.Jobs, "auto-update")require.Equal(t, "0 0 8 * * *", autoUpdateJob.Schedule)require.Equal(t, nextRun, *autoUpdateJob.NextRun)now := time.Now().UTC()failed := scheduler.Run{- ID: "failed", EnvironmentID: "0", Status: scheduler.Failed, CreatedAt: now.Add(-time.Hour), UpdatedAt: now,- Outcome: scheduler.Outcome{Status: scheduler.Failed, Message: "clone failed"},+ ID: "failed",+ EnvironmentID: "0",+ Status: scheduler.Failed,+ CreatedAt: now.Add(-time.Hour),+ UpdatedAt: now,+ Outcome: scheduler.Outcome{Status: scheduler.Failed, Message: "clone failed"},}succeeded := scheduler.Run{ID: "succeeded",@@ -182,83 +196,138 @@ func TestJobService_ListJobs_UsesRuntimeScheduleAndNextRun(t *testing.T) {FinishedAt: new(now.Add(-time.Minute)),}for _, history := range [][]scheduler.Run{{failed, succeeded}, {succeeded, failed}} {- status := jobschedule.JobStatus{LastRun: new(failed), LastError: failed.Outcome.Message}- runtime.ApplyRunStatus(&status, history)+ status := runtime.ApplyRunStatus(jobschedule.JobStatus{LastRun: new(failed), LastError: failed.Outcome.Message}, history)require.Equal(t, succeeded.ID, status.LastRun.ID)require.Empty(t, status.LastError)require.Equal(t, "clone failed", history[0].Outcome.Message+history[1].Outcome.Message)}// An acknowledgement updates an old failure's revision, not its execution time.- failed.UpdatedAt = now.Add(time.Hour)- status := jobschedule.JobStatus{LastRun: new(succeeded)}- runtime.ApplyRunStatus(&status, []scheduler.Run{failed})+ acknowledged := scheduler.Run{+ ID: failed.ID,+ EnvironmentID: failed.EnvironmentID,+ Status: failed.Status,+ CreatedAt: failed.CreatedAt,+ UpdatedAt: now.Add(time.Hour),+ Outcome: failed.Outcome,+ }+ status := runtime.ApplyRunStatus(jobschedule.JobStatus{LastRun: new(succeeded)}, []scheduler.Run{acknowledged})require.Equal(t, succeeded.ID, status.LastRun.ID)require.Empty(t, status.LastError)// A real retry of the older run becomes the latest execution.- failed.StartedAt = new(now.Add(time.Second))- failed.Attempts = []scheduler.Attempt{{Number: 2, StartedAt: *failed.StartedAt}}- runtime.ApplyRunStatus(&status, []scheduler.Run{failed})- require.Equal(t, failed.ID, status.LastRun.ID)+ retried := scheduler.Run{+ ID: acknowledged.ID,+ EnvironmentID: acknowledged.EnvironmentID,+ Status: acknowledged.Status,+ CreatedAt: acknowledged.CreatedAt,+ UpdatedAt: acknowledged.UpdatedAt,+ StartedAt: new(now.Add(time.Second)),+ Outcome: acknowledged.Outcome,+ Attempts: []scheduler.Attempt{{Number: 2, StartedAt: now.Add(time.Second)}},+ }+ status = runtime.ApplyRunStatus(status, []scheduler.Run{retried})+ require.Equal(t, retried.ID, status.LastRun.ID)require.Equal(t, "clone failed", status.LastError)// Merge the same run's terminal revision before choosing an active run.- status.CurrentRun = new(succeeded)- status.CurrentRun.Status = scheduler.Running- succeeded.UpdatedAt = now.Add(2 * time.Second)- runtime.ApplyRunStatus(&status, []scheduler.Run{succeeded})+ status = runtime.ApplyRunStatus(+ jobschedule.JobStatus{+ CurrentRun: &scheduler.Run{+ ID: succeeded.ID,+ EnvironmentID: succeeded.EnvironmentID,+ Status: scheduler.Running,+ CreatedAt: succeeded.CreatedAt,+ UpdatedAt: succeeded.UpdatedAt,+ FinishedAt: succeeded.FinishedAt,+ },+ LastRun: status.LastRun,+ LastSuccess: status.LastSuccess,+ LastError: status.LastError,+ },+ []scheduler.Run{{+ ID: succeeded.ID,+ EnvironmentID: succeeded.EnvironmentID,+ Status: succeeded.Status,+ CreatedAt: succeeded.CreatedAt,+ UpdatedAt: now.Add(2 * time.Second),+ FinishedAt: succeeded.FinishedAt,+ }},+ )require.Nil(t, status.CurrentRun)- failed.Status = scheduler.Running- failed.UpdatedAt = now.Add(2 * time.Hour)- runtime.ApplyRunStatus(&status, []scheduler.Run{failed})- require.Equal(t, failed.ID, status.CurrentRun.ID)+ running := scheduler.Run{+ ID: retried.ID,+ EnvironmentID: retried.EnvironmentID,+ Status: scheduler.Running,+ CreatedAt: retried.CreatedAt,+ UpdatedAt: now.Add(2 * time.Hour),+ StartedAt: retried.StartedAt,+ Outcome: retried.Outcome,+ Attempts: retried.Attempts,+ }+ status = runtime.ApplyRunStatus(status, []scheduler.Run{running})+ require.Equal(t, running.ID, status.CurrentRun.ID)require.Empty(t, status.LastError)// A later manual failure stays visible while older work waits to retry.- retrying := failed- retrying.Status = scheduler.Retrying- retrying.NextAttempt = new(now.Add(time.Hour))- manualFailure := failed- manualFailure.ID = "manual-failure"- manualFailure.Status = scheduler.Failed- manualFailure.CreatedAt = now.Add(time.Minute)- manualFailure.StartedAt = nil- manualFailure.Attempts = nil- manualFailure.Outcome.Message = "latest manual failure"- succeeded.CreatedAt = now.Add(2 * time.Minute)+ retrying := scheduler.Run{+ ID: running.ID,+ EnvironmentID: running.EnvironmentID,+ Status: scheduler.Retrying,+ CreatedAt: running.CreatedAt,+ UpdatedAt: running.UpdatedAt,+ StartedAt: running.StartedAt,+ NextAttempt: new(now.Add(time.Hour)),+ Outcome: running.Outcome,+ Attempts: running.Attempts,+ }+ manualFailure := scheduler.Run{+ ID: "manual-failure",+ EnvironmentID: running.EnvironmentID,+ Status: scheduler.Failed,+ CreatedAt: now.Add(time.Minute),+ UpdatedAt: running.UpdatedAt,+ Outcome: scheduler.Outcome{Status: running.Outcome.Status, Message: "latest manual failure"},+ }+ laterSuccess := scheduler.Run{+ ID: succeeded.ID,+ EnvironmentID: succeeded.EnvironmentID,+ Status: succeeded.Status,+ CreatedAt: now.Add(2 * time.Minute),+ UpdatedAt: now.Add(2 * time.Second),+ FinishedAt: succeeded.FinishedAt,+ }for _, history := range [][]scheduler.Run{{retrying, manualFailure}, {manualFailure, retrying}} {- status = jobschedule.JobStatus{}- runtime.ApplyRunStatus(&status, history)+ status = runtime.ApplyRunStatus(jobschedule.JobStatus{}, history)require.Equal(t, retrying.ID, status.CurrentRun.ID)require.Equal(t, manualFailure.ID, status.LastRun.ID)require.Equal(t, manualFailure.Outcome.Message, status.LastError)- runtime.ApplyRunStatus(&status, []scheduler.Run{succeeded})+ status = runtime.ApplyRunStatus(status, []scheduler.Run{laterSuccess})require.Empty(t, status.LastError)}cloneErr := errors.New("failed to clone repository: network is unreachable")- outcome, runErr := classifyOutcomeInternal("gitops-sync:project", scheduler.Outcome{}, cloneErr)+ outcome, runErr := classifyOutcome("gitops-sync:project", scheduler.Outcome{}, cloneErr)require.ErrorIs(t, runErr, cloneErr)require.Equal(t, scheduler.Failed, outcome.Status)- outcome, runErr = classifyOutcomeInternal("environment-health:0", scheduler.Outcome{}, context.DeadlineExceeded)+ outcome, runErr = classifyOutcome("environment-health:0", scheduler.Outcome{}, context.DeadlineExceeded)require.ErrorIs(t, runErr, context.DeadlineExceeded)require.Equal(t, scheduler.Retrying, outcome.Status)- outcome, runErr = classifyOutcomeInternal("auto-update", scheduler.Outcome{Status: scheduler.Partial}, cloneErr)+ outcome, runErr = classifyOutcome("auto-update", scheduler.Outcome{Status: scheduler.Partial}, cloneErr)require.ErrorIs(t, runErr, cloneErr)require.Equal(t, scheduler.Partial, outcome.Status)}func TestJobService_ListJobs_ImageUpdateWatcherIsContinuousAndRespectsEnabled(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.SetBoolSetting(ctx, "pollingEnabled", false))- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})jobs, err := jobSvc.ListJobs(ctx)require.NoError(t, err)- watcher := findJobStatusByIDInternal(t, jobs.Jobs, "image-polling")+ watcher := findJobStatusByID(t, jobs.Jobs, "image-polling")require.Equal(t, "Image Update Watcher", watcher.Name)require.Equal(t, "0 0 * * * *", watcher.Schedule)require.Equal(t, "pollingInterval", watcher.SettingsKey)@@ -270,13 +339,13 @@ func TestJobService_ListJobs_ImageUpdateWatcherIsContinuousAndRespectsEnabled(tfunc TestJobService_UpdateJobSchedules_ReschedulesChangedJob(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})- fakeScheduler := newFakeJobSchedulerInternal("auto-update")+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})+ fakeScheduler := newFakeJobScheduler("auto-update")jobSvc.SetScheduler(ctx, fakeScheduler)_, err = jobSvc.UpdateJobSchedules(ctx, jobschedule.Update{@@ -289,13 +358,13 @@ func TestJobService_UpdateJobSchedules_ReschedulesChangedJob(t *testing.T) {func TestJobService_UpdateJobSchedules_DeprecatedPollingIntervalDoesNotReschedule(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})- fakeScheduler := newFakeJobSchedulerInternal("auto-update")+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})+ fakeScheduler := newFakeJobScheduler("auto-update")jobSvc.SetScheduler(ctx, fakeScheduler)updated, err := jobSvc.UpdateJobSchedules(ctx, jobschedule.Update{@@ -308,17 +377,17 @@ func TestJobService_UpdateJobSchedules_DeprecatedPollingIntervalDoesNotReschedulfunc TestJobService_UpdateJobSchedules_UsesLifecycleContextForReschedule(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)type lifecycleContextKey struct{}lifecycleCtx := context.WithValue(t.Context(), lifecycleContextKey{}, true)requestCtx, cancelRequest := context.WithCancel(t.Context())- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})- fakeScheduler := newFakeJobSchedulerInternal("auto-update")+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})+ fakeScheduler := newFakeJobScheduler("auto-update")jobSvc.SetScheduler(lifecycleCtx, fakeScheduler)_, err = jobSvc.UpdateJobSchedules(requestCtx, jobschedule.Update{@@ -335,15 +404,15 @@ func TestJobService_UpdateJobSchedules_UsesLifecycleContextForReschedule(t *testfunc TestJobService_UpdateJobSchedules_RejectsInvalidCronWithoutChangingSetting(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})- fakeScheduler := newFakeJobSchedulerInternal()+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})+ fakeScheduler := newFakeJobScheduler()jobSvc.SetScheduler(ctx, fakeScheduler)_, err = jobSvc.UpdateJobSchedules(ctx, jobschedule.Update{@@ -356,12 +425,12 @@ func TestJobService_UpdateJobSchedules_RejectsInvalidCronWithoutChangingSetting(func TestJobService_UpdateJobSchedules_UnchangedScheduleDoesNotReschedule(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})updated, err := jobSvc.UpdateJobSchedules(ctx, jobschedule.Update{PollingInterval: new("0 0 * * * *"),})@@ -371,16 +440,19 @@ func TestJobService_UpdateJobSchedules_UnchangedScheduleDoesNotReschedule(t *tesfunc TestJobService_UpdateJobSchedules_RestoresPreviousScheduleWhenRescheduleFails(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)require.NoError(t, settingsSvc.EnsureDefaultSettings(ctx))require.NoError(t, settingsSvc.LoadDatabaseSettings(ctx))- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})- fakeScheduler := newFakeJobSchedulerInternal("auto-update")- fakeScheduler.rescheduleErr = errors.New("scheduler unavailable")+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})+ fakeScheduler := &fakeJobScheduler{+ jobs: map[string]scheduler.Job{"auto-update": fakeJob{name: "auto-update"}},+ runtimeStates: map[string]scheduler.JobRuntimeState{},+ rescheduleErr: errors.New("scheduler unavailable"),+ }jobSvc.SetScheduler(ctx, fakeScheduler)_, err = jobSvc.UpdateJobSchedules(ctx, jobschedule.Update{@@ -396,13 +468,13 @@ func TestJobService_UpdateJobSchedules_RestoresPreviousScheduleWhenRescheduleFaifunc TestJobService_UpdateJobSchedules_SkipsManagerOnlyJobsInAgentMode(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{AgentMode: true})- fakeScheduler := newFakeJobSchedulerInternal("environment-health")+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{AgentMode: true})+ fakeScheduler := newFakeJobScheduler("environment-health")jobSvc.SetScheduler(ctx, fakeScheduler)_, err = jobSvc.UpdateJobSchedules(ctx, jobschedule.Update{@@ -415,17 +487,17 @@ func TestJobService_UpdateJobSchedules_SkipsManagerOnlyJobsInAgentMode(t *testinfunc TestJobService_UpdateJobSchedules_DelegatesEnvironmentHealthReschedule(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})- fakeScheduler := newFakeJobSchedulerInternal()+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})+ fakeScheduler := newFakeJobScheduler()jobSvc.SetScheduler(ctx, fakeScheduler)rescheduled := 0- jobSvc.OnEnvironmentHealthReschedule = func(context.Context) {+ jobSvc.OnEnvironmentHealthReschedule = func(context.Context) { // keep: bootstrap wires this callback after constructionrescheduled++}@@ -439,18 +511,26 @@ func TestJobService_UpdateJobSchedules_DelegatesEnvironmentHealthReschedule(t *tfunc TestJobService_Submit_PersistsImageUpdateWatcherRun(t *testing.T) {ctx := t.Context()- db := setupSettingsTestDBInternal(t)- settingsSvc, err := newSettingsServiceForTestInternal(t, ctx, db)+ db := setupSettingsTestDB(t)+ settingsSvc, err := newSettingsServiceForTest(t, ctx, db)require.NoError(t, err)- fakeScheduler := newFakeJobSchedulerInternal()- jobSvc := newJobServiceForTestInternal(t, db, settingsSvc, &config.Config{})+ fakeScheduler := newFakeJobScheduler()+ jobSvc := newJobServiceForTest(t, db, settingsSvc, &config.Config{})jobSvc.SetScheduler(ctx, fakeScheduler)require.NoError(t, db.AutoMigrate(&userdomain.User{}, &role.Role{}, &role.UserRoleAssignment{}))require.NoError(t, db.Create(&userdomain.User{Username: "operator", BaseModel: database.BaseModel{ID: "operator"}}).Error)- require.NoError(t, db.Create(&role.Role{ID: "jobs-manager", Name: "Jobs manager", Permissions: database.StringSlice{authz.PermJobsManage}}).Error)+ require.NoError(t, db.Create(&role.Role{+ ID: "jobs-manager",+ Name: "Jobs manager",+ Permissions: database.StringSlice{authz.PermJobsManage},+ }).Error)require.NoError(t, db.Create(&role.UserRoleAssignment{UserID: "operator", RoleID: "jobs-manager"}).Error)- jobSvc.roles = role.NewRoleService(db)- run, err := jobSvc.Submit(ctx, scheduler.Request{JobID: "image-polling", EnvironmentID: "0", Trigger: "manual", RequestedBy: "operator"})+ run, err := jobSvc.Submit(ctx, scheduler.Request{+ JobID: "image-polling",+ EnvironmentID: "0",+ Trigger: "manual",+ RequestedBy: "operator",+ })require.NoError(t, err)require.Equal(t, scheduler.Queued, run.Status)persisted, err := jobSvc.runs.Get(ctx, "0", "image-polling", run.ID)@@ -459,7 +539,7 @@ func TestJobService_Submit_PersistsImageUpdateWatcherRun(t *testing.T) {require.Empty(t, fakeScheduler.busWatcherRuns)}-func findJobStatusByIDInternal(t *testing.T, jobs []jobschedule.JobStatus, id string) jobschedule.JobStatus {+func findJobStatusByID(t *testing.T, jobs []jobschedule.JobStatus, id string) jobschedule.JobStatus {t.Helper()for _, job := range jobs {@@ -471,7 +551,7 @@ func findJobStatusByIDInternal(t *testing.T, jobs []jobschedule.JobStatus, id streturn jobschedule.JobStatus{}}-type fakeJobSchedulerInternal struct {+type fakeJobScheduler struct {jobs map[string]scheduler.JobruntimeStates map[string]scheduler.JobRuntimeStaterescheduled []string@@ -481,29 +561,29 @@ type fakeJobSchedulerInternal struct {busWatcherContexts []context.Context}-func newFakeJobSchedulerInternal(jobIDs ...string) *fakeJobSchedulerInternal {+func newFakeJobScheduler(jobIDs ...string) *fakeJobScheduler {jobs := make(map[string]scheduler.Job, len(jobIDs))for _, jobID := range jobIDs {- jobs[jobID] = fakeJobInternal{name: jobID}+ jobs[jobID] = fakeJob{name: jobID}}- return &fakeJobSchedulerInternal{+ return &fakeJobScheduler{jobs: jobs,runtimeStates: make(map[string]scheduler.JobRuntimeState),}}-func (s *fakeJobSchedulerInternal) GetJob(jobID string) (scheduler.Job, bool) {+func (s *fakeJobScheduler) GetJob(jobID string) (scheduler.Job, bool) {job, ok := s.jobs[jobID]return job, ok}-func (s *fakeJobSchedulerInternal) GetJobRuntimeState(jobID string) (scheduler.JobRuntimeState, bool) {+func (s *fakeJobScheduler) GetJobRuntimeState(jobID string) (scheduler.JobRuntimeState, bool) {state, ok := s.runtimeStates[jobID]return state, ok}-func (s *fakeJobSchedulerInternal) RescheduleJob(ctx context.Context, job scheduler.Job) error {+func (s *fakeJobScheduler) RescheduleJob(ctx context.Context, job scheduler.Job) error {s.rescheduled = append(s.rescheduled, job.Name())s.rescheduleContexts = append(s.rescheduleContexts, ctx)if s.rescheduleErr != nil {@@ -517,44 +597,58 @@ func (s *fakeJobSchedulerInternal) RescheduleJob(ctx context.Context, job schedureturn nil}-func (s *fakeJobSchedulerInternal) RunBusWatcherNow(ctx context.Context, watcherID string) error {+func (s *fakeJobScheduler) RunBusWatcherNow(ctx context.Context, watcherID string) error {s.busWatcherRuns = append(s.busWatcherRuns, watcherID)s.busWatcherContexts = append(s.busWatcherContexts, ctx)return nil}-type fakeJobInternal struct {+type fakeJob struct {name string}-func (j fakeJobInternal) Name() string {+func (j fakeJob) Name() string {return j.name}-func (j fakeJobInternal) Schedule(context.Context) string {+func (j fakeJob) Schedule(context.Context) string {return "0 0 0 * * *"}-func (j fakeJobInternal) Run(context.Context) (scheduler.Outcome, error) {+func (j fakeJob) Run(context.Context) (scheduler.Outcome, error) {return scheduler.Outcome{Status: scheduler.Succeeded}, nil}-func newJobActivityTestServiceInternal(t *testing.T) (*JobService, *activity.ActivityService, *database.DB) {+func newJobActivityTestService(t *testing.T) (*JobService, *activity.ActivityService, *database.DB) {t.Helper()- db := setupSettingsTestDBInternal(t)+ db := setupSettingsTestDB(t)sqlDB, err := db.DB.DB()require.NoError(t, err)sqlDB.SetMaxOpenConns(1)- require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))+ settingsService, err := newSettingsServiceForTest(t, t.Context(), db)+ require.NoError(t, err)activities := activity.NewActivityService(db, nil)- svc := NewJobService(db, nil, &config.Config{}, newJobCoordinatorForTestInternal(t, db), nil, nil, activities)+ svc := NewJobService(+ db,+ settingsService,+ &config.Config{},+ newJobCoordinatorForTest(t, db),+ role.NewRoleService(db),+ environment.NewEnvironmentService(db, nil, nil, nil, nil, nil),+ activities,+ )+ svc.SetScheduler(t.Context(), newFakeJobScheduler())return svc, activities, db}func TestJobActivityLifecycle(t *testing.T) {- svc, activities, db := newJobActivityTestServiceInternal(t)+ svc, activities, db := newJobActivityTestService(t)ctx := t.Context()- run, err := svc.runs.Submit(ctx, scheduler.Request{JobID: "auto-update", RunID: "b6d679c2-bdf5-4af1-985f-49c468983ba0", Trigger: "scheduled"})+ run, err := svc.runs.Submit(ctx, scheduler.Request{+ JobID: "auto-update",+ RunID: "b6d679c2-bdf5-4af1-985f-49c468983ba0",+ Trigger: "scheduled",+ })require.NoError(t, err)activityID, err := uuid.Parse(run.ActivityID)require.NoError(t, err)@@ -569,14 +663,50 @@ func TestJobActivityLifecycle(t *testing.T) {require.Equal(t, run.ID, *detail.Activity.BatchID)}check(activitytypes.StatusQueued, scheduler.Queued)- duplicate, err := svc.runs.Submit(ctx, scheduler.Request{JobID: run.JobID, RunID: run.ID, Trigger: "scheduled"})+ duplicate, err := svc.runs.Submit(ctx, scheduler.Request{+ JobID: run.JobID,+ RunID: run.ID,+ Trigger: "scheduled",+ })require.NoError(t, err)require.Equal(t, run.ActivityID, duplicate.ActivityID)for _, status := range []scheduler.RunStatus{scheduler.Running, scheduler.Waiting, scheduler.Failed} {require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- current.Status = status- current.UpdatedAt = time.Now().UTC()- current.Outcome = scheduler.Outcome{Status: status, Message: "operation detail"}+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: status,+ CreatedAt: current.CreatedAt,+ UpdatedAt: time.Now().UTC(),+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: scheduler.Outcome{ //exhaustruct:enforce+ Status: status,+ Message: "operation detail",+ ActivityID: "",+ Targets: nil,+ Steps: nil,+ },+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,+ }return nil}))}@@ -588,8 +718,35 @@ func TestJobActivityLifecycle(t *testing.T) {later, err := svc.runs.Submit(ctx, scheduler.Request{JobID: run.JobID, Trigger: "scheduled"})require.NoError(t, err)require.NoError(t, svc.runs.UpdateRun(ctx, later, func(current *scheduler.Run) error {- current.Status = scheduler.Succeeded- current.UpdatedAt = time.Now().UTC()+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: scheduler.Succeeded,+ CreatedAt: current.CreatedAt,+ UpdatedAt: time.Now().UTC(),+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,+ }return nil}))failedDetail, err = activities.GetActivityDetail(ctx, "0", run.ActivityID, 10)@@ -603,8 +760,35 @@ func TestJobActivityLifecycle(t *testing.T) {require.ErrorIs(t, err, activity.ErrActivityNotCancelable)check(activitytypes.StatusQueued, scheduler.Queued)require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- current.Status = scheduler.NeedsAttention- current.UpdatedAt = time.Now().UTC()+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: scheduler.NeedsAttention,+ CreatedAt: current.CreatedAt,+ UpdatedAt: time.Now().UTC(),+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,+ }return nil}))_, err = svc.runs.Resolve(ctx, "0", run.JobID, run.ID, "operator")@@ -616,24 +800,78 @@ func TestJobActivityLifecycle(t *testing.T) {}func TestJobActivityVisibilityAndGrouping(t *testing.T) {- svc, activities, _ := newJobActivityTestServiceInternal(t)+ svc, activities, _ := newJobActivityTestService(t)for _, test := range []struct {job, trigger, environment stringvisible bool}{- {"auto-update", "scheduled", "0", true},- {"gitops-sync:project", "scheduled", "0", true},- {"volume-backup:volume", "recovery", "0", true},- {"system-backup:system", "scheduled", "0", true},- {"image-polling", "manual", "0", true},- {"auto-update", "manual", "remote", true},- {"auto-update", "remote", "0", false},- {"image-polling", "scheduled", "0", false},- {"environment-health:0", "scheduled", "0", false},- {"activity-sweep", "scheduled", "0", false},+ {+ job: "auto-update",+ trigger: "scheduled",+ environment: "0",+ visible: true,+ },+ {+ job: "gitops-sync:project",+ trigger: "scheduled",+ environment: "0",+ visible: true,+ },+ {+ job: "volume-backup:volume",+ trigger: "recovery",+ environment: "0",+ visible: true,+ },+ {+ job: "system-backup:system",+ trigger: "scheduled",+ environment: "0",+ visible: true,+ },+ {+ job: "image-polling",+ trigger: "manual",+ environment: "0",+ visible: true,+ },+ {+ job: "auto-update",+ trigger: "manual",+ environment: "remote",+ visible: true,+ },+ {+ job: "auto-update",+ trigger: "remote",+ environment: "0",+ visible: false,+ },+ {+ job: "image-polling",+ trigger: "scheduled",+ environment: "0",+ visible: false,+ },+ {+ job: "environment-health:0",+ trigger: "scheduled",+ environment: "0",+ visible: false,+ },+ {+ job: "activity-sweep",+ trigger: "scheduled",+ environment: "0",+ visible: false,+ },} {t.Run(test.job+"/"+test.trigger+"/"+test.environment, func(t *testing.T) {- run, err := svc.runs.Submit(t.Context(), scheduler.Request{JobID: test.job, Trigger: test.trigger, EnvironmentID: test.environment})+ run, err := svc.runs.Submit(t.Context(), scheduler.Request{+ JobID: test.job,+ Trigger: test.trigger,+ EnvironmentID: test.environment,+ })require.NoError(t, err)require.Equal(t, test.visible, run.ActivityID != "")if test.visible {@@ -650,20 +888,61 @@ func TestJobActivityVisibilityAndGrouping(t *testing.T) {}func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {- svc, activities, db := newJobActivityTestServiceInternal(t)+ svc, activities, db := newJobActivityTestService(t)ctx := t.Context()require.NoError(t, db.Migrator().DropTable(&activity.ActivityMessage{}, &activity.Activity{}))run, err := svc.runs.Submit(ctx, scheduler.Request{JobID: "auto-update", Trigger: "scheduled"})require.NoError(t, err, "activity failure must not reject accepted work")require.NotEmpty(t, run.ActivityID)require.NoError(t, svc.runs.UpdateRun(ctx, run, func(current *scheduler.Run) error {- current.Status = scheduler.NeedsAttention- current.Outcome = scheduler.Outcome{Message: "interrupted before applying updates"}- current.UpdatedAt = time.Now().UTC()+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: scheduler.NeedsAttention,+ CreatedAt: current.CreatedAt,+ UpdatedAt: time.Now().UTC(),+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: scheduler.Outcome{ //exhaustruct:enforce+ Status: "",+ Message: "interrupted before applying updates",+ ActivityID: "",+ Targets: nil,+ Steps: nil,+ },+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,+ }return nil}))require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))- restarted := NewJobService(db, nil, &config.Config{}, svc.runs, nil, nil, activities)+ restarted := NewJobService(+ db,+ svc.settings,+ &config.Config{},+ svc.runs,+ role.NewRoleService(db),+ environment.NewEnvironmentService(db, nil, nil, nil, nil, nil),+ activities,+ )+ restarted.SetScheduler(ctx, newFakeJobScheduler())require.NoError(t, restarted.runs.Start(ctx, ctx))t.Cleanup(func() {stopCtx, cancel := context.WithTimeout(context.WithoutCancel(t.Context()), time.Second)@@ -689,24 +968,52 @@ func TestJobActivityRestartRepairsFailedProjection(t *testing.T) {func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {for _, automatic := range []bool{false, true} {- name := "legacy operator"- if automatic {- name = "automatic settlement"- }- t.Run(name, func(t *testing.T) {- db := setupSettingsTestDBInternal(t)+ t.Run(kit.Ternary(automatic, "automatic settlement", "legacy operator"), func(t *testing.T) {+ db := setupSettingsTestDB(t)require.NoError(t, db.AutoMigrate(&environment.Environment{}))- q := newJobCoordinatorForTestInternal(t, db)- local, err := q.Submit(t.Context(), scheduler.Request{JobID: "auto-update", EnvironmentID: "remote", Trigger: "scheduled"})+ q := newJobCoordinatorForTest(t, db)+ local, err := q.Submit(t.Context(), scheduler.Request{+ JobID: "auto-update",+ EnvironmentID: "remote",+ Trigger: "scheduled",+ })require.NoError(t, err)require.NoError(t, q.UpdateRun(t.Context(), local, func(run *scheduler.Run) error {- run.Status = scheduler.NeedsAttention- if automatic {- run.Status = scheduler.Running+ *run = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: run.ActivityEnvironmentID,+ ActivityID: run.ActivityID,+ Resolution: run.Resolution,+ RequestedWithKey: run.RequestedWithKey,+ ID: run.ID,+ JobID: run.JobID,+ EnvironmentID: run.EnvironmentID,+ Trigger: run.Trigger,+ RequestedBy: run.RequestedBy,+ Status: kit.Ternary(automatic, scheduler.Running, scheduler.NeedsAttention),+ CreatedAt: run.CreatedAt,+ UpdatedAt: run.UpdatedAt,+ StartedAt: run.StartedAt,+ FinishedAt: run.FinishedAt,+ NextAttempt: run.NextAttempt,+ AttemptCount: run.AttemptCount,+ Owner: run.Owner,+ Outcome: scheduler.Outcome{ //exhaustruct:enforce+ Status: scheduler.NeedsAttention,+ Message: "original failure",+ ActivityID: "",+ Targets: nil,+ Steps: nil,+ },+ Attempts: run.Attempts,+ RemoteDeliveryAttempted: true,+ RemoteOutcome: run.RemoteOutcome,+ RemoteRetryRequested: run.RemoteRetryRequested,+ RemoteRetryAttempted: run.RemoteRetryAttempted,+ RemoteAttemptCount: run.RemoteAttemptCount,+ RemoteAccepted: true,+ RemoteSettled: run.RemoteSettled,+ LastConfirmedAt: run.LastConfirmedAt,}- run.RemoteAccepted = true- run.RemoteDeliveryAttempted = true- run.Outcome = scheduler.Outcome{Status: scheduler.NeedsAttention, Message: "original failure"}return nil}))agentRun := scheduler.Run{@@ -740,13 +1047,72 @@ func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {ResolvedBy string `json:"resolvedBy"`}assert.NoError(t, json.UnmarshalRead(r.Body, &body))- agentRun.Status = scheduler.Canceled- agentRun.Resolution = &scheduler.RunResolution{ResolvedBy: body.ResolvedBy, ResolvedAt: time.Now().UTC(), Reason: "resolved_after_review"}+ agentRun = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: agentRun.ActivityEnvironmentID,+ ActivityID: agentRun.ActivityID,+ Resolution: &scheduler.RunResolution{+ ResolvedBy: body.ResolvedBy,+ ResolvedAt: time.Now().UTC(),+ Reason: "resolved_after_review",+ },+ RequestedWithKey: agentRun.RequestedWithKey,+ ID: agentRun.ID,+ JobID: agentRun.JobID,+ EnvironmentID: agentRun.EnvironmentID,+ Trigger: agentRun.Trigger,+ RequestedBy: agentRun.RequestedBy,+ Status: scheduler.Canceled,+ CreatedAt: agentRun.CreatedAt,+ UpdatedAt: agentRun.UpdatedAt,+ StartedAt: agentRun.StartedAt,+ FinishedAt: agentRun.FinishedAt,+ NextAttempt: agentRun.NextAttempt,+ AttemptCount: agentRun.AttemptCount,+ Owner: agentRun.Owner,+ Outcome: agentRun.Outcome,+ Attempts: agentRun.Attempts,+ RemoteDeliveryAttempted: agentRun.RemoteDeliveryAttempted,+ RemoteOutcome: agentRun.RemoteOutcome,+ RemoteRetryRequested: agentRun.RemoteRetryRequested,+ RemoteRetryAttempted: agentRun.RemoteRetryAttempted,+ RemoteAttemptCount: agentRun.RemoteAttemptCount,+ RemoteAccepted: agentRun.RemoteAccepted,+ RemoteSettled: agentRun.RemoteSettled,+ LastConfirmedAt: agentRun.LastConfirmedAt,+ }http.Error(w, "lost response", http.StatusServiceUnavailable)returncase strings.HasSuffix(r.URL.Path, "/ack"):acknowledgements++- agentRun.RemoteSettled = true+ agentRun = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: agentRun.ActivityEnvironmentID,+ ActivityID: agentRun.ActivityID,+ Resolution: agentRun.Resolution,+ RequestedWithKey: agentRun.RequestedWithKey,+ ID: agentRun.ID,+ JobID: agentRun.JobID,+ EnvironmentID: agentRun.EnvironmentID,+ Trigger: agentRun.Trigger,+ RequestedBy: agentRun.RequestedBy,+ Status: agentRun.Status,+ CreatedAt: agentRun.CreatedAt,+ UpdatedAt: agentRun.UpdatedAt,+ StartedAt: agentRun.StartedAt,+ FinishedAt: agentRun.FinishedAt,+ NextAttempt: agentRun.NextAttempt,+ AttemptCount: agentRun.AttemptCount,+ Owner: agentRun.Owner,+ Outcome: agentRun.Outcome,+ Attempts: agentRun.Attempts,+ RemoteDeliveryAttempted: agentRun.RemoteDeliveryAttempted,+ RemoteOutcome: agentRun.RemoteOutcome,+ RemoteRetryRequested: agentRun.RemoteRetryRequested,+ RemoteRetryAttempted: agentRun.RemoteRetryAttempted,+ RemoteAttemptCount: agentRun.RemoteAttemptCount,+ RemoteAccepted: agentRun.RemoteAccepted,+ RemoteSettled: true,+ LastConfirmedAt: agentRun.LastConfirmedAt,+ }if automatic && acknowledgements == 1 {http.Error(w, "lost acknowledgement", http.StatusServiceUnavailable)return@@ -762,9 +1128,24 @@ func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {assert.NoError(t, writeErr)}))defer server.Close()- require.NoError(t, db.Create(&environment.Environment{ID: "remote", Name: "agent", ApiUrl: server.URL, Enabled: true}).Error)+ require.NoError(t, db.Create(&environment.Environment{+ ID: "remote",+ Name: "agent",+ ApiUrl: server.URL,+ Enabled: true,+ }).Error)environments := environment.NewEnvironmentService(db, server.Client(), nil, nil, nil, nil)- svc := &JobService{runs: q, environment: environments, remote: remote.New(q, nil, environments)}+ settingsService, err := newSettingsServiceForTest(t, t.Context(), db)+ require.NoError(t, err)+ svc := NewJobService(+ db,+ settingsService,+ &config.Config{},+ q,+ role.NewRoleService(db),+ environments,+ activity.NewActivityService(db, nil),+ )ctx := context.WithValue(t.Context(), middleware.ContextKeyUserPermissions, authz.EnvironmentPermissionSet("remote"))path := "/api/environments/0/jobs/auto-update/runs/" + local.IDif !automatic {@@ -776,7 +1157,35 @@ func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {require.Equal(t, scheduler.Waiting, outcome.Status)}require.Zero(t, mutations)- agentRun.Status = scheduler.NeedsAttention+ agentRun = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: agentRun.ActivityEnvironmentID,+ ActivityID: agentRun.ActivityID,+ Resolution: agentRun.Resolution,+ RequestedWithKey: agentRun.RequestedWithKey,+ ID: agentRun.ID,+ JobID: agentRun.JobID,+ EnvironmentID: agentRun.EnvironmentID,+ Trigger: agentRun.Trigger,+ RequestedBy: agentRun.RequestedBy,+ Status: scheduler.NeedsAttention,+ CreatedAt: agentRun.CreatedAt,+ UpdatedAt: agentRun.UpdatedAt,+ StartedAt: agentRun.StartedAt,+ FinishedAt: agentRun.FinishedAt,+ NextAttempt: agentRun.NextAttempt,+ AttemptCount: agentRun.AttemptCount,+ Owner: agentRun.Owner,+ Outcome: agentRun.Outcome,+ Attempts: agentRun.Attempts,+ RemoteDeliveryAttempted: agentRun.RemoteDeliveryAttempted,+ RemoteOutcome: agentRun.RemoteOutcome,+ RemoteRetryRequested: agentRun.RemoteRetryRequested,+ RemoteRetryAttempted: agentRun.RemoteRetryAttempted,+ RemoteAttemptCount: agentRun.RemoteAttemptCount,+ RemoteAccepted: agentRun.RemoteAccepted,+ RemoteSettled: agentRun.RemoteSettled,+ LastConfirmedAt: agentRun.LastConfirmedAt,+ }if !automatic {_, err = svc.ResolveRun(ctx, "remote", local.JobID, local.ID, "operator")require.Error(t, err)@@ -806,25 +1215,104 @@ func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {require.Equal(t, scheduler.Failed, stored.RemoteOutcome.Status)require.Equal(t, user.SystemUser.Username, stored.Resolution.ResolvedBy)require.NoError(t, q.UpdateRun(ctx, stored, func(current *scheduler.Run) error {- current.Status = scheduler.Failed+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: scheduler.Failed,+ CreatedAt: current.CreatedAt,+ UpdatedAt: current.UpdatedAt,+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: current.Outcome,+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,+ }return nil}))_, retryErr := svc.RetryRun(ctx, "remote", local.JobID, local.ID)require.ErrorContains(t, retryErr, "upgrade the agent")- agentRun.Status = scheduler.Failed+ agentRun = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: agentRun.ActivityEnvironmentID,+ ActivityID: agentRun.ActivityID,+ Resolution: agentRun.Resolution,+ RequestedWithKey: agentRun.RequestedWithKey,+ ID: agentRun.ID,+ JobID: agentRun.JobID,+ EnvironmentID: agentRun.EnvironmentID,+ Trigger: agentRun.Trigger,+ RequestedBy: agentRun.RequestedBy,+ Status: scheduler.Failed,+ CreatedAt: agentRun.CreatedAt,+ UpdatedAt: agentRun.UpdatedAt,+ StartedAt: agentRun.StartedAt,+ FinishedAt: agentRun.FinishedAt,+ NextAttempt: agentRun.NextAttempt,+ AttemptCount: agentRun.AttemptCount,+ Owner: agentRun.Owner,+ Outcome: agentRun.Outcome,+ Attempts: agentRun.Attempts,+ RemoteDeliveryAttempted: agentRun.RemoteDeliveryAttempted,+ RemoteOutcome: agentRun.RemoteOutcome,+ RemoteRetryRequested: agentRun.RemoteRetryRequested,+ RemoteRetryAttempted: agentRun.RemoteRetryAttempted,+ RemoteAttemptCount: agentRun.RemoteAttemptCount,+ RemoteAccepted: agentRun.RemoteAccepted,+ RemoteSettled: agentRun.RemoteSettled,+ LastConfirmedAt: agentRun.LastConfirmedAt,+ }retried, retryErr := svc.RetryRun(ctx, "remote", local.JobID, local.ID)require.NoError(t, retryErr)require.Equal(t, scheduler.Queued, retried.Status)require.Equal(t, local.ID, retried.ID)// Adopt agent-owned history without coalescing its explicit identity.- agentOwned := agentRun- agentOwned.ID = "d7a9a3c7-d364-4c36-831a-381249e5727e"- agentOwned.Status = scheduler.NeedsAttention- agentOwned.CreatedAt = time.Now().UTC().Add(-time.Hour)- agentOwned.FinishedAt = new(agentOwned.CreatedAt.Add(time.Minute))- agentOwned.ActivityID = "agent-summary"- agentOwned.RemoteSettled = false+ createdAt := time.Now().UTC().Add(-time.Hour)+ agentOwned := scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: agentRun.ActivityEnvironmentID,+ ActivityID: "agent-summary",+ Resolution: agentRun.Resolution,+ RequestedWithKey: agentRun.RequestedWithKey,+ ID: "d7a9a3c7-d364-4c36-831a-381249e5727e",+ JobID: agentRun.JobID,+ EnvironmentID: agentRun.EnvironmentID,+ Trigger: agentRun.Trigger,+ RequestedBy: agentRun.RequestedBy,+ Status: scheduler.NeedsAttention,+ CreatedAt: createdAt,+ UpdatedAt: agentRun.UpdatedAt,+ StartedAt: agentRun.StartedAt,+ FinishedAt: new(createdAt.Add(time.Minute)),+ NextAttempt: agentRun.NextAttempt,+ AttemptCount: agentRun.AttemptCount,+ Owner: agentRun.Owner,+ Outcome: agentRun.Outcome,+ Attempts: agentRun.Attempts,+ RemoteDeliveryAttempted: agentRun.RemoteDeliveryAttempted,+ RemoteOutcome: agentRun.RemoteOutcome,+ RemoteRetryRequested: agentRun.RemoteRetryRequested,+ RemoteRetryAttempted: agentRun.RemoteRetryAttempted,+ RemoteAttemptCount: agentRun.RemoteAttemptCount,+ RemoteAccepted: agentRun.RemoteAccepted,+ RemoteSettled: false,+ LastConfirmedAt: agentRun.LastConfirmedAt,+ }catalog := []jobschedule.JobStatus{{ID: "updates", Children: []jobschedule.JobStatus{{ID: agentOwned.JobID, LastRun: &agentOwned}}}}require.NoError(t, svc.remote.ReconcileLegacyCatalog(ctx, "remote", catalog))require.NoError(t, svc.remote.ReconcileLegacyCatalog(ctx, "remote", catalog))@@ -837,8 +1325,7 @@ func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {require.Equal(t, agentOwned.FinishedAt, adopted.FinishedAt)require.Equal(t, agentOwned.ActivityID, adopted.ActivityID)require.Equal(t, "remote", adopted.ActivityEnvironmentID)- status := jobschedule.JobStatus{LastRun: &agentOwned}- runtime.ApplyRunStatus(&status, []scheduler.Run{adopted})+ status := runtime.ApplyRunStatus(jobschedule.JobStatus{LastRun: &agentOwned}, []scheduler.Run{adopted})require.Nil(t, status.CurrentRun)require.Equal(t, scheduler.Failed, status.LastRun.Status)require.Equal(t, "original failure", status.LastError)@@ -862,13 +1349,48 @@ func TestResolveRemoteRunConfirmsOwnerAfterLostResponse(t *testing.T) {}func TestResolveRunChecksCurrentOperatorPermission(t *testing.T) {- db := setupSettingsTestDBInternal(t)- svc := &JobService{runs: newJobCoordinatorForTestInternal(t, db)}+ db := setupSettingsTestDB(t)+ settingsService, err := newSettingsServiceForTest(t, t.Context(), db)+ require.NoError(t, err)+ svc := newJobServiceForTest(t, db, settingsService, &config.Config{})run, err := svc.runs.Submit(t.Context(), scheduler.Request{JobID: "auto-update", Trigger: "scheduled"})require.NoError(t, err)require.NoError(t, svc.runs.UpdateRun(t.Context(), run, func(current *scheduler.Run) error {- current.Status = scheduler.NeedsAttention- current.Outcome = scheduler.Outcome{Status: scheduler.NeedsAttention, Message: "interrupted"}+ *current = scheduler.Run{ //exhaustruct:enforce+ ActivityEnvironmentID: current.ActivityEnvironmentID,+ ActivityID: current.ActivityID,+ Resolution: current.Resolution,+ RequestedWithKey: current.RequestedWithKey,+ ID: current.ID,+ JobID: current.JobID,+ EnvironmentID: current.EnvironmentID,+ Trigger: current.Trigger,+ RequestedBy: current.RequestedBy,+ Status: scheduler.NeedsAttention,+ CreatedAt: current.CreatedAt,+ UpdatedAt: current.UpdatedAt,+ StartedAt: current.StartedAt,+ FinishedAt: current.FinishedAt,+ NextAttempt: current.NextAttempt,+ AttemptCount: current.AttemptCount,+ Owner: current.Owner,+ Outcome: scheduler.Outcome{ //exhaustruct:enforce+ Status: scheduler.NeedsAttention,+ Message: "interrupted",+ ActivityID: "",+ Targets: nil,+ Steps: nil,+ },+ Attempts: current.Attempts,+ RemoteDeliveryAttempted: current.RemoteDeliveryAttempted,+ RemoteOutcome: current.RemoteOutcome,+ RemoteRetryRequested: current.RemoteRetryRequested,+ RemoteRetryAttempted: current.RemoteRetryAttempted,+ RemoteAttemptCount: current.RemoteAttemptCount,+ RemoteAccepted: current.RemoteAccepted,+ RemoteSettled: current.RemoteSettled,+ LastConfirmedAt: current.LastConfirmedAt,+ }return nil}))_, err = svc.ResolveRun(t.Context(), "0", "auto-update", run.ID, "operator")diff --git a/backend/internal/vulnerability/children/ignores/handler.go b/backend/internal/vulnerability/children/ignores/handler.goindex 246cac286..ba8cdf229 100644--- a/backend/internal/vulnerability/children/ignores/handler.go+++ b/backend/internal/vulnerability/children/ignores/handler.go@@ -8,6 +8,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/base""github.com/getarcaneapp/arcane/types/v2/vulnerability"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/handlerutil")@@ -32,15 +33,17 @@ func (h *Handler) IgnoreVulnerability(ctx context.Context, input *IgnoreVulnerabreturn nil, err}- payload := &input.Body- payload.CreatedBy = user.ID-- ignore, err := h.service.IgnoreVulnerability(ctx, input.EnvironmentID, payload)+ ignore, err := h.service.IgnoreVulnerability(ctx, input.EnvironmentID, &vulnerability.IgnorePayload{+ ImageID: input.Body.ImageID,+ VulnerabilityID: input.Body.VulnerabilityID,+ PkgName: input.Body.PkgName,+ InstalledVersion: input.Body.InstalledVersion,+ Reason: input.Body.Reason,+ CreatedBy: user.ID,+ })if err != nil {- if err.Error() == "vulnerability is already ignored" {- return nil, huma.Error409Conflict(err.Error())- }- return nil, huma.Error500InternalServerError(err.Error())+ apiErr := common.ToAPIError(err)+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}return &handlerutil.Out[vulnerability.IgnoredVulnerability]{@@ -59,10 +62,8 @@ type UnignoreVulnerabilityInput struct {// UnignoreVulnerability removes an ignore record for a vulnerability.func (h *Handler) UnignoreVulnerability(ctx context.Context, input *UnignoreVulnerabilityInput) (*handlerutil.Out[struct{}], error) {if err := h.service.UnignoreVulnerability(ctx, input.EnvironmentID, input.IgnoreID); err != nil {- if err.Error() == "ignore record not found" {- return nil, huma.Error404NotFound(err.Error())- }- return nil, huma.Error500InternalServerError(err.Error())+ apiErr := common.ToAPIError(err)+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}return &handlerutil.Out[struct{}]{@@ -124,10 +125,11 @@ type ListIgnoredVulnerabilitiesInput struct {// ListIgnoredVulnerabilities returns a list of ignored vulnerabilities.func (h *Handler) ListIgnoredVulnerabilities(ctx context.Context, input *ListIgnoredVulnerabilitiesInput) (*handlerutil.Page[vulnerability.IgnoredVulnerability], error) {- params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)- params.Limit = cmp.Or(params.Limit, 20)-- items, paginationResp, err := h.service.ListIgnoredVulnerabilities(ctx, input.EnvironmentID, params)+ items, paginationResp, err := h.service.ListIgnoredVulnerabilities(+ ctx,+ input.EnvironmentID,+ handlerutil.PaginationParams(input.Start, cmp.Or(input.Limit, 20), input.Sort, input.Order, input.Search),+ )if err != nil {return nil, huma.Error500InternalServerError(err.Error())}diff --git a/backend/internal/vulnerability/children/ignores/service.go b/backend/internal/vulnerability/children/ignores/service.goindex cf9c5d7a0..a57b9da0e 100644--- a/backend/internal/vulnerability/children/ignores/service.go+++ b/backend/internal/vulnerability/children/ignores/service.go@@ -1,7 +1,6 @@package ignoresimport (- "cmp""context""errors""fmt"@@ -13,6 +12,7 @@ import ("go.getarcane.app/kit/pkg""gorm.io/gorm"+ "github.com/getarcaneapp/arcane/backend/v2/internal/common""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/dbutil"@@ -44,8 +44,8 @@ type Service struct {activeItemsQuery func(ctx context.Context, db *gorm.DB, envID string, params *pagination.QueryParams) *gorm.DB}-// ignoreTargetInternal identifies one finding to ignore.-type ignoreTargetInternal struct {+// ignoreTarget identifies one finding to ignore.+type ignoreTarget struct {ImageID string `gorm:"column:image_id"`VulnerabilityID string `gorm:"column:vulnerability_id"`PkgName string `gorm:"column:pkg_name"`@@ -53,7 +53,11 @@ type ignoreTargetInternal struct {}func NewService(db *database.DB, records Records, activeItemsQuery func(ctx context.Context, db *gorm.DB, envID string, params *pagination.QueryParams) *gorm.DB) *Service {- return &Service{db: db, records: records, activeItemsQuery: activeItemsQuery}+ return &Service{+ db: db,+ records: records,+ activeItemsQuery: activeItemsQuery,+ }}// FilterForImage drops the findings of imageID that are ignored in envID.@@ -62,7 +66,7 @@ func (s *Service) FilterForImage(envID, imageID string,vulns []vulnerability.Vulnerability,) ([]vulnerability.Vulnerability, error) {- if s.db == nil || len(vulns) == 0 || imageID == "" {+ if len(vulns) == 0 || imageID == "" {return vulns, nil}@@ -76,28 +80,17 @@ func (s *Service) FilterForImage(ignoredKeys := make(map[string]struct{}, len(ignores))for _, ignore := range ignores {- key := vulnerabilityIgnoreKeyInternal(ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion)- ignoredKeys[key] = struct{}{}- }-- filtered := make([]vulnerability.Vulnerability, 0, len(vulns))- for _, vulnEntry := range vulns {- key := vulnerabilityIgnoreKeyInternal(imageID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)- if _, isIgnored := ignoredKeys[key]; isIgnored {- continue- }- filtered = append(filtered, vulnEntry)+ ignoredKeys[ignore.ImageID+":"+ignore.VulnerabilityID+":"+ignore.PkgName+":"+ignore.InstalledVersion] = struct{}{}}- return filtered, nil+ return slices.DeleteFunc(slices.Clone(vulns), func(vulnEntry vulnerability.Vulnerability) bool {+ _, isIgnored := ignoredKeys[imageID+":"+vulnEntry.VulnerabilityID+":"+vulnEntry.PkgName+":"+vulnEntry.InstalledVersion]+ return isIgnored+ }), nil}// IgnoreVulnerability creates a new ignore record for a vulnerabilityfunc (s *Service) IgnoreVulnerability(ctx context.Context, envID string, payload *vulnerability.IgnorePayload) (*vulnerability.IgnoredVulnerability, error) {- if s.db == nil {- return nil, errors.New("database not available")- }-ignore := vulnerability.IgnoredVulnerability{EnvironmentID: envID,ImageID: payload.ImageID,@@ -114,7 +107,7 @@ func (s *Service) IgnoreVulnerability(ctx context.Context, envID string, payloadreturn nil, fmt.Errorf("failed to check existing ignore: %w", err)}if exists {- return nil, errors.New("vulnerability is already ignored")+ return nil, common.Classify(common.ErrConflict, errors.New("vulnerability is already ignored"))}created, createIgnoreErr := s.records.Create(s.db.WithContext(ctx), []vulnerability.IgnoredVulnerability{ignore}, 1)@@ -134,16 +127,12 @@ func (s *Service) IgnoreVulnerability(ctx context.Context, envID string, payload// UnignoreVulnerability removes an ignore record together with any duplicates of its identity, so a finding never stays ignored after a restore.func (s *Service) UnignoreVulnerability(ctx context.Context, envID, ignoreID string) error {- if s.db == nil {- return errors.New("database not available")- }-ignore, found, err := s.records.Get(ctx, envID, ignoreID)if err != nil {return fmt.Errorf("failed to load ignore record: %w", err)}if !found {- return errors.New("ignore record not found")+ return common.Classify(common.ErrNotFound, errors.New("ignore record not found"))}if deleteErr := s.records.DeleteIdentity(ctx, ignore); deleteErr != nil {@@ -155,20 +144,16 @@ func (s *Service) UnignoreVulnerability(ctx context.Context, envID, ignoreID strreturn nil}-// bulkIgnoreBatchSizeInternal bounds how many ignore rows one insert or delete statement carries.-const bulkIgnoreBatchSizeInternal = 500+// bulkIgnoreBatchSize bounds how many ignore rows one insert or delete statement carries.+const bulkIgnoreBatchSize = 500// BulkIgnoreVulnerabilities ignores every active finding matching the filters inside one transaction and returns how many ignore records were created.func (s *Service) BulkIgnoreVulnerabilities(ctx context.Context, envID string, filters *vulnerability.BulkVulnerabilityFilters, createdBy string) (int, error) {- if s.db == nil {- return 0, errors.New("database not available")- }-var ignores []vulnerability.IgnoredVulnerabilityerr := dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {- params := bulkFilterParamsInternal(filters, false)+ params := bulkFilterParams(filters, false)q := s.activeItemsQuery(ctx, tx, envID, ¶ms)- var targets []ignoreTargetInternal+ var targets []ignoreTargetif err := q.Distinct("vulnerability_scan_items.image_id","vulnerability_scan_items.vulnerability_id",@@ -193,7 +178,7 @@ func (s *Service) BulkIgnoreVulnerabilities(ctx context.Context, envID string, fCreatedBy: createdBy,})}- if _, err := s.records.Create(tx, ignores, bulkIgnoreBatchSizeInternal); err != nil {+ if _, err := s.records.Create(tx, ignores, bulkIgnoreBatchSize); err != nil {return fmt.Errorf("failed to create ignore records: %w", err)}return nil@@ -209,19 +194,15 @@ func (s *Service) BulkIgnoreVulnerabilities(ctx context.Context, envID string, f// BulkUnignoreVulnerabilities removes every ignore record matching the filters inside one transaction and returns how many were removed.func (s *Service) BulkUnignoreVulnerabilities(ctx context.Context, envID string, filters *vulnerability.BulkVulnerabilityFilters) (int, error) {- if s.db == nil {- return 0, errors.New("database not available")- }-var affected int64err := dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {- params := bulkFilterParamsInternal(filters, true)+ params := bulkFilterParams(filters, true)q := s.activeItemsQuery(ctx, tx, envID, ¶ms)var ignoreIDs []stringif err := q.Distinct("vulnerability_ignores.id").Pluck("vulnerability_ignores.id", &ignoreIDs).Error; err != nil {return fmt.Errorf("failed to resolve ignored vulnerabilities to restore: %w", err)}- for batch := range slices.Chunk(ignoreIDs, bulkIgnoreBatchSizeInternal) {+ for batch := range slices.Chunk(ignoreIDs, bulkIgnoreBatchSize) {deleted, err := s.records.DeleteIDs(tx, envID, batch)if err != nil {return fmt.Errorf("failed to delete ignore records: %w", err)@@ -239,40 +220,38 @@ func (s *Service) BulkUnignoreVulnerabilities(ctx context.Context, envID string,return int(affected), nil}-func buildIgnoredVulnerabilitiesResponse(total int64, params pagination.QueryParams, ignores []vulnerability.IgnoredVulnerability) pagination.Response {- filtered := pagination.FilterResult[vulnerability.IgnoredVulnerability]{- Items: ignores,- TotalCount: total,- TotalAvailable: total,- }- return pagination.BuildResponse(filtered.TotalCount, filtered.TotalAvailable, params)-}-// ListIgnoredVulnerabilities returns a list of ignored vulnerabilities for an environmentfunc (s *Service) ListIgnoredVulnerabilities(ctx context.Context, envID string, params pagination.QueryParams) ([]vulnerability.IgnoredVulnerability, pagination.Response, error) {- params.Limit = cmp.Or(params.Limit, 20)-- if s.db == nil {- return []vulnerability.IgnoredVulnerability{}, pagination.BuildResponse(0, 0, params), nil- }-- order := func(query *gorm.DB) *gorm.DB { return applyIgnoredVulnerabilitiesSortInternal(query, params.Sort) }- ignores, total, err := s.records.Page(ctx, envID, order, params.Start, params.Limit)+ ignores, total, err := s.records.Page(ctx, envID, func(query *gorm.DB) *gorm.DB {+ if params.Sort == "" {+ return query.Order("created_at DESC")+ }+ for part := range strings.SplitSeq(params.Sort, ",") {+ part = strings.TrimSpace(part)+ if part == "" {+ continue+ }+ desc := strings.HasPrefix(part, "-")+ switch kit.Ternary(desc, strings.TrimPrefix(part, "-"), strings.TrimPrefix(part, "+")) {+ case "createdAt":+ query = query.Order("created_at " + kit.Ternary(desc, "DESC", "ASC"))+ case "vulnerabilityId":+ query = query.Order("vulnerability_id " + kit.Ternary(desc, "DESC", "ASC"))+ default:+ query = query.Order("created_at DESC")+ }+ }+ return query+ }, params.Start, params.Limit)if err != nil {return nil, pagination.Response{}, err}- response := buildIgnoredVulnerabilitiesResponse(total, params, ignores)-- return ignores, response, nil+ return ignores, pagination.BuildResponse(total, total, params), nil}// GetIgnoreRecordsForImage retrieves all ignore records for a specific imagefunc (s *Service) GetIgnoreRecordsForImage(ctx context.Context, envID, imageID string) ([]vulnerability.IgnoredVulnerability, error) {- if s.db == nil {- return nil, nil- }-ignores, err := s.records.ForImage(ctx, envID, imageID)if err != nil {return nil, fmt.Errorf("failed to get ignore records: %w", err)@@ -280,63 +259,23 @@ func (s *Service) GetIgnoreRecordsForImage(ctx context.Context, envID, imageID sreturn ignores, nil}-func applyIgnoredVulnerabilitiesSortPartInternal(query *gorm.DB, part string, desc bool) *gorm.DB {- direction := kit.Ternary(desc, "DESC", "ASC")-- switch part {- case "createdAt":- return query.Order("created_at " + direction)- case "vulnerabilityId":- return query.Order("vulnerability_id " + direction)- default:- return query.Order("created_at DESC")- }-}--func applyIgnoredVulnerabilitiesSortInternal(query *gorm.DB, sort string) *gorm.DB {- if sort == "" {- return query.Order("created_at DESC")- }-- for part := range strings.SplitSeq(sort, ",") {- part = strings.TrimSpace(part)- if part == "" {- continue- }-- desc := false- switch {- case strings.HasPrefix(part, "-"):- desc = true- part = strings.TrimPrefix(part, "-")- case strings.HasPrefix(part, "+"):- part = strings.TrimPrefix(part, "+")- }-- query = applyIgnoredVulnerabilitiesSortPartInternal(query, part, desc)- }-- return query-}--// bulkFilterParamsInternal converts bulk filters into the list query params so matching follows the list and export rules.-func bulkFilterParamsInternal(filters *vulnerability.BulkVulnerabilityFilters, onlyIgnored bool) pagination.QueryParams {- params := pagination.QueryParams{Search: filters.Search, Filters: map[string]string{}}+// bulkFilterParams converts bulk filters into the list query params so matching follows the list and export rules.+func bulkFilterParams(filters *vulnerability.BulkVulnerabilityFilters, onlyIgnored bool) pagination.QueryParams {+ values := map[string]string{}if filters.Severity != "" {- params.Filters["severity"] = filters.Severity+ values["severity"] = filters.Severity}if filters.ImageName != "" {- params.Filters["imageName"] = filters.ImageName+ values["imageName"] = filters.ImageName}if filters.FixAvailable {- params.Filters["fixAvailable"] = "true"+ values["fixAvailable"] = "true"}if onlyIgnored {- params.Filters["ignored"] = "true"+ values["ignored"] = "true"+ }+ return pagination.QueryParams{+ Search: filters.Search,+ Filters: values,}- return params-}--func vulnerabilityIgnoreKeyInternal(imageID, vulnerabilityID, pkgName, installedVersion string) string {- return imageID + ":" + vulnerabilityID + ":" + pkgName + ":" + installedVersion}diff --git a/backend/internal/vulnerability/children/query/handler.go b/backend/internal/vulnerability/children/query/handler.goindex 92b8b4a59..bc29ca76b 100644--- a/backend/internal/vulnerability/children/query/handler.go+++ b/backend/internal/vulnerability/children/query/handler.go@@ -3,14 +3,12 @@ package queryimport ("cmp""context"- "errors""fmt""log/slog""time""github.com/danielgtaylor/huma/v2""github.com/getarcaneapp/arcane/types/v2/base"- "github.com/getarcaneapp/arcane/types/v2/features""github.com/getarcaneapp/arcane/types/v2/vulnerability""github.com/getarcaneapp/arcane/backend/v2/internal/common"@@ -97,8 +95,7 @@ type ListAllVulnerabilityImageOptionsInput struct {// GetScanSummaries retrieves scan summaries for a list of image IDs.func (h *Handler) GetScanSummaries(ctx context.Context, input *GetScanSummariesInput) (*handlerutil.Out[vulnerability.ScanSummariesResponse], error) {- imageIDs := input.Body.ImageIDs- if len(imageIDs) == 0 {+ if len(input.Body.ImageIDs) == 0 {return &handlerutil.Out[vulnerability.ScanSummariesResponse]{Body: base.ApiResponse[vulnerability.ScanSummariesResponse]{Success: true,@@ -109,11 +106,8 @@ func (h *Handler) GetScanSummaries(ctx context.Context, input *GetScanSummariesI}, nil}- summaries, err := h.service.GetScanSummariesByImageIDs(ctx, imageIDs)+ summaries, err := h.service.GetScanSummariesByImageIDs(ctx, input.Body.ImageIDs)if err != nil {- if errors.Is(err, common.ErrFeatureDisabled) {- return nil, handlerutil.FeatureDisabledError(features.VulnerabilityManagement)- }return nil, huma.Error500InternalServerError("Failed to scan image for vulnerabilities: " + err.Error())}@@ -167,18 +161,15 @@ func (h *Handler) GetScanSummary(ctx context.Context, input *GetScanSummaryInput// ListImageVulnerabilities returns a paginated list of vulnerabilities for an image.func (h *Handler) ListImageVulnerabilities(ctx context.Context, input *ListImageVulnerabilitiesInput) (*handlerutil.Page[vulnerability.Vulnerability], error) {- params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)- params.Limit = cmp.Or(params.Limit, 20)+ params := handlerutil.PaginationParams(input.Start, cmp.Or(input.Limit, 20), input.Sort, input.Order, input.Search)if input.Severity != "" {params.Filters["severity"] = input.Severity}items, paginationResp, err := h.service.ListVulnerabilities(ctx, input.ImageID, params)if err != nil {- if errors.Is(err, common.ErrVulnerabilityScanNotFound) {- return nil, huma.Error404NotFound("Vulnerability scan not found")- }- return nil, huma.Error500InternalServerError("Failed to retrieve vulnerability scan: " + err.Error())+ apiErr := common.ToAPIError(err)+ return nil, huma.NewError(apiErr.HTTPStatus(), apiErr.Message)}if items == nil {@@ -216,10 +207,6 @@ func (h *Handler) GetEnvironmentSummary(ctx context.Context, input *GetEnvironmereturn nil, huma.Error500InternalServerError("Failed to retrieve vulnerability scan: " + err.Error())}- if summary == nil {- summary = &vulnerability.EnvironmentVulnerabilitySummary{}- }-return &handlerutil.Out[vulnerability.EnvironmentVulnerabilitySummary]{Body: base.ApiResponse[vulnerability.EnvironmentVulnerabilitySummary]{Success: true,@@ -229,9 +216,8 @@ func (h *Handler) GetEnvironmentSummary(ctx context.Context, input *GetEnvironme}// ListAllVulnerabilities returns a paginated list of vulnerabilities across all images.-func (input *ListAllVulnerabilitiesInput) queryParamsInternal() pagination.QueryParams {- params := handlerutil.PaginationParams(input.Start, input.Limit, input.Sort, input.Order, input.Search)- params.Limit = cmp.Or(params.Limit, 20)+func (input *ListAllVulnerabilitiesInput) queryParams() pagination.QueryParams {+ params := handlerutil.PaginationParams(input.Start, cmp.Or(input.Limit, 20), input.Sort, input.Order, input.Search)if input.Severity != "" {params.Filters["severity"] = input.Severity}@@ -249,13 +235,13 @@ func (input *ListAllVulnerabilitiesInput) queryParamsInternal() pagination.Queryfunc (h *Handler) ListAllVulnerabilities(ctx context.Context, input *ListAllVulnerabilitiesInput) (*handlerutil.Page[vulnerability.VulnerabilityWithImage], error) {if _, ok := httpx.EnvironmentScopeFromContext(ctx); ok {- body, err := h.service.AggregateVulnerabilities(ctx, input.queryParamsInternal())+ body, err := h.service.AggregateVulnerabilities(ctx, input.queryParams())if err != nil {return nil, huma.Error500InternalServerError("Failed to retrieve vulnerability scan: " + err.Error())}return &handlerutil.Page[vulnerability.VulnerabilityWithImage]{Body: body}, nil}- items, paginationResp, err := h.service.ListAllVulnerabilities(ctx, input.EnvironmentID, input.queryParamsInternal())+ items, paginationResp, err := h.service.ListAllVulnerabilities(ctx, input.EnvironmentID, input.queryParams())if err != nil {return nil, huma.Error500InternalServerError("Failed to retrieve vulnerability scan: " + err.Error())}@@ -284,7 +270,7 @@ func (h *Handler) ExportVulnerabilities(ctx context.Context, input *ExportVulnerImageName: input.ImageName,Ignored: input.Ignored,FixAvailable: input.FixAvailable,- }).queryParamsInternal()+ }).queryParams()items, err := h.service.ExportVulnerabilities(ctx, input.EnvironmentID, params)if err != nil {return nil, huma.Error500InternalServerError("Failed to export vulnerabilities: " + err.Error())diff --git a/backend/internal/vulnerability/children/query/service.go b/backend/internal/vulnerability/children/query/service.goindex 733933c96..c9a9b6fc5 100644--- a/backend/internal/vulnerability/children/query/service.go+++ b/backend/internal/vulnerability/children/query/service.go@@ -44,9 +44,9 @@ import ()const (- scansTableInternal = "vulnerability_scans"- scanItemsTableInternal = "vulnerability_scan_items"- threatIntelTableInternal = "vulnerability_threat_intel"+ scansTable = "vulnerability_scans"+ scanItemsTable = "vulnerability_scan_items"+ threatIntelTable = "vulnerability_threat_intel"// riskOverviewCacheTTL bounds staleness from changes that do not invalidate// the cache, such as containers starting or threat intel refreshing.@@ -68,11 +68,11 @@ type Service struct {// riskFlight and riskCache share one risk overview build per environment;// riskGen is bumped by InvalidateRiskOverview to drop cached overviews.riskFlight singleflight.Group- riskCache sync.Map // envID -> *riskOverviewCacheEntryInternal+ riskCache sync.Map // envID -> *riskOverviewCacheEntryriskGen atomic.Uint64}-type riskOverviewCacheEntryInternal struct {+type riskOverviewCacheEntry struct {overview *vulnerability.RiskOverviewbuiltAt time.Timegen uint64@@ -100,8 +100,8 @@ func NewService(}}-// scanSummaryRowInternal is the summary projection of a scan record.-type scanSummaryRowInternal struct {+// scanSummaryRow is the summary projection of a scan record.+type scanSummaryRow struct {ID string `gorm:"column:id"`ScanTime time.Time `gorm:"column:scan_time"`Status string `gorm:"column:status"`@@ -114,14 +114,14 @@ type scanSummaryRowInternal struct {Error *string `gorm:"column:error"`}-// scanImageRowInternal is the image projection of a scan record.-type scanImageRowInternal struct {+// scanImageRow is the image projection of a scan record.+type scanImageRow struct {ID string `gorm:"column:id"`ImageName string `gorm:"column:image_name"`}-// threatIntelRowInternal is the insight projection of a cached threat intel row.-type threatIntelRowInternal struct {+// threatIntelRow is the insight projection of a cached threat intel row.+type threatIntelRow struct {CVEID string `gorm:"column:cve_id"`KnownExploited bool `gorm:"column:known_exploited"`EPSSScore *float64 `gorm:"column:epss_score"`@@ -129,7 +129,10 @@ type threatIntelRowInternal struct {// GetScanResult retrieves the most recent scan result for an image, with risk insights for completed scans.func (s *Service) GetScanResult(ctx context.Context, envID, imageID string) (*vulnerability.ScanResult, error) {- result, err := s.loadScanResultInternal(ctx, imageID)+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ return nil, nil+ }+ result, err := s.loadResult(ctx, imageID)if err != nil || result == nil || result.Status != vulnerability.ScanStatusCompleted || len(result.Vulnerabilities) == 0 {return result, err}@@ -139,42 +142,46 @@ func (s *Service) GetScanResult(ctx context.Context, envID, imageID string) (*vureturn nil, fmt.Errorf("failed to filter ignored vulnerabilities: %w", err)}intel := map[string]vuln.ThreatIntel{}- intelStatus := s.threatIntelStatus(ctx)- if intelStatus.Enabled {+ if s.threatIntelStatus(ctx).Enabled {cveIDs := make([]string, 0, len(vulns))for i := range vulns {cveIDs = append(cveIDs, vulns[i].VulnerabilityID)}- var rows []threatIntelRowInternal- if loadThreatIntelErr := s.db.WithContext(ctx).Table(threatIntelTableInternal).Where("cve_id IN ?", cveIDs).Find(&rows).Error; loadThreatIntelErr != nil {+ var rows []threatIntelRow+ if loadThreatIntelErr := s.db.WithContext(ctx).Table(threatIntelTable).Where("cve_id IN ?", cveIDs).Find(&rows).Error; loadThreatIntelErr != nil {slog.WarnContext(ctx, "failed to load threat intel for image insights", "error", loadThreatIntelErr)}for _, row := range rows {intel[row.CVEID] = vuln.ThreatIntel{KnownExploited: row.KnownExploited, EPSS: row.EPSSScore}}}- usage, usageOK := s.containerUsageInternal(ctx)- result.Insights = vuln.BuildScanInsights(vulns, intel, imageExposureInternal(usage, usageOK, result.ImageID))- return result, nil-}--// loadScanResultInternal reads the stored scan result for an image without computing insights.-func (s *Service) loadScanResultInternal(ctx context.Context, imageID string) (*vulnerability.ScanResult, error) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {- return nil, nil- }- return s.loadResult(ctx, imageID)+ usage, usageOK := s.containerUsage(ctx)+ return &vulnerability.ScanResult{ //exhaustruct:enforce+ ImageID: result.ImageID,+ ImageName: result.ImageName,+ ScanTime: result.ScanTime,+ Status: result.Status,+ ActivityID: result.ActivityID,+ ScanPhase: result.ScanPhase,+ Summary: result.Summary,+ Vulnerabilities: result.Vulnerabilities,+ Error: result.Error,+ Duration: result.Duration,+ ScannerVersion: result.ScannerVersion,+ HasReport: result.HasReport,+ Insights: vuln.BuildScanInsights(vulns, intel, imageExposure(usage, usageOK, result.ImageID)),+ }, nil}// GetScanSummary retrieves just the summary for an image (for list views)func (s *Service) GetScanSummary(ctx context.Context, imageID string) (*vulnerability.ScanSummary, error) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return nil, nil}- var record scanSummaryRowInternal+ var record scanSummaryRowerr := s.db.WithContext(ctx).- Table(scansTableInternal).+ Table(scansTable).Select("id", "scan_time", "status", "critical_count", "high_count", "medium_count", "low_count", "unknown_count", "total_count", "error").Where("id = ?", imageID).First(&record).Error@@ -206,7 +213,10 @@ func (s *Service) GetScanSummary(ctx context.Context, imageID string) (*vulnerab// ListVulnerabilities returns a paginated, filtered list of vulnerabilities for an image.func (s *Service) ListVulnerabilities(ctx context.Context, imageID string, params pagination.QueryParams) ([]vulnerability.Vulnerability, pagination.Response, error) {- result, err := s.loadScanResultInternal(ctx, imageID)+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ return nil, pagination.Response{}, common.ErrVulnerabilityScanNotFound+ }+ result, err := s.loadResult(ctx, imageID)if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to get scan result: %w", err)}@@ -261,35 +271,25 @@ func (s *Service) ListVulnerabilities(ctx context.Context, imageID string, param}filtered := paginationConfig.SearchOrderAndPaginate(result.Vulnerabilities, params)- response := pagination.BuildResponse(filtered.TotalCount, filtered.TotalAvailable, params)-- return filtered.Items, response, nil+ return filtered.Items, pagination.BuildResponse(filtered.TotalCount, filtered.TotalAvailable, params), nil}// GetEnvironmentSummary returns aggregated vulnerability counts across all images.func (s *Service) GetEnvironmentSummary(ctx context.Context) (*vulnerability.EnvironmentVulnerabilitySummary, error) {- summary := &vulnerability.EnvironmentVulnerabilitySummary{- Summary: &vulnerability.SeveritySummary{},+ images, counts, imagesErr := s.dockerService.GetAllImages(ctx)+ if imagesErr != nil {+ slog.WarnContext(ctx, "failed to list images for vulnerability summary", "error", imagesErr)}-- var currentImageIDs []string- haveCurrentImages := false- if s.dockerService != nil {- images, counts, err := s.dockerService.GetAllImages(ctx)- if err != nil {- slog.WarnContext(ctx, "failed to list images for vulnerability summary", "error", err)- } else {- summary.TotalImages = counts.Total- haveCurrentImages = true- currentImageIDs = make([]string, 0, len(images))- for i := range images {- currentImageIDs = append(currentImageIDs, images[i].ID)- }- }+ currentImageIDs := make([]string, 0, len(images))+ for i := range images {+ currentImageIDs = append(currentImageIDs, images[i].ID)}- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {- return summary, nil+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ return &vulnerability.EnvironmentVulnerabilitySummary{+ TotalImages: kit.Ternary(imagesErr == nil, counts.Total, 0),+ Summary: &vulnerability.SeveritySummary{},+ }, nil}type aggregate struct {@@ -304,7 +304,7 @@ func (s *Service) GetEnvironmentSummary(ctx context.Context) (*vulnerability.Envvar agg aggregateaggQuery := s.db.WithContext(ctx).- Table(scansTableInternal).+ Table(scansTable).Select("COALESCE(SUM(critical_count), 0) AS critical","COALESCE(SUM(high_count), 0) AS high",@@ -315,7 +315,7 @@ func (s *Service) GetEnvironmentSummary(ctx context.Context) (*vulnerability.Env"COUNT(*) AS scanned",).Where("status = ?", string(vulnerability.ScanStatusCompleted))- if haveCurrentImages {+ if imagesErr == nil {aggQuery = aggQuery.Where("id IN ?", currentImageIDs)}err := aggQuery.Scan(&agg).Error@@ -323,24 +323,22 @@ func (s *Service) GetEnvironmentSummary(ctx context.Context) (*vulnerability.Envreturn nil, fmt.Errorf("failed to aggregate vulnerability summary: %w", err)}- summary.ScannedImages = agg.Scanned- summary.Summary = &vulnerability.SeveritySummary{- Critical: agg.Critical,- High: agg.High,- Medium: agg.Medium,- Low: agg.Low,- Unknown: agg.Unknown,- Total: agg.Total,- }-- return summary, nil+ return &vulnerability.EnvironmentVulnerabilitySummary{+ TotalImages: kit.Ternary(imagesErr == nil, counts.Total, 0),+ ScannedImages: agg.Scanned,+ Summary: &vulnerability.SeveritySummary{+ Critical: agg.Critical,+ High: agg.High,+ Medium: agg.Medium,+ Low: agg.Low,+ Unknown: agg.Unknown,+ Total: agg.Total,+ },+ }, nil}-// currentImageIDsInternal lists daemon image IDs so queries skip scans of images removed outside Arcane; ok is false when the daemon could not be listed.-func (s *Service) currentImageIDsInternal(ctx context.Context) (ids []string, ok bool) {- if s.dockerService == nil {- return nil, false- }+// currentImageIDs lists daemon image IDs so queries skip scans of images removed outside Arcane; ok is false when the daemon could not be listed.+func (s *Service) currentImageIDs(ctx context.Context) (ids []string, ok bool) {images, err := s.dockerService.ListImages(ctx)if err != nil {slog.WarnContext(ctx, "failed to list images for vulnerability query", "error", err)@@ -361,7 +359,7 @@ func (s *Service) currentImageIDsInternal(ctx context.Context) (ids []string, ok// high_count columns; with ignore records it is a single COUNT over the// per-CVE item rows.func (s *Service) ActionableCountExcludingIgnored(ctx context.Context) (int, error) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return 0, nil}@@ -369,12 +367,12 @@ func (s *Service) ActionableCountExcludingIgnored(ctx context.Context) (int, errif err := s.db.WithContext(ctx).Table("vulnerability_ignores").Count(&ignoreCount).Error; err != nil {return 0, fmt.Errorf("failed to count ignore records: %w", err)}- currentImageIDs, haveCurrentImages := s.currentImageIDsInternal(ctx)+ currentImageIDs, haveCurrentImages := s.currentImageIDs(ctx)if ignoreCount == 0 {var indexedCount int64sumQuery := s.db.WithContext(ctx).- Table(scansTableInternal).+ Table(scansTable).Select("COALESCE(SUM(critical_count + high_count), 0)").Where("status = ?", string(vulnerability.ScanStatusCompleted))if haveCurrentImages {@@ -387,7 +385,7 @@ func (s *Service) ActionableCountExcludingIgnored(ctx context.Context) (int, err}countQuery := s.db.WithContext(ctx).- Table(scanItemsTableInternal).+ Table(scanItemsTable).Joins("JOIN vulnerability_scans ON vulnerability_scans.id = vulnerability_scan_items.image_id AND vulnerability_scans.status = ?", string(vulnerability.ScanStatusCompleted)).Where("vulnerability_scan_items.severity IN ?", []string{string(vulnerability.SeverityCritical), string(vulnerability.SeverityHigh)})if haveCurrentImages {@@ -408,7 +406,7 @@ func (s *Service) ActionableCountExcludingIgnored(ctx context.Context) (int, errreturn int(count), nil}-type vulnerabilityScanItemRowInternal struct {+type vulnerabilityScanItemRow struct {ImageID string `gorm:"column:image_id"`VulnerabilityID string `gorm:"column:vulnerability_id"`PkgName string `gorm:"column:pkg_name"`@@ -423,7 +421,7 @@ type vulnerabilityScanItemRowInternal struct {IgnoreID string `gorm:"column:ignore_id"`}-var listAllVulnerabilitySortColumnsInternal = map[string]string{+var listAllVulnerabilitySortColumns = map[string]string{"vulnerabilityId": "vulnerability_scan_items.vulnerability_id","pkgName": "vulnerability_scan_items.pkg_name","severity": "vulnerability_scan_items.severity_rank",@@ -432,10 +430,10 @@ var listAllVulnerabilitySortColumnsInternal = map[string]string{"imageName": "vulnerability_scans.image_name",}-// scanItemsQueryInternal joins scan items to completed scans and their ignore records, keeping ignored or active rows for the given images.-func scanItemsQueryInternal(ctx context.Context, db *gorm.DB, envID string, onlyIgnored bool, imageIDs mo.Option[[]string]) *gorm.DB {+// scanItemsQuery joins scan items to completed scans and their ignore records, keeping ignored or active rows for the given images.+func scanItemsQuery(ctx context.Context, db *gorm.DB, envID string, onlyIgnored bool, imageIDs mo.Option[[]string]) *gorm.DB {query := db.WithContext(ctx).- Table(scanItemsTableInternal).+ Table(scanItemsTable).Joins("JOIN vulnerability_scans ON vulnerability_scans.id = vulnerability_scan_items.image_id AND vulnerability_scans.status = ?", string(vulnerability.ScanStatusCompleted)).Joins("LEFT JOIN vulnerability_ignores ON vulnerability_ignores.environment_id = ? AND "+@@ -456,14 +454,14 @@ func scanItemsQueryInternal(ctx context.Context, db *gorm.DB, envID string, onlyreturn query}-// listAllVulnerabilitiesQueryInternal returns the base query used for counts and the filtered, ordered query on db; it clears params.Sort once ordering is applied.-func (s *Service) listAllVulnerabilitiesQueryInternal(ctx context.Context, db *gorm.DB, envID string, params *pagination.QueryParams) (*gorm.DB, *gorm.DB) {+// listAllVulnerabilitiesQuery returns the base query used for counts and the filtered, ordered query on db; it clears params.Sort once ordering is applied.+func (s *Service) listAllVulnerabilitiesQuery(ctx context.Context, db *gorm.DB, envID string, params *pagination.QueryParams) (*gorm.DB, *gorm.DB) {// "ignored" flips the list from active to ignored vulnerabilities, the same// way the projects list swaps to archived projects.onlyIgnored := params.Filters["ignored"] == "true"delete(params.Filters, "ignored")- countQuery := scanItemsQueryInternal(ctx, db, envID, onlyIgnored, mo.TupleToOption(s.currentImageIDsInternal(ctx))).+ countQuery := scanItemsQuery(ctx, db, envID, onlyIgnored, mo.TupleToOption(s.currentImageIDs(ctx))).Select("vulnerability_scan_items.*", "vulnerability_scans.image_name AS image_name", "COALESCE(vulnerability_ignores.id, '') AS ignore_id").Session(&gorm.Session{})@@ -487,40 +485,51 @@ func (s *Service) listAllVulnerabilitiesQueryInternal(ctx context.Context, db *g"LOWER(vulnerability_scan_items.fixed_version) LIKE ? OR LOWER(vulnerability_scan_items.title) LIKE "+"? OR LOWER(vulnerability_scans.image_name) LIKE ? OR LOWER(vulnerability_scan_items.image_id) LIKE "+"?)")- if col, ok := listAllVulnerabilitySortColumnsInternal[params.Sort]; ok {+ if col, ok := listAllVulnerabilitySortColumns[params.Sort]; ok {q = q.Order(clause.OrderByColumn{Column: clause.Column{Name: col, Raw: true}, Desc: strings.EqualFold(string(params.Order), "desc")})}- params.Sort = ""+ *params = pagination.QueryParams{+ Search: params.Search,+ Order: params.Order,+ Start: params.Start,+ Limit: params.Limit,+ SkipCount: params.SkipCount,+ Filters: params.Filters,+ }return countQuery, q}// ListAllVulnerabilities returns a paginated list of vulnerabilities across all scanned images.func (s *Service) ListAllVulnerabilities(ctx context.Context, envID string, params pagination.QueryParams) ([]vulnerability.VulnerabilityWithImage, pagination.Response, error) {- params.Limit = cmp.Or(params.Limit, 20)-- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return []vulnerability.VulnerabilityWithImage{}, pagination.BuildResponse(0, 0, params), nil}- countQuery, q := s.listAllVulnerabilitiesQueryInternal(ctx, s.db.DB, envID, ¶ms)+ countQuery, q := s.listAllVulnerabilitiesQuery(ctx, s.db.DB, envID, ¶ms)var totalAvailable int64if err := countQuery.Count(&totalAvailable).Error; err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to count vulnerabilities: %w", err)}- var rows []vulnerabilityScanItemRowInternal+ var rows []vulnerabilityScanItemRowresponse, err := pagination.PaginateAndSortDB(params, q, &rows)if err != nil {return nil, pagination.Response{}, fmt.Errorf("failed to list vulnerabilities: %w", err)}- response.GrandTotalItems = totalAvailableitems := make([]vulnerability.VulnerabilityWithImage, 0, len(rows))for i := range rows {row := &rows[i]- item := vulnerability.VulnerabilityWithImage{+ details := func() vulnerability.Vulnerability {+ var decoded vulnerability.Vulnerability+ if kit.FromPtr(row.Details) == "" || json.Unmarshal([]byte(*row.Details), &decoded) != nil {+ return vulnerability.Vulnerability{}+ }+ return decoded+ }()+ items = append(items, vulnerability.VulnerabilityWithImage{VulnerabilityID: row.VulnerabilityID,PkgName: row.PkgName,InstalledVersion: row.InstalledVersion,@@ -529,25 +538,25 @@ func (s *Service) ListAllVulnerabilities(ctx context.Context, envID string, paraClass: row.PkgClass,Type: row.PkgType,Title: row.Title,+ Description: details.Description,+ References: details.References,+ CVSS: details.CVSS,+ PublishedDate: details.PublishedDate,+ LastModifiedDate: details.LastModifiedDate,Ignored: row.IgnoreID != "",IgnoreID: row.IgnoreID,ImageID: row.ImageID,ImageName: row.ImageName,- }- if row.Details != nil && *row.Details != "" {- var details vulnerability.Vulnerability- if unmarshalErr := json.Unmarshal([]byte(*row.Details), &details); unmarshalErr == nil {- item.Description = details.Description- item.References = details.References- item.CVSS = details.CVSS- item.PublishedDate = details.PublishedDate- item.LastModifiedDate = details.LastModifiedDate- }- }- items = append(items, item)+ })}- return items, response, nil+ return items, pagination.Response{ //exhaustruct:enforce+ TotalPages: response.TotalPages,+ TotalItems: response.TotalItems,+ CurrentPage: response.CurrentPage,+ ItemsPerPage: response.ItemsPerPage,+ GrandTotalItems: totalAvailable,+ }, nil}func (s *Service) AggregateVulnerabilities(ctx context.Context, params pagination.QueryParams) (base.Paginated[vulnerability.VulnerabilityWithImage], error) {@@ -600,11 +609,11 @@ func (s *Service) AggregateVulnerabilities(ctx context.Context, params paginatio// ExportVulnerabilities opens a cursor over every vulnerability matching params and yields rows one at a time in the requested order.func (s *Service) ExportVulnerabilities(ctx context.Context, envID string, params pagination.QueryParams) (iter.Seq2[vulnerability.VulnerabilityWithImage, error], error) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return func(func(vulnerability.VulnerabilityWithImage, error) bool) {}, nil}- _, q := s.listAllVulnerabilitiesQueryInternal(ctx, s.db.DB, envID, ¶ms)+ _, q := s.listAllVulnerabilitiesQuery(ctx, s.db.DB, envID, ¶ms)rows, err := q.Rows()if err != nil {return nil, fmt.Errorf("failed to export vulnerabilities: %w", err)@@ -618,7 +627,7 @@ func (s *Service) ExportVulnerabilities(ctx context.Context, envID string, param}()for rows.Next() {- var row vulnerabilityScanItemRowInternal+ var row vulnerabilityScanItemRowif scanRowsErr := q.ScanRows(rows, &row); scanRowsErr != nil {yield(vulnerability.VulnerabilityWithImage{}, fmt.Errorf("failed to export vulnerabilities: %w", scanRowsErr))return@@ -662,30 +671,30 @@ const (// ListAllVulnerabilityImageOptions returns unique image names (or image IDs when name is empty)// for vulnerability filtering, optionally constrained by severity.func (s *Service) ListAllVulnerabilityImageOptions(ctx context.Context, severityFilter string) ([]string, error) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return []string{}, nil}query := s.db.WithContext(ctx).- Table(scansTableInternal).+ Table(scansTable).Select("id", "image_name").Where("status = ?", string(vulnerability.ScanStatusCompleted)).Where("total_count > 0")- if currentImageIDs, ok := s.currentImageIDsInternal(ctx); ok {+ if currentImageIDs, ok := s.currentImageIDs(ctx); ok {query = query.Where("id IN ?", currentImageIDs)}if strings.TrimSpace(severityFilter) != "" {- var hasPossibleMatches bool- query, hasPossibleMatches = applyListAllVulnerabilityRecordPrefiltersInternal(query, map[string]string{+ filtered, hasPossibleMatches := applyListAllVulnerabilityRecordPrefilters(query, map[string]string{"severity": severityFilter,}).Get()if !hasPossibleMatches {return []string{}, nil}+ query = filtered}- var records []scanImageRowInternal+ var records []scanImageRowif err := query.Find(&records).Error; err != nil {return nil, fmt.Errorf("failed to list vulnerability image options: %w", err)}@@ -709,18 +718,22 @@ func (s *Service) AggregateImageOptions(ctx context.Context) (base.ApiResponse[[options = append(options, result.Value.Data...)}slices.Sort(options)- return base.ApiResponse[[]string]{Success: true, Data: slices.Compact(options), FailedEnvironments: collection.Failures}, nil+ return base.ApiResponse[[]string]{+ Success: true,+ Data: slices.Compact(options),+ FailedEnvironments: collection.Failures,+ }, nil}// GetScanSummariesByImageIDs retrieves scan summaries for multiple imagesfunc (s *Service) GetScanSummariesByImageIDs(ctx context.Context, imageIDs []string) (map[string]*vulnerability.ScanSummary, error) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil || len(imageIDs) == 0 {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || len(imageIDs) == 0 {return make(map[string]*vulnerability.ScanSummary), nil}- var records []scanSummaryRowInternal+ var records []scanSummaryRowerr := s.db.WithContext(ctx).- Table(scansTableInternal).+ Table(scansTable).Select("id", "scan_time", "status", "critical_count", "high_count", "medium_count", "low_count", "unknown_count", "total_count", "error").Where("id IN ?", imageIDs).Find(&records).Error@@ -730,11 +743,6 @@ func (s *Service) GetScanSummariesByImageIDs(ctx context.Context, imageIDs []strresult := make(map[string]*vulnerability.ScanSummary, len(records))for _, record := range records {- var errStr string- if record.Error != nil {- errStr = *record.Error- }-result[record.ID] = &vulnerability.ScanSummary{ImageID: record.ID,ScanTime: record.ScanTime,@@ -747,7 +755,7 @@ func (s *Service) GetScanSummariesByImageIDs(ctx context.Context, imageIDs []strUnknown: record.UnknownCount,Total: record.TotalCount,},- Error: errStr,+ Error: kit.FromPtr(record.Error),}s.refreshSummary(ctx, result[record.ID])}@@ -755,22 +763,19 @@ func (s *Service) GetScanSummariesByImageIDs(ctx context.Context, imageIDs []strreturn result, nil}-type imageUsageInternal struct {+type imageUsage struct {running intstopped int}-// containerUsageInternal counts running and stopped containers per image ID; ok is false when the daemon could not be listed.-func (s *Service) containerUsageInternal(ctx context.Context) (map[string]imageUsageInternal, bool) {- if s.dockerService == nil {- return nil, false- }+// containerUsage counts running and stopped containers per image ID; ok is false when the daemon could not be listed.+func (s *Service) containerUsage(ctx context.Context) (map[string]imageUsage, bool) {containers, err := s.dockerService.ListContainers(ctx)if err != nil {slog.WarnContext(ctx, "failed to list containers for vulnerability risk", "error", err)return nil, false}- usage := make(map[string]imageUsageInternal, len(containers))+ usage := make(map[string]imageUsage, len(containers))for i := range containers {entry := usage[containers[i].ImageID]switch containers[i].State {@@ -786,7 +791,7 @@ func (s *Service) containerUsageInternal(ctx context.Context) (map[string]imageUreturn usage, true}-func imageExposureInternal(usage map[string]imageUsageInternal, ok bool, imageID string) vulnerability.ImageExposure {+func imageExposure(usage map[string]imageUsage, ok bool, imageID string) vulnerability.ImageExposure {switch {case !ok:return vulnerability.ImageExposureUnknown@@ -799,7 +804,7 @@ func imageExposureInternal(usage map[string]imageUsageInternal, ok bool, imageID}}-type riskFindingRowInternal struct {+type riskFindingRow struct {ImageID string `gorm:"column:image_id"`ImageName string `gorm:"column:image_name"`VulnerabilityID string `gorm:"column:vulnerability_id"`@@ -814,64 +819,73 @@ type riskFindingRowInternal struct {EPSSScore *float64 `gorm:"column:epss_score"`}-type riskImageAggInternal struct {+type riskImageAgg struct {image vulnerability.RiskImagescore vuln.RiskScore}-type riskFindingAggInternal struct {+type riskFindingAgg struct {finding vulnerability.RiskFindingimages map[string]struct{}runningImages map[string]struct{}}-// riskAccumulatorInternal aggregates scored findings into environment, per-image and per-CVE totals.-type riskAccumulatorInternal struct {+// riskAccumulator aggregates scored findings into environment, per-image and per-CVE totals.+type riskAccumulator struct {today stringscannedImageIDs []stringexposures map[string]vulnerability.ImageExposure- overview *vulnerability.RiskOverview- images map[string]*riskImageAggInternal- findings map[string]*riskFindingAggInternal+ images map[string]*riskImageAgg+ findings map[string]*riskFindingAgg+ summary vulnerability.SeveritySummary+ exposure vulnerability.RiskExposureBreakdown+ fixable int+ scoreDriver *vulnerability.RiskScoreDriverdriverPriority float64}-func (a *riskAccumulatorInternal) addInternal(row *riskFindingRowInternal, exposure vulnerability.ImageExposure, runningContainers int) {+func (a *riskAccumulator) add(row *riskFindingRow, exposure vulnerability.ImageExposure, runningContainers int) {severity := vulnerability.Severity(row.Severity)baseScore := vuln.BaseScore(*cmp.Or(row.CVSSScore, new(0.0)), severity)fixable := strings.TrimSpace(row.FixedVersion) != ""- summary := &a.overview.Summaryswitch severity {case vulnerability.SeverityCritical:- summary.Critical+++ a.summary.Critical++case vulnerability.SeverityHigh:- summary.High+++ a.summary.High++case vulnerability.SeverityMedium:- summary.Medium+++ a.summary.Medium++case vulnerability.SeverityLow:- summary.Low+++ a.summary.Low++case vulnerability.SeverityUnknown:- summary.Unknown+++ a.summary.Unknown++default:- summary.Unknown+++ a.summary.Unknown++}- summary.Total+++ a.summary.Total++switch exposure {case vulnerability.ImageExposureRunning:- a.overview.Exposure.Running+++ a.exposure.Running++case vulnerability.ImageExposureStopped:- a.overview.Exposure.Stopped+++ a.exposure.Stopped++case vulnerability.ImageExposureUnused:- a.overview.Exposure.Unused+++ a.exposure.Unused++case vulnerability.ImageExposureUnknown:- a.overview.Exposure.Unknown+++ a.exposure.Unknown++}img, ok := a.images[row.ImageID]if !ok {- img = &riskImageAggInternal{image: vulnerability.RiskImage{ImageID: row.ImageID, ImageName: row.ImageName, Exposure: exposure, RunningContainers: runningContainers}}+ img = &riskImageAgg{+ image: vulnerability.RiskImage{+ ImageID: row.ImageID,+ ImageName: row.ImageName,+ Exposure: exposure,+ RunningContainers: runningContainers,+ },+ }a.images[row.ImageID] = img}img.image.Findings++@@ -888,15 +902,15 @@ func (a *riskAccumulatorInternal) addInternal(row *riskFindingRowInternal, expos}if fixable {img.image.Fixable++- a.overview.Drivers.Fixable+++ a.fixable++}if fixable && baseScore > 0 {- a.considerDriverInternal(row, exposure, priority)+ a.considerDriver(row, exposure, priority)}agg, ok := a.findings[row.VulnerabilityID]if !ok {- agg = &riskFindingAggInternal{+ agg = &riskFindingAgg{finding: vulnerability.RiskFinding{VulnerabilityID: row.VulnerabilityID,KnownExploited: row.KnownExploited,@@ -915,21 +929,31 @@ func (a *riskAccumulatorInternal) addInternal(row *riskFindingRowInternal, exposagg.runningImages[row.ImageID] = struct{}{}}// The highest-priority instance of a CVE describes it; any fixed version beats none.- if risk > agg.finding.Risk {- agg.finding.Risk, agg.finding.CVSS, agg.finding.Severity = risk, baseScore, severity- agg.finding.PkgName, agg.finding.Title = row.PkgName, row.Title- if fixable {- agg.finding.FixedVersion = row.FixedVersion- }- }- if fixable && agg.finding.FixedVersion == "" {- agg.finding.FixedVersion = row.FixedVersion+ better := risk > agg.finding.Risk+ *agg = riskFindingAgg{ //exhaustruct:enforce+ finding: vulnerability.RiskFinding{ //exhaustruct:enforce+ VulnerabilityID: agg.finding.VulnerabilityID,+ PkgName: kit.Ternary(better, row.PkgName, agg.finding.PkgName),+ Title: kit.Ternary(better, row.Title, agg.finding.Title),+ Severity: kit.Ternary(better, severity, agg.finding.Severity),+ CVSS: kit.Ternary(better, baseScore, agg.finding.CVSS),+ Risk: kit.Ternary(better, risk, agg.finding.Risk),+ EPSS: agg.finding.EPSS,+ KnownExploited: agg.finding.KnownExploited,+ Ransomware: agg.finding.Ransomware,+ KEVDueDate: agg.finding.KEVDueDate,+ ImagesAffected: agg.finding.ImagesAffected,+ RunningImagesAffected: agg.finding.RunningImagesAffected,+ FixedVersion: kit.Ternary(fixable && (better || agg.finding.FixedVersion == ""), row.FixedVersion, agg.finding.FixedVersion),+ },+ images: agg.images,+ runningImages: agg.runningImages,}}-// considerDriverInternal keeps the highest-priority fixable occurrence so its package, image, fix and score match.-func (a *riskAccumulatorInternal) considerDriverInternal(row *riskFindingRowInternal, exposure vulnerability.ImageExposure, priority float64) {- current := a.overview.ScoreDriver+// considerDriver keeps the highest-priority fixable occurrence so its package, image, fix and score match.+func (a *riskAccumulator) considerDriver(row *riskFindingRow, exposure vulnerability.ImageExposure, priority float64) {+ current := a.scoreDriverif current != nil {switch {case priority != a.driverPriority:@@ -947,7 +971,7 @@ func (a *riskAccumulatorInternal) considerDriverInternal(row *riskFindingRowInte}}a.driverPriority = priority- a.overview.ScoreDriver = &vulnerability.RiskScoreDriver{+ a.scoreDriver = &vulnerability.RiskScoreDriver{VulnerabilityID: row.VulnerabilityID,ImageName: row.ImageName,PkgName: row.PkgName,@@ -958,70 +982,88 @@ func (a *riskAccumulatorInternal) considerDriverInternal(row *riskFindingRowInte}}-// finishInternal derives the drivers and scores, and ranks findings and images.-func (a *riskAccumulatorInternal) finishInternal() {- overview := a.overview- overview.Drivers.Findings = overview.Summary.Total-- findings := make([]*riskFindingAggInternal, 0, len(a.findings))+// finish derives the drivers and scores, and ranks findings and images.+func (a *riskAccumulator) finish(threatIntel vulnerability.ThreatIntelStatus, imagesTotal int) *vulnerability.RiskOverview {+ var knownExploited, overdueKnownExploited, highEPSS, exposedCriticalHigh int+ findings := make([]vulnerability.RiskFinding, 0, len(a.findings))for _, agg := range a.findings {- f := &agg.finding- f.ImagesAffected = len(agg.images)- f.RunningImagesAffected = len(agg.runningImages)- if f.KnownExploited {- overview.Drivers.KnownExploited++- if f.KEVDueDate != "" && f.KEVDueDate < a.today {- overview.Drivers.OverdueKnownExploited+++ if agg.finding.KnownExploited {+ knownExploited+++ if agg.finding.KEVDueDate != "" && agg.finding.KEVDueDate < a.today {+ overdueKnownExploited++}}- if f.EPSS != nil && *f.EPSS >= highEPSSThreshold {- overview.Drivers.HighEPSS+++ if kit.FromPtr(agg.finding.EPSS) >= highEPSSThreshold {+ highEPSS++}- if f.RunningImagesAffected > 0 && (f.Severity == vulnerability.SeverityCritical || f.Severity == vulnerability.SeverityHigh) {- overview.Drivers.ExposedCriticalHigh+++ if len(agg.runningImages) > 0 && (agg.finding.Severity == vulnerability.SeverityCritical || agg.finding.Severity == vulnerability.SeverityHigh) {+ exposedCriticalHigh++}- findings = append(findings, agg)+ findings = append(findings, vulnerability.RiskFinding{ //exhaustruct:enforce+ VulnerabilityID: agg.finding.VulnerabilityID,+ PkgName: agg.finding.PkgName,+ Title: agg.finding.Title,+ Severity: agg.finding.Severity,+ CVSS: agg.finding.CVSS,+ Risk: agg.finding.Risk,+ EPSS: agg.finding.EPSS,+ KnownExploited: agg.finding.KnownExploited,+ Ransomware: agg.finding.Ransomware,+ KEVDueDate: agg.finding.KEVDueDate,+ ImagesAffected: len(agg.images),+ RunningImagesAffected: len(agg.runningImages),+ FixedVersion: agg.finding.FixedVersion,+ })}byRisk := slices.Clone(findings)- slices.SortFunc(byRisk, func(a, b *riskFindingAggInternal) int {- if a.finding.Risk != b.finding.Risk {- return cmp.Compare(b.finding.Risk, a.finding.Risk)+ slices.SortFunc(byRisk, func(a, b vulnerability.RiskFinding) int {+ if a.Risk != b.Risk {+ return cmp.Compare(b.Risk, a.Risk)}- if a.finding.KnownExploited != b.finding.KnownExploited {- return kit.Ternary(b.finding.KnownExploited, 1, -1)+ if a.KnownExploited != b.KnownExploited {+ return kit.Ternary(b.KnownExploited, 1, -1)}return cmp.Or(- cmp.Compare(b.finding.RunningImagesAffected, a.finding.RunningImagesAffected),- cmp.Compare(*cmp.Or(b.finding.EPSS, new(0.0)), *cmp.Or(a.finding.EPSS, new(0.0))),- strings.Compare(a.finding.VulnerabilityID, b.finding.VulnerabilityID),+ cmp.Compare(b.RunningImagesAffected, a.RunningImagesAffected),+ cmp.Compare(*cmp.Or(b.EPSS, new(0.0)), *cmp.Or(a.EPSS, new(0.0))),+ strings.Compare(a.VulnerabilityID, b.VulnerabilityID),)})byPrevalence := slices.Clone(findings)- slices.SortFunc(byPrevalence, func(a, b *riskFindingAggInternal) int {+ slices.SortFunc(byPrevalence, func(a, b vulnerability.RiskFinding) int {return cmp.Or(- cmp.Compare(b.finding.ImagesAffected, a.finding.ImagesAffected),- cmp.Compare(b.finding.RunningImagesAffected, a.finding.RunningImagesAffected),- cmp.Compare(b.finding.Risk, a.finding.Risk),- strings.Compare(a.finding.VulnerabilityID, b.finding.VulnerabilityID),+ cmp.Compare(b.ImagesAffected, a.ImagesAffected),+ cmp.Compare(b.RunningImagesAffected, a.RunningImagesAffected),+ cmp.Compare(b.Risk, a.Risk),+ strings.Compare(a.VulnerabilityID, b.VulnerabilityID),)})- for _, agg := range findings {- agg.finding.Risk = math.Round(agg.finding.Risk*10) / 10- }- top := func(sorted []*riskFindingAggInternal) []vulnerability.RiskFinding {+ top := func(sorted []vulnerability.RiskFinding) []vulnerability.RiskFinding {out := make([]vulnerability.RiskFinding, 0, min(len(sorted), topFindingLimit))- for _, agg := range sorted[:min(len(sorted), topFindingLimit)] {- out = append(out, agg.finding)+ for _, finding := range sorted[:min(len(sorted), topFindingLimit)] {+ out = append(out, vulnerability.RiskFinding{ //exhaustruct:enforce+ VulnerabilityID: finding.VulnerabilityID,+ PkgName: finding.PkgName,+ Title: finding.Title,+ Severity: finding.Severity,+ CVSS: finding.CVSS,+ Risk: math.Round(finding.Risk*10) / 10,+ EPSS: finding.EPSS,+ KnownExploited: finding.KnownExploited,+ Ransomware: finding.Ransomware,+ KEVDueDate: finding.KEVDueDate,+ ImagesAffected: finding.ImagesAffected,+ RunningImagesAffected: finding.RunningImagesAffected,+ FixedVersion: finding.FixedVersion,+ })}return out}- overview.RiskiestFindings = top(byRisk)- overview.PrevalentFindings = top(byPrevalence)// Images average by exposure, clean ones at zero; unavailable images stay out.var environment vuln.RiskScore- overview.RiskiestImages = make([]vulnerability.RiskImage, 0, len(a.images))+ riskiestImages := make([]vulnerability.RiskImage, 0, len(a.images))for _, id := range a.scannedImageIDs {img, ok := a.images[id]if !ok {@@ -1029,11 +1071,21 @@ func (a *riskAccumulatorInternal) finishInternal() {continue}score, priority := img.score.ImageScore()- img.image.PatchPriority = priorityenvironment.AddImage(id, score, priority.ScoreStatus, img.image.Exposure, img.score.ExploitedRunning())- overview.RiskiestImages = append(overview.RiskiestImages, img.image)+ riskiestImages = append(riskiestImages, vulnerability.RiskImage{ //exhaustruct:enforce+ PatchPriority: priority,+ ImageID: img.image.ImageID,+ ImageName: img.image.ImageName,+ Exposure: img.image.Exposure,+ RunningContainers: img.image.RunningContainers,+ KnownExploited: img.image.KnownExploited,+ Critical: img.image.Critical,+ High: img.image.High,+ Fixable: img.image.Fixable,+ Findings: img.image.Findings,+ })}- slices.SortFunc(overview.RiskiestImages, func(a, b vulnerability.RiskImage) int {+ slices.SortFunc(riskiestImages, func(a, b vulnerability.RiskImage) int {return cmp.Or(cmp.Compare(b.RiskScore, a.RiskScore),cmp.Compare(b.KnownExploited, a.KnownExploited),@@ -1041,23 +1093,47 @@ func (a *riskAccumulatorInternal) finishInternal() {strings.Compare(a.ImageName, b.ImageName),)})- overview.RiskiestImages = overview.RiskiestImages[:min(len(overview.RiskiestImages), riskiestImageLimit)]- _, overview.PatchPriority = environment.EnvironmentScore()- overview.Drivers.ScoredImages = environment.Scored()- if overview.Drivers.ScoredImages == 0 {- overview.ScoreStatus = vulnerability.ScoreStatusUnavailable+ _, patchPriority := environment.EnvironmentScore()+ return &vulnerability.RiskOverview{ //exhaustruct:enforce+ PatchPriority: vulnerability.PatchPriority{+ RiskScore: patchPriority.RiskScore,+ RiskBand: patchPriority.RiskBand,+ ScoreStatus: kit.Ternary(environment.Scored() == 0, vulnerability.ScoreStatusUnavailable, patchPriority.ScoreStatus),+ },+ Delta7d: nil,+ Trend: []vulnerability.RiskTrendPoint{},+ Drivers: vulnerability.RiskDrivers{ //exhaustruct:enforce+ KnownExploited: knownExploited,+ OverdueKnownExploited: overdueKnownExploited,+ HighEPSS: highEPSS,+ ExposedCriticalHigh: exposedCriticalHigh,+ Fixable: a.fixable,+ Findings: a.summary.Total,+ ImagesScanned: len(a.scannedImageIDs),+ ImagesTotal: imagesTotal,+ ScoredImages: environment.Scored(),+ },+ Drivers7dAgo: nil,+ Summary: a.summary,+ Exposure: a.exposure,+ ScoreDriver: a.scoreDriver,+ RiskiestImages: riskiestImages[:min(len(riskiestImages), riskiestImageLimit)],+ RiskiestFindings: top(byRisk),+ PrevalentFindings: top(byPrevalence),+ ThreatIntel: threatIntel,+ ComputedAt: time.Time{},}}-// computeRiskInternal scores every non-ignored finding of current images; the trend and deltas are left to callers.-func (s *Service) computeRiskInternal(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {+// computeRisk scores every non-ignored finding of current images; the trend and deltas are left to callers.+func (s *Service) computeRisk(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {intelStatus := s.threatIntelStatus(ctx)- usage, usageOK := s.containerUsageInternal(ctx)- imageIDs, imageIDsOK := s.currentImageIDsInternal(ctx)+ usage, usageOK := s.containerUsage(ctx)+ imageIDs, imageIDsOK := s.currentImageIDs(ctx)// Completed scans are listed first and findings loaded only for them, so a scan finishing in between is absent from both.- scanQuery := s.db.WithContext(ctx).Table(scansTableInternal).Where("status = ?", string(vulnerability.ScanStatusCompleted))+ scanQuery := s.db.WithContext(ctx).Table(scansTable).Where("status = ?", string(vulnerability.ScanStatusCompleted))if imageIDsOK {scanQuery = scanQuery.Where("id IN ?", imageIDs)}@@ -1066,8 +1142,8 @@ func (s *Service) computeRiskInternal(ctx context.Context, envID string) (*vulnereturn nil, fmt.Errorf("failed to list scanned images: %w", err)}- var rows []riskFindingRowInternal- err := scanItemsQueryInternal(ctx, s.db.DB, envID, false, mo.Some(scannedIDs)).+ var rows []riskFindingRow+ err := scanItemsQuery(ctx, s.db.DB, envID, false, mo.Some(scannedIDs)).Joins("LEFT JOIN vulnerability_threat_intel ON vulnerability_threat_intel.cve_id = vulnerability_scan_items.vulnerability_id").Select("vulnerability_scan_items.image_id", "vulnerability_scans.image_name", "vulnerability_scan_items.vulnerability_id",@@ -1083,49 +1159,51 @@ func (s *Service) computeRiskInternal(ctx context.Context, envID string) (*vulnereturn nil, fmt.Errorf("failed to load vulnerability risk findings: %w", err)}- acc := &riskAccumulatorInternal{- today: s.riskNowInternal().Format(riskDateLayout),+ acc := &riskAccumulator{+ today: s.riskNow().Format(riskDateLayout),scannedImageIDs: scannedIDs,exposures: make(map[string]vulnerability.ImageExposure, len(scannedIDs)),- overview: &vulnerability.RiskOverview{Trend: []vulnerability.RiskTrendPoint{}, ThreatIntel: intelStatus},- images: map[string]*riskImageAggInternal{},- findings: map[string]*riskFindingAggInternal{},+ images: map[string]*riskImageAgg{},+ findings: map[string]*riskFindingAgg{},}for _, id := range scannedIDs {- acc.exposures[id] = imageExposureInternal(usage, usageOK, id)- }- acc.overview.Drivers.ImagesScanned = len(scannedIDs)- acc.overview.Drivers.ImagesTotal = len(scannedIDs)- if imageIDsOK {- acc.overview.Drivers.ImagesTotal = len(imageIDs)+ acc.exposures[id] = imageExposure(usage, usageOK, id)}- for i := range rows {- row := &rows[i]+ for _, row := range rows {if !intelStatus.Enabled {- row.KnownExploited, row.KEVRansomware, row.KEVDueDate, row.EPSSScore = false, false, "", nil+ row = riskFindingRow{ //exhaustruct:enforce+ ImageID: row.ImageID,+ ImageName: row.ImageName,+ VulnerabilityID: row.VulnerabilityID,+ PkgName: row.PkgName,+ FixedVersion: row.FixedVersion,+ Severity: row.Severity,+ Title: row.Title,+ CVSSScore: row.CVSSScore,+ KnownExploited: false,+ KEVRansomware: false,+ KEVDueDate: "",+ EPSSScore: nil,+ }}- acc.addInternal(row, acc.exposures[row.ImageID], usage[row.ImageID].running)+ acc.add(&row, acc.exposures[row.ImageID], usage[row.ImageID].running)}- acc.finishInternal()- return acc.overview, nil+ return acc.finish(intelStatus, kit.Ternary(imageIDsOK, len(imageIDs), len(scannedIDs))), nil}-// riskNowInternal returns the current time in the configured timezone, which defines snapshot days.-func (s *Service) riskNowInternal() time.Time {- if s.config == nil {- return time.Now().UTC()- }+// riskNow returns the current time in the configured timezone, which defines snapshot days.+func (s *Service) riskNow() time.Time {return time.Now().In(s.config.GetLocation())}// GetRiskOverview returns the environment patch priority with its trend, drivers, prioritized images and top findings.// Concurrent callers share one build, and the result is reused briefly. Callers must not mutate it.func (s *Service) GetRiskOverview(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.db == nil {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return nil, common.ErrFeatureDisabled}- if overview, ok := s.cachedRiskOverviewInternal(envID); ok {+ if overview, ok := s.cachedRiskOverview(envID); ok {return overview, nil}@@ -1133,16 +1211,20 @@ func (s *Service) GetRiskOverview(ctx context.Context, envID string) (*vulnerabigen := s.riskGen.Load()results := s.riskFlight.DoChan(envID+":"+strconv.FormatUint(gen, 10), func() (any, error) {// A caller that queued behind the winner finds the fresh entry here.- if overview, ok := s.cachedRiskOverviewInternal(envID); ok {+ if overview, ok := s.cachedRiskOverview(envID); ok {return overview, nil}buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), riskOverviewBuildTimeout)defer cancel()- overview, buildErr := s.buildRiskOverviewInternal(buildCtx, envID)+ overview, buildErr := s.buildRiskOverview(buildCtx, envID)if buildErr != nil {return nil, buildErr}- s.riskCache.Store(envID, &riskOverviewCacheEntryInternal{overview: overview, builtAt: time.Now(), gen: gen})+ s.riskCache.Store(envID, &riskOverviewCacheEntry{+ overview: overview,+ builtAt: time.Now(),+ gen: gen,+ })return overview, nil})select {@@ -1165,25 +1247,25 @@ func (s *Service) InvalidateRiskOverview() {s.riskGen.Add(1)}-func (s *Service) cachedRiskOverviewInternal(envID string) (*vulnerability.RiskOverview, bool) {+func (s *Service) cachedRiskOverview(envID string) (*vulnerability.RiskOverview, bool) {value, ok := s.riskCache.Load(envID)if !ok {return nil, false}- entry, ok := value.(*riskOverviewCacheEntryInternal)+ entry, ok := value.(*riskOverviewCacheEntry)if !ok || entry.gen != s.riskGen.Load() || time.Since(entry.builtAt) >= riskOverviewCacheTTL {return nil, false}return entry.overview, true}-func (s *Service) buildRiskOverviewInternal(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {- overview, err := s.computeRiskInternal(ctx, envID)+func (s *Service) buildRiskOverview(ctx context.Context, envID string) (*vulnerability.RiskOverview, error) {+ overview, err := s.computeRisk(ctx, envID)if err != nil {return nil, err}- now := s.riskNowInternal()+ now := s.riskNow()today := now.Format(riskDateLayout)var snapshots []VulnerabilityRiskSnapshotif loadSnapshotsErr := s.db.WithContext(ctx).@@ -1196,15 +1278,17 @@ func (s *Service) buildRiskOverviewInternal(ctx context.Context, envID string) (deltaDay := now.AddDate(0, 0, -riskDeltaDays).Format(riskDateLayout)deltaFloor := now.AddDate(0, 0, -(riskDeltaDays + riskDeltaToleranceDays)).Format(riskDateLayout)+ trend := overview.Trend+ delta7d, drivers7dAgo := overview.Delta7d, overview.Drivers7dAgofor i := range snapshots {snap := &snapshots[i]if snap.SnapshotDate == today {continue}- overview.Trend = append(overview.Trend, vulnerability.RiskTrendPoint{Date: snap.SnapshotDate, RiskScore: snap.RiskScore})+ trend = append(trend, vulnerability.RiskTrendPoint{Date: snap.SnapshotDate, RiskScore: snap.RiskScore})if snap.SnapshotDate <= deltaDay && snap.SnapshotDate >= deltaFloor {- overview.Delta7d = new(overview.RiskScore - snap.RiskScore)- overview.Drivers7dAgo = &vulnerability.RiskDrivers{+ delta7d = new(overview.RiskScore - snap.RiskScore)+ drivers7dAgo = &vulnerability.RiskDrivers{KnownExploited: snap.KnownExploitedCount,OverdueKnownExploited: snap.OverdueKnownExploitedCount,HighEPSS: snap.HighEPSSCount,@@ -1217,13 +1301,25 @@ func (s *Service) buildRiskOverviewInternal(ctx context.Context, envID string) (}}}- overview.Trend = append(overview.Trend, vulnerability.RiskTrendPoint{Date: today, RiskScore: overview.RiskScore})- overview.ComputedAt = time.Now()- return overview, nil+ return &vulnerability.RiskOverview{ //exhaustruct:enforce+ PatchPriority: overview.PatchPriority,+ Delta7d: delta7d,+ Trend: append(trend, vulnerability.RiskTrendPoint{Date: today, RiskScore: overview.RiskScore}),+ Drivers: overview.Drivers,+ Drivers7dAgo: drivers7dAgo,+ Summary: overview.Summary,+ Exposure: overview.Exposure,+ ScoreDriver: overview.ScoreDriver,+ RiskiestImages: overview.RiskiestImages,+ RiskiestFindings: overview.RiskiestFindings,+ PrevalentFindings: overview.PrevalentFindings,+ ThreatIntel: overview.ThreatIntel,+ ComputedAt: time.Now(),+ }, nil}-var vulnerabilityCSVHeaderInternal = []string{"CVE", "Severity", "Package", "Installed Version", "Fixed Version", "Image", "Ignored"}+var vulnerabilityCSVHeader = []string{"CVE", "Severity", "Package", "Installed Version", "Fixed Version", "Image", "Ignored"}// WriteVulnerabilitiesCSV streams items to w as a BOM-prefixed, CRLF-terminated CSV with formula-safe cells.func WriteVulnerabilitiesCSV(w io.Writer, items iter.Seq2[vulnerability.VulnerabilityWithImage, error]) error {@@ -1231,8 +1327,8 @@ func WriteVulnerabilitiesCSV(w io.Writer, items iter.Seq2[vulnerability.Vulnerabreturn fmt.Errorf("failed to write csv: %w", err)}writer := csv.NewWriter(w)- writer.UseCRLF = true- if err := writer.Write(vulnerabilityCSVHeaderInternal); err != nil {+ writer.UseCRLF = true // keep: csv.Writer carries unexported state that only csv.NewWriter can build+ if err := writer.Write(vulnerabilityCSVHeader); err != nil {return fmt.Errorf("failed to write csv: %w", err)}for item, err := range items {@@ -1257,15 +1353,15 @@ func WriteVulnerabilitiesCSV(w io.Writer, items iter.Seq2[vulnerability.Vulnerab// RecordRiskSnapshot upserts today's environment risk snapshot and prunes snapshots past retention.func (s *Service) RecordRiskSnapshot(ctx context.Context) error {- if s.db == nil || !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return nil}- overview, err := s.computeRiskInternal(ctx, types.LocalDockerEnvironmentID)+ overview, err := s.computeRisk(ctx, types.LocalDockerEnvironmentID)if err != nil {return err}- now := s.riskNowInternal()+ now := s.riskNow()snapshot := VulnerabilityRiskSnapshot{SnapshotDate: now.Format(riskDateLayout),ScoringVersion: vuln.ScoringVersion,@@ -1326,11 +1422,11 @@ func (s *Service) RecordRiskSnapshot(ctx context.Context) error {// ActiveItemsQuery returns the filtered, ordered finding query the list and// export use, so bulk ignore operations act on exactly the listed findings.func (s *Service) ActiveItemsQuery(ctx context.Context, db *gorm.DB, envID string, params *pagination.QueryParams) *gorm.DB {- _, q := s.listAllVulnerabilitiesQueryInternal(ctx, db, envID, params)+ _, q := s.listAllVulnerabilitiesQuery(ctx, db, envID, params)return q}-func severityCountColumnsForFilterInternal(value string) []string {+func severityCountColumnsForFilter(value string) []string {parts := kit.TrimNonEmpty(strings.Split(value, ","))if len(parts) == 0 {return nil@@ -1360,13 +1456,13 @@ func severityCountColumnsForFilterInternal(value string) []string {return kit.Unique(columns)}-func applyListAllVulnerabilityRecordPrefiltersInternal(query *gorm.DB, filters map[string]string) mo.Option[*gorm.DB] {+func applyListAllVulnerabilityRecordPrefilters(query *gorm.DB, filters map[string]string) mo.Option[*gorm.DB] {if query == nil || len(filters) == 0 {return mo.Some(query)}if severityFilter, ok := filters["severity"]; ok {- severityColumns := severityCountColumnsForFilterInternal(severityFilter)+ severityColumns := severityCountColumnsForFilter(severityFilter)if len(severityColumns) == 0 {return mo.None[*gorm.DB]()}diff --git a/backend/internal/vulnerability/children/query/service_test.go b/backend/internal/vulnerability/children/query/service_test.goindex 727075387..a666c20dc 100644--- a/backend/internal/vulnerability/children/query/service_test.go+++ b/backend/internal/vulnerability/children/query/service_test.go@@ -22,7 +22,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx")-func TestSeverityCountColumnsForFilterInternal(t *testing.T) {+func TestSeverityCountColumnsForFilter(t *testing.T) {tests := []struct {name stringvalue string@@ -53,7 +53,7 @@ func TestSeverityCountColumnsForFilterInternal(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got := severityCountColumnsForFilterInternal(tt.value)+ got := severityCountColumnsForFilter(tt.value)if tt.wantNil {require.Nil(t, got)return@@ -66,15 +66,15 @@ func TestSeverityCountColumnsForFilterInternal(t *testing.T) {func TestApplyListAllVulnerabilityRecordPrefiltersInternal_NoPossibleMatches(t *testing.T) {db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- scans := db.WithContext(t.Context()).Table(scansTableInternal)+ scans := db.WithContext(t.Context()).Table(scansTable)- _, ok := applyListAllVulnerabilityRecordPrefiltersInternal(scans, map[string]string{"severity": "not-a-severity"}).Get()+ _, ok := applyListAllVulnerabilityRecordPrefilters(scans, map[string]string{"severity": "not-a-severity"}).Get()require.False(t, ok)- _, ok = applyListAllVulnerabilityRecordPrefiltersInternal(scans, map[string]string{"imageName": " , "}).Get()+ _, ok = applyListAllVulnerabilityRecordPrefilters(scans, map[string]string{"imageName": " , "}).Get()require.False(t, ok)- _, ok = applyListAllVulnerabilityRecordPrefiltersInternal(scans, map[string]string{+ _, ok = applyListAllVulnerabilityRecordPrefilters(scans, map[string]string{"severity": "high,bad","imageName": "nginx",}).Get()@@ -92,7 +92,12 @@ func TestWriteVulnerabilitiesCSV(t *testing.T) {ImageID: "img-1",ImageName: "example/security:latest",},- {VulnerabilityID: "CVE-2026-0002", PkgName: "libc, glibc", Severity: vulnerability.SeverityHigh, ImageID: "sha256:fallback"},+ {+ VulnerabilityID: "CVE-2026-0002",+ PkgName: "libc, glibc",+ Severity: vulnerability.SeverityHigh,+ ImageID: "sha256:fallback",+ },{VulnerabilityID: "CVE-2026-0003",PkgName: `pkg "quoted"`,@@ -115,7 +120,7 @@ func TestWriteVulnerabilitiesCSV(t *testing.T) {}var buf bytes.Buffer- require.NoError(t, WriteVulnerabilitiesCSV(&buf, seqInternal(items)))+ require.NoError(t, WriteVulnerabilitiesCSV(&buf, seq(items)))require.Equal(t,"\ufeffCVE,Severity,Package,Installed Version,Fixed Version,Image,Ignored\r\nCVE-2026-0001,HIGH,openssl,3.0."+@@ -127,11 +132,11 @@ func TestWriteVulnerabilitiesCSV(t *testing.T) {)buf.Reset()- require.NoError(t, WriteVulnerabilitiesCSV(&buf, seqInternal(nil)))+ require.NoError(t, WriteVulnerabilitiesCSV(&buf, seq(nil)))require.Equal(t, "\xEF\xBB\xBFCVE,Severity,Package,Installed Version,Fixed Version,Image,Ignored\r\n", buf.String())}-func seqInternal(items []vulnerability.VulnerabilityWithImage) iter.Seq2[vulnerability.VulnerabilityWithImage, error] {+func seq(items []vulnerability.VulnerabilityWithImage) iter.Seq2[vulnerability.VulnerabilityWithImage, error] {return func(yield func(vulnerability.VulnerabilityWithImage, error) bool) {for _, item := range items {if !yield(item, nil) {@@ -172,20 +177,33 @@ func vulnerabilityScopeContext(t *testing.T, target string, bodies map[string]anfunc TestAggregateVulnerabilitiesMergesEnvironments(t *testing.T) {finding := func(id string, severity vulnerability.Severity, imageName string) vulnerability.VulnerabilityWithImage {return vulnerability.VulnerabilityWithImage{- Vulnerability: vulnerability.Vulnerability{VulnerabilityID: id, PkgName: "pkg-" + id, InstalledVersion: "1.0", Severity: severity},- ImageID: "sha256:" + imageName,- ImageName: imageName,+ Vulnerability: vulnerability.Vulnerability{+ VulnerabilityID: id,+ PkgName: "pkg-" + id,+ InstalledVersion: "1.0",+ Severity: severity,+ },+ ImageID: "sha256:" + imageName,+ ImageName: imageName,}}bodies := map[string]any{- "a": base.Paginated[vulnerability.VulnerabilityWithImage]{Success: true, Data: []vulnerability.VulnerabilityWithImage{- finding("CVE-1", vulnerability.SeverityLow, "nginx"),- finding("CVE-2", vulnerability.SeverityCritical, "nginx"),- }, Pagination: base.PaginationResponse{TotalItems: 2, GrandTotalItems: 5}},- "b": base.Paginated[vulnerability.VulnerabilityWithImage]{Success: true, Data: []vulnerability.VulnerabilityWithImage{- finding("CVE-3", vulnerability.SeverityHigh, "redis"),- finding("CVE-4", vulnerability.SeverityUnknown, "redis"),- }, Pagination: base.PaginationResponse{TotalItems: 2}},+ "a": base.Paginated[vulnerability.VulnerabilityWithImage]{+ Success: true,+ Data: []vulnerability.VulnerabilityWithImage{+ finding("CVE-1", vulnerability.SeverityLow, "nginx"),+ finding("CVE-2", vulnerability.SeverityCritical, "nginx"),+ },+ Pagination: base.PaginationResponse{TotalItems: 2, GrandTotalItems: 5},+ },+ "b": base.Paginated[vulnerability.VulnerabilityWithImage]{+ Success: true,+ Data: []vulnerability.VulnerabilityWithImage{+ finding("CVE-3", vulnerability.SeverityHigh, "redis"),+ finding("CVE-4", vulnerability.SeverityUnknown, "redis"),+ },+ Pagination: base.PaginationResponse{TotalItems: 2},+ },}rowLabels := func(items []vulnerability.VulnerabilityWithImage) []string {out := make([]string, 0, len(items))@@ -198,11 +216,19 @@ func TestAggregateVulnerabilitiesMergesEnvironments(t *testing.T) {t.Run("orders severity by rank", func(t *testing.T) {ctx, queries := vulnerabilityScopeContext(t, "/api/environments/a,b,c/vulnerabilities/all?sort=severity&order=asc", bodies)- ascending, err := svc.AggregateVulnerabilities(ctx, pagination.QueryParams{SortParams: pagination.SortParams{Sort: "severity", Order: pagination.SortAsc}, Params: pagination.Params{Limit: -1}})+ ascending, err := svc.AggregateVulnerabilities(ctx, pagination.QueryParams{+ Sort: "severity",+ Order: pagination.SortAsc,+ Limit: -1,+ })require.NoError(t, err)require.Equal(t, []string{"b/CVE-4", "a/CVE-1", "b/CVE-3", "a/CVE-2"}, rowLabels(ascending.Data))- descending, err := svc.AggregateVulnerabilities(ctx, pagination.QueryParams{SortParams: pagination.SortParams{Sort: "severity", Order: pagination.SortDesc}, Params: pagination.Params{Limit: -1}})+ descending, err := svc.AggregateVulnerabilities(ctx, pagination.QueryParams{+ Sort: "severity",+ Order: pagination.SortDesc,+ Limit: -1,+ })require.NoError(t, err)require.Equal(t, []string{"a/CVE-2", "b/CVE-3", "a/CVE-1", "b/CVE-4"}, rowLabels(descending.Data))@@ -219,8 +245,10 @@ func TestAggregateVulnerabilitiesMergesEnvironments(t *testing.T) {t.Run("sorts by image name descending and pages", func(t *testing.T) {ctx, _ := vulnerabilityScopeContext(t, "/api/environments/a,b,c/vulnerabilities/all", bodies)body, err := svc.AggregateVulnerabilities(ctx, pagination.QueryParams{- SortParams: pagination.SortParams{Sort: "imageName", Order: pagination.SortDesc},- Params: pagination.Params{Start: 1, Limit: 2},+ Sort: "imageName",+ Order: pagination.SortDesc,+ Start: 1,+ Limit: 2,})require.NoError(t, err)require.True(t, body.Success)@@ -232,7 +260,11 @@ func TestAggregateVulnerabilitiesMergesEnvironments(t *testing.T) {t.Run("sorts by environment", func(t *testing.T) {ctx, _ := vulnerabilityScopeContext(t, "/api/environments/a,b,c/vulnerabilities/all?sort=environment", bodies)- body, err := svc.AggregateVulnerabilities(ctx, pagination.QueryParams{SortParams: pagination.SortParams{Sort: "environment", Order: pagination.SortDesc}, Params: pagination.Params{Limit: -1}})+ body, err := svc.AggregateVulnerabilities(ctx, pagination.QueryParams{+ Sort: "environment",+ Order: pagination.SortDesc,+ Limit: -1,+ })require.NoError(t, err)require.Equal(t, []string{"b/CVE-3", "b/CVE-4", "a/CVE-1", "a/CVE-2"}, rowLabels(body.Data))})diff --git a/backend/internal/vulnerability/children/scan/children/trivy/handler.go b/backend/internal/vulnerability/children/scan/children/trivy/handler.goindex 595754c41..1c3c5ddbc 100644--- a/backend/internal/vulnerability/children/scan/children/trivy/handler.go+++ b/backend/internal/vulnerability/children/scan/children/trivy/handler.go@@ -25,13 +25,12 @@ type GetScannerStatusInput struct {// GetScannerStatus checks if the vulnerability scanner is available.func (h *Handler) GetScannerStatus(ctx context.Context, input *GetScannerStatusInput) (*handlerutil.Out[vulnerability.ScannerStatus], error) {version := h.service.Version(ctx)- available := version != ""return &handlerutil.Out[vulnerability.ScannerStatus]{Body: base.ApiResponse[vulnerability.ScannerStatus]{Success: true,Data: vulnerability.ScannerStatus{- Available: available,+ Available: version != "",Version: version,},},diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service.go b/backend/internal/vulnerability/children/scan/children/trivy/service.goindex 970d4ab50..eb74b5ae6 100644--- a/backend/internal/vulnerability/children/scan/children/trivy/service.go+++ b/backend/internal/vulnerability/children/scan/children/trivy/service.go@@ -87,14 +87,6 @@ func NewService(}}-// settingsConfig returns the current settings, or nil without a settings service.-func (s *Service) settingsConfig() *settings.Settings {- if s.settingsService == nil {- return nil- }- return s.settingsService.GetSettingsConfig()-}-// A nil release without an error means the scan must wait in the queue.func (s *Service) AcquireSlot(ctx context.Context, imageID string, wait bool) (slotID int, release func(), err error) {if requireFeatureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); requireFeatureErr != nil {@@ -159,10 +151,7 @@ func (s *Service) AcquireSlot(ctx context.Context, imageID string, wait bool) (s}func (s *Service) ConcurrentScanContainers() int {- if cfg := s.settingsConfig(); cfg != nil {- return max(cfg.TrivyConcurrentScanContainers.AsInt(), 1)- }- return 1+ return max(s.settingsService.GetSettingsConfig().TrivyConcurrentScanContainers.AsInt(), 1)}// SetupBatchContainers starts long-running scanner containers that scheduled scans exec into,@@ -181,13 +170,14 @@ func (s *Service) SetupBatchContainers(ctx context.Context, count int) ([]stringreturn nil, nil, nil, fmt.Errorf("trivy scanner not available: %w", err)}- serverURL, _ := s.ServerConfig()- var cacheVolume string- if serverURL == "" {- cacheVolume, err = s.ensureTrivyCacheVolume(ctx)- if err != nil {- return nil, nil, nil, fmt.Errorf("failed to ensure trivy cache volume: %w", err)+ cacheVolume, err := func() (string, error) {+ if serverURL, _ := s.ServerConfig(); serverURL != "" {+ return "", nil}+ return s.ensureTrivyCacheVolume(ctx)+ }()+ if err != nil {+ return nil, nil, nil, fmt.Errorf("failed to ensure trivy cache volume: %w", err)}files := s.trivyContainerFiles(ctx)@@ -195,25 +185,21 @@ func (s *Service) SetupBatchContainers(ctx context.Context, count int) ([]stringif err != nil {return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", err)}- env := append([]string(nil), runtimeOptions.ContainerEnv...)- if files[vuln.RegistryConfigFile] != nil {- // ECR tokens refresh only at batch-container creation; a multi-hour batch can outlive them.- env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)- }containerConfig := &container.Config{Image: scannerImage,Entrypoint: []string{"sh", "-c", "trap 'exit 0' TERM; while :; do sleep 1; done"},- Env: env,+ // ECR tokens refresh only at batch-container creation; a multi-hour batch can outlive them.+ Env: slices.Concat(runtimeOptions.ContainerEnv, kit.Ternary(files[vuln.RegistryConfigFile] != nil, []string{"DOCKER_CONFIG=" + vuln.RegistryConfigDir}, nil)),Labels: map[string]string{libarcane.InternalResourceLabel: "true",},}- var securityOpts []string- privileged := false- if cfg := s.settingsConfig(); cfg != nil {- securityOpts, privileged = vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value), cfg.TrivyPrivileged.IsTrue()- }- hostConfig := vuln.BuildBatchHostConfig(cacheVolume, runtimeOptions, securityOpts, privileged)+ hostConfig := vuln.BuildBatchHostConfig(+ cacheVolume,+ runtimeOptions,+ vuln.ParseSecurityOpts(s.settingsService.GetSettingsConfig().TrivySecurityOpts.Value),+ s.settingsService.GetSettingsConfig().TrivyPrivileged.IsTrue(),+ )resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptions(ctx, dockerClient)if err != nil {return nil, nil, nil, fmt.Errorf("failed to create batch trivy container: %w", err)@@ -269,15 +255,12 @@ func (s *Service) ExecScan(ctx context.Context, containerID, imageName, imageID,defer vuln.CleanupOutputFileInContainer(ctx, dockerClient, containerID, outputPath)serverURL, serverToken := s.ServerConfig()- var cacheDir string- if serverURL == "" {- cacheDir = fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0))- }- execCmd := append([]string{"trivy"}, s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)...)-startTime := time.Now()exitCode, err := docker.ExecInContainer(ctx, dockerClient, containerID, client.ExecCreateOptions{- Cmd: execCmd,+ Cmd: append(+ []string{"trivy"},+ s.buildTrivyCommandArgs(imageName, kit.Ternary(serverURL == "", fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0)), ""), outputPath, files)...,+ ),Env: vuln.ServerTokenEnv(serverToken),AttachStdout: true,AttachStderr: true,@@ -286,21 +269,24 @@ func (s *Service) ExecScan(ctx context.Context, containerID, imageName, imageID,return nil, err}- run := trivyContainerResult{- stdoutFile: stdoutFile, stderrFile: stderrFile, duration: time.Since(startTime).Milliseconds(),- statusCode: int64(exitCode), containerID: containerID,- }- result, err = s.scanResult(ctx, dockerClient, run, outputPath, imageName, imageID, "exec")- if err != nil {- return nil, err- }- result.ScannerVersion = scannerVersion- return result, nil+ return s.scanResult(ctx, dockerClient, trivyContainerResult{+ stdoutFile: stdoutFile,+ stderrFile: stderrFile,+ duration: time.Since(startTime).Milliseconds(),+ statusCode: int64(exitCode),+ containerID: containerID,+ }, outputPath, imageName, imageID, "exec", func(context.Context) string { return scannerVersion })}// scanResult decodes a finished scan's report (output file first, then stdout), persists the raw report for// report-based patching, and records the result counts on the active span.-func (s *Service) scanResult(ctx context.Context, dockerClient *client.Client, run trivyContainerResult, outputPath, imageName, imageID, mode string) (*vulnerability.ScanResult, error) {+func (s *Service) scanResult(+ ctx context.Context,+ dockerClient *client.Client,+ run trivyContainerResult,+ outputPath, imageName, imageID, mode string,+ scannerVersion func(context.Context) string,+) (*vulnerability.ScanResult, error) {diagnose := func(reason string) {slog.WarnContext(ctx, "trivy scan diagnostics", "image", imageName, "imageId", imageID, "reason", mode+" "+reason, "exitCode", run.statusCode,"stdoutBytes", vuln.TempFileSize(run.stdoutFile), "stderrBytes", vuln.TempFileSize(run.stderrFile))@@ -318,12 +304,12 @@ func (s *Service) scanResult(ctx context.Context, dockerClient *client.Client, rraw = nil}- var report *vulnerability.TrivyReport- if len(raw) > 0 {- report, err = vuln.DecodeReportFromBytes(raw)- } else {- report, err = vuln.DecodeReportFromFile(run.stdoutFile)- }+ report, err := func() (*vulnerability.TrivyReport, error) {+ if len(raw) > 0 {+ return vuln.DecodeReportFromBytes(raw)+ }+ return vuln.DecodeReportFromFile(run.stdoutFile)+ }()switch {case errors.Is(err, io.EOF):diagnose("empty output")@@ -353,13 +339,13 @@ func (s *Service) scanResult(ctx context.Context, dockerClient *client.Client, rraw, _ = io.ReadAll(run.stdoutFile)}}- if len(raw) > 0 && s.saveReport != nil {+ hasReport := len(raw) > 0 && func() bool {if saveErr := s.saveReport(ctx, imageID, string(raw)); saveErr != nil {slog.WarnContext(ctx, "failed to persist raw trivy report", "imageId", imageID, "error", saveErr)- } else {- result.HasReport = true+ return false}- }+ return true+ }()summary := cmp.Or(result.Summary, &vulnerability.SeveritySummary{})trace.SpanFromContext(ctx).SetAttributes(@@ -372,7 +358,21 @@ func (s *Service) scanResult(ctx context.Context, dockerClient *client.Client, rattribute.Int("vulnerability.total", summary.Total),)slog.DebugContext(ctx, "trivy scan finished", "imageId", imageID, "imageName", imageName, "durationMs", run.duration, "status", result.Status, "vulnerabilities", summary.Total)- return result, nil+ return &vulnerability.ScanResult{ //exhaustruct:enforce+ ImageID: result.ImageID,+ ImageName: result.ImageName,+ ScanTime: result.ScanTime,+ Status: result.Status,+ ActivityID: result.ActivityID,+ ScanPhase: result.ScanPhase,+ Summary: result.Summary,+ Vulnerabilities: result.Vulnerabilities,+ Error: result.Error,+ Duration: result.Duration,+ ScannerVersion: scannerVersion(ctx),+ HasReport: result.HasReport || hasReport,+ Insights: result.Insights,+ }, nil}// Version returns the Trivy version from the configured scanner image.@@ -387,22 +387,18 @@ func (s *Service) Version(ctx context.Context) string {return ""}- containerConfig := &container.Config{- Image: scannerImage,- Entrypoint: []string{"trivy"},- Cmd: []string{"--version"},- Labels: map[string]string{- libarcane.InternalResourceLabel: "true",- },- }-- hostConfig := &container.HostConfig{- AutoRemove: true,- }-resp, err := dockerClient.ContainerCreate(ctx, client.ContainerCreateOptions{- Config: containerConfig,- HostConfig: hostConfig,+ Config: &container.Config{+ Image: scannerImage,+ Entrypoint: []string{"trivy"},+ Cmd: []string{"--version"},+ Labels: map[string]string{+ libarcane.InternalResourceLabel: "true",+ },+ },+ HostConfig: &container.HostConfig{+ AutoRemove: true,+ },})if err != nil {return ""@@ -470,33 +466,30 @@ func (s *Service) Version(ctx context.Context) string {}func (s *Service) resolveTrivyRuntimeOptions(ctx context.Context, dockerClient *client.Client) (vuln.RuntimeOptions, error) {- var configuredNetwork, configuredDockerHost string- if cfg := s.settingsConfig(); cfg != nil {- configuredNetwork, configuredDockerHost = strings.TrimSpace(cfg.TrivyNetwork.Value), strings.TrimSpace(cfg.DockerHost.Value)- }-- options := vuln.RuntimeOptions{NetworkMode: configuredNetwork}- if options.NetworkMode == "" {- if inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient); inspectErr == nil {- options.NetworkMode = docker.SelectAutoNetworkMode(inspect)- } else {+ networkMode := func() string {+ if configured := strings.TrimSpace(s.settingsService.GetSettingsConfig().TrivyNetwork.Value); configured != "" {+ return configured+ }+ inspect, inspectErr := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)+ if inspectErr != nil {slog.DebugContext(ctx, "failed to auto-detect Trivy network; falling back to bridge", "error", inspectErr)- options.NetworkMode = vuln.DefaultNetworkMode+ return vuln.DefaultNetworkMode}- }+ return docker.SelectAutoNetworkMode(inspect)+ }()- dockerHost := cmp.Or(strings.TrimSpace(s.dockerService.DockerHost()), configuredDockerHost, vuln.DefaultDockerHostURI)+ dockerHost := cmp.Or(strings.TrimSpace(s.dockerService.DockerHost()), strings.TrimSpace(s.settingsService.GetSettingsConfig().DockerHost.Value), vuln.DefaultDockerHostURI)scheme, socketPath, err := vuln.ParseDockerHost(dockerHost)if err != nil {return vuln.RuntimeOptions{}, fmt.Errorf("failed to resolve trivy docker host: %w", err)}- options.ContainerEnv = vuln.BuildDockerHostEnv(dockerHost)- if s.config != nil {- options.ContainerEnv = append(options.ContainerEnv, vuln.ProxyEnv(s.config.HTTPProxy, s.config.HTTPSProxy, s.config.NoProxy)...)- }+ containerEnv := slices.Concat(vuln.BuildDockerHostEnv(dockerHost), vuln.ProxyEnv(s.config.HTTPProxy, s.config.HTTPSProxy, s.config.NoProxy))if scheme != "unix" {- return options, nil+ return vuln.RuntimeOptions{+ NetworkMode: networkMode,+ ContainerEnv: containerEnv,+ }, nil}socketSource, err := vuln.ResolveUnixSocketSource(@@ -514,21 +507,19 @@ func (s *Service) resolveTrivyRuntimeOptions(ctx context.Context, dockerClient *return vuln.RuntimeOptions{}, fmt.Errorf("failed to resolve trivy docker host: %w", err)}- options.Mounts = append(options.Mounts, mount.Mount{- Type: mount.TypeBind,- Source: socketSource,- Target: socketPath,- })-- return options, nil+ return vuln.RuntimeOptions{+ NetworkMode: networkMode,+ ContainerEnv: containerEnv,+ Mounts: []mount.Mount{{+ Type: mount.TypeBind,+ Source: socketSource,+ Target: socketPath,+ }},+ }, nil}func (s *Service) scanTimeoutArg() string {- timeoutSeconds := 0- if cfg := s.settingsConfig(); cfg != nil {- timeoutSeconds = cfg.TrivyScanTimeout.AsInt()- }- return timeouts.GetDuration(timeoutSeconds, timeouts.DefaultTrivyScan).String()+ return timeouts.GetDuration(s.settingsService.GetSettingsConfig().TrivyScanTimeout.AsInt(), timeouts.DefaultTrivyScan).String()}func (s *Service) EnsureImage(ctx context.Context) (string, error) {@@ -537,16 +528,12 @@ func (s *Service) EnsureImage(ctx context.Context) (string, error) {return "", fmt.Errorf("failed to connect to Docker: %w", err)}- toolsRegistry, pullTimeoutSeconds := "", 0- if cfg := s.settingsConfig(); cfg != nil {- toolsRegistry, pullTimeoutSeconds = cfg.ToolsImageRegistry.Value, cfg.DockerImagePullTimeout.AsInt()- }- scannerImage := volumehelper.ToolsImage(toolsRegistry)+ scannerImage := volumehelper.ToolsImage(s.settingsService.GetSettingsConfig().ToolsImageRegistry.Value)if _, imageInspectErr := dockerClient.ImageInspect(ctx, scannerImage); imageInspectErr == nil {return scannerImage, nil}- pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(pullTimeoutSeconds, timeouts.DefaultDockerImagePull))+ pullCtx, pullCancel := context.WithTimeout(ctx, timeouts.GetDuration(s.settingsService.GetSettingsConfig().DockerImagePullTimeout.AsInt(), timeouts.DefaultDockerImagePull))defer pullCancel()pullReader, err := dockerClient.ImagePull(pullCtx, scannerImage, client.ImagePullOptions{})@@ -591,8 +578,8 @@ func (s *Service) ensureTrivyCacheVolume(ctx context.Context) (string, error) {// ServerConfig returns the remote Trivy server URL and optional token when// client/server mode is enabled, or empty strings for standalone (local DB) scanning.func (s *Service) ServerConfig() (serverURL, token string) {- cfg := s.settingsConfig()- if cfg == nil || !cfg.TrivyServerEnabled.IsTrue() {+ cfg := s.settingsService.GetSettingsConfig()+ if !cfg.TrivyServerEnabled.IsTrue() {return "", ""}@@ -602,28 +589,24 @@ func (s *Service) ServerConfig() (serverURL, token string) {// IgnoreUnfixed reports whether scans should only return vulnerabilities// that have a known fix (--ignore-unfixed).func (s *Service) IgnoreUnfixed() bool {- cfg := s.settingsConfig()- return cfg != nil && cfg.TrivyIgnoreUnfixed.IsTrue()+ return s.settingsService.GetSettingsConfig().TrivyIgnoreUnfixed.IsTrue()}func (s *Service) trivyContainerFiles(ctx context.Context) map[string][]byte {files := map[string][]byte{}- if s.registryAuthConfigs != nil {- if authConfigs, err := s.registryAuthConfigs(ctx); err != nil {- slog.WarnContext(ctx, "failed to load registry credentials for trivy scan; continuing anonymously", "error", err)- } else if configJSON, buildErr := vuln.BuildDockerConfigJSON(authConfigs); buildErr != nil {- slog.WarnContext(ctx, "failed to build trivy registry config; continuing anonymously", "error", buildErr)- } else if len(configJSON) > 0 {- files[vuln.RegistryConfigFile] = configJSON- }+ if authConfigs, err := s.registryAuthConfigs(ctx); err != nil {+ slog.WarnContext(ctx, "failed to load registry credentials for trivy scan; continuing anonymously", "error", err)+ } else if configJSON, buildErr := vuln.BuildDockerConfigJSON(authConfigs); buildErr != nil {+ slog.WarnContext(ctx, "failed to build trivy registry config; continuing anonymously", "error", buildErr)+ } else if len(configJSON) > 0 {+ files[vuln.RegistryConfigFile] = configJSON}- if cfg := s.settingsConfig(); cfg != nil {- if strings.TrimSpace(cfg.TrivyConfig.Value) != "" {- files[vuln.TrivyConfigFile] = []byte(cfg.TrivyConfig.Value)- }- if strings.TrimSpace(cfg.TrivyIgnore.Value) != "" {- files[vuln.TrivyIgnoreFile] = []byte(cfg.TrivyIgnore.Value)- }+ cfg := s.settingsService.GetSettingsConfig()+ if strings.TrimSpace(cfg.TrivyConfig.Value) != "" {+ files[vuln.TrivyConfigFile] = []byte(cfg.TrivyConfig.Value)+ }+ if strings.TrimSpace(cfg.TrivyIgnore.Value) != "" {+ files[vuln.TrivyIgnoreFile] = []byte(cfg.TrivyIgnore.Value)}return files}@@ -631,12 +614,8 @@ func (s *Service) trivyContainerFiles(ctx context.Context) map[string][]byte {// Flags passed here take precedence over the user-supplied trivy.yaml.func (s *Service) buildTrivyCommandArgs(imageName, cacheDir, outputPath string, files map[string][]byte) []string {serverURL, _ := s.ServerConfig()- dbRegistry := ""- if cfg := s.settingsConfig(); cfg != nil {- dbRegistry = cfg.TrivyDbRegistry.Value- }cmdArgs := []string{"image", "--format", "json", "--quiet", "--exit-code", "0", "--exit-on-eol", "0", "--timeout", s.scanTimeoutArg()}- cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, dbRegistry)...)+ cmdArgs = append(cmdArgs, vuln.ScanSourceArgs(serverURL, s.settingsService.GetSettingsConfig().TrivyDbRegistry.Value)...)cmdArgs = append(cmdArgs, vuln.IgnoreUnfixedArgs(s.IgnoreUnfixed())...)if files[vuln.TrivyConfigFile] != nil {@@ -715,7 +694,7 @@ func (s *Service) createAndStartTrivyContainer(}func (s *Service) getTrivyContainerResourceOptions(ctx context.Context, dockerClient *client.Client) (container.Resources, string, bool, error) {- resources, applyLimits := buildTrivyContainerResources(s.settingsConfig()).Get()+ resources, applyLimits := buildTrivyContainerResources(s.settingsService.GetSettingsConfig()).Get()if !applyLimits || resources.NanoCPUs == 0 {return resources, "", applyLimits, nil}@@ -728,21 +707,23 @@ func (s *Service) getTrivyContainerResourceOptions(ctx context.Context, dockerCl// Capabilities, not limits, are cached per daemon so a new service instance skips the Info call.key := "trivy:cpu-capabilities:v1:" + kit.SHA256Hex(dockerClient.DaemonHost()+"\x00"+dockerClient.ClientVersion())- var capabilities trivyCPUCapabilities- fresh := false- if s.kvService != nil {+ capabilities, fresh := func() (trivyCPUCapabilities, bool) {raw, found, err := s.kvService.Get(ctx, key)if err != nil {slog.WarnContext(ctx, "failed to read cached Trivy CPU capabilities", "error", err)- } else if found {- if unmarshalErr := json.Unmarshal([]byte(raw), &capabilities); unmarshalErr != nil || capabilities.FetchedAt.IsZero() {- slog.WarnContext(ctx, "ignoring invalid cached Trivy CPU capabilities", "error", unmarshalErr)- capabilities = trivyCPUCapabilities{}- } else if age := time.Since(capabilities.FetchedAt); age >= 0 && age < trivyCPUCapabilitiesCacheTTL {- fresh = true- }+ return trivyCPUCapabilities{}, false}- }+ var cached trivyCPUCapabilities+ if !found {+ return cached, false+ }+ if unmarshalErr := json.Unmarshal([]byte(raw), &cached); unmarshalErr != nil || cached.FetchedAt.IsZero() {+ slog.WarnContext(ctx, "ignoring invalid cached Trivy CPU capabilities", "error", unmarshalErr)+ return trivyCPUCapabilities{}, false+ }+ age := time.Since(cached.FetchedAt)+ return cached, age >= 0 && age < trivyCPUCapabilitiesCacheTTL+ }()if !fresh {infoCtx, cancel := context.WithTimeout(ctx, trivyCPUCapabilitiesLookupTimeout)@@ -763,14 +744,12 @@ func (s *Service) getTrivyContainerResourceOptions(ctx context.Context, dockerClCPUs: info.Info.NCPU,FetchedAt: time.Now().UTC(),}- if s.kvService != nil {- payload, marshalErr := json.Marshal(capabilities)- if marshalErr == nil {- marshalErr = s.kvService.Set(ctx, key, string(payload))- }- if marshalErr != nil {- slog.WarnContext(ctx, "failed to cache Trivy CPU capabilities", "error", marshalErr)- }+ payload, marshalErr := json.Marshal(capabilities)+ if marshalErr == nil {+ marshalErr = s.kvService.Set(ctx, key, string(payload))+ }+ if marshalErr != nil {+ slog.WarnContext(ctx, "failed to cache Trivy CPU capabilities", "error", marshalErr)}}}@@ -779,11 +758,8 @@ func (s *Service) getTrivyContainerResourceOptions(ctx context.Context, dockerClreturn resources, "", applyLimits, nil}- cpuSet := ""- if capabilities.CPUSet {- cpuSet = vuln.BuildCPUSet(resources.NanoCPUs, capabilities.CPUs)- }- resources.NanoCPUs = 0+ cpuSet := kit.Ternary(capabilities.CPUSet, vuln.BuildCPUSet(resources.NanoCPUs, capabilities.CPUs), "")+ resources.NanoCPUs = 0 // keep: copying Docker's ~40-field container.Resources as a literal is unreadableif cpuSet != "" {slog.InfoContext(ctx, "docker host does not support CPU CFS quota; applying Trivy CPU limit via cpuset", "cpuset", cpuSet)@@ -795,32 +771,17 @@ func (s *Service) getTrivyContainerResourceOptions(ctx context.Context, dockerCl}func buildTrivyContainerResources(cfg *settings.Settings) mo.Option[container.Resources] {- limitsEnabled := true- cpuLimitCores := defaultTrivyCPULimitCores- memoryLimitMB := defaultTrivyMemoryLimitMB-- if cfg != nil {- limitsEnabled = cfg.TrivyResourceLimitsEnabled.IsTrue()- cpuLimitCores = vuln.ParseCPULimit(cfg.TrivyCpuLimit.Value, defaultTrivyCPULimitCores)- memoryLimitMB = vuln.ParseMemoryLimitMB(cfg.TrivyMemoryLimitMb.Value, defaultTrivyMemoryLimitMB)- }-- if !limitsEnabled {+ if cfg != nil && !cfg.TrivyResourceLimitsEnabled.IsTrue() {return mo.None[container.Resources]()}- resources := container.Resources{}- if cpuLimitCores > 0 {- resources.NanoCPUs = int64(cpuLimitCores * float64(vuln.NanoCPUsPerCore))- }-- if memoryLimitMB > 0 {- memoryBytes := memoryLimitMB * vuln.BytesPerMB- resources.Memory = memoryBytes- resources.MemorySwap = memoryBytes- }-- return mo.Some(resources)+ cpuLimitCores := vuln.ParseCPULimit(kit.FromPtr(cfg).TrivyCpuLimit.Value, defaultTrivyCPULimitCores)+ memoryBytes := max(vuln.ParseMemoryLimitMB(kit.FromPtr(cfg).TrivyMemoryLimitMb.Value, defaultTrivyMemoryLimitMB), 0) * vuln.BytesPerMB+ return mo.Some(container.Resources{+ NanoCPUs: kit.Ternary(cpuLimitCores > 0, int64(cpuLimitCores*float64(vuln.NanoCPUsPerCore)), 0),+ Memory: memoryBytes,+ MemorySwap: memoryBytes,+ })}type trivyContainerResult struct {@@ -847,44 +808,51 @@ func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID str}serverURL, serverToken := s.ServerConfig()- var cacheVolume, cacheDir string- if serverURL == "" {- if cacheVolume, err = s.ensureTrivyCacheVolume(ctx); err != nil {- return nil, err+ cacheVolume, err := func() (string, error) {+ if serverURL != "" {+ return "", nil}- // A fixed cache dir per slot avoids fs-cache lock contention while bounding growth to the slot count.- cacheDir = fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0))+ return s.ensureTrivyCacheVolume(ctx)+ }()+ if err != nil {+ return nil, err+ }+ // A fixed cache dir per slot avoids fs-cache lock contention while bounding growth to the slot count.+ cacheDir := kit.Ternary(serverURL == "", fmt.Sprintf("%s/slot-%d", trivyCacheSlotRootDir, max(slotID, 0)), "")+ if serverURL == "" {slog.DebugContext(ctx, "using slot-based trivy cache dir for scan", "slotId", slotID, "cacheDir", cacheDir, "concurrentScanContainers", s.ConcurrentScanContainers())}outputPath := vuln.NewOutputPath()files := s.trivyContainerFiles(ctx)- cmdArgs := s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files)runtimeOptions, err := s.resolveTrivyRuntimeOptions(ctx, dockerClient)if err != nil {return nil, err}- env := append([]string(nil), runtimeOptions.ContainerEnv...)- if files[vuln.RegistryConfigFile] != nil {- env = append(env, "DOCKER_CONFIG="+vuln.RegistryConfigDir)- }- if serverToken != "" {- env = append(env, vuln.ServerTokenEnv(serverToken)...)- }-- containerConfig := vuln.BuildContainerConfig(scannerImage, cmdArgs, env)+ containerConfig := vuln.BuildContainerConfig(+ scannerImage,+ s.buildTrivyCommandArgs(imageName, cacheDir, outputPath, files),+ slices.Concat(+ runtimeOptions.ContainerEnv,+ kit.Ternary(files[vuln.RegistryConfigFile] != nil, []string{"DOCKER_CONFIG=" + vuln.RegistryConfigDir}, nil),+ kit.Ternary(serverToken != "", vuln.ServerTokenEnv(serverToken), nil),+ ),+ )resources, cpuSet, applyLimits, err := s.getTrivyContainerResourceOptions(ctx, dockerClient)if err != nil {return nil, err}- var securityOpts []string- privileged := false- if cfg := s.settingsConfig(); cfg != nil {- securityOpts, privileged = vuln.ParseSecurityOpts(cfg.TrivySecurityOpts.Value), cfg.TrivyPrivileged.IsTrue()- }- hostConfig := vuln.BuildHostConfig(cacheVolume, resources, cpuSet, applyLimits, runtimeOptions, securityOpts, privileged)+ hostConfig := vuln.BuildHostConfig(+ cacheVolume,+ resources,+ cpuSet,+ applyLimits,+ runtimeOptions,+ vuln.ParseSecurityOpts(s.settingsService.GetSettingsConfig().TrivySecurityOpts.Value),+ s.settingsService.GetSettingsConfig().TrivyPrivileged.IsTrue(),+ )s.setPhase(imageID, vulnerability.ScanPhaseCreatingContainer)stdoutFile, err := vuln.CreateLogTempFile("trivy-stdout-*")@@ -975,17 +943,13 @@ func (s *Service) Scan(ctx context.Context, scannerImage, imageName, imageID str}}- run := trivyContainerResult{- stdoutFile: stdoutFile, stderrFile: stderrFile, duration: time.Since(startTime).Milliseconds(),- statusCode: statusCode, containerID: containerID,- }-- result, err = s.scanResult(ctx, dockerClient, run, outputPath, imageName, imageID, "container")- if err != nil {- return nil, err- }- result.ScannerVersion = s.Version(ctx)- return result, nil+ return s.scanResult(ctx, dockerClient, trivyContainerResult{+ stdoutFile: stdoutFile,+ stderrFile: stderrFile,+ duration: time.Since(startTime).Milliseconds(),+ statusCode: statusCode,+ containerID: containerID,+ }, outputPath, imageName, imageID, "container", s.Version)}type trivyCPUCapabilities struct {diff --git a/backend/internal/vulnerability/children/scan/children/trivy/service_test.go b/backend/internal/vulnerability/children/scan/children/trivy/service_test.goindex 09feb3a9f..9ea0b1a12 100644--- a/backend/internal/vulnerability/children/scan/children/trivy/service_test.go+++ b/backend/internal/vulnerability/children/scan/children/trivy/service_test.go@@ -14,9 +14,11 @@ import ("github.com/libtnb/sqlite""github.com/moby/moby/client""github.com/stretchr/testify/require"+ "go.getarcane.app/kit/pkg""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database""github.com/getarcaneapp/arcane/backend/v2/internal/docker""github.com/getarcaneapp/arcane/backend/v2/internal/kv"@@ -28,7 +30,7 @@ func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {t.Helper()gdb, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})require.NoError(t, err)- require.NoError(t, gdb.AutoMigrate(&settings.SettingVariable{}))+ require.NoError(t, gdb.AutoMigrate(&settings.SettingVariable{}, &kv.KVEntry{}))db := &database.DB{DB: gdb}svc, err := settings.NewSettingsService(t.Context(), db)if err == nil {@@ -87,20 +89,71 @@ func TestBuildTrivyContainerResources_CustomLimits(t *testing.T) {quota int64cpuSet string}{- {name: "fractional quota", cpu: "0.5", info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4,"OperatingSystem":"Linux"}`, quota: 500_000_000},- {name: "one and a half cores", cpu: "1.5", info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4}`, quota: 1_500_000_000},- {name: "Synology overrides reported quota support", cpu: "0.5", info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4,"OperatingSystem":"Synology NAS"}`, cpuSet: "0"},- {name: "older DiskStation", cpu: "1.5", info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4,"OperatingSystem":"DiskStation"}`, cpuSet: "0"},- {name: "no quotas", cpu: "2.5", info: `{"CPUSet":true,"NCPU":4}`, cpuSet: "0-1"},- {name: "no CPU controls", cpu: "0.5", info: `{"NCPU":4}`},- {name: "CPU limit disabled", cpu: "0", infoFail: true},- {name: "capability lookup fails", cpu: "0.5", infoFail: true, quota: 500_000_000},- {name: "capability lookup times out", cpu: "0.5", slowInfo: true, quota: 500_000_000},- {name: "capability lookup deadline", cpu: "0.5", expired: true},- {name: "no Docker client", cpu: "0.5", noClient: true},+ {+ name: "fractional quota",+ cpu: "0.5",+ info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4,"OperatingSystem":"Linux"}`,+ quota: 500_000_000,+ },+ {+ name: "one and a half cores",+ cpu: "1.5",+ info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4}`,+ quota: 1_500_000_000,+ },+ {+ name: "Synology overrides reported quota support",+ cpu: "0.5",+ info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4,"OperatingSystem":"Synology NAS"}`,+ cpuSet: "0",+ },+ {+ name: "older DiskStation",+ cpu: "1.5",+ info: `{"CPUCfsQuota":true,"CPUSet":true,"NCPU":4,"OperatingSystem":"DiskStation"}`,+ cpuSet: "0",+ },+ {+ name: "no quotas",+ cpu: "2.5",+ info: `{"CPUSet":true,"NCPU":4}`,+ cpuSet: "0-1",+ },+ {+ name: "no CPU controls",+ cpu: "0.5",+ info: `{"NCPU":4}`,+ },+ {+ name: "CPU limit disabled",+ cpu: "0",+ infoFail: true,+ },+ {+ name: "capability lookup fails",+ cpu: "0.5",+ infoFail: true,+ quota: 500_000_000,+ },+ {+ name: "capability lookup times out",+ cpu: "0.5",+ slowInfo: true,+ quota: 500_000_000,+ },+ {+ name: "capability lookup deadline",+ cpu: "0.5",+ expired: true,+ },+ {+ name: "no Docker client",+ cpu: "0.5",+ noClient: true,+ },} {t.Run(tc.name, func(t *testing.T) {- settingsSvc, _ := featureSettings(t)+ settingsSvc, db := featureSettings(t)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), "trivyResourceLimitsEnabled", true))require.NoError(t, settingsSvc.SetStringSetting(t.Context(), "trivyCpuLimit", tc.cpu))require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyMemoryLimitMb", 2048))@@ -127,15 +180,18 @@ func TestBuildTrivyContainerResources_CustomLimits(t *testing.T) {}}))defer server.Close()- httpClient := server.Client()- if tc.slowInfo {- httpClient.Timeout = 20 * time.Millisecond- }- dockerClient, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.41"), client.WithHTTPClient(httpClient))+ dockerClient, err := client.New(+ client.WithHost(server.URL),+ client.WithAPIVersion("1.41"),+ client.WithHTTPClient(&http.Client{+ Transport: server.Client().Transport,+ Timeout: kit.Ternary(tc.slowInfo, 20*time.Millisecond, 0),+ }),+ )require.NoError(t, err)clientToClose := dockerClientt.Cleanup(func() { require.NoError(t, clientToClose.Close()) })- svc := &Service{settingsService: settingsSvc}+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, kv.NewKVService(db), nil, nil, nil)ctx := t.Context()if tc.expired {var cancel context.CancelFunc@@ -171,7 +227,8 @@ func TestBuildTrivyContainerResources_CustomLimits(t *testing.T) {}func TestBuildTrivyCommandArgs_WithCacheDir(t *testing.T) {- svc := &Service{}+ settingsSvc, db := featureSettings(t)+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, kv.NewKVService(db), nil, nil, nil)outputPath := "/tmp/trivy-result-test.json"cmdArgs := svc.buildTrivyCommandArgs("alpine:3.20", "/tmp/trivy-cache", outputPath, nil)@@ -184,7 +241,8 @@ func TestBuildTrivyCommandArgs_WithCacheDir(t *testing.T) {}func TestBuildTrivyCommandArgs_WithoutCacheDir(t *testing.T) {- svc := &Service{}+ settingsSvc, db := featureSettings(t)+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, kv.NewKVService(db), nil, nil, nil)outputPath := "/tmp/trivy-result-test.json"cmdArgs := svc.buildTrivyCommandArgs("alpine:3.20", "", outputPath, nil)@@ -196,7 +254,8 @@ func TestBuildTrivyCommandArgs_WithoutCacheDir(t *testing.T) {}func TestBuildTrivyCommandArgs_IncludesDefaultRepositories(t *testing.T) {- svc := &Service{}+ settingsSvc, db := featureSettings(t)+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, kv.NewKVService(db), nil, nil, nil)cmdArgs := svc.buildTrivyCommandArgs("alpine:3.20", "", "/tmp/out.json", nil)require.Contains(t, cmdArgs, "--db-repository")@@ -208,9 +267,9 @@ func TestBuildTrivyCommandArgs_IncludesDefaultRepositories(t *testing.T) {}func TestAcquireSlot_PrefillsSlots(t *testing.T) {- settingsSvc, _ := featureSettings(t)+ settingsSvc, db := featureSettings(t)require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyConcurrentScanContainers", 3))- svc := &Service{settingsService: settingsSvc}+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, kv.NewKVService(db), nil, nil, nil)seen := map[int]bool{}for i := range 3 {@@ -252,7 +311,7 @@ func assertPersistentTrivyCapabilities(t *testing.T, quotaSupported bool) {dockerClient, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.41"))require.NoError(t, err)t.Cleanup(func() { require.NoError(t, dockerClient.Close()) })- svc := &Service{settingsService: settingsSvc, kvService: store}+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, store, nil, nil, nil)first, firstCPUSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)require.True(t, apply)@@ -266,7 +325,7 @@ func assertPersistentTrivyCapabilities(t *testing.T, quotaSupported bool) {}// A new service instance reuses persisted capabilities without calling Docker.- svc = &Service{settingsService: settingsSvc, kvService: kv.NewKVService(db)}+ svc = NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, kv.NewKVService(db), nil, nil, nil)status.Store(http.StatusForbidden)cached, cpuSet, apply, err := svc.getTrivyContainerResourceOptions(t.Context(), dockerClient)require.NoError(t, err)@@ -289,9 +348,14 @@ func assertPersistentTrivyCapabilities(t *testing.T, quotaSupported bool) {entries, err := store.ListByPrefix(t.Context(), "trivy:cpu-capabilities:v1:")require.NoError(t, err)require.Len(t, entries, 1)- var expired trivyCPUCapabilities- require.NoError(t, json.Unmarshal([]byte(entries[0].Value), &expired))- expired.FetchedAt = time.Now().Add(-2 * trivyCPUCapabilitiesCacheTTL)+ var stored trivyCPUCapabilities+ require.NoError(t, json.Unmarshal([]byte(entries[0].Value), &stored))+ expired := trivyCPUCapabilities{+ Quota: stored.Quota,+ CPUSet: stored.CPUSet,+ CPUs: stored.CPUs,+ FetchedAt: time.Now().Add(-2 * trivyCPUCapabilitiesCacheTTL),+ }payload, err := json.Marshal(expired)require.NoError(t, err)require.NoError(t, store.Set(t.Context(), entries[0].Key, string(payload)))@@ -363,12 +427,20 @@ func TestVulnerabilityFeatureDisabledWhileWaitingForSlot(t *testing.T) {queuedImage stringslots int}{- {name: "waiting for capacity", queuedImage: "other-image", slots: 1},- {name: "waiting for the same image", queuedImage: "active-image", slots: 2},+ {+ name: "waiting for capacity",+ queuedImage: "other-image",+ slots: 1,+ },+ {+ name: "waiting for the same image",+ queuedImage: "active-image",+ slots: 2,+ },}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- settingsSvc, _ := featureSettings(t)+ settingsSvc, db := featureSettings(t)require.NoError(t, settingsSvc.SetIntSetting(t.Context(), "trivyConcurrentScanContainers", tt.slots))var inspections atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {@@ -380,7 +452,7 @@ func TestVulnerabilityFeatureDisabledWhileWaitingForSlot(t *testing.T) {dockerClient, err := client.New(client.WithHost(server.URL), client.WithAPIVersion("1.41"))require.NoError(t, err)t.Cleanup(func() { require.NoError(t, dockerClient.Close()) })- svc := &Service{settingsService: settingsSvc, dockerService: &docker.DockerClientService{Client: dockerClient}}+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient), settingsSvc, kv.NewKVService(db), nil, nil, nil)_, release, err := svc.AcquireSlot(t.Context(), "active-image", false)require.NoError(t, err)require.NotNil(t, release)@@ -423,8 +495,8 @@ func TestVulnerabilityFeatureDisabledWhileWaitingForSlot(t *testing.T) {}func TestVulnerabilityFeatureDisabledRejectsBatchContainers(t *testing.T) {- settingsSvc, _ := featureSettings(t)- svc := &Service{settingsService: settingsSvc}+ settingsSvc, db := featureSettings(t)+ svc := NewService(&config.Config{}, docker.NewDockerClientService(t.Context(), nil, nil, nil), settingsSvc, kv.NewKVService(db), nil, nil, nil)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))_, _, _, err := svc.SetupBatchContainers(t.Context(), 1)diff --git a/backend/internal/vulnerability/children/scan/module.go b/backend/internal/vulnerability/children/scan/module.goindex 020c49745..0cc46c777 100644--- a/backend/internal/vulnerability/children/scan/module.go+++ b/backend/internal/vulnerability/children/scan/module.go@@ -13,11 +13,12 @@ import (// RegisterRoutes registers the vulnerability scan and scanner status endpoints behind featureMiddleware.func RegisterRoutes(api huma.API, h *Handler, featureMiddleware huma.Middlewares) {- var trivyService *trivy.Service- if h.service != nil {- trivyService = h.service.trivy- }- trivy.RegisterRoutes(api, trivy.NewHandler(trivyService), featureMiddleware)+ trivy.RegisterRoutes(api, trivy.NewHandler(func() *trivy.Service {+ if h.service == nil {+ return nil+ }+ return h.service.trivy+ }()), featureMiddleware)middleware.RegisterWithPermission(api, huma.Operation{Middlewares: featureMiddleware,diff --git a/backend/internal/vulnerability/children/scan/service.go b/backend/internal/vulnerability/children/scan/service.goindex dda53a942..3b5b92242 100644--- a/backend/internal/vulnerability/children/scan/service.go+++ b/backend/internal/vulnerability/children/scan/service.go@@ -69,7 +69,7 @@ type Dependencies struct {Notifications *notification.NotificationServiceActivity *activity.ActivityServiceSaveResult func(ctx context.Context, result *vulnerability.ScanResult) error- EnsureSummary func(result *vulnerability.ScanResult)+ EnsureSummary func(result *vulnerability.ScanResult) *vulnerability.SeveritySummaryFilterIgnored func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error)}@@ -84,7 +84,7 @@ type Service struct {notificationService *notification.NotificationServiceactivityService *activity.ActivityServicesaveResult func(ctx context.Context, result *vulnerability.ScanResult) error- ensureSummary func(result *vulnerability.ScanResult)+ ensureSummary func(result *vulnerability.ScanResult) *vulnerability.SeveritySummaryfilterIgnored func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error)scanPhases sync.Map // map[string]vulnerability.ScanPhasepoolMu sync.Mutex@@ -120,9 +120,13 @@ type ()func NewService(deps Dependencies) *Service {- s := &Service{- db: deps.DB,- records: deps.Records,+ var s *Service+ s = &Service{+ db: deps.DB,+ records: deps.Records,+ trivy: trivy.NewService(deps.Config, deps.Docker, deps.Settings, deps.KV, deps.RegistryAuthConfigs, deps.SaveReport, func(imageID string, phase vulnerability.ScanPhase) {+ s.setScanPhase(imageID, phase)+ }),dockerService: deps.Docker,settingsService: deps.Settings,eventService: deps.Events,@@ -132,7 +136,6 @@ func NewService(deps Dependencies) *Service {ensureSummary: deps.EnsureSummary,filterIgnored: deps.FilterIgnored,}- s.trivy = trivy.NewService(deps.Config, deps.Docker, deps.Settings, deps.KV, deps.RegistryAuthConfigs, deps.SaveReport, s.setScanPhase)return s}@@ -190,8 +193,7 @@ func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user usestatus := kit.Ternary(releaseSlot != nil, vulnerability.ScanStatusScanning, vulnerability.ScanStatusPending)slog.InfoContext(scanCtx, "vulnerability scan queued", "environmentId", envID, "imageId", imageID, "imageName", imageName)- activityID := ""- if s.activityService != nil {+ activityID := func() string {scanActivity, startErr := s.activityService.StartActivity(scanCtx, activity.StartActivityRequest{EnvironmentID: envID,Type: activitytypes.TypeVulnerabilityScan,@@ -205,17 +207,24 @@ func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user use})if startErr != nil {slog.WarnContext(scanCtx, "failed to create vulnerability scan activity", "error", startErr, "imageId", imageID)- } else {- activityID = scanActivity.ID+ return ""}- }+ return scanActivity.ID+ }()activityRef := mo.EmptyableToOption(activityID).ToPointer()// Cancelling the activity cancels scanCtx, which stops the Trivy container wait and tears it down.scanCtx = s.activityService.Track(scanCtx, activityID)s.setScanPhase(imageID, vulnerability.ScanPhaseCreatingContainer)- pendingResult := &vulnerability.ScanResult{ImageID: imageID, ImageName: imageName, ScanTime: time.Now(), Status: status, ActivityID: activityRef}- pendingResult.ScanPhase = s.resolveScanPhase(imageID, status)+ pendingResult := &vulnerability.ScanResult{+ ImageID: imageID,+ ImageName: imageName,+ ScanTime: time.Now(),+ Status: status,+ ActivityID: activityRef,+ ScanPhase: s.resolveScanPhase(imageID, status),+ Summary: &vulnerability.SeveritySummary{},+ }if saveErr := s.saveResult(scanCtx, pendingResult); saveErr != nil {slog.WarnContext(scanCtx, "failed to save pending scan result", "error", saveErr, "imageId", imageID, "imageName", imageName)}@@ -236,11 +245,15 @@ func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user usefail := func(message string, duration int64) {s.setScanPhase(imageID, vulnerability.ScanPhaseStoringResults)s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing failed scan result")- failed := &vulnerability.ScanResult{- ImageID: imageID, ImageName: imageName, ScanTime: time.Now(), Status: vulnerability.ScanStatusFailed,- ActivityID: activityRef, Error: message, Duration: duration,- }- if saveErr := s.saveResult(scanCtx, failed); saveErr != nil {+ if saveErr := s.saveResult(scanCtx, &vulnerability.ScanResult{+ ImageID: imageID,+ ImageName: imageName,+ ScanTime: time.Now(),+ Status: vulnerability.ScanStatusFailed,+ ActivityID: activityRef,+ Error: message,+ Duration: duration,+ }); saveErr != nil {slog.WarnContext(scanCtx, "failed to save failed scan result", "error", saveErr)}slog.WarnContext(scanCtx, "background vulnerability scan failed", "environmentId", envID, "imageId", imageID, "imageName", imageName, "error", message)@@ -261,7 +274,7 @@ func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user uses.setScanPhase(imageID, vulnerability.ScanPhaseScanningImage)s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseScanningImage, 45, "Scanning image for vulnerabilities")startTime := time.Now()- result, runErr := s.trivy.Scan(scanCtx, scannerImage, imageName, imageID, slotID)+ scanned, runErr := s.trivy.Scan(scanCtx, scannerImage, imageName, imageID, slotID)duration := time.Since(startTime).Milliseconds()if runErr != nil {s.logScanEvent(scanCtx, event.EventTypeImageScan, "vulnerability_scan", envID, imageID, imageName, user, runErr.Error())@@ -269,9 +282,21 @@ func (s *Service) ScanImage(ctx context.Context, envID, imageID string, user usereturn}- result.Duration = duration- result.ActivityID = activityRef- s.ensureSummary(result)+ result := &vulnerability.ScanResult{ //exhaustruct:enforce+ ImageID: scanned.ImageID,+ ImageName: scanned.ImageName,+ ScanTime: scanned.ScanTime,+ Status: scanned.Status,+ ActivityID: activityRef,+ ScanPhase: scanned.ScanPhase,+ Summary: s.ensureSummary(scanned),+ Vulnerabilities: scanned.Vulnerabilities,+ Error: scanned.Error,+ Duration: duration,+ ScannerVersion: scanned.ScannerVersion,+ HasReport: scanned.HasReport,+ Insights: scanned.Insights,+ }s.setScanPhase(imageID, vulnerability.ScanPhaseStoringResults)s.updateVulnerabilityScanActivity(scanCtx, activityID, vulnerability.ScanPhaseStoringResults, 90, "Storing vulnerability results")if saveErr := s.saveResult(scanCtx, result); saveErr != nil {@@ -299,9 +324,6 @@ func (s *Service) markStale(ctx context.Context, imageID string, status vulneraberrMsg := "Scan timed out or was interrupted. Please retry."duration := time.Since(scanTime).Milliseconds()s.scanPhases.Delete(imageID)- if s.records.MarkFailed == nil {- return errMsg, duration, true- }if err := s.records.MarkFailed(ctx, imageID, errMsg, duration); err != nil {slog.WarnContext(ctx, "failed to mark stale scan as failed", "error", err, "scanId", imageID)}@@ -313,11 +335,16 @@ func (s *Service) RefreshSummary(ctx context.Context, summary *vulnerability.Scaif summary == nil {return}- if errMsg, _, stale := s.markStale(ctx, summary.ImageID, summary.Status, summary.ScanTime); stale {- summary.Status = vulnerability.ScanStatusFailed- summary.Error = errMsg+ errMsg, _, stale := s.markStale(ctx, summary.ImageID, summary.Status, summary.ScanTime)+ status := kit.Ternary(stale, vulnerability.ScanStatusFailed, summary.Status)+ *summary = vulnerability.ScanSummary{ //exhaustruct:enforce+ ImageID: summary.ImageID,+ ScanTime: summary.ScanTime,+ Status: status,+ ScanPhase: s.resolveScanPhase(summary.ImageID, status),+ Summary: summary.Summary,+ Error: kit.Ternary(stale, errMsg, summary.Error),}- summary.ScanPhase = s.resolveScanPhase(summary.ImageID, summary.Status)}// RefreshResult fails stale in-progress scans and attaches the live scan phase.@@ -325,16 +352,27 @@ func (s *Service) RefreshResult(ctx context.Context, result *vulnerability.ScanRif result == nil {return}- if errMsg, duration, stale := s.markStale(ctx, result.ImageID, result.Status, result.ScanTime); stale {- result.Status = vulnerability.ScanStatusFailed- result.Error = errMsg- result.Duration = duration+ errMsg, duration, stale := s.markStale(ctx, result.ImageID, result.Status, result.ScanTime)+ status := kit.Ternary(stale, vulnerability.ScanStatusFailed, result.Status)+ *result = vulnerability.ScanResult{ //exhaustruct:enforce+ ImageID: result.ImageID,+ ImageName: result.ImageName,+ ScanTime: result.ScanTime,+ Status: status,+ ActivityID: result.ActivityID,+ ScanPhase: s.resolveScanPhase(result.ImageID, status),+ Summary: result.Summary,+ Vulnerabilities: result.Vulnerabilities,+ Error: kit.Ternary(stale, errMsg, result.Error),+ Duration: kit.Ternary(stale, duration, result.Duration),+ ScannerVersion: result.ScannerVersion,+ HasReport: result.HasReport,+ Insights: result.Insights,}- result.ScanPhase = s.resolveScanPhase(result.ImageID, result.Status)}func (s *Service) updateVulnerabilityScanActivity(ctx context.Context, activityID string, phase vulnerability.ScanPhase, progress int, message string) {- if s.activityService == nil || activityID == "" {+ if activityID == "" {return}if _, err := s.activityService.AppendMessage(ctx, activityID, activity.AppendActivityMessageRequest{@@ -348,20 +386,19 @@ func (s *Service) updateVulnerabilityScanActivity(ctx context.Context, activityI}func (s *Service) completeVulnerabilityScanActivity(ctx context.Context, activityID string, success bool, errMessage string) {- if s.activityService == nil || activityID == "" {+ if activityID == "" {return}- status := activitytypes.StatusSuccess- message := "Vulnerability scan completed"- var errorPtr *string- switch {- case success:- case activitylib.CancelledByContext(ctx):- status, message = activitytypes.StatusCancelled, "Vulnerability scan cancelled"- default:- status, message = activitytypes.StatusFailed, cmp.Or(errMessage, "Vulnerability scan failed")- errorPtr = mo.EmptyableToOption(errMessage).ToPointer()- }+ status, message, errorPtr := func() (activitytypes.Status, string, *string) {+ switch {+ case success:+ return activitytypes.StatusSuccess, "Vulnerability scan completed", nil+ case activitylib.CancelledByContext(ctx):+ return activitytypes.StatusCancelled, "Vulnerability scan cancelled", nil+ default:+ return activitytypes.StatusFailed, cmp.Or(errMessage, "Vulnerability scan failed"), mo.EmptyableToOption(errMessage).ToPointer()+ }+ }()if _, err := s.activityService.CompleteActivity(utils.ActivityRuntimeContext(ctx, nil), activityID, status, message, errorPtr); err != nil {slog.DebugContext(ctx, "failed to complete vulnerability scan activity", "activityId", activityID, "error", err)}@@ -401,7 +438,11 @@ func (s *Service) Targets(ctx context.Context) ([]Target, error) {imageName := slices.Concat(img.RepoTags, img.RepoDigests, []string{img.ID})[0]// Skip intermediate and dangling images with no usable name.if imageName != "<none>:<none>" && imageName != img.ID {- targets = append(targets, Target{ImageID: img.ID, ImageName: imageName, ScannerVersion: scannerVersion})+ targets = append(targets, Target{+ ImageID: img.ID,+ ImageName: imageName,+ ScannerVersion: scannerVersion,+ })ids = append(ids, img.ID)}}@@ -452,7 +493,12 @@ func (s *Service) joinPool(ctx context.Context) (*scanPool, func(), error) {s.poolMu.Unlock()return nil, nil, common.Classify(common.ErrUnavailable, fmt.Errorf("prepare scan containers: %w", err))}- s.pool = &scanPool{containers: make(chan string, len(containerIDs)), files: files, cleanup: cleanup, retired: make(chan struct{})}+ s.pool = &scanPool{+ containers: make(chan string, len(containerIDs)),+ files: files,+ cleanup: cleanup,+ retired: make(chan struct{}),+ }for _, containerID := range containerIDs {s.pool.containers <- containerID}@@ -460,7 +506,7 @@ func (s *Service) joinPool(ctx context.Context) (*scanPool, func(), error) {pool := s.poolif pool.idle != nil {pool.idle.Stop()- pool.idle = nil+ pool.idle = nil // keep: the pool is shared with in-flight scans under poolMu}pool.users++s.poolMu.Unlock()@@ -473,7 +519,7 @@ func (s *Service) joinPool(ctx context.Context) (*scanPool, func(), error) {s.pool, idle, retired = nil, false, true}if idle {- pool.idle = time.AfterFunc(scanPoolIdle, s.ReleaseScanPool)+ pool.idle = time.AfterFunc(scanPoolIdle, s.ReleaseScanPool) // keep: the pool is shared with in-flight scans under poolMu}s.poolMu.Unlock()// The last scan out of a retired pool removes its containers, the unstoppable one included.@@ -537,46 +583,67 @@ func (s *Service) ScanTarget(ctx context.Context, envID string, target Target) (if featureErr := s.settingsService.RequireFeature(ctx, features.VulnerabilityManagement); featureErr != nil {return Fixable{}, featureErr}- result, err := s.trivy.ExecScan(ctx, containerID, target.ImageName, target.ImageID, target.ScannerVersion, slotID, pool.files)+ scanned, err := s.trivy.ExecScan(ctx, containerID, target.ImageName, target.ImageID, target.ScannerVersion, slotID, pool.files)duration := time.Since(startTime).Milliseconds()if err != nil {- failed := &vulnerability.ScanResult{- ImageID: target.ImageID, ImageName: target.ImageName, ScanTime: time.Now(),- Status: vulnerability.ScanStatusFailed, Error: err.Error(), Duration: duration,- }- if saveErr := s.saveResult(ctx, failed); saveErr != nil {+ if saveErr := s.saveResult(ctx, &vulnerability.ScanResult{+ ImageID: target.ImageID,+ ImageName: target.ImageName,+ ScanTime: time.Now(),+ Status: vulnerability.ScanStatusFailed,+ Error: err.Error(),+ Duration: duration,+ }); saveErr != nil {slog.WarnContext(ctx, "failed to save failed scan result", "error", saveErr)}s.logScanEvent(ctx, event.EventTypeImageVulnerabilityScan, "scheduled_vulnerability_scan", envID, target.ImageID, target.ImageName, scheduledScanUser, err.Error())return Fixable{}, err}- result.Duration = duration- s.ensureSummary(result)+ result := &vulnerability.ScanResult{ //exhaustruct:enforce+ ImageID: scanned.ImageID,+ ImageName: scanned.ImageName,+ ScanTime: scanned.ScanTime,+ Status: scanned.Status,+ ActivityID: scanned.ActivityID,+ ScanPhase: scanned.ScanPhase,+ Summary: s.ensureSummary(scanned),+ Vulnerabilities: scanned.Vulnerabilities,+ Error: scanned.Error,+ Duration: duration,+ ScannerVersion: scanned.ScannerVersion,+ HasReport: scanned.HasReport,+ Insights: scanned.Insights,+ }if saveErr := s.saveResult(ctx, result); saveErr != nil {slog.WarnContext(ctx, "failed to save scan result", "error", saveErr)}s.logScanEvent(ctx, event.EventTypeImageVulnerabilityScan, "scheduled_vulnerability_scan", envID, target.ImageID, target.ImageName, scheduledScanUser, "")- fixable := Fixable{Severity: map[string]int{}}if result.Status != vulnerability.ScanStatusCompleted || len(result.Vulnerabilities) == 0 {- return fixable, nil+ return Fixable{Severity: map[string]int{}}, nil}vulnerabilities, err := s.filterIgnored(ctx, envID, result.ImageID, result.Vulnerabilities)if err != nil {slog.WarnContext(ctx, "failed to filter ignored vulnerabilities for notifications", "error", err)vulnerabilities = result.Vulnerabilities}+ total, severity := 0, map[string]int{}+ var cves []stringfor _, v := range vulnerabilities {if strings.TrimSpace(v.FixedVersion) == "" {continue}- fixable.Total++- fixable.Severity[strings.ToUpper(strings.TrimSpace(string(v.Severity)))]++- if cveID := strings.TrimSpace(v.VulnerabilityID); cveID != "" && len(fixable.CVEs) < 5 && !slices.Contains(fixable.CVEs, cveID) {- fixable.CVEs = append(fixable.CVEs, cveID)+ total+++ severity[strings.ToUpper(strings.TrimSpace(string(v.Severity)))]+++ if cveID := strings.TrimSpace(v.VulnerabilityID); cveID != "" && len(cves) < 5 && !slices.Contains(cves, cveID) {+ cves = append(cves, cveID)}}- return fixable, nil+ return Fixable{+ Total: total,+ Severity: severity,+ CVEs: cves,+ }, nil}// ReleaseScanPool removes the shared scan containers once no scan is using them.@@ -597,7 +664,7 @@ func (s *Service) ReleaseScanPool() {// NotifySummary sends the scheduled scan's daily summary of fixable findings.func (s *Service) NotifySummary(ctx context.Context, scanned int, fixables []Fixable) {- if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || s.notificationService == nil {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) {return}severity := map[string]int{"CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0, "UNKNOWN": 0}diff --git a/backend/internal/vulnerability/children/scan/service_test.go b/backend/internal/vulnerability/children/scan/service_test.goindex 80ffde583..2325d169b 100644--- a/backend/internal/vulnerability/children/scan/service_test.go+++ b/backend/internal/vulnerability/children/scan/service_test.go@@ -35,7 +35,9 @@ func TestDisabledScheduledItemPreservesPreviousResult(t *testing.T) {settingsSvc, db := featureSettings(t)require.NoError(t, db.Table("vulnerability_scans").AutoMigrate(&scanStateRow{}))require.NoError(t, db.Table("vulnerability_scans").Create(&scanStateRow{- ID: "test-image", Status: string(vulnerability.ScanStatusCompleted), ScanTime: time.Now(),+ ID: "test-image",+ Status: string(vulnerability.ScanStatusCompleted),+ ScanTime: time.Now(),}).Error)svc := NewService(Dependencies{DB: db,@@ -56,7 +58,10 @@ func TestDisabledScheduledItemPreservesPreviousResult(t *testing.T) {func TestDisabledFeatureSuppressesScheduledSummaryNotification(t *testing.T) {settingsSvc, _ := featureSettings(t)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))- svc := &Service{settingsService: settingsSvc, notificationService: ¬ification.NotificationService{}}+ svc := NewService(Dependencies{+ Settings: settingsSvc,+ Notifications: ¬ification.NotificationService{},+ })require.NotPanics(t, func() {svc.NotifySummary(t.Context(), 1, []Fixable{{Total: 1}})})diff --git a/backend/internal/vulnerability/children/store/service.go b/backend/internal/vulnerability/children/store/service.goindex ebc2d7793..7a2d961ca 100644--- a/backend/internal/vulnerability/children/store/service.go+++ b/backend/internal/vulnerability/children/store/service.go@@ -1,6 +1,7 @@package storeimport (+ "cmp""context""encoding/json/v2""errors"@@ -8,8 +9,10 @@ import ("io""io/fs""log/slog"+ "maps""net/http""os"+ "slices""strconv""strings""time"@@ -17,6 +20,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/features""github.com/getarcaneapp/arcane/types/v2/vulnerability""go.getarcane.app/acfs"+ "go.getarcane.app/kit/pkg""gorm.io/gorm""github.com/getarcaneapp/arcane/backend/v2/internal/database"@@ -34,9 +38,9 @@ const (kevFeedURL = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json"epssAPIURL = "https://api.first.org/data/v1/epss"- // listAllVulnerabilityRecordBatchSizeInternal bounds how many scan records+ // listAllVulnerabilityRecordBatchSize bounds how many scan records// (each carrying a vulnerability blob of up to ~2 MiB) the backfill loads at once.- listAllVulnerabilityRecordBatchSizeInternal = 25+ listAllVulnerabilityRecordBatchSize = 25threatIntelSettingKey = "vulnerabilityThreatIntelEnabled")@@ -96,10 +100,10 @@ func NewService(}}-// deleteScanChildRowsInternal best-effort removes the raw report and CVE item+// deleteScanChildRows best-effort removes the raw report and CVE item// rows of the given scan records, so neither outlives its scan record.// Failures are logged and never block record deletion.-func (s *Service) deleteScanChildRowsInternal(ctx context.Context, imageIDs []string) {+func (s *Service) deleteScanChildRows(ctx context.Context, imageIDs []string) {if len(imageIDs) == 0 {return}@@ -115,11 +119,7 @@ func (s *Service) deleteScanChildRowsInternal(ctx context.Context, imageIDs []st// recorded before the table existed. Idempotent; failures are logged and never// block startup.func (s *Service) BackfillScanItems(ctx context.Context) {- if s.db == nil {- return- }-- err := s.records.EachMissingItems(ctx, listAllVulnerabilityRecordBatchSizeInternal, func(imageID, vulnerabilitiesJSON string) {+ err := s.records.EachMissingItems(ctx, listAllVulnerabilityRecordBatchSize, func(imageID, vulnerabilitiesJSON string) {var vulns []vulnerability.Vulnerabilityif err := json.Unmarshal([]byte(vulnerabilitiesJSON), &vulns); err != nil {slog.WarnContext(ctx, "failed to unmarshal vulnerabilities for backfill", "error", err, "imageId", imageID)@@ -156,10 +156,6 @@ const (// counts, then removes the on-disk report directory. No-op once imported;// failures are logged and never block startup.func (s *Service) ImportLegacyReportFiles(ctx context.Context) {- if s.db == nil {- return- }-rootDir, logical := ".", "/data/vulnerability-reports"if _, err := os.Stat("/app/data"); err == nil {rootDir, logical = "/app/data", "/vulnerability-reports"@@ -176,33 +172,25 @@ func (s *Service) ImportLegacyReportFiles(ctx context.Context) {}for imageID, fixableCount := range scans {- s.importLegacyReportInternal(ctx, imageID, fixableCount, rootDir, logical)- }-- if removeAllErr := acfs.RemoveAll(ctx, rootDir, logical); removeAllErr != nil {- slog.WarnContext(ctx, "failed to remove legacy vulnerability report directory", "error", removeAllErr)- }-}--func (s *Service) importLegacyReportInternal(ctx context.Context, imageID string, fixableCount *int, rootDir, logical string) {- fileName := strings.ReplaceAll(imageID, ":", "_") + ".json"- data, err := acfs.ReadFile(ctx, rootDir, logical+"/"+fileName)- if err != nil {- if !errors.Is(err, fs.ErrNotExist) {- slog.WarnContext(ctx, "failed to read legacy vulnerability report", "imageId", imageID, "error", err)+ data, readErr := acfs.ReadFile(ctx, rootDir, logical+"/"+strings.ReplaceAll(imageID, ":", "_")+".json")+ if readErr != nil {+ if !errors.Is(readErr, fs.ErrNotExist) {+ slog.WarnContext(ctx, "failed to read legacy vulnerability report", "imageId", imageID, "error", readErr)+ }+ continue+ }+ report, decodeErr := vuln.DecodeReportFromBytes(data)+ if decodeErr != nil || report == nil {+ slog.WarnContext(ctx, "skipping unparseable legacy vulnerability report", "imageId", imageID, "error", decodeErr)+ continue+ }+ if importReportErr := s.records.SaveReport(ctx, imageID, string(data)); importReportErr != nil {+ slog.WarnContext(ctx, "failed to import legacy vulnerability report", "imageId", imageID, "error", importReportErr)+ continue+ }+ if fixableCount != nil {+ continue}- return- }- report, err := vuln.DecodeReportFromBytes(data)- if err != nil || report == nil {- slog.WarnContext(ctx, "skipping unparseable legacy vulnerability report", "imageId", imageID, "error", err)- return- }- if importReportErr := s.records.SaveReport(ctx, imageID, string(data)); importReportErr != nil {- slog.WarnContext(ctx, "failed to import legacy vulnerability report", "imageId", imageID, "error", importReportErr)- return- }- if fixableCount == nil {count := 0for r := range report.Results {res := &report.Results[r]@@ -219,25 +207,25 @@ func (s *Service) importLegacyReportInternal(ctx context.Context, imageID stringslog.WarnContext(ctx, "failed to backfill fixable vulnerability count", "imageId", imageID, "error", updateFixableCountErr)}}++ if removeAllErr := acfs.RemoveAll(ctx, rootDir, logical); removeAllErr != nil {+ slog.WarnContext(ctx, "failed to remove legacy vulnerability report directory", "error", removeAllErr)+ }}// DeleteScanResult deletes the scan result for an imagefunc (s *Service) DeleteScanResult(ctx context.Context, imageID string) error {- if s.db == nil {- return nil- }-- s.deleteScanChildRowsInternal(ctx, []string{imageID})+ s.deleteScanChildRows(ctx, []string{imageID})return s.records.Delete(ctx, []string{imageID})}// DeleteScanResultsByImageIDs deletes scan results for multiple images in one query.func (s *Service) DeleteScanResultsByImageIDs(ctx context.Context, imageIDs []string) error {- if s.db == nil || len(imageIDs) == 0 {+ if len(imageIDs) == 0 {return nil}- s.deleteScanChildRowsInternal(ctx, imageIDs)+ s.deleteScanChildRows(ctx, imageIDs)return s.records.Delete(ctx, imageIDs)}@@ -246,10 +234,6 @@ func (s *Service) DeleteScanResultsByImageIDs(ctx context.Context, imageIDs []st// avoids "5/3" when images were deleted after being scanned). Safe to call// even when no images exist; returns the number of records deleted.func (s *Service) CleanupOrphanedScanRecords(ctx context.Context) (deleted int64, err error) {- if s.db == nil {- return 0, nil- }-images, err := s.dockerService.ListImages(ctx)if err != nil {return 0, fmt.Errorf("failed to list images: %w", err)@@ -261,19 +245,13 @@ func (s *Service) CleanupOrphanedScanRecords(ctx context.Context) (deleted int64}// Build list for NOT IN. If no images, we delete all scan records.- var ids []string- if len(currentIDs) > 0 {- ids = make([]string, 0, len(currentIDs))- for id := range currentIDs {- ids = append(ids, id)- }- }+ ids := slices.Collect(maps.Keys(currentIDs))staleIDs, err := s.records.IDsExcept(ctx, ids)if err != nil {slog.WarnContext(ctx, "failed to list stored vulnerability reports for cleanup", "error", err)} else {- s.deleteScanChildRowsInternal(ctx, staleIDs)+ s.deleteScanChildRows(ctx, staleIDs)}return s.records.DeleteExcept(ctx, ids)}@@ -289,9 +267,43 @@ func (s *Service) SaveResult(retryDelay = defaultScanSaveRetryDelay)+ ensured := &vulnerability.ScanResult{ //exhaustruct:enforce+ ImageID: result.ImageID,+ ImageName: result.ImageName,+ ScanTime: result.ScanTime,+ Status: result.Status,+ ActivityID: result.ActivityID,+ ScanPhase: result.ScanPhase,+ Summary: EnsureSummary(result),+ Vulnerabilities: result.Vulnerabilities,+ Error: result.Error,+ Duration: result.Duration,+ ScannerVersion: result.ScannerVersion,+ HasReport: result.HasReport,+ Insights: result.Insights,+ }+ fixable := 0+ for i := range ensured.Vulnerabilities {+ v := &ensured.Vulnerabilities[i]+ if v.Class == vulnerability.TrivyClassOSPackages && strings.TrimSpace(v.FixedVersion) != "" {+ fixable+++ }+ }+var lastErr errorfor attempt := 1; attempt <= maxAttempts; attempt++ {- lastErr = s.saveResultInternal(ctx, result)+ lastErr = dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {+ if err := s.records.Save(tx, ensured, fixable); err != nil {+ return err+ }+ if err := s.records.DeleteItems(tx, []string{ensured.ImageID}); err != nil {+ return fmt.Errorf("failed to clear scan items: %w", err)+ }+ if ensured.Status != vulnerability.ScanStatusCompleted || len(ensured.Vulnerabilities) == 0 {+ return nil+ }+ return s.records.SaveItems(tx, ensured.ImageID, ensured.Vulnerabilities)+ })if lastErr == nil {if attempt > 1 {slog.DebugContext(ctx,@@ -316,7 +328,7 @@ func (s *Service) SaveResult(select {case <-ctx.Done():timer.Stop()- s.saveScanResultStatusFallbackInternal(ctx, result, ctx.Err())+ s.saveScanResultStatusFallback(ctx, result, ctx.Err())return ctx.Err()case <-timer.C:}@@ -325,28 +337,18 @@ func (s *Service) SaveResult(// Ensure scan status does not stay stuck as scanning when full result// persistence fails. Best-effort minimal status write.- s.saveScanResultStatusFallbackInternal(ctx, result, lastErr)+ s.saveScanResultStatusFallback(ctx, result, lastErr)return lastErr}-func (s *Service) saveScanResultStatusFallbackInternal(ctx context.Context, result *vulnerability.ScanResult, originalErr error) {- if s.records.SaveStatus == nil || result == nil || result.ImageID == "" {+func (s *Service) saveScanResultStatusFallback(ctx context.Context, result *vulnerability.ScanResult, originalErr error) {+ if result == nil || result.ImageID == "" {return}- var summary *vulnerability.SeveritySummary- if result.Summary != nil {- summary = result.Summary- } else {- summary = &vulnerability.SeveritySummary{}- }-- errMsg := strings.TrimSpace(result.Error)- if errMsg == "" && originalErr != nil {- errMsg = fmt.Sprintf("failed to persist full scan result: %v", originalErr)- }- if err := s.records.SaveStatus(ctx, result, summary, errMsg); err != nil {+ errMsg := cmp.Or(strings.TrimSpace(result.Error), kit.Ternary(originalErr != nil, fmt.Sprintf("failed to persist full scan result: %v", originalErr), ""))+ if err := s.records.SaveStatus(ctx, result, cmp.Or(result.Summary, &vulnerability.SeveritySummary{}), errMsg); err != nil {slog.WarnContext(ctx,"failed fallback scan status persistence","imageId", result.ImageID,@@ -365,67 +367,28 @@ func (s *Service) saveScanResultStatusFallbackInternal(ctx context.Context, resu)}-// saveScanResult saves the scan result to the database-func (s *Service) saveResultInternal(ctx context.Context, result *vulnerability.ScanResult) error {- if s.db == nil {- return nil- }-- EnsureSummary(result)-- fixable := 0- for i := range result.Vulnerabilities {- v := &result.Vulnerabilities[i]- if v.Class == vulnerability.TrivyClassOSPackages && strings.TrimSpace(v.FixedVersion) != "" {- fixable++- }- }-- return dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {- if err := s.records.Save(tx, result, fixable); err != nil {- return err- }- if err := s.records.DeleteItems(tx, []string{result.ImageID}); err != nil {- return fmt.Errorf("failed to clear scan items: %w", err)- }- if result.Status != vulnerability.ScanStatusCompleted || len(result.Vulnerabilities) == 0 {- return nil- }- return s.records.SaveItems(tx, result.ImageID, result.Vulnerabilities)- })-}-// EnsureSummary derives the severity summary of result from its findings when missing.-func EnsureSummary(result *vulnerability.ScanResult) {- if result == nil {- return- }-- if len(result.Vulnerabilities) == 0 {- if result.Summary == nil {- result.Summary = &vulnerability.SeveritySummary{}- }- return- }-- if result.Summary != nil && result.Summary.Total > 0 {- return+func EnsureSummary(result *vulnerability.ScanResult) *vulnerability.SeveritySummary {+ switch {+ case result == nil:+ return nil+ case len(result.Vulnerabilities) == 0:+ return cmp.Or(result.Summary, &vulnerability.SeveritySummary{})+ case kit.FromPtr(result.Summary).Total > 0:+ return result.Summary+ default:+ return vuln.SeveritySummary(result.Vulnerabilities)}-- result.Summary = vuln.SeveritySummary(result.Vulnerabilities)}-type threatIntelStateInternal struct {+type threatIntelState struct {LastSyncedAt *time.Time `json:"lastSyncedAt,omitempty"`ETag string `json:"etag,omitempty"`LastError string `json:"lastError,omitempty"`}-func (s *Service) loadThreatIntelStateInternal(ctx context.Context) threatIntelStateInternal {- var state threatIntelStateInternal- if s.kvService == nil {- return state- }+func (s *Service) loadThreatIntelState(ctx context.Context) threatIntelState {+ var state threatIntelStateraw, ok, err := s.kvService.Get(ctx, threatIntelStateKey)if err != nil || !ok {return state@@ -438,42 +401,40 @@ func (s *Service) loadThreatIntelStateInternal(ctx context.Context) threatIntelS// SyncThreatIntel refreshes the CISA KEV catalog and FIRST EPSS scores for scanned CVEs, at most every 20h unless forced.func (s *Service) SyncThreatIntel(ctx context.Context, force bool) error {- if s.db == nil || !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || !s.settingsService.GetBoolSetting(ctx, threatIntelSettingKey, true) {+ if !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement) || !s.settingsService.GetBoolSetting(ctx, threatIntelSettingKey, true) {return nil}- state := s.loadThreatIntelStateInternal(ctx)- if !force && state.LastSyncedAt != nil && time.Since(*state.LastSyncedAt) < threatIntelSyncInterval {+ state := s.loadThreatIntelState(ctx)+ if !force && time.Since(kit.FromPtr(state.LastSyncedAt)) < threatIntelSyncInterval {return nil}- etag, kevErr := s.syncKEVInternal(ctx, state.ETag)- if kevErr == nil {- state.ETag = etag- }- epssErr := s.syncEPSSInternal(ctx)+ etag, kevErr := s.syncKEV(ctx, state.ETag)+ epssErr := s.syncEPSS(ctx)pruneErr := s.records.PruneThreatIntel(ctx)syncErr := errors.Join(kevErr, epssErr, pruneErr)- if syncErr == nil {- state.LastSyncedAt = new(time.Now())- state.LastError = ""- } else {- state.LastError = syncErr.Error()+ raw, err := json.Marshal(threatIntelState{+ LastSyncedAt: kit.Ternary(syncErr == nil, new(time.Now()), state.LastSyncedAt),+ ETag: kit.Ternary(kevErr == nil, etag, state.ETag),+ LastError: func() string {+ if syncErr == nil {+ return ""+ }+ return syncErr.Error()+ }(),+ })+ if err == nil {+ err = s.kvService.Set(ctx, threatIntelStateKey, string(raw))}- if s.kvService != nil {- raw, err := json.Marshal(state)- if err == nil {- err = s.kvService.Set(ctx, threatIntelStateKey, string(raw))- }- if err != nil {- slog.WarnContext(ctx, "failed to store threat intel state", "error", err)- }+ if err != nil {+ slog.WarnContext(ctx, "failed to store threat intel state", "error", err)}return syncErr}-// syncKEVInternal replaces the stored KEV flags with the current catalog and returns the feed ETag.-func (s *Service) syncKEVInternal(ctx context.Context, etag string) (string, error) {+// syncKEV replaces the stored KEV flags with the current catalog and returns the feed ETag.+func (s *Service) syncKEV(ctx context.Context, etag string) (string, error) {req, err := http.NewRequestWithContext(ctx, http.MethodGet, kevFeedURL, http.NoBody)if err != nil {return "", fmt.Errorf("failed to build KEV request: %w", err)@@ -536,8 +497,8 @@ func (s *Service) syncKEVInternal(ctx context.Context, etag string) (string, errreturn resp.Header.Get("ETag"), nil}-// syncEPSSInternal fetches EPSS scores for scanned CVEs that were never checked or were checked over 20h ago.-func (s *Service) syncEPSSInternal(ctx context.Context) error {+// syncEPSS fetches EPSS scores for scanned CVEs that were never checked or were checked over 20h ago.+func (s *Service) syncEPSS(ctx context.Context) error {cveIDs, listErr := s.records.StaleEPSSCVEs(ctx, time.Now().Add(-threatIntelSyncInterval), epssMaxCVEsPerSync)if listErr != nil {return fmt.Errorf("failed to list CVEs for EPSS: %w", listErr)@@ -549,7 +510,7 @@ func (s *Service) syncEPSSInternal(ctx context.Context) error {chars += len(cveIDs[end]) + 1end++}- if err := s.fetchEPSSBatchInternal(ctx, cveIDs[start:end]); err != nil {+ if err := s.fetchEPSSBatch(ctx, cveIDs[start:end]); err != nil {return err}start = end@@ -564,7 +525,7 @@ func (s *Service) syncEPSSInternal(ctx context.Context) error {return nil}-func (s *Service) fetchEPSSBatchInternal(ctx context.Context, cveIDs []string) error {+func (s *Service) fetchEPSSBatch(ctx context.Context, cveIDs []string) error {var resp *http.Responsefor attempt := 0; ; attempt++ {req, err := http.NewRequestWithContext(ctx, http.MethodGet, epssAPIURL+"?cve="+strings.Join(cveIDs, ","), http.NoBody)@@ -608,26 +569,27 @@ func (s *Service) fetchEPSSBatchInternal(ctx context.Context, cveIDs []string) e}now := time.Now()- rows := make(map[string]*vulnerability.ThreatIntelEntry, len(cveIDs))- for _, id := range cveIDs {- rows[strings.ToUpper(id)] = &vulnerability.ThreatIntelEntry{CVEID: id, EPSSCheckedAt: &now, UpdatedAt: &now}- }+ scores := make(map[string][2]*float64, len(payload.Data))for _, entry := range payload.Data {- row, ok := rows[strings.ToUpper(entry.CVE)]- if !ok {- continue- }- if score, err := strconv.ParseFloat(entry.EPSS, 64); err == nil {- row.EPSSScore = &score- }- if percentile, err := strconv.ParseFloat(entry.Percentile, 64); err == nil {- row.EPSSPercentile = &percentile+ score, scoreErr := strconv.ParseFloat(entry.EPSS, 64)+ percentile, percentileErr := strconv.ParseFloat(entry.Percentile, 64)+ previous := scores[strings.ToUpper(entry.CVE)]+ scores[strings.ToUpper(entry.CVE)] = [2]*float64{+ kit.Ternary(scoreErr == nil, &score, previous[0]),+ kit.Ternary(percentileErr == nil, &percentile, previous[1]),}}- batch := make([]vulnerability.ThreatIntelEntry, 0, len(rows))- for _, row := range rows {- batch = append(batch, *row)+ rows := make(map[string]vulnerability.ThreatIntelEntry, len(cveIDs))+ for _, id := range cveIDs {+ rows[strings.ToUpper(id)] = vulnerability.ThreatIntelEntry{+ CVEID: id,+ EPSSScore: scores[strings.ToUpper(id)][0],+ EPSSPercentile: scores[strings.ToUpper(id)][1],+ EPSSCheckedAt: &now,+ UpdatedAt: &now,+ }}+ batch := slices.Collect(maps.Values(rows))if err := s.records.SaveEPSS(ctx, batch); err != nil {return fmt.Errorf("failed to store EPSS scores: %w", err)}@@ -636,12 +598,13 @@ func (s *Service) fetchEPSSBatchInternal(ctx context.Context, cveIDs []string) e// ThreatIntelStatus reports whether threat intelligence is enabled and when it last synced.func (s *Service) ThreatIntelStatus(ctx context.Context) vulnerability.ThreatIntelStatus {- status := vulnerability.ThreatIntelStatus{Enabled: s.settingsService.GetBoolSetting(ctx, threatIntelSettingKey, true)}- if !status.Enabled {- return status- }- state := s.loadThreatIntelStateInternal(ctx)- status.LastSyncedAt = state.LastSyncedAt- status.Stale = state.LastSyncedAt == nil || time.Since(*state.LastSyncedAt) > threatIntelStaleAfter- return status+ if !s.settingsService.GetBoolSetting(ctx, threatIntelSettingKey, true) {+ return vulnerability.ThreatIntelStatus{}+ }+ state := s.loadThreatIntelState(ctx)+ return vulnerability.ThreatIntelStatus{+ Enabled: true,+ LastSyncedAt: state.LastSyncedAt,+ Stale: time.Since(kit.FromPtr(state.LastSyncedAt)) > threatIntelStaleAfter,+ }}diff --git a/backend/internal/vulnerability/service.go b/backend/internal/vulnerability/service.goindex fc66dfe25..34d0eac4d 100644--- a/backend/internal/vulnerability/service.go+++ b/backend/internal/vulnerability/service.go@@ -1,6 +1,7 @@package vulnerabilityimport (+ "cmp""context""encoding/json/v2""errors"@@ -16,7 +17,7 @@ import (usertypes "github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/vulnerability""github.com/italypaleale/francis/builtin/workflow"- dockerregistry "github.com/moby/moby/api/types/registry"+ "github.com/samber/mo"kit "go.getarcane.app/kit/pkg""gorm.io/gorm""gorm.io/gorm/clause"@@ -37,6 +38,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/store"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity""github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/vuln"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/pagination""github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow""github.com/getarcaneapp/arcane/backend/v2/pkg/utils/dbutil")@@ -80,207 +82,229 @@ func NewVulnerabilityService(kvService *kv.KVService,httpClient *http.Client,) *VulnerabilityService {- s := &VulnerabilityService{db: db, settingsService: settingsService}-- var registryAuthConfigs func(context.Context) (map[string]dockerregistry.AuthConfig, error)- if registryService != nil {- registryAuthConfigs = registryService.GetAllRegistryAuthConfigs- }-- s.store = store.NewService(db, store.Records{- Save: s.saveRecord,- SaveStatus: func(ctx context.Context, result *vulnerability.ScanResult, summary *vulnerability.SeveritySummary, errMessage string) error {- updates := map[string]any{- "image_name": result.ImageName,- "status": string(result.Status),- "scan_time": result.ScanTime,- "duration": result.Duration,- "critical_count": summary.Critical,- "high_count": summary.High,- "medium_count": summary.Medium,- "low_count": summary.Low,- "unknown_count": summary.Unknown,- "total_count": summary.Total,- "scanner_version": result.ScannerVersion,- "error": nil,- }- if errMessage != "" {- updates["error"] = errMessage- }- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", result.ImageID).Updates(updates).Error- },- Delete: func(ctx context.Context, imageIDs []string) error {- if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {- return err- }- s.query.InvalidateRiskOverview()- return nil- },- DeleteReports: func(ctx context.Context, imageIDs []string) error {- return s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityReportRecord{}).Error- },- IDsExcept: func(ctx context.Context, keepIDs []string) ([]string, error) {- scope := s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{})- if len(keepIDs) > 0 {- scope = scope.Where("id NOT IN ?", keepIDs)- }- var ids []string- err := scope.Pluck("id", &ids).Error- return ids, err- },- DeleteExcept: func(ctx context.Context, keepIDs []string) (int64, error) {- scope := s.db.WithContext(ctx).Where("1 = 1")- if len(keepIDs) > 0 {- scope = s.db.WithContext(ctx).Where("id NOT IN ?", keepIDs)- }- result := scope.Delete(&VulnerabilityScanRecord{})- if result.Error != nil {- return 0, result.Error- }- return result.RowsAffected, nil- },- EachMissingItems: s.eachRecordMissingItems,- FixableCounts: func(ctx context.Context) (map[string]*int, error) {- var scans []VulnerabilityScanRecord- if err := s.db.WithContext(ctx).Select("id", "fixable_count").Find(&scans).Error; err != nil {- return nil, err- }- counts := make(map[string]*int, len(scans))- for i := range scans {- counts[scans[i].ID] = scans[i].FixableCount- }- return counts, nil- },- SaveReport: s.saveReport,- SetFixableCount: func(ctx context.Context, imageID string, count int) error {- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("fixable_count", count).Error- },- DeleteItems: func(tx *gorm.DB, imageIDs []string) error {- return tx.Where("image_id IN ?", imageIDs).Delete(&VulnerabilityScanItem{}).Error- },- SaveItems: saveItems,- CountKnownExploited: func(ctx context.Context) (int64, error) {- var count int64- err := s.db.WithContext(ctx).Model(&VulnerabilityThreatIntel{}).Where("known_exploited = ?", true).Count(&count).Error- return count, err- },- ReplaceKnownExploited: s.replaceKnownExploited,- StaleEPSSCVEs: func(ctx context.Context, checkedBefore time.Time, limit int) ([]string, error) {- var cveIDs []string- err := s.db.WithContext(ctx).- Model(&VulnerabilityScanItem{}).- Distinct("vulnerability_scan_items.vulnerability_id").- Joins("LEFT JOIN vulnerability_threat_intel ON vulnerability_threat_intel.cve_id = vulnerability_scan_items.vulnerability_id").- Where("vulnerability_scan_items.vulnerability_id LIKE ?", "CVE-%").- Where("vulnerability_threat_intel.epss_checked_at IS NULL OR vulnerability_threat_intel.epss_checked_at < ?", checkedBefore).- Limit(limit).- Pluck("vulnerability_scan_items.vulnerability_id", &cveIDs).Error- return cveIDs, err- },- SaveEPSS: func(ctx context.Context, entries []vulnerability.ThreatIntelEntry) error {- batch := make([]VulnerabilityThreatIntel, 0, len(entries))- for i := range entries {- batch = append(batch, VulnerabilityThreatIntel(entries[i]))- }- return s.db.WithContext(ctx).Clauses(clause.OnConflict{- Columns: []clause.Column{{Name: "cve_id"}},- DoUpdates: clause.AssignmentColumns([]string{"epss_score", "epss_percentile", "epss_checked_at", "updated_at"}),- }).Create(&batch).Error- },- PruneThreatIntel: func(ctx context.Context) error {- return s.db.WithContext(ctx).- Where("known_exploited = ? AND NOT EXISTS (SELECT 1 FROM vulnerability_scan_items WHERE vulnerability_scan_items.vulnerability_id = vulnerability_threat_intel.cve_id)", false).- Delete(&VulnerabilityThreatIntel{}).Error- },- }, dockerService, settingsService, kvService, httpClient)- s.query = query.NewService(db, cfg, dockerService, settingsService, s.loadResult, func(ctx context.Context, summary *vulnerability.ScanSummary) {- s.scan.RefreshSummary(ctx, summary)- }, s.store.ThreatIntelStatus, func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error) {- return s.ignores.FilterForImage(ctx, envID, imageID, vulns)- })- s.ignores = ignores.NewService(db, ignores.Records{- ForImage: func(ctx context.Context, envID, imageID string) ([]vulnerability.IgnoredVulnerability, error) {- var ignoreRecords []VulnerabilityIgnore- if err := s.db.WithContext(ctx).Where("environment_id = ? AND image_id = ?", envID, imageID).Find(&ignoreRecords).Error; err != nil {- return nil, err- }- return ignoredVulnerabilities(ignoreRecords), nil- },- Exists: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) (bool, error) {- var existing VulnerabilityIgnore- err := s.db.WithContext(ctx).Where(- ignoreIdentityQuery,- ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,- ).First(&existing).Error- if errors.Is(err, gorm.ErrRecordNotFound) {- return false, nil- }- return err == nil, err- },- Get: s.getIgnore,- Create: func(tx *gorm.DB, entries []vulnerability.IgnoredVulnerability, batchSize int) ([]vulnerability.IgnoredVulnerability, error) {- records := make([]VulnerabilityIgnore, len(entries))- for i := range entries {- records[i] = VulnerabilityIgnore(entries[i])- }- if err := tx.CreateInBatches(&records, batchSize).Error; err != nil {- return nil, err- }- return ignoredVulnerabilities(records), nil- },- DeleteIdentity: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) error {- return s.db.WithContext(ctx).Where(- ignoreIdentityQuery,- ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,- ).Delete(&VulnerabilityIgnore{}).Error- },- DeleteIDs: func(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error) {- result := tx.Where("environment_id = ? AND id IN ?", envID, ignoreIDs).Delete(&VulnerabilityIgnore{})- return result.RowsAffected, result.Error- },- Page: s.ignorePage,- Changed: s.query.InvalidateRiskOverview,- }, s.query.ActiveItemsQuery)- s.scan = scan.NewService(scan.Dependencies{- DB: db,- Records: scan.Records{- MarkScanning: func(ctx context.Context, imageID string) error {- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("status", ScanStatusScanning).Error+ var s *VulnerabilityService+ s = &VulnerabilityService{+ db: db,+ settingsService: settingsService,+ store: store.NewService(db, store.Records{+ Save: func(tx *gorm.DB, result *vulnerability.ScanResult, fixable int) error {+ return s.saveRecord(tx, result, fixable)},- MarkFailed: func(ctx context.Context, imageID, message string, duration int64) error {- if s.db == nil {- return nil+ SaveStatus: func(ctx context.Context, result *vulnerability.ScanResult, summary *vulnerability.SeveritySummary, errMessage string) error {+ updates := map[string]any{+ "image_name": result.ImageName,+ "status": string(result.Status),+ "scan_time": result.ScanTime,+ "duration": result.Duration,+ "critical_count": summary.Critical,+ "high_count": summary.High,+ "medium_count": summary.Medium,+ "low_count": summary.Low,+ "unknown_count": summary.Unknown,+ "total_count": summary.Total,+ "scanner_version": result.ScannerVersion,+ "error": nil,}- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Updates(map[string]any{- "status": ScanStatusFailed,- "error": message,- "duration": duration,- }).Error+ if errMessage != "" {+ updates["error"] = errMessage+ }+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", result.ImageID).Updates(updates).Error},- SetImageName: func(ctx context.Context, imageID, imageName string) error {- return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("image_name", imageName).Error+ Delete: func(ctx context.Context, imageIDs []string) error {+ if err := s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error; err != nil {+ return err+ }+ s.query.InvalidateRiskOverview()+ return nil},- },- Config: cfg,- KV: kvService,- RegistryAuthConfigs: registryAuthConfigs,- SaveReport: s.saveReport,- Docker: dockerService,- Settings: settingsService,- Events: eventService,- Notifications: notificationService,- Activity: activityService,- // The risk cache is cleared only once the scan's transaction has committed.- SaveResult: func(ctx context.Context, result *vulnerability.ScanResult) error {- if err := s.store.SaveResult(ctx, result); err != nil {- return err- }- s.query.InvalidateRiskOverview()- return nil- },- EnsureSummary: store.EnsureSummary,- FilterIgnored: s.ignores.FilterForImage,- })+ DeleteReports: func(ctx context.Context, imageIDs []string) error {+ return s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityReportRecord{}).Error+ },+ IDsExcept: func(ctx context.Context, keepIDs []string) ([]string, error) {+ scope := s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{})+ if len(keepIDs) > 0 {+ scope = scope.Where("id NOT IN ?", keepIDs)+ }+ var ids []string+ err := scope.Pluck("id", &ids).Error+ return ids, err+ },+ DeleteExcept: func(ctx context.Context, keepIDs []string) (int64, error) {+ scope := s.db.WithContext(ctx).Where("1 = 1")+ if len(keepIDs) > 0 {+ scope = s.db.WithContext(ctx).Where("id NOT IN ?", keepIDs)+ }+ result := scope.Delete(&VulnerabilityScanRecord{})+ if result.Error != nil {+ return 0, result.Error+ }+ return result.RowsAffected, nil+ },+ EachMissingItems: func(ctx context.Context, batchSize int, fn func(imageID, vulnerabilitiesJSON string)) error {+ return s.eachRecordMissingItems(ctx, batchSize, fn)+ },+ FixableCounts: func(ctx context.Context) (map[string]*int, error) {+ var scans []VulnerabilityScanRecord+ if err := s.db.WithContext(ctx).Select("id", "fixable_count").Find(&scans).Error; err != nil {+ return nil, err+ }+ counts := make(map[string]*int, len(scans))+ for i := range scans {+ counts[scans[i].ID] = scans[i].FixableCount+ }+ return counts, nil+ },+ SaveReport: func(ctx context.Context, imageID, data string) error {+ return s.saveReport(ctx, imageID, data)+ },+ SetFixableCount: func(ctx context.Context, imageID string, count int) error {+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("fixable_count", count).Error+ },+ DeleteItems: func(tx *gorm.DB, imageIDs []string) error {+ return tx.Where("image_id IN ?", imageIDs).Delete(&VulnerabilityScanItem{}).Error+ },+ SaveItems: saveItems,+ CountKnownExploited: func(ctx context.Context) (int64, error) {+ var count int64+ err := s.db.WithContext(ctx).Model(&VulnerabilityThreatIntel{}).Where("known_exploited = ?", true).Count(&count).Error+ return count, err+ },+ ReplaceKnownExploited: func(ctx context.Context, entries []vulnerability.ThreatIntelEntry, now time.Time) error {+ return s.replaceKnownExploited(ctx, entries, now)+ },+ StaleEPSSCVEs: func(ctx context.Context, checkedBefore time.Time, limit int) ([]string, error) {+ var cveIDs []string+ err := s.db.WithContext(ctx).+ Model(&VulnerabilityScanItem{}).+ Distinct("vulnerability_scan_items.vulnerability_id").+ Joins("LEFT JOIN vulnerability_threat_intel ON vulnerability_threat_intel.cve_id = vulnerability_scan_items.vulnerability_id").+ Where("vulnerability_scan_items.vulnerability_id LIKE ?", "CVE-%").+ Where("vulnerability_threat_intel.epss_checked_at IS NULL OR vulnerability_threat_intel.epss_checked_at < ?", checkedBefore).+ Limit(limit).+ Pluck("vulnerability_scan_items.vulnerability_id", &cveIDs).Error+ return cveIDs, err+ },+ SaveEPSS: func(ctx context.Context, entries []vulnerability.ThreatIntelEntry) error {+ batch := make([]VulnerabilityThreatIntel, 0, len(entries))+ for i := range entries {+ batch = append(batch, VulnerabilityThreatIntel(entries[i]))+ }+ return s.db.WithContext(ctx).Clauses(clause.OnConflict{+ Columns: []clause.Column{{Name: "cve_id"}},+ DoUpdates: clause.AssignmentColumns([]string{"epss_score", "epss_percentile", "epss_checked_at", "updated_at"}),+ }).Create(&batch).Error+ },+ PruneThreatIntel: func(ctx context.Context) error {+ return s.db.WithContext(ctx).+ Where("known_exploited = ? AND NOT EXISTS (SELECT 1 FROM vulnerability_scan_items WHERE vulnerability_scan_items.vulnerability_id = vulnerability_threat_intel.cve_id)", false).+ Delete(&VulnerabilityThreatIntel{}).Error+ },+ }, dockerService, settingsService, kvService, httpClient),+ query: query.NewService(+ db,+ cfg,+ dockerService,+ settingsService,+ func(ctx context.Context, imageID string) (*vulnerability.ScanResult, error) {+ return s.loadResult(ctx, imageID)+ },+ func(ctx context.Context, summary *vulnerability.ScanSummary) { s.scan.RefreshSummary(ctx, summary) },+ func(ctx context.Context) vulnerability.ThreatIntelStatus { return s.store.ThreatIntelStatus(ctx) },+ func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error) {+ return s.ignores.FilterForImage(ctx, envID, imageID, vulns)+ },+ ),+ ignores: ignores.NewService(db, ignores.Records{+ ForImage: func(ctx context.Context, envID, imageID string) ([]vulnerability.IgnoredVulnerability, error) {+ var ignoreRecords []VulnerabilityIgnore+ if err := s.db.WithContext(ctx).Where("environment_id = ? AND image_id = ?", envID, imageID).Find(&ignoreRecords).Error; err != nil {+ return nil, err+ }+ return ignoredVulnerabilities(ignoreRecords), nil+ },+ Exists: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) (bool, error) {+ var existing VulnerabilityIgnore+ err := s.db.WithContext(ctx).Where(+ ignoreIdentityQuery,+ ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,+ ).First(&existing).Error+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return false, nil+ }+ return err == nil, err+ },+ Get: func(ctx context.Context, envID, ignoreID string) (*vulnerability.IgnoredVulnerability, bool, error) {+ return s.getIgnore(ctx, envID, ignoreID)+ },+ Create: func(tx *gorm.DB, entries []vulnerability.IgnoredVulnerability, batchSize int) ([]vulnerability.IgnoredVulnerability, error) {+ records := make([]VulnerabilityIgnore, len(entries))+ for i := range entries {+ records[i] = VulnerabilityIgnore(entries[i])+ }+ if err := tx.CreateInBatches(&records, batchSize).Error; err != nil {+ return nil, err+ }+ return ignoredVulnerabilities(records), nil+ },+ DeleteIdentity: func(ctx context.Context, ignore *vulnerability.IgnoredVulnerability) error {+ return s.db.WithContext(ctx).Where(+ ignoreIdentityQuery,+ ignore.EnvironmentID, ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion,+ ).Delete(&VulnerabilityIgnore{}).Error+ },+ DeleteIDs: func(tx *gorm.DB, envID string, ignoreIDs []string) (int64, error) {+ result := tx.Where("environment_id = ? AND id IN ?", envID, ignoreIDs).Delete(&VulnerabilityIgnore{})+ return result.RowsAffected, result.Error+ },+ Page: func(ctx context.Context, envID string, order func(*gorm.DB) *gorm.DB, offset, limit int) ([]vulnerability.IgnoredVulnerability, int64, error) {+ return s.ignorePage(ctx, envID, order, offset, limit)+ },+ Changed: func() { s.query.InvalidateRiskOverview() },+ }, func(ctx context.Context, db *gorm.DB, envID string, params *pagination.QueryParams) *gorm.DB {+ return s.query.ActiveItemsQuery(ctx, db, envID, params)+ }),+ scan: scan.NewService(scan.Dependencies{+ DB: db,+ Records: scan.Records{+ MarkScanning: func(ctx context.Context, imageID string) error {+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("status", ScanStatusScanning).Error+ },+ MarkFailed: func(ctx context.Context, imageID, message string, duration int64) error {+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Updates(map[string]any{+ "status": ScanStatusFailed,+ "error": message,+ "duration": duration,+ }).Error+ },+ SetImageName: func(ctx context.Context, imageID, imageName string) error {+ return s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID).Update("image_name", imageName).Error+ },+ },+ Config: cfg,+ KV: kvService,+ RegistryAuthConfigs: registryService.GetAllRegistryAuthConfigs,+ SaveReport: func(ctx context.Context, imageID, data string) error {+ return s.saveReport(ctx, imageID, data)+ },+ Docker: dockerService,+ Settings: settingsService,+ Events: eventService,+ Notifications: notificationService,+ Activity: activityService,+ // The risk cache is cleared only once the scan's transaction has committed.+ SaveResult: func(ctx context.Context, result *vulnerability.ScanResult) error {+ if err := s.store.SaveResult(ctx, result); err != nil {+ return err+ }+ s.query.InvalidateRiskOverview()+ return nil+ },+ EnsureSummary: store.EnsureSummary,+ FilterIgnored: func(ctx context.Context, envID, imageID string, vulns []vulnerability.Vulnerability) ([]vulnerability.Vulnerability, error) {+ return s.ignores.FilterForImage(ctx, envID, imageID, vulns)+ },+ }),+ }return s}@@ -358,13 +382,18 @@ func (s *VulnerabilityService) RegisterWorkflows(engine *flow.Engine) error {if err != nil {return nil, err}- outcome := scheduler.Outcome{Status: scheduler.Succeeded}scanned, skipped := 0, 0+ var failed []scheduler.TargetOutcomefixables := make([]scan.Fixable, 0, len(slots))for index, slot := range slots {switch {case slot.Err != "":- outcome.Targets = append(outcome.Targets, scheduler.TargetOutcome{ResourceType: "image", ID: targets[index].ImageName, Status: scheduler.Failed, Message: slot.Err})+ failed = append(failed, scheduler.TargetOutcome{+ ResourceType: "image",+ ID: targets[index].ImageName,+ Status: scheduler.Failed,+ Message: slot.Err,+ })case slot.Value.Skipped:skipped++default:@@ -372,19 +401,22 @@ func (s *VulnerabilityService) RegisterWorkflows(engine *flow.Engine) error {fixables = append(fixables, slot.Value.Fixable)}}- outcome.Message = fmt.Sprintf("Vulnerability scan completed: %d scanned, %d failed", scanned, len(outcome.Targets))- switch {- case skipped > 0 || (len(targets) == 0 && !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement)):- outcome.Status = kit.Ternary(scanned > 0, scheduler.Partial, scheduler.Skipped)- outcome.Message = fmt.Sprintf("Remaining scans stopped because feature %s was disabled", features.VulnerabilityManagement)- case len(outcome.Targets) > 0:- outcome.Status = scheduler.Partial- }if skipped == 0 {s.scan.NotifySummary(ctx, scanned, fixables)}- slog.InfoContext(ctx, "scheduled vulnerability scan completed", "scanned", scanned, "failed", len(outcome.Targets), "skipped", skipped)- return outcome, nil+ slog.InfoContext(ctx, "scheduled vulnerability scan completed", "scanned", scanned, "failed", len(failed), "skipped", skipped)+ if skipped > 0 || (len(targets) == 0 && !s.settingsService.IsFeatureEnabled(ctx, features.VulnerabilityManagement)) {+ return scheduler.Outcome{+ Status: kit.Ternary(scanned > 0, scheduler.Partial, scheduler.Skipped),+ Message: fmt.Sprintf("Remaining scans stopped because feature %s was disabled", features.VulnerabilityManagement),+ Targets: failed,+ }, nil+ }+ return scheduler.Outcome{+ Status: kit.Ternary(len(failed) > 0, scheduler.Partial, scheduler.Succeeded),+ Message: fmt.Sprintf("Vulnerability scan completed: %d scanned, %d failed", scanned, len(failed)),+ Targets: failed,+ }, nil}), workflow.WithInputFrom("discover")),},})@@ -454,67 +486,66 @@ func (s *VulnerabilityService) loadResult(ctx context.Context, imageID string) (return nil, fmt.Errorf("failed to get scan result: %w", err)}+ vulns := func() []vulnerability.Vulnerability {+ if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {+ return nil+ }+ var decoded []vulnerability.Vulnerability+ if decodeErr := json.Unmarshal([]byte(record.Vulnerabilities[0]), &decoded); decodeErr != nil {+ slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", decodeErr)+ return nil+ }+ return decoded+ }()+ var reportCount int64+ countReportsErr := s.db.WithContext(ctx).Model(&VulnerabilityReportRecord{}).Where("image_id = ?", record.ID).Count(&reportCount).Errorresult := &vulnerability.ScanResult{ImageID: record.ID,ImageName: record.ImageName,ScanTime: record.ScanTime,Status: vulnerability.ScanStatus(record.Status),- Duration: record.Duration,- Summary: &vulnerability.SeveritySummary{- Critical: record.CriticalCount,- High: record.HighCount,- Medium: record.MediumCount,- Low: record.LowCount,- Unknown: record.UnknownCount,- Total: record.TotalCount,- },- ScannerVersion: record.ScannerVersion,- }-- if record.Error != nil {- result.Error = *record.Error- }-- if len(record.Vulnerabilities) > 0 && record.Vulnerabilities[0] != "" {- var vulns []vulnerability.Vulnerability- if decodeErr := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); decodeErr != nil {- slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", decodeErr)- } else {- result.Vulnerabilities = vulns- }+ Summary: store.EnsureSummary(&vulnerability.ScanResult{+ Summary: &vulnerability.SeveritySummary{+ Critical: record.CriticalCount,+ High: record.HighCount,+ Medium: record.MediumCount,+ Low: record.LowCount,+ Unknown: record.UnknownCount,+ Total: record.TotalCount,+ },+ Vulnerabilities: vulns,+ }),+ Vulnerabilities: vulns,+ Error: kit.FromPtr(record.Error),+ Duration: record.Duration,+ ScannerVersion: record.ScannerVersion,+ HasReport: countReportsErr == nil && reportCount > 0,}-- store.EnsureSummary(result)s.scan.RefreshResult(ctx, result)- var reportCount int64- if countReportsErr := s.db.WithContext(ctx).Model(&VulnerabilityReportRecord{}).Where("image_id = ?", record.ID).Count(&reportCount).Error; countReportsErr == nil {- result.HasReport = reportCount > 0- }return result, nil}// saveRecord upserts the scan record of result inside tx.func (s *VulnerabilityService) saveRecord(tx *gorm.DB, result *vulnerability.ScanResult, fixable int) error {- var vulnJSON database.StringSlice- if len(result.Vulnerabilities) > 0 {- vulnBytes, err := json.Marshal(result.Vulnerabilities)- if err != nil {- return fmt.Errorf("failed to marshal vulnerabilities: %w", err)+ vulnJSON, err := func() (database.StringSlice, error) {+ if len(result.Vulnerabilities) == 0 {+ return nil, nil}- vulnJSON = database.StringSlice{string(vulnBytes)}- }-- var errPtr *string- if result.Error != "" {- errPtr = &result.Error- }-- summary := result.Summary- if summary == nil {- summary = &vulnerability.SeveritySummary{}+ vulnBytes, marshalErr := json.Marshal(result.Vulnerabilities)+ if marshalErr != nil {+ return nil, fmt.Errorf("failed to marshal vulnerabilities: %w", marshalErr)+ }+ return database.StringSlice{string(vulnBytes)}, nil+ }()+ if err != nil {+ return err}- record := VulnerabilityScanRecord{+ summary := cmp.Or(result.Summary, &vulnerability.SeveritySummary{})+ return tx.Clauses(clause.OnConflict{+ Columns: []clause.Column{{Name: "id"}},+ UpdateAll: true,+ }).Create(&VulnerabilityScanRecord{ID: result.ImageID,ImageName: result.ImageName,Status: string(result.Status),@@ -527,15 +558,11 @@ func (s *VulnerabilityService) saveRecord(tx *gorm.DB, result *vulnerability.ScaUnknownCount: summary.Unknown,TotalCount: summary.Total,Vulnerabilities: vulnJSON,- Error: errPtr,+ Error: mo.EmptyableToOption(result.Error).ToPointer(),ScannerVersion: result.ScannerVersion,ScanPolicy: s.scan.Policy(),FixableCount: &fixable,- }- return tx.Clauses(clause.OnConflict{- Columns: []clause.Column{{Name: "id"}},- UpdateAll: true,- }).Create(&record).Error+ }).Error}// eachRecordMissingItems visits completed scans with findings but no finding rows.@@ -559,14 +586,10 @@ func (s *VulnerabilityService) eachRecordMissingItems(ctx context.Context, batch// saveReport stores the raw Trivy report of an image, replacing any earlier one.func (s *VulnerabilityService) saveReport(ctx context.Context, imageID, data string) error {- if s.db == nil {- return errors.New("database unavailable")- }- record := VulnerabilityReportRecord{ImageID: imageID, Data: data}return s.db.WithContext(ctx).Clauses(clause.OnConflict{Columns: []clause.Column{{Name: "image_id"}},UpdateAll: true,- }).Create(&record).Error+ }).Create(&VulnerabilityReportRecord{ImageID: imageID, Data: data}).Error}const (@@ -578,7 +601,26 @@ func saveItems(tx *gorm.DB, imageID string, vulns []vulnerability.Vulnerability)items := make([]VulnerabilityScanItem, 0, len(vulns))for i := range vulns {v := &vulns[i]- item := VulnerabilityScanItem{+ details, err := func() (*string, error) {+ if v.Description == "" && len(v.References) == 0 && v.CVSS == nil && v.PublishedDate == nil && v.LastModifiedDate == nil {+ return nil, nil+ }+ raw, marshalErr := json.Marshal(vulnerabilityScanItemDetails{+ Description: v.Description,+ References: v.References,+ CVSS: v.CVSS,+ PublishedDate: v.PublishedDate,+ LastModifiedDate: v.LastModifiedDate,+ })+ if marshalErr != nil {+ return nil, fmt.Errorf("failed to marshal vulnerability details: %w", marshalErr)+ }+ return new(string(raw)), nil+ }()+ if err != nil {+ return err+ }+ items = append(items, VulnerabilityScanItem{ImageID: imageID,VulnerabilityID: v.VulnerabilityID,PkgName: v.PkgName,@@ -589,24 +631,14 @@ func saveItems(tx *gorm.DB, imageID string, vulns []vulnerability.Vulnerability)PkgClass: v.Class,PkgType: v.Type,Title: v.Title,- }- if score := vuln.CVSSScore(v.CVSS); score > 0 {- item.CVSSScore = new(score)- }- if v.Description != "" || len(v.References) > 0 || v.CVSS != nil || v.PublishedDate != nil || v.LastModifiedDate != nil {- raw, err := json.Marshal(vulnerabilityScanItemDetails{- Description: v.Description,- References: v.References,- CVSS: v.CVSS,- PublishedDate: v.PublishedDate,- LastModifiedDate: v.LastModifiedDate,- })- if err != nil {- return fmt.Errorf("failed to marshal vulnerability details: %w", err)- }- item.Details = new(string(raw))- }- items = append(items, item)+ CVSSScore: func() *float64 {+ if score := vuln.CVSSScore(v.CVSS); score > 0 {+ return &score+ }+ return nil+ }(),+ Details: details,+ })}if storeItemsErr := tx.CreateInBatches(items, scanItemInsertBatchSize).Error; storeItemsErr != nil {return fmt.Errorf("failed to store scan items: %w", storeItemsErr)diff --git a/backend/internal/vulnerability/service_test.go b/backend/internal/vulnerability/service_test.goindex 5817aaabc..1ecfc5b25 100644--- a/backend/internal/vulnerability/service_test.go+++ b/backend/internal/vulnerability/service_test.go@@ -17,6 +17,7 @@ import ("github.com/getarcaneapp/arcane/types/v2/user""github.com/getarcaneapp/arcane/types/v2/vulnerability""github.com/libtnb/sqlite"+ "github.com/moby/moby/client""github.com/stretchr/testify/assert""github.com/stretchr/testify/require""go.getarcane.app/kit/pkg"@@ -24,8 +25,12 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/common"+ "github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/docker"+ "github.com/getarcaneapp/arcane/backend/v2/internal/event""github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/internal/notification""github.com/getarcaneapp/arcane/backend/v2/internal/settings""github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability/children/query"activitylib "github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/activity"@@ -34,19 +39,23 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/flow/flowtest")-func setupVulnerabilityScanTestDB(t *testing.T) *database.DB {+func setupVulnerabilityScanTestDB(t *testing.T) (*database.DB, *settings.SettingsService) {t.Helper()dsn := fmt.Sprintf("file:vulnerability-scan-test-%d?mode=memory&cache=shared", time.Now().UnixNano())- db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})+ gdb, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})require.NoError(t, err)- require.NoError(t, db.AutoMigrate(&VulnerabilityScanRecord{}, &VulnerabilityIgnore{}, &VulnerabilityScanItem{}))- return &database.DB{DB: db}+ require.NoError(t, gdb.AutoMigrate(&VulnerabilityScanRecord{}, &VulnerabilityIgnore{}, &VulnerabilityScanItem{}, &settings.SettingVariable{}))+ db := &database.DB{DB: gdb}+ settingsService, err := settings.NewSettingsService(t.Context(), db)+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, settingsService.Stop(context.WithoutCancel(t.Context()))) })+ return db, settingsService}func TestSaveScanResult_FixableCountsOSPackagesOnly(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)result := &vulnerability.ScanResult{ImageID: "sha256:img-fixable",@@ -54,9 +63,17 @@ func TestSaveScanResult_FixableCountsOSPackagesOnly(t *testing.T) {Status: vulnerability.ScanStatusCompleted,ScanTime: time.Now(),Vulnerabilities: []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-1", Class: vulnerability.TrivyClassOSPackages, FixedVersion: "1.2.3"},+ {+ VulnerabilityID: "CVE-1",+ Class: vulnerability.TrivyClassOSPackages,+ FixedVersion: "1.2.3",+ },{VulnerabilityID: "CVE-2", Class: vulnerability.TrivyClassOSPackages},- {VulnerabilityID: "CVE-3", Class: vulnerability.TrivyClassLangPackages, FixedVersion: "2.0.0"},+ {+ VulnerabilityID: "CVE-3",+ Class: vulnerability.TrivyClassLangPackages,+ FixedVersion: "2.0.0",+ },},}require.NoError(t, svc.store.SaveResult(ctx, result))@@ -68,8 +85,8 @@ func TestSaveScanResult_FixableCountsOSPackagesOnly(t *testing.T) {}func TestVulnerabilityService_ListVulnerabilities_NoScanReturnsNotFound(t *testing.T) {- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)_, _, err := svc.query.ListVulnerabilities(t.Context(), "sha256:unknown", pagination.QueryParams{Limit: 20})@@ -78,13 +95,28 @@ func TestVulnerabilityService_ListVulnerabilities_NoScanReturnsNotFound(t *testifunc TestVulnerabilityService_DeleteScanResultsByImageIDs(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)records := []VulnerabilityScanRecord{- {ID: "sha256:img1", ImageName: "nginx:latest", Status: ScanStatusCompleted, ScanTime: time.Now()},- {ID: "sha256:img2", ImageName: "redis:7", Status: ScanStatusCompleted, ScanTime: time.Now()},- {ID: "sha256:img3", ImageName: "postgres:16", Status: ScanStatusCompleted, ScanTime: time.Now()},+ {+ ID: "sha256:img1",+ ImageName: "nginx:latest",+ Status: ScanStatusCompleted,+ ScanTime: time.Now(),+ },+ {+ ID: "sha256:img2",+ ImageName: "redis:7",+ Status: ScanStatusCompleted,+ ScanTime: time.Now(),+ },+ {+ ID: "sha256:img3",+ ImageName: "postgres:16",+ Status: ScanStatusCompleted,+ ScanTime: time.Now(),+ },}for i := range records {require.NoError(t, db.Create(&records[i]).Error)@@ -100,8 +132,8 @@ func TestVulnerabilityService_DeleteScanResultsByImageIDs(t *testing.T) {func TestVulnerabilityService_DeleteScanResultsByImageIDs_EmptyInput(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)require.NoError(t, svc.DeleteScanResultsByImageIDs(ctx, nil))require.NoError(t, svc.DeleteScanResultsByImageIDs(ctx, []string{}))@@ -109,8 +141,8 @@ func TestVulnerabilityService_DeleteScanResultsByImageIDs_EmptyInput(t *testing.func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)vulns := []vulnerability.Vulnerability{{@@ -147,7 +179,7 @@ func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *tesrequire.NoError(t, db.Create(&ignore).Error)items, page, err := svc.query.ListAllVulnerabilities(ctx, "env-1", pagination.QueryParams{- Start: 0, Limit: 20,+ Limit: 20,})require.NoError(t, err)require.Empty(t, items)@@ -157,17 +189,57 @@ func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *tesfunc TestVulnerabilityService_ListAllVulnerabilityImageOptions_SeverityFilter(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)now := time.Now()records := []VulnerabilityScanRecord{- {ID: "sha256:critical-a", ImageName: "zeta:latest", Status: ScanStatusCompleted, ScanTime: now, CriticalCount: 2, TotalCount: 2},- {ID: "sha256:high-a", ImageName: "alpha:latest", Status: ScanStatusCompleted, ScanTime: now, HighCount: 4, TotalCount: 4},- {ID: "sha256:critical-b", ImageName: "", Status: ScanStatusCompleted, ScanTime: now, CriticalCount: 1, TotalCount: 1},- {ID: "sha256:critical-c", ImageName: "zeta:latest", Status: ScanStatusCompleted, ScanTime: now, CriticalCount: 1, TotalCount: 1},- {ID: "sha256:failed", ImageName: "failed:latest", Status: ScanStatusFailed, ScanTime: now, CriticalCount: 9, TotalCount: 9},- {ID: "sha256:zero", ImageName: "zero:latest", Status: ScanStatusCompleted, ScanTime: now, CriticalCount: 1, TotalCount: 0},+ {+ ID: "sha256:critical-a",+ ImageName: "zeta:latest",+ Status: ScanStatusCompleted,+ ScanTime: now,+ CriticalCount: 2,+ TotalCount: 2,+ },+ {+ ID: "sha256:high-a",+ ImageName: "alpha:latest",+ Status: ScanStatusCompleted,+ ScanTime: now,+ HighCount: 4,+ TotalCount: 4,+ },+ {+ ID: "sha256:critical-b",+ Status: ScanStatusCompleted,+ ScanTime: now,+ CriticalCount: 1,+ TotalCount: 1,+ },+ {+ ID: "sha256:critical-c",+ ImageName: "zeta:latest",+ Status: ScanStatusCompleted,+ ScanTime: now,+ CriticalCount: 1,+ TotalCount: 1,+ },+ {+ ID: "sha256:failed",+ ImageName: "failed:latest",+ Status: ScanStatusFailed,+ ScanTime: now,+ CriticalCount: 9,+ TotalCount: 9,+ },+ {+ ID: "sha256:zero",+ ImageName: "zero:latest",+ Status: ScanStatusCompleted,+ ScanTime: now,+ CriticalCount: 1,+ },}for i := range records {@@ -181,14 +253,34 @@ func TestVulnerabilityService_ListAllVulnerabilityImageOptions_SeverityFilter(tfunc TestVulnerabilityService_ListAllVulnerabilityImageOptions_InvalidSeverityAndNoFilter(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)now := time.Now()records := []VulnerabilityScanRecord{- {ID: "sha256:one", ImageName: "beta:latest", Status: ScanStatusCompleted, ScanTime: now, HighCount: 1, TotalCount: 1},- {ID: "sha256:two", ImageName: "", Status: ScanStatusCompleted, ScanTime: now, MediumCount: 2, TotalCount: 2},- {ID: "sha256:three", ImageName: "alpha:latest", Status: ScanStatusCompleted, ScanTime: now, LowCount: 1, TotalCount: 1},+ {+ ID: "sha256:one",+ ImageName: "beta:latest",+ Status: ScanStatusCompleted,+ ScanTime: now,+ HighCount: 1,+ TotalCount: 1,+ },+ {+ ID: "sha256:two",+ Status: ScanStatusCompleted,+ ScanTime: now,+ MediumCount: 2,+ TotalCount: 2,+ },+ {+ ID: "sha256:three",+ ImageName: "alpha:latest",+ Status: ScanStatusCompleted,+ ScanTime: now,+ LowCount: 1,+ TotalCount: 1,+ },}for i := range records {@@ -209,24 +301,59 @@ func TestVulnerabilityService_ListAllVulnerabilityImageOptions_InvalidSeverityAn// produce the same count the old full-decode implementation did.func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)now := time.Now()vulnsA := []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-A1", PkgName: "openssl", InstalledVersion: "1.0.0", Severity: vulnerability.SeverityCritical},- {VulnerabilityID: "CVE-A2", PkgName: "zlib", InstalledVersion: "1.2.0", Severity: vulnerability.SeverityCritical},- {VulnerabilityID: "CVE-A3", PkgName: "curl", InstalledVersion: "8.0.0", Severity: vulnerability.SeverityHigh},- {VulnerabilityID: "CVE-A4", PkgName: "bash", InstalledVersion: "5.0.0", Severity: vulnerability.SeverityMedium},+ {+ VulnerabilityID: "CVE-A1",+ PkgName: "openssl",+ InstalledVersion: "1.0.0",+ Severity: vulnerability.SeverityCritical,+ },+ {+ VulnerabilityID: "CVE-A2",+ PkgName: "zlib",+ InstalledVersion: "1.2.0",+ Severity: vulnerability.SeverityCritical,+ },+ {+ VulnerabilityID: "CVE-A3",+ PkgName: "curl",+ InstalledVersion: "8.0.0",+ Severity: vulnerability.SeverityHigh,+ },+ {+ VulnerabilityID: "CVE-A4",+ PkgName: "bash",+ InstalledVersion: "5.0.0",+ Severity: vulnerability.SeverityMedium,+ },}rawVulnsA, err := json.Marshal(vulnsA)require.NoError(t, err)vulnsB := []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-B1", PkgName: "openssl", InstalledVersion: "1.1.0", Severity: vulnerability.SeverityCritical},- {VulnerabilityID: "CVE-B2", PkgName: "glibc", InstalledVersion: "2.38", Severity: vulnerability.SeverityHigh},- {VulnerabilityID: "CVE-B3", PkgName: "less", InstalledVersion: "600", Severity: vulnerability.SeverityMedium},+ {+ VulnerabilityID: "CVE-B1",+ PkgName: "openssl",+ InstalledVersion: "1.1.0",+ Severity: vulnerability.SeverityCritical,+ },+ {+ VulnerabilityID: "CVE-B2",+ PkgName: "glibc",+ InstalledVersion: "2.38",+ Severity: vulnerability.SeverityHigh,+ },+ {+ VulnerabilityID: "CVE-B3",+ PkgName: "less",+ InstalledVersion: "600",+ Severity: vulnerability.SeverityMedium,+ },}rawVulnsB, err := json.Marshal(vulnsB)require.NoError(t, err)@@ -262,7 +389,15 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {),},},- {ID: "sha256:img-failed", ImageName: "failed:latest", Status: ScanStatusFailed, ScanTime: now, CriticalCount: 9, HighCount: 9, TotalCount: 18},+ {+ ID: "sha256:img-failed",+ ImageName: "failed:latest",+ Status: ScanStatusFailed,+ ScanTime: now,+ CriticalCount: 9,+ HighCount: 9,+ TotalCount: 18,+ },}for i := range records {require.NoError(t, db.Create(&records[i]).Error)@@ -277,11 +412,35 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {ignores := []VulnerabilityIgnore{// Critical on img-b: removes one from the count.- {ID: "ig-1", EnvironmentID: "env-1", ImageID: "sha256:img-b", VulnerabilityID: "CVE-B1", PkgName: "openssl", InstalledVersion: "1.1.0", CreatedBy: "tester"},+ {+ ID: "ig-1",+ EnvironmentID: "env-1",+ ImageID: "sha256:img-b",+ VulnerabilityID: "CVE-B1",+ PkgName: "openssl",+ InstalledVersion: "1.1.0",+ CreatedBy: "tester",+ },// Medium on img-b: never counted, must not subtract anything.- {ID: "ig-2", EnvironmentID: "env-1", ImageID: "sha256:img-b", VulnerabilityID: "CVE-B3", PkgName: "less", InstalledVersion: "600", CreatedBy: "tester"},+ {+ ID: "ig-2",+ EnvironmentID: "env-1",+ ImageID: "sha256:img-b",+ VulnerabilityID: "CVE-B3",+ PkgName: "less",+ InstalledVersion: "600",+ CreatedBy: "tester",+ },// Stale ignore for a CVE no longer in img-b's blob: no effect.- {ID: "ig-3", EnvironmentID: "env-1", ImageID: "sha256:img-b", VulnerabilityID: "CVE-GONE", PkgName: "old", InstalledVersion: "0.1", CreatedBy: "tester"},+ {+ ID: "ig-3",+ EnvironmentID: "env-1",+ ImageID: "sha256:img-b",+ VulnerabilityID: "CVE-GONE",+ PkgName: "old",+ InstalledVersion: "0.1",+ CreatedBy: "tester",+ },}for i := range ignores {require.NoError(t, db.Create(&ignores[i]).Error)@@ -300,27 +459,65 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {// from the sorted matches.func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilters(t *testing.T) {ctx := t.Context()- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)vulnsA := []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-A1", PkgName: "openssl", InstalledVersion: "1.0.0", Severity: vulnerability.SeverityCritical},- {VulnerabilityID: "CVE-A2", PkgName: "zlib", InstalledVersion: "1.2.0", Severity: vulnerability.SeverityLow},+ {+ VulnerabilityID: "CVE-A1",+ PkgName: "openssl",+ InstalledVersion: "1.0.0",+ Severity: vulnerability.SeverityCritical,+ },+ {+ VulnerabilityID: "CVE-A2",+ PkgName: "zlib",+ InstalledVersion: "1.2.0",+ Severity: vulnerability.SeverityLow,+ },}rawVulnsA, err := json.Marshal(vulnsA)require.NoError(t, err)vulnsB := []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-B1", PkgName: "glibc", InstalledVersion: "2.38", Severity: vulnerability.SeverityHigh},- {VulnerabilityID: "CVE-B2", PkgName: "less", InstalledVersion: "600", Severity: vulnerability.SeverityMedium},+ {+ VulnerabilityID: "CVE-B1",+ PkgName: "glibc",+ InstalledVersion: "2.38",+ Severity: vulnerability.SeverityHigh,+ },+ {+ VulnerabilityID: "CVE-B2",+ PkgName: "less",+ InstalledVersion: "600",+ Severity: vulnerability.SeverityMedium,+ },}rawVulnsB, err := json.Marshal(vulnsB)require.NoError(t, err)now := time.Now()records := []VulnerabilityScanRecord{- {ID: "sha256:img-a", ImageName: "nginx:latest", Status: ScanStatusCompleted, ScanTime: now, CriticalCount: 1, LowCount: 1, TotalCount: 2, Vulnerabilities: database.StringSlice{string(rawVulnsA)}},- {ID: "sha256:img-b", ImageName: "redis:7", Status: ScanStatusCompleted, ScanTime: now, HighCount: 1, MediumCount: 1, TotalCount: 2, Vulnerabilities: database.StringSlice{string(rawVulnsB)}},+ {+ ID: "sha256:img-a",+ ImageName: "nginx:latest",+ Status: ScanStatusCompleted,+ ScanTime: now,+ CriticalCount: 1,+ LowCount: 1,+ TotalCount: 2,+ Vulnerabilities: database.StringSlice{string(rawVulnsA)},+ },+ {+ ID: "sha256:img-b",+ ImageName: "redis:7",+ Status: ScanStatusCompleted,+ ScanTime: now,+ HighCount: 1,+ MediumCount: 1,+ TotalCount: 2,+ Vulnerabilities: database.StringSlice{string(rawVulnsB)},+ },}for i := range records {require.NoError(t, db.Create(&records[i]).Error)@@ -330,16 +527,22 @@ func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFil// Ignore the medium CVE on img-b: it must vanish from GrandTotalItems.require.NoError(t, db.Create(&VulnerabilityIgnore{- ID: "ig-1", EnvironmentID: "env-1", ImageID: "sha256:img-b",- VulnerabilityID: "CVE-B2", PkgName: "less", InstalledVersion: "600", CreatedBy: "tester",+ ID: "ig-1",+ EnvironmentID: "env-1",+ ImageID: "sha256:img-b",+ VulnerabilityID: "CVE-B2",+ PkgName: "less",+ InstalledVersion: "600",+ CreatedBy: "tester",}).Error)// Severity filter critical,high with page size 1, sorted by severity desc:// matches are CVE-A1 (critical) and CVE-B1 (high); the first page holds// the critical one.items, page, err := svc.query.ListAllVulnerabilities(ctx, "env-1", pagination.QueryParams{- Start: 0, Limit: 1,- Sort: "severity", Order: pagination.SortDesc,+ Limit: 1,+ Sort: "severity",+ Order: pagination.SortDesc,Filters: map[string]string{"severity": "critical,high"},})require.NoError(t, err)@@ -350,7 +553,7 @@ func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFil// Search matches per-CVE fields decoded from the blobs.items, page, err = svc.query.ListAllVulnerabilities(ctx, "env-1", pagination.QueryParams{- Start: 0, Limit: 20,+ Limit: 20,Search: "glibc",})require.NoError(t, err)@@ -367,7 +570,7 @@ func riskTestService(t *testing.T, httpClient *http.Client) (*VulnerabilityServirequire.NoError(t, err)sqlDB.SetMaxOpenConns(1)require.NoError(t, db.AutoMigrate(&VulnerabilityThreatIntel{}, &query.VulnerabilityRiskSnapshot{}, &kv.KVEntry{}))- svc := NewVulnerabilityService(db, nil, nil, nil, settingsSvc, nil, nil, nil, kv.NewKVService(db), httpClient)+ svc := newTestService(t, db, settingsSvc, httpClient)return svc, settingsSvc, db}@@ -375,22 +578,70 @@ func seedRiskScans(t *testing.T, svc *VulnerabilityService) {t.Helper()scans := []*vulnerability.ScanResult{{- ImageID: "sha256:web", ImageName: "web:latest", Status: vulnerability.ScanStatusCompleted, ScanTime: time.Now(),+ ImageID: "sha256:web",+ ImageName: "web:latest",+ Status: vulnerability.ScanStatusCompleted,+ ScanTime: time.Now(),Vulnerabilities: []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-2024-0001", PkgName: "openssl", InstalledVersion: "3.0.0", FixedVersion: "3.0.1", Severity: vulnerability.SeverityHigh, CVSS: &vulnerability.CVSSInfo{V3Score: 7.5}},- {VulnerabilityID: "CVE-2024-0002", PkgName: "curl", InstalledVersion: "8.0.0", Severity: vulnerability.SeverityMedium, CVSS: &vulnerability.CVSSInfo{V3Score: 5.3}},- {VulnerabilityID: "CVE-2024-0003", PkgName: "zlib", InstalledVersion: "1.2.0", Severity: vulnerability.SeverityCritical, CVSS: &vulnerability.CVSSInfo{V3Score: 9.8}},+ {+ VulnerabilityID: "CVE-2024-0001",+ PkgName: "openssl",+ InstalledVersion: "3.0.0",+ FixedVersion: "3.0.1",+ Severity: vulnerability.SeverityHigh,+ CVSS: &vulnerability.CVSSInfo{V3Score: 7.5},+ },+ {+ VulnerabilityID: "CVE-2024-0002",+ PkgName: "curl",+ InstalledVersion: "8.0.0",+ Severity: vulnerability.SeverityMedium,+ CVSS: &vulnerability.CVSSInfo{V3Score: 5.3},+ },+ {+ VulnerabilityID: "CVE-2024-0003",+ PkgName: "zlib",+ InstalledVersion: "1.2.0",+ Severity: vulnerability.SeverityCritical,+ CVSS: &vulnerability.CVSSInfo{V3Score: 9.8},+ },},},{- ImageID: "sha256:db", ImageName: "db:16", Status: vulnerability.ScanStatusCompleted, ScanTime: time.Now(),+ ImageID: "sha256:db",+ ImageName: "db:16",+ Status: vulnerability.ScanStatusCompleted,+ ScanTime: time.Now(),Vulnerabilities: []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-2024-0002", PkgName: "curl", InstalledVersion: "8.0.0", FixedVersion: "8.1.0", Severity: vulnerability.SeverityMedium, CVSS: &vulnerability.CVSSInfo{V3Score: 5.3}},- {VulnerabilityID: "GHSA-xxxx", PkgName: "lodash", InstalledVersion: "4.0.0", Severity: vulnerability.SeverityLow},+ {+ VulnerabilityID: "CVE-2024-0002",+ PkgName: "curl",+ InstalledVersion: "8.0.0",+ FixedVersion: "8.1.0",+ Severity: vulnerability.SeverityMedium,+ CVSS: &vulnerability.CVSSInfo{V3Score: 5.3},+ },+ {+ VulnerabilityID: "GHSA-xxxx",+ PkgName: "lodash",+ InstalledVersion: "4.0.0",+ Severity: vulnerability.SeverityLow,+ },},},- {ImageID: "sha256:clean", ImageName: "clean:1", Status: vulnerability.ScanStatusCompleted, ScanTime: time.Now()},- {ImageID: "sha256:broken", ImageName: "broken:1", Status: vulnerability.ScanStatusFailed, ScanTime: time.Now(), Error: "scanner exploded"},+ {+ ImageID: "sha256:clean",+ ImageName: "clean:1",+ Status: vulnerability.ScanStatusCompleted,+ ScanTime: time.Now(),+ },+ {+ ImageID: "sha256:broken",+ ImageName: "broken:1",+ Status: vulnerability.ScanStatusFailed,+ ScanTime: time.Now(),+ Error: "scanner exploded",+ },}for _, scan := range scans {require.NoError(t, svc.store.SaveResult(t.Context(), scan))@@ -400,8 +651,20 @@ func seedRiskScans(t *testing.T, svc *VulnerabilityService) {func TestVulnerabilityService_GetRiskOverview(t *testing.T) {svc, _, db := riskTestService(t, nil)seedRiskScans(t, svc)- require.NoError(t, db.Create(&VulnerabilityThreatIntel{CVEID: "CVE-2024-0002", KnownExploited: true, KEVRansomware: true, KEVDueDate: "2024-06-01"}).Error)- require.NoError(t, db.Create(&VulnerabilityIgnore{ID: "ignore-1", EnvironmentID: "0", ImageID: "sha256:web", VulnerabilityID: "CVE-2024-0003", PkgName: "zlib", InstalledVersion: "1.2.0"}).Error)+ require.NoError(t, db.Create(&VulnerabilityThreatIntel{+ CVEID: "CVE-2024-0002",+ KnownExploited: true,+ KEVRansomware: true,+ KEVDueDate: "2024-06-01",+ }).Error)+ require.NoError(t, db.Create(&VulnerabilityIgnore{+ ID: "ignore-1",+ EnvironmentID: "0",+ ImageID: "sha256:web",+ VulnerabilityID: "CVE-2024-0003",+ PkgName: "zlib",+ InstalledVersion: "1.2.0",+ }).Error)overview, err := svc.query.GetRiskOverview(t.Context(), "0")require.NoError(t, err)@@ -490,9 +753,25 @@ func TestVulnerabilityService_RecordRiskSnapshot(t *testing.T) {seedRiskScans(t, svc)now := time.Now().UTC()require.NoError(t, db.Create(&[]query.VulnerabilityRiskSnapshot{- {SnapshotDate: now.AddDate(0, 0, -500).Format(time.DateOnly), ScoringVersion: vuln.ScoringVersion, RiskScore: 10, RiskBand: "low"},- {SnapshotDate: now.AddDate(0, 0, -7).Format(time.DateOnly), ScoringVersion: vuln.ScoringVersion, RiskScore: 60, RiskBand: "medium", KnownExploitedCount: 3},- {SnapshotDate: now.AddDate(0, 0, -3).Format(time.DateOnly), ScoringVersion: 1, RiskScore: 95, RiskBand: "critical"},+ {+ SnapshotDate: now.AddDate(0, 0, -500).Format(time.DateOnly),+ ScoringVersion: vuln.ScoringVersion,+ RiskScore: 10,+ RiskBand: "low",+ },+ {+ SnapshotDate: now.AddDate(0, 0, -7).Format(time.DateOnly),+ ScoringVersion: vuln.ScoringVersion,+ RiskScore: 60,+ RiskBand: "medium",+ KnownExploitedCount: 3,+ },+ {+ SnapshotDate: now.AddDate(0, 0, -3).Format(time.DateOnly),+ ScoringVersion: 1,+ RiskScore: 95,+ RiskBand: "critical",+ },}).Error)require.NoError(t, svc.RecordRiskSnapshot(t.Context()))@@ -576,9 +855,17 @@ func TestVulnerabilityService_SyncThreatIntel(t *testing.T) {vulns := make([]vulnerability.Vulnerability, 0, 151)for i := range 150 {- vulns = append(vulns, vulnerability.Vulnerability{VulnerabilityID: fmt.Sprintf("CVE-2024-%04d", i), PkgName: "pkg", Severity: vulnerability.SeverityMedium})+ vulns = append(vulns, vulnerability.Vulnerability{+ VulnerabilityID: fmt.Sprintf("CVE-2024-%04d", i),+ PkgName: "pkg",+ Severity: vulnerability.SeverityMedium,+ })}- vulns = append(vulns, vulnerability.Vulnerability{VulnerabilityID: "GHSA-abcd", PkgName: "pkg", Severity: vulnerability.SeverityLow})+ vulns = append(vulns, vulnerability.Vulnerability{+ VulnerabilityID: "GHSA-abcd",+ PkgName: "pkg",+ Severity: vulnerability.SeverityLow,+ })require.NoError(t,svc.store.SaveResult(@@ -592,7 +879,11 @@ func TestVulnerabilityService_SyncThreatIntel(t *testing.T) {},),)- require.NoError(t, db.Create(&VulnerabilityThreatIntel{CVEID: "CVE-2019-DELISTED", KnownExploited: true, KEVDueDate: "2020-01-01"}).Error)+ require.NoError(t, db.Create(&VulnerabilityThreatIntel{+ CVEID: "CVE-2019-DELISTED",+ KnownExploited: true,+ KEVDueDate: "2020-01-01",+ }).Error)require.NoError(t, svc.SyncThreatIntel(t.Context(), false))require.EqualValues(t, 1, kevRequests.Load())@@ -645,9 +936,29 @@ func seedBulkIgnoreFixture(t *testing.T, db *database.DB, svc *VulnerabilityServ)}redis := []vulnerability.Vulnerability{- {VulnerabilityID: "CVE-R-1", PkgName: "openssl", InstalledVersion: "3.0.0", FixedVersion: "3.0.1", Severity: vulnerability.SeverityCritical, Title: "openssl overflow"},- {VulnerabilityID: "CVE-R-2", PkgName: "zlib", InstalledVersion: "1.2.0", Severity: vulnerability.SeverityMedium, Title: "zlib bug"},- {VulnerabilityID: "CVE-R-3", PkgName: "curl", InstalledVersion: "8.0.0", FixedVersion: "8.0.1", Severity: vulnerability.SeverityHigh, Title: "curl issue"},+ {+ VulnerabilityID: "CVE-R-1",+ PkgName: "openssl",+ InstalledVersion: "3.0.0",+ FixedVersion: "3.0.1",+ Severity: vulnerability.SeverityCritical,+ Title: "openssl overflow",+ },+ {+ VulnerabilityID: "CVE-R-2",+ PkgName: "zlib",+ InstalledVersion: "1.2.0",+ Severity: vulnerability.SeverityMedium,+ Title: "zlib bug",+ },+ {+ VulnerabilityID: "CVE-R-3",+ PkgName: "curl",+ InstalledVersion: "8.0.0",+ FixedVersion: "8.0.1",+ Severity: vulnerability.SeverityHigh,+ Title: "curl issue",+ },}for _, image := range []struct {id, name string@@ -685,15 +996,40 @@ func TestBulkIgnoreVulnerabilities_AllMatchingHonorsFilters(t *testing.T) {expected int}{{name: "no filters covers every page", expected: 48},- {name: "severity", filters: vulnerability.BulkVulnerabilityFilters{Severity: "critical,medium"}, expected: 2},- {name: "image name", filters: vulnerability.BulkVulnerabilityFilters{ImageName: "redis"}, expected: 3},- {name: "fix available", filters: vulnerability.BulkVulnerabilityFilters{FixAvailable: true}, expected: 25},- {name: "search", filters: vulnerability.BulkVulnerabilityFilters{Search: "openssl"}, expected: 1},- {name: "combined", filters: vulnerability.BulkVulnerabilityFilters{Severity: "high", ImageName: "nginx", FixAvailable: true, Search: "nginx finding"}, expected: 8},+ {+ name: "severity",+ filters: vulnerability.BulkVulnerabilityFilters{Severity: "critical,medium"},+ expected: 2,+ },+ {+ name: "image name",+ filters: vulnerability.BulkVulnerabilityFilters{ImageName: "redis"},+ expected: 3,+ },+ {+ name: "fix available",+ filters: vulnerability.BulkVulnerabilityFilters{FixAvailable: true},+ expected: 25,+ },+ {+ name: "search",+ filters: vulnerability.BulkVulnerabilityFilters{Search: "openssl"},+ expected: 1,+ },+ {+ name: "combined",+ filters: vulnerability.BulkVulnerabilityFilters{+ Severity: "high",+ ImageName: "nginx",+ FixAvailable: true,+ Search: "nginx finding",+ },+ expected: 8,+ },} {t.Run(tc.name, func(t *testing.T) {- db := setupVulnerabilityScanTestDB(t)- svc := newTestService(db, nil)+ db, settingsSvc := setupVulnerabilityScanTestDB(t)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)seedBulkIgnoreFixture(t, db, svc)before, err := svc.ActionableCountExcludingIgnored(ctx)@@ -751,8 +1087,13 @@ func featureSettings(t *testing.T) (*settings.SettingsService, *database.DB) {func TestVulnerabilityFeatureDisabledRejectsScansAndRetainsReports(t *testing.T) {settingsSvc, db := featureSettings(t)- svc := newTestService(db, settingsSvc)- require.NoError(t, db.Create(&VulnerabilityScanRecord{ID: "test-image", ImageName: "test:latest", Status: ScanStatusCompleted, ScanTime: time.Now()}).Error)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)+ require.NoError(t, db.Create(&VulnerabilityScanRecord{+ ID: "test-image",+ ImageName: "test:latest",+ Status: ScanStatusCompleted,+ ScanTime: time.Now(),+ }).Error)require.NoError(t, db.Create(&VulnerabilityReportRecord{ImageID: "test-image", Data: "stored report"}).Error)require.NoError(t, settingsSvc.SetBoolSetting(t.Context(), features.VulnerabilityManagementSettingKey, false))@@ -777,15 +1118,31 @@ func TestVulnerabilityFeatureDisabledRejectsScansAndRetainsReports(t *testing.T)}// newTestService wires a VulnerabilityService and its children over db.-func newTestService(db *database.DB, settingsService *settings.SettingsService) *VulnerabilityService {- return NewVulnerabilityService(db, nil, nil, nil, settingsService, nil, nil, nil, nil, nil)+func newTestService(t *testing.T, db *database.DB, settingsService *settings.SettingsService, httpClient *http.Client) *VulnerabilityService {+ t.Helper()+ dockerClient, err := client.New(client.WithHost("unix:///nonexistent/arcane-vulnerability-test.sock"), client.WithAPIVersion("1.41"))+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, dockerClient.Close()) })+ cfg := &config.Config{}+ return NewVulnerabilityService(+ db,+ cfg,+ docker.NewDockerClientService(t.Context(), nil, nil, nil).WithClient(dockerClient),+ event.NewEventService(db, cfg, nil),+ settingsService,+ notification.NewNotificationService(db, cfg, nil, nil, nil),+ activity.NewActivityService(db, settingsService),+ nil,+ kv.NewKVService(db),+ httpClient,+ )}func TestScanWorkflowSkipsWhenFeatureDisabled(t *testing.T) {settingsSvc, db := featureSettings(t)require.NoError(t, db.AutoMigrate(&activity.Activity{}, &activity.ActivityMessage{}))harness := flowtest.New(t, activity.NewActivityService(db, nil))- svc := newTestService(db, settingsSvc)+ svc := newTestService(t, db, settingsSvc, http.DefaultClient)require.NoError(t, svc.RegisterWorkflows(harness.Engine))harness.Start(t)flowtest.AssertDefinitions(t, harness)diff --git a/backend/pkg/scheduler/job_schedule_integration_test.go b/backend/pkg/scheduler/job_schedule_integration_test.goindex 54714b9b8..7ac4c90b0 100644--- a/backend/pkg/scheduler/job_schedule_integration_test.go+++ b/backend/pkg/scheduler/job_schedule_integration_test.go@@ -11,11 +11,16 @@ import ("gorm.io/gorm""gorm.io/gorm/logger"+ "github.com/getarcaneapp/arcane/backend/v2/internal/activity""github.com/getarcaneapp/arcane/backend/v2/internal/config""github.com/getarcaneapp/arcane/backend/v2/internal/database"+ "github.com/getarcaneapp/arcane/backend/v2/internal/environment""github.com/getarcaneapp/arcane/backend/v2/internal/job""github.com/getarcaneapp/arcane/backend/v2/internal/kv"+ "github.com/getarcaneapp/arcane/backend/v2/internal/role""github.com/getarcaneapp/arcane/backend/v2/internal/settings"+ "github.com/getarcaneapp/arcane/backend/v2/pkg/scheduler/runs"+ francistest "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/francis/testing")func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {@@ -27,7 +32,19 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {lifecycleCtx, cancelLifecycle := context.WithCancel(t.Context())jobScheduler := newJobSchedulerForTestInternal(t, lifecycleCtx, appConfig.GetLocation())- jobService := job.NewJobService(db, settingsService, appConfig, jobScheduler.coordinator, nil, nil, nil)+ jobRuntime := francistest.New(t)+ jobCoordinator := runs.New(kv.NewKVService(db), jobRuntime.Service(), appConfig.GetLocation())+ require.NoError(t, jobCoordinator.Register(jobRuntime))+ francistest.Start(t, jobRuntime)+ jobService := job.NewJobService(+ db,+ settingsService,+ appConfig,+ jobCoordinator,+ role.NewRoleService(db),+ environment.NewEnvironmentService(db, nil, nil, nil, nil, nil),+ activity.NewActivityService(db, settingsService),+ )jobService.SetScheduler(lifecycleCtx, jobScheduler)require.NoError(t, jobScheduler.StartScheduler(t.Context()))@@ -66,7 +83,19 @@ func TestDeprecatedImagePollingSchedulePersistsWithoutRuntimeJob(t *testing.T) {restartDB, restartSettingsService := openJobScheduleTestDatabaseInternal(t, ctx, databasePath)restartLifecycleCtx, cancelRestartLifecycle := context.WithCancel(t.Context())restartScheduler := newJobSchedulerForTestInternal(t, restartLifecycleCtx, appConfig.GetLocation())- restartJobService := job.NewJobService(restartDB, restartSettingsService, appConfig, restartScheduler.coordinator, nil, nil, nil)+ restartRuntime := francistest.New(t)+ restartCoordinator := runs.New(kv.NewKVService(restartDB), restartRuntime.Service(), appConfig.GetLocation())+ require.NoError(t, restartCoordinator.Register(restartRuntime))+ francistest.Start(t, restartRuntime)+ restartJobService := job.NewJobService(+ restartDB,+ restartSettingsService,+ appConfig,+ restartCoordinator,+ role.NewRoleService(restartDB),+ environment.NewEnvironmentService(restartDB, nil, nil, nil, nil, nil),+ activity.NewActivityService(restartDB, restartSettingsService),+ )restartJobService.SetScheduler(restartLifecycleCtx, restartScheduler)require.NoError(t, restartScheduler.StartScheduler(t.Context()))t.Cleanup(func() {