Skip to content

Modern Docker Management, Designed for Everyone

perf: share snapshot production and cheapen badge queries (#3608)

Kyle Mendella4608822ee3b
diff --git a/backend/go.mod b/backend/go.mod
index 8dce5830f..c3b36b736 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -59,7 +59,7 @@ require (
go.getarcane.app/acfs v0.4.0
go.getarcane.app/builds v0.3.0
go.getarcane.app/docker/convert v0.1.0
- go.getarcane.app/streams v0.3.1
+ go.getarcane.app/streams v0.4.0
go.getarcane.app/sys/cgroup v0.1.1
go.getarcane.app/sys/crypto v0.1.0
go.getarcane.app/updater v0.7.1
diff --git a/backend/go.sum b/backend/go.sum
index 33e46c2de..b6e767d77 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -762,8 +762,8 @@ go.getarcane.app/builds v0.3.0 h1:6fCgHy6PaEr3LpCtiF1iCdT4nuV3D3xvUHnV+E/0cqk=
go.getarcane.app/builds v0.3.0/go.mod h1:yZJDSh9XX/sgZlwx7lghHEucgZeQ74Gr/Fx+XdR2bdk=
go.getarcane.app/docker/convert v0.1.0 h1:prlIOs6dMx6CdnetsJFZ6ak0vcistM3DcH5lrZiFHBU=
go.getarcane.app/docker/convert v0.1.0/go.mod h1:o9y4z/vah8HiHFtI+jJVFHbs8nLrkARYt88OaUlvGig=
-go.getarcane.app/streams v0.3.1 h1:Exs9a9G3paM9PnB6DQc8YvIKNbfaUTB7q271SD30zL8=
-go.getarcane.app/streams v0.3.1/go.mod h1:LHMKblQvsL1nqKh/Y+KgQbwFSMI2I7bs9u8ZwZqzjnQ=
+go.getarcane.app/streams v0.4.0 h1:MtBX1SVgt3nrcRNr5SYAvckGp9a5qeSjMTw/LAGwlzk=
+go.getarcane.app/streams v0.4.0/go.mod h1:LHMKblQvsL1nqKh/Y+KgQbwFSMI2I7bs9u8ZwZqzjnQ=
go.getarcane.app/sys/atomic v0.1.0 h1:0HhUeGXYdKAmMW24O+OaqcjDZx6twAuYPBu9T91zKsk=
go.getarcane.app/sys/atomic v0.1.0/go.mod h1:QPb5eUXYKYG5acp9jJBAZ/PTbHbByDAQG6eDNAsZZ2w=
go.getarcane.app/sys/cgroup v0.1.1 h1:c39W8FDqMXlKOZrpH+itaY+t8oC0tva0+6ztTrKvVIg=
diff --git a/backend/internal/activity/handler.go b/backend/internal/activity/handler.go
index df37125d5..7cf449e3f 100644
--- a/backend/internal/activity/handler.go
+++ b/backend/internal/activity/handler.go
@@ -29,6 +29,10 @@ import (
type ActivityHandler struct {
activityService *ActivityService
environment EnvironmentDependencies
+
+ // remoteStreamHub shares one poller per remote environment across every
+ // connected stream client instead of polling per client × environment.
+ remoteStreamHub *agg.Hub[activitytypes.StreamEvent]
}
type EnvironmentDependencies struct {
@@ -98,6 +102,7 @@ func NewHandler(activityService *ActivityService, environment EnvironmentDepende
return &ActivityHandler{
activityService: activityService,
environment: environment,
+ remoteStreamHub: agg.NewHub[activitytypes.StreamEvent](),
}
}
@@ -354,7 +359,17 @@ func (h *ActivityHandler) RunRemoteStreamPollers(ctx context.Context, ps *authz.
activityStreamEnvReconcileInterval,
"activity stream",
func(pollCtx context.Context, environment models.Environment) {
- h.runRemoteActivityStreamPollerInternal(pollCtx, environment, limit, events)
+ // One shared poller per environment (and snapshot limit) serves
+ // every connected client; this subscriber only forwards its
+ // events onto this client's stream.
+ key := activityStreamEnvironmentVersionInternal(environment) + ":" + strconv.Itoa(resolveActivityStreamLimitInternal(limit))
+ h.remoteStreamHub.Subscribe(pollCtx, key,
+ func(runCtx context.Context, publish func(activitytypes.StreamEvent)) {
+ h.runRemoteActivityStreamPollerInternal(runCtx, environment, limit, publish)
+ },
+ func(event activitytypes.StreamEvent) bool {
+ return agg.Send(pollCtx, events, event)
+ })
})
}
@@ -393,7 +408,7 @@ func activitySnapshotFingerprintInternal(items []activitytypes.Activity) string
return strconv.FormatUint(hash.Sum64(), 16)
}
-func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, environment models.Environment, limit int, events chan<- activitytypes.StreamEvent) {
+func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Context, environment models.Environment, limit int, publish func(activitytypes.StreamEvent)) {
environmentID := environment.ID
lastError := ""
lastFingerprint := ""
@@ -424,7 +439,7 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont
// once per distinct message and keep polling.
if msg := err.Error(); msg != lastError {
lastError = msg
- agg.Send(ctx, events, activitytypes.StreamEvent{
+ publish(activitytypes.StreamEvent{
Type: "error",
EnvironmentID: environmentID,
Error: msg,
@@ -440,14 +455,13 @@ func (h *ActivityHandler) runRemoteActivityStreamPollerInternal(ctx context.Cont
if fingerprint == lastFingerprint {
return
}
- if agg.Send(ctx, events, activitytypes.StreamEvent{
+ publish(activitytypes.StreamEvent{
Type: "snapshot",
EnvironmentID: environmentID,
Activities: output.Body.Data,
Timestamp: time.Now(),
- }) {
- lastFingerprint = fingerprint
- }
+ })
+ lastFingerprint = fingerprint
}
poll()
diff --git a/backend/internal/dashboard/handler.go b/backend/internal/dashboard/handler.go
index 8ae9cab55..f5ee3c415 100644
--- a/backend/internal/dashboard/handler.go
+++ b/backend/internal/dashboard/handler.go
@@ -28,6 +28,10 @@ import (
type DashboardHandler struct {
dashboardService *DashboardService
environmentService *environment.EnvironmentService
+
+ // remoteStreamHub shares one poller per remote environment across every
+ // connected stream client instead of polling per client × environment.
+ remoteStreamHub *agg.Hub[dashboardtypes.StreamEvent]
}
type GetDashboardInput struct {
@@ -54,6 +58,7 @@ func NewHandler(dashboardService *DashboardService, environmentService *environm
return &DashboardHandler{
dashboardService: dashboardService,
environmentService: environmentService,
+ remoteStreamHub: agg.NewHub[dashboardtypes.StreamEvent](),
}
}
@@ -77,7 +82,7 @@ func (h *DashboardHandler) GetDashboard(ctx context.Context, input *GetDashboard
snapshot, err := h.dashboardService.GetSnapshot(ctx, DashboardActionItemsOptions{
DebugAllGood: input.DebugAllGood,
- })
+ }, true)
if err != nil {
return nil, huma.Error500InternalServerError(err.Error())
}
@@ -97,7 +102,8 @@ func (h *DashboardHandler) GetDashboard(ctx context.Context, input *GetDashboard
// trimDashboardStreamSnapshotInternal drops the first-page container/image
// tables: the all-environments dashboard only reads the aggregate counters,
// and re-sending table rows for every environment on every poll would bloat
-// the stream.
+// the stream. Only remote snapshots (decoded fresh per poll) pass through
+// here; the local producer gets a snapshot built without tables instead.
func trimDashboardStreamSnapshotInternal(snapshot *dashboardtypes.Snapshot) *dashboardtypes.Snapshot {
if snapshot == nil {
return nil
@@ -113,7 +119,7 @@ func (h *DashboardHandler) RunLocalStreamProducer(ctx context.Context, debugAllG
poll := func() {
snapshot, err := h.dashboardService.GetSnapshot(ctx, DashboardActionItemsOptions{
DebugAllGood: debugAllGood,
- })
+ }, false)
if err == nil && snapshot == nil {
err = common.Classify(common.ErrUnavailable, errors.New("dashboard snapshot not available"))
}
@@ -135,10 +141,12 @@ func (h *DashboardHandler) RunLocalStreamProducer(ctx context.Context, debugAllG
return
}
lastError = ""
+ // Already built without tables; shared with other subscribers, so it
+ // must not be trimmed (mutated) here.
agg.Send(ctx, events, dashboardtypes.StreamEvent{
Type: "snapshot",
EnvironmentID: "0",
- Snapshot: trimDashboardStreamSnapshotInternal(snapshot),
+ Snapshot: snapshot,
Timestamp: time.Now(),
})
}
@@ -183,7 +191,20 @@ func (h *DashboardHandler) RunRemoteStreamPollers(ctx context.Context, ps *authz
dashboardStreamEnvReconcileInterval,
"dashboard stream",
func(pollCtx context.Context, environment models.Environment) {
- h.runRemoteDashboardStreamPollerInternal(pollCtx, environment, debugAllGood, events)
+ // One shared poller per environment (and debug variant) serves
+ // every connected client; this subscriber only forwards its
+ // events onto this client's stream.
+ key := dashboardStreamEnvironmentVersionInternal(environment)
+ if debugAllGood {
+ key += ":debugAllGood"
+ }
+ h.remoteStreamHub.Subscribe(pollCtx, key,
+ func(runCtx context.Context, publish func(dashboardtypes.StreamEvent)) {
+ h.runRemoteDashboardStreamPollerInternal(runCtx, environment, debugAllGood, publish)
+ },
+ func(event dashboardtypes.StreamEvent) bool {
+ return agg.Send(pollCtx, events, event)
+ })
})
}
@@ -194,17 +215,15 @@ func dashboardStreamEnvironmentVersionInternal(environment models.Environment) s
return environment.ID + ":" + environment.UpdatedAt.UTC().Format(time.RFC3339Nano)
}
-func (h *DashboardHandler) runRemoteDashboardStreamPollerInternal(ctx context.Context, environment models.Environment, debugAllGood bool, events chan<- dashboardtypes.StreamEvent) {
+func (h *DashboardHandler) runRemoteDashboardStreamPollerInternal(ctx context.Context, environment models.Environment, debugAllGood bool, publish func(dashboardtypes.StreamEvent)) {
environmentID := environment.ID
// Tell the client this environment is covered before the first poll
// completes so it can hold skeletons instead of assuming no data exists.
- if !agg.Send(ctx, events, dashboardtypes.StreamEvent{
+ publish(dashboardtypes.StreamEvent{
Type: "pending",
EnvironmentID: environmentID,
Timestamp: time.Now(),
- }) {
- return
- }
+ })
lastError := ""
@@ -237,7 +256,7 @@ func (h *DashboardHandler) runRemoteDashboardStreamPollerInternal(ctx context.Co
message, code := classifyDashboardStreamErrorInternal(err)
if message != lastError {
lastError = message
- agg.Send(ctx, events, dashboardtypes.StreamEvent{
+ publish(dashboardtypes.StreamEvent{
Type: "error",
EnvironmentID: environmentID,
Error: message,
@@ -248,7 +267,7 @@ func (h *DashboardHandler) runRemoteDashboardStreamPollerInternal(ctx context.Co
return
}
lastError = ""
- agg.Send(ctx, events, dashboardtypes.StreamEvent{
+ publish(dashboardtypes.StreamEvent{
Type: "snapshot",
EnvironmentID: environmentID,
Snapshot: trimDashboardStreamSnapshotInternal(snapshot),
diff --git a/backend/internal/dashboard/service.go b/backend/internal/dashboard/service.go
index 756cb1ad5..e68d7c7c1 100644
--- a/backend/internal/dashboard/service.go
+++ b/backend/internal/dashboard/service.go
@@ -3,12 +3,15 @@ package dashboard
import (
"context"
"sort"
+ "strconv"
+ "sync/atomic"
"time"
"emperror.dev/errors"
dockercontainer "github.com/moby/moby/api/types/container"
"golang.org/x/sync/errgroup"
+ "golang.org/x/sync/singleflight"
"github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
@@ -23,6 +26,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
dockerutils "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"
"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
+ "github.com/getarcaneapp/arcane/backend/v2/pkg/utils/iconcatalog"
"github.com/getarcaneapp/arcane/types/v2/base"
containertypes "github.com/getarcaneapp/arcane/types/v2/container"
dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"
@@ -36,6 +40,8 @@ import (
const (
defaultDashboardAPIKeyExpiryWindow = 14 * 24 * time.Hour
dashboardSnapshotPreloadLimit = 50
+ dashboardSnapshotCacheTTL = 5 * time.Second
+ dashboardSnapshotBuildTimeout = 30 * time.Second
)
type DashboardService struct {
@@ -49,6 +55,28 @@ type DashboardService struct {
environmentService *environment.EnvironmentService
versionService *version.VersionService
volumeService *volume.VolumeService
+
+ // snapshotFlight + snapshotCache share one snapshot build across all
+ // concurrent dashboard consumers (HTTP first paint + every stream
+ // subscriber). Cached snapshots are shared pointers and must be treated
+ // as immutable by callers. Indexed by
+ // [includeTables][debugAllGood][iconCatalog]: full snapshots resolve
+ // container icons against the requesting user's icon catalog, so each
+ // catalog gets its own entry.
+ snapshotFlight singleflight.Group
+ snapshotCache [2][2][2]atomic.Pointer[dashboardSnapshotCacheEntryInternal]
+}
+
+type dashboardSnapshotCacheEntryInternal struct {
+ snapshot *dashboardtypes.Snapshot
+ builtAt time.Time
+}
+
+func dashboardCacheIndexInternal(flag bool) int {
+ if flag {
+ return 1
+ }
+ return 0
}
type DashboardActionItemsOptions struct {
@@ -81,7 +109,55 @@ func NewDashboardService(
}
}
-func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardActionItemsOptions) (*dashboardtypes.Snapshot, error) {
+// GetSnapshot returns the dashboard snapshot, shared across all concurrent
+// consumers (HTTP first paint + every stream subscriber) through a short-TTL
+// cache; the result is a shared pointer and must not be mutated.
+// includeTables controls whether the first-page container/image tables are
+// built: stream subscribers pass false since the all-environments dashboard
+// only reads the aggregate counters, skipping the per-container DTO builds,
+// sorting, and icon resolution entirely.
+func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboardtypes.Snapshot, error) {
+ // Trimmed snapshots skip icon resolution entirely, so every consumer
+ // shares one entry regardless of the requesting user's catalog.
+ catalog := iconcatalog.DefaultCatalog
+ if includeTables {
+ catalog = iconcatalog.Normalize(project.IconCatalogForContext(ctx))
+ }
+
+ slot := &s.snapshotCache[dashboardCacheIndexInternal(includeTables)][dashboardCacheIndexInternal(options.DebugAllGood)][dashboardCacheIndexInternal(catalog == iconcatalog.CatalogDashboardIcons)]
+ if entry := slot.Load(); entry != nil && time.Since(entry.builtAt) < dashboardSnapshotCacheTTL {
+ return entry.snapshot, nil
+ }
+
+ key := strconv.FormatBool(includeTables) + ":" + strconv.FormatBool(options.DebugAllGood) + ":" + catalog
+ result, err, _ := s.snapshotFlight.Do(key, func() (any, error) {
+ // Re-check under the flight: a caller that queued behind the winner
+ // finds the fresh entry here instead of rebuilding.
+ if entry := slot.Load(); entry != nil && time.Since(entry.builtAt) < dashboardSnapshotCacheTTL {
+ return entry.snapshot, nil
+ }
+
+ buildCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dashboardSnapshotBuildTimeout)
+ defer cancel()
+
+ snapshot, err := s.buildSnapshotInternal(buildCtx, options, includeTables)
+ if err != nil {
+ return nil, err
+ }
+ slot.Store(&dashboardSnapshotCacheEntryInternal{snapshot: snapshot, builtAt: time.Now()})
+ return snapshot, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ snapshot, ok := result.(*dashboardtypes.Snapshot)
+ if !ok {
+ return nil, errors.New("dashboard snapshot cache returned unexpected type")
+ }
+ return snapshot, nil
+}
+
+func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options DashboardActionItemsOptions, includeTables bool) (*dashboardtypes.Snapshot, error) {
if s.dockerService == nil {
return nil, errors.New("docker service not available")
}
@@ -94,57 +170,58 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardAct
dockerImages := dockerSnapshot.Images
filteredContainers := container.FilterInternalContainers(dockerContainers, false)
- containerItems := make([]containertypes.Summary, 0, len(filteredContainers))
- currentContainerID, currentContainerErr := cgroup.CurrentContainerID()
- if s.containerService != nil {
- containerItems = s.containerService.BuildSummaries(filteredContainers, nil, currentContainerID, currentContainerErr)
- } else {
- for _, container := range filteredContainers {
- summary := containertypes.NewSummary(container)
- summary.RedeployDisabled = labels.ShouldDisableArcaneServerRedeploy(summary.Labels, summary.ID, currentContainerID, currentContainerErr)
- containerItems = append(containerItems, summary)
+
+ containerCounts := containertypes.StatusCounts{TotalContainers: len(filteredContainers)}
+ for _, c := range filteredContainers {
+ if c.State == "running" {
+ containerCounts.RunningContainers++
+ } else {
+ containerCounts.StoppedContainers++
}
}
- containerCounts := containertypes.StatusCounts{TotalContainers: len(containerItems)}
- if s.containerService != nil {
- containerCounts = s.containerService.CalculateStatusCounts(containerItems)
- } else {
- for _, item := range containerItems {
- if item.State == "running" {
- containerCounts.RunningContainers++
- } else {
- containerCounts.StoppedContainers++
+ var containerPage []containertypes.Summary
+ var imagePage []imagetypes.Summary
+ if includeTables {
+ containerItems := make([]containertypes.Summary, 0, len(filteredContainers))
+ currentContainerID, currentContainerErr := cgroup.CurrentContainerID()
+ if s.containerService != nil {
+ containerItems = s.containerService.BuildSummaries(filteredContainers, nil, currentContainerID, currentContainerErr)
+ } else {
+ for _, container := range filteredContainers {
+ summary := containertypes.NewSummary(container)
+ summary.RedeployDisabled = labels.ShouldDisableArcaneServerRedeploy(summary.Labels, summary.ID, currentContainerID, currentContainerErr)
+ containerItems = append(containerItems, summary)
}
}
- }
- sort.Slice(containerItems, func(i, j int) bool {
- if containerItems[i].Created == containerItems[j].Created {
- return containerItems[i].ID < containerItems[j].ID
+ sort.Slice(containerItems, func(i, j int) bool {
+ if containerItems[i].Created == containerItems[j].Created {
+ return containerItems[i].ID < containerItems[j].ID
+ }
+ return containerItems[i].Created > containerItems[j].Created
+ })
+ containerPage = limitDashboardItemsInternal(containerItems, dashboardSnapshotPreloadLimit)
+ if s.containerService != nil {
+ s.containerService.ApplySummaryIcons(ctx, containerPage, nil)
}
- return containerItems[i].Created > containerItems[j].Created
- })
- containerPage := limitDashboardItemsInternal(containerItems, dashboardSnapshotPreloadLimit)
- if s.containerService != nil {
- s.containerService.ApplySummaryIcons(ctx, containerPage, nil)
- }
-
- var projectIDByName map[string]string
- if s.imageService != nil {
- projectIDByName = s.imageService.BuildProjectIDMap(ctx, filteredContainers)
- } else {
- projectIDByName = map[string]string{}
- }
- imageUsageMap := image.BuildVolumeUsageMap(filteredContainers, projectIDByName)
- imageItems := image.MapDockerImagesToDTOs(dockerImages, imageUsageMap, nil, nil)
- sort.Slice(imageItems, func(i, j int) bool {
- if imageItems[i].Size == imageItems[j].Size {
- return imageItems[i].ID < imageItems[j].ID
+
+ var projectIDByName map[string]string
+ if s.imageService != nil {
+ projectIDByName = s.imageService.BuildProjectIDMap(ctx, filteredContainers)
+ } else {
+ projectIDByName = map[string]string{}
}
- return imageItems[i].Size > imageItems[j].Size
- })
- imagePage := limitDashboardItemsInternal(imageItems, dashboardSnapshotPreloadLimit)
+ imageUsageMap := image.BuildVolumeUsageMap(filteredContainers, projectIDByName)
+ imageItems := image.MapDockerImagesToDTOs(dockerImages, imageUsageMap, nil, nil)
+ sort.Slice(imageItems, func(i, j int) bool {
+ if imageItems[i].Size == imageItems[j].Size {
+ return imageItems[i].ID < imageItems[j].ID
+ }
+ return imageItems[i].Size > imageItems[j].Size
+ })
+ imagePage = limitDashboardItemsInternal(imageItems, dashboardSnapshotPreloadLimit)
+ }
imageUsageCounts := imagetypes.UsageCounts{}
imageUsageCounts.Inuse, imageUsageCounts.Unused, imageUsageCounts.Total = docker.CountImageUsage(dockerImages, filteredContainers)
@@ -174,11 +251,11 @@ func (s *DashboardService) GetSnapshot(ctx context.Context, options DashboardAct
Containers: dashboardtypes.SnapshotContainers{
Data: containerPage,
Counts: containerCounts,
- Pagination: buildDashboardPaginationResponseInternal(len(containerItems), dashboardSnapshotPreloadLimit),
+ Pagination: buildDashboardPaginationResponseInternal(len(filteredContainers), dashboardSnapshotPreloadLimit),
},
Images: dashboardtypes.SnapshotImages{
Data: imagePage,
- Pagination: buildDashboardPaginationResponseInternal(len(imageItems), dashboardSnapshotPreloadLimit),
+ Pagination: buildDashboardPaginationResponseInternal(len(dockerImages), dashboardSnapshotPreloadLimit),
},
ImageUsageCounts: imageUsageCounts,
VolumeUsageCounts: volumeUsageCounts,
diff --git a/backend/internal/dashboard/service_test.go b/backend/internal/dashboard/service_test.go
index 74320434a..21c8be690 100644
--- a/backend/internal/dashboard/service_test.go
+++ b/backend/internal/dashboard/service_test.go
@@ -13,6 +13,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/actors"
"github.com/getarcaneapp/arcane/backend/v2/internal/config"
+ "github.com/getarcaneapp/arcane/backend/v2/internal/container"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
"github.com/getarcaneapp/arcane/backend/v2/internal/image"
@@ -21,6 +22,7 @@ import (
"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"
+ usertypes "github.com/getarcaneapp/arcane/types/v2/user"
volumetypes "github.com/getarcaneapp/arcane/types/v2/volume"
"github.com/libtnb/sqlite"
dockercontainer "github.com/moby/moby/api/types/container"
@@ -184,7 +186,7 @@ func TestDashboardService_GetSnapshot_ReturnsDashboardSnapshot(t *testing.T) {
projectSvc := project.NewProjectService(db, settingsSvc, nil, image.NewImageService(db, nil, nil, nil, nil, nil), nil, nil, nil, nil, config.Load())
svc := NewDashboardService(db, dockerSvc, nil, projectSvc, nil, settingsSvc, nil, nil, nil, volume.NewVolumeService(db, nil, nil, nil, nil, nil))
- snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{})
+ snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)
require.NoError(t, err)
require.NotNil(t, snapshot)
@@ -237,7 +239,7 @@ func TestDashboardService_GetSnapshot_DebugAllGoodOnlyClearsActionItems(t *testi
dockerSvc := newDashboardTestDockerService(t, settingsSvc, containers, images, nil)
svc := NewDashboardService(db, dockerSvc, nil, nil, nil, settingsSvc, nil, nil, nil, nil)
- snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{DebugAllGood: true})
+ snapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{DebugAllGood: true}, true)
require.NoError(t, err)
require.NotNil(t, snapshot)
@@ -301,3 +303,83 @@ func newSettingsServiceForTestInternal(ctx context.Context, t testing.TB, db *da
})
return settings.NewSettingsService(ctx, db, executor, effects)
}
+
+func TestDashboardService_GetSnapshot_TrimmedOmitsTablesAndSharesBuilds(t *testing.T) {
+ db, settingsSvc := setupDashboardServiceTestDB(t)
+
+ containers := []dockercontainer.Summary{
+ {ID: "container-running", Names: []string{"/running-app"}, Image: "repo/app:stable", ImageID: "sha256:image-a", Created: 1700000000, State: "running", Status: "Up 2 hours", Labels: map[string]string{}},
+ {ID: "container-stopped", Names: []string{"/stopped-app"}, Image: "repo/worker:latest", ImageID: "sha256:image-b", Created: 1800000000, State: "exited", Status: "Exited (0) 1 hour ago", Labels: map[string]string{}},
+ }
+ images := []dockerimage.Summary{
+ {ID: "sha256:image-a", RepoTags: []string{"repo/app:stable"}, Created: 1710000000, Size: 100},
+ {ID: "sha256:image-b", RepoTags: []string{"repo/worker:latest"}, Created: 1720000000, Size: 250},
+ }
+
+ dockerSvc := newDashboardTestDockerService(t, settingsSvc, containers, images, nil)
+ svc := NewDashboardService(db, dockerSvc, nil, nil, nil, settingsSvc, nil, nil, nil, nil)
+
+ trimmed, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, false)
+ require.NoError(t, err)
+ require.NotNil(t, trimmed)
+
+ // No table rows, but counters and pagination totals intact.
+ require.Nil(t, trimmed.Containers.Data)
+ require.Nil(t, trimmed.Images.Data)
+ require.Equal(t, 1, trimmed.Containers.Counts.RunningContainers)
+ require.Equal(t, 1, trimmed.Containers.Counts.StoppedContainers)
+ require.Equal(t, 2, trimmed.Containers.Counts.TotalContainers)
+ require.EqualValues(t, 2, trimmed.Containers.Pagination.TotalItems)
+ require.EqualValues(t, 2, trimmed.Images.Pagination.TotalItems)
+ require.Equal(t, 2, trimmed.ImageUsageCounts.Total)
+
+ // Within the TTL every subscriber shares the same build.
+ again, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, false)
+ require.NoError(t, err)
+ require.Same(t, trimmed, again)
+
+ // The full variant is cached separately and still carries the tables.
+ full, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)
+ require.NoError(t, err)
+ require.NotNil(t, full)
+ require.Len(t, full.Containers.Data, 2)
+ require.Len(t, full.Images.Data, 2)
+}
+
+func TestDashboardService_GetSnapshot_CachesFullSnapshotsPerIconCatalog(t *testing.T) {
+ db, settingsSvc := setupDashboardServiceTestDB(t)
+
+ containers := []dockercontainer.Summary{
+ {ID: "container-running", Names: []string{"/running-app"}, Image: "repo/app:stable", ImageID: "sha256:image-a", Created: 1700000000, State: "running", Status: "Up 2 hours", Labels: map[string]string{"arcane.icon": "myapp"}},
+ }
+ images := []dockerimage.Summary{
+ {ID: "sha256:image-a", RepoTags: []string{"repo/app:stable"}, Created: 1710000000, Size: 100},
+ }
+
+ dockerSvc := newDashboardTestDockerService(t, settingsSvc, containers, images, nil)
+ containerSvc := container.NewContainerService(t.Context(), nil, dockerSvc, nil, settingsSvc, nil)
+ svc := NewDashboardService(db, dockerSvc, containerSvc, nil, nil, settingsSvc, nil, nil, nil, nil)
+
+ defaultSnapshot, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)
+ require.NoError(t, err)
+ require.Len(t, defaultSnapshot.Containers.Data, 1)
+ require.Contains(t, defaultSnapshot.Containers.Data[0].IconLightURL, "selfhst")
+
+ // A user preferring another catalog must not be served the default-catalog
+ // snapshot from the cache.
+ dashboardIconsUser := &models.User{Preferences: usertypes.Preferences{IconCatalog: new("dashboard-icons")}}
+ userCtx := context.WithValue(context.Background(), models.CurrentUserContextKey{}, dashboardIconsUser)
+ userSnapshot, err := svc.GetSnapshot(userCtx, DashboardActionItemsOptions{}, true)
+ require.NoError(t, err)
+ require.NotSame(t, defaultSnapshot, userSnapshot)
+ require.Len(t, userSnapshot.Containers.Data, 1)
+ require.Contains(t, userSnapshot.Containers.Data[0].IconLightURL, "dashboard-icons")
+
+ // Both catalog variants stay cached side by side within the TTL.
+ defaultAgain, err := svc.GetSnapshot(context.Background(), DashboardActionItemsOptions{}, true)
+ require.NoError(t, err)
+ require.Same(t, defaultSnapshot, defaultAgain)
+ userAgain, err := svc.GetSnapshot(userCtx, DashboardActionItemsOptions{}, true)
+ require.NoError(t, err)
+ require.Same(t, userSnapshot, userAgain)
+}
diff --git a/backend/internal/project/project_listing.go b/backend/internal/project/project_listing.go
index 5e67773fa..9ecbd108d 100644
--- a/backend/internal/project/project_listing.go
+++ b/backend/internal/project/project_listing.go
@@ -337,7 +337,7 @@ func (s *ProjectService) appendDiscoveredComposeProjectUpdatesInternal(
return items
}
- knownProjectNames := s.buildKnownComposeProjectNameSetInternal(ctx, projectsArray)
+ knownProjectNames := s.buildKnownComposeProjectNameSetInternal(ctx, projectsArray, false)
discovered := buildDiscoveredComposeProjectUpdateRowsInternal(ctx, composeContainers, knownProjectNames, s.imageService, IconCatalogForContext(ctx))
if len(discovered) == 0 {
return items
@@ -354,7 +354,10 @@ func shouldIncludeDiscoveredComposeProjectUpdatesInternal(params pagination.Quer
return strings.EqualFold(strings.TrimSpace(params.Filters["updates"]), "has_update") && strings.TrimSpace(params.Filters["tags"]) == ""
}
-func (s *ProjectService) buildKnownComposeProjectNameSetInternal(ctx context.Context, projectsArray []models.Project) map[string]struct{} {
+// buildKnownComposeProjectNameSetInternal collects every project name Arcane
+// tracks. projectsArrayIsComplete tells it the caller already loaded the full
+// table (not a filtered page), so the catch-all re-query can be skipped.
+func (s *ProjectService) buildKnownComposeProjectNameSetInternal(ctx context.Context, projectsArray []models.Project, projectsArrayIsComplete bool) map[string]struct{} {
known := make(map[string]struct{}, len(projectsArray)*2)
for _, proj := range projectsArray {
addKnownComposeProjectNameInternal(known, proj.Name)
@@ -363,7 +366,7 @@ func (s *ProjectService) buildKnownComposeProjectNameSetInternal(ctx context.Con
}
}
- if s.db == nil {
+ if s.db == nil || projectsArrayIsComplete {
return known
}
@@ -711,18 +714,28 @@ func (s *ProjectService) CountProjectsWithPendingUpdates(ctx context.Context, al
return 0, nil
}
- var projectsArray []models.Project
- if err := s.db.WithContext(ctx).Where("is_archived = ?", false).Find(&projectsArray).Error; err != nil {
+ // One full scan: archived projects are excluded from the update count but
+ // still mark their compose stacks as known during discovery, so loading
+ // everything here saves the known-name pass its own table scan.
+ var allProjects []models.Project
+ if err := s.db.WithContext(ctx).Find(&allProjects).Error; err != nil {
return 0, errors.WrapIf(err, "failed to list projects for update count")
}
+ activeProjects := make([]models.Project, 0, len(allProjects))
+ for _, proj := range allProjects {
+ if !proj.IsArchived {
+ activeProjects = append(activeProjects, proj)
+ }
+ }
+
// enrichProjectsWithUpdateInfoInternal keys off Details.ID, so the summaries
// only need identity — no status, icons or URLs are read here.
- details := make([]project.Details, len(projectsArray))
- for i, proj := range projectsArray {
+ details := make([]project.Details, len(activeProjects))
+ for i, proj := range activeProjects {
details[i].ID = proj.ID
}
- s.enrichProjectsWithUpdateInfoInternal(ctx, projectsArray, details)
+ s.enrichProjectsWithUpdateInfoInternal(ctx, activeProjects, details)
count := 0
for i := range details {
@@ -731,14 +744,14 @@ func (s *ProjectService) CountProjectsWithPendingUpdates(ctx context.Context, al
}
}
- return count + s.countDiscoveredComposeProjectUpdatesInternal(ctx, projectsArray, allContainers), nil
+ return count + s.countDiscoveredComposeProjectUpdatesInternal(ctx, allProjects, true, allContainers), nil
}
// countDiscoveredComposeProjectUpdatesInternal counts compose projects running on
// the daemon that Arcane does not track but that have a pending image update, so
// the dashboard badge matches the projects table. Errors are logged and counted
// as zero: a missing container list should degrade the badge, not fail the load.
-func (s *ProjectService) countDiscoveredComposeProjectUpdatesInternal(ctx context.Context, projectsArray []models.Project, allContainers []container.Summary) int {
+func (s *ProjectService) countDiscoveredComposeProjectUpdatesInternal(ctx context.Context, projectsArray []models.Project, projectsArrayIsComplete bool, allContainers []container.Summary) int {
if allContainers == nil {
var err error
allContainers, err = s.listGlobalComposeContainersInternal(ctx)
@@ -758,7 +771,7 @@ func (s *ProjectService) countDiscoveredComposeProjectUpdatesInternal(ctx contex
return 0
}
- knownProjectNames := s.buildKnownComposeProjectNameSetInternal(ctx, projectsArray)
+ knownProjectNames := s.buildKnownComposeProjectNameSetInternal(ctx, projectsArray, projectsArrayIsComplete)
// Only rows with a pending update are returned, so the length is the count.
return len(buildDiscoveredComposeProjectUpdateRowsInternal(ctx, composeContainers, knownProjectNames, s.imageService, IconCatalogForContext(ctx)))
}
diff --git a/backend/internal/vulnerability/service_test.go b/backend/internal/vulnerability/service_test.go
index de0e6aafb..10af82e2b 100644
--- a/backend/internal/vulnerability/service_test.go
+++ b/backend/internal/vulnerability/service_test.go
@@ -328,3 +328,62 @@ func TestVulnerabilityService_ListAllVulnerabilityImageOptions_InvalidSeverityAn
require.NoError(t, err)
require.Equal(t, []string{"alpha:latest", "beta:latest", "sha256:two"}, all)
}
+
+// TestVulnerabilityService_ActionableCountExcludingIgnored verifies the
+// indexed-sum fast path and the blob-decode path for images with ignore rows
+// produce the same count the old full-decode implementation did.
+func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {
+ ctx := context.Background()
+ db := setupVulnerabilityScanTestDB(t)
+ svc := &VulnerabilityService{db: db}
+
+ now := time.Now()
+
+ vulnsA := []vulnerability.Vulnerability{
+ {VulnerabilityID: "CVE-A1", PkgName: "openssl", InstalledVersion: "1.0.0", Severity: vulnerability.SeverityCritical},
+ {VulnerabilityID: "CVE-A2", PkgName: "zlib", InstalledVersion: "1.2.0", Severity: vulnerability.SeverityCritical},
+ {VulnerabilityID: "CVE-A3", PkgName: "curl", InstalledVersion: "8.0.0", Severity: vulnerability.SeverityHigh},
+ {VulnerabilityID: "CVE-A4", PkgName: "bash", InstalledVersion: "5.0.0", Severity: vulnerability.SeverityMedium},
+ }
+ rawVulnsA, err := json.Marshal(vulnsA)
+ require.NoError(t, err)
+
+ vulnsB := []vulnerability.Vulnerability{
+ {VulnerabilityID: "CVE-B1", PkgName: "openssl", InstalledVersion: "1.1.0", Severity: vulnerability.SeverityCritical},
+ {VulnerabilityID: "CVE-B2", PkgName: "glibc", InstalledVersion: "2.38", Severity: vulnerability.SeverityHigh},
+ {VulnerabilityID: "CVE-B3", PkgName: "less", InstalledVersion: "600", Severity: vulnerability.SeverityMedium},
+ }
+ rawVulnsB, err := json.Marshal(vulnsB)
+ require.NoError(t, err)
+
+ records := []models.VulnerabilityScanRecord{
+ {ID: "sha256:img-a", ImageName: "nginx:latest", Status: models.ScanStatusCompleted, ScanTime: now, CriticalCount: 2, HighCount: 1, MediumCount: 1, TotalCount: 4, Vulnerabilities: models.StringSlice{string(rawVulnsA)}},
+ {ID: "sha256:img-b", ImageName: "redis:7", Status: models.ScanStatusCompleted, ScanTime: now, CriticalCount: 1, HighCount: 1, MediumCount: 1, TotalCount: 3, Vulnerabilities: models.StringSlice{string(rawVulnsB)}},
+ {ID: "sha256:img-failed", ImageName: "failed:latest", Status: models.ScanStatusFailed, ScanTime: now, CriticalCount: 9, HighCount: 9, TotalCount: 18},
+ }
+ for i := range records {
+ require.NoError(t, db.Create(&records[i]).Error)
+ }
+
+ // No ignores: pure indexed sum, no blob decode.
+ count, err := svc.ActionableCountExcludingIgnored(ctx)
+ require.NoError(t, err)
+ require.Equal(t, 5, count)
+
+ ignores := []models.VulnerabilityIgnore{
+ // Critical on img-b: removes one from the count.
+ {ID: "ig-1", EnvironmentID: "env-1", ImageID: "sha256:img-b", VulnerabilityID: "CVE-B1", PkgName: "openssl", InstalledVersion: "1.1.0", CreatedBy: "tester"},
+ // Medium on img-b: never counted, must not subtract anything.
+ {ID: "ig-2", EnvironmentID: "env-1", ImageID: "sha256:img-b", VulnerabilityID: "CVE-B3", PkgName: "less", InstalledVersion: "600", CreatedBy: "tester"},
+ // Stale ignore for a CVE no longer in img-b's blob: no effect.
+ {ID: "ig-3", EnvironmentID: "env-1", ImageID: "sha256:img-b", VulnerabilityID: "CVE-GONE", PkgName: "old", InstalledVersion: "0.1", CreatedBy: "tester"},
+ }
+ for i := range ignores {
+ require.NoError(t, db.Create(&ignores[i]).Error)
+ }
+
+ // img-a (3 via indexed sum) + img-b (high CVE-B2 only, via blob decode).
+ count, err = svc.ActionableCountExcludingIgnored(ctx)
+ require.NoError(t, err)
+ require.Equal(t, 4, count)
+}
diff --git a/backend/internal/vulnerability/vulnerability_query.go b/backend/internal/vulnerability/vulnerability_query.go
index 75ddd1196..600f2676f 100644
--- a/backend/internal/vulnerability/vulnerability_query.go
+++ b/backend/internal/vulnerability/vulnerability_query.go
@@ -5,6 +5,7 @@ import (
"encoding/json/v2"
"log/slog"
"math"
+ "slices"
"sort"
"strings"
@@ -213,25 +214,16 @@ func (s *VulnerabilityService) GetEnvironmentSummary(ctx context.Context) (*vuln
// ActionableCountExcludingIgnored counts critical and high severity vulnerabilities
// across all completed scans, excluding any that have been ignored. This is used by the dashboard
// to show an accurate action item count that reflects ignore decisions.
+//
+// Images without ignore records are counted from the indexed critical_count/
+// high_count columns; only images that have ignore rows decode their
+// vulnerability blob, so the common no-ignores dashboard poll never
+// unmarshals a scan payload.
func (s *VulnerabilityService) ActionableCountExcludingIgnored(ctx context.Context) (int, error) {
if s.db == nil {
return 0, nil
}
- var records []models.VulnerabilityScanRecord
- err := s.db.WithContext(ctx).
- Select("id", "vulnerabilities").
- Where("status = ?", models.ScanStatusCompleted).
- Where("critical_count + high_count > 0").
- Find(&records).Error
- if err != nil {
- return 0, errors.WrapIf(err, "failed to list vulnerability scans")
- }
-
- if len(records) == 0 {
- return 0, nil
- }
-
// Load all ignore records (not scoped to a single environment, since an
// Arcane instance serves one environment and the dashboard does not carry
// an environment ID through the service layer today).
@@ -241,11 +233,43 @@ func (s *VulnerabilityService) ActionableCountExcludingIgnored(ctx context.Conte
}
ignoredKeys := make(map[string]struct{}, len(ignores))
+ ignoredImageIDs := make([]string, 0, len(ignores))
for _, ig := range ignores {
ignoredKeys[vulnerabilityIgnoreKeyInternal(ig.ImageID, ig.VulnerabilityID, ig.PkgName, ig.InstalledVersion)] = struct{}{}
+ if !slices.Contains(ignoredImageIDs, ig.ImageID) {
+ ignoredImageIDs = append(ignoredImageIDs, ig.ImageID)
+ }
+ }
+
+ sumQuery := s.db.WithContext(ctx).
+ Model(&models.VulnerabilityScanRecord{}).
+ Select("COALESCE(SUM(critical_count + high_count), 0)").
+ Where("status = ?", models.ScanStatusCompleted)
+ if len(ignoredImageIDs) > 0 {
+ sumQuery = sumQuery.Where("id NOT IN ?", ignoredImageIDs)
+ }
+
+ var indexedCount int64
+ if err := sumQuery.Scan(&indexedCount).Error; err != nil {
+ return 0, errors.WrapIf(err, "failed to sum vulnerability counts")
+ }
+
+ if len(ignoredImageIDs) == 0 {
+ return int(indexedCount), nil
}
- count := 0
+ var records []models.VulnerabilityScanRecord
+ err := s.db.WithContext(ctx).
+ Select("id", "vulnerabilities").
+ Where("status = ?", models.ScanStatusCompleted).
+ Where("critical_count + high_count > 0").
+ Where("id IN ?", ignoredImageIDs).
+ Find(&records).Error
+ if err != nil {
+ return 0, errors.WrapIf(err, "failed to list vulnerability scans")
+ }
+
+ count := int(indexedCount)
for _, record := range records {
if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {
continue
@@ -261,11 +285,9 @@ func (s *VulnerabilityService) ActionableCountExcludingIgnored(ctx context.Conte
if vulnEntry.Severity != vulnerability.SeverityCritical && vulnEntry.Severity != vulnerability.SeverityHigh {
continue
}
- if len(ignoredKeys) > 0 {
- key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)
- if _, isIgnored := ignoredKeys[key]; isIgnored {
- continue
- }
+ key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)
+ if _, isIgnored := ignoredKeys[key]; isIgnored {
+ continue
}
count++
}
diff --git a/backend/pkg/utils/iconcatalog/iconcatalog.go b/backend/pkg/utils/iconcatalog/iconcatalog.go
index 379f8323a..96904adae 100644
--- a/backend/pkg/utils/iconcatalog/iconcatalog.go
+++ b/backend/pkg/utils/iconcatalog/iconcatalog.go
@@ -43,7 +43,7 @@ func FirstNonEmpty(sets ...IconSet) IconSet {
}
func Resolve(catalog string, set IconSet) ResolvedIconSet {
- selectedCatalog := normalizeCatalogInternal(catalog)
+ selectedCatalog := Normalize(catalog)
if strings.TrimSpace(set.Light) == "" && strings.TrimSpace(set.Dark) == "" {
generic := resolveValueInternal(selectedCatalog, set.Icon, "")
return ResolvedIconSet{
@@ -70,7 +70,9 @@ func firstNonEmptyValueInternal(values ...string) string {
return ""
}
-func normalizeCatalogInternal(catalog string) string {
+// Normalize maps a raw catalog preference onto one of the known catalog
+// constants, falling back to the default for unknown values.
+func Normalize(catalog string) string {
switch strings.ToLower(strings.TrimSpace(catalog)) {
case CatalogDashboardIcons:
return CatalogDashboardIcons
diff --git a/go.work.sum b/go.work.sum
index 76a383d33..d79acda96 100644
--- a/go.work.sum
+++ b/go.work.sum
@@ -2017,6 +2017,8 @@ go.etcd.io/raft/v3 v3.6.0 h1:5NtvbDVYpnfZWcIHgGRk9DyzkBIXOi8j+DDp1IcnUWQ=
go.etcd.io/raft/v3 v3.6.0/go.mod h1:nLvLevg6+xrVtHUmVaTcTz603gQPHfh7kUAwV6YpfGo=
go.getarcane.app/acfs v0.2.0 h1:m7yXsLIB4MfsAE/AFP54RgtMXUr+AFEVCNiF5784XYI=
go.getarcane.app/acfs v0.2.0/go.mod h1:sU3t21KctTsQM3OW10+RNB0Z6lR+YfaSr5Nxubq3cpI=
+go.getarcane.app/streams v0.4.0 h1:MtBX1SVgt3nrcRNr5SYAvckGp9a5qeSjMTw/LAGwlzk=
+go.getarcane.app/streams v0.4.0/go.mod h1:LHMKblQvsL1nqKh/Y+KgQbwFSMI2I7bs9u8ZwZqzjnQ=
go.getarcane.app/updater v0.3.6 h1:+BqoycaRsHj+cLVSzcYQuqbHxUKbtmaozMNblz5bVSw=
go.getarcane.app/updater v0.3.6/go.mod h1:2ENitZR/lA5MuBn60OZYR5wvU3ZCE+pAgfrebfOlRDA=
go.getarcane.app/updater v0.3.7 h1:WqHsxudvt8gvCR7iSHdxdzwH0+hM0FAh64bSypBX9Mo=