Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: coalesce concurrent Docker image/container list calls to cut duplicate decodes (#3635)
Kyle Mendelldb9469e81eda
diff --git a/backend/internal/dashboard/service.go b/backend/internal/dashboard/service.goindex e68d7c7c1..6ccb9e69a 100644--- a/backend/internal/dashboard/service.go+++ b/backend/internal/dashboard/service.go@@ -223,11 +223,7 @@ func (s *DashboardService) buildSnapshotInternal(ctx context.Context, options DaimagePage = limitDashboardItemsInternal(imageItems, dashboardSnapshotPreloadLimit)}- imageUsageCounts := imagetypes.UsageCounts{}- imageUsageCounts.Inuse, imageUsageCounts.Unused, imageUsageCounts.Total = docker.CountImageUsage(dockerImages, filteredContainers)- for _, img := range dockerImages {- imageUsageCounts.TotalSize += img.Size- }+ imageUsageCounts := docker.CountImageUsage(dockerImages, filteredContainers)// Uses the unfiltered container list so a volume mounted only by an internal// container still counts as in use, matching the volumes page.diff --git a/backend/internal/docker/service.go b/backend/internal/docker/service.goindex 1f7143d96..728bcd09c 100644--- a/backend/internal/docker/service.go+++ b/backend/internal/docker/service.go@@ -19,6 +19,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts""github.com/getarcaneapp/arcane/backend/v2/pkg/utils"dashboardtypes "github.com/getarcaneapp/arcane/types/v2/dashboard"+ imagetypes "github.com/getarcaneapp/arcane/types/v2/image""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/events""github.com/moby/moby/api/types/image"@@ -28,6 +29,7 @@ import ("github.com/moby/moby/client""go.getarcane.app/streams/bus""golang.org/x/sync/errgroup"+ "golang.org/x/sync/singleflight")const dockerClientNegotiationTimeout = 5 * time.Second@@ -42,6 +44,13 @@ type DockerClientService struct {clientLastProbe time.Timemu sync.MutexeventBus *bus.DockerEventBus++ // Coalesce concurrent full-inventory list calls so overlapping requests+ // (e.g. the Homepage widget's simultaneous counts endpoints) decode one+ // Docker response instead of one per caller. Per-instance on purpose:+ // a WithClient copy talks to a different daemon and must not share flights.+ imageListFlight singleflight.Group+ containerListFlight singleflight.Group}func NewDockerClientService(_ context.Context, db *database.DB, cfg *config.Config, settingsService *settings.SettingsService) *DockerClientService {@@ -305,38 +314,72 @@ func sleepDockerEventBackoffInternal(ctx context.Context, eventBackoff *backoff.}}-func (s *DockerClientService) ListContainers(ctx context.Context) ([]container.Summary, error) {- dockerClient, err := s.GetClient(ctx)- if err != nil {- return nil, errors.WrapIf(err, "failed to connect to Docker")+// listCoalescedInternal runs op through the given singleflight group so+// concurrent callers share one Docker request and one decoded result. The+// flight runs on a context detached from the initiating caller, bounded only+// by op's own Docker API timeout, so one canceled waiter cannot cancel work+// other waiters still need; each waiter stops waiting when its own context+// ends. Results and errors are shared only among concurrent callers —+// singleflight drops the key once the flight completes, so nothing is cached.+// Callers must treat the shared result as immutable.+func listCoalescedInternal[T any](ctx context.Context, group *singleflight.Group, op func(context.Context) (T, error)) (T, error) {+ var zero T++ ch := group.DoChan("list", func() (any, error) {+ return op(context.WithoutCancel(ctx))+ })++ select {+ case <-ctx.Done():+ return zero, ctx.Err()+ case res := <-ch:+ if res.Err != nil {+ return zero, res.Err+ }+ val, ok := res.Val.(T)+ if !ok {+ return zero, errors.New("docker list flight returned unexpected type")+ }+ return val, nil}+}- settings := s.settingsService.GetSettingsConfig()- apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(settings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer cancel()+func (s *DockerClientService) ListContainers(ctx context.Context) ([]container.Summary, error) {+ return listCoalescedInternal(ctx, &s.containerListFlight, func(ctx context.Context) ([]container.Summary, error) {+ dockerClient, err := s.GetClient(ctx)+ if err != nil {+ return nil, errors.WrapIf(err, "failed to connect to Docker")+ }- containerList, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})- if err != nil {- return nil, errors.WrapIf(err, "failed to list Docker containers")- }- return containerList.Items, nil+ settings := s.settingsService.GetSettingsConfig()+ apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(settings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))+ defer cancel()++ containerList, err := dockerClient.ContainerList(apiCtx, client.ContainerListOptions{All: true})+ if err != nil {+ return nil, errors.WrapIf(err, "failed to list Docker containers")+ }+ return containerList.Items, nil+ })}func (s *DockerClientService) ListImages(ctx context.Context) ([]image.Summary, error) {- dockerClient, err := s.GetClient(ctx)- if err != nil {- return nil, errors.WrapIf(err, "failed to connect to Docker")- }+ return listCoalescedInternal(ctx, &s.imageListFlight, func(ctx context.Context) ([]image.Summary, error) {+ dockerClient, err := s.GetClient(ctx)+ if err != nil {+ return nil, errors.WrapIf(err, "failed to connect to Docker")+ }- settings := s.settingsService.GetSettingsConfig()- apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(settings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))- defer cancel()+ settings := s.settingsService.GetSettingsConfig()+ apiCtx, cancel := context.WithTimeout(ctx, timeouts.GetDuration(settings.DockerAPITimeout.AsInt(), timeouts.DefaultDockerAPI))+ defer cancel()- imageList, err := dockerClient.ImageList(apiCtx, client.ImageListOptions{All: true})- if err != nil {- return nil, errors.WrapIf(err, "failed to list Docker images")- }- return imageList.Items, nil+ imageList, err := dockerClient.ImageList(apiCtx, client.ImageListOptions{All: true})+ if err != nil {+ return nil, errors.WrapIf(err, "failed to list Docker images")+ }+ return imageList.Items, nil+ })}func (s *DockerClientService) listNetworksInternal(ctx context.Context) ([]network.Summary, error) {@@ -446,23 +489,39 @@ func (s *DockerClientService) GetAllContainers(ctx context.Context) ([]containerreturn containers, running, stopped, total, nil}-func (s *DockerClientService) GetAllImages(ctx context.Context) ([]image.Summary, int, int, int, error) {- images, err := s.ListImages(ctx)- if err != nil {- return nil, 0, 0, 0, err- }+// GetAllImages lists images and containers concurrently and returns the+// images together with their usage counts, including total image size.+func (s *DockerClientService) GetAllImages(ctx context.Context) ([]image.Summary, imagetypes.UsageCounts, error) {+ g, groupCtx := errgroup.WithContext(ctx)- containers, err := s.ListContainers(ctx)- if err != nil {- return nil, 0, 0, 0, err- }+ var images []image.Summary+ var containers []container.Summary- inuse, unused, total := CountImageUsage(images, containers)+ g.Go(func() (workerErr error) {+ defer utils.RecoverToError(&workerErr, "docker image list worker")++ var err error+ images, err = s.ListImages(groupCtx)+ return err+ })+ g.Go(func() (workerErr error) {+ defer utils.RecoverToError(&workerErr, "docker container list worker")++ var err error+ containers, err = s.ListContainers(groupCtx)+ return err+ })+ if err := g.Wait(); err != nil {+ return nil, imagetypes.UsageCounts{}, err+ }- return images, inuse, unused, total, nil+ return images, CountImageUsage(images, containers), nil}-func CountImageUsage(images []image.Summary, containers []container.Summary) (inuse int, unused int, total int) {+// CountImageUsage tallies in-use, unused, and total image counts, plus the+// summed size of all images. An image is in use when a container references+// its ID.+func CountImageUsage(images []image.Summary, containers []container.Summary) imagetypes.UsageCounts {inUseImageIDs := make(map[string]struct{}, len(containers))for _, c := range containers {if c.ImageID == "" {@@ -471,16 +530,18 @@ func CountImageUsage(images []image.Summary, containers []container.Summary) (ininUseImageIDs[c.ImageID] = struct{}{}}+ var counts imagetypes.UsageCountsfor _, img := range images {- total+++ counts.Total+++ counts.TotalSize += img.Sizeif _, ok := inUseImageIDs[img.ID]; ok {- inuse+++ counts.Inuse++continue}- unused+++ counts.Unused++}- return inuse, unused, total+ return counts}func (s *DockerClientService) GetAllNetworks(ctx context.Context) ([]network.Summary, int, int, int, error) {diff --git a/backend/internal/docker/service_test.go b/backend/internal/docker/service_test.goindex 115685bfa..619c4a331 100644--- a/backend/internal/docker/service_test.go+++ b/backend/internal/docker/service_test.go@@ -13,6 +13,7 @@ import ("github.com/getarcaneapp/arcane/backend/v2/internal/actors""github.com/getarcaneapp/arcane/backend/v2/internal/config"docker "github.com/getarcaneapp/arcane/backend/v2/pkg/dockerutil"+ imagetypes "github.com/getarcaneapp/arcane/types/v2/image""github.com/moby/moby/api/types/container""github.com/moby/moby/api/types/events""github.com/moby/moby/api/types/image"@@ -256,9 +257,9 @@ func TestDockerClientService_EventActorStopCancelsAndJoinsStreamInternal(t *testfunc TestCountImageUsage_UsesContainerImageIDs(t *testing.T) {images := []image.Summary{- {ID: "sha256:image-a", Containers: -1},- {ID: "sha256:image-b", Containers: 0},- {ID: "sha256:image-c", Containers: 99},+ {ID: "sha256:image-a", Containers: -1, Size: 100},+ {ID: "sha256:image-b", Containers: 0, Size: 25},+ {ID: "sha256:image-c", Containers: 99, Size: 3},}containers := []container.Summary{@@ -268,19 +269,18 @@ func TestCountImageUsage_UsesContainerImageIDs(t *testing.T) {{ImageID: ""},}- inuse, unused, total := CountImageUsage(images, containers)+ counts := CountImageUsage(images, containers)- assert.Equal(t, 2, inuse)- assert.Equal(t, 1, unused)- assert.Equal(t, 3, total)+ assert.Equal(t, 2, counts.Inuse)+ assert.Equal(t, 1, counts.Unused)+ assert.Equal(t, 3, counts.Total)+ assert.Equal(t, int64(128), counts.TotalSize)}func TestCountImageUsage_NoImages(t *testing.T) {- inuse, unused, total := CountImageUsage(nil, []container.Summary{{ImageID: "sha256:image-a"}})+ counts := CountImageUsage(nil, []container.Summary{{ImageID: "sha256:image-a"}})- assert.Equal(t, 0, inuse)- assert.Equal(t, 0, unused)- assert.Equal(t, 0, total)+ assert.Equal(t, imagetypes.UsageCounts{}, counts)}func newDockerClientServiceForTestInternal(host string) *DockerClientService {diff --git a/backend/internal/image/handler.go b/backend/internal/image/handler.goindex 7f658d939..f4ac7d914 100644--- a/backend/internal/image/handler.go+++ b/backend/internal/image/handler.go@@ -2,7 +2,6 @@ package imageimport ("context"- stderrors "errors""fmt""io""net/http"@@ -826,39 +825,15 @@ func resolveLegacyPruneImageModeInternal(dangling bool) string {// GetImageUsageCounts returns counts of images by usage status.func (h *ImageHandler) GetImageUsageCounts(ctx context.Context, input *GetImageUsageCountsInput) (*GetImageUsageCountsOutput, error) {- var (- inuse, unused, total int- totalSize int64- errs []error- )-- _, iu, un, tot, err := h.dockerService.GetAllImages(ctx)- if err != nil {- errs = append(errs, errors.WrapIf(err, "get images"))- } else {- inuse, unused, total = iu, un, tot- }-- sz, err := h.imageService.GetTotalImageSize(ctx)+ _, counts, err := h.dockerService.GetAllImages(ctx)if err != nil {- errs = append(errs, errors.WrapIf(err, "get total image size"))- } else {- totalSize = sz- }-- if len(errs) > 0 {- return nil, huma.Error500InternalServerError(errors.WithMessage(stderrors.Join(errs...), "Failed to get image usage counts").Error())+ return nil, huma.Error500InternalServerError(errors.WithMessage(err, "Failed to get image usage counts").Error())}return &GetImageUsageCountsOutput{Body: base.ApiResponse[image.UsageCounts]{Success: true,- Data: image.UsageCounts{- Inuse: inuse,- Unused: unused,- Total: total,- TotalSize: totalSize,- },+ Data: counts,},}, nil}diff --git a/backend/internal/image/image.go b/backend/internal/image/image.goindex 4eb0d96ca..d9926efa1 100644--- a/backend/internal/image/image.go+++ b/backend/internal/image/image.go@@ -802,20 +802,6 @@ func convertLabels(labels map[string]string) map[string]any {return result}-func (s *ImageService) GetTotalImageSize(ctx context.Context) (int64, error) {- images, err := s.dockerService.ListImages(ctx)- if err != nil {- return 0, errors.WrapIf(err, "failed to list images")- }-- var total int64- for _, img := range images {- total += img.Size- }-- return total, nil-}-// projectIDCacheTTL bounds how long a snapshot of all (name → id) project rows is reused.// Dashboard polls frequently; the projects table changes rarely, so a short TTL is safe.const projectIDCacheTTL = 5 * time.Seconddiff --git a/backend/internal/vulnerability/vulnerability_query.go b/backend/internal/vulnerability/vulnerability_query.goindex 2c43cb914..bb7ed4727 100644--- a/backend/internal/vulnerability/vulnerability_query.go+++ b/backend/internal/vulnerability/vulnerability_query.go@@ -157,11 +157,11 @@ func (s *VulnerabilityService) GetEnvironmentSummary(ctx context.Context) (*vuln}if s.dockerService != nil {- _, _, _, total, err := s.dockerService.GetAllImages(ctx)+ _, counts, err := s.dockerService.GetAllImages(ctx)if err != nil {slog.WarnContext(ctx, "failed to list images for vulnerability summary", "error", err)} else {- summary.TotalImages = total+ summary.TotalImages = counts.Total}}