Skip to content

Modern Docker Management, Designed for Everyone

fix: stop loading every scan blob to serve one list page (#3610)

Kyle Mendell0e3fa5214bbb
diff --git a/backend/internal/vulnerability/service_test.go b/backend/internal/vulnerability/service_test.go
index 10af82e2b..1baadbba7 100644
--- a/backend/internal/vulnerability/service_test.go
+++ b/backend/internal/vulnerability/service_test.go
@@ -222,17 +222,6 @@ func TestApplyListAllVulnerabilityRecordPrefiltersInternal_NoPossibleMatches(t *
require.True(t, ok)
}
-func TestEstimatedVulnerabilityRecordCountInternal(t *testing.T) {
- records := []models.VulnerabilityScanRecord{
- {TotalCount: 5},
- {TotalCount: 0},
- {TotalCount: -3},
- {TotalCount: 12},
- }
-
- require.Equal(t, 17, estimatedVulnerabilityRecordCountInternal(records))
-}
-
func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *testing.T) {
ctx := context.Background()
db := setupVulnerabilityScanTestDB(t)
@@ -387,3 +376,68 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 4, count)
}
+
+// TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilters
+// verifies the batched decode path preserves the old in-memory pipeline's
+// pagination semantics: TotalItems reflects the search/filter matches,
+// GrandTotalItems reflects every non-ignored CVE, and the page is sliced
+// from the sorted matches.
+func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilters(t *testing.T) {
+ ctx := context.Background()
+ db := setupVulnerabilityScanTestDB(t)
+ svc := &VulnerabilityService{db: db}
+
+ vulnsA := []vulnerability.Vulnerability{
+ {VulnerabilityID: "CVE-A1", PkgName: "openssl", InstalledVersion: "1.0.0", Severity: vulnerability.SeverityCritical},
+ {VulnerabilityID: "CVE-A2", PkgName: "zlib", InstalledVersion: "1.2.0", Severity: vulnerability.SeverityLow},
+ }
+ rawVulnsA, err := json.Marshal(vulnsA)
+ require.NoError(t, err)
+
+ vulnsB := []vulnerability.Vulnerability{
+ {VulnerabilityID: "CVE-B1", PkgName: "glibc", InstalledVersion: "2.38", Severity: vulnerability.SeverityHigh},
+ {VulnerabilityID: "CVE-B2", PkgName: "less", InstalledVersion: "600", Severity: vulnerability.SeverityMedium},
+ }
+ rawVulnsB, err := json.Marshal(vulnsB)
+ require.NoError(t, err)
+
+ now := time.Now()
+ records := []models.VulnerabilityScanRecord{
+ {ID: "sha256:img-a", ImageName: "nginx:latest", Status: models.ScanStatusCompleted, ScanTime: now, CriticalCount: 1, LowCount: 1, TotalCount: 2, Vulnerabilities: models.StringSlice{string(rawVulnsA)}},
+ {ID: "sha256:img-b", ImageName: "redis:7", Status: models.ScanStatusCompleted, ScanTime: now, HighCount: 1, MediumCount: 1, TotalCount: 2, Vulnerabilities: models.StringSlice{string(rawVulnsB)}},
+ }
+ for i := range records {
+ require.NoError(t, db.Create(&records[i]).Error)
+ }
+
+ // Ignore the medium CVE on img-b: it must vanish from GrandTotalItems.
+ require.NoError(t, db.Create(&models.VulnerabilityIgnore{
+ ID: "ig-1", EnvironmentID: "env-1", ImageID: "sha256:img-b",
+ VulnerabilityID: "CVE-B2", PkgName: "less", InstalledVersion: "600", CreatedBy: "tester",
+ }).Error)
+
+ // Severity filter critical,high with page size 1, sorted by severity desc:
+ // matches are CVE-A1 (critical) and CVE-B1 (high); the first page holds
+ // the critical one.
+ items, page, err := svc.ListAllVulnerabilities(ctx, "env-1", pagination.QueryParams{
+ Params: pagination.Params{Start: 0, Limit: 1},
+ SortParams: pagination.SortParams{Sort: "severity", Order: pagination.SortDesc},
+ Filters: map[string]string{"severity": "critical,high"},
+ })
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.Equal(t, "CVE-A1", items[0].VulnerabilityID)
+ require.EqualValues(t, 2, page.TotalItems)
+ require.EqualValues(t, 3, page.GrandTotalItems)
+
+ // Search matches per-CVE fields decoded from the blobs.
+ items, page, err = svc.ListAllVulnerabilities(ctx, "env-1", pagination.QueryParams{
+ Params: pagination.Params{Start: 0, Limit: 20},
+ SearchQuery: pagination.SearchQuery{Search: "glibc"},
+ })
+ require.NoError(t, err)
+ require.Len(t, items, 1)
+ require.Equal(t, "CVE-B1", items[0].VulnerabilityID)
+ require.EqualValues(t, 1, page.TotalItems)
+ require.EqualValues(t, 3, page.GrandTotalItems)
+}
diff --git a/backend/internal/vulnerability/vulnerability_query.go b/backend/internal/vulnerability/vulnerability_query.go
index 600f2676f..2c43cb914 100644
--- a/backend/internal/vulnerability/vulnerability_query.go
+++ b/backend/internal/vulnerability/vulnerability_query.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json/v2"
"log/slog"
- "math"
"slices"
"sort"
"strings"
@@ -321,45 +320,71 @@ func (s *VulnerabilityService) ListAllVulnerabilities(ctx context.Context, envID
return filtered.Items, response, nil
}
- var records []models.VulnerabilityScanRecord
- if err := recordsQuery.Find(&records).Error; err != nil {
- return nil, pagination.Response{}, errors.WrapIf(err, "failed to list vulnerability scans")
- }
-
ignoredKeys, err := s.getIgnoredVulnerabilityKeySetByEnvironmentInternal(ctx, envID)
if err != nil {
slog.WarnContext(ctx, "failed to filter ignored vulnerabilities", "error", err)
}
- items := make([]vulnerability.VulnerabilityWithImage, 0, estimatedVulnerabilityRecordCountInternal(records))
- for _, record := range records {
- if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {
- continue
- }
+ config := listAllVulnerabilitiesPaginationConfigInternal()
- var vulns []vulnerability.Vulnerability
- if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {
- slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err, "image_id", record.ID)
- continue
- }
+ // Stream records in batches so at most one batch of scan blobs (up to
+ // ~2 MiB each) is held at a time, and apply the search term and filters
+ // while decoding so only matching CVEs are retained for sorting — the
+ // old path materialized every CVE of every image to serve one page.
+ var items []vulnerability.VulnerabilityWithImage
+ totalAvailable := 0
+ var batch []models.VulnerabilityScanRecord
+ err = recordsQuery.FindInBatches(&batch, listAllVulnerabilityRecordBatchSizeInternal, func(*gorm.DB, int) error {
+ for _, record := range batch {
+ if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {
+ continue
+ }
- for _, vulnEntry := range vulns {
- if len(ignoredKeys) > 0 {
- key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)
- if _, isIgnored := ignoredKeys[key]; isIgnored {
- continue
- }
+ var vulns []vulnerability.Vulnerability
+ if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {
+ slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err, "image_id", record.ID)
+ continue
}
- items = append(items, vulnerability.VulnerabilityWithImage{
- Vulnerability: vulnEntry,
- ImageID: record.ID,
- ImageName: record.ImageName,
- })
+ for _, vulnEntry := range vulns {
+ if len(ignoredKeys) > 0 {
+ key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)
+ if _, isIgnored := ignoredKeys[key]; isIgnored {
+ continue
+ }
+ }
+
+ item := vulnerability.VulnerabilityWithImage{
+ Vulnerability: vulnEntry,
+ ImageID: record.ID,
+ ImageName: record.ImageName,
+ }
+ totalAvailable++
+ if pagination.MatchesSearchAndFilters(item, params, config) {
+ items = append(items, item)
+ }
+ }
}
+ return nil
+ }).Error
+ if err != nil {
+ return nil, pagination.Response{}, errors.WrapIf(err, "failed to list vulnerability scans")
}
- config := pagination.Config[vulnerability.VulnerabilityWithImage]{
+ // Search and filters were already applied during decode; only sorting
+ // and page slicing remain.
+ page := pagination.OrderAndPaginate(items, params, config)
+ response := pagination.BuildResponse(int64(len(items)), int64(totalAvailable), params)
+
+ return page, response, nil
+}
+
+// listAllVulnerabilityRecordBatchSizeInternal bounds how many scan records
+// (each carrying a vulnerability blob of up to ~2 MiB) are loaded at once.
+const listAllVulnerabilityRecordBatchSizeInternal = 25
+
+func listAllVulnerabilitiesPaginationConfigInternal() pagination.Config[vulnerability.VulnerabilityWithImage] {
+ return pagination.Config[vulnerability.VulnerabilityWithImage]{
SearchAccessors: []pagination.SearchAccessor[vulnerability.VulnerabilityWithImage]{
func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.VulnerabilityID, nil },
func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.PkgName, nil },
@@ -424,11 +449,6 @@ func (s *VulnerabilityService) ListAllVulnerabilities(ctx context.Context, envID
},
},
}
-
- filtered := pagination.SearchOrderAndPaginate(items, params, config)
- response := pagination.BuildResponse(filtered.TotalCount, filtered.TotalAvailable, params)
-
- return filtered.Items, response, nil
}
// ListAllVulnerabilityImageOptions returns unique image names (or image IDs when name is empty)
@@ -567,26 +587,6 @@ func splitNonEmptyCSVInternal(value string) []string {
return parts
}
-func estimatedVulnerabilityRecordCountInternal(records []models.VulnerabilityScanRecord) int {
- if len(records) == 0 {
- return 0
- }
-
- total := 0
- for i := range records {
- if records[i].TotalCount <= 0 {
- continue
- }
-
- if total > math.MaxInt-records[i].TotalCount {
- return math.MaxInt
- }
- total += records[i].TotalCount
- }
-
- return total
-}
-
func severityRankInternal(severity vulnerability.Severity) int {
switch severity {
case vulnerability.SeverityCritical:
diff --git a/backend/pkg/pagination/filters.go b/backend/pkg/pagination/filters.go
index 11379a585..b4caf524d 100644
--- a/backend/pkg/pagination/filters.go
+++ b/backend/pkg/pagination/filters.go
@@ -24,18 +24,49 @@ func SearchOrderAndPaginate[T any](items []T, params QueryParams, searchConfig C
items = searchFn(items, params.SearchQuery, searchConfig.SearchAccessors)
items = filterFn(items, params.Filters, searchConfig.FilterAccessors)
- items = sortFunction(items, params.SortParams, searchConfig.SortBindings)
-
- totalCount := len(items)
- items = paginateItemsFunction(items, params.Params)
return FilterResult[T]{
- Items: items,
- TotalCount: int64(totalCount),
+ Items: OrderAndPaginate(items, params, searchConfig),
+ TotalCount: int64(len(items)),
TotalAvailable: int64(totalAvailable),
}
}
+// OrderAndPaginate sorts items and slices the requested page without applying
+// the search term or filters — for callers that already filtered while
+// building the item list (e.g. dropping non-matches during an incremental
+// decode) and only need ordering plus the page cut.
+func OrderAndPaginate[T any](items []T, params QueryParams, config Config[T]) []T {
+ items = sortFunction(items, params.SortParams, config.SortBindings)
+ return paginateItemsFunction(items, params.Params)
+}
+
+// MatchesSearchAndFilters reports whether a single item passes params' search
+// term and filters under config — the same predicate SearchOrderAndPaginate
+// applies. It lets callers that decode large payloads incrementally drop
+// non-matching items as they go instead of materializing everything first.
+func MatchesSearchAndFilters[T any](item T, params QueryParams, config Config[T]) bool {
+ search := strings.ToLower(strings.TrimSpace(params.Search))
+ if search != "" {
+ matched := false
+ for _, accessor := range config.SearchAccessors {
+ value, err := accessor(item)
+ if err == nil && strings.Contains(strings.ToLower(value), search) {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ return false
+ }
+ }
+
+ if len(params.Filters) == 0 {
+ return true
+ }
+ return itemMatches(item, params.Filters, config.FilterAccessors)
+}
+
func filterFn[T any](items []T, filters map[string]string, accessors []FilterAccessor[T]) []T {
if len(filters) == 0 {
return items