Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: stop loading every scan blob to serve one list page (#3610)
Kyle Mendell0e3fa5214bbb
diff --git a/backend/internal/vulnerability/service_test.go b/backend/internal/vulnerability/service_test.goindex 10af82e2b..1baadbba7 100644--- a/backend/internal/vulnerability/service_test.go+++ b/backend/internal/vulnerability/service_test.go@@ -222,17 +222,6 @@ func TestApplyListAllVulnerabilityRecordPrefiltersInternal_NoPossibleMatches(t *require.True(t, ok)}-func TestEstimatedVulnerabilityRecordCountInternal(t *testing.T) {- records := []models.VulnerabilityScanRecord{- {TotalCount: 5},- {TotalCount: 0},- {TotalCount: -3},- {TotalCount: 12},- }-- require.Equal(t, 17, estimatedVulnerabilityRecordCountInternal(records))-}-func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *testing.T) {ctx := context.Background()db := setupVulnerabilityScanTestDB(t)@@ -387,3 +376,68 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {require.NoError(t, err)require.Equal(t, 4, count)}++// TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilters+// verifies the batched decode path preserves the old in-memory pipeline's+// pagination semantics: TotalItems reflects the search/filter matches,+// GrandTotalItems reflects every non-ignored CVE, and the page is sliced+// from the sorted matches.+func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFilters(t *testing.T) {+ ctx := context.Background()+ db := setupVulnerabilityScanTestDB(t)+ svc := &VulnerabilityService{db: db}++ vulnsA := []vulnerability.Vulnerability{+ {VulnerabilityID: "CVE-A1", PkgName: "openssl", InstalledVersion: "1.0.0", Severity: vulnerability.SeverityCritical},+ {VulnerabilityID: "CVE-A2", PkgName: "zlib", InstalledVersion: "1.2.0", Severity: vulnerability.SeverityLow},+ }+ rawVulnsA, err := json.Marshal(vulnsA)+ require.NoError(t, err)++ vulnsB := []vulnerability.Vulnerability{+ {VulnerabilityID: "CVE-B1", PkgName: "glibc", InstalledVersion: "2.38", Severity: vulnerability.SeverityHigh},+ {VulnerabilityID: "CVE-B2", PkgName: "less", InstalledVersion: "600", Severity: vulnerability.SeverityMedium},+ }+ rawVulnsB, err := json.Marshal(vulnsB)+ require.NoError(t, err)++ now := time.Now()+ records := []models.VulnerabilityScanRecord{+ {ID: "sha256:img-a", ImageName: "nginx:latest", Status: models.ScanStatusCompleted, ScanTime: now, CriticalCount: 1, LowCount: 1, TotalCount: 2, Vulnerabilities: models.StringSlice{string(rawVulnsA)}},+ {ID: "sha256:img-b", ImageName: "redis:7", Status: models.ScanStatusCompleted, ScanTime: now, HighCount: 1, MediumCount: 1, TotalCount: 2, Vulnerabilities: models.StringSlice{string(rawVulnsB)}},+ }+ for i := range records {+ require.NoError(t, db.Create(&records[i]).Error)+ }++ // Ignore the medium CVE on img-b: it must vanish from GrandTotalItems.+ require.NoError(t, db.Create(&models.VulnerabilityIgnore{+ ID: "ig-1", EnvironmentID: "env-1", ImageID: "sha256:img-b",+ VulnerabilityID: "CVE-B2", PkgName: "less", InstalledVersion: "600", CreatedBy: "tester",+ }).Error)++ // Severity filter critical,high with page size 1, sorted by severity desc:+ // matches are CVE-A1 (critical) and CVE-B1 (high); the first page holds+ // the critical one.+ items, page, err := svc.ListAllVulnerabilities(ctx, "env-1", pagination.QueryParams{+ Params: pagination.Params{Start: 0, Limit: 1},+ SortParams: pagination.SortParams{Sort: "severity", Order: pagination.SortDesc},+ Filters: map[string]string{"severity": "critical,high"},+ })+ require.NoError(t, err)+ require.Len(t, items, 1)+ require.Equal(t, "CVE-A1", items[0].VulnerabilityID)+ require.EqualValues(t, 2, page.TotalItems)+ require.EqualValues(t, 3, page.GrandTotalItems)++ // Search matches per-CVE fields decoded from the blobs.+ items, page, err = svc.ListAllVulnerabilities(ctx, "env-1", pagination.QueryParams{+ Params: pagination.Params{Start: 0, Limit: 20},+ SearchQuery: pagination.SearchQuery{Search: "glibc"},+ })+ require.NoError(t, err)+ require.Len(t, items, 1)+ require.Equal(t, "CVE-B1", items[0].VulnerabilityID)+ require.EqualValues(t, 1, page.TotalItems)+ require.EqualValues(t, 3, page.GrandTotalItems)+}diff --git a/backend/internal/vulnerability/vulnerability_query.go b/backend/internal/vulnerability/vulnerability_query.goindex 600f2676f..2c43cb914 100644--- a/backend/internal/vulnerability/vulnerability_query.go+++ b/backend/internal/vulnerability/vulnerability_query.go@@ -4,7 +4,6 @@ import ("context""encoding/json/v2""log/slog"- "math""slices""sort""strings"@@ -321,45 +320,71 @@ func (s *VulnerabilityService) ListAllVulnerabilities(ctx context.Context, envIDreturn filtered.Items, response, nil}- var records []models.VulnerabilityScanRecord- if err := recordsQuery.Find(&records).Error; err != nil {- return nil, pagination.Response{}, errors.WrapIf(err, "failed to list vulnerability scans")- }-ignoredKeys, err := s.getIgnoredVulnerabilityKeySetByEnvironmentInternal(ctx, envID)if err != nil {slog.WarnContext(ctx, "failed to filter ignored vulnerabilities", "error", err)}- items := make([]vulnerability.VulnerabilityWithImage, 0, estimatedVulnerabilityRecordCountInternal(records))- for _, record := range records {- if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {- continue- }+ config := listAllVulnerabilitiesPaginationConfigInternal()- var vulns []vulnerability.Vulnerability- if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {- slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err, "image_id", record.ID)- continue- }+ // Stream records in batches so at most one batch of scan blobs (up to+ // ~2 MiB each) is held at a time, and apply the search term and filters+ // while decoding so only matching CVEs are retained for sorting — the+ // old path materialized every CVE of every image to serve one page.+ var items []vulnerability.VulnerabilityWithImage+ totalAvailable := 0+ var batch []models.VulnerabilityScanRecord+ err = recordsQuery.FindInBatches(&batch, listAllVulnerabilityRecordBatchSizeInternal, func(*gorm.DB, int) error {+ for _, record := range batch {+ if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {+ continue+ }- for _, vulnEntry := range vulns {- if len(ignoredKeys) > 0 {- key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)- if _, isIgnored := ignoredKeys[key]; isIgnored {- continue- }+ var vulns []vulnerability.Vulnerability+ if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {+ slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err, "image_id", record.ID)+ continue}- items = append(items, vulnerability.VulnerabilityWithImage{- Vulnerability: vulnEntry,- ImageID: record.ID,- ImageName: record.ImageName,- })+ for _, vulnEntry := range vulns {+ if len(ignoredKeys) > 0 {+ key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)+ if _, isIgnored := ignoredKeys[key]; isIgnored {+ continue+ }+ }++ item := vulnerability.VulnerabilityWithImage{+ Vulnerability: vulnEntry,+ ImageID: record.ID,+ ImageName: record.ImageName,+ }+ totalAvailable+++ if pagination.MatchesSearchAndFilters(item, params, config) {+ items = append(items, item)+ }+ }}+ return nil+ }).Error+ if err != nil {+ return nil, pagination.Response{}, errors.WrapIf(err, "failed to list vulnerability scans")}- config := pagination.Config[vulnerability.VulnerabilityWithImage]{+ // Search and filters were already applied during decode; only sorting+ // and page slicing remain.+ page := pagination.OrderAndPaginate(items, params, config)+ response := pagination.BuildResponse(int64(len(items)), int64(totalAvailable), params)++ return page, response, nil+}++// listAllVulnerabilityRecordBatchSizeInternal bounds how many scan records+// (each carrying a vulnerability blob of up to ~2 MiB) are loaded at once.+const listAllVulnerabilityRecordBatchSizeInternal = 25++func listAllVulnerabilitiesPaginationConfigInternal() pagination.Config[vulnerability.VulnerabilityWithImage] {+ return pagination.Config[vulnerability.VulnerabilityWithImage]{SearchAccessors: []pagination.SearchAccessor[vulnerability.VulnerabilityWithImage]{func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.VulnerabilityID, nil },func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.PkgName, nil },@@ -424,11 +449,6 @@ func (s *VulnerabilityService) ListAllVulnerabilities(ctx context.Context, envID},},}-- filtered := pagination.SearchOrderAndPaginate(items, params, config)- response := pagination.BuildResponse(filtered.TotalCount, filtered.TotalAvailable, params)-- return filtered.Items, response, nil}// ListAllVulnerabilityImageOptions returns unique image names (or image IDs when name is empty)@@ -567,26 +587,6 @@ func splitNonEmptyCSVInternal(value string) []string {return parts}-func estimatedVulnerabilityRecordCountInternal(records []models.VulnerabilityScanRecord) int {- if len(records) == 0 {- return 0- }-- total := 0- for i := range records {- if records[i].TotalCount <= 0 {- continue- }-- if total > math.MaxInt-records[i].TotalCount {- return math.MaxInt- }- total += records[i].TotalCount- }-- return total-}-func severityRankInternal(severity vulnerability.Severity) int {switch severity {case vulnerability.SeverityCritical:diff --git a/backend/pkg/pagination/filters.go b/backend/pkg/pagination/filters.goindex 11379a585..b4caf524d 100644--- a/backend/pkg/pagination/filters.go+++ b/backend/pkg/pagination/filters.go@@ -24,18 +24,49 @@ func SearchOrderAndPaginate[T any](items []T, params QueryParams, searchConfig Citems = searchFn(items, params.SearchQuery, searchConfig.SearchAccessors)items = filterFn(items, params.Filters, searchConfig.FilterAccessors)- items = sortFunction(items, params.SortParams, searchConfig.SortBindings)-- totalCount := len(items)- items = paginateItemsFunction(items, params.Params)return FilterResult[T]{- Items: items,- TotalCount: int64(totalCount),+ Items: OrderAndPaginate(items, params, searchConfig),+ TotalCount: int64(len(items)),TotalAvailable: int64(totalAvailable),}}+// OrderAndPaginate sorts items and slices the requested page without applying+// the search term or filters — for callers that already filtered while+// building the item list (e.g. dropping non-matches during an incremental+// decode) and only need ordering plus the page cut.+func OrderAndPaginate[T any](items []T, params QueryParams, config Config[T]) []T {+ items = sortFunction(items, params.SortParams, config.SortBindings)+ return paginateItemsFunction(items, params.Params)+}++// MatchesSearchAndFilters reports whether a single item passes params' search+// term and filters under config — the same predicate SearchOrderAndPaginate+// applies. It lets callers that decode large payloads incrementally drop+// non-matching items as they go instead of materializing everything first.+func MatchesSearchAndFilters[T any](item T, params QueryParams, config Config[T]) bool {+ search := strings.ToLower(strings.TrimSpace(params.Search))+ if search != "" {+ matched := false+ for _, accessor := range config.SearchAccessors {+ value, err := accessor(item)+ if err == nil && strings.Contains(strings.ToLower(value), search) {+ matched = true+ break+ }+ }+ if !matched {+ return false+ }+ }++ if len(params.Filters) == 0 {+ return true+ }+ return itemMatches(item, params.Filters, config.FilterAccessors)+}+func filterFn[T any](items []T, filters map[string]string, accessors []FilterAccessor[T]) []T {if len(filters) == 0 {return items