Skip to content

Modern Docker Management, Designed for Everyone

perf: serve the CVE list from a normalized table (#3825)

Kyle Mendell31a9896dfb05
diff --git a/backend/internal/bootstrap/bootstrap.go b/backend/internal/bootstrap/bootstrap.go
index 2a5fe1f58..486cce51e 100644
--- a/backend/internal/bootstrap/bootstrap.go
+++ b/backend/internal/bootstrap/bootstrap.go
@@ -233,6 +233,7 @@ func initializeStartupState(p initializeStartupStateParams) {
}
initializeGitOpsStartupStateInternal(appCtx, p.GitOpsSync)
p.Vulnerability.ImportLegacyReportFiles(appCtx)
+ p.Vulnerability.BackfillScanItems(appCtx)
if p.Project != nil {
if err := p.Project.RecoverProjectRenameJournals(appCtx); err != nil {
slog.WarnContext(appCtx, "Failed to recover interrupted project rename operations on startup", "error", err)
diff --git a/backend/internal/vulnerability/model.go b/backend/internal/vulnerability/model.go
index 6cb83752c..ffc118922 100644
--- a/backend/internal/vulnerability/model.go
+++ b/backend/internal/vulnerability/model.go
@@ -5,6 +5,7 @@ import (
"uuid"
"github.com/getarcaneapp/arcane/backend/v2/internal/database"
+ "github.com/getarcaneapp/arcane/types/v2/vulnerability"
"gorm.io/gorm"
)
@@ -67,6 +68,34 @@ func (VulnerabilityReportRecord) TableName() string {
return "vulnerability_reports"
}
+// VulnerabilityScanItem is one CVE finding from a completed scan, normalized for list queries.
+type VulnerabilityScanItem struct {
+ ID int64 `json:"id" gorm:"primaryKey;autoIncrement;column:id"`
+ ImageID string `json:"imageId" gorm:"column:image_id;index"`
+ VulnerabilityID string `json:"vulnerabilityId" gorm:"column:vulnerability_id;index"`
+ PkgName string `json:"pkgName" gorm:"column:pkg_name"`
+ InstalledVersion string `json:"installedVersion" gorm:"column:installed_version"`
+ FixedVersion string `json:"fixedVersion" gorm:"column:fixed_version"`
+ Severity string `json:"severity" gorm:"column:severity;index"`
+ SeverityRank int `json:"severityRank" gorm:"column:severity_rank"`
+ PkgClass string `json:"pkgClass" gorm:"column:pkg_class"`
+ PkgType string `json:"pkgType" gorm:"column:pkg_type"`
+ Title string `json:"title" gorm:"column:title"`
+ Details *string `json:"-" gorm:"column:details"`
+}
+
+func (VulnerabilityScanItem) TableName() string {
+ return "vulnerability_scan_items"
+}
+
+type vulnerabilityScanItemDetailsInternal struct {
+ Description string `json:"description,omitempty"`
+ References []string `json:"references,omitempty"`
+ CVSS *vulnerability.CVSSInfo `json:"cvss,omitempty"`
+ PublishedDate *time.Time `json:"publishedDate,omitempty"`
+ LastModifiedDate *time.Time `json:"lastModifiedDate,omitempty"`
+}
+
// Scan status constants
const (
ScanStatusPending = "pending"
diff --git a/backend/internal/vulnerability/service_test.go b/backend/internal/vulnerability/service_test.go
index 76e06f659..f9b86e0d7 100644
--- a/backend/internal/vulnerability/service_test.go
+++ b/backend/internal/vulnerability/service_test.go
@@ -133,7 +133,7 @@ func setupVulnerabilityScanTestDB(t *testing.T) *database.DB {
dsn := fmt.Sprintf("file:vulnerability-scan-test-%d?mode=memory&cache=shared", time.Now().UnixNano())
db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{})
require.NoError(t, err)
- require.NoError(t, db.AutoMigrate(&VulnerabilityScanRecord{}, &VulnerabilityIgnore{}))
+ require.NoError(t, db.AutoMigrate(&VulnerabilityScanRecord{}, &VulnerabilityIgnore{}, &VulnerabilityScanItem{}))
return &database.DB{DB: db}
}
@@ -285,6 +285,7 @@ func TestVulnerabilityService_ListAllVulnerabilities_FiltersIgnoredInline(t *tes
Vulnerabilities: database.StringSlice{string(rawVulns)},
}
require.NoError(t, db.Create(&scan).Error)
+ svc.BackfillScanItems(ctx)
ignore := VulnerabilityIgnore{
EnvironmentID: "env-1",
@@ -391,6 +392,8 @@ func TestVulnerabilityService_ActionableCountExcludingIgnored(t *testing.T) {
require.NoError(t, db.Create(&records[i]).Error)
}
+ svc.BackfillScanItems(ctx)
+
// No ignores: pure indexed sum, no blob decode.
count, err := svc.ActionableCountExcludingIgnored(ctx)
require.NoError(t, err)
@@ -447,6 +450,8 @@ func TestVulnerabilityService_ListAllVulnerabilities_PagesAndCountsWithInlineFil
require.NoError(t, db.Create(&records[i]).Error)
}
+ svc.BackfillScanItems(ctx)
+
// Ignore the medium CVE on img-b: it must vanish from GrandTotalItems.
require.NoError(t, db.Create(&VulnerabilityIgnore{
ID: "ig-1", EnvironmentID: "env-1", ImageID: "sha256:img-b",
diff --git a/backend/internal/vulnerability/vulnerability_ignores.go b/backend/internal/vulnerability/vulnerability_ignores.go
index 466c9986c..9304b36d7 100644
--- a/backend/internal/vulnerability/vulnerability_ignores.go
+++ b/backend/internal/vulnerability/vulnerability_ignores.go
@@ -17,31 +17,6 @@ func vulnerabilityIgnoreKeyInternal(imageID, vulnerabilityID, pkgName, installed
return imageID + ":" + vulnerabilityID + ":" + pkgName + ":" + installedVersion
}
-// getIgnoredVulnerabilityKeySetByEnvironmentInternal returns the environment's
-// ignore records keyed by vulnerability identity, mapping to the ignore ID.
-func (s *VulnerabilityService) getIgnoredVulnerabilityKeySetByEnvironmentInternal(ctx context.Context, envID string) (map[string]string, error) {
- if s.db == nil {
- return nil, nil
- }
-
- var ignores []VulnerabilityIgnore
- if err := s.db.WithContext(ctx).Where("environment_id = ?", envID).Find(&ignores).Error; err != nil {
- return nil, errors.WrapIf(err, "failed to get ignore records")
- }
-
- if len(ignores) == 0 {
- return nil, nil
- }
-
- ignoredKeys := make(map[string]string, len(ignores))
- for _, ignore := range ignores {
- key := vulnerabilityIgnoreKeyInternal(ignore.ImageID, ignore.VulnerabilityID, ignore.PkgName, ignore.InstalledVersion)
- ignoredKeys[key] = ignore.ID
- }
-
- return ignoredKeys, nil
-}
-
func (s *VulnerabilityService) filterIgnoredVulnerabilitiesForImage(
ctx context.Context,
imageID string,
diff --git a/backend/internal/vulnerability/vulnerability_query.go b/backend/internal/vulnerability/vulnerability_query.go
index 32b3982f7..513c46c9b 100644
--- a/backend/internal/vulnerability/vulnerability_query.go
+++ b/backend/internal/vulnerability/vulnerability_query.go
@@ -4,7 +4,6 @@ import (
"context"
"encoding/json/v2"
"log/slog"
- "slices"
"sort"
"strings"
@@ -12,6 +11,7 @@ import (
"github.com/samber/mo"
"gorm.io/gorm"
+ "gorm.io/gorm/clause"
"github.com/getarcaneapp/arcane/backend/v2/internal/common"
"github.com/getarcaneapp/arcane/backend/v2/pkg/pagination"
@@ -146,24 +146,23 @@ func (s *VulnerabilityService) ListVulnerabilities(ctx context.Context, imageID
// GetEnvironmentSummary returns aggregated vulnerability counts across all images.
func (s *VulnerabilityService) GetEnvironmentSummary(ctx context.Context) (*vulnerability.EnvironmentVulnerabilitySummary, error) {
- if s.db != nil && s.dockerService != nil {
- if deleted, err := s.CleanupOrphanedScanRecords(ctx); err != nil {
- slog.WarnContext(ctx, "failed to cleanup orphaned vulnerability scan records", "error", err)
- } else if deleted > 0 {
- slog.DebugContext(ctx, "cleaned up orphaned vulnerability scan records", "deleted", deleted)
- }
- }
-
summary := &vulnerability.EnvironmentVulnerabilitySummary{
Summary: &vulnerability.SeveritySummary{},
}
+ var currentImageIDs []string
+ haveCurrentImages := false
if s.dockerService != nil {
- _, counts, err := s.dockerService.GetAllImages(ctx)
+ images, counts, err := s.dockerService.GetAllImages(ctx)
if err != nil {
slog.WarnContext(ctx, "failed to list images for vulnerability summary", "error", err)
} else {
summary.TotalImages = counts.Total
+ haveCurrentImages = true
+ currentImageIDs = make([]string, 0, len(images))
+ for i := range images {
+ currentImageIDs = append(currentImageIDs, images[i].ID)
+ }
}
}
@@ -182,7 +181,7 @@ func (s *VulnerabilityService) GetEnvironmentSummary(ctx context.Context) (*vuln
}
var agg aggregate
- err := s.db.WithContext(ctx).
+ aggQuery := s.db.WithContext(ctx).
Model(&VulnerabilityScanRecord{}).
Select(
"COALESCE(SUM(critical_count), 0) AS critical",
@@ -193,8 +192,11 @@ func (s *VulnerabilityService) GetEnvironmentSummary(ctx context.Context) (*vuln
"COALESCE(SUM(total_count), 0) AS total",
"COUNT(*) AS scanned",
).
- Where("status = ?", ScanStatusCompleted).
- Scan(&agg).Error
+ Where("status = ?", ScanStatusCompleted)
+ if haveCurrentImages {
+ aggQuery = aggQuery.Where("id IN ?", currentImageIDs)
+ }
+ err := aggQuery.Scan(&agg).Error
if err != nil {
return nil, errors.WrapIf(err, "failed to aggregate vulnerability summary")
}
@@ -212,89 +214,87 @@ func (s *VulnerabilityService) GetEnvironmentSummary(ctx context.Context) (*vuln
return summary, nil
}
+// currentImageIDsInternal lists daemon image IDs so queries skip scans of images removed outside Arcane; ok is false when the daemon could not be listed.
+func (s *VulnerabilityService) currentImageIDsInternal(ctx context.Context) (ids []string, ok bool) {
+ if s.dockerService == nil {
+ return nil, false
+ }
+ images, err := s.dockerService.ListImages(ctx)
+ if err != nil {
+ slog.WarnContext(ctx, "failed to list images for vulnerability query", "error", err)
+ return nil, false
+ }
+ ids = make([]string, 0, len(images))
+ for i := range images {
+ ids = append(ids, images[i].ID)
+ }
+ return ids, true
+}
+
// ActionableCountExcludingIgnored counts critical and high severity vulnerabilities
// across all completed scans, excluding any that have been ignored. This is used by the dashboard
// to show an accurate action item count that reflects ignore decisions.
//
-// Images without ignore records are counted from the indexed critical_count/
-// high_count columns; only images that have ignore rows decode their
-// vulnerability blob, so the common no-ignores dashboard poll never
-// unmarshals a scan payload.
+// Without ignore records the count comes from the indexed critical_count/
+// high_count columns; with ignore records it is a single COUNT over the
+// per-CVE item rows.
func (s *VulnerabilityService) ActionableCountExcludingIgnored(ctx context.Context) (int, error) {
if s.db == nil {
return 0, nil
}
- // Load all ignore records (not scoped to a single environment, since an
- // Arcane instance serves one environment and the dashboard does not carry
- // an environment ID through the service layer today).
- var ignores []VulnerabilityIgnore
- if err := s.db.WithContext(ctx).Find(&ignores).Error; err != nil {
- return 0, errors.WrapIf(err, "failed to get ignore records")
+ var ignoreCount int64
+ if err := s.db.WithContext(ctx).Model(&VulnerabilityIgnore{}).Count(&ignoreCount).Error; err != nil {
+ return 0, errors.WrapIf(err, "failed to count ignore records")
}
+ currentImageIDs, haveCurrentImages := s.currentImageIDsInternal(ctx)
- ignoredKeys := make(map[string]struct{}, len(ignores))
- ignoredImageIDs := make([]string, 0, len(ignores))
- for _, ig := range ignores {
- ignoredKeys[vulnerabilityIgnoreKeyInternal(ig.ImageID, ig.VulnerabilityID, ig.PkgName, ig.InstalledVersion)] = struct{}{}
- if !slices.Contains(ignoredImageIDs, ig.ImageID) {
- ignoredImageIDs = append(ignoredImageIDs, ig.ImageID)
+ if ignoreCount == 0 {
+ var indexedCount int64
+ sumQuery := s.db.WithContext(ctx).
+ Model(&VulnerabilityScanRecord{}).
+ Select("COALESCE(SUM(critical_count + high_count), 0)").
+ Where("status = ?", ScanStatusCompleted)
+ if haveCurrentImages {
+ sumQuery = sumQuery.Where("id IN ?", currentImageIDs)
+ }
+ if err := sumQuery.Scan(&indexedCount).Error; err != nil {
+ return 0, errors.WrapIf(err, "failed to sum vulnerability counts")
}
- }
-
- sumQuery := s.db.WithContext(ctx).
- Model(&VulnerabilityScanRecord{}).
- Select("COALESCE(SUM(critical_count + high_count), 0)").
- Where("status = ?", ScanStatusCompleted)
- if len(ignoredImageIDs) > 0 {
- sumQuery = sumQuery.Where("id NOT IN ?", ignoredImageIDs)
- }
-
- var indexedCount int64
- if err := sumQuery.Scan(&indexedCount).Error; err != nil {
- return 0, errors.WrapIf(err, "failed to sum vulnerability counts")
- }
-
- if len(ignoredImageIDs) == 0 {
return int(indexedCount), nil
}
- var records []VulnerabilityScanRecord
- err := s.db.WithContext(ctx).
- Select("id", "vulnerabilities").
- Where("status = ?", ScanStatusCompleted).
- Where("critical_count + high_count > 0").
- Where("id IN ?", ignoredImageIDs).
- Find(&records).Error
+ countQuery := s.db.WithContext(ctx).
+ Model(&VulnerabilityScanItem{}).
+ Joins("JOIN vulnerability_scans ON vulnerability_scans.id = vulnerability_scan_items.image_id AND vulnerability_scans.status = ?", ScanStatusCompleted).
+ Where("vulnerability_scan_items.severity IN ?", []string{string(vulnerability.SeverityCritical), string(vulnerability.SeverityHigh)})
+ if haveCurrentImages {
+ countQuery = countQuery.Where("vulnerability_scan_items.image_id IN ?", currentImageIDs)
+ }
+ var count int64
+ err := countQuery.
+ Where("NOT EXISTS (SELECT 1 FROM vulnerability_ignores WHERE vulnerability_ignores.image_id = vulnerability_scan_items.image_id AND vulnerability_ignores.vulnerability_id = vulnerability_scan_items.vulnerability_id AND vulnerability_ignores.pkg_name = vulnerability_scan_items.pkg_name AND vulnerability_ignores.installed_version = vulnerability_scan_items.installed_version)").
+ Count(&count).Error
if err != nil {
- return 0, errors.WrapIf(err, "failed to list vulnerability scans")
+ return 0, errors.WrapIf(err, "failed to count actionable vulnerabilities")
}
+ return int(count), nil
+}
- count := int(indexedCount)
- for _, record := range records {
- if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {
- continue
- }
+type vulnerabilityScanItemRowInternal struct {
+ VulnerabilityScanItem
- var vulns []vulnerability.Vulnerability
- if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {
- slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err, "image_id", record.ID)
- continue
- }
-
- for _, vulnEntry := range vulns {
- if vulnEntry.Severity != vulnerability.SeverityCritical && vulnEntry.Severity != vulnerability.SeverityHigh {
- continue
- }
- key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)
- if _, isIgnored := ignoredKeys[key]; isIgnored {
- continue
- }
- count++
- }
- }
+ ImageName string `gorm:"column:image_name"`
+ IgnoreID string `gorm:"column:ignore_id"`
+}
- return count, nil
+var listAllVulnerabilitySortColumnsInternal = map[string]string{
+ "vulnerabilityId": "vulnerability_scan_items.vulnerability_id",
+ "pkgName": "vulnerability_scan_items.pkg_name",
+ "severity": "vulnerability_scan_items.severity_rank",
+ "installedVersion": "vulnerability_scan_items.installed_version",
+ "fixedVersion": "vulnerability_scan_items.fixed_version",
+ "imageName": "vulnerability_scans.image_name",
}
// ListAllVulnerabilities returns a paginated list of vulnerabilities across all scanned images.
@@ -307,158 +307,92 @@ func (s *VulnerabilityService) ListAllVulnerabilities(ctx context.Context, envID
return []vulnerability.VulnerabilityWithImage{}, pagination.BuildResponse(0, 0, params), nil
}
- recordsQuery := s.db.WithContext(ctx).
- Select("id", "image_name", "vulnerabilities", "total_count").
- Where("status = ?", ScanStatusCompleted).
- Where("total_count > 0")
- recordsQuery, hasPossibleMatches := applyListAllVulnerabilityRecordPrefiltersInternal(recordsQuery, params.Filters).Get()
- if !hasPossibleMatches {
- filtered := pagination.FilterResult[vulnerability.VulnerabilityWithImage]{
- Items: []vulnerability.VulnerabilityWithImage{},
- TotalCount: 0,
- TotalAvailable: 0,
- }
- response := pagination.BuildResponse(filtered.TotalCount, filtered.TotalAvailable, params)
- return filtered.Items, response, nil
- }
-
// "ignored" flips the list from active to ignored vulnerabilities, the same
// way the projects list swaps to archived projects.
onlyIgnored := params.Filters["ignored"] == "true"
delete(params.Filters, "ignored")
- ignoredKeys, err := s.getIgnoredVulnerabilityKeySetByEnvironmentInternal(ctx, envID)
- if err != nil {
- slog.WarnContext(ctx, "failed to filter ignored vulnerabilities", "error", err)
- }
-
- config := listAllVulnerabilitiesPaginationConfigInternal()
-
- // Stream records in batches so at most one batch of scan blobs (up to
- // ~2 MiB each) is held at a time, and apply the search term and filters
- // while decoding so only matching CVEs are retained for sorting — the
- // old path materialized every CVE of every image to serve one page.
- var items []vulnerability.VulnerabilityWithImage
- totalAvailable := 0
- var batch []VulnerabilityScanRecord
- err = recordsQuery.FindInBatches(&batch, listAllVulnerabilityRecordBatchSizeInternal, func(*gorm.DB, int) error {
- for _, record := range batch {
- if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {
- continue
- }
+ base := s.db.WithContext(ctx).
+ Model(&VulnerabilityScanItem{}).
+ Select("vulnerability_scan_items.*", "vulnerability_scans.image_name AS image_name", "COALESCE(vulnerability_ignores.id, '') AS ignore_id").
+ Joins("JOIN vulnerability_scans ON vulnerability_scans.id = vulnerability_scan_items.image_id AND vulnerability_scans.status = ?", ScanStatusCompleted).
+ Joins("LEFT JOIN vulnerability_ignores ON vulnerability_ignores.environment_id = ? AND vulnerability_ignores.image_id = vulnerability_scan_items.image_id AND vulnerability_ignores.vulnerability_id = vulnerability_scan_items.vulnerability_id AND vulnerability_ignores.pkg_name = vulnerability_scan_items.pkg_name AND vulnerability_ignores.installed_version = vulnerability_scan_items.installed_version", envID)
+ if onlyIgnored {
+ base = base.Where("vulnerability_ignores.id IS NOT NULL")
+ } else {
+ base = base.Where("vulnerability_ignores.id IS NULL")
+ }
+ if currentImageIDs, ok := s.currentImageIDsInternal(ctx); ok {
+ base = base.Where("vulnerability_scan_items.image_id IN ?", currentImageIDs)
+ }
+ base = base.Session(&gorm.Session{})
- var vulns []vulnerability.Vulnerability
- if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {
- slog.WarnContext(ctx, "failed to unmarshal vulnerabilities", "error", err, "image_id", record.ID)
- continue
- }
+ var totalAvailable int64
+ if err := base.Count(&totalAvailable).Error; err != nil {
+ return nil, pagination.Response{}, errors.WrapIf(err, "failed to count vulnerabilities")
+ }
- for _, vulnEntry := range vulns {
- key := vulnerabilityIgnoreKeyInternal(record.ID, vulnEntry.VulnerabilityID, vulnEntry.PkgName, vulnEntry.InstalledVersion)
- ignoreID, isIgnored := ignoredKeys[key]
- if isIgnored != onlyIgnored {
- continue
- }
-
- item := vulnerability.VulnerabilityWithImage{
- Vulnerability: vulnEntry,
- Ignored: isIgnored,
- IgnoreID: ignoreID,
- ImageID: record.ID,
- ImageName: record.ImageName,
- }
- totalAvailable++
- if config.MatchesSearchAndFilters(item, params) {
- items = append(items, item)
- }
- }
+ q := pagination.ApplyFilter(base, "vulnerability_scan_items.severity", strings.ToUpper(params.Filters["severity"]))
+ if imageNameParts := splitNonEmptyCSVInternal(params.Filters["imageName"]); len(imageNameParts) > 0 {
+ clauses := make([]string, 0, len(imageNameParts))
+ args := make([]any, 0, len(imageNameParts))
+ for _, part := range imageNameParts {
+ clauses = append(clauses, "LOWER(vulnerability_scans.image_name) LIKE ?")
+ args = append(args, "%"+strings.ToLower(part)+"%")
}
- return nil
- }).Error
- if err != nil {
- return nil, pagination.Response{}, errors.WrapIf(err, "failed to list vulnerability scans")
+ q = q.Where("("+strings.Join(clauses, " OR ")+")", args...)
+ }
+ q = pagination.ApplyLikeSearch(q, strings.ToLower(params.Search),
+ "(LOWER(vulnerability_scan_items.vulnerability_id) LIKE ? OR LOWER(vulnerability_scan_items.pkg_name) LIKE ? OR LOWER(vulnerability_scan_items.installed_version) LIKE ? OR LOWER(vulnerability_scan_items.fixed_version) LIKE ? OR LOWER(vulnerability_scan_items.title) LIKE ? OR LOWER(vulnerability_scans.image_name) LIKE ? OR LOWER(vulnerability_scan_items.image_id) LIKE ?)")
+ if col, ok := listAllVulnerabilitySortColumnsInternal[params.Sort]; ok {
+ q = q.Order(clause.OrderByColumn{Column: clause.Column{Name: col, Raw: true}, Desc: strings.EqualFold(string(params.Order), "desc")})
}
+ params.Sort = ""
- // Search and filters were already applied during decode; only sorting
- // and page slicing remain.
- page := config.OrderAndPaginate(items, params)
- response := pagination.BuildResponse(int64(len(items)), int64(totalAvailable), params)
+ var rows []vulnerabilityScanItemRowInternal
+ response, err := pagination.PaginateAndSortDB(params, q, &rows)
+ if err != nil {
+ return nil, pagination.Response{}, errors.WrapIf(err, "failed to list vulnerabilities")
+ }
+ response.GrandTotalItems = totalAvailable
+
+ items := make([]vulnerability.VulnerabilityWithImage, 0, len(rows))
+ for i := range rows {
+ row := &rows[i]
+ item := vulnerability.VulnerabilityWithImage{
+ VulnerabilityID: row.VulnerabilityID,
+ PkgName: row.PkgName,
+ InstalledVersion: row.InstalledVersion,
+ FixedVersion: row.FixedVersion,
+ Severity: vulnerability.Severity(row.Severity),
+ Class: row.PkgClass,
+ Type: row.PkgType,
+ Title: row.Title,
+ Ignored: row.IgnoreID != "",
+ IgnoreID: row.IgnoreID,
+ ImageID: row.ImageID,
+ ImageName: row.ImageName,
+ }
+ if row.Details != nil && *row.Details != "" {
+ var details vulnerabilityScanItemDetailsInternal
+ if err := json.Unmarshal([]byte(*row.Details), &details); err == nil {
+ item.Description = details.Description
+ item.References = details.References
+ item.CVSS = details.CVSS
+ item.PublishedDate = details.PublishedDate
+ item.LastModifiedDate = details.LastModifiedDate
+ }
+ }
+ items = append(items, item)
+ }
- return page, response, nil
+ return items, response, nil
}
// listAllVulnerabilityRecordBatchSizeInternal bounds how many scan records
-// (each carrying a vulnerability blob of up to ~2 MiB) are loaded at once.
+// (each carrying a vulnerability blob of up to ~2 MiB) the backfill loads at once.
const listAllVulnerabilityRecordBatchSizeInternal = 25
-func listAllVulnerabilitiesPaginationConfigInternal() pagination.Config[vulnerability.VulnerabilityWithImage] {
- return pagination.Config[vulnerability.VulnerabilityWithImage]{
- SearchAccessors: []pagination.SearchAccessor[vulnerability.VulnerabilityWithImage]{
- func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.VulnerabilityID, nil },
- func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.PkgName, nil },
- func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.InstalledVersion, nil },
- func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.FixedVersion, nil },
- func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.Title, nil },
- func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.ImageName, nil },
- func(item vulnerability.VulnerabilityWithImage) (string, error) { return item.ImageID, nil },
- },
- SortBindings: []pagination.SortBinding[vulnerability.VulnerabilityWithImage]{
- {
- Key: "vulnerabilityId",
- Fn: func(a, b vulnerability.VulnerabilityWithImage) int {
- return strings.Compare(a.VulnerabilityID, b.VulnerabilityID)
- },
- },
- {
- Key: "pkgName",
- Fn: func(a, b vulnerability.VulnerabilityWithImage) int { return strings.Compare(a.PkgName, b.PkgName) },
- },
- {
- Key: "severity",
- Fn: func(a, b vulnerability.VulnerabilityWithImage) int {
- return severityRankInternal(a.Severity) - severityRankInternal(b.Severity)
- },
- },
- {
- Key: "installedVersion",
- Fn: func(a, b vulnerability.VulnerabilityWithImage) int {
- return strings.Compare(a.InstalledVersion, b.InstalledVersion)
- },
- },
- {
- Key: "fixedVersion",
- Fn: func(a, b vulnerability.VulnerabilityWithImage) int {
- return strings.Compare(a.FixedVersion, b.FixedVersion)
- },
- },
- {
- Key: "imageName",
- Fn: func(a, b vulnerability.VulnerabilityWithImage) int { return strings.Compare(a.ImageName, b.ImageName) },
- },
- },
- FilterAccessors: []pagination.FilterAccessor[vulnerability.VulnerabilityWithImage]{
- {
- Key: "severity",
- Fn: func(item vulnerability.VulnerabilityWithImage, value string) bool {
- return strings.EqualFold(string(item.Severity), value)
- },
- },
- {
- Key: "imageName",
- Fn: func(item vulnerability.VulnerabilityWithImage, value string) bool {
- itemLower := strings.ToLower(item.ImageName)
- for part := range strings.SplitSeq(value, ",") {
- if part = strings.TrimSpace(part); part != "" && strings.Contains(itemLower, strings.ToLower(part)) {
- return true
- }
- }
- return false
- },
- },
- },
- }
-}
-
// ListAllVulnerabilityImageOptions returns unique image names (or image IDs when name is empty)
// for vulnerability filtering, optionally constrained by severity.
func (s *VulnerabilityService) ListAllVulnerabilityImageOptions(ctx context.Context, severityFilter string) ([]string, error) {
@@ -471,6 +405,9 @@ func (s *VulnerabilityService) ListAllVulnerabilityImageOptions(ctx context.Cont
Select("id", "image_name").
Where("status = ?", ScanStatusCompleted).
Where("total_count > 0")
+ if currentImageIDs, ok := s.currentImageIDsInternal(ctx); ok {
+ query = query.Where("id IN ?", currentImageIDs)
+ }
if strings.TrimSpace(severityFilter) != "" {
var hasPossibleMatches bool
diff --git a/backend/internal/vulnerability/vulnerability_store.go b/backend/internal/vulnerability/vulnerability_store.go
index bb235cb35..ea4f62056 100644
--- a/backend/internal/vulnerability/vulnerability_store.go
+++ b/backend/internal/vulnerability/vulnerability_store.go
@@ -14,7 +14,6 @@ import (
"emperror.dev/errors"
- "github.com/moby/moby/client"
"go.getarcane.app/acfs"
"gorm.io/gorm"
"gorm.io/gorm/clause"
@@ -24,10 +23,10 @@ import (
"github.com/getarcaneapp/arcane/types/v2/vulnerability"
)
-// deleteStoredReportsInternal best-effort removes the raw report rows of the
-// scan records matched by the given query, so reports never outlive their
-// scan records. Failures are logged and never block record deletion.
-func (s *VulnerabilityService) deleteStoredReportsInternal(ctx context.Context, query *gorm.DB) {
+// deleteScanChildRowsInternal best-effort removes the raw report and CVE item
+// rows of the scan records matched by the given query, so neither outlives its
+// scan record. Failures are logged and never block record deletion.
+func (s *VulnerabilityService) deleteScanChildRowsInternal(ctx context.Context, query *gorm.DB) {
var imageIDs []string
if err := query.Pluck("id", &imageIDs).Error; err != nil {
slog.WarnContext(ctx, "failed to list stored vulnerability reports for cleanup", "error", err)
@@ -39,6 +38,86 @@ func (s *VulnerabilityService) deleteStoredReportsInternal(ctx context.Context,
if err := s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityReportRecord{}).Error; err != nil {
slog.WarnContext(ctx, "failed to delete stored vulnerability reports", "error", err)
}
+ if err := s.db.WithContext(ctx).Where("image_id IN ?", imageIDs).Delete(&VulnerabilityScanItem{}).Error; err != nil {
+ slog.WarnContext(ctx, "failed to delete stored vulnerability scan items", "error", err)
+ }
+}
+
+// BackfillScanItems populates vulnerability_scan_items for completed scans
+// recorded before the table existed. Idempotent; failures are logged and never
+// block startup.
+func (s *VulnerabilityService) BackfillScanItems(ctx context.Context) {
+ if s.db == nil {
+ return
+ }
+
+ var batch []VulnerabilityScanRecord
+ err := s.db.WithContext(ctx).
+ Select("id", "vulnerabilities").
+ Where("status = ?", ScanStatusCompleted).
+ Where("total_count > 0").
+ Where("NOT EXISTS (SELECT 1 FROM vulnerability_scan_items WHERE vulnerability_scan_items.image_id = vulnerability_scans.id)").
+ FindInBatches(&batch, listAllVulnerabilityRecordBatchSizeInternal, func(*gorm.DB, int) error {
+ for _, record := range batch {
+ if len(record.Vulnerabilities) == 0 || record.Vulnerabilities[0] == "" {
+ continue
+ }
+ var vulns []vulnerability.Vulnerability
+ if err := json.Unmarshal([]byte(record.Vulnerabilities[0]), &vulns); err != nil {
+ slog.WarnContext(ctx, "failed to unmarshal vulnerabilities for backfill", "error", err, "image_id", record.ID)
+ continue
+ }
+ items, err := buildScanItemsInternal(record.ID, vulns)
+ if err != nil || len(items) == 0 {
+ continue
+ }
+ if err := dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {
+ return tx.CreateInBatches(items, scanItemInsertBatchSizeInternal).Error
+ }); err != nil {
+ slog.WarnContext(ctx, "failed to backfill vulnerability scan items", "error", err, "image_id", record.ID)
+ }
+ }
+ return nil
+ }).Error
+ if err != nil {
+ slog.WarnContext(ctx, "failed to backfill vulnerability scan items", "error", err)
+ }
+}
+
+const scanItemInsertBatchSizeInternal = 500
+
+func buildScanItemsInternal(imageID string, vulns []vulnerability.Vulnerability) ([]VulnerabilityScanItem, error) {
+ items := make([]VulnerabilityScanItem, 0, len(vulns))
+ for i := range vulns {
+ v := &vulns[i]
+ item := VulnerabilityScanItem{
+ ImageID: imageID,
+ VulnerabilityID: v.VulnerabilityID,
+ PkgName: v.PkgName,
+ InstalledVersion: v.InstalledVersion,
+ FixedVersion: v.FixedVersion,
+ Severity: string(v.Severity),
+ SeverityRank: severityRankInternal(v.Severity),
+ PkgClass: v.Class,
+ PkgType: v.Type,
+ Title: v.Title,
+ }
+ if v.Description != "" || len(v.References) > 0 || v.CVSS != nil || v.PublishedDate != nil || v.LastModifiedDate != nil {
+ raw, err := json.Marshal(vulnerabilityScanItemDetailsInternal{
+ Description: v.Description,
+ References: v.References,
+ CVSS: v.CVSS,
+ PublishedDate: v.PublishedDate,
+ LastModifiedDate: v.LastModifiedDate,
+ })
+ if err != nil {
+ return nil, errors.WrapIf(err, "failed to marshal vulnerability details")
+ }
+ item.Details = new(string(raw))
+ }
+ items = append(items, item)
+ }
+ return items, nil
}
// ImportLegacyReportFiles moves raw Trivy report files written by older
@@ -121,7 +200,7 @@ func (s *VulnerabilityService) DeleteScanResult(ctx context.Context, imageID str
return nil
}
- s.deleteStoredReportsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID))
+ s.deleteScanChildRowsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id = ?", imageID))
return s.db.WithContext(ctx).Where("id = ?", imageID).Delete(&VulnerabilityScanRecord{}).Error
}
@@ -131,7 +210,7 @@ func (s *VulnerabilityService) DeleteScanResultsByImageIDs(ctx context.Context,
return nil
}
- s.deleteStoredReportsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id IN ?", imageIDs))
+ s.deleteScanChildRowsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id IN ?", imageIDs))
return s.db.WithContext(ctx).Where("id IN ?", imageIDs).Delete(&VulnerabilityScanRecord{}).Error
}
@@ -144,16 +223,10 @@ func (s *VulnerabilityService) CleanupOrphanedScanRecords(ctx context.Context) (
return 0, nil
}
- dockerClient, err := s.dockerService.GetClient(ctx)
- if err != nil {
- return 0, errors.WrapIf(err, "failed to connect to Docker")
- }
-
- imageList, err := dockerClient.ImageList(ctx, client.ImageListOptions{})
+ images, err := s.dockerService.ListImages(ctx)
if err != nil {
return 0, errors.WrapIf(err, "failed to list images")
}
- images := imageList.Items
currentIDs := make(map[string]struct{}, len(images))
for _, img := range images {
@@ -171,10 +244,10 @@ func (s *VulnerabilityService) CleanupOrphanedScanRecords(ctx context.Context) (
var result *gorm.DB
if len(ids) == 0 {
- s.deleteStoredReportsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}))
+ s.deleteScanChildRowsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}))
result = s.db.WithContext(ctx).Where("1 = 1").Delete(&VulnerabilityScanRecord{})
} else {
- s.deleteStoredReportsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id NOT IN ?", ids))
+ s.deleteScanChildRowsInternal(ctx, s.db.WithContext(ctx).Model(&VulnerabilityScanRecord{}).Where("id NOT IN ?", ids))
result = s.db.WithContext(ctx).Where("id NOT IN ?", ids).Delete(&VulnerabilityScanRecord{})
}
if result.Error != nil {
@@ -347,11 +420,25 @@ func (s *VulnerabilityService) saveScanResult(ctx context.Context, result *vulne
}
record.FixableCount = &fixable
- // Upsert the record
- return s.db.WithContext(ctx).Clauses(clause.OnConflict{
- Columns: []clause.Column{{Name: "id"}},
- UpdateAll: true,
- }).Create(&record).Error
+ return dbutil.WithTx(ctx, s.db.DB, func(tx *gorm.DB) error {
+ if err := tx.Clauses(clause.OnConflict{
+ Columns: []clause.Column{{Name: "id"}},
+ UpdateAll: true,
+ }).Create(&record).Error; err != nil {
+ return err
+ }
+ if err := tx.Where("image_id = ?", result.ImageID).Delete(&VulnerabilityScanItem{}).Error; err != nil {
+ return errors.WrapIf(err, "failed to clear scan items")
+ }
+ if record.Status != ScanStatusCompleted || len(result.Vulnerabilities) == 0 {
+ return nil
+ }
+ items, err := buildScanItemsInternal(result.ImageID, result.Vulnerabilities)
+ if err != nil {
+ return err
+ }
+ return errors.WrapIf(tx.CreateInBatches(items, scanItemInsertBatchSizeInternal).Error, "failed to store scan items")
+ })
}
// convertRecordToResult converts a database record to a ScanResult
diff --git a/backend/resources/migrations/postgres/078_add_vulnerability_scan_items.sql b/backend/resources/migrations/postgres/078_add_vulnerability_scan_items.sql
new file mode 100644
index 000000000..143eed828
--- /dev/null
+++ b/backend/resources/migrations/postgres/078_add_vulnerability_scan_items.sql
@@ -0,0 +1,27 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS vulnerability_scan_items (
+ id BIGSERIAL PRIMARY KEY,
+ image_id TEXT NOT NULL REFERENCES vulnerability_scans(id) ON DELETE CASCADE,
+ vulnerability_id TEXT NOT NULL,
+ pkg_name TEXT NOT NULL,
+ installed_version TEXT NOT NULL DEFAULT '',
+ fixed_version TEXT NOT NULL DEFAULT '',
+ severity TEXT NOT NULL,
+ severity_rank INTEGER NOT NULL DEFAULT 0,
+ pkg_class TEXT NOT NULL DEFAULT '',
+ pkg_type TEXT NOT NULL DEFAULT '',
+ title TEXT NOT NULL DEFAULT '',
+ details TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_image_id ON vulnerability_scan_items(image_id);
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_severity ON vulnerability_scan_items(severity);
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_vulnerability_id ON vulnerability_scan_items(vulnerability_id);
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_severity_rank_id ON vulnerability_scan_items(severity_rank, id);
+
+-- +goose Down
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_severity_rank_id;
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_vulnerability_id;
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_severity;
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_image_id;
+DROP TABLE IF EXISTS vulnerability_scan_items;
diff --git a/backend/resources/migrations/sqlite/078_add_vulnerability_scan_items.sql b/backend/resources/migrations/sqlite/078_add_vulnerability_scan_items.sql
new file mode 100644
index 000000000..86727b4c7
--- /dev/null
+++ b/backend/resources/migrations/sqlite/078_add_vulnerability_scan_items.sql
@@ -0,0 +1,28 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS vulnerability_scan_items (
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
+ image_id TEXT NOT NULL,
+ vulnerability_id TEXT NOT NULL,
+ pkg_name TEXT NOT NULL,
+ installed_version TEXT NOT NULL DEFAULT '',
+ fixed_version TEXT NOT NULL DEFAULT '',
+ severity TEXT NOT NULL,
+ severity_rank INTEGER NOT NULL DEFAULT 0,
+ pkg_class TEXT NOT NULL DEFAULT '',
+ pkg_type TEXT NOT NULL DEFAULT '',
+ title TEXT NOT NULL DEFAULT '',
+ details TEXT,
+ FOREIGN KEY (image_id) REFERENCES vulnerability_scans(id) ON DELETE CASCADE
+);
+
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_image_id ON vulnerability_scan_items(image_id);
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_severity ON vulnerability_scan_items(severity);
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_vulnerability_id ON vulnerability_scan_items(vulnerability_id);
+CREATE INDEX IF NOT EXISTS idx_vulnerability_scan_items_severity_rank_id ON vulnerability_scan_items(severity_rank, id);
+
+-- +goose Down
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_severity_rank_id;
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_vulnerability_id;
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_severity;
+DROP INDEX IF EXISTS idx_vulnerability_scan_items_image_id;
+DROP TABLE IF EXISTS vulnerability_scan_items;