Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Assemble retained metric series directly by statement-local number
pulse-triage[bot]c36672b2a7cc
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.mdindex 01056f9068..3122fd34a0 100644--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md@@ -2762,8 +2762,17 @@ resource list; never persist or expose it as resource identity. With multiplerequested metrics, return one series number instead of repeatedly decoding themetric name too. Resolve both dimensions against the current bound resourceand metric lists, including on a SQL-template cache hit or a new chunk. Metric-names remain bound data in the projection, not interpolated SQL. Full and-value-only raw projections share this shape; unfiltered, single-resource and+names remain bound data in the projection, not interpolated SQL. Accumulate+filtered raw observations directly by series number, resolving names once per+nonempty series after the read instead of flushing interleaved nested maps for+every point. Empty resources and metrics stay absent. At most 4,096 slice+headers are reserved per chunk; wider sparse filters retain the map assembler.+Both paths return detached observations and restore stable timestamp order+before Docker projection, with no new sampling or source-selection policy.+`TestRetainedNumberedAssemblyMatchesSingleResourceReads` compares compact,+reordered and wide-sparse fleets with single-resource reads on both layouts,+including interleaved series, overlap exclusion, zero and nullable extrema.+Full and value-only raw projections share this shape; unfiltered, single-resource andordered display-bucket reads keep their existing row dimensions.`TestRetainedRawFleetSeriesNumbersBindCurrentMetrics` checks both layouts andprojections, parameter-like identities, reordered metric bindings and exactdiff --git a/pkg/metrics/store.go b/pkg/metrics/store.goindex a46715a8a3..fd847d572a 100644--- a/pkg/metrics/store.go+++ b/pkg/metrics/store.go@@ -1482,6 +1482,10 @@ func (s *Store) QueryAll(resourceType, resourceID string, start, end time.Time,// queries fast.const queryAllBatchChunkSize = 500+// Bound the empty slice headers for direct ordinal assembly. Unusually wide,+// sparse metric filters keep the existing map assembler instead.+const maxRetainedNumberedSeries = 4096+// QueryAllBatch retrieves all metric types for multiple resources of the same// type in a single query. Returns map[resourceID]map[metricType][]MetricPoint.// This avoids N+1 query patterns when loading charts for many resources.@@ -2000,6 +2004,14 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDsresult := make(map[string]map[string][]T, len(resourceIDs))seriesCapacity := estimateQueryAllBatchSeriesCapacity(start, end, stepSecs)+ // Filtered raw fleets already return a statement-local series number.+ // Accumulate by that number directly: time-major rows interleave metrics,+ // so resolving strings and flushing nested maps on every row defeats the+ // compact transport. Bind names once per nonempty series after the read.+ var numberedSeries [][]T+ if seriesOrdinals && len(resourceIDs)*len(metricTypes) <= maxRetainedNumberedSeries {+ numberedSeries = make([][]T, len(resourceIDs)*len(metricTypes))+ }// Consecutive points in one series append directly to its slice. Flush on// a series change so interleaved metrics can resume their existing slices@@ -2077,13 +2089,15 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDsif resourceNumber < 0 || resourceNumber >= len(resourceIDs)*len(metricTypes) {return nil, fmt.Errorf("invalid retained series number %d", resourceNumber)}- resourceIndex /= len(metricTypes)- metricType = metricTypes[resourceNumber%len(metricTypes)]- }- if resourceIndex < 0 || resourceIndex >= len(resourceIDs) {+ if numberedSeries == nil {+ resourceID = resourceIDs[resourceNumber/len(metricTypes)]+ metricType = metricTypes[resourceNumber%len(metricTypes)]+ }+ } else if resourceIndex < 0 || resourceIndex >= len(resourceIDs) {return nil, fmt.Errorf("invalid retained resource number %d for %d resources", resourceIndex, len(resourceIDs))+ } else {+ resourceID = resourceIDs[resourceIndex]}- resourceID = resourceIDs[resourceIndex]}if !projection.valuesOnly {p.Min, p.Max = p.Value, p.Value@@ -2096,7 +2110,11 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs}if !streamBuckets {p.Timestamp = time.Unix(ts, 0)- appendPoint(resourceID, metricType, p)+ if numberedSeries != nil {+ numberedSeries[resourceNumber] = append(numberedSeries[resourceNumber], projection.point(p))+ } else {+ appendPoint(resourceID, metricType, p)+ }continue}start := (ts / stepSecs) * stepSecs@@ -2120,6 +2138,18 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDsflushBucket()flushSeries()+ for number, points := range numberedSeries {+ if len(points) == 0 {+ continue+ }+ id := resourceIDs[number/len(metricTypes)]+ metrics := result[id]+ if metrics == nil {+ metrics = make(map[string][]T, len(metricTypes))+ result[id] = metrics+ }+ metrics[metricTypes[number%len(metricTypes)]] = points+ }if resourceOrdinals {for _, metrics := range result {for _, points := range metrics {diff --git a/pkg/metrics/store_query_plan_test.go b/pkg/metrics/store_query_plan_test.goindex b4c174f1d5..933e8cd7de 100644--- a/pkg/metrics/store_query_plan_test.go+++ b/pkg/metrics/store_query_plan_test.go@@ -736,3 +736,102 @@ func TestRetainedRawFleetSeriesNumbersBindCurrentMetrics(t *testing.T) {}}}++// Direct ordinal accumulation must be indistinguishable from the existing+// single-resource assembler, including interleaved rows and empty series. Wide+// sparse filters exercise the bounded fallback without losing statement scope.+func TestRetainedNumberedAssemblyMatchesSingleResourceReads(t *testing.T) {+ for _, clustered := range []bool{false, true} {+ t.Run(fmt.Sprintf("clustered=%t", clustered), func(t *testing.T) {+ db := newPlanTestDB(t)+ if clustered {+ if _, err := db.Exec(`DROP TABLE metrics; CREATE TABLE metrics (` + metricsTableColumns + `) WITHOUT ROWID; CREATE INDEX idx_metrics_tier_time ON metrics(tier,timestamp)`); err != nil {+ t.Fatal(err)+ }+ }+ store := &Store{db: pdb.Wrap(db, "numbered-series-assembly")}+ store.clusteredMetrics.Store(clustered)+ base := time.Unix(1_800_003_600, 0)+ end := base.Add(6 * time.Hour)+ ids := []string{"b':p2", "a:p4", "missing"}+ names := []string{"memory", "cpu", "netin", "absent"}+ tx, err := db.Begin()+ if err != nil {+ t.Fatal(err)+ }+ insert, err := tx.Prepare(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value,min_value,max_value) VALUES ('node',?,?,?,?,?,?,?)`)+ if err != nil {+ t.Fatal(err)+ }+ for i, id := range ids[:2] {+ for j, name := range names[:3] {+ // Minute-tier rows come first; the raw first/tail points must+ // still be restored to chronological order, never averaged.+ for p := 0; p < 300; p++ {+ value := float64(i*1000 + j*300 + p)+ if _, err := insert.Exec(id, name, "minute", base.Unix()+int64(p*60+60), value, value-7, value+9); err != nil {+ t.Fatal(err)+ }+ }+ for _, p := range []struct{ offset, value int64 }{{1, 12}, {65, 99999}, {18061, 0}} {+ if _, err := insert.Exec(id, name, "raw", base.Unix()+p.offset, p.value, nil, nil); err != nil {+ t.Fatal(err)+ }+ }+ }+ }+ insert.Close()+ if err := tx.Commit(); err != nil {+ t.Fatal(err)+ }+ for _, shape := range []string{"compact", "rebound", "wide-sparse"} {+ t.Run(shape, func(t *testing.T) {+ requested := append([]string(nil), ids...)+ filters := append([]string(nil), names...)+ if shape == "rebound" {+ requested[0], requested[1] = requested[1], requested[0]+ filters[0], filters[2] = filters[2], filters[0]+ }+ if shape == "wide-sparse" {+ for len(requested) < queryAllBatchChunkSize {+ requested = append(requested, fmt.Sprintf("missing-%d", len(requested)))+ }+ for len(filters) <= maxRetainedNumberedSeries/queryAllBatchChunkSize {+ filters = append(filters, fmt.Sprintf("absent-%d", len(filters)))+ }+ }+ full, err := store.QueryMetricTypesBatch("node", requested, filters, base, end, 0)+ if err != nil {+ t.Fatal(err)+ }+ if len(full) != 2 {+ t.Fatalf("empty resource materialised: %d resources", len(full))+ }+ for _, id := range ids[:2] {+ single, err := store.QueryMetricTypesBatch("node", []string{id}, filters, base, end, 0)+ if err != nil {+ t.Fatal(err)+ }+ if !reflect.DeepEqual(full[id], single[id]) {+ t.Fatalf("fleet differs from single-resource read for %s", id)+ }+ for _, points := range full[id] {+ if len(points) != 302 || points[0].Value != 12 || points[len(points)-1].Value != 0 || points[len(points)-1].Min != 0 || points[1].Min != points[1].Value-7 {+ t.Fatalf("chronology/zero/extrema/overlap changed: %d points", len(points))+ }+ }+ }+ values, err := store.QueryMetricValuesBatch("node", requested, filters, base, end, 0)+ if err != nil {+ t.Fatal(err)+ }+ assertMetricValueProjection(t, full, values)+ full[ids[0]][names[0]][0].Value = -999+ if values[ids[0]][names[0]][0].Value != 12 {+ t.Fatal("reader outputs alias each other")+ }+ })+ }+ })+ }+}