Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Transport filtered retained metrics by statement-local series number
pulse-triage[bot]35af9de7f572
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.mdindex 45c155a84b..12958105b3 100644--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md@@ -2750,7 +2750,17 @@ prepared SQL is cached. Layout participates in the bounded SQL-template key;results and tier presence remain uncached. Raw multi-resource reads bind onebounded request table and return statement-local resource numbers across theSQLite row boundary. Resolve each number against that statement's normalized-resource list; never persist or expose it as resource identity. Tier discovery,+resource list; never persist or expose it as resource identity. With multiple+requested metrics, return one series number instead of repeatedly decoding the+metric name too. Resolve both dimensions against the current bound resource+and metric lists, including on a SQL-template cache hit or a new chunk. Metric+names remain bound data in the projection, not interpolated SQL. Full and+value-only raw projections share this shape; unfiltered, single-resource and+ordered display-bucket reads keep their existing row dimensions.+`TestRetainedRawFleetSeriesNumbersBindCurrentMetrics` checks both layouts and+projections, parameter-like identities, reordered metric bindings and exact+observation ownership; the resource-number regression covers filtered chunk+boundaries and mixed-tier chronology. Tier discovery,coverage probes and value ranges retain the same scoped snapshot and indexbounds. These detached raw series explicitly restore stable timestamp order inGo before Docker observation projection, without a fleet-wide SQLite sort ordiff --git a/pkg/metrics/store.go b/pkg/metrics/store.goindex 05662eb846..a46715a8a3 100644--- a/pkg/metrics/store.go+++ b/pkg/metrics/store.go@@ -1718,15 +1718,25 @@ func retainedQueryUsesResourceOrdinals(resources int, stepSecs int64, groupSeriefunc retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier, groupSeries, valuesOnly bool) (string, []interface{}) {params := retainedQueryParameters(resourceType, resourceIDs, metricTypes, start, end, stepSecs, tiers)resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), stepSecs, groupSeries)+ seriesOrdinals := resourceOrdinals && len(metricTypes) > 1identityColumns := ""if len(resourceIDs) != 1 {- if resourceOrdinals {+ if seriesOrdinals {+ // A filtered fleet already binds every metric name. Return one+ // statement-local series number instead of decoding its resource+ // number and the same metric string for every observation.+ metricCases := make([]string, len(metricTypes))+ for i := range metricTypes {+ metricCases[i] = fmt.Sprintf("WHEN :p%d THEN %d", 2+len(resourceIDs)+i, i)+ }+ identityColumns = fmt.Sprintf("(r.resource_number * %d + CASE m.metric_type %s END) AS series_number, ", len(metricTypes), strings.Join(metricCases, " "))+ } else if resourceOrdinals {identityColumns += "r.resource_number, "} else {identityColumns += "resource_id, "}}- if len(metricTypes) != 1 {+ if len(metricTypes) != 1 && !seriesOrdinals {identityColumns += "metric_type, "}// Reuse SQLite's named bindings across every branch and overlap probe.@@ -1955,6 +1965,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs}sqlQuery, params := s.retainedQuerySQLForIndex(index, resourceType, resourceIDs, metricTypes, start, end, queryStep, tiers, streamBuckets, projection.valuesOnly)resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), queryStep, streamBuckets)+ seriesOrdinals := resourceOrdinals && len(metricTypes) > 1queryRows := func() (*sql.Rows, error) { return tx.Query(sqlQuery, params...) }if tx == nil {@@ -2048,7 +2059,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs}if len(metricTypes) == 1 {metricType = metricTypes[0]- } else {+ } else if !seriesOrdinals {destinations = append(destinations, &metricType)}destinations = append(destinations, &ts, &p.Value)@@ -2061,10 +2072,18 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDscontinue}if resourceOrdinals {- if resourceNumber < 0 || resourceNumber >= len(resourceIDs) {- return nil, fmt.Errorf("invalid retained resource number %d for %d resources", resourceNumber, len(resourceIDs))+ resourceIndex := resourceNumber+ if seriesOrdinals {+ if resourceNumber < 0 || resourceNumber >= len(resourceIDs)*len(metricTypes) {+ return nil, fmt.Errorf("invalid retained series number %d", resourceNumber)+ }+ resourceIndex /= len(metricTypes)+ metricType = metricTypes[resourceNumber%len(metricTypes)]+ }+ if resourceIndex < 0 || resourceIndex >= len(resourceIDs) {+ return nil, fmt.Errorf("invalid retained resource number %d for %d resources", resourceIndex, len(resourceIDs))}- resourceID = resourceIDs[resourceNumber]+ resourceID = resourceIDs[resourceIndex]}if !projection.valuesOnly {p.Min, p.Max = p.Value, p.Valuediff --git a/pkg/metrics/store_query_plan_test.go b/pkg/metrics/store_query_plan_test.goindex b7b1a74f3b..b4c174f1d5 100644--- a/pkg/metrics/store_query_plan_test.go+++ b/pkg/metrics/store_query_plan_test.go@@ -666,6 +666,73 @@ func TestRetainedResourceNumbersPreserveScopeAndOrderedSeries(t *testing.T) {}}check(append(append([]string{}, ids...), "missing", ids[0]), nil, 0)+ check(append(append([]string{}, ids...), "missing", ids[0]), []string{"memory", "cpu", "absent"}, 0)})}}++// Filtered raw fleets bind both dimensions already. The row boundary should+// transport one series ordinal, while a cache hit with reordered metric names+// must still resolve against this statement's current bindings.+func TestRetainedRawFleetSeriesNumbersBindCurrentMetrics(t *testing.T) {+ for _, clustered := range []bool{false, true} {+ for _, valuesOnly := range []bool{false, true} {+ t.Run(fmt.Sprintf("clustered=%t/values=%t", clustered, valuesOnly), func(t *testing.T) {+ db := newPlanTestDB(t)+ if clustered {+ if _, err := db.Exec(`DROP TABLE metrics; CREATE TABLE metrics (` + metricsTableColumns + `) WITHOUT ROWID; CREATE INDEX idx_metrics_tier_time ON metrics(tier,timestamp)`); err != nil {+ t.Fatal(err)+ }+ }+ store := &Store{db: pdb.Wrap(db, "series-number-binding")}+ store.clusteredMetrics.Store(clustered)+ base := time.Unix(1_800_003_600, 0)+ ids := []string{"b':p2", "a:p4"}+ names := []string{"memory", "cpu", ":p4"}+ for i, id := range ids {+ for j, metric := range names {+ if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node',?,?,'minute',?,?)`, id, metric, base.Unix(), float64(i*100+j)); err != nil {+ t.Fatal(err)+ }+ }+ }+ for _, filters := range [][]string{names, {":p4", "memory", "cpu"}, {"cpu", "memory", ":p4"}} {+ query, args := store.retainedQuerySQL("node", ids, filters, base.Add(-3*time.Hour), base.Add(time.Minute), 0, []Tier{TierMinute}, false, valuesOnly)+ rows, err := db.Query(query, args...)+ if err != nil {+ t.Fatal(err)+ }+ columns, err := rows.Columns()+ if err != nil {+ t.Fatal(err)+ }+ rows.Close()+ want := []string{"series_number", "timestamp", "value"}+ if !valuesOnly {+ want = append(want, "min_value", "max_value")+ }+ if !reflect.DeepEqual(columns, want) {+ t.Fatalf("repeated series identity still crosses row boundary: got %v, want %v", columns, want)+ }+ full, err := store.QueryMetricTypesBatch("node", ids, filters, base.Add(-3*time.Hour), base.Add(time.Minute), 0)+ if err != nil {+ t.Fatal(err)+ }+ values, err := store.QueryMetricValuesBatch("node", ids, filters, base.Add(-3*time.Hour), base.Add(time.Minute), 0)+ if err != nil {+ t.Fatal(err)+ }+ assertMetricValueProjection(t, full, values)+ for i, id := range ids {+ for j, name := range names {+ points := full[id][name]+ if len(points) != 1 || points[0].Value != float64(i*100+j) || !points[0].Timestamp.Equal(base) {+ t.Fatalf("wrong current series for %q/%q: %+v", id, name, points)+ }+ }+ }+ }+ })+ }+ }+}