Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Transport filtered retained metrics by statement-local series number

pulse-triage[bot]35af9de7f572
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
index 45c155a84b..12958105b3 100644
--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
@@ -2750,7 +2750,17 @@ prepared SQL is cached. Layout participates in the bounded SQL-template key;
results and tier presence remain uncached. Raw multi-resource reads bind one
bounded request table and return statement-local resource numbers across the
SQLite row boundary. Resolve each number against that statement's normalized
-resource list; never persist or expose it as resource identity. Tier discovery,
+resource list; never persist or expose it as resource identity. With multiple
+requested metrics, return one series number instead of repeatedly decoding the
+metric name too. Resolve both dimensions against the current bound resource
+and metric lists, including on a SQL-template cache hit or a new chunk. Metric
+names remain bound data in the projection, not interpolated SQL. Full and
+value-only raw projections share this shape; unfiltered, single-resource and
+ordered display-bucket reads keep their existing row dimensions.
+`TestRetainedRawFleetSeriesNumbersBindCurrentMetrics` checks both layouts and
+projections, parameter-like identities, reordered metric bindings and exact
+observation ownership; the resource-number regression covers filtered chunk
+boundaries and mixed-tier chronology. Tier discovery,
coverage probes and value ranges retain the same scoped snapshot and index
bounds. These detached raw series explicitly restore stable timestamp order in
Go before Docker observation projection, without a fleet-wide SQLite sort or
diff --git a/pkg/metrics/store.go b/pkg/metrics/store.go
index 05662eb846..a46715a8a3 100644
--- a/pkg/metrics/store.go
+++ b/pkg/metrics/store.go
@@ -1718,15 +1718,25 @@ func retainedQueryUsesResourceOrdinals(resources int, stepSecs int64, groupSerie
func retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier, groupSeries, valuesOnly bool) (string, []interface{}) {
params := retainedQueryParameters(resourceType, resourceIDs, metricTypes, start, end, stepSecs, tiers)
resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), stepSecs, groupSeries)
+ seriesOrdinals := resourceOrdinals && len(metricTypes) > 1
identityColumns := ""
if len(resourceIDs) != 1 {
- if resourceOrdinals {
+ if seriesOrdinals {
+ // A filtered fleet already binds every metric name. Return one
+ // statement-local series number instead of decoding its resource
+ // number and the same metric string for every observation.
+ metricCases := make([]string, len(metricTypes))
+ for i := range metricTypes {
+ metricCases[i] = fmt.Sprintf("WHEN :p%d THEN %d", 2+len(resourceIDs)+i, i)
+ }
+ identityColumns = fmt.Sprintf("(r.resource_number * %d + CASE m.metric_type %s END) AS series_number, ", len(metricTypes), strings.Join(metricCases, " "))
+ } else if resourceOrdinals {
identityColumns += "r.resource_number, "
} else {
identityColumns += "resource_id, "
}
}
- if len(metricTypes) != 1 {
+ if len(metricTypes) != 1 && !seriesOrdinals {
identityColumns += "metric_type, "
}
// Reuse SQLite's named bindings across every branch and overlap probe.
@@ -1955,6 +1965,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
}
sqlQuery, params := s.retainedQuerySQLForIndex(index, resourceType, resourceIDs, metricTypes, start, end, queryStep, tiers, streamBuckets, projection.valuesOnly)
resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), queryStep, streamBuckets)
+ seriesOrdinals := resourceOrdinals && len(metricTypes) > 1
queryRows := func() (*sql.Rows, error) { return tx.Query(sqlQuery, params...) }
if tx == nil {
@@ -2048,7 +2059,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
}
if len(metricTypes) == 1 {
metricType = metricTypes[0]
- } else {
+ } else if !seriesOrdinals {
destinations = append(destinations, &metricType)
}
destinations = append(destinations, &ts, &p.Value)
@@ -2061,10 +2072,18 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
continue
}
if resourceOrdinals {
- if resourceNumber < 0 || resourceNumber >= len(resourceIDs) {
- return nil, fmt.Errorf("invalid retained resource number %d for %d resources", resourceNumber, len(resourceIDs))
+ resourceIndex := resourceNumber
+ if seriesOrdinals {
+ if resourceNumber < 0 || resourceNumber >= len(resourceIDs)*len(metricTypes) {
+ return nil, fmt.Errorf("invalid retained series number %d", resourceNumber)
+ }
+ resourceIndex /= len(metricTypes)
+ metricType = metricTypes[resourceNumber%len(metricTypes)]
+ }
+ if resourceIndex < 0 || resourceIndex >= len(resourceIDs) {
+ return nil, fmt.Errorf("invalid retained resource number %d for %d resources", resourceIndex, len(resourceIDs))
}
- resourceID = resourceIDs[resourceNumber]
+ resourceID = resourceIDs[resourceIndex]
}
if !projection.valuesOnly {
p.Min, p.Max = p.Value, p.Value
diff --git a/pkg/metrics/store_query_plan_test.go b/pkg/metrics/store_query_plan_test.go
index b7b1a74f3b..b4c174f1d5 100644
--- a/pkg/metrics/store_query_plan_test.go
+++ b/pkg/metrics/store_query_plan_test.go
@@ -666,6 +666,73 @@ func TestRetainedResourceNumbersPreserveScopeAndOrderedSeries(t *testing.T) {
}
}
check(append(append([]string{}, ids...), "missing", ids[0]), nil, 0)
+ check(append(append([]string{}, ids...), "missing", ids[0]), []string{"memory", "cpu", "absent"}, 0)
})
}
}
+
+// Filtered raw fleets bind both dimensions already. The row boundary should
+// transport one series ordinal, while a cache hit with reordered metric names
+// must still resolve against this statement's current bindings.
+func TestRetainedRawFleetSeriesNumbersBindCurrentMetrics(t *testing.T) {
+ for _, clustered := range []bool{false, true} {
+ for _, valuesOnly := range []bool{false, true} {
+ t.Run(fmt.Sprintf("clustered=%t/values=%t", clustered, valuesOnly), func(t *testing.T) {
+ db := newPlanTestDB(t)
+ if clustered {
+ if _, err := db.Exec(`DROP TABLE metrics; CREATE TABLE metrics (` + metricsTableColumns + `) WITHOUT ROWID; CREATE INDEX idx_metrics_tier_time ON metrics(tier,timestamp)`); err != nil {
+ t.Fatal(err)
+ }
+ }
+ store := &Store{db: pdb.Wrap(db, "series-number-binding")}
+ store.clusteredMetrics.Store(clustered)
+ base := time.Unix(1_800_003_600, 0)
+ ids := []string{"b':p2", "a:p4"}
+ names := []string{"memory", "cpu", ":p4"}
+ for i, id := range ids {
+ for j, metric := range names {
+ if _, err := db.Exec(`INSERT INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node',?,?,'minute',?,?)`, id, metric, base.Unix(), float64(i*100+j)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ for _, filters := range [][]string{names, {":p4", "memory", "cpu"}, {"cpu", "memory", ":p4"}} {
+ query, args := store.retainedQuerySQL("node", ids, filters, base.Add(-3*time.Hour), base.Add(time.Minute), 0, []Tier{TierMinute}, false, valuesOnly)
+ rows, err := db.Query(query, args...)
+ if err != nil {
+ t.Fatal(err)
+ }
+ columns, err := rows.Columns()
+ if err != nil {
+ t.Fatal(err)
+ }
+ rows.Close()
+ want := []string{"series_number", "timestamp", "value"}
+ if !valuesOnly {
+ want = append(want, "min_value", "max_value")
+ }
+ if !reflect.DeepEqual(columns, want) {
+ t.Fatalf("repeated series identity still crosses row boundary: got %v, want %v", columns, want)
+ }
+ full, err := store.QueryMetricTypesBatch("node", ids, filters, base.Add(-3*time.Hour), base.Add(time.Minute), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ values, err := store.QueryMetricValuesBatch("node", ids, filters, base.Add(-3*time.Hour), base.Add(time.Minute), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assertMetricValueProjection(t, full, values)
+ for i, id := range ids {
+ for j, name := range names {
+ points := full[id][name]
+ if len(points) != 1 || points[0].Value != float64(i*100+j) || !points[0].Timestamp.Equal(base) {
+ t.Fatalf("wrong current series for %q/%q: %+v", id, name, points)
+ }
+ }
+ }
+ }
+ })
+ }
+ }
+}