Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Reduce retained chart row decoding without changing observations
rcourtman473926b495eb
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.mdindex c1827263c2..5e5ab57846 100644--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md@@ -553,7 +553,7 @@ which retention tiers have observations in that snapshot. Every present tierparticipates in the shared indexed overlap query. Presence never stands in forper-series coverage. The store retains at most 32 compiled read-statement shapesand 32 SQL templates keyed only by parameter counts, tier order and aggregation-shape, projection and physical layout. Alphabetic named SQLite bindings supply each current identity, window and+shape, projection, physical layout and bounded integer transport. Alphabetic named SQLite bindings supply each current identity, window anddisplay step once across all branches. Bindings use `database/sql.Named` so thedriver need not repeatedly convert ordinals to strings while matching a largeparameter set. Names describe parameter positions only. Resource IDs and all@@ -581,6 +581,15 @@ including all tier priorities and metric filters. Multi-stage rollups preserverecorded minima and maxima instead of taking extrema from bucket averages.Already discarded historical extrema cannot be reconstructed by this change.+Filtered raw value fleets carry each observation's timestamp and statement-local+series number in one SQLite integer when the requested window proves that the+product and sum fit a signed 64-bit integer. The decoder uses floor division for+negative epochs. Extreme timestamp windows retain separate columns. This+transport choice participates in the SQL shape key, so a cached ordinary-window+query cannot be reused for an overflowing window. It changes neither stored+identity nor API representation, metric selection, retention reconciliation,+returned floating-point values or chronological ordering.+The `retained-metric-query-coverage` follow-up addresses the specific first-tierquery defect reproduced during Patrol outcome qualification. Returned pointsstill do not prove continuous collection or a complete requested window.diff --git a/pkg/metrics/store.go b/pkg/metrics/store.goindex baf6dd1a21..4a27ea1c1f 100644--- a/pkg/metrics/store.go+++ b/pkg/metrics/store.go@@ -7,6 +7,7 @@ import ("database/sql""errors""fmt"+ "math""net/url""os""path/filepath"@@ -1645,6 +1646,7 @@ type retainedQueryShape struct {tiers [4]Tieraggregate, groupSeries boolvaluesOnly, clustered bool+ packedSeries bool}func retainedQueryParameters(resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier) []interface{} {@@ -1677,6 +1679,7 @@ func (s *Store) retainedQuerySQL(resourceType string, resourceIDs, metricTypes [func (s *Store) retainedQuerySQLForIndex(index string, resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier, groupSeries, valuesOnly bool) (string, []interface{}) {shape := retainedQueryShape{resources: len(resourceIDs), metrics: len(metricTypes), aggregate: stepSecs > 1, groupSeries: groupSeries, valuesOnly: valuesOnly, clustered: index == clusteredMetricsIndex}+ shape.packedSeries = retainedQueryUsesPackedSeries(len(resourceIDs), len(metricTypes), start, end, stepSecs, groupSeries, valuesOnly)cacheable := len(tiers) <= len(shape.tiers)copy(shape.tiers[:], tiers)if cacheable {@@ -1719,10 +1722,29 @@ func retainedQueryUsesResourceOrdinals(resources int, stepSecs int64, groupSeriereturn resources > 1 && stepSecs <= 1 && !groupSeries}+// A raw value read needs two integers identifying each observation: its+// statement-local series and timestamp. Carry them in one integer while the+// requested bounds prove multiplication and addition cannot overflow SQLite's+// signed integer. This is only row transport, never storage or API encoding.+// Extreme windows use the ordinary separate columns in the same reader.+func retainedQueryUsesPackedSeries(resources, metrics int, start, end time.Time, stepSecs int64, groupSeries, valuesOnly bool) bool {+ if !valuesOnly || metrics < 2 || !retainedQueryUsesResourceOrdinals(resources, stepSecs, groupSeries) {+ return false+ }+ if int64(metrics) > math.MaxInt64/int64(resources) {+ return false+ }+ series := int64(resources) * int64(metrics)+ return end.Unix() >= start.Unix() && start.Unix() >= math.MinInt64/series &&+ end.Unix() <= (math.MaxInt64-series+1)/series+}+func retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier, groupSeries, valuesOnly bool) (string, []interface{}) {params := retainedQueryParameters(resourceType, resourceIDs, metricTypes, start, end, stepSecs, tiers)resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), stepSecs, groupSeries)seriesOrdinals := resourceOrdinals && len(metricTypes) > 1+ packedSeries := retainedQueryUsesPackedSeries(len(resourceIDs), len(metricTypes), start, end, stepSecs, groupSeries, valuesOnly)+ var seriesExpression stringidentityColumns := ""if len(resourceIDs) != 1 {if seriesOrdinals {@@ -1733,7 +1755,8 @@ func retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, mfor i := range metricTypes {metricCases[i] = fmt.Sprintf("WHEN :p%d THEN %d", 2+len(resourceIDs)+i, i)}- identityColumns = fmt.Sprintf("(r.resource_number * %d + CASE m.metric_type %s END) AS series_number, ", len(metricTypes), strings.Join(metricCases, " "))+ seriesExpression = fmt.Sprintf("(r.resource_number * %d + CASE m.metric_type %s END)", len(metricTypes), strings.Join(metricCases, " "))+ identityColumns = seriesExpression + " AS series_number, "} else if resourceOrdinals {identityColumns += "r.resource_number, "} else {@@ -1772,6 +1795,9 @@ func retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, m// value again for each missing bound makes SQLite and its driver decode// three floats for an observation that stores only one.projection := identityColumns + `m.timestamp, m.value`+ if packedSeries {+ projection = fmt.Sprintf("(m.timestamp * %d + %s) AS series_sample, m.value", len(resourceIDs)*len(metricTypes), seriesExpression)+ }if !valuesOnly {projection += `, m.min_value, m.max_value`}@@ -1970,6 +1996,8 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDssqlQuery, params := s.retainedQuerySQLForIndex(index, resourceType, resourceIDs, metricTypes, start, end, queryStep, tiers, streamBuckets, projection.valuesOnly)resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), queryStep, streamBuckets)seriesOrdinals := resourceOrdinals && len(metricTypes) > 1+ packedSeries := retainedQueryUsesPackedSeries(len(resourceIDs), len(metricTypes), start, end, queryStep, streamBuckets, projection.valuesOnly)+ seriesWidth := int64(len(resourceIDs)) * int64(len(metricTypes))queryRows := func() (*sql.Rows, error) { return tx.Query(sqlQuery, params...) }if tx == nil {@@ -2075,7 +2103,10 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs} else if !seriesOrdinals {destinations = append(destinations, &metricType)}- destinations = append(destinations, &ts, &value)+ if !packedSeries {+ destinations = append(destinations, &ts)+ }+ destinations = append(destinations, &value)if !projection.valuesOnly {destinations = append(destinations, &low, &high)}@@ -2084,6 +2115,17 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDslog.Warn().Err(err).Msg("Failed to scan batch metric row")continue}+ if packedSeries {+ encoded := int64(resourceNumber)+ ts = retainedInt64(encoded / seriesWidth)+ resourceNumber = retainedInt64(encoded % seriesWidth)+ // Go division truncates toward zero. Restore floor division for+ // observations before the Unix epoch so the ordinal stays positive.+ if resourceNumber < 0 {+ resourceNumber += retainedInt64(seriesWidth)+ ts--+ }+ }if resourceOrdinals {resourceIndex := resourceNumberif seriesOrdinals {diff --git a/pkg/metrics/store_query_plan_test.go b/pkg/metrics/store_query_plan_test.goindex 933e8cd7de..1aa5f0f2cc 100644--- a/pkg/metrics/store_query_plan_test.go+++ b/pkg/metrics/store_query_plan_test.go@@ -708,7 +708,9 @@ func TestRetainedRawFleetSeriesNumbersBindCurrentMetrics(t *testing.T) {}rows.Close()want := []string{"series_number", "timestamp", "value"}- if !valuesOnly {+ if valuesOnly {+ want = []string{"series_sample", "value"}+ } else {want = append(want, "min_value", "max_value")}if !reflect.DeepEqual(columns, want) {@@ -737,6 +739,67 @@ func TestRetainedRawFleetSeriesNumbersBindCurrentMetrics(t *testing.T) {}}+func TestRetainedPackedSeriesKeepsNegativeTimestampsAndOverflowFallback(t *testing.T) {+ db := newPlanTestDB(t)+ store := &Store{db: pdb.Wrap(db, "packed-series-boundary")}+ ids, names := []string{"second", "first"}, []string{"memory", "cpu", "disk"}+ const width int64 = 6+ const maximum = int64(^uint64(0) >> 1)+ const minimum = -maximum - 1+ for _, tc := range []struct {+ name string+ start int64+ packed bool+ }{+ {"negative epoch", -100, true},+ {"ordinary epoch", 1_800_000_000, true},+ {"near positive bound", (maximum-width+1)/width - 2, true},+ {"positive overflow", (maximum - width + 1) / width, false},+ {"near negative bound", minimum / width, true},+ {"negative overflow", minimum/width - 1, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ start, end := time.Unix(tc.start, 0), time.Unix(tc.start+2, 0)+ if got := retainedQueryUsesPackedSeries(len(ids), len(names), start, end, 0, false, true); got != tc.packed {+ t.Fatalf("packed=%t want %t", got, tc.packed)+ }+ for i, id := range ids {+ for j, metric := range names {+ for offset := int64(0); offset <= 2; offset++ {+ if _, err := db.Exec(`INSERT OR REPLACE INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node',?,?,'raw',?,?)`, id, metric, tc.start+offset, float64(i*100+j*10)+float64(offset)/3); err != nil {+ t.Fatal(err)+ }+ }+ }+ }+ // Reuse one Store and equal parameter counts across packed and+ // overflow windows. Cached SQL must still choose the correct shape.+ full, err := store.QueryMetricTypesBatch("node", ids, names, start, end, 0)+ if err != nil {+ t.Fatal(err)+ }+ values, err := store.QueryMetricValuesBatch("node", ids, names, start, end, 0)+ if err != nil {+ t.Fatal(err)+ }+ assertMetricValueProjection(t, full, values)+ for _, id := range ids {+ for _, name := range names {+ points := values[id][name]+ if len(points) != 3 {+ t.Fatalf("lost observations for %s/%s: %+v", id, name, points)+ }+ for i, point := range points {+ if point.Timestamp.Unix() != tc.start+int64(i) {+ t.Fatalf("moved timestamp: %v", point.Timestamp.Unix())+ }+ }+ }+ }+ })+ }+}+// Direct ordinal accumulation must be indistinguishable from the existing// single-resource assembler, including interleaved rows and empty series. Wide// sparse filters exercise the bounded fallback without losing statement scope.