Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Decode native retained metric numbers without per-point reflection
pulse-triage[bot]105fb05cc3f1
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.mdindex a769962262..a4d605ad5c 100644--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md@@ -2755,7 +2755,16 @@ value points afterward. Full and value projections share scope normalization,filters, tier reconciliation, time ordering, display buckets and Dockerobservation provenance; their distinct SQL column shapes must have distinctkeys within the existing bounded query cache. Neither mode caches results or-tier presence. Full readers continue to preserve extrema. Legacy databases keep+tier presence. Full readers continue to preserve extrema. At the application row-decoding+boundary, native int64 ordinals/timestamps and float64 values are assigned+without reflective per-observation numeric conversion. Atypical numeric text,+blobs and integer-valued observations retain database/sql's ordinary conversion;+NULL, malformed and overflowing values must still fail rather than manufacture+zeroes. This changes neither SQL nor driver behaviour. The compatibility control+`TestRetainedNumericScansMatchDatabaseSQL` compares both decoders bit-for-bit,+including signed zero, large integers and conversion failures; the native scan+allocation control and existing full/value, tier, scope and detached-read tests+remain required. Legacy databases keeptheir existing unique index authoritative while deferred startup maintenancecopies the eight observation columns into the clustered table. `BEGIN IMMEDIATE`serializes writers and independent Store migrations before inspection. Copy,diff --git a/pkg/metrics/retained_numeric.go b/pkg/metrics/retained_numeric.gonew file mode 100644index 0000000000..0a6d52a02e--- /dev/null+++ b/pkg/metrics/retained_numeric.go@@ -0,0 +1,46 @@+package metrics++import (+ "database/sql"+ "fmt"+)++// Retained rows normally carry int64 timestamps/ordinals and float64 values.+// Assign those types without database/sql's per-observation reflection. Keep its+// ordinary numeric conversion for older or atypically typed observations; this+// is an application-side decoder, not a different SQL or driver contract.+type retainedInt64 int64++func (n *retainedInt64) Scan(src any) error {+ if value, ok := src.(int64); ok {+ *n = retainedInt64(value)+ return nil+ }+ var compatible sql.NullInt64+ if err := compatible.Scan(src); err != nil {+ return err+ }+ if !compatible.Valid {+ return fmt.Errorf("converting NULL to int64 is unsupported")+ }+ *n = retainedInt64(compatible.Int64)+ return nil+}++type retainedFloat64 float64++func (n *retainedFloat64) Scan(src any) error {+ if value, ok := src.(float64); ok {+ *n = retainedFloat64(value)+ return nil+ }+ var compatible sql.NullFloat64+ if err := compatible.Scan(src); err != nil {+ return err+ }+ if !compatible.Valid {+ return fmt.Errorf("converting NULL to float64 is unsupported")+ }+ *n = retainedFloat64(compatible.Float64)+ return nil+}diff --git a/pkg/metrics/retained_numeric_test.go b/pkg/metrics/retained_numeric_test.gonew file mode 100644index 0000000000..0f6d213d00--- /dev/null+++ b/pkg/metrics/retained_numeric_test.go@@ -0,0 +1,74 @@+package metrics++import (+ "math"+ "testing"+)++// Read each value through database/sql twice, once with its primitive decoder+// and once through the retained decoder. The fallback must accept and reject+// exactly the same data, not turn malformed/NULL observations into zeroes.+func TestRetainedNumericScansMatchDatabaseSQL(t *testing.T) {+ db := newPlanTestDB(t)+ cases := []struct {+ name string+ value any+ }{+ {"zero", int64(0)}, {"negative", int64(-201)},+ {"max-int64", int64(math.MaxInt64)}, {"min-int64", int64(math.MinInt64)},+ {"outside-exact-float-integer", int64(1<<53 + 1)},+ {"float", 17.125}, {"negative-float", -37.5},+ {"negative-zero", math.Copysign(0, -1)},+ {"tiny-float", math.SmallestNonzeroFloat64}, {"large-float", math.MaxFloat64},+ {"integer-text", "-123"}, {"float-text", "19.25"},+ {"integer-blob", []byte("234")}, {"float-blob", []byte("1.25e3")},+ {"overflow-int", "9223372036854775808"}, {"invalid", "not a number"},+ {"fractional-int", "1.5"}, {"empty", ""}, {"null", nil},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ var wantInt int64+ var gotInt retainedInt64 = -777+ wantErr := db.QueryRow("SELECT ?", tc.value).Scan(&wantInt)+ gotErr := db.QueryRow("SELECT ?", tc.value).Scan(&gotInt)+ if (wantErr == nil) != (gotErr == nil) || (wantErr == nil && int64(gotInt) != wantInt) {+ t.Fatalf("integer: standard=%d/%v retained=%d/%v", wantInt, wantErr, gotInt, gotErr)+ }+ if wantErr != nil && gotInt != -777 {+ t.Fatalf("invalid integer changed destination to %d", gotInt)+ }+ var wantFloat float64+ var gotFloat retainedFloat64 = -777+ wantErr = db.QueryRow("SELECT ?", tc.value).Scan(&wantFloat)+ gotErr = db.QueryRow("SELECT ?", tc.value).Scan(&gotFloat)+ if (wantErr == nil) != (gotErr == nil) || (wantErr == nil && math.Float64bits(float64(gotFloat)) != math.Float64bits(wantFloat)) {+ t.Fatalf("float: standard=%g/%v retained=%g/%v", wantFloat, wantErr, gotFloat, gotErr)+ }+ if wantErr != nil && gotFloat != -777 {+ t.Fatalf("invalid float changed destination to %g", gotFloat)+ }+ })+ }+}++func TestRetainedNativeNumericScansDoNotAllocate(t *testing.T) {+ var number retainedInt64+ var value retainedFloat64+ var ordinal any = int64(149)+ var timestamp any = int64(1792854756)+ var observation any = 12.5+ allocations := testing.AllocsPerRun(100, func() {+ if err := number.Scan(ordinal); err != nil {+ panic(err)+ }+ if err := number.Scan(timestamp); err != nil {+ panic(err)+ }+ if err := value.Scan(observation); err != nil {+ panic(err)+ }+ })+ if allocations != 0 || number != 1792854756 || value != 12.5 {+ t.Fatalf("native values: allocations=%g number=%d value=%g", allocations, number, value)+ }+}diff --git a/pkg/metrics/store.go b/pkg/metrics/store.goindex fd847d572a..baf6dd1a21 100644--- a/pkg/metrics/store.go+++ b/pkg/metrics/store.go@@ -2057,8 +2057,9 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDsbucketCount = 0}var resourceID, metricType string- var resourceNumber int- var ts int64+ var resourceNumber retainedInt64+ var ts retainedInt64+ var value retainedFloat64var p MetricPointvar low, high sql.NullFloat64destinations := make([]interface{}, 0, 6)@@ -2074,7 +2075,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs} else if !seriesOrdinals {destinations = append(destinations, &metricType)}- destinations = append(destinations, &ts, &p.Value)+ destinations = append(destinations, &ts, &value)if !projection.valuesOnly {destinations = append(destinations, &low, &high)}@@ -2086,19 +2087,20 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDsif resourceOrdinals {resourceIndex := resourceNumberif seriesOrdinals {- if resourceNumber < 0 || resourceNumber >= len(resourceIDs)*len(metricTypes) {+ if resourceNumber < 0 || resourceNumber >= retainedInt64(len(resourceIDs)*len(metricTypes)) {return nil, fmt.Errorf("invalid retained series number %d", resourceNumber)}if numberedSeries == nil {- resourceID = resourceIDs[resourceNumber/len(metricTypes)]- metricType = metricTypes[resourceNumber%len(metricTypes)]+ resourceID = resourceIDs[resourceNumber/retainedInt64(len(metricTypes))]+ metricType = metricTypes[resourceNumber%retainedInt64(len(metricTypes))]}- } else if resourceIndex < 0 || resourceIndex >= len(resourceIDs) {+ } else if resourceIndex < 0 || resourceIndex >= retainedInt64(len(resourceIDs)) {return nil, fmt.Errorf("invalid retained resource number %d for %d resources", resourceIndex, len(resourceIDs))} else {resourceID = resourceIDs[resourceIndex]}}+ p.Value = float64(value)if !projection.valuesOnly {p.Min, p.Max = p.Value, p.Valueif low.Valid {@@ -2109,7 +2111,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs}}if !streamBuckets {- p.Timestamp = time.Unix(ts, 0)+ p.Timestamp = time.Unix(int64(ts), 0)if numberedSeries != nil {numberedSeries[resourceNumber] = append(numberedSeries[resourceNumber], projection.point(p))} else {@@ -2117,7 +2119,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs}continue}- start := (ts / stepSecs) * stepSecs+ start := (int64(ts) / stepSecs) * stepSecsif bucketCount > 0 && (bucketResource != resourceID || bucketMetric != metricType || bucketStart != start) {flushBucket()}