Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Decode native retained metric numbers without per-point reflection

pulse-triage[bot]105fb05cc3f1
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
index a769962262..a4d605ad5c 100644
--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
@@ -2755,7 +2755,16 @@ value points afterward. Full and value projections share scope normalization,
filters, tier reconciliation, time ordering, display buckets and Docker
observation provenance; their distinct SQL column shapes must have distinct
keys within the existing bounded query cache. Neither mode caches results or
-tier presence. Full readers continue to preserve extrema. Legacy databases keep
+tier presence. Full readers continue to preserve extrema. At the application row-decoding
+boundary, native int64 ordinals/timestamps and float64 values are assigned
+without reflective per-observation numeric conversion. Atypical numeric text,
+blobs and integer-valued observations retain database/sql's ordinary conversion;
+NULL, malformed and overflowing values must still fail rather than manufacture
+zeroes. This changes neither SQL nor driver behaviour. The compatibility control
+`TestRetainedNumericScansMatchDatabaseSQL` compares both decoders bit-for-bit,
+including signed zero, large integers and conversion failures; the native scan
+allocation control and existing full/value, tier, scope and detached-read tests
+remain required. Legacy databases keep
their existing unique index authoritative while deferred startup maintenance
copies the eight observation columns into the clustered table. `BEGIN IMMEDIATE`
serializes writers and independent Store migrations before inspection. Copy,
diff --git a/pkg/metrics/retained_numeric.go b/pkg/metrics/retained_numeric.go
new file mode 100644
index 0000000000..0a6d52a02e
--- /dev/null
+++ b/pkg/metrics/retained_numeric.go
@@ -0,0 +1,46 @@
+package metrics
+
+import (
+ "database/sql"
+ "fmt"
+)
+
+// Retained rows normally carry int64 timestamps/ordinals and float64 values.
+// Assign those types without database/sql's per-observation reflection. Keep its
+// ordinary numeric conversion for older or atypically typed observations; this
+// is an application-side decoder, not a different SQL or driver contract.
+type retainedInt64 int64
+
+func (n *retainedInt64) Scan(src any) error {
+ if value, ok := src.(int64); ok {
+ *n = retainedInt64(value)
+ return nil
+ }
+ var compatible sql.NullInt64
+ if err := compatible.Scan(src); err != nil {
+ return err
+ }
+ if !compatible.Valid {
+ return fmt.Errorf("converting NULL to int64 is unsupported")
+ }
+ *n = retainedInt64(compatible.Int64)
+ return nil
+}
+
+type retainedFloat64 float64
+
+func (n *retainedFloat64) Scan(src any) error {
+ if value, ok := src.(float64); ok {
+ *n = retainedFloat64(value)
+ return nil
+ }
+ var compatible sql.NullFloat64
+ if err := compatible.Scan(src); err != nil {
+ return err
+ }
+ if !compatible.Valid {
+ return fmt.Errorf("converting NULL to float64 is unsupported")
+ }
+ *n = retainedFloat64(compatible.Float64)
+ return nil
+}
diff --git a/pkg/metrics/retained_numeric_test.go b/pkg/metrics/retained_numeric_test.go
new file mode 100644
index 0000000000..0f6d213d00
--- /dev/null
+++ b/pkg/metrics/retained_numeric_test.go
@@ -0,0 +1,74 @@
+package metrics
+
+import (
+ "math"
+ "testing"
+)
+
+// Read each value through database/sql twice, once with its primitive decoder
+// and once through the retained decoder. The fallback must accept and reject
+// exactly the same data, not turn malformed/NULL observations into zeroes.
+func TestRetainedNumericScansMatchDatabaseSQL(t *testing.T) {
+ db := newPlanTestDB(t)
+ cases := []struct {
+ name string
+ value any
+ }{
+ {"zero", int64(0)}, {"negative", int64(-201)},
+ {"max-int64", int64(math.MaxInt64)}, {"min-int64", int64(math.MinInt64)},
+ {"outside-exact-float-integer", int64(1<<53 + 1)},
+ {"float", 17.125}, {"negative-float", -37.5},
+ {"negative-zero", math.Copysign(0, -1)},
+ {"tiny-float", math.SmallestNonzeroFloat64}, {"large-float", math.MaxFloat64},
+ {"integer-text", "-123"}, {"float-text", "19.25"},
+ {"integer-blob", []byte("234")}, {"float-blob", []byte("1.25e3")},
+ {"overflow-int", "9223372036854775808"}, {"invalid", "not a number"},
+ {"fractional-int", "1.5"}, {"empty", ""}, {"null", nil},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ var wantInt int64
+ var gotInt retainedInt64 = -777
+ wantErr := db.QueryRow("SELECT ?", tc.value).Scan(&wantInt)
+ gotErr := db.QueryRow("SELECT ?", tc.value).Scan(&gotInt)
+ if (wantErr == nil) != (gotErr == nil) || (wantErr == nil && int64(gotInt) != wantInt) {
+ t.Fatalf("integer: standard=%d/%v retained=%d/%v", wantInt, wantErr, gotInt, gotErr)
+ }
+ if wantErr != nil && gotInt != -777 {
+ t.Fatalf("invalid integer changed destination to %d", gotInt)
+ }
+ var wantFloat float64
+ var gotFloat retainedFloat64 = -777
+ wantErr = db.QueryRow("SELECT ?", tc.value).Scan(&wantFloat)
+ gotErr = db.QueryRow("SELECT ?", tc.value).Scan(&gotFloat)
+ if (wantErr == nil) != (gotErr == nil) || (wantErr == nil && math.Float64bits(float64(gotFloat)) != math.Float64bits(wantFloat)) {
+ t.Fatalf("float: standard=%g/%v retained=%g/%v", wantFloat, wantErr, gotFloat, gotErr)
+ }
+ if wantErr != nil && gotFloat != -777 {
+ t.Fatalf("invalid float changed destination to %g", gotFloat)
+ }
+ })
+ }
+}
+
+func TestRetainedNativeNumericScansDoNotAllocate(t *testing.T) {
+ var number retainedInt64
+ var value retainedFloat64
+ var ordinal any = int64(149)
+ var timestamp any = int64(1792854756)
+ var observation any = 12.5
+ allocations := testing.AllocsPerRun(100, func() {
+ if err := number.Scan(ordinal); err != nil {
+ panic(err)
+ }
+ if err := number.Scan(timestamp); err != nil {
+ panic(err)
+ }
+ if err := value.Scan(observation); err != nil {
+ panic(err)
+ }
+ })
+ if allocations != 0 || number != 1792854756 || value != 12.5 {
+ t.Fatalf("native values: allocations=%g number=%d value=%g", allocations, number, value)
+ }
+}
diff --git a/pkg/metrics/store.go b/pkg/metrics/store.go
index fd847d572a..baf6dd1a21 100644
--- a/pkg/metrics/store.go
+++ b/pkg/metrics/store.go
@@ -2057,8 +2057,9 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
bucketCount = 0
}
var resourceID, metricType string
- var resourceNumber int
- var ts int64
+ var resourceNumber retainedInt64
+ var ts retainedInt64
+ var value retainedFloat64
var p MetricPoint
var low, high sql.NullFloat64
destinations := make([]interface{}, 0, 6)
@@ -2074,7 +2075,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
} else if !seriesOrdinals {
destinations = append(destinations, &metricType)
}
- destinations = append(destinations, &ts, &p.Value)
+ destinations = append(destinations, &ts, &value)
if !projection.valuesOnly {
destinations = append(destinations, &low, &high)
}
@@ -2086,19 +2087,20 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
if resourceOrdinals {
resourceIndex := resourceNumber
if seriesOrdinals {
- if resourceNumber < 0 || resourceNumber >= len(resourceIDs)*len(metricTypes) {
+ if resourceNumber < 0 || resourceNumber >= retainedInt64(len(resourceIDs)*len(metricTypes)) {
return nil, fmt.Errorf("invalid retained series number %d", resourceNumber)
}
if numberedSeries == nil {
- resourceID = resourceIDs[resourceNumber/len(metricTypes)]
- metricType = metricTypes[resourceNumber%len(metricTypes)]
+ resourceID = resourceIDs[resourceNumber/retainedInt64(len(metricTypes))]
+ metricType = metricTypes[resourceNumber%retainedInt64(len(metricTypes))]
}
- } else if resourceIndex < 0 || resourceIndex >= len(resourceIDs) {
+ } else if resourceIndex < 0 || resourceIndex >= retainedInt64(len(resourceIDs)) {
return nil, fmt.Errorf("invalid retained resource number %d for %d resources", resourceIndex, len(resourceIDs))
} else {
resourceID = resourceIDs[resourceIndex]
}
}
+ p.Value = float64(value)
if !projection.valuesOnly {
p.Min, p.Max = p.Value, p.Value
if low.Valid {
@@ -2109,7 +2111,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
}
}
if !streamBuckets {
- p.Timestamp = time.Unix(ts, 0)
+ p.Timestamp = time.Unix(int64(ts), 0)
if numberedSeries != nil {
numberedSeries[resourceNumber] = append(numberedSeries[resourceNumber], projection.point(p))
} else {
@@ -2117,7 +2119,7 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
}
continue
}
- start := (ts / stepSecs) * stepSecs
+ start := (int64(ts) / stepSecs) * stepSecs
if bucketCount > 0 && (bucketResource != resourceID || bucketMetric != metricType || bucketStart != start) {
flushBucket()
}