Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Reduce retained chart row decoding without changing observations

rcourtman473926b495eb
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
index c1827263c2..5e5ab57846 100644
--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
@@ -553,7 +553,7 @@ which retention tiers have observations in that snapshot. Every present tier
participates in the shared indexed overlap query. Presence never stands in for
per-series coverage. The store retains at most 32 compiled read-statement shapes
and 32 SQL templates keyed only by parameter counts, tier order and aggregation
-shape, projection and physical layout. Alphabetic named SQLite bindings supply each current identity, window and
+shape, projection, physical layout and bounded integer transport. Alphabetic named SQLite bindings supply each current identity, window and
display step once across all branches. Bindings use `database/sql.Named` so the
driver need not repeatedly convert ordinals to strings while matching a large
parameter set. Names describe parameter positions only. Resource IDs and all
@@ -581,6 +581,15 @@ including all tier priorities and metric filters. Multi-stage rollups preserve
recorded minima and maxima instead of taking extrema from bucket averages.
Already discarded historical extrema cannot be reconstructed by this change.
+Filtered raw value fleets carry each observation's timestamp and statement-local
+series number in one SQLite integer when the requested window proves that the
+product and sum fit a signed 64-bit integer. The decoder uses floor division for
+negative epochs. Extreme timestamp windows retain separate columns. This
+transport choice participates in the SQL shape key, so a cached ordinary-window
+query cannot be reused for an overflowing window. It changes neither stored
+identity nor API representation, metric selection, retention reconciliation,
+returned floating-point values or chronological ordering.
+
The `retained-metric-query-coverage` follow-up addresses the specific first-tier
query defect reproduced during Patrol outcome qualification. Returned points
still do not prove continuous collection or a complete requested window.
diff --git a/pkg/metrics/store.go b/pkg/metrics/store.go
index baf6dd1a21..4a27ea1c1f 100644
--- a/pkg/metrics/store.go
+++ b/pkg/metrics/store.go
@@ -7,6 +7,7 @@ import (
"database/sql"
"errors"
"fmt"
+ "math"
"net/url"
"os"
"path/filepath"
@@ -1645,6 +1646,7 @@ type retainedQueryShape struct {
tiers [4]Tier
aggregate, groupSeries bool
valuesOnly, clustered bool
+ packedSeries bool
}
func retainedQueryParameters(resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier) []interface{} {
@@ -1677,6 +1679,7 @@ func (s *Store) retainedQuerySQL(resourceType string, resourceIDs, metricTypes [
func (s *Store) retainedQuerySQLForIndex(index string, resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier, groupSeries, valuesOnly bool) (string, []interface{}) {
shape := retainedQueryShape{resources: len(resourceIDs), metrics: len(metricTypes), aggregate: stepSecs > 1, groupSeries: groupSeries, valuesOnly: valuesOnly, clustered: index == clusteredMetricsIndex}
+ shape.packedSeries = retainedQueryUsesPackedSeries(len(resourceIDs), len(metricTypes), start, end, stepSecs, groupSeries, valuesOnly)
cacheable := len(tiers) <= len(shape.tiers)
copy(shape.tiers[:], tiers)
if cacheable {
@@ -1719,10 +1722,29 @@ func retainedQueryUsesResourceOrdinals(resources int, stepSecs int64, groupSerie
return resources > 1 && stepSecs <= 1 && !groupSeries
}
+// A raw value read needs two integers identifying each observation: its
+// statement-local series and timestamp. Carry them in one integer while the
+// requested bounds prove multiplication and addition cannot overflow SQLite's
+// signed integer. This is only row transport, never storage or API encoding.
+// Extreme windows use the ordinary separate columns in the same reader.
+func retainedQueryUsesPackedSeries(resources, metrics int, start, end time.Time, stepSecs int64, groupSeries, valuesOnly bool) bool {
+ if !valuesOnly || metrics < 2 || !retainedQueryUsesResourceOrdinals(resources, stepSecs, groupSeries) {
+ return false
+ }
+ if int64(metrics) > math.MaxInt64/int64(resources) {
+ return false
+ }
+ series := int64(resources) * int64(metrics)
+ return end.Unix() >= start.Unix() && start.Unix() >= math.MinInt64/series &&
+ end.Unix() <= (math.MaxInt64-series+1)/series
+}
+
func retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, metricTypes []string, start, end time.Time, stepSecs int64, tiers []Tier, groupSeries, valuesOnly bool) (string, []interface{}) {
params := retainedQueryParameters(resourceType, resourceIDs, metricTypes, start, end, stepSecs, tiers)
resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), stepSecs, groupSeries)
seriesOrdinals := resourceOrdinals && len(metricTypes) > 1
+ packedSeries := retainedQueryUsesPackedSeries(len(resourceIDs), len(metricTypes), start, end, stepSecs, groupSeries, valuesOnly)
+ var seriesExpression string
identityColumns := ""
if len(resourceIDs) != 1 {
if seriesOrdinals {
@@ -1733,7 +1755,8 @@ func retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, m
for i := range metricTypes {
metricCases[i] = fmt.Sprintf("WHEN :p%d THEN %d", 2+len(resourceIDs)+i, i)
}
- identityColumns = fmt.Sprintf("(r.resource_number * %d + CASE m.metric_type %s END) AS series_number, ", len(metricTypes), strings.Join(metricCases, " "))
+ seriesExpression = fmt.Sprintf("(r.resource_number * %d + CASE m.metric_type %s END)", len(metricTypes), strings.Join(metricCases, " "))
+ identityColumns = seriesExpression + " AS series_number, "
} else if resourceOrdinals {
identityColumns += "r.resource_number, "
} else {
@@ -1772,6 +1795,9 @@ func retainedQuerySQLWithIndex(index string, resourceType string, resourceIDs, m
// value again for each missing bound makes SQLite and its driver decode
// three floats for an observation that stores only one.
projection := identityColumns + `m.timestamp, m.value`
+ if packedSeries {
+ projection = fmt.Sprintf("(m.timestamp * %d + %s) AS series_sample, m.value", len(resourceIDs)*len(metricTypes), seriesExpression)
+ }
if !valuesOnly {
projection += `, m.min_value, m.max_value`
}
@@ -1970,6 +1996,8 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
sqlQuery, params := s.retainedQuerySQLForIndex(index, resourceType, resourceIDs, metricTypes, start, end, queryStep, tiers, streamBuckets, projection.valuesOnly)
resourceOrdinals := retainedQueryUsesResourceOrdinals(len(resourceIDs), queryStep, streamBuckets)
seriesOrdinals := resourceOrdinals && len(metricTypes) > 1
+ packedSeries := retainedQueryUsesPackedSeries(len(resourceIDs), len(metricTypes), start, end, queryStep, streamBuckets, projection.valuesOnly)
+ seriesWidth := int64(len(resourceIDs)) * int64(len(metricTypes))
queryRows := func() (*sql.Rows, error) { return tx.Query(sqlQuery, params...) }
if tx == nil {
@@ -2075,7 +2103,10 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
} else if !seriesOrdinals {
destinations = append(destinations, &metricType)
}
- destinations = append(destinations, &ts, &value)
+ if !packedSeries {
+ destinations = append(destinations, &ts)
+ }
+ destinations = append(destinations, &value)
if !projection.valuesOnly {
destinations = append(destinations, &low, &high)
}
@@ -2084,6 +2115,17 @@ func queryRetainedChunkAttempt[T any](s *Store, resourceType string, resourceIDs
log.Warn().Err(err).Msg("Failed to scan batch metric row")
continue
}
+ if packedSeries {
+ encoded := int64(resourceNumber)
+ ts = retainedInt64(encoded / seriesWidth)
+ resourceNumber = retainedInt64(encoded % seriesWidth)
+ // Go division truncates toward zero. Restore floor division for
+ // observations before the Unix epoch so the ordinal stays positive.
+ if resourceNumber < 0 {
+ resourceNumber += retainedInt64(seriesWidth)
+ ts--
+ }
+ }
if resourceOrdinals {
resourceIndex := resourceNumber
if seriesOrdinals {
diff --git a/pkg/metrics/store_query_plan_test.go b/pkg/metrics/store_query_plan_test.go
index 933e8cd7de..1aa5f0f2cc 100644
--- a/pkg/metrics/store_query_plan_test.go
+++ b/pkg/metrics/store_query_plan_test.go
@@ -708,7 +708,9 @@ func TestRetainedRawFleetSeriesNumbersBindCurrentMetrics(t *testing.T) {
}
rows.Close()
want := []string{"series_number", "timestamp", "value"}
- if !valuesOnly {
+ if valuesOnly {
+ want = []string{"series_sample", "value"}
+ } else {
want = append(want, "min_value", "max_value")
}
if !reflect.DeepEqual(columns, want) {
@@ -737,6 +739,67 @@ func TestRetainedRawFleetSeriesNumbersBindCurrentMetrics(t *testing.T) {
}
}
+func TestRetainedPackedSeriesKeepsNegativeTimestampsAndOverflowFallback(t *testing.T) {
+ db := newPlanTestDB(t)
+ store := &Store{db: pdb.Wrap(db, "packed-series-boundary")}
+ ids, names := []string{"second", "first"}, []string{"memory", "cpu", "disk"}
+ const width int64 = 6
+ const maximum = int64(^uint64(0) >> 1)
+ const minimum = -maximum - 1
+ for _, tc := range []struct {
+ name string
+ start int64
+ packed bool
+ }{
+ {"negative epoch", -100, true},
+ {"ordinary epoch", 1_800_000_000, true},
+ {"near positive bound", (maximum-width+1)/width - 2, true},
+ {"positive overflow", (maximum - width + 1) / width, false},
+ {"near negative bound", minimum / width, true},
+ {"negative overflow", minimum/width - 1, false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ start, end := time.Unix(tc.start, 0), time.Unix(tc.start+2, 0)
+ if got := retainedQueryUsesPackedSeries(len(ids), len(names), start, end, 0, false, true); got != tc.packed {
+ t.Fatalf("packed=%t want %t", got, tc.packed)
+ }
+ for i, id := range ids {
+ for j, metric := range names {
+ for offset := int64(0); offset <= 2; offset++ {
+ if _, err := db.Exec(`INSERT OR REPLACE INTO metrics(resource_type,resource_id,metric_type,tier,timestamp,value) VALUES ('node',?,?,'raw',?,?)`, id, metric, tc.start+offset, float64(i*100+j*10)+float64(offset)/3); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ }
+ // Reuse one Store and equal parameter counts across packed and
+ // overflow windows. Cached SQL must still choose the correct shape.
+ full, err := store.QueryMetricTypesBatch("node", ids, names, start, end, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ values, err := store.QueryMetricValuesBatch("node", ids, names, start, end, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ assertMetricValueProjection(t, full, values)
+ for _, id := range ids {
+ for _, name := range names {
+ points := values[id][name]
+ if len(points) != 3 {
+ t.Fatalf("lost observations for %s/%s: %+v", id, name, points)
+ }
+ for i, point := range points {
+ if point.Timestamp.Unix() != tc.start+int64(i) {
+ t.Fatalf("moved timestamp: %v", point.Timestamp.Unix())
+ }
+ }
+ }
+ }
+ })
+ }
+}
+
// Direct ordinal accumulation must be indistinguishable from the existing
// single-resource assembler, including interleaved rows and empty series. Wide
// sparse filters exercise the bounded fallback without losing statement scope.