Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bound rolling-alert history retention independently of fleet projection

pulse-triage[bot]eb4460ffd9f3
diff --git a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
index f91138480c..c2843f6f58 100644
--- a/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
+++ b/docs/release-control/v6/internal/subsystems/performance-and-scalability.md
@@ -4054,3 +4054,29 @@ projection are unchanged. `GuestRow.test.tsx` covers both nested row paths and
`BackupStatusAccess.test.tsx` covers semantics, focus and observation changes.
Browser proof exercises production rows with synthetic data, not native backup
or guest-recovery acceptance.
+
+### Bounded rolling-alert window copies
+
+The existing persistent-window cache is one optional working set per metrics
+history. It admits at most 1,024 entries and 16 MiB of charged storage, including
+keys, conservative entry bookkeeping and copied timestamp/value points. Admission
+precedes the copy. Expired entries release their charge; expiry sweeps follow
+the earliest retained expiry rather than scanning the cache on every live miss.
+Replacement and Reset release old storage and accounting. Readers receive
+independently owned points, and the existing 30-second hit expiry is unchanged.
+
+At either ceiling an uncached request still returns its complete source window;
+the query, live-tail authority, interval merge, alert calculation and persistent
+history are unchanged. This bounds optional retention, not query allocations or
+whole-process RSS. Overflow can increase query work: no installed CPU or memory
+relief, or attribution of a reporter's memory sample, follows from the bound.
+
+`metric_window_cache_bounds_test.go` checks live-entry and dense/key byte bounds,
+oversized admission, replacement, zero/empty points, independent ownership,
+staggered expiry, re-admission and concurrent readers/writers/Reset without a
+database or collector. `metric_window_provider_test.go` retains cache Reset,
+request-scoped history ownership and fresh in-memory duplicate authority.
+Its connected monitor control fills each ceiling independently and checks
+complete stored/live windows, measured zero, unchanged original times, reader
+ownership and renewed admission after Reset. It uses an ordinary temporary
+history store, not an installed workload or process-memory measurement.
diff --git a/internal/monitoring/metric_window_cache_bounds_test.go b/internal/monitoring/metric_window_cache_bounds_test.go
new file mode 100644
index 0000000000..499d3c5e00
--- /dev/null
+++ b/internal/monitoring/metric_window_cache_bounds_test.go
@@ -0,0 +1,215 @@
+package monitoring
+
+import (
+ "fmt"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+ "unsafe"
+
+ "github.com/rcourtman/pulse-go-rewrite/internal/alerts"
+)
+
+// These controls use only the in-process cache. No database, collector,
+// listener, process profile, forced GC or installed workload is involved.
+func TestMetricWindowCacheBoundsLiveEntries(t *testing.T) {
+ history := NewMetricsHistory(32, time.Hour)
+ now := time.Now()
+ for i := 0; i < 1100; i++ {
+ history.cacheMetricWindow(fmt.Sprintf("vm-%04d", i), []MetricPoint{{Timestamp: now, Value: float64(i)}}, now)
+ }
+ if got := len(history.metricWindowCache); got > 1024 {
+ t.Fatalf("live cache entries = %d, want at most 1024; expiry is not a capacity bound", got)
+ }
+ if got := len(history.metricWindowCache); got != metricWindowCacheMaxEntries {
+ t.Fatalf("retained %d entries, want the admitted working set %d", got, metricWindowCacheMaxEntries)
+ }
+ // Refusing a new copy must not evict or corrupt an unexpired accepted one.
+ if got, ok := history.cachedMetricWindow("vm-0000", now); !ok || len(got) != 1 || got[0].Value != 0 {
+ t.Fatalf("capacity refusal lost an accepted measured-zero window: %+v, hit=%v", got, ok)
+ }
+ if _, ok := history.cachedMetricWindow("vm-1099", now); ok {
+ t.Fatal("a refused window was retained")
+ }
+ assertMetricWindowCacheAccounting(t, history)
+}
+
+func TestMetricWindowCacheBoundsDenseWindows(t *testing.T) {
+ history := NewMetricsHistory(32, time.Hour)
+ now := time.Now()
+ points := make([]MetricPoint, 4096)
+ for i := range points {
+ points[i] = MetricPoint{Timestamp: now.Add(time.Duration(i) * time.Millisecond), Value: float64(i)}
+ }
+ for i := 0; i < 200; i++ {
+ history.cacheMetricWindow(fmt.Sprintf("vm-%04d", i), points, now)
+ }
+ retained := 0
+ for _, entry := range history.metricWindowCache {
+ retained += len(entry.points)
+ }
+ // Charge points conservatively at 64 bytes, plus keys/map/entry overhead.
+ if retained*64 > 16<<20 {
+ t.Fatalf("retained %d dense points (%d charged point bytes), want at most 16 MiB", retained, retained*64)
+ }
+ if retained == 0 {
+ t.Fatal("budget disabled the ordinary dense working set")
+ }
+ assertMetricWindowCacheAccounting(t, history)
+}
+
+func assertMetricWindowCacheAccounting(t *testing.T, history *MetricsHistory) {
+ t.Helper()
+ history.metricWindowMu.Lock()
+ defer history.metricWindowMu.Unlock()
+ sum := 0
+ for key, entry := range history.metricWindowCache {
+ want := metricWindowCacheEntryBytes + len(key) + len(entry.points)*metricWindowCachePointBytes
+ if entry.bytes != want || cap(entry.points) != len(entry.points) {
+ t.Fatalf("entry charge/backing capacity diverged: bytes=%d want=%d len=%d cap=%d", entry.bytes, want, len(entry.points), cap(entry.points))
+ }
+ sum += entry.bytes
+ }
+ if sum != history.metricWindowBytes || sum > metricWindowCacheMaxBytes || len(history.metricWindowCache) > metricWindowCacheMaxEntries {
+ t.Fatalf("cache budget diverged: summed=%d accounted=%d entries=%d", sum, history.metricWindowBytes, len(history.metricWindowCache))
+ }
+}
+
+func TestMetricWindowCacheBudgetIncludesKeysAndEmptyWindows(t *testing.T) {
+ if metricWindowCachePointBytes < int(unsafe.Sizeof(alerts.MetricWindowPoint{})) {
+ t.Fatal("point budget no longer covers the native point representation")
+ }
+ history := NewMetricsHistory(32, time.Hour)
+ now := time.Now()
+ // No point bytes: large keys alone must be bounded, too.
+ prefix := strings.Repeat("x", 32<<10)
+ for i := 0; i < 1024; i++ {
+ history.cacheMetricWindow(fmt.Sprintf("%s-%04d", prefix, i), nil, now)
+ }
+ assertMetricWindowCacheAccounting(t, history)
+ if len(history.metricWindowCache) == 0 || len(history.metricWindowCache) >= metricWindowCacheMaxEntries {
+ t.Fatal("the byte budget did not admit/bound long empty-window keys")
+ }
+}
+
+func TestMetricWindowCacheOversizedReplacementDoesNotReviveOldReadings(t *testing.T) {
+ now := time.Now()
+ for _, kind := range []string{"points", "key"} {
+ t.Run(kind, func(t *testing.T) {
+ history := NewMetricsHistory(32, time.Hour)
+ key := "vm-1"
+ points := []MetricPoint{{Timestamp: now, Value: 5}}
+ if kind == "key" {
+ key = strings.Repeat("x", metricWindowCacheMaxBytes)
+ } else {
+ history.cacheMetricWindow(key, points, now)
+ points = make([]MetricPoint, metricWindowCacheMaxBytes/metricWindowCachePointBytes+1)
+ }
+ history.cacheMetricWindow(key, points, now)
+ if _, ok := history.cachedMetricWindow(key, now); ok {
+ t.Fatal("an oversized window revived a previous reading or kept its extra copy")
+ }
+ if history.metricWindowCache != nil || history.metricWindowBytes != 0 {
+ t.Fatal("oversized admission allocated or retained the cache working set")
+ }
+ })
+ }
+}
+
+func TestMetricWindowCacheReplacementAndIndependentOwnership(t *testing.T) {
+ history := NewMetricsHistory(32, time.Hour)
+ now := time.Now()
+ points := []MetricPoint{{Timestamp: now, Value: 42}, {Timestamp: now.Add(time.Second), Value: 99}}
+ history.cacheMetricWindow("vm-1", points, now)
+ points[0].Value = -1
+ got, ok := history.cachedMetricWindow("vm-1", now)
+ if !ok || len(got) != 2 || got[0].Value != 42 || got[1].Value != 99 {
+ t.Fatalf("admitted window changed with query storage: %+v, hit=%v", got, ok)
+ }
+ got[0].Value = -2
+ again, _ := history.cachedMetricWindow("vm-1", now)
+ if again[0].Value != 42 {
+ t.Fatal("a reader changed another reader's retained window")
+ }
+ history.cacheMetricWindow("vm-1", []MetricPoint{{Timestamp: now, Value: 0}}, now.Add(time.Second))
+ assertMetricWindowCacheAccounting(t, history)
+ if len(history.metricWindowCache) != 1 || len(history.metricWindowCache["vm-1"].points) != 1 {
+ t.Fatal("replacement double-counted or retained the old backing array")
+ }
+}
+
+func TestMetricWindowCacheExpiryReleasesAndReadmitsAtCapacity(t *testing.T) {
+ history := NewMetricsHistory(32, time.Hour)
+ now := time.Now()
+ for i := 0; i < metricWindowCacheMaxEntries; i++ {
+ history.cacheMetricWindow(fmt.Sprintf("vm-%04d", i), nil, now)
+ }
+ expiry := now.Add(metricWindowPersistentCacheTTL)
+ if _, ok := history.cachedMetricWindow("vm-0000", expiry); ok {
+ t.Fatal("exact-expiry lookup renewed stale evidence")
+ }
+ // An ordinary insert sweeps the other expired windows and frees budget.
+ history.cacheMetricWindow("new", []MetricPoint{{Timestamp: expiry, Value: 7}}, expiry)
+ assertMetricWindowCacheAccounting(t, history)
+ if len(history.metricWindowCache) != 1 || history.metricWindowSweepAt != expiry.Add(metricWindowPersistentCacheTTL) {
+ t.Fatal("expired windows prevented ordinary re-admission or reset the wrong expiry")
+ }
+ if _, ok := history.cachedMetricWindow("new", expiry.Add(metricWindowPersistentCacheTTL)); ok {
+ t.Fatal("replacement outlived its own TTL")
+ }
+ if history.metricWindowCache != nil || history.metricWindowBytes != 0 || !history.metricWindowSweepAt.IsZero() {
+ t.Fatal("the last expired entry kept map storage or accounting")
+ }
+}
+
+func TestMetricWindowCacheSweepKeepsUnexpiredStaggeredWindows(t *testing.T) {
+ history := NewMetricsHistory(32, time.Hour)
+ now := time.Now()
+ history.cacheMetricWindow("old", nil, now)
+ history.cacheMetricWindow("later", nil, now.Add(10*time.Second))
+ expiry := now.Add(metricWindowPersistentCacheTTL)
+ history.cacheMetricWindow("new", nil, expiry)
+ if _, ok := history.cachedMetricWindow("old", expiry); ok {
+ t.Fatal("sweep retained expired history")
+ }
+ if _, ok := history.cachedMetricWindow("later", expiry); !ok {
+ t.Fatal("sweep discarded an unexpired independent window")
+ }
+ if history.metricWindowSweepAt != now.Add(10*time.Second+metricWindowPersistentCacheTTL) {
+ t.Fatal("sweep did not follow the earliest live expiry")
+ }
+ assertMetricWindowCacheAccounting(t, history)
+}
+
+func TestMetricWindowCacheResetAndConcurrentReadersWriters(t *testing.T) {
+ history := NewMetricsHistory(32, time.Hour)
+ now := time.Now()
+ var workers sync.WaitGroup
+ for worker := 0; worker < 16; worker++ {
+ workers.Add(1)
+ go func() {
+ defer workers.Done()
+ for i := 0; i < 128; i++ {
+ key := fmt.Sprintf("worker-%02d-%03d", worker, i)
+ history.cacheMetricWindow(key, []MetricPoint{{Timestamp: now, Value: float64(worker)}}, now)
+ if got, ok := history.cachedMetricWindow(key, now); ok {
+ if len(got) != 1 || got[0].Value != float64(worker) {
+ t.Error("concurrent caller received another window's values")
+ continue
+ }
+ got[0].Value = -1
+ }
+ if i%31 == 0 {
+ history.Reset()
+ }
+ }
+ }()
+ }
+ workers.Wait()
+ assertMetricWindowCacheAccounting(t, history)
+ history.Reset()
+ if history.metricWindowCache != nil || history.metricWindowBytes != 0 || !history.metricWindowSweepAt.IsZero() {
+ t.Fatal("reset retained history cache storage/accounting")
+ }
+}
diff --git a/internal/monitoring/metric_window_provider.go b/internal/monitoring/metric_window_provider.go
index 7129dca78a..bb01b77d0b 100644
--- a/internal/monitoring/metric_window_provider.go
+++ b/internal/monitoring/metric_window_provider.go
@@ -10,11 +10,20 @@ import (
"github.com/rs/zerolog/log"
)
-const metricWindowPersistentCacheTTL = 30 * time.Second
+const (
+ metricWindowPersistentCacheTTL = 30 * time.Second
+ metricWindowCacheMaxEntries = 1024
+ metricWindowCacheMaxBytes = 16 << 20
+ // Charge conservatively for the key, map/entry bookkeeping and each
+ // copied timestamp/value. This is a cache working-set budget, not RSS.
+ metricWindowCacheEntryBytes = 256
+ metricWindowCachePointBytes = 64
+)
type metricWindowCacheEntry struct {
points []alerts.MetricWindowPoint
expiresAt time.Time
+ bytes int
}
func metricHistoryName(metric string) string {
@@ -96,16 +105,9 @@ func (m *Monitor) persistentMetricWindow(history *MetricsHistory, resourceType,
cacheKey := strings.Join([]string{resourceType, resourceID, metric, fmt.Sprint(end.Sub(start).Seconds())}, "\x00")
now := time.Now()
if history != nil {
- history.metricWindowMu.Lock()
- if cached, ok := history.metricWindowCache[cacheKey]; ok && now.Before(cached.expiresAt) {
- points := make([]MetricPoint, len(cached.points))
- for i, point := range cached.points {
- points[i] = MetricPoint{Timestamp: point.Timestamp, Value: point.Value}
- }
- history.metricWindowMu.Unlock()
+ if points, ok := history.cachedMetricWindow(cacheKey, now); ok {
return points
}
- history.metricWindowMu.Unlock()
}
var best []MetricPoint
@@ -124,25 +126,83 @@ func (m *Monitor) persistentMetricWindow(history *MetricsHistory, resourceType,
}
}
if history != nil {
- cached := make([]alerts.MetricWindowPoint, len(best))
- for i, point := range best {
- cached[i] = alerts.MetricWindowPoint{Timestamp: point.Timestamp, Value: point.Value}
+ history.cacheMetricWindow(cacheKey, best, now)
+ }
+ return best
+}
+
+func (mh *MetricsHistory) cachedMetricWindow(key string, now time.Time) ([]MetricPoint, bool) {
+ mh.metricWindowMu.Lock()
+ defer mh.metricWindowMu.Unlock()
+ if cached, ok := mh.metricWindowCache[key]; ok {
+ if !now.Before(cached.expiresAt) {
+ mh.dropMetricWindowLocked(key)
+ return nil, false
}
- history.metricWindowMu.Lock()
- if history.metricWindowCache == nil {
- history.metricWindowCache = make(map[string]metricWindowCacheEntry)
+ points := make([]MetricPoint, len(cached.points))
+ for i, point := range cached.points {
+ points[i] = MetricPoint{Timestamp: point.Timestamp, Value: point.Value}
}
- if len(history.metricWindowCache) >= 1024 {
- for key, entry := range history.metricWindowCache {
- if !now.Before(entry.expiresAt) {
- delete(history.metricWindowCache, key)
- }
+ return points, true
+ }
+ return nil, false
+}
+
+// Overflow windows still return the complete query result through the caller;
+// only the optional private copy is refused. Neither persistent history nor
+// alert coverage is truncated. Admit before copying a potentially large window.
+func (mh *MetricsHistory) cacheMetricWindow(key string, points []MetricPoint, now time.Time) {
+ mh.metricWindowMu.Lock()
+ defer mh.metricWindowMu.Unlock()
+ mh.dropMetricWindowLocked(key)
+ // Sweep when the earliest retained entry can have expired, not on every
+ // miss at capacity. A wide live fleet otherwise scans 1024 keys per miss.
+ if !mh.metricWindowSweepAt.IsZero() && !now.Before(mh.metricWindowSweepAt) {
+ var next time.Time
+ for cachedKey, entry := range mh.metricWindowCache {
+ if !now.Before(entry.expiresAt) {
+ mh.dropMetricWindowLocked(cachedKey)
+ } else if next.IsZero() || entry.expiresAt.Before(next) {
+ next = entry.expiresAt
}
}
- history.metricWindowCache[cacheKey] = metricWindowCacheEntry{points: cached, expiresAt: now.Add(metricWindowPersistentCacheTTL)}
- history.metricWindowMu.Unlock()
+ mh.metricWindowSweepAt = next
+ }
+ // Check division before multiplication so an oversized key/window cannot
+ // overflow the charge or allocate an uncapped second copy.
+ if len(key) > metricWindowCacheMaxBytes-metricWindowCacheEntryBytes ||
+ len(points) > (metricWindowCacheMaxBytes-metricWindowCacheEntryBytes-len(key))/metricWindowCachePointBytes {
+ return
+ }
+ charge := metricWindowCacheEntryBytes + len(key) + len(points)*metricWindowCachePointBytes
+ if len(mh.metricWindowCache) >= metricWindowCacheMaxEntries || charge > metricWindowCacheMaxBytes-mh.metricWindowBytes {
+ return
+ }
+ cached := make([]alerts.MetricWindowPoint, len(points))
+ for i, point := range points {
+ cached[i] = alerts.MetricWindowPoint{Timestamp: point.Timestamp, Value: point.Value}
+ }
+ if mh.metricWindowCache == nil {
+ mh.metricWindowCache = make(map[string]metricWindowCacheEntry)
+ }
+ expiresAt := now.Add(metricWindowPersistentCacheTTL)
+ mh.metricWindowCache[key] = metricWindowCacheEntry{points: cached, expiresAt: expiresAt, bytes: charge}
+ mh.metricWindowBytes += charge
+ if mh.metricWindowSweepAt.IsZero() || expiresAt.Before(mh.metricWindowSweepAt) {
+ mh.metricWindowSweepAt = expiresAt
+ }
+}
+
+func (mh *MetricsHistory) dropMetricWindowLocked(key string) {
+ if entry, ok := mh.metricWindowCache[key]; ok {
+ delete(mh.metricWindowCache, key)
+ mh.metricWindowBytes -= entry.bytes
+ if len(mh.metricWindowCache) == 0 {
+ mh.metricWindowCache = nil
+ mh.metricWindowBytes = 0
+ mh.metricWindowSweepAt = time.Time{}
+ }
}
- return best
}
func metricWindowCoverage(points []MetricPoint) time.Duration {
diff --git a/internal/monitoring/metric_window_provider_test.go b/internal/monitoring/metric_window_provider_test.go
index ef74ee88d4..4d035fea4b 100644
--- a/internal/monitoring/metric_window_provider_test.go
+++ b/internal/monitoring/metric_window_provider_test.go
@@ -1,6 +1,7 @@
package monitoring
import (
+ "fmt"
"strings"
"sync"
"testing"
@@ -114,15 +115,11 @@ func TestMetricWindowMergePrefersFreshInMemoryDuplicate(t *testing.T) {
func TestMetricsHistoryResetClearsMetricWindowCache(t *testing.T) {
history := NewMetricsHistory(32, time.Hour)
- history.metricWindowCache = map[string]metricWindowCacheEntry{
- "vm\x00vm-1\x00cpu\x00300": {
- points: []alerts.MetricWindowPoint{{Timestamp: time.Now().UTC(), Value: 42}},
- expiresAt: time.Now().Add(time.Minute),
- },
- }
+ now := time.Now().UTC()
+ history.cacheMetricWindow("vm\x00vm-1\x00cpu\x00300", []MetricPoint{{Timestamp: now, Value: 42}}, now)
history.Reset()
- if history.metricWindowCache != nil {
+ if history.metricWindowCache != nil || history.metricWindowBytes != 0 || !history.metricWindowSweepAt.IsZero() {
t.Fatalf("metric window cache survived reset: %+v", history.metricWindowCache)
}
}
@@ -132,12 +129,7 @@ func TestMetricWindowPointsKeepsHistorySnapshotDuringReplacement(t *testing.T) {
monitor := newChartFallbackTestMonitor(t)
history := monitor.metricsHistory
cacheKey := strings.Join([]string{"vm", "vm-1", "cpu", "300"}, "\x00")
- history.metricWindowCache = map[string]metricWindowCacheEntry{
- cacheKey: {
- points: []alerts.MetricWindowPoint{{Timestamp: now.Add(-time.Minute), Value: 42}},
- expiresAt: now.Add(time.Minute),
- },
- }
+ history.cacheMetricWindow(cacheKey, []MetricPoint{{Timestamp: now.Add(-time.Minute), Value: 42}}, now)
resume := make(chan struct{})
var resumeOnce sync.Once
@@ -183,3 +175,90 @@ func TestMetricWindowPointsKeepsHistorySnapshotDuringReplacement(t *testing.T) {
t.Fatal("metricWindowPoints did not finish after history replacement")
}
}
+
+func TestMetricWindowPointsPreservesCompleteWindowAtCacheCapacity(t *testing.T) {
+ for _, pressure := range []string{"none", "entries", "bytes"} {
+ t.Run(pressure, func(t *testing.T) {
+ monitor := newChartFallbackTestMonitor(t)
+ history := monitor.metricsHistory
+ now := time.Now().UTC().Truncate(time.Second)
+ writeRawMetricBatch(t, monitor.metricsStore, "vm", "vm-1", "cpu", []MetricPoint{
+ {Timestamp: now.Add(-4 * time.Minute), Value: 0},
+ {Timestamp: now.Add(-2 * time.Minute), Value: 99},
+ {Timestamp: now.Add(-time.Minute), Value: 30},
+ })
+ history.AddGuestMetric("vm-1", "cpu", 55, now.Add(-2*time.Minute))
+ history.AddGuestMetric("vm-1", "cpu", 60, now.Add(-30*time.Second))
+
+ // Fill the optional cache, not the query source or live history.
+ // Its complete source result must still reach alert evaluation.
+ cacheNow := time.Now()
+ switch pressure {
+ case "entries":
+ for i := 0; i < metricWindowCacheMaxEntries; i++ {
+ history.cacheMetricWindow(fmt.Sprintf("other-%04d", i), nil, cacheNow)
+ }
+ case "bytes":
+ points := make([]MetricPoint, 4096)
+ for i := 0; i < 63; i++ {
+ history.cacheMetricWindow(fmt.Sprintf("other-%04d", i), points, cacheNow)
+ }
+ remaining := metricWindowCacheMaxBytes - history.metricWindowBytes
+ if remaining <= metricWindowCacheEntryBytes {
+ t.Fatal("fixture has no room for the final exact-budget entry")
+ }
+ history.cacheMetricWindow(strings.Repeat("x", remaining-metricWindowCacheEntryBytes), nil, cacheNow)
+ if history.metricWindowBytes != metricWindowCacheMaxBytes {
+ t.Fatalf("fixture filled %d bytes, want %d", history.metricWindowBytes, metricWindowCacheMaxBytes)
+ }
+ }
+ retainedEntries, retainedBytes := len(history.metricWindowCache), history.metricWindowBytes
+ request := alerts.MetricWindowRequest{
+ ResourceID: "vm-1", ResourceType: "vm", Metric: "cpu",
+ Start: now.Add(-5 * time.Minute), End: now,
+ }
+ want := []alerts.MetricWindowPoint{
+ {Timestamp: now.Add(-4 * time.Minute), Value: 0},
+ {Timestamp: now.Add(-2 * time.Minute), Value: 55},
+ {Timestamp: now.Add(-time.Minute), Value: 30},
+ {Timestamp: now.Add(-30 * time.Second), Value: 60},
+ }
+ assertWindow := func() {
+ t.Helper()
+ got, err := monitor.metricWindowPoints(request)
+ if err != nil || len(got) != len(want) {
+ t.Fatalf("window lost source coverage: got=%+v err=%v want=%+v", got, err, want)
+ }
+ for i := range want {
+ if !got[i].Timestamp.Equal(want[i].Timestamp) || got[i].Value != want[i].Value {
+ t.Fatalf("window changed source time/value or live-tail authority: got=%+v want=%+v", got, want)
+ }
+ }
+ // A returned result never owns the cache or next query's data.
+ got[0].Value = -1
+ }
+ assertWindow()
+ assertWindow()
+ cacheKey := strings.Join([]string{"vm", "vm-1", "cpu", "300"}, "\x00")
+ _, cached := history.cachedMetricWindow(cacheKey, time.Now())
+ if cached != (pressure == "none") {
+ t.Fatalf("optional copy admitted=%v under %q pressure", cached, pressure)
+ }
+ if pressure != "none" && (len(history.metricWindowCache) != retainedEntries || history.metricWindowBytes != retainedBytes) {
+ t.Fatal("refused optional copy changed the retained working set")
+ }
+ assertMetricWindowCacheAccounting(t, history)
+
+ // Reset releases capacity and live history, not the persistent source.
+ // The ordinary reader can cache again and still returns every point.
+ history.Reset()
+ history.AddGuestMetric("vm-1", "cpu", 55, now.Add(-2*time.Minute))
+ history.AddGuestMetric("vm-1", "cpu", 60, now.Add(-30*time.Second))
+ assertWindow()
+ if _, ok := history.cachedMetricWindow(cacheKey, time.Now()); !ok {
+ t.Fatal("reset did not restore ordinary cache admission")
+ }
+ assertMetricWindowCacheAccounting(t, history)
+ })
+ }
+}
diff --git a/internal/monitoring/metrics_history.go b/internal/monitoring/metrics_history.go
index d32f1b9e54..01550c5df0 100644
--- a/internal/monitoring/metrics_history.go
+++ b/internal/monitoring/metrics_history.go
@@ -58,6 +58,8 @@ type MetricsHistory struct {
capacityForecastCache map[string]storageCapacityForecastCacheEntry
metricWindowMu sync.Mutex
metricWindowCache map[string]metricWindowCacheEntry
+ metricWindowBytes int
+ metricWindowSweepAt time.Time
}
// NewMetricsHistory creates a new metrics history tracker
@@ -145,6 +147,8 @@ func (mh *MetricsHistory) Reset() {
mh.metricWindowMu.Lock()
mh.metricWindowCache = nil
+ mh.metricWindowBytes = 0
+ mh.metricWindowSweepAt = time.Time{}
mh.metricWindowMu.Unlock()
}