SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
fix: fix time series caching edge cases (#12813)
Naman Verma368efd2afedb
diff --git a/docs/api/openapi.yml b/docs/api/openapi.ymlindex 7ebd05fe0d..6b28d542ee 100644--- a/docs/api/openapi.yml+++ b/docs/api/openapi.yml@@ -8616,6 +8616,8 @@ components:type: stringquery:type: string+ skipStepAlignment:+ type: booleanstats:type: booleanstep:diff --git a/frontend/src/api/generated/services/sigNoz.schemas.ts b/frontend/src/api/generated/services/sigNoz.schemas.tsindex 5638598dac..6650d2c9e7 100644--- a/frontend/src/api/generated/services/sigNoz.schemas.ts+++ b/frontend/src/api/generated/services/sigNoz.schemas.ts@@ -6116,6 +6116,10 @@ export interface Querybuildertypesv5PromQueryDTO {* @type string*/query?: string;+ /**+ * @type boolean+ */+ skipStepAlignment?: boolean;/*** @type boolean*/diff --git a/pkg/querier/bucket_cache.go b/pkg/querier/bucket_cache.goindex c637184a62..bc5adfdeef 100644--- a/pkg/querier/bucket_cache.go+++ b/pkg/querier/bucket_cache.go@@ -11,6 +11,7 @@ import ("github.com/SigNoz/signoz/pkg/cache""github.com/SigNoz/signoz/pkg/errors""github.com/SigNoz/signoz/pkg/factory"+ "github.com/SigNoz/signoz/pkg/types/metrictypes"qbtypes "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5""github.com/SigNoz/signoz/pkg/valuer")@@ -89,14 +90,15 @@ func (bc *bucketCache) GetMissRanges(// Extract relevant buckets and merge themrelevantBuckets := bc.filterRelevantBuckets(data.Buckets, startMs, endMs)if len(relevantBuckets) == 0 {- return nil, missing+ // nothing cached overlaps the window, so all of it has to be fetched.+ return nil, []*qbtypes.TimeRange{{From: startMs, To: endMs}}}// Merge buckets into a single resultmergedResult := bc.mergeBuckets(ctx, relevantBuckets, data.Warnings)// Filter the merged result to only include values within the requested time range- mergedResult = bc.filterResultToTimeRange(mergedResult, startMs, endMs)+ mergedResult = bc.filterResultToTimeRange(mergedResult, q, startMs, endMs, stepMs)return mergedResult, missing}@@ -208,8 +210,8 @@ func (bc *bucketCache) generateCacheKey(q qbtypes.Query) string {// findMissingRangesWithStep identifies time ranges not covered by cached buckets with step alignment.func (bc *bucketCache) findMissingRangesWithStep(buckets []*qbtypes.CachedBucket, startMs, endMs uint64, stepMs uint64) []*qbtypes.TimeRange {- // When step is 0 or window is too small to be cached, use simple algorithm- if stepMs == 0 || (startMs+stepMs) > endMs {+ // When step is 0, use simple algorithm+ if stepMs == 0 {return bc.findMissingRangesBasic(buckets, startMs, endMs)}@@ -304,7 +306,8 @@ func (bc *bucketCache) findMissingRangesWithStep(buckets []*qbtypes.CachedBucket// Update current position to the end of this bucket// But ensure it's aligned to step boundarybucketEnd := min(bucket.EndMs, endMs)- if bucketEnd%stepMs != 0 && bucketEnd < endMs {+ // The step the window ends inside is missing however far the bucket runs.+ if bucketEnd%stepMs != 0 {// Round down to step boundarybucketEnd = bucketEnd - (bucketEnd % stepMs)}@@ -791,12 +794,26 @@ func max(a, b uint64) uint64 {return b}-// filterResultToTimeRange filters the result to only include values within the requested time range.-func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, startMs, endMs uint64) *qbtypes.Result {+// filterResultToTimeRange narrows the cached result to the requested window, both+// the values in it and the heatmap axis under them.+func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, q qbtypes.Query, startMs, endMs, stepMs uint64) *qbtypes.Result {if result == nil || result.Value == nil {return result}+ _, isPromQL := q.(*promqlQuery)+ maxTimestampMs := endMs+ // A promql value at T is the query evaluated at T, so T == endMs is inside the+ // requested range. For every other query type the value at T aggregates+ // [T, T+stepMs), which the requested range contains only when T <= endMs-stepMs.+ if !isPromQL {+ if stepMs > 0 {+ maxTimestampMs = endMs - stepMs+ } else {+ maxTimestampMs = endMs - 1+ }+ }+switch result.Type {case qbtypes.RequestTypeTimeSeries, qbtypes.RequestTypeHeatmap:if tsData, ok := result.Value.(*qbtypes.TimeSeriesData); ok {@@ -821,7 +838,7 @@ func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, startMs,// Filter values to only include those within the requested time rangefor _, value := range series.Values {timestampMs := uint64(value.Timestamp)- if timestampMs >= startMs && timestampMs < endMs {+ if timestampMs >= startMs && timestampMs <= maxTimestampMs {filteredSeries.Values = append(filteredSeries.Values, value)}}@@ -836,6 +853,8 @@ func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, startMs,}}+ bc.trimHeatmapAxisToTheWindow(q, filteredData)+// Create a new result with the filtered datareturn &qbtypes.Result{Type: result.Type,@@ -849,3 +868,25 @@ func (bc *bucketCache) filterResultToTimeRange(result *qbtypes.Result, startMs,// For non-time series data, return as isreturn result}++// A cache entry can hold a longer time range than the one now requested, and its+// bucket axis spans every sample value seen across that longer range. Once the+// points outside the requested range are dropped, the axis may still list+// buckets that only those dropped points counted into. Left there, they render+// as empty rows that the same request never has when it bypasses the cache.+//+// PromQL and ClickHouse queries name their own buckets, and a histogram's are+// its `le` labels, so only a gauge, sum or summary axis is cut.+func (bc *bucketCache) trimHeatmapAxisToTheWindow(q qbtypes.Query, tsData *qbtypes.TimeSeriesData) {+ metricQuery, isMetricQuery := q.(*builderQuery[qbtypes.MetricAggregation])+ if !isMetricQuery {+ return+ }+ if metricQuery.spec.Aggregations[0].Type == metrictypes.HistogramType {+ return+ }++ for _, aggBucket := range tsData.Aggregations {+ aggBucket.TrimAxisToCountedBuckets()+ }+}diff --git a/pkg/querier/bucket_cache_bench_test.go b/pkg/querier/bucket_cache_bench_test.goindex 78aebfd5a2..8e17b14cf4 100644--- a/pkg/querier/bucket_cache_bench_test.go+++ b/pkg/querier/bucket_cache_bench_test.go@@ -327,7 +327,7 @@ func BenchmarkBucketCache_FilterResultToTimeRange(b *testing.B) {b.ReportAllocs()for i := 0; i < b.N; i++ {- filtered := bc.filterResultToTimeRange(result, startMs, endMs)+ filtered := bc.filterResultToTimeRange(result, &promqlQuery{}, startMs, endMs, 0)_ = filtered}})diff --git a/pkg/querier/bucket_cache_test.go b/pkg/querier/bucket_cache_test.goindex c4cd6db9a8..4d55960ded 100644--- a/pkg/querier/bucket_cache_test.go+++ b/pkg/querier/bucket_cache_test.go@@ -3,6 +3,7 @@ package querierimport ("context""fmt"+ "log/slog""testing""time"@@ -1069,8 +1070,11 @@ func TestBucketCache_FilteredCachedResults(t *testing.T) {// Get cached data - should be filtered to requested rangecached, missing := bc.GetMissRanges(ctx, orgID, query2, qbtypes.Step{Duration: 1000 * time.Millisecond})- // Should have no missing ranges- assert.Len(t, missing, 0)+ // The value at 3000 stands for the whole step to 4000, which reaches past the+ // window, so it is left to be recomputed as a partial rather than served.+ require.Len(t, missing, 1)+ assert.Equal(t, uint64(3000), missing[0].From)+ assert.Equal(t, uint64(3500), missing[0].To)assert.NotNil(t, cached)// Verify the cached result only contains values within the requested range@@ -1080,18 +1084,65 @@ func TestBucketCache_FilteredCachedResults(t *testing.T) {require.Len(t, tsData.Aggregations[0].Series, 1)series := tsData.Aggregations[0].Series[0]- assert.Len(t, series.Values, 2) // Only values at 2000 and 3000 should be included+ require.Len(t, series.Values, 1)// Verify the exact valuesassert.Equal(t, int64(2000), series.Values[0].Timestamp)assert.Equal(t, float64(20), series.Values[0].Value)- assert.Equal(t, int64(3000), series.Values[1].Timestamp)- assert.Equal(t, float64(30), series.Values[1].Value)// Value at 1000 should not be included (before requested range)// Value at 4000 should not be included (after requested range)}+// A promql value is the query evaluated at a single moment rather than over a+// span, so the one at the window's end belongs to it and has to survive caching.+func TestBucketCache_PromQLKeepsTheValueAtTheWindowEnd(t *testing.T) {+ bc := createTestBucketCache(t)+ ctx := context.Background()+ orgID := valuer.UUID{}+ step := qbtypes.Step{Duration: time.Minute}++ query := &promqlQuery{+ logger: slog.Default(),+ query: qbtypes.PromQuery{Query: "up", Step: step},+ tr: qbtypes.TimeRange{From: 600_000, To: 780_000},+ requestType: qbtypes.RequestTypeTimeSeries,+ }++ bc.Put(ctx, orgID, query, step, &qbtypes.Result{+ Type: qbtypes.RequestTypeTimeSeries,+ Value: &qbtypes.TimeSeriesData{+ QueryName: "A",+ Aggregations: []*qbtypes.AggregationBucket{{+ Series: []*qbtypes.TimeSeries{{+ Values: []*qbtypes.TimeSeriesValue{+ {Timestamp: 600_000, Value: 1},+ {Timestamp: 660_000, Value: 2},+ {Timestamp: 720_000, Value: 3},+ {Timestamp: 780_000, Value: 4},+ },+ }},+ }},+ },+ })+ time.Sleep(10 * time.Millisecond)++ cached, missing := bc.GetMissRanges(ctx, orgID, query, step)+ assert.Empty(t, missing)+ require.NotNil(t, cached)++ tsData, ok := cached.Value.(*qbtypes.TimeSeriesData)+ require.True(t, ok)+ require.Len(t, tsData.Aggregations, 1)+ require.Len(t, tsData.Aggregations[0].Series, 1)++ timestamps := []int64{}+ for _, value := range tsData.Aggregations[0].Series[0].Values {+ timestamps = append(timestamps, value.Timestamp)+ }+ assert.Equal(t, []int64{600_000, 660_000, 720_000, 780_000}, timestamps)+}+func TestBucketCache_FindMissingRangesWithStep(t *testing.T) {bc := createTestBucketCache(t)@@ -1150,7 +1201,20 @@ func TestBucketCache_FindMissingRangesWithStep(t *testing.T) {expectedMiss: []*qbtypes.TimeRange{{From: 1000, To: 1500},},- description: "Window smaller than step should use basic algorithm",+ description: "Window smaller than step is fetched whole",+ },+ {+ name: "small_window_less_than_step_inside_a_bucket",+ buckets: []*qbtypes.CachedBucket{+ {StartMs: 0, EndMs: 3000},+ },+ startMs: 1000,+ endMs: 1500,+ stepMs: 1000,+ expectedMiss: []*qbtypes.TimeRange{+ {From: 1000, To: 1500},+ },+ description: "A cached point spans its whole step, so it cannot answer a window shorter than the step",},{name: "zero_step_uses_basic_algorithm",diff --git a/pkg/querier/promql_heatmap.go b/pkg/querier/promql_heatmap.goindex 1a66660b18..466ec127e8 100644--- a/pkg/querier/promql_heatmap.go+++ b/pkg/querier/promql_heatmap.go@@ -22,7 +22,7 @@ import (const promHistogramBucketLabel = "le"// cumulativeColumn maps a bucket's upper bound to the cumulative count at it.-// Differencing turns it into the per-band counts a heatmapColumn holds.+// Differencing turns it into the per-bucket counts a heatmapColumn holds.type cumulativeColumn map[float64]float64// promHeatmapGroup assembles one group across the several matrix series its `le`@@ -34,8 +34,8 @@ type promHeatmapGroup struct {}// foldMatrixAsHeatmap folds a matrix of one cumulative series per (group, `le`)-// into one series per group whose points hold a count per band.-func foldMatrixAsHeatmap(matrix promql.Matrix, queryWindow *qbv5.TimeRange, stepMs uint64, queryName string) (*qbv5.TimeSeriesData, error) {+// into one series per group whose points hold a count per bucket.+func foldMatrixAsHeatmap(matrix promql.Matrix, queryName string) (*qbv5.TimeSeriesData, error) {groups, groupOrder := collectCumulativeGroups(matrix)// An empty matrix is only ever the window having no data, but series that@@ -53,11 +53,12 @@ func foldMatrixAsHeatmap(matrix promql.Matrix, queryWindow *qbv5.TimeRange, step}}- return accumulator.foldSeries(queryWindow, stepMs, queryName)+ // a promql data point can never be partial, hence nil and 0 are sent here+ return accumulator.foldSeries(nil, 0, queryName)}// collectCumulativeGroups reads the matrix into one group per label set. A series-// without `le` has no band to sit in, so an expression that dropped the label+// without `le` has no bucket to sit in, so an expression that dropped the label// draws nothing.func collectCumulativeGroups(matrix promql.Matrix) (groups map[string]*promHeatmapGroup, groupOrder []string) {groups = map[string]*promHeatmapGroup{}diff --git a/pkg/querier/promql_heatmap_test.go b/pkg/querier/promql_heatmap_test.goindex cdd73a0181..bd2f7db0e0 100644--- a/pkg/querier/promql_heatmap_test.go+++ b/pkg/querier/promql_heatmap_test.go@@ -16,7 +16,7 @@ import (// The cache key is the fingerprint alone, so two request types over one// expression must not produce the same one — a time series payload served to a-// heatmap request has no axis and reads back as a single collapsed band.+// heatmap request has no axis and reads back as a single collapsed bucket.func TestFingerprintSeparatesHeatmapFromTimeSeries(t *testing.T) {fingerprintFor := func(requestType qbv5.RequestType) string {q := &promqlQuery{@@ -50,7 +50,7 @@ func TestFoldMatrixAsHeatmapClampsADecreasingCumulativeCount(t *testing.T) {},}- data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")+ data, err := foldMatrixAsHeatmap(matrix, "A")require.NoError(t, err)require.Len(t, data.Aggregations, 1)@@ -76,7 +76,7 @@ func TestFoldMatrixAsHeatmapWidensTheBandOverAMissingUpperBound(t *testing.T) {},}- data, err := foldMatrixAsHeatmap(matrix, &qbv5.TimeRange{From: 1710000000000, To: 1710000060000}, uint64(time.Minute.Milliseconds()), "A")+ data, err := foldMatrixAsHeatmap(matrix, "A")require.NoError(t, err)require.Len(t, data.Aggregations, 1)diff --git a/pkg/querier/promql_query.go b/pkg/querier/promql_query.goindex f5c02b6649..0d2501c8f1 100644--- a/pkg/querier/promql_query.go+++ b/pkg/querier/promql_query.go@@ -115,6 +115,13 @@ func newPromqlQuery(requestType qbv5.RequestType,variables map[string]qbv5.VariableItem,) *promqlQuery {+ // promql evaluates at tr.From and every step after it, so two windows a few+ // seconds apart ask for different instants and could never share a cache+ // entry. Rounding both ends down to the step makes them the same query.+ if stepMs := uint64(query.Step.Milliseconds()); stepMs > 0 && !query.SkipStepAlignment {+ tr.From -= tr.From % stepMs+ tr.To -= tr.To % stepMs+ }return &promqlQuery{logger: logger,promEngine: promEngine,@@ -133,6 +140,12 @@ func (q *promqlQuery) Fingerprint() string {return ""}+ // an unaligned window evaluates at instants no other window asks for, so+ // there is nothing to share+ if q.query.SkipStepAlignment {+ return ""+ }+query, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To)if err != nil {q.logger.ErrorContext(context.TODO(), "failed render template variables", slog.String("query", q.query.Query))@@ -352,7 +365,7 @@ func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began ti}func (q *promqlQuery) toResultForHeatmap(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) (*qbv5.Result, error) {- tsData, err := foldMatrixAsHeatmap(matrix, &q.tr, uint64(q.query.Step.Milliseconds()), q.query.Name)+ tsData, err := foldMatrixAsHeatmap(matrix, q.query.Name)if err != nil {return nil, err}diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/prom_query.go b/pkg/types/querybuildertypes/querybuildertypesv5/prom_query.goindex 3644427fe0..0ece5bf664 100644--- a/pkg/types/querybuildertypes/querybuildertypesv5/prom_query.go+++ b/pkg/types/querybuildertypes/querybuildertypesv5/prom_query.go@@ -11,6 +11,9 @@ type PromQuery struct {Step Step `json:"step"`// stats if true, the query will return statsStats bool `json:"stats"`+ // skipStepAlignment if true, start and end are used as given instead of+ // rounded down to the step, and the result is never cached+ SkipStepAlignment bool `json:"skipStepAlignment,omitempty"`Legend string `json:"legend"`}diff --git a/pkg/types/querybuildertypes/querybuildertypesv5/resp.go b/pkg/types/querybuildertypes/querybuildertypesv5/resp.goindex d864bb4633..aad62a4183 100644--- a/pkg/types/querybuildertypes/querybuildertypesv5/resp.go+++ b/pkg/types/querybuildertypes/querybuildertypesv5/resp.go@@ -189,6 +189,51 @@ func (a *AggregationBucket) ReindexValuesToNewUpperBounds(onto []float64) {a.Meta.Buckets = onto}+// TrimAxisToCountedBuckets drops the buckets at either end of Meta.Buckets that hold+// no counts, since an axis runs from the lowest value in the window to the highest.+// Call it only when the buckets were chosen from the values. A histogram's `le`+// buckets are fixed, so an empty one of those still has to stay on the axis.+func (a *AggregationBucket) TrimAxisToCountedBuckets() {+ if a == nil || len(a.Meta.Buckets) == 0 {+ return+ }++ lowestCounted, highestCounted := len(a.Meta.Buckets), -1+ for _, series := range a.Series {+ for _, point := range series.Values {+ for slot := 0; slot < len(a.Meta.Buckets) && slot < len(point.Values); slot++ {+ if point.Values[slot] != 0 {+ lowestCounted = min(lowestCounted, slot)+ highestCounted = max(highestCounted, slot)+ }+ }+ }+ }+ if highestCounted < 0 {+ return+ }++ if lowestCounted == 0 && highestCounted == len(a.Meta.Buckets)-1 {+ return+ }++ trimmed := a.Meta.Buckets[lowestCounted : highestCounted+1]+ for _, series := range a.Series {+ for _, point := range series.Values {+ if len(point.Values) == 0 {+ continue+ }+ counts := make([]float64, len(trimmed)+1)+ for slot, count := range point.Values {+ counts[min(max(slot-lowestCounted, 0), len(trimmed))] += count+ }+ point.Values = counts+ }+ }++ a.Meta.Buckets = trimmed+}+type AggregationMeta struct {Unit string `json:"unit,omitempty"`// Buckets holds ascending upper bounds shared by every series in thediff --git a/tests/integration/tests/promqlconformance/01_upstream_corpus.py b/tests/integration/tests/promqlconformance/01_upstream_corpus.pyindex cd27271f55..bcd109fb08 100644--- a/tests/integration/tests/promqlconformance/01_upstream_corpus.py+++ b/tests/integration/tests/promqlconformance/01_upstream_corpus.py@@ -51,7 +51,9 @@ def test_upstream_promqltest_corpus(req_start_ms = start_ms - step_s * 1000query = {"type": "promql",- "spec": {"name": "A", "query": case["expr"], "step": step_s},+ # the corpus expects points at exactly start plus whole steps, as+ # Prometheus itself evaluates them+ "spec": {"name": "A", "query": case["expr"], "step": step_s, "skipStepAlignment": True},}case_id = f"{case['source']}[{case['variant']}]"diff --git a/tests/integration/tests/promqlconformance/02_fingerprint_probe.py b/tests/integration/tests/promqlconformance/02_fingerprint_probe.pyindex d0f5eed5f3..18c9880d7d 100644--- a/tests/integration/tests/promqlconformance/02_fingerprint_probe.py+++ b/tests/integration/tests/promqlconformance/02_fingerprint_probe.py@@ -73,7 +73,7 @@ def test_identical_labelsets_merge_and_grouping(token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)def run(promql: str, start_ms: int, end_ms: int) -> list[tuple[dict, list]]:- q = {"type": "promql", "spec": {"name": "A", "query": promql, "step": 60}}+ q = {"type": "promql", "spec": {"name": "A", "query": promql, "step": 60, "skipStepAlignment": True}}resp = make_query_request(signoz, token, start_ms, end_ms, [q])assert resp.status_code == HTTPStatus.OK, f"{promql!r}: {resp.text[:300]}"out = []diff --git a/tests/integration/tests/promqlconformance/03_nonfinite_cache.py b/tests/integration/tests/promqlconformance/03_nonfinite_cache.pyindex 9d813e6c21..0e6a5c1686 100644--- a/tests/integration/tests/promqlconformance/03_nonfinite_cache.py+++ b/tests/integration/tests/promqlconformance/03_nonfinite_cache.py@@ -58,8 +58,7 @@ def test_promql_ratio_with_zero_denominator_is_dropped_and_cached(assert set(first["active_job"].values()) == {25.0}, sorted(set(first["active_job"].values()))assert len(first["active_job"]) == expected_points, f"expected {expected_points} points, got {len(first['active_job'])}"- # The cached read excludes end_ms, the one legitimate difference.+ # Both reads must agree exactly, including the point promql reports at end_ms.assert set(second) == set(first), sorted(second)for job_name, points in first.items():- expected = {ts: value for ts, value in points.items() if ts < end_ms}- assert second[job_name] == expected, f"{job_name}: got {len(second[job_name])} of {len(expected)} points"+ assert second[job_name] == points, f"{job_name}: got {len(second[job_name])} of {len(points)} points"diff --git a/tests/integration/tests/promqlconformance/04_no_step_subquery.py b/tests/integration/tests/promqlconformance/04_no_step_subquery.pyindex 66052f47de..d8f3953eac 100644--- a/tests/integration/tests/promqlconformance/04_no_step_subquery.py+++ b/tests/integration/tests/promqlconformance/04_no_step_subquery.py@@ -40,7 +40,7 @@ def test_promql_subquery_without_step_evaluates(token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)- query = {"type": "promql", "spec": {"name": "A", "query": f"max_over_time({metric}[5m:])"}}+ query = {"type": "promql", "spec": {"name": "A", "query": f"max_over_time({metric}[5m:])", "skipStepAlignment": True}}response = make_query_request(signoz, token, start_ms, end_ms, [query])assert response.status_code == HTTPStatus.OK, response.text[:300]series = get_all_series(response.json(), "A")@@ -54,6 +54,6 @@ def test_promql_subquery_without_step_evaluates(token,start_ms,end_ms,- [{"type": "promql", "spec": {"name": "A", "query": metric}}],+ [{"type": "promql", "spec": {"name": "A", "query": metric, "skipStepAlignment": True}}],)assert response.status_code == HTTPStatus.OK, response.text[:300]diff --git a/tests/integration/tests/promqlconformance/05_or_subquery_duplicate_labelset.py b/tests/integration/tests/promqlconformance/05_or_subquery_duplicate_labelset.pyindex 5fcd73899d..cc58dd0f23 100644--- a/tests/integration/tests/promqlconformance/05_or_subquery_duplicate_labelset.py+++ b/tests/integration/tests/promqlconformance/05_or_subquery_duplicate_labelset.py@@ -72,7 +72,7 @@ def test_or_arms_merge_under_subquery_name_drop(assert set(instant_values.values()) == {1.0}, instant_values# query_range over the same grid must agree point for point.- spec = {"name": "A", "query": query, "step": 300}+ spec = {"name": "A", "query": query, "step": 300, "skipStepAlignment": True}response = make_query_request(signoz, token, start_ms, end_ms, [{"type": "promql", "spec": spec}])assert response.status_code == HTTPStatus.OK, response.text[:300]series = get_all_series(response.json(), "A")diff --git a/tests/integration/tests/queriermetrics/17_cache.py b/tests/integration/tests/queriermetrics/17_cache.pynew file mode 100644index 0000000000..c82446d628--- /dev/null+++ b/tests/integration/tests/queriermetrics/17_cache.py@@ -0,0 +1,444 @@+from collections.abc import Callable+from datetime import UTC, datetime, timedelta+from http import HTTPStatus+from uuid import uuid4++from fixtures import types+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD+from fixtures.metrics import Metrics+from fixtures.querier import (+ assert_results_equal,+ build_builder_query,+ get_series_values,+ make_query_request,+)++MINUTE_MS = 60_000+++def test_builder_shortening_the_time_range_at_the_end(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ # the cache outlives the run, so a fixed name would serve the previous run's+ # points back to this one+ metric_name = f"cache_end_shortened_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache. Flooring to a multiple of the 5m step makes the base query span+ # two whole steps, so both its points are complete+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 300 * 300, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms_base_query = start_time_ms + 10 * MINUTE_MS+ end_time_ms_shortened_query = start_time_ms + 7 * MINUTE_MS++ query = [build_builder_query("A", metric_name, "max", "max", step_interval=300)]++ # the 5m step splits the ten minutes into two points, each the max over its+ # own step: minutes 0-4 and minutes 5-9. The second changes partway through,+ # 256 until minute 7 and then 4096, so ending the range at minute 7 has to+ # reach a different value than ending it at minute 10+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=(16, 16, 16, 16, 16, 256, 256, 4096, 4096, 4096)[minute],+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(10)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ base_query = make_query_request(signoz, token, start_time_ms, end_time_ms_base_query, query, no_cache=False)+ assert base_query.status_code == HTTPStatus.OK, base_query.text+ points = sorted(get_series_values(base_query.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["value"], point.get("partial", False)) for point in points]+ assert returned_points == [(16, False), (4096, False)]++ from_cache = make_query_request(signoz, token, start_time_ms, end_time_ms_shortened_query, query, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, start_time_ms, end_time_ms_shortened_query, query, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ assert_results_equal(from_cache.json(), uncached.json(), "A", "shortened end")++ # the shortened end reaches only minutes 5-6 of the second point, so it comes+ # back as 256 and partial, where the cached one spans all five minutes+ for label, response in (("from cache", from_cache), ("uncached", uncached)):+ points = sorted(get_series_values(response.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["value"], point.get("partial", False)) for point in points]+ assert returned_points == [(16, False), (256, True)], label+++def test_builder_shortening_the_time_range_at_the_start(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"cache_start_shortened_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache. Flooring to a multiple of the 5m step makes the base query span+ # two whole steps, so both its points are complete+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 300 * 300, tz=UTC)+ start_time_ms_base_query = int(start_time.timestamp() * 1000)+ start_time_ms_shortened_query = start_time_ms_base_query + 3 * MINUTE_MS+ end_time_ms = start_time_ms_base_query + 10 * MINUTE_MS++ query = [build_builder_query("A", metric_name, "max", "max", step_interval=300)]++ # the 5m step splits the ten minutes into two points, each the max over its+ # own step: minutes 0-4 and minutes 5-9. Only minute 0 holds 65536, so a first+ # point reaching it says the whole step was read even though the shortened+ # range opens at minute 3+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=(65536, 16, 16, 16, 16, 4096, 4096, 4096, 4096, 4096)[minute],+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(10)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ base_query = make_query_request(signoz, token, start_time_ms_base_query, end_time_ms, query, no_cache=False)+ assert base_query.status_code == HTTPStatus.OK, base_query.text+ points = sorted(get_series_values(base_query.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["value"], point.get("partial", False)) for point in points]+ assert returned_points == [(65536, False), (4096, False)]++ from_cache = make_query_request(signoz, token, start_time_ms_shortened_query, end_time_ms, query, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, start_time_ms_shortened_query, end_time_ms, query, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ assert_results_equal(from_cache.json(), uncached.json(), "A", "shortened start")++ # starting inside the first point's step flags that point partial without+ # clipping its value, which still covers the whole step and so reaches the+ # 65536 at minute 0+ for label, response in (("from cache", from_cache), ("uncached", uncached)):+ points = sorted(get_series_values(response.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["value"], point.get("partial", False)) for point in points]+ assert returned_points == [(65536, True), (4096, False)], label+++def test_builder_time_range_shorter_than_the_step(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"cache_range_below_step_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache. Flooring to a multiple of the 10m step makes the base query span+ # two whole steps, so both its points are complete+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 600 * 600, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms_base_query = start_time_ms + 20 * MINUTE_MS+ start_time_ms_short_query = start_time_ms + 10 * MINUTE_MS+ end_time_ms_short_query = start_time_ms + 14 * MINUTE_MS++ query = [build_builder_query("A", metric_name, "max", "max", step_interval=600)]++ # the 10m step splits the twenty minutes into two points, each the max over+ # its own step: minutes 0-9 and minutes 10-19. The second changes partway+ # through, 256 until minute 14 and then 4096, so a four minute range opening+ # at minute 10 has to reach a different value than the whole step does+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=(16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 256, 256, 256, 256, 4096, 4096, 4096, 4096, 4096, 4096)[minute],+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(20)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ base_query = make_query_request(signoz, token, start_time_ms, end_time_ms_base_query, query, no_cache=False)+ assert base_query.status_code == HTTPStatus.OK, base_query.text+ points = sorted(get_series_values(base_query.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["value"], point.get("partial", False)) for point in points]+ assert returned_points == [(16, False), (4096, False)]++ from_cache = make_query_request(signoz, token, start_time_ms_short_query, end_time_ms_short_query, query, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, start_time_ms_short_query, end_time_ms_short_query, query, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ assert_results_equal(from_cache.json(), uncached.json(), "A", "range shorter than the step")++ # a range shorter than its step holds a single point, and this one reaches+ # only minutes 10-13 of the second step, so it comes back as 256 and partial,+ # where the cached point spans all ten minutes+ for label, response in (("from cache", from_cache), ("uncached", uncached)):+ points = sorted(get_series_values(response.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["value"], point.get("partial", False)) for point in points]+ assert returned_points == [(256, True)], label+++def test_promql_running_the_same_query_twice(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"cache_repeat_total_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 60 * 60, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms = start_time_ms + 2 * MINUTE_MS++ query = [{"type": "promql", "spec": {"name": "A", "query": f"sum(increase({metric_name}[2m]))", "step": 60}}]++ # the counter opens a minute before the query so its first point has something+ # to increase over, and starts far above its own rise across the range, below+ # which increase clips its back-extrapolation at the counter's zero point. It+ # rises by a different amount each minute, so every point is its own number+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=(1000, 1010, 1030, 1060, 1100)[minute + 1],+ temporality="Cumulative",+ type_="Sum",+ is_monotonic=True,+ )+ for minute in range(-1, 4)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ first = make_query_request(signoz, token, start_time_ms, end_time_ms, query, no_cache=False)+ assert first.status_code == HTTPStatus.OK, first.text++ second = make_query_request(signoz, token, start_time_ms, end_time_ms, query, no_cache=False)+ assert second.status_code == HTTPStatus.OK, second.text++ assert_results_equal(first.json(), second.json(), "A", "the same query twice")++ # promql reports a point at the instant the range closes, and the second run,+ # answered out of what the first one cached, has to keep it+ for run, response in (("first", first), ("second", second)):+ points = sorted(get_series_values(response.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["timestamp"], point["value"]) for point in points]+ ## at each timestamp t, promql looks at points in (t-2minutes, t].+ assert returned_points == [+ (start_time_ms, 20), # t = 0, points taken 1000, 1010. hence diff over 1m is 10, extrapolated to 20.+ (start_time_ms + MINUTE_MS, 40), # t = 1m, points taken 1010, 1030. hence diff over 1m is 20, extrapolated to 40.+ (end_time_ms, 60), # t = 2m, points taken 1030, 1060. hence diff over 1m is 30, extrapolated to 60.+ ], f"{run} run"+++def test_promql_shifting_the_time_range(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"cache_shift_gauge_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache. Flooring to a whole minute is what makes the first query aligned+ # to its 1m step, and the unaligned one half a step off it+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 60 * 60, tz=UTC)+ aligned_start_time_ms = int(start_time.timestamp() * 1000)+ aligned_end_time_ms = aligned_start_time_ms + 3 * MINUTE_MS+ unaligned_start_time_ms = aligned_start_time_ms + MINUTE_MS // 2+ unaligned_end_time_ms = aligned_end_time_ms + MINUTE_MS // 2++ query = [{"type": "promql", "spec": {"name": "A", "query": f"max_over_time({metric_name}[2m])", "step": 60}}]++ # a sample every 30s, rising by 100 each time. The two queries report 30s+ # apart, so they land on different samples and share no value between them+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(seconds=30 * half_minute),+ value=100 * (half_minute + 4),+ type_="Gauge",+ is_monotonic=False,+ )+ for half_minute in range(-3, 8)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ aligned_and_cached = make_query_request(signoz, token, aligned_start_time_ms, aligned_end_time_ms, query, no_cache=False)+ assert aligned_and_cached.status_code == HTTPStatus.OK, aligned_and_cached.text++ ## at each timestamp t, promql takes the highest sample in (t-2minutes, t],+ ## which is the one at t itself since the gauge only rises.+ aligned_points = [+ (aligned_start_time_ms, 400), # t = 0+ (aligned_start_time_ms + MINUTE_MS, 600), # t = 1m+ (aligned_start_time_ms + 2 * MINUTE_MS, 800), # t = 2m+ (aligned_end_time_ms, 1000), # t = 3m+ ]+ points = sorted(get_series_values(aligned_and_cached.json(), "A"), key=lambda point: point["timestamp"])+ assert [(point["timestamp"], point["value"]) for point in points] == aligned_points++ # an unaligned range is rounded down to the step, so this asks for the same+ # instants as the aligned query and gets the same answer+ unaligned_and_cached = make_query_request(signoz, token, unaligned_start_time_ms, unaligned_end_time_ms, query, no_cache=False)+ assert unaligned_and_cached.status_code == HTTPStatus.OK, unaligned_and_cached.text+ points = sorted(get_series_values(unaligned_and_cached.json(), "A"), key=lambda point: point["timestamp"])+ assert [(point["timestamp"], point["value"]) for point in points] == aligned_points, "unaligned query, rounded to the step"+ assert_results_equal(unaligned_and_cached.json(), aligned_and_cached.json(), "A", "unaligned query, rounded to the step")++ # skipping the alignment keeps the range as asked, so promql reports 30s off+ # the cached points, on samples the aligned query never saw. Such a query is+ # never cached, so repeating it never gets the aligned answer either+ unaligned_query = [{"type": "promql", "spec": {"name": "A", "query": f"max_over_time({metric_name}[2m])", "step": 60, "skipStepAlignment": True}}]+ unaligned_points = [+ (unaligned_start_time_ms, 500), # t = 30s+ (unaligned_start_time_ms + MINUTE_MS, 700), # t = 1m30s+ (unaligned_start_time_ms + 2 * MINUTE_MS, 900), # t = 2m30s+ (unaligned_end_time_ms, 1100), # t = 3m30s+ ]+ for run in ("first", "second"):+ unaligned_as_asked = make_query_request(signoz, token, unaligned_start_time_ms, unaligned_end_time_ms, unaligned_query, no_cache=False)+ assert unaligned_as_asked.status_code == HTTPStatus.OK, unaligned_as_asked.text+ points = sorted(get_series_values(unaligned_as_asked.json(), "A"), key=lambda point: point["timestamp"])+ assert [(point["timestamp"], point["value"]) for point in points] == unaligned_points, f"unaligned query as asked, {run} run"+++def test_promql_time_range_shorter_than_the_step(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"cache_promql_range_below_step_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache. Flooring to a multiple of the 10m step aligns the base query to it+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 600 * 600, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms_base_query = start_time_ms + 10 * MINUTE_MS+ start_time_ms_short_query = start_time_ms + 12 * MINUTE_MS+ end_time_ms_short_query = start_time_ms + 14 * MINUTE_MS++ query = [{"type": "promql", "spec": {"name": "A", "query": f"max_over_time({metric_name}[2m])", "step": 600}}]++ # a sample every minute, rising by 100 each time, so every instant promql+ # reports at has its own value+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=100 * (minute + 1),+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(15)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ base_query = make_query_request(signoz, token, start_time_ms, end_time_ms_base_query, query, no_cache=False)+ assert base_query.status_code == HTTPStatus.OK, base_query.text+ points = sorted(get_series_values(base_query.json(), "A"), key=lambda point: point["timestamp"])+ assert [(point["timestamp"], point["value"]) for point in points] == [(start_time_ms, 100), (end_time_ms_base_query, 1100)]++ # a range shorter than the step rounds down to the single instant at minute+ # 10, the instant the base query closed on+ from_cache = make_query_request(signoz, token, start_time_ms_short_query, end_time_ms_short_query, query, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, start_time_ms_short_query, end_time_ms_short_query, query, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ assert_results_equal(from_cache.json(), uncached.json(), "A", "range shorter than the step")++ for label, response in (("from cache", from_cache), ("uncached", uncached)):+ points = sorted(get_series_values(response.json(), "A"), key=lambda point: point["timestamp"])+ assert [(point["timestamp"], point["value"]) for point in points] == [(end_time_ms_base_query, 1100)], label+++def test_builder_refreshing_a_sliding_time_range(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"cache_sliding_{uuid4().hex[:8]}"++ # 90 minutes back so even the twentieth refresh closes clear of the flux+ # interval, which holds recent data out of the cache+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=90)).timestamp()) // 60 * 60, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)++ query = [build_builder_query("A", metric_name, "max", "max")]++ # the 1m step gives one point per seeded minute, and a value no other minute+ # carries, so a point stitched in from the wrong range reads as the wrong minute+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=1000 + minute,+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(80)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ # a dashboard left open on a one hour range, re-running a minute later each time+ for refresh in range(20):+ refresh_start_ms = start_time_ms + refresh * MINUTE_MS+ from_cache = make_query_request(signoz, token, refresh_start_ms, refresh_start_ms + 60 * MINUTE_MS, query, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ # each refresh is stitched out of overlapping cached ranges, so this catches+ # a point served twice, dropped, or carried over from an earlier refresh+ points = sorted(get_series_values(from_cache.json(), "A"), key=lambda point: point["timestamp"])+ returned_points = [(point["timestamp"], point["value"], point.get("partial", False)) for point in points]+ expected_points = [(start_time_ms + minute * MINUTE_MS, 1000 + minute, False) for minute in range(refresh, refresh + 60)]+ assert returned_points == expected_points, f"refresh {refresh} did not return the minutes it covers"++ last_refresh_start_ms = start_time_ms + 19 * MINUTE_MS+ uncached = make_query_request(signoz, token, last_refresh_start_ms, last_refresh_start_ms + 60 * MINUTE_MS, query, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ assert_results_equal(from_cache.json(), uncached.json(), "A", "the twentieth refresh")diff --git a/tests/integration/tests/queriermetrics/18_heatmap_cache.py b/tests/integration/tests/queriermetrics/18_heatmap_cache.pynew file mode 100644index 0000000000..471966e510--- /dev/null+++ b/tests/integration/tests/queriermetrics/18_heatmap_cache.py@@ -0,0 +1,546 @@+from collections.abc import Callable+from datetime import UTC, datetime, timedelta+from http import HTTPStatus+from uuid import uuid4++import pytest++from fixtures import types+from fixtures.auth import USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD+from fixtures.metrics import Metrics+from fixtures.querier import (+ RequestType,+ assert_identical_query_response,+ build_builder_query,+ build_linear_bucket_options,+ get_heatmap_buckets,+ get_heatmap_columns,+ make_query_request,+)++MINUTE_MS = 60_000+++ "first_minute, expected_buckets",+ [+ pytest.param(0, [100, 200], id="the_lower_half"),+ pytest.param(5, [800, 900], id="the_upper_half"),+ ],+)+def test_builder_narrowing_to_half_the_range(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+ first_minute: int,+ expected_buckets: list[int],+) -> None:+ metric_name = f"heatmap_cache_narrowed_{uuid4().hex[:8]}"++ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 60 * 60, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms = start_time_ms + 10 * MINUTE_MS++ # 100 wide buckets, and the first five minutes sit seven buckets under the+ # last five, so the axis over all ten covers a stretch neither half reaches+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=150 if minute < 5 else 850,+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(10)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ query = [build_builder_query("A", metric_name, "max", "max", bucket_options=build_linear_bucket_options(1000, 10))]++ # the whole range first, which is what puts its axis in the cache+ whole_range = make_query_request(signoz, token, start_time_ms, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert whole_range.status_code == HTTPStatus.OK, whole_range.text+ assert get_heatmap_buckets(whole_range.json(), "A") == pytest.approx([100, 200, 300, 400, 500, 600, 700, 800, 900])+ assert [column["values"] for column in get_heatmap_columns(whole_range.json(), "A")] == [+ [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],+ [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],+ [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],+ [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],+ [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],+ [0, 0, 0, 0, 0, 0, 0, 0, 1, 0],+ [0, 0, 0, 0, 0, 0, 0, 0, 1, 0],+ [0, 0, 0, 0, 0, 0, 0, 0, 1, 0],+ [0, 0, 0, 0, 0, 0, 0, 0, 1, 0],+ [0, 0, 0, 0, 0, 0, 0, 0, 1, 0],+ ]++ half_start_ms = start_time_ms + first_minute * MINUTE_MS+ half_end_ms = half_start_ms + 5 * MINUTE_MS+ from_cache = make_query_request(signoz, token, half_start_ms, half_end_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, half_start_ms, half_end_ms, query, request_type=RequestType.HEATMAP, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ for source, response in (("uncached", uncached), ("from cache", from_cache)):+ assert get_heatmap_buckets(response.json(), "A") == pytest.approx(expected_buckets), source+ assert [column["values"] for column in get_heatmap_columns(response.json(), "A")] == [+ [0, 1, 0],+ [0, 1, 0],+ [0, 1, 0],+ [0, 1, 0],+ [0, 1, 0],+ ], source++ assert_identical_query_response(from_cache, uncached)+++def test_builder_narrowing_a_histogram(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"heatmap_cache_histogram_{uuid4().hex[:8]}_bucket"++ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 60 * 60, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms = start_time_ms + 10 * MINUTE_MS++ # the count each `le` reports every minute, cumulative across `le` as a+ # histogram is. For the first five minutes the ten arrivals are all at or+ # below 1, for the last five they are all between 4 and 8, and the buckets+ # holding none of them report a count of 0 rather than going unreported+ le_to_counts = {+ "1": [10, 10, 10, 10, 10, 0, 0, 0, 0, 0],+ "2": [10, 10, 10, 10, 10, 0, 0, 0, 0, 0],+ "4": [10, 10, 10, 10, 10, 0, 0, 0, 0, 0],+ "8": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10],+ "+Inf": [10, 10, 10, 10, 10, 10, 10, 10, 10, 10],+ }+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"le": le},+ timestamp=start_time + timedelta(minutes=minute),+ value=count,+ temporality="Delta",+ type_="Histogram",+ )+ for le, counts in le_to_counts.items()+ for minute, count in enumerate(counts)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ query = [build_builder_query("A", metric_name, "increase", "p50", temporality="delta", group_by=["le"])]++ # the whole range first, which is what puts its axis in the cache+ whole_range = make_query_request(signoz, token, start_time_ms, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert whole_range.status_code == HTTPStatus.OK, whole_range.text+ assert get_heatmap_buckets(whole_range.json(), "A") == [1, 2, 4, 8]+ assert [column["values"] for column in get_heatmap_columns(whole_range.json(), "A")] == [+ [10, 0, 0, 0, 0],+ [10, 0, 0, 0, 0],+ [10, 0, 0, 0, 0],+ [10, 0, 0, 0, 0],+ [10, 0, 0, 0, 0],+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ ]++ # even though this shortened time range has no data below 4, all histogram+ # buckets are still returned back+ half_start_ms = start_time_ms + 5 * MINUTE_MS+ from_cache = make_query_request(signoz, token, half_start_ms, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, half_start_ms, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ for source, response in (("uncached", uncached), ("from cache", from_cache)):+ assert get_heatmap_buckets(response.json(), "A") == [1, 2, 4, 8], source+ assert [column["values"] for column in get_heatmap_columns(response.json(), "A")] == [+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ [0, 0, 0, 10, 0],+ ], source++ assert_identical_query_response(from_cache, uncached)+++def test_builder_shortening_the_time_range_at_the_end(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"heatmap_cache_end_shortened_{uuid4().hex[:8]}"++ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 300 * 300, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms_base_query = start_time_ms + 10 * MINUTE_MS+ end_time_ms_shortened_query = start_time_ms + 7 * MINUTE_MS++ query = [build_builder_query("A", metric_name, "max", "max", step_interval=300, bucket_options=build_linear_bucket_options(1000, 10))]++ # the 5m step splits the ten minutes into two columns, each the max over its+ # own step: minutes 0-4 and minutes 5-9. The second changes partway through,+ # 250 until minute 7 and then 850, which fall six buckets apart, so ending+ # the range at minute 7 has to reach a different bucket than ending it at+ # minute 10 and an axis that stops well below it+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=(150, 150, 150, 150, 150, 250, 250, 850, 850, 850)[minute],+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(10)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ base_query = make_query_request(signoz, token, start_time_ms, end_time_ms_base_query, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert base_query.status_code == HTTPStatus.OK, base_query.text++ # 100 wide buckets, and the two maxes are 150 and 850, so the axis runs from+ # the bottom of (100, 200] to the top of (800, 900]+ assert get_heatmap_buckets(base_query.json(), "A") == pytest.approx([100, 200, 300, 400, 500, 600, 700, 800, 900])+ base_columns = get_heatmap_columns(base_query.json(), "A")+ assert [column["values"] for column in base_columns] == [+ [0, 1, 0, 0, 0, 0, 0, 0, 0, 0],+ [0, 0, 0, 0, 0, 0, 0, 0, 1, 0],+ ]+ assert [column.get("partial", False) for column in base_columns] == [False, False]++ from_cache = make_query_request(signoz, token, start_time_ms, end_time_ms_shortened_query, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, start_time_ms, end_time_ms_shortened_query, query, request_type=RequestType.HEATMAP, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ # the shortened end reaches only minutes 5-6 of the second column, whose max+ # is 250 and which comes back partial. Nothing in this window passes 300, so+ # the axis stops there rather than carrying the buckets above it+ for label, response in (("uncached", uncached), ("from cache", from_cache)):+ assert get_heatmap_buckets(response.json(), "A") == pytest.approx([100, 200, 300]), label+ columns = get_heatmap_columns(response.json(), "A")+ assert [column["values"] for column in columns] == [+ [0, 1, 0, 0],+ [0, 0, 1, 0],+ ], label+ assert [column.get("partial", False) for column in columns] == [False, True], label++ assert_identical_query_response(from_cache, uncached)+++def test_builder_shortening_the_time_range_at_the_start(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"heatmap_cache_start_shortened_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache. Flooring to a multiple of the 5m step makes the base query span+ # two whole steps, so both its columns are complete+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 300 * 300, tz=UTC)+ start_time_ms_base_query = int(start_time.timestamp() * 1000)+ start_time_ms_shortened_query = start_time_ms_base_query + 3 * MINUTE_MS+ end_time_ms = start_time_ms_base_query + 10 * MINUTE_MS++ query = [build_builder_query("A", metric_name, "max", "max", step_interval=300, bucket_options=build_linear_bucket_options(1000, 10))]++ # the 5m step splits the ten minutes into two columns, each the max over its+ # own step: minutes 0-4 and minutes 5-9. Only minute 0 reaches 950, so a+ # first column counted in (900, 1000] says the whole step was read even+ # though the shortened range opens at minute 3+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=(950, 150, 150, 150, 150, 350, 350, 350, 350, 350)[minute],+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(10)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ base_query = make_query_request(signoz, token, start_time_ms_base_query, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert base_query.status_code == HTTPStatus.OK, base_query.text++ # 100 wide buckets, and the two maxes are 950 and 350, so the axis runs from+ # the bottom of (300, 400] to the top of (900, 1000]+ assert get_heatmap_buckets(base_query.json(), "A") == pytest.approx([300, 400, 500, 600, 700, 800, 900, 1000])+ base_columns = get_heatmap_columns(base_query.json(), "A")+ assert [column["values"] for column in base_columns] == [+ [0, 0, 0, 0, 0, 0, 0, 1, 0],+ [0, 1, 0, 0, 0, 0, 0, 0, 0],+ ]+ assert [column.get("partial", False) for column in base_columns] == [False, False]++ from_cache = make_query_request(signoz, token, start_time_ms_shortened_query, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ uncached = make_query_request(signoz, token, start_time_ms_shortened_query, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ # starting inside the first column's step flags that column partial without+ # clipping its counts, which still cover the whole step and so reach the 950+ # at minute 0, leaving the axis where the base query drew it+ for label, response in (("uncached", uncached), ("from cache", from_cache)):+ assert get_heatmap_buckets(response.json(), "A") == pytest.approx([300, 400, 500, 600, 700, 800, 900, 1000]), label+ columns = get_heatmap_columns(response.json(), "A")+ assert [column["values"] for column in columns] == [+ [0, 0, 0, 0, 0, 0, 0, 1, 0],+ [0, 1, 0, 0, 0, 0, 0, 0, 0],+ ], label+ assert [column.get("partial", False) for column in columns] == [True, False], label++ assert_identical_query_response(from_cache, uncached)+++def test_builder_refreshing_a_sliding_time_range(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"heatmap_cache_sliding_{uuid4().hex[:8]}"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 60 * 60, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)++ query = [build_builder_query("A", metric_name, "max", "max", bucket_options=build_linear_bucket_options(1000, 10))]++ # the 1m step gives one column per seeded minute, and 100 wide buckets give+ # every minute a bucket no other minute reaches, so a column stitched in from+ # the wrong range is counted in the wrong bucket+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"service": "api"},+ timestamp=start_time + timedelta(minutes=minute),+ value=100 * minute + 50,+ type_="Gauge",+ is_monotonic=False,+ )+ for minute in range(7)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ # the window slides onto a bucket a minute higher each refresh, so an axis+ # carried over from an earlier one is off by as many buckets+ expected_buckets_by_refresh = [+ [0, 100, 200, 300, 400],+ [100, 200, 300, 400, 500],+ [200, 300, 400, 500, 600],+ [300, 400, 500, 600, 700],+ ]++ # whichever four minutes a refresh reads, each is in a bucket of its own and+ # they arrive in order, so the counts run down the diagonal+ expected_columns = [+ [0, 1, 0, 0, 0, 0],+ [0, 0, 1, 0, 0, 0],+ [0, 0, 0, 1, 0, 0],+ [0, 0, 0, 0, 1, 0],+ ]++ # a dashboard left open on a four minute range, re-running a minute later each+ # time, so every refresh is stitched out of the ranges the ones before it cached+ for refresh, expected_buckets in enumerate(expected_buckets_by_refresh):+ refresh_start_ms = start_time_ms + refresh * MINUTE_MS+ from_cache = make_query_request(signoz, token, refresh_start_ms, refresh_start_ms + 4 * MINUTE_MS, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert from_cache.status_code == HTTPStatus.OK, from_cache.text++ assert get_heatmap_buckets(from_cache.json(), "A") == pytest.approx(expected_buckets), f"refresh {refresh}"++ # a column served twice, dropped, or carried over from an earlier refresh+ # breaks the diagonal or the run of timestamps+ columns = get_heatmap_columns(from_cache.json(), "A")+ assert [column["timestamp"] for column in columns] == [+ refresh_start_ms,+ refresh_start_ms + MINUTE_MS,+ refresh_start_ms + 2 * MINUTE_MS,+ refresh_start_ms + 3 * MINUTE_MS,+ ], f"refresh {refresh}"+ assert [column["values"] for column in columns] == expected_columns, f"refresh {refresh}"++ uncached = make_query_request(signoz, token, refresh_start_ms, refresh_start_ms + 4 * MINUTE_MS, query, request_type=RequestType.HEATMAP, no_cache=True)+ assert uncached.status_code == HTTPStatus.OK, uncached.text++ assert_identical_query_response(from_cache, uncached)+++def test_promql_running_the_same_query_twice(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"heatmap_cache_repeat_{uuid4().hex[:8]}_bucket"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 60 * 60, tz=UTC)+ start_time_ms = int(start_time.timestamp() * 1000)+ end_time_ms = start_time_ms + 2 * MINUTE_MS++ query = [{"type": "promql", "spec": {"name": "A", "query": f"sum by (le) (increase({metric_name}[2m]))", "step": 60}}]++ # the cumulative count of each `le`, one entry per minute. The counters open+ # a minute before the query so its first column has something to increase+ # over, and start far above their own rise across the range, below which+ # increase clips its back-extrapolation at a counter's zero point+ le_to_counts = {+ "1": [1000, 1005, 1010, 1020],+ "2": [2000, 2010, 2025, 2040],+ "4": [3000, 3015, 3040, 3070],+ "+Inf": [4000, 4022, 4050, 4090],+ }+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"__temporality__": "Cumulative", "service": "api", "le": le},+ timestamp=start_time + timedelta(minutes=minute),+ value=count,+ temporality="Cumulative",+ type_="Histogram",+ )+ for le, counts in le_to_counts.items()+ for minute, count in enumerate(counts, start=-1)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ first = make_query_request(signoz, token, start_time_ms, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert first.status_code == HTTPStatus.OK, first.text++ second = make_query_request(signoz, token, start_time_ms, end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert second.status_code == HTTPStatus.OK, second.text++ # promql reports a column at the instant the range closes, and the second+ # run, answered out of what the first one cached, has to keep it+ for run, response in (("first", first), ("second", second)):+ assert get_heatmap_buckets(response.json(), "A") == [1, 2, 4], run+ ## what the query returns per `le` is cumulative across `le`, so each+ ## count is its own minus the one below it, and `le=+Inf` has no finite+ ## bound to sit on and lands in the trailing slot. increase over a 2m+ ## window of minutely samples extrapolates one minute's rise to two.+ assert [(column["timestamp"], column["values"]) for column in get_heatmap_columns(response.json(), "A")] == [+ (start_time_ms, [10, 10, 10, 14]), # t = 0, the minute brings 5, 10, 15 and 22 arrivals at or below each `le`+ (start_time_ms + MINUTE_MS, [10, 20, 20, 6]), # t = 1m, 5, 15, 25 and 28+ (end_time_ms, [20, 10, 30, 20]), # t = 2m, 10, 15, 30 and 40+ ], f"{run} run"++ assert_identical_query_response(first, second)+++def test_promql_shifting_the_time_range(+ signoz: types.SigNoz,+ create_user_admin: None, # pylint: disable=unused-argument+ get_token: Callable[[str, str], str],+ insert_metrics: Callable[[list[Metrics]], None],+) -> None:+ metric_name = f"heatmap_cache_shift_{uuid4().hex[:8]}_bucket"++ # 40 minutes back clears the flux interval, which holds recent data out of+ # the cache. Flooring to a whole minute is what makes the first query aligned+ # to its 1m step, and the unaligned one half a step off it+ start_time = datetime.fromtimestamp(int((datetime.now(tz=UTC) - timedelta(minutes=40)).timestamp()) // 60 * 60, tz=UTC)+ aligned_start_time_ms = int(start_time.timestamp() * 1000)+ aligned_end_time_ms = aligned_start_time_ms + 3 * MINUTE_MS+ unaligned_start_time_ms = aligned_start_time_ms + MINUTE_MS // 2+ unaligned_end_time_ms = aligned_end_time_ms + MINUTE_MS // 2++ query = [{"type": "promql", "spec": {"name": "A", "query": f"sum by (le) (max_over_time({metric_name}[2m]))", "step": 60}}]++ # a sample every 30s, each `le` counting up by its own fixed amount every+ # time. The two queries report 30s apart, so they land on different samples+ # and share no count between them+ le_to_arrivals_per_sample = {"1": 100, "2": 300, "4": 600, "+Inf": 1000}+ insert_metrics(+ [+ Metrics(+ metric_name=metric_name,+ labels={"__temporality__": "Cumulative", "service": "api", "le": le},+ timestamp=start_time + timedelta(seconds=30 * half_minute),+ value=arrivals_per_sample * (half_minute + 4),+ temporality="Cumulative",+ type_="Histogram",+ )+ for le, arrivals_per_sample in le_to_arrivals_per_sample.items()+ for half_minute in range(-3, 8)+ ]+ )++ token = get_token(USER_ADMIN_EMAIL, USER_ADMIN_PASSWORD)++ aligned_and_cached = make_query_request(signoz, token, aligned_start_time_ms, aligned_end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert aligned_and_cached.status_code == HTTPStatus.OK, aligned_and_cached.text++ assert get_heatmap_buckets(aligned_and_cached.json(), "A") == [1, 2, 4]+ ## each column reads the counters at their latest sample at or before its+ ## timestamp, and a bucket holds its own `le`'s count less the one below it.+ aligned_columns = [+ (aligned_start_time_ms, [400, 800, 1200, 1600]), # t = 0, the fourth sample+ (aligned_start_time_ms + MINUTE_MS, [600, 1200, 1800, 2400]), # t = 1m, the sixth+ (aligned_start_time_ms + 2 * MINUTE_MS, [800, 1600, 2400, 3200]), # t = 2m, the eighth+ (aligned_end_time_ms, [1000, 2000, 3000, 4000]), # t = 3m, the tenth+ ]+ assert [(column["timestamp"], column["values"]) for column in get_heatmap_columns(aligned_and_cached.json(), "A")] == aligned_columns++ # an unaligned range is rounded down to the step, so this asks for the same+ # instants as the aligned query and gets the same answer+ unaligned_and_cached = make_query_request(signoz, token, unaligned_start_time_ms, unaligned_end_time_ms, query, request_type=RequestType.HEATMAP, no_cache=False)+ assert unaligned_and_cached.status_code == HTTPStatus.OK, unaligned_and_cached.text+ assert get_heatmap_buckets(unaligned_and_cached.json(), "A") == [1, 2, 4], "unaligned query, rounded to the step"+ assert [(column["timestamp"], column["values"]) for column in get_heatmap_columns(unaligned_and_cached.json(), "A")] == aligned_columns, "unaligned query, rounded to the step"+ assert_identical_query_response(unaligned_and_cached, aligned_and_cached)++ # skipping the alignment keeps the range as asked, so every column sits 30s+ # off the cached ones, on samples the aligned query never saw. Such a query+ # is never cached, so repeating it never gets the aligned answer either+ unaligned_query = [{"type": "promql", "spec": {"name": "A", "query": f"sum by (le) (max_over_time({metric_name}[2m]))", "step": 60, "skipStepAlignment": True}}]+ unaligned_columns = [+ (unaligned_start_time_ms, [500, 1000, 1500, 2000]), # t = 30s, the fifth sample+ (unaligned_start_time_ms + MINUTE_MS, [700, 1400, 2100, 2800]), # t = 1m30s, the seventh+ (unaligned_start_time_ms + 2 * MINUTE_MS, [900, 1800, 2700, 3600]), # t = 2m30s, the ninth+ (unaligned_end_time_ms, [1100, 2200, 3300, 4400]), # t = 3m30s, the eleventh+ ]+ for run in ("first", "second"):+ unaligned_as_asked = make_query_request(signoz, token, unaligned_start_time_ms, unaligned_end_time_ms, unaligned_query, request_type=RequestType.HEATMAP, no_cache=False)+ assert unaligned_as_asked.status_code == HTTPStatus.OK, unaligned_as_asked.text+ assert get_heatmap_buckets(unaligned_as_asked.json(), "A") == [1, 2, 4], f"unaligned query as asked, {run} run"+ assert [(column["timestamp"], column["values"]) for column in get_heatmap_columns(unaligned_as_asked.json(), "A")] == unaligned_columns, f"unaligned query as asked, {run} run"