SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.
TypeScript apmapplication-monitoringdistributed-tracinggogood-first-issuejaegerloglogsmetricsmonitoringnextjsobservabilityopen-sourceopentelemetryprometheusreactreactjsself-hostedtracingtypescript
perf(clickhouseprometheusv2): skip the series lookup for statically named transpiled units (#12728)
Srikanth Chekuri4d015a927e82
diff --git a/docs/contributing/prometheus.md b/docs/contributing/prometheus.mdindex 7f4573ac1d..f9a9d72ea3 100644--- a/docs/contributing/prometheus.md+++ b/docs/contributing/prometheus.md@@ -299,8 +299,11 @@ substituted. One subtlety makes it exact: we write stale markers at absentgrid points. Without them, the engine's lookback would resurrect a pointfrom up to `lookback` earlier. The marker encodes "absent here" the way theengine itself encodes it. Units evaluate concurrently. Each unit is one-series lookup plus one grid statement. A step of 0 is an instant query: a-single evaluation at `end`.+grid statement: the group-key join resolves the matchers, and the samples+primary key takes the metric name straight from the selector. Only a+selector without a static `__name__` runs the series lookup first, to learn+the concrete metric names. A step of 0 is an instant query: a single+evaluation at `end`.A note on the window sliver: when the window is narrower than the step, thegrid windows cover only `window/step` of the timeline. A sample in a gap@@ -315,8 +318,9 @@ selectors and `last_over_time` transpile at window < step too.## Series lookup-Both paths resolve matchers the same way, once per selector-(`selectSeries`). The series tables hold one row per (fingerprint, bucket)+The engine path resolves matchers once per selector (`selectSeries`); the+transpiled path builds the same conditions into its group-key join. Both+read the same tables. The series tables hold one row per (fingerprint, bucket)at 1h/6h/1d/1w granularities. The shared schema package(`pkg/telemetryschema/metricstelemetryschema`) picks the table whose bucketfits the window. It rounds the window start down to the bucket boundary, sodiff --git a/pkg/prometheus/clickhouseprometheusv2/capture.go b/pkg/prometheus/clickhouseprometheusv2/capture.goindex 118716841b..c6413de90f 100644--- a/pkg/prometheus/clickhouseprometheusv2/capture.go+++ b/pkg/prometheus/clickhouseprometheusv2/capture.go@@ -73,8 +73,8 @@ func (c *captureQuerier) LabelNames(context.Context, *storage.LabelHints, ...*la}// metricNamesFromMatchers extracts the statically known metric name, if any.-// The live path derives names from the matched series; the capture path has-// no execution results, so only a __name__ equality contributes.+// Only a __name__ equality contributes; a regex selector needs a series+// lookup to learn the concrete names.func metricNamesFromMatchers(matchers []*labels.Matcher) []string {for _, m := range matchers {if m.Name == metricNameLabel && m.Type == labels.MatchEqual && m.Value != "" {diff --git a/pkg/prometheus/clickhouseprometheusv2/transpiler_exec.go b/pkg/prometheus/clickhouseprometheusv2/transpiler_exec.goindex 87fea487d6..77007bded6 100644--- a/pkg/prometheus/clickhouseprometheusv2/transpiler_exec.go+++ b/pkg/prometheus/clickhouseprometheusv2/transpiler_exec.go@@ -88,7 +88,8 @@ func (e *executor) TryExecuteRange(ctx context.Context, qs string, start, end ti}// Evaluate every unit concurrently on its own grid (the query grid, or a- // subquery grid); each is one series lookup plus one grid query.+ // subquery grid); each is one grid query (see executeUnit for when a+ // series lookup precedes it).results := make([][]transpiledSeries, len(plan.units))eg, egCtx := errgroup.WithContext(ctx)for i, unit := range plan.units {@@ -142,19 +143,27 @@ func (e *executor) executeUnit(ctx context.Context, unit *coreUnit, grid gridCondataStart := startMs - unit.offsetMs - windowMsdataEnd := endMs - unit.offsetMs- seriesQuery, seriesArgs, err := buildSeriesQuery(dataStart, dataEnd, unit.matchers)- if err != nil {- return nil, err- }- lookup, err := e.client.selectSeries(ctx, seriesQuery, seriesArgs)- if err != nil {- return nil, err- }- if len(lookup.fingerprints) == 0 {- return nil, nil+ // The group-key join resolves the matchers on its own, so the unit+ // statement only needs concrete metric names for the samples+ // primary-key prefix. A selector without a static __name__ learns them+ // through the series lookup; every other selector skips the roundtrip.+ metricNames := metricNamesFromMatchers(unit.matchers)+ if metricNames == nil {+ seriesQuery, seriesArgs, err := buildSeriesQuery(dataStart, dataEnd, unit.matchers)+ if err != nil {+ return nil, err+ }+ lookup, err := e.client.selectSeries(ctx, seriesQuery, seriesArgs)+ if err != nil {+ return nil, err+ }+ if len(lookup.fingerprints) == 0 {+ return nil, nil+ }+ metricNames = lookup.metricNames}- query, args, err := buildUnitSQL(unit, lookup.metricNames, dataStart, dataEnd, startMs, endMs, stepMs, e.client.lookbackMs)+ query, args, err := buildUnitSQL(unit, metricNames, dataStart, dataEnd, startMs, endMs, stepMs, e.client.lookbackMs)if err != nil {return nil, err}diff --git a/pkg/prometheus/clickhouseprometheusv2/transpiler_test.go b/pkg/prometheus/clickhouseprometheusv2/transpiler_test.goindex a042ae5cd4..a14e7c7923 100644--- a/pkg/prometheus/clickhouseprometheusv2/transpiler_test.go+++ b/pkg/prometheus/clickhouseprometheusv2/transpiler_test.go@@ -27,9 +27,15 @@ func newTestClient(t *testing.T) (*client, *telemetrystoretest.Provider) {return newClient(settings, store, prometheus.Config{}), store}-var seriesCols = []cmock.ColumnType{- {Name: "fingerprint", Type: "UInt64"},- {Name: "labels", Type: "String"},+var unitCols = []cmock.ColumnType{+ {Name: "gkey", Type: "String"},+ {Name: "grid", Type: "Array(Nullable(Float64))"},+}++// anyArgs matches a bound-argument list by count alone: the mock treats a+// nil expected argument as a wildcard.+func anyArgs(n int) []any {+ return make([]any, n)}func parse(t *testing.T, q string) parser.Expr {@@ -553,7 +559,7 @@ func TestTryExecuteRange_WindowedGateFallsBack(t *testing.T) {// 1m range at 5m step: the windows are disjoint slivers — no// divisibility or width requirement, so this transpiles.- store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("up", int64(1_699_999_200_000), int64(1_700_003_600_000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{}))+ store.Mock().ExpectQuery("FROM signoz_metrics\\.distributed_samples_v4").WithArgs(anyArgs(9)...).WillReturnRows(cmock.NewRows(unitCols, [][]any{}))_, ok, err = e.TryExecuteRange(context.Background(), `avg_over_time(up[1m])`, start, end, 5*time.Minute)require.NoError(t, err)assert.True(t, ok, "range below step is the disjoint form and must transpile")@@ -637,12 +643,12 @@ func TestTryExecuteRange_LastStyleWindowBelowStepTranspiles(t *testing.T) {start := time.UnixMilli(1_700_000_000_000)end := time.UnixMilli(1_700_003_600_000)- store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("up", int64(1_699_999_200_000), int64(1_700_003_600_000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{}))+ store.Mock().ExpectQuery("timeSeriesLastToGrid").WithArgs(anyArgs(10)...).WillReturnRows(cmock.NewRows(unitCols, [][]any{}))_, ok, err := e.TryExecuteRange(context.Background(), `sum by (pod) (up)`, start, end, time.Hour)require.NoError(t, err)assert.True(t, ok, "instant selection at step > lookback must transpile")- store.Mock().ExpectQuery("SELECT fingerprint, any\\(labels\\)").WithArgs("up", int64(1_699_999_200_000), int64(1_700_003_600_000)).WillReturnRows(cmock.NewRows(seriesCols, [][]any{}))+ store.Mock().ExpectQuery("timeSeriesLastToGrid").WithArgs(anyArgs(9)...).WillReturnRows(cmock.NewRows(unitCols, [][]any{}))_, ok, err = e.TryExecuteRange(context.Background(), `last_over_time(up[10m])`, start, end, time.Hour)require.NoError(t, err)assert.True(t, ok, "last_over_time at range < step must transpile")