Skip to content

SigNoz is an open-source, OpenTelemetry-native observability platform for your team and their AI agents. Get logs, metrics, and traces in one tool with features like APM, distributed tracing, log management, infra monitoring, etc. Combined with SigNoz MCP and a native AI teammate (in SigNoz Cloud) it helps you build more resilient apps.

15.5 KB
Raw Blame
package querier

import (
	"bytes"
	"context"
	"fmt"
	"log/slog"
	"math"
	"regexp"
	"sort"
	"strings"
	"sync"
	"text/template"
	"time"

	"github.com/ClickHouse/clickhouse-go/v2"

	"github.com/prometheus/prometheus/model/labels"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/promql/parser"

	"github.com/SigNoz/signoz/pkg/errors"
	"github.com/SigNoz/signoz/pkg/prometheus"
	"github.com/SigNoz/signoz/pkg/querybuilder"
	"github.com/SigNoz/signoz/pkg/types/ctxtypes"
	"github.com/SigNoz/signoz/pkg/types/instrumentationtypes"
	qbv5 "github.com/SigNoz/signoz/pkg/types/querybuildertypes/querybuildertypesv5"
	"github.com/SigNoz/signoz/pkg/types/telemetrytypes"
)

// unquotedDottedNamePattern matches unquoted identifiers containing dots
// that appear in metric or label name positions. This helps detect queries
// using the old syntax that needs migration to UTF-8 quoted syntax.
// Examples it matches: k8s.pod.name, deployment.environment, http.status_code.
var unquotedDottedNamePattern = regexp.MustCompile(`(?:^|[{,(\s])([a-zA-Z_][a-zA-Z0-9_]*(?:\.[a-zA-Z0-9_]+)+)(?:[}\s,=!~)\[]|$)`)

// quotedMetricOutsideBracesPattern matches the incorrect syntax where a quoted
// metric name appears outside of braces followed by a selector block.
// Example: "kube_pod_status_ready_time"{"condition"="true"}
// This is a common mistake when migrating to UTF-8 syntax.
var quotedMetricOutsideBracesPattern = regexp.MustCompile(`"([^"]+)"\s*\{`)

// tryEnhancePromQLExecError attempts to convert a PromQL execution error into
// a properly typed error. Returns nil if the error is not a recognized execution error.
func tryEnhancePromQLExecError(execErr error) error {
	var eqc promql.ErrQueryCanceled
	var eqt promql.ErrQueryTimeout
	var es promql.ErrStorage
	switch {
	case errors.As(execErr, &eqc):
		return errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled").WithAdditional(eqc.Error())
	case errors.As(execErr, &eqt):
		return errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timed out").WithAdditional(eqt.Error())
	case errors.Is(execErr, context.DeadlineExceeded):
		return errors.Newf(errors.TypeTimeout, errors.CodeTimeout, "query timed out")
	case errors.Is(execErr, context.Canceled):
		return errors.Newf(errors.TypeCanceled, errors.CodeCanceled, "query canceled")
	case errors.As(execErr, &es):
		return errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", execErr)
	default:
		return nil
	}
}

// enhancePromQLError adds helpful context to PromQL parse errors,
// particularly for UTF-8 syntax migration issues where metric and label
// names containing dots need to be quoted.
func enhancePromQLError(query string, parseErr error) error {
	errMsg := parseErr.Error()

	if matches := quotedMetricOutsideBracesPattern.FindStringSubmatch(query); len(matches) > 1 {
		metricName := matches[1]
		return errors.NewInvalidInputf(
			errors.CodeInvalidInput,
			"invalid promql query: %s. Hint: The metric name should be inside the braces. Use {\"__name__\"=\"%s\", ...} or {\"%s\", ...} instead of \"%s\"{...}",
			errMsg,
			metricName,
			metricName,
			metricName,
		)
	}

	if matches := unquotedDottedNamePattern.FindStringSubmatch(query); len(matches) > 1 {
		dottedName := matches[1]
		return errors.NewInvalidInputf(
			errors.CodeInvalidInput,
			"invalid promql query: %s. Hint: Metric and label names containing dots require quoted notation in the new UTF-8 syntax, e.g., use \"%s\" instead of %s",
			errMsg,
			dottedName,
			dottedName,
		)
	}

	return errors.NewInvalidInputf(errors.CodeInvalidInput, "invalid promql query: %s", errMsg)
}

type promqlQuery struct {
	logger      *slog.Logger
	promEngine  prometheus.Prometheus
	parser      parser.Parser
	query       qbv5.PromQuery
	tr          qbv5.TimeRange
	requestType qbv5.RequestType
	vars        map[string]qbv5.VariableItem
}

var _ qbv5.Query = (*promqlQuery)(nil)
var _ qbv5.StatementProvider = (*promqlQuery)(nil)

func newPromqlQuery(
	logger *slog.Logger,
	promEngine prometheus.Prometheus,
	query qbv5.PromQuery,
	tr qbv5.TimeRange,
	requestType qbv5.RequestType,
	variables map[string]qbv5.VariableItem,
) *promqlQuery {
	// promql evaluates at tr.From and every step after it, so two windows a few
	// seconds apart ask for different instants and could never share a cache
	// entry. Rounding both ends down to the step makes them the same query.
	if stepMs := uint64(query.Step.Milliseconds()); stepMs > 0 && !query.SkipStepAlignment {
		tr.From -= tr.From % stepMs
		tr.To -= tr.To % stepMs
	}
	return &promqlQuery{
		logger:      logger,
		promEngine:  promEngine,
		parser:      prometheus.NewParser(),
		query:       query,
		tr:          tr,
		requestType: requestType,
		vars:        variables,
	}
}

func (q *promqlQuery) Fingerprint() string {
	switch q.requestType {
	case qbv5.RequestTypeTimeSeries, qbv5.RequestTypeHeatmap:
	default:
		return ""
	}

	// an unaligned window evaluates at instants no other window asks for, so
	// there is nothing to share
	if q.query.SkipStepAlignment {
		return ""
	}

	query, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To)
	if err != nil {
		q.logger.ErrorContext(context.TODO(), "failed render template variables", slog.String("query", q.query.Query))
		return ""
	}
	parts := []string{
		"promql",
		// one expression returns a different shape per request type
		fmt.Sprintf("requestType=%s", q.requestType.StringValue()),
		query,
		q.query.Step.String(),
	}

	return strings.Join(parts, "&")
}

func (q *promqlQuery) Window() (uint64, uint64) {
	return q.tr.From, q.tr.To
}

// removeAllVarMatchers removes label matchers from a PromQL query that reference variables with __all__ value.
// This method parses the query, walks the AST to remove matching matchers, and returns the modified query string.
// If parsing or walking fails, it returns an error.
func (q *promqlQuery) removeAllVarMatchers(query string, vars map[string]qbv5.VariableItem) (string, error) {
	// Find all variables that have __all__ value
	allVars := make(map[string]bool)
	for k, v := range vars {
		if v.Type == qbv5.DynamicVariableType {
			if allVal, ok := v.Value.(string); ok && allVal == "__all__" {
				allVars[k] = true
			}
		}
	}

	// If no variables have __all__ value, return the query unchanged
	if len(allVars) == 0 {
		return query, nil
	}

	expr, err := q.parser.ParseExpr(query)
	if err != nil {
		return "", enhancePromQLError(query, err)
	}

	// Create visitor and walk the AST
	visitor := &allVarRemover{allVars: allVars}
	if err := parser.Walk(visitor, expr, nil); err != nil {
		q.logger.ErrorContext(context.TODO(), "unexpected error while removing __all__ variable matchers", errors.Attr(err), slog.String("query", query))
		return "", errors.WrapInternalf(err, errors.CodeInternal, "error while removing __all__ variable matchers")
	}

	// Convert the modified AST back to a string
	return expr.String(), nil
}

// TODO(srikanthccv): cleanup the templating logic.
func (q *promqlQuery) renderVars(query string, vars map[string]qbv5.VariableItem, start, end uint64) (string, error) {
	// First, remove label matchers that use variables with __all__ value.
	// This must happen before variable substitution so we can detect variable references
	// in their original form ($var, {{var}}, [[var]]).
	query, err := q.removeAllVarMatchers(query, vars)
	if err != nil {
		return "", err
	}
	varsData := map[string]any{}
	for k, v := range vars {
		varsData[k] = formatValueForProm(v.Value)
	}

	querybuilder.AssignReservedVars(varsData, start, end)

	keys := make([]string, 0, len(varsData))
	for k := range varsData {
		keys = append(keys, k)
	}
	sort.Slice(keys, func(i, j int) bool {
		return len(keys[i]) > len(keys[j])
	})

	for _, k := range keys {
		query = strings.ReplaceAll(query, fmt.Sprintf("{{%s}}", k), fmt.Sprint(varsData[k]))
		query = strings.ReplaceAll(query, fmt.Sprintf("[[%s]]", k), fmt.Sprint(varsData[k]))
		query = strings.ReplaceAll(query, fmt.Sprintf("$%s", k), fmt.Sprint(varsData[k]))
	}

	tmpl := template.New("promql-query")
	tmpl, err = tmpl.Parse(query)
	if err != nil {
		return "", errors.WrapInternalf(err, errors.CodeInternal, "error while replacing template variables")
	}
	var newQuery bytes.Buffer

	// replace go template variables
	err = tmpl.Execute(&newQuery, varsData)
	if err != nil {
		return "", errors.WrapInternalf(err, errors.CodeInternal, "error while replacing template variables")
	}
	return newQuery.String(), nil
}

// Statement renders the PromQL string (no SQL args) without executing it, for
// the preview path.
func (q *promqlQuery) Statement(_ context.Context) (*qbv5.Statement, error) {
	rendered, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To)
	if err != nil {
		return nil, err
	}
	return &qbv5.Statement{Query: rendered}, nil
}

// PreviewStatements returns the ClickHouse statement(s) this PromQL query
// would run on the engine path, captured without executing them.
func (q *promqlQuery) PreviewStatements(ctx context.Context) ([]prometheus.CapturedStatement, error) {
	rendered, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To)
	if err != nil {
		return nil, err
	}

	start := int64(querybuilder.ToNanoSecs(q.tr.From))
	end := int64(querybuilder.ToNanoSecs(q.tr.To))

	statements, err := q.promEngine.Statements(ctx, rendered, time.Unix(0, start), time.Unix(0, end), q.query.Step.Duration)
	if err != nil {
		return nil, q.evalError(rendered, err)
	}
	return statements, nil
}

func (q *promqlQuery) Execute(ctx context.Context) (*qbv5.Result, error) {

	ctx = ctxtypes.NewContextWithCommentVals(ctx, map[string]string{
		instrumentationtypes.TelemetrySignal: telemetrytypes.SignalMetrics.StringValue(),
		instrumentationtypes.QueryDuration:   instrumentationtypes.DurationBucket(q.tr.From, q.tr.To),
	})

	start := int64(querybuilder.ToNanoSecs(q.tr.From))
	end := int64(querybuilder.ToNanoSecs(q.tr.To))

	query, err := q.renderVars(q.query.Query, q.vars, q.tr.From, q.tr.To)
	if err != nil {
		return nil, err
	}

	// Accumulate ClickHouse-side scan stats across every storage query this
	// evaluation issues (engine selectors or the compiled executor): progress
	// options propagate to each ClickHouse query through the context.
	var statsMu sync.Mutex
	var rowsScanned, bytesScanned uint64
	ctx = clickhouse.Context(ctx, clickhouse.WithProgress(func(p *clickhouse.Progress) {
		statsMu.Lock()
		rowsScanned += p.Rows
		bytesScanned += p.Bytes
		statsMu.Unlock()
	}))

	began := time.Now()

	res, err := q.promEngine.QueryRange(ctx, query, time.Unix(0, start), time.Unix(0, end), q.query.Step.Duration)
	if err != nil {
		return nil, q.evalError(query, err)
	}

	matrix, ok := res.Value.(promql.Matrix)
	if !ok {
		return nil, errors.Newf(errors.TypeInternal, errors.CodeInternal, "promql query %q returned %T, expected a matrix", query, res.Value)
	}

	warnings, _ := res.Warnings.AsStrings(query, 10, 0)
	return q.toResult(matrix, warnings, began, &statsMu, &rowsScanned, &bytesScanned)
}

// evalError types an evaluation error: engine execution classes first, then
// parse errors with the migration hints, everything else internal.
func (q *promqlQuery) evalError(query string, err error) error {
	if enhanced := tryEnhancePromQLExecError(err); enhanced != nil {
		return enhanced
	}
	var parseErrs parser.ParseErrors
	if errors.As(err, &parseErrs) {
		return enhancePromQLError(query, err)
	}
	// The transpiled path raises typed user errors of its own; keep them.
	if errors.Ast(err, errors.TypeInvalidInput) {
		return err
	}
	return errors.Newf(errors.TypeInternal, errors.CodeInternal, "query execution error: %v", err)
}

// excludePromLabel hides only known SigNoz storage keys: label names are user
// data and may legitimately start with "__" (e.g. __address__), so a blanket
// dunder strip mangles user labelsets. The __scope./__resource. prefixes cover
// every exporter version's keys.
func excludePromLabel(labelName string) bool {
	return labelName == "__temporality__" ||
		strings.HasPrefix(labelName, "__scope.") ||
		strings.HasPrefix(labelName, "__resource.")
}

// collectExecStats snapshots the scan counters a query accumulated. Callers take
// it at the point they are done with the matrix, so the duration covers the
// shaping they did.
func collectExecStats(began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) qbv5.ExecStats {
	statsMu.Lock()
	defer statsMu.Unlock()
	return qbv5.ExecStats{
		RowsScanned:  *rowsScanned,
		BytesScanned: *bytesScanned,
		DurationMS:   uint64(time.Since(began).Milliseconds()),
	}
}

func (q *promqlQuery) toResult(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) (*qbv5.Result, error) {
	if q.requestType == qbv5.RequestTypeHeatmap {
		return q.toResultForHeatmap(matrix, warnings, began, statsMu, rowsScanned, bytesScanned)
	}
	return q.toResultForTimeSeriesAndScalar(matrix, warnings, began, statsMu, rowsScanned, bytesScanned), nil
}

func (q *promqlQuery) toResultForHeatmap(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) (*qbv5.Result, error) {
	tsData, err := foldMatrixAsHeatmap(matrix, q.query.Name)
	if err != nil {
		return nil, err
	}

	return &qbv5.Result{
		Type:     q.requestType,
		Value:    tsData,
		Warnings: warnings,
		Stats:    collectExecStats(began, statsMu, rowsScanned, bytesScanned),
	}, nil
}

func (q *promqlQuery) toResultForTimeSeriesAndScalar(matrix promql.Matrix, warnings []string, began time.Time, statsMu *sync.Mutex, rowsScanned, bytesScanned *uint64) *qbv5.Result {
	var series []*qbv5.TimeSeries
	for _, v := range matrix {
		var s qbv5.TimeSeries
		lbls := make([]*qbv5.Label, 0, v.Metric.Len())
		v.Metric.Range(func(l labels.Label) {
			if excludePromLabel(l.Name) {
				return
			}
			lbls = append(lbls, &qbv5.Label{
				Key:   telemetrytypes.TelemetryFieldKey{Name: l.Name},
				Value: l.Value,
			})
		})
		s.Labels = lbls

		for idx := range v.Floats {
			p := v.Floats[idx]
			// NaN and +/-Inf have no JSON number form and nothing to plot; the
			// builder path drops them while scanning rows (see consume.go).
			if math.IsNaN(p.F) || math.IsInf(p.F, 0) {
				continue
			}
			s.Values = append(s.Values, &qbv5.TimeSeriesValue{
				Timestamp: p.T,
				Value:     p.F,
			})
		}
		if len(s.Values) == 0 {
			continue
		}
		series = append(series, &s)
	}

	stats := collectExecStats(began, statsMu, rowsScanned, bytesScanned)

	tsData := &qbv5.TimeSeriesData{QueryName: q.query.Name}
	// No bucket at all when nothing survived: a bucket holding no series reads
	// as "filtered to empty" to the cache, which stores it as a real result.
	if len(series) > 0 {
		tsData.Aggregations = []*qbv5.AggregationBucket{{Series: series}}
	}

	var payload any = tsData
	// Scalar requests must return scalar data; reduce each series to its
	// last point. This approximates an instant query evaluated at the end
	// of the window: the final range-eval step sees the same samples an
	// instant query at that timestamp would, except a series that went
	// stale mid-window still surfaces its most recent value instead of
	// dropping out of the result.
	// TODO(srikanthccv): "last" mirrors the instant-query semantics we
	// have always followed for PromQL scalar requests, but it should be
	// configurable on the PromQuery spec just like the builder reduceTo.
	if q.requestType == qbv5.RequestTypeScalar {
		for _, aggBucket := range tsData.Aggregations {
			for i, s := range aggBucket.Series {
				aggBucket.Series[i] = qbv5.FunctionReduceTo(s, qbv5.ReduceToLast)
			}
		}
		payload = convertTimeSeriesDataToScalar(tsData, q.query.Name)
	}

	return &qbv5.Result{
		Type:     q.requestType,
		Value:    payload,
		Warnings: warnings,
		Stats:    stats,
	}
}