Skip to content

The only tool you need to know what is happening and how to fix it.

24.4 KB
Raw Blame
//go:build !telemetry_ch

package notifications

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"maps"
	"slices"
	"time"

	"github.com/google/uuid"
	"github.com/tracewayapp/traceway/backend/app/db"
	"github.com/tracewayapp/traceway/backend/app/models"
	"github.com/tracewayapp/traceway/backend/app/repositories/telemetry"
)

type EvalResult struct {
	Fired     bool
	Recovered bool
	Message   Message
	Messages  []Message
}

type RuleEvaluator func(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error)

var polledEvaluators = map[string]RuleEvaluator{
	"error_rate_threshold":    evaluateErrorRateThreshold,
	"endpoint_p95_threshold":  evaluateEndpointP95Threshold,
	"endpoint_p99_threshold":  evaluateEndpointP99Threshold,
	"apdex_drop":              evaluateApdexDrop,
	"metric_threshold":        evaluateMetricThreshold,
	"no_data":                 evaluateNoData,
	"error_count_threshold":   evaluateErrorCountThreshold,
	"task_duration_threshold": evaluateTaskDurationThreshold,
	"task_failure_rate":       evaluateTaskFailureRate,
	"throughput_drop":         evaluateThroughputDrop,
	"endpoint_error_rate":     evaluateEndpointErrorRate,
	"impact_score_critical":   evaluateImpactScoreCritical,
	"impact_score_high":       evaluateImpactScoreHigh,
	"impact_score_medium":     evaluateImpactScoreMedium,
	"slo_burn_rate":           evaluateSLOBurnRate,
	"slo_budget_consumed":     evaluateSLOBudgetConsumed,
}

// --- Error Rate Threshold ---

type errorRateConfig struct {
	ThresholdPercent float64 `json:"thresholdPercent"`
	LookbackMinutes  int     `json:"lookbackMinutes"`
	MinRequests      int     `json:"minRequests"`
}

func evaluateErrorRateThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg errorRateConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid error_rate_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 5
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	var total, errors int64
	err := db.TelemetryDB.QueryRowContext(ctx,
		"SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN status_code >= 500 THEN 1 ELSE 0 END), 0) as errors FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
		projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &errors)
	if err != nil {
		return nil, err
	}

	if total < int64(cfg.MinRequests) {
		return &EvalResult{Fired: false}, nil
	}

	rate := float64(errors) / float64(total) * 100
	if rate < cfg.ThresholdPercent {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildErrorRateMessage(rate, cfg.ThresholdPercent, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Endpoint P95 Threshold ---

type endpointLatencyConfig struct {
	Endpoint        string  `json:"endpoint"`
	ThresholdMs     float64 `json:"thresholdMs"`
	LookbackMinutes int     `json:"lookbackMinutes"`
}

func evaluateEndpointP95Threshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg endpointLatencyConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid endpoint_p95_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 5
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	p95 := queryPercentile(ctx, projectId, cfg.Endpoint, from, now, 0.95)

	if p95 < cfg.ThresholdMs {
		return &EvalResult{Fired: false}, nil
	}

	endpoint := cfg.Endpoint
	if endpoint == "" || endpoint == "*" {
		endpoint = "all endpoints"
	}
	projectName := getProjectName(projectId)
	msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Endpoint P99 Threshold ---

func evaluateEndpointP99Threshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg endpointLatencyConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid endpoint_p99_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 5
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	p99 := queryPercentile(ctx, projectId, cfg.Endpoint, from, now, 0.99)

	if p99 < cfg.ThresholdMs {
		return &EvalResult{Fired: false}, nil
	}

	endpoint := cfg.Endpoint
	if endpoint == "" || endpoint == "*" {
		endpoint = "all endpoints"
	}
	projectName := getProjectName(projectId)
	msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

func queryPercentile(ctx context.Context, projectId uuid.UUID, endpoint string, from, to time.Time, pct float64) float64 {
	query := "SELECT duration FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId.String(), from.Format(time.RFC3339Nano), to.Format(time.RFC3339Nano)}
	if endpoint != "" && endpoint != "*" {
		query += " AND endpoint = ?"
		args = append(args, endpoint)
	}
	query += " ORDER BY duration ASC"

	rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
	if err != nil {
		return 0
	}
	defer rows.Close()

	var durations []float64
	for rows.Next() {
		var d int64
		if err := rows.Scan(&d); err != nil {
			continue
		}
		durations = append(durations, float64(d)/1000000) // ns to ms
	}

	return computePercentile(durations, pct)
}

func computePercentile(sorted []float64, p float64) float64 {
	n := len(sorted)
	if n == 0 {
		return 0
	}
	if n == 1 {
		return sorted[0]
	}
	idx := p * float64(n-1)
	lower := int(idx)
	frac := idx - float64(lower)
	if lower+1 >= n {
		return sorted[lower]
	}
	return sorted[lower]*(1-frac) + sorted[lower+1]*frac
}

// --- Apdex Drop ---

type apdexConfig struct {
	ThresholdApdex  float64 `json:"thresholdApdex"`
	LookbackMinutes int     `json:"lookbackMinutes"`
	MinRequests     int     `json:"minRequests"`
}

func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg apdexConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid apdex_drop config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 15
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	var total, satisfied, tolerating int64
	err := db.TelemetryDB.QueryRowContext(ctx,
		`SELECT COUNT(*) as total,
			COALESCE(SUM(CASE WHEN duration <= 750000000 AND status_code < 500 THEN 1 ELSE 0 END), 0) as satisfied,
			COALESCE(SUM(CASE WHEN duration > 750000000 AND duration <= 1500000000 AND status_code < 500 THEN 1 ELSE 0 END), 0) as tolerating
		FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?`,
		projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &satisfied, &tolerating)
	if err != nil {
		return nil, err
	}

	if total < int64(cfg.MinRequests) {
		return &EvalResult{Fired: false}, nil
	}

	apdex := (float64(satisfied) + float64(tolerating)/2.0) / float64(total)
	if apdex >= cfg.ThresholdApdex {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, total, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Metric Threshold ---

func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	cfg, err := parseMetricThresholdConfig(rule.Config)
	if err != nil {
		return nil, err
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	tagExtract := "json_extract"
	if db.IsDuckDBTelemetry() {
		tagExtract = "json_extract_string"
	}
	groupExpr, args := metricGroupExpr(cfg.GroupBy, "COALESCE("+tagExtract+"(tags, '$.\"' || ? || '\"'), '')")
	where := "project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?"
	args = append(args, projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
	for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {
		where += " AND " + tagExtract + "(tags, '$.\"' || ? || '\"') = ?"
		args = append(args, k, cfg.Tags[k])
	}

	var series []metricSeries
	switch cfg.Aggregation {
	case "last", "p95", "p99":
		order := "recorded_at DESC"
		if cfg.Aggregation != "last" {
			order = "value ASC"
		}
		series, err = scanMetricSeriesValues(ctx, cfg.Aggregation,
			fmt.Sprintf("SELECT %s AS group_key, value FROM metric_points WHERE %s ORDER BY group_key, %s", groupExpr, where, order), args)
	default:
		aggFunc := "avg"
		switch cfg.Aggregation {
		case "max":
			aggFunc = "max"
		case "min":
			aggFunc = "min"
		case "sum":
			aggFunc = "sum"
		}
		query := fmt.Sprintf("SELECT %s AS group_key, COALESCE(%s(value), 0) FROM metric_points WHERE %s GROUP BY group_key ORDER BY group_key LIMIT ?", groupExpr, aggFunc, where)
		series, err = scanMetricSeries(ctx, query, append(args, maxMetricRuleSeriesScanned))
	}
	if err != nil {
		return nil, err
	}

	return metricThresholdResult(rule, cfg, series, projectId), nil
}

func scanMetricSeries(ctx context.Context, query string, args []interface{}) ([]metricSeries, error) {
	rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var series []metricSeries
	for rows.Next() {
		var s metricSeries
		if err := rows.Scan(&s.Key, &s.Value); err != nil {
			return nil, err
		}
		series = append(series, s)
	}
	return series, rows.Err()
}

// scanMetricSeriesValues reduces rows ordered by group key to one value per
// series: the first row for last (newest first), a percentile otherwise
// (values ascending).
func scanMetricSeriesValues(ctx context.Context, aggregation string, query string, args []interface{}) ([]metricSeries, error) {
	rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	pct := 0.95
	if aggregation == "p99" {
		pct = 0.99
	}
	var series []metricSeries
	var vals []float64
	flush := func(key string) {
		if len(vals) == 0 {
			return
		}
		value := vals[0]
		if aggregation != "last" {
			value = computePercentile(vals, pct)
		}
		series = append(series, metricSeries{Key: key, Value: value})
		vals = vals[:0]
	}

	current := ""
	for rows.Next() {
		var key string
		var v float64
		if err := rows.Scan(&key, &v); err != nil {
			return nil, err
		}
		if key != current || len(vals) == 0 {
			flush(current)
			if len(series) >= maxMetricRuleSeriesScanned {
				return series, nil
			}
			current = key
		}
		vals = append(vals, v)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	flush(current)
	return series, nil
}

// --- No Data ---

type noDataConfig struct {
	DataType       string `json:"dataType"`
	SilenceMinutes int    `json:"silenceMinutes"`
}

func evaluateNoData(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg noDataConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid no_data config: %w", err)
	}
	if cfg.SilenceMinutes <= 0 {
		cfg.SilenceMinutes = 10
	}

	threshold := time.Now().UTC().Add(-time.Duration(cfg.SilenceMinutes) * time.Minute)
	pid := projectId.String()

	if cfg.DataType == "any" {
		tables := []string{"endpoints_v2", "exceptions_v2", "metric_points", "tasks_v2"}
		for _, t := range tables {
			var maxTs sql.NullString
			err := db.TelemetryDB.QueryRowContext(ctx,
				fmt.Sprintf("SELECT MAX(recorded_at) FROM %s WHERE project_id = ?", t),
				pid).Scan(&maxTs)
			if err == nil && maxTs.Valid && maxTs.String != "" {
				if parsed, pErr := time.Parse(time.RFC3339Nano, maxTs.String); pErr == nil && parsed.After(threshold) {
					return &EvalResult{Fired: false}, nil
				}
			}
		}
		projectName := getProjectName(projectId)
		msg := buildNoDataMessage("any", cfg.SilenceMinutes, projectName)
		return &EvalResult{Fired: true, Message: msg}, nil
	}

	table := ""
	switch cfg.DataType {
	case "endpoints":
		table = "endpoints_v2"
	case "exceptions":
		table = "exceptions_v2"
	case "metrics":
		table = "metric_points"
	case "tasks":
		table = "tasks_v2"
	default:
		return nil, fmt.Errorf("unknown data type: %s", cfg.DataType)
	}

	var maxTs sql.NullString
	err := db.TelemetryDB.QueryRowContext(ctx,
		fmt.Sprintf("SELECT MAX(recorded_at) FROM %s WHERE project_id = ?", table),
		pid).Scan(&maxTs)
	if err != nil {
		return nil, err
	}

	if maxTs.Valid && maxTs.String != "" {
		if parsed, pErr := time.Parse(time.RFC3339Nano, maxTs.String); pErr == nil && parsed.After(threshold) {
			return &EvalResult{Fired: false}, nil
		}
	}

	projectName := getProjectName(projectId)
	msg := buildNoDataMessage(cfg.DataType, cfg.SilenceMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Error Count Threshold ---

type errorCountConfig struct {
	ThresholdCount  int64 `json:"thresholdCount"`
	LookbackMinutes int   `json:"lookbackMinutes"`
}

func evaluateErrorCountThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg errorCountConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid error_count_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 60
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	var count int64
	err := db.TelemetryDB.QueryRowContext(ctx,
		"SELECT COUNT(*) FROM exceptions_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_message = 0",
		projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&count)
	if err != nil {
		return nil, err
	}

	if count < cfg.ThresholdCount {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildErrorCountMessage(count, cfg.ThresholdCount, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Task Duration Threshold ---

type taskDurationConfig struct {
	TaskName        string  `json:"taskName"`
	ThresholdMs     float64 `json:"thresholdMs"`
	LookbackMinutes int     `json:"lookbackMinutes"`
}

func evaluateTaskDurationThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg taskDurationConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid task_duration_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 30
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	query := "SELECT duration FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)}
	if cfg.TaskName != "" && cfg.TaskName != "*" {
		query += " AND task_name = ?"
		args = append(args, cfg.TaskName)
	}
	query += " ORDER BY duration ASC"

	rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var durations []float64
	for rows.Next() {
		var d int64
		if err := rows.Scan(&d); err != nil {
			continue
		}
		durations = append(durations, float64(d)/1000000) // ns to ms
	}

	p95 := computePercentile(durations, 0.95)

	if p95 < cfg.ThresholdMs {
		return &EvalResult{Fired: false}, nil
	}

	taskName := cfg.TaskName
	if taskName == "" || taskName == "*" {
		taskName = "all tasks"
	}
	projectName := getProjectName(projectId)
	msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Task Failure Rate ---

func countTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
	query := "SELECT COUNT(*) FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId.String(), from.Format(time.RFC3339Nano), to.Format(time.RFC3339Nano)}
	if named {
		query += " AND task_name = ?"
		args = append(args, taskName)
	}

	var total int64
	if err := db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&total); err != nil {
		return 0, err
	}
	return total, nil
}

func countFailedTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
	pid := projectId.String()
	fromStr := from.Format(time.RFC3339Nano)
	toStr := to.Format(time.RFC3339Nano)

	// A task failed when an exception was recorded on its own span. One caught further down did not fail it.
	query := "SELECT COUNT(*) FROM (SELECT DISTINCT trace_id, span_id FROM exceptions_v2 WHERE project_id = ? AND trace_type = 'task' AND recorded_at >= ? AND recorded_at <= ?" +
		" AND (trace_id, span_id) IN (SELECT trace_id, span_id FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{pid, fromStr, toStr, pid, fromStr, toStr}
	if named {
		query += " AND task_name = ?"
		args = append(args, taskName)
	}
	query += ")) AS failed_tasks"

	var failed int64
	if err := db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&failed); err != nil {
		return 0, err
	}
	return failed, nil
}

// --- Throughput Drop ---

type throughputDropConfig struct {
	DropPercent           float64 `json:"dropPercent"`
	LookbackMinutes       int     `json:"lookbackMinutes"`
	BaselineWindowMinutes int     `json:"baselineWindowMinutes"`
}

func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg throughputDropConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid throughput_drop config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 15
	}
	if cfg.BaselineWindowMinutes <= 0 {
		cfg.BaselineWindowMinutes = 60
	}

	now := time.Now().UTC()
	lookbackFrom := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
	baselineFrom := lookbackFrom.Add(-time.Duration(cfg.BaselineWindowMinutes) * time.Minute)
	pid := projectId.String()

	var currentCount int64
	err := db.TelemetryDB.QueryRowContext(ctx,
		"SELECT COUNT(*) FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
		pid, lookbackFrom.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&currentCount)
	if err != nil {
		return nil, err
	}

	var baselineCount int64
	err = db.TelemetryDB.QueryRowContext(ctx,
		"SELECT COUNT(*) FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
		pid, baselineFrom.Format(time.RFC3339Nano), lookbackFrom.Format(time.RFC3339Nano)).Scan(&baselineCount)
	if err != nil {
		return nil, err
	}

	if baselineCount == 0 {
		return &EvalResult{Fired: false}, nil
	}

	normalizedBaseline := float64(baselineCount) * float64(cfg.LookbackMinutes) / float64(cfg.BaselineWindowMinutes)
	if normalizedBaseline == 0 {
		return &EvalResult{Fired: false}, nil
	}

	dropPercent := (1 - float64(currentCount)/normalizedBaseline) * 100
	if dropPercent < cfg.DropPercent {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildThroughputDropMessage(dropPercent, currentCount, normalizedBaseline, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Endpoint Error Rate ---

type endpointErrorRateConfig struct {
	Endpoint         string  `json:"endpoint"`
	ThresholdPercent float64 `json:"thresholdPercent"`
	LookbackMinutes  int     `json:"lookbackMinutes"`
	MinRequests      int     `json:"minRequests"`
}

func evaluateEndpointErrorRate(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg endpointErrorRateConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid endpoint_error_rate config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 10
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
	pid := projectId.String()

	var total, errors int64
	err := db.TelemetryDB.QueryRowContext(ctx,
		"SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN status_code >= 500 THEN 1 ELSE 0 END), 0) as errors FROM endpoints_v2 WHERE project_id = ? AND endpoint = ? AND recorded_at >= ? AND recorded_at <= ?",
		pid, cfg.Endpoint, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &errors)
	if err != nil {
		return nil, err
	}

	if total < int64(cfg.MinRequests) {
		return &EvalResult{Fired: false}, nil
	}

	rate := float64(errors) / float64(total) * 100
	if rate < cfg.ThresholdPercent {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildEndpointErrorRateMessage(cfg.Endpoint, rate, cfg.ThresholdPercent, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Impact Score ---

func computeImpactEndpoints(ctx context.Context, projectId uuid.UUID, minRequests int) ([]impactEndpointData, error) {
	now := time.Now().UTC()
	from := now.Add(-24 * time.Hour)
	pid := projectId.String()
	fromStr := from.Format(time.RFC3339Nano)
	nowStr := now.Format(time.RFC3339Nano)

	rows, err := db.TelemetryDB.QueryContext(ctx, `SELECT
		e.endpoint,
		COUNT(*) as total_count,
		COALESCE(s.offset_ms, 0) as offset_ms,
		COALESCE(SUM(CASE WHEN e.duration <= (750000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as satisfied_count,
		COALESCE(SUM(CASE WHEN e.duration > (750000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.duration <= (1500000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as tolerating_count,
		COALESCE(SUM(CASE WHEN e.duration > (1500000000 + COALESCE(s.offset_ms, 0) * 1000000) OR e.status_code >= 500 THEN 1 ELSE 0 END), 0) as bad_count,
		COALESCE(SUM(CASE WHEN e.status_code >= 400 AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as client_error_count
	FROM endpoints_v2 e
	LEFT JOIN slow_endpoints s ON e.endpoint = s.endpoint AND e.project_id = s.project_id
	WHERE e.project_id = ? AND e.recorded_at >= ? AND e.recorded_at <= ? AND e.is_stream = 0
	GROUP BY e.endpoint, COALESCE(s.offset_ms, 0)`,
		pid, fromStr, nowStr)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var candidates []impactEndpointData
	for rows.Next() {
		var ep string
		var totalCount, satisfied, tolerating, bad, clientErrors int64
		var offsetMs int64
		if err := rows.Scan(&ep, &totalCount, &offsetMs, &satisfied, &tolerating, &bad, &clientErrors); err != nil {
			return nil, err
		}

		if totalCount < int64(minRequests) {
			continue
		}

		candidates = append(candidates, impactEndpointData{
			endpoint:     ep,
			totalCount:   uint64(totalCount),
			offsetMs:     uint32(offsetMs),
			satisfied:    uint64(satisfied),
			tolerating:   uint64(tolerating),
			bad:          uint64(bad),
			clientErrors: uint64(clientErrors),
		})
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	if len(candidates) == 0 {
		return nil, nil
	}

	p99Query := `SELECT endpoint, duration FROM (
		SELECT endpoint, duration,
			ROW_NUMBER() OVER (PARTITION BY endpoint ORDER BY duration) AS rn,
			COUNT(*) OVER (PARTITION BY endpoint) AS cnt
		FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_stream = 0
	) WHERE rn = CAST(0.99 * (cnt - 1) AS INTEGER) + 1`
	if db.IsDuckDBTelemetry() {
		p99Query = `SELECT endpoint, CAST(quantile_cont(duration, 0.99) AS BIGINT) as duration
		FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_stream = 0
		GROUP BY endpoint`
	}

	p99s := make(map[string]float64, len(candidates))
	pRows, err := db.TelemetryDB.QueryContext(ctx, p99Query, pid, fromStr, nowStr)
	if err != nil {
		return nil, err
	}
	defer pRows.Close()

	for pRows.Next() {
		var ep string
		var d int64
		if err := pRows.Scan(&ep, &d); err != nil {
			return nil, err
		}
		p99s[ep] = float64(d)
	}
	if err := pRows.Err(); err != nil {
		return nil, err
	}

	var result []impactEndpointData
	for _, c := range candidates {
		c.p99 = p99s[c.endpoint]

		impact := telemetry.ComputeImpactScore(c.endpoint, c.totalCount, c.satisfied, c.tolerating, c.bad, c.clientErrors, c.p99, c.offsetMs)
		if impact >= minImpactThreshold {
			c.impact = impact
			result = append(result, c)
		}
	}

	return result, nil
}