Skip to content

The only tool you need to know what is happening and how to fix it.

21.0 KB
Raw Blame
//go:build telemetry_ch

package notifications

import (
	"context"
	"encoding/json"
	"fmt"
	"maps"
	"slices"
	"time"

	"github.com/google/uuid"
	"github.com/tracewayapp/traceway/backend/app/chdb"
	"github.com/tracewayapp/traceway/backend/app/models"
)

type EvalResult struct {
	Fired     bool
	Recovered bool
	Message   Message
	Messages  []Message
}

type RuleEvaluator func(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error)

var polledEvaluators = map[string]RuleEvaluator{
	"error_rate_threshold":    evaluateErrorRateThreshold,
	"endpoint_p95_threshold":  evaluateEndpointP95Threshold,
	"endpoint_p99_threshold":  evaluateEndpointP99Threshold,
	"apdex_drop":              evaluateApdexDrop,
	"metric_threshold":        evaluateMetricThreshold,
	"no_data":                 evaluateNoData,
	"error_count_threshold":   evaluateErrorCountThreshold,
	"task_duration_threshold": evaluateTaskDurationThreshold,
	"task_failure_rate":       evaluateTaskFailureRate,
	"throughput_drop":         evaluateThroughputDrop,
	"endpoint_error_rate":     evaluateEndpointErrorRate,
	"impact_score_critical":   evaluateImpactScoreCritical,
	"impact_score_high":       evaluateImpactScoreHigh,
	"impact_score_medium":     evaluateImpactScoreMedium,
	"slo_burn_rate":           evaluateSLOBurnRate,
	"slo_budget_consumed":     evaluateSLOBudgetConsumed,
}

// --- Error Rate Threshold ---

type errorRateConfig struct {
	ThresholdPercent float64 `json:"thresholdPercent"`
	LookbackMinutes  int     `json:"lookbackMinutes"`
	MinRequests      int     `json:"minRequests"`
}

func evaluateErrorRateThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg errorRateConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid error_rate_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 5
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	var total, errors uint64
	err := chdb.Conn.QueryRow(ctx,
		"SELECT count() as total, countIf(status_code >= 500) as errors FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
		projectId, from, now).Scan(&total, &errors)
	if err != nil {
		return nil, err
	}

	if total < uint64(cfg.MinRequests) {
		return &EvalResult{Fired: false}, nil
	}

	rate := float64(errors) / float64(total) * 100
	if rate < cfg.ThresholdPercent {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildErrorRateMessage(rate, cfg.ThresholdPercent, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Endpoint P95 Threshold ---

type endpointLatencyConfig struct {
	Endpoint        string  `json:"endpoint"`
	ThresholdMs     float64 `json:"thresholdMs"`
	LookbackMinutes int     `json:"lookbackMinutes"`
}

func evaluateEndpointP95Threshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg endpointLatencyConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid endpoint_p95_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 5
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	// duration is stored in nanoseconds, convert to milliseconds
	query := "SELECT quantile(0.95)(duration / 1000000) as p95 FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId, from, now}
	if cfg.Endpoint != "" && cfg.Endpoint != "*" {
		query += " AND endpoint = ?"
		args = append(args, cfg.Endpoint)
	}

	var p95 float64
	err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&p95)
	if err != nil {
		return nil, err
	}

	if p95 < cfg.ThresholdMs {
		return &EvalResult{Fired: false}, nil
	}

	endpoint := cfg.Endpoint
	if endpoint == "" || endpoint == "*" {
		endpoint = "all endpoints"
	}
	projectName := getProjectName(projectId)
	msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Endpoint P99 Threshold ---

func evaluateEndpointP99Threshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg endpointLatencyConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid endpoint_p99_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 5
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	query := "SELECT quantile(0.99)(duration / 1000000) as p99 FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId, from, now}
	if cfg.Endpoint != "" && cfg.Endpoint != "*" {
		query += " AND endpoint = ?"
		args = append(args, cfg.Endpoint)
	}

	var p99 float64
	err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&p99)
	if err != nil {
		return nil, err
	}

	if p99 < cfg.ThresholdMs {
		return &EvalResult{Fired: false}, nil
	}

	endpoint := cfg.Endpoint
	if endpoint == "" || endpoint == "*" {
		endpoint = "all endpoints"
	}
	projectName := getProjectName(projectId)
	msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Apdex Drop ---

type apdexConfig struct {
	ThresholdApdex  float64 `json:"thresholdApdex"`
	LookbackMinutes int     `json:"lookbackMinutes"`
	MinRequests     int     `json:"minRequests"`
}

func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg apdexConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid apdex_drop config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 15
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	// Apdex thresholds: Good <= 750ms (750000000ns), Tolerable <= 1500ms (1500000000ns)
	var total, satisfied, tolerating uint64
	err := chdb.Conn.QueryRow(ctx,
		`SELECT count() as total,
			countIf(duration <= 750000000 AND status_code < 500) as satisfied,
			countIf(duration > 750000000 AND duration <= 1500000000 AND status_code < 500) as tolerating
		FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?`,
		projectId, from, now).Scan(&total, &satisfied, &tolerating)
	if err != nil {
		return nil, err
	}

	if total < uint64(cfg.MinRequests) {
		return &EvalResult{Fired: false}, nil
	}

	apdex := (float64(satisfied) + float64(tolerating)/2.0) / float64(total)
	if apdex >= cfg.ThresholdApdex {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, int64(total), cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Metric Threshold ---

func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	cfg, err := parseMetricThresholdConfig(rule.Config)
	if err != nil {
		return nil, err
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	selectExpr := "avg(value)"
	switch cfg.Aggregation {
	case "max":
		selectExpr = "max(value)"
	case "min":
		selectExpr = "min(value)"
	case "sum":
		selectExpr = "sum(value)"
	case "p95":
		selectExpr = "quantile(0.95)(value)"
	case "p99":
		selectExpr = "quantile(0.99)(value)"
	case "last":
		selectExpr = "argMax(value, recorded_at)"
	}

	groupExpr, args := metricGroupExpr(cfg.GroupBy, "tags[?]")
	query := fmt.Sprintf("SELECT %s AS group_key, %s FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", groupExpr, selectExpr)
	args = append(args, projectId, cfg.MetricName, from, now)
	for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {
		query += " AND tags[?] = ?"
		args = append(args, k, cfg.Tags[k])
	}
	query += " GROUP BY group_key ORDER BY group_key LIMIT ?"
	args = append(args, maxMetricRuleSeriesScanned)

	rows, err := chdb.Conn.Query(ctx, query, args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var series []metricSeries
	for rows.Next() {
		var s metricSeries
		if err := rows.Scan(&s.Key, &s.Value); err != nil {
			return nil, err
		}
		series = append(series, s)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return metricThresholdResult(rule, cfg, series, projectId), nil
}

// --- No Data ---

type noDataConfig struct {
	DataType       string `json:"dataType"`
	SilenceMinutes int    `json:"silenceMinutes"`
}

func evaluateNoData(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg noDataConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid no_data config: %w", err)
	}
	if cfg.SilenceMinutes <= 0 {
		cfg.SilenceMinutes = 10
	}

	threshold := time.Now().UTC().Add(-time.Duration(cfg.SilenceMinutes) * time.Minute)

	if cfg.DataType == "any" {
		tables := []string{"endpoints_v2", "exceptions_v2", "metric_points", "tasks_v2"}
		for _, t := range tables {
			var maxTs time.Time
			err := chdb.Conn.QueryRow(ctx,
				fmt.Sprintf("SELECT max(recorded_at) FROM %s WHERE project_id = ?", t),
				projectId).Scan(&maxTs)
			if err == nil && maxTs.After(threshold) {
				return &EvalResult{Fired: false}, nil
			}
		}
		projectName := getProjectName(projectId)
		msg := buildNoDataMessage("any", cfg.SilenceMinutes, projectName)
		return &EvalResult{Fired: true, Message: msg}, nil
	}

	table := ""
	switch cfg.DataType {
	case "endpoints":
		table = "endpoints_v2"
	case "exceptions":
		table = "exceptions_v2"
	case "metrics":
		table = "metric_points"
	case "tasks":
		table = "tasks_v2"
	default:
		return nil, fmt.Errorf("unknown data type: %s", cfg.DataType)
	}

	var maxTs time.Time
	err := chdb.Conn.QueryRow(ctx,
		fmt.Sprintf("SELECT max(recorded_at) FROM %s WHERE project_id = ?", table),
		projectId).Scan(&maxTs)
	if err != nil {
		return nil, err
	}

	if maxTs.After(threshold) {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildNoDataMessage(cfg.DataType, cfg.SilenceMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Error Count Threshold ---

type errorCountConfig struct {
	ThresholdCount  int64 `json:"thresholdCount"`
	LookbackMinutes int   `json:"lookbackMinutes"`
}

func evaluateErrorCountThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg errorCountConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid error_count_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 60
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	var count uint64
	err := chdb.Conn.QueryRow(ctx,
		"SELECT count() FROM exceptions_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_message = 0",
		projectId, from, now).Scan(&count)
	if err != nil {
		return nil, err
	}

	if int64(count) < cfg.ThresholdCount {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildErrorCountMessage(int64(count), cfg.ThresholdCount, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Task Duration Threshold ---

type taskDurationConfig struct {
	TaskName        string  `json:"taskName"`
	ThresholdMs     float64 `json:"thresholdMs"`
	LookbackMinutes int     `json:"lookbackMinutes"`
}

func evaluateTaskDurationThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg taskDurationConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid task_duration_threshold config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 30
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	// duration is stored in nanoseconds, convert to milliseconds
	query := "SELECT quantile(0.95)(duration / 1000000) as p95 FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId, from, now}
	if cfg.TaskName != "" && cfg.TaskName != "*" {
		query += " AND task_name = ?"
		args = append(args, cfg.TaskName)
	}

	var p95 float64
	err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&p95)
	if err != nil {
		return nil, err
	}

	if p95 < cfg.ThresholdMs {
		return &EvalResult{Fired: false}, nil
	}

	taskName := cfg.TaskName
	if taskName == "" || taskName == "*" {
		taskName = "all tasks"
	}
	projectName := getProjectName(projectId)
	msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Task Failure Rate ---

func countTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
	query := "SELECT count() FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId, from, to}
	if named {
		query += " AND task_name = ?"
		args = append(args, taskName)
	}

	var total uint64
	if err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&total); err != nil {
		return 0, err
	}
	return int64(total), nil
}

func countFailedTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
	// A task failed when an exception was recorded on its own span. One caught further down did not fail it.
	query := "SELECT count() FROM (SELECT DISTINCT trace_id, span_id FROM exceptions_v2 WHERE project_id = ? AND trace_type = 'task' AND recorded_at >= ? AND recorded_at <= ?" +
		" AND (trace_id, span_id) IN (SELECT trace_id, span_id FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
	args := []interface{}{projectId, from, to, projectId, from, to}
	if named {
		query += " AND task_name = ?"
		args = append(args, taskName)
	}
	query += ")) AS failed_tasks"

	var failed uint64
	if err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&failed); err != nil {
		return 0, err
	}
	return int64(failed), nil
}

// --- Throughput Drop ---

type throughputDropConfig struct {
	DropPercent           float64 `json:"dropPercent"`
	LookbackMinutes       int     `json:"lookbackMinutes"`
	BaselineWindowMinutes int     `json:"baselineWindowMinutes"`
}

func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg throughputDropConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid throughput_drop config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 15
	}
	if cfg.BaselineWindowMinutes <= 0 {
		cfg.BaselineWindowMinutes = 60
	}

	now := time.Now().UTC()
	lookbackFrom := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
	baselineFrom := lookbackFrom.Add(-time.Duration(cfg.BaselineWindowMinutes) * time.Minute)

	var currentCount uint64
	err := chdb.Conn.QueryRow(ctx,
		"SELECT count() FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
		projectId, lookbackFrom, now).Scan(&currentCount)
	if err != nil {
		return nil, err
	}

	var baselineCount uint64
	err = chdb.Conn.QueryRow(ctx,
		"SELECT count() FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
		projectId, baselineFrom, lookbackFrom).Scan(&baselineCount)
	if err != nil {
		return nil, err
	}

	if baselineCount == 0 {
		return &EvalResult{Fired: false}, nil
	}

	normalizedBaseline := float64(baselineCount) * float64(cfg.LookbackMinutes) / float64(cfg.BaselineWindowMinutes)
	if normalizedBaseline == 0 {
		return &EvalResult{Fired: false}, nil
	}

	dropPercent := (1 - float64(currentCount)/normalizedBaseline) * 100
	if dropPercent < cfg.DropPercent {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildThroughputDropMessage(dropPercent, int64(currentCount), normalizedBaseline, cfg.LookbackMinutes, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Endpoint Error Rate ---

type endpointErrorRateConfig struct {
	Endpoint         string  `json:"endpoint"`
	ThresholdPercent float64 `json:"thresholdPercent"`
	LookbackMinutes  int     `json:"lookbackMinutes"`
	MinRequests      int     `json:"minRequests"`
}

func evaluateEndpointErrorRate(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
	var cfg endpointErrorRateConfig
	if err := json.Unmarshal(rule.Config, &cfg); err != nil {
		return nil, fmt.Errorf("invalid endpoint_error_rate config: %w", err)
	}
	if cfg.LookbackMinutes <= 0 {
		cfg.LookbackMinutes = 10
	}

	now := time.Now().UTC()
	from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)

	var total, errors uint64
	err := chdb.Conn.QueryRow(ctx,
		"SELECT count() as total, countIf(status_code >= 500) as errors FROM endpoints_v2 WHERE project_id = ? AND endpoint = ? AND recorded_at >= ? AND recorded_at <= ?",
		projectId, cfg.Endpoint, from, now).Scan(&total, &errors)
	if err != nil {
		return nil, err
	}

	if total < uint64(cfg.MinRequests) {
		return &EvalResult{Fired: false}, nil
	}

	rate := float64(errors) / float64(total) * 100
	if rate < cfg.ThresholdPercent {
		return &EvalResult{Fired: false}, nil
	}

	projectName := getProjectName(projectId)
	msg := buildEndpointErrorRateMessage(cfg.Endpoint, rate, cfg.ThresholdPercent, projectName)
	return &EvalResult{Fired: true, Message: msg}, nil
}

// --- Impact Score ---

func computeImpactEndpoints(ctx context.Context, projectId uuid.UUID, minRequests int) ([]impactEndpointData, error) {
	now := time.Now().UTC()
	from := now.Add(-24 * time.Hour)

	query := `SELECT
		endpoint, total_count, p99_duration, offset_ms,
		satisfied_count, tolerating_count, bad_count, client_error_count,
		greatest(
			if(total_count > 0,
				1.0 - ((satisfied_count + tolerating_count * 0.5) / total_count), 0.0),
			multiIf(
				bad_count / total_count > 0.33, 0.75,
				bad_count / total_count > 0.20, 0.50,
				bad_count / total_count > 0.10, 0.25, 0.0),
			multiIf(
				toFloat64(p99_duration) - toFloat64(offset_ms) * 1000000 > 8000000000, 0.75,
				toFloat64(p99_duration) - toFloat64(offset_ms) * 1000000 > 6000000000, 0.50,
				toFloat64(p99_duration) - toFloat64(offset_ms) * 1000000 > 3000000000, 0.25, 0.0),
			if(endpoint != 'UNMATCHED' AND total_count > 10,
				multiIf(
					client_error_count / total_count > 0.50, 0.75,
					client_error_count / total_count > 0.25, 0.50, 0.0),
				0.0),
			multiIf(
				bad_count / total_count > 0.10 AND bad_count >= 500, 0.75,
				bad_count / total_count > 0.10 AND bad_count >= 50, 0.50,
				bad_count / total_count > 0.05 AND bad_count >= 2000, 0.75,
				bad_count / total_count > 0.05 AND bad_count >= 500, 0.50,
				bad_count / total_count > 0.05 AND bad_count >= 50, 0.25,
				bad_count / total_count > 0.01 AND bad_count >= 10000, 0.75,
				bad_count / total_count > 0.01 AND bad_count >= 2000, 0.50,
				bad_count / total_count > 0.01 AND bad_count >= 500, 0.25,
				0.0)
		) as impact
	FROM (
		SELECT
			endpoint,
			offset_ms,
			count() as total_count,
			quantile(0.99)(duration) as p99_duration,
			countIf(duration <= (750000000 + toInt64(offset_ms) * 1000000)
				AND status_code < 500) as satisfied_count,
			countIf(duration > (750000000 + toInt64(offset_ms) * 1000000)
				AND duration <= (1500000000 + toInt64(offset_ms) * 1000000)
				AND status_code < 500) as tolerating_count,
			countIf(duration > (1500000000 + toInt64(offset_ms) * 1000000)
				OR status_code >= 500) as bad_count,
			countIf(status_code >= 400 AND status_code < 500) as client_error_count
		FROM (
			SELECT e.endpoint, e.duration, e.status_code, e.recorded_at,
				   s.offset_ms as offset_ms
			FROM endpoints_v2 e
			LEFT JOIN (SELECT * FROM slow_endpoints FINAL) AS s
				ON e.endpoint = s.endpoint AND e.project_id = s.project_id
			WHERE e.project_id = ? AND e.recorded_at >= ? AND e.recorded_at <= ? AND e.is_stream = 0
		)
		GROUP BY endpoint, offset_ms
	)
	WHERE impact >= ? AND total_count >= ?`

	rows, err := chdb.Conn.Query(ctx, query, projectId, from, now, minImpactThreshold, uint64(minRequests))
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var result []impactEndpointData
	for rows.Next() {
		var ep string
		var totalCount, satisfied, tolerating, bad, clientErrors uint64
		var p99 float64
		var offsetMs uint32
		var impact float64
		if err := rows.Scan(&ep, &totalCount, &p99, &offsetMs,
			&satisfied, &tolerating, &bad, &clientErrors, &impact); err != nil {
			return nil, err
		}
		result = append(result, impactEndpointData{
			endpoint:     ep,
			impact:       impact,
			totalCount:   totalCount,
			p99:          p99,
			offsetMs:     offsetMs,
			satisfied:    satisfied,
			tolerating:   tolerating,
			bad:          bad,
			clientErrors: clientErrors,
		})
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return result, nil
}