The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
//go:build !telemetry_ch
package notifications
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"maps"
"slices"
"time"
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/repositories/telemetry"
)
type EvalResult struct {
Fired bool
Recovered bool
Message Message
Messages []Message
}
type RuleEvaluator func(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error)
var polledEvaluators = map[string]RuleEvaluator{
"error_rate_threshold": evaluateErrorRateThreshold,
"endpoint_p95_threshold": evaluateEndpointP95Threshold,
"endpoint_p99_threshold": evaluateEndpointP99Threshold,
"apdex_drop": evaluateApdexDrop,
"metric_threshold": evaluateMetricThreshold,
"no_data": evaluateNoData,
"error_count_threshold": evaluateErrorCountThreshold,
"task_duration_threshold": evaluateTaskDurationThreshold,
"task_failure_rate": evaluateTaskFailureRate,
"throughput_drop": evaluateThroughputDrop,
"endpoint_error_rate": evaluateEndpointErrorRate,
"impact_score_critical": evaluateImpactScoreCritical,
"impact_score_high": evaluateImpactScoreHigh,
"impact_score_medium": evaluateImpactScoreMedium,
"slo_burn_rate": evaluateSLOBurnRate,
"slo_budget_consumed": evaluateSLOBudgetConsumed,
}
// --- Error Rate Threshold ---
type errorRateConfig struct {
ThresholdPercent float64 `json:"thresholdPercent"`
LookbackMinutes int `json:"lookbackMinutes"`
MinRequests int `json:"minRequests"`
}
func evaluateErrorRateThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg errorRateConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid error_rate_threshold config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 5
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
var total, errors int64
err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN status_code >= 500 THEN 1 ELSE 0 END), 0) as errors FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &errors)
if err != nil {
return nil, err
}
if total < int64(cfg.MinRequests) {
return &EvalResult{Fired: false}, nil
}
rate := float64(errors) / float64(total) * 100
if rate < cfg.ThresholdPercent {
return &EvalResult{Fired: false}, nil
}
projectName := getProjectName(projectId)
msg := buildErrorRateMessage(rate, cfg.ThresholdPercent, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Endpoint P95 Threshold ---
type endpointLatencyConfig struct {
Endpoint string `json:"endpoint"`
ThresholdMs float64 `json:"thresholdMs"`
LookbackMinutes int `json:"lookbackMinutes"`
}
func evaluateEndpointP95Threshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg endpointLatencyConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid endpoint_p95_threshold config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 5
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
p95 := queryPercentile(ctx, projectId, cfg.Endpoint, from, now, 0.95)
if p95 < cfg.ThresholdMs {
return &EvalResult{Fired: false}, nil
}
endpoint := cfg.Endpoint
if endpoint == "" || endpoint == "*" {
endpoint = "all endpoints"
}
projectName := getProjectName(projectId)
msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Endpoint P99 Threshold ---
func evaluateEndpointP99Threshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg endpointLatencyConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid endpoint_p99_threshold config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 5
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
p99 := queryPercentile(ctx, projectId, cfg.Endpoint, from, now, 0.99)
if p99 < cfg.ThresholdMs {
return &EvalResult{Fired: false}, nil
}
endpoint := cfg.Endpoint
if endpoint == "" || endpoint == "*" {
endpoint = "all endpoints"
}
projectName := getProjectName(projectId)
msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
func queryPercentile(ctx context.Context, projectId uuid.UUID, endpoint string, from, to time.Time, pct float64) float64 {
query := "SELECT duration FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
args := []interface{}{projectId.String(), from.Format(time.RFC3339Nano), to.Format(time.RFC3339Nano)}
if endpoint != "" && endpoint != "*" {
query += " AND endpoint = ?"
args = append(args, endpoint)
}
query += " ORDER BY duration ASC"
rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
if err != nil {
return 0
}
defer rows.Close()
var durations []float64
for rows.Next() {
var d int64
if err := rows.Scan(&d); err != nil {
continue
}
durations = append(durations, float64(d)/1000000) // ns to ms
}
return computePercentile(durations, pct)
}
func computePercentile(sorted []float64, p float64) float64 {
n := len(sorted)
if n == 0 {
return 0
}
if n == 1 {
return sorted[0]
}
idx := p * float64(n-1)
lower := int(idx)
frac := idx - float64(lower)
if lower+1 >= n {
return sorted[lower]
}
return sorted[lower]*(1-frac) + sorted[lower+1]*frac
}
// --- Apdex Drop ---
type apdexConfig struct {
ThresholdApdex float64 `json:"thresholdApdex"`
LookbackMinutes int `json:"lookbackMinutes"`
MinRequests int `json:"minRequests"`
}
func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg apdexConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid apdex_drop config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 15
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
var total, satisfied, tolerating int64
err := db.TelemetryDB.QueryRowContext(ctx,
`SELECT COUNT(*) as total,
COALESCE(SUM(CASE WHEN duration <= 750000000 AND status_code < 500 THEN 1 ELSE 0 END), 0) as satisfied,
COALESCE(SUM(CASE WHEN duration > 750000000 AND duration <= 1500000000 AND status_code < 500 THEN 1 ELSE 0 END), 0) as tolerating
FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?`,
projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &satisfied, &tolerating)
if err != nil {
return nil, err
}
if total < int64(cfg.MinRequests) {
return &EvalResult{Fired: false}, nil
}
apdex := (float64(satisfied) + float64(tolerating)/2.0) / float64(total)
if apdex >= cfg.ThresholdApdex {
return &EvalResult{Fired: false}, nil
}
projectName := getProjectName(projectId)
msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, total, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Metric Threshold ---
func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
cfg, err := parseMetricThresholdConfig(rule.Config)
if err != nil {
return nil, err
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
tagExtract := "json_extract"
if db.IsDuckDBTelemetry() {
tagExtract = "json_extract_string"
}
groupExpr, args := metricGroupExpr(cfg.GroupBy, "COALESCE("+tagExtract+"(tags, '$.\"' || ? || '\"'), '')")
where := "project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?"
args = append(args, projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {
where += " AND " + tagExtract + "(tags, '$.\"' || ? || '\"') = ?"
args = append(args, k, cfg.Tags[k])
}
var series []metricSeries
switch cfg.Aggregation {
case "last", "p95", "p99":
order := "recorded_at DESC"
if cfg.Aggregation != "last" {
order = "value ASC"
}
series, err = scanMetricSeriesValues(ctx, cfg.Aggregation,
fmt.Sprintf("SELECT %s AS group_key, value FROM metric_points WHERE %s ORDER BY group_key, %s", groupExpr, where, order), args)
default:
aggFunc := "avg"
switch cfg.Aggregation {
case "max":
aggFunc = "max"
case "min":
aggFunc = "min"
case "sum":
aggFunc = "sum"
}
query := fmt.Sprintf("SELECT %s AS group_key, COALESCE(%s(value), 0) FROM metric_points WHERE %s GROUP BY group_key ORDER BY group_key LIMIT ?", groupExpr, aggFunc, where)
series, err = scanMetricSeries(ctx, query, append(args, maxMetricRuleSeriesScanned))
}
if err != nil {
return nil, err
}
return metricThresholdResult(rule, cfg, series, projectId), nil
}
func scanMetricSeries(ctx context.Context, query string, args []interface{}) ([]metricSeries, error) {
rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var series []metricSeries
for rows.Next() {
var s metricSeries
if err := rows.Scan(&s.Key, &s.Value); err != nil {
return nil, err
}
series = append(series, s)
}
return series, rows.Err()
}
// scanMetricSeriesValues reduces rows ordered by group key to one value per
// series: the first row for last (newest first), a percentile otherwise
// (values ascending).
func scanMetricSeriesValues(ctx context.Context, aggregation string, query string, args []interface{}) ([]metricSeries, error) {
rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
pct := 0.95
if aggregation == "p99" {
pct = 0.99
}
var series []metricSeries
var vals []float64
flush := func(key string) {
if len(vals) == 0 {
return
}
value := vals[0]
if aggregation != "last" {
value = computePercentile(vals, pct)
}
series = append(series, metricSeries{Key: key, Value: value})
vals = vals[:0]
}
current := ""
for rows.Next() {
var key string
var v float64
if err := rows.Scan(&key, &v); err != nil {
return nil, err
}
if key != current || len(vals) == 0 {
flush(current)
if len(series) >= maxMetricRuleSeriesScanned {
return series, nil
}
current = key
}
vals = append(vals, v)
}
if err := rows.Err(); err != nil {
return nil, err
}
flush(current)
return series, nil
}
// --- No Data ---
type noDataConfig struct {
DataType string `json:"dataType"`
SilenceMinutes int `json:"silenceMinutes"`
}
func evaluateNoData(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg noDataConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid no_data config: %w", err)
}
if cfg.SilenceMinutes <= 0 {
cfg.SilenceMinutes = 10
}
threshold := time.Now().UTC().Add(-time.Duration(cfg.SilenceMinutes) * time.Minute)
pid := projectId.String()
if cfg.DataType == "any" {
tables := []string{"endpoints_v2", "exceptions_v2", "metric_points", "tasks_v2"}
for _, t := range tables {
var maxTs sql.NullString
err := db.TelemetryDB.QueryRowContext(ctx,
fmt.Sprintf("SELECT MAX(recorded_at) FROM %s WHERE project_id = ?", t),
pid).Scan(&maxTs)
if err == nil && maxTs.Valid && maxTs.String != "" {
if parsed, pErr := time.Parse(time.RFC3339Nano, maxTs.String); pErr == nil && parsed.After(threshold) {
return &EvalResult{Fired: false}, nil
}
}
}
projectName := getProjectName(projectId)
msg := buildNoDataMessage("any", cfg.SilenceMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
table := ""
switch cfg.DataType {
case "endpoints":
table = "endpoints_v2"
case "exceptions":
table = "exceptions_v2"
case "metrics":
table = "metric_points"
case "tasks":
table = "tasks_v2"
default:
return nil, fmt.Errorf("unknown data type: %s", cfg.DataType)
}
var maxTs sql.NullString
err := db.TelemetryDB.QueryRowContext(ctx,
fmt.Sprintf("SELECT MAX(recorded_at) FROM %s WHERE project_id = ?", table),
pid).Scan(&maxTs)
if err != nil {
return nil, err
}
if maxTs.Valid && maxTs.String != "" {
if parsed, pErr := time.Parse(time.RFC3339Nano, maxTs.String); pErr == nil && parsed.After(threshold) {
return &EvalResult{Fired: false}, nil
}
}
projectName := getProjectName(projectId)
msg := buildNoDataMessage(cfg.DataType, cfg.SilenceMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Error Count Threshold ---
type errorCountConfig struct {
ThresholdCount int64 `json:"thresholdCount"`
LookbackMinutes int `json:"lookbackMinutes"`
}
func evaluateErrorCountThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg errorCountConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid error_count_threshold config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 60
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
var count int64
err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM exceptions_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_message = 0",
projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&count)
if err != nil {
return nil, err
}
if count < cfg.ThresholdCount {
return &EvalResult{Fired: false}, nil
}
projectName := getProjectName(projectId)
msg := buildErrorCountMessage(count, cfg.ThresholdCount, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Task Duration Threshold ---
type taskDurationConfig struct {
TaskName string `json:"taskName"`
ThresholdMs float64 `json:"thresholdMs"`
LookbackMinutes int `json:"lookbackMinutes"`
}
func evaluateTaskDurationThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg taskDurationConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid task_duration_threshold config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 30
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
query := "SELECT duration FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
args := []interface{}{projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)}
if cfg.TaskName != "" && cfg.TaskName != "*" {
query += " AND task_name = ?"
args = append(args, cfg.TaskName)
}
query += " ORDER BY duration ASC"
rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var durations []float64
for rows.Next() {
var d int64
if err := rows.Scan(&d); err != nil {
continue
}
durations = append(durations, float64(d)/1000000) // ns to ms
}
p95 := computePercentile(durations, 0.95)
if p95 < cfg.ThresholdMs {
return &EvalResult{Fired: false}, nil
}
taskName := cfg.TaskName
if taskName == "" || taskName == "*" {
taskName = "all tasks"
}
projectName := getProjectName(projectId)
msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Task Failure Rate ---
func countTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
query := "SELECT COUNT(*) FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
args := []interface{}{projectId.String(), from.Format(time.RFC3339Nano), to.Format(time.RFC3339Nano)}
if named {
query += " AND task_name = ?"
args = append(args, taskName)
}
var total int64
if err := db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&total); err != nil {
return 0, err
}
return total, nil
}
func countFailedTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
pid := projectId.String()
fromStr := from.Format(time.RFC3339Nano)
toStr := to.Format(time.RFC3339Nano)
// A task failed when an exception was recorded on its own span. One caught further down did not fail it.
query := "SELECT COUNT(*) FROM (SELECT DISTINCT trace_id, span_id FROM exceptions_v2 WHERE project_id = ? AND trace_type = 'task' AND recorded_at >= ? AND recorded_at <= ?" +
" AND (trace_id, span_id) IN (SELECT trace_id, span_id FROM tasks_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
args := []interface{}{pid, fromStr, toStr, pid, fromStr, toStr}
if named {
query += " AND task_name = ?"
args = append(args, taskName)
}
query += ")) AS failed_tasks"
var failed int64
if err := db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&failed); err != nil {
return 0, err
}
return failed, nil
}
// --- Throughput Drop ---
type throughputDropConfig struct {
DropPercent float64 `json:"dropPercent"`
LookbackMinutes int `json:"lookbackMinutes"`
BaselineWindowMinutes int `json:"baselineWindowMinutes"`
}
func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg throughputDropConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid throughput_drop config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 15
}
if cfg.BaselineWindowMinutes <= 0 {
cfg.BaselineWindowMinutes = 60
}
now := time.Now().UTC()
lookbackFrom := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
baselineFrom := lookbackFrom.Add(-time.Duration(cfg.BaselineWindowMinutes) * time.Minute)
pid := projectId.String()
var currentCount int64
err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
pid, lookbackFrom.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(¤tCount)
if err != nil {
return nil, err
}
var baselineCount int64
err = db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
pid, baselineFrom.Format(time.RFC3339Nano), lookbackFrom.Format(time.RFC3339Nano)).Scan(&baselineCount)
if err != nil {
return nil, err
}
if baselineCount == 0 {
return &EvalResult{Fired: false}, nil
}
normalizedBaseline := float64(baselineCount) * float64(cfg.LookbackMinutes) / float64(cfg.BaselineWindowMinutes)
if normalizedBaseline == 0 {
return &EvalResult{Fired: false}, nil
}
dropPercent := (1 - float64(currentCount)/normalizedBaseline) * 100
if dropPercent < cfg.DropPercent {
return &EvalResult{Fired: false}, nil
}
projectName := getProjectName(projectId)
msg := buildThroughputDropMessage(dropPercent, currentCount, normalizedBaseline, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Endpoint Error Rate ---
type endpointErrorRateConfig struct {
Endpoint string `json:"endpoint"`
ThresholdPercent float64 `json:"thresholdPercent"`
LookbackMinutes int `json:"lookbackMinutes"`
MinRequests int `json:"minRequests"`
}
func evaluateEndpointErrorRate(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
var cfg endpointErrorRateConfig
if err := json.Unmarshal(rule.Config, &cfg); err != nil {
return nil, fmt.Errorf("invalid endpoint_error_rate config: %w", err)
}
if cfg.LookbackMinutes <= 0 {
cfg.LookbackMinutes = 10
}
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
pid := projectId.String()
var total, errors int64
err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN status_code >= 500 THEN 1 ELSE 0 END), 0) as errors FROM endpoints_v2 WHERE project_id = ? AND endpoint = ? AND recorded_at >= ? AND recorded_at <= ?",
pid, cfg.Endpoint, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &errors)
if err != nil {
return nil, err
}
if total < int64(cfg.MinRequests) {
return &EvalResult{Fired: false}, nil
}
rate := float64(errors) / float64(total) * 100
if rate < cfg.ThresholdPercent {
return &EvalResult{Fired: false}, nil
}
projectName := getProjectName(projectId)
msg := buildEndpointErrorRateMessage(cfg.Endpoint, rate, cfg.ThresholdPercent, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
// --- Impact Score ---
func computeImpactEndpoints(ctx context.Context, projectId uuid.UUID, minRequests int) ([]impactEndpointData, error) {
now := time.Now().UTC()
from := now.Add(-24 * time.Hour)
pid := projectId.String()
fromStr := from.Format(time.RFC3339Nano)
nowStr := now.Format(time.RFC3339Nano)
rows, err := db.TelemetryDB.QueryContext(ctx, `SELECT
e.endpoint,
COUNT(*) as total_count,
COALESCE(s.offset_ms, 0) as offset_ms,
COALESCE(SUM(CASE WHEN e.duration <= (750000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as satisfied_count,
COALESCE(SUM(CASE WHEN e.duration > (750000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.duration <= (1500000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as tolerating_count,
COALESCE(SUM(CASE WHEN e.duration > (1500000000 + COALESCE(s.offset_ms, 0) * 1000000) OR e.status_code >= 500 THEN 1 ELSE 0 END), 0) as bad_count,
COALESCE(SUM(CASE WHEN e.status_code >= 400 AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as client_error_count
FROM endpoints_v2 e
LEFT JOIN slow_endpoints s ON e.endpoint = s.endpoint AND e.project_id = s.project_id
WHERE e.project_id = ? AND e.recorded_at >= ? AND e.recorded_at <= ? AND e.is_stream = 0
GROUP BY e.endpoint, COALESCE(s.offset_ms, 0)`,
pid, fromStr, nowStr)
if err != nil {
return nil, err
}
defer rows.Close()
var candidates []impactEndpointData
for rows.Next() {
var ep string
var totalCount, satisfied, tolerating, bad, clientErrors int64
var offsetMs int64
if err := rows.Scan(&ep, &totalCount, &offsetMs, &satisfied, &tolerating, &bad, &clientErrors); err != nil {
return nil, err
}
if totalCount < int64(minRequests) {
continue
}
candidates = append(candidates, impactEndpointData{
endpoint: ep,
totalCount: uint64(totalCount),
offsetMs: uint32(offsetMs),
satisfied: uint64(satisfied),
tolerating: uint64(tolerating),
bad: uint64(bad),
clientErrors: uint64(clientErrors),
})
}
if err := rows.Err(); err != nil {
return nil, err
}
if len(candidates) == 0 {
return nil, nil
}
p99Query := `SELECT endpoint, duration FROM (
SELECT endpoint, duration,
ROW_NUMBER() OVER (PARTITION BY endpoint ORDER BY duration) AS rn,
COUNT(*) OVER (PARTITION BY endpoint) AS cnt
FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_stream = 0
) WHERE rn = CAST(0.99 * (cnt - 1) AS INTEGER) + 1`
if db.IsDuckDBTelemetry() {
p99Query = `SELECT endpoint, CAST(quantile_cont(duration, 0.99) AS BIGINT) as duration
FROM endpoints_v2 WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_stream = 0
GROUP BY endpoint`
}
p99s := make(map[string]float64, len(candidates))
pRows, err := db.TelemetryDB.QueryContext(ctx, p99Query, pid, fromStr, nowStr)
if err != nil {
return nil, err
}
defer pRows.Close()
for pRows.Next() {
var ep string
var d int64
if err := pRows.Scan(&ep, &d); err != nil {
return nil, err
}
p99s[ep] = float64(d)
}
if err := pRows.Err(); err != nil {
return nil, err
}
var result []impactEndpointData
for _, c := range candidates {
c.p99 = p99s[c.endpoint]
impact := telemetry.ComputeImpactScore(c.endpoint, c.totalCount, c.satisfied, c.tolerating, c.bad, c.clientErrors, c.p99, c.offsetMs)
if impact >= minImpactThreshold {
c.impact = impact
result = append(result, c)
}
}
return result, nil
}