The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
//go:build !telemetry_ch
package notifications
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"path/filepath"
"sort"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/tracewayapp/lit/v2"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/models"
_ "modernc.org/sqlite"
)
func TestCountOccurrences(t *testing.T) {
result := countOccurrences([]string{"a", "b", "a", "c", "a", "b"})
if len(result) != 3 {
t.Fatalf("expected 3 distinct hashes, got %d", len(result))
}
if result["a"] != 3 {
t.Errorf("expected a counted 3 times, got %d", result["a"])
}
if result["b"] != 2 {
t.Errorf("expected b counted 2 times, got %d", result["b"])
}
if result["c"] != 1 {
t.Errorf("expected c counted 1 time, got %d", result["c"])
}
unique := countOccurrences([]string{"x", "y"})
if len(unique) != 2 || unique["x"] != 1 || unique["y"] != 1 {
t.Errorf("unique hashes not preserved: %v", unique)
}
empty := countOccurrences(nil)
if len(empty) != 0 {
t.Errorf("expected empty map for empty slice, got %v", empty)
}
}
func TestCooldownSeed(t *testing.T) {
tracker := &cooldownTracker{fired: make(map[int]time.Time)}
t1 := time.Now().Add(-2 * time.Hour)
t2 := time.Now().Add(-1 * time.Hour)
tracker.seed(map[int]time.Time{1: t1, 2: t2})
if got := tracker.fired[1]; !got.Equal(t1) {
t.Errorf("seed on empty tracker: fired[1] = %v, expected %v", got, t1)
}
if got := tracker.fired[2]; !got.Equal(t2) {
t.Errorf("seed on empty tracker: fired[2] = %v, expected %v", got, t2)
}
tracker.recordFire(3)
recorded := tracker.fired[3]
tracker.seed(map[int]time.Time{3: time.Now().Add(-3 * time.Hour)})
if got := tracker.fired[3]; !got.Equal(recorded) {
t.Errorf("seed overwrote newer recordFire time: got %v, expected %v", got, recorded)
}
newer := time.Now().Add(time.Hour)
tracker.seed(map[int]time.Time{1: newer})
if got := tracker.fired[1]; !got.Equal(newer) {
t.Errorf("seed did not update with newer time: got %v, expected %v", got, newer)
}
}
func TestWindowP99MatchesNearestRank(t *testing.T) {
dbPath := filepath.Join(t.TempDir(), "p99.db")
conn, err := sql.Open("sqlite", dbPath)
if err != nil {
t.Fatalf("failed to open sqlite: %v", err)
}
defer conn.Close()
conn.SetMaxOpenConns(1)
if _, err := conn.Exec(`CREATE TABLE endpoints (
endpoint TEXT NOT NULL,
duration INTEGER NOT NULL,
project_id TEXT NOT NULL,
recorded_at DATETIME NOT NULL
)`); err != nil {
t.Fatalf("failed to create endpoints table: %v", err)
}
projectId := uuid.New().String()
now := time.Now().UTC()
from := now.Add(-time.Hour)
ts := now.Add(-30 * time.Minute).Format(time.RFC3339Nano)
durations := map[string][]int64{}
for i := 0; i < 100; i++ {
j := (i * 37) % 100
durations["GET /alpha"] = append(durations["GET /alpha"], int64(1000+j*10))
durations["GET /beta"] = append(durations["GET /beta"], int64(5000+j*37))
}
for ep, ds := range durations {
for _, d := range ds {
if _, err := conn.Exec(
"INSERT INTO endpoints (endpoint, duration, project_id, recorded_at) VALUES (?, ?, ?, ?)",
ep, d, projectId, ts); err != nil {
t.Fatalf("failed to insert row: %v", err)
}
}
}
rows, err := conn.Query(`SELECT endpoint, duration FROM (
SELECT endpoint, duration,
ROW_NUMBER() OVER (PARTITION BY endpoint ORDER BY duration) AS rn,
COUNT(*) OVER (PARTITION BY endpoint) AS cnt
FROM endpoints WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?
) WHERE rn = CAST(0.99 * (cnt - 1) AS INTEGER) + 1`,
projectId, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
if err != nil {
t.Fatalf("window query failed: %v", err)
}
defer rows.Close()
got := map[string]int64{}
for rows.Next() {
var ep string
var d int64
if err := rows.Scan(&ep, &d); err != nil {
t.Fatalf("failed to scan row: %v", err)
}
got[ep] = d
}
if err := rows.Err(); err != nil {
t.Fatalf("rows error: %v", err)
}
if len(got) != 2 {
t.Fatalf("expected p99 for 2 endpoints, got %d: %v", len(got), got)
}
for ep, ds := range durations {
sorted := append([]int64(nil), ds...)
sort.Slice(sorted, func(a, b int) bool { return sorted[a] < sorted[b] })
expected := sorted[int(0.99*float64(len(sorted)-1))]
if got[ep] != expected {
t.Errorf("endpoint %s: window p99 = %d, nearest-rank-lower = %d", ep, got[ep], expected)
}
}
}
func setupNotificationsTestDB(t *testing.T) {
t.Helper()
prevDB := db.DB
prevTelemetryDB := db.TelemetryDB
prevDriver := db.Driver
t.Cleanup(func() {
db.DB = prevDB
db.TelemetryDB = prevTelemetryDB
db.Driver = prevDriver
})
dir := t.TempDir()
mainDB, err := sql.Open("sqlite", filepath.Join(dir, "main.db"))
if err != nil {
t.Fatalf("failed to open main sqlite: %v", err)
}
mainDB.SetMaxOpenConns(1)
telemetryDB, err := sql.Open("sqlite", filepath.Join(dir, "telemetry.db"))
if err != nil {
t.Fatalf("failed to open telemetry sqlite: %v", err)
}
telemetryDB.SetMaxOpenConns(1)
t.Cleanup(func() {
mainDB.Close()
telemetryDB.Close()
})
db.DB = mainDB
db.TelemetryDB = telemetryDB
db.Driver = lit.SQLite
models.Init(db.Driver)
if _, err := telemetryDB.Exec(`CREATE TABLE tasks_v2 (
id TEXT NOT NULL,
project_id TEXT NOT NULL,
trace_id TEXT NOT NULL DEFAULT '',
span_id TEXT NOT NULL DEFAULT '',
task_name TEXT NOT NULL DEFAULT '',
duration INTEGER NOT NULL DEFAULT 0,
recorded_at DATETIME NOT NULL
)`); err != nil {
t.Fatalf("failed to create tasks_v2 table: %v", err)
}
if _, err := telemetryDB.Exec(`CREATE TABLE exceptions_v2 (
id TEXT NOT NULL,
project_id TEXT NOT NULL,
trace_id TEXT NOT NULL DEFAULT '',
span_id TEXT NOT NULL DEFAULT '',
trace_type TEXT NOT NULL DEFAULT '',
recorded_at DATETIME NOT NULL
)`); err != nil {
t.Fatalf("failed to create exceptions_v2 table: %v", err)
}
if _, err := telemetryDB.Exec(`CREATE TABLE metric_points (
project_id TEXT NOT NULL,
name TEXT NOT NULL DEFAULT '',
value REAL NOT NULL DEFAULT 0,
tags TEXT NOT NULL DEFAULT '{}',
recorded_at DATETIME NOT NULL
)`); err != nil {
t.Fatalf("failed to create metric_points table: %v", err)
}
}
func TestEvaluateMetricThresholdTagFilter(t *testing.T) {
setupNotificationsTestDB(t)
ctx := context.Background()
projectId := uuid.New()
ts := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano)
for _, p := range []struct {
value float64
tags string
}{
{3, `{"outcome":"healed","host":"fw1"}`},
{4, `{"outcome":"healed","host":"fw2"}`},
{2, `{"outcome":"failed","host":"fw1"}`},
} {
if _, err := db.TelemetryDB.Exec(
"INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (?, ?, ?, ?, ?)",
projectId.String(), "syslogd_heal_total", p.value, p.tags, ts); err != nil {
t.Fatalf("failed to insert metric point: %v", err)
}
}
cases := []struct {
name string
agg string
tags string
threshold float64
fired bool
}{
{"no filter sums every series", "sum", `{}`, 8, true},
{"filter keeps only matching series", "sum", `{"outcome":"failed"}`, 2, false},
{"filter fires on matching series", "sum", `{"outcome":"failed"}`, 1, true},
{"every filter must match", "max", `{"outcome":"healed","host":"fw1"}`, 3, false},
{"no matching series never fires", "sum", `{"outcome":"unknown"}`, -1, false},
{"last honors the filter", "last", `{"host":"fw2"}`, 3.5, true},
{"percentile honors the filter", "p95", `{"outcome":"failed"}`, 2, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
rule := &models.NotificationRule{
Config: json.RawMessage(fmt.Sprintf(`{"metricName":"syslogd_heal_total","aggregation":%q,"operator":"gt","thresholdValue":%v,"lookbackMinutes":5,"tags":%s}`, tc.agg, tc.threshold, tc.tags)),
}
result, err := evaluateMetricThreshold(ctx, rule, projectId)
if err != nil {
t.Fatalf("evaluateMetricThreshold failed: %v", err)
}
if result.Fired != tc.fired {
t.Errorf("fired = %v, want %v", result.Fired, tc.fired)
}
})
}
}
func TestEvaluateMetricThresholdGroupBy(t *testing.T) {
setupNotificationsTestDB(t)
ctx := context.Background()
projectId := uuid.New()
ts := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano)
insert := func(value float64, tags string) {
t.Helper()
if _, err := db.TelemetryDB.Exec(
"INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (?, ?, ?, ?, ?)",
projectId.String(), "radius.freshness", value, tags, ts); err != nil {
t.Fatalf("failed to insert metric point: %v", err)
}
}
insert(22166, `{"site":"RUINS","nas":"10.0.0.1"}`)
insert(10, `{"site":"RUINS","nas":"10.0.0.2"}`)
insert(30, `{"site":"HILL","nas":"10.0.0.3"}`)
insert(950, `{"nas":"10.0.0.4"}`)
ruleId := 9100
newRule := func(agg, groupBy string) *models.NotificationRule {
ruleId++
return &models.NotificationRule{
Id: ruleId,
RuleType: "metric_threshold",
CooldownMinutes: 30,
Config: json.RawMessage(fmt.Sprintf(`{"metricName":"radius.freshness","aggregation":%q,"operator":"gt","thresholdValue":900,"lookbackMinutes":5,"groupBy":%s}`, agg, groupBy)),
}
}
subjects := func(r *EvalResult) []string {
var out []string
for _, m := range r.Messages {
out = append(out, m.Subject)
}
return out
}
for _, agg := range []string{"max", "last", "p95"} {
t.Run("one message per breaching series ("+agg+")", func(t *testing.T) {
rule := newRule(agg, `["site"]`)
result, err := evaluateMetricThreshold(ctx, rule, projectId)
if err != nil {
t.Fatalf("evaluateMetricThreshold failed: %v", err)
}
got := subjects(result)
if !result.Fired || len(got) != 2 {
t.Fatalf("messages = %v, want two", got)
}
if !strings.Contains(got[0], "radius.freshness{site=RUINS}") || !strings.Contains(got[1], "radius.freshness{site=(empty)}") {
t.Errorf("subjects = %v, want RUINS first, then the untagged series", got)
}
if result.Messages[0].DedupToken != "radius.freshness{site=RUINS}" {
t.Errorf("dedup token = %q", result.Messages[0].DedupToken)
}
})
}
t.Run("several group keys name every value", func(t *testing.T) {
result, err := evaluateMetricThreshold(ctx, newRule("max", `["site","nas"]`), projectId)
if err != nil {
t.Fatalf("evaluateMetricThreshold failed: %v", err)
}
got := subjects(result)
if len(got) != 2 || !strings.Contains(got[0], "{site=RUINS, nas=10.0.0.1}") {
t.Errorf("subjects = %v", got)
}
})
t.Run("a breaching series stays quiet for the cooldown, a new one still fires", func(t *testing.T) {
rule := newRule("max", `["site"]`)
if _, err := evaluateMetricThreshold(ctx, rule, projectId); err != nil {
t.Fatalf("evaluateMetricThreshold failed: %v", err)
}
insert(5000, `{"site":"HILL","nas":"10.0.0.3"}`)
result, err := evaluateMetricThreshold(ctx, rule, projectId)
if err != nil {
t.Fatalf("evaluateMetricThreshold failed: %v", err)
}
got := subjects(result)
if len(got) != 1 || !strings.Contains(got[0], "{site=HILL}") {
t.Errorf("subjects = %v, want only HILL", got)
}
if !isPerSeriesRule(rule.RuleType, rule.Config) {
t.Error("a grouped rule must bypass the rule-level cooldown")
}
})
t.Run("without groupBy the rule reduces every series to one value", func(t *testing.T) {
rule := newRule("max", `[]`)
result, err := evaluateMetricThreshold(ctx, rule, projectId)
if err != nil {
t.Fatalf("evaluateMetricThreshold failed: %v", err)
}
if !result.Fired || len(result.Messages) != 0 || !strings.Contains(result.Message.Subject, "Metric radius.freshness is 22166.00") {
t.Errorf("result = %+v", result)
}
if isPerSeriesRule(rule.RuleType, rule.Config) {
t.Error("an ungrouped rule keeps the rule-level cooldown")
}
})
}
func TestMetricThresholdResultCapsNotifiedSeries(t *testing.T) {
setupNotificationsTestDB(t)
cfg := metricThresholdConfig{MetricName: "m", Operator: "gt", ThresholdValue: 1, LookbackMinutes: 5, GroupBy: []string{"host"}}
var series []metricSeries
for i := range maxMetricRuleSeriesNotified + 7 {
series = append(series, metricSeries{Key: fmt.Sprintf("h%03d", i), Value: float64(i + 2)})
}
rule := &models.NotificationRule{Id: 9300, CooldownMinutes: 30}
result := metricThresholdResult(rule, cfg, series, uuid.New())
if len(result.Messages) != maxMetricRuleSeriesNotified+1 {
t.Fatalf("messages = %d, want %d", len(result.Messages), maxMetricRuleSeriesNotified+1)
}
if !strings.Contains(result.Messages[0].Subject, "m{host=h056}") {
t.Errorf("first subject = %q, want the worst series first", result.Messages[0].Subject)
}
if last := result.Messages[maxMetricRuleSeriesNotified].Subject; !strings.Contains(last, "7 more series") {
t.Errorf("overflow subject = %q", last)
}
if again := metricThresholdResult(rule, cfg, series, uuid.New()); again.Fired {
t.Errorf("series summarized in the overflow notice must also wait out the cooldown, got %d messages", len(again.Messages))
}
}
func TestEvaluateTaskFailureRate(t *testing.T) {
setupNotificationsTestDB(t)
ctx := context.Background()
projectId := uuid.New()
pid := projectId.String()
now := time.Now().UTC()
trace := func(i int) string { return fmt.Sprintf("%032x", i+1) }
span := func(i int) string { return fmt.Sprintf("%016x", i+1) }
for i := 0; i < 5; i++ {
ts := now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339Nano)
if _, err := db.TelemetryDB.Exec(
"INSERT INTO tasks_v2 (id, project_id, trace_id, span_id, task_name, duration, recorded_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
uuid.New().String(), pid, trace(i), span(i), "etl-job", 1000000, ts); err != nil {
t.Fatalf("failed to insert task: %v", err)
}
}
exception := func(i int, spanId, traceType string) {
ts := now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339Nano)
if _, err := db.TelemetryDB.Exec(
"INSERT INTO exceptions_v2 (id, project_id, trace_id, span_id, trace_type, recorded_at) VALUES (?, ?, ?, ?, ?, ?)",
uuid.New().String(), pid, trace(i), spanId, traceType, ts); err != nil {
t.Fatalf("failed to insert exception: %v", err)
}
}
// Two tasks failed, one of them twice on a retried export. A third caught an exception further down and finished.
exception(0, span(0), "task")
exception(0, span(0), "task")
exception(1, span(1), "task")
exception(2, "00000000000000ff", "")
t.Run("fires above threshold", func(t *testing.T) {
rule := &models.NotificationRule{
Config: json.RawMessage(`{"taskName":"etl-job","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":5}`),
}
result, err := evaluateTaskFailureRate(ctx, rule, projectId)
if err != nil {
t.Fatalf("evaluateTaskFailureRate failed: %v", err)
}
if !result.Fired {
t.Fatal("expected rule to fire")
}
if !strings.Contains(result.Message.Subject, "40.0%") {
t.Errorf("subject %q does not contain 40.0%%", result.Message.Subject)
}
if !strings.Contains(result.Message.Body, "2 of 5") {
t.Errorf("body %q does not contain 2 of 5", result.Message.Body)
}
})
t.Run("gated below min executions", func(t *testing.T) {
rule := &models.NotificationRule{
Config: json.RawMessage(`{"taskName":"etl-job","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":10}`),
}
result, err := evaluateTaskFailureRate(ctx, rule, projectId)
if err != nil {
t.Fatalf("evaluateTaskFailureRate failed: %v", err)
}
if result.Fired {
t.Error("expected rule not to fire below minExecutions")
}
})
t.Run("wildcard task name", func(t *testing.T) {
rule := &models.NotificationRule{
Config: json.RawMessage(`{"taskName":"*","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":5}`),
}
result, err := evaluateTaskFailureRate(ctx, rule, projectId)
if err != nil {
t.Fatalf("evaluateTaskFailureRate failed: %v", err)
}
if !result.Fired {
t.Fatal("expected wildcard rule to fire")
}
if !strings.Contains(result.Message.Subject, "all tasks") {
t.Errorf("subject %q does not mention all tasks", result.Message.Subject)
}
if !strings.Contains(result.Message.Body, "2 of 5") {
t.Errorf("body %q does not contain 2 of 5", result.Message.Body)
}
})
}