Skip to content

The only tool you need to know what is happening and how to fix it.

15.8 KB
Raw Blame
//go:build !telemetry_ch

package notifications

import (
	"context"
	"database/sql"
	"encoding/json"
	"fmt"
	"path/filepath"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/google/uuid"
	"github.com/tracewayapp/lit/v2"
	"github.com/tracewayapp/traceway/backend/app/db"
	"github.com/tracewayapp/traceway/backend/app/models"
	_ "modernc.org/sqlite"
)

func TestCountOccurrences(t *testing.T) {
	result := countOccurrences([]string{"a", "b", "a", "c", "a", "b"})
	if len(result) != 3 {
		t.Fatalf("expected 3 distinct hashes, got %d", len(result))
	}
	if result["a"] != 3 {
		t.Errorf("expected a counted 3 times, got %d", result["a"])
	}
	if result["b"] != 2 {
		t.Errorf("expected b counted 2 times, got %d", result["b"])
	}
	if result["c"] != 1 {
		t.Errorf("expected c counted 1 time, got %d", result["c"])
	}

	unique := countOccurrences([]string{"x", "y"})
	if len(unique) != 2 || unique["x"] != 1 || unique["y"] != 1 {
		t.Errorf("unique hashes not preserved: %v", unique)
	}

	empty := countOccurrences(nil)
	if len(empty) != 0 {
		t.Errorf("expected empty map for empty slice, got %v", empty)
	}
}

func TestCooldownSeed(t *testing.T) {
	tracker := &cooldownTracker{fired: make(map[int]time.Time)}
	t1 := time.Now().Add(-2 * time.Hour)
	t2 := time.Now().Add(-1 * time.Hour)

	tracker.seed(map[int]time.Time{1: t1, 2: t2})
	if got := tracker.fired[1]; !got.Equal(t1) {
		t.Errorf("seed on empty tracker: fired[1] = %v, expected %v", got, t1)
	}
	if got := tracker.fired[2]; !got.Equal(t2) {
		t.Errorf("seed on empty tracker: fired[2] = %v, expected %v", got, t2)
	}

	tracker.recordFire(3)
	recorded := tracker.fired[3]
	tracker.seed(map[int]time.Time{3: time.Now().Add(-3 * time.Hour)})
	if got := tracker.fired[3]; !got.Equal(recorded) {
		t.Errorf("seed overwrote newer recordFire time: got %v, expected %v", got, recorded)
	}

	newer := time.Now().Add(time.Hour)
	tracker.seed(map[int]time.Time{1: newer})
	if got := tracker.fired[1]; !got.Equal(newer) {
		t.Errorf("seed did not update with newer time: got %v, expected %v", got, newer)
	}
}

func TestWindowP99MatchesNearestRank(t *testing.T) {
	dbPath := filepath.Join(t.TempDir(), "p99.db")
	conn, err := sql.Open("sqlite", dbPath)
	if err != nil {
		t.Fatalf("failed to open sqlite: %v", err)
	}
	defer conn.Close()
	conn.SetMaxOpenConns(1)

	if _, err := conn.Exec(`CREATE TABLE endpoints (
		endpoint TEXT NOT NULL,
		duration INTEGER NOT NULL,
		project_id TEXT NOT NULL,
		recorded_at DATETIME NOT NULL
	)`); err != nil {
		t.Fatalf("failed to create endpoints table: %v", err)
	}

	projectId := uuid.New().String()
	now := time.Now().UTC()
	from := now.Add(-time.Hour)
	ts := now.Add(-30 * time.Minute).Format(time.RFC3339Nano)

	durations := map[string][]int64{}
	for i := 0; i < 100; i++ {
		j := (i * 37) % 100
		durations["GET /alpha"] = append(durations["GET /alpha"], int64(1000+j*10))
		durations["GET /beta"] = append(durations["GET /beta"], int64(5000+j*37))
	}

	for ep, ds := range durations {
		for _, d := range ds {
			if _, err := conn.Exec(
				"INSERT INTO endpoints (endpoint, duration, project_id, recorded_at) VALUES (?, ?, ?, ?)",
				ep, d, projectId, ts); err != nil {
				t.Fatalf("failed to insert row: %v", err)
			}
		}
	}

	rows, err := conn.Query(`SELECT endpoint, duration FROM (
		SELECT endpoint, duration,
			ROW_NUMBER() OVER (PARTITION BY endpoint ORDER BY duration) AS rn,
			COUNT(*) OVER (PARTITION BY endpoint) AS cnt
		FROM endpoints WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?
	) WHERE rn = CAST(0.99 * (cnt - 1) AS INTEGER) + 1`,
		projectId, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
	if err != nil {
		t.Fatalf("window query failed: %v", err)
	}
	defer rows.Close()

	got := map[string]int64{}
	for rows.Next() {
		var ep string
		var d int64
		if err := rows.Scan(&ep, &d); err != nil {
			t.Fatalf("failed to scan row: %v", err)
		}
		got[ep] = d
	}
	if err := rows.Err(); err != nil {
		t.Fatalf("rows error: %v", err)
	}

	if len(got) != 2 {
		t.Fatalf("expected p99 for 2 endpoints, got %d: %v", len(got), got)
	}

	for ep, ds := range durations {
		sorted := append([]int64(nil), ds...)
		sort.Slice(sorted, func(a, b int) bool { return sorted[a] < sorted[b] })
		expected := sorted[int(0.99*float64(len(sorted)-1))]
		if got[ep] != expected {
			t.Errorf("endpoint %s: window p99 = %d, nearest-rank-lower = %d", ep, got[ep], expected)
		}
	}
}

func setupNotificationsTestDB(t *testing.T) {
	t.Helper()

	prevDB := db.DB
	prevTelemetryDB := db.TelemetryDB
	prevDriver := db.Driver
	t.Cleanup(func() {
		db.DB = prevDB
		db.TelemetryDB = prevTelemetryDB
		db.Driver = prevDriver
	})

	dir := t.TempDir()

	mainDB, err := sql.Open("sqlite", filepath.Join(dir, "main.db"))
	if err != nil {
		t.Fatalf("failed to open main sqlite: %v", err)
	}
	mainDB.SetMaxOpenConns(1)

	telemetryDB, err := sql.Open("sqlite", filepath.Join(dir, "telemetry.db"))
	if err != nil {
		t.Fatalf("failed to open telemetry sqlite: %v", err)
	}
	telemetryDB.SetMaxOpenConns(1)

	t.Cleanup(func() {
		mainDB.Close()
		telemetryDB.Close()
	})

	db.DB = mainDB
	db.TelemetryDB = telemetryDB
	db.Driver = lit.SQLite
	models.Init(db.Driver)

	if _, err := telemetryDB.Exec(`CREATE TABLE tasks_v2 (
		id TEXT NOT NULL,
		project_id TEXT NOT NULL,
		trace_id TEXT NOT NULL DEFAULT '',
		span_id TEXT NOT NULL DEFAULT '',
		task_name TEXT NOT NULL DEFAULT '',
		duration INTEGER NOT NULL DEFAULT 0,
		recorded_at DATETIME NOT NULL
	)`); err != nil {
		t.Fatalf("failed to create tasks_v2 table: %v", err)
	}

	if _, err := telemetryDB.Exec(`CREATE TABLE exceptions_v2 (
		id TEXT NOT NULL,
		project_id TEXT NOT NULL,
		trace_id TEXT NOT NULL DEFAULT '',
		span_id TEXT NOT NULL DEFAULT '',
		trace_type TEXT NOT NULL DEFAULT '',
		recorded_at DATETIME NOT NULL
	)`); err != nil {
		t.Fatalf("failed to create exceptions_v2 table: %v", err)
	}

	if _, err := telemetryDB.Exec(`CREATE TABLE metric_points (
		project_id TEXT NOT NULL,
		name TEXT NOT NULL DEFAULT '',
		value REAL NOT NULL DEFAULT 0,
		tags TEXT NOT NULL DEFAULT '{}',
		recorded_at DATETIME NOT NULL
	)`); err != nil {
		t.Fatalf("failed to create metric_points table: %v", err)
	}
}

func TestEvaluateMetricThresholdTagFilter(t *testing.T) {
	setupNotificationsTestDB(t)

	ctx := context.Background()
	projectId := uuid.New()
	ts := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano)
	for _, p := range []struct {
		value float64
		tags  string
	}{
		{3, `{"outcome":"healed","host":"fw1"}`},
		{4, `{"outcome":"healed","host":"fw2"}`},
		{2, `{"outcome":"failed","host":"fw1"}`},
	} {
		if _, err := db.TelemetryDB.Exec(
			"INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (?, ?, ?, ?, ?)",
			projectId.String(), "syslogd_heal_total", p.value, p.tags, ts); err != nil {
			t.Fatalf("failed to insert metric point: %v", err)
		}
	}

	cases := []struct {
		name      string
		agg       string
		tags      string
		threshold float64
		fired     bool
	}{
		{"no filter sums every series", "sum", `{}`, 8, true},
		{"filter keeps only matching series", "sum", `{"outcome":"failed"}`, 2, false},
		{"filter fires on matching series", "sum", `{"outcome":"failed"}`, 1, true},
		{"every filter must match", "max", `{"outcome":"healed","host":"fw1"}`, 3, false},
		{"no matching series never fires", "sum", `{"outcome":"unknown"}`, -1, false},
		{"last honors the filter", "last", `{"host":"fw2"}`, 3.5, true},
		{"percentile honors the filter", "p95", `{"outcome":"failed"}`, 2, false},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			rule := &models.NotificationRule{
				Config: json.RawMessage(fmt.Sprintf(`{"metricName":"syslogd_heal_total","aggregation":%q,"operator":"gt","thresholdValue":%v,"lookbackMinutes":5,"tags":%s}`, tc.agg, tc.threshold, tc.tags)),
			}
			result, err := evaluateMetricThreshold(ctx, rule, projectId)
			if err != nil {
				t.Fatalf("evaluateMetricThreshold failed: %v", err)
			}
			if result.Fired != tc.fired {
				t.Errorf("fired = %v, want %v", result.Fired, tc.fired)
			}
		})
	}
}

func TestEvaluateMetricThresholdGroupBy(t *testing.T) {
	setupNotificationsTestDB(t)

	ctx := context.Background()
	projectId := uuid.New()
	ts := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano)
	insert := func(value float64, tags string) {
		t.Helper()
		if _, err := db.TelemetryDB.Exec(
			"INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (?, ?, ?, ?, ?)",
			projectId.String(), "radius.freshness", value, tags, ts); err != nil {
			t.Fatalf("failed to insert metric point: %v", err)
		}
	}
	insert(22166, `{"site":"RUINS","nas":"10.0.0.1"}`)
	insert(10, `{"site":"RUINS","nas":"10.0.0.2"}`)
	insert(30, `{"site":"HILL","nas":"10.0.0.3"}`)
	insert(950, `{"nas":"10.0.0.4"}`)

	ruleId := 9100
	newRule := func(agg, groupBy string) *models.NotificationRule {
		ruleId++
		return &models.NotificationRule{
			Id:              ruleId,
			RuleType:        "metric_threshold",
			CooldownMinutes: 30,
			Config:          json.RawMessage(fmt.Sprintf(`{"metricName":"radius.freshness","aggregation":%q,"operator":"gt","thresholdValue":900,"lookbackMinutes":5,"groupBy":%s}`, agg, groupBy)),
		}
	}
	subjects := func(r *EvalResult) []string {
		var out []string
		for _, m := range r.Messages {
			out = append(out, m.Subject)
		}
		return out
	}

	for _, agg := range []string{"max", "last", "p95"} {
		t.Run("one message per breaching series ("+agg+")", func(t *testing.T) {
			rule := newRule(agg, `["site"]`)
			result, err := evaluateMetricThreshold(ctx, rule, projectId)
			if err != nil {
				t.Fatalf("evaluateMetricThreshold failed: %v", err)
			}
			got := subjects(result)
			if !result.Fired || len(got) != 2 {
				t.Fatalf("messages = %v, want two", got)
			}
			if !strings.Contains(got[0], "radius.freshness{site=RUINS}") || !strings.Contains(got[1], "radius.freshness{site=(empty)}") {
				t.Errorf("subjects = %v, want RUINS first, then the untagged series", got)
			}
			if result.Messages[0].DedupToken != "radius.freshness{site=RUINS}" {
				t.Errorf("dedup token = %q", result.Messages[0].DedupToken)
			}
		})
	}

	t.Run("several group keys name every value", func(t *testing.T) {
		result, err := evaluateMetricThreshold(ctx, newRule("max", `["site","nas"]`), projectId)
		if err != nil {
			t.Fatalf("evaluateMetricThreshold failed: %v", err)
		}
		got := subjects(result)
		if len(got) != 2 || !strings.Contains(got[0], "{site=RUINS, nas=10.0.0.1}") {
			t.Errorf("subjects = %v", got)
		}
	})

	t.Run("a breaching series stays quiet for the cooldown, a new one still fires", func(t *testing.T) {
		rule := newRule("max", `["site"]`)
		if _, err := evaluateMetricThreshold(ctx, rule, projectId); err != nil {
			t.Fatalf("evaluateMetricThreshold failed: %v", err)
		}
		insert(5000, `{"site":"HILL","nas":"10.0.0.3"}`)
		result, err := evaluateMetricThreshold(ctx, rule, projectId)
		if err != nil {
			t.Fatalf("evaluateMetricThreshold failed: %v", err)
		}
		got := subjects(result)
		if len(got) != 1 || !strings.Contains(got[0], "{site=HILL}") {
			t.Errorf("subjects = %v, want only HILL", got)
		}
		if !isPerSeriesRule(rule.RuleType, rule.Config) {
			t.Error("a grouped rule must bypass the rule-level cooldown")
		}
	})

	t.Run("without groupBy the rule reduces every series to one value", func(t *testing.T) {
		rule := newRule("max", `[]`)
		result, err := evaluateMetricThreshold(ctx, rule, projectId)
		if err != nil {
			t.Fatalf("evaluateMetricThreshold failed: %v", err)
		}
		if !result.Fired || len(result.Messages) != 0 || !strings.Contains(result.Message.Subject, "Metric radius.freshness is 22166.00") {
			t.Errorf("result = %+v", result)
		}
		if isPerSeriesRule(rule.RuleType, rule.Config) {
			t.Error("an ungrouped rule keeps the rule-level cooldown")
		}
	})
}

func TestMetricThresholdResultCapsNotifiedSeries(t *testing.T) {
	setupNotificationsTestDB(t)

	cfg := metricThresholdConfig{MetricName: "m", Operator: "gt", ThresholdValue: 1, LookbackMinutes: 5, GroupBy: []string{"host"}}
	var series []metricSeries
	for i := range maxMetricRuleSeriesNotified + 7 {
		series = append(series, metricSeries{Key: fmt.Sprintf("h%03d", i), Value: float64(i + 2)})
	}
	rule := &models.NotificationRule{Id: 9300, CooldownMinutes: 30}
	result := metricThresholdResult(rule, cfg, series, uuid.New())
	if len(result.Messages) != maxMetricRuleSeriesNotified+1 {
		t.Fatalf("messages = %d, want %d", len(result.Messages), maxMetricRuleSeriesNotified+1)
	}
	if !strings.Contains(result.Messages[0].Subject, "m{host=h056}") {
		t.Errorf("first subject = %q, want the worst series first", result.Messages[0].Subject)
	}
	if last := result.Messages[maxMetricRuleSeriesNotified].Subject; !strings.Contains(last, "7 more series") {
		t.Errorf("overflow subject = %q", last)
	}
	if again := metricThresholdResult(rule, cfg, series, uuid.New()); again.Fired {
		t.Errorf("series summarized in the overflow notice must also wait out the cooldown, got %d messages", len(again.Messages))
	}
}

func TestEvaluateTaskFailureRate(t *testing.T) {
	setupNotificationsTestDB(t)

	ctx := context.Background()
	projectId := uuid.New()
	pid := projectId.String()
	now := time.Now().UTC()

	trace := func(i int) string { return fmt.Sprintf("%032x", i+1) }
	span := func(i int) string { return fmt.Sprintf("%016x", i+1) }
	for i := 0; i < 5; i++ {
		ts := now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339Nano)
		if _, err := db.TelemetryDB.Exec(
			"INSERT INTO tasks_v2 (id, project_id, trace_id, span_id, task_name, duration, recorded_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
			uuid.New().String(), pid, trace(i), span(i), "etl-job", 1000000, ts); err != nil {
			t.Fatalf("failed to insert task: %v", err)
		}
	}

	exception := func(i int, spanId, traceType string) {
		ts := now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339Nano)
		if _, err := db.TelemetryDB.Exec(
			"INSERT INTO exceptions_v2 (id, project_id, trace_id, span_id, trace_type, recorded_at) VALUES (?, ?, ?, ?, ?, ?)",
			uuid.New().String(), pid, trace(i), spanId, traceType, ts); err != nil {
			t.Fatalf("failed to insert exception: %v", err)
		}
	}
	// Two tasks failed, one of them twice on a retried export. A third caught an exception further down and finished.
	exception(0, span(0), "task")
	exception(0, span(0), "task")
	exception(1, span(1), "task")
	exception(2, "00000000000000ff", "")

	t.Run("fires above threshold", func(t *testing.T) {
		rule := &models.NotificationRule{
			Config: json.RawMessage(`{"taskName":"etl-job","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":5}`),
		}
		result, err := evaluateTaskFailureRate(ctx, rule, projectId)
		if err != nil {
			t.Fatalf("evaluateTaskFailureRate failed: %v", err)
		}
		if !result.Fired {
			t.Fatal("expected rule to fire")
		}
		if !strings.Contains(result.Message.Subject, "40.0%") {
			t.Errorf("subject %q does not contain 40.0%%", result.Message.Subject)
		}
		if !strings.Contains(result.Message.Body, "2 of 5") {
			t.Errorf("body %q does not contain 2 of 5", result.Message.Body)
		}
	})

	t.Run("gated below min executions", func(t *testing.T) {
		rule := &models.NotificationRule{
			Config: json.RawMessage(`{"taskName":"etl-job","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":10}`),
		}
		result, err := evaluateTaskFailureRate(ctx, rule, projectId)
		if err != nil {
			t.Fatalf("evaluateTaskFailureRate failed: %v", err)
		}
		if result.Fired {
			t.Error("expected rule not to fire below minExecutions")
		}
	})

	t.Run("wildcard task name", func(t *testing.T) {
		rule := &models.NotificationRule{
			Config: json.RawMessage(`{"taskName":"*","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":5}`),
		}
		result, err := evaluateTaskFailureRate(ctx, rule, projectId)
		if err != nil {
			t.Fatalf("evaluateTaskFailureRate failed: %v", err)
		}
		if !result.Fired {
			t.Fatal("expected wildcard rule to fire")
		}
		if !strings.Contains(result.Message.Subject, "all tasks") {
			t.Errorf("subject %q does not mention all tasks", result.Message.Subject)
		}
		if !strings.Contains(result.Message.Body, "2 of 5") {
			t.Errorf("body %q does not contain 2 of 5", result.Message.Body)
		}
	})
}