The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
#421
narrow-adventuree20b898e7751
diff --git a/backend/app/controllers/notification_rule.controller.go b/backend/app/controllers/notification_rule.controller.goindex 6f932e26..9e641bc4 100644--- a/backend/app/controllers/notification_rule.controller.go+++ b/backend/app/controllers/notification_rule.controller.go@@ -423,13 +423,17 @@ func (ctrl *notificationRuleController) Snooze(ctx *gin.Context) {var NotificationRuleController = notificationRuleController{}-const maxMetricRuleTagFilters = 20+const (+ maxMetricRuleTagFilters = 20+ maxMetricRuleGroupBy = 5+)func validateRuleConfig(tx *sql.Tx, projectId uuid.UUID, ruleType string, config json.RawMessage) (string, error) {switch ruleType {case "metric_threshold":var cfg struct {- Tags map[string]string `json:"tags"`+ Tags map[string]string `json:"tags"`+ GroupBy []string `json:"groupBy"`}if err := json.Unmarshal(config, &cfg); err != nil {return "The rule configuration is not valid JSON.", nil@@ -442,6 +446,19 @@ func validateRuleConfig(tx *sql.Tx, projectId uuid.UUID, ruleType string, configreturn "Every tag filter needs a key and a value.", nil}}+ if len(cfg.GroupBy) > maxMetricRuleGroupBy {+ return fmt.Sprintf("A metric rule can group by at most %d tags.", maxMetricRuleGroupBy), nil+ }+ seen := make(map[string]bool, len(cfg.GroupBy))+ for _, k := range cfg.GroupBy {+ if strings.TrimSpace(k) == "" {+ return "Group by tag keys cannot be empty.", nil+ }+ if seen[k] {+ return fmt.Sprintf("The tag %q is listed twice in group by.", k), nil+ }+ seen[k] = true+ }case "slo_burn_rate":var cfg struct {SloId int `json:"sloId"`diff --git a/backend/app/notifications/cooldown.go b/backend/app/notifications/cooldown.goindex 36c5731c..35f3ece0 100644--- a/backend/app/notifications/cooldown.go+++ b/backend/app/notifications/cooldown.go@@ -57,6 +57,10 @@ func checkDownDedupKey(ruleId int, checkId int) string {return ruleStatePrefix(ruleId) + fmt.Sprintf("check_down:%d", checkId)}+func metricSeriesDedupKey(ruleId int, groupKey string) string {+ return ruleStatePrefix(ruleId) + "metric_series:" + groupKey+}+func ClearRuleState(ruleId int) {cooldowns.mu.Lock()delete(cooldowns.fired, ruleId)diff --git a/backend/app/notifications/evaluator.go b/backend/app/notifications/evaluator.goindex 34f1f80e..d5cedccb 100644--- a/backend/app/notifications/evaluator.go+++ b/backend/app/notifications/evaluator.go@@ -77,7 +77,7 @@ func evaluatePolledRules(ctx context.Context) {continue}- if !statefulRuleTypes[rule.RuleType] && !cooldowns.canFire(rule.Id, rule.CooldownMinutes) {+ if !statefulRuleTypes[rule.RuleType] && !isPerSeriesRule(rule.RuleType, rule.Config) && !cooldowns.canFire(rule.Id, rule.CooldownMinutes) {continue}diff --git a/backend/app/notifications/evaluator_polled.go b/backend/app/notifications/evaluator_polled.goindex fe80aed8..272af60d 100644--- a/backend/app/notifications/evaluator_polled.go+++ b/backend/app/notifications/evaluator_polled.go@@ -220,22 +220,10 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje// --- Metric Threshold ----type metricThresholdConfig struct {- MetricName string `json:"metricName"`- Operator string `json:"operator"`- ThresholdValue float64 `json:"thresholdValue"`- Aggregation string `json:"aggregation"`- LookbackMinutes int `json:"lookbackMinutes"`- Tags map[string]string `json:"tags"`-}-func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {- var cfg metricThresholdConfig- if err := json.Unmarshal(rule.Config, &cfg); err != nil {- return nil, fmt.Errorf("invalid metric_threshold config: %w", err)- }- if cfg.LookbackMinutes <= 0 {- cfg.LookbackMinutes = 5+ cfg, err := parseMetricThresholdConfig(rule.Config)+ if err != nil {+ return nil, err}now := time.Now().UTC()@@ -257,44 +245,35 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,selectExpr = "argMax(value, recorded_at)"}- query := fmt.Sprintf("SELECT count(), %s FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", selectExpr)- args := []interface{}{projectId, cfg.MetricName, from, now}+ groupExpr, args := metricGroupExpr(cfg.GroupBy, "tags[?]")+ query := fmt.Sprintf("SELECT %s AS group_key, %s FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", groupExpr, selectExpr)+ args = append(args, projectId, cfg.MetricName, from, now)for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {query += " AND tags[?] = ?"args = append(args, k, cfg.Tags[k])}+ query += " GROUP BY group_key ORDER BY group_key LIMIT ?"+ args = append(args, maxMetricRuleSeriesScanned)- var count uint64- var value float64- err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&count, &value)+ rows, err := chdb.Conn.Query(ctx, query, args...)if err != nil {return nil, err}- if count == 0 {- return &EvalResult{Fired: false}, nil- }+ defer rows.Close()- fired := false- switch cfg.Operator {- case "gt":- fired = value > cfg.ThresholdValue- case "gte":- fired = value >= cfg.ThresholdValue- case "lt":- fired = value < cfg.ThresholdValue- case "lte":- fired = value <= cfg.ThresholdValue- case "eq":- fired = value == cfg.ThresholdValue+ var series []metricSeries+ for rows.Next() {+ var s metricSeries+ if err := rows.Scan(&s.Key, &s.Value); err != nil {+ return nil, err+ }+ series = append(series, s)}-- if !fired {- return &EvalResult{Fired: false}, nil+ if err := rows.Err(); err != nil {+ return nil, err}- projectName := getProjectName(projectId)- msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, cfg.Aggregation, cfg.LookbackMinutes, projectName)- return &EvalResult{Fired: true, Message: msg}, nil+ return metricThresholdResult(rule, cfg, series, projectId), nil}// --- No Data ---diff --git a/backend/app/notifications/evaluator_polled_sqlite.go b/backend/app/notifications/evaluator_polled_sqlite.goindex b1f6c25c..58224ba9 100644--- a/backend/app/notifications/evaluator_polled_sqlite.go+++ b/backend/app/notifications/evaluator_polled_sqlite.go@@ -242,74 +242,36 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje// --- Metric Threshold ----type metricThresholdConfig struct {- MetricName string `json:"metricName"`- Operator string `json:"operator"`- ThresholdValue float64 `json:"thresholdValue"`- Aggregation string `json:"aggregation"`- LookbackMinutes int `json:"lookbackMinutes"`- Tags map[string]string `json:"tags"`-}-func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {- var cfg metricThresholdConfig- if err := json.Unmarshal(rule.Config, &cfg); err != nil {- return nil, fmt.Errorf("invalid metric_threshold config: %w", err)- }- if cfg.LookbackMinutes <= 0 {- cfg.LookbackMinutes = 5+ cfg, err := parseMetricThresholdConfig(rule.Config)+ if err != nil {+ return nil, err}now := time.Now().UTC()from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)- where := "project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?"- args := []interface{}{projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)}tagExtract := "json_extract"if db.IsDuckDBTelemetry() {tagExtract = "json_extract_string"}+ groupExpr, args := metricGroupExpr(cfg.GroupBy, "COALESCE("+tagExtract+"(tags, '$.\"' || ? || '\"'), '')")+ where := "project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?"+ args = append(args, projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {where += " AND " + tagExtract + "(tags, '$.\"' || ? || '\"') = ?"args = append(args, k, cfg.Tags[k])}- var value float64- var err error-+ var series []metricSeriesswitch cfg.Aggregation {- case "last":- err = db.TelemetryDB.QueryRowContext(ctx,- "SELECT value FROM metric_points WHERE "+where+" ORDER BY recorded_at DESC LIMIT 1", args...).Scan(&value)- if err == sql.ErrNoRows {- return &EvalResult{Fired: false}, nil- }- if err != nil {- return nil, err- }- case "p95", "p99":- pct := 0.95- if cfg.Aggregation == "p99" {- pct = 0.99+ case "last", "p95", "p99":+ order := "recorded_at DESC"+ if cfg.Aggregation != "last" {+ order = "value ASC"}- rows, qErr := db.TelemetryDB.QueryContext(ctx,- "SELECT value FROM metric_points WHERE "+where+" ORDER BY value ASC", args...)- if qErr != nil {- return nil, qErr- }- defer rows.Close()- var vals []float64- for rows.Next() {- var v float64- if err := rows.Scan(&v); err != nil {- continue- }- vals = append(vals, v)- }- if len(vals) == 0 {- return &EvalResult{Fired: false}, nil- }- value = computePercentile(vals, pct)+ series, err = scanMetricSeriesValues(ctx, cfg.Aggregation,+ fmt.Sprintf("SELECT %s AS group_key, value FROM metric_points WHERE %s ORDER BY group_key, %s", groupExpr, where, order), args)default:aggFunc := "avg"switch cfg.Aggregation {@@ -320,38 +282,83 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,case "sum":aggFunc = "sum"}- query := fmt.Sprintf("SELECT COUNT(value), COALESCE(%s(value), 0) FROM metric_points WHERE %s", aggFunc, where)- var count int64- err = db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&count, &value)- if err != nil {+ query := fmt.Sprintf("SELECT %s AS group_key, COALESCE(%s(value), 0) FROM metric_points WHERE %s GROUP BY group_key ORDER BY group_key LIMIT ?", groupExpr, aggFunc, where)+ series, err = scanMetricSeries(ctx, query, append(args, maxMetricRuleSeriesScanned))+ }+ if err != nil {+ return nil, err+ }++ return metricThresholdResult(rule, cfg, series, projectId), nil+}++func scanMetricSeries(ctx context.Context, query string, args []interface{}) ([]metricSeries, error) {+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)+ if err != nil {+ return nil, err+ }+ defer rows.Close()++ var series []metricSeries+ for rows.Next() {+ var s metricSeries+ if err := rows.Scan(&s.Key, &s.Value); err != nil {return nil, err}- if count == 0 {- return &EvalResult{Fired: false}, nil- }+ series = append(series, s)}+ return series, rows.Err()+}- fired := false- switch cfg.Operator {- case "gt":- fired = value > cfg.ThresholdValue- case "gte":- fired = value >= cfg.ThresholdValue- case "lt":- fired = value < cfg.ThresholdValue- case "lte":- fired = value <= cfg.ThresholdValue- case "eq":- fired = value == cfg.ThresholdValue+// scanMetricSeriesValues reduces rows ordered by group key to one value per+// series: the first row for last (newest first), a percentile otherwise+// (values ascending).+func scanMetricSeriesValues(ctx context.Context, aggregation string, query string, args []interface{}) ([]metricSeries, error) {+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)+ if err != nil {+ return nil, err}+ defer rows.Close()- if !fired {- return &EvalResult{Fired: false}, nil+ pct := 0.95+ if aggregation == "p99" {+ pct = 0.99+ }+ var series []metricSeries+ var vals []float64+ flush := func(key string) {+ if len(vals) == 0 {+ return+ }+ value := vals[0]+ if aggregation != "last" {+ value = computePercentile(vals, pct)+ }+ series = append(series, metricSeries{Key: key, Value: value})+ vals = vals[:0]}- projectName := getProjectName(projectId)- msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, cfg.Aggregation, cfg.LookbackMinutes, projectName)- return &EvalResult{Fired: true, Message: msg}, nil+ current := ""+ for rows.Next() {+ var key string+ var v float64+ if err := rows.Scan(&key, &v); err != nil {+ return nil, err+ }+ if key != current || len(vals) == 0 {+ flush(current)+ if len(series) >= maxMetricRuleSeriesScanned {+ return series, nil+ }+ current = key+ }+ vals = append(vals, v)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ flush(current)+ return series, nil}// --- No Data ---diff --git a/backend/app/notifications/messages.go b/backend/app/notifications/messages.goindex cd1bf949..41f66aad 100644--- a/backend/app/notifications/messages.go+++ b/backend/app/notifications/messages.go@@ -94,7 +94,7 @@ func buildApdexDropMessage(apdex float64, threshold float64, total int64, window}}-func buildMetricThresholdMessage(metricName string, value float64, operator string, threshold float64, aggregation string, window int, projectName string) Message {+func buildMetricThresholdMessage(metricName string, seriesLabel string, value float64, operator string, threshold float64, aggregation string, window int, projectName string) Message {severity := SeverityWarningdiff := value - thresholdif diff < 0 {@@ -106,18 +106,19 @@ func buildMetricThresholdMessage(metricName string, value float64, operator striif aggregation == "" {aggregation = "avg"}- headline := fmt.Sprintf("The metric %s has a %s of %.2f over the last %d minutes which violates the threshold %s %.2f.", metricName, aggregation, value, window, operator, threshold)+ series := metricName + seriesLabel+ headline := fmt.Sprintf("The metric %s has a %s of %.2f over the last %d minutes which violates the threshold %s %.2f.", series, aggregation, value, window, operator, threshold)return Message{- Subject: fmt.Sprintf("[%s] Metric %s is %.2f (threshold: %s %.2f)", projectName, metricName, value, operator, threshold),+ Subject: fmt.Sprintf("[%s] Metric %s is %.2f (threshold: %s %.2f)", projectName, series, value, operator, threshold),Body: headline,Severity: severity,URL: "/metrics?preset=1h",- DedupToken: metricName,+ DedupToken: series,Email: alertEmail(models.EmailAlert{ProjectName: projectName,Headline: headline,ScopeLabel: "Metric",- Scope: metricName,+ Scope: series,Observed: fmt.Sprintf("%s %.2f", aggregation, value),Threshold: fmt.Sprintf("%s %.2f", operator, threshold),WindowMins: window,@@ -125,6 +126,26 @@ func buildMetricThresholdMessage(metricName string, value float64, operator stri}}+func buildMetricThresholdOverflowMessage(metricName string, count int, operator string, threshold float64, window int, projectName string) Message {+ headline := fmt.Sprintf("%d more series of the metric %s violate the threshold %s %.2f over the last %d minutes. Group the metric on the metrics page to see them.", count, metricName, operator, threshold, window)+ return Message{+ Subject: fmt.Sprintf("[%s] Metric %s: %d more series breach the threshold (%s %.2f)", projectName, metricName, count, operator, threshold),+ Body: headline,+ Severity: SeverityWarning,+ URL: "/metrics?preset=1h",+ DedupToken: metricName + ":overflow",+ Email: alertEmail(models.EmailAlert{+ ProjectName: projectName,+ Headline: headline,+ ScopeLabel: "Metric",+ Scope: metricName,+ Observed: fmt.Sprintf("%d more series", count),+ Threshold: fmt.Sprintf("%s %.2f", operator, threshold),+ WindowMins: window,+ }),+ }+}+func buildNoDataMessage(dataType string, silenceMinutes int, projectName string) Message {headline := fmt.Sprintf("No %s data has been received for the last %d minutes.", dataType, silenceMinutes)return Message{diff --git a/backend/app/notifications/metric_series.go b/backend/app/notifications/metric_series.gonew file mode 100644index 00000000..069a0a39--- /dev/null+++ b/backend/app/notifications/metric_series.go@@ -0,0 +1,168 @@+package notifications++import (+ "cmp"+ "encoding/json"+ "fmt"+ "math"+ "slices"+ "strings"+ "time"++ "github.com/google/uuid"+ "github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/repositories/telemetry/shared"+)++type metricThresholdConfig struct {+ MetricName string `json:"metricName"`+ Operator string `json:"operator"`+ ThresholdValue float64 `json:"thresholdValue"`+ Aggregation string `json:"aggregation"`+ LookbackMinutes int `json:"lookbackMinutes"`+ Tags map[string]string `json:"tags"`+ GroupBy []string `json:"groupBy"`+}++const (+ maxMetricRuleSeriesNotified = 50+ // Series are read in group key order, so a metric with more series than+ // this has the rest left unevaluated rather than unbounded memory.+ maxMetricRuleSeriesScanned = 5000+)++type metricSeries struct {+ Key string+ Value float64+}++func parseMetricThresholdConfig(raw json.RawMessage) (metricThresholdConfig, error) {+ var cfg metricThresholdConfig+ if err := json.Unmarshal(raw, &cfg); err != nil {+ return cfg, fmt.Errorf("invalid metric_threshold config: %w", err)+ }+ if cfg.LookbackMinutes <= 0 {+ cfg.LookbackMinutes = 5+ }+ return cfg, nil+}++// isPerSeriesRule reports whether a rule keeps its cooldown per series instead+// of per rule, so one breaching series cannot silence another.+func isPerSeriesRule(ruleType string, config json.RawMessage) bool {+ if ruleType != "metric_threshold" {+ return false+ }+ var cfg struct {+ GroupBy []string `json:"groupBy"`+ }+ return json.Unmarshal(config, &cfg) == nil && len(cfg.GroupBy) > 0+}++// metricGroupExpr joins one tagExpr per groupBy key into a single group key+// column. tagExpr takes the tag key as its only placeholder.+func metricGroupExpr(groupBy []string, tagExpr string) (string, []interface{}) {+ if len(groupBy) == 0 {+ return "''", nil+ }+ parts := make([]string, len(groupBy))+ var args []interface{}+ for i, key := range groupBy {+ parts[i] = tagExpr+ if i > 0 {+ args = append(args, shared.GroupKeySeparator)+ }+ args = append(args, key)+ }+ return strings.Join(parts, " || ? || "), args+}++func metricThresholdBreached(value float64, operator string, threshold float64) bool {+ switch operator {+ case "gt":+ return value > threshold+ case "gte":+ return value >= threshold+ case "lt":+ return value < threshold+ case "lte":+ return value <= threshold+ case "eq":+ return value == threshold+ }+ return false+}++func metricSeriesLabel(groupBy []string, key string) string {+ if len(groupBy) == 0 {+ return ""+ }+ values := shared.SplitGroupKey(key)+ pairs := make([]string, len(groupBy))+ for i, k := range groupBy {+ v := ""+ if i < len(values) {+ v = values[i]+ }+ if v == "" {+ v = "(empty)"+ }+ pairs[i] = k + "=" + v+ }+ return "{" + strings.Join(pairs, ", ") + "}"+}++func metricThresholdResult(rule *models.NotificationRule, cfg metricThresholdConfig, series []metricSeries, projectId uuid.UUID) *EvalResult {+ var breaching []metricSeries+ for _, s := range series {+ if metricThresholdBreached(s.Value, cfg.Operator, cfg.ThresholdValue) {+ breaching = append(breaching, s)+ }+ }+ if len(breaching) == 0 {+ return &EvalResult{Fired: false}+ }++ if len(cfg.GroupBy) == 0 {+ projectName := getProjectName(projectId)+ msg := buildMetricThresholdMessage(cfg.MetricName, "", breaching[0].Value, cfg.Operator, cfg.ThresholdValue, cfg.Aggregation, cfg.LookbackMinutes, projectName)+ return &EvalResult{Fired: true, Message: msg}+ }++ cooldown := time.Duration(rule.CooldownMinutes) * time.Minute+ var fresh []metricSeries+ for _, s := range breaching {+ key := metricSeriesDedupKey(rule.Id, s.Key)+ if dedup.isDuplicate(key, cooldown) {+ continue+ }+ // Recorded before dispatch, like check_down: a failing channel retries+ // once per cooldown window, not on every poll.+ dedup.record(key)+ fresh = append(fresh, s)+ }+ if len(fresh) == 0 {+ return &EvalResult{Fired: false}+ }++ slices.SortFunc(fresh, func(a, b metricSeries) int {+ da := math.Abs(a.Value - cfg.ThresholdValue)+ db := math.Abs(b.Value - cfg.ThresholdValue)+ if c := cmp.Compare(db, da); c != 0 {+ return c+ }+ return cmp.Compare(a.Key, b.Key)+ })++ projectName := getProjectName(projectId)+ notified := fresh[:min(len(fresh), maxMetricRuleSeriesNotified)]+ messages := make([]Message, 0, len(notified)+1)+ for _, s := range notified {+ label := metricSeriesLabel(cfg.GroupBy, s.Key)+ messages = append(messages, buildMetricThresholdMessage(cfg.MetricName, label, s.Value, cfg.Operator, cfg.ThresholdValue, cfg.Aggregation, cfg.LookbackMinutes, projectName))+ }+ if rest := len(fresh) - len(notified); rest > 0 {+ messages = append(messages, buildMetricThresholdOverflowMessage(cfg.MetricName, rest, cfg.Operator, cfg.ThresholdValue, cfg.LookbackMinutes, projectName))+ }+ return &EvalResult{Fired: true, Messages: messages}+}diff --git a/backend/app/notifications/notifications_sqlite_test.go b/backend/app/notifications/notifications_sqlite_test.goindex e866de88..b49e6e0f 100644--- a/backend/app/notifications/notifications_sqlite_test.go+++ b/backend/app/notifications/notifications_sqlite_test.go@@ -274,6 +274,132 @@ func TestEvaluateMetricThresholdTagFilter(t *testing.T) {}}+func TestEvaluateMetricThresholdGroupBy(t *testing.T) {+ setupNotificationsTestDB(t)++ ctx := context.Background()+ projectId := uuid.New()+ ts := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano)+ insert := func(value float64, tags string) {+ t.Helper()+ if _, err := db.TelemetryDB.Exec(+ "INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (?, ?, ?, ?, ?)",+ projectId.String(), "radius.freshness", value, tags, ts); err != nil {+ t.Fatalf("failed to insert metric point: %v", err)+ }+ }+ insert(22166, `{"site":"RUINS","nas":"10.0.0.1"}`)+ insert(10, `{"site":"RUINS","nas":"10.0.0.2"}`)+ insert(30, `{"site":"HILL","nas":"10.0.0.3"}`)+ insert(950, `{"nas":"10.0.0.4"}`)++ ruleId := 9100+ newRule := func(agg, groupBy string) *models.NotificationRule {+ ruleId+++ return &models.NotificationRule{+ Id: ruleId,+ RuleType: "metric_threshold",+ CooldownMinutes: 30,+ Config: json.RawMessage(fmt.Sprintf(`{"metricName":"radius.freshness","aggregation":%q,"operator":"gt","thresholdValue":900,"lookbackMinutes":5,"groupBy":%s}`, agg, groupBy)),+ }+ }+ subjects := func(r *EvalResult) []string {+ var out []string+ for _, m := range r.Messages {+ out = append(out, m.Subject)+ }+ return out+ }++ for _, agg := range []string{"max", "last", "p95"} {+ t.Run("one message per breaching series ("+agg+")", func(t *testing.T) {+ rule := newRule(agg, `["site"]`)+ result, err := evaluateMetricThreshold(ctx, rule, projectId)+ if err != nil {+ t.Fatalf("evaluateMetricThreshold failed: %v", err)+ }+ got := subjects(result)+ if !result.Fired || len(got) != 2 {+ t.Fatalf("messages = %v, want two", got)+ }+ if !strings.Contains(got[0], "radius.freshness{site=RUINS}") || !strings.Contains(got[1], "radius.freshness{site=(empty)}") {+ t.Errorf("subjects = %v, want RUINS first, then the untagged series", got)+ }+ if result.Messages[0].DedupToken != "radius.freshness{site=RUINS}" {+ t.Errorf("dedup token = %q", result.Messages[0].DedupToken)+ }+ })+ }++ t.Run("several group keys name every value", func(t *testing.T) {+ result, err := evaluateMetricThreshold(ctx, newRule("max", `["site","nas"]`), projectId)+ if err != nil {+ t.Fatalf("evaluateMetricThreshold failed: %v", err)+ }+ got := subjects(result)+ if len(got) != 2 || !strings.Contains(got[0], "{site=RUINS, nas=10.0.0.1}") {+ t.Errorf("subjects = %v", got)+ }+ })++ t.Run("a breaching series stays quiet for the cooldown, a new one still fires", func(t *testing.T) {+ rule := newRule("max", `["site"]`)+ if _, err := evaluateMetricThreshold(ctx, rule, projectId); err != nil {+ t.Fatalf("evaluateMetricThreshold failed: %v", err)+ }+ insert(5000, `{"site":"HILL","nas":"10.0.0.3"}`)+ result, err := evaluateMetricThreshold(ctx, rule, projectId)+ if err != nil {+ t.Fatalf("evaluateMetricThreshold failed: %v", err)+ }+ got := subjects(result)+ if len(got) != 1 || !strings.Contains(got[0], "{site=HILL}") {+ t.Errorf("subjects = %v, want only HILL", got)+ }+ if !isPerSeriesRule(rule.RuleType, rule.Config) {+ t.Error("a grouped rule must bypass the rule-level cooldown")+ }+ })++ t.Run("without groupBy the rule reduces every series to one value", func(t *testing.T) {+ rule := newRule("max", `[]`)+ result, err := evaluateMetricThreshold(ctx, rule, projectId)+ if err != nil {+ t.Fatalf("evaluateMetricThreshold failed: %v", err)+ }+ if !result.Fired || len(result.Messages) != 0 || !strings.Contains(result.Message.Subject, "Metric radius.freshness is 22166.00") {+ t.Errorf("result = %+v", result)+ }+ if isPerSeriesRule(rule.RuleType, rule.Config) {+ t.Error("an ungrouped rule keeps the rule-level cooldown")+ }+ })+}++func TestMetricThresholdResultCapsNotifiedSeries(t *testing.T) {+ setupNotificationsTestDB(t)++ cfg := metricThresholdConfig{MetricName: "m", Operator: "gt", ThresholdValue: 1, LookbackMinutes: 5, GroupBy: []string{"host"}}+ var series []metricSeries+ for i := range maxMetricRuleSeriesNotified + 7 {+ series = append(series, metricSeries{Key: fmt.Sprintf("h%03d", i), Value: float64(i + 2)})+ }+ rule := &models.NotificationRule{Id: 9300, CooldownMinutes: 30}+ result := metricThresholdResult(rule, cfg, series, uuid.New())+ if len(result.Messages) != maxMetricRuleSeriesNotified+1 {+ t.Fatalf("messages = %d, want %d", len(result.Messages), maxMetricRuleSeriesNotified+1)+ }+ if !strings.Contains(result.Messages[0].Subject, "m{host=h056}") {+ t.Errorf("first subject = %q, want the worst series first", result.Messages[0].Subject)+ }+ if last := result.Messages[maxMetricRuleSeriesNotified].Subject; !strings.Contains(last, "7 more series") {+ t.Errorf("overflow subject = %q", last)+ }+ if again := metricThresholdResult(rule, cfg, series, uuid.New()); again.Fired {+ t.Errorf("series summarized in the overflow notice must also wait out the cooldown, got %d messages", len(again.Messages))+ }+}+func TestEvaluateTaskFailureRate(t *testing.T) {setupNotificationsTestDB(t)diff --git a/cli/pkg/mcpserver/knowledge/notifications.md b/cli/pkg/mcpserver/knowledge/notifications.mdindex b9209e49..7e768c3d 100644--- a/cli/pkg/mcpserver/knowledge/notifications.md+++ b/cli/pkg/mcpserver/knowledge/notifications.md@@ -27,7 +27,7 @@ These fire from latency/throughput rules, not from an error. There is no hash, n| `endpoint_p95_threshold` / `endpoint_p99_threshold` | `P95 latency 1250ms on <endpoint>` · "reached 1250ms over the last N minutes (threshold: 1000ms)" | `/endpoints?from=<iso>&to=<iso>` || `apdex_drop` | `Apdex dropped to 0.62 (threshold: 0.80)` · "across N requests over the last N minutes" | `/endpoints?from=<iso>&to=<iso>` || `impact_score_critical\|high\|medium` | `Endpoint <endpoint> impact became critical` · "impact score: 0.82. Reason: <reason>" | `/endpoints?from=<iso>&to=<iso>` |-| `metric_threshold` | `Metric <name> is <value> (threshold: gt 100)` | `/metrics?preset=1h` |+| `metric_threshold` | `Metric <name> is <value> (threshold: gt 100)`, or `Metric <name>{<tag>=<value>} is ...` per breaching series when the rule sets `groupBy` | `/metrics?preset=1h` |What the body gives you: the **endpoint name** (parse it from the subject/body; it is not a separate field), the **metric/percentile**, the **current value**, the **threshold**, and the **lookback window**. The `/endpoints?from=&to=` link is only a ~3-minute window around the fire time, so treat the fire time as the onset hint, not the incident window.diff --git a/docs/pages/learn/alerts.mdx b/docs/pages/learn/alerts.mdxindex 7257f97a..8a7a520e 100644--- a/docs/pages/learn/alerts.mdx+++ b/docs/pages/learn/alerts.mdx@@ -272,11 +272,26 @@ Fires when an aggregated [metric](/learn/metrics) value violates the configured| Threshold Value | 90 | The value compared against the aggregate || Lookback Minutes | 5 | Window the aggregate is computed over || Tag Filters | | Optional `key=value` pairs, comma-separated. Only points whose tags match every pair count. Up to 20 |+| Group By Tags | | Optional tag keys, comma-separated. Each combination of their values is evaluated as its own series. Up to 5 |Without tag filters the aggregation spans every series of the metric name. Use a filter to watch one slice, such as `outcome=failed` or `host.name=web-1`. For metrics tagged per host, per disk, or per table, an `avg` can hide one bad series behind healthy ones. Prefer `max` (or `min` for lower-is-bad metrics) so the worst series drives the alert.Over the API the filters are the `tags` object of the rule config, for example `"tags": {"outcome": "failed"}`. An empty key or value answers 422.+#### Alerting per series++Set Group By Tags to alert on each series separately, for example `site_id` for a gauge reported once per gateway. The aggregate is then computed per combination of the listed tags, and every series that breaches sends its own notification naming it:++```+[HS API] Metric radius.accounting_freshness_seconds{site_id=RUINS} is 22166.00 (threshold: gt 900.00)+```++A point without one of the tags falls into a series where that value reads `(empty)`. Tag filters still apply first, so `tags` and `groupBy` combine.++The cooldown is kept per series instead of per rule. A series that keeps breaching is notified again once its cooldown has passed, and a second series that starts breaching meanwhile is notified right away. On an escalation channel each series opens its own page. One evaluation sends at most 50 series notifications, worst first. Any further breaching series are summarized in a single "N more series" notice. Series are read in tag value order and at most 5,000 are evaluated. The per-series cooldowns are held in memory, so after a backend restart a series that is still breaching is notified again.++Over the API the keys are the `groupBy` array of the rule config, for example `"groupBy": ["site_id"]`. An empty or repeated key answers 422.+### No DataFires when no telemetry of the selected kind has been received for the configured number of minutes. Use it to detect a stopped service, a broken deploy, or a misconfigured SDK rather than an application-level problem.diff --git a/frontend/src/lib/types/notifications.ts b/frontend/src/lib/types/notifications.tsindex 115b9268..178377af 100644--- a/frontend/src/lib/types/notifications.ts+++ b/frontend/src/lib/types/notifications.ts@@ -38,6 +38,7 @@ export type NotificationRuleConfig = {thresholdValue?: number;aggregation?: string;tags?: Record<string, string>;+ groupBy?: string[];dataType?: string;silenceMinutes?: number;thresholdCount?: number;diff --git a/frontend/src/routes/notifications/rule-dialog.svelte b/frontend/src/routes/notifications/rule-dialog.svelteindex 3013ca08..9702e7ec 100644--- a/frontend/src/routes/notifications/rule-dialog.svelte+++ b/frontend/src/routes/notifications/rule-dialog.svelte@@ -78,6 +78,7 @@let thresholdValue = $state(90);let aggregation = $state('avg');let metricTags = $state('');+ let metricGroupBy = $state('');let dataType = $state('any');let silenceMinutes = $state(10);let thresholdCount = $state(100);@@ -265,6 +266,7 @@thresholdValue = 90;aggregation = 'avg';metricTags = '';+ metricGroupBy = '';dataType = 'any';silenceMinutes = 10;thresholdCount = 100;@@ -315,6 +317,7 @@metricTags = Object.entries(cfg.tags ?? {}).map(([k, v]) => `${k}=${v}`).join(', ');+ metricGroupBy = (cfg.groupBy ?? []).join(', ');lookbackMinutes = cfg.lookbackMinutes ?? 5;break;case 'no_data':@@ -405,7 +408,11 @@thresholdValue,aggregation,lookbackMinutes,- tags: parseMetricTags(metricTags)+ tags: parseMetricTags(metricTags),+ groupBy: metricGroupBy+ .split(',')+ .map((k) => k.trim())+ .filter(Boolean)};case 'no_data':return { dataType, silenceMinutes };@@ -674,6 +681,13 @@placeholder="outcome=failed, host.name=web-1"/></div>+ <div class="space-y-2">+ <Label for="cfg-metric-group-by">Group By Tags (optional, comma-separated)</Label>+ <Input id="cfg-metric-group-by" bind:value={metricGroupBy} placeholder="host.name" />+ <p class="text-xs text-muted-foreground">+ Evaluates every series separately and names the breaching one in the alert.+ </p>+ </div>{:else if ruleType === 'no_data'}<div class="space-y-2"><Label for="cfg-data-type">Data Type</Label>diff --git a/skills/traceway/SKILL.md b/skills/traceway/SKILL.mdindex 7cc0c6bb..74d8aed6 100644--- a/skills/traceway/SKILL.md+++ b/skills/traceway/SKILL.md@@ -106,7 +106,7 @@ These fire from latency/throughput rules, not from an error. There is no hash, n| `endpoint_p95_threshold` / `endpoint_p99_threshold` | `P95 latency 1250ms on <endpoint>` · "reached 1250ms over the last N minutes (threshold: 1000ms)" | `/endpoints?from=<iso>&to=<iso>` || `apdex_drop` | `Apdex dropped to 0.62 (threshold: 0.80)` · "across N requests over the last N minutes" | `/endpoints?from=<iso>&to=<iso>` || `impact_score_critical\|high\|medium` | `Endpoint <endpoint> impact became critical` · "impact score: 0.82. Reason: <reason>" | `/endpoints?from=<iso>&to=<iso>` |-| `metric_threshold` | `Metric <name> is <value> (threshold: gt 100)` | `/metrics?preset=1h` |+| `metric_threshold` | `Metric <name> is <value> (threshold: gt 100)`, or `Metric <name>{<tag>=<value>} is ...` per breaching series when the rule sets `groupBy` | `/metrics?preset=1h` |What the body gives you: the **endpoint name** (parse it from the subject/body; it is not a separate field), the **metric/percentile**, the **current value**, the **threshold**, and the **lookback window**. The `/endpoints?from=&to=` link is only a ~3-minute window around the fire time, so treat the fire time as the onset hint, not the incident window.