The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
#420
narrow-adventure5533033095e4
diff --git a/backend/app/controllers/notification_rule.controller.go b/backend/app/controllers/notification_rule.controller.goindex 4f93c1bc..6f932e26 100644--- a/backend/app/controllers/notification_rule.controller.go+++ b/backend/app/controllers/notification_rule.controller.go@@ -4,6 +4,7 @@ import ("database/sql""encoding/json""errors"+ "fmt""io""net/http""reflect"@@ -127,9 +128,9 @@ func (ctrl *notificationRuleController) Create(ctx *gin.Context) {ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Channel not found in this project."})return}- message, err := validateSLORuleConfig(tx, projectId, req.RuleType, req.Config)+ message, err := validateRuleConfig(tx, projectId, req.RuleType, req.Config)if err != nil {- ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the SLO rule: %w", err))+ ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the rule: %w", err))return}if message != "" {@@ -231,9 +232,9 @@ func (ctrl *notificationRuleController) Update(ctx *gin.Context) {ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Channel not found in this project."})return}- message, err := validateSLORuleConfig(tx, projectId, req.RuleType, req.Config)+ message, err := validateRuleConfig(tx, projectId, req.RuleType, req.Config)if err != nil {- ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the SLO rule: %w", err))+ ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the rule: %w", err))return}if message != "" {@@ -422,8 +423,25 @@ func (ctrl *notificationRuleController) Snooze(ctx *gin.Context) {var NotificationRuleController = notificationRuleController{}-func validateSLORuleConfig(tx *sql.Tx, projectId uuid.UUID, ruleType string, config json.RawMessage) (string, error) {+const maxMetricRuleTagFilters = 20++func validateRuleConfig(tx *sql.Tx, projectId uuid.UUID, ruleType string, config json.RawMessage) (string, error) {switch ruleType {+ case "metric_threshold":+ var cfg struct {+ Tags map[string]string `json:"tags"`+ }+ if err := json.Unmarshal(config, &cfg); err != nil {+ return "The rule configuration is not valid JSON.", nil+ }+ if len(cfg.Tags) > maxMetricRuleTagFilters {+ return fmt.Sprintf("A metric rule can filter on at most %d tags.", maxMetricRuleTagFilters), nil+ }+ for k, v := range cfg.Tags {+ if strings.TrimSpace(k) == "" || strings.TrimSpace(v) == "" {+ return "Every tag filter needs a key and a value.", nil+ }+ }case "slo_burn_rate":var cfg struct {SloId int `json:"sloId"`diff --git a/backend/app/notifications/evaluator_polled.go b/backend/app/notifications/evaluator_polled.goindex 58a50597..fe80aed8 100644--- a/backend/app/notifications/evaluator_polled.go+++ b/backend/app/notifications/evaluator_polled.go@@ -6,6 +6,8 @@ import ("context""encoding/json""fmt"+ "maps"+ "slices""time""github.com/google/uuid"@@ -219,11 +221,12 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje// --- Metric Threshold ---type metricThresholdConfig struct {- MetricName string `json:"metricName"`- Operator string `json:"operator"`- ThresholdValue float64 `json:"thresholdValue"`- Aggregation string `json:"aggregation"`- LookbackMinutes int `json:"lookbackMinutes"`+ MetricName string `json:"metricName"`+ Operator string `json:"operator"`+ ThresholdValue float64 `json:"thresholdValue"`+ Aggregation string `json:"aggregation"`+ LookbackMinutes int `json:"lookbackMinutes"`+ Tags map[string]string `json:"tags"`}func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {@@ -256,6 +259,10 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,query := fmt.Sprintf("SELECT count(), %s FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", selectExpr)args := []interface{}{projectId, cfg.MetricName, from, now}+ for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {+ query += " AND tags[?] = ?"+ args = append(args, k, cfg.Tags[k])+ }var count uint64var value float64diff --git a/backend/app/notifications/evaluator_polled_sqlite.go b/backend/app/notifications/evaluator_polled_sqlite.goindex a94ed7d9..b1f6c25c 100644--- a/backend/app/notifications/evaluator_polled_sqlite.go+++ b/backend/app/notifications/evaluator_polled_sqlite.go@@ -7,6 +7,8 @@ import ("database/sql""encoding/json""fmt"+ "maps"+ "slices""time""github.com/google/uuid"@@ -241,11 +243,12 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje// --- Metric Threshold ---type metricThresholdConfig struct {- MetricName string `json:"metricName"`- Operator string `json:"operator"`- ThresholdValue float64 `json:"thresholdValue"`- Aggregation string `json:"aggregation"`- LookbackMinutes int `json:"lookbackMinutes"`+ MetricName string `json:"metricName"`+ Operator string `json:"operator"`+ ThresholdValue float64 `json:"thresholdValue"`+ Aggregation string `json:"aggregation"`+ LookbackMinutes int `json:"lookbackMinutes"`+ Tags map[string]string `json:"tags"`}func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {@@ -260,14 +263,24 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,now := time.Now().UTC()from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)+ where := "project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?"+ args := []interface{}{projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)}+ tagExtract := "json_extract"+ if db.IsDuckDBTelemetry() {+ tagExtract = "json_extract_string"+ }+ for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {+ where += " AND " + tagExtract + "(tags, '$.\"' || ? || '\"') = ?"+ args = append(args, k, cfg.Tags[k])+ }+var value float64var err errorswitch cfg.Aggregation {case "last":err = db.TelemetryDB.QueryRowContext(ctx,- "SELECT value FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ? ORDER BY recorded_at DESC LIMIT 1",- projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&value)+ "SELECT value FROM metric_points WHERE "+where+" ORDER BY recorded_at DESC LIMIT 1", args...).Scan(&value)if err == sql.ErrNoRows {return &EvalResult{Fired: false}, nil}@@ -280,8 +293,7 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,pct = 0.99}rows, qErr := db.TelemetryDB.QueryContext(ctx,- "SELECT value FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ? ORDER BY value ASC",- projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))+ "SELECT value FROM metric_points WHERE "+where+" ORDER BY value ASC", args...)if qErr != nil {return nil, qErr}@@ -308,9 +320,9 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,case "sum":aggFunc = "sum"}- query := fmt.Sprintf("SELECT COUNT(value), COALESCE(%s(value), 0) FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", aggFunc)+ query := fmt.Sprintf("SELECT COUNT(value), COALESCE(%s(value), 0) FROM metric_points WHERE %s", aggFunc, where)var count int64- err = db.TelemetryDB.QueryRowContext(ctx, query, projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&count, &value)+ err = db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&count, &value)if err != nil {return nil, err}diff --git a/backend/app/notifications/notifications_sqlite_test.go b/backend/app/notifications/notifications_sqlite_test.goindex 84a19859..e866de88 100644--- a/backend/app/notifications/notifications_sqlite_test.go+++ b/backend/app/notifications/notifications_sqlite_test.go@@ -210,6 +210,68 @@ func setupNotificationsTestDB(t *testing.T) {)`); err != nil {t.Fatalf("failed to create exceptions_v2 table: %v", err)}++ if _, err := telemetryDB.Exec(`CREATE TABLE metric_points (+ project_id TEXT NOT NULL,+ name TEXT NOT NULL DEFAULT '',+ value REAL NOT NULL DEFAULT 0,+ tags TEXT NOT NULL DEFAULT '{}',+ recorded_at DATETIME NOT NULL+ )`); err != nil {+ t.Fatalf("failed to create metric_points table: %v", err)+ }+}++func TestEvaluateMetricThresholdTagFilter(t *testing.T) {+ setupNotificationsTestDB(t)++ ctx := context.Background()+ projectId := uuid.New()+ ts := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano)+ for _, p := range []struct {+ value float64+ tags string+ }{+ {3, `{"outcome":"healed","host":"fw1"}`},+ {4, `{"outcome":"healed","host":"fw2"}`},+ {2, `{"outcome":"failed","host":"fw1"}`},+ } {+ if _, err := db.TelemetryDB.Exec(+ "INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (?, ?, ?, ?, ?)",+ projectId.String(), "syslogd_heal_total", p.value, p.tags, ts); err != nil {+ t.Fatalf("failed to insert metric point: %v", err)+ }+ }++ cases := []struct {+ name string+ agg string+ tags string+ threshold float64+ fired bool+ }{+ {"no filter sums every series", "sum", `{}`, 8, true},+ {"filter keeps only matching series", "sum", `{"outcome":"failed"}`, 2, false},+ {"filter fires on matching series", "sum", `{"outcome":"failed"}`, 1, true},+ {"every filter must match", "max", `{"outcome":"healed","host":"fw1"}`, 3, false},+ {"no matching series never fires", "sum", `{"outcome":"unknown"}`, -1, false},+ {"last honors the filter", "last", `{"host":"fw2"}`, 3.5, true},+ {"percentile honors the filter", "p95", `{"outcome":"failed"}`, 2, false},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ rule := &models.NotificationRule{+ Config: json.RawMessage(fmt.Sprintf(`{"metricName":"syslogd_heal_total","aggregation":%q,"operator":"gt","thresholdValue":%v,"lookbackMinutes":5,"tags":%s}`, tc.agg, tc.threshold, tc.tags)),+ }+ result, err := evaluateMetricThreshold(ctx, rule, projectId)+ if err != nil {+ t.Fatalf("evaluateMetricThreshold failed: %v", err)+ }+ if result.Fired != tc.fired {+ t.Errorf("fired = %v, want %v", result.Fired, tc.fired)+ }+ })+ }}func TestEvaluateTaskFailureRate(t *testing.T) {diff --git a/docs/pages/learn/alerts.mdx b/docs/pages/learn/alerts.mdxindex 7e33545b..7257f97a 100644--- a/docs/pages/learn/alerts.mdx+++ b/docs/pages/learn/alerts.mdx@@ -271,8 +271,11 @@ Fires when an aggregated [metric](/learn/metrics) value violates the configured| Operator | gt | One of `gt`, `gte`, `lt`, `lte`, `eq` || Threshold Value | 90 | The value compared against the aggregate || Lookback Minutes | 5 | Window the aggregate is computed over |+| Tag Filters | | Optional `key=value` pairs, comma-separated. Only points whose tags match every pair count. Up to 20 |-The aggregation spans every series of the metric name, with no tag filter. For metrics tagged per host, per disk, or per table, an `avg` can hide one bad series behind healthy ones; prefer `max` (or `min` for lower-is-bad metrics) so the worst series drives the alert.+Without tag filters the aggregation spans every series of the metric name. Use a filter to watch one slice, such as `outcome=failed` or `host.name=web-1`. For metrics tagged per host, per disk, or per table, an `avg` can hide one bad series behind healthy ones. Prefer `max` (or `min` for lower-is-bad metrics) so the worst series drives the alert.++Over the API the filters are the `tags` object of the rule config, for example `"tags": {"outcome": "failed"}`. An empty key or value answers 422.### No Datadiff --git a/frontend/src/routes/notifications/rule-dialog.svelte b/frontend/src/routes/notifications/rule-dialog.svelteindex 9c358a6f..3013ca08 100644--- a/frontend/src/routes/notifications/rule-dialog.svelte+++ b/frontend/src/routes/notifications/rule-dialog.svelte@@ -77,6 +77,7 @@let operator = $state('gt');let thresholdValue = $state(90);let aggregation = $state('avg');+ let metricTags = $state('');let dataType = $state('any');let silenceMinutes = $state(10);let thresholdCount = $state(100);@@ -263,6 +264,7 @@operator = 'gt';thresholdValue = 90;aggregation = 'avg';+ metricTags = '';dataType = 'any';silenceMinutes = 10;thresholdCount = 100;@@ -310,6 +312,9 @@operator = cfg.operator ?? 'gt';thresholdValue = cfg.thresholdValue ?? 90;aggregation = cfg.aggregation ?? 'avg';+ metricTags = Object.entries(cfg.tags ?? {})+ .map(([k, v]) => `${k}=${v}`)+ .join(', ');lookbackMinutes = cfg.lookbackMinutes ?? 5;break;case 'no_data':@@ -373,6 +378,17 @@}}+ function parseMetricTags(value: string): Record<string, string> {+ const tags: Record<string, string> = {};+ for (const part of value.split(',')) {+ if (!part.trim()) continue;+ const eq = part.indexOf('=');+ const key = (eq === -1 ? part : part.slice(0, eq)).trim();+ tags[key] = eq === -1 ? '' : part.slice(eq + 1).trim();+ }+ return tags;+ }+function buildConfig(): NotificationRuleConfig {switch (ruleType) {case 'error_rate_threshold':@@ -383,7 +399,14 @@case 'apdex_drop':return { thresholdApdex, lookbackMinutes, minRequests };case 'metric_threshold':- return { metricName, operator, thresholdValue, aggregation, lookbackMinutes, tags: {} };+ return {+ metricName,+ operator,+ thresholdValue,+ aggregation,+ lookbackMinutes,+ tags: parseMetricTags(metricTags)+ };case 'no_data':return { dataType, silenceMinutes };case 'error_count_threshold':@@ -643,6 +666,14 @@<Label for="cfg-lookback-m">Lookback (minutes)</Label><Input id="cfg-lookback-m" type="number" bind:value={lookbackMinutes} min="1" /></div>+ <div class="space-y-2">+ <Label for="cfg-metric-tags">Tag Filters (optional, comma-separated)</Label>+ <Input+ id="cfg-metric-tags"+ bind:value={metricTags}+ placeholder="outcome=failed, host.name=web-1"+ />+ </div>{:else if ruleType === 'no_data'}<div class="space-y-2"><Label for="cfg-data-type">Data Type</Label>