Skip to content

The only tool you need to know what is happening and how to fix it.

#420

narrow-adventure5533033095e4
diff --git a/backend/app/controllers/notification_rule.controller.go b/backend/app/controllers/notification_rule.controller.go
index 4f93c1bc..6f932e26 100644
--- a/backend/app/controllers/notification_rule.controller.go
+++ b/backend/app/controllers/notification_rule.controller.go
@@ -4,6 +4,7 @@ import (
"database/sql"
"encoding/json"
"errors"
+ "fmt"
"io"
"net/http"
"reflect"
@@ -127,9 +128,9 @@ func (ctrl *notificationRuleController) Create(ctx *gin.Context) {
ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Channel not found in this project."})
return
}
- message, err := validateSLORuleConfig(tx, projectId, req.RuleType, req.Config)
+ message, err := validateRuleConfig(tx, projectId, req.RuleType, req.Config)
if err != nil {
- ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the SLO rule: %w", err))
+ ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the rule: %w", err))
return
}
if message != "" {
@@ -231,9 +232,9 @@ func (ctrl *notificationRuleController) Update(ctx *gin.Context) {
ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Channel not found in this project."})
return
}
- message, err := validateSLORuleConfig(tx, projectId, req.RuleType, req.Config)
+ message, err := validateRuleConfig(tx, projectId, req.RuleType, req.Config)
if err != nil {
- ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the SLO rule: %w", err))
+ ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to validate the rule: %w", err))
return
}
if message != "" {
@@ -422,8 +423,25 @@ func (ctrl *notificationRuleController) Snooze(ctx *gin.Context) {
var NotificationRuleController = notificationRuleController{}
-func validateSLORuleConfig(tx *sql.Tx, projectId uuid.UUID, ruleType string, config json.RawMessage) (string, error) {
+const maxMetricRuleTagFilters = 20
+
+func validateRuleConfig(tx *sql.Tx, projectId uuid.UUID, ruleType string, config json.RawMessage) (string, error) {
switch ruleType {
+ case "metric_threshold":
+ var cfg struct {
+ Tags map[string]string `json:"tags"`
+ }
+ if err := json.Unmarshal(config, &cfg); err != nil {
+ return "The rule configuration is not valid JSON.", nil
+ }
+ if len(cfg.Tags) > maxMetricRuleTagFilters {
+ return fmt.Sprintf("A metric rule can filter on at most %d tags.", maxMetricRuleTagFilters), nil
+ }
+ for k, v := range cfg.Tags {
+ if strings.TrimSpace(k) == "" || strings.TrimSpace(v) == "" {
+ return "Every tag filter needs a key and a value.", nil
+ }
+ }
case "slo_burn_rate":
var cfg struct {
SloId int `json:"sloId"`
diff --git a/backend/app/notifications/evaluator_polled.go b/backend/app/notifications/evaluator_polled.go
index 58a50597..fe80aed8 100644
--- a/backend/app/notifications/evaluator_polled.go
+++ b/backend/app/notifications/evaluator_polled.go
@@ -6,6 +6,8 @@ import (
"context"
"encoding/json"
"fmt"
+ "maps"
+ "slices"
"time"
"github.com/google/uuid"
@@ -219,11 +221,12 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje
// --- Metric Threshold ---
type metricThresholdConfig struct {
- MetricName string `json:"metricName"`
- Operator string `json:"operator"`
- ThresholdValue float64 `json:"thresholdValue"`
- Aggregation string `json:"aggregation"`
- LookbackMinutes int `json:"lookbackMinutes"`
+ MetricName string `json:"metricName"`
+ Operator string `json:"operator"`
+ ThresholdValue float64 `json:"thresholdValue"`
+ Aggregation string `json:"aggregation"`
+ LookbackMinutes int `json:"lookbackMinutes"`
+ Tags map[string]string `json:"tags"`
}
func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
@@ -256,6 +259,10 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
query := fmt.Sprintf("SELECT count(), %s FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", selectExpr)
args := []interface{}{projectId, cfg.MetricName, from, now}
+ for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {
+ query += " AND tags[?] = ?"
+ args = append(args, k, cfg.Tags[k])
+ }
var count uint64
var value float64
diff --git a/backend/app/notifications/evaluator_polled_sqlite.go b/backend/app/notifications/evaluator_polled_sqlite.go
index a94ed7d9..b1f6c25c 100644
--- a/backend/app/notifications/evaluator_polled_sqlite.go
+++ b/backend/app/notifications/evaluator_polled_sqlite.go
@@ -7,6 +7,8 @@ import (
"database/sql"
"encoding/json"
"fmt"
+ "maps"
+ "slices"
"time"
"github.com/google/uuid"
@@ -241,11 +243,12 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje
// --- Metric Threshold ---
type metricThresholdConfig struct {
- MetricName string `json:"metricName"`
- Operator string `json:"operator"`
- ThresholdValue float64 `json:"thresholdValue"`
- Aggregation string `json:"aggregation"`
- LookbackMinutes int `json:"lookbackMinutes"`
+ MetricName string `json:"metricName"`
+ Operator string `json:"operator"`
+ ThresholdValue float64 `json:"thresholdValue"`
+ Aggregation string `json:"aggregation"`
+ LookbackMinutes int `json:"lookbackMinutes"`
+ Tags map[string]string `json:"tags"`
}
func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
@@ -260,14 +263,24 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
+ where := "project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?"
+ args := []interface{}{projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)}
+ tagExtract := "json_extract"
+ if db.IsDuckDBTelemetry() {
+ tagExtract = "json_extract_string"
+ }
+ for _, k := range slices.Sorted(maps.Keys(cfg.Tags)) {
+ where += " AND " + tagExtract + "(tags, '$.\"' || ? || '\"') = ?"
+ args = append(args, k, cfg.Tags[k])
+ }
+
var value float64
var err error
switch cfg.Aggregation {
case "last":
err = db.TelemetryDB.QueryRowContext(ctx,
- "SELECT value FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ? ORDER BY recorded_at DESC LIMIT 1",
- projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&value)
+ "SELECT value FROM metric_points WHERE "+where+" ORDER BY recorded_at DESC LIMIT 1", args...).Scan(&value)
if err == sql.ErrNoRows {
return &EvalResult{Fired: false}, nil
}
@@ -280,8 +293,7 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
pct = 0.99
}
rows, qErr := db.TelemetryDB.QueryContext(ctx,
- "SELECT value FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ? ORDER BY value ASC",
- projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
+ "SELECT value FROM metric_points WHERE "+where+" ORDER BY value ASC", args...)
if qErr != nil {
return nil, qErr
}
@@ -308,9 +320,9 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
case "sum":
aggFunc = "sum"
}
- query := fmt.Sprintf("SELECT COUNT(value), COALESCE(%s(value), 0) FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", aggFunc)
+ query := fmt.Sprintf("SELECT COUNT(value), COALESCE(%s(value), 0) FROM metric_points WHERE %s", aggFunc, where)
var count int64
- err = db.TelemetryDB.QueryRowContext(ctx, query, projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&count, &value)
+ err = db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&count, &value)
if err != nil {
return nil, err
}
diff --git a/backend/app/notifications/notifications_sqlite_test.go b/backend/app/notifications/notifications_sqlite_test.go
index 84a19859..e866de88 100644
--- a/backend/app/notifications/notifications_sqlite_test.go
+++ b/backend/app/notifications/notifications_sqlite_test.go
@@ -210,6 +210,68 @@ func setupNotificationsTestDB(t *testing.T) {
)`); err != nil {
t.Fatalf("failed to create exceptions_v2 table: %v", err)
}
+
+ if _, err := telemetryDB.Exec(`CREATE TABLE metric_points (
+ project_id TEXT NOT NULL,
+ name TEXT NOT NULL DEFAULT '',
+ value REAL NOT NULL DEFAULT 0,
+ tags TEXT NOT NULL DEFAULT '{}',
+ recorded_at DATETIME NOT NULL
+ )`); err != nil {
+ t.Fatalf("failed to create metric_points table: %v", err)
+ }
+}
+
+func TestEvaluateMetricThresholdTagFilter(t *testing.T) {
+ setupNotificationsTestDB(t)
+
+ ctx := context.Background()
+ projectId := uuid.New()
+ ts := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339Nano)
+ for _, p := range []struct {
+ value float64
+ tags string
+ }{
+ {3, `{"outcome":"healed","host":"fw1"}`},
+ {4, `{"outcome":"healed","host":"fw2"}`},
+ {2, `{"outcome":"failed","host":"fw1"}`},
+ } {
+ if _, err := db.TelemetryDB.Exec(
+ "INSERT INTO metric_points (project_id, name, value, tags, recorded_at) VALUES (?, ?, ?, ?, ?)",
+ projectId.String(), "syslogd_heal_total", p.value, p.tags, ts); err != nil {
+ t.Fatalf("failed to insert metric point: %v", err)
+ }
+ }
+
+ cases := []struct {
+ name string
+ agg string
+ tags string
+ threshold float64
+ fired bool
+ }{
+ {"no filter sums every series", "sum", `{}`, 8, true},
+ {"filter keeps only matching series", "sum", `{"outcome":"failed"}`, 2, false},
+ {"filter fires on matching series", "sum", `{"outcome":"failed"}`, 1, true},
+ {"every filter must match", "max", `{"outcome":"healed","host":"fw1"}`, 3, false},
+ {"no matching series never fires", "sum", `{"outcome":"unknown"}`, -1, false},
+ {"last honors the filter", "last", `{"host":"fw2"}`, 3.5, true},
+ {"percentile honors the filter", "p95", `{"outcome":"failed"}`, 2, false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ rule := &models.NotificationRule{
+ Config: json.RawMessage(fmt.Sprintf(`{"metricName":"syslogd_heal_total","aggregation":%q,"operator":"gt","thresholdValue":%v,"lookbackMinutes":5,"tags":%s}`, tc.agg, tc.threshold, tc.tags)),
+ }
+ result, err := evaluateMetricThreshold(ctx, rule, projectId)
+ if err != nil {
+ t.Fatalf("evaluateMetricThreshold failed: %v", err)
+ }
+ if result.Fired != tc.fired {
+ t.Errorf("fired = %v, want %v", result.Fired, tc.fired)
+ }
+ })
+ }
}
func TestEvaluateTaskFailureRate(t *testing.T) {
diff --git a/docs/pages/learn/alerts.mdx b/docs/pages/learn/alerts.mdx
index 7e33545b..7257f97a 100644
--- a/docs/pages/learn/alerts.mdx
+++ b/docs/pages/learn/alerts.mdx
@@ -271,8 +271,11 @@ Fires when an aggregated [metric](/learn/metrics) value violates the configured
| Operator | gt | One of `gt`, `gte`, `lt`, `lte`, `eq` |
| Threshold Value | 90 | The value compared against the aggregate |
| Lookback Minutes | 5 | Window the aggregate is computed over |
+| Tag Filters | | Optional `key=value` pairs, comma-separated. Only points whose tags match every pair count. Up to 20 |
-The aggregation spans every series of the metric name, with no tag filter. For metrics tagged per host, per disk, or per table, an `avg` can hide one bad series behind healthy ones; prefer `max` (or `min` for lower-is-bad metrics) so the worst series drives the alert.
+Without tag filters the aggregation spans every series of the metric name. Use a filter to watch one slice, such as `outcome=failed` or `host.name=web-1`. For metrics tagged per host, per disk, or per table, an `avg` can hide one bad series behind healthy ones. Prefer `max` (or `min` for lower-is-bad metrics) so the worst series drives the alert.
+
+Over the API the filters are the `tags` object of the rule config, for example `"tags": {"outcome": "failed"}`. An empty key or value answers 422.
### No Data
diff --git a/frontend/src/routes/notifications/rule-dialog.svelte b/frontend/src/routes/notifications/rule-dialog.svelte
index 9c358a6f..3013ca08 100644
--- a/frontend/src/routes/notifications/rule-dialog.svelte
+++ b/frontend/src/routes/notifications/rule-dialog.svelte
@@ -77,6 +77,7 @@
let operator = $state('gt');
let thresholdValue = $state(90);
let aggregation = $state('avg');
+ let metricTags = $state('');
let dataType = $state('any');
let silenceMinutes = $state(10);
let thresholdCount = $state(100);
@@ -263,6 +264,7 @@
operator = 'gt';
thresholdValue = 90;
aggregation = 'avg';
+ metricTags = '';
dataType = 'any';
silenceMinutes = 10;
thresholdCount = 100;
@@ -310,6 +312,9 @@
operator = cfg.operator ?? 'gt';
thresholdValue = cfg.thresholdValue ?? 90;
aggregation = cfg.aggregation ?? 'avg';
+ metricTags = Object.entries(cfg.tags ?? {})
+ .map(([k, v]) => `${k}=${v}`)
+ .join(', ');
lookbackMinutes = cfg.lookbackMinutes ?? 5;
break;
case 'no_data':
@@ -373,6 +378,17 @@
}
}
+ function parseMetricTags(value: string): Record<string, string> {
+ const tags: Record<string, string> = {};
+ for (const part of value.split(',')) {
+ if (!part.trim()) continue;
+ const eq = part.indexOf('=');
+ const key = (eq === -1 ? part : part.slice(0, eq)).trim();
+ tags[key] = eq === -1 ? '' : part.slice(eq + 1).trim();
+ }
+ return tags;
+ }
+
function buildConfig(): NotificationRuleConfig {
switch (ruleType) {
case 'error_rate_threshold':
@@ -383,7 +399,14 @@
case 'apdex_drop':
return { thresholdApdex, lookbackMinutes, minRequests };
case 'metric_threshold':
- return { metricName, operator, thresholdValue, aggregation, lookbackMinutes, tags: {} };
+ return {
+ metricName,
+ operator,
+ thresholdValue,
+ aggregation,
+ lookbackMinutes,
+ tags: parseMetricTags(metricTags)
+ };
case 'no_data':
return { dataType, silenceMinutes };
case 'error_count_threshold':
@@ -643,6 +666,14 @@
<Label for="cfg-lookback-m">Lookback (minutes)</Label>
<Input id="cfg-lookback-m" type="number" bind:value={lookbackMinutes} min="1" />
</div>
+ <div class="space-y-2">
+ <Label for="cfg-metric-tags">Tag Filters (optional, comma-separated)</Label>
+ <Input
+ id="cfg-metric-tags"
+ bind:value={metricTags}
+ placeholder="outcome=failed, host.name=web-1"
+ />
+ </div>
{:else if ruleType === 'no_data'}
<div class="space-y-2">
<Label for="cfg-data-type">Data Type</Label>