Skip to content

The only tool you need to know what is happening and how to fix it.

feat: added additional notification types (#201)

narrow-adventure69221788b03f
diff --git a/CLAUDE.md b/CLAUDE.md
index 99838234..ddd5f32f 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -251,6 +251,9 @@ POSTGRES_USERNAME=traceway
POSTGRES_PASSWORD=
POSTGRES_SSLMODE=disable
+# Notifications
+NOTIFICATION_POLL_SECONDS=60 # polled rule evaluation interval; minimum 5, invalid values fall back to 60
+
# Retention (see "Data Retention" section below)
SQLITE_RETENTION_DAYS=30 # 0 to disable; only applies in SQLite mode
SESSION_RECORDING_RETENTION_DAYS=30 # 0 to disable; only applies when STORAGE_TYPE=local
@@ -706,7 +709,7 @@ In SQLite mode (`DB_TYPE=sqlite`), the backend uses **two separate SQLite databa
**Main DB tables** (`db.DB` — transactional, uses lit with `*sql.Tx`):
- `users`, `organizations`, `organization_users`, `projects`, `invitations`
- `source_maps`, `metric_registry`, `widget_groups`, `widget_group_widgets`
-- `notification_channels`, `notification_rules`, `notification_history`
+- `notification_channels`, `notification_rules`
**Telemetry DB tables** (`db.TelemetryDB` — non-transactional, uses lit with `db.TelemetryDB` directly):
- `endpoints`, `tasks`, `exception_stack_traces`, `spans`, `metric_points`
@@ -752,7 +755,7 @@ Retention is handled in three different ways depending on the deployment.
| `log_records` | **30 days** | `0045_create_log_records.up.sql` |
| All other CH tables (`transactions`, `exception_stack_traces`, `tasks`, `spans`, `sessions`, `ai_traces`, `session_recordings`, `fired_notifications`, `archived_exceptions`, `slow_endpoints`, `endpoints`, etc.) | **No TTL — retained indefinitely** | — |
-**2. SQLite — `retention.Start` worker** (`backend/app/retention/sqlite.go`). In SQLite mode, neither `db.DB` nor `db.TelemetryDB` has any built-in expiry, so a background worker fires once at startup and then every hour and runs a `DELETE FROM <table> WHERE <time_column> < cutoff` against each telemetry table plus `notification_history`.
+**2. SQLite — `retention.Start` worker** (`backend/app/retention/sqlite.go`). In SQLite mode, neither `db.DB` nor `db.TelemetryDB` has any built-in expiry, so a background worker fires once at startup and then every hour and runs a `DELETE FROM <table> WHERE <time_column> < cutoff` against each telemetry table.
| Variable | Default | Notes |
|----------|---------|-------|
@@ -766,7 +769,6 @@ Tables it prunes (and the column used):
| Telemetry | `log_records` | `timestamp` |
| Telemetry | `sessions` | `started_at` |
| Telemetry | `fired_notifications` | `fired_at` |
-| Main (`db.DB`) | `notification_history` | `created_at` |
`archived_exceptions` (per-hash flags) and `slow_endpoints` (per-endpoint config) are intentionally skipped — they are not time-series data.
diff --git a/backend/app/config/config.go b/backend/app/config/config.go
index 49cb45a6..798a0122 100644
--- a/backend/app/config/config.go
+++ b/backend/app/config/config.go
@@ -40,6 +40,8 @@ type Cfg struct {
SourceMapDiskCacheMaxMB string
SymbolicatorParser string
+ NotificationPollSeconds string
+
SMTPEnabled string
SMTPHost string
SMTPPort string
@@ -119,6 +121,8 @@ func LoadFromEnv() *Cfg {
SourceMapDiskCacheMaxMB: os.Getenv("SOURCEMAP_DISK_CACHE_MAX_MB"),
SymbolicatorParser: os.Getenv("SYMBOLICATOR_PARSER"),
+ NotificationPollSeconds: os.Getenv("NOTIFICATION_POLL_SECONDS"),
+
SMTPEnabled: os.Getenv("SMTP_ENABLED"),
SMTPHost: os.Getenv("SMTP_HOST"),
SMTPPort: os.Getenv("SMTP_PORT"),
diff --git a/backend/app/controllers/notification_history.controller.go b/backend/app/controllers/notification_history.controller.go
index f8024336..801ef165 100644
--- a/backend/app/controllers/notification_history.controller.go
+++ b/backend/app/controllers/notification_history.controller.go
@@ -1,12 +1,10 @@
package controllers
import (
- "database/sql"
"net/http"
"time"
"github.com/gin-gonic/gin"
- "github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/repositories"
@@ -56,36 +54,27 @@ func (ctrl *notificationHistoryController) List(ctx *gin.Context) {
toTime = &t
}
- type historyResult struct {
- Items []*models.NotificationHistory
- Total int64
- }
-
- result, err := db.ExecuteTransaction(func(tx *sql.Tx) (historyResult, error) {
- items, total, err := repositories.NotificationHistoryRepository.FindByProject(tx, projectId, page, pageSize, request.Search, fromTime, toTime)
- return historyResult{Items: items, Total: total}, err
- })
+ items, total, err := repositories.FiredNotificationRepository.FindByProject(ctx.Request.Context(), projectId, page, pageSize, request.Search, fromTime, toTime)
if err != nil {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to list notification history: %w", err))
return
}
- totalPages := result.Total / int64(pageSize)
- if result.Total%int64(pageSize) != 0 {
+ totalPages := total / int64(pageSize)
+ if total%int64(pageSize) != 0 {
totalPages++
}
- data := result.Items
- if data == nil {
- data = []*models.NotificationHistory{}
+ if items == nil {
+ items = []*models.NotificationHistoryEntry{}
}
- ctx.JSON(http.StatusOK, PaginatedResponse[*models.NotificationHistory]{
- Data: data,
+ ctx.JSON(http.StatusOK, PaginatedResponse[*models.NotificationHistoryEntry]{
+ Data: items,
Pagination: Pagination{
Page: page,
PageSize: pageSize,
- Total: result.Total,
+ Total: total,
TotalPages: totalPages,
},
})
diff --git a/backend/app/controllers/notification_rule.controller.go b/backend/app/controllers/notification_rule.controller.go
index 10ecef6c..df83530a 100644
--- a/backend/app/controllers/notification_rule.controller.go
+++ b/backend/app/controllers/notification_rule.controller.go
@@ -11,6 +11,7 @@ import (
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
+ "github.com/tracewayapp/traceway/backend/app/notifications"
"github.com/tracewayapp/traceway/backend/app/repositories"
traceway "go.tracewayapp.com"
)
@@ -228,6 +229,7 @@ func (ctrl *notificationRuleController) Update(ctx *gin.Context) {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to update notification rule: %w", err))
return
}
+ notifications.ClearRuleState(id)
ctx.JSON(http.StatusOK, existing)
}
@@ -261,6 +263,7 @@ func (ctrl *notificationRuleController) Delete(ctx *gin.Context) {
ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("failed to delete notification rule: %w", err))
return
}
+ notifications.ClearRuleState(id)
ctx.JSON(http.StatusOK, gin.H{"deleted": true})
}
diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.go
index 2a27805c..58de7bbf 100644
--- a/backend/app/controllers/otelcontrollers/otel.controller.go
+++ b/backend/app/controllers/otelcontrollers/otel.controller.go
@@ -136,6 +136,11 @@ func (o otelController) ExportTraces(c *gin.Context) {
exceptionHashes = append(exceptionHashes, ex.ExceptionHash)
}
+ var aiTraceInfos []hooks.AiTraceInfo
+ for _, at := range aiTraces {
+ aiTraceInfos = append(aiTraceInfos, hooks.AiTraceInfo{TraceName: at.TraceName, TotalCost: at.TotalCost})
+ }
+
if project != nil && project.OrganizationId != nil {
hooks.BroadcastReport(hooks.ReportEvent{
OrganizationId: *project.OrganizationId,
@@ -144,6 +149,7 @@ func (o otelController) ExportTraces(c *gin.Context) {
ErrorCount: len(exceptions),
TaskCount: len(tasks),
ExceptionHashes: exceptionHashes,
+ AiTraces: aiTraceInfos,
})
}
diff --git a/backend/app/migrations/ch/0064_add_url_to_fired_notifications.up.sql b/backend/app/migrations/ch/0064_add_url_to_fired_notifications.up.sql
new file mode 100644
index 00000000..30b5ff0a
--- /dev/null
+++ b/backend/app/migrations/ch/0064_add_url_to_fired_notifications.up.sql
@@ -0,0 +1 @@
+ALTER TABLE fired_notifications ADD COLUMN url String DEFAULT ''
diff --git a/backend/app/migrations/pg/0040_drop_notification_history.up.sql b/backend/app/migrations/pg/0040_drop_notification_history.up.sql
new file mode 100644
index 00000000..326c4587
--- /dev/null
+++ b/backend/app/migrations/pg/0040_drop_notification_history.up.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS notification_history;
diff --git a/backend/app/migrations/sqlite/0014_drop_notification_history.up.sql b/backend/app/migrations/sqlite/0014_drop_notification_history.up.sql
new file mode 100644
index 00000000..fac987b2
--- /dev/null
+++ b/backend/app/migrations/sqlite/0014_drop_notification_history.up.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS notification_history
diff --git a/backend/app/migrations/sqlite_telemetry/0011_add_url_to_fired_notifications.up.sql b/backend/app/migrations/sqlite_telemetry/0011_add_url_to_fired_notifications.up.sql
new file mode 100644
index 00000000..f88df507
--- /dev/null
+++ b/backend/app/migrations/sqlite_telemetry/0011_add_url_to_fired_notifications.up.sql
@@ -0,0 +1 @@
+ALTER TABLE fired_notifications ADD COLUMN url TEXT NOT NULL DEFAULT ''
diff --git a/backend/app/models/models.go b/backend/app/models/models.go
index 6501cf08..3a8710db 100644
--- a/backend/app/models/models.go
+++ b/backend/app/models/models.go
@@ -12,12 +12,6 @@ func (metricRegistryNaming) GetTableNameFromStructName(string) string {
return "metric_registry"
}
-type notificationHistoryNaming struct{ lit.DefaultDbNamingStrategy }
-
-func (notificationHistoryNaming) GetTableNameFromStructName(string) string {
- return "notification_history"
-}
-
func Init(driver lit.Driver) {
lit.RegisterModel[Project](driver)
lit.RegisterModel[User](driver)
@@ -36,7 +30,6 @@ func Init(driver lit.Driver) {
lit.RegisterModel[WidgetGroupWidget](driver)
lit.RegisterModel[NotificationChannel](driver)
lit.RegisterModel[NotificationRule](driver)
- lit.RegisterModelWithNaming[NotificationHistory](driver, notificationHistoryNaming{})
lit.RegisterModel[NotificationRuleWithChannel](driver)
for _, register := range ExtensionModelRegistrations {
diff --git a/backend/app/models/notification.model.go b/backend/app/models/notification.model.go
index 4e597923..2b15a00f 100644
--- a/backend/app/models/notification.model.go
+++ b/backend/app/models/notification.model.go
@@ -35,21 +35,19 @@ type NotificationRule struct {
UpdatedAt time.Time `json:"updatedAt" lit:"updated_at"`
}
-type NotificationHistory struct {
- Id int `json:"id" lit:"id"`
- ProjectId uuid.UUID `json:"projectId" lit:"project_id"`
- RuleId *int `json:"ruleId" lit:"rule_id"`
- ChannelId *int `json:"channelId" lit:"channel_id"`
- RuleType string `json:"ruleType" lit:"rule_type"`
- RuleName string `json:"ruleName" lit:"rule_name"`
- ChannelName string `json:"channelName" lit:"channel_name"`
- Severity string `json:"severity" lit:"severity"`
- Subject string `json:"subject" lit:"subject"`
- Body string `json:"body" lit:"body"`
- Status string `json:"status" lit:"status"`
- ErrorMessage *string `json:"errorMessage" lit:"error_message"`
- URL string `json:"url" lit:"url"`
- CreatedAt time.Time `json:"createdAt" lit:"created_at"`
+type NotificationHistoryEntry struct {
+ RuleId int `json:"ruleId"`
+ RuleType string `json:"ruleType"`
+ RuleName string `json:"ruleName"`
+ ChannelType string `json:"channelType"`
+ ChannelName string `json:"channelName"`
+ Severity string `json:"severity"`
+ Subject string `json:"subject"`
+ Body string `json:"body"`
+ Status string `json:"status"`
+ ErrorMessage string `json:"errorMessage"`
+ URL string `json:"url"`
+ CreatedAt time.Time `json:"createdAt"`
}
type NotificationRuleWithChannel struct {
diff --git a/backend/app/notifications/cooldown.go b/backend/app/notifications/cooldown.go
index 05fd5440..e5ba5e2c 100644
--- a/backend/app/notifications/cooldown.go
+++ b/backend/app/notifications/cooldown.go
@@ -2,6 +2,8 @@ package notifications
import (
"context"
+ "fmt"
+ "strings"
"sync"
"time"
@@ -31,6 +33,51 @@ func (m *cooldownTracker) recordFire(ruleId int) {
m.fired[ruleId] = time.Now()
}
+func ruleStatePrefix(ruleId int) string {
+ return fmt.Sprintf("%d:", ruleId)
+}
+
+func errorDedupKey(ruleId int, hash string) string {
+ return ruleStatePrefix(ruleId) + hash
+}
+
+func aiCostDedupKey(ruleId int, traceName string) string {
+ return ruleStatePrefix(ruleId) + "ai_cost:" + traceName
+}
+
+func ClearRuleState(ruleId int) {
+ cooldowns.mu.Lock()
+ delete(cooldowns.fired, ruleId)
+ cooldowns.mu.Unlock()
+
+ prefix := ruleStatePrefix(ruleId)
+ dedup.mu.Lock()
+ for k := range dedup.seen {
+ if strings.HasPrefix(k, prefix) {
+ delete(dedup.seen, k)
+ }
+ }
+ dedup.mu.Unlock()
+
+ impactStateMu.Lock()
+ for k := range impactState {
+ if strings.HasPrefix(k, prefix) {
+ delete(impactState, k)
+ }
+ }
+ impactStateMu.Unlock()
+}
+
+func (m *cooldownTracker) seed(entries map[int]time.Time) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ for id, t := range entries {
+ if existing, ok := m.fired[id]; !ok || t.After(existing) {
+ m.fired[id] = t
+ }
+ }
+}
+
type dedupTracker struct {
mu sync.RWMutex
seen map[string]time.Time
diff --git a/backend/app/notifications/dispatch.go b/backend/app/notifications/dispatch.go
index 440cd296..1a8dc5a5 100644
--- a/backend/app/notifications/dispatch.go
+++ b/backend/app/notifications/dispatch.go
@@ -27,14 +27,12 @@ func dispatch(rule *models.NotificationRuleWithChannel, msg Message) {
return repositories.NotificationChannelRepository.FindById(tx, rule.ChannelId)
})
if dbErr != nil || channel == nil {
- recordHistory(rule, msg, "failed", "failed to load channel")
recordFiredNotification(rule, msg, "failed", "failed to load channel")
return
}
adapter, err := NewAdapter(channel.ChannelType, channel.Config)
if err != nil {
- recordHistory(rule, msg, "failed", err.Error())
recordFiredNotification(rule, msg, "failed", err.Error())
return
}
@@ -50,15 +48,13 @@ func dispatch(rule *models.NotificationRuleWithChannel, msg Message) {
err = adapter.Send(ctx, msg)
if err != nil {
- recordHistory(rule, msg, "failed", err.Error())
recordFiredNotification(rule, msg, "failed", err.Error())
traceway.CaptureException(fmt.Errorf("notification dispatch failed (rule=%d, channel=%s): %w", rule.Id, rule.ChannelName, err))
return
}
- recordHistory(rule, msg, "sent", "")
- recordFiredNotification(rule, msg, "sent", "")
cooldowns.recordFire(rule.Id)
+ recordFiredNotification(rule, msg, "sent", "")
}
func recordFiredNotification(rule *models.NotificationRuleWithChannel, msg Message, status string, errorMsg string) {
@@ -78,6 +74,7 @@ func recordFiredNotification(rule *models.NotificationRuleWithChannel, msg Messa
Status: status,
ErrorMsg: sanitizeForDB(errorMsg),
Endpoint: msg.Endpoint,
+ URL: msg.URL,
FiredAt: time.Now().UTC(),
})
if err != nil {
@@ -86,41 +83,3 @@ func recordFiredNotification(rule *models.NotificationRuleWithChannel, msg Messa
}()
}
-func recordHistory(rule *models.NotificationRuleWithChannel, msg Message, status string, errorMsg string) {
- var errMsgPtr *string
- if errorMsg != "" {
- errMsgPtr = &errorMsg
- }
-
- ruleId := rule.Id
- channelId := rule.ChannelId
-
- history := &models.NotificationHistory{
- ProjectId: rule.ProjectId,
- RuleId: &ruleId,
- ChannelId: &channelId,
- RuleType: rule.RuleType,
- RuleName: rule.Name,
- ChannelName: rule.ChannelName,
- Severity: string(msg.Severity),
- Subject: sanitizeForDB(msg.Subject),
- Body: sanitizeForDB(msg.Body),
- Status: status,
- ErrorMessage: errMsgPtr,
- URL: msg.URL,
- CreatedAt: time.Now().UTC(),
- }
-
- if errMsgPtr != nil {
- sanitized := sanitizeForDB(*errMsgPtr)
- errMsgPtr = &sanitized
- history.ErrorMessage = errMsgPtr
- }
-
- _, dbErr := db.ExecuteTransaction(func(tx *sql.Tx) (int, error) {
- return repositories.NotificationHistoryRepository.Create(tx, history)
- })
- if dbErr != nil {
- traceway.CaptureException(fmt.Errorf("failed to record notification history: %w", dbErr))
- }
-}
diff --git a/backend/app/notifications/evaluator.go b/backend/app/notifications/evaluator.go
index cb861f12..4b7a1f48 100644
--- a/backend/app/notifications/evaluator.go
+++ b/backend/app/notifications/evaluator.go
@@ -4,6 +4,7 @@ import (
"context"
"database/sql"
"fmt"
+ "strconv"
"time"
"github.com/tracewayapp/traceway/backend/app/config"
@@ -15,15 +16,35 @@ import (
func StartEvaluator(ctx context.Context) {
config.Logln("Starting notification evaluator")
+ seedCooldowns(ctx)
startDedupPurger(ctx)
registerReportHook()
go startPolledLoop(ctx)
}
+func pollInterval() time.Duration {
+ seconds := 60
+ if v := config.Config.NotificationPollSeconds; v != "" {
+ if parsed, err := strconv.Atoi(v); err == nil && parsed >= 5 {
+ seconds = parsed
+ }
+ }
+ return time.Duration(seconds) * time.Second
+}
+
+func seedCooldowns(ctx context.Context) {
+ entries, err := repositories.FiredNotificationRepository.FindLastFiredPerRule(ctx)
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("failed to seed notification cooldowns: %w", err))
+ return
+ }
+ cooldowns.seed(entries)
+}
+
func startPolledLoop(ctx context.Context) {
defer traceway.Recover()
- ticker := time.NewTicker(60 * time.Second)
+ ticker := time.NewTicker(pollInterval())
defer ticker.Stop()
for {
diff --git a/backend/app/notifications/evaluator_event.go b/backend/app/notifications/evaluator_event.go
index 53c1a016..26034195 100644
--- a/backend/app/notifications/evaluator_event.go
+++ b/backend/app/notifications/evaluator_event.go
@@ -19,8 +19,8 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
var cfg newErrorConfig
json.Unmarshal(rule.Config, &cfg)
- for _, hash := range event.ExceptionHashes {
- dedupKey := fmt.Sprintf("%d:%s", rule.Id, hash)
+ for hash, batchOccurrences := range countOccurrences(event.ExceptionHashes) {
+ dedupKey := errorDedupKey(rule.Id, hash)
if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
continue
}
@@ -34,9 +34,7 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
continue
}
- // count > 1 means this hash already existed before this batch
- if count > 1 {
- // Check if this hash was archived (resolved) — if so, a re-occurrence should be treated as new
+ if count > uint64(batchOccurrences) {
var archivedCount uint64
archErr := chdb.Conn.QueryRow(ctx,
"SELECT count() FROM archived_exceptions FINAL WHERE project_id = ? AND exception_hash = ?",
@@ -45,7 +43,6 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
continue
}
- // Archived — only fire if this is the first re-occurrence after archiving
var postArchiveCount uint64
archErr = chdb.Conn.QueryRow(ctx,
"SELECT count() FROM exception_stack_traces WHERE project_id = ? AND exception_hash = ? AND recorded_at > (SELECT max(archived_at) FROM archived_exceptions FINAL WHERE project_id = ? AND exception_hash = ?)",
@@ -55,7 +52,7 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
continue
}
- if postArchiveCount > 1 {
+ if postArchiveCount > uint64(batchOccurrences) {
continue
}
}
@@ -74,8 +71,8 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
}
func evaluateErrorRegression(ctx context.Context, rule *models.NotificationRuleWithChannel, event hooks.ReportEvent) {
- for _, hash := range event.ExceptionHashes {
- dedupKey := fmt.Sprintf("%d:%s", rule.Id, hash)
+ for hash := range countOccurrences(event.ExceptionHashes) {
+ dedupKey := errorDedupKey(rule.Id, hash)
if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
continue
}
diff --git a/backend/app/notifications/evaluator_event_sqlite.go b/backend/app/notifications/evaluator_event_sqlite.go
index 0c6bd6db..4f56c9a5 100644
--- a/backend/app/notifications/evaluator_event_sqlite.go
+++ b/backend/app/notifications/evaluator_event_sqlite.go
@@ -20,14 +20,14 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
var cfg newErrorConfig
json.Unmarshal(rule.Config, &cfg)
- for _, hash := range event.ExceptionHashes {
- dedupKey := fmt.Sprintf("%d:%s", rule.Id, hash)
+ for hash, batchOccurrences := range countOccurrences(event.ExceptionHashes) {
+ dedupKey := errorDedupKey(rule.Id, hash)
if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
continue
}
var count int64
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM exception_stack_traces WHERE project_id = ? AND exception_hash = ?",
event.ProjectId.String(), hash).Scan(&count)
if err != nil {
@@ -35,9 +35,9 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
continue
}
- if count > 1 {
+ if count > int64(batchOccurrences) {
var archivedCount int64
- archErr := db.DB.QueryRowContext(ctx,
+ archErr := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM archived_exceptions WHERE project_id = ? AND exception_hash = ?",
event.ProjectId.String(), hash).Scan(&archivedCount)
if archErr != nil || archivedCount == 0 {
@@ -45,7 +45,7 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
}
var postArchiveCount int64
- archErr = db.DB.QueryRowContext(ctx,
+ archErr = db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM exception_stack_traces WHERE project_id = ? AND exception_hash = ? AND recorded_at > (SELECT MAX(archived_at) FROM archived_exceptions WHERE project_id = ? AND exception_hash = ?)",
event.ProjectId.String(), hash, event.ProjectId.String(), hash).Scan(&postArchiveCount)
if archErr != nil {
@@ -53,7 +53,7 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
continue
}
- if postArchiveCount > 1 {
+ if postArchiveCount > int64(batchOccurrences) {
continue
}
}
@@ -72,14 +72,14 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
}
func evaluateErrorRegression(ctx context.Context, rule *models.NotificationRuleWithChannel, event hooks.ReportEvent) {
- for _, hash := range event.ExceptionHashes {
- dedupKey := fmt.Sprintf("%d:%s", rule.Id, hash)
+ for hash := range countOccurrences(event.ExceptionHashes) {
+ dedupKey := errorDedupKey(rule.Id, hash)
if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
continue
}
var count int64
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM archived_exceptions WHERE project_id = ? AND exception_hash = ?",
event.ProjectId.String(), hash).Scan(&count)
if err != nil {
@@ -103,7 +103,7 @@ func getExceptionDetails(ctx context.Context, projectId uuid.UUID, hash string)
var idStr, stackTrace, traceType, appVersion, serverName, attributesJSON, recordedAtStr string
var traceIdStr sql.NullString
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT id, trace_id, trace_type, stack_trace, attributes, app_version, server_name, recorded_at FROM exception_stack_traces WHERE project_id = ? AND exception_hash = ? ORDER BY recorded_at DESC LIMIT 1",
projectId.String(), hash).Scan(&idStr, &traceIdStr, &traceType, &stackTrace, &attributesJSON, &appVersion, &serverName, &recordedAtStr)
diff --git a/backend/app/notifications/evaluator_helpers.go b/backend/app/notifications/evaluator_helpers.go
index 3758a470..d4c8643a 100644
--- a/backend/app/notifications/evaluator_helpers.go
+++ b/backend/app/notifications/evaluator_helpers.go
@@ -86,17 +86,25 @@ func evaluateAiTraceCostEvent(rule *models.NotificationRuleWithChannel, event ho
continue
}
- dedupKey := fmt.Sprintf("ai_cost:%d:%s", rule.Id, at.TraceName)
+ dedupKey := aiCostDedupKey(rule.Id, at.TraceName)
if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
continue
}
dedup.record(dedupKey)
- msg := buildAiTraceCostMessage(at.TraceName, at.TotalCost, cfg.ThresholdCost, 0, projectName)
+ msg := buildAiTraceCostMessage(at.TraceName, at.TotalCost, cfg.ThresholdCost, projectName)
dispatch(rule, msg)
}
}
+func countOccurrences(hashes []string) map[string]int {
+ occurrences := make(map[string]int, len(hashes))
+ for _, h := range hashes {
+ occurrences[h]++
+ }
+ return occurrences
+}
+
func extractErrorType(stackTrace string) string {
if stackTrace == "" {
return "Unknown Error"
diff --git a/backend/app/notifications/evaluator_polled.go b/backend/app/notifications/evaluator_polled.go
index 76e81b0c..3cd7f699 100644
--- a/backend/app/notifications/evaluator_polled.go
+++ b/backend/app/notifications/evaluator_polled.go
@@ -6,13 +6,11 @@ import (
"context"
"encoding/json"
"fmt"
- "sync"
"time"
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/chdb"
"github.com/tracewayapp/traceway/backend/app/models"
- "github.com/tracewayapp/traceway/backend/app/repositories"
)
type EvalResult struct {
@@ -32,6 +30,7 @@ var polledEvaluators = map[string]RuleEvaluator{
"no_data": evaluateNoData,
"error_count_threshold": evaluateErrorCountThreshold,
"task_duration_threshold": evaluateTaskDurationThreshold,
+ "task_failure_rate": evaluateTaskFailureRate,
"throughput_drop": evaluateThroughputDrop,
"endpoint_error_rate": evaluateEndpointErrorRate,
"impact_score_critical": evaluateImpactScoreCritical,
@@ -124,7 +123,7 @@ func evaluateEndpointP95Threshold(ctx context.Context, rule *models.Notification
endpoint = "all endpoints"
}
projectName := getProjectName(projectId)
- msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, projectName)
+ msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -164,7 +163,7 @@ func evaluateEndpointP99Threshold(ctx context.Context, rule *models.Notification
endpoint = "all endpoints"
}
projectName := getProjectName(projectId)
- msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, projectName)
+ msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -210,7 +209,7 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje
}
projectName := getProjectName(projectId)
- msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, projectName)
+ msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, int64(total), cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -236,28 +235,34 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
now := time.Now().UTC()
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
- aggFunc := "avg"
+ selectExpr := "avg(value)"
switch cfg.Aggregation {
case "max":
- aggFunc = "max"
+ selectExpr = "max(value)"
case "min":
- aggFunc = "min"
+ selectExpr = "min(value)"
case "sum":
- aggFunc = "sum"
+ selectExpr = "sum(value)"
case "p95":
- aggFunc = "quantile(0.95)"
+ selectExpr = "quantile(0.95)(value)"
case "p99":
- aggFunc = "quantile(0.99)"
+ selectExpr = "quantile(0.99)(value)"
+ case "last":
+ selectExpr = "argMax(value, recorded_at)"
}
- query := fmt.Sprintf("SELECT %s(value) FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", aggFunc)
+ query := fmt.Sprintf("SELECT count(), %s FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", selectExpr)
args := []interface{}{projectId, cfg.MetricName, from, now}
+ var count uint64
var value float64
- err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&value)
+ err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&count, &value)
if err != nil {
return nil, err
}
+ if count == 0 {
+ return &EvalResult{Fired: false}, nil
+ }
fired := false
switch cfg.Operator {
@@ -278,7 +283,7 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
}
projectName := getProjectName(projectId)
- msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, projectName)
+ msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, cfg.Aggregation, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -426,10 +431,44 @@ func evaluateTaskDurationThreshold(ctx context.Context, rule *models.Notificatio
taskName = "all tasks"
}
projectName := getProjectName(projectId)
- msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, projectName)
+ msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
+// --- Task Failure Rate ---
+
+func countTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
+ query := "SELECT count() FROM tasks WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
+ args := []interface{}{projectId, from, to}
+ if named {
+ query += " AND task_name = ?"
+ args = append(args, taskName)
+ }
+
+ var total uint64
+ if err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&total); err != nil {
+ return 0, err
+ }
+ return int64(total), nil
+}
+
+func countFailedTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
+ query := "SELECT countDistinct(trace_id) FROM exception_stack_traces WHERE project_id = ? AND trace_type = 'task' AND recorded_at >= ? AND recorded_at <= ?" +
+ " AND trace_id IN (SELECT id FROM tasks WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
+ args := []interface{}{projectId, from, to, projectId, from, to}
+ if named {
+ query += " AND task_name = ?"
+ args = append(args, taskName)
+ }
+ query += ")"
+
+ var failed uint64
+ if err := chdb.Conn.QueryRow(ctx, query, args...).Scan(&failed); err != nil {
+ return 0, err
+ }
+ return int64(failed), nil
+}
+
// --- Throughput Drop ---
type throughputDropConfig struct {
@@ -485,7 +524,7 @@ func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule,
}
projectName := getProjectName(projectId)
- msg := buildThroughputDropMessage(dropPercent, projectName)
+ msg := buildThroughputDropMessage(dropPercent, int64(currentCount), normalizedBaseline, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -534,30 +573,11 @@ func evaluateEndpointErrorRate(ctx context.Context, rule *models.NotificationRul
// --- Impact Score ---
-var (
- impactStateMu sync.RWMutex
- impactState = make(map[string]map[string]bool)
-)
-
-type impactScoreConfig struct {
- MinRequests int `json:"minRequests"`
-}
-
-type impactMessageBuilder func(endpoint string, score float64, reason string, projectName string) Message
-
-func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID, threshold float64, buildMsg impactMessageBuilder) (*EvalResult, error) {
- var cfg impactScoreConfig
- if err := json.Unmarshal(rule.Config, &cfg); err != nil {
- return nil, fmt.Errorf("invalid impact_score config: %w", err)
- }
- if cfg.MinRequests <= 0 {
- cfg.MinRequests = 50
- }
-
+func computeImpactEndpoints(ctx context.Context, projectId uuid.UUID, minRequests int) ([]impactEndpointData, error) {
now := time.Now().UTC()
from := now.Add(-24 * time.Hour)
- query := fmt.Sprintf(`SELECT
+ query := `SELECT
endpoint, total_count, p99_duration, offset_ms,
satisfied_count, tolerating_count, bad_count, client_error_count,
greatest(
@@ -607,31 +627,19 @@ func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, pro
FROM endpoints e
LEFT JOIN (SELECT * FROM slow_endpoints FINAL) AS s
ON e.endpoint = s.endpoint AND e.project_id = s.project_id
- WHERE e.project_id = ? AND e.recorded_at >= ? AND e.recorded_at <= ?
+ WHERE e.project_id = ? AND e.recorded_at >= ? AND e.recorded_at <= ? AND e.is_stream = 0
)
GROUP BY endpoint, offset_ms
)
- WHERE impact >= %.2f AND total_count >= ?`, threshold)
+ WHERE impact >= ? AND total_count >= ?`
- rows, err := chdb.Conn.Query(ctx, query, projectId, from, now, uint64(cfg.MinRequests))
+ rows, err := chdb.Conn.Query(ctx, query, projectId, from, now, minImpactThreshold, uint64(minRequests))
if err != nil {
return nil, err
}
defer rows.Close()
- type impactEndpoint struct {
- endpoint string
- impact float64
- totalCount uint64
- p99 float64
- offsetMs uint32
- satisfied uint64
- tolerating uint64
- bad uint64
- clientErrors uint64
- }
-
- currentSet := make(map[string]impactEndpoint)
+ var result []impactEndpointData
for rows.Next() {
var ep string
var totalCount, satisfied, tolerating, bad, clientErrors uint64
@@ -642,7 +650,7 @@ func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, pro
&satisfied, &tolerating, &bad, &clientErrors, &impact); err != nil {
return nil, err
}
- currentSet[ep] = impactEndpoint{
+ result = append(result, impactEndpointData{
endpoint: ep,
impact: impact,
totalCount: totalCount,
@@ -652,55 +660,11 @@ func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, pro
tolerating: tolerating,
bad: bad,
clientErrors: clientErrors,
- }
- }
-
- stateKey := fmt.Sprintf("%d:%s", rule.Id, projectId.String())
-
- impactStateMu.RLock()
- prevSet := impactState[stateKey]
- impactStateMu.RUnlock()
-
- newSet := make(map[string]bool)
- for ep := range currentSet {
- newSet[ep] = true
- }
-
- impactStateMu.Lock()
- impactState[stateKey] = newSet
- impactStateMu.Unlock()
-
- projectName := getProjectName(projectId)
-
- var messages []Message
- if prevSet != nil {
- for ep, data := range currentSet {
- if prevSet[ep] {
- continue
- }
- reason := repositories.ComputeImpactReason(
- ep, data.totalCount, data.satisfied, data.tolerating,
- data.bad, data.clientErrors, data.p99, data.offsetMs,
- )
- messages = append(messages, buildMsg(ep, data.impact, reason, projectName))
- }
+ })
}
-
- if len(messages) == 0 {
- return &EvalResult{Fired: false}, nil
+ if err := rows.Err(); err != nil {
+ return nil, err
}
- return &EvalResult{Fired: true, Messages: messages}, nil
-}
-
-func evaluateImpactScoreCritical(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
- return evaluateImpactScore(ctx, rule, projectId, 0.75, buildImpactScoreCriticalMessage)
-}
-func evaluateImpactScoreHigh(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
- return evaluateImpactScore(ctx, rule, projectId, 0.50, buildImpactScoreHighMessage)
+ return result, nil
}
-
-func evaluateImpactScoreMedium(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
- return evaluateImpactScore(ctx, rule, projectId, 0.25, buildImpactScoreMediumMessage)
-}
-
diff --git a/backend/app/notifications/evaluator_polled_sqlite.go b/backend/app/notifications/evaluator_polled_sqlite.go
index b688d497..0d096e78 100644
--- a/backend/app/notifications/evaluator_polled_sqlite.go
+++ b/backend/app/notifications/evaluator_polled_sqlite.go
@@ -4,10 +4,9 @@ package notifications
import (
"context"
+ "database/sql"
"encoding/json"
"fmt"
- "math"
- "sync"
"time"
"github.com/google/uuid"
@@ -33,6 +32,7 @@ var polledEvaluators = map[string]RuleEvaluator{
"no_data": evaluateNoData,
"error_count_threshold": evaluateErrorCountThreshold,
"task_duration_threshold": evaluateTaskDurationThreshold,
+ "task_failure_rate": evaluateTaskFailureRate,
"throughput_drop": evaluateThroughputDrop,
"endpoint_error_rate": evaluateEndpointErrorRate,
"impact_score_critical": evaluateImpactScoreCritical,
@@ -61,7 +61,7 @@ func evaluateErrorRateThreshold(ctx context.Context, rule *models.NotificationRu
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
var total, errors int64
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN status_code >= 500 THEN 1 ELSE 0 END), 0) as errors FROM endpoints WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &errors)
if err != nil {
@@ -113,7 +113,7 @@ func evaluateEndpointP95Threshold(ctx context.Context, rule *models.Notification
endpoint = "all endpoints"
}
projectName := getProjectName(projectId)
- msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, projectName)
+ msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -142,7 +142,7 @@ func evaluateEndpointP99Threshold(ctx context.Context, rule *models.Notification
endpoint = "all endpoints"
}
projectName := getProjectName(projectId)
- msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, projectName)
+ msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -155,7 +155,7 @@ func queryPercentile(ctx context.Context, projectId uuid.UUID, endpoint string,
}
query += " ORDER BY duration ASC"
- rows, err := db.DB.QueryContext(ctx, query, args...)
+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
if err != nil {
return 0
}
@@ -211,7 +211,7 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
var total, satisfied, tolerating int64
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
`SELECT COUNT(*) as total,
COALESCE(SUM(CASE WHEN duration <= 750000000 AND status_code < 500 THEN 1 ELSE 0 END), 0) as satisfied,
COALESCE(SUM(CASE WHEN duration > 750000000 AND duration <= 1500000000 AND status_code < 500 THEN 1 ELSE 0 END), 0) as tolerating
@@ -231,7 +231,7 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje
}
projectName := getProjectName(projectId)
- msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, projectName)
+ msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, total, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -261,12 +261,22 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
var err error
switch cfg.Aggregation {
+ case "last":
+ err = db.TelemetryDB.QueryRowContext(ctx,
+ "SELECT value FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ? ORDER BY recorded_at DESC LIMIT 1",
+ projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&value)
+ if err == sql.ErrNoRows {
+ return &EvalResult{Fired: false}, nil
+ }
+ if err != nil {
+ return nil, err
+ }
case "p95", "p99":
pct := 0.95
if cfg.Aggregation == "p99" {
pct = 0.99
}
- rows, qErr := db.DB.QueryContext(ctx,
+ rows, qErr := db.TelemetryDB.QueryContext(ctx,
"SELECT value FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ? ORDER BY value ASC",
projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
if qErr != nil {
@@ -281,6 +291,9 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
}
vals = append(vals, v)
}
+ if len(vals) == 0 {
+ return &EvalResult{Fired: false}, nil
+ }
value = computePercentile(vals, pct)
default:
aggFunc := "avg"
@@ -292,11 +305,15 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
case "sum":
aggFunc = "sum"
}
- query := fmt.Sprintf("SELECT COALESCE(%s(value), 0) FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", aggFunc)
- err = db.DB.QueryRowContext(ctx, query, projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&value)
+ query := fmt.Sprintf("SELECT COUNT(value), COALESCE(%s(value), 0) FROM metric_points WHERE project_id = ? AND name = ? AND recorded_at >= ? AND recorded_at <= ?", aggFunc)
+ var count int64
+ err = db.TelemetryDB.QueryRowContext(ctx, query, projectId.String(), cfg.MetricName, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&count, &value)
if err != nil {
return nil, err
}
+ if count == 0 {
+ return &EvalResult{Fired: false}, nil
+ }
}
fired := false
@@ -318,7 +335,7 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
}
projectName := getProjectName(projectId)
- msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, projectName)
+ msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, cfg.Aggregation, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -345,7 +362,7 @@ func evaluateNoData(ctx context.Context, rule *models.NotificationRule, projectI
tables := []string{"endpoints", "exception_stack_traces", "metric_points", "tasks"}
for _, t := range tables {
var maxTs string
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
fmt.Sprintf("SELECT COALESCE(MAX(recorded_at), '') FROM %s WHERE project_id = ?", t),
pid).Scan(&maxTs)
if err == nil && maxTs != "" {
@@ -374,7 +391,7 @@ func evaluateNoData(ctx context.Context, rule *models.NotificationRule, projectI
}
var maxTs string
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
fmt.Sprintf("SELECT COALESCE(MAX(recorded_at), '') FROM %s WHERE project_id = ?", table),
pid).Scan(&maxTs)
if err != nil {
@@ -412,7 +429,7 @@ func evaluateErrorCountThreshold(ctx context.Context, rule *models.NotificationR
from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
var count int64
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM exception_stack_traces WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_message = 0",
projectId.String(), from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&count)
if err != nil {
@@ -456,7 +473,7 @@ func evaluateTaskDurationThreshold(ctx context.Context, rule *models.Notificatio
}
query += " ORDER BY duration ASC"
- rows, err := db.DB.QueryContext(ctx, query, args...)
+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
if err != nil {
return nil, err
}
@@ -482,10 +499,48 @@ func evaluateTaskDurationThreshold(ctx context.Context, rule *models.Notificatio
taskName = "all tasks"
}
projectName := getProjectName(projectId)
- msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, projectName)
+ msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
+// --- Task Failure Rate ---
+
+func countTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
+ query := "SELECT COUNT(*) FROM tasks WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
+ args := []interface{}{projectId.String(), from.Format(time.RFC3339Nano), to.Format(time.RFC3339Nano)}
+ if named {
+ query += " AND task_name = ?"
+ args = append(args, taskName)
+ }
+
+ var total int64
+ if err := db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&total); err != nil {
+ return 0, err
+ }
+ return total, nil
+}
+
+func countFailedTaskExecutions(ctx context.Context, projectId uuid.UUID, taskName string, named bool, from, to time.Time) (int64, error) {
+ pid := projectId.String()
+ fromStr := from.Format(time.RFC3339Nano)
+ toStr := to.Format(time.RFC3339Nano)
+
+ query := "SELECT COUNT(DISTINCT trace_id) FROM exception_stack_traces WHERE project_id = ? AND trace_type = 'task' AND recorded_at >= ? AND recorded_at <= ?" +
+ " AND trace_id IN (SELECT id FROM tasks WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?"
+ args := []interface{}{pid, fromStr, toStr, pid, fromStr, toStr}
+ if named {
+ query += " AND task_name = ?"
+ args = append(args, taskName)
+ }
+ query += ")"
+
+ var failed int64
+ if err := db.TelemetryDB.QueryRowContext(ctx, query, args...).Scan(&failed); err != nil {
+ return 0, err
+ }
+ return failed, nil
+}
+
// --- Throughput Drop ---
type throughputDropConfig struct {
@@ -512,7 +567,7 @@ func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule,
pid := projectId.String()
var currentCount int64
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM endpoints WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
pid, lookbackFrom.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&currentCount)
if err != nil {
@@ -520,7 +575,7 @@ func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule,
}
var baselineCount int64
- err = db.DB.QueryRowContext(ctx,
+ err = db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM endpoints WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?",
pid, baselineFrom.Format(time.RFC3339Nano), lookbackFrom.Format(time.RFC3339Nano)).Scan(&baselineCount)
if err != nil {
@@ -542,7 +597,7 @@ func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule,
}
projectName := getProjectName(projectId)
- msg := buildThroughputDropMessage(dropPercent, projectName)
+ msg := buildThroughputDropMessage(dropPercent, currentCount, normalizedBaseline, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -569,7 +624,7 @@ func evaluateEndpointErrorRate(ctx context.Context, rule *models.NotificationRul
pid := projectId.String()
var total, errors int64
- err := db.DB.QueryRowContext(ctx,
+ err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) as total, COALESCE(SUM(CASE WHEN status_code >= 500 THEN 1 ELSE 0 END), 0) as errors FROM endpoints WHERE project_id = ? AND endpoint = ? AND recorded_at >= ? AND recorded_at <= ?",
pid, cfg.Endpoint, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano)).Scan(&total, &errors)
if err != nil {
@@ -592,35 +647,16 @@ func evaluateEndpointErrorRate(ctx context.Context, rule *models.NotificationRul
// --- Impact Score ---
-var (
- impactStateMu sync.RWMutex
- impactState = make(map[string]map[string]bool)
-)
-
-type impactScoreConfig struct {
- MinRequests int `json:"minRequests"`
-}
-
-type impactMessageBuilder func(endpoint string, score float64, reason string, projectName string) Message
-
-func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID, threshold float64, buildMsg impactMessageBuilder) (*EvalResult, error) {
- var cfg impactScoreConfig
- if err := json.Unmarshal(rule.Config, &cfg); err != nil {
- return nil, fmt.Errorf("invalid impact_score config: %w", err)
- }
- if cfg.MinRequests <= 0 {
- cfg.MinRequests = 50
- }
-
+func computeImpactEndpoints(ctx context.Context, projectId uuid.UUID, minRequests int) ([]impactEndpointData, error) {
now := time.Now().UTC()
from := now.Add(-24 * time.Hour)
pid := projectId.String()
+ fromStr := from.Format(time.RFC3339Nano)
+ nowStr := now.Format(time.RFC3339Nano)
- // Get all endpoints with basic aggregates
- rows, err := db.DB.QueryContext(ctx, `SELECT
+ rows, err := db.TelemetryDB.QueryContext(ctx, `SELECT
e.endpoint,
COUNT(*) as total_count,
- MAX(e.recorded_at) as last_seen,
COALESCE(s.offset_ms, 0) as offset_ms,
COALESCE(SUM(CASE WHEN e.duration <= (750000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as satisfied_count,
COALESCE(SUM(CASE WHEN e.duration > (750000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.duration <= (1500000000 + COALESCE(s.offset_ms, 0) * 1000000) AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as tolerating_count,
@@ -628,40 +664,28 @@ func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, pro
COALESCE(SUM(CASE WHEN e.status_code >= 400 AND e.status_code < 500 THEN 1 ELSE 0 END), 0) as client_error_count
FROM endpoints e
LEFT JOIN slow_endpoints s ON e.endpoint = s.endpoint AND e.project_id = s.project_id
- WHERE e.project_id = ? AND e.recorded_at >= ? AND e.recorded_at <= ?
+ WHERE e.project_id = ? AND e.recorded_at >= ? AND e.recorded_at <= ? AND e.is_stream = 0
GROUP BY e.endpoint, COALESCE(s.offset_ms, 0)`,
- pid, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
+ pid, fromStr, nowStr)
if err != nil {
return nil, err
}
defer rows.Close()
- type impactEndpoint struct {
- endpoint string
- impact float64
- totalCount uint64
- p99 float64
- offsetMs uint32
- satisfied uint64
- tolerating uint64
- bad uint64
- clientErrors uint64
- }
-
- var candidates []impactEndpoint
+ var candidates []impactEndpointData
for rows.Next() {
- var ep, lastSeenStr string
+ var ep string
var totalCount, satisfied, tolerating, bad, clientErrors int64
var offsetMs int64
- if err := rows.Scan(&ep, &totalCount, &lastSeenStr, &offsetMs, &satisfied, &tolerating, &bad, &clientErrors); err != nil {
+ if err := rows.Scan(&ep, &totalCount, &offsetMs, &satisfied, &tolerating, &bad, &clientErrors); err != nil {
return nil, err
}
- if totalCount < int64(cfg.MinRequests) {
+ if totalCount < int64(minRequests) {
continue
}
- candidates = append(candidates, impactEndpoint{
+ candidates = append(candidates, impactEndpointData{
endpoint: ep,
totalCount: uint64(totalCount),
offsetMs: uint32(offsetMs),
@@ -671,138 +695,49 @@ func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, pro
clientErrors: uint64(clientErrors),
})
}
-
- // Compute P99 per endpoint and impact score
- currentSet := make(map[string]impactEndpoint)
- for _, c := range candidates {
- dRows, err := db.DB.QueryContext(ctx,
- "SELECT duration FROM endpoints WHERE project_id = ? AND endpoint = ? AND recorded_at >= ? AND recorded_at <= ? ORDER BY duration ASC",
- pid, c.endpoint, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
- if err != nil {
- continue
- }
- var durations []float64
- for dRows.Next() {
- var d int64
- if err := dRows.Scan(&d); err != nil {
- continue
- }
- durations = append(durations, float64(d))
- }
- dRows.Close()
-
- p99 := computePercentile(durations, 0.99)
- c.p99 = p99
-
- totalF := float64(c.totalCount)
- badF := float64(c.bad)
- clientF := float64(c.clientErrors)
- offsetNs := float64(c.offsetMs) * 1_000_000
- badRate := badF / totalF
-
- apdexScore := 1.0 - (float64(c.satisfied)+float64(c.tolerating)*0.5)/totalF
- var errorRateScore float64
- switch {
- case badRate > 0.33:
- errorRateScore = 0.75
- case badRate > 0.20:
- errorRateScore = 0.50
- case badRate > 0.10:
- errorRateScore = 0.25
- }
- adjustedP99 := p99 - offsetNs
- var p99Score float64
- switch {
- case adjustedP99 > 8_000_000_000:
- p99Score = 0.75
- case adjustedP99 > 6_000_000_000:
- p99Score = 0.50
- case adjustedP99 > 3_000_000_000:
- p99Score = 0.25
- }
- var clientErrorScore float64
- if c.endpoint != "UNMATCHED" && c.totalCount > 10 {
- clientRate := clientF / totalF
- switch {
- case clientRate > 0.50:
- clientErrorScore = 0.75
- case clientRate > 0.25:
- clientErrorScore = 0.50
- }
- }
- var volumeScore float64
- switch {
- case badRate > 0.10 && c.bad >= 500:
- volumeScore = 0.75
- case badRate > 0.10 && c.bad >= 50:
- volumeScore = 0.50
- case badRate > 0.05 && c.bad >= 2000:
- volumeScore = 0.75
- case badRate > 0.05 && c.bad >= 500:
- volumeScore = 0.50
- case badRate > 0.05 && c.bad >= 50:
- volumeScore = 0.25
- case badRate > 0.01 && c.bad >= 10000:
- volumeScore = 0.75
- case badRate > 0.01 && c.bad >= 2000:
- volumeScore = 0.50
- case badRate > 0.01 && c.bad >= 500:
- volumeScore = 0.25
- }
-
- impact := math.Max(apdexScore, math.Max(errorRateScore, math.Max(p99Score, math.Max(clientErrorScore, volumeScore))))
- if impact >= threshold {
- c.impact = impact
- currentSet[c.endpoint] = c
- }
+ if err := rows.Err(); err != nil {
+ return nil, err
}
- stateKey := fmt.Sprintf("%d:%s", rule.Id, projectId.String())
-
- impactStateMu.RLock()
- prevSet := impactState[stateKey]
- impactStateMu.RUnlock()
-
- newSet := make(map[string]bool)
- for ep := range currentSet {
- newSet[ep] = true
+ if len(candidates) == 0 {
+ return nil, nil
}
- impactStateMu.Lock()
- impactState[stateKey] = newSet
- impactStateMu.Unlock()
-
- projectName := getProjectName(projectId)
+ p99s := make(map[string]float64, len(candidates))
+ pRows, err := db.TelemetryDB.QueryContext(ctx, `SELECT endpoint, duration FROM (
+ SELECT endpoint, duration,
+ ROW_NUMBER() OVER (PARTITION BY endpoint ORDER BY duration) AS rn,
+ COUNT(*) OVER (PARTITION BY endpoint) AS cnt
+ FROM endpoints WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ? AND is_stream = 0
+ ) WHERE rn = CAST(0.99 * (cnt - 1) AS INTEGER) + 1`,
+ pid, fromStr, nowStr)
+ if err != nil {
+ return nil, err
+ }
+ defer pRows.Close()
- var messages []Message
- if prevSet != nil {
- for ep, data := range currentSet {
- if prevSet[ep] {
- continue
- }
- reason := repositories.ComputeImpactReason(
- ep, data.totalCount, data.satisfied, data.tolerating,
- data.bad, data.clientErrors, data.p99, data.offsetMs,
- )
- messages = append(messages, buildMsg(ep, data.impact, reason, projectName))
+ for pRows.Next() {
+ var ep string
+ var d int64
+ if err := pRows.Scan(&ep, &d); err != nil {
+ return nil, err
}
+ p99s[ep] = float64(d)
}
-
- if len(messages) == 0 {
- return &EvalResult{Fired: false}, nil
+ if err := pRows.Err(); err != nil {
+ return nil, err
}
- return &EvalResult{Fired: true, Messages: messages}, nil
-}
-func evaluateImpactScoreCritical(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
- return evaluateImpactScore(ctx, rule, projectId, 0.75, buildImpactScoreCriticalMessage)
-}
+ var result []impactEndpointData
+ for _, c := range candidates {
+ c.p99 = p99s[c.endpoint]
-func evaluateImpactScoreHigh(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
- return evaluateImpactScore(ctx, rule, projectId, 0.50, buildImpactScoreHighMessage)
-}
+ impact := repositories.ComputeImpactScore(c.endpoint, c.totalCount, c.satisfied, c.tolerating, c.bad, c.clientErrors, c.p99, c.offsetMs)
+ if impact >= minImpactThreshold {
+ c.impact = impact
+ result = append(result, c)
+ }
+ }
-func evaluateImpactScoreMedium(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
- return evaluateImpactScore(ctx, rule, projectId, 0.25, buildImpactScoreMediumMessage)
+ return result, nil
}
-
diff --git a/backend/app/notifications/impact_shared.go b/backend/app/notifications/impact_shared.go
new file mode 100644
index 00000000..28460bf9
--- /dev/null
+++ b/backend/app/notifications/impact_shared.go
@@ -0,0 +1,147 @@
+package notifications
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "sync"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/tracewayapp/traceway/backend/app/models"
+ "github.com/tracewayapp/traceway/backend/app/repositories"
+)
+
+const minImpactThreshold = 0.25
+
+type impactScoreConfig struct {
+ MinRequests int `json:"minRequests"`
+}
+
+type impactMessageBuilder func(endpoint string, score float64, reason string, projectName string) Message
+
+type impactEndpointData struct {
+ endpoint string
+ impact float64
+ totalCount uint64
+ p99 float64
+ offsetMs uint32
+ satisfied uint64
+ tolerating uint64
+ bad uint64
+ clientErrors uint64
+}
+
+var (
+ impactStateMu sync.RWMutex
+ impactState = make(map[string]map[string]bool)
+)
+
+type cachedImpact struct {
+ endpoints []impactEndpointData
+ computedAt time.Time
+}
+
+var (
+ impactCacheMu sync.Mutex
+ impactCache = make(map[string]cachedImpact)
+)
+
+func getImpactEndpoints(ctx context.Context, projectId uuid.UUID, minRequests int) ([]impactEndpointData, error) {
+ key := fmt.Sprintf("%s:%d", projectId.String(), minRequests)
+
+ impactCacheMu.Lock()
+ cached, ok := impactCache[key]
+ impactCacheMu.Unlock()
+ if ok && time.Since(cached.computedAt) < 30*time.Second {
+ return cached.endpoints, nil
+ }
+
+ endpoints, err := computeImpactEndpoints(ctx, projectId, minRequests)
+ if err != nil {
+ return nil, err
+ }
+
+ impactCacheMu.Lock()
+ for k, v := range impactCache {
+ if time.Since(v.computedAt) > 5*time.Minute {
+ delete(impactCache, k)
+ }
+ }
+ impactCache[key] = cachedImpact{endpoints: endpoints, computedAt: time.Now()}
+ impactCacheMu.Unlock()
+
+ return endpoints, nil
+}
+
+func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID, threshold float64, buildMsg impactMessageBuilder) (*EvalResult, error) {
+ var cfg impactScoreConfig
+ if err := json.Unmarshal(rule.Config, &cfg); err != nil {
+ return nil, fmt.Errorf("invalid impact_score config: %w", err)
+ }
+ if cfg.MinRequests <= 0 {
+ cfg.MinRequests = 50
+ }
+
+ endpoints, err := getImpactEndpoints(ctx, projectId, cfg.MinRequests)
+ if err != nil {
+ return nil, err
+ }
+
+ currentSet := make(map[string]impactEndpointData)
+ for _, e := range endpoints {
+ if e.impact >= threshold {
+ currentSet[e.endpoint] = e
+ }
+ }
+
+ stateKey := fmt.Sprintf("%d:%s", rule.Id, projectId.String())
+
+ impactStateMu.RLock()
+ prevSet := impactState[stateKey]
+ impactStateMu.RUnlock()
+
+ newSet := make(map[string]bool)
+ for ep := range currentSet {
+ newSet[ep] = true
+ }
+
+ impactStateMu.Lock()
+ impactState[stateKey] = newSet
+ impactStateMu.Unlock()
+
+ if prevSet == nil {
+ return &EvalResult{Fired: false}, nil
+ }
+
+ projectName := getProjectName(projectId)
+
+ var messages []Message
+ for ep, data := range currentSet {
+ if prevSet[ep] {
+ continue
+ }
+ reason := repositories.ComputeImpactReason(
+ ep, data.totalCount, data.satisfied, data.tolerating,
+ data.bad, data.clientErrors, data.p99, data.offsetMs,
+ )
+ messages = append(messages, buildMsg(ep, data.impact, reason, projectName))
+ }
+
+ if len(messages) == 0 {
+ return &EvalResult{Fired: false}, nil
+ }
+ return &EvalResult{Fired: true, Messages: messages}, nil
+}
+
+func evaluateImpactScoreCritical(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
+ return evaluateImpactScore(ctx, rule, projectId, 0.75, buildImpactScoreCriticalMessage)
+}
+
+func evaluateImpactScoreHigh(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
+ return evaluateImpactScore(ctx, rule, projectId, 0.50, buildImpactScoreHighMessage)
+}
+
+func evaluateImpactScoreMedium(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
+ return evaluateImpactScore(ctx, rule, projectId, minImpactThreshold, buildImpactScoreMediumMessage)
+}
diff --git a/backend/app/notifications/messages.go b/backend/app/notifications/messages.go
index dc89fe07..9b5f27b7 100644
--- a/backend/app/notifications/messages.go
+++ b/backend/app/notifications/messages.go
@@ -26,29 +26,29 @@ func endpointTimeRangeURL(now time.Time) string {
return fmt.Sprintf("/endpoints?from=%s&to=%s", from, to)
}
-func buildEndpointLatencyMessage(percentile string, latencyMs float64, thresholdMs float64, endpoint string, projectName string) Message {
+func buildEndpointLatencyMessage(percentile string, latencyMs float64, thresholdMs float64, endpoint string, window int, projectName string) Message {
return Message{
Subject: fmt.Sprintf("[%s] %s latency %.0fms on %s", projectName, percentile, latencyMs, endpoint),
- Body: fmt.Sprintf("The %s latency for %s has reached %.0fms (threshold: %.0fms).", percentile, endpoint, latencyMs, thresholdMs),
+ Body: fmt.Sprintf("The %s latency for %s has reached %.0fms over the last %d minutes (threshold: %.0fms).", percentile, endpoint, latencyMs, window, thresholdMs),
Severity: SeverityWarning,
URL: endpointTimeRangeURL(time.Now()),
}
}
-func buildApdexDropMessage(apdex float64, threshold float64, projectName string) Message {
+func buildApdexDropMessage(apdex float64, threshold float64, total int64, window int, projectName string) Message {
severity := SeverityWarning
if apdex < 0.5 {
severity = SeverityCritical
}
return Message{
Subject: fmt.Sprintf("[%s] Apdex dropped to %.2f (threshold: %.2f)", projectName, apdex, threshold),
- Body: fmt.Sprintf("The Apdex score has dropped to %.2f (threshold: %.2f).", apdex, threshold),
+ Body: fmt.Sprintf("The Apdex score has dropped to %.2f across %d requests over the last %d minutes (threshold: %.2f).", apdex, total, window, threshold),
Severity: severity,
URL: endpointTimeRangeURL(time.Now()),
}
}
-func buildMetricThresholdMessage(metricName string, value float64, operator string, threshold float64, projectName string) Message {
+func buildMetricThresholdMessage(metricName string, value float64, operator string, threshold float64, aggregation string, window int, projectName string) Message {
severity := SeverityWarning
diff := value - threshold
if diff < 0 {
@@ -57,9 +57,12 @@ func buildMetricThresholdMessage(metricName string, value float64, operator stri
if diff > threshold*0.2 {
severity = SeverityCritical
}
+ if aggregation == "" {
+ aggregation = "avg"
+ }
return Message{
Subject: fmt.Sprintf("[%s] Metric %s is %.2f (threshold: %s %.2f)", projectName, metricName, value, operator, threshold),
- Body: fmt.Sprintf("The metric %s has a value of %.2f which violates the threshold %s %.2f.", metricName, value, operator, threshold),
+ Body: fmt.Sprintf("The metric %s has a %s of %.2f over the last %d minutes which violates the threshold %s %.2f.", metricName, aggregation, value, window, operator, threshold),
Severity: severity,
URL: "/metrics?preset=1h",
}
@@ -87,23 +90,36 @@ func buildErrorCountMessage(count int64, threshold int64, window int, projectNam
}
}
-func buildTaskDurationMessage(taskName string, p95Ms float64, thresholdMs float64, projectName string) Message {
+func buildTaskDurationMessage(taskName string, p95Ms float64, thresholdMs float64, window int, projectName string) Message {
return Message{
Subject: fmt.Sprintf("[%s] Task %s P95 %.0fms exceeds %.0fms", projectName, taskName, p95Ms, thresholdMs),
- Body: fmt.Sprintf("The task %s P95 duration is %.0fms (threshold: %.0fms).", taskName, p95Ms, thresholdMs),
+ Body: fmt.Sprintf("The task %s P95 duration is %.0fms over the last %d minutes (threshold: %.0fms).", taskName, p95Ms, window, thresholdMs),
Severity: SeverityWarning,
URL: "/tasks?preset=1h",
}
}
-func buildThroughputDropMessage(dropPercent float64, projectName string) Message {
+func buildTaskFailureRateMessage(taskName string, rate float64, threshold float64, failed int64, total int64, window int, projectName string) Message {
+ severity := SeverityWarning
+ if rate >= threshold*2 {
+ severity = SeverityCritical
+ }
+ return Message{
+ Subject: fmt.Sprintf("[%s] Task %s failure rate %.1f%% exceeds %.1f%%", projectName, taskName, rate, threshold),
+ Body: fmt.Sprintf("The task %s failed %d of %d executions (%.1f%%) over the last %d minutes (threshold: %.1f%%).", taskName, failed, total, rate, window, threshold),
+ Severity: severity,
+ URL: "/tasks?preset=1h",
+ }
+}
+
+func buildThroughputDropMessage(dropPercent float64, current int64, expected float64, window int, projectName string) Message {
severity := SeverityWarning
if dropPercent >= 80 {
severity = SeverityCritical
}
return Message{
Subject: fmt.Sprintf("[%s] Throughput dropped %.0f%% vs baseline", projectName, dropPercent),
- Body: fmt.Sprintf("Request throughput has dropped by %.0f%% compared to the baseline window.", dropPercent),
+ Body: fmt.Sprintf("Request throughput has dropped by %.0f%%: %d requests in the last %d minutes vs an expected %.0f based on the baseline window.", dropPercent, current, window, expected),
Severity: severity,
URL: endpointTimeRangeURL(time.Now()),
}
@@ -159,14 +175,14 @@ func formatCostForMessage(cost float64) string {
return fmt.Sprintf("$%.2f", cost)
}
-func buildAiTraceCostMessage(traceName string, cost float64, threshold float64, window int, projectName string) Message {
+func buildAiTraceCostMessage(traceName string, cost float64, threshold float64, projectName string) Message {
severity := SeverityWarning
if cost >= threshold*3 {
severity = SeverityCritical
}
return Message{
Subject: fmt.Sprintf("[%s] AI trace %s cost %s exceeds %s", projectName, traceName, formatCostForMessage(cost), formatCostForMessage(threshold)),
- Body: fmt.Sprintf("The AI trace \"%s\" cost %s in the last %d minutes (threshold: %s).", traceName, formatCostForMessage(cost), window, formatCostForMessage(threshold)),
+ Body: fmt.Sprintf("The AI trace \"%s\" cost %s, exceeding the threshold of %s.", traceName, formatCostForMessage(cost), formatCostForMessage(threshold)),
Severity: severity,
URL: "/ai-traces?preset=1h",
}
diff --git a/backend/app/notifications/notifications_sqlite_test.go b/backend/app/notifications/notifications_sqlite_test.go
new file mode 100644
index 00000000..466c56aa
--- /dev/null
+++ b/backend/app/notifications/notifications_sqlite_test.go
@@ -0,0 +1,290 @@
+//go:build !pgch
+
+package notifications
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "path/filepath"
+ "sort"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/tracewayapp/lit/v2"
+ "github.com/tracewayapp/traceway/backend/app/db"
+ "github.com/tracewayapp/traceway/backend/app/models"
+ _ "modernc.org/sqlite"
+)
+
+func TestCountOccurrences(t *testing.T) {
+ result := countOccurrences([]string{"a", "b", "a", "c", "a", "b"})
+ if len(result) != 3 {
+ t.Fatalf("expected 3 distinct hashes, got %d", len(result))
+ }
+ if result["a"] != 3 {
+ t.Errorf("expected a counted 3 times, got %d", result["a"])
+ }
+ if result["b"] != 2 {
+ t.Errorf("expected b counted 2 times, got %d", result["b"])
+ }
+ if result["c"] != 1 {
+ t.Errorf("expected c counted 1 time, got %d", result["c"])
+ }
+
+ unique := countOccurrences([]string{"x", "y"})
+ if len(unique) != 2 || unique["x"] != 1 || unique["y"] != 1 {
+ t.Errorf("unique hashes not preserved: %v", unique)
+ }
+
+ empty := countOccurrences(nil)
+ if len(empty) != 0 {
+ t.Errorf("expected empty map for empty slice, got %v", empty)
+ }
+}
+
+func TestCooldownSeed(t *testing.T) {
+ tracker := &cooldownTracker{fired: make(map[int]time.Time)}
+ t1 := time.Now().Add(-2 * time.Hour)
+ t2 := time.Now().Add(-1 * time.Hour)
+
+ tracker.seed(map[int]time.Time{1: t1, 2: t2})
+ if got := tracker.fired[1]; !got.Equal(t1) {
+ t.Errorf("seed on empty tracker: fired[1] = %v, expected %v", got, t1)
+ }
+ if got := tracker.fired[2]; !got.Equal(t2) {
+ t.Errorf("seed on empty tracker: fired[2] = %v, expected %v", got, t2)
+ }
+
+ tracker.recordFire(3)
+ recorded := tracker.fired[3]
+ tracker.seed(map[int]time.Time{3: time.Now().Add(-3 * time.Hour)})
+ if got := tracker.fired[3]; !got.Equal(recorded) {
+ t.Errorf("seed overwrote newer recordFire time: got %v, expected %v", got, recorded)
+ }
+
+ newer := time.Now().Add(time.Hour)
+ tracker.seed(map[int]time.Time{1: newer})
+ if got := tracker.fired[1]; !got.Equal(newer) {
+ t.Errorf("seed did not update with newer time: got %v, expected %v", got, newer)
+ }
+}
+
+func TestWindowP99MatchesNearestRank(t *testing.T) {
+ dbPath := filepath.Join(t.TempDir(), "p99.db")
+ conn, err := sql.Open("sqlite", dbPath)
+ if err != nil {
+ t.Fatalf("failed to open sqlite: %v", err)
+ }
+ defer conn.Close()
+ conn.SetMaxOpenConns(1)
+
+ if _, err := conn.Exec(`CREATE TABLE endpoints (
+ endpoint TEXT NOT NULL,
+ duration INTEGER NOT NULL,
+ project_id TEXT NOT NULL,
+ recorded_at DATETIME NOT NULL
+ )`); err != nil {
+ t.Fatalf("failed to create endpoints table: %v", err)
+ }
+
+ projectId := uuid.New().String()
+ now := time.Now().UTC()
+ from := now.Add(-time.Hour)
+ ts := now.Add(-30 * time.Minute).Format(time.RFC3339Nano)
+
+ durations := map[string][]int64{}
+ for i := 0; i < 100; i++ {
+ j := (i * 37) % 100
+ durations["GET /alpha"] = append(durations["GET /alpha"], int64(1000+j*10))
+ durations["GET /beta"] = append(durations["GET /beta"], int64(5000+j*37))
+ }
+
+ for ep, ds := range durations {
+ for _, d := range ds {
+ if _, err := conn.Exec(
+ "INSERT INTO endpoints (endpoint, duration, project_id, recorded_at) VALUES (?, ?, ?, ?)",
+ ep, d, projectId, ts); err != nil {
+ t.Fatalf("failed to insert row: %v", err)
+ }
+ }
+ }
+
+ rows, err := conn.Query(`SELECT endpoint, duration FROM (
+ SELECT endpoint, duration,
+ ROW_NUMBER() OVER (PARTITION BY endpoint ORDER BY duration) AS rn,
+ COUNT(*) OVER (PARTITION BY endpoint) AS cnt
+ FROM endpoints WHERE project_id = ? AND recorded_at >= ? AND recorded_at <= ?
+ ) WHERE rn = CAST(0.99 * (cnt - 1) AS INTEGER) + 1`,
+ projectId, from.Format(time.RFC3339Nano), now.Format(time.RFC3339Nano))
+ if err != nil {
+ t.Fatalf("window query failed: %v", err)
+ }
+ defer rows.Close()
+
+ got := map[string]int64{}
+ for rows.Next() {
+ var ep string
+ var d int64
+ if err := rows.Scan(&ep, &d); err != nil {
+ t.Fatalf("failed to scan row: %v", err)
+ }
+ got[ep] = d
+ }
+ if err := rows.Err(); err != nil {
+ t.Fatalf("rows error: %v", err)
+ }
+
+ if len(got) != 2 {
+ t.Fatalf("expected p99 for 2 endpoints, got %d: %v", len(got), got)
+ }
+
+ for ep, ds := range durations {
+ sorted := append([]int64(nil), ds...)
+ sort.Slice(sorted, func(a, b int) bool { return sorted[a] < sorted[b] })
+ expected := sorted[int(0.99*float64(len(sorted)-1))]
+ if got[ep] != expected {
+ t.Errorf("endpoint %s: window p99 = %d, nearest-rank-lower = %d", ep, got[ep], expected)
+ }
+ }
+}
+
+func setupNotificationsTestDB(t *testing.T) {
+ t.Helper()
+
+ prevDB := db.DB
+ prevTelemetryDB := db.TelemetryDB
+ prevDriver := db.Driver
+ t.Cleanup(func() {
+ db.DB = prevDB
+ db.TelemetryDB = prevTelemetryDB
+ db.Driver = prevDriver
+ })
+
+ dir := t.TempDir()
+
+ mainDB, err := sql.Open("sqlite", filepath.Join(dir, "main.db"))
+ if err != nil {
+ t.Fatalf("failed to open main sqlite: %v", err)
+ }
+ mainDB.SetMaxOpenConns(1)
+
+ telemetryDB, err := sql.Open("sqlite", filepath.Join(dir, "telemetry.db"))
+ if err != nil {
+ t.Fatalf("failed to open telemetry sqlite: %v", err)
+ }
+ telemetryDB.SetMaxOpenConns(1)
+
+ t.Cleanup(func() {
+ mainDB.Close()
+ telemetryDB.Close()
+ })
+
+ db.DB = mainDB
+ db.TelemetryDB = telemetryDB
+ db.Driver = lit.SQLite
+ models.Init(db.Driver)
+
+ if _, err := telemetryDB.Exec(`CREATE TABLE tasks (
+ id TEXT NOT NULL,
+ project_id TEXT NOT NULL,
+ task_name TEXT NOT NULL DEFAULT '',
+ duration INTEGER NOT NULL DEFAULT 0,
+ recorded_at DATETIME NOT NULL
+ )`); err != nil {
+ t.Fatalf("failed to create tasks table: %v", err)
+ }
+
+ if _, err := telemetryDB.Exec(`CREATE TABLE exception_stack_traces (
+ id TEXT NOT NULL,
+ project_id TEXT NOT NULL,
+ trace_id TEXT,
+ trace_type TEXT NOT NULL DEFAULT 'endpoint',
+ recorded_at DATETIME NOT NULL
+ )`); err != nil {
+ t.Fatalf("failed to create exception_stack_traces table: %v", err)
+ }
+}
+
+func TestEvaluateTaskFailureRate(t *testing.T) {
+ setupNotificationsTestDB(t)
+
+ ctx := context.Background()
+ projectId := uuid.New()
+ pid := projectId.String()
+ now := time.Now().UTC()
+
+ var taskIds []string
+ for i := 0; i < 5; i++ {
+ id := uuid.New().String()
+ taskIds = append(taskIds, id)
+ ts := now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339Nano)
+ if _, err := db.TelemetryDB.Exec(
+ "INSERT INTO tasks (id, project_id, task_name, duration, recorded_at) VALUES (?, ?, ?, ?, ?)",
+ id, pid, "etl-job", 1000000, ts); err != nil {
+ t.Fatalf("failed to insert task: %v", err)
+ }
+ }
+
+ for i := 0; i < 2; i++ {
+ ts := now.Add(-time.Duration(i+1) * time.Minute).Format(time.RFC3339Nano)
+ if _, err := db.TelemetryDB.Exec(
+ "INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, recorded_at) VALUES (?, ?, ?, ?, ?)",
+ uuid.New().String(), pid, taskIds[i], "task", ts); err != nil {
+ t.Fatalf("failed to insert exception: %v", err)
+ }
+ }
+
+ t.Run("fires above threshold", func(t *testing.T) {
+ rule := &models.NotificationRule{
+ Config: json.RawMessage(`{"taskName":"etl-job","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":5}`),
+ }
+ result, err := evaluateTaskFailureRate(ctx, rule, projectId)
+ if err != nil {
+ t.Fatalf("evaluateTaskFailureRate failed: %v", err)
+ }
+ if !result.Fired {
+ t.Fatal("expected rule to fire")
+ }
+ if !strings.Contains(result.Message.Subject, "40.0%") {
+ t.Errorf("subject %q does not contain 40.0%%", result.Message.Subject)
+ }
+ if !strings.Contains(result.Message.Body, "2 of 5") {
+ t.Errorf("body %q does not contain 2 of 5", result.Message.Body)
+ }
+ })
+
+ t.Run("gated below min executions", func(t *testing.T) {
+ rule := &models.NotificationRule{
+ Config: json.RawMessage(`{"taskName":"etl-job","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":10}`),
+ }
+ result, err := evaluateTaskFailureRate(ctx, rule, projectId)
+ if err != nil {
+ t.Fatalf("evaluateTaskFailureRate failed: %v", err)
+ }
+ if result.Fired {
+ t.Error("expected rule not to fire below minExecutions")
+ }
+ })
+
+ t.Run("wildcard task name", func(t *testing.T) {
+ rule := &models.NotificationRule{
+ Config: json.RawMessage(`{"taskName":"*","thresholdPercent":20,"lookbackMinutes":60,"minExecutions":5}`),
+ }
+ result, err := evaluateTaskFailureRate(ctx, rule, projectId)
+ if err != nil {
+ t.Fatalf("evaluateTaskFailureRate failed: %v", err)
+ }
+ if !result.Fired {
+ t.Fatal("expected wildcard rule to fire")
+ }
+ if !strings.Contains(result.Message.Subject, "all tasks") {
+ t.Errorf("subject %q does not mention all tasks", result.Message.Subject)
+ }
+ if !strings.Contains(result.Message.Body, "2 of 5") {
+ t.Errorf("body %q does not contain 2 of 5", result.Message.Body)
+ }
+ })
+}
diff --git a/backend/app/notifications/task_failure_shared.go b/backend/app/notifications/task_failure_shared.go
new file mode 100644
index 00000000..ae2edbbe
--- /dev/null
+++ b/backend/app/notifications/task_failure_shared.go
@@ -0,0 +1,61 @@
+package notifications
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/tracewayapp/traceway/backend/app/models"
+)
+
+type taskFailureRateConfig struct {
+ TaskName string `json:"taskName"`
+ ThresholdPercent float64 `json:"thresholdPercent"`
+ LookbackMinutes int `json:"lookbackMinutes"`
+ MinExecutions int `json:"minExecutions"`
+}
+
+func evaluateTaskFailureRate(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
+ var cfg taskFailureRateConfig
+ if err := json.Unmarshal(rule.Config, &cfg); err != nil {
+ return nil, fmt.Errorf("invalid task_failure_rate config: %w", err)
+ }
+ if cfg.LookbackMinutes <= 0 {
+ cfg.LookbackMinutes = 60
+ }
+ if cfg.MinExecutions <= 0 {
+ cfg.MinExecutions = 5
+ }
+
+ now := time.Now().UTC()
+ from := now.Add(-time.Duration(cfg.LookbackMinutes) * time.Minute)
+ named := cfg.TaskName != "" && cfg.TaskName != "*"
+
+ total, err := countTaskExecutions(ctx, projectId, cfg.TaskName, named, from, now)
+ if err != nil {
+ return nil, err
+ }
+ if total < int64(cfg.MinExecutions) {
+ return &EvalResult{Fired: false}, nil
+ }
+
+ failed, err := countFailedTaskExecutions(ctx, projectId, cfg.TaskName, named, from, now)
+ if err != nil {
+ return nil, err
+ }
+
+ rate := float64(failed) / float64(total) * 100
+ if rate < cfg.ThresholdPercent {
+ return &EvalResult{Fired: false}, nil
+ }
+
+ taskName := cfg.TaskName
+ if taskName == "" || taskName == "*" {
+ taskName = "all tasks"
+ }
+ projectName := getProjectName(projectId)
+ msg := buildTaskFailureRateMessage(taskName, rate, cfg.ThresholdPercent, failed, total, cfg.LookbackMinutes, projectName)
+ return &EvalResult{Fired: true, Message: msg}, nil
+}
diff --git a/backend/app/repositories/ai_trace.repository_sqlite.go b/backend/app/repositories/ai_trace.repository_sqlite.go
index 0035a0bf..7d4c6cc9 100644
--- a/backend/app/repositories/ai_trace.repository_sqlite.go
+++ b/backend/app/repositories/ai_trace.repository_sqlite.go
@@ -15,6 +15,12 @@ import (
"github.com/tracewayapp/traceway/backend/app/models"
)
+type aiTraceRowNaming struct{ lit.DefaultDbNamingStrategy }
+
+func (aiTraceRowNaming) GetTableNameFromStructName(string) string {
+ return "ai_traces"
+}
+
type aiTraceRow struct {
Id uuid.UUID `lit:"id"`
ProjectId uuid.UUID `lit:"project_id"`
@@ -72,7 +78,7 @@ type aiTraceDetailStatsRow struct {
func init() {
models.ExtensionModelRegistrations = append(models.ExtensionModelRegistrations, func(driver lit.Driver) {
- lit.RegisterModel[aiTraceRow](driver)
+ lit.RegisterModelWithNaming[aiTraceRow](driver, aiTraceRowNaming{})
lit.RegisterModel[groupedAiTraceRow](driver)
lit.RegisterModel[aiTraceDurationRow](driver)
lit.RegisterModel[aiTraceDetailStatsRow](driver)
diff --git a/backend/app/repositories/endpoint.repository_sqlite.go b/backend/app/repositories/endpoint.repository_sqlite.go
index c3b79ac0..d854422f 100644
--- a/backend/app/repositories/endpoint.repository_sqlite.go
+++ b/backend/app/repositories/endpoint.repository_sqlite.go
@@ -317,7 +317,7 @@ func (e *endpointRepository) FindGroupedByEndpoint(ctx context.Context, projectI
p95 := computePercentile(durations, 0.95)
p99 := computePercentile(durations, 0.99)
- impact := computeImpactScore(g.Endpoint, g.TotalCount, g.SatisfiedCount, g.ToleratingCount, g.BadCount, g.ClientErrorCount, p99, g.OffsetMs)
+ impact := ComputeImpactScore(g.Endpoint, g.TotalCount, g.SatisfiedCount, g.ToleratingCount, g.BadCount, g.ClientErrorCount, p99, g.OffsetMs)
stats = append(stats, models.EndpointStats{
Endpoint: g.Endpoint,
@@ -537,7 +537,7 @@ func (e *endpointRepository) FindWorstEndpoints(ctx context.Context, projectId u
p95 := computePercentile(durations, 0.95)
p99 := computePercentile(durations, 0.99)
- impact := computeImpactScore(g.Endpoint, g.TotalCount, g.SatisfiedCount, g.ToleratingCount, g.BadCount, g.ClientErrorCount, p99, g.OffsetMs)
+ impact := ComputeImpactScore(g.Endpoint, g.TotalCount, g.SatisfiedCount, g.ToleratingCount, g.BadCount, g.ClientErrorCount, p99, g.OffsetMs)
lastSeen, _ := time.Parse(time.RFC3339Nano, g.LastSeen)
@@ -843,7 +843,7 @@ func fetchSortedDurations(ctx context.Context, projectId uuid.UUID, endpoint str
return durations, nil
}
-func computeImpactScore(endpoint string, total, satisfiedCount, toleratingCount, badCount, clientErrorCount uint64, p99Ns float64, offsetMs uint32) float64 {
+func ComputeImpactScore(endpoint string, total, satisfiedCount, toleratingCount, badCount, clientErrorCount uint64, p99Ns float64, offsetMs uint32) float64 {
if total == 0 {
return 0
}
diff --git a/backend/app/repositories/fired_notification.repository.go b/backend/app/repositories/fired_notification.repository.go
index 83c0baa1..868fdf43 100644
--- a/backend/app/repositories/fired_notification.repository.go
+++ b/backend/app/repositories/fired_notification.repository.go
@@ -8,6 +8,7 @@ import (
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/chdb"
+ "github.com/tracewayapp/traceway/backend/app/models"
)
type FiredNotification struct {
@@ -23,6 +24,7 @@ type FiredNotification struct {
Status string
ErrorMsg string
Endpoint string
+ URL string
FiredAt time.Time
}
@@ -31,7 +33,7 @@ type firedNotificationRepository struct{}
func (r *firedNotificationRepository) Insert(ctx context.Context, n FiredNotification) error {
batch, err := chdb.Conn.PrepareBatch(
chdb.BatchCtx(),
- "INSERT INTO fired_notifications (project_id, rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, endpoint, fired_at)",
+ "INSERT INTO fired_notifications (project_id, rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, endpoint, url, fired_at)",
)
if err != nil {
return err
@@ -39,11 +41,88 @@ func (r *firedNotificationRepository) Insert(ctx context.Context, n FiredNotific
if err := batch.Append(
n.ProjectId, int32(n.RuleId), n.RuleType, n.RuleName,
n.ChannelType, n.ChannelName, n.Severity, n.Subject, n.Body,
- n.Status, n.ErrorMsg, n.Endpoint, n.FiredAt,
+ n.Status, n.ErrorMsg, n.Endpoint, n.URL, n.FiredAt,
); err != nil {
return err
}
return batch.Send()
}
+func (r *firedNotificationRepository) FindByProject(ctx context.Context, projectId uuid.UUID, page int, pageSize int, search string, from *time.Time, to *time.Time) ([]*models.NotificationHistoryEntry, int64, error) {
+ where := "WHERE project_id = ?"
+ args := []interface{}{projectId}
+
+ if search != "" {
+ where += " AND (rule_name ILIKE ? OR channel_name ILIKE ? OR subject ILIKE ?)"
+ pattern := "%" + search + "%"
+ args = append(args, pattern, pattern, pattern)
+ }
+ if from != nil {
+ where += " AND fired_at >= ?"
+ args = append(args, *from)
+ }
+ if to != nil {
+ where += " AND fired_at <= ?"
+ args = append(args, *to)
+ }
+
+ var total uint64
+ if err := chdb.Conn.QueryRow(ctx, "SELECT count() FROM fired_notifications "+where, args...).Scan(&total); err != nil {
+ return nil, 0, err
+ }
+
+ query := "SELECT rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, url, fired_at FROM fired_notifications " +
+ where + " ORDER BY fired_at DESC LIMIT ? OFFSET ?"
+ args = append(args, pageSize, (page-1)*pageSize)
+
+ rows, err := chdb.Conn.Query(ctx, query, args...)
+ if err != nil {
+ return nil, 0, err
+ }
+ defer rows.Close()
+
+ var items []*models.NotificationHistoryEntry
+ for rows.Next() {
+ var ruleId int32
+ entry := &models.NotificationHistoryEntry{}
+ if err := rows.Scan(
+ &ruleId, &entry.RuleType, &entry.RuleName, &entry.ChannelType, &entry.ChannelName,
+ &entry.Severity, &entry.Subject, &entry.Body, &entry.Status, &entry.ErrorMessage,
+ &entry.URL, &entry.CreatedAt,
+ ); err != nil {
+ return nil, 0, err
+ }
+ entry.RuleId = int(ruleId)
+ items = append(items, entry)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, 0, err
+ }
+
+ return items, int64(total), nil
+}
+
+func (r *firedNotificationRepository) FindLastFiredPerRule(ctx context.Context) (map[int]time.Time, error) {
+ rows, err := chdb.Conn.Query(ctx, "SELECT rule_id, max(fired_at) FROM fired_notifications WHERE status = 'sent' GROUP BY rule_id")
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result := make(map[int]time.Time)
+ for rows.Next() {
+ var ruleId int32
+ var firedAt time.Time
+ if err := rows.Scan(&ruleId, &firedAt); err != nil {
+ return nil, err
+ }
+ result[int(ruleId)] = firedAt
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ return result, nil
+}
+
var FiredNotificationRepository = firedNotificationRepository{}
diff --git a/backend/app/repositories/fired_notification.repository_sqlite.go b/backend/app/repositories/fired_notification.repository_sqlite.go
index b19b7af9..fa2bb85f 100644
--- a/backend/app/repositories/fired_notification.repository_sqlite.go
+++ b/backend/app/repositories/fired_notification.repository_sqlite.go
@@ -4,6 +4,7 @@ package repositories
import (
"context"
+ "strings"
"time"
"github.com/google/uuid"
@@ -25,6 +26,7 @@ type FiredNotification struct {
Status string
ErrorMsg string
Endpoint string
+ URL string
FiredAt time.Time
}
@@ -41,6 +43,7 @@ type firedNotificationRow struct {
Status string `lit:"status"`
ErrorMsg string `lit:"error_message"`
Endpoint string `lit:"endpoint"`
+ URL string `lit:"url"`
FiredAt SQLiteTime `lit:"fired_at"`
}
@@ -66,12 +69,13 @@ func (r *firedNotificationRepository) Insert(ctx context.Context, n FiredNotific
Status: n.Status,
ErrorMsg: n.ErrorMsg,
Endpoint: n.Endpoint,
+ URL: n.URL,
FiredAt: NewSQLiteTime(n.FiredAt),
}
query, args, err := lit.ParseNamedQuery(db.Driver,
- `INSERT INTO fired_notifications (project_id, rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, endpoint, fired_at)
- VALUES (:project_id, :rule_id, :rule_type, :rule_name, :channel_type, :channel_name, :severity, :subject, :body, :status, :error_message, :endpoint, :fired_at)`,
+ `INSERT INTO fired_notifications (project_id, rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, endpoint, url, fired_at)
+ VALUES (:project_id, :rule_id, :rule_type, :rule_name, :channel_type, :channel_name, :severity, :subject, :body, :status, :error_message, :endpoint, :url, :fired_at)`,
lit.P{
"project_id": row.ProjectId,
"rule_id": row.RuleId,
@@ -85,6 +89,7 @@ func (r *firedNotificationRepository) Insert(ctx context.Context, n FiredNotific
"status": row.Status,
"error_message": row.ErrorMsg,
"endpoint": row.Endpoint,
+ "url": row.URL,
"fired_at": row.FiredAt,
})
if err != nil {
@@ -94,4 +99,81 @@ func (r *firedNotificationRepository) Insert(ctx context.Context, n FiredNotific
return err
}
+func (r *firedNotificationRepository) FindByProject(ctx context.Context, projectId uuid.UUID, page int, pageSize int, search string, from *time.Time, to *time.Time) ([]*models.NotificationHistoryEntry, int64, error) {
+ where := "WHERE project_id = ?"
+ args := []interface{}{projectId.String()}
+
+ if search != "" {
+ where += " AND (LOWER(rule_name) LIKE ? OR LOWER(channel_name) LIKE ? OR LOWER(subject) LIKE ?)"
+ pattern := "%" + strings.ToLower(search) + "%"
+ args = append(args, pattern, pattern, pattern)
+ }
+ if from != nil {
+ where += " AND fired_at >= ?"
+ args = append(args, from.UTC().Format(time.RFC3339Nano))
+ }
+ if to != nil {
+ where += " AND fired_at <= ?"
+ args = append(args, to.UTC().Format(time.RFC3339Nano))
+ }
+
+ var total int64
+ if err := db.TelemetryDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM fired_notifications "+where, args...).Scan(&total); err != nil {
+ return nil, 0, err
+ }
+
+ query := "SELECT rule_id, rule_type, rule_name, channel_type, channel_name, severity, subject, body, status, error_message, url, fired_at FROM fired_notifications " +
+ where + " ORDER BY fired_at DESC LIMIT ? OFFSET ?"
+ args = append(args, pageSize, (page-1)*pageSize)
+
+ rows, err := db.TelemetryDB.QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, 0, err
+ }
+ defer rows.Close()
+
+ var items []*models.NotificationHistoryEntry
+ for rows.Next() {
+ entry := &models.NotificationHistoryEntry{}
+ var firedAt SQLiteTime
+ if err := rows.Scan(
+ &entry.RuleId, &entry.RuleType, &entry.RuleName, &entry.ChannelType, &entry.ChannelName,
+ &entry.Severity, &entry.Subject, &entry.Body, &entry.Status, &entry.ErrorMessage,
+ &entry.URL, &firedAt,
+ ); err != nil {
+ return nil, 0, err
+ }
+ entry.CreatedAt = firedAt.Time
+ items = append(items, entry)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, 0, err
+ }
+
+ return items, total, nil
+}
+
+func (r *firedNotificationRepository) FindLastFiredPerRule(ctx context.Context) (map[int]time.Time, error) {
+ rows, err := db.TelemetryDB.QueryContext(ctx, "SELECT rule_id, MAX(fired_at) FROM fired_notifications WHERE status = 'sent' GROUP BY rule_id")
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ result := make(map[int]time.Time)
+ for rows.Next() {
+ var ruleId int
+ var firedAt SQLiteTime
+ if err := rows.Scan(&ruleId, &firedAt); err != nil {
+ return nil, err
+ }
+ result[ruleId] = firedAt.Time
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ return result, nil
+}
+
var FiredNotificationRepository = firedNotificationRepository{}
diff --git a/backend/app/repositories/impact.go b/backend/app/repositories/impact.go
index 7348c46b..7b27b98f 100644
--- a/backend/app/repositories/impact.go
+++ b/backend/app/repositories/impact.go
@@ -149,7 +149,7 @@ func ComputeStreamImpactReason(endpoint string, total, serverErrorCount, clientE
}
// errorRateFloor / clientErrorFloor / volumeAwareErrorFloor are the tier
-// definitions from ComputeImpactReason (mirrored in computeImpactScore),
+// definitions from ComputeImpactReason (mirrored in ComputeImpactScore),
// factored out so the stream variants reuse identical thresholds.
func errorRateFloor(badRate float64) float64 {
switch {
diff --git a/backend/app/repositories/notification_history.repository.go b/backend/app/repositories/notification_history.repository.go
deleted file mode 100644
index 24c138b4..00000000
--- a/backend/app/repositories/notification_history.repository.go
+++ /dev/null
@@ -1,65 +0,0 @@
-package repositories
-
-import (
- "database/sql"
- "time"
-
- "github.com/tracewayapp/traceway/backend/app/models"
-
- "github.com/google/uuid"
- "github.com/tracewayapp/lit/v2"
-)
-
-type notificationHistoryRepository struct{}
-
-func (r *notificationHistoryRepository) FindByProject(tx *sql.Tx, projectId uuid.UUID, page, pageSize int, search string, fromDate, toDate *time.Time) ([]*models.NotificationHistory, int64, error) {
- params := lit.P{"project_id": projectId}
- whereClause := "WHERE project_id = :project_id"
-
- if search != "" {
- whereClause += " AND (rule_name ILIKE :search OR channel_name ILIKE :search OR subject ILIKE :search)"
- params["search"] = "%" + search + "%"
- }
-
- if fromDate != nil {
- whereClause += " AND created_at >= :from_date"
- params["from_date"] = *fromDate
- }
- if toDate != nil {
- whereClause += " AND created_at <= :to_date"
- params["to_date"] = *toDate
- }
-
- countResult, err := lit.SelectSingleNamed[models.CountResult](
- tx,
- "SELECT COUNT(*) as count FROM notification_history "+whereClause,
- params,
- )
- if err != nil {
- return nil, 0, err
- }
- var total int64
- if countResult != nil {
- total = int64(countResult.Count)
- }
-
- offset := (page - 1) * pageSize
- params["limit"] = pageSize
- params["offset"] = offset
- items, err := lit.SelectNamed[models.NotificationHistory](
- tx,
- "SELECT id, project_id, rule_id, channel_id, rule_type, rule_name, channel_name, severity, subject, body, status, error_message, url, created_at FROM notification_history "+whereClause+" ORDER BY created_at DESC LIMIT :limit OFFSET :offset",
- params,
- )
- if err != nil {
- return nil, 0, err
- }
-
- return items, total, nil
-}
-
-func (r *notificationHistoryRepository) Create(tx *sql.Tx, history *models.NotificationHistory) (int, error) {
- return lit.Insert[models.NotificationHistory](tx, history)
-}
-
-var NotificationHistoryRepository = notificationHistoryRepository{}
diff --git a/backend/app/repositories/notification_rule.repository.go b/backend/app/repositories/notification_rule.repository.go
index 5345e2c8..f3063b45 100644
--- a/backend/app/repositories/notification_rule.repository.go
+++ b/backend/app/repositories/notification_rule.repository.go
@@ -72,7 +72,7 @@ func (r *notificationRuleRepository) Delete(tx *sql.Tx, id int) error {
}
func (r *notificationRuleRepository) UpdateEnabled(tx *sql.Tx, id int, enabled bool) error {
- q, a, err := lit.ParseNamedQuery(db.Driver, "UPDATE notification_rules SET enabled = :enabled, updated_at = NOW() WHERE id = :id", lit.P{"enabled": enabled, "id": id})
+ q, a, err := lit.ParseNamedQuery(db.Driver, "UPDATE notification_rules SET enabled = :enabled, updated_at = :updated_at WHERE id = :id", lit.P{"enabled": enabled, "updated_at": time.Now().UTC(), "id": id})
if err != nil {
return err
}
@@ -80,7 +80,7 @@ func (r *notificationRuleRepository) UpdateEnabled(tx *sql.Tx, id int, enabled b
}
func (r *notificationRuleRepository) UpdateSnoozedUntil(tx *sql.Tx, id int, snoozedUntil *time.Time) error {
- q, a, err := lit.ParseNamedQuery(db.Driver, "UPDATE notification_rules SET snoozed_until = :snoozed_until, updated_at = NOW() WHERE id = :id", lit.P{"snoozed_until": snoozedUntil, "id": id})
+ q, a, err := lit.ParseNamedQuery(db.Driver, "UPDATE notification_rules SET snoozed_until = :snoozed_until, updated_at = :updated_at WHERE id = :id", lit.P{"snoozed_until": snoozedUntil, "updated_at": time.Now().UTC(), "id": id})
if err != nil {
return err
}
diff --git a/backend/app/repositories/project.repository.go b/backend/app/repositories/project.repository.go
index 0f778d80..ae4e1bf5 100644
--- a/backend/app/repositories/project.repository.go
+++ b/backend/app/repositories/project.repository.go
@@ -215,7 +215,6 @@ func (p *projectRepository) Update(tx *sql.Tx, id uuid.UUID, name string, framew
func (p *projectRepository) Delete(tx *sql.Tx, id uuid.UUID) error {
related := []string{
- "notification_history",
"notification_rules",
"notification_channels",
"widget_groups",
diff --git a/backend/app/repositories/sqlite_types.go b/backend/app/repositories/sqlite_types.go
index 8b91359f..b3283af2 100644
--- a/backend/app/repositories/sqlite_types.go
+++ b/backend/app/repositories/sqlite_types.go
@@ -49,7 +49,7 @@ func (t *SQLiteTime) parseString(s string) error {
}
func (t SQLiteTime) Value() (driver.Value, error) {
- return t.Time.UTC().Format(time.RFC3339Nano), nil
+ return t.Time.UTC().Format("2006-01-02T15:04:05.000000000Z07:00"), nil
}
// SQLiteJSONMap handles scanning map[string]string from SQLite TEXT JSON columns
diff --git a/backend/app/repositories/testhelper_sqlite_test.go b/backend/app/repositories/testhelper_sqlite_test.go
index 1a796079..5ea67b7a 100644
--- a/backend/app/repositories/testhelper_sqlite_test.go
+++ b/backend/app/repositories/testhelper_sqlite_test.go
@@ -200,6 +200,7 @@ CREATE TABLE IF NOT EXISTS fired_notifications (
status TEXT NOT NULL DEFAULT '',
error_message TEXT NOT NULL DEFAULT '',
endpoint TEXT NOT NULL DEFAULT '',
+ url TEXT NOT NULL DEFAULT '',
fired_at DATETIME NOT NULL
);
`
diff --git a/backend/app/retention/sqlite.go b/backend/app/retention/sqlite.go
index ba8f1a44..5bfcbee1 100644
--- a/backend/app/retention/sqlite.go
+++ b/backend/app/retention/sqlite.go
@@ -27,13 +27,6 @@ var telemetryRetentionTargets = []struct {
{"sessions", "started_at"},
}
-var mainRetentionTargets = []struct {
- table string
- column string
-}{
- {"notification_history", "created_at"},
-}
-
func startSQLiteRetention(ctx context.Context, days int) {
if !db.IsSQLite() {
return
@@ -78,16 +71,4 @@ func runSQLiteRetention(ctx context.Context, days int) {
}
}
}
-
- if db.DB != nil {
- for _, tgt := range mainRetentionTargets {
- if ctx.Err() != nil {
- return
- }
- query := fmt.Sprintf("DELETE FROM %s WHERE %s < :cutoff", tgt.table, tgt.column)
- if err := lit.DeleteNamed(db.Driver, db.DB, query, params); err != nil {
- traceway.CaptureException(fmt.Errorf("retention: delete from main.%s failed: %w", tgt.table, err))
- }
- }
- }
}
diff --git a/backend/app/retention/sqlite_test.go b/backend/app/retention/sqlite_test.go
index 741eb128..148a083e 100644
--- a/backend/app/retention/sqlite_test.go
+++ b/backend/app/retention/sqlite_test.go
@@ -40,16 +40,6 @@ func setupRetentionTestDB(t *testing.T) {
}
}
- for _, tgt := range mainRetentionTargets {
- ddl := fmt.Sprintf(
- "CREATE TABLE %s (id INTEGER PRIMARY KEY AUTOINCREMENT, project_id TEXT, %s DATETIME NOT NULL)",
- tgt.table, tgt.column,
- )
- if _, err := mainDB.Exec(ddl); err != nil {
- t.Fatalf("main ddl %s: %v", tgt.table, err)
- }
- }
-
prevDB, prevTelDB, prevDriver := db.DB, db.TelemetryDB, db.Driver
db.DB = mainDB
db.TelemetryDB = telDB
@@ -105,16 +95,12 @@ func TestRunSQLiteRetention_DeletesOldKeepsFresh(t *testing.T) {
insertWithTime(t, db.TelemetryDB, "fired_notifications", "fired_at", old)
insertWithTime(t, db.TelemetryDB, "fired_notifications", "fired_at", fresh)
- insertWithTime(t, db.DB, "notification_history", "created_at", old)
- insertWithTime(t, db.DB, "notification_history", "created_at", fresh)
-
runSQLiteRetention(context.Background(), 30)
assertCount(t, db.TelemetryDB, "endpoints", 3)
assertCount(t, db.TelemetryDB, "log_records", 2)
assertCount(t, db.TelemetryDB, "sessions", 1)
assertCount(t, db.TelemetryDB, "fired_notifications", 1)
- assertCount(t, db.DB, "notification_history", 1)
}
func TestRunSQLiteRetention_BoundaryRowAtCutoffStays(t *testing.T) {
diff --git a/backend/cmd/run.go b/backend/cmd/run.go
index 95c976d7..df462670 100644
--- a/backend/cmd/run.go
+++ b/backend/cmd/run.go
@@ -264,6 +264,7 @@ func applyEnvOverrides(cfg *config.Cfg) {
{"OIDC_TOKEN_URL", &cfg.OIDCTokenURL},
{"OIDC_USER_INFO_URL", &cfg.OIDCUserInfoURL},
{"DISABLE_PASSWORD_LOGIN", &cfg.DisablePasswordLogin},
+ {"NOTIFICATION_POLL_SECONDS", &cfg.NotificationPollSeconds},
{"SOURCEMAP_CACHE_MAX_ENTRIES", &cfg.SourceMapCacheMaxEntries},
{"SOURCEMAP_CACHE_MAX_BYTES_MB", &cfg.SourceMapCacheMaxBytesMB},
{"SOURCEMAP_CACHE_TYPE", &cfg.SourceMapCacheType},
diff --git a/docs/pages/learn/alerts.mdx b/docs/pages/learn/alerts.mdx
index 691f1457..e0dd44ab 100644
--- a/docs/pages/learn/alerts.mdx
+++ b/docs/pages/learn/alerts.mdx
@@ -112,51 +112,247 @@ If you configure an HMAC secret on the channel, each request includes an `X-Trac
## Rules
-### Rule Types
+A rule connects a condition to a channel. Every rule has:
-| Rule | What It Monitors |
-|------|-----------------|
-| New Issue | A never-before-seen exception appears |
-| Impact Score Critical | An endpoint's [impact score](/learn/endpoints#impact-score) reaches critical level |
-| Impact Score High | An endpoint's impact score reaches high level |
-| Impact Score Medium | An endpoint's impact score reaches medium level |
+| Setting | Description |
+|---------|-------------|
+| Name | A label you choose, included in notifications |
+| Type | One of the rule types below, with type-specific configuration |
+| Channel | The channel notifications are delivered through (exactly one per rule) |
+| Cooldown | Minimum time between notifications for the same condition (default 15 minutes) |
+| Severity | Auto (derived from the condition) or an explicit override |
-### Coming Soon
+Rules belong to a project and only evaluate that project's data. Frontend projects only support the New Issue rule type; all other types monitor backend telemetry.
-The following rule types are planned but not yet available:
+## Rule Types
-- **Error Regression** — A previously resolved (archived) [issue](/learn/issues) reappears
-- **Error Rate** — Overall 5xx error rate exceeds a percentage threshold
-- **Error Count** — Total number of errors exceeds a count in a time window
-- **Endpoint P95** — P95 latency for an endpoint exceeds a millisecond threshold
-- **Endpoint P99** — P99 latency for an endpoint exceeds a millisecond threshold
-- **Endpoint Error Rate** — A specific endpoint's error rate exceeds a percentage threshold
-- **Apdex Drop** — Apdex score falls below a threshold
-- **Metric Threshold** — A custom or system [metric](/learn/metrics) crosses a boundary (above or below)
-- **No Data** — No telemetry data received for a specified duration
-- **Task Duration** — A background task's P95 duration exceeds a threshold
-- **Task Failure Rate** — A background task's failure rate exceeds a percentage threshold
-- **Throughput Drop** — Request volume drops compared to baseline
+| Rule | Fires When | Evaluation |
+|------|-----------|------------|
+| [New Issue](#new-issue) | A never-before-seen error appears | On ingest |
+| [Error Regression](#error-regression) | A resolved (archived) error reappears | On ingest |
+| [Error Rate](#error-rate) | The 5xx rate across all endpoints exceeds a percentage | Polled |
+| [Error Count](#error-count) | Total errors in a window reach a count | Polled |
+| [Endpoint Error Rate](#endpoint-error-rate) | One endpoint's 5xx rate exceeds a percentage | Polled |
+| [Endpoint P95 / P99](#endpoint-p95--p99) | Latency percentile exceeds a millisecond threshold | Polled |
+| [Apdex Drop](#apdex-drop) | The Apdex score falls below a threshold | Polled |
+| [Throughput Drop](#throughput-drop) | Request volume drops versus the preceding baseline | Polled |
+| [Impact Score](#impact-score-critical--high--medium) | An endpoint newly crosses an impact level | Polled |
+| [Task Duration](#task-duration) | A background task's P95 duration exceeds a threshold | Polled |
+| [Task Failure Rate](#task-failure-rate) | The share of failed task executions exceeds a percentage | Polled |
+| [Metric Threshold](#metric-threshold) | An aggregated metric violates a comparison | Polled |
+| [No Data](#no-data) | No telemetry received for a configured duration | Polled |
+| [AI Trace Cost](#ai-trace-cost) | A single AI trace exceeds a cost threshold | On ingest |
-### Evaluation
+### New Issue
-New Issue rules are evaluated immediately when new data arrives during report ingestion. Impact Score rules are evaluated every 60 seconds by a background process.
+Fires the first time an error with a given fingerprint is seen. Errors are grouped by a normalized stack trace hash (see [Issues](/learn/issues)), so the same logical error reported many times produces a single alert. If you archive an issue and it occurs again, that first re-occurrence counts as new and fires the rule.
+
+The notification includes the error type, stack trace, app version, server name, the endpoint or task it occurred on, and a deep link to the issue.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Ignore Patterns | empty | Comma-separated, case-insensitive substring matches against the error type, e.g. `*timeout*, *context canceled*`. Matching errors never alert. |
+
+The cooldown applies per error, not per rule: two different new errors arriving a minute apart both alert, but the same error will not alert again until its cooldown expires.
+
+### Error Regression
+
+Fires when an error whose issue was archived (resolved) occurs again. Use this to catch fixes that did not stick. The cooldown applies per error. There is no extra configuration.
+
+### Error Rate
+
+Fires when the percentage of requests returning a 5xx status across all endpoints exceeds the threshold within the lookback window.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Threshold % | 5 | Error rate that triggers the alert |
+| Lookback Minutes | 5 | Window the rate is computed over |
+| Min Requests | 10 | Minimum requests in the window; below this the rule never fires, which avoids noise from a single failed request in quiet periods |
+
+### Error Count
+
+Fires when the total number of errors recorded in the lookback window reaches the threshold. Captured messages (log-style events without an exception) are not counted.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Threshold Count | 100 | Number of errors that triggers the alert |
+| Lookback Minutes | 60 | Window the count is computed over |
+
+### Endpoint Error Rate
+
+Like Error Rate, but scoped to a single endpoint.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Endpoint | | The endpoint to monitor, e.g. `GET /api/users` |
+| Threshold % | 2 | Error rate that triggers the alert |
+| Lookback Minutes | 10 | Window the rate is computed over |
+| Min Requests | 20 | Minimum requests in the window before the rule can fire |
+
+### Endpoint P95 / P99
+
+Fires when the P95 (or P99) response time over the lookback window exceeds the threshold. Two separate rule types so you can alert on each percentile independently.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Endpoint | `*` | A specific endpoint, or `*` for all endpoints combined |
+| Threshold ms | 500 | Latency that triggers the alert |
+| Lookback Minutes | 5 | Window the percentile is computed over |
+
+### Apdex Drop
+
+Fires when the Apdex score over the lookback window falls below the threshold. Traceway computes Apdex with fixed bands: requests up to 750 ms are satisfied, up to 1500 ms are tolerating, and anything slower or returning 5xx is bad. The score is `(satisfied + tolerating / 2) / total`, so 1.0 is perfect and anything below roughly 0.85 means users are noticing.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Threshold Apdex | 0.85 | Score below which the alert fires |
+| Lookback Minutes | 15 | Window the score is computed over |
+| Min Requests | 50 | Minimum requests in the window before the rule can fire |
+
+### Throughput Drop
+
+Fires when request volume in the lookback window drops by more than the configured percentage compared to the baseline window that immediately precedes it. The baseline count is normalized to the lookback length, so a 15-minute window is compared against the expected 15-minute share of a 60-minute baseline. If the baseline window has no traffic at all, the rule does not fire.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Drop % | 50 | Drop versus the normalized baseline that triggers the alert |
+| Lookback Minutes | 15 | The current window being measured |
+| Baseline Window Minutes | 60 | The preceding window used as the baseline |
+
+### Impact Score (Critical / High / Medium)
+
+Fires when an endpoint's impact score crosses into the configured level. The impact score is a 0 to 1 rating computed over the trailing 24 hours as the worst of five components: Apdex degradation, server error rate, P99 latency, client error rate, and error volume. Medium starts at 0.25, High at 0.50, and Critical at 0.75. Streaming endpoints are excluded.
+
+These rules are transition-based: they fire once when an endpoint newly enters the level, not continuously while it stays there. The endpoint alerts again only if it drops out of the level and later crosses back in. After a server restart the first evaluation establishes a baseline without firing.
+
+The notification names the endpoint, its score, and the dominant reason (for example a P99 of 9.2 s, or a 12% server error rate).
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Min Requests | 50 | Endpoints with fewer requests in 24 hours are ignored |
+
+### Task Duration
+
+Fires when the P95 duration of background task executions over the lookback window exceeds the threshold.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Task Name | `*` | A specific task, or `*` for all tasks combined |
+| Threshold ms | 30000 | P95 duration that triggers the alert |
+| Lookback Minutes | 30 | Window the percentile is computed over |
+
+### Task Failure Rate
+
+Fires when the percentage of task executions that recorded an error exceeds the threshold. An execution counts as failed when at least one exception was captured during it, so one execution with three errors counts once.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Task Name | `*` | A specific task, or `*` for all tasks combined |
+| Threshold % | 10 | Failure rate that triggers the alert |
+| Lookback Minutes | 60 | Window the rate is computed over |
+| Min Executions | 5 | Minimum executions in the window before the rule can fire |
+
+### Metric Threshold
+
+Fires when an aggregated [metric](/learn/metrics) value violates the configured comparison over the lookback window. Works for both system metrics (`cpu.used_pcnt`, `mem.used_pcnt`, ...) and custom metrics. If the window contains no data points the rule does not fire; pair it with a No Data rule if you also want to know when the metric stops reporting.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Metric Name | | The metric to monitor |
+| Aggregation | avg | One of `avg`, `max`, `min`, `sum`, `p95`, `p99`, `last` (most recent value) |
+| Operator | gt | One of `gt`, `gte`, `lt`, `lte`, `eq` |
+| Threshold Value | 90 | The value compared against the aggregate |
+| Lookback Minutes | 5 | Window the aggregate is computed over |
+
+The aggregation spans every series of the metric name, with no tag filter. For metrics tagged per host, per disk, or per table, an `avg` can hide one bad series behind healthy ones; prefer `max` (or `min` for lower-is-bad metrics) so the worst series drives the alert.
+
+### No Data
+
+Fires when no telemetry of the selected kind has been received for the configured number of minutes. Use it to detect a stopped service, a broken deploy, or a misconfigured SDK rather than an application-level problem.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Data Type | any | One of `any`, `endpoints`, `exceptions`, `metrics`, `tasks`. `any` fires only when all four kinds are silent |
+| Silence Minutes | 10 | How long the data must be absent before the alert fires |
+
+### AI Trace Cost
+
+Fires when a single [AI trace](/learn/ai-tracing) exceeds a cost threshold, catching runaway agent loops or unexpectedly expensive calls as they are ingested rather than at the end of the month.
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| Trace Name | `*` | A specific trace name, or `*` for all traces |
+| Threshold Cost | 0.5 | Cost in USD a single trace must reach to trigger the alert |
+
+The cooldown applies per trace name, so two different expensive workflows alert independently.
+
+## Evaluation Model and Cost
+
+Rules are evaluated in one of two ways:
+
+- **On ingest** (New Issue, Error Regression, AI Trace Cost): evaluated immediately as reports arrive, so alerts fire within seconds of the triggering event.
+- **Polled** (everything else): a background loop evaluates all enabled polled rules on an interval. The default is every 60 seconds; self-hosted deployments can tune it with `NOTIFICATION_POLL_SECONDS` (minimum 5). Detection latency for polled rules is at most one poll interval.
+
+Each rule type has a different evaluation cost against the telemetry database. This matters mostly for self-hosted deployments with many rules or high data volume:
+
+| Rule | Work per evaluation | Relative cost |
+|------|--------------------|---------------|
+| AI Trace Cost | In-memory check of the ingested batch, no queries | Minimal |
+| New Issue | 1 to 3 indexed lookups per distinct new error in the batch | Low |
+| Error Regression | 1 indexed lookup per distinct error in the batch | Low |
+| Error Rate | One aggregate over the lookback window | Low |
+| Error Count | One count over the lookback window | Low |
+| Endpoint Error Rate | One aggregate over one endpoint's window | Low |
+| Endpoint P95 / P99 | One percentile over the lookback window | Low |
+| Apdex Drop | One aggregate over the lookback window | Low |
+| Metric Threshold | One aggregate over one metric's window | Low |
+| No Data | One freshness check per data type, up to four for `any` | Low |
+| Task Duration | One percentile over the lookback window | Low |
+| Throughput Drop | Two counts: lookback plus baseline window | Low |
+| Task Failure Rate | Two queries joining tasks and exceptions over the window | Moderate |
+| Impact Score | One scan of the trailing 24 hours of requests | Highest |
+
+Cost-relevant behavior:
+
+- Rules that are snoozed or still in cooldown are skipped entirely, including their queries. Longer cooldowns directly reduce evaluation load.
+- The three Impact Score levels share one 24-hour scan per project per poll cycle through a 30-second cache, so adding all three costs the same as adding one.
+- On-ingest rules deduplicate per error hash, so a busy error stops generating lookups once it has alerted until its cooldown expires.
+- Polled query cost scales with the lookback window. The defaults keep windows short; a 24-hour lookback on a high-traffic project costs proportionally more.
+- In SQLite mode percentile rules (Endpoint P95/P99, Task Duration, Metric Threshold with `p95`/`p99`) load the window's values into memory to compute the percentile, so prefer short lookback windows on busy projects.
## Cooldown and Snooze
-**Cooldown** prevents a rule from firing repeatedly for the same ongoing condition. After an alert fires, the rule will not fire again until the cooldown period expires. The default cooldown is 15 minutes, and it can be configured per rule.
+**Cooldown** prevents a rule from firing repeatedly for the same ongoing condition. The default is 15 minutes, configurable per rule.
-**Snooze** lets you temporarily mute a rule for a set duration from the dashboard. This is useful during deployments, maintenance windows, or known incidents where you don't want to be notified.
+- For polled rules the cooldown is rule-wide: after a notification is delivered, the rule is not evaluated again until the cooldown expires.
+- For on-ingest rules the cooldown is per condition: per error hash for New Issue and Error Regression, per trace name for AI Trace Cost. Different errors can alert back-to-back; the same error cannot.
-## Severity
+Rule-level cooldowns are persisted with the notification history, so a server restart does not cause every rule to re-fire. Per-error cooldown state is held in memory and resets on restart, meaning an already-alerted error can alert once more after a restart. Editing a rule resets its cooldown and per-error state so the new configuration takes effect immediately.
+
+**Snooze** temporarily mutes a rule for a set duration from the dashboard. Snoozed rules are not evaluated at all. Use it during deployments, maintenance windows, or known incidents.
-Each rule has a **Severity** setting that controls the severity level attached to its notifications. The default is **Auto**, which assigns severity based on the condition:
+## Severity
-| Severity | Auto Triggers |
-|----------|--------------|
-| Critical | New errors, critical impact score, error regressions |
-| Warning | High impact score, elevated error/latency thresholds |
-| Info | Medium impact score |
+Each rule has a **Severity** setting that controls the severity attached to its notifications. The default is **Auto**, which derives severity from how badly the condition is violated:
+
+| Rule | Auto Severity |
+|------|--------------|
+| New Issue | Critical |
+| Error Regression | Critical |
+| No Data | Critical |
+| Error Rate | Warning; Critical at twice the threshold |
+| Error Count | Warning; Critical at five times the threshold |
+| Endpoint Error Rate | Warning; Critical at twice the threshold |
+| Endpoint P95 / P99 | Warning |
+| Apdex Drop | Warning; Critical when Apdex falls below 0.5 |
+| Throughput Drop | Warning; Critical at an 80% drop or more |
+| Impact Score Critical | Critical |
+| Impact Score High | Warning |
+| Impact Score Medium | Info |
+| Task Duration | Warning |
+| Task Failure Rate | Warning; Critical at twice the threshold |
+| Metric Threshold | Warning; Critical when the value deviates from the threshold by more than 20% |
+| AI Trace Cost | Warning; Critical at three times the threshold |
You can override the automatic severity by setting it explicitly on a rule to **Critical**, **Warning**, or **Info**. This is useful when you want a normally-warning condition to page as critical, or to downgrade noisy rules to info.
diff --git a/frontend/src/routes/notifications/+page.svelte b/frontend/src/routes/notifications/+page.svelte
index 7ba6f448..32a8bb39 100644
--- a/frontend/src/routes/notifications/+page.svelte
+++ b/frontend/src/routes/notifications/+page.svelte
@@ -33,6 +33,7 @@
import ChannelDialog from './channel-dialog.svelte';
import RuleDialog from './rule-dialog.svelte';
import SnoozeDialog from './snooze-dialog.svelte';
+ import { ruleTypeLabels } from './rule-types';
interface NotificationChannel {
id: number;
@@ -61,38 +62,20 @@
}
interface NotificationHistory {
- id: number;
+ ruleId: number;
ruleType: string;
ruleName: string;
+ channelType: string;
channelName: string;
severity: string;
subject: string;
body: string;
status: string;
- errorMessage: string | null;
+ errorMessage: string;
url: string;
createdAt: string;
}
- const ruleTypeLabels: Record<string, string> = {
- error_rate_threshold: 'Error Rate',
- endpoint_p95_threshold: 'Endpoint P95',
- endpoint_p99_threshold: 'Endpoint P99',
- apdex_drop: 'Apdex Drop',
- metric_threshold: 'Metric Threshold',
- no_data: 'No Data',
- error_count_threshold: 'Error Count',
- task_duration_threshold: 'Task Duration',
- task_failure_rate: 'Task Failure Rate',
- throughput_drop: 'Throughput Drop',
- endpoint_error_rate: 'Endpoint Error Rate',
- new_error: 'New Issue',
- error_regression: 'Error Regression',
- impact_score_critical: 'Impact Score Critical',
- impact_score_high: 'Impact Score High',
- impact_score_medium: 'Impact Score Medium'
- };
-
const channelTypeLabels: Record<string, string> = {
email: 'Email',
webhook: 'Webhook',
diff --git a/frontend/src/routes/notifications/rule-dialog.svelte b/frontend/src/routes/notifications/rule-dialog.svelte
index b985fbf7..1d80bebd 100644
--- a/frontend/src/routes/notifications/rule-dialog.svelte
+++ b/frontend/src/routes/notifications/rule-dialog.svelte
@@ -9,6 +9,7 @@
import { toast } from 'svelte-sonner';
import { api } from '$lib/api';
import { projectsState, isFrontendFramework } from '$lib/state/projects.svelte';
+ import { ruleTypeOptions } from './rule-types';
interface NotificationChannel {
id: number;
@@ -78,47 +79,18 @@
let dropPercent = $state(50);
let baselineWindowMinutes = $state(60);
let ignorePatterns = $state('');
- let includeArchived = $state(true);
let traceName = $state('*');
let thresholdCost = $state(0.5);
const isEditing = $derived(rule !== null);
- const ruleTypeOptions = [
- { value: 'new_error', label: 'New Issue' },
- { value: 'impact_score_critical', label: 'Impact Score Critical' },
- { value: 'impact_score_high', label: 'Impact Score High' },
- { value: 'impact_score_medium', label: 'Impact Score Medium' },
- { value: 'error_regression', label: 'Error Regression' },
- { value: 'error_rate_threshold', label: 'Error Rate' },
- { value: 'error_count_threshold', label: 'Error Count' },
- { value: 'endpoint_p95_threshold', label: 'Endpoint P95' },
- { value: 'endpoint_p99_threshold', label: 'Endpoint P99' },
- { value: 'endpoint_error_rate', label: 'Endpoint Error Rate' },
- { value: 'apdex_drop', label: 'Apdex Drop' },
- { value: 'metric_threshold', label: 'Metric Threshold' },
- { value: 'no_data', label: 'No Data' },
- { value: 'task_duration_threshold', label: 'Task Duration' },
- { value: 'task_failure_rate', label: 'Task Failure Rate' },
- { value: 'throughput_drop', label: 'Throughput Drop' },
- { value: 'ai_trace_cost', label: 'AI Trace Cost' }
- ];
-
const isFrontendProject = $derived(
!!projectsState.currentProject &&
isFrontendFramework(projectsState.currentProject.framework)
);
const visibleRuleTypes = $derived(
- isFrontendProject
- ? [{ value: 'new_error', label: 'New Issue' }]
- : [
- { value: 'new_error', label: 'New Issue' },
- { value: 'impact_score_critical', label: 'Impact Score Critical' },
- { value: 'impact_score_high', label: 'Impact Score High' },
- { value: 'impact_score_medium', label: 'Impact Score Medium' },
- { value: 'ai_trace_cost', label: 'AI Trace Cost' }
- ]
+ isFrontendProject ? [{ value: 'new_error', label: 'New Issue' }] : ruleTypeOptions
);
$effect(() => {
@@ -137,7 +109,31 @@
impact_score_medium:
'Fires when an endpoint\u2019s impact score reaches medium level (\u2265 0.25), an early warning of emerging performance or reliability issues.',
ai_trace_cost:
- 'Fires when an individual AI trace exceeds a cost threshold. Monitors per-call spending across AI providers.'
+ 'Fires when an individual AI trace exceeds a cost threshold. Monitors per-call spending across AI providers.',
+ error_regression:
+ 'Fires when an error that was previously archived (resolved) occurs again.',
+ error_rate_threshold:
+ 'Fires when the percentage of requests returning 5xx across all endpoints exceeds the threshold within the lookback window.',
+ error_count_threshold:
+ 'Fires when the total number of errors recorded within the lookback window reaches the threshold.',
+ endpoint_p95_threshold:
+ 'Fires when the P95 response time of an endpoint (or all endpoints) exceeds the threshold within the lookback window.',
+ endpoint_p99_threshold:
+ 'Fires when the P99 response time of an endpoint (or all endpoints) exceeds the threshold within the lookback window.',
+ endpoint_error_rate:
+ 'Fires when the percentage of 5xx responses on a specific endpoint exceeds the threshold within the lookback window.',
+ apdex_drop:
+ 'Fires when the Apdex satisfaction score drops below the threshold within the lookback window.',
+ metric_threshold:
+ 'Fires when an aggregated metric value violates the configured comparison against the threshold within the lookback window.',
+ no_data:
+ 'Fires when no telemetry of the selected kind (endpoints, exceptions, metrics or tasks) has been received for the configured number of minutes.',
+ task_duration_threshold:
+ 'Fires when the P95 duration of a background task exceeds the threshold within the lookback window.',
+ task_failure_rate:
+ 'Fires when the percentage of background task executions that recorded an error exceeds the threshold within the lookback window.',
+ throughput_drop:
+ 'Fires when request throughput drops by more than the configured percentage compared to the preceding baseline window.'
};
const operatorOptions = [
@@ -191,7 +187,6 @@
dropPercent = 50;
baselineWindowMinutes = 60;
ignorePatterns = '';
- includeArchived = true;
traceName = '*';
thresholdCost = 0.5;
}
@@ -261,9 +256,6 @@
case 'new_error':
ignorePatterns = (cfg.ignorePatterns || []).join(', ');
break;
- case 'error_regression':
- includeArchived = cfg.includeArchived ?? true;
- break;
case 'impact_score_critical':
case 'impact_score_high':
case 'impact_score_medium':
@@ -306,8 +298,6 @@
.filter((p) => p);
return { ignorePatterns: patterns };
}
- case 'error_regression':
- return { includeArchived };
case 'impact_score_critical':
case 'impact_score_high':
case 'impact_score_medium':
@@ -748,16 +738,6 @@
placeholder="*timeout*, *context canceled*"
/>
</div>
- {:else if ruleType === 'error_regression'}
- <div class="flex items-center gap-2">
- <input
- id="cfg-include-archived"
- type="checkbox"
- bind:checked={includeArchived}
- class="h-4 w-4 rounded border-border"
- />
- <Label for="cfg-include-archived">Include archived exceptions</Label>
- </div>
{:else if ruleType === 'impact_score_critical' || ruleType === 'impact_score_high' || ruleType === 'impact_score_medium'}
<div class="space-y-2">
<Label for="cfg-min-req-impact">Min Requests</Label>
diff --git a/frontend/src/routes/notifications/rule-types.ts b/frontend/src/routes/notifications/rule-types.ts
new file mode 100644
index 00000000..cf720bfa
--- /dev/null
+++ b/frontend/src/routes/notifications/rule-types.ts
@@ -0,0 +1,23 @@
+export const ruleTypeOptions = [
+ { value: 'new_error', label: 'New Issue' },
+ { value: 'impact_score_critical', label: 'Impact Score Critical' },
+ { value: 'impact_score_high', label: 'Impact Score High' },
+ { value: 'impact_score_medium', label: 'Impact Score Medium' },
+ { value: 'error_regression', label: 'Error Regression' },
+ { value: 'error_rate_threshold', label: 'Error Rate' },
+ { value: 'error_count_threshold', label: 'Error Count' },
+ { value: 'endpoint_p95_threshold', label: 'Endpoint P95' },
+ { value: 'endpoint_p99_threshold', label: 'Endpoint P99' },
+ { value: 'endpoint_error_rate', label: 'Endpoint Error Rate' },
+ { value: 'apdex_drop', label: 'Apdex Drop' },
+ { value: 'metric_threshold', label: 'Metric Threshold' },
+ { value: 'no_data', label: 'No Data' },
+ { value: 'task_duration_threshold', label: 'Task Duration' },
+ { value: 'task_failure_rate', label: 'Task Failure Rate' },
+ { value: 'throughput_drop', label: 'Throughput Drop' },
+ { value: 'ai_trace_cost', label: 'AI Trace Cost' }
+];
+
+export const ruleTypeLabels: Record<string, string> = Object.fromEntries(
+ ruleTypeOptions.map((o) => [o.value, o.label])
+);