Skip to content

The only tool you need to know what is happening and how to fix it.

notifications finally done

ddux2f8702744a2a
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.go
index 7022f2b3..384f6f09 100644
--- a/backend/app/controllers/clientcontrollers/client.controller.go
+++ b/backend/app/controllers/clientcontrollers/client.controller.go
@@ -14,6 +14,7 @@ import (
"database/sql"
"encoding/hex"
"fmt"
+ "log"
"net/http"
"regexp"
"strings"
@@ -195,6 +196,7 @@ func (e clientController) Report(c *gin.Context) {
exceptionHashes = append(exceptionHashes, est.ExceptionHash)
}
+ log.Printf("[notif] BroadcastReport: projectId=%s exceptionHashes=%d hashes=%v", projectId, len(exceptionHashes), exceptionHashes)
if project, exists := c.Get(middleware.ProjectContextKey); exists {
if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {
hooks.BroadcastReport(hooks.ReportEvent{
diff --git a/backend/app/controllers/notification_channel.controller.go b/backend/app/controllers/notification_channel.controller.go
index 64b72a42..70e221ac 100644
--- a/backend/app/controllers/notification_channel.controller.go
+++ b/backend/app/controllers/notification_channel.controller.go
@@ -254,7 +254,7 @@ func (ctrl *notificationChannelController) Test(ctx *gin.Context) {
}
if err := adapter.Send(ctx.Request.Context(), testMsg); err != nil {
- ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()})
+ ctx.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("test notification failed: %w", err))
return
}
diff --git a/backend/app/controllers/notification_history.controller.go b/backend/app/controllers/notification_history.controller.go
index 93b70fdb..f8024336 100644
--- a/backend/app/controllers/notification_history.controller.go
+++ b/backend/app/controllers/notification_history.controller.go
@@ -3,6 +3,7 @@ package controllers
import (
"database/sql"
"net/http"
+ "time"
"github.com/gin-gonic/gin"
"github.com/tracewayapp/traceway/backend/app/db"
@@ -17,6 +18,8 @@ type notificationHistoryController struct{}
type NotificationHistorySearchRequest struct {
Pagination PaginationParams `json:"pagination"`
Search string `json:"search"`
+ FromDate string `json:"fromDate"`
+ ToDate string `json:"toDate"`
}
func (ctrl *notificationHistoryController) List(ctx *gin.Context) {
@@ -35,13 +38,31 @@ func (ctrl *notificationHistoryController) List(ctx *gin.Context) {
page := request.Pagination.Page
pageSize := request.Pagination.PageSize
+ var fromTime, toTime *time.Time
+ if request.FromDate != "" {
+ t, err := time.Parse(time.RFC3339, request.FromDate)
+ if err != nil {
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": "Invalid fromDate format"})
+ return
+ }
+ fromTime = &t
+ }
+ if request.ToDate != "" {
+ t, err := time.Parse(time.RFC3339, request.ToDate)
+ if err != nil {
+ ctx.JSON(http.StatusBadRequest, gin.H{"error": "Invalid toDate format"})
+ return
+ }
+ toTime = &t
+ }
+
type historyResult struct {
Items []*models.NotificationHistory
Total int64
}
result, err := db.ExecuteTransaction(func(tx *sql.Tx) (historyResult, error) {
- items, total, err := repositories.NotificationHistoryRepository.FindByProject(tx, projectId, page, pageSize, request.Search)
+ items, total, err := repositories.NotificationHistoryRepository.FindByProject(tx, projectId, page, pageSize, request.Search, fromTime, toTime)
return historyResult{Items: items, Total: total}, err
})
if err != nil {
diff --git a/backend/app/controllers/notification_rule.controller.go b/backend/app/controllers/notification_rule.controller.go
index 18a89e6a..2011e62d 100644
--- a/backend/app/controllers/notification_rule.controller.go
+++ b/backend/app/controllers/notification_rule.controller.go
@@ -58,6 +58,14 @@ type createRuleRequest struct {
RuleType string `json:"ruleType"`
Config json.RawMessage `json:"config"`
CooldownMinutes int `json:"cooldownMinutes"`
+ Severity string `json:"severity"`
+}
+
+var validSeverities = map[string]bool{
+ "": true,
+ "critical": true,
+ "warning": true,
+ "info": true,
}
func (ctrl *notificationRuleController) Create(ctx *gin.Context) {
@@ -86,6 +94,10 @@ func (ctrl *notificationRuleController) Create(ctx *gin.Context) {
ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Invalid rule type."})
return
}
+ if !validSeverities[req.Severity] {
+ ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Severity must be critical, warning, info, or empty for auto."})
+ return
+ }
if req.CooldownMinutes < 0 {
req.CooldownMinutes = 15
}
@@ -121,6 +133,7 @@ func (ctrl *notificationRuleController) Create(ctx *gin.Context) {
Config: req.Config,
Enabled: true,
CooldownMinutes: cooldown,
+ Severity: req.Severity,
CreatedBy: createdBy,
CreatedAt: now,
UpdatedAt: now,
@@ -169,6 +182,10 @@ func (ctrl *notificationRuleController) Update(ctx *gin.Context) {
ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Invalid rule type."})
return
}
+ if !validSeverities[req.Severity] {
+ ctx.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Severity must be critical, warning, info, or empty for auto."})
+ return
+ }
tx := middleware.GetTx(ctx)
@@ -202,6 +219,7 @@ func (ctrl *notificationRuleController) Update(ctx *gin.Context) {
existing.RuleType = req.RuleType
existing.Config = req.Config
existing.CooldownMinutes = cooldown
+ existing.Severity = req.Severity
existing.UpdatedAt = time.Now().UTC()
if err := repositories.NotificationRuleRepository.Update(tx, existing); err != nil {
diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.go
index d2ca6a68..aee1ec95 100644
--- a/backend/app/controllers/otelcontrollers/otel.controller.go
+++ b/backend/app/controllers/otelcontrollers/otel.controller.go
@@ -64,13 +64,20 @@ func (o otelController) ExportTraces(c *gin.Context) {
return
}
+ var exceptionHashes []string
+ for _, ex := range exceptions {
+ exceptionHashes = append(exceptionHashes, ex.ExceptionHash)
+ }
+
if project, exists := c.Get(middleware.ProjectContextKey); exists {
if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {
hooks.BroadcastReport(hooks.ReportEvent{
- OrganizationId: *p.OrganizationId,
- EndpointCount: len(endpoints),
- ErrorCount: len(exceptions),
- TaskCount: len(tasks),
+ OrganizationId: *p.OrganizationId,
+ ProjectId: projectId,
+ EndpointCount: len(endpoints),
+ ErrorCount: len(exceptions),
+ TaskCount: len(tasks),
+ ExceptionHashes: exceptionHashes,
})
}
}
diff --git a/backend/app/migrations/pg/0028_add_url_to_notification_history.up.sql b/backend/app/migrations/pg/0028_add_url_to_notification_history.up.sql
new file mode 100644
index 00000000..b02abe35
--- /dev/null
+++ b/backend/app/migrations/pg/0028_add_url_to_notification_history.up.sql
@@ -0,0 +1 @@
+ALTER TABLE notification_history ADD COLUMN url TEXT NOT NULL DEFAULT '';
diff --git a/backend/app/migrations/pg/0029_add_severity_to_notification_rules.up.sql b/backend/app/migrations/pg/0029_add_severity_to_notification_rules.up.sql
new file mode 100644
index 00000000..e72048d8
--- /dev/null
+++ b/backend/app/migrations/pg/0029_add_severity_to_notification_rules.up.sql
@@ -0,0 +1 @@
+ALTER TABLE notification_rules ADD COLUMN severity VARCHAR(20) NOT NULL DEFAULT '';
diff --git a/backend/app/models/models.go b/backend/app/models/models.go
index 00f52b5a..420ad9dc 100644
--- a/backend/app/models/models.go
+++ b/backend/app/models/models.go
@@ -12,6 +12,12 @@ func (metricRegistryNaming) GetTableNameFromStructName(string) string {
return "metric_registry"
}
+type notificationHistoryNaming struct{ lit.DefaultDbNamingStrategy }
+
+func (notificationHistoryNaming) GetTableNameFromStructName(string) string {
+ return "notification_history"
+}
+
func Init(driver lit.Driver) {
lit.RegisterModel[Project](driver)
lit.RegisterModel[User](driver)
@@ -28,7 +34,7 @@ func Init(driver lit.Driver) {
lit.RegisterModel[WidgetGroupWidget](driver)
lit.RegisterModel[NotificationChannel](driver)
lit.RegisterModel[NotificationRule](driver)
- lit.RegisterModel[NotificationHistory](driver)
+ lit.RegisterModelWithNaming[NotificationHistory](driver, notificationHistoryNaming{})
lit.RegisterModel[NotificationRuleWithChannel](driver)
for _, register := range ExtensionModelRegistrations {
diff --git a/backend/app/models/notification.model.go b/backend/app/models/notification.model.go
index a6114b99..4e597923 100644
--- a/backend/app/models/notification.model.go
+++ b/backend/app/models/notification.model.go
@@ -28,6 +28,7 @@ type NotificationRule struct {
Config json.RawMessage `json:"config" lit:"config"`
Enabled bool `json:"enabled" lit:"enabled"`
CooldownMinutes int `json:"cooldownMinutes" lit:"cooldown_minutes"`
+ Severity string `json:"severity" lit:"severity"`
SnoozedUntil *time.Time `json:"snoozedUntil" lit:"snoozed_until"`
CreatedBy *int `json:"createdBy" lit:"created_by"`
CreatedAt time.Time `json:"createdAt" lit:"created_at"`
@@ -47,6 +48,7 @@ type NotificationHistory struct {
Body string `json:"body" lit:"body"`
Status string `json:"status" lit:"status"`
ErrorMessage *string `json:"errorMessage" lit:"error_message"`
+ URL string `json:"url" lit:"url"`
CreatedAt time.Time `json:"createdAt" lit:"created_at"`
}
@@ -59,6 +61,7 @@ type NotificationRuleWithChannel struct {
Config json.RawMessage `json:"config" lit:"config"`
Enabled bool `json:"enabled" lit:"enabled"`
CooldownMinutes int `json:"cooldownMinutes" lit:"cooldown_minutes"`
+ Severity string `json:"severity" lit:"severity"`
SnoozedUntil *time.Time `json:"snoozedUntil" lit:"snoozed_until"`
CreatedBy *int `json:"createdBy" lit:"created_by"`
CreatedAt time.Time `json:"createdAt" lit:"created_at"`
diff --git a/backend/app/notifications/adapter_email.go b/backend/app/notifications/adapter_email.go
index 287c61d4..963d82e7 100644
--- a/backend/app/notifications/adapter_email.go
+++ b/backend/app/notifications/adapter_email.go
@@ -2,9 +2,12 @@ package notifications
import (
"context"
+ "crypto/tls"
"fmt"
+ "net"
"net/smtp"
"strings"
+ "time"
"github.com/tracewayapp/traceway/backend/app/config"
"github.com/tracewayapp/traceway/backend/app/services"
@@ -62,5 +65,54 @@ func (a *EmailAdapter) Send(ctx context.Context, msg Message) error {
emailMsg := fmt.Sprintf("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s",
from, strings.Join(a.Recipients, ", "), subject, msg.Body)
- return smtp.SendMail(addr, auth, from, a.Recipients, []byte(emailMsg))
+ return sendMailWithTimeout(ctx, addr, auth, from, a.Recipients, []byte(emailMsg))
+}
+
+func sendMailWithTimeout(ctx context.Context, addr string, auth smtp.Auth, from string, to []string, msg []byte) error {
+ conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
+ if err != nil {
+ return fmt.Errorf("SMTP dial failed: %w", err)
+ }
+
+ host, _, _ := net.SplitHostPort(addr)
+ client, err := smtp.NewClient(conn, host)
+ if err != nil {
+ conn.Close()
+ return fmt.Errorf("SMTP client failed: %w", err)
+ }
+ defer client.Close()
+
+ conn.SetDeadline(time.Now().Add(10 * time.Second))
+
+ if ok, _ := client.Extension("STARTTLS"); ok {
+ if err := client.StartTLS(&tls.Config{ServerName: host}); err != nil {
+ return fmt.Errorf("SMTP STARTTLS failed: %w", err)
+ }
+ }
+
+ if auth != nil {
+ if err := client.Auth(auth); err != nil {
+ return fmt.Errorf("SMTP auth failed: %w", err)
+ }
+ }
+ if err := client.Mail(from); err != nil {
+ return fmt.Errorf("SMTP MAIL failed: %w", err)
+ }
+ for _, r := range to {
+ if err := client.Rcpt(r); err != nil {
+ return fmt.Errorf("SMTP RCPT failed: %w", err)
+ }
+ }
+ w, err := client.Data()
+ if err != nil {
+ return fmt.Errorf("SMTP DATA failed: %w", err)
+ }
+ if _, err := w.Write(msg); err != nil {
+ return fmt.Errorf("SMTP write failed: %w", err)
+ }
+ if err := w.Close(); err != nil {
+ return fmt.Errorf("SMTP close data failed: %w", err)
+ }
+ client.Quit()
+ return nil
}
diff --git a/backend/app/notifications/dispatch.go b/backend/app/notifications/dispatch.go
index 8e7298cf..ff50c694 100644
--- a/backend/app/notifications/dispatch.go
+++ b/backend/app/notifications/dispatch.go
@@ -4,7 +4,10 @@ import (
"context"
"database/sql"
"fmt"
+ "log"
+ "strings"
"time"
+ "unicode/utf8"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/models"
@@ -12,6 +15,14 @@ import (
traceway "go.tracewayapp.com"
)
+func sanitizeForDB(s string) string {
+ s = strings.ReplaceAll(s, "\x00", "")
+ if !utf8.ValidString(s) {
+ s = strings.ToValidUTF8(s, "\uFFFD")
+ }
+ return s
+}
+
func dispatch(rule *models.NotificationRuleWithChannel, msg Message) {
channel, dbErr := db.ExecuteTransaction(func(tx *sql.Tx) (*models.NotificationChannel, error) {
return repositories.NotificationChannelRepository.FindById(tx, rule.ChannelId)
@@ -24,6 +35,7 @@ func dispatch(rule *models.NotificationRuleWithChannel, msg Message) {
adapter, err := NewAdapter(channel.ChannelType, channel.Config)
if err != nil {
+ log.Printf("[notif] dispatch: ruleId=%d adapter error: %v", rule.Id, err)
recordHistory(rule, msg, "failed", err.Error())
recordFiredNotification(rule, msg, "failed", err.Error())
return
@@ -34,15 +46,20 @@ func dispatch(rule *models.NotificationRuleWithChannel, msg Message) {
msg.RuleType = rule.RuleType
msg.RuleName = rule.Name
+ if rule.Severity != "" {
+ msg.Severity = Severity(rule.Severity)
+ }
err = adapter.Send(ctx, msg)
if err != nil {
+ log.Printf("[notif] dispatch: ruleId=%d send FAILED: %v", rule.Id, err)
recordHistory(rule, msg, "failed", err.Error())
recordFiredNotification(rule, msg, "failed", err.Error())
traceway.CaptureException(fmt.Errorf("notification dispatch failed (rule=%d, channel=%s): %w", rule.Id, rule.ChannelName, err))
return
}
+ log.Printf("[notif] dispatch: ruleId=%d sent successfully via %s", rule.Id, channel.ChannelType)
recordHistory(rule, msg, "sent", "")
recordFiredNotification(rule, msg, "sent", "")
cooldowns.recordFire(rule.Id)
@@ -58,10 +75,10 @@ func recordFiredNotification(rule *models.NotificationRuleWithChannel, msg Messa
ChannelType: rule.ChannelType,
ChannelName: rule.ChannelName,
Severity: string(msg.Severity),
- Subject: msg.Subject,
- Body: msg.Body,
+ Subject: sanitizeForDB(msg.Subject),
+ Body: sanitizeForDB(msg.Body),
Status: status,
- ErrorMsg: errorMsg,
+ ErrorMsg: sanitizeForDB(errorMsg),
Endpoint: msg.Endpoint,
FiredAt: time.Now().UTC(),
})
@@ -88,17 +105,25 @@ func recordHistory(rule *models.NotificationRuleWithChannel, msg Message, status
RuleName: rule.Name,
ChannelName: rule.ChannelName,
Severity: string(msg.Severity),
- Subject: msg.Subject,
- Body: msg.Body,
+ Subject: sanitizeForDB(msg.Subject),
+ Body: sanitizeForDB(msg.Body),
Status: status,
ErrorMessage: errMsgPtr,
+ URL: msg.URL,
CreatedAt: time.Now().UTC(),
}
+ if errMsgPtr != nil {
+ sanitized := sanitizeForDB(*errMsgPtr)
+ errMsgPtr = &sanitized
+ history.ErrorMessage = errMsgPtr
+ }
+
_, dbErr := db.ExecuteTransaction(func(tx *sql.Tx) (int, error) {
return repositories.NotificationHistoryRepository.Create(tx, history)
})
if dbErr != nil {
+ log.Printf("[notif] recordHistory FAILED: ruleId=%d subject=%q err=%v", rule.Id, msg.Subject, dbErr)
traceway.CaptureException(fmt.Errorf("failed to record notification history: %w", dbErr))
}
}
diff --git a/backend/app/notifications/evaluator.go b/backend/app/notifications/evaluator.go
index f64bd403..bf636b86 100644
--- a/backend/app/notifications/evaluator.go
+++ b/backend/app/notifications/evaluator.go
@@ -78,7 +78,13 @@ func evaluatePolledRules(ctx context.Context) {
}
if result != nil && result.Fired {
- dispatch(rule, result.Message)
+ if len(result.Messages) > 0 {
+ for _, msg := range result.Messages {
+ dispatch(rule, msg)
+ }
+ } else {
+ dispatch(rule, result.Message)
+ }
}
}
}
diff --git a/backend/app/notifications/evaluator_event.go b/backend/app/notifications/evaluator_event.go
index 2abd81ce..6d2516d8 100644
--- a/backend/app/notifications/evaluator_event.go
+++ b/backend/app/notifications/evaluator_event.go
@@ -5,6 +5,7 @@ import (
"database/sql"
"encoding/json"
"fmt"
+ "log"
"strings"
"time"
@@ -76,17 +77,39 @@ func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChan
// count > 1 means this hash already existed before this batch
if count > 1 {
- continue
+ // Check if this hash was archived (resolved) — if so, a re-occurrence should be treated as new
+ var archivedCount uint64
+ archErr := chdb.Conn.QueryRow(ctx,
+ "SELECT count() FROM archived_exceptions FINAL WHERE project_id = ? AND exception_hash = ?",
+ event.ProjectId, hash).Scan(&archivedCount)
+ if archErr != nil || archivedCount == 0 {
+ continue
+ }
+
+ // Archived — only fire if this is the first re-occurrence after archiving
+ var postArchiveCount uint64
+ archErr = chdb.Conn.QueryRow(ctx,
+ "SELECT count() FROM exception_stack_traces WHERE project_id = ? AND exception_hash = ? AND recorded_at > (SELECT max(archived_at) FROM archived_exceptions FINAL WHERE project_id = ? AND exception_hash = ?)",
+ event.ProjectId, hash, event.ProjectId, hash).Scan(&postArchiveCount)
+ if archErr != nil {
+ traceway.CaptureException(fmt.Errorf("new_error post-archive count failed: %w", archErr))
+ continue
+ }
+
+ if postArchiveCount > 1 {
+ continue
+ }
}
- errorType := getErrorTypeForHash(ctx, event.ProjectId, hash)
+ details := getExceptionDetails(ctx, event.ProjectId, hash)
- if shouldIgnore(errorType, cfg.IgnorePatterns) {
+ if shouldIgnore(details.ErrorType, cfg.IgnorePatterns) {
continue
}
dedup.record(dedupKey)
- msg := buildNewErrorMessage(errorType, hash, "")
+ projectName := getProjectName(rule.ProjectId)
+ msg := buildNewErrorMessage(details, projectName)
dispatch(rule, msg)
}
}
@@ -112,19 +135,51 @@ func evaluateErrorRegression(ctx context.Context, rule *models.NotificationRuleW
continue
}
- errorType := getErrorTypeForHash(ctx, event.ProjectId, hash)
+ details := getExceptionDetails(ctx, event.ProjectId, hash)
dedup.record(dedupKey)
- msg := buildErrorRegressionMessage(errorType, hash, "")
+ projectName := getProjectName(rule.ProjectId)
+ msg := buildErrorRegressionMessage(details, projectName)
dispatch(rule, msg)
}
}
-func getErrorTypeForHash(ctx context.Context, projectId uuid.UUID, hash string) string {
- var stackTrace string
+func getExceptionDetails(ctx context.Context, projectId uuid.UUID, hash string) ExceptionDetails {
+ var id uuid.UUID
+ var stackTrace, appVersion, serverName, attributesJSON string
+ var recordedAt time.Time
+
err := chdb.Conn.QueryRow(ctx,
- "SELECT stack_trace FROM exception_stack_traces WHERE project_id = ? AND exception_hash = ? ORDER BY recorded_at DESC LIMIT 1",
- projectId, hash).Scan(&stackTrace)
- if err != nil || stackTrace == "" {
+ "SELECT id, stack_trace, attributes, app_version, server_name, recorded_at FROM exception_stack_traces WHERE project_id = ? AND exception_hash = ? ORDER BY recorded_at DESC LIMIT 1",
+ projectId, hash).Scan(&id, &stackTrace, &attributesJSON, &appVersion, &serverName, &recordedAt)
+
+ details := ExceptionDetails{
+ Hash: hash,
+ }
+
+ if err != nil {
+ details.ErrorType = "Unknown Error"
+ return details
+ }
+
+ details.Id = id.String()
+ details.StackTrace = stackTrace
+ details.AppVersion = appVersion
+ details.ServerName = serverName
+ details.RecordedAt = recordedAt
+
+ if attributesJSON != "" && attributesJSON != "{}" {
+ attrs := make(map[string]string)
+ if jsonErr := json.Unmarshal([]byte(attributesJSON), &attrs); jsonErr == nil {
+ details.Attributes = attrs
+ }
+ }
+
+ details.ErrorType = extractErrorType(stackTrace)
+ return details
+}
+
+func extractErrorType(stackTrace string) string {
+ if stackTrace == "" {
return "Unknown Error"
}
lines := strings.SplitN(stackTrace, "\n", 2)
@@ -138,6 +193,16 @@ func getErrorTypeForHash(ctx context.Context, projectId uuid.UUID, hash string)
return "Unknown Error"
}
+func getProjectName(projectId uuid.UUID) string {
+ project, err := db.ExecuteTransaction(func(tx *sql.Tx) (*models.Project, error) {
+ return repositories.ProjectRepository.FindById(tx, projectId)
+ })
+ if err != nil || project == nil {
+ return ""
+ }
+ return project.Name
+}
+
func shouldIgnore(errorType string, patterns []string) bool {
lower := strings.ToLower(errorType)
for _, pattern := range patterns {
diff --git a/backend/app/notifications/evaluator_polled.go b/backend/app/notifications/evaluator_polled.go
index 3c4bf060..e7e39544 100644
--- a/backend/app/notifications/evaluator_polled.go
+++ b/backend/app/notifications/evaluator_polled.go
@@ -14,8 +14,9 @@ import (
)
type EvalResult struct {
- Fired bool
- Message Message
+ Fired bool
+ Message Message
+ Messages []Message
}
type RuleEvaluator func(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error)
@@ -73,7 +74,8 @@ func evaluateErrorRateThreshold(ctx context.Context, rule *models.NotificationRu
return &EvalResult{Fired: false}, nil
}
- msg := buildErrorRateMessage(rate, cfg.ThresholdPercent, cfg.LookbackMinutes, "")
+ projectName := getProjectName(projectId)
+ msg := buildErrorRateMessage(rate, cfg.ThresholdPercent, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -119,7 +121,8 @@ func evaluateEndpointP95Threshold(ctx context.Context, rule *models.Notification
if endpoint == "" || endpoint == "*" {
endpoint = "all endpoints"
}
- msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, "")
+ projectName := getProjectName(projectId)
+ msg := buildEndpointLatencyMessage("P95", p95, cfg.ThresholdMs, endpoint, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -158,7 +161,8 @@ func evaluateEndpointP99Threshold(ctx context.Context, rule *models.Notification
if endpoint == "" || endpoint == "*" {
endpoint = "all endpoints"
}
- msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, "")
+ projectName := getProjectName(projectId)
+ msg := buildEndpointLatencyMessage("P99", p99, cfg.ThresholdMs, endpoint, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -203,7 +207,8 @@ func evaluateApdexDrop(ctx context.Context, rule *models.NotificationRule, proje
return &EvalResult{Fired: false}, nil
}
- msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, "")
+ projectName := getProjectName(projectId)
+ msg := buildApdexDropMessage(apdex, cfg.ThresholdApdex, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -270,7 +275,8 @@ func evaluateMetricThreshold(ctx context.Context, rule *models.NotificationRule,
return &EvalResult{Fired: false}, nil
}
- msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, "")
+ projectName := getProjectName(projectId)
+ msg := buildMetricThresholdMessage(cfg.MetricName, value, cfg.Operator, cfg.ThresholdValue, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -303,7 +309,8 @@ func evaluateNoData(ctx context.Context, rule *models.NotificationRule, projectI
return &EvalResult{Fired: false}, nil
}
}
- msg := buildNoDataMessage("any", cfg.SilenceMinutes, "")
+ projectName := getProjectName(projectId)
+ msg := buildNoDataMessage("any", cfg.SilenceMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -333,7 +340,8 @@ func evaluateNoData(ctx context.Context, rule *models.NotificationRule, projectI
return &EvalResult{Fired: false}, nil
}
- msg := buildNoDataMessage(cfg.DataType, cfg.SilenceMinutes, "")
+ projectName := getProjectName(projectId)
+ msg := buildNoDataMessage(cfg.DataType, cfg.SilenceMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -368,7 +376,8 @@ func evaluateErrorCountThreshold(ctx context.Context, rule *models.NotificationR
return &EvalResult{Fired: false}, nil
}
- msg := buildErrorCountMessage(int64(count), cfg.ThresholdCount, cfg.LookbackMinutes, "")
+ projectName := getProjectName(projectId)
+ msg := buildErrorCountMessage(int64(count), cfg.ThresholdCount, cfg.LookbackMinutes, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -414,7 +423,8 @@ func evaluateTaskDurationThreshold(ctx context.Context, rule *models.Notificatio
if taskName == "" || taskName == "*" {
taskName = "all tasks"
}
- msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, "")
+ projectName := getProjectName(projectId)
+ msg := buildTaskDurationMessage(taskName, p95, cfg.ThresholdMs, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -472,7 +482,8 @@ func evaluateThroughputDrop(ctx context.Context, rule *models.NotificationRule,
return &EvalResult{Fired: false}, nil
}
- msg := buildThroughputDropMessage(dropPercent, "")
+ projectName := getProjectName(projectId)
+ msg := buildThroughputDropMessage(dropPercent, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -514,7 +525,8 @@ func evaluateEndpointErrorRate(ctx context.Context, rule *models.NotificationRul
return &EvalResult{Fired: false}, nil
}
- msg := buildEndpointErrorRateMessage(cfg.Endpoint, rate, cfg.ThresholdPercent, "")
+ projectName := getProjectName(projectId)
+ msg := buildEndpointErrorRateMessage(cfg.Endpoint, rate, cfg.ThresholdPercent, projectName)
return &EvalResult{Fired: true, Message: msg}, nil
}
@@ -656,19 +668,26 @@ func evaluateImpactScore(ctx context.Context, rule *models.NotificationRule, pro
impactState[stateKey] = newSet
impactStateMu.Unlock()
- for ep, data := range currentSet {
- if prevSet != nil && prevSet[ep] {
- continue
+ projectName := getProjectName(projectId)
+
+ var messages []Message
+ if prevSet != nil {
+ for ep, data := range currentSet {
+ if prevSet[ep] {
+ continue
+ }
+ reason := repositories.ComputeImpactReason(
+ ep, data.totalCount, data.satisfied, data.tolerating,
+ data.bad, data.clientErrors, data.p99, data.offsetMs,
+ )
+ messages = append(messages, buildMsg(ep, data.impact, reason, projectName))
}
- reason := repositories.ComputeImpactReason(
- ep, data.totalCount, data.satisfied, data.tolerating,
- data.bad, data.clientErrors, data.p99, data.offsetMs,
- )
- msg := buildMsg(ep, data.impact, reason, "")
- return &EvalResult{Fired: true, Message: msg}, nil
}
- return &EvalResult{Fired: false}, nil
+ if len(messages) == 0 {
+ return &EvalResult{Fired: false}, nil
+ }
+ return &EvalResult{Fired: true, Messages: messages}, nil
}
func evaluateImpactScoreCritical(ctx context.Context, rule *models.NotificationRule, projectId uuid.UUID) (*EvalResult, error) {
diff --git a/backend/app/notifications/messages.go b/backend/app/notifications/messages.go
index 720776de..f632c311 100644
--- a/backend/app/notifications/messages.go
+++ b/backend/app/notifications/messages.go
@@ -2,6 +2,9 @@ package notifications
import (
"fmt"
+ "sort"
+ "strings"
+ "time"
)
func buildErrorRateMessage(rate float64, threshold float64, window int, projectName string) Message {
@@ -17,12 +20,18 @@ func buildErrorRateMessage(rate float64, threshold float64, window int, projectN
}
}
+func endpointTimeRangeURL(now time.Time) string {
+ from := now.Add(-1 * time.Minute).UTC().Format(time.RFC3339)
+ to := now.Add(2 * time.Minute).UTC().Format(time.RFC3339)
+ return fmt.Sprintf("/endpoints?from=%s&to=%s", from, to)
+}
+
func buildEndpointLatencyMessage(percentile string, latencyMs float64, thresholdMs float64, endpoint string, projectName string) Message {
return Message{
Subject: fmt.Sprintf("[%s] %s latency %.0fms on %s", projectName, percentile, latencyMs, endpoint),
Body: fmt.Sprintf("The %s latency for %s has reached %.0fms (threshold: %.0fms).", percentile, endpoint, latencyMs, thresholdMs),
Severity: SeverityWarning,
- URL: "/endpoints?preset=1h",
+ URL: endpointTimeRangeURL(time.Now()),
}
}
@@ -35,7 +44,7 @@ func buildApdexDropMessage(apdex float64, threshold float64, projectName string)
Subject: fmt.Sprintf("[%s] Apdex dropped to %.2f (threshold: %.2f)", projectName, apdex, threshold),
Body: fmt.Sprintf("The Apdex score has dropped to %.2f (threshold: %.2f).", apdex, threshold),
Severity: severity,
- URL: "/endpoints?preset=1h",
+ URL: endpointTimeRangeURL(time.Now()),
}
}
@@ -96,7 +105,7 @@ func buildThroughputDropMessage(dropPercent float64, projectName string) Message
Subject: fmt.Sprintf("[%s] Throughput dropped %.0f%% vs baseline", projectName, dropPercent),
Body: fmt.Sprintf("Request throughput has dropped by %.0f%% compared to the baseline window.", dropPercent),
Severity: severity,
- URL: "/endpoints?preset=1h",
+ URL: endpointTimeRangeURL(time.Now()),
}
}
@@ -109,7 +118,7 @@ func buildEndpointErrorRateMessage(endpoint string, rate float64, threshold floa
Subject: fmt.Sprintf("[%s] %s error rate %.1f%%", projectName, endpoint, rate),
Body: fmt.Sprintf("The endpoint %s has an error rate of %.1f%% (threshold: %.1f%%).", endpoint, rate, threshold),
Severity: severity,
- URL: "/endpoints?preset=1h",
+ URL: endpointTimeRangeURL(time.Now()),
}
}
@@ -118,7 +127,7 @@ func buildImpactScoreCriticalMessage(endpoint string, score float64, reason stri
Subject: fmt.Sprintf("[%s] Endpoint %s impact became critical", projectName, endpoint),
Body: fmt.Sprintf("The endpoint %s has become critical (impact score: %.2f). Reason: %s", endpoint, score, reason),
Severity: SeverityCritical,
- URL: "/endpoints?preset=24h",
+ URL: endpointTimeRangeURL(time.Now()),
Endpoint: endpoint,
}
}
@@ -128,7 +137,7 @@ func buildImpactScoreHighMessage(endpoint string, score float64, reason string,
Subject: fmt.Sprintf("[%s] Endpoint %s impact became high", projectName, endpoint),
Body: fmt.Sprintf("The endpoint %s has become high impact (impact score: %.2f). Reason: %s", endpoint, score, reason),
Severity: SeverityWarning,
- URL: "/endpoints?preset=24h",
+ URL: endpointTimeRangeURL(time.Now()),
Endpoint: endpoint,
}
}
@@ -138,25 +147,80 @@ func buildImpactScoreMediumMessage(endpoint string, score float64, reason string
Subject: fmt.Sprintf("[%s] Endpoint %s impact became medium", projectName, endpoint),
Body: fmt.Sprintf("The endpoint %s has become medium impact (impact score: %.2f). Reason: %s", endpoint, score, reason),
Severity: SeverityInfo,
- URL: "/endpoints?preset=24h",
+ URL: endpointTimeRangeURL(time.Now()),
Endpoint: endpoint,
}
}
-func buildNewErrorMessage(errorType string, hash string, projectName string) Message {
+type ExceptionDetails struct {
+ Id string
+ Hash string
+ ErrorType string
+ StackTrace string
+ Attributes map[string]string
+ AppVersion string
+ ServerName string
+ RecordedAt time.Time
+}
+
+func buildNewErrorMessage(details ExceptionDetails, projectName string) Message {
return Message{
- Subject: fmt.Sprintf("[%s] New error: %s", projectName, errorType),
- Body: fmt.Sprintf("A new error has been detected: %s", errorType),
+ Subject: fmt.Sprintf("[%s] New error: %s", projectName, details.ErrorType),
+ Body: buildExceptionBody("A new error has been detected: "+details.ErrorType, details),
Severity: SeverityCritical,
- URL: fmt.Sprintf("/issues/%s", hash),
+ URL: fmt.Sprintf("/issues/%s", details.Hash),
}
}
-func buildErrorRegressionMessage(errorType string, hash string, projectName string) Message {
+func buildErrorRegressionMessage(details ExceptionDetails, projectName string) Message {
return Message{
- Subject: fmt.Sprintf("[%s] Resolved error reappeared: %s", projectName, errorType),
- Body: fmt.Sprintf("A previously resolved error has reappeared: %s", errorType),
+ Subject: fmt.Sprintf("[%s] Resolved error reappeared: %s", projectName, details.ErrorType),
+ Body: buildExceptionBody("A previously resolved error has reappeared: "+details.ErrorType, details),
Severity: SeverityCritical,
- URL: fmt.Sprintf("/issues/%s", hash),
+ URL: fmt.Sprintf("/issues/%s", details.Hash),
+ }
+}
+
+func buildExceptionBody(headline string, d ExceptionDetails) string {
+ var b strings.Builder
+ b.WriteString(headline)
+ b.WriteString("\n")
+
+ if d.Id != "" {
+ fmt.Fprintf(&b, "\nException ID: %s", d.Id)
+ }
+ if d.Hash != "" {
+ fmt.Fprintf(&b, "\nHash: %s", d.Hash)
+ }
+ if !d.RecordedAt.IsZero() {
+ fmt.Fprintf(&b, "\nOccurred at: %s", d.RecordedAt.UTC().Format("2006-01-02 15:04:05 UTC"))
}
+ if d.AppVersion != "" {
+ fmt.Fprintf(&b, "\nApp version: %s", d.AppVersion)
+ }
+ if d.ServerName != "" {
+ fmt.Fprintf(&b, "\nServer: %s", d.ServerName)
+ }
+
+ if d.StackTrace != "" {
+ fmt.Fprintf(&b, "\n\nStack trace:\n%s", d.StackTrace)
+ }
+
+ if len(d.Attributes) > 0 {
+ b.WriteString("\n\nAttributes:")
+ keys := make([]string, 0, len(d.Attributes))
+ for k := range d.Attributes {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ fmt.Fprintf(&b, "\n %s: %s", k, d.Attributes[k])
+ }
+ }
+
+ if d.Hash != "" {
+ fmt.Fprintf(&b, "\n\nView details: /issues/%s", d.Hash)
+ }
+
+ return b.String()
}
diff --git a/backend/app/repositories/notification_history.repository.go b/backend/app/repositories/notification_history.repository.go
index caf7e71a..24c138b4 100644
--- a/backend/app/repositories/notification_history.repository.go
+++ b/backend/app/repositories/notification_history.repository.go
@@ -2,6 +2,7 @@ package repositories
import (
"database/sql"
+ "time"
"github.com/tracewayapp/traceway/backend/app/models"
@@ -11,7 +12,7 @@ import (
type notificationHistoryRepository struct{}
-func (r *notificationHistoryRepository) FindByProject(tx *sql.Tx, projectId uuid.UUID, page, pageSize int, search string) ([]*models.NotificationHistory, int64, error) {
+func (r *notificationHistoryRepository) FindByProject(tx *sql.Tx, projectId uuid.UUID, page, pageSize int, search string, fromDate, toDate *time.Time) ([]*models.NotificationHistory, int64, error) {
params := lit.P{"project_id": projectId}
whereClause := "WHERE project_id = :project_id"
@@ -20,6 +21,15 @@ func (r *notificationHistoryRepository) FindByProject(tx *sql.Tx, projectId uuid
params["search"] = "%" + search + "%"
}
+ if fromDate != nil {
+ whereClause += " AND created_at >= :from_date"
+ params["from_date"] = *fromDate
+ }
+ if toDate != nil {
+ whereClause += " AND created_at <= :to_date"
+ params["to_date"] = *toDate
+ }
+
countResult, err := lit.SelectSingleNamed[models.CountResult](
tx,
"SELECT COUNT(*) as count FROM notification_history "+whereClause,
@@ -38,7 +48,7 @@ func (r *notificationHistoryRepository) FindByProject(tx *sql.Tx, projectId uuid
params["offset"] = offset
items, err := lit.SelectNamed[models.NotificationHistory](
tx,
- "SELECT id, project_id, rule_id, channel_id, rule_type, rule_name, channel_name, severity, subject, body, status, error_message, created_at FROM notification_history "+whereClause+" ORDER BY created_at DESC LIMIT :limit OFFSET :offset",
+ "SELECT id, project_id, rule_id, channel_id, rule_type, rule_name, channel_name, severity, subject, body, status, error_message, url, created_at FROM notification_history "+whereClause+" ORDER BY created_at DESC LIMIT :limit OFFSET :offset",
params,
)
if err != nil {
diff --git a/backend/app/repositories/notification_rule.repository.go b/backend/app/repositories/notification_rule.repository.go
index cb29a6bd..7317c4b6 100644
--- a/backend/app/repositories/notification_rule.repository.go
+++ b/backend/app/repositories/notification_rule.repository.go
@@ -16,7 +16,7 @@ type notificationRuleRepository struct{}
func (r *notificationRuleRepository) FindByProjectWithChannel(tx *sql.Tx, projectId uuid.UUID) ([]*models.NotificationRuleWithChannel, error) {
return lit.SelectNamed[models.NotificationRuleWithChannel](
tx,
- `SELECT r.id, r.project_id, r.channel_id, r.name, r.rule_type, r.config, r.enabled, r.cooldown_minutes, r.snoozed_until, r.created_by, r.created_at, r.updated_at,
+ `SELECT r.id, r.project_id, r.channel_id, r.name, r.rule_type, r.config, r.enabled, r.cooldown_minutes, r.severity, r.snoozed_until, r.created_by, r.created_at, r.updated_at,
c.name as channel_name, c.channel_type as channel_type
FROM notification_rules r
JOIN notification_channels c ON c.id = r.channel_id
@@ -29,7 +29,7 @@ func (r *notificationRuleRepository) FindByProjectWithChannel(tx *sql.Tx, projec
func (r *notificationRuleRepository) FindById(tx *sql.Tx, id int) (*models.NotificationRule, error) {
return lit.SelectSingleNamed[models.NotificationRule](
tx,
- "SELECT id, project_id, channel_id, name, rule_type, config, enabled, cooldown_minutes, snoozed_until, created_by, created_at, updated_at FROM notification_rules WHERE id = :id",
+ "SELECT id, project_id, channel_id, name, rule_type, config, enabled, cooldown_minutes, severity, snoozed_until, created_by, created_at, updated_at FROM notification_rules WHERE id = :id",
lit.P{"id": id},
)
}
@@ -37,7 +37,7 @@ func (r *notificationRuleRepository) FindById(tx *sql.Tx, id int) (*models.Notif
func (r *notificationRuleRepository) FindEnabledPolledRules(tx *sql.Tx) ([]*models.NotificationRuleWithChannel, error) {
return lit.Select[models.NotificationRuleWithChannel](
tx,
- `SELECT r.id, r.project_id, r.channel_id, r.name, r.rule_type, r.config, r.enabled, r.cooldown_minutes, r.snoozed_until, r.created_by, r.created_at, r.updated_at,
+ `SELECT r.id, r.project_id, r.channel_id, r.name, r.rule_type, r.config, r.enabled, r.cooldown_minutes, r.severity, r.snoozed_until, r.created_by, r.created_at, r.updated_at,
c.name as channel_name, c.channel_type as channel_type
FROM notification_rules r
JOIN notification_channels c ON c.id = r.channel_id
@@ -49,7 +49,7 @@ func (r *notificationRuleRepository) FindEnabledPolledRules(tx *sql.Tx) ([]*mode
func (r *notificationRuleRepository) FindEnabledEventRules(tx *sql.Tx, projectId uuid.UUID) ([]*models.NotificationRuleWithChannel, error) {
return lit.SelectNamed[models.NotificationRuleWithChannel](
tx,
- `SELECT r.id, r.project_id, r.channel_id, r.name, r.rule_type, r.config, r.enabled, r.cooldown_minutes, r.snoozed_until, r.created_by, r.created_at, r.updated_at,
+ `SELECT r.id, r.project_id, r.channel_id, r.name, r.rule_type, r.config, r.enabled, r.cooldown_minutes, r.severity, r.snoozed_until, r.created_by, r.created_at, r.updated_at,
c.name as channel_name, c.channel_type as channel_type
FROM notification_rules r
JOIN notification_channels c ON c.id = r.channel_id
diff --git a/backend/app/services/email.service.go b/backend/app/services/email.service.go
index b29da795..6dd0867f 100644
--- a/backend/app/services/email.service.go
+++ b/backend/app/services/email.service.go
@@ -1,10 +1,13 @@
package services
import (
- "github.com/tracewayapp/traceway/backend/app/config"
"fmt"
+ "net"
"net/smtp"
"strconv"
+ "time"
+
+ "github.com/tracewayapp/traceway/backend/app/config"
)
type emailService struct {
@@ -77,10 +80,7 @@ The Traceway Team
msg := fmt.Sprintf("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s",
e.from, toEmail, subject, body)
- auth := smtp.PlainAuth("", e.username, e.password, e.host)
- addr := fmt.Sprintf("%s:%d", e.host, e.port)
-
- err := smtp.SendMail(addr, auth, e.from, []string{toEmail}, []byte(msg))
+ err := e.sendMail([]string{toEmail}, []byte(msg))
if err != nil {
config.Logf("Failed to send invitation email to %s: %v", toEmail, err)
return err
@@ -121,10 +121,7 @@ The Traceway Team
msg := fmt.Sprintf("From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n%s",
e.from, toEmail, subject, body)
- auth := smtp.PlainAuth("", e.username, e.password, e.host)
- addr := fmt.Sprintf("%s:%d", e.host, e.port)
-
- err := smtp.SendMail(addr, auth, e.from, []string{toEmail}, []byte(msg))
+ err := e.sendMail([]string{toEmail}, []byte(msg))
if err != nil {
config.Logf("Failed to send password reset email to %s: %v", toEmail, err)
return err
@@ -133,3 +130,45 @@ The Traceway Team
config.Logf("Password reset email sent to %s", toEmail)
return nil
}
+
+func (e *emailService) sendMail(to []string, msg []byte) error {
+ addr := fmt.Sprintf("%s:%d", e.host, e.port)
+ auth := smtp.PlainAuth("", e.username, e.password, e.host)
+
+ conn, err := net.DialTimeout("tcp", addr, 10*time.Second)
+ if err != nil {
+ return fmt.Errorf("SMTP dial failed: %w", err)
+ }
+
+ client, err := smtp.NewClient(conn, e.host)
+ if err != nil {
+ conn.Close()
+ return fmt.Errorf("SMTP client failed: %w", err)
+ }
+ defer client.Close()
+
+ conn.SetDeadline(time.Now().Add(10 * time.Second))
+
+ if err := client.Auth(auth); err != nil {
+ return fmt.Errorf("SMTP auth failed: %w", err)
+ }
+ if err := client.Mail(e.from); err != nil {
+ return fmt.Errorf("SMTP MAIL failed: %w", err)
+ }
+ for _, r := range to {
+ if err := client.Rcpt(r); err != nil {
+ return fmt.Errorf("SMTP RCPT failed: %w", err)
+ }
+ }
+ w, err := client.Data()
+ if err != nil {
+ return fmt.Errorf("SMTP DATA failed: %w", err)
+ }
+ if _, err := w.Write(msg); err != nil {
+ return fmt.Errorf("SMTP write failed: %w", err)
+ }
+ if err := w.Close(); err != nil {
+ return fmt.Errorf("SMTP close data failed: %w", err)
+ }
+ return client.Quit()
+}
diff --git a/docs/pages/learn/alerts.mdx b/docs/pages/learn/alerts.mdx
index 3402e701..49756a35 100644
--- a/docs/pages/learn/alerts.mdx
+++ b/docs/pages/learn/alerts.mdx
@@ -24,6 +24,55 @@ A channel represents a notification destination. Traceway supports four channel
Each channel has a **Test** button that sends a sample notification so you can verify the configuration before attaching rules.
+### Setting Up Email
+
+To receive alerts via email, add one or more recipient email addresses. Traceway will send an email to all listed recipients when a rule fires.
+
+<img src="/channel-email.png" alt="Email channel configuration dialog" style={{ maxWidth: '100%', height: 'auto', marginTop: '16px', borderRadius: '8px', border: '1px solid #e5e7eb' }} />
+
+### Setting Up Slack
+
+To send alerts to a Slack channel, you need an **Incoming Webhook URL**:
+
+1. Go to [api.slack.com/apps](https://api.slack.com/apps) and click **Create New App**
+2. Choose **From scratch**, give it a name (e.g. "Traceway Alerts"), and select your workspace
+3. In the app settings, navigate to **Incoming Webhooks** and toggle it **on**
+4. Click **Add New Webhook to Workspace**, select the channel you want alerts posted to, and click **Allow**
+5. Copy the **Webhook URL** — it will look like `https://hooks.slack.com/services/T.../B.../xxx`
+
+Then configure the Slack channel in Traceway:
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| Webhook URL | Yes | The URL you copied from Slack |
+| Channel Override | No | Post to a different channel than the webhook default (e.g. `#incidents`) |
+| Username | No | Override the bot display name (defaults to your Slack app name) |
+
+<img src="/channel-slack.png" alt="Slack channel configuration dialog" style={{ maxWidth: '100%', height: 'auto', marginTop: '16px', borderRadius: '8px', border: '1px solid #e5e7eb' }} />
+
+### Setting Up GitHub
+
+To create GitHub issues from alerts, you need a **Personal Access Token** with issue permissions:
+
+1. Go to [GitHub Settings → Developer settings → Personal access tokens → Fine-grained tokens](https://github.com/settings/tokens?type=beta)
+2. Click **Generate new token**
+3. Under **Resource owner**, select the account or organization that owns the target repository. If your organization doesn't appear, an org admin needs to enable fine-grained tokens under **Organization Settings → Third-party access → Personal access tokens → Settings**
+4. Set a descriptive name (e.g. "Traceway Alerts") and choose an expiration
+5. Under **Repository access**, select **Only select repositories** and choose the repo where issues should be created
+6. Under **Permissions → Repository permissions**, set **Issues** to **Read and write**
+7. Click **Generate token** and copy it — you won't be able to see it again
+
+Then configure the GitHub channel in Traceway:
+
+| Field | Required | Description |
+|-------|----------|-------------|
+| Personal Access Token | Yes | The fine-grained token you generated |
+| Owner | Yes | The repository owner (user or organization), e.g. `my-org` |
+| Repo | Yes | The repository name, e.g. `backend` |
+| Labels | No | Comma-separated labels to apply to created issues, e.g. `bug,traceway` |
+
+<img src="/channel-github.png" alt="GitHub channel configuration dialog" style={{ maxWidth: '100%', height: 'auto', marginTop: '16px', borderRadius: '8px', border: '1px solid #e5e7eb' }} />
+
### Webhook Payload
When a webhook channel fires, Traceway sends a JSON POST request with the following fields:
@@ -40,6 +89,8 @@ When a webhook channel fires, Traceway sends a JSON POST request with the follow
If you configure an HMAC secret on the channel, each request includes an `X-Traceway-Signature` header containing `sha256=<hex>` of the request body, which you can use to verify that the request came from Traceway.
+<img src="/channel-webhook.png" alt="Webhook channel configuration dialog" style={{ maxWidth: '100%', height: 'auto', marginTop: '16px', borderRadius: '8px', border: '1px solid #e5e7eb' }} />
+
## Rules
### Rule Types
@@ -80,14 +131,16 @@ New Issue rules are evaluated immediately when new data arrives during report in
## Severity
-Each alert is assigned a severity level based on the condition:
+Each rule has a **Severity** setting that controls the severity level attached to its notifications. The default is **Auto**, which assigns severity based on the condition:
-| Severity | Typical Triggers |
-|----------|-----------------|
-| Critical | New errors, critical impact score |
-| Warning | High impact score |
+| Severity | Auto Triggers |
+|----------|--------------|
+| Critical | New errors, critical impact score, error regressions |
+| Warning | High impact score, elevated error/latency thresholds |
| Info | Medium impact score |
+You can override the automatic severity by setting it explicitly on a rule to **Critical**, **Warning**, or **Info**. This is useful when you want a normally-warning condition to page as critical, or to downgrade noisy rules to info.
+
## History
All fired notifications are recorded and visible in the **History** tab on the Alerts page. Each entry shows:
@@ -100,9 +153,3 @@ All fired notifications are recorded and visible in the **History** tab on the A
| Channel | Which channel delivered the notification |
| Status | Whether delivery succeeded or failed |
| Timestamp | When the notification was sent |
-erity | Critical, warning, or info |
-| Rule Name | The rule that triggered |
-| Subject | Alert title summarizing the condition |
-| Channel | Which channel delivered the notification |
-| Status | Whether delivery succeeded or failed |
-| Timestamp | When the notification was sent |
diff --git a/docs/public/channel-email.png b/docs/public/channel-email.png
new file mode 100644
index 00000000..9c6cdb90
Binary files /dev/null and b/docs/public/channel-email.png differ
diff --git a/docs/public/channel-github.png b/docs/public/channel-github.png
new file mode 100644
index 00000000..82fe8c03
Binary files /dev/null and b/docs/public/channel-github.png differ
diff --git a/docs/public/channel-slack.png b/docs/public/channel-slack.png
new file mode 100644
index 00000000..84d07ebe
Binary files /dev/null and b/docs/public/channel-slack.png differ
diff --git a/docs/public/channel-webhook.png b/docs/public/channel-webhook.png
new file mode 100644
index 00000000..00991de7
Binary files /dev/null and b/docs/public/channel-webhook.png differ
diff --git a/frontend/src/lib/components/app-sidebar.svelte b/frontend/src/lib/components/app-sidebar.svelte
index 123297c1..83b664fe 100644
--- a/frontend/src/lib/components/app-sidebar.svelte
+++ b/frontend/src/lib/components/app-sidebar.svelte
@@ -41,7 +41,7 @@
title: 'Metrics',
stickyParams: ['preset', 'from', 'to']
},
- { Icon: Bell, href: '/notifications', title: 'Alerts', stickyParams: [] },
+ { Icon: Bell, href: '/notifications', title: 'Alerts', stickyParams: ['preset', 'from', 'to'] },
{ Icon: Link2, href: '/connection', title: 'Connection', stickyParams: [] }
];
diff --git a/frontend/src/routes/notifications/+page.svelte b/frontend/src/routes/notifications/+page.svelte
index 8f5ae6d0..3ad699d2 100644
--- a/frontend/src/routes/notifications/+page.svelte
+++ b/frontend/src/routes/notifications/+page.svelte
@@ -1,9 +1,11 @@
<script lang="ts">
- import { onMount } from 'svelte';
+ import { onMount, onDestroy } from 'svelte';
+ import { browser } from '$app/environment';
import { page } from '$app/state';
import { goto } from '$app/navigation';
import { api } from '$lib/api';
import { projectsState } from '$lib/state/projects.svelte';
+ import { getTimezone } from '$lib/state/timezone.svelte';
import { LoadingCircle } from '$lib/components/ui/loading-circle';
import { Button } from '$lib/components/ui/button';
import { Badge } from '$lib/components/ui/badge';
@@ -16,6 +18,17 @@
import { Plus, Pencil, Trash2, Zap, ZapOff, Clock, Send, Info } from '@lucide/svelte';
import { SearchBar } from '$lib/components/ui/search-bar';
import { TableEmptyState } from '$lib/components/ui/table-empty-state';
+ import { TimeRangePicker } from '$lib/components/ui/time-range-picker';
+ import { CalendarDate } from '@internationalized/date';
+ import {
+ parseTimeRangeFromUrl,
+ getResolvedTimeRange,
+ getTimeRangeFromPreset,
+ dateToCalendarDate,
+ dateToTimeString,
+ updateUrl
+ } from '$lib/utils/url-params';
+ import { calendarDateTimeToLuxon, toUTCISO } from '$lib/utils/formatters';
import ChannelDialog from './channel-dialog.svelte';
import RuleDialog from './rule-dialog.svelte';
@@ -40,6 +53,7 @@
config: any;
enabled: boolean;
cooldownMinutes: number;
+ severity: string;
snoozedUntil: string | null;
channelName: string;
channelType: string;
@@ -56,6 +70,7 @@
body: string;
status: string;
errorMessage: string | null;
+ url: string;
createdAt: string;
}
@@ -98,7 +113,15 @@
function setTab(tab: string) {
const url = new URL(window.location.href);
url.searchParams.set('tab', tab);
+ if (tab !== 'history') {
+ url.searchParams.delete('preset');
+ url.searchParams.delete('from');
+ url.searchParams.delete('to');
+ }
goto(url.toString(), { replaceState: true, noScroll: true });
+ if (tab === 'history') {
+ loadHistory(false);
+ }
}
let channels = $state<NotificationChannel[]>([]);
@@ -115,6 +138,52 @@
let historyTotalPages = $state(0);
let searchQuery = $state('');
+ const timezone = $derived(getTimezone());
+
+ function parseHistoryUrlParams() {
+ if (!browser) return { preset: '7d', from: null, to: null };
+ return parseTimeRangeFromUrl(timezone, '7d');
+ }
+
+ const initialUrlParams = parseHistoryUrlParams();
+ const initialRange = getResolvedTimeRange(initialUrlParams, timezone);
+
+ let selectedPreset = $state<string | null>(initialUrlParams.preset);
+ let fromDate = $state<CalendarDate>(dateToCalendarDate(initialRange.from, timezone));
+ let toDate = $state<CalendarDate>(dateToCalendarDate(initialRange.to, timezone));
+ let fromTime = $state(dateToTimeString(initialRange.from, timezone));
+ let toTime = $state(dateToTimeString(initialRange.to, timezone));
+
+ function getFromDateTimeUTC(): string {
+ const [hour, minute] = fromTime.split(':').map(Number);
+ const luxonDt = calendarDateTimeToLuxon(
+ { year: fromDate.year, month: fromDate.month, day: fromDate.day, hour, minute },
+ timezone
+ );
+ return toUTCISO(luxonDt);
+ }
+
+ function getToDateTimeUTC(): string {
+ const [hour, minute] = toTime.split(':').map(Number);
+ const luxonDt = calendarDateTimeToLuxon(
+ { year: toDate.year, month: toDate.month, day: toDate.day, hour, minute },
+ timezone
+ ).endOf('minute');
+ return toUTCISO(luxonDt);
+ }
+
+ function updateHistoryUrl(pushToHistory = true) {
+ const params: Record<string, string | null | undefined> = {};
+ params.tab = 'history';
+ if (selectedPreset) {
+ params.preset = selectedPreset;
+ } else {
+ params.from = getFromDateTimeUTC();
+ params.to = getToDateTimeUTC();
+ }
+ updateUrl(params, { pushToHistory });
+ }
+
let channelDialogOpen = $state(false);
let editingChannel = $state<NotificationChannel | null>(null);
let ruleDialogOpen = $state(false);
@@ -128,6 +197,15 @@
let showDeleteRuleDialog = $state(false);
let deletingRule = $state<NotificationRule | null>(null);
+ let showToggleRuleDialog = $state(false);
+ let togglingRule = $state<NotificationRule | null>(null);
+ let togglingLoading = $state(false);
+
+ let showTestChannelDialog = $state(false);
+ let testingChannel = $state<NotificationChannel | null>(null);
+ let testingLoading = $state(false);
+ let testError = $state('');
+
async function loadChannels() {
channelsLoading = true;
try {
@@ -156,14 +234,27 @@
}
}
- async function loadHistory() {
+ async function loadHistory(pushToHistory = true) {
historyLoading = true;
+
+ if (selectedPreset) {
+ const range = getTimeRangeFromPreset(selectedPreset, timezone);
+ fromDate = dateToCalendarDate(range.from, timezone);
+ toDate = dateToCalendarDate(range.to, timezone);
+ fromTime = dateToTimeString(range.from, timezone);
+ toTime = dateToTimeString(range.to, timezone);
+ }
+
+ updateHistoryUrl(pushToHistory);
+
try {
const res = await api.post(
'/notification-history',
{
pagination: { page: historyPage, pageSize: historyPageSize },
- search: searchQuery.trim()
+ search: searchQuery.trim(),
+ fromDate: getFromDateTimeUTC(),
+ toDate: getToDateTimeUTC()
},
{ projectId: projectsState.currentProjectId ?? undefined }
);
@@ -179,7 +270,7 @@
function handleHistorySearch() {
historyPage = 1;
- loadHistory();
+ loadHistory(true);
}
function openDeleteChannel(channel: NotificationChannel) {
@@ -203,18 +294,35 @@
}
}
- async function testChannel(id: number) {
+ function openTestChannel(channel: NotificationChannel) {
+ testingChannel = channel;
+ testError = '';
+ showTestChannelDialog = true;
+ }
+
+ async function confirmTestChannel() {
+ if (!testingChannel) return;
+ testingLoading = true;
+ testError = '';
try {
await api.post(
- `/notification-channels/${id}/test`,
+ `/notification-channels/${testingChannel.id}/test`,
{},
- {
- projectId: projectsState.currentProjectId ?? undefined
- }
+ { projectId: projectsState.currentProjectId ?? undefined }
);
toast.success('Test notification sent', { position: 'top-center' });
+ showTestChannelDialog = false;
+ testingChannel = null;
} catch (e: any) {
- toast.error(e.message || 'Test failed', { position: 'top-center' });
+ if (e.status === 422) {
+ testError = e.message || 'Validation failed';
+ } else {
+ toast.error('An unexpected error has occurred', { position: 'top-center' });
+ showTestChannelDialog = false;
+ testingChannel = null;
+ }
+ } finally {
+ testingLoading = false;
}
}
@@ -238,18 +346,29 @@
}
}
- async function toggleRule(id: number) {
+ function openToggleRule(rule: NotificationRule) {
+ togglingRule = rule;
+ showToggleRuleDialog = true;
+ }
+
+ async function confirmToggleRule() {
+ if (!togglingRule) return;
+ togglingLoading = true;
try {
+ const action = togglingRule.enabled ? 'disabled' : 'enabled';
await api.post(
- `/notification-rules/${id}/toggle`,
+ `/notification-rules/${togglingRule.id}/toggle`,
{},
- {
- projectId: projectsState.currentProjectId ?? undefined
- }
+ { projectId: projectsState.currentProjectId ?? undefined }
);
+ toast.success(`Successfully ${action} the Rule`, { position: 'top-center' });
+ showToggleRuleDialog = false;
+ togglingRule = null;
loadRules();
} catch {
toast.error('Failed to toggle rule', { position: 'top-center' });
+ } finally {
+ togglingLoading = false;
}
}
@@ -297,19 +416,52 @@
function handleHistoryPageChange(newPage: number) {
historyPage = newPage;
- loadHistory();
+ loadHistory(true);
}
function handleHistoryPageSizeChange(newSize: number) {
historyPageSize = newSize;
historyPage = 1;
- loadHistory();
+ loadHistory(true);
+ }
+
+ function handleTimeRangeChange(
+ from: { date: CalendarDate; time: string },
+ to: { date: CalendarDate; time: string },
+ preset: string | null
+ ) {
+ fromDate = from.date;
+ toDate = to.date;
+ fromTime = from.time;
+ toTime = to.time;
+ selectedPreset = preset;
+ historyPage = 1;
+ loadHistory(true);
+ }
+
+ function handlePopState() {
+ const urlParams = parseHistoryUrlParams();
+ const range = getResolvedTimeRange(urlParams, timezone);
+ selectedPreset = urlParams.preset;
+ fromDate = dateToCalendarDate(range.from, timezone);
+ toDate = dateToCalendarDate(range.to, timezone);
+ fromTime = dateToTimeString(range.from, timezone);
+ toTime = dateToTimeString(range.to, timezone);
+ historyPage = 1;
+ loadHistory(false);
}
onMount(() => {
+ window.addEventListener('popstate', handlePopState);
loadChannels();
loadRules();
- loadHistory();
+ loadHistory(false);
+ });
+
+ onDestroy(() => {
+ if (typeof window !== 'undefined') {
+ window.removeEventListener('popstate', handlePopState);
+ }
});
</script>
@@ -392,7 +544,7 @@
<Button
variant="ghost"
size="icon"
- onclick={() => testChannel(channel.id)}
+ onclick={() => openTestChannel(channel)}
title="Test"
>
<Send class="h-4 w-4" />
@@ -478,7 +630,7 @@
<Button
variant="ghost"
size="icon"
- onclick={() => toggleRule(rule.id)}
+ onclick={() => openToggleRule(rule)}
title={rule.enabled ? 'Disable' : 'Enable'}
>
{#if rule.enabled}
@@ -512,13 +664,23 @@
</div>
{/if}
{:else if activeTab === 'history'}
- <div class="pt-2">
+ <div class="flex flex-col gap-2 pt-2 sm:flex-row sm:items-center sm:justify-between">
<SearchBar
placeholder="Search Historic Alerts..."
bind:value={searchQuery}
onSearch={handleHistorySearch}
disabled={historyLoading}
/>
+ <div class="w-full sm:w-auto">
+ <TimeRangePicker
+ bind:fromDate
+ bind:toDate
+ bind:fromTime
+ bind:toTime
+ bind:preset={selectedPreset}
+ onApply={handleTimeRangeChange}
+ />
+ </div>
</div>
<div class="overflow-hidden rounded-md border">
@@ -561,7 +723,13 @@
{/if}
</Table.Cell>
<Table.Cell class="font-medium">{item.ruleName}</Table.Cell>
- <Table.Cell class="max-w-xs truncate">{item.subject}</Table.Cell>
+ <Table.Cell class="max-w-xs truncate">
+ {#if item.url}
+ <a href={item.url} class="text-blue-600 hover:underline dark:text-blue-400">{item.subject}</a>
+ {:else}
+ {item.subject}
+ {/if}
+ </Table.Cell>
<Table.Cell>{item.channelName}</Table.Cell>
<Table.Cell>
{#if item.status === 'sent'}
@@ -651,3 +819,46 @@
</AlertDialog.Footer>
</AlertDialog.Content>
</AlertDialog.Root>
+
+<AlertDialog.Root bind:open={showToggleRuleDialog}>
+ <AlertDialog.Content
+ interactOutsideBehavior={togglingLoading ? 'ignore' : 'close'}
+ escapeKeydownBehavior={togglingLoading ? 'ignore' : 'close'}
+ >
+ <AlertDialog.Header>
+ <AlertDialog.Title>{togglingRule?.enabled ? 'Disable' : 'Enable'} Rule</AlertDialog.Title>
+ <AlertDialog.Description>
+ Are you sure you want to {togglingRule?.enabled ? 'disable' : 'enable'} "{togglingRule?.name}"?
+ </AlertDialog.Description>
+ </AlertDialog.Header>
+ <AlertDialog.Footer>
+ <Button variant="outline" onclick={() => (showToggleRuleDialog = false)} disabled={togglingLoading}>Cancel</Button>
+ <Button onclick={confirmToggleRule} disabled={togglingLoading}>
+ {togglingRule?.enabled ? 'Disable' : 'Enable'}
+ </Button>
+ </AlertDialog.Footer>
+ </AlertDialog.Content>
+</AlertDialog.Root>
+
+<AlertDialog.Root bind:open={showTestChannelDialog}>
+ <AlertDialog.Content
+ interactOutsideBehavior={testingLoading ? 'ignore' : 'close'}
+ escapeKeydownBehavior={testingLoading ? 'ignore' : 'close'}
+ >
+ <AlertDialog.Header>
+ <AlertDialog.Title>Test Channel</AlertDialog.Title>
+ <AlertDialog.Description>
+ Send a test notification to "{testingChannel?.name}"? This will deliver a test message through the configured channel.
+ </AlertDialog.Description>
+ </AlertDialog.Header>
+ {#if testError}
+ <p class="text-sm text-destructive">{testError}</p>
+ {/if}
+ <AlertDialog.Footer>
+ <Button variant="outline" onclick={() => (showTestChannelDialog = false)} disabled={testingLoading}>Cancel</Button>
+ <Button onclick={confirmTestChannel} disabled={testingLoading}>
+ <Send class="h-4 w-4" /> {testingLoading ? 'Sending...' : 'Send Test'}
+ </Button>
+ </AlertDialog.Footer>
+ </AlertDialog.Content>
+</AlertDialog.Root>
diff --git a/frontend/src/routes/notifications/rule-dialog.svelte b/frontend/src/routes/notifications/rule-dialog.svelte
index ded4a376..692fd1aa 100644
--- a/frontend/src/routes/notifications/rule-dialog.svelte
+++ b/frontend/src/routes/notifications/rule-dialog.svelte
@@ -29,6 +29,7 @@
config: any;
enabled: boolean;
cooldownMinutes: number;
+ severity: string;
snoozedUntil: string | null;
channelName: string;
channelType: string;
@@ -48,9 +49,17 @@
let channelId = $state('');
let ruleType = $state('new_error');
let cooldownMinutes = $state(15);
+ let severity = $state('');
let loading = $state(false);
let error = $state('');
+ const severityOptions = [
+ { value: '', label: 'Auto (default)' },
+ { value: 'critical', label: 'Critical' },
+ { value: 'warning', label: 'Warning' },
+ { value: 'info', label: 'Info' }
+ ];
+
let thresholdPercent = $state(5);
let lookbackMinutes = $state(5);
let minRequests = $state(10);
@@ -134,6 +143,7 @@
channelId = '';
ruleType = 'new_error';
cooldownMinutes = 15;
+ severity = '';
error = '';
thresholdPercent = 5;
lookbackMinutes = 5;
@@ -161,6 +171,7 @@
channelId = r.channelId.toString();
ruleType = r.ruleType;
cooldownMinutes = r.cooldownMinutes;
+ severity = r.severity || '';
const cfg = r.config || {};
switch (r.ruleType) {
@@ -282,7 +293,8 @@
name,
ruleType,
config: buildConfig(),
- cooldownMinutes
+ cooldownMinutes,
+ severity
};
if (isEditing) {
@@ -747,6 +759,20 @@
/>
</div>
+ <div class="space-y-2">
+ <Label for="rule-severity">Severity</Label>
+ <Select.Root type="single" bind:value={severity}>
+ <Select.Trigger class="w-full">
+ {severityOptions.find((o) => o.value === severity)?.label || 'Auto (default)'}
+ </Select.Trigger>
+ <Select.Content>
+ {#each severityOptions as option}
+ <Select.Item value={option.value}>{option.label}</Select.Item>
+ {/each}
+ </Select.Content>
+ </Select.Root>
+ </div>
+
{#if error}
<p class="text-sm text-destructive">{error}</p>
{/if}
diff --git a/testing/devtesting-symfony/src/Controller/TestController.php b/testing/devtesting-symfony/src/Controller/TestController.php
index 2ec17130..4f50b82d 100644
--- a/testing/devtesting-symfony/src/Controller/TestController.php
+++ b/testing/devtesting-symfony/src/Controller/TestController.php
@@ -16,6 +16,7 @@ class TestController
#[Route('/test-ok', methods: ['GET'])]
public function testOk(): JsonResponse
{
+ usleep(random_int(0, 3000) * 1000);
return new JsonResponse(['status' => 'ok']);
}
@@ -31,13 +32,6 @@ class TestController
return new JsonResponse(['param' => $param]);
}
- #[Route('/test-exception', methods: ['GET'])]
- public function testException(): never
- {
- usleep(random_int(0, 2000) * 1000);
- throw new \RuntimeException('Cool');
- }
-
#[Route('/test-spans', methods: ['GET'])]
public function testSpans(): JsonResponse
{
@@ -72,6 +66,14 @@ class TestController
return new JsonResponse(['status' => 'ok', 'message' => 'Spans captured']);
}
+ #[Route('/test-exception/best', methods: ['GET'])]
+ public function testExceptionTest(): never
+ {
+ usleep(random_int(0, 2000) * 1000);
+ throw new \RuntimeException('Damn nice');
+ }
+
+
#[Route('/test-metrics', methods: ['GET'])]
public function testMetrics(): JsonResponse
{