The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package notifications
import (
"fmt"
"sort"
"strings"
"time"
"github.com/tracewayapp/traceway/backend/app/config"
"github.com/tracewayapp/traceway/backend/app/models"
)
// dashboardURL makes a dashboard path absolute when APP_BASE_URL is set.
// Without it the path stays relative: a guessed localhost origin would be
// wrong for everyone reading a Slack post or a GitHub issue, whereas the bare
// path still identifies the page.
func dashboardURL(path string) string {
base := config.Config.PublicBaseURL()
if base == "" || !strings.HasPrefix(path, "/") {
return path
}
return base + path
}
func alertEmail(alert models.EmailAlert) *models.NotificationEmail {
return &models.NotificationEmail{Template: models.EmailTemplateAlert, Alert: &alert}
}
func buildErrorRateMessage(rate float64, threshold float64, window int, projectName string) Message {
severity := SeverityWarning
if rate >= threshold*2 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The error rate has reached %.1f%% over the last %d minutes (threshold: %.1f%%).", rate, window, threshold)
return Message{
Subject: fmt.Sprintf("[%s] Error rate %.1f%% exceeds %.1f%%", projectName, rate, threshold),
Body: headline,
Severity: severity,
URL: "/issues?preset=1h",
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
Observed: fmt.Sprintf("%.1f%% of requests", rate),
Threshold: fmt.Sprintf("%.1f%%", threshold),
WindowMins: window,
}),
}
}
func endpointTimeRangeURL(now time.Time) string {
from := now.Add(-1 * time.Minute).UTC().Format(time.RFC3339)
to := now.Add(2 * time.Minute).UTC().Format(time.RFC3339)
return fmt.Sprintf("/endpoints?from=%s&to=%s", from, to)
}
func buildEndpointLatencyMessage(percentile string, latencyMs float64, thresholdMs float64, endpoint string, window int, projectName string) Message {
headline := fmt.Sprintf("The %s latency for %s has reached %.0fms over the last %d minutes (threshold: %.0fms).", percentile, endpoint, latencyMs, window, thresholdMs)
return Message{
Subject: fmt.Sprintf("[%s] %s latency %.0fms on %s", projectName, percentile, latencyMs, endpoint),
Body: headline,
Severity: SeverityWarning,
URL: endpointTimeRangeURL(time.Now()),
DedupToken: endpoint,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Endpoint",
Scope: endpoint,
Observed: fmt.Sprintf("%s %.0fms", percentile, latencyMs),
Threshold: fmt.Sprintf("%.0fms", thresholdMs),
WindowMins: window,
}),
}
}
func buildApdexDropMessage(apdex float64, threshold float64, total int64, window int, projectName string) Message {
severity := SeverityWarning
if apdex < 0.5 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The Apdex score has dropped to %.2f across %d requests over the last %d minutes (threshold: %.2f).", apdex, total, window, threshold)
return Message{
Subject: fmt.Sprintf("[%s] Apdex dropped to %.2f (threshold: %.2f)", projectName, apdex, threshold),
Body: headline,
Severity: severity,
URL: endpointTimeRangeURL(time.Now()),
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
Observed: fmt.Sprintf("%.2f across %d requests", apdex, total),
Threshold: fmt.Sprintf("%.2f", threshold),
WindowMins: window,
}),
}
}
func buildMetricThresholdMessage(metricName string, seriesLabel string, value float64, operator string, threshold float64, aggregation string, window int, projectName string) Message {
severity := SeverityWarning
diff := value - threshold
if diff < 0 {
diff = -diff
}
if diff > threshold*0.2 {
severity = SeverityCritical
}
if aggregation == "" {
aggregation = "avg"
}
series := metricName + seriesLabel
headline := fmt.Sprintf("The metric %s has a %s of %.2f over the last %d minutes which violates the threshold %s %.2f.", series, aggregation, value, window, operator, threshold)
return Message{
Subject: fmt.Sprintf("[%s] Metric %s is %.2f (threshold: %s %.2f)", projectName, series, value, operator, threshold),
Body: headline,
Severity: severity,
URL: "/metrics?preset=1h",
DedupToken: series,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Metric",
Scope: series,
Observed: fmt.Sprintf("%s %.2f", aggregation, value),
Threshold: fmt.Sprintf("%s %.2f", operator, threshold),
WindowMins: window,
}),
}
}
func buildMetricThresholdOverflowMessage(metricName string, count int, operator string, threshold float64, window int, projectName string) Message {
headline := fmt.Sprintf("%d more series of the metric %s violate the threshold %s %.2f over the last %d minutes. Group the metric on the metrics page to see them.", count, metricName, operator, threshold, window)
return Message{
Subject: fmt.Sprintf("[%s] Metric %s: %d more series breach the threshold (%s %.2f)", projectName, metricName, count, operator, threshold),
Body: headline,
Severity: SeverityWarning,
URL: "/metrics?preset=1h",
DedupToken: metricName + ":overflow",
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Metric",
Scope: metricName,
Observed: fmt.Sprintf("%d more series", count),
Threshold: fmt.Sprintf("%s %.2f", operator, threshold),
WindowMins: window,
}),
}
}
func buildNoDataMessage(dataType string, silenceMinutes int, projectName string) Message {
headline := fmt.Sprintf("No %s data has been received for the last %d minutes.", dataType, silenceMinutes)
return Message{
Subject: fmt.Sprintf("[%s] No %s data for %d minutes", projectName, dataType, silenceMinutes),
Body: headline,
Severity: SeverityCritical,
URL: "/",
DedupToken: dataType,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Signal",
Scope: dataType,
Observed: fmt.Sprintf("silent for %d minutes", silenceMinutes),
}),
}
}
func buildErrorCountMessage(count int64, threshold int64, window int, projectName string) Message {
severity := SeverityWarning
if count >= threshold*5 {
severity = SeverityCritical
}
headline := fmt.Sprintf("There have been %d errors in the last %d minutes (threshold: %d).", count, window, threshold)
return Message{
Subject: fmt.Sprintf("[%s] %d errors in last %d minutes", projectName, count, window),
Body: headline,
Severity: severity,
URL: "/issues?preset=1h",
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
Observed: fmt.Sprintf("%d errors", count),
Threshold: fmt.Sprintf("%d errors", threshold),
WindowMins: window,
}),
}
}
func buildTaskDurationMessage(taskName string, p95Ms float64, thresholdMs float64, window int, projectName string) Message {
headline := fmt.Sprintf("The task %s P95 duration is %.0fms over the last %d minutes (threshold: %.0fms).", taskName, p95Ms, window, thresholdMs)
return Message{
Subject: fmt.Sprintf("[%s] Task %s P95 %.0fms exceeds %.0fms", projectName, taskName, p95Ms, thresholdMs),
Body: headline,
Severity: SeverityWarning,
URL: "/tasks?preset=1h",
DedupToken: taskName,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Task",
Scope: taskName,
Observed: fmt.Sprintf("P95 %.0fms", p95Ms),
Threshold: fmt.Sprintf("%.0fms", thresholdMs),
WindowMins: window,
}),
}
}
func buildTaskFailureRateMessage(taskName string, rate float64, threshold float64, failed int64, total int64, window int, projectName string) Message {
severity := SeverityWarning
if rate >= threshold*2 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The task %s failed %d of %d executions (%.1f%%) over the last %d minutes (threshold: %.1f%%).", taskName, failed, total, rate, window, threshold)
return Message{
Subject: fmt.Sprintf("[%s] Task %s failure rate %.1f%% exceeds %.1f%%", projectName, taskName, rate, threshold),
Body: headline,
Severity: severity,
URL: "/tasks?preset=1h",
DedupToken: taskName,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Task",
Scope: taskName,
Observed: fmt.Sprintf("%d of %d failed (%.1f%%)", failed, total, rate),
Threshold: fmt.Sprintf("%.1f%%", threshold),
WindowMins: window,
}),
}
}
func buildThroughputDropMessage(dropPercent float64, current int64, expected float64, window int, projectName string) Message {
severity := SeverityWarning
if dropPercent >= 80 {
severity = SeverityCritical
}
headline := fmt.Sprintf("Request throughput has dropped by %.0f%%: %d requests in the last %d minutes vs an expected %.0f based on the baseline window.", dropPercent, current, window, expected)
return Message{
Subject: fmt.Sprintf("[%s] Throughput dropped %.0f%% vs baseline", projectName, dropPercent),
Body: headline,
Severity: severity,
URL: endpointTimeRangeURL(time.Now()),
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
Observed: fmt.Sprintf("%d requests", current),
Threshold: fmt.Sprintf("%.0f expected", expected),
WindowMins: window,
}),
}
}
func buildEndpointErrorRateMessage(endpoint string, rate float64, threshold float64, projectName string) Message {
severity := SeverityWarning
if rate >= threshold*2 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The endpoint %s has an error rate of %.1f%% (threshold: %.1f%%).", endpoint, rate, threshold)
return Message{
Subject: fmt.Sprintf("[%s] %s error rate %.1f%%", projectName, endpoint, rate),
Body: headline,
Severity: severity,
URL: endpointTimeRangeURL(time.Now()),
DedupToken: endpoint,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Endpoint",
Scope: endpoint,
Observed: fmt.Sprintf("%.1f%% of requests", rate),
Threshold: fmt.Sprintf("%.1f%%", threshold),
}),
}
}
func buildImpactScoreMessage(level string, severity Severity, endpoint string, score float64, reason string, projectName string) Message {
headline := fmt.Sprintf("The endpoint %s has become %s impact (impact score: %.2f). Reason: %s", endpoint, level, score, reason)
if level == "critical" {
headline = fmt.Sprintf("The endpoint %s has become critical (impact score: %.2f). Reason: %s", endpoint, score, reason)
}
return Message{
Subject: fmt.Sprintf("[%s] Endpoint %s impact became %s", projectName, endpoint, level),
Body: headline,
Severity: severity,
URL: endpointTimeRangeURL(time.Now()),
Endpoint: endpoint,
DedupToken: endpoint,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Endpoint",
Scope: endpoint,
Observed: fmt.Sprintf("impact score %.2f", score),
Threshold: reason,
}),
}
}
func buildImpactScoreCriticalMessage(endpoint string, score float64, reason string, projectName string) Message {
return buildImpactScoreMessage("critical", SeverityCritical, endpoint, score, reason, projectName)
}
func buildImpactScoreHighMessage(endpoint string, score float64, reason string, projectName string) Message {
return buildImpactScoreMessage("high", SeverityWarning, endpoint, score, reason, projectName)
}
func buildImpactScoreMediumMessage(endpoint string, score float64, reason string, projectName string) Message {
return buildImpactScoreMessage("medium", SeverityInfo, endpoint, score, reason, projectName)
}
func formatCostForMessage(cost float64) string {
if cost < 0.01 {
return fmt.Sprintf("$%.4f", cost)
}
return fmt.Sprintf("$%.2f", cost)
}
func buildAiTraceCostMessage(traceName string, cost float64, threshold float64, projectName string) Message {
severity := SeverityWarning
if cost >= threshold*3 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The AI trace %q cost %s, exceeding the threshold of %s.", traceName, formatCostForMessage(cost), formatCostForMessage(threshold))
return Message{
Subject: fmt.Sprintf("[%s] AI trace %s cost %s exceeds %s", projectName, traceName, formatCostForMessage(cost), formatCostForMessage(threshold)),
Body: headline,
Severity: severity,
URL: "/ai-traces?preset=1h",
DedupToken: traceName,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "AI trace",
Scope: traceName,
Observed: formatCostForMessage(cost),
Threshold: formatCostForMessage(threshold),
}),
}
}
func buildAiConversationCostMessage(conversationId string, cost, threshold float64, projectName string) Message {
severity := SeverityWarning
if cost >= threshold*3 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The AI conversation %q has cost %s over the last 24 hours, exceeding the threshold of %s.", conversationId, formatCostForMessage(cost), formatCostForMessage(threshold))
return Message{
Subject: fmt.Sprintf("[%s] AI conversation cost %s exceeds %s", projectName, formatCostForMessage(cost), formatCostForMessage(threshold)),
Body: headline,
Severity: severity,
URL: "/ai-traces/conversations?preset=24h",
DedupToken: conversationId,
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "Conversation",
Scope: conversationId,
Observed: formatCostForMessage(cost) + " in 24 hours",
Threshold: formatCostForMessage(threshold),
}),
}
}
func buildAiFlaggedContentMessage(conversationId, userId string, terms []string, projectName string) Message {
termList := strings.Join(terms, ", ")
body := fmt.Sprintf("An AI conversation matched flagged terms: %s.", termList)
if conversationId != "" {
body = fmt.Sprintf("AI conversation %q matched flagged terms: %s.", conversationId, termList)
}
if userId != "" {
body += fmt.Sprintf(" User: %s.", userId)
}
return Message{
Subject: fmt.Sprintf("[%s] AI conversation flagged: %s", projectName, termList),
Body: body,
Severity: SeverityWarning,
URL: "/ai-traces/conversations?preset=24h&flagged=1",
Email: &models.NotificationEmail{
Template: models.EmailTemplateAiFlagged,
Flagged: &models.EmailFlagged{
ProjectName: projectName,
ConversationId: conversationId,
UserId: userId,
Terms: terms,
},
},
}
}
type ExceptionDetails struct {
Id string
Hash string
ErrorType string
StackTrace string
Attributes map[string]string
AppVersion string
ServerName string
RecordedAt time.Time
TraceType string
TraceName string
}
func (d ExceptionDetails) endpointForMessage() string {
if d.TraceType == "endpoint" {
return d.TraceName
}
return ""
}
func buildNewErrorMessage(details ExceptionDetails, projectName string) Message {
return Message{
Subject: fmt.Sprintf("[%s] New error: %s", projectName, details.ErrorType),
Body: buildExceptionBody("A new error has been detected: "+details.ErrorType, details),
Severity: SeverityCritical,
URL: fmt.Sprintf("/issues/%s", details.Hash),
Endpoint: details.endpointForMessage(),
DedupToken: details.Hash,
Email: exceptionEmail(models.EmailTemplateNewError, details, projectName),
}
}
func buildErrorRegressionMessage(details ExceptionDetails, projectName string) Message {
return Message{
Subject: fmt.Sprintf("[%s] Resolved error reappeared: %s", projectName, details.ErrorType),
Body: buildExceptionBody("A previously resolved error has reappeared: "+details.ErrorType, details),
Severity: SeverityCritical,
URL: fmt.Sprintf("/issues/%s", details.Hash),
Endpoint: details.endpointForMessage(),
DedupToken: details.Hash,
Email: exceptionEmail(models.EmailTemplateErrorRegression, details, projectName),
}
}
func exceptionEmail(template string, d ExceptionDetails, projectName string) *models.NotificationEmail {
payload := &models.EmailException{
ProjectName: projectName,
ErrorType: d.ErrorType,
ExceptionId: d.Id,
Hash: d.Hash,
AppVersion: d.AppVersion,
ServerName: d.ServerName,
TraceLabel: "Endpoint",
TraceName: d.TraceName,
StackTrace: d.StackTrace,
}
if !d.RecordedAt.IsZero() {
payload.OccurredAt = d.RecordedAt.UTC().Format("2006-01-02 15:04:05 UTC")
}
switch d.TraceType {
case "task":
payload.TraceLabel = "Task"
case "ai_trace":
payload.TraceLabel = "AI trace"
}
for _, key := range sortedKeys(d.Attributes) {
payload.Attributes = append(payload.Attributes, models.EmailField{Label: key, Value: d.Attributes[key]})
}
return &models.NotificationEmail{Template: template, Exception: payload}
}
func sortedKeys(values map[string]string) []string {
keys := make([]string, 0, len(values))
for key := range values {
keys = append(keys, key)
}
sort.Strings(keys)
return keys
}
func buildExceptionBody(headline string, d ExceptionDetails) string {
var b strings.Builder
b.WriteString(headline)
b.WriteString("\n")
if d.Id != "" {
fmt.Fprintf(&b, "\nException ID: %s", d.Id)
}
if d.Hash != "" {
fmt.Fprintf(&b, "\nHash: %s", d.Hash)
}
if !d.RecordedAt.IsZero() {
fmt.Fprintf(&b, "\nOccurred at: %s", d.RecordedAt.UTC().Format("2006-01-02 15:04:05 UTC"))
}
if d.AppVersion != "" {
fmt.Fprintf(&b, "\nApp version: %s", d.AppVersion)
}
if d.ServerName != "" {
fmt.Fprintf(&b, "\nServer: %s", d.ServerName)
}
if d.TraceName != "" {
switch d.TraceType {
case "task":
fmt.Fprintf(&b, "\nTask: %s", d.TraceName)
case "ai_trace":
fmt.Fprintf(&b, "\nAI trace: %s", d.TraceName)
default:
fmt.Fprintf(&b, "\nEndpoint: %s", d.TraceName)
}
}
if d.StackTrace != "" {
fmt.Fprintf(&b, "\n\nStack trace:\n%s", d.StackTrace)
}
if len(d.Attributes) > 0 {
b.WriteString("\n\nAttributes:")
for _, k := range sortedKeys(d.Attributes) {
fmt.Fprintf(&b, "\n %s: %s", k, d.Attributes[k])
}
}
if d.Hash != "" {
fmt.Fprintf(&b, "\n\nView details: %s", dashboardURL("/issues/"+d.Hash))
}
return b.String()
}
func buildCheckDownMessage(check *models.SyntheticCheck, errorMsg string, projectName string) Message {
body := fmt.Sprintf("The synthetic check %q is failing (%d consecutive failures).", check.Name, check.ConsecutiveFailures)
if errorMsg != "" {
body += "\n\nLast error: " + errorMsg
}
return Message{
Subject: fmt.Sprintf("[%s] Check %q is down", projectName, check.Name),
Body: body,
Severity: SeverityCritical,
URL: fmt.Sprintf("/monitors/%d", check.Id),
DedupToken: fmt.Sprintf("check:%d", check.Id),
Email: &models.NotificationEmail{
Template: models.EmailTemplateCheckDown,
Check: &models.EmailCheck{
ProjectName: projectName,
CheckName: check.Name,
ConsecutiveFailures: check.ConsecutiveFailures,
LastError: errorMsg,
},
},
}
}
func buildCheckRecoveredMessage(check *models.SyntheticCheck, projectName string) Message {
return Message{
Subject: fmt.Sprintf("[%s] Check %q recovered", projectName, check.Name),
Body: fmt.Sprintf("The synthetic check %q is passing again.", check.Name),
Severity: SeverityInfo,
Recovered: true,
URL: fmt.Sprintf("/monitors/%d", check.Id),
DedupToken: fmt.Sprintf("check:%d", check.Id),
Email: &models.NotificationEmail{
Template: models.EmailTemplateCheckRecovered,
Check: &models.EmailCheck{
ProjectName: projectName,
CheckName: check.Name,
},
},
}
}
func TestChannelMessage() Message {
return Message{
Subject: "Traceway Test Notification",
Body: "This is a test notification from Traceway. If you received this, your notification channel is configured correctly.",
Severity: SeverityInfo,
RuleType: "test",
RuleName: "Test",
Email: &models.NotificationEmail{
Template: models.EmailTemplateTest,
Test: &models.EmailTest{Target: "notification channel"},
},
}
}
func TestContactMethodMessage() Message {
return Message{
Subject: "Traceway on-call test",
Body: "This is a test of your on-call contact method. If you received this, pages will reach you here.",
Severity: SeverityInfo,
RuleType: "test",
RuleName: "Test",
Email: &models.NotificationEmail{
Template: models.EmailTemplateTest,
Test: &models.EmailTest{Target: "on-call contact method"},
},
}
}
func buildSLOBurnRateMessage(entity *models.SLO, longBurn, shortBurn float64, longMinutes, shortMinutes int, threshold float64, projectName string) Message {
severity := SeverityWarning
if longBurn >= threshold*2 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The SLO %q is burning its error budget at %.1fx over the last %s and %.1fx over the last %s (threshold %.1fx).",
entity.Name, longBurn, windowLabel(longMinutes), shortBurn, windowLabel(shortMinutes), threshold)
return Message{
Subject: fmt.Sprintf("[%s] SLO %q burn rate %.1fx", projectName, entity.Name, longBurn),
Body: headline,
Severity: severity,
URL: fmt.Sprintf("/slos/%d", entity.Id),
DedupToken: SLODedupToken(entity.Id),
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "SLO",
Scope: entity.Name,
Observed: fmt.Sprintf("%.1fx over %s, %.1fx over %s", longBurn, windowLabel(longMinutes), shortBurn, windowLabel(shortMinutes)),
Threshold: fmt.Sprintf("%.1fx", threshold),
WindowMins: longMinutes,
}),
}
}
func buildSLOBudgetConsumedMessage(entity *models.SLO, consumed, threshold float64, projectName string) Message {
severity := SeverityWarning
if consumed >= 100 {
severity = SeverityCritical
}
headline := fmt.Sprintf("The SLO %q has consumed %.1f%% of its %d-day error budget (threshold %.0f%%).", entity.Name, consumed, entity.WindowDays, threshold)
subject := fmt.Sprintf("[%s] SLO %q budget %.0f%% consumed", projectName, entity.Name, consumed)
if consumed >= 100 {
headline = fmt.Sprintf("The SLO %q has exhausted its %d-day error budget: %.0f%% consumed against a threshold of %.0f%%.", entity.Name, entity.WindowDays, consumed, threshold)
subject = fmt.Sprintf("[%s] SLO %q error budget exhausted", projectName, entity.Name)
}
return Message{
Subject: subject,
Body: headline,
Severity: severity,
URL: fmt.Sprintf("/slos/%d", entity.Id),
DedupToken: SLODedupToken(entity.Id),
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "SLO",
Scope: entity.Name,
Observed: fmt.Sprintf("%.1f%% consumed", consumed),
Threshold: fmt.Sprintf("%.0f%%", threshold),
WindowMins: entity.WindowDays * 1440,
}),
}
}
func buildSLORecoveredMessage(entity *models.SLO, detail string, projectName string) Message {
headline := fmt.Sprintf("The SLO %q recovered. %s", entity.Name, detail)
return Message{
Subject: fmt.Sprintf("[%s] SLO %q recovered", projectName, entity.Name),
Body: headline,
Severity: SeverityInfo,
Recovered: true,
URL: fmt.Sprintf("/slos/%d", entity.Id),
DedupToken: SLODedupToken(entity.Id),
Email: alertEmail(models.EmailAlert{
ProjectName: projectName,
Headline: headline,
ScopeLabel: "SLO",
Scope: entity.Name,
}),
}
}