The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
//go:build !telemetry_ch
package notifications
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"time"
"github.com/google/uuid"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/hooks"
"github.com/tracewayapp/traceway/backend/app/models"
traceway "go.tracewayapp.com"
)
func evaluateNewError(ctx context.Context, rule *models.NotificationRuleWithChannel, event hooks.ReportEvent) {
var cfg newErrorConfig
json.Unmarshal(rule.Config, &cfg)
for hash, batchOccurrences := range countOccurrences(event.ExceptionHashes) {
dedupKey := errorDedupKey(rule.Id, hash)
if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
continue
}
var count int64
err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM exceptions_v2 WHERE project_id = ? AND exception_hash = ?",
event.ProjectId.String(), hash).Scan(&count)
if err != nil {
traceway.CaptureException(fmt.Errorf("new_error check failed: %w", err))
continue
}
if count > int64(batchOccurrences) {
var archivedCount int64
archErr := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM archived_exceptions WHERE project_id = ? AND exception_hash = ?",
event.ProjectId.String(), hash).Scan(&archivedCount)
if archErr != nil || archivedCount == 0 {
continue
}
var postArchiveCount int64
archErr = db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM exceptions_v2 WHERE project_id = ? AND exception_hash = ? AND recorded_at > (SELECT MAX(archived_at) FROM archived_exceptions WHERE project_id = ? AND exception_hash = ?)",
event.ProjectId.String(), hash, event.ProjectId.String(), hash).Scan(&postArchiveCount)
if archErr != nil {
traceway.CaptureException(fmt.Errorf("new_error post-archive count failed: %w", archErr))
continue
}
if postArchiveCount > int64(batchOccurrences) {
continue
}
}
details := getExceptionDetails(ctx, event.ProjectId, hash)
if shouldIgnore(details.ErrorType, cfg.IgnorePatterns) {
continue
}
projectName := getProjectName(rule.ProjectId)
msg := buildNewErrorMessage(details, projectName)
// Record before dispatch: a persistently failing dispatch retries once
// per cooldown window, never on every ingest event.
dedup.record(dedupKey)
dispatch(rule, msg)
}
}
func evaluateErrorRegression(ctx context.Context, rule *models.NotificationRuleWithChannel, event hooks.ReportEvent) {
for hash := range countOccurrences(event.ExceptionHashes) {
dedupKey := errorDedupKey(rule.Id, hash)
if dedup.isDuplicate(dedupKey, time.Duration(rule.CooldownMinutes)*time.Minute) {
continue
}
var count int64
err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT COUNT(*) FROM archived_exceptions WHERE project_id = ? AND exception_hash = ?",
event.ProjectId.String(), hash).Scan(&count)
if err != nil {
traceway.CaptureException(fmt.Errorf("error_regression check failed: %w", err))
continue
}
if count == 0 {
continue
}
details := getExceptionDetails(ctx, event.ProjectId, hash)
projectName := getProjectName(rule.ProjectId)
msg := buildErrorRegressionMessage(details, projectName)
// Record before dispatch: a persistently failing dispatch retries once
// per cooldown window, never on every ingest event.
dedup.record(dedupKey)
dispatch(rule, msg)
}
}
func getExceptionDetails(ctx context.Context, projectId uuid.UUID, hash string) ExceptionDetails {
var idStr, stackTrace, traceId, spanId, traceType, appVersion, serverName, attributesJSON, recordedAtStr string
err := db.TelemetryDB.QueryRowContext(ctx,
"SELECT id, trace_id, span_id, trace_type, stack_trace, attributes, app_version, server_name, recorded_at FROM exceptions_v2 WHERE project_id = ? AND exception_hash = ? ORDER BY recorded_at DESC LIMIT 1",
projectId.String(), hash).Scan(&idStr, &traceId, &spanId, &traceType, &stackTrace, &attributesJSON, &appVersion, &serverName, &recordedAtStr)
details := ExceptionDetails{
Hash: hash,
}
if err != nil {
if err == sql.ErrNoRows {
details.ErrorType = "Unknown Error"
return details
}
details.ErrorType = "Unknown Error"
return details
}
details.Id = idStr
details.StackTrace = stackTrace
details.AppVersion = appVersion
details.ServerName = serverName
details.RecordedAt, _ = time.Parse(time.RFC3339Nano, recordedAtStr)
exceptionId, _ := uuid.Parse(idStr)
details.TraceType, details.TraceName = resolveExceptionOwner(ctx, models.ExceptionStackTrace{
Id: exceptionId, ProjectId: projectId, TraceId: traceId, SpanId: spanId, RecordedAt: details.RecordedAt,
})
if attributesJSON != "" && attributesJSON != "{}" {
attrs := make(map[string]string)
if jsonErr := json.Unmarshal([]byte(attributesJSON), &attrs); jsonErr == nil {
details.Attributes = attrs
}
}
details.ErrorType = extractErrorType(stackTrace)
return details
}