The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
proper recovers for non critical paths
ddux2a6169aa07d2
diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.goindex c76e88c8..2224f044 100644--- a/backend/app/controllers/otelcontrollers/otel.controller.go+++ b/backend/app/controllers/otelcontrollers/otel.controller.go@@ -88,6 +88,8 @@ func (o otelController) ExportTraces(c *gin.Context) {if len(aiConversations) > 0 {convs := aiConversationsgo func() {+ defer traceway.Recover()+for _, conv := range convs {if err := storage.Store.Write(context.Background(), conv.StorageKey, conv.Content); err != nil {traceway.CaptureException(fmt.Errorf("failed to write AI trace conversation (key=%s): %w", conv.StorageKey, err))diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.goindex 8fa9e11e..c0affc3e 100644--- a/backend/app/monitoring/ch_reporter.go+++ b/backend/app/monitoring/ch_reporter.go@@ -26,6 +26,8 @@ var telemetryTables = []string{func StartClickHouseReporter(ctx context.Context) {go func() {+ defer traceway.Recover()+ticker := time.NewTicker(reportInterval)defer ticker.Stop()diff --git a/backend/app/notifications/cooldown.go b/backend/app/notifications/cooldown.goindex 5686c0bf..05fd5440 100644--- a/backend/app/notifications/cooldown.go+++ b/backend/app/notifications/cooldown.go@@ -4,6 +4,8 @@ import ("context""sync""time"++ traceway "go.tracewayapp.com")type cooldownTracker struct {@@ -62,6 +64,8 @@ func (m *dedupTracker) purgeExpired(maxAge time.Duration) {func startDedupPurger(ctx context.Context) {go func() {+ defer traceway.Recover()+ticker := time.NewTicker(10 * time.Minute)defer ticker.Stop()for {diff --git a/backend/app/notifications/dispatch.go b/backend/app/notifications/dispatch.goindex 4e1f904c..440cd296 100644--- a/backend/app/notifications/dispatch.go+++ b/backend/app/notifications/dispatch.go@@ -63,6 +63,8 @@ func dispatch(rule *models.NotificationRuleWithChannel, msg Message) {func recordFiredNotification(rule *models.NotificationRuleWithChannel, msg Message, status string, errorMsg string) {go func() {+ defer traceway.Recover()+err := repositories.FiredNotificationRepository.Insert(context.Background(), repositories.FiredNotification{ProjectId: rule.ProjectId,RuleId: rule.Id,diff --git a/backend/app/notifications/evaluator.go b/backend/app/notifications/evaluator.goindex bf636b86..cb861f12 100644--- a/backend/app/notifications/evaluator.go+++ b/backend/app/notifications/evaluator.go@@ -21,6 +21,8 @@ func StartEvaluator(ctx context.Context) {}func startPolledLoop(ctx context.Context) {+ defer traceway.Recover()+ticker := time.NewTicker(60 * time.Second)defer ticker.Stop()diff --git a/backend/app/notifications/evaluator_helpers.go b/backend/app/notifications/evaluator_helpers.goindex 116c928a..4443ac06 100644--- a/backend/app/notifications/evaluator_helpers.go+++ b/backend/app/notifications/evaluator_helpers.go@@ -30,6 +30,8 @@ func registerReportHook() {}func evaluateEventRules(event hooks.ReportEvent) {+ defer traceway.Recover()+rules, err := db.ExecuteTransaction(func(tx *sql.Tx) ([]*models.NotificationRuleWithChannel, error) {return repositories.NotificationRuleRepository.FindEnabledEventRules(tx, event.ProjectId)})diff --git a/backend/app/retention/recordings.go b/backend/app/retention/recordings.goindex 62ff92a9..fafd6d28 100644--- a/backend/app/retention/recordings.go+++ b/backend/app/retention/recordings.go@@ -50,6 +50,8 @@ func startRecordingDiskCleanup(ctx context.Context, days int) {config.Logf("Starting session recording disk cleanup worker (TTL: %d days, dir: %s)", days, recordingsDir)go func() {+ defer traceway.Recover()+runRecordingDiskCleanup(recordingsDir, days)ticker := time.NewTicker(tickInterval)diff --git a/backend/app/retention/sqlite.go b/backend/app/retention/sqlite.goindex 0e9f43bf..ba8f1a44 100644--- a/backend/app/retention/sqlite.go+++ b/backend/app/retention/sqlite.go@@ -46,6 +46,8 @@ func startSQLiteRetention(ctx context.Context, days int) {config.Logf("Starting SQLite retention worker (TTL: %d days, interval: %s)", days, tickInterval)go func() {+ defer traceway.Recover()+runSQLiteRetention(ctx, days)ticker := time.NewTicker(tickInterval)diff --git a/backend/cmd/run.go b/backend/cmd/run.goindex 02572630..c2061107 100644--- a/backend/cmd/run.go+++ b/backend/cmd/run.go@@ -25,6 +25,7 @@ import ("github.com/coreos/go-systemd/v22/daemon""github.com/gin-gonic/gin""github.com/joho/godotenv"+ traceway "go.tracewayapp.com"tracewaygin "go.tracewayapp.com/tracewaygin")@@ -187,6 +188,8 @@ func Run(opts ...Option) {continue}go func() {+ defer traceway.Recover()+port := ":" + portsList[i]config.Logln("Starting server on " + port)if err := router.Run(port); err != nil {@@ -211,6 +214,8 @@ func notifySystemd() {}go func() {+ defer traceway.Recover()+ticker := time.NewTicker(15 * time.Second)defer ticker.Stop()for range ticker.C {