The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
Reapply "better metrics for the backend"
dduxb04a40fb0a64
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.goindex b255a112..8ff913d6 100644--- a/backend/app/controllers/clientcontrollers/client.controller.go+++ b/backend/app/controllers/clientcontrollers/client.controller.go@@ -16,6 +16,7 @@ import ("github.com/tracewayapp/traceway/backend/app/middleware""github.com/tracewayapp/traceway/backend/app/models""github.com/tracewayapp/traceway/backend/app/models/clientmodels"+ "github.com/tracewayapp/traceway/backend/app/monitoring""github.com/tracewayapp/traceway/backend/app/repositories""github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"@@ -43,6 +44,7 @@ func (e clientController) Report(c *gin.Context) {if project, exists := c.Get(middleware.ProjectContextKey); exists {if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {if !hooks.CanReport(*p.OrganizationId) {+ monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -58,6 +60,8 @@ func (e clientController) Report(c *gin.Context) {}parseSpan.End()+ convertStart := time.Now()+endpointsToInsert := []models.Endpoint{}tasksToInsert := []models.Task{}exceptionStackTraceToInsert := []models.ExceptionStackTrace{}@@ -160,6 +164,9 @@ func (e clientController) Report(c *gin.Context) {}convertSpan.End()+ convertMs := float64(time.Since(convertStart).Microseconds()) / 1000.0+ insertStart := time.Now()+if len(endpointsToInsert) > 0 {insertSpan := traceway.StartSpan(c, "report.insert.endpoints")err := repositories.EndpointRepository.InsertAsync(c, endpointsToInsert)@@ -211,6 +218,9 @@ func (e clientController) Report(c *gin.Context) {return}+ insertMs := float64(time.Since(insertStart).Microseconds()) / 1000.0+ totalSize := len(endpointsToInsert) + len(tasksToInsert) + len(spansToInsert) + len(exceptionStackTraceToInsert) + len(metricPointsToInsert)+var exceptionHashes []stringfor _, est := range exceptionStackTraceToInsert {exceptionHashes = append(exceptionHashes, est.ExceptionHash)diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.goindex 7c42a80c..c76e88c8 100644--- a/backend/app/controllers/otelcontrollers/otel.controller.go+++ b/backend/app/controllers/otelcontrollers/otel.controller.go@@ -4,17 +4,23 @@ import ("context""fmt""net/http"+ "time""github.com/gin-gonic/gin""github.com/tracewayapp/traceway/backend/app/hooks""github.com/tracewayapp/traceway/backend/app/middleware""github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/monitoring""github.com/tracewayapp/traceway/backend/app/repositories""github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"traceway "go.tracewayapp.com")+func msSince(t time.Time) float64 {+ return float64(time.Since(t).Microseconds()) / 1000.0+}+type otelController struct{}var OtelController = otelController{}@@ -31,6 +37,7 @@ func (o otelController) ExportTraces(c *gin.Context) {attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))}if !hooks.CanReport(*p.OrganizationId) {+ monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -42,7 +49,11 @@ func (o otelController) ExportTraces(c *gin.Context) {return}+ convertStart := time.Now()endpoints, tasks, spans, exceptions, aiTraces, aiConversations := convertTraces(projectId, req)+ convertMs := msSince(convertStart)++ insertStart := time.Now()if len(endpoints) > 0 {if err := repositories.EndpointRepository.InsertAsync(c, endpoints); err != nil {@@ -86,6 +97,10 @@ func (o otelController) ExportTraces(c *gin.Context) {}}+ insertMs := msSince(insertStart)+ totalSize := len(endpoints) + len(tasks) + len(spans) + len(exceptions) + len(aiTraces)+ monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize)+var exceptionHashes []stringfor _, ex := range exceptions {exceptionHashes = append(exceptionHashes, ex.ExceptionHash)@@ -120,6 +135,7 @@ func (o otelController) ExportMetrics(c *gin.Context) {attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))}if !hooks.CanReport(*p.OrganizationId) {+ monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -132,18 +148,26 @@ func (o otelController) ExportMetrics(c *gin.Context) {return}+ convertStart := time.Now()result := convertMetricPoints(projectId, req)+ convertMs := msSince(convertStart)++ insertMs := 0.0if len(result.Points) > 0 {+ insertStart := time.Now()if err := repositories.MetricPointRepository.InsertAsync(c, result.Points); err != nil {c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL metric points: %w", err))return}+ insertMs = msSince(insertStart)if len(result.Entries) > 0 {go services.AutoRegisterMetricsWithUnits(projectId, result.Entries)}}+ monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points))+writeMetricsResponse(c)}@@ -160,6 +184,7 @@ func (o otelController) ExportLogs(c *gin.Context) {attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))}if !hooks.CanReport(*p.OrganizationId) {+ monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -172,13 +197,21 @@ func (o otelController) ExportLogs(c *gin.Context) {return}+ convertStart := time.Now()records := convertLogs(projectId, req)+ convertMs := msSince(convertStart)++ insertMs := 0.0if len(records) > 0 {+ insertStart := time.Now()if err := repositories.LogRecordRepository.InsertAsync(c, records); err != nil {c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL log records: %w", err))return}+ insertMs = msSince(insertStart)}+ monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records))+writeLogsResponse(c)}diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.gonew file mode 100644index 00000000..8fa9e11e--- /dev/null+++ b/backend/app/monitoring/ch_reporter.go@@ -0,0 +1,133 @@+//go:build pgch++package monitoring++import (+ "context"+ "fmt"+ "time"++ traceway "go.tracewayapp.com"++ "github.com/tracewayapp/traceway/backend/app/chdb"+)++const reportInterval = 60 * time.Second++var telemetryTables = []string{+ "traces",+ "log_records",+ "metric_points",+ "spans",+ "exception_stack_traces",+ "endpoints",+ "tasks",+}++func StartClickHouseReporter(ctx context.Context) {+ go func() {+ ticker := time.NewTicker(reportInterval)+ defer ticker.Stop()++ var prevInserted uint64+ var prevFailed uint64+ first := true++ // Fire once immediately so we don't wait 60s for the first data point.+ reportOnce(ctx, &prevInserted, &prevFailed, &first)++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ reportOnce(ctx, &prevInserted, &prevFailed, &first)+ }+ }+ }()+}++func reportOnce(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {+ // Bound each tick so a stuck CH doesn't wedge the reporter forever.+ qCtx, cancel := context.WithTimeout(ctx, 10*time.Second)+ defer cancel()++ reportPartsByTable(qCtx)+ reportMergesRunning(qCtx)+ reportEventDeltas(qCtx, prevInserted, prevFailed, first)+}++func reportPartsByTable(ctx context.Context) {+ for _, table := range telemetryTables {+ var parts uint64+ var rows uint64+ err := chdb.Conn.QueryRow(+ ctx,+ "SELECT coalesce(count(), 0), coalesce(sum(rows), 0) FROM system.parts WHERE database = currentDatabase() AND table = ? AND active",+ table,+ ).Scan(&parts, &rows)+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts query for %q failed: %w", table, err))+ continue+ }+ tags := map[string]string{"table": table}+ traceway.CaptureMetricWithTags("traceway.ch.parts.active", float64(parts), tags)+ traceway.CaptureMetricWithTags("traceway.ch.parts.rows", float64(rows), tags)+ }+}++func reportMergesRunning(ctx context.Context) {+ var merges uint64+ err := chdb.Conn.QueryRow(+ ctx,+ "SELECT coalesce(count(), 0) FROM system.merges WHERE database = currentDatabase()",+ ).Scan(&merges)+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.merges query failed: %w", err))+ return+ }+ traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))+}++func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {+ curInserted, err := scanEventValue(ctx, "InsertedRows")+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))+ return+ }+ curFailed, err := scanEventValue(ctx, "FailedInsertQuery")+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))+ return+ }++ // First tick has no baseline — the "delta" would be the lifetime counter.+ if !*first {+ inserted := safeDelta(*prevInserted, curInserted)+ failed := safeDelta(*prevFailed, curFailed)+ traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(inserted))+ traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(failed))+ }+ *prevInserted = curInserted+ *prevFailed = curFailed+ *first = false+}++func scanEventValue(ctx context.Context, event string) (uint64, error) {+ var value uint64+ err := chdb.Conn.QueryRow(+ ctx,+ "SELECT coalesce(sum(value), 0) FROM system.events WHERE event = ?",+ event,+ ).Scan(&value)+ return value, err+}++// Guards against counter resets across process/CH restart producing a huge+// uint64 underflow.+func safeDelta(prev, cur uint64) uint64 {+ if cur < prev {+ return cur+ }+ return cur - prev+}diff --git a/backend/app/monitoring/ch_reporter_sqlite.go b/backend/app/monitoring/ch_reporter_sqlite.gonew file mode 100644index 00000000..dedd7814--- /dev/null+++ b/backend/app/monitoring/ch_reporter_sqlite.go@@ -0,0 +1,7 @@+//go:build !pgch++package monitoring++import "context"++func StartClickHouseReporter(_ context.Context) {}diff --git a/backend/app/monitoring/metrics.go b/backend/app/monitoring/metrics.gonew file mode 100644index 00000000..259ecca8--- /dev/null+++ b/backend/app/monitoring/metrics.go@@ -0,0 +1,30 @@+package monitoring++import (+ "strconv"++ traceway "go.tracewayapp.com"+)++const (+ SignalTraces = "traces"+ SignalMetrics = "metrics"+ SignalLogs = "logs"+ SignalNative = "native"+)++func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size int) {+ tags := map[string]string{+ "signal": signal,+ "table": table,+ }+ traceway.CaptureMetricWithTags("traceway.ingest.batch.convert_ms", convertMs, tags)+ traceway.CaptureMetricWithTags("traceway.ingest.batch.insert_ms", insertMs, tags)+ traceway.CaptureMetricWithTags("traceway.ingest.batch.size", float64(size), tags)+}++func RecordRateLimited(orgID int) {+ traceway.CaptureMetricWithTags("traceway.ingest.rate_limited", 1, map[string]string{+ "org_id": strconv.Itoa(orgID),+ })+}diff --git a/backend/cmd/run.go b/backend/cmd/run.goindex b80ab4d3..04c2a9ae 100644--- a/backend/cmd/run.go+++ b/backend/cmd/run.go@@ -9,6 +9,7 @@ import ("github.com/tracewayapp/traceway/backend/app/middleware""github.com/tracewayapp/traceway/backend/app/migrations""github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/monitoring""github.com/tracewayapp/traceway/backend/app/notifications""github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"@@ -131,6 +132,7 @@ func Run(opts ...Option) {monitoringTracewayUrl,tracewaygin.WithOnErrorRecording(tracewaygin.RecordingQuery|tracewaygin.RecordingBody|tracewaygin.RecordingHeader|tracewaygin.RecordingUrl),))+ monitoring.StartClickHouseReporter(ctx)}router.GET("/health", func(c *gin.Context) {