The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
Revert "better metrics for the backend"
ddux0323e3e53798
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.goindex 7ff2c146..0cd27bd2 100644--- a/backend/app/controllers/clientcontrollers/client.controller.go+++ b/backend/app/controllers/clientcontrollers/client.controller.go@@ -16,7 +16,6 @@ import ("github.com/tracewayapp/traceway/backend/app/middleware""github.com/tracewayapp/traceway/backend/app/models""github.com/tracewayapp/traceway/backend/app/models/clientmodels"- "github.com/tracewayapp/traceway/backend/app/monitoring""github.com/tracewayapp/traceway/backend/app/repositories""github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"@@ -44,7 +43,6 @@ func (e clientController) Report(c *gin.Context) {if project, exists := c.Get(middleware.ProjectContextKey); exists {if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {if !hooks.CanReport(*p.OrganizationId) {- monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -57,8 +55,6 @@ func (e clientController) Report(c *gin.Context) {return}- convertStart := time.Now()-endpointsToInsert := []models.Endpoint{}tasksToInsert := []models.Task{}exceptionStackTraceToInsert := []models.ExceptionStackTrace{}@@ -155,9 +151,6 @@ func (e clientController) Report(c *gin.Context) {}}- convertMs := float64(time.Since(convertStart).Microseconds()) / 1000.0- insertStart := time.Now()-if len(endpointsToInsert) > 0 {err := repositories.EndpointRepository.InsertAsync(c, endpointsToInsert)if err != nil {@@ -198,9 +191,6 @@ func (e clientController) Report(c *gin.Context) {return}- insertMs := float64(time.Since(insertStart).Microseconds()) / 1000.0- totalSize := len(endpointsToInsert) + len(tasksToInsert) + len(spansToInsert) + len(exceptionStackTraceToInsert) + len(metricPointsToInsert)-var exceptionHashes []stringfor _, est := range exceptionStackTraceToInsert {exceptionHashes = append(exceptionHashes, est.ExceptionHash)diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.goindex c76e88c8..7c42a80c 100644--- a/backend/app/controllers/otelcontrollers/otel.controller.go+++ b/backend/app/controllers/otelcontrollers/otel.controller.go@@ -4,23 +4,17 @@ import ("context""fmt""net/http"- "time""github.com/gin-gonic/gin""github.com/tracewayapp/traceway/backend/app/hooks""github.com/tracewayapp/traceway/backend/app/middleware""github.com/tracewayapp/traceway/backend/app/models"- "github.com/tracewayapp/traceway/backend/app/monitoring""github.com/tracewayapp/traceway/backend/app/repositories""github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"traceway "go.tracewayapp.com")-func msSince(t time.Time) float64 {- return float64(time.Since(t).Microseconds()) / 1000.0-}-type otelController struct{}var OtelController = otelController{}@@ -37,7 +31,6 @@ func (o otelController) ExportTraces(c *gin.Context) {attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))}if !hooks.CanReport(*p.OrganizationId) {- monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -49,11 +42,7 @@ func (o otelController) ExportTraces(c *gin.Context) {return}- convertStart := time.Now()endpoints, tasks, spans, exceptions, aiTraces, aiConversations := convertTraces(projectId, req)- convertMs := msSince(convertStart)-- insertStart := time.Now()if len(endpoints) > 0 {if err := repositories.EndpointRepository.InsertAsync(c, endpoints); err != nil {@@ -97,10 +86,6 @@ func (o otelController) ExportTraces(c *gin.Context) {}}- insertMs := msSince(insertStart)- totalSize := len(endpoints) + len(tasks) + len(spans) + len(exceptions) + len(aiTraces)- monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize)-var exceptionHashes []stringfor _, ex := range exceptions {exceptionHashes = append(exceptionHashes, ex.ExceptionHash)@@ -135,7 +120,6 @@ func (o otelController) ExportMetrics(c *gin.Context) {attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))}if !hooks.CanReport(*p.OrganizationId) {- monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -148,26 +132,18 @@ func (o otelController) ExportMetrics(c *gin.Context) {return}- convertStart := time.Now()result := convertMetricPoints(projectId, req)- convertMs := msSince(convertStart)-- insertMs := 0.0if len(result.Points) > 0 {- insertStart := time.Now()if err := repositories.MetricPointRepository.InsertAsync(c, result.Points); err != nil {c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL metric points: %w", err))return}- insertMs = msSince(insertStart)if len(result.Entries) > 0 {go services.AutoRegisterMetricsWithUnits(projectId, result.Entries)}}- monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points))-writeMetricsResponse(c)}@@ -184,7 +160,6 @@ func (o otelController) ExportLogs(c *gin.Context) {attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))}if !hooks.CanReport(*p.OrganizationId) {- monitoring.RecordRateLimited(*p.OrganizationId)c.AbortWithStatus(http.StatusTooManyRequests)return}@@ -197,21 +172,13 @@ func (o otelController) ExportLogs(c *gin.Context) {return}- convertStart := time.Now()records := convertLogs(projectId, req)- convertMs := msSince(convertStart)-- insertMs := 0.0if len(records) > 0 {- insertStart := time.Now()if err := repositories.LogRecordRepository.InsertAsync(c, records); err != nil {c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL log records: %w", err))return}- insertMs = msSince(insertStart)}- monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records))-writeLogsResponse(c)}diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.godeleted file mode 100644index 8fa9e11e..00000000--- a/backend/app/monitoring/ch_reporter.go+++ /dev/null@@ -1,133 +0,0 @@-//go:build pgch--package monitoring--import (- "context"- "fmt"- "time"-- traceway "go.tracewayapp.com"-- "github.com/tracewayapp/traceway/backend/app/chdb"-)--const reportInterval = 60 * time.Second--var telemetryTables = []string{- "traces",- "log_records",- "metric_points",- "spans",- "exception_stack_traces",- "endpoints",- "tasks",-}--func StartClickHouseReporter(ctx context.Context) {- go func() {- ticker := time.NewTicker(reportInterval)- defer ticker.Stop()-- var prevInserted uint64- var prevFailed uint64- first := true-- // Fire once immediately so we don't wait 60s for the first data point.- reportOnce(ctx, &prevInserted, &prevFailed, &first)-- for {- select {- case <-ctx.Done():- return- case <-ticker.C:- reportOnce(ctx, &prevInserted, &prevFailed, &first)- }- }- }()-}--func reportOnce(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {- // Bound each tick so a stuck CH doesn't wedge the reporter forever.- qCtx, cancel := context.WithTimeout(ctx, 10*time.Second)- defer cancel()-- reportPartsByTable(qCtx)- reportMergesRunning(qCtx)- reportEventDeltas(qCtx, prevInserted, prevFailed, first)-}--func reportPartsByTable(ctx context.Context) {- for _, table := range telemetryTables {- var parts uint64- var rows uint64- err := chdb.Conn.QueryRow(- ctx,- "SELECT coalesce(count(), 0), coalesce(sum(rows), 0) FROM system.parts WHERE database = currentDatabase() AND table = ? AND active",- table,- ).Scan(&parts, &rows)- if err != nil {- traceway.CaptureException(fmt.Errorf("monitoring: system.parts query for %q failed: %w", table, err))- continue- }- tags := map[string]string{"table": table}- traceway.CaptureMetricWithTags("traceway.ch.parts.active", float64(parts), tags)- traceway.CaptureMetricWithTags("traceway.ch.parts.rows", float64(rows), tags)- }-}--func reportMergesRunning(ctx context.Context) {- var merges uint64- err := chdb.Conn.QueryRow(- ctx,- "SELECT coalesce(count(), 0) FROM system.merges WHERE database = currentDatabase()",- ).Scan(&merges)- if err != nil {- traceway.CaptureException(fmt.Errorf("monitoring: system.merges query failed: %w", err))- return- }- traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))-}--func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {- curInserted, err := scanEventValue(ctx, "InsertedRows")- if err != nil {- traceway.CaptureException(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))- return- }- curFailed, err := scanEventValue(ctx, "FailedInsertQuery")- if err != nil {- traceway.CaptureException(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))- return- }-- // First tick has no baseline — the "delta" would be the lifetime counter.- if !*first {- inserted := safeDelta(*prevInserted, curInserted)- failed := safeDelta(*prevFailed, curFailed)- traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(inserted))- traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(failed))- }- *prevInserted = curInserted- *prevFailed = curFailed- *first = false-}--func scanEventValue(ctx context.Context, event string) (uint64, error) {- var value uint64- err := chdb.Conn.QueryRow(- ctx,- "SELECT coalesce(sum(value), 0) FROM system.events WHERE event = ?",- event,- ).Scan(&value)- return value, err-}--// Guards against counter resets across process/CH restart producing a huge-// uint64 underflow.-func safeDelta(prev, cur uint64) uint64 {- if cur < prev {- return cur- }- return cur - prev-}diff --git a/backend/app/monitoring/ch_reporter_sqlite.go b/backend/app/monitoring/ch_reporter_sqlite.godeleted file mode 100644index dedd7814..00000000--- a/backend/app/monitoring/ch_reporter_sqlite.go+++ /dev/null@@ -1,7 +0,0 @@-//go:build !pgch--package monitoring--import "context"--func StartClickHouseReporter(_ context.Context) {}diff --git a/backend/app/monitoring/metrics.go b/backend/app/monitoring/metrics.godeleted file mode 100644index 259ecca8..00000000--- a/backend/app/monitoring/metrics.go+++ /dev/null@@ -1,30 +0,0 @@-package monitoring--import (- "strconv"-- traceway "go.tracewayapp.com"-)--const (- SignalTraces = "traces"- SignalMetrics = "metrics"- SignalLogs = "logs"- SignalNative = "native"-)--func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size int) {- tags := map[string]string{- "signal": signal,- "table": table,- }- traceway.CaptureMetricWithTags("traceway.ingest.batch.convert_ms", convertMs, tags)- traceway.CaptureMetricWithTags("traceway.ingest.batch.insert_ms", insertMs, tags)- traceway.CaptureMetricWithTags("traceway.ingest.batch.size", float64(size), tags)-}--func RecordRateLimited(orgID int) {- traceway.CaptureMetricWithTags("traceway.ingest.rate_limited", 1, map[string]string{- "org_id": strconv.Itoa(orgID),- })-}diff --git a/backend/cmd/run.go b/backend/cmd/run.goindex 04c2a9ae..b80ab4d3 100644--- a/backend/cmd/run.go+++ b/backend/cmd/run.go@@ -9,7 +9,6 @@ import ("github.com/tracewayapp/traceway/backend/app/middleware""github.com/tracewayapp/traceway/backend/app/migrations""github.com/tracewayapp/traceway/backend/app/models"- "github.com/tracewayapp/traceway/backend/app/monitoring""github.com/tracewayapp/traceway/backend/app/notifications""github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"@@ -132,7 +131,6 @@ func Run(opts ...Option) {monitoringTracewayUrl,tracewaygin.WithOnErrorRecording(tracewaygin.RecordingQuery|tracewaygin.RecordingBody|tracewaygin.RecordingHeader|tracewaygin.RecordingUrl),))- monitoring.StartClickHouseReporter(ctx)}router.GET("/health", func(c *gin.Context) {