Skip to content

The only tool you need to know what is happening and how to fix it.

Reapply "better metrics for the backend"

dduxb04a40fb0a64
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.go
index b255a112..8ff913d6 100644
--- a/backend/app/controllers/clientcontrollers/client.controller.go
+++ b/backend/app/controllers/clientcontrollers/client.controller.go
@@ -16,6 +16,7 @@ import (
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/models/clientmodels"
+ "github.com/tracewayapp/traceway/backend/app/monitoring"
"github.com/tracewayapp/traceway/backend/app/repositories"
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
@@ -43,6 +44,7 @@ func (e clientController) Report(c *gin.Context) {
if project, exists := c.Get(middleware.ProjectContextKey); exists {
if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {
if !hooks.CanReport(*p.OrganizationId) {
+ monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -58,6 +60,8 @@ func (e clientController) Report(c *gin.Context) {
}
parseSpan.End()
+ convertStart := time.Now()
+
endpointsToInsert := []models.Endpoint{}
tasksToInsert := []models.Task{}
exceptionStackTraceToInsert := []models.ExceptionStackTrace{}
@@ -160,6 +164,9 @@ func (e clientController) Report(c *gin.Context) {
}
convertSpan.End()
+ convertMs := float64(time.Since(convertStart).Microseconds()) / 1000.0
+ insertStart := time.Now()
+
if len(endpointsToInsert) > 0 {
insertSpan := traceway.StartSpan(c, "report.insert.endpoints")
err := repositories.EndpointRepository.InsertAsync(c, endpointsToInsert)
@@ -211,6 +218,9 @@ func (e clientController) Report(c *gin.Context) {
return
}
+ insertMs := float64(time.Since(insertStart).Microseconds()) / 1000.0
+ totalSize := len(endpointsToInsert) + len(tasksToInsert) + len(spansToInsert) + len(exceptionStackTraceToInsert) + len(metricPointsToInsert)
+
var exceptionHashes []string
for _, est := range exceptionStackTraceToInsert {
exceptionHashes = append(exceptionHashes, est.ExceptionHash)
diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.go
index 7c42a80c..c76e88c8 100644
--- a/backend/app/controllers/otelcontrollers/otel.controller.go
+++ b/backend/app/controllers/otelcontrollers/otel.controller.go
@@ -4,17 +4,23 @@ import (
"context"
"fmt"
"net/http"
+ "time"
"github.com/gin-gonic/gin"
"github.com/tracewayapp/traceway/backend/app/hooks"
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
+ "github.com/tracewayapp/traceway/backend/app/monitoring"
"github.com/tracewayapp/traceway/backend/app/repositories"
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
traceway "go.tracewayapp.com"
)
+func msSince(t time.Time) float64 {
+ return float64(time.Since(t).Microseconds()) / 1000.0
+}
+
type otelController struct{}
var OtelController = otelController{}
@@ -31,6 +37,7 @@ func (o otelController) ExportTraces(c *gin.Context) {
attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))
}
if !hooks.CanReport(*p.OrganizationId) {
+ monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -42,7 +49,11 @@ func (o otelController) ExportTraces(c *gin.Context) {
return
}
+ convertStart := time.Now()
endpoints, tasks, spans, exceptions, aiTraces, aiConversations := convertTraces(projectId, req)
+ convertMs := msSince(convertStart)
+
+ insertStart := time.Now()
if len(endpoints) > 0 {
if err := repositories.EndpointRepository.InsertAsync(c, endpoints); err != nil {
@@ -86,6 +97,10 @@ func (o otelController) ExportTraces(c *gin.Context) {
}
}
+ insertMs := msSince(insertStart)
+ totalSize := len(endpoints) + len(tasks) + len(spans) + len(exceptions) + len(aiTraces)
+ monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize)
+
var exceptionHashes []string
for _, ex := range exceptions {
exceptionHashes = append(exceptionHashes, ex.ExceptionHash)
@@ -120,6 +135,7 @@ func (o otelController) ExportMetrics(c *gin.Context) {
attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))
}
if !hooks.CanReport(*p.OrganizationId) {
+ monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -132,18 +148,26 @@ func (o otelController) ExportMetrics(c *gin.Context) {
return
}
+ convertStart := time.Now()
result := convertMetricPoints(projectId, req)
+ convertMs := msSince(convertStart)
+
+ insertMs := 0.0
if len(result.Points) > 0 {
+ insertStart := time.Now()
if err := repositories.MetricPointRepository.InsertAsync(c, result.Points); err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL metric points: %w", err))
return
}
+ insertMs = msSince(insertStart)
if len(result.Entries) > 0 {
go services.AutoRegisterMetricsWithUnits(projectId, result.Entries)
}
}
+ monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points))
+
writeMetricsResponse(c)
}
@@ -160,6 +184,7 @@ func (o otelController) ExportLogs(c *gin.Context) {
attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))
}
if !hooks.CanReport(*p.OrganizationId) {
+ monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -172,13 +197,21 @@ func (o otelController) ExportLogs(c *gin.Context) {
return
}
+ convertStart := time.Now()
records := convertLogs(projectId, req)
+ convertMs := msSince(convertStart)
+
+ insertMs := 0.0
if len(records) > 0 {
+ insertStart := time.Now()
if err := repositories.LogRecordRepository.InsertAsync(c, records); err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL log records: %w", err))
return
}
+ insertMs = msSince(insertStart)
}
+ monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records))
+
writeLogsResponse(c)
}
diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.go
new file mode 100644
index 00000000..8fa9e11e
--- /dev/null
+++ b/backend/app/monitoring/ch_reporter.go
@@ -0,0 +1,133 @@
+//go:build pgch
+
+package monitoring
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ traceway "go.tracewayapp.com"
+
+ "github.com/tracewayapp/traceway/backend/app/chdb"
+)
+
+const reportInterval = 60 * time.Second
+
+var telemetryTables = []string{
+ "traces",
+ "log_records",
+ "metric_points",
+ "spans",
+ "exception_stack_traces",
+ "endpoints",
+ "tasks",
+}
+
+func StartClickHouseReporter(ctx context.Context) {
+ go func() {
+ ticker := time.NewTicker(reportInterval)
+ defer ticker.Stop()
+
+ var prevInserted uint64
+ var prevFailed uint64
+ first := true
+
+ // Fire once immediately so we don't wait 60s for the first data point.
+ reportOnce(ctx, &prevInserted, &prevFailed, &first)
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ reportOnce(ctx, &prevInserted, &prevFailed, &first)
+ }
+ }
+ }()
+}
+
+func reportOnce(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {
+ // Bound each tick so a stuck CH doesn't wedge the reporter forever.
+ qCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+ defer cancel()
+
+ reportPartsByTable(qCtx)
+ reportMergesRunning(qCtx)
+ reportEventDeltas(qCtx, prevInserted, prevFailed, first)
+}
+
+func reportPartsByTable(ctx context.Context) {
+ for _, table := range telemetryTables {
+ var parts uint64
+ var rows uint64
+ err := chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(count(), 0), coalesce(sum(rows), 0) FROM system.parts WHERE database = currentDatabase() AND table = ? AND active",
+ table,
+ ).Scan(&parts, &rows)
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts query for %q failed: %w", table, err))
+ continue
+ }
+ tags := map[string]string{"table": table}
+ traceway.CaptureMetricWithTags("traceway.ch.parts.active", float64(parts), tags)
+ traceway.CaptureMetricWithTags("traceway.ch.parts.rows", float64(rows), tags)
+ }
+}
+
+func reportMergesRunning(ctx context.Context) {
+ var merges uint64
+ err := chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(count(), 0) FROM system.merges WHERE database = currentDatabase()",
+ ).Scan(&merges)
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.merges query failed: %w", err))
+ return
+ }
+ traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))
+}
+
+func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {
+ curInserted, err := scanEventValue(ctx, "InsertedRows")
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))
+ return
+ }
+ curFailed, err := scanEventValue(ctx, "FailedInsertQuery")
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))
+ return
+ }
+
+ // First tick has no baseline — the "delta" would be the lifetime counter.
+ if !*first {
+ inserted := safeDelta(*prevInserted, curInserted)
+ failed := safeDelta(*prevFailed, curFailed)
+ traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(inserted))
+ traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(failed))
+ }
+ *prevInserted = curInserted
+ *prevFailed = curFailed
+ *first = false
+}
+
+func scanEventValue(ctx context.Context, event string) (uint64, error) {
+ var value uint64
+ err := chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(sum(value), 0) FROM system.events WHERE event = ?",
+ event,
+ ).Scan(&value)
+ return value, err
+}
+
+// Guards against counter resets across process/CH restart producing a huge
+// uint64 underflow.
+func safeDelta(prev, cur uint64) uint64 {
+ if cur < prev {
+ return cur
+ }
+ return cur - prev
+}
diff --git a/backend/app/monitoring/ch_reporter_sqlite.go b/backend/app/monitoring/ch_reporter_sqlite.go
new file mode 100644
index 00000000..dedd7814
--- /dev/null
+++ b/backend/app/monitoring/ch_reporter_sqlite.go
@@ -0,0 +1,7 @@
+//go:build !pgch
+
+package monitoring
+
+import "context"
+
+func StartClickHouseReporter(_ context.Context) {}
diff --git a/backend/app/monitoring/metrics.go b/backend/app/monitoring/metrics.go
new file mode 100644
index 00000000..259ecca8
--- /dev/null
+++ b/backend/app/monitoring/metrics.go
@@ -0,0 +1,30 @@
+package monitoring
+
+import (
+ "strconv"
+
+ traceway "go.tracewayapp.com"
+)
+
+const (
+ SignalTraces = "traces"
+ SignalMetrics = "metrics"
+ SignalLogs = "logs"
+ SignalNative = "native"
+)
+
+func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size int) {
+ tags := map[string]string{
+ "signal": signal,
+ "table": table,
+ }
+ traceway.CaptureMetricWithTags("traceway.ingest.batch.convert_ms", convertMs, tags)
+ traceway.CaptureMetricWithTags("traceway.ingest.batch.insert_ms", insertMs, tags)
+ traceway.CaptureMetricWithTags("traceway.ingest.batch.size", float64(size), tags)
+}
+
+func RecordRateLimited(orgID int) {
+ traceway.CaptureMetricWithTags("traceway.ingest.rate_limited", 1, map[string]string{
+ "org_id": strconv.Itoa(orgID),
+ })
+}
diff --git a/backend/cmd/run.go b/backend/cmd/run.go
index b80ab4d3..04c2a9ae 100644
--- a/backend/cmd/run.go
+++ b/backend/cmd/run.go
@@ -9,6 +9,7 @@ import (
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/migrations"
"github.com/tracewayapp/traceway/backend/app/models"
+ "github.com/tracewayapp/traceway/backend/app/monitoring"
"github.com/tracewayapp/traceway/backend/app/notifications"
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
@@ -131,6 +132,7 @@ func Run(opts ...Option) {
monitoringTracewayUrl,
tracewaygin.WithOnErrorRecording(tracewaygin.RecordingQuery|tracewaygin.RecordingBody|tracewaygin.RecordingHeader|tracewaygin.RecordingUrl),
))
+ monitoring.StartClickHouseReporter(ctx)
}
router.GET("/health", func(c *gin.Context) {