Skip to content

The only tool you need to know what is happening and how to fix it.

Revert "better metrics for the backend"

ddux0323e3e53798
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.go
index 7ff2c146..0cd27bd2 100644
--- a/backend/app/controllers/clientcontrollers/client.controller.go
+++ b/backend/app/controllers/clientcontrollers/client.controller.go
@@ -16,7 +16,6 @@ import (
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/models/clientmodels"
- "github.com/tracewayapp/traceway/backend/app/monitoring"
"github.com/tracewayapp/traceway/backend/app/repositories"
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
@@ -44,7 +43,6 @@ func (e clientController) Report(c *gin.Context) {
if project, exists := c.Get(middleware.ProjectContextKey); exists {
if p, ok := project.(*models.Project); ok && p.OrganizationId != nil {
if !hooks.CanReport(*p.OrganizationId) {
- monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -57,8 +55,6 @@ func (e clientController) Report(c *gin.Context) {
return
}
- convertStart := time.Now()
-
endpointsToInsert := []models.Endpoint{}
tasksToInsert := []models.Task{}
exceptionStackTraceToInsert := []models.ExceptionStackTrace{}
@@ -155,9 +151,6 @@ func (e clientController) Report(c *gin.Context) {
}
}
- convertMs := float64(time.Since(convertStart).Microseconds()) / 1000.0
- insertStart := time.Now()
-
if len(endpointsToInsert) > 0 {
err := repositories.EndpointRepository.InsertAsync(c, endpointsToInsert)
if err != nil {
@@ -198,9 +191,6 @@ func (e clientController) Report(c *gin.Context) {
return
}
- insertMs := float64(time.Since(insertStart).Microseconds()) / 1000.0
- totalSize := len(endpointsToInsert) + len(tasksToInsert) + len(spansToInsert) + len(exceptionStackTraceToInsert) + len(metricPointsToInsert)
-
var exceptionHashes []string
for _, est := range exceptionStackTraceToInsert {
exceptionHashes = append(exceptionHashes, est.ExceptionHash)
diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.go
index c76e88c8..7c42a80c 100644
--- a/backend/app/controllers/otelcontrollers/otel.controller.go
+++ b/backend/app/controllers/otelcontrollers/otel.controller.go
@@ -4,23 +4,17 @@ import (
"context"
"fmt"
"net/http"
- "time"
"github.com/gin-gonic/gin"
"github.com/tracewayapp/traceway/backend/app/hooks"
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/models"
- "github.com/tracewayapp/traceway/backend/app/monitoring"
"github.com/tracewayapp/traceway/backend/app/repositories"
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
traceway "go.tracewayapp.com"
)
-func msSince(t time.Time) float64 {
- return float64(time.Since(t).Microseconds()) / 1000.0
-}
-
type otelController struct{}
var OtelController = otelController{}
@@ -37,7 +31,6 @@ func (o otelController) ExportTraces(c *gin.Context) {
attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))
}
if !hooks.CanReport(*p.OrganizationId) {
- monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -49,11 +42,7 @@ func (o otelController) ExportTraces(c *gin.Context) {
return
}
- convertStart := time.Now()
endpoints, tasks, spans, exceptions, aiTraces, aiConversations := convertTraces(projectId, req)
- convertMs := msSince(convertStart)
-
- insertStart := time.Now()
if len(endpoints) > 0 {
if err := repositories.EndpointRepository.InsertAsync(c, endpoints); err != nil {
@@ -97,10 +86,6 @@ func (o otelController) ExportTraces(c *gin.Context) {
}
}
- insertMs := msSince(insertStart)
- totalSize := len(endpoints) + len(tasks) + len(spans) + len(exceptions) + len(aiTraces)
- monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize)
-
var exceptionHashes []string
for _, ex := range exceptions {
exceptionHashes = append(exceptionHashes, ex.ExceptionHash)
@@ -135,7 +120,6 @@ func (o otelController) ExportMetrics(c *gin.Context) {
attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))
}
if !hooks.CanReport(*p.OrganizationId) {
- monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -148,26 +132,18 @@ func (o otelController) ExportMetrics(c *gin.Context) {
return
}
- convertStart := time.Now()
result := convertMetricPoints(projectId, req)
- convertMs := msSince(convertStart)
-
- insertMs := 0.0
if len(result.Points) > 0 {
- insertStart := time.Now()
if err := repositories.MetricPointRepository.InsertAsync(c, result.Points); err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL metric points: %w", err))
return
}
- insertMs = msSince(insertStart)
if len(result.Entries) > 0 {
go services.AutoRegisterMetricsWithUnits(projectId, result.Entries)
}
}
- monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points))
-
writeMetricsResponse(c)
}
@@ -184,7 +160,6 @@ func (o otelController) ExportLogs(c *gin.Context) {
attrs.SetTag("organization_id", fmt.Sprintf("%d", *p.OrganizationId))
}
if !hooks.CanReport(*p.OrganizationId) {
- monitoring.RecordRateLimited(*p.OrganizationId)
c.AbortWithStatus(http.StatusTooManyRequests)
return
}
@@ -197,21 +172,13 @@ func (o otelController) ExportLogs(c *gin.Context) {
return
}
- convertStart := time.Now()
records := convertLogs(projectId, req)
- convertMs := msSince(convertStart)
-
- insertMs := 0.0
if len(records) > 0 {
- insertStart := time.Now()
if err := repositories.LogRecordRepository.InsertAsync(c, records); err != nil {
c.AbortWithError(500, traceway.NewStackTraceErrorf("error inserting OTEL log records: %w", err))
return
}
- insertMs = msSince(insertStart)
}
- monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records))
-
writeLogsResponse(c)
}
diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.go
deleted file mode 100644
index 8fa9e11e..00000000
--- a/backend/app/monitoring/ch_reporter.go
+++ /dev/null
@@ -1,133 +0,0 @@
-//go:build pgch
-
-package monitoring
-
-import (
- "context"
- "fmt"
- "time"
-
- traceway "go.tracewayapp.com"
-
- "github.com/tracewayapp/traceway/backend/app/chdb"
-)
-
-const reportInterval = 60 * time.Second
-
-var telemetryTables = []string{
- "traces",
- "log_records",
- "metric_points",
- "spans",
- "exception_stack_traces",
- "endpoints",
- "tasks",
-}
-
-func StartClickHouseReporter(ctx context.Context) {
- go func() {
- ticker := time.NewTicker(reportInterval)
- defer ticker.Stop()
-
- var prevInserted uint64
- var prevFailed uint64
- first := true
-
- // Fire once immediately so we don't wait 60s for the first data point.
- reportOnce(ctx, &prevInserted, &prevFailed, &first)
-
- for {
- select {
- case <-ctx.Done():
- return
- case <-ticker.C:
- reportOnce(ctx, &prevInserted, &prevFailed, &first)
- }
- }
- }()
-}
-
-func reportOnce(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {
- // Bound each tick so a stuck CH doesn't wedge the reporter forever.
- qCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
- defer cancel()
-
- reportPartsByTable(qCtx)
- reportMergesRunning(qCtx)
- reportEventDeltas(qCtx, prevInserted, prevFailed, first)
-}
-
-func reportPartsByTable(ctx context.Context) {
- for _, table := range telemetryTables {
- var parts uint64
- var rows uint64
- err := chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(count(), 0), coalesce(sum(rows), 0) FROM system.parts WHERE database = currentDatabase() AND table = ? AND active",
- table,
- ).Scan(&parts, &rows)
- if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.parts query for %q failed: %w", table, err))
- continue
- }
- tags := map[string]string{"table": table}
- traceway.CaptureMetricWithTags("traceway.ch.parts.active", float64(parts), tags)
- traceway.CaptureMetricWithTags("traceway.ch.parts.rows", float64(rows), tags)
- }
-}
-
-func reportMergesRunning(ctx context.Context) {
- var merges uint64
- err := chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(count(), 0) FROM system.merges WHERE database = currentDatabase()",
- ).Scan(&merges)
- if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.merges query failed: %w", err))
- return
- }
- traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))
-}
-
-func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {
- curInserted, err := scanEventValue(ctx, "InsertedRows")
- if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))
- return
- }
- curFailed, err := scanEventValue(ctx, "FailedInsertQuery")
- if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))
- return
- }
-
- // First tick has no baseline — the "delta" would be the lifetime counter.
- if !*first {
- inserted := safeDelta(*prevInserted, curInserted)
- failed := safeDelta(*prevFailed, curFailed)
- traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(inserted))
- traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(failed))
- }
- *prevInserted = curInserted
- *prevFailed = curFailed
- *first = false
-}
-
-func scanEventValue(ctx context.Context, event string) (uint64, error) {
- var value uint64
- err := chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(sum(value), 0) FROM system.events WHERE event = ?",
- event,
- ).Scan(&value)
- return value, err
-}
-
-// Guards against counter resets across process/CH restart producing a huge
-// uint64 underflow.
-func safeDelta(prev, cur uint64) uint64 {
- if cur < prev {
- return cur
- }
- return cur - prev
-}
diff --git a/backend/app/monitoring/ch_reporter_sqlite.go b/backend/app/monitoring/ch_reporter_sqlite.go
deleted file mode 100644
index dedd7814..00000000
--- a/backend/app/monitoring/ch_reporter_sqlite.go
+++ /dev/null
@@ -1,7 +0,0 @@
-//go:build !pgch
-
-package monitoring
-
-import "context"
-
-func StartClickHouseReporter(_ context.Context) {}
diff --git a/backend/app/monitoring/metrics.go b/backend/app/monitoring/metrics.go
deleted file mode 100644
index 259ecca8..00000000
--- a/backend/app/monitoring/metrics.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package monitoring
-
-import (
- "strconv"
-
- traceway "go.tracewayapp.com"
-)
-
-const (
- SignalTraces = "traces"
- SignalMetrics = "metrics"
- SignalLogs = "logs"
- SignalNative = "native"
-)
-
-func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size int) {
- tags := map[string]string{
- "signal": signal,
- "table": table,
- }
- traceway.CaptureMetricWithTags("traceway.ingest.batch.convert_ms", convertMs, tags)
- traceway.CaptureMetricWithTags("traceway.ingest.batch.insert_ms", insertMs, tags)
- traceway.CaptureMetricWithTags("traceway.ingest.batch.size", float64(size), tags)
-}
-
-func RecordRateLimited(orgID int) {
- traceway.CaptureMetricWithTags("traceway.ingest.rate_limited", 1, map[string]string{
- "org_id": strconv.Itoa(orgID),
- })
-}
diff --git a/backend/cmd/run.go b/backend/cmd/run.go
index 04c2a9ae..b80ab4d3 100644
--- a/backend/cmd/run.go
+++ b/backend/cmd/run.go
@@ -9,7 +9,6 @@ import (
"github.com/tracewayapp/traceway/backend/app/middleware"
"github.com/tracewayapp/traceway/backend/app/migrations"
"github.com/tracewayapp/traceway/backend/app/models"
- "github.com/tracewayapp/traceway/backend/app/monitoring"
"github.com/tracewayapp/traceway/backend/app/notifications"
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
@@ -132,7 +131,6 @@ func Run(opts ...Option) {
monitoringTracewayUrl,
tracewaygin.WithOnErrorRecording(tracewaygin.RecordingQuery|tracewaygin.RecordingBody|tracewaygin.RecordingHeader|tracewaygin.RecordingUrl),
))
- monitoring.StartClickHouseReporter(ctx)
}
router.GET("/health", func(c *gin.Context) {