Skip to content

The only tool you need to know what is happening and how to fix it.

feat: more metrics regarding the backend

ddux65cfdbe05130
diff --git a/backend/app/cache/project_cache.go b/backend/app/cache/project_cache.go
index f24f0c9e..7b3bcedd 100644
--- a/backend/app/cache/project_cache.go
+++ b/backend/app/cache/project_cache.go
@@ -147,3 +147,9 @@ func (c *projectCache) LastRefresh() time.Time {
defer c.mu.RUnlock()
return c.lastRefresh
}
+
+func (c *projectCache) Entries() int {
+ c.mu.RLock()
+ defer c.mu.RUnlock()
+ return len(c.projectsById)
+}
diff --git a/backend/app/cache/sourcemap_cache.go b/backend/app/cache/sourcemap_cache.go
index 2ef1bd56..66d57cb7 100644
--- a/backend/app/cache/sourcemap_cache.go
+++ b/backend/app/cache/sourcemap_cache.go
@@ -11,12 +11,25 @@ type sourceMapCacheEntry struct {
}
type sourceMapCache struct {
- maxCount int
- maxBytes int64
- mu sync.Mutex
- items map[string]*list.Element
- order *list.List
- curBytes int64
+ maxCount int
+ maxBytes int64
+ mu sync.Mutex
+ items map[string]*list.Element
+ order *list.List
+ curBytes int64
+ hits uint64
+ misses uint64
+ evictions uint64
+}
+
+type SourceMapCacheStats struct {
+ Entries int
+ Bytes int64
+ MaxEntries int
+ MaxBytes int64
+ Hits uint64
+ Misses uint64
+ Evictions uint64
}
var SourceMapCache *sourceMapCache
@@ -35,9 +48,11 @@ func (c *sourceMapCache) Get(key string) ([]byte, bool) {
defer c.mu.Unlock()
if el, ok := c.items[key]; ok {
+ c.hits++
c.order.MoveToFront(el)
return el.Value.(*sourceMapCacheEntry).data, true
}
+ c.misses++
return nil, false
}
@@ -67,5 +82,21 @@ func (c *sourceMapCache) Put(key string, data []byte) {
evicted := c.order.Remove(back).(*sourceMapCacheEntry)
delete(c.items, evicted.key)
c.curBytes -= int64(len(evicted.data))
+ c.evictions++
+ }
+}
+
+func (c *sourceMapCache) Stats() SourceMapCacheStats {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+
+ return SourceMapCacheStats{
+ Entries: len(c.items),
+ Bytes: c.curBytes,
+ MaxEntries: c.maxCount,
+ MaxBytes: c.maxBytes,
+ Hits: c.hits,
+ Misses: c.misses,
+ Evictions: c.evictions,
}
}
diff --git a/backend/app/chdb/chdb.go b/backend/app/chdb/chdb.go
index 5b12bc33..debdab7a 100644
--- a/backend/app/chdb/chdb.go
+++ b/backend/app/chdb/chdb.go
@@ -19,6 +19,7 @@ type ChConn interface {
QueryRow(ctx context.Context, query string, args ...any) driver.Row
PrepareBatch(ctx context.Context, query string, opts ...driver.PrepareBatchOption) (driver.Batch, error)
Exec(ctx context.Context, query string, args ...any) error
+ Stats() driver.Stats
}
var Conn ChConn
diff --git a/backend/app/config/config.go b/backend/app/config/config.go
index fa15cbba..7f81af4c 100644
--- a/backend/app/config/config.go
+++ b/backend/app/config/config.go
@@ -33,6 +33,10 @@ type Cfg struct {
SessionRecordingUploadWorkers string
SessionRecordingUploadQueueSize string
+ SourceMapCacheMaxEntries string
+ SourceMapCacheMaxBytesMB string
+ SourceMapParsedCacheMax string
+
SMTPEnabled string
SMTPHost string
SMTPPort string
@@ -105,6 +109,10 @@ func LoadFromEnv() *Cfg {
SessionRecordingUploadWorkers: os.Getenv("SESSION_RECORDING_UPLOAD_WORKERS"),
SessionRecordingUploadQueueSize: os.Getenv("SESSION_RECORDING_UPLOAD_QUEUE_SIZE"),
+ SourceMapCacheMaxEntries: os.Getenv("SOURCEMAP_CACHE_MAX_ENTRIES"),
+ SourceMapCacheMaxBytesMB: os.Getenv("SOURCEMAP_CACHE_MAX_BYTES_MB"),
+ SourceMapParsedCacheMax: os.Getenv("SOURCEMAP_PARSED_CACHE_MAX"),
+
SMTPEnabled: os.Getenv("SMTP_ENABLED"),
SMTPHost: os.Getenv("SMTP_HOST"),
SMTPPort: os.Getenv("SMTP_PORT"),
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.go
index c8ef56a8..a05e6222 100644
--- a/backend/app/controllers/clientcontrollers/client.controller.go
+++ b/backend/app/controllers/clientcontrollers/client.controller.go
@@ -49,6 +49,9 @@ type ReportRequest struct {
}
func (e clientController) Report(c *gin.Context) {
+ monitoring.IngestStarted()
+ defer monitoring.IngestFinished()
+
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))
@@ -74,6 +77,13 @@ func (e clientController) Report(c *gin.Context) {
}
parseSpan.End()
+ bodyBytes := 0
+ if cb, ok := c.Get(gin.BodyBytesKey); ok {
+ if b, ok := cb.([]byte); ok {
+ bodyBytes = len(b)
+ }
+ }
+
convertStart := time.Now()
endpointsToInsert := []models.Endpoint{}
@@ -298,7 +308,7 @@ func (e clientController) Report(c *gin.Context) {
insertMs := float64(time.Since(insertStart).Microseconds()) / 1000.0
totalSize := len(endpointsToInsert) + len(tasksToInsert) + len(spansToInsert) + len(exceptionStackTraceToInsert) + len(metricPointsToInsert)
- monitoring.RecordIngestBatch(monitoring.SignalNative, "report", convertMs, insertMs, totalSize)
+ monitoring.RecordIngestBatch(monitoring.SignalNative, "report", convertMs, insertMs, totalSize, bodyBytes)
var exceptionHashes []string
for _, est := range exceptionStackTraceToInsert {
diff --git a/backend/app/controllers/otelcontrollers/codec.go b/backend/app/controllers/otelcontrollers/codec.go
index ac1bc3af..0f6585e2 100644
--- a/backend/app/controllers/otelcontrollers/codec.go
+++ b/backend/app/controllers/otelcontrollers/codec.go
@@ -35,40 +35,40 @@ func isProtobuf(c *gin.Context) bool {
return strings.Contains(ct, "application/x-protobuf") || strings.Contains(ct, "application/protobuf")
}
-func decodeTraceRequest(c *gin.Context) (*coltracepb.ExportTraceServiceRequest, error) {
+func decodeTraceRequest(c *gin.Context) (*coltracepb.ExportTraceServiceRequest, int, error) {
body, err := readBody(c)
if err != nil {
- return nil, err
+ return nil, 0, err
}
req := &coltracepb.ExportTraceServiceRequest{}
if isProtobuf(c) {
if err := proto.Unmarshal(body, req); err != nil {
- return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err)
+ return nil, 0, fmt.Errorf("failed to unmarshal protobuf: %w", err)
}
} else {
if err := protojson.Unmarshal(body, req); err != nil {
- return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
+ return nil, 0, fmt.Errorf("failed to unmarshal JSON: %w", err)
}
}
- return req, nil
+ return req, len(body), nil
}
-func decodeMetricsRequest(c *gin.Context) (*colmetricspb.ExportMetricsServiceRequest, error) {
+func decodeMetricsRequest(c *gin.Context) (*colmetricspb.ExportMetricsServiceRequest, int, error) {
body, err := readBody(c)
if err != nil {
- return nil, err
+ return nil, 0, err
}
req := &colmetricspb.ExportMetricsServiceRequest{}
if isProtobuf(c) {
if err := proto.Unmarshal(body, req); err != nil {
- return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err)
+ return nil, 0, fmt.Errorf("failed to unmarshal protobuf: %w", err)
}
} else {
if err := protojson.Unmarshal(body, req); err != nil {
- return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
+ return nil, 0, fmt.Errorf("failed to unmarshal JSON: %w", err)
}
}
- return req, nil
+ return req, len(body), nil
}
func writeTraceResponse(c *gin.Context) {
@@ -93,22 +93,22 @@ func writeMetricsResponse(c *gin.Context) {
}
}
-func decodeLogsRequest(c *gin.Context) (*collogspb.ExportLogsServiceRequest, error) {
+func decodeLogsRequest(c *gin.Context) (*collogspb.ExportLogsServiceRequest, int, error) {
body, err := readBody(c)
if err != nil {
- return nil, err
+ return nil, 0, err
}
req := &collogspb.ExportLogsServiceRequest{}
if isProtobuf(c) {
if err := proto.Unmarshal(body, req); err != nil {
- return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err)
+ return nil, 0, fmt.Errorf("failed to unmarshal protobuf: %w", err)
}
} else {
if err := protojson.Unmarshal(body, req); err != nil {
- return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
+ return nil, 0, fmt.Errorf("failed to unmarshal JSON: %w", err)
}
}
- return req, nil
+ return req, len(body), nil
}
func writeLogsResponse(c *gin.Context) {
diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.go
index 2224f044..5ad1318c 100644
--- a/backend/app/controllers/otelcontrollers/otel.controller.go
+++ b/backend/app/controllers/otelcontrollers/otel.controller.go
@@ -26,6 +26,9 @@ type otelController struct{}
var OtelController = otelController{}
func (o otelController) ExportTraces(c *gin.Context) {
+ monitoring.IngestStarted()
+ defer monitoring.IngestFinished()
+
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))
@@ -43,7 +46,7 @@ func (o otelController) ExportTraces(c *gin.Context) {
}
}
}
- req, err := decodeTraceRequest(c)
+ req, bodyBytes, err := decodeTraceRequest(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
@@ -101,7 +104,7 @@ func (o otelController) ExportTraces(c *gin.Context) {
insertMs := msSince(insertStart)
totalSize := len(endpoints) + len(tasks) + len(spans) + len(exceptions) + len(aiTraces)
- monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize)
+ monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize, bodyBytes)
var exceptionHashes []string
for _, ex := range exceptions {
@@ -125,6 +128,9 @@ func (o otelController) ExportTraces(c *gin.Context) {
}
func (o otelController) ExportMetrics(c *gin.Context) {
+ monitoring.IngestStarted()
+ defer monitoring.IngestFinished()
+
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))
@@ -144,7 +150,7 @@ func (o otelController) ExportMetrics(c *gin.Context) {
}
}
- req, err := decodeMetricsRequest(c)
+ req, bodyBytes, err := decodeMetricsRequest(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
@@ -168,12 +174,15 @@ func (o otelController) ExportMetrics(c *gin.Context) {
}
}
- monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points))
+ monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points), bodyBytes)
writeMetricsResponse(c)
}
func (o otelController) ExportLogs(c *gin.Context) {
+ monitoring.IngestStarted()
+ defer monitoring.IngestFinished()
+
projectId, err := middleware.GetProjectId(c)
if err != nil {
c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))
@@ -193,7 +202,7 @@ func (o otelController) ExportLogs(c *gin.Context) {
}
}
- req, err := decodeLogsRequest(c)
+ req, bodyBytes, err := decodeLogsRequest(c)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
return
@@ -213,7 +222,7 @@ func (o otelController) ExportLogs(c *gin.Context) {
insertMs = msSince(insertStart)
}
- monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records))
+ monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records), bodyBytes)
writeLogsResponse(c)
}
diff --git a/backend/app/monitoring/backend_reporter.go b/backend/app/monitoring/backend_reporter.go
new file mode 100644
index 00000000..5f07604a
--- /dev/null
+++ b/backend/app/monitoring/backend_reporter.go
@@ -0,0 +1,87 @@
+package monitoring
+
+import (
+ "context"
+ "time"
+
+ traceway "go.tracewayapp.com"
+
+ "github.com/tracewayapp/traceway/backend/app/cache"
+ "github.com/tracewayapp/traceway/backend/app/repositories"
+ "github.com/tracewayapp/traceway/backend/app/services"
+)
+
+const backendReportInterval = 30 * time.Second
+
+type backendBaselines struct {
+ rawHits uint64
+ rawMisses uint64
+ rawEvictions uint64
+ parsedHits uint64
+ parsedMisses uint64
+ first bool
+}
+
+func StartBackendReporter(ctx context.Context) {
+ go func() {
+ defer traceway.Recover()
+
+ ticker := time.NewTicker(backendReportInterval)
+ defer ticker.Stop()
+
+ baselines := &backendBaselines{first: true}
+
+ reportBackendOnce(baselines)
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ reportBackendOnce(baselines)
+ }
+ }
+ }()
+}
+
+func reportBackendOnce(b *backendBaselines) {
+ traceway.CaptureMetric("traceway.ingest.in_flight", float64(InFlightIngest()))
+
+ if rss, ok := ReadRSSBytes(); ok {
+ traceway.CaptureMetric("traceway.proc.rss_mb", float64(rss)/1024.0/1024.0)
+ }
+
+ rawStats := cache.SourceMapCache.Stats()
+ traceway.CaptureMetric("traceway.sourcemap.raw.entries", float64(rawStats.Entries))
+ traceway.CaptureMetric("traceway.sourcemap.raw.bytes", float64(rawStats.Bytes))
+
+ parsedStats := services.ParsedSourceMapStats()
+ traceway.CaptureMetric("traceway.sourcemap.parsed.entries", float64(parsedStats.Entries))
+ traceway.CaptureMetric("traceway.sourcemap.parsed.parse_ms", parsedStats.LastParseMs)
+
+ if !b.first {
+ traceway.CaptureMetric("traceway.sourcemap.raw.hits.delta", float64(safeDelta(b.rawHits, rawStats.Hits)))
+ traceway.CaptureMetric("traceway.sourcemap.raw.misses.delta", float64(safeDelta(b.rawMisses, rawStats.Misses)))
+ traceway.CaptureMetric("traceway.sourcemap.raw.evictions.delta", float64(safeDelta(b.rawEvictions, rawStats.Evictions)))
+ traceway.CaptureMetric("traceway.sourcemap.parsed.hits.delta", float64(safeDelta(b.parsedHits, parsedStats.Hits)))
+ traceway.CaptureMetric("traceway.sourcemap.parsed.misses.delta", float64(safeDelta(b.parsedMisses, parsedStats.Misses)))
+ }
+ b.rawHits = rawStats.Hits
+ b.rawMisses = rawStats.Misses
+ b.rawEvictions = rawStats.Evictions
+ b.parsedHits = parsedStats.Hits
+ b.parsedMisses = parsedStats.Misses
+ b.first = false
+
+ traceway.CaptureMetric("traceway.cache.projects.entries", float64(cache.ProjectCache.Entries()))
+ traceway.CaptureMetric("traceway.cache.metric_registry.entries", float64(repositories.MetricRegistryRepository.KnownCount()))
+}
+
+// Guards against counter resets across process/CH restart producing a huge
+// uint64 underflow.
+func safeDelta(prev, cur uint64) uint64 {
+ if cur < prev {
+ return cur
+ }
+ return cur - prev
+}
diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.go
index c0affc3e..fec534ea 100644
--- a/backend/app/monitoring/ch_reporter.go
+++ b/backend/app/monitoring/ch_reporter.go
@@ -24,6 +24,14 @@ var telemetryTables = []string{
"tasks",
}
+type chBaselines struct {
+ insertedRows uint64
+ failedInserts uint64
+ delayedInserts uint64
+ rejectedInserts uint64
+ first bool
+}
+
func StartClickHouseReporter(ctx context.Context) {
go func() {
defer traceway.Recover()
@@ -31,32 +39,40 @@ func StartClickHouseReporter(ctx context.Context) {
ticker := time.NewTicker(reportInterval)
defer ticker.Stop()
- var prevInserted uint64
- var prevFailed uint64
- first := true
+ baselines := &chBaselines{first: true}
// Fire once immediately so we don't wait 60s for the first data point.
- reportOnce(ctx, &prevInserted, &prevFailed, &first)
+ reportOnce(ctx, baselines)
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
- reportOnce(ctx, &prevInserted, &prevFailed, &first)
+ reportOnce(ctx, baselines)
}
}
}()
}
-func reportOnce(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {
+func reportOnce(ctx context.Context, b *chBaselines) {
// Bound each tick so a stuck CH doesn't wedge the reporter forever.
qCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
+ reportPoolStats()
reportPartsByTable(qCtx)
+ reportMaxPartsPerPartition(qCtx)
reportMergesRunning(qCtx)
- reportEventDeltas(qCtx, prevInserted, prevFailed, first)
+ reportEventDeltas(qCtx, b)
+ reportDiskUsage(qCtx)
+ reportServerMemory(qCtx)
+}
+
+func reportPoolStats() {
+ stats := chdb.Conn.Stats()
+ traceway.CaptureMetric("traceway.ch.pool.open", float64(stats.Open))
+ traceway.CaptureMetric("traceway.ch.pool.idle", float64(stats.Idle))
}
func reportPartsByTable(ctx context.Context) {
@@ -78,6 +94,31 @@ func reportPartsByTable(ctx context.Context) {
}
}
+func reportMaxPartsPerPartition(ctx context.Context) {
+ rows, err := chdb.Conn.Query(
+ ctx,
+ "SELECT table, max(parts) FROM (SELECT table, partition, count() AS parts FROM system.parts WHERE database = currentDatabase() AND active GROUP BY table, partition) GROUP BY table",
+ )
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition query failed: %w", err))
+ return
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var table string
+ var maxParts uint64
+ if err := rows.Scan(&table, &maxParts); err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition scan failed: %w", err))
+ return
+ }
+ traceway.CaptureMetricWithTags("traceway.ch.parts.max_per_partition", float64(maxParts), map[string]string{"table": table})
+ }
+ if err := rows.Err(); err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition rows failed: %w", err))
+ }
+}
+
func reportMergesRunning(ctx context.Context) {
var merges uint64
err := chdb.Conn.QueryRow(
@@ -91,7 +132,7 @@ func reportMergesRunning(ctx context.Context) {
traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))
}
-func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {
+func reportEventDeltas(ctx context.Context, b *chBaselines) {
curInserted, err := scanEventValue(ctx, "InsertedRows")
if err != nil {
traceway.CaptureException(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))
@@ -102,17 +143,29 @@ func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, fi
traceway.CaptureException(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))
return
}
+ curDelayed, err := scanEventValue(ctx, "DelayedInserts")
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.events DelayedInserts failed: %w", err))
+ return
+ }
+ curRejected, err := scanEventValue(ctx, "RejectedInserts")
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.events RejectedInserts failed: %w", err))
+ return
+ }
// First tick has no baseline — the "delta" would be the lifetime counter.
- if !*first {
- inserted := safeDelta(*prevInserted, curInserted)
- failed := safeDelta(*prevFailed, curFailed)
- traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(inserted))
- traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(failed))
- }
- *prevInserted = curInserted
- *prevFailed = curFailed
- *first = false
+ if !b.first {
+ traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(safeDelta(b.insertedRows, curInserted)))
+ traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(safeDelta(b.failedInserts, curFailed)))
+ traceway.CaptureMetric("traceway.ch.delayed_inserts.delta", float64(safeDelta(b.delayedInserts, curDelayed)))
+ traceway.CaptureMetric("traceway.ch.rejected_inserts.delta", float64(safeDelta(b.rejectedInserts, curRejected)))
+ }
+ b.insertedRows = curInserted
+ b.failedInserts = curFailed
+ b.delayedInserts = curDelayed
+ b.rejectedInserts = curRejected
+ b.first = false
}
func scanEventValue(ctx context.Context, event string) (uint64, error) {
@@ -125,11 +178,56 @@ func scanEventValue(ctx context.Context, event string) (uint64, error) {
return value, err
}
-// Guards against counter resets across process/CH restart producing a huge
-// uint64 underflow.
-func safeDelta(prev, cur uint64) uint64 {
- if cur < prev {
- return cur
+func reportDiskUsage(ctx context.Context) {
+ rows, err := chdb.Conn.Query(ctx, "SELECT name, free_space, total_space FROM system.disks")
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.disks query failed: %w", err))
+ return
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var name string
+ var free, total uint64
+ if err := rows.Scan(&name, &free, &total); err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.disks scan failed: %w", err))
+ return
+ }
+ if total == 0 {
+ continue
+ }
+ tags := map[string]string{"disk": name}
+ traceway.CaptureMetricWithTags("traceway.ch.disk.free_bytes", float64(free), tags)
+ traceway.CaptureMetricWithTags("traceway.ch.disk.total_bytes", float64(total), tags)
+ traceway.CaptureMetricWithTags("traceway.ch.disk.used_pcnt", float64(total-free)/float64(total)*100, tags)
+ }
+ if err := rows.Err(); err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.disks rows failed: %w", err))
+ }
+}
+
+func reportServerMemory(ctx context.Context) {
+ var tracking int64
+ err := chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(sum(value), 0) FROM system.metrics WHERE metric = 'MemoryTracking'",
+ ).Scan(&tracking)
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.metrics MemoryTracking failed: %w", err))
+ } else {
+ traceway.CaptureMetric("traceway.ch.memory.tracking_bytes", float64(tracking))
+ }
+
+ var available float64
+ err = chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(sum(value), 0) FROM system.asynchronous_metrics WHERE metric = 'OSMemoryAvailable'",
+ ).Scan(&available)
+ if err != nil {
+ traceway.CaptureException(fmt.Errorf("monitoring: system.asynchronous_metrics OSMemoryAvailable failed: %w", err))
+ return
+ }
+ if available > 0 {
+ traceway.CaptureMetric("traceway.ch.memory.os_available_bytes", available)
}
- return cur - prev
}
diff --git a/backend/app/monitoring/metrics.go b/backend/app/monitoring/metrics.go
index 259ecca8..784a1781 100644
--- a/backend/app/monitoring/metrics.go
+++ b/backend/app/monitoring/metrics.go
@@ -2,6 +2,7 @@ package monitoring
import (
"strconv"
+ "sync/atomic"
traceway "go.tracewayapp.com"
)
@@ -13,7 +14,21 @@ const (
SignalNative = "native"
)
-func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size int) {
+var inFlightIngest atomic.Int64
+
+func IngestStarted() {
+ inFlightIngest.Add(1)
+}
+
+func IngestFinished() {
+ inFlightIngest.Add(-1)
+}
+
+func InFlightIngest() int64 {
+ return inFlightIngest.Load()
+}
+
+func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size, bytes int) {
tags := map[string]string{
"signal": signal,
"table": table,
@@ -21,6 +36,7 @@ func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size i
traceway.CaptureMetricWithTags("traceway.ingest.batch.convert_ms", convertMs, tags)
traceway.CaptureMetricWithTags("traceway.ingest.batch.insert_ms", insertMs, tags)
traceway.CaptureMetricWithTags("traceway.ingest.batch.size", float64(size), tags)
+ traceway.CaptureMetricWithTags("traceway.ingest.batch.bytes", float64(bytes), tags)
}
func RecordRateLimited(orgID int) {
diff --git a/backend/app/monitoring/proc_rss_linux.go b/backend/app/monitoring/proc_rss_linux.go
new file mode 100644
index 00000000..e3011eb5
--- /dev/null
+++ b/backend/app/monitoring/proc_rss_linux.go
@@ -0,0 +1,25 @@
+//go:build linux
+
+package monitoring
+
+import (
+ "os"
+ "strconv"
+ "strings"
+)
+
+func ReadRSSBytes() (uint64, bool) {
+ data, err := os.ReadFile("/proc/self/statm")
+ if err != nil {
+ return 0, false
+ }
+ fields := strings.Fields(string(data))
+ if len(fields) < 2 {
+ return 0, false
+ }
+ pages, err := strconv.ParseUint(fields[1], 10, 64)
+ if err != nil {
+ return 0, false
+ }
+ return pages * uint64(os.Getpagesize()), true
+}
diff --git a/backend/app/monitoring/proc_rss_other.go b/backend/app/monitoring/proc_rss_other.go
new file mode 100644
index 00000000..031e82a2
--- /dev/null
+++ b/backend/app/monitoring/proc_rss_other.go
@@ -0,0 +1,7 @@
+//go:build !linux
+
+package monitoring
+
+func ReadRSSBytes() (uint64, bool) {
+ return 0, false
+}
diff --git a/backend/app/monitoring/recordings.go b/backend/app/monitoring/recordings.go
index 72c16f26..58282af9 100644
--- a/backend/app/monitoring/recordings.go
+++ b/backend/app/monitoring/recordings.go
@@ -2,10 +2,10 @@ package monitoring
import traceway "go.tracewayapp.com"
-func RecordRecordingUploader(queueDepth, inFlight int, uploaded, dropped, failed uint64) {
+func RecordRecordingUploader(queueDepth, inFlight int, uploadedDelta, droppedDelta, failedDelta uint64) {
traceway.CaptureMetric("traceway.recordings.queue_depth", float64(queueDepth))
traceway.CaptureMetric("traceway.recordings.in_flight", float64(inFlight))
- traceway.CaptureMetric("traceway.recordings.uploaded", float64(uploaded))
- traceway.CaptureMetric("traceway.recordings.dropped", float64(dropped))
- traceway.CaptureMetric("traceway.recordings.failed", float64(failed))
+ traceway.CaptureMetric("traceway.recordings.uploaded.delta", float64(uploadedDelta))
+ traceway.CaptureMetric("traceway.recordings.dropped.delta", float64(droppedDelta))
+ traceway.CaptureMetric("traceway.recordings.failed.delta", float64(failedDelta))
}
diff --git a/backend/app/recordings/uploader.go b/backend/app/recordings/uploader.go
index b907e53a..fc9de56c 100644
--- a/backend/app/recordings/uploader.go
+++ b/backend/app/recordings/uploader.go
@@ -237,6 +237,8 @@ func (p *pool) batcher(ctx context.Context) {
func (p *pool) metricsLoop(ctx context.Context) {
defer traceway.Recover()
+ var prevUploaded, prevDropped, prevFailed uint64
+
ticker := time.NewTicker(metricsTickInterval)
defer ticker.Stop()
for {
@@ -244,13 +246,19 @@ func (p *pool) metricsLoop(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
+ curUploaded := p.uploaded.Load()
+ curDropped := p.dropped.Load()
+ curFailed := p.failed.Load()
monitoring.RecordRecordingUploader(
len(p.jobs),
int(p.inFlight.Load()),
- p.uploaded.Load(),
- p.dropped.Load(),
- p.failed.Load(),
+ curUploaded-prevUploaded,
+ curDropped-prevDropped,
+ curFailed-prevFailed,
)
+ prevUploaded = curUploaded
+ prevDropped = curDropped
+ prevFailed = curFailed
}
}
}
diff --git a/backend/app/repositories/metric_registry.repository.go b/backend/app/repositories/metric_registry.repository.go
index 6980a67b..e1556449 100644
--- a/backend/app/repositories/metric_registry.repository.go
+++ b/backend/app/repositories/metric_registry.repository.go
@@ -149,4 +149,13 @@ func defaultUnit(name string) string {
}
}
+func (r *metricRegistryRepository) KnownCount() int {
+ count := 0
+ r.knownMetrics.Range(func(_, _ any) bool {
+ count++
+ return true
+ })
+ return count
+}
+
var MetricRegistryRepository = metricRegistryRepository{}
diff --git a/backend/app/services/sourcemap_resolver.go b/backend/app/services/sourcemap_resolver.go
index 91a6fbd8..5e54c5ca 100644
--- a/backend/app/services/sourcemap_resolver.go
+++ b/backend/app/services/sourcemap_resolver.go
@@ -9,6 +9,7 @@ import (
"strconv"
"strings"
"sync"
+ "time"
"github.com/tracewayapp/traceway/backend/app/cache"
"github.com/tracewayapp/traceway/backend/app/models"
@@ -26,10 +27,13 @@ import (
//
// Consumers are safe for concurrent reads per go-sourcemap docs.
type parsedSourceMapCache struct {
- mu sync.Mutex
- items map[string]*list.Element
- order *list.List
- max int
+ mu sync.Mutex
+ items map[string]*list.Element
+ order *list.List
+ max int
+ hits uint64
+ misses uint64
+ lastParseMs float64
}
type parsedSourceMapEntry struct {
@@ -43,16 +47,30 @@ var parsedSourceMaps = &parsedSourceMapCache{
max: 5,
}
+func InitParsedSourceMapCache(max int) {
+ parsedSourceMaps.mu.Lock()
+ defer parsedSourceMaps.mu.Unlock()
+ parsedSourceMaps.max = max
+}
+
func (c *parsedSourceMapCache) get(key string) (*sourcemap.Consumer, bool) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.items[key]; ok {
+ c.hits++
c.order.MoveToFront(el)
return el.Value.(*parsedSourceMapEntry).consumer, true
}
+ c.misses++
return nil, false
}
+func (c *parsedSourceMapCache) recordParse(ms float64) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.lastParseMs = ms
+}
+
func (c *parsedSourceMapCache) put(key string, consumer *sourcemap.Consumer) {
c.mu.Lock()
defer c.mu.Unlock()
@@ -73,10 +91,24 @@ func (c *parsedSourceMapCache) put(key string, consumer *sourcemap.Consumer) {
}
}
-func ParsedSourceMapStats() (items int, max int) {
+type ParsedSourceMapCacheStats struct {
+ Entries int
+ Max int
+ Hits uint64
+ Misses uint64
+ LastParseMs float64
+}
+
+func ParsedSourceMapStats() ParsedSourceMapCacheStats {
parsedSourceMaps.mu.Lock()
defer parsedSourceMaps.mu.Unlock()
- return parsedSourceMaps.order.Len(), parsedSourceMaps.max
+ return ParsedSourceMapCacheStats{
+ Entries: parsedSourceMaps.order.Len(),
+ Max: parsedSourceMaps.max,
+ Hits: parsedSourceMaps.hits,
+ Misses: parsedSourceMaps.misses,
+ LastParseMs: parsedSourceMaps.lastParseMs,
+ }
}
var stackFrameRe = regexp.MustCompile(`^(\s{4})(.+):(\d+):(\d+)$`)
@@ -227,10 +259,12 @@ func getParsedSourceMap(ctx context.Context, storageKey string, localConsumers m
cache.SourceMapCache.Put(storageKey, data)
}
+ parseStart := time.Now()
consumer, err := sourcemap.Parse("", data)
if err != nil {
return nil, err
}
+ parsedSourceMaps.recordParse(float64(time.Since(parseStart).Microseconds()) / 1000.0)
parsedSourceMaps.put(storageKey, consumer)
localConsumers[storageKey] = consumer
return consumer, nil
diff --git a/backend/cmd/run.go b/backend/cmd/run.go
index 47b9b3c1..8f95b60c 100644
--- a/backend/cmd/run.go
+++ b/backend/cmd/run.go
@@ -1,6 +1,15 @@
package cmd
import (
+ "context"
+ "fmt"
+ "io/fs"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
"github.com/tracewayapp/traceway/backend/app/cache"
"github.com/tracewayapp/traceway/backend/app/chdb"
"github.com/tracewayapp/traceway/backend/app/config"
@@ -16,13 +25,6 @@ import (
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
"github.com/tracewayapp/traceway/backend/static"
- "context"
- "fmt"
- "io/fs"
- "net/http"
- "os"
- "strings"
- "time"
"github.com/coreos/go-systemd/v22/daemon"
"github.com/gin-gonic/gin"
@@ -107,7 +109,11 @@ func Run(opts ...Option) {
if err := cache.ProjectCache.Init(ctx); err != nil {
panic(fmt.Errorf("projects cache could not be initialized: %w", err))
}
- cache.InitSourceMapCache(200, 500*1024*1024)
+ cache.InitSourceMapCache(
+ parsePositiveInt(cfg.SourceMapCacheMaxEntries, 200),
+ int64(parsePositiveInt(cfg.SourceMapCacheMaxBytesMB, 500))*1024*1024,
+ )
+ services.InitParsedSourceMapCache(parsePositiveInt(cfg.SourceMapParsedCacheMax, 5))
middleware.InitUseClientAuth()
middleware.InitUseAppAuth()
@@ -142,6 +148,7 @@ func Run(opts ...Option) {
tracewaygin.WithOnErrorRecording(tracewaygin.RecordingQuery|tracewaygin.RecordingBody|tracewaygin.RecordingHeader|tracewaygin.RecordingUrl),
))
monitoring.StartClickHouseReporter(ctx)
+ monitoring.StartBackendReporter(ctx)
}
router.GET("/health", func(c *gin.Context) {
@@ -243,6 +250,18 @@ func applyOAuthEnvOverrides(cfg *config.Cfg) {
}
}
+func parsePositiveInt(s string, def int) int {
+ s = strings.TrimSpace(s)
+ if s == "" {
+ return def
+ }
+ v, err := strconv.Atoi(s)
+ if err != nil || v < 1 {
+ return def
+ }
+ return v
+}
+
func notifySystemd() {
sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)
if err != nil {
diff --git a/docs/pages/server/_meta.json b/docs/pages/server/_meta.json
index 8e683557..c4193d3c 100644
--- a/docs/pages/server/_meta.json
+++ b/docs/pages/server/_meta.json
@@ -5,5 +5,6 @@
"minimal": "Minimal Container",
"sqlite": "SQLite",
"local-setup": "Local Setup",
- "haloy": "Haloy"
+ "haloy": "Haloy",
+ "capacity-monitoring": "Capacity & Self-Monitoring"
}
diff --git a/docs/pages/server/capacity-monitoring.mdx b/docs/pages/server/capacity-monitoring.mdx
new file mode 100644
index 00000000..27e8a291
--- /dev/null
+++ b/docs/pages/server/capacity-monitoring.mdx
@@ -0,0 +1,173 @@
+# Capacity & Self-Monitoring
+
+Traceway's backend can monitor itself with the same Go SDK your applications use. When enabled, the server continuously reports ingestion pressure, ClickHouse health, cache sizes, and process resource usage into a regular Traceway project — so you can dashboard them, and more importantly, alert on them before you hit a wall.
+
+This page lists every self-monitoring metric, recommended alert thresholds, and a decision tree for the three scaling questions that matter in production: **do I scale ClickHouse, do I scale the backend server, or do I put Kafka in front of ingestion?**
+
+## Enabling self-monitoring
+
+1. Create a project in your Traceway dashboard (a dedicated "monitoring" project keeps things tidy). For larger setups, point self-monitoring at a *separate* Traceway instance so an outage doesn't take down its own monitoring.
+2. Copy the project token and set:
+
+```bash
+MONITORING_TRACEWAY_URL="<project_token>@https://your-traceway-host/api/report"
+```
+
+That single variable turns everything on:
+
+- Every HTTP request to the backend is captured as a transaction (so endpoint latency/error-rate alert rules work on `POST /api/report` out of the box).
+- The Go SDK reports runtime metrics every 30s: `cpu.used_pcnt`, `mem.used`, `mem.total`, `go.go_routines`, `go.heap_objects`, `go.num_gc`, `go.gc_pause`.
+- The backend reporters described below start emitting `traceway.*` metrics.
+
+Note: `mem.used` is Go **heap** memory only. The real OOM-relevant number is `traceway.proc.rss_mb` (resident set size, Linux only).
+
+## Metric reference
+
+### Ingestion (every batch)
+
+Emitted once per `/api/report` and OTLP request, tagged with `signal` and `table`:
+
+| Metric | Meaning |
+|--------|---------|
+| `traceway.ingest.batch.convert_ms` | Time converting the client payload into rows (backend CPU work) |
+| `traceway.ingest.batch.insert_ms` | Time writing the batch into ClickHouse (inside the request — clients wait on this) |
+| `traceway.ingest.batch.size` | Rows in the batch |
+| `traceway.ingest.batch.bytes` | Decompressed payload size in bytes |
+| `traceway.ingest.rate_limited` | Fired when an org hits its ingest quota (tagged `org_id`) |
+
+### Backend process (every 30s)
+
+| Metric | Meaning |
+|--------|---------|
+| `traceway.ingest.in_flight` | Ingest requests currently being processed |
+| `traceway.proc.rss_mb` | Process resident memory in MB (Linux only) |
+| `traceway.cache.projects.entries` | Projects held in the in-memory token cache |
+| `traceway.cache.metric_registry.entries` | Known metric names cached in memory |
+
+### ClickHouse health (every 60s, ClickHouse deployments only)
+
+Not emitted in SQLite mode.
+
+| Metric | Meaning |
+|--------|---------|
+| `traceway.ch.pool.open` / `traceway.ch.pool.idle` | Connection pool usage (max is 15) |
+| `traceway.ch.parts.active` / `traceway.ch.parts.rows` | Active parts and rows per telemetry table (tagged `table`) |
+| `traceway.ch.parts.max_per_partition` | Largest part count in any single partition, per table (tagged `table`). ClickHouse delays inserts at 1000 parts/partition and rejects them at 3000 |
+| `traceway.ch.merges.running` | Background merges currently running |
+| `traceway.ch.inserted_rows.delta` | Rows inserted since the last tick |
+| `traceway.ch.failed_inserts.delta` | Failed insert queries since the last tick |
+| `traceway.ch.delayed_inserts.delta` | Inserts ClickHouse throttled since the last tick (parts back-pressure) |
+| `traceway.ch.rejected_inserts.delta` | Inserts ClickHouse rejected since the last tick (`parts_to_throw` hit) |
+| `traceway.ch.disk.free_bytes` / `total_bytes` / `used_pcnt` | Disk usage per ClickHouse disk (tagged `disk`) |
+| `traceway.ch.memory.tracking_bytes` | Memory the ClickHouse server is currently tracking |
+| `traceway.ch.memory.os_available_bytes` | Available OS memory on the ClickHouse host |
+
+### Session recording uploader (every 10s)
+
+| Metric | Meaning |
+|--------|---------|
+| `traceway.recordings.queue_depth` | Segments waiting in the upload queue (cap default 2048) |
+| `traceway.recordings.in_flight` | Workers mid-upload |
+| `traceway.recordings.uploaded.delta` | Segments uploaded since the last tick |
+| `traceway.recordings.dropped.delta` | Segments dropped (queue full) since the last tick — data loss |
+| `traceway.recordings.failed.delta` | Upload/insert failures since the last tick |
+
+### Source map caches (every 30s)
+
+| Metric | Meaning |
+|--------|---------|
+| `traceway.sourcemap.raw.entries` / `traceway.sourcemap.raw.bytes` | Raw `.map` files held in memory |
+| `traceway.sourcemap.raw.hits.delta` / `misses.delta` / `evictions.delta` | Raw cache traffic since the last tick |
+| `traceway.sourcemap.parsed.entries` | Parsed source maps held in memory |
+| `traceway.sourcemap.parsed.hits.delta` / `misses.delta` | Parsed cache traffic since the last tick |
+| `traceway.sourcemap.parsed.parse_ms` | Duration of the most recent source map parse |
+
+## Source map cache sizing
+
+Source maps are cached at two levels: raw file bytes, and parsed lookup structures. **A parsed source map is roughly 5–10× its raw size** — a 20 MB `.map` file becomes 100–200 MB in memory. The parsed cache is capped at 5 entries by default for exactly this reason; with five large maps that's still up to ~1 GB.
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `SOURCEMAP_CACHE_MAX_ENTRIES` | `200` | Max raw `.map` files kept in memory |
+| `SOURCEMAP_CACHE_MAX_BYTES_MB` | `500` | Max total raw bytes kept in memory |
+| `SOURCEMAP_PARSED_CACHE_MAX` | `5` | Max parsed source maps kept in memory |
+
+Sizing guidance:
+
+- **Small box (1–2 GB RAM):** `SOURCEMAP_CACHE_MAX_BYTES_MB=100`, `SOURCEMAP_PARSED_CACHE_MAX=2`.
+- **Plenty of RAM, many active source maps:** raise `SOURCEMAP_PARSED_CACHE_MAX` until `traceway.sourcemap.parsed.misses.delta` stays near zero in steady state — every parsed-cache miss re-parses a map, which burns CPU and briefly doubles its memory footprint.
+- Watch `traceway.proc.rss_mb` against your host RAM; if it climbs after exception views with stack traces, the parsed cache is the usual suspect.
+
+## Alerting on these metrics
+
+Create rules in the monitoring project under **Notifications** using the **Metric threshold** rule type. A rule is defined by: metric name, aggregation (`avg`, `max`, `min`, `sum`, `p95`, `p99`), operator (`gt`, `gte`, `lt`, `lte`, `eq`), threshold value, and lookback window in minutes. Rules are evaluated every 60 seconds.
+
+**Tag caveat:** metric threshold rules aggregate across *all* tag combinations of a metric name. That is what you want for the metrics on this page — `max` on `traceway.ch.parts.max_per_partition` gives you the worst partition across every table, and `max` on `traceway.ch.disk.used_pcnt` gives you the fullest disk. Just avoid `avg` on per-table/per-disk metrics, where mixing series would dilute the signal.
+
+Two more rule types complete the picture:
+
+- **Endpoint P95 threshold** on `POST /api/report` — client-visible ingest latency (the backend's own HTTP requests are transactions in the monitoring project).
+- **No data** with data type `metrics` — fires when the backend stops self-reporting entirely (crash, network partition).
+
+## Recommended starting alerts
+
+| Metric | Agg | Op | Threshold | Lookback | Urgency | Meaning |
+|--------|-----|----|-----------|----------|---------|---------|
+| `traceway.ch.failed_inserts.delta` | max | gt | 0 | 5 | critical | Inserts are failing right now |
+| `traceway.ch.rejected_inserts.delta` | max | gt | 0 | 5 | critical | ClickHouse is rejecting inserts (3000 parts/partition hit) |
+| `traceway.ch.inserted_rows.delta` | max | lte | 0 | 10 | critical | Ingestion stalled — nothing written for 10 minutes |
+| `traceway.ch.disk.used_pcnt` | max | gt | 90 | 5 | critical | ClickHouse disk nearly full |
+| `traceway.ch.parts.max_per_partition` | max | gt | 2000 | 5 | critical | Approaching the insert-rejection threshold |
+| `traceway.ingest.batch.insert_ms` | p95 | gt | 500 | 10 | warning | ClickHouse write latency rising — clients feel this directly |
+| `traceway.ch.delayed_inserts.delta` | max | gt | 0 | 10 | warning | ClickHouse is throttling inserts (merge back-pressure) |
+| `traceway.ch.parts.max_per_partition` | max | gt | 700 | 10 | warning | Approaching the insert-delay threshold (1000) |
+| `traceway.ch.disk.used_pcnt` | max | gt | 80 | 15 | warning | ClickHouse disk filling up |
+| `traceway.ch.pool.open` | avg | gte | 15 | 10 | warning | Connection pool saturated — check `insert_ms` next |
+| `traceway.recordings.dropped.delta` | max | gt | 0 | 5 | warning | Recording segments being dropped — ingestion overload |
+| `cpu.used_pcnt` | avg | gt | 80 | 10 | warning | Backend CPU-bound |
+| `traceway.proc.rss_mb` | avg | gt | 80% of host RAM | 10 | warning | Memory pressure / OOM risk (e.g. 800 on a 1 GB box) |
+| `traceway.sourcemap.raw.bytes` | avg | gt | ~half of `SOURCEMAP_CACHE_MAX_BYTES_MB`×1MB | 30 | info | Source map cache growing — revisit sizing |
+
+Plus the two non-metric rules: **Endpoint P95 threshold** on `POST /api/report` (e.g. 1000 ms over 10 minutes) and **No data** on `metrics` with 10 minutes of silence.
+
+## Scale ClickHouse, scale the backend, or add Kafka?
+
+Ingestion in Traceway is synchronous: each `/api/report` request converts the payload and writes it to ClickHouse before responding. That makes the diagnosis straightforward — the metrics tell you which side of that pipeline is saturated, and whether the problem is sustained or bursty.
+
+### Scale ClickHouse when the pressure is sustained and CH-side
+
+ClickHouse can't merge or store as fast as you're writing. Signals, in rough order of severity:
+
+- `traceway.ch.parts.max_per_partition` climbing steadily, `traceway.ch.merges.running` persistently high
+- `traceway.ch.delayed_inserts.delta` > 0 across multiple intervals (then `rejected_inserts` if ignored)
+- `traceway.ch.disk.used_pcnt` rising, `traceway.ch.memory.tracking_bytes` near the host's RAM
+- `traceway.ingest.batch.insert_ms` p95 high **while the ClickHouse host's CPU is also high**
+
+Fix: more CPU/RAM/disk for ClickHouse, faster disks, or sharding. Kafka won't help here — a buffer in front of a database that's saturated on average just delays the inevitable.
+
+### Scale the backend server when the Go process is the bottleneck
+
+- `cpu.used_pcnt` avg > 80% together with high `traceway.ingest.batch.convert_ms` — payload conversion (JSON/OTLP decoding, hashing) is CPU work that scales with traffic
+- `traceway.proc.rss_mb` approaching host RAM — before resizing, tune the source map cache variables above; they're the largest tunable consumer
+- `go.go_routines` growing without bound
+
+Fix: a bigger box, or multiple backend instances behind a load balancer (the backend is stateless apart from in-memory caches).
+
+### Add Kafka when the load is bursty and ClickHouse averages look healthy
+
+Because inserts happen inside the request, every ClickHouse hiccup or burst becomes client-visible `/api/report` latency. The tell-tale combination:
+
+- `traceway.ingest.batch.insert_ms` p95 spiking while ClickHouse CPU, disk, and merges look calm on average
+- `traceway.ch.pool.open` pinned at 15 (pool saturated) during peaks
+- `traceway.recordings.dropped.delta` > 0 — the one buffered path is already overflowing during bursts
+- P95 on `POST /api/report` rising even though total `traceway.ch.inserted_rows.delta` is well within what ClickHouse handles at steady state
+
+That pattern means ClickHouse can absorb your *average* write rate but not your *peaks*, and the synchronous path turns peaks into client errors and data loss. A queue (Kafka) between the HTTP handlers and ClickHouse decouples client latency from insert latency, absorbs spikes, and enables larger, better-shaped batches.
+
+### Rule of thumb
+
+| Pattern | Action |
+|---------|--------|
+| High `convert_ms` + high backend CPU | Scale the backend |
+| Sustained parts/merge/disk/memory pressure on CH | Scale ClickHouse |
+| Bursty `insert_ms` spikes + drops + pool saturation while CH averages are calm | Add Kafka |