The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
feat: more metrics regarding the backend
ddux65cfdbe05130
diff --git a/backend/app/cache/project_cache.go b/backend/app/cache/project_cache.goindex f24f0c9e..7b3bcedd 100644--- a/backend/app/cache/project_cache.go+++ b/backend/app/cache/project_cache.go@@ -147,3 +147,9 @@ func (c *projectCache) LastRefresh() time.Time {defer c.mu.RUnlock()return c.lastRefresh}++func (c *projectCache) Entries() int {+ c.mu.RLock()+ defer c.mu.RUnlock()+ return len(c.projectsById)+}diff --git a/backend/app/cache/sourcemap_cache.go b/backend/app/cache/sourcemap_cache.goindex 2ef1bd56..66d57cb7 100644--- a/backend/app/cache/sourcemap_cache.go+++ b/backend/app/cache/sourcemap_cache.go@@ -11,12 +11,25 @@ type sourceMapCacheEntry struct {}type sourceMapCache struct {- maxCount int- maxBytes int64- mu sync.Mutex- items map[string]*list.Element- order *list.List- curBytes int64+ maxCount int+ maxBytes int64+ mu sync.Mutex+ items map[string]*list.Element+ order *list.List+ curBytes int64+ hits uint64+ misses uint64+ evictions uint64+}++type SourceMapCacheStats struct {+ Entries int+ Bytes int64+ MaxEntries int+ MaxBytes int64+ Hits uint64+ Misses uint64+ Evictions uint64}var SourceMapCache *sourceMapCache@@ -35,9 +48,11 @@ func (c *sourceMapCache) Get(key string) ([]byte, bool) {defer c.mu.Unlock()if el, ok := c.items[key]; ok {+ c.hits++c.order.MoveToFront(el)return el.Value.(*sourceMapCacheEntry).data, true}+ c.misses++return nil, false}@@ -67,5 +82,21 @@ func (c *sourceMapCache) Put(key string, data []byte) {evicted := c.order.Remove(back).(*sourceMapCacheEntry)delete(c.items, evicted.key)c.curBytes -= int64(len(evicted.data))+ c.evictions+++ }+}++func (c *sourceMapCache) Stats() SourceMapCacheStats {+ c.mu.Lock()+ defer c.mu.Unlock()++ return SourceMapCacheStats{+ Entries: len(c.items),+ Bytes: c.curBytes,+ MaxEntries: c.maxCount,+ MaxBytes: c.maxBytes,+ Hits: c.hits,+ Misses: c.misses,+ Evictions: c.evictions,}}diff --git a/backend/app/chdb/chdb.go b/backend/app/chdb/chdb.goindex 5b12bc33..debdab7a 100644--- a/backend/app/chdb/chdb.go+++ b/backend/app/chdb/chdb.go@@ -19,6 +19,7 @@ type ChConn interface {QueryRow(ctx context.Context, query string, args ...any) driver.RowPrepareBatch(ctx context.Context, query string, opts ...driver.PrepareBatchOption) (driver.Batch, error)Exec(ctx context.Context, query string, args ...any) error+ Stats() driver.Stats}var Conn ChConndiff --git a/backend/app/config/config.go b/backend/app/config/config.goindex fa15cbba..7f81af4c 100644--- a/backend/app/config/config.go+++ b/backend/app/config/config.go@@ -33,6 +33,10 @@ type Cfg struct {SessionRecordingUploadWorkers stringSessionRecordingUploadQueueSize string+ SourceMapCacheMaxEntries string+ SourceMapCacheMaxBytesMB string+ SourceMapParsedCacheMax string+SMTPEnabled stringSMTPHost stringSMTPPort string@@ -105,6 +109,10 @@ func LoadFromEnv() *Cfg {SessionRecordingUploadWorkers: os.Getenv("SESSION_RECORDING_UPLOAD_WORKERS"),SessionRecordingUploadQueueSize: os.Getenv("SESSION_RECORDING_UPLOAD_QUEUE_SIZE"),+ SourceMapCacheMaxEntries: os.Getenv("SOURCEMAP_CACHE_MAX_ENTRIES"),+ SourceMapCacheMaxBytesMB: os.Getenv("SOURCEMAP_CACHE_MAX_BYTES_MB"),+ SourceMapParsedCacheMax: os.Getenv("SOURCEMAP_PARSED_CACHE_MAX"),+SMTPEnabled: os.Getenv("SMTP_ENABLED"),SMTPHost: os.Getenv("SMTP_HOST"),SMTPPort: os.Getenv("SMTP_PORT"),diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.goindex c8ef56a8..a05e6222 100644--- a/backend/app/controllers/clientcontrollers/client.controller.go+++ b/backend/app/controllers/clientcontrollers/client.controller.go@@ -49,6 +49,9 @@ type ReportRequest struct {}func (e clientController) Report(c *gin.Context) {+ monitoring.IngestStarted()+ defer monitoring.IngestFinished()+projectId, err := middleware.GetProjectId(c)if err != nil {c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))@@ -74,6 +77,13 @@ func (e clientController) Report(c *gin.Context) {}parseSpan.End()+ bodyBytes := 0+ if cb, ok := c.Get(gin.BodyBytesKey); ok {+ if b, ok := cb.([]byte); ok {+ bodyBytes = len(b)+ }+ }+convertStart := time.Now()endpointsToInsert := []models.Endpoint{}@@ -298,7 +308,7 @@ func (e clientController) Report(c *gin.Context) {insertMs := float64(time.Since(insertStart).Microseconds()) / 1000.0totalSize := len(endpointsToInsert) + len(tasksToInsert) + len(spansToInsert) + len(exceptionStackTraceToInsert) + len(metricPointsToInsert)- monitoring.RecordIngestBatch(monitoring.SignalNative, "report", convertMs, insertMs, totalSize)+ monitoring.RecordIngestBatch(monitoring.SignalNative, "report", convertMs, insertMs, totalSize, bodyBytes)var exceptionHashes []stringfor _, est := range exceptionStackTraceToInsert {diff --git a/backend/app/controllers/otelcontrollers/codec.go b/backend/app/controllers/otelcontrollers/codec.goindex ac1bc3af..0f6585e2 100644--- a/backend/app/controllers/otelcontrollers/codec.go+++ b/backend/app/controllers/otelcontrollers/codec.go@@ -35,40 +35,40 @@ func isProtobuf(c *gin.Context) bool {return strings.Contains(ct, "application/x-protobuf") || strings.Contains(ct, "application/protobuf")}-func decodeTraceRequest(c *gin.Context) (*coltracepb.ExportTraceServiceRequest, error) {+func decodeTraceRequest(c *gin.Context) (*coltracepb.ExportTraceServiceRequest, int, error) {body, err := readBody(c)if err != nil {- return nil, err+ return nil, 0, err}req := &coltracepb.ExportTraceServiceRequest{}if isProtobuf(c) {if err := proto.Unmarshal(body, req); err != nil {- return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err)+ return nil, 0, fmt.Errorf("failed to unmarshal protobuf: %w", err)}} else {if err := protojson.Unmarshal(body, req); err != nil {- return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)+ return nil, 0, fmt.Errorf("failed to unmarshal JSON: %w", err)}}- return req, nil+ return req, len(body), nil}-func decodeMetricsRequest(c *gin.Context) (*colmetricspb.ExportMetricsServiceRequest, error) {+func decodeMetricsRequest(c *gin.Context) (*colmetricspb.ExportMetricsServiceRequest, int, error) {body, err := readBody(c)if err != nil {- return nil, err+ return nil, 0, err}req := &colmetricspb.ExportMetricsServiceRequest{}if isProtobuf(c) {if err := proto.Unmarshal(body, req); err != nil {- return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err)+ return nil, 0, fmt.Errorf("failed to unmarshal protobuf: %w", err)}} else {if err := protojson.Unmarshal(body, req); err != nil {- return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)+ return nil, 0, fmt.Errorf("failed to unmarshal JSON: %w", err)}}- return req, nil+ return req, len(body), nil}func writeTraceResponse(c *gin.Context) {@@ -93,22 +93,22 @@ func writeMetricsResponse(c *gin.Context) {}}-func decodeLogsRequest(c *gin.Context) (*collogspb.ExportLogsServiceRequest, error) {+func decodeLogsRequest(c *gin.Context) (*collogspb.ExportLogsServiceRequest, int, error) {body, err := readBody(c)if err != nil {- return nil, err+ return nil, 0, err}req := &collogspb.ExportLogsServiceRequest{}if isProtobuf(c) {if err := proto.Unmarshal(body, req); err != nil {- return nil, fmt.Errorf("failed to unmarshal protobuf: %w", err)+ return nil, 0, fmt.Errorf("failed to unmarshal protobuf: %w", err)}} else {if err := protojson.Unmarshal(body, req); err != nil {- return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)+ return nil, 0, fmt.Errorf("failed to unmarshal JSON: %w", err)}}- return req, nil+ return req, len(body), nil}func writeLogsResponse(c *gin.Context) {diff --git a/backend/app/controllers/otelcontrollers/otel.controller.go b/backend/app/controllers/otelcontrollers/otel.controller.goindex 2224f044..5ad1318c 100644--- a/backend/app/controllers/otelcontrollers/otel.controller.go+++ b/backend/app/controllers/otelcontrollers/otel.controller.go@@ -26,6 +26,9 @@ type otelController struct{}var OtelController = otelController{}func (o otelController) ExportTraces(c *gin.Context) {+ monitoring.IngestStarted()+ defer monitoring.IngestFinished()+projectId, err := middleware.GetProjectId(c)if err != nil {c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))@@ -43,7 +46,7 @@ func (o otelController) ExportTraces(c *gin.Context) {}}}- req, err := decodeTraceRequest(c)+ req, bodyBytes, err := decodeTraceRequest(c)if err != nil {c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})return@@ -101,7 +104,7 @@ func (o otelController) ExportTraces(c *gin.Context) {insertMs := msSince(insertStart)totalSize := len(endpoints) + len(tasks) + len(spans) + len(exceptions) + len(aiTraces)- monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize)+ monitoring.RecordIngestBatch(monitoring.SignalTraces, "traces", convertMs, insertMs, totalSize, bodyBytes)var exceptionHashes []stringfor _, ex := range exceptions {@@ -125,6 +128,9 @@ func (o otelController) ExportTraces(c *gin.Context) {}func (o otelController) ExportMetrics(c *gin.Context) {+ monitoring.IngestStarted()+ defer monitoring.IngestFinished()+projectId, err := middleware.GetProjectId(c)if err != nil {c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))@@ -144,7 +150,7 @@ func (o otelController) ExportMetrics(c *gin.Context) {}}- req, err := decodeMetricsRequest(c)+ req, bodyBytes, err := decodeMetricsRequest(c)if err != nil {c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})return@@ -168,12 +174,15 @@ func (o otelController) ExportMetrics(c *gin.Context) {}}- monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points))+ monitoring.RecordIngestBatch(monitoring.SignalMetrics, "metric_points", convertMs, insertMs, len(result.Points), bodyBytes)writeMetricsResponse(c)}func (o otelController) ExportLogs(c *gin.Context) {+ monitoring.IngestStarted()+ defer monitoring.IngestFinished()+projectId, err := middleware.GetProjectId(c)if err != nil {c.AbortWithError(http.StatusInternalServerError, traceway.NewStackTraceErrorf("UseClientAuth middleware must be applied: %w", err))@@ -193,7 +202,7 @@ func (o otelController) ExportLogs(c *gin.Context) {}}- req, err := decodeLogsRequest(c)+ req, bodyBytes, err := decodeLogsRequest(c)if err != nil {c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})return@@ -213,7 +222,7 @@ func (o otelController) ExportLogs(c *gin.Context) {insertMs = msSince(insertStart)}- monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records))+ monitoring.RecordIngestBatch(monitoring.SignalLogs, "log_records", convertMs, insertMs, len(records), bodyBytes)writeLogsResponse(c)}diff --git a/backend/app/monitoring/backend_reporter.go b/backend/app/monitoring/backend_reporter.gonew file mode 100644index 00000000..5f07604a--- /dev/null+++ b/backend/app/monitoring/backend_reporter.go@@ -0,0 +1,87 @@+package monitoring++import (+ "context"+ "time"++ traceway "go.tracewayapp.com"++ "github.com/tracewayapp/traceway/backend/app/cache"+ "github.com/tracewayapp/traceway/backend/app/repositories"+ "github.com/tracewayapp/traceway/backend/app/services"+)++const backendReportInterval = 30 * time.Second++type backendBaselines struct {+ rawHits uint64+ rawMisses uint64+ rawEvictions uint64+ parsedHits uint64+ parsedMisses uint64+ first bool+}++func StartBackendReporter(ctx context.Context) {+ go func() {+ defer traceway.Recover()++ ticker := time.NewTicker(backendReportInterval)+ defer ticker.Stop()++ baselines := &backendBaselines{first: true}++ reportBackendOnce(baselines)++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ reportBackendOnce(baselines)+ }+ }+ }()+}++func reportBackendOnce(b *backendBaselines) {+ traceway.CaptureMetric("traceway.ingest.in_flight", float64(InFlightIngest()))++ if rss, ok := ReadRSSBytes(); ok {+ traceway.CaptureMetric("traceway.proc.rss_mb", float64(rss)/1024.0/1024.0)+ }++ rawStats := cache.SourceMapCache.Stats()+ traceway.CaptureMetric("traceway.sourcemap.raw.entries", float64(rawStats.Entries))+ traceway.CaptureMetric("traceway.sourcemap.raw.bytes", float64(rawStats.Bytes))++ parsedStats := services.ParsedSourceMapStats()+ traceway.CaptureMetric("traceway.sourcemap.parsed.entries", float64(parsedStats.Entries))+ traceway.CaptureMetric("traceway.sourcemap.parsed.parse_ms", parsedStats.LastParseMs)++ if !b.first {+ traceway.CaptureMetric("traceway.sourcemap.raw.hits.delta", float64(safeDelta(b.rawHits, rawStats.Hits)))+ traceway.CaptureMetric("traceway.sourcemap.raw.misses.delta", float64(safeDelta(b.rawMisses, rawStats.Misses)))+ traceway.CaptureMetric("traceway.sourcemap.raw.evictions.delta", float64(safeDelta(b.rawEvictions, rawStats.Evictions)))+ traceway.CaptureMetric("traceway.sourcemap.parsed.hits.delta", float64(safeDelta(b.parsedHits, parsedStats.Hits)))+ traceway.CaptureMetric("traceway.sourcemap.parsed.misses.delta", float64(safeDelta(b.parsedMisses, parsedStats.Misses)))+ }+ b.rawHits = rawStats.Hits+ b.rawMisses = rawStats.Misses+ b.rawEvictions = rawStats.Evictions+ b.parsedHits = parsedStats.Hits+ b.parsedMisses = parsedStats.Misses+ b.first = false++ traceway.CaptureMetric("traceway.cache.projects.entries", float64(cache.ProjectCache.Entries()))+ traceway.CaptureMetric("traceway.cache.metric_registry.entries", float64(repositories.MetricRegistryRepository.KnownCount()))+}++// Guards against counter resets across process/CH restart producing a huge+// uint64 underflow.+func safeDelta(prev, cur uint64) uint64 {+ if cur < prev {+ return cur+ }+ return cur - prev+}diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.goindex c0affc3e..fec534ea 100644--- a/backend/app/monitoring/ch_reporter.go+++ b/backend/app/monitoring/ch_reporter.go@@ -24,6 +24,14 @@ var telemetryTables = []string{"tasks",}+type chBaselines struct {+ insertedRows uint64+ failedInserts uint64+ delayedInserts uint64+ rejectedInserts uint64+ first bool+}+func StartClickHouseReporter(ctx context.Context) {go func() {defer traceway.Recover()@@ -31,32 +39,40 @@ func StartClickHouseReporter(ctx context.Context) {ticker := time.NewTicker(reportInterval)defer ticker.Stop()- var prevInserted uint64- var prevFailed uint64- first := true+ baselines := &chBaselines{first: true}// Fire once immediately so we don't wait 60s for the first data point.- reportOnce(ctx, &prevInserted, &prevFailed, &first)+ reportOnce(ctx, baselines)for {select {case <-ctx.Done():returncase <-ticker.C:- reportOnce(ctx, &prevInserted, &prevFailed, &first)+ reportOnce(ctx, baselines)}}}()}-func reportOnce(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {+func reportOnce(ctx context.Context, b *chBaselines) {// Bound each tick so a stuck CH doesn't wedge the reporter forever.qCtx, cancel := context.WithTimeout(ctx, 10*time.Second)defer cancel()+ reportPoolStats()reportPartsByTable(qCtx)+ reportMaxPartsPerPartition(qCtx)reportMergesRunning(qCtx)- reportEventDeltas(qCtx, prevInserted, prevFailed, first)+ reportEventDeltas(qCtx, b)+ reportDiskUsage(qCtx)+ reportServerMemory(qCtx)+}++func reportPoolStats() {+ stats := chdb.Conn.Stats()+ traceway.CaptureMetric("traceway.ch.pool.open", float64(stats.Open))+ traceway.CaptureMetric("traceway.ch.pool.idle", float64(stats.Idle))}func reportPartsByTable(ctx context.Context) {@@ -78,6 +94,31 @@ func reportPartsByTable(ctx context.Context) {}}+func reportMaxPartsPerPartition(ctx context.Context) {+ rows, err := chdb.Conn.Query(+ ctx,+ "SELECT table, max(parts) FROM (SELECT table, partition, count() AS parts FROM system.parts WHERE database = currentDatabase() AND active GROUP BY table, partition) GROUP BY table",+ )+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition query failed: %w", err))+ return+ }+ defer rows.Close()++ for rows.Next() {+ var table string+ var maxParts uint64+ if err := rows.Scan(&table, &maxParts); err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition scan failed: %w", err))+ return+ }+ traceway.CaptureMetricWithTags("traceway.ch.parts.max_per_partition", float64(maxParts), map[string]string{"table": table})+ }+ if err := rows.Err(); err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition rows failed: %w", err))+ }+}+func reportMergesRunning(ctx context.Context) {var merges uint64err := chdb.Conn.QueryRow(@@ -91,7 +132,7 @@ func reportMergesRunning(ctx context.Context) {traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))}-func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, first *bool) {+func reportEventDeltas(ctx context.Context, b *chBaselines) {curInserted, err := scanEventValue(ctx, "InsertedRows")if err != nil {traceway.CaptureException(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))@@ -102,17 +143,29 @@ func reportEventDeltas(ctx context.Context, prevInserted, prevFailed *uint64, fitraceway.CaptureException(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))return}+ curDelayed, err := scanEventValue(ctx, "DelayedInserts")+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.events DelayedInserts failed: %w", err))+ return+ }+ curRejected, err := scanEventValue(ctx, "RejectedInserts")+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.events RejectedInserts failed: %w", err))+ return+ }// First tick has no baseline — the "delta" would be the lifetime counter.- if !*first {- inserted := safeDelta(*prevInserted, curInserted)- failed := safeDelta(*prevFailed, curFailed)- traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(inserted))- traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(failed))- }- *prevInserted = curInserted- *prevFailed = curFailed- *first = false+ if !b.first {+ traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(safeDelta(b.insertedRows, curInserted)))+ traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(safeDelta(b.failedInserts, curFailed)))+ traceway.CaptureMetric("traceway.ch.delayed_inserts.delta", float64(safeDelta(b.delayedInserts, curDelayed)))+ traceway.CaptureMetric("traceway.ch.rejected_inserts.delta", float64(safeDelta(b.rejectedInserts, curRejected)))+ }+ b.insertedRows = curInserted+ b.failedInserts = curFailed+ b.delayedInserts = curDelayed+ b.rejectedInserts = curRejected+ b.first = false}func scanEventValue(ctx context.Context, event string) (uint64, error) {@@ -125,11 +178,56 @@ func scanEventValue(ctx context.Context, event string) (uint64, error) {return value, err}-// Guards against counter resets across process/CH restart producing a huge-// uint64 underflow.-func safeDelta(prev, cur uint64) uint64 {- if cur < prev {- return cur+func reportDiskUsage(ctx context.Context) {+ rows, err := chdb.Conn.Query(ctx, "SELECT name, free_space, total_space FROM system.disks")+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.disks query failed: %w", err))+ return+ }+ defer rows.Close()++ for rows.Next() {+ var name string+ var free, total uint64+ if err := rows.Scan(&name, &free, &total); err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.disks scan failed: %w", err))+ return+ }+ if total == 0 {+ continue+ }+ tags := map[string]string{"disk": name}+ traceway.CaptureMetricWithTags("traceway.ch.disk.free_bytes", float64(free), tags)+ traceway.CaptureMetricWithTags("traceway.ch.disk.total_bytes", float64(total), tags)+ traceway.CaptureMetricWithTags("traceway.ch.disk.used_pcnt", float64(total-free)/float64(total)*100, tags)+ }+ if err := rows.Err(); err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.disks rows failed: %w", err))+ }+}++func reportServerMemory(ctx context.Context) {+ var tracking int64+ err := chdb.Conn.QueryRow(+ ctx,+ "SELECT coalesce(sum(value), 0) FROM system.metrics WHERE metric = 'MemoryTracking'",+ ).Scan(&tracking)+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.metrics MemoryTracking failed: %w", err))+ } else {+ traceway.CaptureMetric("traceway.ch.memory.tracking_bytes", float64(tracking))+ }++ var available float64+ err = chdb.Conn.QueryRow(+ ctx,+ "SELECT coalesce(sum(value), 0) FROM system.asynchronous_metrics WHERE metric = 'OSMemoryAvailable'",+ ).Scan(&available)+ if err != nil {+ traceway.CaptureException(fmt.Errorf("monitoring: system.asynchronous_metrics OSMemoryAvailable failed: %w", err))+ return+ }+ if available > 0 {+ traceway.CaptureMetric("traceway.ch.memory.os_available_bytes", available)}- return cur - prev}diff --git a/backend/app/monitoring/metrics.go b/backend/app/monitoring/metrics.goindex 259ecca8..784a1781 100644--- a/backend/app/monitoring/metrics.go+++ b/backend/app/monitoring/metrics.go@@ -2,6 +2,7 @@ package monitoringimport ("strconv"+ "sync/atomic"traceway "go.tracewayapp.com")@@ -13,7 +14,21 @@ const (SignalNative = "native")-func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size int) {+var inFlightIngest atomic.Int64++func IngestStarted() {+ inFlightIngest.Add(1)+}++func IngestFinished() {+ inFlightIngest.Add(-1)+}++func InFlightIngest() int64 {+ return inFlightIngest.Load()+}++func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size, bytes int) {tags := map[string]string{"signal": signal,"table": table,@@ -21,6 +36,7 @@ func RecordIngestBatch(signal, table string, convertMs, insertMs float64, size itraceway.CaptureMetricWithTags("traceway.ingest.batch.convert_ms", convertMs, tags)traceway.CaptureMetricWithTags("traceway.ingest.batch.insert_ms", insertMs, tags)traceway.CaptureMetricWithTags("traceway.ingest.batch.size", float64(size), tags)+ traceway.CaptureMetricWithTags("traceway.ingest.batch.bytes", float64(bytes), tags)}func RecordRateLimited(orgID int) {diff --git a/backend/app/monitoring/proc_rss_linux.go b/backend/app/monitoring/proc_rss_linux.gonew file mode 100644index 00000000..e3011eb5--- /dev/null+++ b/backend/app/monitoring/proc_rss_linux.go@@ -0,0 +1,25 @@+//go:build linux++package monitoring++import (+ "os"+ "strconv"+ "strings"+)++func ReadRSSBytes() (uint64, bool) {+ data, err := os.ReadFile("/proc/self/statm")+ if err != nil {+ return 0, false+ }+ fields := strings.Fields(string(data))+ if len(fields) < 2 {+ return 0, false+ }+ pages, err := strconv.ParseUint(fields[1], 10, 64)+ if err != nil {+ return 0, false+ }+ return pages * uint64(os.Getpagesize()), true+}diff --git a/backend/app/monitoring/proc_rss_other.go b/backend/app/monitoring/proc_rss_other.gonew file mode 100644index 00000000..031e82a2--- /dev/null+++ b/backend/app/monitoring/proc_rss_other.go@@ -0,0 +1,7 @@+//go:build !linux++package monitoring++func ReadRSSBytes() (uint64, bool) {+ return 0, false+}diff --git a/backend/app/monitoring/recordings.go b/backend/app/monitoring/recordings.goindex 72c16f26..58282af9 100644--- a/backend/app/monitoring/recordings.go+++ b/backend/app/monitoring/recordings.go@@ -2,10 +2,10 @@ package monitoringimport traceway "go.tracewayapp.com"-func RecordRecordingUploader(queueDepth, inFlight int, uploaded, dropped, failed uint64) {+func RecordRecordingUploader(queueDepth, inFlight int, uploadedDelta, droppedDelta, failedDelta uint64) {traceway.CaptureMetric("traceway.recordings.queue_depth", float64(queueDepth))traceway.CaptureMetric("traceway.recordings.in_flight", float64(inFlight))- traceway.CaptureMetric("traceway.recordings.uploaded", float64(uploaded))- traceway.CaptureMetric("traceway.recordings.dropped", float64(dropped))- traceway.CaptureMetric("traceway.recordings.failed", float64(failed))+ traceway.CaptureMetric("traceway.recordings.uploaded.delta", float64(uploadedDelta))+ traceway.CaptureMetric("traceway.recordings.dropped.delta", float64(droppedDelta))+ traceway.CaptureMetric("traceway.recordings.failed.delta", float64(failedDelta))}diff --git a/backend/app/recordings/uploader.go b/backend/app/recordings/uploader.goindex b907e53a..fc9de56c 100644--- a/backend/app/recordings/uploader.go+++ b/backend/app/recordings/uploader.go@@ -237,6 +237,8 @@ func (p *pool) batcher(ctx context.Context) {func (p *pool) metricsLoop(ctx context.Context) {defer traceway.Recover()+ var prevUploaded, prevDropped, prevFailed uint64+ticker := time.NewTicker(metricsTickInterval)defer ticker.Stop()for {@@ -244,13 +246,19 @@ func (p *pool) metricsLoop(ctx context.Context) {case <-ctx.Done():returncase <-ticker.C:+ curUploaded := p.uploaded.Load()+ curDropped := p.dropped.Load()+ curFailed := p.failed.Load()monitoring.RecordRecordingUploader(len(p.jobs),int(p.inFlight.Load()),- p.uploaded.Load(),- p.dropped.Load(),- p.failed.Load(),+ curUploaded-prevUploaded,+ curDropped-prevDropped,+ curFailed-prevFailed,)+ prevUploaded = curUploaded+ prevDropped = curDropped+ prevFailed = curFailed}}}diff --git a/backend/app/repositories/metric_registry.repository.go b/backend/app/repositories/metric_registry.repository.goindex 6980a67b..e1556449 100644--- a/backend/app/repositories/metric_registry.repository.go+++ b/backend/app/repositories/metric_registry.repository.go@@ -149,4 +149,13 @@ func defaultUnit(name string) string {}}+func (r *metricRegistryRepository) KnownCount() int {+ count := 0+ r.knownMetrics.Range(func(_, _ any) bool {+ count+++ return true+ })+ return count+}+var MetricRegistryRepository = metricRegistryRepository{}diff --git a/backend/app/services/sourcemap_resolver.go b/backend/app/services/sourcemap_resolver.goindex 91a6fbd8..5e54c5ca 100644--- a/backend/app/services/sourcemap_resolver.go+++ b/backend/app/services/sourcemap_resolver.go@@ -9,6 +9,7 @@ import ("strconv""strings""sync"+ "time""github.com/tracewayapp/traceway/backend/app/cache""github.com/tracewayapp/traceway/backend/app/models"@@ -26,10 +27,13 @@ import (//// Consumers are safe for concurrent reads per go-sourcemap docs.type parsedSourceMapCache struct {- mu sync.Mutex- items map[string]*list.Element- order *list.List- max int+ mu sync.Mutex+ items map[string]*list.Element+ order *list.List+ max int+ hits uint64+ misses uint64+ lastParseMs float64}type parsedSourceMapEntry struct {@@ -43,16 +47,30 @@ var parsedSourceMaps = &parsedSourceMapCache{max: 5,}+func InitParsedSourceMapCache(max int) {+ parsedSourceMaps.mu.Lock()+ defer parsedSourceMaps.mu.Unlock()+ parsedSourceMaps.max = max+}+func (c *parsedSourceMapCache) get(key string) (*sourcemap.Consumer, bool) {c.mu.Lock()defer c.mu.Unlock()if el, ok := c.items[key]; ok {+ c.hits++c.order.MoveToFront(el)return el.Value.(*parsedSourceMapEntry).consumer, true}+ c.misses++return nil, false}+func (c *parsedSourceMapCache) recordParse(ms float64) {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.lastParseMs = ms+}+func (c *parsedSourceMapCache) put(key string, consumer *sourcemap.Consumer) {c.mu.Lock()defer c.mu.Unlock()@@ -73,10 +91,24 @@ func (c *parsedSourceMapCache) put(key string, consumer *sourcemap.Consumer) {}}-func ParsedSourceMapStats() (items int, max int) {+type ParsedSourceMapCacheStats struct {+ Entries int+ Max int+ Hits uint64+ Misses uint64+ LastParseMs float64+}++func ParsedSourceMapStats() ParsedSourceMapCacheStats {parsedSourceMaps.mu.Lock()defer parsedSourceMaps.mu.Unlock()- return parsedSourceMaps.order.Len(), parsedSourceMaps.max+ return ParsedSourceMapCacheStats{+ Entries: parsedSourceMaps.order.Len(),+ Max: parsedSourceMaps.max,+ Hits: parsedSourceMaps.hits,+ Misses: parsedSourceMaps.misses,+ LastParseMs: parsedSourceMaps.lastParseMs,+ }}var stackFrameRe = regexp.MustCompile(`^(\s{4})(.+):(\d+):(\d+)$`)@@ -227,10 +259,12 @@ func getParsedSourceMap(ctx context.Context, storageKey string, localConsumers mcache.SourceMapCache.Put(storageKey, data)}+ parseStart := time.Now()consumer, err := sourcemap.Parse("", data)if err != nil {return nil, err}+ parsedSourceMaps.recordParse(float64(time.Since(parseStart).Microseconds()) / 1000.0)parsedSourceMaps.put(storageKey, consumer)localConsumers[storageKey] = consumerreturn consumer, nildiff --git a/backend/cmd/run.go b/backend/cmd/run.goindex 47b9b3c1..8f95b60c 100644--- a/backend/cmd/run.go+++ b/backend/cmd/run.go@@ -1,6 +1,15 @@package cmdimport (+ "context"+ "fmt"+ "io/fs"+ "net/http"+ "os"+ "strconv"+ "strings"+ "time"+"github.com/tracewayapp/traceway/backend/app/cache""github.com/tracewayapp/traceway/backend/app/chdb""github.com/tracewayapp/traceway/backend/app/config"@@ -16,13 +25,6 @@ import ("github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage""github.com/tracewayapp/traceway/backend/static"- "context"- "fmt"- "io/fs"- "net/http"- "os"- "strings"- "time""github.com/coreos/go-systemd/v22/daemon""github.com/gin-gonic/gin"@@ -107,7 +109,11 @@ func Run(opts ...Option) {if err := cache.ProjectCache.Init(ctx); err != nil {panic(fmt.Errorf("projects cache could not be initialized: %w", err))}- cache.InitSourceMapCache(200, 500*1024*1024)+ cache.InitSourceMapCache(+ parsePositiveInt(cfg.SourceMapCacheMaxEntries, 200),+ int64(parsePositiveInt(cfg.SourceMapCacheMaxBytesMB, 500))*1024*1024,+ )+ services.InitParsedSourceMapCache(parsePositiveInt(cfg.SourceMapParsedCacheMax, 5))middleware.InitUseClientAuth()middleware.InitUseAppAuth()@@ -142,6 +148,7 @@ func Run(opts ...Option) {tracewaygin.WithOnErrorRecording(tracewaygin.RecordingQuery|tracewaygin.RecordingBody|tracewaygin.RecordingHeader|tracewaygin.RecordingUrl),))monitoring.StartClickHouseReporter(ctx)+ monitoring.StartBackendReporter(ctx)}router.GET("/health", func(c *gin.Context) {@@ -243,6 +250,18 @@ func applyOAuthEnvOverrides(cfg *config.Cfg) {}}+func parsePositiveInt(s string, def int) int {+ s = strings.TrimSpace(s)+ if s == "" {+ return def+ }+ v, err := strconv.Atoi(s)+ if err != nil || v < 1 {+ return def+ }+ return v+}+func notifySystemd() {sent, err := daemon.SdNotify(false, daemon.SdNotifyReady)if err != nil {diff --git a/docs/pages/server/_meta.json b/docs/pages/server/_meta.jsonindex 8e683557..c4193d3c 100644--- a/docs/pages/server/_meta.json+++ b/docs/pages/server/_meta.json@@ -5,5 +5,6 @@"minimal": "Minimal Container","sqlite": "SQLite","local-setup": "Local Setup",- "haloy": "Haloy"+ "haloy": "Haloy",+ "capacity-monitoring": "Capacity & Self-Monitoring"}diff --git a/docs/pages/server/capacity-monitoring.mdx b/docs/pages/server/capacity-monitoring.mdxnew file mode 100644index 00000000..27e8a291--- /dev/null+++ b/docs/pages/server/capacity-monitoring.mdx@@ -0,0 +1,173 @@+# Capacity & Self-Monitoring++Traceway's backend can monitor itself with the same Go SDK your applications use. When enabled, the server continuously reports ingestion pressure, ClickHouse health, cache sizes, and process resource usage into a regular Traceway project — so you can dashboard them, and more importantly, alert on them before you hit a wall.++This page lists every self-monitoring metric, recommended alert thresholds, and a decision tree for the three scaling questions that matter in production: **do I scale ClickHouse, do I scale the backend server, or do I put Kafka in front of ingestion?**++## Enabling self-monitoring++1. Create a project in your Traceway dashboard (a dedicated "monitoring" project keeps things tidy). For larger setups, point self-monitoring at a *separate* Traceway instance so an outage doesn't take down its own monitoring.+2. Copy the project token and set:++```bash+MONITORING_TRACEWAY_URL="<project_token>@https://your-traceway-host/api/report"+```++That single variable turns everything on:++- Every HTTP request to the backend is captured as a transaction (so endpoint latency/error-rate alert rules work on `POST /api/report` out of the box).+- The Go SDK reports runtime metrics every 30s: `cpu.used_pcnt`, `mem.used`, `mem.total`, `go.go_routines`, `go.heap_objects`, `go.num_gc`, `go.gc_pause`.+- The backend reporters described below start emitting `traceway.*` metrics.++Note: `mem.used` is Go **heap** memory only. The real OOM-relevant number is `traceway.proc.rss_mb` (resident set size, Linux only).++## Metric reference++### Ingestion (every batch)++Emitted once per `/api/report` and OTLP request, tagged with `signal` and `table`:++| Metric | Meaning |+|--------|---------|+| `traceway.ingest.batch.convert_ms` | Time converting the client payload into rows (backend CPU work) |+| `traceway.ingest.batch.insert_ms` | Time writing the batch into ClickHouse (inside the request — clients wait on this) |+| `traceway.ingest.batch.size` | Rows in the batch |+| `traceway.ingest.batch.bytes` | Decompressed payload size in bytes |+| `traceway.ingest.rate_limited` | Fired when an org hits its ingest quota (tagged `org_id`) |++### Backend process (every 30s)++| Metric | Meaning |+|--------|---------|+| `traceway.ingest.in_flight` | Ingest requests currently being processed |+| `traceway.proc.rss_mb` | Process resident memory in MB (Linux only) |+| `traceway.cache.projects.entries` | Projects held in the in-memory token cache |+| `traceway.cache.metric_registry.entries` | Known metric names cached in memory |++### ClickHouse health (every 60s, ClickHouse deployments only)++Not emitted in SQLite mode.++| Metric | Meaning |+|--------|---------|+| `traceway.ch.pool.open` / `traceway.ch.pool.idle` | Connection pool usage (max is 15) |+| `traceway.ch.parts.active` / `traceway.ch.parts.rows` | Active parts and rows per telemetry table (tagged `table`) |+| `traceway.ch.parts.max_per_partition` | Largest part count in any single partition, per table (tagged `table`). ClickHouse delays inserts at 1000 parts/partition and rejects them at 3000 |+| `traceway.ch.merges.running` | Background merges currently running |+| `traceway.ch.inserted_rows.delta` | Rows inserted since the last tick |+| `traceway.ch.failed_inserts.delta` | Failed insert queries since the last tick |+| `traceway.ch.delayed_inserts.delta` | Inserts ClickHouse throttled since the last tick (parts back-pressure) |+| `traceway.ch.rejected_inserts.delta` | Inserts ClickHouse rejected since the last tick (`parts_to_throw` hit) |+| `traceway.ch.disk.free_bytes` / `total_bytes` / `used_pcnt` | Disk usage per ClickHouse disk (tagged `disk`) |+| `traceway.ch.memory.tracking_bytes` | Memory the ClickHouse server is currently tracking |+| `traceway.ch.memory.os_available_bytes` | Available OS memory on the ClickHouse host |++### Session recording uploader (every 10s)++| Metric | Meaning |+|--------|---------|+| `traceway.recordings.queue_depth` | Segments waiting in the upload queue (cap default 2048) |+| `traceway.recordings.in_flight` | Workers mid-upload |+| `traceway.recordings.uploaded.delta` | Segments uploaded since the last tick |+| `traceway.recordings.dropped.delta` | Segments dropped (queue full) since the last tick — data loss |+| `traceway.recordings.failed.delta` | Upload/insert failures since the last tick |++### Source map caches (every 30s)++| Metric | Meaning |+|--------|---------|+| `traceway.sourcemap.raw.entries` / `traceway.sourcemap.raw.bytes` | Raw `.map` files held in memory |+| `traceway.sourcemap.raw.hits.delta` / `misses.delta` / `evictions.delta` | Raw cache traffic since the last tick |+| `traceway.sourcemap.parsed.entries` | Parsed source maps held in memory |+| `traceway.sourcemap.parsed.hits.delta` / `misses.delta` | Parsed cache traffic since the last tick |+| `traceway.sourcemap.parsed.parse_ms` | Duration of the most recent source map parse |++## Source map cache sizing++Source maps are cached at two levels: raw file bytes, and parsed lookup structures. **A parsed source map is roughly 5–10× its raw size** — a 20 MB `.map` file becomes 100–200 MB in memory. The parsed cache is capped at 5 entries by default for exactly this reason; with five large maps that's still up to ~1 GB.++| Variable | Default | Description |+|----------|---------|-------------|+| `SOURCEMAP_CACHE_MAX_ENTRIES` | `200` | Max raw `.map` files kept in memory |+| `SOURCEMAP_CACHE_MAX_BYTES_MB` | `500` | Max total raw bytes kept in memory |+| `SOURCEMAP_PARSED_CACHE_MAX` | `5` | Max parsed source maps kept in memory |++Sizing guidance:++- **Small box (1–2 GB RAM):** `SOURCEMAP_CACHE_MAX_BYTES_MB=100`, `SOURCEMAP_PARSED_CACHE_MAX=2`.+- **Plenty of RAM, many active source maps:** raise `SOURCEMAP_PARSED_CACHE_MAX` until `traceway.sourcemap.parsed.misses.delta` stays near zero in steady state — every parsed-cache miss re-parses a map, which burns CPU and briefly doubles its memory footprint.+- Watch `traceway.proc.rss_mb` against your host RAM; if it climbs after exception views with stack traces, the parsed cache is the usual suspect.++## Alerting on these metrics++Create rules in the monitoring project under **Notifications** using the **Metric threshold** rule type. A rule is defined by: metric name, aggregation (`avg`, `max`, `min`, `sum`, `p95`, `p99`), operator (`gt`, `gte`, `lt`, `lte`, `eq`), threshold value, and lookback window in minutes. Rules are evaluated every 60 seconds.++**Tag caveat:** metric threshold rules aggregate across *all* tag combinations of a metric name. That is what you want for the metrics on this page — `max` on `traceway.ch.parts.max_per_partition` gives you the worst partition across every table, and `max` on `traceway.ch.disk.used_pcnt` gives you the fullest disk. Just avoid `avg` on per-table/per-disk metrics, where mixing series would dilute the signal.++Two more rule types complete the picture:++- **Endpoint P95 threshold** on `POST /api/report` — client-visible ingest latency (the backend's own HTTP requests are transactions in the monitoring project).+- **No data** with data type `metrics` — fires when the backend stops self-reporting entirely (crash, network partition).++## Recommended starting alerts++| Metric | Agg | Op | Threshold | Lookback | Urgency | Meaning |+|--------|-----|----|-----------|----------|---------|---------|+| `traceway.ch.failed_inserts.delta` | max | gt | 0 | 5 | critical | Inserts are failing right now |+| `traceway.ch.rejected_inserts.delta` | max | gt | 0 | 5 | critical | ClickHouse is rejecting inserts (3000 parts/partition hit) |+| `traceway.ch.inserted_rows.delta` | max | lte | 0 | 10 | critical | Ingestion stalled — nothing written for 10 minutes |+| `traceway.ch.disk.used_pcnt` | max | gt | 90 | 5 | critical | ClickHouse disk nearly full |+| `traceway.ch.parts.max_per_partition` | max | gt | 2000 | 5 | critical | Approaching the insert-rejection threshold |+| `traceway.ingest.batch.insert_ms` | p95 | gt | 500 | 10 | warning | ClickHouse write latency rising — clients feel this directly |+| `traceway.ch.delayed_inserts.delta` | max | gt | 0 | 10 | warning | ClickHouse is throttling inserts (merge back-pressure) |+| `traceway.ch.parts.max_per_partition` | max | gt | 700 | 10 | warning | Approaching the insert-delay threshold (1000) |+| `traceway.ch.disk.used_pcnt` | max | gt | 80 | 15 | warning | ClickHouse disk filling up |+| `traceway.ch.pool.open` | avg | gte | 15 | 10 | warning | Connection pool saturated — check `insert_ms` next |+| `traceway.recordings.dropped.delta` | max | gt | 0 | 5 | warning | Recording segments being dropped — ingestion overload |+| `cpu.used_pcnt` | avg | gt | 80 | 10 | warning | Backend CPU-bound |+| `traceway.proc.rss_mb` | avg | gt | 80% of host RAM | 10 | warning | Memory pressure / OOM risk (e.g. 800 on a 1 GB box) |+| `traceway.sourcemap.raw.bytes` | avg | gt | ~half of `SOURCEMAP_CACHE_MAX_BYTES_MB`×1MB | 30 | info | Source map cache growing — revisit sizing |++Plus the two non-metric rules: **Endpoint P95 threshold** on `POST /api/report` (e.g. 1000 ms over 10 minutes) and **No data** on `metrics` with 10 minutes of silence.++## Scale ClickHouse, scale the backend, or add Kafka?++Ingestion in Traceway is synchronous: each `/api/report` request converts the payload and writes it to ClickHouse before responding. That makes the diagnosis straightforward — the metrics tell you which side of that pipeline is saturated, and whether the problem is sustained or bursty.++### Scale ClickHouse when the pressure is sustained and CH-side++ClickHouse can't merge or store as fast as you're writing. Signals, in rough order of severity:++- `traceway.ch.parts.max_per_partition` climbing steadily, `traceway.ch.merges.running` persistently high+- `traceway.ch.delayed_inserts.delta` > 0 across multiple intervals (then `rejected_inserts` if ignored)+- `traceway.ch.disk.used_pcnt` rising, `traceway.ch.memory.tracking_bytes` near the host's RAM+- `traceway.ingest.batch.insert_ms` p95 high **while the ClickHouse host's CPU is also high**++Fix: more CPU/RAM/disk for ClickHouse, faster disks, or sharding. Kafka won't help here — a buffer in front of a database that's saturated on average just delays the inevitable.++### Scale the backend server when the Go process is the bottleneck++- `cpu.used_pcnt` avg > 80% together with high `traceway.ingest.batch.convert_ms` — payload conversion (JSON/OTLP decoding, hashing) is CPU work that scales with traffic+- `traceway.proc.rss_mb` approaching host RAM — before resizing, tune the source map cache variables above; they're the largest tunable consumer+- `go.go_routines` growing without bound++Fix: a bigger box, or multiple backend instances behind a load balancer (the backend is stateless apart from in-memory caches).++### Add Kafka when the load is bursty and ClickHouse averages look healthy++Because inserts happen inside the request, every ClickHouse hiccup or burst becomes client-visible `/api/report` latency. The tell-tale combination:++- `traceway.ingest.batch.insert_ms` p95 spiking while ClickHouse CPU, disk, and merges look calm on average+- `traceway.ch.pool.open` pinned at 15 (pool saturated) during peaks+- `traceway.recordings.dropped.delta` > 0 — the one buffered path is already overflowing during bursts+- P95 on `POST /api/report` rising even though total `traceway.ch.inserted_rows.delta` is well within what ClickHouse handles at steady state++That pattern means ClickHouse can absorb your *average* write rate but not your *peaks*, and the synchronous path turns peaks into client errors and data loss. A queue (Kafka) between the HTTP handlers and ClickHouse decouples client latency from insert latency, absorbs spikes, and enables larger, better-shaped batches.++### Rule of thumb++| Pattern | Action |+|---------|--------|+| High `convert_ms` + high backend CPU | Scale the backend |+| Sustained parts/merge/disk/memory pressure on CH | Scale ClickHouse |+| Bursty `insert_ms` spikes + drops + pool saturation while CH averages are calm | Add Kafka |