The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
progress
dduxc2bd5a86449a
diff --git a/CLAUDE.md b/CLAUDE.mdindex 1180cfbe..801cb727 100644--- a/CLAUDE.md+++ b/CLAUDE.md@@ -248,6 +248,10 @@ POSTGRES_SSLMODE=disable# Retention (see "Data Retention" section below)SQLITE_RETENTION_DAYS=30 # 0 to disable; only applies in SQLite modeSESSION_RECORDING_RETENTION_DAYS=30 # 0 to disable; only applies when STORAGE_TYPE=local++# Session recording uploads (see "Session Recording Uploader" section below)+SESSION_RECORDING_UPLOAD_WORKERS=32 # 0 to disable uploads entirely+SESSION_RECORDING_UPLOAD_QUEUE_SIZE=2048```---@@ -762,6 +766,27 @@ Tables it prunes (and the column used):The DB rows in `session_recordings` are pruned by the SQLite retention worker (above) or by ClickHouse TTL — they are intentionally not coupled to the disk cleanup. Controllers that read recordings already log a non-fatal `traceway.CaptureException` when a referenced file is missing.+#### Session Recording Uploader++Session recording segments arriving on `/api/report` are not uploaded inline. The handler enqueues each segment onto a bounded worker pool (`backend/app/recordings/uploader.go`, started from `cmd/run.go` next to `retention.Start`). Workers drain the queue and write the body via `storage.Store.Write` (S3 or local disk); successful writes are handed to a single batcher goroutine that calls `SessionRecordingRepository.InsertAsync` once per ~1000 rows or every 2 s, whichever comes first — single-row inserts are an anti-pattern for ClickHouse. Enqueue is non-blocking: when the queue is full the segment is dropped (newest-first) so a burst of `/api/report` traffic cannot spawn unbounded goroutines or saturate S3.++| Variable | Default | Notes |+|----------|---------|-------|+| `SESSION_RECORDING_UPLOAD_WORKERS` | `32` | Concurrent uploaders. Set to `0` to drop every segment (uploads disabled). |+| `SESSION_RECORDING_UPLOAD_QUEUE_SIZE` | `2048` | Max queued segments. Overflow is dropped. |++Observability (emitted every 10s via `traceway.CaptureMetric`):++| Metric | Type | Meaning |+|--------|------|---------|+| `traceway.recordings.queue_depth` | gauge | Segments currently waiting in the channel. |+| `traceway.recordings.in_flight` | gauge | Workers mid-upload. |+| `traceway.recordings.uploaded` | counter | Successful S3/local + DB writes since startup. |+| `traceway.recordings.dropped` | counter | Segments dropped on overflow since startup. |+| `traceway.recordings.failed` | counter | Upload or DB-insert errors since startup. |++Sustained drops also fire a rate-limited (1/min) `traceway.CaptureException` so overload is visible in the issues feed without flooding it.+#### Organization Roles| Role | Description ||------|-------------|diff --git a/backend/app/config/config.go b/backend/app/config/config.goindex 049d3e45..2661e1ae 100644--- a/backend/app/config/config.go+++ b/backend/app/config/config.go@@ -28,8 +28,10 @@ type Cfg struct {S3SecretKey stringS3Endpoint string- SQLiteRetentionDays string- SessionRecordingRetentionDays string+ SQLiteRetentionDays string+ SessionRecordingRetentionDays string+ SessionRecordingUploadWorkers string+ SessionRecordingUploadQueueSize stringSMTPEnabled stringSMTPHost string@@ -83,8 +85,10 @@ func LoadFromEnv() *Cfg {S3SecretKey: os.Getenv("S3_SECRET_KEY"),S3Endpoint: os.Getenv("S3_ENDPOINT"),- SQLiteRetentionDays: os.Getenv("SQLITE_RETENTION_DAYS"),- SessionRecordingRetentionDays: os.Getenv("SESSION_RECORDING_RETENTION_DAYS"),+ SQLiteRetentionDays: os.Getenv("SQLITE_RETENTION_DAYS"),+ SessionRecordingRetentionDays: os.Getenv("SESSION_RECORDING_RETENTION_DAYS"),+ SessionRecordingUploadWorkers: os.Getenv("SESSION_RECORDING_UPLOAD_WORKERS"),+ SessionRecordingUploadQueueSize: os.Getenv("SESSION_RECORDING_UPLOAD_QUEUE_SIZE"),SMTPEnabled: os.Getenv("SMTP_ENABLED"),SMTPHost: os.Getenv("SMTP_HOST"),diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.goindex ebac5896..e66ee6fe 100644--- a/backend/app/controllers/clientcontrollers/client.controller.go+++ b/backend/app/controllers/clientcontrollers/client.controller.go@@ -2,7 +2,6 @@ package clientcontrollersimport ("bytes"- "context""crypto/sha256""database/sql""encoding/hex"@@ -19,9 +18,9 @@ import ("github.com/tracewayapp/traceway/backend/app/models""github.com/tracewayapp/traceway/backend/app/models/clientmodels""github.com/tracewayapp/traceway/backend/app/monitoring"+ "github.com/tracewayapp/traceway/backend/app/recordings""github.com/tracewayapp/traceway/backend/app/repositories""github.com/tracewayapp/traceway/backend/app/services"- "github.com/tracewayapp/traceway/backend/app/storage""github.com/gin-gonic/gin""github.com/google/uuid"@@ -84,22 +83,7 @@ func (e clientController) Report(c *gin.Context) {spansToInsert := []models.Span{}sessionsToUpsert := []models.Session{}- type recordingWork struct {- Id uuid.UUID- ProjectId uuid.UUID- ExceptionId uuid.UUID- SessionId *uuid.UUID- SegmentIndex int32- // Key is the S3 object key the body is written to.- Key string- // Body is the marshaled JSON of the entire ClientSessionRecording- // sub-document — events + logs + actions + startedAt/endedAt — exactly- // as it lands in S3. App console logs in `logs` are intentionally not- // inserted into the OTel logs ClickHouse table; they live only here.- Body []byte- RecordedAt time.Time- }- var recordingsWork []recordingWork+ var recordingsWork []recordings.Job// Map frontend sessionRecordingId → backend-generated exception UUID.// Used only for the legacy exception-bound recording path; the always-on@@ -234,7 +218,7 @@ func (e clientController) Report(c *gin.Context) {} else {key = fmt.Sprintf("recordings/%s/%s.json", projectId, exceptionId)}- recordingsWork = append(recordingsWork, recordingWork{+ recordingsWork = append(recordingsWork, recordings.Job{Id: uuid.New(),ProjectId: projectId,ExceptionId: exceptionId,@@ -335,31 +319,8 @@ func (e clientController) Report(c *gin.Context) {}}- if len(recordingsWork) > 0 {- work := recordingsWork- go func() {- var successful []models.SessionRecording- for _, rw := range work {- if err := storage.Store.Write(context.Background(), rw.Key, rw.Body); err != nil {- traceway.CaptureException(traceway.NewStackTraceErrorf("failed to write session recording (key=%s): %w", rw.Key, err))- continue- }- successful = append(successful, models.SessionRecording{- Id: rw.Id,- ProjectId: rw.ProjectId,- ExceptionId: rw.ExceptionId,- SessionId: rw.SessionId,- SegmentIndex: rw.SegmentIndex,- FilePath: rw.Key,- RecordedAt: rw.RecordedAt,- })- }- if len(successful) > 0 {- if err := repositories.SessionRecordingRepository.InsertAsync(context.Background(), successful); err != nil {- traceway.CaptureException(traceway.NewStackTraceErrorf("failed to batch insert %d session recording refs: %w", len(successful), err))- }- }- }()+ for _, rw := range recordingsWork {+ recordings.Enqueue(rw)}c.JSON(http.StatusOK, gin.H{})diff --git a/backend/app/monitoring/recordings.go b/backend/app/monitoring/recordings.gonew file mode 100644index 00000000..72c16f26--- /dev/null+++ b/backend/app/monitoring/recordings.go@@ -0,0 +1,11 @@+package monitoring++import traceway "go.tracewayapp.com"++func RecordRecordingUploader(queueDepth, inFlight int, uploaded, dropped, failed uint64) {+ traceway.CaptureMetric("traceway.recordings.queue_depth", float64(queueDepth))+ traceway.CaptureMetric("traceway.recordings.in_flight", float64(inFlight))+ traceway.CaptureMetric("traceway.recordings.uploaded", float64(uploaded))+ traceway.CaptureMetric("traceway.recordings.dropped", float64(dropped))+ traceway.CaptureMetric("traceway.recordings.failed", float64(failed))+}diff --git a/backend/app/recordings/uploader.go b/backend/app/recordings/uploader.gonew file mode 100644index 00000000..b907e53a--- /dev/null+++ b/backend/app/recordings/uploader.go@@ -0,0 +1,257 @@+package recordings++import (+ "context"+ "strconv"+ "strings"+ "sync"+ "sync/atomic"+ "time"++ "github.com/google/uuid"+ "github.com/tracewayapp/traceway/backend/app/config"+ "github.com/tracewayapp/traceway/backend/app/models"+ "github.com/tracewayapp/traceway/backend/app/monitoring"+ "github.com/tracewayapp/traceway/backend/app/repositories"+ "github.com/tracewayapp/traceway/backend/app/storage"+ traceway "go.tracewayapp.com"+)++const (+ defaultWorkers = 32+ defaultQueueSize = 2048+ metricsTickInterval = 10 * time.Second+ dropReportInterval = time.Minute+ insertBatchSize = 1000+ insertFlushInterval = 2 * time.Second+)++type Job struct {+ Id uuid.UUID+ ProjectId uuid.UUID+ ExceptionId uuid.UUID+ SessionId *uuid.UUID+ SegmentIndex int32+ Key string+ Body []byte+ RecordedAt time.Time+}++type storer interface {+ Write(ctx context.Context, key string, data []byte) error+}++type pool struct {+ workers int+ jobs chan Job+ inserts chan models.SessionRecording++ storage storer++ inFlight atomic.Int64+ uploaded atomic.Uint64+ dropped atomic.Uint64+ failed atomic.Uint64++ dropMu sync.Mutex+ lastDropReportAt time.Time+ droppedSinceReport uint64+}++var singleton *pool++func Start(ctx context.Context) {+ if singleton != nil {+ return+ }++ workers := defaultWorkers+ workersStr := strings.TrimSpace(config.Config.SessionRecordingUploadWorkers)+ if workersStr != "" {+ if v, err := strconv.Atoi(workersStr); err == nil && v >= 0 {+ workers = v+ }+ }++ queueSize := defaultQueueSize+ queueStr := strings.TrimSpace(config.Config.SessionRecordingUploadQueueSize)+ if queueStr != "" {+ if v, err := strconv.Atoi(queueStr); err == nil && v >= 1 {+ queueSize = v+ }+ }++ singleton = &pool{+ workers: workers,+ jobs: make(chan Job, queueSize),+ inserts: make(chan models.SessionRecording, insertBatchSize),+ storage: storage.Store,+ }+ singleton.start(ctx)++ if workers == 0 {+ config.Logln("Session recording uploader disabled (SESSION_RECORDING_UPLOAD_WORKERS=0); segments will be dropped")+ } else {+ config.Logf("Started session recording uploader (workers=%d, queue=%d)", workers, queueSize)+ }+}++func Enqueue(j Job) {+ if singleton == nil {+ return+ }+ singleton.enqueue(j)+}++func (p *pool) start(ctx context.Context) {+ for i := 0; i < p.workers; i++ {+ go p.worker(ctx)+ }+ go p.metricsLoop(ctx)+ go p.batcher(ctx)+}++func (p *pool) enqueue(j Job) {+ select {+ case p.jobs <- j:+ return+ default:+ }++ p.dropped.Add(1)++ var report uint64+ p.dropMu.Lock()+ p.droppedSinceReport+++ if time.Since(p.lastDropReportAt) >= dropReportInterval {+ report = p.droppedSinceReport+ p.droppedSinceReport = 0+ p.lastDropReportAt = time.Now()+ }+ p.dropMu.Unlock()++ if report > 0 {+ traceway.CaptureException(traceway.NewStackTraceErrorf(+ "session recording uploader dropped %d segments due to full queue (cap=%d)", report, cap(p.jobs)))+ }+}++func (p *pool) worker(ctx context.Context) {+ defer traceway.Recover()++ for {+ select {+ case <-ctx.Done():+ return+ case j, ok := <-p.jobs:+ if !ok {+ return+ }+ p.handle(ctx, j)+ }+ }+}++func (p *pool) handle(ctx context.Context, j Job) {+ p.inFlight.Add(1)+ defer p.inFlight.Add(-1)++ if err := p.storage.Write(ctx, j.Key, j.Body); err != nil {+ p.failed.Add(1)+ traceway.CaptureException(traceway.NewStackTraceErrorf("failed to write session recording (key=%s): %w", j.Key, err))+ return+ }++ p.uploaded.Add(1)++ rec := models.SessionRecording{+ Id: j.Id,+ ProjectId: j.ProjectId,+ ExceptionId: j.ExceptionId,+ SessionId: j.SessionId,+ SegmentIndex: j.SegmentIndex,+ FilePath: j.Key,+ RecordedAt: j.RecordedAt,+ }+ select {+ case p.inserts <- rec:+ case <-ctx.Done():+ }+}++func (p *pool) batcher(ctx context.Context) {+ defer traceway.Recover()++ batch := make([]models.SessionRecording, 0, insertBatchSize)+ timer := time.NewTimer(insertFlushInterval)+ if !timer.Stop() {+ <-timer.C+ }+ timerActive := false++ flush := func() {+ if len(batch) == 0 {+ return+ }+ if err := repositories.SessionRecordingRepository.InsertAsync(ctx, batch); err != nil {+ p.failed.Add(uint64(len(batch)))+ traceway.CaptureException(traceway.NewStackTraceErrorf("failed to insert batch of %d session recording rows: %w", len(batch), err))+ }+ batch = batch[:0]+ if timerActive {+ if !timer.Stop() {+ select {+ case <-timer.C:+ default:+ }+ }+ timerActive = false+ }+ }++ for {+ select {+ case <-ctx.Done():+ flush()+ return+ case rec, ok := <-p.inserts:+ if !ok {+ flush()+ return+ }+ batch = append(batch, rec)+ if !timerActive {+ timer.Reset(insertFlushInterval)+ timerActive = true+ }+ if len(batch) >= insertBatchSize {+ flush()+ }+ case <-timer.C:+ timerActive = false+ flush()+ }+ }+}++func (p *pool) metricsLoop(ctx context.Context) {+ defer traceway.Recover()++ ticker := time.NewTicker(metricsTickInterval)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ monitoring.RecordRecordingUploader(+ len(p.jobs),+ int(p.inFlight.Load()),+ p.uploaded.Load(),+ p.dropped.Load(),+ p.failed.Load(),+ )+ }+ }+}+diff --git a/backend/app/recordings/uploader_test.go b/backend/app/recordings/uploader_test.gonew file mode 100644index 00000000..2b6b7ad9--- /dev/null+++ b/backend/app/recordings/uploader_test.go@@ -0,0 +1,128 @@+package recordings++import (+ "context"+ "sync/atomic"+ "testing"+ "time"++ "github.com/google/uuid"+ "github.com/tracewayapp/traceway/backend/app/models"+)++type stubStorage struct {+ delay time.Duration+ written atomic.Uint64+}++func (s *stubStorage) Write(ctx context.Context, key string, data []byte) error {+ if s.delay > 0 {+ select {+ case <-time.After(s.delay):+ case <-ctx.Done():+ return ctx.Err()+ }+ }+ s.written.Add(1)+ return nil+}++func makeJob() Job {+ return Job{+ Id: uuid.New(),+ ProjectId: uuid.New(),+ Key: "recordings/test/" + uuid.NewString() + ".json",+ Body: []byte("{}"),+ RecordedAt: time.Now().UTC(),+ SegmentIndex: 0,+ }+}++// TestEnqueue_NeverBlocks_AndDropsAccountedFor exercises the backpressure+// path: the channel must never block enqueue, and uploaded+dropped+failed+// must always equal the number of submitted jobs. The batcher is wired up+// to the real repository singleton so this test only runs as far as the+// successful storage write — it discards the inserts channel via cancel.+func TestEnqueue_NeverBlocks_AndDropsAccountedFor(t *testing.T) {+ storage := &stubStorage{delay: 50 * time.Millisecond}++ const workers = 2+ const queue = 4+ const total = 100++ p := &pool{+ workers: workers,+ jobs: make(chan Job, queue),+ inserts: make(chan models.SessionRecording, insertBatchSize),+ storage: storage,+ }+ ctx, cancel := context.WithCancel(context.Background())+ for i := 0; i < workers; i++ {+ go p.worker(ctx)+ }+ go func() {+ for {+ select {+ case <-ctx.Done():+ return+ case <-p.inserts:+ }+ }+ }()+ defer cancel()++ start := time.Now()+ for i := 0; i < total; i++ {+ p.enqueue(makeJob())+ }+ enqueueElapsed := time.Since(start)++ if enqueueElapsed > 50*time.Millisecond {+ t.Fatalf("Enqueue blocked: %d calls took %v (expected < 50ms)", total, enqueueElapsed)+ }++ deadline := time.Now().Add(2 * time.Second)+ for time.Now().Before(deadline) {+ if p.uploaded.Load()+p.dropped.Load() >= total {+ break+ }+ time.Sleep(20 * time.Millisecond)+ }++ uploaded := p.uploaded.Load()+ dropped := p.dropped.Load()+ failed := p.failed.Load()++ if uploaded+dropped+failed != total {+ t.Fatalf("expected uploaded+dropped+failed == %d, got uploaded=%d dropped=%d failed=%d",+ total, uploaded, dropped, failed)+ }+ if dropped == 0 {+ t.Fatalf("expected drops with workers=%d queue=%d under slow storage, got dropped=0", workers, queue)+ }+ if uploaded == 0 {+ t.Fatalf("expected at least some uploads to succeed, got uploaded=0")+ }++ if p.inFlight.Load() != 0 {+ deadline = time.Now().Add(time.Second)+ for time.Now().Before(deadline) {+ if p.inFlight.Load() == 0 {+ break+ }+ time.Sleep(20 * time.Millisecond)+ }+ }+ if p.inFlight.Load() != 0 {+ t.Errorf("expected inFlight=0 after drain, got %d", p.inFlight.Load())+ }+}++func TestEnqueue_NoSingleton_NoOps(t *testing.T) {+ saved := singleton+ singleton = nil+ defer func() { singleton = saved }()++ Enqueue(makeJob())+}+diff --git a/backend/cmd/run.go b/backend/cmd/run.goindex c2061107..039ab8ac 100644--- a/backend/cmd/run.go+++ b/backend/cmd/run.go@@ -11,6 +11,7 @@ import ("github.com/tracewayapp/traceway/backend/app/models""github.com/tracewayapp/traceway/backend/app/monitoring""github.com/tracewayapp/traceway/backend/app/notifications"+ "github.com/tracewayapp/traceway/backend/app/recordings""github.com/tracewayapp/traceway/backend/app/retention""github.com/tracewayapp/traceway/backend/app/services""github.com/tracewayapp/traceway/backend/app/storage"@@ -122,6 +123,7 @@ func Run(opts ...Option) {notifications.StartEvaluator(ctx)retention.Start(ctx)+ recordings.Start(ctx)var router *gin.Engineif o != nil && o.disableLogging {