Skip to content

The only tool you need to know what is happening and how to fix it.

progress

dduxc2bd5a86449a
diff --git a/CLAUDE.md b/CLAUDE.md
index 1180cfbe..801cb727 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -248,6 +248,10 @@ POSTGRES_SSLMODE=disable
# Retention (see "Data Retention" section below)
SQLITE_RETENTION_DAYS=30 # 0 to disable; only applies in SQLite mode
SESSION_RECORDING_RETENTION_DAYS=30 # 0 to disable; only applies when STORAGE_TYPE=local
+
+# Session recording uploads (see "Session Recording Uploader" section below)
+SESSION_RECORDING_UPLOAD_WORKERS=32 # 0 to disable uploads entirely
+SESSION_RECORDING_UPLOAD_QUEUE_SIZE=2048
```
---
@@ -762,6 +766,27 @@ Tables it prunes (and the column used):
The DB rows in `session_recordings` are pruned by the SQLite retention worker (above) or by ClickHouse TTL — they are intentionally not coupled to the disk cleanup. Controllers that read recordings already log a non-fatal `traceway.CaptureException` when a referenced file is missing.
+#### Session Recording Uploader
+
+Session recording segments arriving on `/api/report` are not uploaded inline. The handler enqueues each segment onto a bounded worker pool (`backend/app/recordings/uploader.go`, started from `cmd/run.go` next to `retention.Start`). Workers drain the queue and write the body via `storage.Store.Write` (S3 or local disk); successful writes are handed to a single batcher goroutine that calls `SessionRecordingRepository.InsertAsync` once per ~1000 rows or every 2 s, whichever comes first — single-row inserts are an anti-pattern for ClickHouse. Enqueue is non-blocking: when the queue is full the segment is dropped (newest-first) so a burst of `/api/report` traffic cannot spawn unbounded goroutines or saturate S3.
+
+| Variable | Default | Notes |
+|----------|---------|-------|
+| `SESSION_RECORDING_UPLOAD_WORKERS` | `32` | Concurrent uploaders. Set to `0` to drop every segment (uploads disabled). |
+| `SESSION_RECORDING_UPLOAD_QUEUE_SIZE` | `2048` | Max queued segments. Overflow is dropped. |
+
+Observability (emitted every 10s via `traceway.CaptureMetric`):
+
+| Metric | Type | Meaning |
+|--------|------|---------|
+| `traceway.recordings.queue_depth` | gauge | Segments currently waiting in the channel. |
+| `traceway.recordings.in_flight` | gauge | Workers mid-upload. |
+| `traceway.recordings.uploaded` | counter | Successful S3/local + DB writes since startup. |
+| `traceway.recordings.dropped` | counter | Segments dropped on overflow since startup. |
+| `traceway.recordings.failed` | counter | Upload or DB-insert errors since startup. |
+
+Sustained drops also fire a rate-limited (1/min) `traceway.CaptureException` so overload is visible in the issues feed without flooding it.
+
#### Organization Roles
| Role | Description |
|------|-------------|
diff --git a/backend/app/config/config.go b/backend/app/config/config.go
index 049d3e45..2661e1ae 100644
--- a/backend/app/config/config.go
+++ b/backend/app/config/config.go
@@ -28,8 +28,10 @@ type Cfg struct {
S3SecretKey string
S3Endpoint string
- SQLiteRetentionDays string
- SessionRecordingRetentionDays string
+ SQLiteRetentionDays string
+ SessionRecordingRetentionDays string
+ SessionRecordingUploadWorkers string
+ SessionRecordingUploadQueueSize string
SMTPEnabled string
SMTPHost string
@@ -83,8 +85,10 @@ func LoadFromEnv() *Cfg {
S3SecretKey: os.Getenv("S3_SECRET_KEY"),
S3Endpoint: os.Getenv("S3_ENDPOINT"),
- SQLiteRetentionDays: os.Getenv("SQLITE_RETENTION_DAYS"),
- SessionRecordingRetentionDays: os.Getenv("SESSION_RECORDING_RETENTION_DAYS"),
+ SQLiteRetentionDays: os.Getenv("SQLITE_RETENTION_DAYS"),
+ SessionRecordingRetentionDays: os.Getenv("SESSION_RECORDING_RETENTION_DAYS"),
+ SessionRecordingUploadWorkers: os.Getenv("SESSION_RECORDING_UPLOAD_WORKERS"),
+ SessionRecordingUploadQueueSize: os.Getenv("SESSION_RECORDING_UPLOAD_QUEUE_SIZE"),
SMTPEnabled: os.Getenv("SMTP_ENABLED"),
SMTPHost: os.Getenv("SMTP_HOST"),
diff --git a/backend/app/controllers/clientcontrollers/client.controller.go b/backend/app/controllers/clientcontrollers/client.controller.go
index ebac5896..e66ee6fe 100644
--- a/backend/app/controllers/clientcontrollers/client.controller.go
+++ b/backend/app/controllers/clientcontrollers/client.controller.go
@@ -2,7 +2,6 @@ package clientcontrollers
import (
"bytes"
- "context"
"crypto/sha256"
"database/sql"
"encoding/hex"
@@ -19,9 +18,9 @@ import (
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/models/clientmodels"
"github.com/tracewayapp/traceway/backend/app/monitoring"
+ "github.com/tracewayapp/traceway/backend/app/recordings"
"github.com/tracewayapp/traceway/backend/app/repositories"
"github.com/tracewayapp/traceway/backend/app/services"
- "github.com/tracewayapp/traceway/backend/app/storage"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
@@ -84,22 +83,7 @@ func (e clientController) Report(c *gin.Context) {
spansToInsert := []models.Span{}
sessionsToUpsert := []models.Session{}
- type recordingWork struct {
- Id uuid.UUID
- ProjectId uuid.UUID
- ExceptionId uuid.UUID
- SessionId *uuid.UUID
- SegmentIndex int32
- // Key is the S3 object key the body is written to.
- Key string
- // Body is the marshaled JSON of the entire ClientSessionRecording
- // sub-document — events + logs + actions + startedAt/endedAt — exactly
- // as it lands in S3. App console logs in `logs` are intentionally not
- // inserted into the OTel logs ClickHouse table; they live only here.
- Body []byte
- RecordedAt time.Time
- }
- var recordingsWork []recordingWork
+ var recordingsWork []recordings.Job
// Map frontend sessionRecordingId → backend-generated exception UUID.
// Used only for the legacy exception-bound recording path; the always-on
@@ -234,7 +218,7 @@ func (e clientController) Report(c *gin.Context) {
} else {
key = fmt.Sprintf("recordings/%s/%s.json", projectId, exceptionId)
}
- recordingsWork = append(recordingsWork, recordingWork{
+ recordingsWork = append(recordingsWork, recordings.Job{
Id: uuid.New(),
ProjectId: projectId,
ExceptionId: exceptionId,
@@ -335,31 +319,8 @@ func (e clientController) Report(c *gin.Context) {
}
}
- if len(recordingsWork) > 0 {
- work := recordingsWork
- go func() {
- var successful []models.SessionRecording
- for _, rw := range work {
- if err := storage.Store.Write(context.Background(), rw.Key, rw.Body); err != nil {
- traceway.CaptureException(traceway.NewStackTraceErrorf("failed to write session recording (key=%s): %w", rw.Key, err))
- continue
- }
- successful = append(successful, models.SessionRecording{
- Id: rw.Id,
- ProjectId: rw.ProjectId,
- ExceptionId: rw.ExceptionId,
- SessionId: rw.SessionId,
- SegmentIndex: rw.SegmentIndex,
- FilePath: rw.Key,
- RecordedAt: rw.RecordedAt,
- })
- }
- if len(successful) > 0 {
- if err := repositories.SessionRecordingRepository.InsertAsync(context.Background(), successful); err != nil {
- traceway.CaptureException(traceway.NewStackTraceErrorf("failed to batch insert %d session recording refs: %w", len(successful), err))
- }
- }
- }()
+ for _, rw := range recordingsWork {
+ recordings.Enqueue(rw)
}
c.JSON(http.StatusOK, gin.H{})
diff --git a/backend/app/monitoring/recordings.go b/backend/app/monitoring/recordings.go
new file mode 100644
index 00000000..72c16f26
--- /dev/null
+++ b/backend/app/monitoring/recordings.go
@@ -0,0 +1,11 @@
+package monitoring
+
+import traceway "go.tracewayapp.com"
+
+func RecordRecordingUploader(queueDepth, inFlight int, uploaded, dropped, failed uint64) {
+ traceway.CaptureMetric("traceway.recordings.queue_depth", float64(queueDepth))
+ traceway.CaptureMetric("traceway.recordings.in_flight", float64(inFlight))
+ traceway.CaptureMetric("traceway.recordings.uploaded", float64(uploaded))
+ traceway.CaptureMetric("traceway.recordings.dropped", float64(dropped))
+ traceway.CaptureMetric("traceway.recordings.failed", float64(failed))
+}
diff --git a/backend/app/recordings/uploader.go b/backend/app/recordings/uploader.go
new file mode 100644
index 00000000..b907e53a
--- /dev/null
+++ b/backend/app/recordings/uploader.go
@@ -0,0 +1,257 @@
+package recordings
+
+import (
+ "context"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/tracewayapp/traceway/backend/app/config"
+ "github.com/tracewayapp/traceway/backend/app/models"
+ "github.com/tracewayapp/traceway/backend/app/monitoring"
+ "github.com/tracewayapp/traceway/backend/app/repositories"
+ "github.com/tracewayapp/traceway/backend/app/storage"
+ traceway "go.tracewayapp.com"
+)
+
+const (
+ defaultWorkers = 32
+ defaultQueueSize = 2048
+ metricsTickInterval = 10 * time.Second
+ dropReportInterval = time.Minute
+ insertBatchSize = 1000
+ insertFlushInterval = 2 * time.Second
+)
+
+type Job struct {
+ Id uuid.UUID
+ ProjectId uuid.UUID
+ ExceptionId uuid.UUID
+ SessionId *uuid.UUID
+ SegmentIndex int32
+ Key string
+ Body []byte
+ RecordedAt time.Time
+}
+
+type storer interface {
+ Write(ctx context.Context, key string, data []byte) error
+}
+
+type pool struct {
+ workers int
+ jobs chan Job
+ inserts chan models.SessionRecording
+
+ storage storer
+
+ inFlight atomic.Int64
+ uploaded atomic.Uint64
+ dropped atomic.Uint64
+ failed atomic.Uint64
+
+ dropMu sync.Mutex
+ lastDropReportAt time.Time
+ droppedSinceReport uint64
+}
+
+var singleton *pool
+
+func Start(ctx context.Context) {
+ if singleton != nil {
+ return
+ }
+
+ workers := defaultWorkers
+ workersStr := strings.TrimSpace(config.Config.SessionRecordingUploadWorkers)
+ if workersStr != "" {
+ if v, err := strconv.Atoi(workersStr); err == nil && v >= 0 {
+ workers = v
+ }
+ }
+
+ queueSize := defaultQueueSize
+ queueStr := strings.TrimSpace(config.Config.SessionRecordingUploadQueueSize)
+ if queueStr != "" {
+ if v, err := strconv.Atoi(queueStr); err == nil && v >= 1 {
+ queueSize = v
+ }
+ }
+
+ singleton = &pool{
+ workers: workers,
+ jobs: make(chan Job, queueSize),
+ inserts: make(chan models.SessionRecording, insertBatchSize),
+ storage: storage.Store,
+ }
+ singleton.start(ctx)
+
+ if workers == 0 {
+ config.Logln("Session recording uploader disabled (SESSION_RECORDING_UPLOAD_WORKERS=0); segments will be dropped")
+ } else {
+ config.Logf("Started session recording uploader (workers=%d, queue=%d)", workers, queueSize)
+ }
+}
+
+func Enqueue(j Job) {
+ if singleton == nil {
+ return
+ }
+ singleton.enqueue(j)
+}
+
+func (p *pool) start(ctx context.Context) {
+ for i := 0; i < p.workers; i++ {
+ go p.worker(ctx)
+ }
+ go p.metricsLoop(ctx)
+ go p.batcher(ctx)
+}
+
+func (p *pool) enqueue(j Job) {
+ select {
+ case p.jobs <- j:
+ return
+ default:
+ }
+
+ p.dropped.Add(1)
+
+ var report uint64
+ p.dropMu.Lock()
+ p.droppedSinceReport++
+ if time.Since(p.lastDropReportAt) >= dropReportInterval {
+ report = p.droppedSinceReport
+ p.droppedSinceReport = 0
+ p.lastDropReportAt = time.Now()
+ }
+ p.dropMu.Unlock()
+
+ if report > 0 {
+ traceway.CaptureException(traceway.NewStackTraceErrorf(
+ "session recording uploader dropped %d segments due to full queue (cap=%d)", report, cap(p.jobs)))
+ }
+}
+
+func (p *pool) worker(ctx context.Context) {
+ defer traceway.Recover()
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case j, ok := <-p.jobs:
+ if !ok {
+ return
+ }
+ p.handle(ctx, j)
+ }
+ }
+}
+
+func (p *pool) handle(ctx context.Context, j Job) {
+ p.inFlight.Add(1)
+ defer p.inFlight.Add(-1)
+
+ if err := p.storage.Write(ctx, j.Key, j.Body); err != nil {
+ p.failed.Add(1)
+ traceway.CaptureException(traceway.NewStackTraceErrorf("failed to write session recording (key=%s): %w", j.Key, err))
+ return
+ }
+
+ p.uploaded.Add(1)
+
+ rec := models.SessionRecording{
+ Id: j.Id,
+ ProjectId: j.ProjectId,
+ ExceptionId: j.ExceptionId,
+ SessionId: j.SessionId,
+ SegmentIndex: j.SegmentIndex,
+ FilePath: j.Key,
+ RecordedAt: j.RecordedAt,
+ }
+ select {
+ case p.inserts <- rec:
+ case <-ctx.Done():
+ }
+}
+
+func (p *pool) batcher(ctx context.Context) {
+ defer traceway.Recover()
+
+ batch := make([]models.SessionRecording, 0, insertBatchSize)
+ timer := time.NewTimer(insertFlushInterval)
+ if !timer.Stop() {
+ <-timer.C
+ }
+ timerActive := false
+
+ flush := func() {
+ if len(batch) == 0 {
+ return
+ }
+ if err := repositories.SessionRecordingRepository.InsertAsync(ctx, batch); err != nil {
+ p.failed.Add(uint64(len(batch)))
+ traceway.CaptureException(traceway.NewStackTraceErrorf("failed to insert batch of %d session recording rows: %w", len(batch), err))
+ }
+ batch = batch[:0]
+ if timerActive {
+ if !timer.Stop() {
+ select {
+ case <-timer.C:
+ default:
+ }
+ }
+ timerActive = false
+ }
+ }
+
+ for {
+ select {
+ case <-ctx.Done():
+ flush()
+ return
+ case rec, ok := <-p.inserts:
+ if !ok {
+ flush()
+ return
+ }
+ batch = append(batch, rec)
+ if !timerActive {
+ timer.Reset(insertFlushInterval)
+ timerActive = true
+ }
+ if len(batch) >= insertBatchSize {
+ flush()
+ }
+ case <-timer.C:
+ timerActive = false
+ flush()
+ }
+ }
+}
+
+func (p *pool) metricsLoop(ctx context.Context) {
+ defer traceway.Recover()
+
+ ticker := time.NewTicker(metricsTickInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ monitoring.RecordRecordingUploader(
+ len(p.jobs),
+ int(p.inFlight.Load()),
+ p.uploaded.Load(),
+ p.dropped.Load(),
+ p.failed.Load(),
+ )
+ }
+ }
+}
+
diff --git a/backend/app/recordings/uploader_test.go b/backend/app/recordings/uploader_test.go
new file mode 100644
index 00000000..2b6b7ad9
--- /dev/null
+++ b/backend/app/recordings/uploader_test.go
@@ -0,0 +1,128 @@
+package recordings
+
+import (
+ "context"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/tracewayapp/traceway/backend/app/models"
+)
+
+type stubStorage struct {
+ delay time.Duration
+ written atomic.Uint64
+}
+
+func (s *stubStorage) Write(ctx context.Context, key string, data []byte) error {
+ if s.delay > 0 {
+ select {
+ case <-time.After(s.delay):
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+ s.written.Add(1)
+ return nil
+}
+
+func makeJob() Job {
+ return Job{
+ Id: uuid.New(),
+ ProjectId: uuid.New(),
+ Key: "recordings/test/" + uuid.NewString() + ".json",
+ Body: []byte("{}"),
+ RecordedAt: time.Now().UTC(),
+ SegmentIndex: 0,
+ }
+}
+
+// TestEnqueue_NeverBlocks_AndDropsAccountedFor exercises the backpressure
+// path: the channel must never block enqueue, and uploaded+dropped+failed
+// must always equal the number of submitted jobs. The batcher is wired up
+// to the real repository singleton so this test only runs as far as the
+// successful storage write — it discards the inserts channel via cancel.
+func TestEnqueue_NeverBlocks_AndDropsAccountedFor(t *testing.T) {
+ storage := &stubStorage{delay: 50 * time.Millisecond}
+
+ const workers = 2
+ const queue = 4
+ const total = 100
+
+ p := &pool{
+ workers: workers,
+ jobs: make(chan Job, queue),
+ inserts: make(chan models.SessionRecording, insertBatchSize),
+ storage: storage,
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ for i := 0; i < workers; i++ {
+ go p.worker(ctx)
+ }
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-p.inserts:
+ }
+ }
+ }()
+ defer cancel()
+
+ start := time.Now()
+ for i := 0; i < total; i++ {
+ p.enqueue(makeJob())
+ }
+ enqueueElapsed := time.Since(start)
+
+ if enqueueElapsed > 50*time.Millisecond {
+ t.Fatalf("Enqueue blocked: %d calls took %v (expected < 50ms)", total, enqueueElapsed)
+ }
+
+ deadline := time.Now().Add(2 * time.Second)
+ for time.Now().Before(deadline) {
+ if p.uploaded.Load()+p.dropped.Load() >= total {
+ break
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+
+ uploaded := p.uploaded.Load()
+ dropped := p.dropped.Load()
+ failed := p.failed.Load()
+
+ if uploaded+dropped+failed != total {
+ t.Fatalf("expected uploaded+dropped+failed == %d, got uploaded=%d dropped=%d failed=%d",
+ total, uploaded, dropped, failed)
+ }
+ if dropped == 0 {
+ t.Fatalf("expected drops with workers=%d queue=%d under slow storage, got dropped=0", workers, queue)
+ }
+ if uploaded == 0 {
+ t.Fatalf("expected at least some uploads to succeed, got uploaded=0")
+ }
+
+ if p.inFlight.Load() != 0 {
+ deadline = time.Now().Add(time.Second)
+ for time.Now().Before(deadline) {
+ if p.inFlight.Load() == 0 {
+ break
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ }
+ if p.inFlight.Load() != 0 {
+ t.Errorf("expected inFlight=0 after drain, got %d", p.inFlight.Load())
+ }
+}
+
+func TestEnqueue_NoSingleton_NoOps(t *testing.T) {
+ saved := singleton
+ singleton = nil
+ defer func() { singleton = saved }()
+
+ Enqueue(makeJob())
+}
+
diff --git a/backend/cmd/run.go b/backend/cmd/run.go
index c2061107..039ab8ac 100644
--- a/backend/cmd/run.go
+++ b/backend/cmd/run.go
@@ -11,6 +11,7 @@ import (
"github.com/tracewayapp/traceway/backend/app/models"
"github.com/tracewayapp/traceway/backend/app/monitoring"
"github.com/tracewayapp/traceway/backend/app/notifications"
+ "github.com/tracewayapp/traceway/backend/app/recordings"
"github.com/tracewayapp/traceway/backend/app/retention"
"github.com/tracewayapp/traceway/backend/app/services"
"github.com/tracewayapp/traceway/backend/app/storage"
@@ -122,6 +123,7 @@ func Run(opts ...Option) {
notifications.StartEvaluator(ctx)
retention.Start(ctx)
+ recordings.Start(ctx)
var router *gin.Engine
if o != nil && o.disableLogging {