The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
feat: fixed ch inserts for retryable errors (#228)
narrow-adventurea2a01081588d
diff --git a/backend/app/chdb/batch_retry.go b/backend/app/chdb/batch_retry.gonew file mode 100644index 00000000..fd0380c1--- /dev/null+++ b/backend/app/chdb/batch_retry.go@@ -0,0 +1,74 @@+//go:build pgch++package chdb++import (+ "errors"+ "io"+ "net"+ "strings"+ "syscall"+ "time"++ "github.com/ClickHouse/clickhouse-go/v2"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"+)++const batchSendMaxAttempts = 3++func SendBatch(query string, fill func(driver.Batch) error) error {+ var batch driver.Batch+ if err := RetryOnConn(batchSendMaxAttempts, IsConnError, func() error {+ b, err := Conn.PrepareBatch(BatchCtx(), query)+ batch = b+ return err+ }); err != nil {+ return err+ }+ if err := fill(batch); err != nil {+ batch.Abort()+ return err+ }+ return batch.Send()+}++func RetryOnConn(maxAttempts int, retryable func(error) bool, fn func() error) error {+ for attempt := 1; ; attempt++ {+ err := fn()+ if err == nil || attempt >= maxAttempts || !retryable(err) {+ return err+ }+ time.Sleep(time.Duration(attempt) * 50 * time.Millisecond)+ }+}++func IsConnError(err error) bool {+ if err == nil {+ return false+ }+ var chErr *clickhouse.Exception+ if errors.As(err, &chErr) {+ return false+ }+ if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) ||+ errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.EPIPE) ||+ errors.Is(err, net.ErrClosed) {+ return true+ }+ msg := strings.ToLower(err.Error())+ for _, s := range []string{+ "eof",+ "broken pipe",+ "connection reset",+ "reset by peer",+ "connection refused",+ "no route to host",+ "use of closed",+ "first block packet",+ } {+ if strings.Contains(msg, s) {+ return true+ }+ }+ return false+}diff --git a/backend/app/chdb/batch_retry_test.go b/backend/app/chdb/batch_retry_test.gonew file mode 100644index 00000000..b69e6d1d--- /dev/null+++ b/backend/app/chdb/batch_retry_test.go@@ -0,0 +1,206 @@+//go:build pgch++package chdb++import (+ "context"+ "errors"+ "io"+ "testing"++ "github.com/ClickHouse/clickhouse-go/v2"+ "github.com/ClickHouse/clickhouse-go/v2/lib/column"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"+)++type fakeBatch struct {+ sendErr error+ sent bool+ aborted bool+}++func (b *fakeBatch) Abort() error { b.aborted = true; return nil }+func (b *fakeBatch) Append(v ...any) error { return nil }+func (b *fakeBatch) AppendStruct(v any) error { return nil }+func (b *fakeBatch) Column(int) driver.BatchColumn { return nil }+func (b *fakeBatch) Flush() error { return nil }+func (b *fakeBatch) Send() error { b.sent = true; return b.sendErr }+func (b *fakeBatch) IsSent() bool { return b.sent }+func (b *fakeBatch) Rows() int { return 0 }+func (b *fakeBatch) Columns() []column.Interface { return nil }+func (b *fakeBatch) Close() error { return nil }++type fakeConn struct {+ prepareErrs []error+ sendErr error+ calls int+ lastBatch *fakeBatch+}++func (c *fakeConn) Query(ctx context.Context, query string, args ...any) (driver.Rows, error) {+ return nil, nil+}+func (c *fakeConn) QueryRow(ctx context.Context, query string, args ...any) driver.Row { return nil }+func (c *fakeConn) Exec(ctx context.Context, query string, args ...any) error { return nil }+func (c *fakeConn) Stats() driver.Stats { return driver.Stats{} }+func (c *fakeConn) PrepareBatch(ctx context.Context, query string, opts ...driver.PrepareBatchOption) (driver.Batch, error) {+ i := c.calls+ c.calls+++ var e error+ if i < len(c.prepareErrs) {+ e = c.prepareErrs[i]+ }+ if e != nil {+ return nil, e+ }+ b := &fakeBatch{sendErr: c.sendErr}+ c.lastBatch = b+ return b, nil+}++func TestSendBatchRecoversFromStalePrepareEOF(t *testing.T) {+ orig := Conn+ defer func() { Conn = orig }()++ eof := errors.New("query processing: failed to read first block packet from 1.2.3.4:9440 (conn_id=1): read: EOF")+ fc := &fakeConn{prepareErrs: []error{eof, eof, nil}}+ Conn = fc++ fills := 0+ err := SendBatch("INSERT INTO x", func(b driver.Batch) error {+ fills+++ return b.Append(1)+ })+ if err != nil {+ t.Fatalf("expected success after prepare retries, got %v", err)+ }+ if fc.calls != 3 {+ t.Fatalf("expected 3 prepare attempts, got %d", fc.calls)+ }+ if fills != 1 {+ t.Fatalf("expected fill invoked once (after the successful prepare), got %d", fills)+ }+}++func TestSendBatchDoesNotRetrySendFailure(t *testing.T) {+ orig := Conn+ defer func() { Conn = orig }()++ sendEOF := errors.New("query processing: failed to read packet from 1.2.3.4:9440 (conn_id=1): read: EOF")+ fc := &fakeConn{prepareErrs: []error{nil}, sendErr: sendEOF}+ Conn = fc++ err := SendBatch("INSERT INTO x", func(b driver.Batch) error { return nil })+ if !errors.Is(err, sendEOF) {+ t.Fatalf("expected the send error returned without retry, got %v", err)+ }+ if fc.calls != 1 {+ t.Fatalf("send failures must not be retried (would risk duplicate inserts); got %d prepare calls", fc.calls)+ }+}++func TestSendBatchDoesNotRetryServerException(t *testing.T) {+ orig := Conn+ defer func() { Conn = orig }()++ boom := &clickhouse.Exception{Code: 27, Message: "Cannot parse input: unexpected EOF"}+ fc := &fakeConn{prepareErrs: []error{boom, nil}}+ Conn = fc++ err := SendBatch("INSERT INTO x", func(b driver.Batch) error { return nil })+ if !errors.Is(err, boom) {+ t.Fatalf("a server exception must not be retried even if its message contains EOF, got %v", err)+ }+ if fc.calls != 1 {+ t.Fatalf("expected no retry on server exception, got %d attempts", fc.calls)+ }+}++func TestSendBatchExhaustsPrepareAttempts(t *testing.T) {+ orig := Conn+ defer func() { Conn = orig }()++ eof := errors.New("read: EOF")+ fc := &fakeConn{prepareErrs: []error{eof, eof, eof, nil}}+ Conn = fc++ err := SendBatch("INSERT INTO x", func(b driver.Batch) error { return nil })+ if err == nil {+ t.Fatal("expected error after exhausting attempts")+ }+ if fc.calls != batchSendMaxAttempts {+ t.Fatalf("expected %d attempts, got %d", batchSendMaxAttempts, fc.calls)+ }+}++func TestSendBatchAbortsOnFillError(t *testing.T) {+ orig := Conn+ defer func() { Conn = orig }()++ fc := &fakeConn{prepareErrs: []error{nil}}+ Conn = fc++ fillErr := errors.New("bad row")+ err := SendBatch("INSERT INTO x", func(b driver.Batch) error { return fillErr })+ if !errors.Is(err, fillErr) {+ t.Fatalf("expected fill error back, got %v", err)+ }+ if fc.calls != 1 {+ t.Fatalf("fill error must not retry, got %d attempts", fc.calls)+ }+ if fc.lastBatch == nil || !fc.lastBatch.aborted {+ t.Fatal("expected the batch to be aborted on fill error")+ }+}++func TestRetryOnConn(t *testing.T) {+ boom := errors.New("boom")++ calls := 0+ err := RetryOnConn(3, func(error) bool { return true }, func() error { calls++; return boom })+ if !errors.Is(err, boom) || calls != 3 {+ t.Fatalf("expected last error after 3 attempts, got err=%v calls=%d", err, calls)+ }++ calls = 0+ err = RetryOnConn(3, func(error) bool { return false }, func() error { calls++; return boom })+ if !errors.Is(err, boom) || calls != 1 {+ t.Fatalf("non-retryable error must not retry, got err=%v calls=%d", err, calls)+ }++ calls = 0+ err = RetryOnConn(3, func(error) bool { return true }, func() error {+ calls+++ if calls < 2 {+ return boom+ }+ return nil+ })+ if err != nil || calls != 2 {+ t.Fatalf("expected success on second attempt, got err=%v calls=%d", err, calls)+ }+}++func TestIsConnError(t *testing.T) {+ cases := []struct {+ err error+ want bool+ }{+ {nil, false},+ {io.EOF, true},+ {io.ErrUnexpectedEOF, true},+ {errors.New("failed to read first block packet from x: read: EOF"), true},+ {errors.New("write: broken pipe"), true},+ {errors.New("read: connection reset by peer"), true},+ {errors.New("dial tcp 1.2.3.4:9440: connect: connection refused"), true},+ {errors.New("dial tcp: no route to host"), true},+ {errors.New("use of closed network connection"), true},+ {&clickhouse.Exception{Code: 62, Message: "syntax error near EOF"}, false},+ {errors.New("code: 241, memory limit exceeded"), false},+ }+ for _, tc := range cases {+ if got := IsConnError(tc.err); got != tc.want {+ t.Errorf("IsConnError(%v) = %v, want %v", tc.err, got, tc.want)+ }+ }+}diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.goindex d7bd01d6..b0c28311 100644--- a/backend/app/monitoring/ch_reporter.go+++ b/backend/app/monitoring/ch_reporter.go@@ -6,7 +6,6 @@ import ("context""errors""fmt"- "io""net""strings""sync"@@ -49,31 +48,21 @@ func transientCHError(err error) bool {if err == nil {return false}- if errors.Is(err, io.EOF) ||- errors.Is(err, net.ErrClosed) ||- errors.Is(err, context.DeadlineExceeded) ||- errors.Is(err, context.Canceled) {+ if chdb.IsConnError(err) {+ return true+ }+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {return true}var netErr net.Errorif errors.As(err, &netErr) && netErr.Timeout() {return true}- msg := strings.ToLower(err.Error())- for _, s := range []string{"eof", "connection reset", "broken pipe", "use of closed", "connection refused", "reset by peer", "i/o timeout", "no route to host"} {- if strings.Contains(msg, s) {- return true- }- }- return false+ return strings.Contains(strings.ToLower(err.Error()), "i/o timeout")}func withRetry(fn func() error) error {- err := fn()- if err != nil && transientCHError(err) {- return fn()- }- return err+ return chdb.RetryOnConn(2, transientCHError, fn)}func captureMonitoringErr(err error) {diff --git a/backend/app/repositories/endpoint.repository.go b/backend/app/repositories/endpoint.repository.goindex 37fdc9db..af3486f0 100644--- a/backend/app/repositories/endpoint.repository.go+++ b/backend/app/repositories/endpoint.repository.go@@ -11,36 +11,38 @@ import ("errors""time"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver""github.com/google/uuid")type endpointRepository struct{}func (e *endpointRepository) InsertAsync(ctx context.Context, lines []models.Endpoint) error {- batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO endpoints (id, project_id, endpoint, duration, recorded_at, status_code, body_size, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_stream, is_root)")- if err != nil {- return err- }- for _, t := range lines {- attributesJSON := "{}"- if len(t.Attributes) != 0 {- if attributesBytes, err := json.Marshal(t.Attributes); err == nil {- attributesJSON = string(attributesBytes)+ if len(lines) == 0 {+ return nil+ }+ return chdb.SendBatch("INSERT INTO endpoints (id, project_id, endpoint, duration, recorded_at, status_code, body_size, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_stream, is_root)", func(batch driver.Batch) error {+ for _, t := range lines {+ attributesJSON := "{}"+ if len(t.Attributes) != 0 {+ if attributesBytes, err := json.Marshal(t.Attributes); err == nil {+ attributesJSON = string(attributesBytes)+ }+ }+ isStream := uint8(0)+ if t.IsStream {+ isStream = 1+ }+ isRoot := uint8(0)+ if t.IsRoot {+ isRoot = 1+ }+ if err := batch.Append(t.Id, t.ProjectId, t.Endpoint, int64(t.Duration), t.RecordedAt, t.StatusCode, t.BodySize, t.ClientIP, attributesJSON, t.AppVersion, t.ServerName, t.DistributedTraceId, t.SpanId, isStream, isRoot); err != nil {+ return err}}- isStream := uint8(0)- if t.IsStream {- isStream = 1- }- isRoot := uint8(0)- if t.IsRoot {- isRoot = 1- }- if err := batch.Append(t.Id, t.ProjectId, t.Endpoint, int64(t.Duration), t.RecordedAt, t.StatusCode, t.BodySize, t.ClientIP, attributesJSON, t.AppVersion, t.ServerName, t.DistributedTraceId, t.SpanId, isStream, isRoot); err != nil {- return err- }- }- return batch.Send()+ return nil+ })}func (e *endpointRepository) CountBetween(ctx context.Context, projectId uuid.UUID, start, end time.Time) (int64, error) {diff --git a/backend/app/repositories/exception_stack_trace.repository.go b/backend/app/repositories/exception_stack_trace.repository.goindex 06919c79..e7507aa3 100644--- a/backend/app/repositories/exception_stack_trace.repository.go+++ b/backend/app/repositories/exception_stack_trace.repository.go@@ -12,36 +12,38 @@ import ("strings""time"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver""github.com/google/uuid")type exceptionStackTraceRepository struct{}func (e *exceptionStackTraceRepository) InsertAsync(ctx context.Context, lines []models.ExceptionStackTrace) error {- batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id)")- if err != nil {- return err+ if len(lines) == 0 {+ return nil}- for _, est := range lines {- attributesJSON := "{}"- if len(est.Attributes) != 0 {- if attributesBytes, err := json.Marshal(est.Attributes); err == nil {- attributesJSON = string(attributesBytes)+ return chdb.SendBatch("INSERT INTO exception_stack_traces (id, project_id, trace_id, trace_type, exception_hash, stack_trace, recorded_at, attributes, app_version, server_name, is_message, distributed_trace_id, session_id)", func(batch driver.Batch) error {+ for _, est := range lines {+ attributesJSON := "{}"+ if len(est.Attributes) != 0 {+ if attributesBytes, err := json.Marshal(est.Attributes); err == nil {+ attributesJSON = string(attributesBytes)+ }+ }+ isMessage := uint8(0)+ if est.IsMessage {+ isMessage = 1+ }+ traceType := est.TraceType+ if traceType == "" {+ traceType = "endpoint"+ }+ if err := batch.Append(est.Id, est.ProjectId, est.TraceId, traceType, est.ExceptionHash, est.StackTrace, est.RecordedAt, attributesJSON, est.AppVersion, est.ServerName, isMessage, est.DistributedTraceId, est.SessionId); err != nil {+ return err}}- isMessage := uint8(0)- if est.IsMessage {- isMessage = 1- }- traceType := est.TraceType- if traceType == "" {- traceType = "endpoint"- }- if err := batch.Append(est.Id, est.ProjectId, est.TraceId, traceType, est.ExceptionHash, est.StackTrace, est.RecordedAt, attributesJSON, est.AppVersion, est.ServerName, isMessage, est.DistributedTraceId, est.SessionId); err != nil {- return err- }- }- return batch.Send()+ return nil+ })}// FindAllBySessionId returns all exceptions/messages stamped with the given session_id.diff --git a/backend/app/repositories/metric_point.repository.go b/backend/app/repositories/metric_point.repository.goindex 76ca39e9..14f55bc4 100644--- a/backend/app/repositories/metric_point.repository.go+++ b/backend/app/repositories/metric_point.repository.go@@ -8,22 +8,24 @@ import ("context""time"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver""github.com/google/uuid")type metricPointRepository struct{}func (r *metricPointRepository) InsertAsync(ctx context.Context, points []models.MetricPoint) error {- batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO metric_points (project_id, name, value, tags, recorded_at)")- if err != nil {- return err+ if len(points) == 0 {+ return nil}- for _, p := range points {- if err := batch.Append(p.ProjectId, p.Name, p.Value, p.Tags, p.RecordedAt); err != nil {- return err+ return chdb.SendBatch("INSERT INTO metric_points (project_id, name, value, tags, recorded_at)", func(batch driver.Batch) error {+ for _, p := range points {+ if err := batch.Append(p.ProjectId, p.Name, p.Value, p.Tags, p.RecordedAt); err != nil {+ return err+ }}- }- return batch.Send()+ return nil+ })}func (r *metricPointRepository) QueryTimeSeries(ctx context.Context, projectId uuid.UUID, name string, from, to time.Time, intervalMinutes int, aggregation string, tagFilters map[string]string, groupBy string) (map[string][]models.TimeSeriesPoint, error) {diff --git a/backend/app/repositories/session.repository.go b/backend/app/repositories/session.repository.goindex d083ed62..cac713ad 100644--- a/backend/app/repositories/session.repository.go+++ b/backend/app/repositories/session.repository.go@@ -9,6 +9,7 @@ import ("strings""time"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver""github.com/google/uuid""github.com/tracewayapp/traceway/backend/app/chdb""github.com/tracewayapp/traceway/backend/app/models"@@ -20,21 +21,18 @@ func (r *sessionRepository) Upsert(ctx context.Context, sessions []models.Sessioif len(sessions) == 0 {return nil}- batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),- "INSERT INTO sessions (id, project_id, started_at, ended_at, duration, client_ip, attributes, app_version, server_name, distributed_trace_id, version)")- if err != nil {- return err- }- for _, s := range sessions {- attrs := s.Attributes- if attrs == nil {- attrs = map[string]string{}- }- if err := batch.Append(s.Id, s.ProjectId, s.StartedAt, s.EndedAt, s.Duration, s.ClientIP, attrs, s.AppVersion, s.ServerName, s.DistributedTraceId, time.Now()); err != nil {- return err+ return chdb.SendBatch("INSERT INTO sessions (id, project_id, started_at, ended_at, duration, client_ip, attributes, app_version, server_name, distributed_trace_id, version)", func(batch driver.Batch) error {+ for _, s := range sessions {+ attrs := s.Attributes+ if attrs == nil {+ attrs = map[string]string{}+ }+ if err := batch.Append(s.Id, s.ProjectId, s.StartedAt, s.EndedAt, s.Duration, s.ClientIP, attrs, s.AppVersion, s.ServerName, s.DistributedTraceId, time.Now()); err != nil {+ return err+ }}- }- return batch.Send()+ return nil+ })}func (r *sessionRepository) CountBetween(ctx context.Context, projectId uuid.UUID, start, end time.Time) (int64, error) {diff --git a/backend/app/repositories/span.repository.go b/backend/app/repositories/span.repository.goindex 775d01d6..cee72ef5 100644--- a/backend/app/repositories/span.repository.go+++ b/backend/app/repositories/span.repository.go@@ -10,6 +10,7 @@ import ("github.com/tracewayapp/traceway/backend/app/chdb""github.com/tracewayapp/traceway/backend/app/models"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver""github.com/google/uuid")@@ -20,35 +21,30 @@ func (r *spanRepository) InsertAsync(ctx context.Context, spans []models.Span) ereturn nil}- batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(),- "INSERT INTO spans (id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id, attributes)")- if err != nil {- return err- }-- for _, s := range spans {- attributesJSON := "{}"- if len(s.Attributes) != 0 {- if attributesBytes, err := json.Marshal(s.Attributes); err == nil {- attributesJSON = string(attributesBytes)+ return chdb.SendBatch("INSERT INTO spans (id, trace_id, project_id, name, start_time, duration, recorded_at, parent_span_id, attributes)", func(batch driver.Batch) error {+ for _, s := range spans {+ attributesJSON := "{}"+ if len(s.Attributes) != 0 {+ if attributesBytes, err := json.Marshal(s.Attributes); err == nil {+ attributesJSON = string(attributesBytes)+ }+ }+ if err := batch.Append(+ s.Id,+ s.TraceId,+ s.ProjectId,+ s.Name,+ s.StartTime,+ int64(s.Duration),+ s.RecordedAt,+ s.ParentSpanId,+ attributesJSON,+ ); err != nil {+ return err}}- if err := batch.Append(- s.Id,- s.TraceId,- s.ProjectId,- s.Name,- s.StartTime,- int64(s.Duration),- s.RecordedAt,- s.ParentSpanId,- attributesJSON,- ); err != nil {- return err- }- }-- return batch.Send()+ return nil+ })}func (r *spanRepository) FindByTraceId(ctx context.Context, projectId, traceId uuid.UUID, recordedAt *time.Time) ([]models.Span, error) {diff --git a/backend/app/repositories/task.repository.go b/backend/app/repositories/task.repository.goindex 1acbb7d6..de198355 100644--- a/backend/app/repositories/task.repository.go+++ b/backend/app/repositories/task.repository.go@@ -11,32 +11,34 @@ import ("errors""time"+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver""github.com/google/uuid")type taskRepository struct{}func (e *taskRepository) InsertAsync(ctx context.Context, lines []models.Task) error {- batch, err := chdb.Conn.PrepareBatch(chdb.BatchCtx(), "INSERT INTO tasks (id, project_id, task_name, duration, recorded_at, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_root)")- if err != nil {- return err- }- for _, t := range lines {- attributesJSON := "{}"- if len(t.Attributes) != 0 {- if attributesBytes, err := json.Marshal(t.Attributes); err == nil {- attributesJSON = string(attributesBytes)+ if len(lines) == 0 {+ return nil+ }+ return chdb.SendBatch("INSERT INTO tasks (id, project_id, task_name, duration, recorded_at, client_ip, attributes, app_version, server_name, distributed_trace_id, span_id, is_root)", func(batch driver.Batch) error {+ for _, t := range lines {+ attributesJSON := "{}"+ if len(t.Attributes) != 0 {+ if attributesBytes, err := json.Marshal(t.Attributes); err == nil {+ attributesJSON = string(attributesBytes)+ }+ }+ isRoot := uint8(0)+ if t.IsRoot {+ isRoot = 1+ }+ if err := batch.Append(t.Id, t.ProjectId, t.TaskName, int64(t.Duration), t.RecordedAt, t.ClientIP, attributesJSON, t.AppVersion, t.ServerName, t.DistributedTraceId, t.SpanId, isRoot); err != nil {+ return err}}- isRoot := uint8(0)- if t.IsRoot {- isRoot = 1- }- if err := batch.Append(t.Id, t.ProjectId, t.TaskName, int64(t.Duration), t.RecordedAt, t.ClientIP, attributesJSON, t.AppVersion, t.ServerName, t.DistributedTraceId, t.SpanId, isRoot); err != nil {- return err- }- }- return batch.Send()+ return nil+ })}func (e *taskRepository) CountBetween(ctx context.Context, projectId uuid.UUID, start, end time.Time) (int64, error) {