Skip to content

The only tool you need to know what is happening and how to fix it.

feat: added retry to ch queries for reporting metrics for non important errors like eof or conn closed (#220)

narrow-adventure5a8b4397ce71
diff --git a/backend/app/monitoring/ch_reporter.go b/backend/app/monitoring/ch_reporter.go
index fec534ea..d7bd01d6 100644
--- a/backend/app/monitoring/ch_reporter.go
+++ b/backend/app/monitoring/ch_reporter.go
@@ -4,11 +4,17 @@ package monitoring
import (
"context"
+ "errors"
"fmt"
+ "io"
+ "net"
+ "strings"
+ "sync"
"time"
traceway "go.tracewayapp.com"
+ "github.com/ClickHouse/clickhouse-go/v2/lib/driver"
"github.com/tracewayapp/traceway/backend/app/chdb"
)
@@ -32,6 +38,59 @@ type chBaselines struct {
first bool
}
+const transientReportInterval = 10 * time.Minute
+
+var (
+ transientReportMu sync.Mutex
+ lastTransientReport time.Time
+)
+
+func transientCHError(err error) bool {
+ if err == nil {
+ return false
+ }
+ if errors.Is(err, io.EOF) ||
+ errors.Is(err, net.ErrClosed) ||
+ errors.Is(err, context.DeadlineExceeded) ||
+ errors.Is(err, context.Canceled) {
+ return true
+ }
+ var netErr net.Error
+ if errors.As(err, &netErr) && netErr.Timeout() {
+ return true
+ }
+ msg := strings.ToLower(err.Error())
+ for _, s := range []string{"eof", "connection reset", "broken pipe", "use of closed", "connection refused", "reset by peer", "i/o timeout", "no route to host"} {
+ if strings.Contains(msg, s) {
+ return true
+ }
+ }
+ return false
+}
+
+func withRetry(fn func() error) error {
+ err := fn()
+ if err != nil && transientCHError(err) {
+ return fn()
+ }
+ return err
+}
+
+func captureMonitoringErr(err error) {
+ if transientCHError(err) {
+ transientReportMu.Lock()
+ report := time.Since(lastTransientReport) >= transientReportInterval
+ if report {
+ lastTransientReport = time.Now()
+ }
+ transientReportMu.Unlock()
+ if !report {
+ return
+ }
+ }
+ traceway.CaptureException(err)
+}
+
func StartClickHouseReporter(ctx context.Context) {
go func() {
defer traceway.Recover()
@@ -79,13 +138,15 @@ func reportPartsByTable(ctx context.Context) {
for _, table := range telemetryTables {
var parts uint64
var rows uint64
- err := chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(count(), 0), coalesce(sum(rows), 0) FROM system.parts WHERE database = currentDatabase() AND table = ? AND active",
- table,
- ).Scan(&parts, &rows)
+ err := withRetry(func() error {
+ return chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(count(), 0), coalesce(sum(rows), 0) FROM system.parts WHERE database = currentDatabase() AND table = ? AND active",
+ table,
+ ).Scan(&parts, &rows)
+ })
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.parts query for %q failed: %w", table, err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.parts query for %q failed: %w", table, err))
continue
}
tags := map[string]string{"table": table}
@@ -95,12 +156,17 @@ func reportPartsByTable(ctx context.Context) {
}
func reportMaxPartsPerPartition(ctx context.Context) {
- rows, err := chdb.Conn.Query(
- ctx,
- "SELECT table, max(parts) FROM (SELECT table, partition, count() AS parts FROM system.parts WHERE database = currentDatabase() AND active GROUP BY table, partition) GROUP BY table",
- )
+ var rows driver.Rows
+ err := withRetry(func() error {
+ var e error
+ rows, e = chdb.Conn.Query(
+ ctx,
+ "SELECT table, max(parts) FROM (SELECT table, partition, count() AS parts FROM system.parts WHERE database = currentDatabase() AND active GROUP BY table, partition) GROUP BY table",
+ )
+ return e
+ })
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition query failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.parts per-partition query failed: %w", err))
return
}
defer rows.Close()
@@ -109,24 +175,26 @@ func reportMaxPartsPerPartition(ctx context.Context) {
var table string
var maxParts uint64
if err := rows.Scan(&table, &maxParts); err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition scan failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.parts per-partition scan failed: %w", err))
return
}
traceway.CaptureMetricWithTags("traceway.ch.parts.max_per_partition", float64(maxParts), map[string]string{"table": table})
}
if err := rows.Err(); err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.parts per-partition rows failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.parts per-partition rows failed: %w", err))
}
}
func reportMergesRunning(ctx context.Context) {
var merges uint64
- err := chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(count(), 0) FROM system.merges WHERE database = currentDatabase()",
- ).Scan(&merges)
+ err := withRetry(func() error {
+ return chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(count(), 0) FROM system.merges WHERE database = currentDatabase()",
+ ).Scan(&merges)
+ })
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.merges query failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.merges query failed: %w", err))
return
}
traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))
@@ -135,22 +203,22 @@ func reportMergesRunning(ctx context.Context) {
func reportEventDeltas(ctx context.Context, b *chBaselines) {
curInserted, err := scanEventValue(ctx, "InsertedRows")
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))
return
}
curFailed, err := scanEventValue(ctx, "FailedInsertQuery")
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))
return
}
curDelayed, err := scanEventValue(ctx, "DelayedInserts")
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.events DelayedInserts failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.events DelayedInserts failed: %w", err))
return
}
curRejected, err := scanEventValue(ctx, "RejectedInserts")
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.events RejectedInserts failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.events RejectedInserts failed: %w", err))
return
}
@@ -170,18 +238,25 @@ func reportEventDeltas(ctx context.Context, b *chBaselines) {
func scanEventValue(ctx context.Context, event string) (uint64, error) {
var value uint64
- err := chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(sum(value), 0) FROM system.events WHERE event = ?",
- event,
- ).Scan(&value)
+ err := withRetry(func() error {
+ return chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(sum(value), 0) FROM system.events WHERE event = ?",
+ event,
+ ).Scan(&value)
+ })
return value, err
}
func reportDiskUsage(ctx context.Context) {
- rows, err := chdb.Conn.Query(ctx, "SELECT name, free_space, total_space FROM system.disks")
+ var rows driver.Rows
+ err := withRetry(func() error {
+ var e error
+ rows, e = chdb.Conn.Query(ctx, "SELECT name, free_space, total_space FROM system.disks")
+ return e
+ })
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.disks query failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.disks query failed: %w", err))
return
}
defer rows.Close()
@@ -190,7 +265,7 @@ func reportDiskUsage(ctx context.Context) {
var name string
var free, total uint64
if err := rows.Scan(&name, &free, &total); err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.disks scan failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.disks scan failed: %w", err))
return
}
if total == 0 {
@@ -202,29 +277,33 @@ func reportDiskUsage(ctx context.Context) {
traceway.CaptureMetricWithTags("traceway.ch.disk.used_pcnt", float64(total-free)/float64(total)*100, tags)
}
if err := rows.Err(); err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.disks rows failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.disks rows failed: %w", err))
}
}
func reportServerMemory(ctx context.Context) {
var tracking int64
- err := chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(sum(value), 0) FROM system.metrics WHERE metric = 'MemoryTracking'",
- ).Scan(&tracking)
+ err := withRetry(func() error {
+ return chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(sum(value), 0) FROM system.metrics WHERE metric = 'MemoryTracking'",
+ ).Scan(&tracking)
+ })
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.metrics MemoryTracking failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.metrics MemoryTracking failed: %w", err))
} else {
traceway.CaptureMetric("traceway.ch.memory.tracking_bytes", float64(tracking))
}
var available float64
- err = chdb.Conn.QueryRow(
- ctx,
- "SELECT coalesce(sum(value), 0) FROM system.asynchronous_metrics WHERE metric = 'OSMemoryAvailable'",
- ).Scan(&available)
+ err = withRetry(func() error {
+ return chdb.Conn.QueryRow(
+ ctx,
+ "SELECT coalesce(sum(value), 0) FROM system.asynchronous_metrics WHERE metric = 'OSMemoryAvailable'",
+ ).Scan(&available)
+ })
if err != nil {
- traceway.CaptureException(fmt.Errorf("monitoring: system.asynchronous_metrics OSMemoryAvailable failed: %w", err))
+ captureMonitoringErr(fmt.Errorf("monitoring: system.asynchronous_metrics OSMemoryAvailable failed: %w", err))
return
}
if available > 0 {