Skip to content

The only tool you need to know what is happening and how to fix it.

8.4 KB
Raw Blame
//go:build telemetry_ch

package monitoring

import (
	"context"
	"errors"
	"fmt"
	"net"
	"strings"
	"sync"
	"time"

	traceway "go.tracewayapp.com"

	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
	"github.com/tracewayapp/traceway/backend/app/chdb"
)

const reportInterval = 60 * time.Second

var telemetryTables = []string{
	"traces",
	"log_records",
	"metric_points",
	"spans_v2",
	"exceptions_v2",
	"endpoints_v2",
	"tasks_v2",
	"ai_traces_v2",
	// Backfill retains the legacy tables; their parts still consume capacity.
	"spans",
	"exception_stack_traces",
	"endpoints",
	"tasks",
	"ai_traces",
}

type chBaselines struct {
	insertedRows    uint64
	failedInserts   uint64
	delayedInserts  uint64
	rejectedInserts uint64
	first           bool
}

const transientReportInterval = 10 * time.Minute

var (
	transientReportMu   sync.Mutex
	lastTransientReport time.Time
)

func transientCHError(err error) bool {
	if err == nil {
		return false
	}
	if chdb.IsConnError(err) {
		return true
	}
	if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) {
		return true
	}
	var netErr net.Error
	if errors.As(err, &netErr) && netErr.Timeout() {
		return true
	}
	return strings.Contains(strings.ToLower(err.Error()), "i/o timeout")
}

func withRetry(fn func() error) error {
	return chdb.RetryOnConn(2, transientCHError, fn)
}

func captureMonitoringErr(err error) {
	if transientCHError(err) {
		transientReportMu.Lock()
		report := time.Since(lastTransientReport) >= transientReportInterval
		if report {
			lastTransientReport = time.Now()
		}
		transientReportMu.Unlock()
		if !report {
			return
		}
	}
	traceway.CaptureException(err)
}

func StartClickHouseReporter(ctx context.Context) {
	go func() {
		defer traceway.Recover()

		ticker := time.NewTicker(reportInterval)
		defer ticker.Stop()

		baselines := &chBaselines{first: true}

		// Fire once immediately so we don't wait 60s for the first data point.
		reportOnce(ctx, baselines)

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				reportOnce(ctx, baselines)
			}
		}
	}()
}

func reportOnce(ctx context.Context, b *chBaselines) {
	// Bound each tick so a stuck CH doesn't wedge the reporter forever.
	qCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
	defer cancel()

	reportPoolStats()
	reportPartsByTable(qCtx)
	reportMaxPartsPerPartition(qCtx)
	reportMergesRunning(qCtx)
	reportEventDeltas(qCtx, b)
	reportDiskUsage(qCtx)
	reportServerMemory(qCtx)
}

func reportPoolStats() {
	stats := chdb.Conn.Stats()
	traceway.CaptureMetric("traceway.ch.pool.open", float64(stats.Open))
	traceway.CaptureMetric("traceway.ch.pool.idle", float64(stats.Idle))
}

func reportPartsByTable(ctx context.Context) {
	for _, table := range telemetryTables {
		var parts uint64
		var rows uint64
		err := withRetry(func() error {
			return chdb.Conn.QueryRow(
				ctx,
				"SELECT coalesce(count(), 0), coalesce(sum(rows), 0) FROM system.parts WHERE database = currentDatabase() AND table = ? AND active",
				table,
			).Scan(&parts, &rows)
		})
		if err != nil {
			captureMonitoringErr(fmt.Errorf("monitoring: system.parts query for %q failed: %w", table, err))
			continue
		}
		tags := map[string]string{"table": table}
		traceway.CaptureMetricWithTags("traceway.ch.parts.active", float64(parts), tags)
		traceway.CaptureMetricWithTags("traceway.ch.parts.rows", float64(rows), tags)
	}
}

func reportMaxPartsPerPartition(ctx context.Context) {
	var rows driver.Rows
	err := withRetry(func() error {
		var e error
		rows, e = chdb.Conn.Query(
			ctx,
			"SELECT table, max(parts) FROM (SELECT table, partition, count() AS parts FROM system.parts WHERE database = currentDatabase() AND active GROUP BY table, partition) GROUP BY table",
		)
		return e
	})
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.parts per-partition query failed: %w", err))
		return
	}
	defer rows.Close()

	for rows.Next() {
		var table string
		var maxParts uint64
		if err := rows.Scan(&table, &maxParts); err != nil {
			captureMonitoringErr(fmt.Errorf("monitoring: system.parts per-partition scan failed: %w", err))
			return
		}
		traceway.CaptureMetricWithTags("traceway.ch.parts.max_per_partition", float64(maxParts), map[string]string{"table": table})
	}
	if err := rows.Err(); err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.parts per-partition rows failed: %w", err))
	}
}

func reportMergesRunning(ctx context.Context) {
	var merges uint64
	err := withRetry(func() error {
		return chdb.Conn.QueryRow(
			ctx,
			"SELECT coalesce(count(), 0) FROM system.merges WHERE database = currentDatabase()",
		).Scan(&merges)
	})
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.merges query failed: %w", err))
		return
	}
	traceway.CaptureMetric("traceway.ch.merges.running", float64(merges))
}

func reportEventDeltas(ctx context.Context, b *chBaselines) {
	curInserted, err := scanEventValue(ctx, "InsertedRows")
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.events InsertedRows failed: %w", err))
		return
	}
	curFailed, err := scanEventValue(ctx, "FailedInsertQuery")
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.events FailedInsertQuery failed: %w", err))
		return
	}
	curDelayed, err := scanEventValue(ctx, "DelayedInserts")
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.events DelayedInserts failed: %w", err))
		return
	}
	curRejected, err := scanEventValue(ctx, "RejectedInserts")
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.events RejectedInserts failed: %w", err))
		return
	}

	// First tick has no baseline — the "delta" would be the lifetime counter.
	if !b.first {
		traceway.CaptureMetric("traceway.ch.inserted_rows.delta", float64(safeDelta(b.insertedRows, curInserted)))
		traceway.CaptureMetric("traceway.ch.failed_inserts.delta", float64(safeDelta(b.failedInserts, curFailed)))
		traceway.CaptureMetric("traceway.ch.delayed_inserts.delta", float64(safeDelta(b.delayedInserts, curDelayed)))
		traceway.CaptureMetric("traceway.ch.rejected_inserts.delta", float64(safeDelta(b.rejectedInserts, curRejected)))
	}
	b.insertedRows = curInserted
	b.failedInserts = curFailed
	b.delayedInserts = curDelayed
	b.rejectedInserts = curRejected
	b.first = false
}

func scanEventValue(ctx context.Context, event string) (uint64, error) {
	var value uint64
	err := withRetry(func() error {
		return chdb.Conn.QueryRow(
			ctx,
			"SELECT coalesce(sum(value), 0) FROM system.events WHERE event = ?",
			event,
		).Scan(&value)
	})
	return value, err
}

func reportDiskUsage(ctx context.Context) {
	var rows driver.Rows
	err := withRetry(func() error {
		var e error
		rows, e = chdb.Conn.Query(ctx, "SELECT name, free_space, total_space FROM system.disks")
		return e
	})
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.disks query failed: %w", err))
		return
	}
	defer rows.Close()

	for rows.Next() {
		var name string
		var free, total uint64
		if err := rows.Scan(&name, &free, &total); err != nil {
			captureMonitoringErr(fmt.Errorf("monitoring: system.disks scan failed: %w", err))
			return
		}
		if total == 0 {
			continue
		}
		tags := map[string]string{"disk": name}
		traceway.CaptureMetricWithTags("traceway.ch.disk.free_bytes", float64(free), tags)
		traceway.CaptureMetricWithTags("traceway.ch.disk.total_bytes", float64(total), tags)
		traceway.CaptureMetricWithTags("traceway.ch.disk.used_pcnt", float64(total-free)/float64(total)*100, tags)
	}
	if err := rows.Err(); err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.disks rows failed: %w", err))
	}
}

func reportServerMemory(ctx context.Context) {
	var tracking int64
	err := withRetry(func() error {
		return chdb.Conn.QueryRow(
			ctx,
			"SELECT coalesce(sum(value), 0) FROM system.metrics WHERE metric = 'MemoryTracking'",
		).Scan(&tracking)
	})
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.metrics MemoryTracking failed: %w", err))
	} else {
		traceway.CaptureMetric("traceway.ch.memory.tracking_bytes", float64(tracking))
	}

	var available float64
	err = withRetry(func() error {
		return chdb.Conn.QueryRow(
			ctx,
			"SELECT coalesce(sum(value), 0) FROM system.asynchronous_metrics WHERE metric = 'OSMemoryAvailable'",
		).Scan(&available)
	})
	if err != nil {
		captureMonitoringErr(fmt.Errorf("monitoring: system.asynchronous_metrics OSMemoryAvailable failed: %w", err))
		return
	}
	if available > 0 {
		traceway.CaptureMetric("traceway.ch.memory.os_available_bytes", available)
	}
}