The only tool you need to know what is happening and how to fix it.
Go alertingexception-trackerissue-trackinglogsmetricsmonitoringobservability-platformopen-telemetryotelsession-replaytelemetrytraces
package controllers
import (
"fmt"
"net/http"
"github.com/gin-gonic/gin"
"github.com/tracewayapp/traceway/backend/app/db"
"github.com/tracewayapp/traceway/backend/app/outbox"
"github.com/tracewayapp/traceway/backend/app/slo"
"github.com/tracewayapp/traceway/backend/app/synthetics"
traceway "go.tracewayapp.com"
)
type healthDeepController struct{}
type TableParts struct {
Table string `json:"table"`
Parts int64 `json:"parts"`
}
type CHError struct {
Name string `json:"name"`
Value int64 `json:"value"`
LastErrorTime string `json:"lastErrorTime,omitempty"`
}
type HealthDeepResponse struct {
CHReachable bool `json:"chReachable"`
CHUptimeSec int64 `json:"chUptimeSec"`
PartsCount int64 `json:"partsCount"`
PartsByTable []TableParts `json:"partsByTable,omitempty"`
ActiveMerges int64 `json:"activeMerges"`
LongestMergeSec float64 `json:"longestMergeSec"`
ErrorsRecent []CHError `json:"errorsRecent,omitempty"`
MemoryUsageBytes int64 `json:"memoryUsageBytes,omitempty"`
MemoryTotalBytes int64 `json:"memoryTotalBytes,omitempty"`
TelemetryBackend string `json:"telemetryBackend"`
DroppedRows map[string]uint64 `json:"droppedRows"`
DroppedRowsTotal uint64 `json:"droppedRowsTotal"`
InsertFailures uint64 `json:"insertFailures"`
IngestRejected uint64 `json:"ingestRejected"`
// Spans whose source start was missing or unrepresentable, so their partition time is the ingest time.
SpanPartitionTimeFallbacks uint64 `json:"spanPartitionTimeFallbacks"`
Engine *db.TelemetryEngineStats `json:"engine,omitempty"`
Outbox *outbox.HealthStats `json:"outbox,omitempty"`
Synthetics *synthetics.HealthStats `json:"synthetics,omitempty"`
SLO *slo.HealthStats `json:"slo,omitempty"`
}
// 503 means the configured telemetry backend is down. On the embedded
// backends chReachable stays false with a 200, which is what the loadgen's
// merge-idle fast path keys on.
func (h healthDeepController) Get(c *gin.Context) {
resp := fetchCHHealth(c.Request.Context())
dropped, droppedTotal, insertFailures := db.GetTelemetryIngestCounters()
resp.TelemetryBackend = db.TelemetryBackendName()
resp.DroppedRows = dropped
resp.DroppedRowsTotal = droppedTotal
resp.InsertFailures = insertFailures
resp.IngestRejected = db.GetIngestRejects()
resp.SpanPartitionTimeFallbacks = db.GetSpanPartitionTimeFallbacks()
if engine, ok := db.GetTelemetryEngineStats(c.Request.Context()); ok {
resp.Engine = &engine
}
// Best-effort: health must not 500 because the main DB hiccuped.
if outboxStats, err := outbox.HealthSnapshot(); err == nil {
resp.Outbox = outboxStats
} else {
traceway.CaptureException(fmt.Errorf("failed to load outbox health: %w", err))
}
if syntheticsStats, err := synthetics.HealthSnapshot(); err == nil {
resp.Synthetics = syntheticsStats
} else {
traceway.CaptureException(fmt.Errorf("failed to load synthetics health: %w", err))
}
sloStats, err := slo.HealthSnapshot(c.Request.Context())
if err != nil {
traceway.CaptureException(fmt.Errorf("failed to load slo health: %w", err))
}
resp.SLO = sloStats
if resp.TelemetryBackend == "clickhouse" && !resp.CHReachable {
c.JSON(http.StatusServiceUnavailable, resp)
return
}
c.JSON(http.StatusOK, resp)
}
var HealthDeepController = healthDeepController{}