Skip to content

The only tool you need to know what is happening and how to fix it.

16.9 KB
Raw Blame
package main

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"time"
)

type endpointProbeResult struct {
	Name      string  `json:"name"`
	Path      string  `json:"path"`
	LatencyMs float64 `json:"latencyMs"`
	Ok        bool    `json:"ok"`
	Error     string  `json:"error,omitempty"`
}

type readProbeStep struct {
	FillLevelTarget      int64   `json:"fillLevelTarget"`
	RowsIngested         int64   `json:"rowsIngested"`
	IngestSecondsElapsed float64 `json:"ingestSecondsElapsed"`
	// DroppedRows is cumulative since scenario start, mirroring RowsIngested.
	// Nonzero means the SUT silently discarded rows during fill, so the true
	// table size is RowsIngested minus DroppedRows.
	DroppedRows int64 `json:"droppedRows,omitempty"`
	// DigestSeconds is how long the SUT's engine needed after the fill before
	// its db+WAL file sizes went stable (the WAL-checkpoint gate). Probes run
	// after this wait, so latencies measure steady-state query cost, not a
	// mid-checkpoint wedge. DigestTimedOut means the cap expired first and
	// the probes ran against a possibly still-digesting engine.
	DigestSeconds   float64               `json:"digestSeconds,omitempty"`
	DigestTimedOut  bool                  `json:"digestTimedOut,omitempty"`
	Probes          []endpointProbeResult `json:"probes"`
	MedianLatencyMs float64               `json:"medianLatencyMs"`
	// Back-compat: equals MedianLatencyMs and AND-of-all-probe-Ok respectively.
	// Existing chart.py renderers (and post-1 readers) consume these.
	ReadLatencyMs float64 `json:"readLatencyMs"`
	ReadOk        bool    `json:"readOk"`
	Passed        bool    `json:"passed"`
	FailReason    string  `json:"failReason,omitempty"`
}

type readProbeEndpointDescriptor struct {
	Name string `json:"name"`
	Path string `json:"path"`
}

type readProbeResult struct {
	Endpoints []readProbeEndpointDescriptor `json:"endpoints"`
	// Back-compat: first endpoint's path. Older chart.py renderers read this.
	ReadPath           string          `json:"readPath"`
	ReadThresholdMs    int             `json:"readThresholdMs"`
	SettleSeconds      int             `json:"settleSeconds"`
	FillBatchSize      int             `json:"fillBatchSize"`
	FillRequestRate    float64         `json:"fillRequestRate"`
	Steps              []readProbeStep `json:"steps"`
	MaxFillLevelPassed int64           `json:"maxFillLevelPassed"`
}

// readProbeCheckpoint is invoked after every fill-level step so the caller
// can persist a partial report — we never want to lose progress when the
// process dies mid-run.
type readProbeCheckpoint func(readProbeResult)

// runReadProbe walks the configured fill levels. For each level it ingests
// rows until totalIngested >= target, settles for cfg.settleSeconds, then
// issues one read probe **per configured endpoint** for the signal, records
// per-endpoint latencies, and uses the median to pass/fail against
// cfg.readThresholdMs. Any HTTP error on any probe fails the step.
func runReadProbe(ctx context.Context, cfg config, ing *ingester, ingestStats *latencyTracker, client *http.Client, checkpoint readProbeCheckpoint) readProbeResult {
	endpoints := endpointSetForSignal(cfg.signal)
	descriptors := make([]readProbeEndpointDescriptor, len(endpoints))
	for i, ep := range endpoints {
		descriptors[i] = readProbeEndpointDescriptor{Name: ep.Name, Path: ep.Path}
	}
	res := readProbeResult{
		Endpoints:       descriptors,
		ReadThresholdMs: cfg.readThresholdMs,
		SettleSeconds:   int(cfg.settleSeconds.Seconds()),
		FillBatchSize:   cfg.fillBatchSize,
		FillRequestRate: cfg.fillRequestRate,
	}
	if len(endpoints) > 0 {
		res.ReadPath = endpoints[0].Path
	}

	// The shared client carries a 30s Timeout that would silently cap probes
	// below a raised --read-threshold-ms; probes get their own client so the
	// per-probe context (threshold + 1s) is the only deadline.
	probeClient := &http.Client{Transport: client.Transport}

	ing.SetBatchSize(cfg.fillBatchSize)
	ing.SetRequestRate(cfg.fillRequestRate)

	// Reset the ingester counters before starting so totalIngested reflects
	// only what this scenario sent.
	ing.SnapshotAndResetItems()
	ing.SnapshotAndResetOkItems()
	ingestStats.SnapshotAndReset()

	_, sutStatsBase := fetchDeepHealth(ctx, cfg, client)

	var totalIngested int64

	for _, target := range cfg.fillLevels {
		if ctx.Err() != nil {
			break
		}
		step := readProbeStep{FillLevelTarget: target}

		if totalIngested < target {
			fillStart := time.Now()
			ing.Start(ctx)
			pollFillProgress(ctx, ing, &totalIngested, target)
			ing.Stop()
			step.IngestSecondsElapsed = time.Since(fillStart).Seconds()
		}
		step.RowsIngested = totalIngested

		// Drain whatever bumped between Stop and Snapshot.
		totalIngested += ing.SnapshotAndResetOkItems()
		step.RowsIngested = totalIngested

		if sutStatsBase.Reachable {
			if _, cur := fetchDeepHealth(ctx, cfg, client); cur.Reachable {
				if dropped := cur.DroppedRowsTotal - sutStatsBase.DroppedRowsTotal; dropped > 0 {
					step.DroppedRows = dropped
					fmt.Fprintf(stderrPrefix(), "read-probe: SUT silently dropped %d rows during fill; true table size is below the fill target\n", dropped)
				}
			}
		}

		step.DigestSeconds, step.DigestTimedOut = waitForIngestDigestion(ctx, cfg, client.Transport)
		if step.DigestTimedOut {
			fmt.Fprintf(stderrPrefix(), "read-probe: engine still digesting after %.0fs cap — probing anyway\n", step.DigestSeconds)
		}
		// ClickHouse's post-ingest work is merges rather than a WAL
		// checkpoint; give it the same courtesy before probing. Skips
		// instantly on backends where CH isn't reachable.
		waitForMergesIdle(ctx, cfg, client, fmt.Sprintf("fill %d -> probe", target))

		fmt.Fprintf(stderrPrefix(), "read-probe fill=%d rows reached in %.1fs (signal=%s) — digested in %.1fs, settling %ds\n",
			step.RowsIngested, step.IngestSecondsElapsed, cfg.signal, step.DigestSeconds, res.SettleSeconds)

		select {
		case <-time.After(cfg.settleSeconds):
		case <-ctx.Done():
		}
		if ctx.Err() != nil {
			res.Steps = append(res.Steps, step)
			if checkpoint != nil {
				checkpoint(res)
			}
			break
		}

		probeResults := make([]endpointProbeResult, 0, len(endpoints))
		allOk := true
		latencies := make([]float64, 0, len(endpoints))
		for _, ep := range endpoints {
			latency, err := probeReadEndpoint(ctx, probeClient, cfg, ep)
			pr := endpointProbeResult{
				Name:      ep.Name,
				Path:      ep.Path,
				LatencyMs: latency,
				Ok:        err == nil,
			}
			if err != nil {
				pr.Error = err.Error()
				allOk = false
			}
			probeResults = append(probeResults, pr)
			latencies = append(latencies, latency)
		}
		step.Probes = probeResults

		median := medianFloat(latencies)
		step.MedianLatencyMs = median
		step.ReadLatencyMs = median
		step.ReadOk = allOk

		switch {
		case !allOk:
			step.Passed = false
			// Find the first failed probe for the reason.
			for _, pr := range probeResults {
				if !pr.Ok {
					step.FailReason = fmt.Sprintf("probe %s failed: %s (latency %.0fms)", pr.Name, pr.Error, pr.LatencyMs)
					break
				}
			}
		case median > float64(cfg.readThresholdMs):
			step.Passed = false
			step.FailReason = fmt.Sprintf("median latency %.0fms > %dms threshold across %d probes", median, cfg.readThresholdMs, len(probeResults))
		default:
			step.Passed = true
			res.MaxFillLevelPassed = target
		}

		fmt.Fprintf(stderrPrefix(), "read-probe target=%d median=%.0fms probes=%d allOk=%t passed=%t %s\n",
			target, median, len(probeResults), allOk, step.Passed, step.FailReason)
		res.Steps = append(res.Steps, step)
		if checkpoint != nil {
			checkpoint(res)
		}
		if !step.Passed {
			break
		}
	}

	return res
}

// pollFillProgress drains confirmed-item counters from the ingester until the
// target is reached or the context cancels. Counting only 2xx-confirmed items
// (not attempts) keeps the fill honest when the SUT sheds load with 503s.
// Sub-second polling keeps overshoot small (at fill-batch=8192 × 100 req/s
// the loadgen sends ~800k items/sec, so 200ms granularity overshoots by
// ≤160k items — negligible at 100M fill).
func pollFillProgress(ctx context.Context, ing *ingester, totalIngested *int64, target int64) {
	tick := time.NewTicker(200 * time.Millisecond)
	defer tick.Stop()
	for *totalIngested < target {
		select {
		case <-ctx.Done():
			return
		case <-tick.C:
			*totalIngested += ing.SnapshotAndResetOkItems()
		}
	}
}

func medianFloat(values []float64) float64 {
	if len(values) == 0 {
		return 0
	}
	sorted := make([]float64, len(values))
	copy(sorted, values)
	sort.Float64s(sorted)
	mid := len(sorted) / 2
	if len(sorted)%2 == 1 {
		return sorted[mid]
	}
	return (sorted[mid-1] + sorted[mid]) / 2
}

// readProbeEndpoint describes one HTTP probe target.
type readProbeEndpoint struct {
	Name  string
	Path  string
	Build func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error)
}

// endpointSetForSignal returns the 3 endpoints probed per fill-level for a
// given signal. Locked 2026-06-08 (see POSTS.md decision log). Order matters:
// the first entry's path becomes the back-compat readProbeResult.ReadPath.
func endpointSetForSignal(signal string) []readProbeEndpoint {
	switch signal {
	case "spans":
		return []readProbeEndpoint{
			{Name: "endpoints-grouped", Path: "/api/endpoints/grouped", Build: buildPostJSON("/api/endpoints/grouped", spansGroupedBody)},
			{Name: "endpoints-chart", Path: "/api/endpoints/chart", Build: buildPostJSON("/api/endpoints/chart", spansChartBody)},
			{Name: "exception-stack-traces", Path: "/api/exception-stack-traces", Build: buildPostJSON("/api/exception-stack-traces", spansExceptionsBody)},
		}
	case "metrics":
		return []readProbeEndpoint{
			{Name: "metrics-server", Path: "/api/metrics/server", Build: buildGetQuery("/api/metrics/server", metricsTimeRangeQuery)},
			{Name: "metrics-application", Path: "/api/metrics/application", Build: buildGetQuery("/api/metrics/application", metricsTimeRangeQuery)},
			{Name: "dashboard", Path: "/api/dashboard", Build: buildGetQuery("/api/dashboard", metricsTimeRangeQuery)},
		}
	case "logs":
		return []readProbeEndpoint{
			{Name: "logs-severity-error", Path: "/api/logs", Build: buildPostJSON("/api/logs", logsSeverityErrorBody)},
			{Name: "logs-body-search", Path: "/api/logs", Build: buildPostJSON("/api/logs", logsBodySearchBody)},
			{Name: "logs-trace-id", Path: "/api/logs", Build: buildPostJSON("/api/logs", logsTraceIdBody)},
		}
	}
	return nil
}

// waitForIngestDigestion blocks until the SUT's storage engine has absorbed
// the fill burst, or cfg.maxDigestWait expires. DuckDB checkpoints the WAL
// asynchronously after ingest stops; probing mid-checkpoint on a small box
// measures a wedged engine, not query cost — the symptom is sub-millisecond
// queries timing out uniformly. /api/health/deep is the canary: on DuckDB it
// executes real engine queries, so it only answers promptly when a dashboard
// query would too. "Digested" means two consecutive successful polls, 5s
// apart, reporting identical db+WAL file sizes. Backends without engine
// gauges (SQLite, ClickHouse) return on the first successful poll, keeping
// their timing behavior identical to before this gate existed. Returns the
// seconds waited and whether the cap expired first.
func waitForIngestDigestion(ctx context.Context, cfg config, transport http.RoundTripper) (float64, bool) {
	if cfg.maxDigestWait <= 0 {
		return 0, false
	}
	pollClient := &http.Client{Timeout: 5 * time.Second, Transport: transport}
	start := time.Now()
	deadline := start.Add(cfg.maxDigestWait)
	var prev *ingestStatsSnapshot
	for time.Now().Before(deadline) {
		if ctx.Err() != nil {
			return time.Since(start).Seconds(), false
		}
		_, cur := fetchDeepHealth(ctx, cfg, pollClient)
		if cur.Reachable {
			if cur.DBSizeBytes == 0 && cur.WALSizeBytes == 0 {
				return time.Since(start).Seconds(), false
			}
			if prev != nil && prev.DBSizeBytes == cur.DBSizeBytes && prev.WALSizeBytes == cur.WALSizeBytes {
				return time.Since(start).Seconds(), false
			}
			snap := cur
			prev = &snap
		} else {
			// A slow or failed poll is the engine (or the box) not answering —
			// it breaks the stability streak rather than counting toward it.
			prev = nil
		}
		select {
		case <-time.After(5 * time.Second):
		case <-ctx.Done():
			return time.Since(start).Seconds(), false
		}
	}
	return time.Since(start).Seconds(), true
}

// probeReadEndpoint issues one HTTP request for the given endpoint and returns
// wall-clock latency in ms plus any error. Hard-capped at threshold + 1s so a
// hanging SUT can't deadlock the loop.
func probeReadEndpoint(ctx context.Context, client *http.Client, cfg config, ep readProbeEndpoint) (float64, error) {
	now := time.Now().UTC()
	fromDate := now.Add(-24 * time.Hour).Format(time.RFC3339)
	toDate := now.Format(time.RFC3339)

	probeCtx, cancel := context.WithTimeout(ctx, time.Duration(cfg.readThresholdMs+1000)*time.Millisecond)
	defer cancel()

	req, err := ep.Build(cfg, fromDate, toDate, probeCtx)
	if err != nil {
		return 0, err
	}
	req.Header.Set("Authorization", "Bearer "+cfg.jwt)

	start := time.Now()
	resp, err := client.Do(req)
	elapsed := time.Since(start).Seconds() * 1000
	if err != nil {
		return elapsed, err
	}
	defer resp.Body.Close()
	_, _ = io.Copy(io.Discard, resp.Body)
	if resp.StatusCode >= 400 {
		return elapsed, fmt.Errorf("status %d", resp.StatusCode)
	}
	return elapsed, nil
}

// buildPostJSON returns a request-builder that POSTs a JSON body. The body
// callback receives the date range so it can populate fromDate/toDate.
func buildPostJSON(path string, body func(fromDate, toDate string) map[string]any) func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {
	return func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {
		payload, err := json.Marshal(body(fromDate, toDate))
		if err != nil {
			return nil, err
		}
		u, err := url.Parse(cfg.target + path)
		if err != nil {
			return nil, err
		}
		q := u.Query()
		q.Set("projectId", cfg.projectId)
		u.RawQuery = q.Encode()
		req, err := http.NewRequestWithContext(probeCtx, http.MethodPost, u.String(), bytes.NewReader(payload))
		if err != nil {
			return nil, err
		}
		req.Header.Set("Content-Type", "application/json")
		return req, nil
	}
}

// buildGetQuery returns a request-builder that GETs with extra query params.
func buildGetQuery(path string, query func(fromDate, toDate string) map[string]string) func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {
	return func(cfg config, fromDate, toDate string, probeCtx context.Context) (*http.Request, error) {
		u, err := url.Parse(cfg.target + path)
		if err != nil {
			return nil, err
		}
		q := u.Query()
		q.Set("projectId", cfg.projectId)
		for k, v := range query(fromDate, toDate) {
			q.Set(k, v)
		}
		u.RawQuery = q.Encode()
		return http.NewRequestWithContext(probeCtx, http.MethodGet, u.String(), nil)
	}
}

// --- Per-endpoint body / query builders ------------------------------------

func spansGroupedBody(fromDate, toDate string) map[string]any {
	return map[string]any{
		"fromDate":      fromDate,
		"toDate":        toDate,
		"orderBy":       "count",
		"sortDirection": "desc",
		"pagination":    map[string]int{"page": 1, "pageSize": 50},
		"search":        "",
	}
}

func spansChartBody(fromDate, toDate string) map[string]any {
	return map[string]any{
		"fromDate":        fromDate,
		"toDate":          toDate,
		"metricType":      "p95",
		"intervalMinutes": 5,
	}
}

func spansExceptionsBody(fromDate, toDate string) map[string]any {
	return map[string]any{
		"fromDate":        fromDate,
		"toDate":          toDate,
		"orderBy":         "count",
		"pagination":      map[string]int{"page": 1, "pageSize": 50},
		"search":          "",
		"searchType":      "message",
		"includeArchived": false,
	}
}

func metricsTimeRangeQuery(fromDate, toDate string) map[string]string {
	return map[string]string{
		"fromDate": fromDate,
		"toDate":   toDate,
	}
}

func logsSeverityErrorBody(fromDate, toDate string) map[string]any {
	// MinSeverity 17 = OTel SeverityNumber for ERROR.
	return map[string]any{
		"fromDate":      fromDate,
		"toDate":        toDate,
		"orderBy":       "timestamp",
		"sortDirection": "desc",
		"minSeverity":   17,
		"pagination":    map[string]int{"page": 1, "pageSize": 50},
	}
}

func logsBodySearchBody(fromDate, toDate string) map[string]any {
	// "error" is short, common, and inside the 24h unscoped-body-search
	// window allowed by /api/logs.
	return map[string]any{
		"fromDate":      fromDate,
		"toDate":        toDate,
		"orderBy":       "timestamp",
		"sortDirection": "desc",
		"search":        "error",
		"searchType":    "body",
		"pagination":    map[string]int{"page": 1, "pageSize": 50},
	}
}

func logsTraceIdBody(fromDate, toDate string) map[string]any {
	// Fixed dummy trace_id — the query exercises the trace_id index path;
	// result count doesn't matter for latency measurement.
	return map[string]any{
		"fromDate":      fromDate,
		"toDate":        toDate,
		"orderBy":       "timestamp",
		"sortDirection": "desc",
		"traceId":       "00000000000000000000000000000001",
		"pagination":    map[string]int{"page": 1, "pageSize": 50},
	}
}