Skip to content

The only tool you need to know what is happening and how to fix it.

54.5 KB
Raw Blame
#!/usr/bin/env python3
"""Render benchmark charts from a directory of loadgen JSONs.

Inputs: a directory containing N JSON files (one per matrix entry), each in
the schema emitted by benchmarks/loadgen/. Files without a "signal" field
(old pre-OTLP-split format) are skipped with a log line.

The script auto-detects scenarios per file (each JSON has `scenario`). Run it
against `benchmarks/results-throughput/` for throughput charts, or
`benchmarks/results-probe/` for read-probe charts. Mixed folders also work —
each render function filters to its own scenario.

See benchmarks/charts.md for a reader's guide to each output.
"""

from __future__ import annotations

import json
import sys
from pathlib import Path

import matplotlib

matplotlib.use("Agg")
import matplotlib.pyplot as plt
from matplotlib.patches import Patch, Rectangle


TIER_ORDER = ["ccx13", "ccx23", "ccx33", "ccx43"]
MODE_ORDER = ["sqlite", "pgch", "managed-ch"]
SIGNALS = ["spans", "metrics", "logs"]

MODE_COLORS = {
    "sqlite": "#4f9fff",
    "pgch": "#ff9f4f",
    "managed-ch": "#7c4fff",
}
SIGNAL_COLORS = {
    "spans": "#1f77b4",
    "metrics": "#2ca02c",
    "logs": "#d62728",
}
PHASE_LABEL = {"phase1": "P1", "phase2": "P2", "phase3": "P3"}

ITEM_LABEL = {
    "spans": "spans/sec",
    "metrics": "data points/sec",
    "logs": "log records/sec",
}
ROW_LABEL = {
    "spans": "spans rows",
    "metrics": "metric_points rows",
    "logs": "log_records rows",
}
TIER_META = {
    "ccx13": "2 vCPU / 8 GB",
    "ccx23": "4 vCPU / 16 GB",
    "ccx33": "8 vCPU / 32 GB",
    "ccx43": "16 vCPU / 64 GB",
}
TIER_VCPU = {"ccx13": 2, "ccx23": 4, "ccx33": 8, "ccx43": 16}

# Match loadgen defaults so chart thresholds line up with the JSON's `passed` bit.
SOFT_CLIFF_RATIO = 0.7
ERR_THRESHOLD_PCT = 5.0

STATUS_COLOR = {
    "pass": "#4caf50",
    "soft-cliff": "#ffc107",
    "hard-fail": "#f44336",
    "dead": "#616161",
    "missing": "#f5f5f5",
}


# ---------- Loading + small helpers ----------


def load_runs(results_dir: Path) -> list[dict]:
    runs = []
    for p in sorted(results_dir.glob("*.json")):
        with p.open() as fh:
            doc = json.load(fh)
        if "signal" not in doc:
            print(f"skipping {p.name}: missing 'signal' field (pre-OTLP-split format)", file=sys.stderr)
            continue
        if doc["signal"] not in SIGNALS:
            print(f"skipping {p.name}: unknown signal {doc['signal']!r}", file=sys.stderr)
            continue
        doc.setdefault("scenario", "throughput")
        if doc["scenario"] not in ("throughput", "read-probe"):
            print(f"skipping {p.name}: unknown scenario {doc['scenario']!r}", file=sys.stderr)
            continue
        doc["_path"] = p
        runs.append(doc)
    return runs


def tier_rank(t: str) -> int:
    return TIER_ORDER.index(t) if t in TIER_ORDER else len(TIER_ORDER)


def mode_rank(m: str) -> int:
    return MODE_ORDER.index(m) if m in MODE_ORDER else len(MODE_ORDER)


def mode_color(m: str) -> str:
    return MODE_COLORS.get(m, "#777")


def read_path_for_signal(signal: str) -> str:
    return {
        "spans": "/api/endpoints/grouped",
        "metrics": "/api/metrics/application",
        "logs": "/api/logs",
    }.get(signal, "")


def fmt_count(n: float) -> str:
    n = int(n or 0)
    if n >= 1_000_000_000:
        return f"{n/1_000_000_000:.1f}B"
    if n >= 1_000_000:
        return f"{n/1_000_000:.1f}M"
    if n >= 1_000:
        return f"{n/1_000:.0f}K"
    return f"{n}"


def run_label(run: dict) -> str:
    return f"{run['tier']} / {run['mode']}"


def iter_runs(runs: list[dict], signal: str, scenario: str) -> list[dict]:
    out = [r for r in runs if r["signal"] == signal and r.get("scenario", "throughput") == scenario]
    return sorted(out, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"])))


def phase_steps(run: dict, phase_key: str, sort_by: str = "step") -> list[dict]:
    """Return phase steps in a chosen sort order. Phase 2/3 record execution
    order, not rate order, so sorting before plotting keeps lines monotonic."""
    phase = run.get(phase_key) or {}
    steps = phase.get("steps") or []
    if not steps:
        return []
    if sort_by == "rate":
        return sorted(steps, key=lambda s: (s.get("requestRate", 0), s.get("step", 0)))
    if sort_by == "batch":
        return sorted(steps, key=lambda s: (s.get("batchSize", 0), s.get("step", 0)))
    return sorted(steps, key=lambda s: s.get("step", 0))


def step_status(step: dict) -> str:
    """Categorize a step as pass / soft-cliff / hard-fail / dead.
    `pass` = loadgen marked it passed.
    `dead` = no requests completed (actual=0 or attempted=0).
    `soft-cliff` = achieved fell below SOFT_CLIFF_RATIO × attempted with no error spike.
    `hard-fail` = combined error rate above threshold (default 5%).
    """
    if step.get("passed"):
        return "pass"
    actual = step.get("actualItemsPerSec", 0) or 0
    attempted = step.get("attemptedItemsPerSec", 0) or 0
    err_rate_pct = ((step.get("ingest") or {}).get("errRate", 0) or 0) * 100
    if attempted == 0 or actual == 0:
        return "dead"
    if err_rate_pct > ERR_THRESHOLD_PCT:
        return "hard-fail"
    if attempted > 0 and actual / attempted < SOFT_CLIFF_RATIO:
        return "soft-cliff"
    return "hard-fail"


def headline_source(run: dict) -> tuple[str, dict] | None:
    """Return (phase_key, step) whose actualItemsPerSec equals the headline.
    Returns None if the run has no passing step."""
    headline = run.get("maxSustainableItemsPerSec", 0) or 0
    if headline <= 0:
        return None
    best = None
    best_delta = float("inf")
    for phase_key in ("phase1", "phase2", "phase3"):
        for step in phase_steps(run, phase_key):
            if not step.get("passed"):
                continue
            actual = step.get("actualItemsPerSec", 0) or 0
            delta = abs(actual - headline)
            if delta < best_delta:
                best = (phase_key, step)
                best_delta = delta
    return best


def first_failure(run: dict) -> tuple[str, dict] | None:
    for phase_key in ("phase1", "phase2", "phase3"):
        for step in phase_steps(run, phase_key):
            if not step.get("passed"):
                return (phase_key, step)
    return None


def phase1_p99_at_batch(run: dict, batch_size: int) -> float | None:
    for step in phase_steps(run, "phase1"):
        if step.get("batchSize") == batch_size and step.get("passed"):
            return (step.get("ingest") or {}).get("p99")
    return None


def worst_phase_for_run(run: dict) -> str | None:
    """Pick the phase that produced the lowest passing actualItemsPerSec, or the
    phase that first failed. Used by the CH-pressure chart so the visualisation
    targets the phase where the SUT actually struggled."""
    failing = first_failure(run)
    if failing:
        return failing[0]
    worst_phase = None
    worst_value = float("inf")
    for phase_key in ("phase1", "phase2", "phase3"):
        for step in phase_steps(run, phase_key):
            if not step.get("passed"):
                continue
            actual = step.get("actualItemsPerSec", 0) or 0
            if actual < worst_value:
                worst_value = actual
                worst_phase = phase_key
    return worst_phase


# ---------- Throughput charts ----------


def render_headline_bar(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = iter_runs(runs, signal, "throughput")
    if not sig_runs:
        return

    tiers_present = sorted({r["tier"] for r in sig_runs}, key=tier_rank)
    modes_present = sorted({r["mode"] for r in sig_runs}, key=mode_rank)
    if not tiers_present or not modes_present:
        return

    fig, ax = plt.subplots(figsize=(max(8, len(tiers_present) * 2.5), 6))

    values = [r.get("maxSustainableItemsPerSec", 0) or 0 for r in sig_runs]
    ymax = max(values) if values else 1
    ghost_height = max(ymax * 0.02, 1)

    width = 0.8 / max(len(modes_present), 1)
    x_centers = list(range(len(tiers_present)))

    any_ch_restart = False
    for mi, mode in enumerate(modes_present):
        for ti, tier in enumerate(tiers_present):
            match = [r for r in sig_runs if r["tier"] == tier and r["mode"] == mode]
            x = ti + (mi - (len(modes_present) - 1) / 2) * width
            if not match:
                continue
            run = match[0]
            value = run.get("maxSustainableItemsPerSec", 0) or 0
            ch_restart = bool(run.get("chRestarted"))
            if ch_restart:
                any_ch_restart = True
            if value > 0:
                bar_kwargs = {"width": width, "color": mode_color(mode)}
                if ch_restart:
                    bar_kwargs.update({"edgecolor": "#aa3333", "hatch": "xx", "linewidth": 1.2})
                ax.bar(x, value, **bar_kwargs)
                src = headline_source(run)
                phase_tag = f" ({PHASE_LABEL[src[0]]})" if src else ""
                restart_tag = " CH↻" if ch_restart else ""
                ax.annotate(
                    f"{int(value):,}{phase_tag}{restart_tag}",
                    xy=(x, value), xytext=(0, 4), textcoords="offset points",
                    ha="center", va="bottom", fontsize=8,
                    color="#aa3333" if ch_restart else "black",
                )
            else:
                ax.bar(x, ghost_height, width=width, color="#dddddd",
                       edgecolor="#999999", hatch="///", linewidth=0.7)
                fail = first_failure(run)
                if fail:
                    pkey, step = fail
                    label = f"failed @ {PHASE_LABEL[pkey]} step {step.get('step', '?')}"
                else:
                    label = "failed"
                ax.annotate(
                    label, xy=(x, ghost_height), xytext=(0, 4), textcoords="offset points",
                    ha="center", va="bottom", fontsize=7, color="#aa3333",
                )

    ax.set_xticks(x_centers)
    ax.set_xticklabels(tiers_present)
    ax.set_ylabel(f"Max sustainable {ITEM_LABEL[signal]}")
    ax.set_title(
        f"Traceway: max sustainable {signal} ingest by hardware tier\n"
        "Phase tag (P1/P2/P3) shows which workload shape produced the headline"
    )

    legend_handles = [Patch(facecolor=mode_color(m), label=m) for m in modes_present]
    legend_handles.append(Patch(facecolor="#dddddd", edgecolor="#999999", hatch="///", label="all phases failed"))
    if any_ch_restart:
        legend_handles.append(Patch(facecolor="white", edgecolor="#aa3333", hatch="xx", label="CH restart during run (result invalid)"))
    ax.legend(handles=legend_handles, title="DB mode", loc="upper left")

    ax.grid(axis="y", linestyle=":", alpha=0.4)
    ax.set_ylim(0, ymax * 1.3 if ymax > 0 else 1)

    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_phase1_latency(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = [r for r in iter_runs(runs, signal, "throughput") if phase_steps(r, "phase1")]
    if not sig_runs:
        return

    fig, ax = plt.subplots(figsize=(11, 6))
    plotted = False

    for run in sig_runs:
        color = mode_color(run["mode"])
        steps = phase_steps(run, "phase1", sort_by="batch")
        batches = [s["batchSize"] for s in steps]
        p50 = [(s.get("ingest") or {}).get("p50", 0) for s in steps]
        p95 = [(s.get("ingest") or {}).get("p95", 0) for s in steps]
        p99 = [(s.get("ingest") or {}).get("p99", 0) for s in steps]
        passed = [bool(s.get("passed")) for s in steps]

        def _safe(xs, ys):
            return [(x, y) for x, y in zip(xs, ys) if y and y > 0]

        for percs, ls, alpha in [(p50, ":", 0.5), (p95, "-.", 0.7), (p99, "-", 1.0)]:
            pts = _safe(batches, percs)
            if not pts:
                continue
            xs, ys = zip(*pts)
            label = run_label(run) if ls == "-" else None
            ax.plot(xs, ys, linestyle=ls, color=color, alpha=alpha, label=label)
            plotted = True

        for b, lat, p in zip(batches, p99, passed):
            if lat and lat > 0:
                ax.plot(b, lat,
                        marker=("o" if p else "x"),
                        color=color, markersize=8,
                        markeredgewidth=2 if not p else 1, linestyle="")

    if not plotted:
        plt.close(fig)
        return

    ax.set_xlabel("Batch size (items/request, log)")
    ax.set_ylabel("Ingest latency (ms, log)\nP50 ··  P95 -·  P99 —")
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_title(
        f"Phase 1 — batch-size scaling — {signal}\n"
        "Fixed 5 req/sec.  Solid line = P99 per run, dot/dash = P50/P95.  × = step failed."
    )
    ax.grid(True, which="both", linestyle=":", alpha=0.4)
    ax.legend(loc="upper left", fontsize=9, title="P99 line per run")
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def _render_rate_ramp(runs: list[dict], signal: str, phase_key: str, subtitle: str, out: Path) -> None:
    sig_runs = [r for r in iter_runs(runs, signal, "throughput") if phase_steps(r, phase_key)]
    if not sig_runs:
        return

    fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(11, 12), sharex=True)
    plotted = False

    for run in sig_runs:
        color = mode_color(run["mode"])
        label = run_label(run)
        steps = phase_steps(run, phase_key, sort_by="rate")

        rates = [s["requestRate"] for s in steps]
        actual = [s["actualItemsPerSec"] for s in steps]
        attempted = [s["attemptedItemsPerSec"] for s in steps]
        p50 = [(s.get("ingest") or {}).get("p50", 0) for s in steps]
        p95 = [(s.get("ingest") or {}).get("p95", 0) for s in steps]
        p99 = [(s.get("ingest") or {}).get("p99", 0) for s in steps]
        errs_pct = [((s.get("ingest") or {}).get("errRate", 0) or 0) * 100 for s in steps]
        passed = [bool(s.get("passed")) for s in steps]

        # Top: throughput. Drop zeros for log scale.
        act_pts = [(r, a) for r, a in zip(rates, actual) if a and a > 0]
        att_pts = [(r, a) for r, a in zip(rates, attempted) if a and a > 0]
        if act_pts:
            xs, ys = zip(*act_pts)
            ax1.plot(xs, ys, linestyle="-", color=color, label=label)
            plotted = True
        if att_pts:
            xs, ys = zip(*att_pts)
            ax1.plot(xs, ys, linestyle="--", color=color, alpha=0.4)
        for r, a, p in zip(rates, actual, passed):
            if a and a > 0:
                ax1.plot(r, a, marker=("o" if p else "x"), color=color,
                         markersize=8, markeredgewidth=2 if not p else 1, linestyle="")

        # Middle: latency percentiles.
        for percs, ls, alpha in [(p50, ":", 0.5), (p95, "-.", 0.7), (p99, "-", 1.0)]:
            pts = [(r, l) for r, l in zip(rates, percs) if l and l > 0]
            if pts:
                xs, ys = zip(*pts)
                ax2.plot(xs, ys, linestyle=ls, color=color, alpha=alpha)
        # P1 floor overlay
        fixed_batch = (run.get(phase_key) or {}).get("fixedBatchSize", 0)
        if fixed_batch:
            floor = phase1_p99_at_batch(run, fixed_batch)
            if floor:
                ax2.axhline(floor, linestyle=":", color=color, alpha=0.35, linewidth=1)
        for r, lat, p in zip(rates, p99, passed):
            if lat and lat > 0:
                ax2.plot(r, lat, marker=("o" if p else "x"), color=color,
                         markersize=8, markeredgewidth=2 if not p else 1, linestyle="")

        # Bottom: error %
        ax3.plot(rates, errs_pct, linestyle="-", color=color, alpha=0.5)
        for r, e, p in zip(rates, errs_pct, passed):
            ax3.plot(r, e, marker=("o" if p else "x"), color=color,
                     markersize=8, markeredgewidth=2 if not p else 1, linestyle="")

        # Bottom right twin: rejected
        rej = [s.get("rejected", 0) for s in steps]
        if any(rej):
            if not hasattr(ax3, "_rej_twin"):
                ax3._rej_twin = ax3.twinx()
            ax3._rej_twin.plot(rates, rej, linestyle=":", color=color, alpha=0.4)

    if not plotted:
        plt.close(fig)
        return

    ax1.set_ylabel(f"items/sec (log)\n— actual,  -- attempted")
    ax1.set_xscale("log")
    ax1.set_yscale("log")
    ax1.grid(True, which="both", linestyle=":", alpha=0.4)
    ax1.legend(loc="lower right", fontsize=8)

    ax2.set_ylabel("Ingest latency (ms, log)\nP50 ··  P95 -·  P99 —\n(thin dotted = P1 floor)")
    ax2.set_yscale("log")
    ax2.grid(True, which="both", linestyle=":", alpha=0.4)

    ax3.axhline(ERR_THRESHOLD_PCT, linestyle="--", color="#cc3333", alpha=0.5, linewidth=1)
    ax3.set_ylabel("Error rate (%)\n× = failed step")
    ax3.set_xlabel("Request rate (req/sec, log)")
    ax3.set_xscale("log")
    ax3.grid(True, which="both", linestyle=":", alpha=0.4)
    if hasattr(ax3, "_rej_twin"):
        ax3._rej_twin.set_ylabel("OTLP rejected items (dotted)", color="#666666")

    fig.suptitle(f"{subtitle} — {signal}\n× marker = failed step (err > 5% or soft-cliff). Dotted P1 floor = no-concurrency latency.")
    fig.tight_layout(rect=(0, 0, 1, 0.97))
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_phase2_rate(runs: list[dict], signal: str, out: Path) -> None:
    _render_rate_ramp(runs, signal, "phase2",
                      "Phase 2 — collector-shape rate ramp (fixed fat batch)", out)


def render_phase3_rate(runs: list[dict], signal: str, out: Path) -> None:
    # Only render when at least one run has phase3 data.
    any_p3 = any(phase_steps(r, "phase3") for r in iter_runs(runs, signal, "throughput"))
    if not any_p3:
        return
    _render_rate_ramp(runs, signal, "phase3",
                      "Phase 3 — SDK-fleet rate ramp (small batch, high rate)", out)


def render_pareto(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = iter_runs(runs, signal, "throughput")
    if not sig_runs:
        return

    fig, ax = plt.subplots(figsize=(11, 7))
    plotted = False
    dead_runs = []  # (run_label, list_of_phase_keys+step_n)

    for run in sig_runs:
        color = mode_color(run["mode"])
        label = run_label(run)

        passed_pts = []
        failed_pts = []
        for phase_key in ("phase2", "phase3"):
            for step in phase_steps(run, phase_key):
                p99 = (step.get("ingest") or {}).get("p99", 0)
                actual = step.get("actualItemsPerSec", 0) or 0
                if p99 <= 0 or actual <= 0:
                    if step.get("passed") is False:
                        dead_runs.append((label, f"{PHASE_LABEL[phase_key]}s{step.get('step', '?')}"))
                    continue
                if step.get("passed"):
                    passed_pts.append((p99, actual))
                else:
                    failed_pts.append((p99, actual))

        if passed_pts:
            sorted_pts = sorted(passed_pts, key=lambda p: p[0])
            xs, ys = zip(*sorted_pts)
            ax.plot(xs, ys, linestyle=":", color=color, alpha=0.4)
            ax.scatter(xs, ys, marker="o", color=color, s=70, edgecolor="black",
                       linewidth=0.5, label=label, zorder=3)
            plotted = True
        if failed_pts:
            xs, ys = zip(*failed_pts)
            ax.scatter(xs, ys, marker="x", color=color, s=100, linewidth=2, zorder=4)
            plotted = True

    if not plotted and not dead_runs:
        plt.close(fig)
        return

    ax.set_xlabel("Ingest P99 latency (ms, log)")
    ax.set_ylabel(f"Achieved {ITEM_LABEL[signal]} (log)")
    if plotted:
        ax.set_xscale("log")
        ax.set_yscale("log")
    ax.set_title(
        f"Latency–throughput Pareto — {signal}\n"
        "Every Phase 2/3 step.  ○ = passed,  × = failed.  Up-and-left is better."
    )
    ax.grid(True, which="both", linestyle=":", alpha=0.4)

    # Annotate runs that had only dead (0-actual) failures
    if dead_runs:
        dead_by_run: dict[str, list[str]] = {}
        for lab, key in dead_runs:
            dead_by_run.setdefault(lab, []).append(key)
        text = "Zero-throughput failures: " + "; ".join(
            f"{lab}@{','.join(keys)}" for lab, keys in dead_by_run.items()
        )
        ax.text(0.5, -0.15, text, transform=ax.transAxes, ha="center", va="top",
                fontsize=8, color="#aa3333")
        fig.subplots_adjust(bottom=0.18)

    if plotted:
        ax.legend(loc="upper left", fontsize=9)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_tier_scaling(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = iter_runs(runs, signal, "throughput")
    if not sig_runs:
        return

    tiers_present = sorted({r["tier"] for r in sig_runs}, key=tier_rank)
    modes_present = sorted({r["mode"] for r in sig_runs}, key=mode_rank)
    if not tiers_present:
        return

    fig, ax = plt.subplots(figsize=(10, 6))
    plotted = False
    x_positions = list(range(len(tiers_present)))

    for mode in modes_present:
        xs, ys = [], []
        for i, tier in enumerate(tiers_present):
            match = [r for r in sig_runs if r["tier"] == tier and r["mode"] == mode]
            if not match:
                continue
            value = match[0].get("maxSustainableItemsPerSec", 0) or 0
            if value <= 0:
                continue
            xs.append(i)
            ys.append(value)
        if not xs:
            continue
        ax.plot(xs, ys, marker="o", color=mode_color(mode), markersize=9,
                linewidth=2, label=mode)
        for x, y in zip(xs, ys):
            ax.annotate(fmt_count(y), xy=(x, y), xytext=(0, 8),
                        textcoords="offset points", ha="center", va="bottom", fontsize=9)
        # Linear-vCPU reference, anchored at the smallest tier of THIS mode
        base_x = xs[0]
        base_y = ys[0]
        base_vcpu = TIER_VCPU.get(tiers_present[base_x], 1)
        ref_xs = list(range(len(tiers_present)))
        ref_ys = [base_y * (TIER_VCPU.get(tiers_present[i], 1) / base_vcpu) for i in ref_xs]
        ax.plot(ref_xs, ref_ys, linestyle="--", color=mode_color(mode), alpha=0.3, linewidth=1)
        plotted = True

    if not plotted:
        plt.close(fig)
        return

    ax.set_xticks(x_positions)
    ax.set_xticklabels([f"{t}\n{TIER_META.get(t, '')}" for t in tiers_present])
    ax.set_ylabel(f"Max sustainable {ITEM_LABEL[signal]}")
    ax.set_title(
        f"Tier scaling — {signal}\n"
        "Dashed = linear-vCPU reference (anchored at smallest present tier).  "
        "Distance below dashed = scaling penalty."
    )
    ax.legend(title="DB mode", loc="upper left")
    ax.grid(True, linestyle=":", alpha=0.4)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_cliff_grid(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = iter_runs(runs, signal, "throughput")
    if not sig_runs:
        return

    n_p1 = max((len(phase_steps(r, "phase1")) for r in sig_runs), default=0)
    n_p2 = max((len(phase_steps(r, "phase2")) for r in sig_runs), default=0)
    n_p3 = max((len(phase_steps(r, "phase3")) for r in sig_runs), default=0)
    total_cols = n_p1 + n_p2 + n_p3
    if total_cols == 0:
        return

    n_rows = len(sig_runs)
    fig_w = max(11, total_cols * 1.55 + 4)
    fig_h = max(4.5, n_rows * 0.95 + 2.2)
    fig, ax = plt.subplots(figsize=(fig_w, fig_h))

    for ri, run in enumerate(sig_runs):
        y = n_rows - ri - 1
        ax.text(-0.5, y + 0.5, run_label(run), ha="right", va="center", fontsize=10, fontweight="bold")

        col_x = 0
        for pk, n_cols in [("phase1", n_p1), ("phase2", n_p2), ("phase3", n_p3)]:
            steps = phase_steps(run, pk, sort_by="step")
            for ci in range(n_cols):
                x = col_x + ci
                if ci < len(steps):
                    step = steps[ci]
                    color = STATUS_COLOR[step_status(step)]
                    bs = step.get("batchSize", 0)
                    r = step.get("requestRate", 0)
                    rect = Rectangle((x, y), 1, 1, facecolor=color,
                                     edgecolor="white", linewidth=1.5)
                    ax.add_patch(rect)
                    ax.text(x + 0.5, y + 0.5, f"bs={fmt_count(bs)}\nr={r:g}",
                            ha="center", va="center", fontsize=7.5)
                else:
                    rect = Rectangle((x, y), 1, 1, facecolor=STATUS_COLOR["missing"],
                                     edgecolor="white", linewidth=1.5)
                    ax.add_patch(rect)
            col_x += n_cols

        headline = run.get("maxSustainableItemsPerSec", 0) or 0
        margin = total_cols + 0.35
        if headline > 0:
            ax.text(margin, y + 0.5, f"{fmt_count(headline)}/s",
                    ha="left", va="center", fontsize=10, fontweight="bold")
        else:
            ax.text(margin, y + 0.5, "failed",
                    ha="left", va="center", fontsize=10, color="#cc3333", fontweight="bold")

    # Phase headers + separators
    headers = []
    if n_p1 > 0:
        headers.append((n_p1 / 2.0, "Phase 1 (batch ramp)"))
    if n_p2 > 0:
        headers.append((n_p1 + n_p2 / 2.0, "Phase 2 (collector rate ramp)"))
    if n_p3 > 0:
        headers.append((n_p1 + n_p2 + n_p3 / 2.0, "Phase 3 (SDK-fleet rate ramp)"))
    for cx, label in headers:
        ax.text(cx, n_rows + 0.25, label, ha="center", va="bottom",
                fontsize=10.5, fontweight="bold")

    for sep in [n_p1, n_p1 + n_p2]:
        if 0 < sep < total_cols:
            ax.plot([sep, sep], [0, n_rows], color="black", linewidth=2)

    ax.set_xlim(-4, total_cols + 5)
    ax.set_ylim(-0.6, n_rows + 0.9)
    ax.set_aspect("equal")
    ax.axis("off")

    legend = [
        Patch(facecolor=STATUS_COLOR["pass"], label="pass"),
        Patch(facecolor=STATUS_COLOR["soft-cliff"], label=f"soft-cliff (<{int(SOFT_CLIFF_RATIO*100)}% achieved)"),
        Patch(facecolor=STATUS_COLOR["hard-fail"], label=f"hard fail (err > {ERR_THRESHOLD_PCT:.0f}%)"),
        Patch(facecolor=STATUS_COLOR["dead"], label="0 requests completed"),
        Patch(facecolor=STATUS_COLOR["missing"], edgecolor="#cccccc", label="not run"),
    ]
    ax.legend(handles=legend, loc="lower center", bbox_to_anchor=(0.5, -0.08),
              ncol=5, fontsize=9, frameon=False)
    fig.suptitle(f"Step-status grid — {signal}\nbs = batch size, r = request rate (req/sec)",
                 fontsize=12)
    fig.tight_layout(rect=(0, 0.04, 1, 0.95))
    fig.savefig(out, dpi=130, bbox_inches="tight")
    plt.close(fig)


def render_batch_efficiency(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = [r for r in iter_runs(runs, signal, "throughput") if phase_steps(r, "phase1")]
    if not sig_runs:
        return

    fig, ax = plt.subplots(figsize=(11, 6))
    plotted = False

    for run in sig_runs:
        color = mode_color(run["mode"])
        steps = phase_steps(run, "phase1", sort_by="batch")
        batches = [s["batchSize"] for s in steps]
        actual = [s["actualItemsPerSec"] for s in steps]
        attempted = [s["attemptedItemsPerSec"] for s in steps]
        passed = [bool(s.get("passed")) for s in steps]

        # log scale; drop zeros
        act_pts = [(b, a) for b, a in zip(batches, actual) if a and a > 0]
        att_pts = [(b, a) for b, a in zip(batches, attempted) if a and a > 0]

        if act_pts:
            xs, ys = zip(*act_pts)
            ax.plot(xs, ys, linestyle="-", color=color, marker="o", label=run_label(run))
            plotted = True
        if att_pts:
            xs, ys = zip(*att_pts)
            ax.plot(xs, ys, linestyle="--", color=color, alpha=0.35)

        for b, a, p in zip(batches, actual, passed):
            if not p and a and a > 0:
                ax.plot(b, a, marker="x", color=color, markersize=11,
                        markeredgewidth=2.2, linestyle="")

    if not plotted:
        plt.close(fig)
        return

    ax.set_xlabel("Batch size (items/request, log)")
    ax.set_ylabel(f"{ITEM_LABEL[signal]} (log)\n— actual,  -- attempted (batch × 5 req/s)")
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_title(
        f"Batch efficiency — {signal}\n"
        "Phase 1 at fixed 5 req/sec.  Solid below dashed = SUT dropping batches."
    )
    ax.grid(True, which="both", linestyle=":", alpha=0.4)
    ax.legend(loc="upper left", fontsize=9)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_ch_pressure(runs: list[dict], signal: str, out: Path) -> None:
    """For each (tier, mode) of a signal, plot CH parts count and active merges
    across the steps of its worst-performing phase. Makes the CH-side story —
    "parts piled up before the cliff" or "merges were idle and the cliff was
    backend-side" — visible at a glance."""
    sig_runs = iter_runs(runs, signal, "throughput")
    sig_runs = [r for r in sig_runs if any(
        (step.get("ch") or {}).get("reachable")
        for phase_key in ("phase1", "phase2", "phase3")
        for step in phase_steps(r, phase_key)
    )]
    if not sig_runs:
        return

    cols = min(len(sig_runs), 3)
    rows = (len(sig_runs) + cols - 1) // cols
    fig, axes = plt.subplots(rows, cols, figsize=(cols * 5.5, rows * 3.5), squeeze=False)

    for idx, run in enumerate(sig_runs):
        r, c = divmod(idx, cols)
        ax_parts = axes[r][c]
        ax_merges = ax_parts.twinx()

        phase_key = worst_phase_for_run(run)
        if not phase_key:
            ax_parts.set_visible(False)
            ax_merges.set_visible(False)
            continue
        steps = phase_steps(run, phase_key)
        if not steps:
            ax_parts.set_visible(False)
            ax_merges.set_visible(False)
            continue

        xs = [s.get("step", i + 1) for i, s in enumerate(steps)]
        parts = [(s.get("ch") or {}).get("partsCount", 0) for s in steps]
        merges = [(s.get("ch") or {}).get("activeMerges", 0) for s in steps]
        first_failed_step = next((s.get("step") for s in steps if not s.get("passed")), None)

        ax_parts.plot(xs, parts, marker="o", color="#1f77b4", label="parts")
        ax_merges.plot(xs, merges, marker="s", color="#d62728", linestyle="--", label="active merges")

        if first_failed_step is not None:
            ax_parts.axvline(first_failed_step, color="#aa3333", linestyle=":", alpha=0.6)

        title = f"{run_label(run)} — {PHASE_LABEL[phase_key]}"
        if run.get("chRestarted"):
            title += " (CH↻)"
        ax_parts.set_title(title, fontsize=10)
        ax_parts.set_xlabel("step")
        ax_parts.set_ylabel("parts (total, active)", color="#1f77b4")
        ax_merges.set_ylabel("active merges", color="#d62728")
        ax_parts.grid(True, linestyle=":", alpha=0.4)

    for idx in range(len(sig_runs), rows * cols):
        r, c = divmod(idx, cols)
        axes[r][c].set_visible(False)

    fig.suptitle(f"ClickHouse pressure during the worst-performing phase — {signal}", y=1.02)
    fig.tight_layout()
    fig.savefig(out, dpi=130, bbox_inches="tight")
    plt.close(fig)


def render_signal_mix(runs: list[dict], out: Path) -> None:
    """Cross-signal throughput comparison: bars per (tier, mode), one bar per signal."""
    thr_runs = [r for r in runs if r.get("scenario", "throughput") == "throughput"]
    if not thr_runs:
        return

    keys = sorted({(r["tier"], r["mode"]) for r in thr_runs},
                  key=lambda k: (tier_rank(k[0]), mode_rank(k[1])))
    if not keys:
        return

    fig, ax = plt.subplots(figsize=(max(8, len(keys) * 1.9), 6))

    width = 0.25
    x = list(range(len(keys)))
    any_value = False

    for si, signal in enumerate(SIGNALS):
        ys = []
        for tier, mode in keys:
            match = [r for r in thr_runs if r["tier"] == tier and r["mode"] == mode and r["signal"] == signal]
            ys.append((match[0].get("maxSustainableItemsPerSec", 0) or 0) if match else 0)
        offsets = [xi + (si - 1) * width for xi in x]
        bars = ax.bar(offsets, ys, width=width, color=SIGNAL_COLORS[signal], label=signal)
        for b, y in zip(bars, ys):
            if y > 0:
                ax.annotate(fmt_count(y), xy=(b.get_x() + b.get_width() / 2, y),
                            xytext=(0, 3), textcoords="offset points",
                            ha="center", va="bottom", fontsize=7)
                any_value = True

    if not any_value:
        plt.close(fig)
        return

    ax.set_xticks(x)
    ax.set_xticklabels([f"{t}\n{m}" for t, m in keys], fontsize=9)
    ax.set_ylabel("Max sustainable items/sec (Y unit varies by signal)")
    ax.set_title("Cross-signal comparison — per (tier × mode), spans vs metrics vs logs")
    ax.legend(title="Signal")
    ax.grid(axis="y", linestyle=":", alpha=0.4)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


# ---------- Read-probe charts ----------


def _readprobe_steps(run: dict) -> list[dict]:
    return ((run.get("readProbe") or {}).get("steps") or [])


def render_readprobe_headline(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = iter_runs(runs, signal, "read-probe")
    if not sig_runs:
        return

    tiers_present = sorted({r["tier"] for r in sig_runs}, key=tier_rank)
    modes_present = sorted({r["mode"] for r in sig_runs}, key=mode_rank)
    if not tiers_present:
        return

    fig, ax = plt.subplots(figsize=(max(8, len(tiers_present) * 2.5), 6))

    values = [r.get("maxFillLevelPassed", 0) or 0 for r in sig_runs]
    ymax = max(values) if values else 1
    ghost_height = max(ymax * 0.02, 1)

    width = 0.8 / max(len(modes_present), 1)
    x_centers = list(range(len(tiers_present)))

    for mi, mode in enumerate(modes_present):
        for ti, tier in enumerate(tiers_present):
            match = [r for r in sig_runs if r["tier"] == tier and r["mode"] == mode]
            x = ti + (mi - (len(modes_present) - 1) / 2) * width
            if not match:
                continue
            run = match[0]
            value = run.get("maxFillLevelPassed", 0) or 0
            if value > 0:
                ax.bar(x, value, width=width, color=mode_color(mode))
                # latency at max
                last_pass = None
                for step in _readprobe_steps(run):
                    if step.get("passed"):
                        last_pass = step
                lat_str = f"\n{int(last_pass['readLatencyMs'])}ms" if last_pass else ""
                ax.annotate(
                    f"{fmt_count(value)}{lat_str}",
                    xy=(x, value), xytext=(0, 4), textcoords="offset points",
                    ha="center", va="bottom", fontsize=8,
                )
            else:
                ax.bar(x, ghost_height, width=width, color="#dddddd",
                       edgecolor="#999999", hatch="///", linewidth=0.7)
                ax.annotate(
                    "no fill passed",
                    xy=(x, ghost_height), xytext=(0, 4), textcoords="offset points",
                    ha="center", va="bottom", fontsize=7, color="#aa3333",
                )

    ax.set_xticks(x_centers)
    ax.set_xticklabels(tiers_present)
    ax.set_yscale("log")
    ax.set_ylabel(f"Max passing fill level — {ROW_LABEL[signal]} (log)")
    ax.set_title(
        f"Read-probe headline — {signal} via {read_path_for_signal(signal)}\n"
        "Bar label: max passing fill + read latency at that fill"
    )

    legend = [Patch(facecolor=mode_color(m), label=m) for m in modes_present]
    legend.append(Patch(facecolor="#dddddd", edgecolor="#999999", hatch="///", label="no fill level passed"))
    ax.legend(handles=legend, title="DB mode", loc="upper left")
    ax.grid(axis="y", which="both", linestyle=":", alpha=0.4)

    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_readprobe(runs: list[dict], signal: str, out: Path) -> None:
    """Read latency vs rows ingested — improved over legacy chart-readprobe."""
    sig_runs = [r for r in iter_runs(runs, signal, "read-probe") if _readprobe_steps(r)]
    if not sig_runs:
        return

    fig, ax = plt.subplots(figsize=(11, 6))
    plotted = False
    threshold_ms = 5000

    for run in sig_runs:
        color = mode_color(run["mode"])
        rp = run.get("readProbe") or {}
        threshold_ms = rp.get("readThresholdMs", threshold_ms)
        steps = _readprobe_steps(run)
        xs = [s["rowsIngested"] for s in steps]
        ys = [s["readLatencyMs"] for s in steps]
        passed = [bool(s.get("passed")) for s in steps]
        if not xs:
            continue
        ax.plot(xs, ys, linestyle="-", color=color, label=run_label(run))
        for x, y, p in zip(xs, ys, passed):
            ax.plot(x, y, marker=("o" if p else "x"), color=color,
                    markersize=9, markeredgewidth=2 if not p else 1, linestyle="")
        plotted = True

    if not plotted:
        plt.close(fig)
        return

    ax.axhline(threshold_ms, linestyle="--", color="#cc3333", alpha=0.7,
               label=f"threshold ({threshold_ms}ms)")
    ax.set_xlabel(f"{ROW_LABEL[signal]} ingested (log)")
    ax.set_ylabel("Read latency (ms, log)")
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_title(
        f"Read latency vs table size — {signal} via {read_path_for_signal(signal)}\n"
        "○ = passed (latency under threshold),  × = failed step"
    )
    ax.grid(True, which="both", linestyle=":", alpha=0.4)
    ax.legend(loc="upper left", fontsize=9)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_readprobe_per_endpoint(runs: list[dict], signal: str, out: Path) -> None:
    """Latency vs rows ingested, **one line per probed endpoint**, per (tier, mode).
    Shows which specific query slows down first as the table grows. Replaces
    the single-line render_readprobe view when the JSON has per-endpoint
    probe data (T2+). Older JSON without `step["probes"]` falls back to the
    median-only view in render_readprobe."""
    sig_runs = [r for r in iter_runs(runs, signal, "read-probe") if _readprobe_steps(r)]
    sig_runs = [r for r in sig_runs if any(s.get("probes") for s in _readprobe_steps(r))]
    if not sig_runs:
        return

    fig, ax = plt.subplots(figsize=(12, 7))
    plotted = False
    threshold_ms = 5000
    # Deterministic palette per endpoint name. Falls back to grey if more than 6.
    endpoint_palette = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b"]
    endpoint_color: dict[str, str] = {}

    for run in sig_runs:
        rp = run.get("readProbe") or {}
        threshold_ms = rp.get("readThresholdMs", threshold_ms)
        steps = _readprobe_steps(run)
        if not steps:
            continue

        # Collect per-endpoint (rowsIngested, latencyMs, ok) series.
        by_endpoint: dict[str, list[tuple[float, float, bool]]] = {}
        for s in steps:
            rows = s.get("rowsIngested", 0)
            for p in (s.get("probes") or []):
                name = p.get("name") or p.get("path") or "unknown"
                by_endpoint.setdefault(name, []).append((rows, p.get("latencyMs", 0), bool(p.get("ok", False))))

        run_lbl = run_label(run)
        for name, series in by_endpoint.items():
            if name not in endpoint_color:
                endpoint_color[name] = endpoint_palette[len(endpoint_color) % len(endpoint_palette)]
            color = endpoint_color[name]
            xs = [pt[0] for pt in series]
            ys = [pt[1] for pt in series]
            oks = [pt[2] for pt in series]
            ax.plot(xs, ys, linestyle="-", color=color, alpha=0.85,
                    label=f"{run_lbl} · {name}")
            for x, y, ok in zip(xs, ys, oks):
                ax.plot(x, y, marker=("o" if ok else "x"), color=color,
                        markersize=8, markeredgewidth=2 if not ok else 1, linestyle="")
            plotted = True

    if not plotted:
        plt.close(fig)
        return

    ax.axhline(threshold_ms, linestyle="--", color="#cc3333", alpha=0.7,
               label=f"threshold ({threshold_ms}ms)")
    ax.set_xlabel(f"{ROW_LABEL[signal]} ingested (log)")
    ax.set_ylabel("Read latency (ms, log)")
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_title(
        f"Per-endpoint read latency vs table size — {signal}\n"
        "○ = HTTP ok,  × = HTTP error.  Step pass uses median across endpoints."
    )
    ax.grid(True, which="both", linestyle=":", alpha=0.4)
    ax.legend(loc="upper left", fontsize=8, ncol=1)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_readprobe_ingest_time(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = [r for r in iter_runs(runs, signal, "read-probe") if _readprobe_steps(r)]
    if not sig_runs:
        return

    fig, ax = plt.subplots(figsize=(11, 6))
    plotted = False

    for run in sig_runs:
        color = mode_color(run["mode"])
        steps = _readprobe_steps(run)
        xs = [s["fillLevelTarget"] for s in steps]
        ys = [s.get("ingestSecondsElapsed", 0) for s in steps]
        pts = [(x, y) for x, y in zip(xs, ys) if y and y > 0]
        if not pts:
            continue
        xs2, ys2 = zip(*pts)
        ax.plot(xs2, ys2, linestyle="-", marker="o", color=color, label=run_label(run))
        plotted = True

    if not plotted:
        plt.close(fig)
        return

    ax.set_xlabel(f"Target fill level — {ROW_LABEL[signal]} (log)")
    ax.set_ylabel("Time to reach fill level (seconds, log)")
    ax.set_xscale("log")
    ax.set_yscale("log")
    ax.set_title(
        f"Fill time vs target rows — {signal}\n"
        "Curve sag = compaction/merge cost rising with table size."
    )
    ax.grid(True, which="both", linestyle=":", alpha=0.4)
    ax.legend(loc="upper left", fontsize=9)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_readprobe_tier_scaling(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = iter_runs(runs, signal, "read-probe")
    if not sig_runs:
        return

    tiers_present = sorted({r["tier"] for r in sig_runs}, key=tier_rank)
    modes_present = sorted({r["mode"] for r in sig_runs}, key=mode_rank)
    if not tiers_present:
        return

    fig, ax = plt.subplots(figsize=(10, 6))
    plotted = False
    x_positions = list(range(len(tiers_present)))

    for mode in modes_present:
        xs, ys = [], []
        for i, tier in enumerate(tiers_present):
            match = [r for r in sig_runs if r["tier"] == tier and r["mode"] == mode]
            if not match:
                continue
            value = match[0].get("maxFillLevelPassed", 0) or 0
            if value <= 0:
                continue
            xs.append(i)
            ys.append(value)
        if not xs:
            continue
        ax.plot(xs, ys, marker="o", color=mode_color(mode), markersize=9,
                linewidth=2, label=mode)
        for x, y in zip(xs, ys):
            ax.annotate(fmt_count(y), xy=(x, y), xytext=(0, 8),
                        textcoords="offset points", ha="center", va="bottom", fontsize=9)
        base_x = xs[0]
        base_y = ys[0]
        base_vcpu = TIER_VCPU.get(tiers_present[base_x], 1)
        ref_xs = list(range(len(tiers_present)))
        ref_ys = [base_y * (TIER_VCPU.get(tiers_present[i], 1) / base_vcpu) for i in ref_xs]
        ax.plot(ref_xs, ref_ys, linestyle="--", color=mode_color(mode), alpha=0.3, linewidth=1)
        plotted = True

    if not plotted:
        plt.close(fig)
        return

    ax.set_xticks(x_positions)
    ax.set_xticklabels([f"{t}\n{TIER_META.get(t, '')}" for t in tiers_present])
    ax.set_yscale("log")
    ax.set_ylabel(f"Max passing fill level — {ROW_LABEL[signal]} (log)")
    ax.set_title(
        f"Read-probe tier scaling — {signal} via {read_path_for_signal(signal)}\n"
        "Dashed = linear-vCPU reference anchored at smallest tier."
    )
    ax.legend(title="DB mode", loc="upper left")
    ax.grid(True, which="both", linestyle=":", alpha=0.4)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


def render_readprobe_cliff(runs: list[dict], signal: str, out: Path) -> None:
    sig_runs = iter_runs(runs, signal, "read-probe")
    if not sig_runs:
        return

    # Build the union of all fill levels seen across runs, sorted.
    all_targets = sorted({s["fillLevelTarget"] for r in sig_runs for s in _readprobe_steps(r)})
    if not all_targets:
        return

    n_cols = len(all_targets)
    n_rows = len(sig_runs)

    fig_w = max(10, n_cols * 1.8 + 3.5)
    fig_h = max(4, n_rows * 0.9 + 2)
    fig, ax = plt.subplots(figsize=(fig_w, fig_h))

    def _probe_status(step: dict) -> str:
        if step.get("passed"):
            return "pass"
        # over-threshold but read OK
        if step.get("readOk"):
            return "soft-cliff"
        return "hard-fail"

    for ri, run in enumerate(sig_runs):
        y = n_rows - ri - 1
        ax.text(-0.5, y + 0.5, run_label(run), ha="right", va="center",
                fontsize=10, fontweight="bold")

        by_target = {s["fillLevelTarget"]: s for s in _readprobe_steps(run)}
        for ci, target in enumerate(all_targets):
            x = ci
            step = by_target.get(target)
            if step is None:
                color = STATUS_COLOR["missing"]
                label_text = ""
            else:
                color = STATUS_COLOR[_probe_status(step)]
                lat = int(step.get("readLatencyMs", 0))
                label_text = f"{lat}ms"
            ax.add_patch(Rectangle((x, y), 1, 1, facecolor=color,
                                   edgecolor="white", linewidth=1.5))
            if label_text:
                ax.text(x + 0.5, y + 0.5, label_text, ha="center", va="center", fontsize=8)

        max_passed = run.get("maxFillLevelPassed", 0) or 0
        margin = n_cols + 0.35
        if max_passed > 0:
            ax.text(margin, y + 0.5, f"max {fmt_count(max_passed)}",
                    ha="left", va="center", fontsize=10, fontweight="bold")
        else:
            ax.text(margin, y + 0.5, "no fill passed",
                    ha="left", va="center", fontsize=10, color="#cc3333", fontweight="bold")

    # Column header (fill level)
    for ci, target in enumerate(all_targets):
        ax.text(ci + 0.5, n_rows + 0.25, fmt_count(target),
                ha="center", va="bottom", fontsize=10, fontweight="bold")
    ax.text(n_cols / 2.0, n_rows + 0.85, f"Fill level ({ROW_LABEL[signal]})",
            ha="center", va="bottom", fontsize=11, fontweight="bold")

    ax.set_xlim(-4, n_cols + 5)
    ax.set_ylim(-0.6, n_rows + 1.5)
    ax.set_aspect("equal")
    ax.axis("off")

    legend = [
        Patch(facecolor=STATUS_COLOR["pass"], label="passed (read OK, under threshold)"),
        Patch(facecolor=STATUS_COLOR["soft-cliff"], label="over threshold (read OK but slow)"),
        Patch(facecolor=STATUS_COLOR["hard-fail"], label="read errored"),
        Patch(facecolor=STATUS_COLOR["missing"], edgecolor="#cccccc", label="not run"),
    ]
    ax.legend(handles=legend, loc="lower center", bbox_to_anchor=(0.5, -0.05),
              ncol=4, fontsize=9, frameon=False)

    fig.suptitle(f"Read-probe step-status grid — {signal}\nread path: {read_path_for_signal(signal)}",
                 fontsize=12)
    fig.tight_layout(rect=(0, 0.04, 1, 0.94))
    fig.savefig(out, dpi=130, bbox_inches="tight")
    plt.close(fig)


def render_readprobe_signal_mix(runs: list[dict], out: Path) -> None:
    rp_runs = [r for r in runs if r.get("scenario") == "read-probe"]
    if not rp_runs:
        return

    keys = sorted({(r["tier"], r["mode"]) for r in rp_runs},
                  key=lambda k: (tier_rank(k[0]), mode_rank(k[1])))
    if not keys:
        return

    fig, ax = plt.subplots(figsize=(max(8, len(keys) * 1.9), 6))

    width = 0.25
    x = list(range(len(keys)))
    any_value = False

    for si, signal in enumerate(SIGNALS):
        ys = []
        for tier, mode in keys:
            match = [r for r in rp_runs if r["tier"] == tier and r["mode"] == mode and r["signal"] == signal]
            ys.append((match[0].get("maxFillLevelPassed", 0) or 0) if match else 0)
        offsets = [xi + (si - 1) * width for xi in x]
        bars = ax.bar(offsets, ys, width=width,
                      color=SIGNAL_COLORS[signal],
                      label=f"{signal} ({read_path_for_signal(signal)})")
        for b, y in zip(bars, ys):
            if y > 0:
                ax.annotate(fmt_count(y), xy=(b.get_x() + b.get_width() / 2, y),
                            xytext=(0, 3), textcoords="offset points",
                            ha="center", va="bottom", fontsize=7)
                any_value = True

    if not any_value:
        plt.close(fig)
        return

    ax.set_xticks(x)
    ax.set_xticklabels([f"{t}\n{m}" for t, m in keys], fontsize=9)
    ax.set_yscale("log")
    ax.set_ylabel("Max passing fill level (rows, log)")
    ax.set_title("Read-probe cross-signal comparison — which read path scales worst?")
    ax.legend(title="Signal (read path)", loc="upper left", fontsize=8)
    ax.grid(axis="y", which="both", linestyle=":", alpha=0.4)
    fig.tight_layout()
    fig.savefig(out, dpi=130)
    plt.close(fig)


# ---------- summary.md ----------


def render_summary(runs: list[dict], out: Path) -> None:
    lines = [
        "# Traceway hardware benchmark — summary",
        "",
        "> See [../charts.md](../charts.md) for a guide to reading these charts.",
        "",
        f"Runs analyzed: {len(runs)}",
        "",
    ]

    throughput_runs = [r for r in runs if r["scenario"] == "throughput"]
    if throughput_runs:
        lines.append("## Throughput scenario")
        lines.append("")
        lines.append("Failure threshold: combined HTTP error rate + OTLP rejected items > 5% of attempted.")
        lines.append("")
        for signal in SIGNALS:
            sig_runs = [r for r in throughput_runs if r["signal"] == signal]
            if not sig_runs:
                continue
            lines.append(f"### {signal.capitalize()}")
            lines.append("")
            lines.append(f"| Tier | Mode | Max {ITEM_LABEL[signal]} | P1 max batch | P2 max req/sec @ batch | P3 max req/sec @ batch=100 |")
            lines.append("|------|------|---------|--------------|-------------------------|-------------------------------|")
            for run in sorted(sig_runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):
                max_items = int(run.get("maxSustainableItemsPerSec", 0) or 0)
                max_batch = (run.get("phase1") or {}).get("maxBatchSize", 0)
                p2 = run.get("phase2") or {}
                p2_rate = p2.get("maxRequestRate", 0)
                p2_batch = p2.get("fixedBatchSize", 0)
                p2_cell = f"{p2_rate:g} @ {p2_batch:,}" if p2_rate else "—"
                p3 = run.get("phase3") or {}
                p3_rate = p3.get("maxRequestRate", 0)
                p3_cell = f"{p3_rate:g}" if p3_rate else "—"
                lines.append(f"| {run['tier']} | {run['mode']} | {max_items:,} | {max_batch:,} | {p2_cell} | {p3_cell} |")
            lines.append("")

    readprobe_runs = [r for r in runs if r["scenario"] == "read-probe"]
    if readprobe_runs:
        lines.append("## Read-probe scenario")
        lines.append("")
        lines.append("Failure: a read probe exceeded the configured threshold (default 5000ms) or returned an error. "
                     "Max fill level passed is the largest row count at which the read still came back in time.")
        lines.append("")
        for signal in SIGNALS:
            sig_runs = [r for r in readprobe_runs if r["signal"] == signal]
            if not sig_runs:
                continue
            lines.append(f"### {signal.capitalize()} — probing `{read_path_for_signal(signal)}`")
            lines.append("")
            lines.append(f"| Tier | Mode | Max {ROW_LABEL[signal]} passed | Read latency @ max (ms) | First failing fill | Read latency @ failing (ms) |")
            lines.append("|------|------|----------------|-------------------|--------------------|------------------------|")
            for run in sorted(sig_runs, key=lambda r: (tier_rank(r["tier"]), mode_rank(r["mode"]))):
                steps = _readprobe_steps(run)
                max_passed = run.get("maxFillLevelPassed", 0) or 0
                latency_at_max = 0
                first_fail = ""
                latency_at_fail = ""
                for s in steps:
                    if s.get("passed"):
                        latency_at_max = int(s.get("readLatencyMs", 0))
                    elif not first_fail:
                        first_fail = f"{int(s.get('rowsIngested', 0)):,}"
                        latency_at_fail = f"{int(s.get('readLatencyMs', 0))}"
                lines.append(f"| {run['tier']} | {run['mode']} | {int(max_passed):,} | {latency_at_max} | {first_fail or '—'} | {latency_at_fail or '—'} |")
            lines.append("")

    lines.append("Generated by `benchmarks/scripts/chart.py`.")
    out.write_text("\n".join(lines) + "\n")


# ---------- Entry point ----------


def main() -> int:
    if len(sys.argv) < 2:
        print("usage: chart.py <results-dir>", file=sys.stderr)
        return 2
    results_dir = Path(sys.argv[1])
    if not results_dir.is_dir():
        print(f"not a directory: {results_dir}", file=sys.stderr)
        return 2

    runs = load_runs(results_dir)
    if not runs:
        print(f"no usable *.json files in {results_dir}", file=sys.stderr)
        return 1

    for signal in SIGNALS:
        # Throughput charts (no-op if no throughput runs for this signal)
        render_headline_bar(runs, signal, results_dir / f"chart-{signal}.png")
        render_phase1_latency(runs, signal, results_dir / f"chart-phase1-batch-{signal}.png")
        render_phase2_rate(runs, signal, results_dir / f"chart-phase2-rate-{signal}.png")
        render_phase3_rate(runs, signal, results_dir / f"chart-phase3-rate-{signal}.png")
        render_pareto(runs, signal, results_dir / f"chart-pareto-{signal}.png")
        render_tier_scaling(runs, signal, results_dir / f"chart-tier-scaling-{signal}.png")
        render_cliff_grid(runs, signal, results_dir / f"chart-cliff-{signal}.png")
        render_batch_efficiency(runs, signal, results_dir / f"chart-batch-efficiency-{signal}.png")
        render_ch_pressure(runs, signal, results_dir / f"chart-ch-pressure-{signal}.png")

        # Read-probe charts (no-op if no read-probe runs for this signal)
        render_readprobe_headline(runs, signal, results_dir / f"chart-readprobe-headline-{signal}.png")
        render_readprobe(runs, signal, results_dir / f"chart-readprobe-{signal}.png")
        render_readprobe_per_endpoint(runs, signal, results_dir / f"chart-readprobe-per-endpoint-{signal}.png")
        render_readprobe_ingest_time(runs, signal, results_dir / f"chart-readprobe-ingest-time-{signal}.png")
        render_readprobe_tier_scaling(runs, signal, results_dir / f"chart-readprobe-tier-scaling-{signal}.png")
        render_readprobe_cliff(runs, signal, results_dir / f"chart-readprobe-cliff-{signal}.png")

    render_signal_mix(runs, results_dir / "chart-signal-mix.png")
    render_readprobe_signal_mix(runs, results_dir / "chart-readprobe-signal-mix.png")
    render_summary(runs, results_dir / "summary.md")
    print(f"wrote charts and summary.md into {results_dir}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())