Skip to content

Git designed for AI agents.

33.8 KB
Raw Blame
#!/usr/bin/env python3
"""Render a Markdown summary from a k6 benchmark run.

Reads ``k6-summary.json`` produced by ``k6 run --summary-export`` and emits a
human-readable report to stdout (or ``--output``). The report is intended for
posting as a GitLab MR note.

Optional inputs in the same results directory are linked in if present:

  - ``gitaly-stats/all-gitalies-latency{,-overall}.json`` - server-side
    per-RPC latency aggregated by agrind from the harvested Gitaly logs;
    the sole latency source (absent stats render as empty/zero)
  - ``plots/*.png``       - PNG plots produced by an experiment's plot.py

The ``--artifact-base-url`` flag, if set, turns relative plot paths into
clickable links suitable for embedding in an MR comment. Typically passed as
``--artifact-base-url=${CI_JOB_URL}/artifacts/raw/_support/benchmarking/experiments/${EXPERIMENT}/results/${RUN_NAME}``.

This script intentionally has zero non-stdlib dependencies so it can run in any
CI image without ``pip install``.
"""

from __future__ import annotations

import argparse
import json
import os
import sys
from pathlib import Path

# Latency stat labels exposed by the report, mapped to the keys agrind's
# -o json output uses for them.
_STAT_LABELS = ["p50", "p90", "p95", "max", "avg"]
_AGRIND_STAT_KEYS = {"p50": "med", "p90": "p90", "p95": "p95",
                     "max": "_max", "avg": "_average"}
# agrind emits the group-by column under the escaped field-reference key.
_AGRIND_RPC_KEY = '["grpc.method"]'


def fmt_ms(value_ms: float) -> str:
    """Render a millisecond value compactly. k6 reports durations in ms."""
    if value_ms < 1:
        return f"{value_ms * 1000:.0f}\u00b5s"
    if value_ms < 1000:
        return f"{value_ms:.1f}ms"
    return f"{value_ms / 1000:.2f}s"


def fmt_rate(value_per_s: float) -> str:
    if value_per_s >= 1000:
        return f"{value_per_s / 1000:.1f}k/s"
    return f"{value_per_s:.1f}/s"


def fmt_pct(numerator: int, denominator: int) -> str:
    if denominator == 0:
        return "n/a"
    return f"{(numerator / denominator) * 100:.1f}%"


def _normalize_stats(row: dict, keys: dict) -> dict:
    return {label: row.get(key, 0) for label, key in keys.items()}


def _load_agrind_rows(path: Path) -> list:
    if not path.is_file():
        return []
    try:
        rows = json.loads(path.read_text())
    except (OSError, ValueError):
        return []
    return rows if isinstance(rows, list) else []


def agrind_latency(results_dir: Path) -> tuple[dict, dict]:
    """(overall, per-RPC) latency from the agrind stats in gitaly-stats/.

    These are Gitaly's server-side ``grpc.time_ms`` figures aggregated
    across every harvested pod log by the benchmark entrypoint. Returns
    ``({}, {})`` when the run has no agrind stats.
    """
    stats_dir = results_dir / "gitaly-stats"
    overall = {}
    rows = _load_agrind_rows(stats_dir / "all-gitalies-latency-overall.json")
    if rows:
        overall = _normalize_stats(rows[0], _AGRIND_STAT_KEYS)
    per_rpc = {}
    for row in _load_agrind_rows(stats_dir / "all-gitalies-latency.json"):
        rpc = row.get(_AGRIND_RPC_KEY)
        if rpc:
            per_rpc[rpc] = _normalize_stats(row, _AGRIND_STAT_KEYS)
    return overall, per_rpc


def per_rpc_checks(root_group: dict) -> dict:
    """Aggregate check passes/fails by check-name prefix.

    Names follow ``"<prefix> - <description>"``; a prefix may have several
    checks (e.g. UserMergeBranch), so sum them by prefix.
    """
    out: dict[str, dict] = {}
    for name, data in root_group.get("checks", {}).items():
        rpc = name.split(" - ", 1)[0].strip()
        entry = out.setdefault(rpc, {"passes": 0, "fails": 0})
        entry["passes"] += data.get("passes", 0)
        entry["fails"] += data.get("fails", 0)
    return out


def _om_value(value: float) -> str:
    f = float(value)
    if f == int(f):
        return str(int(f))
    return f"{f:.6g}"


def _om_labels(labels: dict) -> str:
    parts = []
    for key, val in labels.items():
        if val is None:
            continue
        escaped = str(val).replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
        parts.append(f'{key}="{escaped}"')
    return "{" + ",".join(parts) + "}" if parts else ""


def render_openmetrics(metrics: dict, root_group: dict,
                       overall: dict, per_rpc: dict,
                       experiment: str | None,
                       gitaly_version: str | None = None) -> str:
    """Render the metrics in the OpenMetrics text format.

    Consumed by GitLab's ``artifacts:reports:metrics``, which renders the
    values in the MR widget and diffs each ``name{labels}`` line against the
    target branch's latest run.

    Conforms to the OpenMetrics text format: every family is preceded by
    ``# TYPE`` (and ``# HELP``), a family's samples are contiguous, counters
    carry the ``_total`` suffix (the family name in ``# TYPE`` omits it), and
    the exposition is terminated by ``# EOF``.

    Latency is emitted overall (no ``rpc`` label) and per RPC (``rpc`` label)
    in one gauge family, distinguished by a ``stat`` label
    (p50/p90/p95/max/avg). Check pass ratio is likewise overall and per RPC.
    """
    base = {"experiment": experiment} if experiment else {}
    iters = metrics.get("iterations", {})
    checks = metrics.get("checks", {})
    dropped = metrics.get("dropped_iterations", {})

    def latency_samples(stat_source: dict, extra: dict | None = None) -> list:
        extra = extra or {}
        return [(dict(base, **extra, stat=label), stat_source.get(label, 0))
                for label in _STAT_LABELS]

    duration = latency_samples(overall) if overall else []
    for rpc, stats in sorted(per_rpc.items()):
        duration += latency_samples(stats, {"rpc": rpc})

    passes = checks.get("passes", 0)
    fails = checks.get("fails", 0)
    total = passes + fails
    pass_ratio = [(dict(base), passes / total if total else 0.0)]
    rpc_passes, rpc_fails = [], []
    for rpc, counts in sorted(per_rpc_checks(root_group).items()):
        p, f = counts["passes"], counts["fails"]
        t = p + f
        rpc_passes.append((dict(base, rpc=rpc), p))
        rpc_fails.append((dict(base, rpc=rpc), f))
        pass_ratio.append((dict(base, rpc=rpc), p / t if t else 0.0))

    # (family name, type, help, samples, sample name [defaults to family]).
    families = [
        ("gitaly_rpc_duration_ms", "gauge",
         "Server-side RPC latency in milliseconds from the Gitaly logs "
         "(stat=p50|p90|p95|max|avg).",
         duration, None),
        ("gitaly_benchmark_iterations", "counter",
         "Completed k6 iterations.",
         [(dict(base), iters.get("count", 0))],
         "gitaly_benchmark_iterations_total"),
        ("gitaly_benchmark_iterations_per_second", "gauge",
         "k6 iteration throughput per second.",
         [(dict(base), iters.get("rate", 0))], None),
        ("gitaly_benchmark_dropped_iterations", "counter",
         "k6 iterations dropped before starting.",
         [(dict(base), dropped.get("count", 0))],
         "gitaly_benchmark_dropped_iterations_total"),
        ("gitaly_benchmark_checks_pass_ratio", "gauge",
         "Fraction of k6 checks passing (overall and per RPC).",
         pass_ratio, None),
        ("gitaly_benchmark_checks_passes", "counter",
         "k6 checks passed per RPC.", rpc_passes,
         "gitaly_benchmark_checks_passes_total"),
        ("gitaly_benchmark_checks_fails", "counter",
         "k6 checks failed per RPC.", rpc_fails,
         "gitaly_benchmark_checks_fails_total"),
    ]
    if gitaly_version:
        families.append(
            ("gitaly_benchmark_build_info", "gauge",
             "Deployed Gitaly under test; value is always 1.",
             [(dict(base, gitaly_version=gitaly_version), 1)], None))

    lines = []
    for name, mtype, help_text, samples, sample_name in families:
        if not samples:
            continue
        lines.append(f"# HELP {name} {help_text}")
        lines.append(f"# TYPE {name} {mtype}")
        for labels, value in samples:
            lines.append(
                f"{sample_name or name}{_om_labels(labels)} {_om_value(value)}")
    lines.append("# EOF")
    return "\n".join(lines) + "\n"


def _pct_delta_value(current: float, baseline: float) -> float | None:
    """Signed % change; callers pass (B, A). None when baseline is 0."""
    if baseline == 0:
        return None
    return (current - baseline) / baseline * 100


def _pct_delta(current: float, baseline: float) -> str:
    """Signed %-change string, e.g. ``+12.3%``."""
    delta = _pct_delta_value(current, baseline)
    if delta is None:
        return "n/a" if current == 0 else "new"
    return f"{delta:+.1f}%"


# Deltas below this (% or %-points) count as noise, not a win.
_VERDICT_EPS = 5.0


def _verdict(delta: float | None, higher_is_better: bool,
             eps: float = _VERDICT_EPS) -> str:
    """Marker saying whether B beats A. ``delta`` is relative % for ratio
    metrics, %-points for pass rates; "" when undefined."""
    if delta is None:
        return ""
    if abs(delta) < eps:
        return "≈ similar"
    improved = delta > 0 if higher_is_better else delta < 0
    return "✅ better" if improved else "🔴 worse"


def render_compare_summary_table(metrics_a: dict, metrics_b: dict,
                                 lat_a: dict, lat_b: dict,
                                 label_a: str, label_b: str) -> str:
    """Side-by-side aggregate metrics for two runs (A vs B)."""
    iters_a = metrics_a.get("iterations", {})
    iters_b = metrics_b.get("iterations", {})
    checks_a = metrics_a.get("checks", {})
    checks_b = metrics_b.get("checks", {})

    def pass_pct(checks: dict) -> str:
        return fmt_pct(checks.get("passes", 0),
                       checks.get("passes", 0) + checks.get("fails", 0))

    def pass_ratio(checks: dict) -> float | None:
        total = checks.get("passes", 0) + checks.get("fails", 0)
        return checks.get("passes", 0) / total if total else None

    ra, rb = pass_ratio(checks_a), pass_ratio(checks_b)
    # Pass-rate delta is in %-points, not relative %; leave Δ blank but judge.
    checks_pts = (rb - ra) * 100 if ra is not None and rb is not None else None

    rows = [
        ("Throughput",
         fmt_rate(iters_a.get("rate", 0)), fmt_rate(iters_b.get("rate", 0)),
         _pct_delta(iters_b.get("rate", 0), iters_a.get("rate", 0)),
         _verdict(_pct_delta_value(iters_b.get("rate", 0),
                                   iters_a.get("rate", 0)), True)),
        ("Iterations",
         f"{iters_a.get('count', 0):,}", f"{iters_b.get('count', 0):,}",
         _pct_delta(iters_b.get("count", 0), iters_a.get("count", 0)),
         _verdict(_pct_delta_value(iters_b.get("count", 0),
                                   iters_a.get("count", 0)), True)),
        ("Checks passed", pass_pct(checks_a), pass_pct(checks_b), "",
         _verdict(checks_pts, True)),
    ]
    for stat_label in _STAT_LABELS:
        a_val, b_val = lat_a.get(stat_label, 0), lat_b.get(stat_label, 0)
        rows.append(
            (f"RPC latency {stat_label}", fmt_ms(a_val), fmt_ms(b_val),
             _pct_delta(b_val, a_val),
             _verdict(_pct_delta_value(b_val, a_val), higher_is_better=False)))

    lines = [f"| Metric | {label_a} | {label_b} | Δ (B vs A) | {label_b} better? |",
             "| --- | ---: | ---: | ---: | :--- |"]
    lines.extend(f"| {name} | {a} | {b} | {d} | {v} |"
                 for name, a, b, d, v in rows)
    return "\n".join(lines)


def render_compare_rpc_latency_table(per_a: dict, per_b: dict,
                                     label_a: str, label_b: str) -> str:
    """Per-RPC p95 latency, B vs A, sorted alphabetically by RPC name."""
    rpcs = sorted(set(per_a) | set(per_b))
    if not rpcs:
        return ""

    def p95(stats: dict) -> float:
        return stats.get("p95", 0)

    rows = []
    for rpc in rpcs:
        a = p95(per_a.get(rpc, {}))
        b = p95(per_b.get(rpc, {}))
        # Zero p95 = no sample on that side; nothing to compare.
        comparable = a > 0 and b > 0
        rows.append((rpc, a, b, comparable))

    lines = ["### Per-RPC latency (p95)", "",
             f"| RPC | {label_a} | {label_b} | Δ (B vs A) | {label_b} better? |",
             "| --- | ---: | ---: | ---: | :--- |"]
    for rpc, a, b, comparable in rows:
        if comparable:
            delta = _pct_delta(b, a)
            verdict = _verdict(_pct_delta_value(b, a), higher_is_better=False)
        else:
            # A sample on at most one side; "new" when it's B's.
            delta, verdict = ("new" if b > 0 else "n/a"), ""
        lines.append(
            f"| {rpc} | {fmt_ms(a)} | {fmt_ms(b)} | {delta} | {verdict} |")
    return "\n".join(lines)


def render_compare_checks_table(root_group_a: dict, root_group_b: dict,
                                label_a: str, label_b: str) -> str:
    """k6 check pass rates grouped by check-name prefix, B vs A."""
    per_a = per_rpc_checks(root_group_a)
    per_b = per_rpc_checks(root_group_b)
    rpcs = sorted(set(per_a) | set(per_b))
    if not rpcs:
        return ""

    def rate(counts: dict) -> float | None:
        total = counts.get("passes", 0) + counts.get("fails", 0)
        return counts.get("passes", 0) / total if total else None

    rows = []
    for rpc in rpcs:
        a = rate(per_a.get(rpc, {}))
        b = rate(per_b.get(rpc, {}))
        rows.append((rpc, a, b))

    lines = ["### Check pass rate", "",
             "k6's client-side checks, grouped by check-name prefix: a "
             "workload section (e.g. `pushSmallCommit-ssh`), an RPC, or "
             "`<section> <RPC>` where a section checks one RPC. `n/a` means "
             "that side did not run the check.", "",
             f"| Check | {label_a} | {label_b} | Δ (%points) | {label_b} better? |",
             "| --- | ---: | ---: | ---: | :--- |"]
    for rpc, a, b in rows:
        a_s = f"{a * 100:.1f}%" if a is not None else "n/a"
        b_s = f"{b * 100:.1f}%" if b is not None else "n/a"
        if a is not None and b is not None:
            pts = (b - a) * 100
            d_s = f"{pts:+.1f}"
            v_s = _verdict(pts, higher_is_better=True)
        else:
            d_s, v_s = "n/a", ""
        lines.append(f"| {rpc} | {a_s} | {b_s} | {d_s} | {v_s} |")
    return "\n".join(lines)


def render_compare_report(summary_a: dict, summary_b: dict,
                          latency_a: tuple[dict, dict],
                          latency_b: tuple[dict, dict],
                          label_a: str, label_b: str,
                          experiment: str | None,
                          version_a: str | None, version_b: str | None,
                          run_a: str | None = None,
                          run_b: str | None = None,
                          grafana: str | None = None) -> str:
    """A/B comparison note: each metric gets a direction-aware "B better?"
    marker (B = --compare side)."""
    metrics_a = summary_a.get("metrics", {})
    metrics_b = summary_b.get("metrics", {})
    overall_a, per_rpc_a = latency_a
    overall_b, per_rpc_b = latency_b

    def side(letter: str, label: str, run: str | None, version: str | None) -> str:
        parts = [f"**{letter} = {label}**"]
        if run:
            parts.append(f"run `{run}`")
        if version:
            parts.append(f"(`{version}`)")
        return " · ".join(parts)

    header = ["## Gitaly backend comparison"]
    if experiment:
        header.append(f"**Experiment:** `{experiment}`")
    header.append(side("A", label_a, run_a, version_a))
    header.append(side("B", label_b, run_b, version_b))

    note = (
        "> `A` typically serves repositories from "
        "a local disk; `B` from network object storage (MVCC/elastic). Δ is "
        "`(B - A) / A`, i.e. the B side measured against the A baseline, and the "
        f"**{label_b} better?** column judges each metric in its good direction."
    )

    sections = [
        "\n".join(header),
        note,
        "### Summary",
        render_compare_summary_table(metrics_a, metrics_b,
                                     overall_a, overall_b, label_a, label_b),
    ]

    rpc = render_compare_rpc_latency_table(per_rpc_a, per_rpc_b,
                                           label_a, label_b)
    if rpc:
        sections.append(rpc)

    checks = render_compare_checks_table(
        summary_a.get("root_group", {}), summary_b.get("root_group", {}),
        label_a, label_b)
    if checks:
        sections.append(checks)

    if grafana:
        sections.append(grafana)

    sections.append(
        "_Single-run comparison; numbers reflect one k6 execution per backend "
        "and may have run-to-run variance. Run multiple interleaved rounds for "
        "a stable delta. Latency and throughput are only comparable where both "
        "sides' check pass rates are high and similar; failed RPCs return fast "
        "and skew both metrics._"
    )
    return "\n\n".join(sections) + "\n"


def render_metrics_table(metrics: dict, latency: dict) -> str:
    """Aggregate latency + iteration metrics across all scenarios."""
    iters = metrics.get("iterations", {})
    checks = metrics.get("checks", {})
    dropped = metrics.get("dropped_iterations", {})

    rows = [
        ("Iterations", f"{iters.get('count', 0):,}"),
        ("Throughput", fmt_rate(iters.get("rate", 0))),
        ("Dropped iterations",
         f"{dropped.get('count', 0):,} ({fmt_rate(dropped.get('rate', 0))})"),
        ("Checks passed",
         fmt_pct(checks.get("passes", 0),
                 checks.get("passes", 0) + checks.get("fails", 0))),
        ("RPC latency p50", fmt_ms(latency.get("p50", 0))),
        ("RPC latency p90", fmt_ms(latency.get("p90", 0))),
        ("RPC latency p95", fmt_ms(latency.get("p95", 0))),
        ("RPC latency max", fmt_ms(latency.get("max", 0))),
        ("RPC latency avg", fmt_ms(latency.get("avg", 0))),
    ]

    lines = ["| Metric | Value |", "| --- | --- |"]
    lines.extend(f"| {name} | {value} |" for name, value in rows)
    return "\n".join(lines)


def render_rpc_latency_table(per_rpc: dict) -> str:
    """Per-RPC latency breakdown."""
    if not per_rpc:
        return ""

    lines = ["### Per-RPC latency", "",
             "| RPC | p50 | p90 | p95 | max | avg |",
             "| --- | ---: | ---: | ---: | ---: | ---: |"]
    for rpc, stats in sorted(per_rpc.items()):
        lines.append(
            f"| {rpc} | {fmt_ms(stats.get('p50', 0))} "
            f"| {fmt_ms(stats.get('p90', 0))} | {fmt_ms(stats.get('p95', 0))} "
            f"| {fmt_ms(stats.get('max', 0))} | {fmt_ms(stats.get('avg', 0))} |")
    return "\n".join(lines)


def render_checks_table(root_group: dict) -> str:
    """Per-RPC pass/fail breakdown from k6 ``check()`` calls."""
    checks = root_group.get("checks", {})
    if not checks:
        return "_No per-RPC checks recorded._"

    lines = ["| Check | Passes | Fails | Pass rate |",
             "| --- | ---: | ---: | ---: |"]
    for name, data in sorted(checks.items()):
        passes = data.get("passes", 0)
        fails = data.get("fails", 0)
        lines.append(f"| {name} | {passes:,} | {fails:,} "
                     f"| {fmt_pct(passes, passes + fails)} |")
    return "\n".join(lines)


# Grafana dashboard the benchmark panels live on; <uid>/<slug>, overridable.
_GRAFANA_DASHBOARD = "efneoyur0u4u8f/gitaly-benchmark-testing"


class GrafanaLink:
    """Builds the dashboard deep link and the port-forward command to reach it."""

    def __init__(self, base_url: str, dashboard: str, ns: str, service: str,
                 local_port: int, service_port: int) -> None:
        self.base_url = base_url.rstrip("/")
        self.dashboard = dashboard
        self.ns = ns
        self.service = service
        self.local_port = local_port
        self.service_port = service_port

    def url(self, start_s: float | None, end_s: float | None,
            pad_s: int = 60) -> str:
        # Pad the window so the run's annotation edges stay visible; fall
        # back to the last hour when the window is unknown.
        if start_s and end_s:
            span = (f"&from={int((start_s - pad_s) * 1000)}"
                    f"&to={int((end_s + pad_s) * 1000)}")
        else:
            span = "&from=now-1h&to=now"
        return f"{self.base_url}/d/{self.dashboard}?orgId=1{span}"

    def port_forward_cmd(self) -> str:
        return (f"kubectl -n {self.ns} port-forward "
                f"svc/{self.service} {self.local_port}:{self.service_port}")


def _grafana_link(args) -> GrafanaLink:
    return GrafanaLink(args.grafana_url, args.grafana_dashboard,
                       args.grafana_namespace, args.grafana_service,
                       args.grafana_local_port, args.grafana_service_port)


def read_run_window(run_dir: Path) -> tuple[float | None, float | None]:
    """(start, end) epoch seconds from a run's grafana-run.env, else (None, None)."""
    env_file = run_dir / "grafana-run.env"
    if not env_file.is_file():
        return None, None
    values = dict(ln.split("=", 1) for ln in env_file.read_text().splitlines()
                  if "=" in ln)

    def num(key: str) -> float | None:
        try:
            return float(values[key].strip())
        except (KeyError, ValueError):
            return None

    return num("RUN_START_EPOCH"), num("RUN_END_EPOCH")


def _default_screenshot(explicit: Path | None, results_dir: Path) -> Path | None:
    if explicit:
        return explicit
    candidate = results_dir / "grafana" / "dashboard.png"
    return candidate if candidate.is_file() else None


def render_grafana_section(link: GrafanaLink,
                           windows: list[tuple[str, float | None, float | None]],
                           screenshot: Path | None, results_dir: Path,
                           base_url: str | None) -> str:
    """'View in Grafana' section: a link per run window (plus a combined one),
    the port-forward command, and a grafana screenshot."""
    lines = ["### View in Grafana", ""]
    # Embed the snapshot only with an absolute URL; a relative image path
    # would render as a dead image in an MR note.
    if screenshot and screenshot.is_file() and base_url:
        rel = screenshot.relative_to(results_dir) \
            if screenshot.is_relative_to(results_dir) else screenshot.name
        lines += [f"![Grafana benchmark dashboard]({base_url}/{rel})", ""]
    lines += [
        "Each run is marked with a purple **Benchmark runs** region annotation, "
        "and every k6 section within it with a cyan **k6 sections** region. "
        "Hovering a cyan region shows the section name and the check it "
        "drives, named as in the check tables above (e.g. "
        "`refUpdateBatched UpdateReferences` or `pushSmallCommit-ssh`), "
        "so the panels can be read against the exact window each phase "
        "occupied. Port-forward Grafana, then open a link below:",
        "", "```sh", link.port_forward_cmd(), "```", "",
    ]
    for label, start_s, end_s in windows:
        lines.append(f"- **{label}**: [open dashboard]({link.url(start_s, end_s)})")
    spans = [(s, e) for _, s, e in windows if s and e]
    if len(spans) > 1:
        combined = link.url(min(s for s, _ in spans), max(e for _, e in spans))
        lines.append(f"- **Both runs**: [open dashboard]({combined})")
    return "\n".join(lines).rstrip() + "\n"


def render_plots_section(results_dir: Path, base_url: str | None) -> str:
    """List PNG plots from the experiment's plot.py output."""
    plots_dir = results_dir / "plots"
    if not plots_dir.is_dir():
        return ""

    pngs = sorted(plots_dir.glob("*.png"))
    if not pngs:
        return ""

    lines = ["### Plots", ""]
    for png in pngs:
        rel = png.relative_to(results_dir)
        if base_url:
            url = f"{base_url}/{rel}"
            lines.append(f"![{png.stem}]({url})")
        else:
            lines.append(f"- `{rel}`")
        lines.append("")
    return "\n".join(lines)


def render_report(summary: dict, results_dir: Path, base_url: str | None,
                  experiment: str | None, run_name: str | None,
                  gitaly_version: str | None,
                  overall: dict, per_rpc: dict,
                  job_url: str | None = None,
                  grafana: str | None = None) -> str:
    metrics = summary.get("metrics", {})
    root_group = summary.get("root_group", {})

    header_parts = ["## Gitaly benchmark results"]
    if experiment:
        header_parts.append(f"**Experiment:** `{experiment}`")
    if run_name:
        header_parts.append(f"**Run:** `{run_name}`")
    if gitaly_version:
        # The deployed Gitaly is usually not the MR's code; say what ran.
        header_parts.append(f"**Gitaly under test:** `{gitaly_version}`")

    sections = [
        "\n".join(header_parts),
        "### Summary",
        render_metrics_table(metrics, overall),
    ]

    rpc_latency = render_rpc_latency_table(per_rpc)
    if rpc_latency:
        sections.append(rpc_latency)

    sections.extend([
        "### Per-RPC checks",
        render_checks_table(root_group),
    ])

    plots = render_plots_section(results_dir, base_url)
    if plots:
        sections.append(plots)

    if grafana:
        sections.append(grafana)

    sections.append(
        "_Single-run results; no baseline comparison available yet. "
        "Numbers reflect one k6 execution and may have run-to-run variance._"
    )

    if job_url:
        sections.append(
            f"[View CI job]({job_url}) \u00b7 "
            f"[Browse artifacts]({job_url}/artifacts/browse)")

    return "\n\n".join(sections) + "\n"


def _run_compare(args) -> int:
    if not args.summary.is_file():
        print(f"error: summary A not found: {args.summary}", file=sys.stderr)
        return 1
    if not args.compare.is_file():
        print(f"error: summary B not found: {args.compare}", file=sys.stderr)
        return 1
    # Labels can't be inferred (both backends run the same image), and a
    # wrong default would mislabel every column and verdict.
    if not args.label_a or not args.label_b:
        print("error: --label-a and --label-b are required with --compare",
              file=sys.stderr)
        return 1

    with args.summary.open() as f:
        summary_a = json.load(f)
    with args.compare.open() as f:
        summary_b = json.load(f)

    run_a_dir, run_b_dir = args.summary.parent, args.compare.parent
    grafana = None
    if not args.no_grafana:
        a_start = args.a_start or args.run_start
        a_end = args.a_end or args.run_end
        if not (a_start and a_end):
            a_start, a_end = read_run_window(run_a_dir)
        b_start, b_end = args.b_start, args.b_end
        if not (b_start and b_end):
            b_start, b_end = read_run_window(run_b_dir)
        results_dir = run_a_dir.parent
        base_url = args.artifact_base_url.rstrip("/") \
            if args.artifact_base_url else None
        grafana = render_grafana_section(
            _grafana_link(args),
            [(f"A · {args.label_a} · `{run_a_dir.name}`", a_start, a_end),
             (f"B · {args.label_b} · `{run_b_dir.name}`", b_start, b_end)],
            _default_screenshot(args.grafana_screenshot, results_dir),
            results_dir, base_url)

    markdown = render_compare_report(
        summary_a=summary_a,
        summary_b=summary_b,
        latency_a=agrind_latency(run_a_dir),
        latency_b=agrind_latency(run_b_dir),
        label_a=args.label_a,
        label_b=args.label_b,
        experiment=args.experiment,
        version_a=args.gitaly_version,
        version_b=args.gitaly_version_b,
        run_a=run_a_dir.name,
        run_b=run_b_dir.name,
        grafana=grafana,
    )

    # Default output: comparisons/<A-run>_<B-run>.md next to the run dirs;
    # explicit --output wins.
    if args.output:
        out_path = args.output
    else:
        results_dir = args.summary.parent.parent
        out_path = (results_dir / "comparisons" /
                    f"{args.summary.parent.name}_{args.compare.parent.name}.md")
    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(markdown)
    print(f"wrote comparison report to {out_path}", file=sys.stderr)
    return 0


def main() -> int:
    parser = argparse.ArgumentParser(
        description="Render a Markdown report from a k6 benchmark run.")
    parser.add_argument(
        "summary",
        type=Path,
        help="Path to k6-summary.json (the file produced by --summary-export)")
    parser.add_argument(
        "--output", "-o",
        type=Path,
        help="Write Markdown here instead of stdout")
    parser.add_argument(
        "--artifact-base-url",
        help="Base URL for plot/pprof links. Typically "
             "${CI_JOB_URL}/artifacts/raw/<path-to-run-dir>")
    parser.add_argument(
        "--job-url",
        default=os.environ.get("CI_JOB_URL"),
        help="CI job URL; adds job and artifacts links at the bottom of "
             "the report. Defaults to $CI_JOB_URL.")
    parser.add_argument(
        "--experiment",
        default=os.environ.get("EXPERIMENT"),
        help="Experiment name (for the report header). Defaults to $EXPERIMENT.")
    parser.add_argument(
        "--run-name",
        default=os.environ.get("RUN_NAME"),
        help="Run identifier (for the report header). Defaults to $RUN_NAME.")
    parser.add_argument(
        "--metrics-output",
        type=Path,
        help="Also write an OpenMetrics file here for GitLab's metrics report.")
    parser.add_argument(
        "--gitaly-version",
        default=os.environ.get("GITALY_VERSION"),
        help="Deployed Gitaly image/version under test (for provenance). "
             "Defaults to $GITALY_VERSION, then gitaly-version.txt in the "
             "results dir if present.")
    parser.add_argument(
        "--compare",
        type=Path,
        help="Second k6-summary.json (B side); renders an A/B comparison "
             "report instead.")
    parser.add_argument(
        "--label-a",
        help="Backend name for the A side (the positional summary). Required "
             "with --compare.")
    parser.add_argument(
        "--label-b",
        help="Backend name for the B side (the --compare summary). Required "
             "with --compare.")
    parser.add_argument(
        "--gitaly-version-b",
        help="Gitaly image/version of the B side (for provenance).")

    grp = parser.add_argument_group("Grafana dashboard link")
    grp.add_argument("--no-grafana", action="store_true",
                     help="Omit the 'View in Grafana' section.")
    grp.add_argument("--grafana-url",
                     default=os.environ.get("GRAFANA_URL", "http://localhost:3000"),
                     help="Base URL opened after port-forwarding. "
                          "Default $GRAFANA_URL, else http://localhost:3000.")
    grp.add_argument("--grafana-dashboard", default=_GRAFANA_DASHBOARD,
                     help="Dashboard <uid>/<slug> for the deep link.")
    grp.add_argument("--grafana-namespace",
                     default=os.environ.get("GRAFANA_NAMESPACE", "prometheus"),
                     help="Grafana namespace (for the port-forward command).")
    grp.add_argument("--grafana-service",
                     default=os.environ.get("GRAFANA_SERVICE", "prometheus-grafana"),
                     help="Grafana Service name (for the port-forward command).")
    grp.add_argument("--grafana-local-port", type=int, default=3000)
    grp.add_argument("--grafana-service-port", type=int, default=80)
    grp.add_argument("--grafana-screenshot", type=Path,
                     help="PNG to embed. Default grafana/dashboard.png in the "
                          "results dir if present.")
    grp.add_argument("--run-start", type=float, help="Single-run start (epoch s).")
    grp.add_argument("--run-end", type=float, help="Single-run end (epoch s).")
    grp.add_argument("--a-start", type=float, help="A-run start (epoch s).")
    grp.add_argument("--a-end", type=float, help="A-run end (epoch s).")
    grp.add_argument("--b-start", type=float, help="B-run start (epoch s).")
    grp.add_argument("--b-end", type=float, help="B-run end (epoch s).")

    args = parser.parse_args()

    if not args.summary.is_file():
        print(f"error: summary file not found: {args.summary}", file=sys.stderr)
        return 1

    if args.compare is not None:
        return _run_compare(args)

    with args.summary.open() as f:
        summary = json.load(f)

    results_dir = args.summary.parent
    gitaly_version = args.gitaly_version
    if not gitaly_version:
        version_file = results_dir / "gitaly-version.txt"
        if version_file.is_file():
            gitaly_version = version_file.read_text().strip()
    gitaly_version = gitaly_version or None

    overall, per_rpc = agrind_latency(results_dir)
    base_url = args.artifact_base_url.rstrip("/") if args.artifact_base_url else None

    grafana = None
    if not args.no_grafana:
        run_start, run_end = args.run_start, args.run_end
        if not (run_start and run_end):
            run_start, run_end = read_run_window(results_dir)
        label = f"`{args.run_name}`" if args.run_name else "This run"
        grafana = render_grafana_section(
            _grafana_link(args), [(label, run_start, run_end)],
            _default_screenshot(args.grafana_screenshot, results_dir),
            results_dir, base_url)

    markdown = render_report(
        summary=summary,
        results_dir=results_dir,
        base_url=base_url,
        experiment=args.experiment,
        run_name=args.run_name,
        gitaly_version=gitaly_version,
        overall=overall,
        per_rpc=per_rpc,
        job_url=args.job_url.rstrip("/") if args.job_url else None,
        grafana=grafana,
    )

    if args.output:
        args.output.write_text(markdown)
    else:
        sys.stdout.write(markdown)

    if args.metrics_output:
        args.metrics_output.write_text(
            render_openmetrics(summary.get("metrics", {}),
                               summary.get("root_group", {}),
                               overall, per_rpc, args.experiment,
                               gitaly_version))

    return 0


if __name__ == "__main__":
    sys.exit(main())