Skip to content

Make Codex and Claude Code subscriptions go further: a local model router that gives routine agent work to economical models and keeps your strongest model for the hard parts.

27.6 KB
Raw Blame
#!/usr/bin/env python3
"""Summarize exact-base versus candidate fixed-runner Gateway measurements."""

from __future__ import annotations

import argparse
import json
import math
import statistics
from pathlib import Path
from typing import Any


MIN_ROUNDS = 5
THRESHOLD_PERCENT = 5.0
# Per-request variance and mean CI are evidence-quality signals, rather than a
# second latency SLO.  A tenfold ceiling over the exact-base value catches
# broken measurement; a p50-derived 1% noise floor applies only when the base
# reports exact zero, leaving ordinary fixed-run scheduler jitter to the
# primary throughput/latency comparison.
DISPERSION_MAX_MULTIPLIER = 10.0
DISPERSION_NOISE_FLOOR_RATIO = 0.01
DYNAMIC_RESOURCE_DETAIL_METRICS = frozenset(
    {"queue_mean_ns", "run_mean_ns", "resume_mean_ns"}
)


class BenchmarkError(RuntimeError):
    pass


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument("--base-revision", required=True)
    parser.add_argument("--candidate-revision", required=True)
    parser.add_argument("--harness-revision", required=True)
    parser.add_argument("--base-core", type=Path, required=True)
    parser.add_argument("--candidate-core", type=Path, required=True)
    parser.add_argument("--base-resource", type=Path, required=True)
    parser.add_argument("--candidate-resource", type=Path, required=True)
    parser.add_argument("--base-replay", type=Path, required=True)
    parser.add_argument("--candidate-replay", type=Path, required=True)
    parser.add_argument("--base-product", type=Path, required=True)
    parser.add_argument("--candidate-product", type=Path, required=True)
    parser.add_argument("--base-hirouted-sha256", required=True)
    parser.add_argument("--candidate-hirouted-sha256", required=True)
    parser.add_argument("--output", type=Path, required=True)
    return parser.parse_args()


def text(path: Path) -> str:
    if not path.is_file():
        raise BenchmarkError(f"missing benchmark output: {path}")
    return path.read_text(encoding="utf-8", errors="replace")


def numeric(value: str, field: str) -> float:
    try:
        parsed = float(value)
    except ValueError as error:
        raise BenchmarkError(f"invalid {field}: {value!r}") from error
    if not math.isfinite(parsed):
        raise BenchmarkError(f"invalid {field}: expected a finite number")
    return parsed


def nonnegative_numeric(value: str, field: str) -> float:
    parsed = numeric(value, field)
    if parsed < 0:
        raise BenchmarkError(f"invalid {field}: expected a non-negative number")
    return parsed


def integer(value: str, field: str) -> int:
    try:
        return int(value)
    except ValueError as error:
        raise BenchmarkError(f"invalid {field}: {value!r}") from error


def aggregate(values: list[float]) -> dict[str, float | int]:
    if len(values) < MIN_ROUNDS:
        raise BenchmarkError(f"expected at least {MIN_ROUNDS} independent rounds, got {len(values)}")
    mean = statistics.fmean(values)
    variance = statistics.variance(values) if len(values) > 1 else 0.0
    return {
        "round_count": len(values),
        "mean": mean,
        "round_variance": variance,
        "round_mean_ci95": 1.96 * math.sqrt(variance / len(values)),
    }


def parse_detail(value: str) -> dict[str, str]:
    detail: dict[str, str] = {}
    for item in value.split(";"):
        if not item:
            continue
        key, separator, item_value = item.partition("=")
        if separator:
            detail[key] = item_value
    return detail


def parse_core(path: Path) -> dict[str, Any]:
    rows: dict[str, list[dict[str, float | int]]] = {"h1": [], "h2": []}
    for line in text(path).splitlines():
        fields = line.split("\t")
        if len(fields) != 10 or fields[0] not in rows or fields[2] != "gateway-production-driver":
            continue
        protocol, round_text, _, throughput, ttft_p50, ttft_p99, p50, p99, variance, ci = fields
        rows[protocol].append(
            {
                "round": integer(round_text, "round"),
                "throughput_ops_s": numeric(throughput, "throughput_ops_s"),
                "ttft_p50_ns": integer(ttft_p50, "ttft_p50_ns"),
                "ttft_p99_ns": integer(ttft_p99, "ttft_p99_ns"),
                "p50_ns": integer(p50, "p50_ns"),
                "p99_ns": integer(p99, "p99_ns"),
                "request_sample_variance_ns2": numeric(variance, "variance_ns2"),
                "request_mean_ci95_ns": numeric(ci, "mean_ci95_ns"),
            }
        )
    summary: dict[str, Any] = {}
    for protocol, protocol_rows in rows.items():
        if len(protocol_rows) < MIN_ROUNDS:
            raise BenchmarkError(f"{path}: {protocol} lacks {MIN_ROUNDS} gateway-production-driver rounds")
        rounds = {int(row["round"]) for row in protocol_rows}
        if len(rounds) != len(protocol_rows):
            raise BenchmarkError(f"{path}: {protocol} has duplicate benchmark rounds")
        metrics = {
            key: aggregate([float(row[key]) for row in protocol_rows])
            for key in ("throughput_ops_s", "ttft_p50_ns", "ttft_p99_ns", "p50_ns", "p99_ns")
        }
        metrics["request_sample_variance_ns2"] = {
            "mean": statistics.fmean(float(row["request_sample_variance_ns2"]) for row in protocol_rows)
        }
        metrics["request_mean_ci95_ns"] = {
            "mean": statistics.fmean(float(row["request_mean_ci95_ns"]) for row in protocol_rows)
        }
        summary[protocol] = {"rounds": protocol_rows, "metrics": metrics}
    return summary


def parse_replay(path: Path) -> dict[str, Any]:
    rows: dict[str, list[dict[str, Any]]] = {"memory-retained": [], "disk-spill": []}
    for line in text(path).splitlines():
        fields = line.split("\t")
        if len(fields) != 7 or fields[0] != "gateway_replay" or fields[1] not in rows:
            continue
        _, variant, round_text, payload_text, elapsed_text, ns_per_operation, detail_text = fields
        detail = parse_detail(detail_text)
        required = {"copy_bytes", "scan_bytes", "peak_retained_bytes", "peak_spill_bytes", "disk_backed"}
        if not required.issubset(detail):
            raise BenchmarkError(f"{path}: replay row is missing required resource metrics")
        row = {
            "round": integer(round_text, "replay round"),
            "payload_bytes": integer(payload_text, "payload_bytes"),
            "elapsed_ns": integer(elapsed_text, "elapsed_ns"),
            "ns_per_operation": integer(ns_per_operation, "ns_per_operation"),
            "copy_bytes": integer(detail["copy_bytes"], "copy_bytes"),
            "scan_bytes": integer(detail["scan_bytes"], "scan_bytes"),
            "peak_retained_bytes": integer(detail["peak_retained_bytes"], "peak_retained_bytes"),
            "peak_spill_bytes": integer(detail["peak_spill_bytes"], "peak_spill_bytes"),
            "disk_backed": detail["disk_backed"] == "true",
        }
        if row["copy_bytes"] != row["payload_bytes"] or row["scan_bytes"] != row["payload_bytes"]:
            raise BenchmarkError(f"{path}: replay row did not copy and verify the whole payload")
        if row["disk_backed"] != (variant == "disk-spill"):
            raise BenchmarkError(f"{path}: replay backing mode disagrees with its variant")
        if row["disk_backed"] and row["peak_spill_bytes"] == 0:
            raise BenchmarkError(f"{path}: disk-spill row reports zero backing bytes")
        if not row["disk_backed"] and row["peak_spill_bytes"] != 0:
            raise BenchmarkError(f"{path}: memory row reports unexpected spill bytes")
        rows[variant].append(row)
    summary: dict[str, Any] = {}
    for variant, variant_rows in rows.items():
        if len(variant_rows) < MIN_ROUNDS:
            raise BenchmarkError(f"{path}: {variant} lacks {MIN_ROUNDS} independent rounds")
        rounds = {int(row["round"]) for row in variant_rows}
        if len(rounds) != len(variant_rows):
            raise BenchmarkError(f"{path}: {variant} has duplicate benchmark rounds")
        summary[variant] = {
            "rounds": variant_rows,
            "metrics": {
                key: aggregate([float(row[key]) for row in variant_rows])
                for key in ("elapsed_ns", "copy_bytes", "scan_bytes", "peak_retained_bytes", "peak_spill_bytes")
            },
        }
    return summary


def parse_product(path: Path, expected_hirouted_sha256: str) -> dict[str, Any]:
    rows: dict[str, list[dict[str, Any]]] = {"memory-retained": [], "disk-spill": []}
    for line in text(path).splitlines():
        fields = line.split("\t")
        if len(fields) != 12 or fields[0] != "gateway_product" or fields[1] not in rows:
            continue
        (
            _,
            variant,
            round_text,
            iterations_text,
            throughput,
            ttft_p50,
            ttft_p99,
            p50,
            p99,
            variance,
            ci,
            detail_text,
        ) = fields
        detail = parse_detail(detail_text)
        required = {
            "copy_bytes",
            "scan_bytes",
            "peak_rss_delta_kib",
            "peak_replay_bytes",
            "terminal_replay_bytes",
            "hirouted_sha256",
        }
        if not required.issubset(detail):
            raise BenchmarkError(f"{path}: product row is missing required subprocess resource metrics")
        row = {
            "round": integer(round_text, "product round"),
            "iterations": integer(iterations_text, "product iterations"),
            "throughput_ops_s": numeric(throughput, "product throughput_ops_s"),
            "ttft_p50_ns": integer(ttft_p50, "product ttft_p50_ns"),
            "ttft_p99_ns": integer(ttft_p99, "product ttft_p99_ns"),
            "p50_ns": integer(p50, "product p50_ns"),
            "p99_ns": integer(p99, "product p99_ns"),
            "request_sample_variance_ns2": nonnegative_numeric(variance, "product variance_ns2"),
            "request_mean_ci95_ns": nonnegative_numeric(ci, "product mean_ci95_ns"),
            "copy_bytes": integer(detail["copy_bytes"], "product copy_bytes"),
            "scan_bytes": integer(detail["scan_bytes"], "product scan_bytes"),
            "peak_rss_delta_kib": integer(detail["peak_rss_delta_kib"], "product peak_rss_delta_kib"),
            "peak_replay_bytes": integer(detail["peak_replay_bytes"], "product peak_replay_bytes"),
            "terminal_replay_bytes": integer(detail["terminal_replay_bytes"], "product terminal_replay_bytes"),
            "hirouted_sha256": detail["hirouted_sha256"],
        }
        if row["iterations"] <= 0 or row["throughput_ops_s"] <= 0:
            raise BenchmarkError(f"{path}: product row did not execute a positive production workload")
        if row["copy_bytes"] <= 0 or row["scan_bytes"] <= 0:
            raise BenchmarkError(f"{path}: product row did not report request copy/scan evidence")
        if row["terminal_replay_bytes"] != 0:
            raise BenchmarkError(f"{path}: product row retained replay backing after terminal cleanup")
        if len(row["hirouted_sha256"]) != 64:
            raise BenchmarkError(f"{path}: product row has an invalid release hirouted digest")
        if row["hirouted_sha256"] != expected_hirouted_sha256:
            raise BenchmarkError(f"{path}: product row did not execute the release hirouted binary built for this revision")
        if variant == "memory-retained" and row["peak_replay_bytes"] != 0:
            raise BenchmarkError(f"{path}: memory-retained product row wrote replay backing")
        if variant == "disk-spill" and row["peak_replay_bytes"] == 0:
            raise BenchmarkError(f"{path}: disk-spill product row did not observe replay backing")
        rows[variant].append(row)
    summary: dict[str, Any] = {}
    for variant, variant_rows in rows.items():
        if len(variant_rows) < MIN_ROUNDS:
            raise BenchmarkError(f"{path}: {variant} lacks {MIN_ROUNDS} independent product rounds")
        rounds = {int(row["round"]) for row in variant_rows}
        if len(rounds) != len(variant_rows):
            raise BenchmarkError(f"{path}: {variant} has duplicate product benchmark rounds")
        summary[variant] = {
            "rounds": variant_rows,
            "metrics": {
                key: aggregate([float(row[key]) for row in variant_rows])
                for key in (
                    "throughput_ops_s",
                    "ttft_p50_ns",
                    "ttft_p99_ns",
                    "p50_ns",
                    "p99_ns",
                    "copy_bytes",
                    "scan_bytes",
                    "peak_rss_delta_kib",
                    "peak_replay_bytes",
                    "request_sample_variance_ns2",
                    "request_mean_ci95_ns",
                )
            },
        }
    return summary


def parse_resource(path: Path) -> dict[str, Any]:
    rows: dict[str, list[dict[str, Any]]] = {}
    for line in text(path).splitlines():
        fields = line.split("\t")
        if len(fields) < 6 or fields[0] in {"resource", "release", "rounds"}:
            continue
        try:
            resource = fields[0]
            variant = fields[1]
            row = {
                "iterations": integer(fields[2], "resource iterations"),
                "elapsed_ns": integer(fields[3], "resource elapsed_ns"),
                "ns_per_operation": integer(fields[4], "resource ns_per_operation"),
                "detail": parse_detail(fields[5]),
            }
        except BenchmarkError:
            continue
        rows.setdefault(f"{resource}:{variant}", []).append(row)
    if not rows:
        raise BenchmarkError(f"{path}: no resource benchmark rows")
    summary: dict[str, Any] = {}
    for key, values in rows.items():
        if len(values) < MIN_ROUNDS:
            raise BenchmarkError(f"{path}: {key} lacks {MIN_ROUNDS} independent resource rounds")
        detail_keys = set.intersection(*(set(row["detail"]) for row in values))
        detail_metrics = {
            detail_key: aggregate([numeric(row["detail"][detail_key], f"{key} {detail_key}") for row in values])
            for detail_key in sorted(detail_keys)
            if all(_is_numeric(row["detail"][detail_key]) for row in values)
        }
        summary[key] = {
            "rounds": values,
            "metrics": {
                "ns_per_operation": aggregate([float(row["ns_per_operation"]) for row in values]),
                "detail": detail_metrics,
            },
        }
    return summary


def _is_numeric(value: str) -> bool:
    try:
        float(value)
    except ValueError:
        return False
    return True


def percent_regression(base: float, candidate: float, higher_is_worse: bool) -> float:
    if base < 0 or candidate < 0:
        raise BenchmarkError("benchmark metric must not be negative")
    if base == 0:
        return 0.0 if candidate == 0 else math.inf
    delta = (candidate - base) / base * 100.0
    return max(delta if higher_is_worse else -delta, 0.0)


def mean(metrics: dict[str, Any], key: str) -> float:
    return float(metrics[key]["mean"])


def add_threshold_error(
    errors: list[str],
    label: str,
    base: float,
    candidate: float,
    higher_is_worse: bool,
) -> float:
    regression = percent_regression(base, candidate, higher_is_worse)
    if regression > THRESHOLD_PERCENT:
        errors.append(f"{label} regression exceeds {THRESHOLD_PERCENT}%")
    return regression


def confidence_aware_timing_comparison(
    base_metrics: dict[str, Any], candidate_metrics: dict[str, Any], metric: str = "ns_per_operation"
) -> dict[str, float]:
    """Compare timing means only after allowing each five-round 95% interval.

    This preserves the 5% fixed-run limit while avoiding an exact-value check on
    scheduler-dependent measurements.  A candidate must be slower even at its
    favorable confidence bound than the base at its unfavorable bound.
    """

    base_mean = mean(base_metrics, metric)
    candidate_mean = mean(candidate_metrics, metric)
    base_ci95 = float(base_metrics[metric]["round_mean_ci95"])
    candidate_ci95 = float(candidate_metrics[metric]["round_mean_ci95"])
    base_upper = base_mean + base_ci95
    candidate_lower = max(candidate_mean - candidate_ci95, 0.0)
    return {
        "base_mean": base_mean,
        "candidate_mean": candidate_mean,
        "base_round_mean_ci95": base_ci95,
        "candidate_round_mean_ci95": candidate_ci95,
        "observed_regression_pct": percent_regression(base_mean, candidate_mean, higher_is_worse=True),
        "confidence_aware_regression_pct": percent_regression(
            base_upper, candidate_lower, higher_is_worse=True
        ),
    }


def add_confidence_aware_timing_error(
    errors: list[str],
    label: str,
    base_metrics: dict[str, Any],
    candidate_metrics: dict[str, Any],
    metric: str = "ns_per_operation",
) -> dict[str, float]:
    comparison = confidence_aware_timing_comparison(base_metrics, candidate_metrics, metric)
    comparison["threshold_pct"] = THRESHOLD_PERCENT
    if comparison["confidence_aware_regression_pct"] > THRESHOLD_PERCENT:
        errors.append(f"{label} confidence-aware regression exceeds {THRESHOLD_PERCENT}%")
    return comparison


def compare_dispersion(
    errors: list[str],
    label: str,
    base_value: float,
    candidate_value: float,
    latency_scale_ns: float,
    *,
    squared_units: bool,
) -> dict[str, float]:
    """Gate unusable product variance/CI evidence without treating normal jitter as a regression."""

    noise_floor = max(latency_scale_ns * DISPERSION_NOISE_FLOOR_RATIO, 1.0)
    if squared_units:
        noise_floor *= noise_floor
    reference = base_value if base_value > 0 else noise_floor
    multiplier = candidate_value / reference
    if multiplier > DISPERSION_MAX_MULTIPLIER:
        errors.append(
            f"{label} exceeds {DISPERSION_MAX_MULTIPLIER:g}x exact-base/noise-floor dispersion policy"
        )
    return {
        "base_mean": base_value,
        "candidate_mean": candidate_value,
        "noise_floor": noise_floor,
        "reference": reference,
        "candidate_multiplier": multiplier,
        "max_multiplier": DISPERSION_MAX_MULTIPLIER,
    }


def compare_core(base: dict[str, Any], candidate: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
    comparison: dict[str, Any] = {}
    errors: list[str] = []
    for protocol in ("h1", "h2"):
        base_metrics = base[protocol]["metrics"]
        candidate_metrics = candidate[protocol]["metrics"]
        throughput_regression = add_threshold_error(
            errors,
            f"{protocol} throughput",
            mean(base_metrics, "throughput_ops_s"),
            mean(candidate_metrics, "throughput_ops_s"),
            higher_is_worse=False,
        )
        p99_increase = add_threshold_error(
            errors,
            f"{protocol} p99",
            mean(base_metrics, "p99_ns"),
            mean(candidate_metrics, "p99_ns"),
            higher_is_worse=True,
        )
        comparison[protocol] = {
            "throughput_regression_pct": throughput_regression,
            "p99_increase_pct": p99_increase,
            "threshold_pct": THRESHOLD_PERCENT,
        }
    return comparison, errors


def compare_replay(base: dict[str, Any], candidate: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
    result: dict[str, Any] = {}
    errors: list[str] = []
    for variant in ("memory-retained", "disk-spill"):
        base_metrics = base[variant]["metrics"]
        candidate_metrics = candidate[variant]["metrics"]
        comparison: dict[str, Any] = {"threshold_pct": THRESHOLD_PERCENT}
        for metric in ("elapsed_ns", "peak_retained_bytes", "peak_spill_bytes"):
            base_value = mean(base_metrics, metric)
            candidate_value = mean(candidate_metrics, metric)
            comparison[metric] = {
                "base_mean": base_value,
                "candidate_mean": candidate_value,
                "regression_pct": add_threshold_error(
                    errors,
                    f"gateway_replay {variant} {metric}",
                    base_value,
                    candidate_value,
                    higher_is_worse=True,
                ),
            }
        for metric in ("copy_bytes", "scan_bytes"):
            base_value = mean(base_metrics, metric)
            candidate_value = mean(candidate_metrics, metric)
            comparison[metric] = {"base_mean": base_value, "candidate_mean": candidate_value}
            if candidate_value != base_value:
                errors.append(f"gateway_replay {variant} {metric} differs from exact base")
        result[variant] = comparison
    return result, errors


def compare_product(base: dict[str, Any], candidate: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
    result: dict[str, Any] = {}
    errors: list[str] = []
    for variant in ("memory-retained", "disk-spill"):
        base_metrics = base[variant]["metrics"]
        candidate_metrics = candidate[variant]["metrics"]
        comparison: dict[str, Any] = {
            "threshold_pct": THRESHOLD_PERCENT,
            "dispersion_policy": {
                "max_multiplier": DISPERSION_MAX_MULTIPLIER,
                "noise_floor_ratio_of_base_p50": DISPERSION_NOISE_FLOOR_RATIO,
            },
        }
        for metric, higher_is_worse in (
            ("throughput_ops_s", False),
            ("ttft_p50_ns", True),
            ("ttft_p99_ns", True),
            ("p50_ns", True),
            ("p99_ns", True),
            ("peak_rss_delta_kib", True),
            ("peak_replay_bytes", True),
        ):
            base_value = mean(base_metrics, metric)
            candidate_value = mean(candidate_metrics, metric)
            comparison[metric] = {
                "base_mean": base_value,
                "candidate_mean": candidate_value,
                "regression_pct": add_threshold_error(
                    errors,
                    f"gateway_product {variant} {metric}",
                    base_value,
                    candidate_value,
                    higher_is_worse=higher_is_worse,
                ),
            }
        for metric in ("copy_bytes", "scan_bytes"):
            base_value = mean(base_metrics, metric)
            candidate_value = mean(candidate_metrics, metric)
            comparison[metric] = {"base_mean": base_value, "candidate_mean": candidate_value}
            if candidate_value != base_value:
                errors.append(f"gateway_product {variant} {metric} differs from exact base")
        latency_scale_ns = max(mean(base_metrics, "p50_ns"), 1.0)
        comparison["request_sample_variance_ns2"] = compare_dispersion(
            errors,
            f"gateway_product {variant} request_sample_variance_ns2",
            mean(base_metrics, "request_sample_variance_ns2"),
            mean(candidate_metrics, "request_sample_variance_ns2"),
            latency_scale_ns,
            squared_units=True,
        )
        comparison["request_mean_ci95_ns"] = compare_dispersion(
            errors,
            f"gateway_product {variant} request_mean_ci95_ns",
            mean(base_metrics, "request_mean_ci95_ns"),
            mean(candidate_metrics, "request_mean_ci95_ns"),
            latency_scale_ns,
            squared_units=False,
        )
        result[variant] = comparison
    return result, errors


def compare_resource(base: dict[str, Any], candidate: dict[str, Any]) -> tuple[dict[str, Any], list[str]]:
    result: dict[str, Any] = {}
    errors: list[str] = []
    if set(base) != set(candidate):
        errors.append("resource benchmark paths differ between exact base and candidate")
    for key in sorted(set(base) & set(candidate)):
        base_metrics = base[key]["metrics"]
        candidate_metrics = candidate[key]["metrics"]
        comparison: dict[str, Any] = {
            "threshold_pct": THRESHOLD_PERCENT,
            "timing_policy": "five-round 95% confidence-aware relative regression",
            "ns_per_operation": add_confidence_aware_timing_error(
                errors,
                f"resource {key} ns_per_operation",
                base_metrics,
                candidate_metrics,
            ),
            "detail": {},
        }
        base_detail = base_metrics["detail"]
        candidate_detail = candidate_metrics["detail"]
        if set(base_detail) != set(candidate_detail):
            errors.append(f"resource {key} detail metrics differ between exact base and candidate")
        for detail_key in sorted(set(base_detail) & set(candidate_detail)):
            base_value = mean(base_detail, detail_key)
            candidate_value = mean(candidate_detail, detail_key)
            if detail_key in DYNAMIC_RESOURCE_DETAIL_METRICS:
                comparison["detail"][detail_key] = add_confidence_aware_timing_error(
                    errors,
                    f"resource {key} {detail_key}",
                    base_detail,
                    candidate_detail,
                    detail_key,
                )
            else:
                comparison["detail"][detail_key] = {
                    "base_mean": base_value,
                    "candidate_mean": candidate_value,
                }
                if candidate_value != base_value:
                    errors.append(f"resource {key} {detail_key} differs from exact base")
        result[key] = comparison
    return result, errors


def run(args: argparse.Namespace) -> dict[str, Any]:
    base_core = parse_core(args.base_core)
    candidate_core = parse_core(args.candidate_core)
    core_comparison, core_errors = compare_core(base_core, candidate_core)
    base_replay = parse_replay(args.base_replay)
    candidate_replay = parse_replay(args.candidate_replay)
    replay_comparison, replay_errors = compare_replay(base_replay, candidate_replay)
    base_product = parse_product(args.base_product, args.base_hirouted_sha256)
    candidate_product = parse_product(args.candidate_product, args.candidate_hirouted_sha256)
    product_comparison, product_errors = compare_product(base_product, candidate_product)
    base_resource = parse_resource(args.base_resource)
    candidate_resource = parse_resource(args.candidate_resource)
    resource_comparison, resource_errors = compare_resource(base_resource, candidate_resource)
    errors = core_errors + replay_errors + product_errors + resource_errors
    return {
        "schema_version": "hiroute.p0-gateway-fixed-runner-benchmark/v2",
        "base_revision": args.base_revision,
        "candidate_revision": args.candidate_revision,
        "benchmark_harness_revision": args.harness_revision,
        "release_mode": True,
        "semantic_status": "green" if not errors else "red",
        "errors": errors,
        "core": {"base": base_core, "candidate": candidate_core, "comparison": core_comparison},
        "gateway_replay": {"base": base_replay, "candidate": candidate_replay, "comparison": replay_comparison},
        "gateway_product": {"base": base_product, "candidate": candidate_product, "comparison": product_comparison},
        "resource_paths": {"base": base_resource, "candidate": candidate_resource, "comparison": resource_comparison},
    }


def main() -> int:
    args = parse_args()
    try:
        payload = run(args)
    except (BenchmarkError, OSError, ValueError) as error:
        payload = {
            "schema_version": "hiroute.p0-gateway-fixed-runner-benchmark/v2",
            "base_revision": args.base_revision,
            "candidate_revision": args.candidate_revision,
            "benchmark_harness_revision": args.harness_revision,
            "release_mode": True,
            "semantic_status": "red",
            "errors": [str(error)],
        }
    args.output.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
    print(json.dumps({"semantic_status": payload["semantic_status"], "errors": payload["errors"]}, sort_keys=True))
    return 0 if payload["semantic_status"] == "green" else 1


if __name__ == "__main__":
    raise SystemExit(main())