Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

15.9 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Render docs/source/get-started/performance.rst from benchmarks/results/**.json.

The same result files feed the landing page: ``render_hero_svg`` draws the CPU-vs-accelerator bar
chart in the hero's "GPU-accelerated" tab, so the figures there are read from the committed run
rather than typed in.
"""

from __future__ import annotations

import json
import re
import sys
from html import escape
from pathlib import Path

REPO = Path(__file__).resolve().parents[1]
RESULTS = REPO / "benchmarks" / "results"
OUT = REPO / "docs" / "source" / "get-started" / "performance.rst"
HERO_OUT = REPO / "docs" / "source" / "_generated" / "hero-benchmark.html"
LLMS_FULL = REPO / "docs" / "source" / "_extra" / "llms-full.txt"

# The one comparison the landing page draws: a kornia op at one batch size, CPU against the
# accelerator of the same machine. ``machine`` is a preference; any machine with both a CPU and an
# accelerator result set for the latest version is used when that one is absent.
HERO = {
    "suite": "augmentation",
    "machine": "i7-14700k-rtx-4090",
    "op": "RandomGaussianBlur",
    "batch": 32,
    "backend": "kornia (eager)",
}

BEGIN, END = "<!-- BENCH:BEGIN -->", "<!-- BENCH:END -->"

#: Directory under ``benchmarks/results/`` holding snapshots that are kept for history but are no
#: longer published. A snapshot is identified by its kornia version **and** its ``git_commit``: a
#: version directory spans many commits, so a run taken before a change that altered an op's speed
#: is stale even though it sits in the current version's directory and carries a recent date.
SUPERSEDED_DIR = "superseded"

INTRO = """\
Performance
===========

Numbers below are generated at docs-build time from the committed benchmark result files in
``benchmarks/results/`` β€” median wall clock over repeated runs with device synchronization and
recorded hardware/software metadata (see ``benchmarks/README.md`` for methodology). Kornia's
regime is batched float tensors on an accelerator, differentiable end-to-end; OpenCV and
albumentations win the single-image uint8 CPU regime and we publish those losses too.

.. warning::

   Each table is a snapshot of one machine. Before contributing a run: close other applications,
   use mains power, and let the machine cool. Expect run-to-run variance (about Β±5% on CPU and
   up to Β±30% on MPS). Never compare numbers across machines β€” only across columns of one table.
   Only aggregate load metrics (load average, memory) are recorded, never process names.

Reproduce or contribute a machine with::

   python benchmarks/<dir>/<script>.py --device <cpu|cuda|mps> --contribute benchmarks/results

Most suites are a ``flagship.py``; ``benchmarks/README.md`` maps each suite to its script.

CPU: large batches and the memory allocator
-------------------------------------------

With glibc on Linux, an operation whose temporaries are large gets them as fresh memory from the operating system on
every call and faults it in page by page: glibc serves a buffer above its mmap threshold (which rises to at most 32 MiB
on 64-bit systems) with a fresh mapping and unmaps it when it is freed, and it returns the free top of its heap to the
system once that exceeds the trim threshold. Smaller temporaries stop faulting once those thresholds have adapted to the
process, so a fresh process overstates their cost. Comparing ``resource.getrusage(resource.RUSAGE_SELF).ru_minflt``
before and after a call counts its faults. At batch 32 of 256 x 256 float32 images with 4 threads, on an i7-14700K under
WSL2 with torch 2.14, ``bilateral_blur`` took 544 ms per call, ``ssim`` 270 ms and ``dice_loss`` 390 ms (the median of
three processes), faulting 440,000 to 730,000 pages per call; with those per-call faults removed by::

   GLIBC_TUNABLES=glibc.malloc.mmap_max=0:glibc.malloc.trim_threshold=68719476736 python train.py

they took 344 ms, 135 ms and 214 ms, with no faults per call after the first. The setting keeps freed memory in the
process, so its resident memory stays at its peak. Loading jemalloc or tcmalloc through ``LD_PRELOAD`` is an
alternative, not measured here. Faults cost less on native Linux than under WSL2. This is glibc's behaviour: macOS uses
another allocator and is not covered here, and CUDA tensors come from PyTorch's caching allocator, which keeps freed
blocks.

"""


def load_results(results_root: Path) -> dict[str, dict[str, dict]]:
    """Committed snapshots by version directory, excluding anything under ``superseded/``.

    Superseded runs keep their version directory name inside ``superseded/`` so the schema rules
    still apply to them; skipping them by path keeps them out of the page and the digest.
    """
    root = Path(results_root)
    out: dict[str, dict[str, dict]] = {}
    for path in sorted(root.rglob("*.json")):
        if SUPERSEDED_DIR in path.relative_to(root).parts[:-1]:
            continue
        out.setdefault(path.parent.name, {})[path.name] = json.loads(path.read_text())
    return out


def _version_key(v: str) -> tuple:
    # pad to three numeric parts so digit-less dirs (e.g. "unknown") stay comparable
    nums = ([int(x) for x in re.findall(r"\d+", v)[:3]] + [0, 0, 0])[:3]
    is_final = not re.search(r"(rc|a|b|dev)\d*$", v)
    return (*nums, is_final, v)


def latest_version(versions: list[str]) -> str:
    return max(versions, key=_version_key)


#: Row fields that identify the measurement rather than the configuration it was taken at.
MEASUREMENT_FIELDS = frozenset({"op", "backend", "median_us", "iqr_us", "throughput_per_s", "error"})


def _extra_config_fields(rows: list[dict]) -> list[str]:
    """Config fields besides ``batch`` that differ between rows of one result file.

    ``op`` plus ``batch`` identifies a row in a suite that sweeps batch size alone, which is what
    the flagship suites do; a suite sweeping a second axis (``feature-laf-ops`` varies ``n_lafs``
    at a fixed batch) would otherwise collapse two configs onto one row and render whichever came
    last. Fields that are constant across the file -- image size, dtype -- stay out of the label
    and are already disclosed in the metadata line above the table.
    """
    keys = [k for k in dict.fromkeys(k for r in rows for k in r) if k not in MEASUREMENT_FIELDS and k != "batch"]
    return [k for k in keys if len({repr(r.get(k)) for r in rows}) > 1]


def _config_key(row: dict, extra: list[str]) -> tuple:
    return (row["op"], row["batch"], *(repr(row.get(k)) for k in extra))


def _config_label(row: dict, extra: list[str]) -> str:
    label = f"{row['op']} @ {row['batch']}"
    if extra:
        label += " (" + ", ".join(f"{k}={row.get(k)}" for k in extra) + ")"
    return label


def _table(payload: dict) -> str:
    rows = payload["results"]
    backends = list(dict.fromkeys(r["backend"] for r in rows))
    extra = _extra_config_fields(rows)
    lines = [".. list-table::", "   :header-rows: 1", "", "   * - op @ batch"]
    lines += [f"     - {b}" for b in backends]
    cells = {(_config_key(r, extra), r["backend"]): r.get("throughput_per_s") for r in rows}
    labels = {_config_key(r, extra): _config_label(r, extra) for r in rows}
    for key in dict.fromkeys(_config_key(r, extra) for r in rows):
        lines.append(f"   * - {labels[key]}")
        for b in backends:
            v = cells.get((key, b))
            lines.append(f"     - {v:.0f}" if isinstance(v, (int, float)) else "     - β€”")
    return "\n".join(lines) + "\n"


def render_page(results_root: Path) -> str:
    data = load_results(results_root)
    if not data:
        return INTRO + "\nNo benchmark results are committed yet.\n"
    version = latest_version(list(data))
    parts = [INTRO, f"Results for kornia {version}\n{'-' * (20 + len(version))}\n"]
    for fname, payload in sorted(data[version].items()):
        suite, slug, device = fname[:-5].split("--")
        meta = payload["metadata"]
        parts.append(f"\n{suite} β€” {slug} ({device})\n{'^' * (len(suite) + len(slug) + len(device) + 6)}\n")
        parts.append(
            f"``{meta['platform']}`` β€” torch {meta['torch']}, kornia {meta['kornia']}, "
            f"commit ``{meta['git_commit']}``, {meta['timestamp_utc'][:10]}, "
            f"throughput in {meta.get('units') or 'items/s'}\n\n"
        )
        parts.append(_table(payload))
    older = sorted(set(data) - {version})
    if older:
        parts.append("\nOlder result sets in git: " + ", ".join(f"``benchmarks/results/{v}/``" for v in older) + "\n")
    if (Path(results_root) / SUPERSEDED_DIR).is_dir():
        parts.append(
            f"\nSnapshots measured before a change that altered the numbers they report are kept, unpublished, in "
            f"``benchmarks/results/{SUPERSEDED_DIR}/`` β€” see the README there for what replaced each one.\n"
        )
    return "\n".join(parts)


def _throughput(payload: dict, op: str, batch: int, backend: str) -> float | None:
    for row in payload["results"]:
        if row["op"] == op and row["batch"] == batch and row["backend"] == backend:
            value = row.get("throughput_per_s")
            return float(value) if isinstance(value, (int, float)) else None
    return None


def hero_figures(results_root: Path) -> dict | None:
    """CPU and accelerator throughput of the ``HERO`` op from the latest committed result set.

    Returns ``None`` when no machine has both measurements, so the caller can leave the chart out.
    """
    data = load_results(results_root)
    if not data:
        return None
    version = latest_version(list(data))
    by_machine: dict[str, dict[str, dict]] = {}
    for fname, payload in data[version].items():
        suite, slug, device = fname[:-5].split("--")
        if suite == HERO["suite"]:
            by_machine.setdefault(slug, {})[device] = payload
    machines = sorted(by_machine, key=lambda slug: (slug != HERO["machine"], slug))
    for slug in machines:
        devices = by_machine[slug]
        accelerators = sorted(device for device in devices if device != "cpu")
        if "cpu" not in devices or not accelerators:
            continue
        cpu = _throughput(devices["cpu"], HERO["op"], HERO["batch"], HERO["backend"])
        gpu = _throughput(devices[accelerators[0]], HERO["op"], HERO["batch"], HERO["backend"])
        if cpu is None or gpu is None or cpu <= 0 or gpu <= 0:
            continue
        meta = devices[accelerators[0]]["metadata"]
        return {
            "machine": slug.replace("-", " ").title(),  # "apple-m1" -> "Apple M1"
            "device": accelerators[0],
            "cpu": cpu,
            "gpu": gpu,
            "speedup": gpu / cpu,
            "kornia": meta["kornia"],
            "torch": meta["torch"],
        }
    return None


def render_hero_svg(results_root: Path) -> str:
    """The landing page's CPU-vs-accelerator bar chart, as an HTML fragment; empty when there is no data."""
    fig = hero_figures(results_root)
    if fig is None:
        return ""
    full, left = 516, 80  # the longer bar spans the chart; the other is scaled to it
    longest = max(fig["cpu"], fig["gpu"])

    def bar(y: int, value: float, text: str, fill: str, text_class: str) -> str:
        width = round(full * value / longest)
        rect = f'<rect x="{left}" y="{y}" width="{width}" height="32" rx="6" class="{fill}"/>'
        if width == full:  # no room to the right: print the value inside the bar, right-aligned
            anchor = f'x="{left + width - 12}" y="{y + 23}" class="illo-value {text_class}" text-anchor="end"'
        else:
            anchor = f'x="{left + width + 12}" y="{y + 23}" class="illo-value"'
        return f"{rect}\n    <text {anchor}>{text}</text>"

    faster = "GPU" if fig["gpu"] >= fig["cpu"] else "CPU"
    ratio = max(fig["gpu"], fig["cpu"]) / min(fig["gpu"], fig["cpu"])
    speedup = f" Β· {ratio:.1f}\N{MULTIPLICATION SIGN} faster"
    cpu_text = f"{fig['cpu']:,.0f} img/s" + (speedup if faster == "CPU" else "")
    gpu_text = f"{fig['gpu']:,.0f} img/s" + (speedup if faster == "GPU" else "")
    title = escape(f"{HERO['op']} Β· batch {HERO['batch']} Β· {fig['machine']}")
    label = escape(
        f"{HERO['op']} at batch {HERO['batch']} on an {fig['machine']}: {fig['cpu']:,.0f} images per second on "
        f"the CPU, {fig['gpu']:,.0f} on the GPU"
    )
    note = escape(f"items/s, eager mode β€” committed benchmark run, kornia {fig['kornia']} / torch {fig['torch']}")
    return f"""\
<div class="kornia-tab-visual">
  <svg class="kornia-illo" viewBox="0 0 640 220" role="img" aria-label="{label}">
    <text x="24" y="36" class="illo-title">{title}</text>
    <text x="24" y="90" class="illo-label">CPU</text>
    {bar(68, fig["cpu"], cpu_text, "illo-muted", "")}
    <text x="24" y="152" class="illo-label">GPU</text>
    {bar(130, fig["gpu"], gpu_text, "illo-primary", "illo-on-primary")}
    <text x="24" y="204" class="illo-note">{note}</text>
  </svg>
</div>
"""


def _digest(results_root: Path) -> str:
    data = load_results(results_root)
    if not data:
        return "- No committed benchmark results yet.\n"
    version = latest_version(list(data))
    lines = [f"- Result set: kornia {version}, committed in `benchmarks/results/{version}/` (per-machine"]
    lines += ["  snapshots; reproduce with `python benchmarks/<dir>/<script>.py --contribute benchmarks/results`,"]
    lines += ["  the script each suite maps to in `benchmarks/README.md`)."]
    for fname, payload in sorted(data[version].items()):
        suite, slug, device = fname[:-5].split("--")
        meta = payload["metadata"]
        rows = [r for r in payload["results"] if isinstance(r.get("throughput_per_s"), (int, float))]
        if not rows:
            continue
        best = max(rows, key=lambda r: r["throughput_per_s"])
        kornia_rows = [r for r in rows if r["backend"].startswith("kornia")]
        worst = min(kornia_rows, key=lambda r: r["throughput_per_s"]) if kornia_rows else None
        extra = _extra_config_fields(rows)
        units = meta.get("units") or "items/s"

        def _at(row: dict, extra: list[str] = extra) -> str:
            tail = "".join(f",{k}={row.get(k)}" for k in extra)
            return f"{row['op']}@{row['batch']}{tail}"

        # kornia version + commit, not only the date: one version directory spans many commits, so
        # the date alone cannot tell a reader which implementation a line describes.
        line = (
            f"- {suite} on {slug}/{device} (kornia {meta['kornia']} @ {meta['git_commit']}, "
            f"{meta['timestamp_utc'][:10]}): fastest overall "
            f"{best['backend']} {_at(best)} at {best['throughput_per_s']:.0f} {units}"
        )
        if worst is not None:
            line += f"; slowest kornia op {_at(worst)} at {worst['throughput_per_s']:.0f} {units}"
        lines.append(line + ".")
    return "\n".join(lines) + "\n"


def refresh_llms(llms_path: Path, results_root: Path) -> None:
    text = Path(llms_path).read_text()
    if BEGIN not in text or END not in text or text.index(BEGIN) > text.index(END):
        raise RuntimeError(f"markers {BEGIN}/{END} not found (in order) in {llms_path}")
    head, rest = text.split(BEGIN, 1)
    _, tail = rest.split(END, 1)
    Path(llms_path).write_text(f"{head}{BEGIN}\n{_digest(results_root)}{END}{tail}")


def main() -> None:
    OUT.write_text(render_page(RESULTS))
    HERO_OUT.parent.mkdir(exist_ok=True)
    HERO_OUT.write_text(render_hero_svg(RESULTS))


if __name__ == "__main__":
    if "--refresh-llms" in sys.argv:
        refresh_llms(LLMS_FULL, RESULTS)
    else:
        main()