π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Shared harness of the library-wide graph-capture survey behind the ONNX / torch.compile / torch.export
survey page (docs/source/get-started/export-support.rst, rendered by docs/generate_export_support.py).
A *case* is one public kornia callable (function or nn.Module instance) plus a concrete set of
tensor inputs. Every tensor input becomes an ONNX graph input; Python-only kwargs are baked into
the graph. The harness:
1. runs the target eagerly (reference),
2. exports with ``torch.onnx.export(dynamo=True, opset_version=18)`` (or, with ``dynamo=False``, the legacy
TorchScript exporter at its highest opset up to 18),
3. runs ``onnx.checker.check_model`` (full check),
4. runs the graph in onnxruntime (CPU) and compares every tensor output against the reference: the same dtype,
and the values within the case's ``atol`` / ``rtol``,
5. records op types, size, wall time and (on failure) the root-cause error.
Importing this module (or a registry) leaves the process alone: the survey entry points (``run_cases``,
``probe_compile.main``) call ``configure_survey_process`` for their quieter, fixed-thread-count settings, so
``tests/onnx/test_conformance.py`` can drive ``run_case`` inside a pytest session. ``onnx`` and ``onnxruntime`` are
imported where they are used, so the compile probe and ``tests/test_inductor_conformance.py`` do not need them.
Usage from a registry file::
from harness import case, run_cases
CASES = [case("rgb_to_grayscale", "color", K.color.rgb_to_grayscale, [torch.rand(1, 3, 32, 32)]), ...]
if __name__ == "__main__":
run_cases(CASES, sys.argv[1], only=sys.argv[2:] or None)
Statuses: ok | ok-unverified (check=False, nondeterministic op: output count, shapes and finiteness checked) |
eager-fail | export-fail | checker-fail | ort-load-fail | ort-run-fail | mismatch | no-tensor-output | skipped.
"""
from __future__ import annotations
import contextlib
import copy
import inspect
import io
import json
import logging
import os
import subprocess
import time
import traceback
import warnings
from typing import TYPE_CHECKING, Any
import numpy as np
import torch
from torch import nn
from torch.utils._pytree import tree_flatten
if TYPE_CHECKING:
import onnx
SEED = 1234
OPSET = 18 # the opset the dynamo export requests, and the ceiling of the legacy one
def configure_survey_process() -> None:
"""Quiet logs and warnings and fix the thread count, for a process that runs survey probes and nothing else."""
logging.disable(logging.CRITICAL)
warnings.filterwarnings("ignore")
try:
import onnxruntime as ort
except ImportError: # the compile probe runs without it
pass
else:
ort.set_default_logger_severity(3)
torch.set_num_threads(4)
def case(
name: str,
group: str,
target: Any,
inputs: list[torch.Tensor],
kwargs: dict[str, Any] | None = None,
*,
note: str = "",
check: bool = True,
atol: float = 2e-4,
rtol: float = 1e-3,
skip: str | None = None,
tags: tuple[str, ...] = (),
method: str | None = None,
) -> dict[str, Any]:
"""Describe one survey case.
Args:
name: unique display name (usually the public callable, e.g. ``"filters.filter3d"``).
group: report grouping (``"filters"``, ``"geometry.epipolar"``, ``"feature.descriptors"``...).
target: nn.Module instance or plain callable ``fn(*inputs, **kwargs)``.
inputs: tensors; each becomes a live ONNX input. Non-tensor positional args go in ``kwargs``
only when the callable accepts them by keyword; otherwise wrap with a lambda.
kwargs: Python-only keyword args baked as constants.
note: free text shown in the report (e.g. "kernel_size baked", "weights: hardnet liberty").
check: False for ops with randomness inside the graph -> export+run only, output unverified.
skip: reason string; the case is reported as skipped without running.
tags: e.g. ("3d", "model", "pretrained").
method: call this method on the module instead of ``forward`` (e.g. ``"detect"``).
"""
return {
"name": name,
"group": group,
"target": target,
"inputs": list(inputs),
"kwargs": dict(kwargs or {}),
"note": note,
"check": check,
"atol": atol,
"rtol": rtol,
"skip": skip,
"tags": list(tags),
"method": method,
}
# ----------------------------------------------------------------------------- wrapping
def _flatten_outputs(out: Any) -> list[torch.Tensor]:
# A tensor first, before any attribute probe: on torch 2.5 ``hasattr(tensor, "data")`` inside a ``torch.export``
# trace raises "Attempting to use FunctionalTensor on its own", which fails the export of every operator that
# returns a plain tensor (the dynamo ONNX exporter then falls back to its TorchScript-trace strategy).
if isinstance(out, torch.Tensor):
return [out]
# dataclass-like kornia containers (Boxes, Keypoints, Se3, Quaternion, Image...) expose tensors via attributes
if hasattr(out, "data") and isinstance(out.data, torch.Tensor):
return [out.data]
if hasattr(out, "__dict__") and not isinstance(out, (torch.Tensor, dict, list, tuple)):
vals = []
for v in vars(out).values():
vals += _flatten_outputs(v)
if vals:
return vals
leaves, _ = tree_flatten(out)
res = []
for x in leaves:
if isinstance(x, torch.Tensor):
res.append(x)
elif hasattr(x, "data") and isinstance(x.data, torch.Tensor):
res.append(x.data)
return res
def make_wrapper(target: Any, kwargs: dict[str, Any], n_inputs: int, method: str | None) -> nn.Module:
"""Return an nn.Module with a fixed-arity forward(in0, ..., inN-1) -> tuple[Tensor, ...]."""
args = ", ".join(f"in{i}" for i in range(n_inputs))
src = f"def forward(self, {args}):\n return self._call(({args},))\n"
ns: dict[str, Any] = {}
exec(src, ns) # noqa: S102
class Wrap(nn.Module):
def __init__(self) -> None:
super().__init__()
if isinstance(target, nn.Module):
self.target = target
else:
self._fn = target
self.kw = kwargs
def _call(self, xs: tuple) -> tuple:
t = self.target if isinstance(target, nn.Module) else self._fn
fn = getattr(t, method) if method else t
out = fn(*xs, **self.kw)
outs = _flatten_outputs(out)
if not outs:
raise RuntimeError("NO_TENSOR_OUTPUT")
return tuple(outs)
Wrap.forward = ns["forward"]
# in eval mode: the legacy exporter puts the wrapper back into the mode it found, children included
return Wrap().eval()
# ----------------------------------------------------------------------------- utilities
def to_np(x: torch.Tensor) -> np.ndarray:
x = x.detach().cpu()
if x.dtype == torch.bfloat16:
x = x.float()
return x.numpy()
def _dtype_name(x: Any) -> str:
"""The dtype of a tensor or an array as numpy names it (``float32``, ``bfloat16``, ``bool``)."""
return str(x.dtype).removeprefix("torch.") if isinstance(x, torch.Tensor) else str(np.asarray(x).dtype)
def close(a: Any, b: Any, atol: float, rtol: float) -> tuple[bool, str]:
"""Compare a reference ``a`` with a candidate ``b``, tensors or arrays: the same shape and dtype, then the values.
The dtypes are compared before any conversion (``to_np`` widens bfloat16), the values within ``atol`` / ``rtol``.
Complex values are compared componentwise, including their NaN patterns: a NaN in one component must not hide
a mismatch in the other.
"""
dtype_a, dtype_b = _dtype_name(a), _dtype_name(b)
a = to_np(a) if isinstance(a, torch.Tensor) else np.asarray(a)
b = to_np(b) if isinstance(b, torch.Tensor) else np.asarray(b)
if a.shape != b.shape:
return False, f"shape {a.shape} vs {b.shape}"
if dtype_a != dtype_b:
return False, f"dtype {dtype_a} vs {dtype_b}"
if bool in (a.dtype, b.dtype):
ok = bool(np.array_equal(a, b))
return ok, "" if ok else f"{int((a != b).sum())} bool mismatches"
if np.iscomplexobj(a):
for component in ("real", "imag"):
ok, why = close(getattr(a, component), getattr(b, component), atol, rtol)
if not ok:
return False, f"{component}: {why}"
return True, ""
a64, b64 = a.astype(np.float64), b.astype(np.float64)
nan_a, nan_b = np.isnan(a64), np.isnan(b64)
if nan_a.any() or nan_b.any():
if not np.array_equal(nan_a, nan_b):
return False, f"NaN pattern differs (reference {int(nan_a.sum())}, candidate {int(nan_b.sum())} NaNs)"
a64, b64 = a64[~nan_a], b64[~nan_b]
ok = bool(np.allclose(a64, b64, atol=atol, rtol=rtol))
if ok:
return True, ""
diff = np.abs(a64 - b64)
return (
False,
f"max abs diff {diff.max():.3e} at {int((diff > atol + rtol * np.abs(b64)).sum())}/{diff.size} elements",
)
def shape_and_finite(ref: list[torch.Tensor], out: list[Any]) -> str | None:
"""Why ``out`` cannot stand in for ``ref``: another output count or shape, or a non-finite value; else None.
The check of a random operator, whose values cannot be compared with an eager run.
"""
if len(out) != len(ref):
return f"{len(out)} outputs vs {len(ref)}"
for i, (a, b) in enumerate(zip(ref, out)):
b = to_np(b) if isinstance(b, torch.Tensor) else np.asarray(b)
if tuple(a.shape) != tuple(b.shape):
return f"out{i}: shape {tuple(b.shape)} vs {tuple(a.shape)}"
if np.issubdtype(b.dtype, np.floating) and not np.isfinite(b).all():
return f"out{i}: non-finite values"
return None
def op_types(model: onnx.ModelProto) -> list[str]:
import onnx
s: set[str] = set()
def walk(graph: onnx.GraphProto) -> None:
for n in graph.node:
s.add(n.op_type if n.domain in ("", "ai.onnx") else f"{n.domain}.{n.op_type}")
for a in n.attribute:
if a.type == onnx.AttributeProto.GRAPH:
walk(a.g)
for gg in a.graphs:
walk(gg)
walk(model.graph)
for f in model.functions:
for n in f.node:
s.add(n.op_type if n.domain in ("", "ai.onnx") else f"{n.domain}.{n.op_type}")
return sorted(x for x in s if not x.startswith("pkg."))
def err_str(e: BaseException) -> str:
root = e
while root.__cause__ is not None or root.__context__ is not None:
root = root.__cause__ or root.__context__ # type: ignore[assignment]
parts = [f"{type(e).__name__}"]
msg_src = root if root is not e else e
msg = [ln for ln in str(msg_src).splitlines() if ln.strip()]
first = (msg[0] if msg else "")[:240]
parts.append(f"<- {type(root).__name__}: {first}" if root is not e else f": {first}")
return " ".join(parts)
def err_full(e: BaseException) -> str:
out = []
x: BaseException | None = e
while x is not None:
out.append(f"{type(x).__name__}: {str(x)[:2000]}")
x = x.__cause__ or x.__context__
return "\n-- caused by --\n".join(out)[-4000:]
def _kornia_frames(e: BaseException) -> list[str]:
"""Locate the kornia source lines involved in a failure (root cause first)."""
frames: list[str] = []
x: BaseException | None = e
while x is not None:
for fr in traceback.extract_tb(x.__traceback__):
if "/kornia/kornia/" in fr.filename:
short = fr.filename.split("/kornia/kornia/")[-1]
frames.append(f"kornia/{short}:{fr.lineno} {fr.line.strip() if fr.line else ''}")
x = x.__cause__ or x.__context__
# dedupe, keep order, innermost last
seen: set[str] = set()
res = [f for f in frames if not (f in seen or seen.add(f))]
return res[-6:]
def legacy_opset() -> int:
"""The opset a legacy (TorchScript) export requests: the highest that exporter has on this torch, at most OPSET."""
from torch.onnx import _constants
return min(OPSET, getattr(_constants, "ONNX_TORCHSCRIPT_EXPORTER_MAX_OPSET", _constants.ONNX_MAX_OPSET))
def export(m: nn.Module, inputs: list[torch.Tensor], dynamo: bool = True) -> onnx.ModelProto:
"""Export ``m`` called on ``inputs`` with the dynamo exporter (opset 18) or the legacy TorchScript one."""
import onnx
buf = io.BytesIO()
if dynamo:
kwargs: dict[str, Any] = {
"dynamo": True,
"opset_version": OPSET,
"verbose": False,
"report": False,
"optimize": os.environ.get("KORNIA_SURVEY_OPTIMIZE", "1") == "1",
}
else:
kwargs = {
"dynamo": False,
"opset_version": legacy_opset(),
"verbose": False,
"input_names": [f"in{i}" for i in range(len(inputs))],
}
# an older torch.onnx.export lacks some of these keywords (``optimize`` arrived in torch 2.6)
accepted = inspect.signature(torch.onnx.export).parameters
kwargs = {k: v for k, v in kwargs.items() if k in accepted}
# dynamo prints the whole FX graph on data-dependent failures; keep the survey log readable
with torch.no_grad(), open(os.devnull, "w") as dn, contextlib.redirect_stdout(dn), contextlib.redirect_stderr(dn):
if dynamo:
torch.onnx.export(m, tuple(inputs), **kwargs).save(buf)
else:
torch.onnx.export(m, tuple(inputs), buf, **kwargs)
return onnx.load_from_string(buf.getvalue())
# ----------------------------------------------------------------------------- runner
def run_case(c: dict[str, Any], dynamo: bool = True) -> dict[str, Any]:
"""Probe one case (see the module docstring) with the dynamo or the legacy exporter; return its record."""
import onnx
import onnxruntime as ort
rec: dict[str, Any] = {k: c[k] for k in ("name", "group", "note", "tags", "check")}
rec["n_inputs"] = len(c["inputs"])
rec["input_shapes"] = [list(x.shape) + [str(x.dtype).replace("torch.", "")] for x in c["inputs"]]
rec["baked"] = {k: repr(v)[:60] for k, v in c["kwargs"].items()}
if c["skip"]:
rec.update(status="skipped", error=c["skip"])
return rec
t0 = time.time()
try:
target = c["target"]
if isinstance(target, nn.Module):
target.eval()
wrap = make_wrapper(target, c["kwargs"], len(c["inputs"]), c["method"])
inputs = [x.detach().clone() for x in c["inputs"]]
torch.manual_seed(SEED)
# Eager and the export each run on their own copy: the export reads none of the state the eager call leaves
# on a module, and neither leaves state (a dynamo export's fake tensors) on the case, which later tests reuse.
with torch.no_grad():
ref = copy.deepcopy(wrap)(*copy.deepcopy(inputs))
rec["n_outputs"] = len(ref)
rec["output_shapes"] = [list(x.shape) + [str(x.dtype).replace("torch.", "")] for x in ref]
except Exception as e:
status = "no-tensor-output" if "NO_TENSOR_OUTPUT" in str(e) else "eager-fail"
rec.update(status=status, error=err_str(e), error_full=err_full(e), where=_kornia_frames(e))
return rec
try:
torch.manual_seed(SEED)
# the legacy exporter traces with the real tensors, so an operator that updates its input in place would
# otherwise leave onnxruntime fed with the updated values
model = export(copy.deepcopy(wrap), copy.deepcopy(inputs), dynamo=dynamo)
except Exception as e:
rec.update(
status="export-fail",
error=err_str(e),
error_full=err_full(e),
where=_kornia_frames(e),
time_s=round(time.time() - t0, 1),
)
return rec
blob = model.SerializeToString()
rec["size_kb"] = len(blob) // 1024
rec["ops"] = op_types(model)
rec["time_s"] = round(time.time() - t0, 1)
try:
onnx.checker.check_model(model, full_check=True)
except Exception as e:
rec.update(status="checker-fail", error=err_str(e), error_full=err_full(e))
return rec
try:
sess = ort.InferenceSession(blob, providers=["CPUExecutionProvider"])
except Exception as e:
rec.update(status="ort-load-fail", error=err_str(e), error_full=err_full(e))
return rec
names = [i.name for i in sess.get_inputs()]
rec["graph_inputs"] = names
feed = {}
dropped = []
for i, x in enumerate(inputs):
n = f"in{i}"
if n in names:
feed[n] = to_np(x)
else:
dropped.append(n)
if len(feed) != len(names):
# exporter renamed inputs: fall back to positional feed when the count matches
if len(names) == len(inputs):
feed = {n: to_np(x) for n, x in zip(names, inputs)}
dropped = []
else:
rec.update(
status="ort-run-fail", error=f"cannot map inputs: graph has {names}, case has {len(inputs)} tensors"
)
return rec
rec["dropped_inputs"] = dropped
try:
outs = sess.run(None, feed)
except Exception as e:
rec.update(status="ort-run-fail", error=err_str(e), error_full=err_full(e))
return rec
if not c["check"]:
why = shape_and_finite(list(ref), outs)
if why is None:
rec["status"] = "ok-unverified"
else:
rec.update(status="mismatch", error=why)
return rec
if len(outs) != len(ref):
rec.update(status="mismatch", error=f"{len(outs)} ORT outputs vs {len(ref)} torch outputs")
return rec
bad = []
for i, (a, b) in enumerate(zip(ref, outs)):
ok, why = close(a, b, c["atol"], c["rtol"])
if not ok:
bad.append(f"out{i}: {why}")
if bad:
rec.update(status="mismatch", error="; ".join(bad)[:400])
return rec
rec["status"] = "ok"
return rec
HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(os.path.dirname(HERE))
def provenance() -> dict[str, str]:
"""The kornia revision and torch version a probe run measures, stamped into its inventory."""
def git(*args: str) -> str:
return subprocess.run(["git", *args], cwd=REPO, capture_output=True, text=True, check=False).stdout.strip()
dirty = "-dirty" if git("status", "--porcelain", "--", "kornia") else ""
return {"revision": git("rev-parse", "--short", "HEAD") + dirty, "torch": torch.__version__}
def inventory_path(out_json: str) -> str:
"""``results/inventory_<name>.json`` next to ``results/<name>.json``."""
return os.path.join(os.path.dirname(out_json), "inventory_" + os.path.basename(out_json))
def load_inventory(out_json: str) -> dict[str, Any] | None:
try:
return json.load(open(inventory_path(out_json)))
except (OSError, ValueError):
return None
def prepare_resume(cases: list[dict[str, Any]], out_json: str) -> dict[str, dict[str, Any]]:
"""Records of ``out_json`` worth reusing, then (re)write the inventory this run is measured against.
Records left by another kornia revision or torch version are discarded rather than resumed:
the inventory that accompanied them names what produced them, and a file without one predates
the stamp, so it is not trusted either.
"""
done: dict[str, dict[str, Any]] = {}
if os.path.exists(out_json):
try:
with open(out_json) as fh:
done = {r["name"]: r for r in json.load(fh)}
except Exception:
done = {}
inv = load_inventory(out_json)
stamp = provenance()
if done and (inv is None or {k: inv.get(k) for k in stamp} != stamp):
was = inv and (inv.get("revision"), inv.get("torch"))
print(f"{out_json}: records are from {was}, starting over", flush=True)
done = {}
os.makedirs(os.path.dirname(os.path.abspath(out_json)), exist_ok=True)
with open(inventory_path(out_json), "w") as fh:
json.dump({"names": [c["name"] for c in cases], **stamp}, fh, indent=1)
return done
def run_cases(cases: list[dict[str, Any]], out_json: str, only: list[str] | None = None, resume: bool = True) -> None:
configure_survey_process()
names = [c["name"] for c in cases]
dup = {n for n in names if names.count(n) > 1}
if dup:
raise SystemExit(f"duplicate case names: {sorted(dup)}")
done = prepare_resume(cases, out_json) if resume else {}
results: list[dict[str, Any]] = []
for c in cases:
if only and not any(o == c["name"] or o == c["group"] or c["name"].startswith(o) for o in only):
continue
if c["name"] in done and done[c["name"]].get("status") not in (None, "crashed") and not only:
results.append(done[c["name"]])
continue
# mark as crashed first so a hard crash (segfault) is visible after restart
with open(out_json, "w") as fh:
json.dump(
results + [{"name": c["name"], "group": c["group"], "status": "crashed", "note": c["note"]}],
fh,
indent=1,
default=str,
)
rec = run_case(c)
results.append(rec)
with open(out_json, "w") as fh:
json.dump(results, fh, indent=1, default=str)
extra = rec.get("error", "")
if rec["status"] == "ok" and rec.get("dropped_inputs"):
extra = f"inputs folded away: {rec['dropped_inputs']}"
print(
f"{rec['name']:48s} {rec['status']:14s} {rec.get('size_kb', ''):>6} {rec.get('time_s', ''):>6} {extra}",
flush=True,
)
if only:
# merge into existing file without disturbing other records
merged = {r["name"]: r for r in done.values()}
for r in results:
merged[r["name"]] = r
order = [c["name"] for c in cases]
with open(out_json, "w") as fh:
json.dump(
[merged[n] for n in order if n in merged] + [r for n, r in merged.items() if n not in order],
fh,
indent=1,
default=str,
)
from collections import Counter
print("SUMMARY", dict(Counter(r["status"] for r in results)), flush=True)