Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

14.1 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""ONNX export regression tests for kornia.augmentation modules.

These tests pin the set of augmentations that successfully export through
``torch.onnx.export`` (legacy TorchScript-tracer path) at opset 20, so future
changes to the augmentation base classes do not silently regress export
support.

Opset 20 was chosen as the floor because:
- ``aten::affine_grid_generator`` (used by every geometric augmentation that
  goes through ``warp_affine``) only lowers from opset 20 upward.
- ``aten::grid_sampler`` is supported from opset 16, well below 20.
- Earlier opsets work for non-geometric augmentations, but standardising on a
  single opset keeps the test matrix small.

What this file does NOT test:
- Numerical equivalence between the exported ONNX graph and the eager forward.
  That requires onnxruntime, which is not a hard dependency of kornia. Add a
  separate test file (e.g. ``test_onnx_export_numerical.py``) gated on the
  optional ``onnxruntime`` import when that work lands.
- The ``dynamo=True`` export path. That is tracked separately because it has
  different op coverage (notably, ``aten::linalg_solve`` lowers cleanly under
  dynamo but not under the legacy tracer).
- Augmentations that currently cannot export under the legacy tracer due to
  missing ONNX op support, see ``XFAIL_OPS`` below.

Adding an augmentation to ``EXPORTABLE_OPS`` is the way to advertise that it
cleanly exports today; adding to ``XFAIL_OPS`` marks a known gap with the
underlying op so we don't lose the signal that it's still pending.
"""

from __future__ import annotations

import io
import warnings
from typing import Any, Callable, Tuple

import pytest
import torch

import kornia.augmentation as K
from kornia.core._compat import torch_version_ge, torch_version_lt

pytestmark = pytest.mark.skipif(
    torch_version_lt(2, 4, 0), reason="augmentation ONNX regression tests require opset 20 support from PyTorch 2.4"
)


def _hflip() -> torch.nn.Module:
    return K.RandomHorizontalFlip(p=1.0)


def _vflip() -> torch.nn.Module:
    return K.RandomVerticalFlip(p=1.0)


def _color_jiggle() -> torch.nn.Module:
    return K.ColorJiggle(0.1, 0.1, 0.1, 0.1, p=1.0)


def _brightness() -> torch.nn.Module:
    return K.RandomBrightness(brightness=(0.5, 1.5), p=1.0)


def _grayscale() -> torch.nn.Module:
    return K.RandomGrayscale(p=1.0)


def _invert() -> torch.nn.Module:
    return K.RandomInvert(p=1.0)


def _gaussian_blur() -> torch.nn.Module:
    return K.RandomGaussianBlur((3, 3), (0.1, 2.0), p=1.0)


def _erasing() -> torch.nn.Module:
    return K.RandomErasing(p=1.0)


@pytest.mark.parametrize(
    "parameter_dtype,image_dtype",
    [
        (torch.float16, torch.float32),
        (torch.float16, torch.float16),
        (torch.bfloat16, torch.float32),
    ],
    ids=["float16-params-float32-image", "float16-params-float16-image", "bfloat16-params-float32-image"],
)
@pytest.mark.device_agnostic
def test_onnx_export_random_erasing_tensor_parameters(parameter_dtype, image_dtype) -> None:
    module = K.RandomErasing(
        scale=torch.tensor((0.25, 0.25), dtype=parameter_dtype),
        ratio=torch.tensor((1.0, 1.0), dtype=parameter_dtype),
        p=1.0,
    )
    size = _try_export(module, torch.randn(2, 3, 16, 16, dtype=image_dtype))
    assert size > 0


def _posterize() -> torch.nn.Module:
    return K.RandomPosterize(3, p=1.0)


def _solarize() -> torch.nn.Module:
    return K.RandomSolarize(0.1, p=1.0)


def _normalize() -> torch.nn.Module:
    return K.Normalize(mean=torch.zeros(3), std=torch.ones(3))


def _affine() -> torch.nn.Module:
    return K.RandomAffine(degrees=10.0, p=1.0)


def _rotation() -> torch.nn.Module:
    return K.RandomRotation(degrees=10.0, p=1.0)


def _perspective() -> torch.nn.Module:
    return K.RandomPerspective(p=1.0)


def _augmentation_sequential() -> torch.nn.Module:
    """A torchgeo-typical pipeline: per-element random flips followed by normalize."""
    return K.AugmentationSequential(
        K.RandomHorizontalFlip(p=0.5),
        K.RandomVerticalFlip(p=0.5),
        K.Normalize(mean=torch.zeros(3), std=torch.ones(3)),
        data_keys=["input"],
    )


# Augmentations that export today through the legacy TorchScript tracer at opset 20.
EXPORTABLE_OPS: list[Tuple[str, Callable[[], torch.nn.Module]]] = [
    ("RandomHorizontalFlip", _hflip),
    ("RandomVerticalFlip", _vflip),
    ("ColorJiggle", _color_jiggle),
    ("RandomBrightness", _brightness),
    ("RandomGrayscale", _grayscale),
    ("RandomInvert", _invert),
    ("RandomGaussianBlur", _gaussian_blur),
    ("RandomErasing", _erasing),
    ("RandomPosterize", _posterize),
    ("RandomSolarize", _solarize),
    ("Normalize", _normalize),
    # Geometric augmentations enabled by:
    #   - bumping opset to 20 (for ``affine_grid_generator``)
    #   - the closed-form 3x3 inverse in ``kornia.core.utils._inverse_3x3_closed_form``
    #     that replaces ``aten::linalg_inv`` during ONNX export.
    ("RandomAffine", _affine),
    ("RandomRotation", _rotation),
    # ``RandomPerspective`` exports via the closed-form Heckbert decomposition in
    # ``kornia.geometry.transform.imgwarp._get_perspective_transform_closed_form``,
    # which replaces the 8x8 ``torch.linalg.solve`` with two unit-square-to-quad
    # constructions and one closed-form 3x3 inverse.
    ("RandomPerspective", _perspective),
    # Container: pinned because torchgeo and similar callers wrap their pipelines
    # in ``AugmentationSequential``. The container's ``forward`` strips trailing
    # ``None`` positional args injected by the legacy ONNX tracer.
    ("AugmentationSequential", _augmentation_sequential),
]

# Currently no augmentations are blocked at export time. Augmentations that export
# but produce numerically different results from eager are tracked separately in
# ``ONNX_NUMERICAL_KNOWN_DRIFT`` below.
# Future entries must record the actual exception type and message pattern as well as the reason.
XFAIL_OPS: list[Tuple[str, Callable[[], torch.nn.Module], str, type[Exception], str]] = []


def _legacy_onnx_export(module: torch.nn.Module, x: torch.Tensor, buf: io.BytesIO) -> None:
    """Export through the legacy tracer across supported PyTorch versions."""
    kwargs: dict[str, Any] = {
        "opset_version": 20,
        "input_names": ["input"],
        "output_names": ["output"],
    }
    if torch_version_ge(2, 5, 0):
        kwargs["dynamo"] = False
    torch.onnx.export(module, (x,), buf, **kwargs)


def _try_export(module: torch.nn.Module, x: torch.Tensor) -> int:
    """Run ``torch.onnx.export`` against an in-memory buffer and return the byte count."""
    module.eval()
    buf = io.BytesIO()
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        _legacy_onnx_export(module, x, buf)
    return len(buf.getvalue())


@pytest.mark.parametrize("name,factory", EXPORTABLE_OPS, ids=[n for n, _ in EXPORTABLE_OPS])
@pytest.mark.device_agnostic
def test_onnx_export_exportable(name: str, factory: Callable[[], torch.nn.Module]) -> None:
    """Augmentation exports cleanly via ``torch.onnx.export`` (legacy tracer, opset 20)."""
    torch.manual_seed(0)
    x = torch.randn(2, 3, 32, 32)
    size = _try_export(factory(), x)
    assert size > 0, f"{name}: ONNX graph was empty"


@pytest.mark.parametrize("name,factory,reason,error_type,match", XFAIL_OPS, ids=[n for n, *_ in XFAIL_OPS])
@pytest.mark.device_agnostic
def test_onnx_export_known_blocked(
    name: str, factory: Callable[[], torch.nn.Module], reason: str, error_type: type[Exception], match: str
) -> None:
    """Augmentation cannot export today; pinned so we notice if it starts working."""
    torch.manual_seed(0)
    x = torch.randn(2, 3, 32, 32)
    with pytest.raises(error_type, match=match):
        _try_export(factory(), x)


# ----------------------------------------------------------------------------
# Numerical-equivalence checks (require onnxruntime).
#
# Each entry pins a concrete deterministic configuration of the augmentation is
# fixed angles, fixed brightness, etc. So comparing eager and ONNX is a
# clean apples-to-apples test of "did the export faithfully capture the
# computation". Augmentations with random parameter ranges are deliberately
# excluded from the numerical check because ``torch.onnx.export`` consumes RNG
# during tracing in ways that don't line up with a separate eager call, even
# under the same ``torch.manual_seed``, that mismatch is RNG bookkeeping and
# would mask real bugs. The export-success tests above cover the random case.
# ----------------------------------------------------------------------------

ort = pytest.importorskip("onnxruntime", reason="onnxruntime is required for numerical-equivalence checks")
np = pytest.importorskip("numpy")


# Deterministic factories for numerical tests. Tuple ranges of the form ``(v, v)``
# evaluate to a single fixed value under uniform sampling. Geometric augmentations
# additionally rely on the in-place-mutation fix in
# ``kornia.geometry.conversions.normal_transform_pixel`` and the ``deg2rad``
# inlining in the affine/shear ``compute_transformation`` paths.
def _hflip_det() -> torch.nn.Module:
    return K.RandomHorizontalFlip(p=1.0)


def _vflip_det() -> torch.nn.Module:
    return K.RandomVerticalFlip(p=1.0)


def _grayscale_det() -> torch.nn.Module:
    return K.RandomGrayscale(p=1.0)


def _invert_det() -> torch.nn.Module:
    return K.RandomInvert(p=1.0)


def _normalize_det() -> torch.nn.Module:
    return K.Normalize(mean=torch.zeros(3), std=torch.ones(3))


def _brightness_det() -> torch.nn.Module:
    return K.RandomBrightness(brightness=(1.2, 1.2), p=1.0)


def _color_jiggle_det() -> torch.nn.Module:
    return K.ColorJiggle(brightness=(1.2, 1.2), p=1.0)


def _gaussian_blur_det() -> torch.nn.Module:
    return K.RandomGaussianBlur((3, 3), (1.5, 1.5), p=1.0)


def _solarize_det() -> torch.nn.Module:
    return K.RandomSolarize(thresholds=(0.5, 0.5), additions=(0.0, 0.0), p=1.0)


def _rotation_det() -> torch.nn.Module:
    return K.RandomRotation(degrees=(15.0, 15.0), p=1.0)


def _affine_det() -> torch.nn.Module:
    return K.RandomAffine(degrees=(15.0, 15.0), translate=(0.0, 0.0), scale=(1.0, 1.0), shear=(0.0, 0.0), p=1.0)


def _affine_with_shear_det() -> torch.nn.Module:
    return K.RandomAffine(degrees=(0.0, 0.0), shear=(5.0, 5.0), p=1.0)


# Augmentations whose exported graph produces numerically equivalent results to
# eager when configured with deterministic parameters. ``max diff < 1e-3``
# threshold is conservative, most are float32 precision (< 1e-5).
ONNX_NUMERICAL_EQUIVALENT: list[Tuple[str, Callable[[], torch.nn.Module]]] = [
    # Parameter-free / no random sampling
    ("RandomHorizontalFlip", _hflip_det),
    ("RandomVerticalFlip", _vflip_det),
    ("RandomGrayscale", _grayscale_det),
    ("RandomInvert", _invert_det),
    ("Normalize", _normalize_det),
    # Random params pinned to deterministic ranges
    ("RandomBrightness", _brightness_det),
    ("ColorJiggle", _color_jiggle_det),
    ("RandomGaussianBlur", _gaussian_blur_det),
    ("RandomSolarize", _solarize_det),
    # Geometric augmentations, exercise the warp_affine / grid_sample path.
    # Previously diverged due to in-place mutation in normal_transform_pixel
    # losing the scale factors under tracing.
    ("RandomRotation", _rotation_det),
    ("RandomAffine", _affine_det),
    ("RandomAffine_with_shear", _affine_with_shear_det),
]


def _run_onnx(module: torch.nn.Module, x: torch.Tensor) -> Any:
    module.eval()
    buf = io.BytesIO()
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        _legacy_onnx_export(module, x, buf)
    sess = ort.InferenceSession(buf.getvalue())
    return sess.run(["output"], {"input": x.numpy()})[0]


@pytest.mark.parametrize("shape", [(1, 1, 1, 8), (1, 1, 8, 1), (1, 1, 1, 1)])
@pytest.mark.device_agnostic
def test_onnx_random_perspective_singleton_axis_is_identity_5000(shape: Tuple[int, ...]) -> None:
    """A size-1 axis exported through the legacy tracer is the identity at zero distortion, as in eager."""
    # Start at 1: an all-zero 1x1 image cannot tell the identity from the zeros the broken export returned.
    image = torch.arange(1, shape[-2] * shape[-1] + 1, dtype=torch.float32).reshape(shape)
    aug = K.RandomPerspective(0.0, p=1.0).eval()
    eager = aug(image).numpy()
    onnx_out = _run_onnx(aug, image)
    assert eager.shape == onnx_out.shape
    assert float(np.abs(eager - image.numpy()).max()) < 1e-5
    assert float(np.abs(onnx_out - eager).max()) < 1e-5


@pytest.mark.parametrize("name,factory", ONNX_NUMERICAL_EQUIVALENT, ids=[n for n, _ in ONNX_NUMERICAL_EQUIVALENT])
@pytest.mark.device_agnostic
def test_onnx_export_numerically_matches_eager(name: str, factory: Callable[[], torch.nn.Module]) -> None:
    """Exported graph produces the same numbers as eager for the deterministic configuration."""
    torch.manual_seed(0)
    x = torch.randn(2, 3, 32, 32)
    aug = factory()
    aug.eval()
    eager = aug(x).numpy()
    onnx_out = _run_onnx(aug, x)
    assert eager.shape == onnx_out.shape, f"{name}: shape mismatch {eager.shape} vs {onnx_out.shape}"
    max_diff = float(np.abs(eager - onnx_out).max())
    assert max_diff < 1e-3, f"{name}: max abs diff {max_diff:.4f} exceeds 1e-3"


@pytest.mark.parametrize("size", [(9, 21), (5, 7)])
@pytest.mark.device_agnostic
def test_onnx_nearest_resize_exports_through_the_legacy_tracer(size: Tuple[int, int]) -> None:
    """Nearest ``Resize`` samples ``nearest-exact``, which the legacy tracer cannot lower; export gathers instead."""
    image = torch.arange(3 * 7 * 12, dtype=torch.float32).reshape(1, 3, 7, 12)
    aug = K.Resize(size, resample="nearest").eval()
    eager = aug(image).numpy()
    onnx_out = _run_onnx(aug, image)
    assert np.array_equal(onnx_out, eager)