Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

22.5 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Tests for the closed-form small-matrix kernels in ``kornia.core._small_linalg``.

These are **raw kernels**: exact shape and a real floating dtype are caller preconditions,
not runtime checks, and behavior on anything else is unspecified. So nothing here asserts
that a kernel rejects an integer or complex tensor, and nothing pins what
``_adjugate_3x3`` does when handed a 4x4 (it silently uses the leading 3x3 block) or a 2x2
(it raises an incidental ``IndexError``). Those are recorded as known warts in the design
note, deliberately not as behavior. The size guard that *is* contractual belongs to the
dispatcher ``_adjugate_closed_form`` and is tested in ``test_helpers.py``.
"""

import io
import itertools
import re

import pytest
import torch

from kornia.core._compat import torch_version_ge, torch_version_lt
from kornia.core._small_linalg import (
    _adjugate_2x2,
    _adjugate_3x3,
    _adjugate_4x4,
    _det_perm_2x2,
    _det_perm_3x3,
    _det_perm_4x4,
    _inverse_3x3_cross,
    _inverse_3x3_scalar,
)

from testing.base import BaseTester

ADJUGATE = {2: _adjugate_2x2, 3: _adjugate_3x3, 4: _adjugate_4x4}
DET_PERM = {2: _det_perm_2x2, 3: _det_perm_3x3, 4: _det_perm_4x4}

# Op-type substrings that mean a linalg decomposition reached the graph. The point of these
# kernels is that an exported graph is basic arithmetic and nothing else, so this is what a
# regression would look like.
#
# Note what does the real work in the ONNX tests below: a MISSING lowering raises rather than
# emitting a node -- measured on torch 2.9.1, ``torch.linalg.inv`` gives
# ``UnsupportedOperatorError`` from the legacy exporter and ``ConversionError`` from the dynamo
# one -- so the export call succeeding is itself the gate. This check covers the other case: a
# future decomposition that exports *successfully* through something heavier than arithmetic.
# Matched as a substring rather than a prefix, because the dynamo exporter names a fallback node
# ``aten_linalg_inv``, which a ``startswith("linalg")`` test would miss.
# Long markers are unambiguous as bare substrings. The short ones are anchored to a name
# boundary, or they swallow basic arithmetic: "qr" matches ``Sqrt`` and "lu" matches the whole
# ``Relu``/``Selu``/``Elu``/``Celu``/``PRelu``/``LeakyRelu``/``ThresholdedRelu`` family -- all of
# them exactly the category this check exists to allow.
_LINALG_OP_RE = re.compile(
    r"(?:linalg|inverse|matmul|gemm|einsum)|(?:^|_)(?:det|lu|svd|qr|cholesky|eig)(?:$|_)", re.IGNORECASE
)


def _assert_basic_arithmetic_only(ops):
    offenders = sorted(op for op in ops if _LINALG_OP_RE.search(op))
    assert not offenders, f"linalg-shaped ops in the exported graph: {offenders} (full set: {sorted(ops)})"


def _export_kwargs(use_dynamo_exporter: bool) -> dict:
    """Export kwargs that are valid on every torch kornia declares support for.

    ``dynamo=`` does not exist before torch 2.4 -- measured: ``TypeError: export() got an
    unexpected keyword argument 'dynamo'`` on 2.0.1 and 2.2.2, present in 2.4.1. Passing it
    unconditionally would be a ``TypeError`` on an older pin rather than a skip. Gated at 2.5
    to match ``tests/filters/test_gaussian.py::test_onnx_export_legacy``, which is conservative
    by one release and costs nothing.
    """
    kwargs = {"opset_version": 18}
    if torch_version_ge(2, 5, 0):
        kwargs["dynamo"] = use_dynamo_exporter
    if use_dynamo_exporter and torch_version_ge(2, 6, 0):
        # These cases export into a ``BytesIO``, and the dynamo exporter's default
        # ``external_data=True`` hands that destination straight to onnxscript's
        # ``save_model_with_external_data``, which takes a filesystem path. On torch 2.6 that is
        # ``TypeError: expected str, bytes or os.PathLike object, not BytesIO`` after a successful
        # translation -- measured; 2.9.1 and 2.14.0 accept the buffer through a newer onnxscript
        # framework API. These graphs are a handful of nodes with no initializers, so there is no
        # external data to write out on any version.
        #
        # Gated because ``external_data=`` does not exist before 2.6, and this function promises
        # kwargs valid on every supported torch. Every call site happens to run
        # ``_require_exporter`` first, which skips below 2.6, but that is an ordering at three
        # call sites rather than a property of the function that claims the invariant -- and
        # ``test_cross_kernel_onnx_export_is_informational`` re-raises ``TypeError``, so getting
        # it wrong is a hard failure rather than a skip.
        kwargs["external_data"] = False
    return kwargs


def _require_exporter(use_dynamo_exporter: bool) -> None:
    """Skip unless this build can actually run the requested ONNX exporter.

    ``dynamo=`` first exists in torch 2.5, but there it still routes through the experimental
    ``_compat.export_compat`` shim and needs ``onnxscript`` -- which the 2.5.1 CI legs do not
    install, so an unguarded call is a hard failure rather than a skip there. Same reasoning and
    same guard as ``tests/filters/test_gaussian.py::test_onnx_export_modern``.
    """
    pytest.importorskip("onnx")
    if use_dynamo_exporter:
        if torch_version_lt(2, 6, 0):
            pytest.skip("the dynamo ONNX exporter is only non-experimental from PyTorch 2.6")
        pytest.importorskip("onnxscript")


def _well_conditioned(n, device, dtype, batch=()):
    """A deterministic, well-conditioned matrix family.

    Built without touching the global RNG: a seeded ``torch.rand`` would still mutate it, and
    a fresh draw per run turns a tolerance assertion into a coin flip.

    Each batch entry gets a distinct scale. A bare ``expand`` would make every slice identical,
    and then a kernel that broadcast element 0 across the batch -- an indexing slip in the
    trailing-dim arithmetic -- would pass the batched cases. Scaling leaves the condition number
    untouched, so the tolerance argument above still holds.
    """
    ramp = torch.arange(1, n * n + 1, device=device, dtype=dtype).reshape(n, n) * 0.1
    x = (torch.eye(n, device=device, dtype=dtype) + ramp).expand(*batch, n, n).clone()
    if batch:
        idx = torch.arange(x.numel() // (n * n), device=device, dtype=dtype).reshape(*batch, 1, 1)
        x = x * (1 + idx * 0.05)
    return x


class TestAdjugateKernels(BaseTester):
    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_smoke(self, device, dtype, n):
        adj, det = ADJUGATE[n](_well_conditioned(n, device, dtype))
        assert adj.shape == (n, n)
        assert det.shape == ()

    @pytest.mark.parametrize("n", [2, 3, 4])
    @pytest.mark.parametrize("batch", [(), (2,), (2, 3)])
    def test_cardinality(self, device, dtype, n, batch):
        adj, det = ADJUGATE[n](_well_conditioned(n, device, dtype, batch))
        assert adj.shape == (*batch, n, n)
        assert det.shape == batch

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_matches_linalg_inv_on_well_conditioned_input(self, device, dtype, n):
        # A smoke-test invariant on a controlled family, evaluated in a stated dtype -- not an
        # accuracy oracle. linalg.inv has no half kernel, so the reference is computed in
        # float32 and compared back.
        x = _well_conditioned(n, device, dtype, (2,))
        ref_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
        adj, det = ADJUGATE[n](x)
        tol = 1e-2 if dtype in (torch.float16, torch.bfloat16) else 1e-4
        self.assert_close(
            adj / det[..., None, None],
            torch.linalg.inv(x.to(ref_dtype)).to(dtype),
            atol=tol,
            rtol=tol,
        )
        self.assert_close(det, torch.linalg.det(x.to(ref_dtype)).to(dtype), atol=tol, rtol=tol)

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_adjugate_identity_is_exact_on_integral_float64(self, device, n):
        # A @ adj(A) == det(A) * I holds identically. Small integers held in float64 make every
        # intermediate exactly representable, so this is an equality, not a tolerance -- and it
        # is the only check here that exercises the float64 path against something other than a
        # same-precision torch.linalg call, which would be no reference at all.
        if device.type == "mps":
            pytest.skip("MPS has no float64")
        values = torch.arange(1, n * n + 1, device=device, dtype=torch.float64).reshape(n, n)
        x = values + torch.eye(n, device=device, dtype=torch.float64) * (n * n)
        adj, det = ADJUGATE[n](x)
        expected = det * torch.eye(n, device=device, dtype=torch.float64)
        assert torch.equal(x @ adj, expected)

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_gradcheck(self, device, n):
        x = _well_conditioned(n, device, torch.float64)
        self.gradcheck(lambda t: ADJUGATE[n](t)[0], (x,))
        self.gradcheck(lambda t: ADJUGATE[n](t)[1], (x,))

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_dynamo(self, device, dtype, n, torch_optimizer):
        x = _well_conditioned(n, device, dtype, (2,))
        op = ADJUGATE[n]
        op_optimized = torch_optimizer(op)
        expected_adj, expected_det = op(x)
        actual_adj, actual_det = op_optimized(x)
        self.assert_close(actual_adj, expected_adj)
        self.assert_close(actual_det, expected_det)

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_scripts(self, device, dtype, n):
        x = _well_conditioned(n, device, dtype, (2,))
        scripted = torch.jit.script(ADJUGATE[n])
        expected_adj, expected_det = ADJUGATE[n](x)
        actual_adj, actual_det = scripted(x)
        self.assert_close(actual_adj, expected_adj)
        self.assert_close(actual_det, expected_det)

    @pytest.mark.device_agnostic
    @pytest.mark.filterwarnings("ignore::DeprecationWarning")
    @pytest.mark.parametrize("use_dynamo_exporter", [False, True], ids=["torchscript", "torchexport"])
    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_onnx_export_uses_basic_arithmetic(self, device, n, use_dynamo_exporter):
        # Capability matrix: the scalar kernels are the ONNX-safe path, and that is the whole
        # reason they exist. Assert the graph carries no linalg decomposition, on both exporters.
        _require_exporter(use_dynamo_exporter)
        onnx = pytest.importorskip("onnx")
        x = _well_conditioned(n, device, torch.float32, (2,))
        buffer = io.BytesIO()
        torch.onnx.export(_AdjugateModule(n), (x,), buffer, **_export_kwargs(use_dynamo_exporter))
        ops = {node.op_type for node in onnx.load_from_string(buffer.getvalue()).graph.node}
        _assert_basic_arithmetic_only(ops)


class _AdjugateModule(torch.nn.Module):
    """Wrapper so the kernels can be handed to ``torch.onnx.export``."""

    def __init__(self, n: int) -> None:
        super().__init__()
        self.n = n

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        adj, det = ADJUGATE[self.n](x)
        return adj / det[..., None, None]


def _signed(n, device, dtype):
    """A batch of two: ``_well_conditioned``, all positive, so terms of the determinant cancel and the permanent
    of the absolute values is strictly larger than ``|det|``, and the same with mixed signs."""
    x = _well_conditioned(n, device, dtype)
    signs = torch.tensor([[(-1.0) ** (r * c) for c in range(n)] for r in range(n)], device=device, dtype=dtype)
    return torch.stack([x, x * signs])


def _permanent_of_abs(x):
    """The permanent of ``|x|`` summed over the permutations, in float64 on the CPU (MPS has no float64)."""
    n = x.shape[-1]
    x = x.abs().cpu().to(torch.float64)
    terms = [x[..., range(n), perm].prod(-1) for perm in itertools.permutations(range(n))]
    return torch.stack(terms, -1).sum(-1)


class TestDetPermKernels(BaseTester):
    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_smoke(self, device, dtype, n):
        det, perm = DET_PERM[n](_well_conditioned(n, device, dtype))
        assert det.shape == ()
        assert perm.shape == ()
        assert det.dtype == dtype
        assert perm.dtype == dtype

    @pytest.mark.parametrize("n", [2, 3, 4])
    @pytest.mark.parametrize("batch", [(), (2,), (2, 3)])
    def test_cardinality(self, device, dtype, n, batch):
        det, perm = DET_PERM[n](_well_conditioned(n, device, dtype, batch))
        assert det.shape == batch
        assert perm.shape == batch

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_matches_linalg_det_and_the_permanent_of_the_absolute_values(self, device, dtype, n):
        # The determinant is checked against torch within the rounding scale the permanent names, and the
        # permanent against its definition, the sum over the permutations of the products of ``|x|``.
        x = _signed(n, device, dtype)
        det, perm = DET_PERM[n](x)
        ref_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
        eps = torch.finfo(dtype).eps
        det_ref = torch.linalg.det(x.to(ref_dtype)).to(dtype)
        assert ((det - det_ref).abs() <= 8 * n * eps * perm).all()
        self.assert_close(perm.cpu().to(torch.float64), _permanent_of_abs(x), rtol=8 * n * eps, atol=0.0)
        assert (perm >= det.abs()).all()
        assert (perm > det.abs()).any()
        if n == 4:
            # the same Laplace pairing of the minors, in the same order, as the adjugate kernel's determinant
            assert torch.equal(det, ADJUGATE[n](x)[1])

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_scaling_a_row_scales_both_and_keeps_their_ratio(self, device, dtype, n):
        # Both are homogeneous of degree one in every row and column, so their ratio, which is what
        # ``safe_inverse_with_mask`` reads, does not depend on the units of the matrix. Powers of two keep
        # the scaling exact in every dtype.
        x = _signed(n, device, dtype)
        det, perm = DET_PERM[n](x)
        scale = torch.ones(n, 1, device=device, dtype=dtype)
        scale[0] = 2.0**6
        scale[-1] = 2.0**-9
        det_s, perm_s = DET_PERM[n](x * scale)
        factor = 2.0 ** (6 - 9)
        assert torch.equal(det_s, det * factor)
        assert torch.equal(perm_s, perm * factor)

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_dependent_rows_give_a_zero_determinant_and_a_positive_permanent(self, device, dtype, n):
        # Small integers keep every product and sum exact, so the determinant of a matrix with a row equal
        # to the sum of two others is exactly zero while the permanent of the absolute values is not.
        x = torch.arange(1, n * n + 1, device=device, dtype=dtype).reshape(n, n)
        x = x * torch.tensor([[(-1.0) ** (r * n + c) for c in range(n)] for r in range(n)], device=device, dtype=dtype)
        x[-1] = x[0] + x[1] if n > 2 else -2 * x[0]
        det, perm = DET_PERM[n](x)
        assert torch.equal(det, torch.zeros_like(det))
        assert perm.item() > 0

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_dynamo(self, device, dtype, n, torch_optimizer):
        x = _signed(n, device, dtype)
        op = DET_PERM[n]
        op_optimized = torch_optimizer(op)
        expected_det, expected_perm = op(x)
        actual_det, actual_perm = op_optimized(x)
        self.assert_close(actual_det, expected_det)
        self.assert_close(actual_perm, expected_perm)

    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_scripts(self, device, dtype, n):
        x = _signed(n, device, dtype)
        scripted = torch.jit.script(DET_PERM[n])
        expected_det, expected_perm = DET_PERM[n](x)
        actual_det, actual_perm = scripted(x)
        self.assert_close(actual_det, expected_det)
        self.assert_close(actual_perm, expected_perm)

    @pytest.mark.device_agnostic
    @pytest.mark.filterwarnings("ignore::DeprecationWarning")
    @pytest.mark.parametrize("use_dynamo_exporter", [False, True], ids=["torchscript", "torchexport"])
    @pytest.mark.parametrize("n", [2, 3, 4])
    def test_onnx_export_uses_basic_arithmetic(self, device, n, use_dynamo_exporter):
        # The singular verdict of ``safe_inverse_with_mask`` under capture is built from these kernels, so
        # the exported graph has to stay basic arithmetic as the adjugate's does.
        _require_exporter(use_dynamo_exporter)
        onnx = pytest.importorskip("onnx")
        x = _signed(n, device, torch.float32)
        buffer = io.BytesIO()
        torch.onnx.export(_DetPermModule(n), (x,), buffer, **_export_kwargs(use_dynamo_exporter))
        ops = {node.op_type for node in onnx.load_from_string(buffer.getvalue()).graph.node}
        _assert_basic_arithmetic_only(ops)


class _DetPermModule(torch.nn.Module):
    """Wrapper so the kernels can be handed to ``torch.onnx.export``."""

    def __init__(self, n: int) -> None:
        super().__init__()
        self.n = n

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        det, perm = DET_PERM[self.n](x)
        return det.abs() <= 1e-6 * perm


class TestInverse3x3Kernels(BaseTester):
    def test_smoke(self, device, dtype):
        x = _well_conditioned(3, device, dtype)
        assert _inverse_3x3_cross(x).shape == (3, 3)
        assert _inverse_3x3_scalar(x).shape == (3, 3)

    @pytest.mark.parametrize("batch", [(), (2,), (2, 3)])
    def test_cardinality(self, device, dtype, batch):
        x = _well_conditioned(3, device, dtype, batch)
        assert _inverse_3x3_cross(x).shape == (*batch, 3, 3)
        assert _inverse_3x3_scalar(x).shape == (*batch, 3, 3)

    def test_cross_and_scalar_kernels_agree(self, device, dtype):
        # The mode split exists so that export avoids ``aten::linalg_inv``, not so that the two
        # paths compute different answers. They are different orderings of the same algebra, so
        # they agree to rounding rather than bit-for-bit.
        x = _well_conditioned(3, device, dtype, (2,))
        tol = 1e-2 if dtype in (torch.float16, torch.bfloat16) else 1e-4
        self.assert_close(_inverse_3x3_cross(x), _inverse_3x3_scalar(x), atol=tol, rtol=tol)

    @pytest.mark.parametrize("kernel", [_inverse_3x3_cross, _inverse_3x3_scalar])
    def test_matches_linalg_inv_on_well_conditioned_input(self, device, dtype, kernel):
        x = _well_conditioned(3, device, dtype, (2,))
        ref_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
        tol = 1e-2 if dtype in (torch.float16, torch.bfloat16) else 1e-4
        self.assert_close(kernel(x), torch.linalg.inv(x.to(ref_dtype)).to(dtype), atol=tol, rtol=tol)

    @pytest.mark.parametrize("kernel", [_inverse_3x3_cross, _inverse_3x3_scalar])
    def test_gradcheck(self, device, kernel):
        self.gradcheck(kernel, (_well_conditioned(3, device, torch.float64),))

    @pytest.mark.parametrize("kernel", [_inverse_3x3_cross, _inverse_3x3_scalar])
    def test_dynamo(self, device, dtype, kernel, torch_optimizer):
        x = _well_conditioned(3, device, dtype, (2,))
        self.assert_close(torch_optimizer(kernel)(x), kernel(x))

    @pytest.mark.parametrize("kernel", [_inverse_3x3_cross, _inverse_3x3_scalar])
    def test_scripts(self, device, dtype, kernel):
        x = _well_conditioned(3, device, dtype, (2,))
        self.assert_close(torch.jit.script(kernel)(x), kernel(x))

    @pytest.mark.device_agnostic
    @pytest.mark.filterwarnings("ignore::DeprecationWarning")
    @pytest.mark.parametrize("use_dynamo_exporter", [False, True], ids=["torchscript", "torchexport"])
    def test_scalar_kernel_exports_to_onnx(self, device, use_dynamo_exporter):
        # Contractual: the scalar kernel is the path the dispatcher takes under export, so its
        # graph must contain no linalg decomposition. Both exporters.
        _require_exporter(use_dynamo_exporter)
        onnx = pytest.importorskip("onnx")
        x = _well_conditioned(3, device, torch.float32, (2,))
        buffer = io.BytesIO()
        torch.onnx.export(_ScalarInverseModule(), (x,), buffer, **_export_kwargs(use_dynamo_exporter))
        ops = {node.op_type for node in onnx.load_from_string(buffer.getvalue()).graph.node}
        _assert_basic_arithmetic_only(ops)

    @pytest.mark.device_agnostic
    @pytest.mark.filterwarnings("ignore::DeprecationWarning")
    @pytest.mark.parametrize("use_dynamo_exporter", [False, True], ids=["torchscript", "torchexport"])
    def test_cross_kernel_onnx_export_is_informational(self, device, use_dynamo_exporter):
        # NOT a gate, deliberately -- but the measurement behind it is now stronger than it was.
        # ``torch.linalg.cross`` lowers on every torch version CI runs: 2.14.0 and 2.9.1 on both
        # exporters, and 2.5.1 on the legacy one (2.5.1's dynamo path needs onnxscript, which
        # those CI legs do not install, so it is out of reach rather than broken). Nothing below
        # 2.5.1 is measured, and nothing below it is supported either.
        # It stays a skip rather than a hard assertion for two reasons: the dispatcher never takes
        # this kernel under export, so nothing in kornia depends on it lowering; and a future torch
        # that stops lowering ``cross`` should surface as a visible skip here rather than as a red
        # test for something kornia does not rely on.
        _require_exporter(use_dynamo_exporter)
        onnx = pytest.importorskip("onnx")
        x = _well_conditioned(3, device, torch.float32, (2,))
        buffer = io.BytesIO()
        try:
            torch.onnx.export(_CrossInverseModule(), (x,), buffer, **_export_kwargs(use_dynamo_exporter))
        except TypeError:
            # An API misuse is not a lowering failure. Re-raised rather than skipped, or a wrong
            # kwarg would masquerade forever as "cross does not lower here".
            raise
        except Exception as err:
            pytest.skip(f"torch.linalg.cross does not lower on torch {torch.__version__}: {err}")
        ops = {node.op_type for node in onnx.load_from_string(buffer.getvalue()).graph.node}
        _assert_basic_arithmetic_only(ops)


class _ScalarInverseModule(torch.nn.Module):
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return _inverse_3x3_scalar(x)


class _CrossInverseModule(torch.nn.Module):
    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return _inverse_3x3_cross(x)