Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

21.5 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import math

import pytest
import torch

import kornia

from testing.base import BaseTester


class TestZCA(BaseTester):
    @pytest.mark.parametrize("dim", [0, -1])
    @pytest.mark.parametrize("unbiased", [True, False])
    @pytest.mark.parametrize("scale,repeats", [(127.5, 2), (0.5, 65536)])
    def test_covariance_accumulation(self, device, dtype, dim, unbiased, scale, repeats):
        data = torch.tensor([[-1.0, -0.5], [-1.0, 0.5], [1.0, -0.5], [1.0, 0.5]], device=device, dtype=dtype)
        mean = torch.tensor([[scale, scale / 2]], device=device, dtype=dtype)
        data = data.repeat(repeats, 1) * scale + mean
        sample_count = data.shape[0]
        divisor = sample_count - 1 if unbiased else sample_count
        # The two features have zero cross covariance and variances N * scale**2 / divisor and a quarter of it.
        std = [math.sqrt(sample_count * s**2 / divisor + 1e-6) for s in (scale, scale / 2)]
        compute_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
        expected_transform = torch.diag(torch.tensor([1 / s for s in std], device=device, dtype=compute_dtype))
        expected_inverse = torch.diag(torch.tensor(std, device=device, dtype=compute_dtype))
        expected = (data - mean) * torch.tensor([1 / s for s in std], device=device, dtype=dtype)
        if dim != 0:
            data, expected = data.t(), expected.t()

        transform, actual_mean, inverse = kornia.enhance.zca_mean(data, dim=dim, unbiased=unbiased, return_inverse=True)
        self.assert_close(transform, expected_transform)
        self.assert_close(actual_mean, mean.to(actual_mean.dtype))
        self.assert_close(inverse, expected_inverse)
        self.assert_close(kornia.enhance.zca_whiten(data, dim=dim, unbiased=unbiased), expected)
        model = kornia.enhance.ZCAWhitening(dim=dim, unbiased=unbiased)
        self.assert_close(model(data, include_fit=True), expected)

    @pytest.mark.parametrize("unbiased", [True, False])
    def test_covariance_accumulation_gradient(self, device, dtype, unbiased):
        data = torch.tensor([[0.0], [255.0]], device=device, dtype=dtype).repeat(4, 1).requires_grad_()
        transform, _, _ = kornia.enhance.zca_mean(data, unbiased=unbiased)
        gradient = torch.autograd.grad(1000 * transform.sum(), data)[0]
        divisor = data.shape[0] - 1 if unbiased else data.shape[0]
        variance = data.shape[0] * 127.5**2 / divisor
        # Differentiate (variance + eps)**(-1/2), with d(variance)/dx = 2 * (x - mean) / divisor.
        expected = (data.detach() - 127.5) * (-1000 / (divisor * (variance + 1e-6) ** 1.5))
        self.assert_close(gradient, expected)

    def test_dynamo_covariance_accumulation(self, device, dtype, torch_optimizer):
        data = torch.tensor([[0.0, 0.0], [0.0, 127.5], [255.0, 0.0], [255.0, 127.5]], device=device, dtype=dtype)
        data = data.repeat(2, 1)
        op = kornia.enhance.zca_whiten
        self.assert_close(torch_optimizer(op)(data), op(data))

    @pytest.mark.parametrize("unbiased", [True, False])
    def test_covariance_beyond_half_range(self, device, dtype, unbiased):
        """A covariance above 65504 overflows float16, while the transform and its inverse fit."""
        data = torch.tensor([[-300.0], [300.0]], device=device, dtype=dtype).repeat(2, 1)
        std = math.sqrt(4 * 300.0**2 / (3 if unbiased else 4) + 1e-6)
        transform, mean, inverse = kornia.enhance.zca_mean(data, unbiased=unbiased, return_inverse=True)
        assert (
            transform.dtype
            == mean.dtype
            == inverse.dtype
            == (torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype)
        )
        # Compare dimensionless values so an absolute tolerance cannot accept a zero transform.
        self.assert_close(transform * std, torch.ones_like(transform))
        self.assert_close(mean, torch.zeros_like(mean))
        self.assert_close(inverse / std, torch.ones_like(inverse))

    @pytest.mark.parametrize("unbiased", [True, False])
    def test_covariance_accumulation_amp(self, device, dtype, unbiased):
        """Fitting leaves float16 autocast, whose matrix products would overflow a covariance above 65504."""
        if device.type not in ("cpu", "cuda", "mps"):
            pytest.skip("CPU/CUDA/MPS mixed-precision coverage")
        # The MPS job runs float32 only, so it fits a float16 input here as well.
        input_dtypes = [dtype, torch.float16] if device.type == "mps" and dtype != torch.float16 else [dtype]
        std = math.sqrt(4 * 300.0**2 / (3 if unbiased else 4) + 1e-6)
        for input_dtype in input_dtypes:
            data = torch.tensor([[-300.0], [300.0]], device=device, dtype=input_dtype).repeat(2, 1)
            with torch.autocast(device_type=device.type, dtype=torch.float16):
                transform, mean, inverse = kornia.enhance.zca_mean(data, unbiased=unbiased, return_inverse=True)
                assert torch.is_autocast_enabled(device.type)
            assert (
                transform.dtype
                == mean.dtype
                == inverse.dtype
                == (torch.float32 if input_dtype in (torch.float16, torch.bfloat16) else input_dtype)
            )
            self.assert_close(transform * std, torch.ones_like(transform))
            self.assert_close(mean, torch.zeros_like(mean))
            self.assert_close(inverse / std, torch.ones_like(inverse))

    @pytest.mark.device_agnostic
    def test_xpu_covariance_accumulation_amp(self):
        """XPU autocast must not downcast a finite float32 covariance to float16."""
        if not torch.xpu.is_available():
            pytest.skip("XPU is unavailable")
        data = torch.tensor([[-255.0], [255.0]], device="xpu", dtype=torch.float16).repeat(2, 1)
        std = math.sqrt(4 * 255.0**2 / 3 + 1e-6)
        with torch.autocast(device_type="xpu", dtype=torch.float16):
            transform, mean, inverse = kornia.enhance.zca_mean(data, return_inverse=True)
            assert torch.is_autocast_enabled("xpu")
            self.assert_close(transform, torch.full_like(transform, 1 / std))
            self.assert_close(mean, torch.zeros_like(mean))
            self.assert_close(inverse, torch.full_like(inverse, std))
            self.assert_close(kornia.enhance.zca_whiten(data), data / std)

    def test_covariance_accumulation_jit(self, device, dtype):
        """Scripted fitting matches eager fitting outside autocast."""
        data = torch.tensor([[-127.5], [127.5]], device=device, dtype=dtype).repeat(4, 1)
        scripted = torch.jit.script(kornia.enhance.zca_mean)
        actual = scripted(data, return_inverse=True)
        expected = kornia.enhance.zca_mean(data, return_inverse=True)
        for actual_tensor, expected_tensor in zip(actual, expected):
            self.assert_close(actual_tensor, expected_tensor)

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize("half_dtype", [torch.float16, torch.bfloat16])
    def test_covariance_accumulation_meta(self, half_dtype):
        """Shape-only fitting works on devices without an autocast context."""
        data = torch.empty(4, 2, device="meta", dtype=half_dtype)
        transform, mean, inverse = kornia.enhance.zca_mean(data, return_inverse=True)
        assert transform.shape == inverse.shape == (2, 2)
        assert mean.shape == (1, 2)
        assert transform.device == mean.device == inverse.device == data.device
        assert transform.dtype == mean.dtype == inverse.dtype == torch.float32

    @pytest.mark.parametrize("unbiased", [True, False])
    def test_covariance_accumulation_bfloat16_range(self, device, dtype, unbiased):
        """A finite covariance must not overflow its unnormalized scatter."""
        if dtype != torch.bfloat16:
            pytest.skip("Bfloat16 exponent-range regression")
        data = torch.tensor([[-1e19], [1e19]], device=device, dtype=dtype).repeat(2, 1)
        scale = data[1, 0]
        divisor = data.shape[0] - 1 if unbiased else data.shape[0]
        normalized_std = math.sqrt(data.shape[0] / divisor)
        transform, mean, inverse = kornia.enhance.zca_mean(data, unbiased=unbiased, return_inverse=True)
        assert (
            transform.dtype
            == mean.dtype
            == inverse.dtype
            == (torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype)
        )
        assert transform.device == mean.device == inverse.device == device
        # Compare dimensionless values so an absolute tolerance cannot accept a zero transform.
        self.assert_close(transform * scale, torch.full_like(transform, 1 / normalized_std))
        self.assert_close(mean, torch.zeros_like(mean))
        self.assert_close(inverse / scale, torch.full_like(inverse, normalized_std))
        self.assert_close(kornia.enhance.zca_whiten(data, unbiased=unbiased), data.sign() / normalized_std)

    @pytest.mark.parametrize("dim", [0, 1, -1])
    @pytest.mark.parametrize("sample_count", [0, 1])
    def test_unbiased_sample_count(self, device, dtype, dim, sample_count):
        """Unbiased covariance is undefined with fewer than two samples (gh-5313)."""
        shape = (sample_count, 3) if dim == 0 else (3, sample_count)
        data = torch.ones(shape, device=device, dtype=dtype)
        with pytest.raises(ValueError, match="at least two samples"):
            kornia.enhance.zca_mean(data, dim=dim, unbiased=True)

    @pytest.mark.parametrize("dim", [0, 1, -1])
    def test_biased_single_sample(self, device, dtype, dim):
        """The population covariance of one sample is zero and remains supported."""
        data = torch.tensor([[1.0, 2.0, 3.0]], device=device, dtype=dtype)
        if dim != 0:
            data = data.t()
        actual = kornia.enhance.zca_whiten(data, dim=dim, unbiased=False)
        self.assert_close(actual, torch.zeros_like(data))

    @pytest.mark.parametrize("dim", [0, 1, -1])
    def test_biased_empty_sample_axis(self, device, dtype, dim):
        """The biased covariance divides by N, so an empty sample axis is rejected as well."""
        shape = (0, 3) if dim == 0 else (3, 0)
        data = torch.ones(shape, device=device, dtype=dtype)
        with pytest.raises(ValueError, match="at least one sample"):
            kornia.enhance.zca_mean(data, dim=dim, unbiased=False)

    @pytest.mark.parametrize("unbiased", [True, False])
    @pytest.mark.parametrize("dim", [0, 1, -1])
    def test_two_samples(self, device, dtype, dim, unbiased):
        """Two samples are the fewest that unbiased whitening accepts; the divisor is N - 1 = 1, or N = 2 if biased."""
        data = torch.tensor([[2.0, 1.0], [0.0, 3.0]], device=device, dtype=dtype)
        # The centred samples (1, -1) and (-1, 1) have scatter eigenvalue 4 along (1, -1) and 0 across it, so with
        # eps = 1 the output is the centred data divided by sqrt(4 / divisor + 1).
        expected = (
            torch.tensor([[1.0, -1.0], [-1.0, 1.0]], device=device, dtype=dtype) / (5.0 if unbiased else 3.0) ** 0.5
        )
        if dim != 0:
            data, expected = data.t(), expected.t()
        actual = kornia.enhance.ZCAWhitening(dim=dim, unbiased=unbiased, eps=1.0)(data, include_fit=True)
        self.assert_close(actual, expected)

    @pytest.mark.parametrize("shape", [(3, 5, 2), (3, 3, 2)])
    @pytest.mark.parametrize("dim", [0, 1, -2, -1])
    def test_inverse_sample_axis(self, device, dtype, shape, dim):
        """Inverse uses the fitted sample axis, including equal-sized axis aliases (gh-5311)."""
        data = torch.arange(30 if shape[1] == 5 else 18, device=device, dtype=dtype).reshape(shape) / 100
        zca = kornia.enhance.ZCAWhitening(dim=dim, compute_inv=True, eps=1.0).fit(data)
        # A held-out query also exercises the fitted mean, rather than refitting.
        query = (10 * data).cos() / 4
        actual = zca.inverse_transform(zca(query))
        assert actual.shape == query.shape
        self.assert_close(actual, query)

    @pytest.mark.parametrize("unbiased", [True, False])
    def test_zca_unbiased(self, unbiased, device, dtype):
        data = torch.tensor([[0, 1], [1, 0], [-1, 0], [0, -1]], device=device, dtype=dtype)

        unbiased_val = 1.5 if unbiased else 2.0

        expected = torch.sqrt(unbiased_val * torch.abs(data)) * torch.sign(data)

        zca = kornia.enhance.ZCAWhitening(unbiased=unbiased).fit(data)

        actual = zca(data)

        self.assert_close(actual, expected, low_tolerance=True)

    @pytest.mark.parametrize("dim", [0, 1])
    def test_dim_args(self, dim, device, dtype):
        if "xla" in device.type:
            pytest.skip("buggy with XLA devices.")

        data = torch.tensor([[0, 1], [1, 0], [-1, 0], [0, -1]], device=device, dtype=dtype)

        if dim == 1:
            expected = torch.tensor(
                [
                    [-0.35360718, 0.35360718],
                    [0.35351562, -0.35351562],
                    [-0.35353088, 0.35353088],
                    [0.35353088, -0.35353088],
                ],
                device=device,
                dtype=dtype,
            )
        elif dim == 0:
            expected = torch.tensor(
                [[0.0, 1.2247448], [1.2247448, 0.0], [-1.2247448, 0.0], [0.0, -1.2247448]], device=device, dtype=dtype
            )

        zca = kornia.enhance.ZCAWhitening(dim=dim)
        actual = zca(data, True)

        self.assert_close(actual, expected, low_tolerance=True)

    @pytest.mark.parametrize("input_shape,eps", [((15, 2, 2, 2), 1e-6), ((10, 4), 0.1), ((20, 3, 2, 2), 1e-3)])
    def test_identity(self, input_shape, eps, device, dtype):
        """Assert that data can be recovered by the inverse transform."""
        data = torch.randn(*input_shape, device=device, dtype=dtype)

        zca = kornia.enhance.ZCAWhitening(compute_inv=True, eps=eps).fit(data)

        data_w = zca(data)

        data_hat = zca.inverse_transform(data_w)

        self.assert_close(data, data_hat, low_tolerance=True)

    def test_grad_zca_individual_transforms(self, device):
        """Check if the gradients of the transforms are correct w.r.t to the input data."""
        if device.type == "mps":
            pytest.skip("MPS does not support float64 required for gradcheck")
        data = torch.tensor([[2, 0], [0, 1], [-2, 0], [0, -1]], device=device, dtype=torch.float64)

        def zca_T(x):
            return kornia.enhance.zca_mean(x)[0]

        def zca_mu(x):
            return kornia.enhance.zca_mean(x)[1]

        def zca_T_inv(x):
            return kornia.enhance.zca_mean(x, return_inverse=True)[2]

        self.gradcheck(zca_T, (data,))
        self.gradcheck(zca_mu, (data,))
        self.gradcheck(zca_T_inv, (data,))

    def test_grad_zca_with_fit(self, device):
        if device.type == "mps":
            pytest.skip("MPS does not support float64 required for gradcheck")
        data = torch.tensor([[2, 0], [0, 1], [-2, 0], [0, -1]], device=device, dtype=torch.float64)

        def zca_fit(x):
            zca = kornia.enhance.ZCAWhitening(detach_transforms=False)
            return zca(x, include_fit=True)

        self.gradcheck(zca_fit, (data,))

    def test_grad_detach_zca(self, device):
        if device.type == "mps":
            pytest.skip("MPS does not support float64 required for gradcheck")
        data = torch.tensor([[1, 0], [0, 1], [-2, 0], [0, -1]], device=device, dtype=torch.float64)

        zca = kornia.enhance.ZCAWhitening()

        zca.fit(data)

        self.gradcheck(zca, (data,))

    def test_fitted_state_is_serialized(self, device, dtype):
        data = torch.randn(8, 3, device=device, dtype=dtype)
        zca = kornia.enhance.ZCAWhitening().fit(data)

        state_dict = zca.state_dict()

        assert "mean_vector" in state_dict
        assert "transform_matrix" in state_dict
        assert "transform_inv" in state_dict
        assert state_dict["mean_vector"].shape == zca.mean_vector.shape
        assert state_dict["transform_matrix"].shape == zca.transform_matrix.shape
        # Without compute_inv the inverse is an empty placeholder in the fit's working dtype.
        assert state_dict["transform_inv"].shape == (0,)
        compute_dtype = torch.float32 if dtype in (torch.float16, torch.bfloat16) else dtype
        assert all(tensor.device == device and tensor.dtype == compute_dtype for tensor in state_dict.values())

    def test_unfitted_state_is_empty(self):
        zca = kornia.enhance.ZCAWhitening()

        assert zca.state_dict() == {}

    def test_fitted_state_follows_dtype_conversion(self, device):
        if device.type == "mps":
            pytest.skip("MPS does not support float64")
        data = torch.randn(8, 3, device=device, dtype=torch.float32)
        zca = kornia.enhance.ZCAWhitening().fit(data)

        expected = zca(data)
        zca.double()

        actual = zca(data.double())

        assert zca.mean_vector.dtype == torch.float64
        assert zca.transform_matrix.dtype == torch.float64
        assert zca.transform_inv.dtype == torch.float64
        self.assert_close(actual, expected.double(), low_tolerance=True)

    def test_fitted_state_follows_device_conversion(self, device, dtype):
        data = torch.randn(8, 3, device=device, dtype=dtype)
        zca = kornia.enhance.ZCAWhitening(compute_inv=True).fit(data)

        # The meta device needs no hardware, so every test leg moves the fit off the device it was made on.
        zca.to("meta")

        assert zca.mean_vector.device.type == "meta"
        assert zca.transform_matrix.device.type == "meta"
        assert zca.transform_inv.device.type == "meta"

        output = zca(data.to("meta"))

        assert output.device.type == "meta"

    def test_fitted_state_round_trip(self, device, dtype):
        data = torch.randn(8, 3, device=device, dtype=dtype)

        zca = kornia.enhance.ZCAWhitening(compute_inv=True).fit(data)
        expected = zca(data)

        loaded = kornia.enhance.ZCAWhitening(compute_inv=True)
        result = loaded.load_state_dict(zca.state_dict())

        assert result.missing_keys == []
        assert result.unexpected_keys == []
        assert loaded.fitted
        self.assert_close(loaded(data), expected, low_tolerance=True)
        self.assert_close(loaded.inverse_transform(expected), zca.inverse_transform(expected), low_tolerance=True)

    def test_checkpoint_without_fitted_state_keeps_the_fit(self, device, dtype):
        data = torch.randn(8, 3, device=device, dtype=dtype)
        model = torch.nn.Sequential(kornia.enhance.ZCAWhitening().fit(data))
        expected = model(data)

        # A checkpoint saved before the fitted state was persisted: no ZCA keys and module version 1.
        checkpoint = torch.nn.Sequential(kornia.enhance.ZCAWhitening()).state_dict()
        checkpoint._metadata["0"]["version"] = 1
        result = model.load_state_dict(checkpoint)

        assert result.missing_keys == []
        assert model[0].fitted
        self.assert_close(model(data), expected)

        # A plain dict, e.g. one rebuilt with renamed keys, carries no version and is read the same way.
        assert model.load_state_dict({}).missing_keys == []
        self.assert_close(model(data), expected)

        # An unfitted module stays unfitted.
        unfitted = torch.nn.Sequential(kornia.enhance.ZCAWhitening())
        result = unfitted.load_state_dict(checkpoint)
        assert result.missing_keys == []
        assert result.unexpected_keys == []
        assert not unfitted[0].fitted

        # A checkpoint of this version saved from an unfitted module lacks the keys, and strict loading says so.
        with pytest.raises(RuntimeError, match="Missing key"):
            model.load_state_dict(torch.nn.Sequential(kornia.enhance.ZCAWhitening()).state_dict())

    def test_unfitted_state_round_trip(self):
        zca = kornia.enhance.ZCAWhitening()
        loaded = kornia.enhance.ZCAWhitening()

        result = loaded.load_state_dict(zca.state_dict())

        assert result.missing_keys == []
        assert result.unexpected_keys == []
        assert not loaded.fitted

    def test_not_fitted(self, device, dtype):
        data = torch.rand(10, 2, device=device, dtype=dtype)
        zca = kornia.enhance.ZCAWhitening()
        with pytest.raises(RuntimeError):
            zca(data)

    def test_not_fitted_inv(self, device, dtype):
        data = torch.rand(10, 2, device=device, dtype=dtype)
        zca = kornia.enhance.ZCAWhitening()
        with pytest.raises(RuntimeError):
            zca.inverse_transform(data)

    def test_jit(self, device, dtype):
        data = torch.rand(10, 3, 1, 2, device=device, dtype=dtype)
        zca = kornia.enhance.ZCAWhitening().fit(data)
        zca_jit = kornia.enhance.ZCAWhitening().fit(data)
        zca_jit = torch.jit.script(zca_jit)
        self.assert_close(zca_jit(data), zca(data))

    @pytest.mark.parametrize("unbiased", [True, False])
    def test_zca_whiten_func_unbiased(self, unbiased, device, dtype):
        data = torch.tensor([[0, 1], [1, 0], [-1, 0], [0, -1]], device=device, dtype=dtype)

        unbiased_val = 1.5 if unbiased else 2.0

        expected = torch.sqrt(unbiased_val * torch.abs(data)) * torch.sign(data)

        actual = kornia.enhance.zca_whiten(data, unbiased=unbiased)

        self.assert_close(actual, expected, low_tolerance=True)