Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

14.3 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import pytest
import torch

import kornia
from kornia.core.exceptions import BaseError

from testing.base import BaseTester


class TestTotalVariation(BaseTester):
    @pytest.mark.parametrize("shape", [(1, 1), (1, 4), (4, 1), (2, 3, 1, 4), (2, 3, 4, 1)])
    @pytest.mark.parametrize("reduction", ["sum", "mean"])
    def test_singleton_spatial_axis(self, device, dtype, shape, reduction):
        height, width = shape[-2:]
        image = torch.arange(height * width, device=device, dtype=dtype).reshape(height, width).expand(shape)
        expected_value = 0.0 if height == width == 1 else 1.0 if reduction == "mean" else 3.0
        expected = torch.full(shape[:-2], expected_value, device=device, dtype=dtype)

        self.assert_close(kornia.losses.total_variation(image, reduction), expected)

    @pytest.mark.parametrize("shape", [(1, 1), (1, 4), (4, 1)])
    @pytest.mark.parametrize("input_dtype", [torch.uint8, torch.int8, torch.int16, torch.int32])
    def test_singleton_spatial_axis_integer(self, device, input_dtype, shape):
        height, width = shape
        image = torch.arange(height * width, device=device, dtype=input_dtype).reshape(shape)
        expected = torch.tensor(0.0 if height == width == 1 else 1.0, device=device)

        self.assert_close(kornia.losses.total_variation(image, reduction="mean"), expected)

    @pytest.mark.parametrize("shape", [(1, 1), (1, 4), (4, 1)])
    def test_singleton_spatial_axis_gradient(self, device, dtype, shape):
        height, width = shape
        image = torch.arange(height * width, device=device, dtype=dtype).reshape(shape).requires_grad_()
        loss = kornia.losses.total_variation(image, reduction="mean")
        loss.backward()
        expected = torch.zeros_like(image).flatten()
        if height * width > 1:
            expected[0], expected[-1] = -1 / 3, 1 / 3

        self.assert_close(loss, image.new_tensor(0.0 if height == width == 1 else 1.0))
        self.assert_close(image.grad, expected.reshape(shape))

    @pytest.mark.parametrize("shape", [(1, 1), (1, 4), (4, 1)])
    def test_dynamo_singleton_spatial_axis(self, device, dtype, shape, torch_optimizer):
        height, width = shape
        image = torch.arange(height * width, device=device, dtype=dtype).reshape(shape)
        expected = image.new_tensor(0.0 if height == width == 1 else 1.0)
        op_optimized = torch_optimizer(kornia.losses.total_variation)

        self.assert_close(op_optimized(image, reduction="mean"), expected)

    # Total variation of constant vectors is 0
    @pytest.mark.parametrize(
        "pred, expected",
        [
            (torch.ones(3, 4, 5), torch.tensor([0.0, 0.0, 0.0])),
            (2 * torch.ones(2, 3, 4, 5), torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])),
        ],
    )
    def test_tv_on_constant(self, device, dtype, pred, expected):
        actual = kornia.losses.total_variation(pred.to(device, dtype))
        self.assert_close(actual, expected.to(device, dtype))

    # Total variation of constant vectors is 0
    @pytest.mark.parametrize(
        "pred, expected",
        [
            (torch.ones(3, 4, 5), torch.tensor([0.0, 0.0, 0.0])),
            (2 * torch.ones(2, 3, 4, 5), torch.tensor([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0]])),
        ],
    )
    def test_tv_on_constant_int(self, device, pred, expected):
        actual = kornia.losses.total_variation(pred.to(device, dtype=torch.int32), reduction="mean")
        self.assert_close(actual, expected.to(device))

    @pytest.mark.parametrize("input_dtype", [torch.uint8, torch.int8, torch.int16, torch.int32])
    @pytest.mark.parametrize("shape", [(2, 3), (2, 3, 2, 3)])
    @pytest.mark.parametrize("reduction", ["sum", "mean"])
    def test_tv_integer_extrema(self, device, input_dtype, shape, reduction):
        limits = torch.iinfo(input_dtype)
        pred = torch.tensor(
            [[limits.max, limits.min, 0], [limits.max, limits.min, 0]], device=device, dtype=input_dtype
        ).expand(shape)
        # Four horizontal edges: two span the full range, two have magnitude abs(min).
        expected_sum = 2 * (limits.max - limits.min + abs(limits.min))
        expected = torch.tensor(expected_sum, device=device, dtype=torch.int64)
        if reduction == "mean":
            expected = expected.float() / 4
        expected = expected.expand(shape[:-2])

        self.assert_close(kornia.losses.total_variation(pred, reduction), expected)
        self.assert_close(kornia.losses.total_variation(pred.flip(-1), reduction), expected)
        self.assert_close(kornia.losses.total_variation(pred.transpose(-2, -1), reduction), expected)
        if reduction == "sum":
            self.assert_close(kornia.losses.TotalVariation()(pred), expected)

    # Total variation for 3D tensors
    @pytest.mark.parametrize(
        "pred, expected",
        [
            (
                torch.tensor(
                    [
                        [
                            [0.11747694, 0.5717714, 0.89223915, 0.2929412, 0.63556224],
                            [0.5371079, 0.13416398, 0.7782737, 0.21392655, 0.1757018],
                            [0.62360305, 0.8563448, 0.25304103, 0.68539226, 0.6956515],
                            [0.9350611, 0.01694632, 0.78724295, 0.4760313, 0.73099905],
                        ],
                        [
                            [0.4788819, 0.45253807, 0.932798, 0.5721999, 0.7612051],
                            [0.5455887, 0.8836531, 0.79551977, 0.6677338, 0.74293613],
                            [0.4830376, 0.16420758, 0.15784949, 0.21445751, 0.34168917],
                            [0.8675162, 0.5468113, 0.6117004, 0.01305223, 0.17554593],
                        ],
                        [
                            [0.6423703, 0.5561105, 0.54304767, 0.20339686, 0.8553698],
                            [0.98024786, 0.31562763, 0.10122144, 0.17686582, 0.26260805],
                            [0.20522952, 0.14523649, 0.8601968, 0.02593213, 0.7382898],
                            [0.71935296, 0.9625162, 0.42287344, 0.07979459, 0.9149871],
                        ],
                    ]
                ),
                torch.tensor([12.6647, 7.9527, 12.3838]),
            ),
            (
                torch.tensor([[[0.09094203, 0.32630223, 0.8066123], [0.10921168, 0.09534764, 0.48588026]]]),
                torch.tensor([1.6900]),
            ),
        ],
    )
    def test_tv_on_3d(self, device, dtype, pred, expected):
        actual = kornia.losses.total_variation(pred.to(device, dtype))
        self.assert_close(actual, expected.to(device, dtype), rtol=1e-3, atol=1e-3)

    # Total variation for 4D tensors
    @pytest.mark.parametrize(
        "pred, expected",
        [
            (
                torch.tensor(
                    [
                        [
                            [[0.8756, 0.0920], [0.8034, 0.3107]],
                            [[0.3069, 0.2981], [0.9399, 0.7944]],
                            [[0.6269, 0.1494], [0.2493, 0.8490]],
                        ],
                        [
                            [[0.3256, 0.9923], [0.2856, 0.9104]],
                            [[0.4107, 0.4387], [0.2742, 0.0095]],
                            [[0.7064, 0.3674], [0.6139, 0.2487]],
                        ],
                    ]
                ),
                torch.tensor([[1.5672, 1.2836, 2.1544], [1.4134, 0.8584, 0.9154]]),
            ),
            (
                torch.tensor(
                    [
                        [[[0.1104, 0.2284, 0.4371], [0.4569, 0.1906, 0.8035]]],
                        [[[0.0552, 0.6831, 0.8310], [0.3589, 0.5044, 0.0802]]],
                        [[[0.5078, 0.5703, 0.9110], [0.4765, 0.8401, 0.2754]]],
                    ]
                ),
                torch.tensor([[1.9566], [2.5787], [2.2682]]),
            ),
        ],
    )
    def test_tv_on_4d(self, device, dtype, pred, expected):
        actual = kornia.losses.total_variation(pred.to(device, dtype))
        self.assert_close(actual, expected.to(device, dtype), rtol=1e-3, atol=1e-3)

    @pytest.mark.parametrize("pred", [torch.rand(3, 5, 5), torch.rand(4, 3, 5, 5), torch.rand(4, 2, 3, 5, 5)])
    def test_tv_shapes(self, device, dtype, pred):
        pred = pred.to(device, dtype)
        actual_lesser_dims = []
        for slice in torch.unbind(pred, dim=0):
            slice_tv = kornia.losses.total_variation(slice)
            actual_lesser_dims.append(slice_tv)
        actual_lesser_dims = torch.stack(actual_lesser_dims, dim=0)
        actual_higher_dims = kornia.losses.total_variation(pred)
        self.assert_close(actual_lesser_dims, actual_higher_dims.to(device, dtype), rtol=1e-3, atol=1e-3)

    @pytest.mark.parametrize("reduction, expected", [("sum", torch.tensor(20)), ("mean", torch.tensor(1))])
    def test_tv_reduction(self, device, dtype, reduction, expected):
        pred, _ = torch.meshgrid([torch.arange(5), torch.arange(5)], indexing="ij")
        pred = pred.to(device, dtype)
        actual = kornia.losses.total_variation(pred, reduction=reduction)
        self.assert_close(actual, expected.to(device, dtype), rtol=1e-3, atol=1e-3)

    @pytest.mark.parametrize("layout", ["contiguous", "channels_last", "permuted"])
    @pytest.mark.parametrize("reduction", ["sum", "mean"])
    def test_tv_matches_the_two_dim_reduction(self, device, dtype, layout, reduction):
        # The reduction runs over flatten(-2) rather than dim=(-2, -1); the two must agree
        # for any memory layout the flatten has to copy, not only a contiguous input.
        pred = torch.rand(2, 3, 6, 7, device=device, dtype=dtype)
        if layout == "channels_last":
            pred = pred.contiguous(memory_format=torch.channels_last)
        elif layout == "permuted":
            pred = pred.permute(0, 1, 3, 2)

        dif1 = (pred[..., 1:, :] - pred[..., :-1, :]).abs()
        dif2 = (pred[..., :, 1:] - pred[..., :, :-1]).abs()
        if reduction == "sum":
            expected = dif1.sum(dim=(-2, -1)) + dif2.sum(dim=(-2, -1))
        else:
            expected = dif1.mean(dim=(-2, -1)) + dif2.mean(dim=(-2, -1))

        actual = kornia.losses.total_variation(pred, reduction=reduction)
        self.assert_close(actual, expected)

    # Expect TypeError to be raised when non-torch tensors are passed
    @pytest.mark.parametrize("pred", [1, [1, 2]])
    def test_tv_on_invalid_types(self, device, dtype, pred):
        with pytest.raises(TypeError):
            kornia.losses.total_variation(pred)

    def test_dynamo(self, device, dtype, torch_optimizer):
        image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype)

        op = kornia.losses.total_variation
        op_optimized = torch_optimizer(op)

        self.assert_close(op(image), op_optimized(image))

    @pytest.mark.parametrize("input_dtype", [torch.uint8, torch.int8, torch.int16, torch.int32])
    @pytest.mark.parametrize("reduction", ["sum", "mean"])
    def test_dynamo_integer(self, device, input_dtype, reduction, torch_optimizer):
        limits = torch.iinfo(input_dtype)
        image = torch.tensor([[limits.max, limits.min], [limits.max, limits.min]], device=device, dtype=input_dtype)
        op = kornia.losses.total_variation
        op_optimized = torch_optimizer(op)

        self.assert_close(op(image, reduction), op_optimized(image, reduction))

    def test_module(self, device, dtype):
        image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype)

        op = kornia.losses.total_variation
        op_module = kornia.losses.TotalVariation()

        self.assert_close(op(image), op_module(image))

    def test_gradcheck(self, device, dtype):
        dtype = torch.float64
        image = torch.rand(1, 2, 3, 4, device=device, dtype=dtype)
        self.gradcheck(kornia.losses.total_variation, (image,))


class TestConventionsTotalVariation(BaseTester):
    @staticmethod
    def _edges(device, dtype):
        # H = 5, W = 8. Channel 0 holds one vertical edge (a step along W in each of the 5 rows), channel 1 one
        # horizontal edge (a step along H in each of the 8 columns), channel 2 is constant; sample 1 doubles sample 0.
        img = torch.zeros(2, 3, 5, 8)
        img[:, 0, :, 3:] = 1.0
        img[:, 1, 2:, :] = 1.0
        img[:, 2] = 0.5
        img[1] *= 2
        return img.to(device, dtype)

    def test_convention_total_variation_sums_over_the_last_two_axes(self, device, dtype):
        # Anisotropic L1 total variation, sum |d/dy| + sum |d/dx| over the last two axes only: (B, C, H, W) gives
        # (B, C), with the channels kept apart. The default reduction is 'sum', and TotalVariation always sums.
        img = self._edges(device, dtype)
        expected = torch.tensor([[5.0, 8.0, 0.0], [10.0, 16.0, 0.0]], device=device, dtype=dtype)
        self.assert_close(kornia.losses.total_variation(img), expected)
        self.assert_close(kornia.losses.TotalVariation()(img), expected)
        self.assert_close(kornia.losses.total_variation(img[0, 1]), expected[0, 1])

    def test_convention_total_variation_mean_divides_each_term_by_its_own_count(self, device, dtype):
        # 'mean' is mean |d/dy| over the (H - 1) W vertical differences plus mean |d/dx| over the H (W - 1) horizontal
        # ones: a vertical edge gives 1 / (W - 1) = 1/7 and a horizontal edge 1 / (H - 1) = 1/4, where dividing the sum
        # by H W would give 1/8 and 1/5. Only 'mean' and 'sum' are accepted.
        img = self._edges(device, dtype)
        expected = torch.tensor([[1 / 7, 1 / 4, 0.0], [2 / 7, 2 / 4, 0.0]], device=device, dtype=dtype)
        self.assert_close(kornia.losses.total_variation(img, reduction="mean"), expected)
        # Relabelling check: transposing the image transposes both counts with it, so the values stay.
        self.assert_close(kornia.losses.total_variation(img.transpose(-2, -1), reduction="mean"), expected)
        with pytest.raises((ValueError, BaseError)):
            kornia.losses.total_variation(img, reduction="none")