Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

16.3 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import warnings

import pytest
import torch

import kornia
from kornia.core.check import are_checks_enabled, disable_checks, enable_checks
from kornia.core.exceptions import ValueCheckError

from testing.base import BaseTester


class TestExtractTensorPatches(BaseTester):
    @pytest.mark.parametrize("argument", ["window_size", "stride"])
    @pytest.mark.parametrize("value", [0, (1, 0), -1])
    def test_function_rejects_non_positive_patch_params(self, argument, value):
        params = {"window_size": 2, "stride": 1}
        params[argument] = value
        with pytest.raises(ValueCheckError, match=argument):
            kornia.contrib.extract_tensor_patches(torch.ones(1, 1, 4, 4), **params)

    @pytest.mark.parametrize("argument", ["window_size", "stride"])
    def test_rejects_non_positive_patch_params_with_checks_disabled(self, argument):
        # A zero window_size returned empty patches without this guard, so it must not follow disable_checks().
        params = {"window_size": 2, "stride": 1}
        params[argument] = 0
        was_enabled = are_checks_enabled()
        disable_checks()
        try:
            with pytest.raises(ValueCheckError, match=argument):
                kornia.contrib.extract_tensor_patches(torch.ones(1, 1, 4, 4), **params)
        finally:
            if was_enabled:
                enable_checks()

    @pytest.mark.parametrize("argument", ["window_size", "stride"])
    @pytest.mark.parametrize("value", [0, (1, 0), -1])
    def test_module_rejects_non_positive_patch_params(self, argument, value):
        params = {"window_size": 2, "stride": 1}
        params[argument] = value
        with pytest.raises(ValueCheckError, match=argument):
            kornia.contrib.ExtractTensorPatches(**params)

    def test_smoke(self, device):
        img = torch.arange(16.0, device=device).view(1, 1, 4, 4)
        m = kornia.contrib.ExtractTensorPatches(3)
        assert m(img).shape == (1, 4, 1, 3, 3)

    def test_b1_ch1_h4w4_ws3(self, device):
        img = torch.arange(16.0, device=device).view(1, 1, 4, 4)
        m = kornia.contrib.ExtractTensorPatches(3)
        patches = m(img)
        assert patches.shape == (1, 4, 1, 3, 3)
        self.assert_close(img[0, :, :3, :3], patches[0, 0])
        self.assert_close(img[0, :, :3, 1:], patches[0, 1])
        self.assert_close(img[0, :, 1:, :3], patches[0, 2])
        self.assert_close(img[0, :, 1:, 1:], patches[0, 3])

    def test_b1_ch2_h4w4_ws3(self, device):
        img = torch.arange(16.0, device=device).view(1, 1, 4, 4)
        img = img.expand(-1, 2, -1, -1)  # copy all channels
        m = kornia.contrib.ExtractTensorPatches(3)
        patches = m(img)
        assert patches.shape == (1, 4, 2, 3, 3)
        self.assert_close(img[0, :, :3, :3], patches[0, 0])
        self.assert_close(img[0, :, :3, 1:], patches[0, 1])
        self.assert_close(img[0, :, 1:, :3], patches[0, 2])
        self.assert_close(img[0, :, 1:, 1:], patches[0, 3])

    def test_b1_ch1_h4w4_ws2(self, device):
        img = torch.arange(16.0, device=device).view(1, 1, 4, 4)
        m = kornia.contrib.ExtractTensorPatches(2)
        patches = m(img)
        assert patches.shape == (1, 9, 1, 2, 2)
        self.assert_close(img[0, :, 0:2, 1:3], patches[0, 1])
        self.assert_close(img[0, :, 0:2, 2:4], patches[0, 2])
        self.assert_close(img[0, :, 1:3, 1:3], patches[0, 4])
        self.assert_close(img[0, :, 2:4, 1:3], patches[0, 7])

    def test_b1_ch1_h4w4_ws2_stride2(self, device):
        img = torch.arange(16.0, device=device).view(1, 1, 4, 4)
        m = kornia.contrib.ExtractTensorPatches(2, stride=2)
        patches = m(img)
        assert patches.shape == (1, 4, 1, 2, 2)
        self.assert_close(img[0, :, 0:2, 0:2], patches[0, 0])
        self.assert_close(img[0, :, 0:2, 2:4], patches[0, 1])
        self.assert_close(img[0, :, 2:4, 0:2], patches[0, 2])
        self.assert_close(img[0, :, 2:4, 2:4], patches[0, 3])

    def test_b1_ch1_h4w4_ws2_stride21(self, device):
        img = torch.arange(16.0, device=device).view(1, 1, 4, 4)
        m = kornia.contrib.ExtractTensorPatches(2, stride=(2, 1))
        patches = m(img)
        assert patches.shape == (1, 6, 1, 2, 2)
        self.assert_close(img[0, :, 0:2, 1:3], patches[0, 1])
        self.assert_close(img[0, :, 0:2, 2:4], patches[0, 2])
        self.assert_close(img[0, :, 2:4, 0:2], patches[0, 3])
        self.assert_close(img[0, :, 2:4, 2:4], patches[0, 5])

    def test_b1_ch1_h3w3_ws2_stride1_padding1(self, device):
        img = torch.arange(9.0).view(1, 1, 3, 3).to(device)
        m = kornia.contrib.ExtractTensorPatches(2, stride=1, padding=1)
        patches = m(img)
        assert patches.shape == (1, 16, 1, 2, 2)
        self.assert_close(img[0, :, 0:2, 0:2], patches[0, 5])
        self.assert_close(img[0, :, 0:2, 1:3], patches[0, 6])
        self.assert_close(img[0, :, 1:3, 0:2], patches[0, 9])
        self.assert_close(img[0, :, 1:3, 1:3], patches[0, 10])

    def test_b2_ch1_h3w3_ws2_stride1_padding1(self, device):
        batch_size = 2
        img = torch.arange(9.0).view(1, 1, 3, 3).to(device)
        img = img.expand(batch_size, -1, -1, -1)
        m = kornia.contrib.ExtractTensorPatches(2, stride=1, padding=1)
        patches = m(img)
        assert patches.shape == (batch_size, 16, 1, 2, 2)
        for i in range(batch_size):
            self.assert_close(img[i, :, 0:2, 0:2], patches[i, 5])
            self.assert_close(img[i, :, 0:2, 1:3], patches[i, 6])
            self.assert_close(img[i, :, 1:3, 0:2], patches[i, 9])
            self.assert_close(img[i, :, 1:3, 1:3], patches[i, 10])

    def test_b1_ch1_h3w3_ws23(self, device):
        img = torch.arange(9.0).view(1, 1, 3, 3).to(device)
        m = kornia.contrib.ExtractTensorPatches((2, 3))
        patches = m(img)
        assert patches.shape == (1, 2, 1, 2, 3)
        self.assert_close(img[0, :, 0:2, 0:3], patches[0, 0])
        self.assert_close(img[0, :, 1:3, 0:3], patches[0, 1])

    def test_b1_ch1_h3w4_ws23(self, device):
        img = torch.arange(12.0).view(1, 1, 3, 4).to(device)
        m = kornia.contrib.ExtractTensorPatches((2, 3))
        patches = m(img)
        assert patches.shape == (1, 4, 1, 2, 3)
        self.assert_close(img[0, :, 0:2, 0:3], patches[0, 0])
        self.assert_close(img[0, :, 0:2, 1:4], patches[0, 1])
        self.assert_close(img[0, :, 1:3, 0:3], patches[0, 2])
        self.assert_close(img[0, :, 1:3, 1:4], patches[0, 3])

    @pytest.mark.skip(reason="turn off all jit for a while")
    def test_jit(self, device):
        @torch.jit.script
        def op_script(img: torch.Tensor, height: int, width: int) -> torch.Tensor:
            return kornia.geometry.denormalize_pixel_coordinates(img, height, width)

        height, width = 3, 4
        grid = kornia.geometry.create_meshgrid(height, width, normalized_coordinates=True).to(device)

        actual = op_script(grid, height, width)
        expected = kornia.denormalize_pixel_coordinates(grid, height, width)

        self.assert_close(actual, expected)

    def test_gradcheck(self, device):
        img = torch.rand(2, 3, 4, 4, device=device, dtype=torch.float64)
        self.gradcheck(kornia.contrib.extract_tensor_patches, (img, 3))

    def test_auto_padding_stride(self, device, dtype):
        img_shape = (11, 14)
        window_size = (3, 3)
        stride = 2
        rnge = img_shape[0] * img_shape[1]
        img = torch.arange(rnge, device=device, dtype=dtype).view(1, 1, *img_shape)
        patches = kornia.contrib.extract_tensor_patches(
            img, window_size=window_size, stride=stride, allow_auto_padding=True
        )
        # 5 patches vertical, 6 2/3 = 7 horizontal = 35 patches
        assert patches.shape == (1, 35, 1, *window_size)

    @pytest.mark.parametrize("img_shape, fits", [((4, 4), True), ((5, 4), False), ((4, 5), False), ((2, 2), True)])
    def test_warns_when_the_window_does_not_fit(self, device, dtype, img_shape, fits):
        # With window 2 and stride 2, a remainder along either axis alone leaves pixels uncovered.
        img = torch.zeros(1, 1, *img_shape, device=device, dtype=dtype)
        with warnings.catch_warnings(record=True) as caught:
            warnings.simplefilter("always")
            kornia.contrib.extract_tensor_patches(img, window_size=2, stride=2)
        assert any("will not fit" in str(w.message) for w in caught) is not fits

    @pytest.mark.parametrize("batch_size", [0, 1, 2])
    @pytest.mark.parametrize("noncontiguous", [False, True])
    @pytest.mark.parametrize(
        "image_size,window,stride,padding,auto_padding,pad,grid",
        [
            ((8, 12), (4, 4), (4, 4), 0, False, (0, 0, 0, 0), (2, 3)),
            ((7, 11), (3, 5), (2, 3), 0, False, (0, 0, 0, 0), (3, 3)),
            ((7, 11), (3, 5), (2, 3), (1, 1, 2, 1), False, (2, 1, 1, 1), (4, 4)),
            ((9, 13), (4, 5), (4, 5), 0, True, (1, 1, 1, 2), (3, 3)),
        ],
    )
    def test_patch_layout_and_empty_batch_4429(
        self, batch_size, noncontiguous, image_size, window, stride, padding, auto_padding, pad, grid, device, dtype
    ):
        height, width = image_size
        storage_size = (width, height) if noncontiguous else image_size
        image = (torch.arange(batch_size * 3 * height * width, device=device) % 251).to(dtype)
        image = image.reshape(batch_size, 3, *storage_size)
        if noncontiguous:
            image = image.transpose(-2, -1)
        image = image.detach().requires_grad_()
        original = image.detach().clone()
        module = kornia.contrib.ExtractTensorPatches(window, stride, padding, auto_padding)
        patches = module(image)
        functional = kornia.contrib.extract_tensor_patches(image, window, stride, padding, auto_padding)
        # Independent spatial slices pin both the window count and row-major ordering.
        padded = torch.nn.functional.pad(image, pad)
        expected = torch.stack(
            [
                padded[
                    ..., row * stride[0] : row * stride[0] + window[0], col * stride[1] : col * stride[1] + window[1]
                ]
                for row in range(grid[0])
                for col in range(grid[1])
            ],
            dim=1,
        )
        assert patches.shape == (batch_size, grid[0] * grid[1], 3, *window)
        assert patches.dtype == dtype
        assert patches.device == device
        self.assert_close(patches, expected, rtol=0, atol=0)
        self.assert_close(functional, expected, rtol=0, atol=0)
        self.assert_close(image, original, rtol=0, atol=0)
        weights = (torch.arange(patches.numel(), device=device) % 7 + 1).to(dtype).reshape(patches.shape)
        actual_grad = torch.autograd.grad((patches * weights).sum(), image)[0]
        expected_grad = torch.autograd.grad((expected * weights).sum(), image)[0]
        assert actual_grad.shape == image.shape
        self.assert_close(actual_grad, expected_grad, rtol=0, atol=0)

    @pytest.mark.parametrize("batch_size", [0, 1])
    def test_invalid_empty_planes_and_window_4429(self, batch_size, device, dtype):
        with pytest.raises(ValueError, match="non-empty channel"):
            kornia.contrib.extract_tensor_patches(torch.empty(batch_size, 0, 8, 12, device=device, dtype=dtype), 4, 4)
        with pytest.raises(ValueError, match=r"window_size.*positive"):
            kornia.contrib.extract_tensor_patches(
                torch.empty(batch_size, 3, 8, 12, device=device, dtype=dtype), (0, 4), (4, 4)
            )
        with pytest.raises(ValueCheckError, match=r"stride.*positive"):
            kornia.contrib.extract_tensor_patches(
                torch.empty(batch_size, 3, 8, 12, device=device, dtype=dtype), 4, 0, padding=(1, 1, 1, 1)
            )

    @pytest.mark.parametrize("batch_size", [0, 2])
    def test_dynamo_patch_extraction_4429(self, batch_size, device, dtype, torch_optimizer):
        image = torch.arange(batch_size * 3 * 8 * 12, device=device).to(dtype).reshape(batch_size, 3, 8, 12)
        module = kornia.contrib.ExtractTensorPatches((4, 4), (4, 4))
        actual = torch_optimizer(module)(image)
        assert actual.shape == (batch_size, 6, 3, 4, 4)
        self.assert_close(actual, module(image), rtol=0, atol=0)

    @pytest.mark.parametrize("padding", [(0, 0), (0, 0, 0, 0)])
    def test_all_zero_padding_means_no_padding(self, padding, device, dtype):
        # A truthy zero tuple switched off allow_auto_padding (N 6 instead of 12, last row/column dropped).
        img = torch.arange(1, 36, device=device, dtype=dtype).view(1, 1, 5, 7)
        expected = kornia.contrib.extract_tensor_patches(img, 2, 2, padding=0, allow_auto_padding=True)
        out = kornia.contrib.extract_tensor_patches(img, 2, 2, padding=padding, allow_auto_padding=True)
        assert out.shape == (1, 12, 1, 2, 2)
        self.assert_close(out, expected)

    def test_convention_patches_are_stacked_row_major_like_unfold(self, device, dtype):
        img = torch.arange(1, 57, device=device, dtype=dtype).view(1, 2, 4, 7)
        patches = kornia.contrib.extract_tensor_patches(img, (2, 3), (1, 2))
        # F.unfold's column order, reshaped to (B, N, C, h, w)
        unfold = torch.nn.functional.unfold(img, (2, 3), stride=(1, 2)).transpose(1, 2).reshape(1, -1, 2, 2, 3)
        assert patches.shape == (1, 9, 2, 2, 3)
        self.assert_close(patches, unfold)
        # Patch n starts at row (n // 3) * 1 and column (n % 3) * 2: left to right, then top to bottom.
        assert patches[0, :, 0, 0, 0].tolist() == [1, 3, 5, 8, 10, 12, 15, 17, 19]

    def test_convention_four_tuple_padding_is_top_bottom_left_right(self, device, dtype):
        img = torch.arange(1, 13, device=device, dtype=dtype).view(1, 1, 3, 4)
        out = kornia.contrib.extract_tensor_patches(img, (4, 6), padding=(1, 0, 0, 2))
        # (top, bottom, left, right) = (1, 0, 0, 2) is F.pad's (left, right, top, bottom) = (0, 2, 1, 0)
        self.assert_close(out[:, 0], torch.nn.functional.pad(img, (0, 2, 1, 0)))
        out = kornia.contrib.extract_tensor_patches(img, (5, 8), padding=(1, 2))  # (ph, pw) on each side
        self.assert_close(out[:, 0], torch.nn.functional.pad(img, (2, 2, 1, 1)))

    def test_convention_auto_padding_puts_the_odd_pixel_bottom_right(self, device, dtype):
        assert kornia.contrib.compute_padding((4, 5), 3, 3) == (1, 1, 0, 1)
        assert kornia.contrib.compute_padding((5, 4), 3, 3) == (0, 1, 1, 1)
        assert kornia.contrib.compute_padding((2, 3), (4, 4), 1) == (1, 1, 0, 1)  # smaller than the window: one window
        img = torch.arange(1, 21, device=device, dtype=dtype).view(1, 1, 4, 5)
        patches = kornia.contrib.extract_tensor_patches(img, 3, 3, allow_auto_padding=True)
        assert patches.shape == (1, 4, 1, 3, 3)
        assert patches[0, 0, 0].tolist() == [[0, 0, 0], [1, 2, 3], [6, 7, 8]]
        assert patches[0, 3, 0].tolist() == [[14, 15, 0], [19, 20, 0], [0, 0, 0]]

    def test_convention_explicit_padding_switches_off_auto_padding_and_the_fit_warning(self, device, dtype):
        img = torch.arange(1, 36, device=device, dtype=dtype).view(1, 1, 5, 7)
        with warnings.catch_warnings():
            warnings.simplefilter("error")
            # One row of bottom padding: the 7 columns still do not fit stride 2, and the last one is dropped silently.
            out = kornia.contrib.extract_tensor_patches(img, 2, 2, padding=(0, 1, 0, 0), allow_auto_padding=True)
        assert out.shape == (1, 9, 1, 2, 2)
        assert not (out == 35).any()
        assert kornia.contrib.extract_tensor_patches(img, 2, 2, allow_auto_padding=True).shape == (1, 12, 1, 2, 2)


@pytest.mark.parametrize("argument", ["window_size", "stride"])
@pytest.mark.parametrize("value", [0, (1, 0), -1])
def test_compute_padding_rejects_non_positive_patch_params(argument, value):
    params = {"window_size": 2, "stride": 1}
    params[argument] = value
    with pytest.raises(ValueCheckError, match=argument):
        kornia.contrib.compute_padding((4, 4), **params)