Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

22.3 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from unittest.mock import PropertyMock, patch

import pytest
import torch
from torch import nn

import kornia
from kornia.feature import LocalFeatureMatcher

from testing.base import BaseTester, supports_reflect_padding, supports_replicate_padding


def _render(x0: float, w: int, h: int = 48, dtype: torch.dtype = torch.float32) -> torch.Tensor:  # float32: MPS too
    ys, xs = torch.meshgrid(torch.arange(h, dtype=dtype), torch.arange(w, dtype=dtype) + x0, indexing="ij")
    return torch.stack(
        [
            0.5 + 0.2 * torch.sin(0.21 * xs + 0.13 * ys + c) + 0.15 * torch.cos(0.17 * ys - 0.07 * xs + 2 * c)
            for c in range(3)
        ]
    )[None]


class _ShiftMatcher(LocalFeatureMatcher):
    """Exact correspondences: right pixel p is left pixel p + (shift_b, 0); samples in `empty` get none.

    The right image is sampled every 5 px along x and every `ystep` px along y.
    """

    def __init__(self, shifts, empty=(), float32_keypoints=False, ystep=5.0):
        nn.Module.__init__(self)
        self.shifts, self.empty, self.float32_keypoints, self.ystep = shifts, set(empty), float32_keypoints, ystep

    def forward(self, data):
        im0, im1 = data["image0"], data["image1"]
        kp0, kp1, idx = [torch.empty(0, 2)], [torch.empty(0, 2)], [torch.empty(0, dtype=torch.long)]
        for b in range(im1.shape[0]):
            if b in self.empty:
                continue
            ys, xs = torch.meshgrid(
                torch.arange(2, im1.shape[-2] - 2, self.ystep), torch.arange(2, im1.shape[-1] - 2, 5.0), indexing="ij"
            )
            p1 = torch.stack([xs.flatten(), ys.flatten()], -1)
            p0 = p1 + torch.tensor([self.shifts[b], 0.0])
            keep = (p0[:, 0] >= 0) & (p0[:, 0] <= im0.shape[-1] - 1)
            kp0.append(p0[keep])
            kp1.append(p1[keep])
            idx.append(torch.full((int(keep.sum()),), b))
        return {
            "keypoints0": torch.cat(kp0).to(im0.device if self.float32_keypoints else im0),
            "keypoints1": torch.cat(kp1).to(im0.device if self.float32_keypoints else im0),
            "batch_indexes": torch.cat(idx),
        }


class TestImageStitcher(BaseTester):
    @pytest.mark.parametrize(
        "columns, width",
        [
            ([1, 0, 1, 1, 0, 0], 4),
            ([0, 1, 1, 0, 0, 0], 3),
            ([1, 0, 1, 0, 1, 1], 6),
            ([1, 1, 1, 0, 0, 0], 3),
            ([1, 1, 1, 1, 1, 0], 5),
            ([1, 1, 1, 1, 1, 1], 6),
            ([0, 0, 0, 0, 0, 0], 6),
        ],
    )
    def test_postprocess_trims_only_empty_suffix(self, columns, width, device, dtype):
        stitcher = kornia.contrib.ImageStitcher(torch.nn.Identity())
        image = torch.arange(72, device=device, dtype=dtype).reshape(2, 3, 2, 6)
        mask = torch.tensor(columns, device=device, dtype=dtype).reshape(1, 1, 1, 6).expand_as(image)
        image_before, mask_before = image.clone(), mask.clone()

        actual = stitcher.postprocess(image, mask)

        self.assert_close(actual, image[..., :width])
        assert actual.device == image.device
        assert actual.dtype == image.dtype
        self.assert_close(image, image_before)
        self.assert_close(mask, mask_before)

    def test_postprocess_uses_all_mask_pixels(self, device, dtype):
        stitcher = kornia.contrib.ImageStitcher(torch.nn.Identity())
        image = torch.arange(72, device=device, dtype=dtype).reshape(2, 3, 2, 6)
        mask = torch.zeros_like(image, dtype=torch.bool)
        mask[0, 0, 0, 0] = True
        mask[1, 2, 1, 3] = True

        self.assert_close(stitcher.postprocess(image, mask), image[..., :4])

    def test_postprocess_gradcheck(self, device):
        stitcher = kornia.contrib.ImageStitcher(torch.nn.Identity())
        image = torch.rand(1, 1, 2, 6, device=device, dtype=torch.float64, requires_grad=True)
        mask = torch.tensor([1, 0, 1, 1, 0, 0], device=device, dtype=torch.bool).reshape(1, 1, 1, 6)
        self.gradcheck(lambda value: stitcher.postprocess(value, mask), (image,))

    @pytest.mark.parametrize("img_dtype", [torch.float32, torch.float16, torch.bfloat16])
    @pytest.mark.parametrize("shift", [40.0, 40.5, 40.7])
    def test_subpixel_shift_has_no_dark_seam(self, shift, img_dtype, device):
        # A mask warped with nearest and an image warped bilinearly: s = 40.5 left a dark column (error 0.43),
        # s = 40.7 an extra dark trailing column (width 97, truth 96). The mask and the image share one warp, so they
        # share the sampling grid in half precision too (on MPS a half image keeps a half grid).
        torch.manual_seed(0)
        left, right = _render(0, 64).to(device, img_dtype), _render(shift, 56).to(device, img_dtype)
        with torch.no_grad():
            out = kornia.contrib.ImageStitcher(_ShiftMatcher([shift]))(left, right)
        assert out.shape[-1] == int(shift) + 56
        truth = _render(0, out.shape[-1]).to(device)
        # bilinear resampling of the smooth texture, no black mixed in and no covered pixel dropped: a dark or dropped
        # pixel is off by 0.3 or more for these shifts; the bound is interpolation error plus the bf16 keypoint rounding
        tol = {torch.float32: 0.02, torch.float16: 0.02, torch.bfloat16: 0.04}[img_dtype]
        assert (out.float() - truth).abs().max().item() < tol

    @pytest.mark.parametrize("img_dtype", [torch.float16, torch.bfloat16])
    def test_float32_homography_with_half_image(self, img_dtype, device):
        # The warp runs in float32, so a float32 homography (exact float32 keypoints) works with a half image.
        left, right = _render(0, 64).to(device, img_dtype), _render(40.5, 56).to(device, img_dtype)
        matcher = _ShiftMatcher([40.5], float32_keypoints=True)
        with torch.no_grad():
            out = kornia.contrib.ImageStitcher(matcher)(left, right)
        assert out.dtype == img_dtype
        tol = {torch.float16: 0.02, torch.bfloat16: 0.04}[img_dtype]
        assert (out.float() - _render(0, out.shape[-1]).to(device)).abs().max().item() < tol

    @pytest.mark.parametrize("img_dtype", [torch.float32, torch.float16])
    def test_image_and_mask_share_one_warp(self, img_dtype, device):
        # On MPS a half image is sampled on a half grid; a mask warped in a separate (float32) call lands on another
        # grid and marks pixels the image samples outside of as covered. One call keeps both on the same grid.
        left, right = _render(0, 64).to(device, img_dtype), _render(40.5, 56).to(device, img_dtype)
        stitcher = kornia.contrib.ImageStitcher(_ShiftMatcher([40.5]))
        with (
            patch(
                "kornia.contrib.image_stitching.warp_perspective", wraps=kornia.geometry.transform.warp_perspective
            ) as w,
            torch.no_grad(),
        ):
            stitcher.stitch_pair(left, right)
        assert w.call_count == 1
        assert w.call_args.args[0].shape[1] == 6  # three image channels and three mask channels
        assert w.call_args.args[0].dtype == torch.float32  # also for a half image

    def test_batch_of_two_equals_each_sample_alone(self, device):
        left = torch.cat([_render(0, 64), _render(3, 64)]).to(device)
        right = torch.cat([_render(40, 56), _render(31.5, 56)]).to(device)
        stitcher = kornia.contrib.ImageStitcher(_ShiftMatcher([40.0, 28.5]))
        with torch.no_grad():
            pair = stitcher.stitch_pair(left, right)[0]
            for b in range(2):
                alone = kornia.contrib.ImageStitcher(_ShiftMatcher([[40.0, 28.5][b]])).stitch_pair(
                    left[b : b + 1], right[b : b + 1]
                )[0]
                self.assert_close(pair[b : b + 1], alone)

    def test_estimate_transform_names_a_gap_without_batch_size(self):
        # Samples 0 and 2 matched, sample 1 not: the sample count comes from the largest batch index.
        stitcher = kornia.contrib.ImageStitcher(_ShiftMatcher([0.0]))
        kp = torch.rand(8, 2)
        idx = torch.tensor([0, 0, 0, 0, 2, 2, 2, 2])
        with pytest.raises(RuntimeError, match="sample 1"):
            stitcher.estimate_transform(keypoints0=kp, keypoints1=kp, batch_indexes=idx)

    def test_unmatched_sample_is_named(self, device):
        # One unmatched sample failed the batch with grid_sampler's batch-size error.
        left = torch.cat([_render(0, 64)] * 2).to(device)
        right = torch.cat([_render(40, 56), _render(30, 56)]).to(device)
        with pytest.raises(RuntimeError, match="sample 1"):
            kornia.contrib.ImageStitcher(_ShiftMatcher([40.0, 30.0], empty=(1,)))(left, right)

    def test_every_sample_unmatched_names_sample_0(self, device):
        # With the batch size known, an all-empty match set is reported per sample, like a single unmatched one.
        left = torch.cat([_render(0, 64)] * 2).to(device)
        right = torch.cat([_render(40, 56), _render(30, 56)]).to(device)
        with pytest.raises(RuntimeError, match="sample 0: 0 matches"):
            kornia.contrib.ImageStitcher(_ShiftMatcher([40.0, 30.0], empty=(0, 1)))(left, right)

    def test_no_matches_without_batch_size(self):
        # Without a batch size there is no sample to name.
        stitcher = kornia.contrib.ImageStitcher(_ShiftMatcher([0.0]))
        empty = torch.empty(0, 2)
        with pytest.raises(RuntimeError, match="No matched keypoints found"):
            stitcher.estimate_transform(keypoints0=empty, keypoints1=empty, batch_indexes=torch.empty(0).long())

    @pytest.mark.parametrize(
        "idx, batch_size, message",
        [
            ([0, 1, 2], 2, "batch index 2 is outside the batch of 2 samples"),
            ([0, 1], 1, "batch index 1 is outside the batch of 1 sample\\."),
            ([0, 1, -1], 2, "batch index -1 is negative"),
            ([-2], None, "batch index -2 is negative"),
        ],
    )
    def test_batch_index_outside_the_batch_is_named(self, idx, batch_size, message):
        # Matches of a batch index outside [0, batch_size) were silently left out.
        stitcher = kornia.contrib.ImageStitcher(_ShiftMatcher([0.0]))
        idx = torch.tensor(idx).repeat_interleave(4)
        kp = torch.rand(len(idx), 2) * 50
        with pytest.raises(RuntimeError, match=message):
            stitcher.estimate_transform(batch_size=batch_size, keypoints0=kp, keypoints1=kp, batch_indexes=idx)

    def test_integer_shift_at_large_coordinates_keeps_the_border(self, device):
        # For integer shifts the warp's float32 sampling positions are off by up to ~1.5 * eps * max(H_out, W_out) px
        # (general homographies up to ~2.9), so the coverage of a completely covered border pixel reads below 1 - 1e-4
        # on a wide panorama. Here 8 x 4096 images shifted by 3 px: the right image's last column read 0.99951 and was
        # dropped (the panorama was one column short).
        left, right = _render(0, 4096, h=8).to(device), _render(3, 4096, h=8).to(device)
        with torch.no_grad():
            out = kornia.contrib.ImageStitcher(_ShiftMatcher([3.0], ystep=1.0))(left, right)
        assert out.shape[-1] == 3 + 4096
        assert (out - _render(0, 3 + 4096, h=8).to(device)).abs().max().item() < 0.02

    def test_coverage_tolerance_scales_with_the_panorama(self, device):
        # The tolerance sits between the warp's round-off and a genuinely partial pixel. On this 8 x 8192 panorama a
        # coverage 3 * eps * 8192 below 1 is kept (above the worst round-off measured: 2.86 * eps * max(H_out, W_out)
        # for general homographies, 1.5 for integer shifts, on canvases too large for a test) and 0.99 is dropped.
        left, right = _render(0, 4096, h=8).to(device), _render(3, 4096, h=8).to(device)
        real_warp = kornia.geometry.transform.warp_perspective

        def warp(*args, **kwargs):
            out = real_warp(*args, **kwargs)
            out[:, 3:, 0, 3 + 4095] = 1 - 3 * torch.finfo(torch.float32).eps * 8192
            out[:, 3:, 1, 3 + 4095] = 0.99
            return out

        with patch("kornia.contrib.image_stitching.warp_perspective", warp), torch.no_grad():
            _, mask = kornia.contrib.ImageStitcher(_ShiftMatcher([3.0], ystep=1.0)).stitch_pair(left, right)
        assert mask[0, 0, 0, 3 + 4095].item() == 1
        assert mask[0, 0, 1, 3 + 4095].item() == 0

    @pytest.mark.parametrize("autocast_dtype", [torch.float16, torch.bfloat16])
    def test_float32_warp_under_autocast_keeps_the_border(self, autocast_dtype, device):
        # Under autocast the warp's homography matmuls ran in half precision, the coverage of completely covered
        # border pixels fell below the float32 tolerance, and they were dropped: black pixels in the new columns and
        # the panorama one column short (66, truth 67). The conftest skips tests named "autocast" on MPS, where the
        # same case applies.
        if not torch.amp.is_autocast_available(device.type):
            pytest.skip(f"autocast is not available on {device.type}")
        left, right = torch.ones(1, 3, 48, 64, device=device), torch.ones(1, 3, 48, 64, device=device)
        stitcher = kornia.contrib.ImageStitcher(_ShiftMatcher([3.0]))
        homography = torch.tensor([[[1.0, 0.0, 3.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device)
        stitcher.estimate_transform = lambda *args, **kwargs: homography  # exact, to isolate the resampling
        with torch.autocast(device_type=device.type, dtype=autocast_dtype), torch.no_grad():
            out = stitcher(left, right)
        assert out.shape[-1] == 64 + 3
        assert out[..., 64:].min().item() > 0.99999

    @pytest.mark.parametrize("estimator", ["ransac", "vanilla"])
    def test_smoke(self, estimator, device, dtype):
        B, C, H, W = 1, 3, 6, 6
        sample1 = torch.tensor(
            [
                [0.5349, 0.1988, 0.6592, 0.6569, 0.2328, 0.4251],
                [0.2071, 0.6297, 0.3653, 0.8513, 0.8549, 0.5509],
                [0.2868, 0.2063, 0.4451, 0.3593, 0.7204, 0.0731],
                [0.9699, 0.1078, 0.8829, 0.4132, 0.7572, 0.6948],
                [0.5209, 0.5932, 0.8797, 0.6286, 0.7653, 0.1132],
                [0.8559, 0.6721, 0.6267, 0.5691, 0.7437, 0.9592],
            ],
            dtype=dtype,
            device=device,
        )
        sample2 = torch.tensor(
            [
                [0.3887, 0.2214, 0.3742, 0.1953, 0.7405, 0.2529],
                [0.2332, 0.9314, 0.9575, 0.5575, 0.4134, 0.4355],
                [0.7369, 0.0331, 0.0914, 0.8994, 0.9936, 0.4703],
                [0.1049, 0.5137, 0.2674, 0.4990, 0.7447, 0.7213],
                [0.4414, 0.5550, 0.6361, 0.1081, 0.3305, 0.5196],
                [0.2147, 0.2816, 0.6679, 0.7878, 0.5070, 0.3055],
            ],
            dtype=dtype,
            device=device,
        )
        sample1 = sample1.expand((B, C, H, W))
        sample2 = sample2.expand((B, C, H, W))
        return_value = {
            "keypoints0": torch.tensor(
                [
                    [0.1546, 0.9391],
                    [0.8077, 0.1051],
                    [0.6768, 0.5596],
                    [0.5092, 0.7195],
                    [0.2856, 0.8889],
                    [0.4342, 0.0203],
                    [0.6701, 0.0585],
                    [0.3828, 0.9038],
                    [0.7301, 0.0762],
                    [0.7864, 0.4490],
                    [0.3509, 0.0756],
                    [0.6782, 0.9297],
                    [0.4132, 0.3664],
                    [0.3134, 0.5039],
                    [0.2073, 0.2552],
                ],
                device=device,
                dtype=dtype,
            ),
            "keypoints1": torch.tensor(
                [
                    [0.2076, 0.2669],
                    [0.9679, 0.8137],
                    [0.9536, 0.8317],
                    [0.3718, 0.2456],
                    [0.3875, 0.8450],
                    [0.7592, 0.1687],
                    [0.5173, 0.6760],
                    [0.9446, 0.4570],
                    [0.6164, 0.1867],
                    [0.4732, 0.1786],
                    [0.4090, 0.8089],
                    [0.9742, 0.8943],
                    [0.5996, 0.7427],
                    [0.7038, 0.9210],
                    [0.6272, 0.0796],
                ],
                device=device,
                dtype=dtype,
            ),
            "confidence": torch.tensor(
                [
                    0.9314,
                    0.5951,
                    0.4187,
                    0.0318,
                    0.1434,
                    0.7952,
                    0.8306,
                    0.7511,
                    0.6407,
                    0.7379,
                    0.4363,
                    0.9220,
                    0.8453,
                    0.5075,
                    0.8141,
                ],
                device=device,
                dtype=dtype,
            ),
            "batch_indexes": torch.zeros((15,), device=device, dtype=dtype),
        }
        with patch(
            "kornia.contrib.ImageStitcher.on_matcher", new_callable=PropertyMock, return_value=lambda x: return_value
        ):
            # NOTE: This will need to download the pretrained weights.
            # To avoid that, we mock as below
            matcher = kornia.feature.LoFTR(None)
            stitcher = kornia.contrib.ImageStitcher(matcher, estimator=estimator).to(device=device, dtype=dtype)
            torch.manual_seed(1)  # issue kornia#2027
            out = stitcher(sample1, sample2)
            assert out.shape[:-1] == torch.Size([1, 3, 6])
            assert out.shape[-1] <= 12

    @pytest.mark.parametrize("estimator", ["ransac", "vanilla"])
    def test_real_matcher_in_the_image_dtype(self, estimator, device, dtype):
        # A real matcher, not keypoints mocked in the test dtype: a float16 or bfloat16 SIFT matcher returns float32
        # keypoints and so a float32 homography, which the stitcher must warp the half-precision image with. The right
        # crop overlaps the left one by 64 px of a 96 x 224 scene, so the panorama is about 224 px wide.
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS has no float64")
        if not (supports_reflect_padding(device, dtype) and supports_replicate_padding(device, dtype)):
            pytest.skip(f"SIFTFeature pads with reflection and replication, absent for {dtype} on {device.type}")
        g = torch.Generator().manual_seed(0)
        scene = torch.nn.functional.avg_pool2d(torch.rand(1, 1, 96, 224, generator=g), 5, 1, 2).expand(-1, 3, -1, -1)
        left, right = scene[..., :144].to(device, dtype), scene[..., 80:].to(device, dtype)
        matcher = kornia.feature.LocalFeatureMatcher(
            kornia.feature.SIFTFeature(200), kornia.feature.DescriptorMatcher("snn", 0.8)
        ).to(device, dtype)
        stitcher = kornia.contrib.ImageStitcher(matcher, estimator=estimator)
        torch.manual_seed(0)  # RANSAC sampling
        with torch.no_grad():
            pano = stitcher(left, right)
            _, mask = stitcher.stitch_pair(left, right)
        assert pano.dtype == mask.dtype == dtype
        assert pano.shape[:3] == (1, 3, 96)
        assert 220 <= pano.shape[-1] <= 236
        assert bool(torch.isfinite(pano).all())
        width = min(pano.shape[-1], 224)
        assert (pano[..., :width].cpu().double() - scene[..., :width].double()).abs().median() < 0.05

    @pytest.mark.slow
    def test_exception(self, device, dtype):
        B, C, H, W = 1, 3, 224, 224
        sample1 = torch.rand(B, C, H, W, device=device, dtype=dtype)
        sample2 = torch.rand(B, C, H, W, device=device, dtype=dtype)
        # NOTE: This will need to download the pretrained weights.
        matcher = kornia.feature.LoFTR(None).eval()

        with pytest.raises(NotImplementedError):
            stitcher = kornia.contrib.ImageStitcher(matcher, estimator="random").to(device=device, dtype=dtype)

        stitcher = kornia.contrib.ImageStitcher(matcher).to(device=device, dtype=dtype)
        with pytest.raises(RuntimeError):
            stitcher(sample1, sample2)

    def test_convention_panorama_is_in_the_left_frame_and_the_right_image_wins_the_overlap(self, device, dtype):
        left, right = _render(0, 64).to(device, dtype), _render(40, 56).to(device, dtype)
        right[..., :24] = 0.25  # columns 40..63 of the panorama, where the left image also has content
        with torch.no_grad():
            out = kornia.contrib.ImageStitcher(_ShiftMatcher([40.0]))(left, right)
        assert out.shape == (1, 3, 48, 96)  # the left image's height; the empty trailing columns are cropped
        self.assert_close(out[..., :40], left[..., :40])
        self.assert_close(out[..., 40:64], torch.full_like(out[..., 40:64], 0.25))
        self.assert_close(out[..., 64:], right[..., 24:], rtol=2e-2, atol=2e-2)  # resampled through the homography

    def test_convention_a_batch_is_cropped_to_its_widest_sample(self, device):
        left = torch.cat([_render(0, 64)] * 2).to(device)
        right = torch.cat([_render(20, 56), _render(40, 56)]).to(device)
        with torch.no_grad():
            out = kornia.contrib.ImageStitcher(_ShiftMatcher([20.0, 40.0]))(left, right)
        assert out.shape == (2, 3, 48, 96)
        assert (out[0, ..., 76:] == 0).all()  # the narrower sample keeps empty trailing columns

    def test_convention_image_stitcher_postprocess_gap_5971(self, device, dtype):
        stitcher = kornia.contrib.ImageStitcher(nn.Identity())
        image = torch.arange(6, device=device, dtype=dtype).view(1, 1, 1, 6).expand(1, 3, 4, 6)
        gap = torch.tensor([1.0, 0, 1, 1, 0, 0], device=device, dtype=dtype).view(1, 1, 1, 6).expand(1, 3, 4, 6)
        # a gap in the coverage is kept: the crop ends at the last covered column
        assert stitcher.postprocess(image, gap).shape[-1] == 4
        empty_first = torch.tensor([0.0, 1, 1, 0, 0, 0], device=device, dtype=dtype).view(1, 1, 1, 6).expand(1, 3, 4, 6)
        # an uncovered first column is kept and the empty suffix is still dropped
        assert stitcher.postprocess(image, empty_first).shape[-1] == 3