Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

16.4 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


import math
from unittest.mock import patch

import pytest
import torch
import torch.nn.functional as F

from kornia.augmentation import RandomAffine
from kornia.core.utils import _torch_grid_sample
from kornia.geometry import DepthWarper, PinholeCamera, warp_frame_depth
from kornia.geometry.grid import create_meshgrid
from kornia.geometry.transform import (
    HomographyWarper,
    elastic_transform2d,
    homography_warp,
    homography_warp3d,
    remap,
    rotate,
    warp_affine,
    warp_affine3d,
    warp_image_tps,
    warp_perspective,
    warp_perspective3d,
)
from kornia.geometry.transform.imgwarp import _runs_in_float32

from testing.base import BaseTester, supports_bilinear_3d_grid_sample


class TestHalfSampling(BaseTester):
    @staticmethod
    def _warp(image, operation, align_corners):
        batch, _, height, width = image.shape
        matrix = torch.eye(3, device=image.device, dtype=image.dtype)[None].expand(batch, -1, -1)
        options = {"align_corners": align_corners}
        if operation == "identity":
            return warp_affine(image, matrix[:, :2], (height, width), **options)
        if operation == "rotate":
            return rotate(image, image.new_tensor([25.0]), **options)
        if operation == "fill":
            matrix = matrix.clone()
            matrix[:, 0, 2] = 13.5
            return warp_affine(
                image,
                matrix[:, :2],
                (height, width),
                padding_mode="fill",
                fill_value=image.new_tensor([0.25, 0.5, 0.75]),
                **options,
            )
        if operation == "perspective":
            return warp_perspective(image, matrix, (height, width), **options)
        if operation == "remap":
            grid = create_meshgrid(height, width, device=image.device, dtype=image.dtype)
            return remap(
                image,
                grid[..., 0],
                grid[..., 1],
                normalized_coordinates=True,
                mode="nearest",
                padding_mode="reflection",
                **options,
            )
        if operation == "homography":
            return homography_warp(image, matrix, (height, width), padding_mode="border", **options)
        if operation == "cached_homography":
            warper = HomographyWarper(height, width, **options)
            warper.precompute_warp_grid(matrix)
            return warper(image)
        if operation == "elastic":
            noise = image.new_zeros(batch, 2, height, width)
            return elastic_transform2d(image, noise, kernel_size=(3, 3), sigma=(1.0, 1.0), **options)
        if operation == "tps":
            centers = image.new_tensor([[[-0.5, -0.5], [0.5, -0.5], [0.0, 0.5]]])
            weights = torch.zeros_like(centers)
            affine = image.new_tensor([[[0.0, 0.0], [1.0, 0.0], [0.0, 1.0]]])
            return warp_image_tps(image, centers, weights, affine, **options)
        if operation == "depth":
            return warp_frame_depth(
                image, torch.ones_like(image[:, :1]), torch.eye(4, device=image.device, dtype=image.dtype)[None], matrix
            )
        if operation == "depth_warper":
            identity = torch.eye(4, device=image.device, dtype=image.dtype)[None]
            camera = PinholeCamera(identity, identity.clone(), image.new_tensor([height]), image.new_tensor([width]))
            warper = DepthWarper(camera, height, width, **options)
            warper.compute_projection_matrix(camera)
            return warper(torch.ones_like(image[:, :1]), image)
        if operation == "augmentation":
            aug = RandomAffine(degrees=(25.0, 25.0), p=1.0, **options)
            return aug(image)
        raise AssertionError(operation)

    # These operations hand grid_sample a grid they built in the image dtype; only the sampling moves to float32.
    @pytest.mark.parametrize(
        "operation", ["homography", "cached_homography", "elastic", "tps", "depth", "depth_warper"]
    )
    @pytest.mark.parametrize("align_corners", [True, False])
    def test_float32_sampling_reference(self, device, dtype, operation, align_corners):
        if device.type != "cpu":
            pytest.skip("CPU sampling regression")
        image = ((torch.arange(3 * 256 * 256, device=device) % 251).float() / 251).reshape(1, 3, 256, 256).to(dtype)
        sample = F.grid_sample

        def reference(src, grid, **kwargs):
            work_dtype = torch.float64 if src.dtype == torch.float64 else torch.float32
            return sample(src.to(work_dtype), grid.to(work_dtype), **kwargs).to(src.dtype)

        with patch.object(F, "grid_sample", side_effect=reference):
            expected = self._warp(image, operation, align_corners)
        actual = self._warp(image, operation, align_corners)
        assert actual.dtype == dtype
        assert torch.isfinite(actual).all()
        self.assert_close(actual, expected, atol=0, rtol=0)

    @staticmethod
    def _float32_warp(image, operation, align_corners, param_dtype):
        # The transform is rounded to ``param_dtype`` first, so a float64 run of the same inputs is the reference.
        # Transforms and maps are built on the CPU in float64 and then moved, as MPS has no float64.
        batch, _, height, width = image.shape
        c = math.cos(math.radians(25.0))
        s = math.sin(math.radians(25.0))
        cx, cy = (width - 1) / 2, (height - 1) / 2
        rotation = (
            torch.tensor([[[c, -s, cx - c * cx + s * cy], [s, c, cy - s * cx - c * cy]]], dtype=torch.float64)
            .to(param_dtype)
            .to(image)
        )
        homography = torch.tensor([[[0.9, 0.05, 7.0], [-0.04, 1.1, -3.0], [1e-4, -2e-4, 1.0]]], dtype=torch.float64)
        homography = homography.to(param_dtype).to(image)
        options = {"align_corners": align_corners}
        if operation == "affine":
            return warp_affine(image, rotation, (height, width), **options)
        if operation == "affine_fill":
            fill = torch.tensor([0.25, 0.5, 0.75], dtype=torch.float64).to(param_dtype).to(image)
            return warp_affine(image, rotation, (height, width), padding_mode="fill", fill_value=fill, **options)
        if operation == "perspective":
            return warp_perspective(image, homography, (height, width), **options)
        if operation == "homography_pixel":
            return homography_warp(image, homography, (height, width), normalized_homography=False, **options)
        if operation == "remap_pixel":
            grid = create_meshgrid(height, width, False, dtype=torch.float64)
            points = torch.cat([grid, torch.ones_like(grid[..., :1])], -1) @ rotation.cpu().double()[0].T
            points = points.to(param_dtype).to(image)
            return remap(image, points[..., 0], points[..., 1], **options)
        if operation == "affine3d":
            volume = image[:, :1].reshape(batch, 1, 64, 32, 32)
            matrix = torch.tensor([[[c, -s, 0.0, 9.0], [s, c, 0.0, -6.0], [0.0, 0.0, 1.0, 1.5]]], dtype=torch.float64)
            return warp_affine3d(volume, matrix.to(param_dtype).to(image), (64, 32, 32), align_corners=align_corners)
        if operation == "perspective3d":
            volume = image[:, :1].reshape(batch, 1, 64, 32, 32)
            matrix = torch.eye(4, dtype=torch.float64)[None]
            matrix[:, :3, :3] = torch.tensor([[c, -s, 0.0], [s, c, 0.0], [0.0, 0.0, 1.0]], dtype=torch.float64)
            matrix[:, :3, 3] = torch.tensor([9.0, -6.0, 1.5], dtype=torch.float64)
            return warp_perspective3d(
                volume, matrix.to(param_dtype).to(image), (64, 32, 32), align_corners=align_corners
            )
        if operation == "homography3d":
            volume = image[:, :1].reshape(batch, 1, 64, 32, 32)
            matrix = torch.eye(4, dtype=torch.float64)[None]
            matrix[:, :3, :3] = torch.tensor([[c, -s, 0.0], [s, c, 0.0], [0.0, 0.0, 1.0]], dtype=torch.float64)
            matrix[:, :3, 3] = torch.tensor([0.1, -0.05, 0.02], dtype=torch.float64)
            return homography_warp3d(
                volume, matrix.to(param_dtype).to(image), (64, 32, 32), align_corners=align_corners
            )
        raise AssertionError(operation)

    @pytest.mark.parametrize(
        "operation",
        [
            "affine",
            "affine_fill",
            "perspective",
            "homography_pixel",
            "remap_pixel",
            "affine3d",
            "perspective3d",
            "homography3d",
        ],
    )
    @pytest.mark.parametrize("align_corners", [True, False])
    def test_half_warp_matches_float64(self, device, dtype, operation, align_corners):
        # torch's CPU half grid_sample is corrupt from about 128 px, and on every device a half grid is off by more
        # than half a pixel at 256 px in bfloat16 (#5751, #5814): the matrix warps and remap build their grid and
        # sample in float32, so they agree with a float64 run of the same half inputs to the output rounding. The
        # float64 reference runs on the CPU, as MPS has no float64.
        if dtype not in (torch.float16, torch.bfloat16):
            pytest.skip("float16/bfloat16 warps run in float32")
        if operation.endswith("3d") and not supports_bilinear_3d_grid_sample(device, torch.float32):
            pytest.skip("bilinear 3D grid_sample is unavailable for this device")
        image = ((torch.arange(3 * 256 * 256, device=device) % 251).float() / 251).reshape(1, 3, 256, 256).to(dtype)
        actual = self._float32_warp(image, operation, align_corners, dtype)
        expected = self._float32_warp(image.cpu().double(), operation, align_corners, dtype)
        assert actual.dtype == dtype
        self.assert_close(actual.cpu().double(), expected, rtol=0, atol=torch.finfo(dtype).eps)

    def test_half_augmentation_matches_float64(self, device, dtype):
        if dtype not in (torch.float16, torch.bfloat16):
            pytest.skip("float16/bfloat16 warps run in float32")
        image = ((torch.arange(3 * 256 * 256, device=device) % 251).float() / 251).reshape(1, 3, 256, 256).to(dtype)
        aug = RandomAffine(degrees=(25.0, 25.0), p=1.0)
        actual = aug(image)
        # The float64 reference runs on the CPU, as MPS has no float64.
        matrix = aug.transform_matrix[:, :2].cpu().double()
        expected = warp_affine(image.cpu().double(), matrix, (256, 256), align_corners=False)
        assert actual.dtype == dtype
        self.assert_close(actual.cpu().double(), expected, rtol=0, atol=torch.finfo(dtype).eps)

    @pytest.mark.parametrize("operation", ["affine", "perspective", "remap_pixel"])
    def test_cpu_half_warp_gradients_run_in_float32(self, device, dtype, operation):
        # Forward and backward both run in float32: the half gradients are the float32 ones, rounded.
        if device.type != "cpu" or dtype not in (torch.float16, torch.bfloat16):
            pytest.skip("CPU float16/bfloat16 warps run in float32")
        image = torch.linspace(0, 1, 3 * 160 * 144, device=device).reshape(1, 3, 160, 144).to(dtype)
        half_image = image.clone().requires_grad_()
        float_image = image.float().requires_grad_()
        actual = self._float32_warp(half_image, operation, True, dtype)
        expected = self._float32_warp(float_image, operation, True, dtype).to(dtype)
        self.assert_close(actual, expected, rtol=0, atol=0)
        (actual_grad,) = torch.autograd.grad(actual.float().square().sum(), half_image)
        (expected_grad,) = torch.autograd.grad(expected.float().square().sum(), float_image)
        assert actual_grad.dtype == dtype
        self.assert_close(actual_grad, expected_grad.to(dtype), rtol=0, atol=0)

    @pytest.mark.parametrize("mode", ["bilinear", "nearest"])
    @pytest.mark.parametrize("padding_mode", ["zeros", "border", "reflection"])
    @pytest.mark.parametrize("align_corners", [True, False])
    def test_sampling_gradients(self, device, dtype, mode, padding_mode, align_corners):
        if device.type != "cpu":
            pytest.skip("CPU sampling regression")
        image = torch.linspace(0, 1, 17 * 19, device=device, dtype=dtype).reshape(1, 1, 17, 19)
        image.requires_grad_()
        grid = create_meshgrid(13, 11, device=device, dtype=dtype).mul(1.2).add(0.03).requires_grad_()
        options = {"mode": mode, "padding_mode": padding_mode, "align_corners": align_corners}
        actual = remap(image, grid[..., 0], grid[..., 1], normalized_coordinates=True, **options)
        work_dtype = torch.float64 if dtype == torch.float64 else torch.float32
        expected = F.grid_sample(image.to(work_dtype), grid.to(work_dtype).contiguous(), **options).to(dtype)
        actual_grads = torch.autograd.grad(actual.sum(), (image, grid))
        expected_grads = torch.autograd.grad(expected.sum(), (image, grid))
        self.assert_close(actual, expected, atol=0, rtol=0)
        for actual_grad, expected_grad in zip(actual_grads, expected_grads):
            assert actual_grad.dtype == dtype
            assert torch.isfinite(actual_grad).all()
            self.assert_close(actual_grad, expected_grad, atol=0, rtol=0)

    @staticmethod
    def _rejects_mismatched_grid(device, dtype):
        # torch 2.5.1's MPS grid_sample samples a half input with a float32 grid instead of rejecting it.
        try:
            F.grid_sample(torch.zeros(1, 1, 2, 2, device=device, dtype=dtype), torch.zeros(1, 1, 1, 2, device=device))
        except RuntimeError:
            return True
        return False

    @pytest.mark.parametrize("size", [0, 4])
    @pytest.mark.parametrize("map_dtype", [torch.float32, torch.int64])
    def test_mismatched_grid_dtype(self, device, dtype, size, map_dtype):
        if dtype not in (torch.float16, torch.bfloat16):
            pytest.skip("half sampling dtype validation")
        if not self._rejects_mismatched_grid(device, dtype):
            pytest.skip("torch's grid_sample accepts a grid of another dtype on this device")
        image = torch.ones(1, 1, 4, 4, device=device, dtype=dtype)
        coords = torch.zeros(1, size, 4, device=device, dtype=map_dtype)
        with pytest.raises(RuntimeError, match=r"(dtype|scalar type|same type|grid_sampler_2d_cpu not implemented)"):
            remap(image, coords, coords, normalized_coordinates=True)

    def test_mismatched_matrix_dtype(self, device, dtype):
        # A float32 matrix next to a half image is rejected as before; only matching dtypes move to float32.
        if dtype not in (torch.float16, torch.bfloat16):
            pytest.skip("half sampling dtype validation")
        if not self._rejects_mismatched_grid(device, dtype):
            pytest.skip("torch's grid_sample accepts a grid of another dtype on this device")
        image = torch.ones(1, 1, 4, 4, device=device, dtype=dtype)
        matrix = torch.eye(3, device=device)[None]
        with pytest.raises(RuntimeError, match=r"(dtype|scalar type|same type|grid_sampler_2d_cpu not implemented)"):
            warp_affine(image, matrix[:, :2], (4, 4))
        with pytest.raises(RuntimeError, match=r"(dtype|scalar type|same type|grid_sampler_2d_cpu not implemented)"):
            warp_perspective(image, matrix, (4, 4))

    @pytest.mark.parametrize("half", [torch.float16, torch.bfloat16], ids=["float16", "bfloat16"])
    def test_every_device_samples_half_in_float32(self, half):
        # Half warps move to float32 on every device, not only on the CPU and CUDA legs CI runs (#5751, #5814).
        # The meta device stands in for the others, so every CI leg checks that the policy has no device gate.
        image = torch.zeros(1, 1, 4, 4, device="meta", dtype=half)
        grid = torch.zeros(1, 4, 4, 2, device="meta", dtype=half)
        assert _runs_in_float32(image, image[:, 0])
        seen = []

        def record(src, *args, **kwargs):
            seen.append(src.dtype)
            return src

        with patch.object(F, "grid_sample", side_effect=record):
            output = _torch_grid_sample(image, grid)
        assert seen == [torch.float32]
        assert output.dtype == half