Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

16.3 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import annotations

import os
import pickle
import re
from unittest.mock import patch

import numpy as np
import pytest
import torch
from torch import nn

from kornia.contrib.edge_detection import EdgeDetectorBuilder
from kornia.io import ImageLoadType, load_image
from kornia.models.base import ModelBase, ModelBaseMixin

from testing.base import BaseTester
from testing.pickle_payload import CreatesMarkerOnLoad, load_without_running_payload


class DummyMixin(ModelBaseMixin):
    name = "dummy"


class TestModelBaseMixinTensorToType:
    def test_torch_output_returns_tensor_unchanged(self):
        mixin = DummyMixin()
        t = torch.rand(1, 3, 8, 8)
        out = mixin._tensor_to_type(t, "torch")
        assert out is t

    def test_torch_output_returns_list_unchanged(self):
        mixin = DummyMixin()
        tensors = [torch.rand(1, 3, 8, 8), torch.rand(1, 3, 8, 8)]
        out = mixin._tensor_to_type(tensors, "torch")
        assert out is tensors

    @staticmethod
    def _assert_pil_images(images, tensors, modes):
        """Each image is one (C, H, W) slice, converted as ``ImageModuleMixIn.to_pil`` does."""
        Image = pytest.importorskip("PIL.Image")
        slices = [chw for t in tensors for chw in t.reshape(-1, *t.shape[-3:])]
        assert isinstance(images, list)
        assert len(images) == len(slices) == len(modes)
        for image, chw, mode in zip(images, slices, modes):
            assert isinstance(image, Image.Image)
            assert image.mode == mode
            # PIL sizes are (W, H)
            assert image.size == (chw.shape[-1], chw.shape[-2])
            # clamped to [0, 1], scaled to 0..255 and rounded; (H, W, 3) for "RGB", (H, W) for "L"
            expected = (chw.clamp(0, 1) * 255).round().byte().permute(1, 2, 0)
            if mode == "L":
                expected = expected[..., 0]
            assert np.array_equal(np.asarray(image), expected.numpy())

    @pytest.mark.parametrize(
        ("shape", "modes"), [((3, 4, 5), ["RGB"]), ((1, 4, 5), ["L"]), ((2, 3, 4, 5), ["RGB", "RGB"])]
    )
    def test_pil_output_single_tensor(self, shape, modes):
        mixin = DummyMixin()
        # values beyond [0, 1] check the clamp
        t = torch.rand(*shape) * 1.2 - 0.1
        out = mixin._tensor_to_type(t, "pil")
        self._assert_pil_images(out, [t], modes)

    def test_pil_output_list_of_tensors(self):
        mixin = DummyMixin()
        tensors = [torch.rand(3, 4, 5), torch.rand(1, 4, 5), torch.rand(2, 3, 4, 5)]
        out = mixin._tensor_to_type(tensors, "pil")
        self._assert_pil_images(out, tensors, ["RGB", "L", "RGB", "RGB"])

    def test_pil_output_2d_items(self):
        mixin = DummyMixin()
        # values beyond [0, 1] check the clamp
        t = torch.rand(4, 5) * 1.2 - 0.1
        # an (H, W) map converts as the same map with one channel, (1, H, W): one mode "L" image
        self._assert_pil_images(mixin._tensor_to_type(t, "pil"), [t[None]], ["L"])
        self._assert_pil_images(mixin._tensor_to_type([t, t[None]], "pil"), [t[None], t[None]], ["L", "L"])

    def test_unsupported_output_type_raises(self):
        mixin = DummyMixin()
        t = torch.rand(1, 3, 8, 8)
        with pytest.raises(RuntimeError, match=r"Output type.*is not supported"):
            mixin._tensor_to_type(t, "numpy")


def _decoded_files(directory):
    """The decoded images of a directory, keyed by file name without its ``_YYYYmmdd_HHMMSS`` timestamp."""
    return {
        re.sub(r"_\d{8}_\d{6}", "", p.name): load_image(str(p), ImageLoadType.UNCHANGED)
        for p in sorted(directory.iterdir())
    }


class TestModelBaseWrappersPilOutput:
    """``output_type="pil"`` in a wrapper's ``visualize`` and ``save``, on a random-init ``EdgeDetector``."""

    # edge maps in the forms EdgeDetector.visualize accepts: the (B, 1, H, W) tensor and a list of (1, H, W) maps,
    # whose items it reduces to (H, W), and a list of (1, 1, H, W) maps as its forward returns, reduced to (3, H, W)
    _FORMS = ("tensor (B, 1, H, W)", "list of (1, H, W)", "list of (1, 1, H, W)")

    @staticmethod
    def _edge_maps(form):
        maps = torch.rand(2, 1, 8, 10)
        if form == "list of (1, H, W)":
            return list(maps)
        if form == "list of (1, 1, H, W)":
            return [m[None] for m in maps]
        return maps

    @pytest.mark.parametrize("form", _FORMS)
    def test_edge_detector_visualize_pil(self, form):
        Image = pytest.importorskip("PIL.Image")
        detector = EdgeDetectorBuilder.build(pretrained=False)
        images = torch.rand(2, 3, 8, 10)
        edge_maps = self._edge_maps(form)

        out = detector.visualize(images, edge_maps, output_type="pil")

        drawn = detector.visualize(images, edge_maps)
        assert isinstance(out, list)
        assert len(out) == len(drawn) == 2
        for image, tensor in zip(out, drawn):
            assert isinstance(image, Image.Image)
            # (H, W) items become mode "L", (3, H, W) items "RGB", each clamped, scaled to 0..255 and rounded
            expected = (tensor.clamp(0, 1) * 255).round().byte()
            expected = expected.permute(1, 2, 0) if expected.dim() == 3 else expected
            assert image.mode == ("RGB" if tensor.dim() == 3 else "L")
            assert np.array_equal(np.asarray(image), expected.numpy())

    @pytest.mark.parametrize("form", _FORMS)
    def test_edge_detector_save_pil_writes_the_torch_pixels(self, form, tmp_path):
        pytest.importorskip("PIL.Image")
        detector = EdgeDetectorBuilder.build(pretrained=False)
        images = torch.rand(2, 3, 8, 10)
        edge_maps = self._edge_maps(form)

        for output_type in ("torch", "pil"):
            detector.save(images, edge_maps, directory=str(tmp_path / output_type), output_type=output_type)
        written = {output_type: _decoded_files(tmp_path / output_type) for output_type in ("torch", "pil")}

        # source and edge map of each of the two images, under the same names and with the same pixels
        assert len(written["torch"]) == 4
        assert written["pil"].keys() == written["torch"].keys()
        for name, decoded in written["torch"].items():
            assert torch.equal(written["pil"][name], decoded), name


class TestModelBaseMixinSave:
    def test_save_single_tensor_calls_write_image_once(self, tmp_path):
        mixin = DummyMixin()
        t = torch.rand(3, 8, 8)
        with patch("kornia.models.base.write_image") as mock_write:
            mixin.save(t, str(tmp_path))
            assert mock_write.call_count == 1
            # path is the first positional arg (matches write_image(path_file, image, ...))
            saved_path = mock_write.call_args[0][0]
            assert os.path.normcase(str(tmp_path)) in os.path.normcase(saved_path)

    def test_save_list_of_tensors_calls_write_image_per_item(self, tmp_path):
        mixin = DummyMixin()
        tensors = [torch.rand(3, 8, 8), torch.rand(3, 8, 8), torch.rand(3, 8, 8)]
        with patch("kornia.models.base.write_image") as mock_write:
            mixin.save(tensors, str(tmp_path))
            assert mock_write.call_count == 3

    def test_save_creates_directory(self, tmp_path):
        mixin = DummyMixin()
        t = torch.rand(3, 8, 8)
        new_dir = str(tmp_path / "new_subdir")
        assert not os.path.exists(new_dir)
        with patch("kornia.models.base.write_image"):
            mixin.save(t, new_dir)
        assert os.path.exists(new_dir)


class TestModelBaseMixinSaveOutputs:
    def test_save_outputs_with_explicit_dir_single_tensor(self, tmp_path):
        mixin = DummyMixin()
        t = torch.rand(3, 8, 8)
        with patch("kornia.models.base.write_image") as mock_write:
            mixin._save_outputs(t, directory=str(tmp_path), suffix="_test")
            assert mock_write.call_count == 1
            # path is the first positional arg; verify suffix appears in filename
            saved_path = mock_write.call_args[0][0]
            assert "_test_" in saved_path

    def test_save_outputs_with_explicit_dir_list(self, tmp_path):
        mixin = DummyMixin()
        tensors = [torch.rand(3, 8, 8), torch.rand(3, 8, 8)]
        with patch("kornia.models.base.write_image") as mock_write:
            mixin._save_outputs(tensors, directory=str(tmp_path))
            assert mock_write.call_count == 2

    def test_save_outputs_none_dir_creates_default(self, tmp_path, monkeypatch):
        # Run from tmp_path so we don't pollute the repo
        monkeypatch.chdir(tmp_path)
        mixin = DummyMixin()
        t = torch.rand(3, 8, 8)
        with patch("kornia.models.base.write_image"):
            mixin._save_outputs(t, directory=None)
        kornia_outputs = tmp_path / "kornia_outputs"
        assert kornia_outputs.exists()
        subdirs = list(kornia_outputs.iterdir())
        assert len(subdirs) == 1
        assert subdirs[0].name.startswith("dummy")


class TestModelBaseMixinSaveWritesRealFiles:
    """`save` against the real `write_image`, not a mock.

    Every other test in this file patches `write_image`, which is why #4322
    shipped: the mock accepts any dtype and any rank, so nothing checked that
    what the containers hand it is something it can actually write. `visualize`
    returns float images and the containers document batched input, and
    `write_image` accepts neither -- PNG is uint8/uint16 only, and it takes one
    image: (3, H, W), (1, H, W), (H, W) or, for PNG, (4, H, W).
    """

    @pytest.mark.parametrize("dtype", [torch.float32, torch.float64, torch.float16])
    def test_a_float_visualization_is_written(self, tmp_path, dtype):
        mixin = DummyMixin()
        mixin.save(torch.rand(3, 8, 8, dtype=dtype), str(tmp_path))
        written = list(tmp_path.iterdir())
        assert len(written) == 1, f"expected one file, got {written}"
        assert written[0].stat().st_size > 0

    def test_a_batch_is_written_as_one_file_per_item(self, tmp_path):
        """The containers document (B, 3, H, W); write_image takes (3, H, W)."""
        mixin = DummyMixin()
        mixin.save(torch.rand(4, 3, 8, 8), str(tmp_path))
        assert len(list(tmp_path.iterdir())) == 4

    def test_a_batch_of_one_is_still_indexed(self, tmp_path):
        """So a caller does not have to guess between name.png and name_0.png."""
        mixin = DummyMixin()
        mixin.save(torch.rand(1, 3, 8, 8), str(tmp_path))
        written = list(tmp_path.iterdir())
        assert len(written) == 1
        assert written[0].name.endswith("_0.png")

    def test_the_written_file_is_a_readable_image(self, tmp_path):
        """Writing something unreadable would satisfy a file-count check."""
        from kornia.io import ImageLoadType, load_image

        mixin = DummyMixin()
        mixin.save(torch.rand(3, 8, 8), str(tmp_path))
        written = next(iter(tmp_path.iterdir()))
        image = load_image(str(written), ImageLoadType.UNCHANGED)
        assert image.shape == (3, 8, 8)
        assert image.dtype == torch.uint8

    def test_values_survive_the_conversion(self, tmp_path):
        """A float in [0, 1] must land on the matching 0-255 level."""
        from kornia.io import ImageLoadType, load_image

        mixin = DummyMixin()
        source = torch.tensor([[[0.0, 1.0], [0.5, 0.25]]]).repeat(3, 1, 1)
        mixin.save(source, str(tmp_path))
        written = next(iter(tmp_path.iterdir()))
        image = load_image(str(written), ImageLoadType.UNCHANGED)
        assert image[0].tolist() == [[0, 255], [128, 64]]

    def test_out_of_range_values_are_clamped_not_wrapped(self, tmp_path):
        """Without the clamp, 1.5 * 255 overflows uint8 and comes back dark."""
        from kornia.io import ImageLoadType, load_image

        mixin = DummyMixin()
        source = torch.tensor([[[1.5, -0.5]]]).repeat(3, 1, 1)
        mixin.save(source, str(tmp_path))
        written = next(iter(tmp_path.iterdir()))
        image = load_image(str(written), ImageLoadType.UNCHANGED)
        assert image[0].tolist() == [[255, 0]]

    def test_an_integer_image_is_passed_through(self, tmp_path):
        from kornia.io import ImageLoadType, load_image

        mixin = DummyMixin()
        source = torch.tensor([[[0, 255], [7, 64]]], dtype=torch.uint8).repeat(3, 1, 1)
        mixin.save(source, str(tmp_path))
        written = next(iter(tmp_path.iterdir()))
        image = load_image(str(written), ImageLoadType.UNCHANGED)
        assert image[0].tolist() == [[0, 255], [7, 64]]

    def test_save_outputs_writes_real_files_too(self, tmp_path):
        mixin = DummyMixin()
        mixin._save_outputs(torch.rand(2, 3, 8, 8), directory=str(tmp_path), suffix="_mask")
        written = sorted(p.name for p in tmp_path.iterdir())
        assert len(written) == 2
        assert all("_mask_" in name for name in written)


class _TinyModel(ModelBase[None]):
    def __init__(self) -> None:
        super().__init__()
        self.linear = nn.Linear(2, 3)

    @staticmethod
    def from_config(config: None) -> _TinyModel:
        return _TinyModel()


class TestModelBaseLoadCheckpoint(BaseTester):
    """A local checkpoint is loaded as data, as a downloaded one is (``weights_only=True``)."""

    def test_a_pickled_callable_in_a_local_file_is_refused_and_never_run(self, tmp_path) -> None:
        marker = tmp_path / "marker"
        path = tmp_path / "model.pth"
        torch.save({**_TinyModel().state_dict(), "extra": CreatesMarkerOnLoad(marker)}, path)

        model = _TinyModel()
        with pytest.raises(pickle.UnpicklingError):
            load_without_running_payload(marker, lambda: model.load_checkpoint(str(path)))

    def test_a_home_relative_path_loads_the_local_file(self, tmp_path, monkeypatch, dtype) -> None:
        source = _TinyModel().to(dtype)
        torch.save(source.state_dict(), tmp_path / "x.pt")
        monkeypatch.setenv("HOME", str(tmp_path))
        monkeypatch.setenv("USERPROFILE", str(tmp_path))

        model = _TinyModel().to(dtype)
        model.load_checkpoint("~/x.pt")

        self.assert_close(model.linear.weight, source.linear.weight)
        self.assert_close(model.linear.bias, source.linear.bias)

    def test_a_local_state_dict_loads(self, tmp_path, dtype) -> None:
        source = _TinyModel().to(dtype)
        path = tmp_path / "model.pth"
        torch.save(source.state_dict(), path)

        model = _TinyModel().to(dtype)
        model.load_checkpoint(str(path))

        self.assert_close(model.linear.weight, source.linear.weight)
        self.assert_close(model.linear.bias, source.linear.bias)

    def test_load_checkpoint_moves_the_model(self, tmp_path, device) -> None:
        path = tmp_path / "model.pth"
        torch.save(_TinyModel().state_dict(), path)

        model = _TinyModel()
        model.load_checkpoint(str(path), device=device)

        assert {p.device for p in model.parameters()} == {device}

    @pytest.mark.device_agnostic
    def test_load_checkpoint_calls_to_with_the_device(self, tmp_path) -> None:
        path = tmp_path / "model.pth"
        torch.save(_TinyModel().state_dict(), path)
        model = _TinyModel()

        with patch.object(_TinyModel, "to", autospec=True, side_effect=nn.Module.to) as to:
            model.load_checkpoint(str(path))
            to.assert_not_called()

            model.load_checkpoint(str(path), device=torch.device("cpu"))
            assert to.call_count == 1
            assert torch.device("cpu") in (*to.call_args.args[1:], *to.call_args.kwargs.values())

    def test_load_checkpoint_keeps_the_mode(self, tmp_path) -> None:
        path = tmp_path / "model.pth"
        torch.save(_TinyModel().state_dict(), path)

        # ``training`` of the model and of every submodule: ``{False}`` is evaluation mode throughout.
        model = _TinyModel()
        model.load_checkpoint(str(path))
        assert {m.training for m in model.modules()} == {True}

        model.eval()
        model.load_checkpoint(str(path))
        assert {m.training for m in model.modules()} == {False}