π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import annotations
import os
import pickle
import re
from unittest.mock import patch
import numpy as np
import pytest
import torch
from torch import nn
from kornia.contrib.edge_detection import EdgeDetectorBuilder
from kornia.io import ImageLoadType, load_image
from kornia.models.base import ModelBase, ModelBaseMixin
from testing.base import BaseTester
from testing.pickle_payload import CreatesMarkerOnLoad, load_without_running_payload
class DummyMixin(ModelBaseMixin):
name = "dummy"
class TestModelBaseMixinTensorToType:
def test_torch_output_returns_tensor_unchanged(self):
mixin = DummyMixin()
t = torch.rand(1, 3, 8, 8)
out = mixin._tensor_to_type(t, "torch")
assert out is t
def test_torch_output_returns_list_unchanged(self):
mixin = DummyMixin()
tensors = [torch.rand(1, 3, 8, 8), torch.rand(1, 3, 8, 8)]
out = mixin._tensor_to_type(tensors, "torch")
assert out is tensors
@staticmethod
def _assert_pil_images(images, tensors, modes):
"""Each image is one (C, H, W) slice, converted as ``ImageModuleMixIn.to_pil`` does."""
Image = pytest.importorskip("PIL.Image")
slices = [chw for t in tensors for chw in t.reshape(-1, *t.shape[-3:])]
assert isinstance(images, list)
assert len(images) == len(slices) == len(modes)
for image, chw, mode in zip(images, slices, modes):
assert isinstance(image, Image.Image)
assert image.mode == mode
# PIL sizes are (W, H)
assert image.size == (chw.shape[-1], chw.shape[-2])
# clamped to [0, 1], scaled to 0..255 and rounded; (H, W, 3) for "RGB", (H, W) for "L"
expected = (chw.clamp(0, 1) * 255).round().byte().permute(1, 2, 0)
if mode == "L":
expected = expected[..., 0]
assert np.array_equal(np.asarray(image), expected.numpy())
@pytest.mark.parametrize(
("shape", "modes"), [((3, 4, 5), ["RGB"]), ((1, 4, 5), ["L"]), ((2, 3, 4, 5), ["RGB", "RGB"])]
)
def test_pil_output_single_tensor(self, shape, modes):
mixin = DummyMixin()
# values beyond [0, 1] check the clamp
t = torch.rand(*shape) * 1.2 - 0.1
out = mixin._tensor_to_type(t, "pil")
self._assert_pil_images(out, [t], modes)
def test_pil_output_list_of_tensors(self):
mixin = DummyMixin()
tensors = [torch.rand(3, 4, 5), torch.rand(1, 4, 5), torch.rand(2, 3, 4, 5)]
out = mixin._tensor_to_type(tensors, "pil")
self._assert_pil_images(out, tensors, ["RGB", "L", "RGB", "RGB"])
def test_pil_output_2d_items(self):
mixin = DummyMixin()
# values beyond [0, 1] check the clamp
t = torch.rand(4, 5) * 1.2 - 0.1
# an (H, W) map converts as the same map with one channel, (1, H, W): one mode "L" image
self._assert_pil_images(mixin._tensor_to_type(t, "pil"), [t[None]], ["L"])
self._assert_pil_images(mixin._tensor_to_type([t, t[None]], "pil"), [t[None], t[None]], ["L", "L"])
def test_unsupported_output_type_raises(self):
mixin = DummyMixin()
t = torch.rand(1, 3, 8, 8)
with pytest.raises(RuntimeError, match=r"Output type.*is not supported"):
mixin._tensor_to_type(t, "numpy")
def _decoded_files(directory):
"""The decoded images of a directory, keyed by file name without its ``_YYYYmmdd_HHMMSS`` timestamp."""
return {
re.sub(r"_\d{8}_\d{6}", "", p.name): load_image(str(p), ImageLoadType.UNCHANGED)
for p in sorted(directory.iterdir())
}
class TestModelBaseWrappersPilOutput:
"""``output_type="pil"`` in a wrapper's ``visualize`` and ``save``, on a random-init ``EdgeDetector``."""
# edge maps in the forms EdgeDetector.visualize accepts: the (B, 1, H, W) tensor and a list of (1, H, W) maps,
# whose items it reduces to (H, W), and a list of (1, 1, H, W) maps as its forward returns, reduced to (3, H, W)
_FORMS = ("tensor (B, 1, H, W)", "list of (1, H, W)", "list of (1, 1, H, W)")
@staticmethod
def _edge_maps(form):
maps = torch.rand(2, 1, 8, 10)
if form == "list of (1, H, W)":
return list(maps)
if form == "list of (1, 1, H, W)":
return [m[None] for m in maps]
return maps
@pytest.mark.parametrize("form", _FORMS)
def test_edge_detector_visualize_pil(self, form):
Image = pytest.importorskip("PIL.Image")
detector = EdgeDetectorBuilder.build(pretrained=False)
images = torch.rand(2, 3, 8, 10)
edge_maps = self._edge_maps(form)
out = detector.visualize(images, edge_maps, output_type="pil")
drawn = detector.visualize(images, edge_maps)
assert isinstance(out, list)
assert len(out) == len(drawn) == 2
for image, tensor in zip(out, drawn):
assert isinstance(image, Image.Image)
# (H, W) items become mode "L", (3, H, W) items "RGB", each clamped, scaled to 0..255 and rounded
expected = (tensor.clamp(0, 1) * 255).round().byte()
expected = expected.permute(1, 2, 0) if expected.dim() == 3 else expected
assert image.mode == ("RGB" if tensor.dim() == 3 else "L")
assert np.array_equal(np.asarray(image), expected.numpy())
@pytest.mark.parametrize("form", _FORMS)
def test_edge_detector_save_pil_writes_the_torch_pixels(self, form, tmp_path):
pytest.importorskip("PIL.Image")
detector = EdgeDetectorBuilder.build(pretrained=False)
images = torch.rand(2, 3, 8, 10)
edge_maps = self._edge_maps(form)
for output_type in ("torch", "pil"):
detector.save(images, edge_maps, directory=str(tmp_path / output_type), output_type=output_type)
written = {output_type: _decoded_files(tmp_path / output_type) for output_type in ("torch", "pil")}
# source and edge map of each of the two images, under the same names and with the same pixels
assert len(written["torch"]) == 4
assert written["pil"].keys() == written["torch"].keys()
for name, decoded in written["torch"].items():
assert torch.equal(written["pil"][name], decoded), name
class TestModelBaseMixinSave:
def test_save_single_tensor_calls_write_image_once(self, tmp_path):
mixin = DummyMixin()
t = torch.rand(3, 8, 8)
with patch("kornia.models.base.write_image") as mock_write:
mixin.save(t, str(tmp_path))
assert mock_write.call_count == 1
# path is the first positional arg (matches write_image(path_file, image, ...))
saved_path = mock_write.call_args[0][0]
assert os.path.normcase(str(tmp_path)) in os.path.normcase(saved_path)
def test_save_list_of_tensors_calls_write_image_per_item(self, tmp_path):
mixin = DummyMixin()
tensors = [torch.rand(3, 8, 8), torch.rand(3, 8, 8), torch.rand(3, 8, 8)]
with patch("kornia.models.base.write_image") as mock_write:
mixin.save(tensors, str(tmp_path))
assert mock_write.call_count == 3
def test_save_creates_directory(self, tmp_path):
mixin = DummyMixin()
t = torch.rand(3, 8, 8)
new_dir = str(tmp_path / "new_subdir")
assert not os.path.exists(new_dir)
with patch("kornia.models.base.write_image"):
mixin.save(t, new_dir)
assert os.path.exists(new_dir)
class TestModelBaseMixinSaveOutputs:
def test_save_outputs_with_explicit_dir_single_tensor(self, tmp_path):
mixin = DummyMixin()
t = torch.rand(3, 8, 8)
with patch("kornia.models.base.write_image") as mock_write:
mixin._save_outputs(t, directory=str(tmp_path), suffix="_test")
assert mock_write.call_count == 1
# path is the first positional arg; verify suffix appears in filename
saved_path = mock_write.call_args[0][0]
assert "_test_" in saved_path
def test_save_outputs_with_explicit_dir_list(self, tmp_path):
mixin = DummyMixin()
tensors = [torch.rand(3, 8, 8), torch.rand(3, 8, 8)]
with patch("kornia.models.base.write_image") as mock_write:
mixin._save_outputs(tensors, directory=str(tmp_path))
assert mock_write.call_count == 2
def test_save_outputs_none_dir_creates_default(self, tmp_path, monkeypatch):
# Run from tmp_path so we don't pollute the repo
monkeypatch.chdir(tmp_path)
mixin = DummyMixin()
t = torch.rand(3, 8, 8)
with patch("kornia.models.base.write_image"):
mixin._save_outputs(t, directory=None)
kornia_outputs = tmp_path / "kornia_outputs"
assert kornia_outputs.exists()
subdirs = list(kornia_outputs.iterdir())
assert len(subdirs) == 1
assert subdirs[0].name.startswith("dummy")
class TestModelBaseMixinSaveWritesRealFiles:
"""`save` against the real `write_image`, not a mock.
Every other test in this file patches `write_image`, which is why #4322
shipped: the mock accepts any dtype and any rank, so nothing checked that
what the containers hand it is something it can actually write. `visualize`
returns float images and the containers document batched input, and
`write_image` accepts neither -- PNG is uint8/uint16 only, and it takes one
image: (3, H, W), (1, H, W), (H, W) or, for PNG, (4, H, W).
"""
@pytest.mark.parametrize("dtype", [torch.float32, torch.float64, torch.float16])
def test_a_float_visualization_is_written(self, tmp_path, dtype):
mixin = DummyMixin()
mixin.save(torch.rand(3, 8, 8, dtype=dtype), str(tmp_path))
written = list(tmp_path.iterdir())
assert len(written) == 1, f"expected one file, got {written}"
assert written[0].stat().st_size > 0
def test_a_batch_is_written_as_one_file_per_item(self, tmp_path):
"""The containers document (B, 3, H, W); write_image takes (3, H, W)."""
mixin = DummyMixin()
mixin.save(torch.rand(4, 3, 8, 8), str(tmp_path))
assert len(list(tmp_path.iterdir())) == 4
def test_a_batch_of_one_is_still_indexed(self, tmp_path):
"""So a caller does not have to guess between name.png and name_0.png."""
mixin = DummyMixin()
mixin.save(torch.rand(1, 3, 8, 8), str(tmp_path))
written = list(tmp_path.iterdir())
assert len(written) == 1
assert written[0].name.endswith("_0.png")
def test_the_written_file_is_a_readable_image(self, tmp_path):
"""Writing something unreadable would satisfy a file-count check."""
from kornia.io import ImageLoadType, load_image
mixin = DummyMixin()
mixin.save(torch.rand(3, 8, 8), str(tmp_path))
written = next(iter(tmp_path.iterdir()))
image = load_image(str(written), ImageLoadType.UNCHANGED)
assert image.shape == (3, 8, 8)
assert image.dtype == torch.uint8
def test_values_survive_the_conversion(self, tmp_path):
"""A float in [0, 1] must land on the matching 0-255 level."""
from kornia.io import ImageLoadType, load_image
mixin = DummyMixin()
source = torch.tensor([[[0.0, 1.0], [0.5, 0.25]]]).repeat(3, 1, 1)
mixin.save(source, str(tmp_path))
written = next(iter(tmp_path.iterdir()))
image = load_image(str(written), ImageLoadType.UNCHANGED)
assert image[0].tolist() == [[0, 255], [128, 64]]
def test_out_of_range_values_are_clamped_not_wrapped(self, tmp_path):
"""Without the clamp, 1.5 * 255 overflows uint8 and comes back dark."""
from kornia.io import ImageLoadType, load_image
mixin = DummyMixin()
source = torch.tensor([[[1.5, -0.5]]]).repeat(3, 1, 1)
mixin.save(source, str(tmp_path))
written = next(iter(tmp_path.iterdir()))
image = load_image(str(written), ImageLoadType.UNCHANGED)
assert image[0].tolist() == [[255, 0]]
def test_an_integer_image_is_passed_through(self, tmp_path):
from kornia.io import ImageLoadType, load_image
mixin = DummyMixin()
source = torch.tensor([[[0, 255], [7, 64]]], dtype=torch.uint8).repeat(3, 1, 1)
mixin.save(source, str(tmp_path))
written = next(iter(tmp_path.iterdir()))
image = load_image(str(written), ImageLoadType.UNCHANGED)
assert image[0].tolist() == [[0, 255], [7, 64]]
def test_save_outputs_writes_real_files_too(self, tmp_path):
mixin = DummyMixin()
mixin._save_outputs(torch.rand(2, 3, 8, 8), directory=str(tmp_path), suffix="_mask")
written = sorted(p.name for p in tmp_path.iterdir())
assert len(written) == 2
assert all("_mask_" in name for name in written)
class _TinyModel(ModelBase[None]):
def __init__(self) -> None:
super().__init__()
self.linear = nn.Linear(2, 3)
@staticmethod
def from_config(config: None) -> _TinyModel:
return _TinyModel()
class TestModelBaseLoadCheckpoint(BaseTester):
"""A local checkpoint is loaded as data, as a downloaded one is (``weights_only=True``)."""
def test_a_pickled_callable_in_a_local_file_is_refused_and_never_run(self, tmp_path) -> None:
marker = tmp_path / "marker"
path = tmp_path / "model.pth"
torch.save({**_TinyModel().state_dict(), "extra": CreatesMarkerOnLoad(marker)}, path)
model = _TinyModel()
with pytest.raises(pickle.UnpicklingError):
load_without_running_payload(marker, lambda: model.load_checkpoint(str(path)))
def test_a_home_relative_path_loads_the_local_file(self, tmp_path, monkeypatch, dtype) -> None:
source = _TinyModel().to(dtype)
torch.save(source.state_dict(), tmp_path / "x.pt")
monkeypatch.setenv("HOME", str(tmp_path))
monkeypatch.setenv("USERPROFILE", str(tmp_path))
model = _TinyModel().to(dtype)
model.load_checkpoint("~/x.pt")
self.assert_close(model.linear.weight, source.linear.weight)
self.assert_close(model.linear.bias, source.linear.bias)
def test_a_local_state_dict_loads(self, tmp_path, dtype) -> None:
source = _TinyModel().to(dtype)
path = tmp_path / "model.pth"
torch.save(source.state_dict(), path)
model = _TinyModel().to(dtype)
model.load_checkpoint(str(path))
self.assert_close(model.linear.weight, source.linear.weight)
self.assert_close(model.linear.bias, source.linear.bias)
def test_load_checkpoint_moves_the_model(self, tmp_path, device) -> None:
path = tmp_path / "model.pth"
torch.save(_TinyModel().state_dict(), path)
model = _TinyModel()
model.load_checkpoint(str(path), device=device)
assert {p.device for p in model.parameters()} == {device}
@pytest.mark.device_agnostic
def test_load_checkpoint_calls_to_with_the_device(self, tmp_path) -> None:
path = tmp_path / "model.pth"
torch.save(_TinyModel().state_dict(), path)
model = _TinyModel()
with patch.object(_TinyModel, "to", autospec=True, side_effect=nn.Module.to) as to:
model.load_checkpoint(str(path))
to.assert_not_called()
model.load_checkpoint(str(path), device=torch.device("cpu"))
assert to.call_count == 1
assert torch.device("cpu") in (*to.call_args.args[1:], *to.call_args.kwargs.values())
def test_load_checkpoint_keeps_the_mode(self, tmp_path) -> None:
path = tmp_path / "model.pth"
torch.save(_TinyModel().state_dict(), path)
# ``training`` of the model and of every submodule: ``{False}`` is evaluation mode throughout.
model = _TinyModel()
model.load_checkpoint(str(path))
assert {m.training for m in model.modules()} == {True}
model.eval()
model.load_checkpoint(str(path))
assert {m.training for m in model.modules()} == {False}