π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pickle
import pytest
import torch
from kornia.core._compat import torch_version_lt
from kornia.models.efficient_vit import EfficientViT, EfficientViTConfig
from kornia.models.efficient_vit import backbone as vit
class _NotAWeight:
"""An arbitrary class, which ``torch.load(..., weights_only=True)`` refuses to unpickle."""
class TestEfficientViT:
@staticmethod
def _fake_checkpoint() -> dict[str, torch.Tensor]:
"""Mimic the hosted checkpoints: backbone weights under a ``backbone.`` prefix plus a ``head.`` classifier."""
model = vit.efficientvit_backbone_b1()
state_dict = {
f"backbone.{key}": val.clone()
for key, val in model.state_dict().items()
if "num_batches_tracked" not in key
}
state_dict["head.classifier.weight"] = torch.randn(1000, 128)
state_dict["head.classifier.bias"] = torch.randn(1000)
return state_dict
def test_load_pretrained_loads_the_checkpoint_weights(self, monkeypatch):
# GH#5276: strict=False over the raw checkpoint silently loaded no weight at all
state_dict = self._fake_checkpoint()
monkeypatch.setattr(
"kornia.models.efficient_vit.model.load_state_dict_from_url",
lambda *args, **kwargs: {"state_dict": state_dict},
)
model = EfficientViT.from_config(EfficientViTConfig())
for key, val in state_dict.items():
if key.startswith("backbone."):
assert torch.equal(model.backbone.state_dict()[key[len("backbone.") :]], val)
def test_load_local_checkpoint(self, tmp_path, monkeypatch):
state_dict = self._fake_checkpoint()
checkpoint = tmp_path / "b1-local.pt"
torch.save(state_dict, checkpoint)
monkeypatch.setattr(
"kornia.models.efficient_vit.model.load_state_dict_from_url",
lambda *args, **kwargs: pytest.fail("local checkpoints must not be downloaded"),
)
model = EfficientViT.from_config(EfficientViTConfig(checkpoint=str(checkpoint)))
for key, val in state_dict.items():
if key.startswith("backbone."):
assert torch.equal(model.backbone.state_dict()[key[len("backbone.") :]], val)
def test_load_local_checkpoint_expands_user(self, tmp_path, monkeypatch):
# a ``~`` path must not reach the URL loader, which resolves a name already in the hub cache to that file
state_dict = self._fake_checkpoint()
torch.save(state_dict, tmp_path / "b1-r224.pt")
monkeypatch.setenv("HOME", str(tmp_path))
monkeypatch.setenv("USERPROFILE", str(tmp_path))
monkeypatch.setattr(
"kornia.models.efficient_vit.model.load_state_dict_from_url",
lambda *args, **kwargs: pytest.fail("local checkpoints must not be downloaded"),
)
model = EfficientViT.from_config(EfficientViTConfig(checkpoint="~/b1-r224.pt"))
key = "input_stem.op_list.0.conv.weight"
assert torch.equal(model.backbone.state_dict()[key], state_dict[f"backbone.{key}"])
def test_load_local_checkpoint_is_weights_only(self, tmp_path, monkeypatch):
# a local file is unpickled with weights_only=True, as the URL path and ModelBase.load_checkpoint do
checkpoint = tmp_path / "b1-local.pt"
torch.save({"payload": _NotAWeight()}, checkpoint)
monkeypatch.setattr(
"kornia.models.efficient_vit.model.load_state_dict_from_url",
lambda *args, **kwargs: pytest.fail("local checkpoints must not be downloaded"),
)
with pytest.raises(pickle.UnpicklingError):
EfficientViT.from_config(EfficientViTConfig(checkpoint=str(checkpoint)))
def test_load_failure_preserves_cause(self, monkeypatch):
cause = RuntimeError("download failed")
def fail_download(*args, **kwargs):
raise cause
monkeypatch.setattr("kornia.models.efficient_vit.model.load_state_dict_from_url", fail_download)
with pytest.raises(RuntimeError, match="Unable to load the model") as exc_info:
EfficientViT.from_config(EfficientViTConfig())
assert exc_info.value.__cause__ is cause
@pytest.mark.parametrize("drift", ["missing", "unexpected"])
def test_load_pretrained_is_strict(self, monkeypatch, drift):
# a checkpoint that lacks a backbone entry, or holds one the backbone has no slot for, raises
state_dict = self._fake_checkpoint()
if drift == "missing":
del state_dict["backbone.input_stem.op_list.0.conv.weight"]
else:
state_dict["backbone.input_stem.extra.weight"] = torch.zeros(1)
monkeypatch.setattr(
"kornia.models.efficient_vit.model.load_state_dict_from_url",
lambda *args, **kwargs: {"state_dict": state_dict},
)
with pytest.raises(RuntimeError, match=f"{drift.capitalize()} key"):
EfficientViT.from_config(EfficientViTConfig())
def test_load_pretrained_accepts_a_backbone_state_dict(self, monkeypatch):
# a state dict saved from the backbone itself has no "backbone." prefix
state_dict = {key: val + 1 for key, val in vit.efficientvit_backbone_b1().state_dict().items()}
monkeypatch.setattr(
"kornia.models.efficient_vit.model.load_state_dict_from_url", lambda *args, **kwargs: state_dict
)
model = EfficientViT.from_config(EfficientViTConfig())
loaded = model.backbone.state_dict()
assert loaded.keys() == state_dict.keys()
for key, val in state_dict.items():
assert torch.equal(loaded[key], val)
def test_load_pretrained_returns_eval_mode(self, monkeypatch):
state_dict = self._fake_checkpoint()
monkeypatch.setattr(
"kornia.models.efficient_vit.model.load_state_dict_from_url",
lambda *args, **kwargs: {"state_dict": state_dict},
)
model = EfficientViT.from_config(EfficientViTConfig.from_pretrained("b1", 224))
# ``training`` of the model and of every submodule: ``{False}`` is evaluation mode throughout.
assert {m.training for m in model.modules()} == {False}
@pytest.mark.slow
def test_load_pretrained_is_batch_independent(self):
model = EfficientViT.from_config(EfficientViTConfig.from_pretrained("b1", 224))
running_means = [m.running_mean.clone() for m in model.modules() if isinstance(m, torch.nn.BatchNorm2d)]
generator = torch.Generator().manual_seed(0)
x1 = torch.rand(1, 3, 224, 256, generator=generator)
x2 = torch.rand(1, 3, 224, 256, generator=generator)
with torch.no_grad():
alone = model(x1)["stage_final"][0]
batched = model(torch.cat([x1, x2]))["stage_final"][0]
# In evaluation mode sample 0 does not see sample 1; only the kernels' summation order may change with the
# batch size, so the float32 tolerance is widened to 1e-4.
torch.testing.assert_close(alone, batched, rtol=1e-4, atol=1e-4)
after = [m.running_mean for m in model.modules() if isinstance(m, torch.nn.BatchNorm2d)]
assert all(torch.equal(a, b) for a, b in zip(after, running_means))
def _test_smoke(self, device, dtype, img_size: int, expected_resolution: int, model_name: str):
model = getattr(vit, f"efficientvit_backbone_{model_name}")()
model = model.to(device=device, dtype=dtype)
image = torch.randn(1, 3, img_size, img_size, device=device, dtype=dtype)
out = model(image)
assert "input" in out
assert out["input"].shape == image.shape
assert "stage_final" in out
assert out["stage_final"].shape[-2:] == torch.Size([expected_resolution, expected_resolution])
@pytest.mark.parametrize("model_name", ["b3"])
@pytest.mark.parametrize("img_size,expected_resolution", [(224, 7), (256, 8), (288, 9)])
@pytest.mark.slow
def test_smoke_slow(self, device, dtype, img_size: int, expected_resolution: int, model_name: str):
self._test_smoke(device, dtype, img_size, expected_resolution, model_name)
@pytest.mark.parametrize("model_name", ["b0", "b1", "b2"])
@pytest.mark.parametrize("img_size,expected_resolution", [(64, 2), (96, 3), (128, 4)])
def test_smoke(self, device, dtype, img_size: int, expected_resolution: int, model_name: str):
self._test_smoke(device, dtype, img_size, expected_resolution, model_name)
@pytest.mark.parametrize("model_name", ["b0", "l0"])
def test_empty_batch_6266(self, device, dtype, model_name: str):
"""An empty batch goes through the linear attention and keeps every stage's channels and size."""
if device.type == "mps" and dtype == torch.float64:
pytest.skip("MPS does not support float64")
model = getattr(vit, f"efficientvit_backbone_{model_name}")().to(device, dtype).eval()
feats = model(torch.rand(0, 3, 64, 64, device=device, dtype=dtype))
ref = model(torch.rand(1, 3, 64, 64, device=device, dtype=dtype))
assert {k: v.shape[1:] for k, v in feats.items()} == {k: v.shape[1:] for k, v in ref.items()}
assert all(v.shape[0] == 0 for v in feats.values())
@pytest.mark.slow
@pytest.mark.skipif(torch_version_lt(2, 0, 0), reason="requires torch 2.0.0 or higher")
@pytest.mark.parametrize("model_name", ["l0", "l1", "l2", "l3"])
@pytest.mark.parametrize("img_size,expected_resolution", [(224, 7), (256, 8), (288, 9), (320, 10), (384, 12)])
def test_smoke_large(self, device, dtype, img_size: int, expected_resolution: int, model_name: str):
self._test_smoke(device, dtype, img_size, expected_resolution, model_name)
@pytest.mark.slow
def test_load_pretrained(self, device, dtype):
model = EfficientViT.from_config(EfficientViTConfig())
model = model.to(device=device, dtype=dtype)
image = torch.randn(1, 3, 224, 224, device=device, dtype=dtype)
feats = model(image)
assert feats["stage_final"].shape == torch.Size([1, 256, 7, 7])
@pytest.mark.parametrize("model_type", ["b1", "b2", "b3"])
@pytest.mark.parametrize("resolution", [224, 256, 288])
def test_config(self, model_type, resolution):
config = EfficientViTConfig.from_pretrained(model_type, resolution)
assert model_type in config.checkpoint
assert str(resolution) in config.checkpoint
def test_forward_returns_a_dict(self, device, dtype):
if device.type == "mps" and dtype == torch.float64:
pytest.skip("MPS does not support float64")
model = EfficientViT(backbone=vit.efficientvit_backbone_b0()).to(device, dtype).eval()
image = torch.rand(1, 3, 64, 96, device=device, dtype=dtype)
with torch.no_grad():
out = model(image)
assert isinstance(out, dict)
# stride 32: (64 // 32, 96 // 32)
assert out["stage_final"].shape[-2:] == (2, 3)