Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

20.7 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import copy
import sys

import pytest
import torch
from torch import nn

from kornia.models.tiny_vit import Attention, TinyViT, urls

from testing.base import BaseTester


def _attentions(model: nn.Module) -> list[Attention]:
    return [m for m in model.modules() if isinstance(m, Attention)]


def _random_biases(model: TinyViT) -> TinyViT:
    # ``attention_biases`` starts as zeros, so a bias read from a stale copy would still be right. Seeded N(0, 1)
    # values make every relative-position bias table distinct from its initial value.
    generator = torch.Generator().manual_seed(0)
    with torch.no_grad():
        for attention in _attentions(model):
            attention.attention_biases.copy_(torch.randn(attention.attention_biases.shape, generator=generator))
    return model


def _modes(model: nn.Module) -> set[bool]:
    # The ``training`` flags of the model and every submodule: ``{False}`` is evaluation mode throughout.
    return {m.training for m in model.modules()}


def _fake_5m_checkpoint(url: str | list[str], *args: object, **kwargs: object) -> dict[str, torch.Tensor]:
    # Stands in for ``load_state_dict_from_url``. The ImageNet-22k checkpoints (``*_22k_distill.pth``) carry a
    # 21841-class head and the ImageNet-1k ones (``*_22kto1k_distill.pth``) a 1000-class head
    # (docs/source/models/tiny_vit.rst).
    candidates = [url] if isinstance(url, str) else url
    num_classes = 21841 if any("_22k_distill" in candidate for candidate in candidates) else 1000
    return TinyViT.from_config("5m", num_classes=num_classes).state_dict()


def _windows(model: TinyViT) -> list[int]:
    # The window size of each attention stage; stage 0 is convolutional.
    return [layer.blocks[0].window_size for layer in model.layers[1:]]


# TinyViT-21M's layout, from Cream tiny_vit_21m_224/_384/_512 (microsoft/Cream@8dc3882, TinyViT/models/tiny_vit.py).
_21M_LAYOUT = {"embed_dims": [96, 192, 384, 576], "depths": [2, 2, 6, 2], "num_heads": [3, 6, 12, 18]}


def _image(batch_size: int = 1, seed: int = 0) -> torch.Tensor:
    # TinyViT takes only its square ``img_size`` (224 by default).
    return torch.rand(batch_size, 3, 224, 224, generator=torch.Generator().manual_seed(seed))


class TestTinyViT(BaseTester):
    # 128 is not a multiple of the 7-pixel attention window at any stage, so it takes the padded
    # window-attention path; 224 divides evenly and takes the unpadded one. Both belong here: the
    # only other coverage of the exact-multiple path is marked slow, which PR CI skips.
    @pytest.mark.parametrize("img_size", [128, 224])
    def test_smoke(self, device, dtype, img_size):
        model = TinyViT(img_size=img_size).to(device=device, dtype=dtype)
        data = torch.randn(1, 3, img_size, img_size, device=device, dtype=dtype)

        out = model(data)
        assert isinstance(out, torch.Tensor)

    @pytest.mark.slow
    @pytest.mark.parametrize("num_classes", [10, 100])
    @pytest.mark.parametrize("batch_size", [1, 3])
    def test_cardinality(self, device, dtype, batch_size, num_classes):
        model = TinyViT(num_classes=num_classes).to(device=device, dtype=dtype)
        data = torch.rand(batch_size, 3, model.img_size, model.img_size, device=device, dtype=dtype)

        out = model(data)
        assert out.shape == (batch_size, num_classes)

    @pytest.mark.parametrize("img_size", [128, 224])
    def test_empty_batch_6266(self, device, dtype, img_size):
        """An empty batch goes through the padded and the unpadded window attention."""
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS does not support float64")
        model = TinyViT(img_size=img_size).to(device=device, dtype=dtype).eval()
        out = model(torch.rand(0, 3, img_size, img_size, device=device, dtype=dtype))
        assert out.shape == (0, 1000)

    @pytest.mark.skip("not implemented")
    def test_exception(self): ...

    @pytest.mark.skip("not implemented")
    def test_gradcheck(self): ...

    @pytest.mark.skip("not implemented")
    def test_module(self): ...

    @pytest.mark.skipif(sys.version_info.major == 3 and sys.version_info.minor == 8, reason="not working for py3.8")
    def test_dynamo(self, device, dtype, torch_optimizer, cudnn_tf32_follows_option):
        op = TinyViT().to(device=device, dtype=dtype)
        img = torch.rand(1, 3, op.img_size, op.img_size, device=device, dtype=dtype)

        op_optimized = torch_optimizer(op)
        # As in test_dynamo_eval_mode: inductor keeps fused half-precision intermediates in float32.
        tol = {torch.float16: 1e-2, torch.bfloat16: 5e-2}.get(dtype)
        self.assert_close(op(img), op_optimized(img), rtol=tol, atol=tol)

    @pytest.mark.slow
    @pytest.mark.parametrize("pretrained", [False, True])
    @pytest.mark.parametrize("variant", ["5m", "11m", "21m"])
    def test_from_config(self, variant, pretrained):
        model = TinyViT.from_config(variant, pretrained=pretrained)
        assert isinstance(model, TinyViT)

    @pytest.mark.slow
    @pytest.mark.parametrize("num_classes", [1000, 8])
    @pytest.mark.parametrize("img_size", [224, 256])
    def test_pretrained(self, img_size, num_classes):
        model = TinyViT.from_config("5m", img_size=img_size, num_classes=num_classes, pretrained=True)
        assert isinstance(model, TinyViT)

    @pytest.mark.slow
    def test_mobile_sam_backbone(self, device, dtype):
        img_size = 1024
        batch_size = 1
        model = TinyViT.from_config("5m", img_size=img_size, mobile_sam=True).to(device=device, dtype=dtype)
        data = torch.randn(batch_size, 3, img_size, img_size, device=device, dtype=dtype)

        out = model(data)
        assert out.shape == (batch_size, 256, img_size // 16, img_size // 16)

    @pytest.mark.device_agnostic
    def test_eval_bias_follows_load_state_dict(self):
        src = _random_biases(TinyViT.from_config("5m"))
        dst = TinyViT.from_config("5m").eval()
        x = _image()
        with torch.no_grad():
            dst(x)  # an evaluation-mode forward before the load, so a bias kept from it would go stale
        dst.load_state_dict(src.state_dict())
        src.eval()

        with torch.no_grad():
            assert torch.equal(dst(x), src(x))

    @pytest.mark.device_agnostic
    def test_eval_mode_bias_gets_gradients(self):
        model = _random_biases(TinyViT.from_config("5m")).eval()

        model(_image()).sum().backward()

        for attention in _attentions(model):
            assert attention.attention_biases.grad is not None
            assert attention.attention_biases.grad.abs().sum() > 0

    @pytest.mark.device_agnostic
    def test_eval_bias_follows_parameter_update(self):
        model = _random_biases(TinyViT.from_config("5m")).eval()
        x = _image()
        with torch.no_grad():
            y0 = model(x)
            # Doubling N(0, 1) biases changes the gaps between the logits of a row, which the softmax keeps; a
            # constant shift of every bias would cancel in the softmax and could not show a stale bias.
            for attention in _attentions(model):
                attention.attention_biases.mul_(2.0)
            y1 = model(x)

            fresh = TinyViT.from_config("5m")
            fresh.load_state_dict(model.state_dict())
            expected = fresh.eval()(x)

        self.assert_close(y1, expected)
        # The update moves the logits by 0.11 at most; float32 rounding alone moves them by about 4e-7.
        assert (y1 - y0).abs().max() > 1e-2

    def test_eval_then_cast_matches_cast_then_eval(self, device, dtype):
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS does not support float64")
        evaluated_first = _random_biases(TinyViT.from_config("5m"))
        cast_first = copy.deepcopy(evaluated_first)
        evaluated_first.eval().to(device, dtype)
        cast_first.to(device, dtype).eval()
        x = _image().to(device, dtype)

        with torch.no_grad():
            assert torch.equal(evaluated_first(x), cast_first(x))

    @pytest.mark.device_agnostic
    def test_from_config_mode(self, monkeypatch):
        assert _modes(TinyViT.from_config("5m")) == {True}
        assert _modes(TinyViT.from_config("5m", pretrained=False)) == {True}

        monkeypatch.setattr("kornia.models.tiny_vit.load_state_dict_from_url", _fake_5m_checkpoint)
        assert _modes(TinyViT.from_config("5m", pretrained=True)) == {False}
        assert _modes(TinyViT.from_config("5m", pretrained="in1k", num_classes=1000)) == {False}
        assert _modes(TinyViT.from_config("5m", pretrained="in22k", num_classes=21841)) == {False}

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize(
        ("pretrained", "num_classes"),
        [(True, 8), (True, 1001), ("in22k", 1000)],
        ids=["head-narrower", "head-wider", "in22k-default-classes"],
    )
    def test_from_config_partial_weights_mode(self, monkeypatch, pretrained, num_classes):
        monkeypatch.setattr("kornia.models.tiny_vit.load_state_dict_from_url", _fake_5m_checkpoint)

        # A class count other than the checkpoint's (1000 for in1k, 21841 for in22k) re-initialises the head: a
        # pretrained backbone to fine-tune, so training mode.
        with pytest.warns(UserWarning, match="Resetting classification head"):
            model = TinyViT.from_config("5m", pretrained=pretrained, num_classes=num_classes)
        assert model.head.out_features == num_classes
        assert not model.head.weight.any()
        assert not model.head.bias.any()
        assert _modes(model) == {True}

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize(
        ("img_size", "key", "window", "drop_path", "pretrained_values", "num_classes"),
        [
            # True picks in1k below 384 px; the in22k checkpoint has a 21841-class head and is only reached by name.
            (224, "in1k", 7, 0.2, (True, "in1k"), 1000),
            (224, "in22k", 7, 0.2, ("in22k",), 21841),
            (384, "in1k_384", 12, 0.1, (True, "in1k_384"), 1000),
            (512, "in1k_512", 16, 0.1, (True, "in1k_512"), 1000),
        ],
    )
    def test_21m_checkpoints_keep_their_windows(
        self, monkeypatch, img_size, key, window, drop_path, pretrained_values, num_classes
    ):
        # Cream builds tiny_vit_21m_224 with window_sizes [7, 7, 14, 7], tiny_vit_21m_384 with [12, 12, 24, 12] and
        # tiny_vit_21m_512 with [16, 16, 32, 16] (microsoft/Cream@8dc3882, TinyViT/models/tiny_vit.py); each checkpoint
        # holds the attention biases of its windows. The stand-in has that shape and N(0, 1) biases. The same builders
        # set drop_path_rate 0.2, 0.1 and 0.1, the rate of the 384 and 512 fine-tuning configurations.
        windows = [window, window, 2 * window, window]
        stand_in = TinyViT(img_size=img_size, window_sizes=windows, num_classes=num_classes, **_21M_LAYOUT)
        checkpoint = _random_biases(stand_in).state_dict()
        biases = {k: v.clone() for k, v in checkpoint.items() if "attention_biases" in k}
        fetched: list[str | list[str]] = []

        def fake_loader(url: str | list[str], *args: object, **kwargs: object) -> dict[str, torch.Tensor]:
            fetched.append(url)
            return dict(checkpoint)

        monkeypatch.setattr("kornia.models.tiny_vit.load_state_dict_from_url", fake_loader)

        for pretrained in pretrained_values:
            model = TinyViT.from_config("21m", pretrained=pretrained, img_size=img_size, num_classes=num_classes)
            assert fetched[-1] == urls["21m"][key]
            assert _windows(model) == windows[1:]
            # The stochastic depth rate rises linearly to drop_path_rate at the last block.
            assert model.layers[-1].blocks[-1].drop_path1.drop_prob == pytest.approx(drop_path)
            # Loaded as they are: interpolating them to other windows would change their shape.
            state_dict = model.state_dict()
            for name, bias in biases.items():
                assert torch.equal(state_dict[name], bias), name

        # An explicit ``window_sizes`` or ``drop_path_rate`` wins; the biases are then interpolated to the windows.
        model = TinyViT.from_config(
            "21m",
            pretrained=key,
            img_size=img_size,
            num_classes=num_classes,
            window_sizes=[8, 8, 16, 8],
            drop_path_rate=0.3,
        )
        assert _windows(model) == [8, 16, 8]
        assert model.layers[-1].blocks[-1].drop_path1.drop_prob == pytest.approx(0.3)
        # Random init has no checkpoint to follow and keeps the 224-px windows and rate.
        model = TinyViT.from_config("21m", img_size=img_size)
        assert _windows(model) == [7, 14, 7]
        assert model.layers[-1].blocks[-1].drop_path1.drop_prob == pytest.approx(0.2)

    @pytest.mark.device_agnostic
    @pytest.mark.parametrize(
        ("variant", "embed_dims", "num_heads"),
        # Each stage's dimension stays divisible by its number of heads, which TinyViTBlock checks; the defaults are
        # 5m [2, 4, 5, 10], 11m [2, 4, 8, 14] and 21m [3, 6, 12, 18] heads.
        [
            ("5m", [32, 64, 160, 320], [2, 4, 4, 8]),
            ("11m", [32, 64, 256, 448], [1, 2, 4, 7]),
            ("21m", [48, 96, 384, 576], [2, 4, 8, 12]),
        ],
    )
    def test_builder_arguments_can_be_overridden(self, variant, embed_dims, num_heads):
        model = TinyViT.from_config(
            variant,
            window_sizes=[8, 8, 16, 8],
            drop_path_rate=0.3,
            embed_dims=embed_dims,
            depths=[1, 1, 2, 1],
            num_heads=num_heads,
        )

        assert _windows(model) == [8, 16, 8]
        stage_dims = [model.layers[0].blocks[0].conv3.c.out_channels]
        stage_dims += [layer.blocks[0].attn.norm.normalized_shape[0] for layer in model.layers[1:]]
        assert stage_dims == embed_dims
        assert [len(layer.blocks) for layer in model.layers] == [1, 1, 2, 1]
        # Stage 0 is convolutional and has no heads.
        assert [layer.blocks[0].attn.num_heads for layer in model.layers[1:]] == num_heads[1:]
        # The stochastic depth rate rises linearly to drop_path_rate at the last block.
        assert model.layers[-1].blocks[-1].drop_path1.drop_prob == pytest.approx(0.3)

    @pytest.mark.parametrize(
        ("img_size", "mobile_sam", "size"),
        [
            (224, False, (224, 224)),
            (224, False, (220, 220)),
            (224, False, (217, 224)),
            (64, True, (64, 64)),
            (64, True, (57, 64)),
        ],
    )
    def test_forward_takes_sizes_of_its_token_grid(self, device, dtype, img_size, mobile_sam, size):
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS does not support float64")
        # The stem and the first patch merging downsample by 8 with ceil rounding, so sides from img_size - 7 to
        # img_size give the img_size // 8 grid the attention stages are built for, and their result is correct.
        model = TinyViT.from_config("5m", img_size=img_size, mobile_sam=mobile_sam).to(device, dtype).eval()

        with torch.no_grad():
            out = model(torch.rand(1, 3, *size, device=device, dtype=dtype))

        assert out.shape == ((1, 256, img_size // 16, img_size // 16) if mobile_sam else (1, 1000))

    @pytest.mark.parametrize(
        ("img_size", "mobile_sam", "size"),
        [
            (224, False, (112, 448)),
            (224, False, (448, 112)),
            (224, False, (216, 216)),
            (224, False, (225, 225)),
            (224, False, (224, 256)),
            (224, False, (256, 224)),
            (64, True, (32, 128)),
        ],
    )
    def test_forward_rejects_other_token_grids(self, device, dtype, img_size, mobile_sam, size):
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS does not support float64")
        # 112 x 448 and 448 x 112 give 14 x 56 and 56 x 14 grids of the 784 tokens of 28 x 28, which the stages would
        # read as 28 x 28; 32 x 128 gives MobileSAM's 4 x 16 grid of the 64 tokens of 8 x 8. 216 and 225 give 27 x 27
        # and 29 x 29; 224 x 256 and 256 x 224 have one side right (28 x 32 and 32 x 28).
        model = TinyViT.from_config("5m", img_size=img_size, mobile_sam=mobile_sam).to(device, dtype).eval()

        with torch.no_grad(), pytest.raises(ValueError, match=f"{img_size // 8}x{img_size // 8}"):
            model(torch.rand(1, 3, *size, device=device, dtype=dtype))

    @pytest.mark.parametrize(("img_size", "mobile_sam"), [(80, False), (240, False), (72, True)])
    def test_img_size_off_the_merging_grid_is_named_in_the_error(self, device, dtype, img_size, mobile_sam):
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS does not support float64")
        # The stride-2 convolutions round an odd grid up where the stages were built for it rounded down, so such an
        # img_size rejects its own img_size x img_size input: the error names the img_size rule.
        model = TinyViT.from_config("5m", img_size=img_size, mobile_sam=mobile_sam).to(device, dtype).eval()

        with (
            torch.no_grad(),
            pytest.raises(ValueError, match="img_size must be a multiple of 32, of 16 with mobile_sam"),
        ):
            model(torch.rand(1, 3, img_size, img_size, device=device, dtype=dtype))

    @pytest.mark.parametrize("mobile_sam", [False, True])
    def test_dynamo_fullgraph(self, device, dtype, torch_optimizer, mobile_sam, cudnn_tf32_follows_option):
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS does not support float64")
        # The token-grid check reads static shapes: it must compile without a graph break.
        op = _random_biases(TinyViT.from_config("5m", img_size=64, mobile_sam=mobile_sam)).eval().to(device, dtype)
        img = torch.rand(1, 3, 64, 64, device=device, dtype=dtype)

        op_optimized = torch_optimizer(op, fullgraph=True)
        # As in test_dynamo_eval_mode: inductor keeps fused half-precision intermediates in float32. MobileSAM's feature
        # map reaches about 4.7, where a bfloat16 ulp is 0.031: over 128 seeds on CUDA the largest difference was 0.080,
        # so that case gets an absolute 0.125 (four ulps there) instead of 5e-2 + 5e-2 * |value|.
        tol = {torch.float16: 1e-2, torch.bfloat16: 5e-2}.get(dtype)
        rtol, atol = (0.0, 0.125) if mobile_sam and dtype == torch.bfloat16 else (tol, tol)
        with torch.no_grad():
            self.assert_close(op(img), op_optimized(img), rtol=rtol, atol=atol)

    @pytest.mark.skipif(sys.version_info.major == 3 and sys.version_info.minor == 8, reason="not working for py3.8")
    def test_dynamo_eval_mode(self, device, dtype, torch_optimizer, cudnn_tf32_follows_option):
        if device.type == "mps" and dtype == torch.float64:
            pytest.skip("MPS does not support float64")
        # Evaluation mode first, then the cast: the order of a pretrained model moved to its device.
        op = _random_biases(TinyViT.from_config("5m")).eval().to(device, dtype)
        img = _image().to(device, dtype)

        op_optimized = torch_optimizer(op)
        # Inductor keeps fused intermediates in float32 where eager rounds every op to the half dtype. Over the network
        # the two differ on MPS by up to 1.1e-3 in float16 and 1.6e-2 (two bfloat16 steps at the logit scale of 2) in
        # bfloat16, past the default tolerances of 1e-3 and 7.8e-3.
        tol = {torch.float16: 1e-2, torch.bfloat16: 5e-2}.get(dtype)
        with torch.no_grad():
            self.assert_close(op(img), op_optimized(img), rtol=tol, atol=tol)

    @pytest.mark.slow
    def test_pretrained_is_batch_independent(self):
        model = TinyViT.from_config("5m", pretrained=True)
        running_means = [m.running_mean.clone() for m in model.modules() if isinstance(m, nn.BatchNorm2d)]
        x1, x2 = _image(seed=0), _image(seed=1)

        with torch.no_grad():
            alone = model(x1)[0]
            batched = model(torch.cat([x1, x2]))[0]

        # In evaluation mode sample 0 does not see sample 1; only the kernels' summation order may change with the
        # batch size, so the float32 tolerance is widened to 1e-4.
        self.assert_close(alone, batched, rtol=1e-4, atol=1e-4)
        after = [m.running_mean for m in model.modules() if isinstance(m, nn.BatchNorm2d)]
        assert all(torch.equal(a, b) for a, b in zip(after, running_means))