Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(models): accept empty batches where reshape/view(B, -1) flattened the batch (#6266) (#6350)

Viranshul-Pefe43c1d0272
diff --git a/changelog.d/6350.fixed.md b/changelog.d/6350.fixed.md
new file mode 100644
index 000000000..bded5936a
--- /dev/null
+++ b/changelog.d/6350.fixed.md
@@ -0,0 +1,4 @@
+Models that reshaped a batch with a `-1` next to the batch dimension now accept empty batches instead of raising
+"cannot reshape tensor of 0 elements" (#6266): `TinyViT`, `VisionTransformer`, `YuNet` and `FaceDetector`, the SAM
+image encoder and mask decoder (an image with no prompts), the EfficientViT backbones, `MoonViT` and `KimiVLModel`, and
+`get_intermediate_layers(reshape=True)` of DeDoDe's DINOv2 backbone.
diff --git a/kornia/feature/dedode/transformer/dinov2.py b/kornia/feature/dedode/transformer/dinov2.py
index 3fdd13d8a..ad8161874 100644
--- a/kornia/feature/dedode/transformer/dinov2.py
+++ b/kornia/feature/dedode/transformer/dinov2.py
@@ -339,7 +339,9 @@ class DinoVisionTransformer(nn.Module):
if reshape:
B, _, w, h = x.shape
outputs = [
- out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous()
+ out.reshape(B, w // self.patch_size, h // self.patch_size, out.shape[-1])
+ .permute(0, 3, 1, 2)
+ .contiguous()
for out in outputs
]
if return_class_token:
diff --git a/kornia/models/common.py b/kornia/models/common.py
index 11a36a9ed..ded17dc03 100644
--- a/kornia/models/common.py
+++ b/kornia/models/common.py
@@ -208,7 +208,7 @@ def window_partition(x: torch.Tensor, window_size: int) -> tuple[torch.Tensor, t
Hp, Wp = H + pad_h, W + pad_w
x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C)
- windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+ windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().flatten(0, 2)
return windows, (Hp, Wp)
@@ -230,8 +230,9 @@ def window_unpartition(
Hp, Wp = pad_hw
H, W = hw
B = windows.shape[0] // (Hp * Wp // window_size // window_size)
- x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1)
- x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1)
+ C = windows.shape[-1]
+ x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, C)
+ x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, C)
if Hp > H or Wp > W:
x = x[:, :H, :W, :].contiguous()
diff --git a/kornia/models/efficient_vit/nn/ops.py b/kornia/models/efficient_vit/nn/ops.py
index d95a8df3a..10d10038b 100644
--- a/kornia/models/efficient_vit/nn/ops.py
+++ b/kornia/models/efficient_vit/nn/ops.py
@@ -469,14 +469,14 @@ class LiteMLA(nn.Module):
@torch.amp.autocast(device_type="cuda", enabled=False)
def relu_linear_att(self, qkv: torch.Tensor) -> torch.Tensor:
- B, _, H, W = list(qkv.size())
+ _, _, H, W = list(qkv.size())
dtype = qkv.dtype
if qkv.dtype == torch.float16:
qkv = qkv.float()
- qkv = torch.reshape(qkv, (B, -1, 3 * self.dim, H * W))
+ qkv = qkv.unflatten(1, (-1, 3 * self.dim)).flatten(-2)
qkv = torch.transpose(qkv, -1, -2)
q, k, v = (qkv[..., 0 : self.dim], qkv[..., self.dim : 2 * self.dim], qkv[..., 2 * self.dim :])
@@ -493,7 +493,7 @@ class LiteMLA(nn.Module):
out = out[..., :-1] / (out[..., -1:] + self.eps)
out = torch.transpose(out, -1, -2)
- return torch.reshape(out, (B, -1, H, W)).to(dtype)
+ return out.flatten(1, 2).unflatten(-1, (H, W)).to(dtype)
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Apply lightweight multi-scale linear attention.
diff --git a/kornia/models/kimi_vl/moonvit.py b/kornia/models/kimi_vl/moonvit.py
index 323c90018..5148e2025 100644
--- a/kornia/models/kimi_vl/moonvit.py
+++ b/kornia/models/kimi_vl/moonvit.py
@@ -41,7 +41,7 @@ def apply_rotary_pos_emb(x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor)
"""
# Complex ops require at least float32; preserve float64 inputs for numerical accuracy.
compute_dtype = torch.float64 if x.dtype == torch.float64 else torch.float32
- x_complex = torch.view_as_complex(x.to(compute_dtype).reshape(*x.shape[:-1], -1, 2))
+ x_complex = torch.view_as_complex(x.to(compute_dtype).reshape(*x.shape[:-1], x.shape[-1] // 2, 2))
freqs_cis = torch.complex(cos.to(compute_dtype), sin.to(compute_dtype))
return torch.view_as_real(x_complex * freqs_cis).flatten(-2).to(x.dtype)
diff --git a/kornia/models/sam/architecture/image_encoder.py b/kornia/models/sam/architecture/image_encoder.py
index 3792a16d7..2d33370d4 100644
--- a/kornia/models/sam/architecture/image_encoder.py
+++ b/kornia/models/sam/architecture/image_encoder.py
@@ -277,11 +277,12 @@ class Attention(nn.Module):
Tensor with shape :math:`(B, H, W, C)` after attention, optional
decomposed relative positional bias, and output projection.
"""
- B, H, W, _ = x.shape
+ _, H, W, C = x.shape
+ head_dim = C // self.num_heads
# qkv with shape (3, B, nHead, H * W, C)
- qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
+ qkv = self.qkv(x).reshape(-1, H * W, 3, self.num_heads, head_dim).permute(2, 0, 3, 1, 4)
# q, k, v with shape (B * nHead, H * W, C)
- q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0)
+ q, k, v = qkv.reshape(3, -1, H * W, head_dim).unbind(0)
attn = (q * self.scale) @ k.transpose(-2, -1)
@@ -289,7 +290,7 @@ class Attention(nn.Module):
attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W))
attn = attn.softmax(dim=-1)
- x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1)
+ x = (attn @ v).view(-1, self.num_heads, H, W, head_dim).permute(0, 2, 3, 1, 4).reshape(-1, H, W, C)
return self.proj(x)
diff --git a/kornia/models/sam/architecture/mask_decoder.py b/kornia/models/sam/architecture/mask_decoder.py
index da3979e3f..dfcc4cd10 100644
--- a/kornia/models/sam/architecture/mask_decoder.py
+++ b/kornia/models/sam/architecture/mask_decoder.py
@@ -148,7 +148,7 @@ class MaskDecoder(nn.Module):
hyper_in_list.append(self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]))
hyper_in = torch.stack(hyper_in_list, dim=1)
b, c, h, w = upscaled_embedding.shape
- masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w)
+ masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, hyper_in.shape[1], h, w)
# Generate mask quality predictions
iou_pred = self.iou_prediction_head(iou_token_out)
diff --git a/kornia/models/tiny_vit.py b/kornia/models/tiny_vit.py
index 82a46cfc0..610334af6 100644
--- a/kornia/models/tiny_vit.py
+++ b/kornia/models/tiny_vit.py
@@ -347,7 +347,7 @@ class Attention(nn.Module):
B, N, _ = x.shape
x = self.norm(x)
qkv = self.qkv(x)
- qkv = qkv.view(B, N, self.num_heads, -1).permute(0, 2, 1, 3)
+ qkv = qkv.unflatten(-1, (self.num_heads, -1)).permute(0, 2, 1, 3)
q, k, v = qkv.split([self.key_dim, self.key_dim, self.d], dim=3)
bias = self.attention_biases[:, self.attention_bias_idxs]
@@ -416,7 +416,7 @@ class TinyViTBlock(nn.Module):
x = x.view(B, H, W, C)
x, pad_hw = window_partition(x, self.window_size) # (B * num_windows, window_size, window_size, C)
- x = self.attn(x.flatten(1, 2))
+ x = self.attn(x.flatten(1, 2)).unflatten(1, (self.window_size, self.window_size))
x = window_unpartition(x, self.window_size, pad_hw, (H, W))
x = x.view(B, L, C)
diff --git a/kornia/models/vit.py b/kornia/models/vit.py
index 474253652..fc5068f0c 100644
--- a/kornia/models/vit.py
+++ b/kornia/models/vit.py
@@ -96,7 +96,7 @@ class MultiHeadAttention(nn.Module):
# sum up over the third axis
out = torch.einsum("bhal, bhlv -> bhav ", att, v)
- out = out.permute(0, 2, 1, 3).contiguous().view(B, N, -1)
+ out = out.permute(0, 2, 1, 3).flatten(2)
out = self.projection(out)
return self.projection_drop(out)
@@ -183,13 +183,13 @@ class PatchEmbedding(nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.backbone(x)
- B, N, height, width = x.shape
+ B, _, height, width = x.shape
if (height, width) != self.grid_size:
raise ValueError(
f"The patch embedding gives a {height}x{width} token grid, but the position embeddings are built for "
f"the {self.grid_size[0]}x{self.grid_size[1]} grid of an image_size input."
)
- x = x.view(B, N, -1).permute(0, 2, 1) # BxNxE
+ x = x.flatten(2).permute(0, 2, 1) # BxNxE
cls_tokens = self.cls_token.repeat(B, 1, 1) # Bx1xE
# prepend the cls token to the input
x = torch.cat([cls_tokens, x], dim=1) # Bx(N+1)xE
diff --git a/kornia/models/yunet/model.py b/kornia/models/yunet/model.py
index 756b0a3da..951457356 100644
--- a/kornia/models/yunet/model.py
+++ b/kornia/models/yunet/model.py
@@ -157,16 +157,12 @@ class YuNet(nn.Module):
x_tmp = h(detection_sources[i])
head_list.append(x_tmp.permute(0, 2, 3, 1).contiguous())
- head_data = torch.cat([o.view(o.size(0), -1) for o in head_list], 1)
- head_data = head_data.view(head_data.size(0), -1, 17)
+ head_data = torch.cat([o.flatten(1) for o in head_list], 1)
+ head_data = head_data.view(head_data.shape[0], head_data.shape[1] // 17, 17)
loc_data, conf_data, iou_data = head_data.split((14, 2, 1), dim=-1)
if self.phase == "test":
conf_data = torch.softmax(conf_data, dim=-1)
- else:
- loc_data = loc_data.view(loc_data.size(0), -1, 14)
- conf_data = conf_data.view(conf_data.size(0), -1, self.num_classes)
- iou_data = iou_data.view(iou_data.size(0), -1, 1)
return {"loc": loc_data, "conf": conf_data, "iou": iou_data}
diff --git a/tests/contrib/test_face_detection.py b/tests/contrib/test_face_detection.py
index b0342224f..32796a2d4 100644
--- a/tests/contrib/test_face_detection.py
+++ b/tests/contrib/test_face_detection.py
@@ -15,6 +15,7 @@
# limitations under the License.
#
+import io
import warnings
import pytest
@@ -109,6 +110,34 @@ class TestFaceDetection(BaseTester):
)
assert model_path.is_file()
+ @pytest.mark.device_agnostic
+ def test_yunet_legacy_onnx_declares_dynamic_output_sizes(self):
+ """The TorchScript exporter declares each output's batch and anchor count dynamic, as the input sizes are."""
+ onnx = pytest.importorskip("onnx")
+ ort = pytest.importorskip("onnxruntime")
+ model = YuNet("test", pretrained=False).eval()
+ buffer = io.BytesIO()
+ torch.onnx.export(
+ model,
+ torch.rand(2, 3, 64, 64),
+ buffer,
+ input_names=["images"],
+ output_names=["loc", "conf", "iou"],
+ dynamic_axes={"images": {0: "B", 2: "H", 3: "W"}},
+ dynamo=False,
+ )
+ for output in onnx.load_from_string(buffer.getvalue()).graph.output:
+ batch, anchors, _ = output.type.tensor_type.shape.dim
+ assert batch.dim_param, output
+ assert anchors.dim_param, output
+ # The graph also runs at another batch and a non-square size, and matches eager there.
+ session = ort.InferenceSession(buffer.getvalue(), providers=["CPUExecutionProvider"])
+ images = torch.rand(1, 3, 96, 80)
+ with torch.no_grad():
+ expected = model(images)
+ for name, actual in zip(["loc", "conf", "iou"], session.run(None, {"images": images.numpy()})):
+ torch.testing.assert_close(torch.from_numpy(actual), expected[name], rtol=1e-4, atol=1e-5)
+
@pytest.mark.device_agnostic
def test_yunet_mode(self, monkeypatch):
# ``training`` of the model and of every submodule: ``{False}`` is evaluation mode throughout.
@@ -118,6 +147,23 @@ class TestFaceDetection(BaseTester):
monkeypatch.setattr("kornia.models.yunet.model.load_state_dict_from_url", lambda *args, **kwargs: state_dict)
assert {m.training for m in YuNet("test", pretrained=True).modules()} == {False}
+ @pytest.mark.parametrize("phase", ["test", "train"])
+ def test_yunet_empty_batch_6266(self, device, dtype, phase):
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ model = YuNet(phase, pretrained=False).to(device, dtype).eval()
+ out = model(torch.rand(0, 3, 64, 64, device=device, dtype=dtype))
+ assert {k: v.shape for k, v in out.items()} == {"loc": (0, 235, 14), "conf": (0, 235, 2), "iou": (0, 235, 1)}
+
+ def test_empty_batch_6266(self, device, dtype, monkeypatch):
+ """An empty batch gives one detection tensor per image: none."""
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ state_dict = YuNet("test", pretrained=False).state_dict()
+ monkeypatch.setattr("kornia.models.yunet.model.load_state_dict_from_url", lambda *args, **kwargs: state_dict)
+ face_detector = kornia.contrib.FaceDetector().to(device, dtype)
+ assert face_detector(torch.rand(0, 3, 64, 64, device=device, dtype=dtype)) == []
+
def test_exception(self, face_detector, device, dtype):
img = torch.rand(1, 4, 320, 320, device=device, dtype=dtype)
diff --git a/tests/feature/test_dedode.py b/tests/feature/test_dedode.py
index 0b216bd85..904e788c8 100644
--- a/tests/feature/test_dedode.py
+++ b/tests/feature/test_dedode.py
@@ -96,6 +96,15 @@ class TestNestedTensorBlock(BaseTester):
class TestDinoVisionTransformer(BaseTester):
+ def test_get_intermediate_layers_empty_batch_6266(self, device, dtype):
+ """``reshape=True`` turns the token grid of an empty batch into an empty feature map."""
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ model = DinoVisionTransformer(img_size=56, patch_size=14, embed_dim=32, depth=2, num_heads=2)
+ model = model.to(device, dtype).eval()
+ (out,) = model.get_intermediate_layers(torch.rand(0, 3, 56, 56, device=device, dtype=dtype), reshape=True)
+ assert out.shape == (0, 32, 4, 4)
+
def test_forward_features_tensor(self, device, dtype):
model = DinoVisionTransformer(
img_size=8,
diff --git a/tests/models/test_efficient_vit.py b/tests/models/test_efficient_vit.py
index 23d9d7786..00e9e05cf 100644
--- a/tests/models/test_efficient_vit.py
+++ b/tests/models/test_efficient_vit.py
@@ -199,6 +199,17 @@ class TestEfficientViT:
def test_smoke(self, device, dtype, img_size: int, expected_resolution: int, model_name: str):
self._test_smoke(device, dtype, img_size, expected_resolution, model_name)
+ @pytest.mark.parametrize("model_name", ["b0", "l0"])
+ def test_empty_batch_6266(self, device, dtype, model_name: str):
+ """An empty batch goes through the linear attention and keeps every stage's channels and size."""
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ model = getattr(vit, f"efficientvit_backbone_{model_name}")().to(device, dtype).eval()
+ feats = model(torch.rand(0, 3, 64, 64, device=device, dtype=dtype))
+ ref = model(torch.rand(1, 3, 64, 64, device=device, dtype=dtype))
+ assert {k: v.shape[1:] for k, v in feats.items()} == {k: v.shape[1:] for k, v in ref.items()}
+ assert all(v.shape[0] == 0 for v in feats.values())
+
@pytest.mark.slow
@pytest.mark.skipif(torch_version_lt(2, 0, 0), reason="requires torch 2.0.0 or higher")
@pytest.mark.parametrize("model_name", ["l0", "l1", "l2", "l3"])
diff --git a/tests/models/test_kimi_vl.py b/tests/models/test_kimi_vl.py
index 2701ab645..774369c43 100644
--- a/tests/models/test_kimi_vl.py
+++ b/tests/models/test_kimi_vl.py
@@ -135,6 +135,14 @@ class TestKimiVLModel(BaseTester):
expected_patches = 16
assert output.shape == (batch_size, expected_patches, config.projector_config.output_dim)
+ def test_empty_batch_6266(self, device, dtype, config):
+ """An empty batch goes through the rotary position embedding of every attention layer."""
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ model = KimiVLModel(config).to(device, dtype)
+ output = model(torch.randn(0, 3, 32, 32, device=device, dtype=dtype))
+ assert output.shape == (0, 16, config.projector_config.output_dim)
+
def test_variable_resolution(self, device, dtype, model, config):
batch_size = 1
# 48x48 image -> 12x12 patches -> 6x6 after shuffle -> 36 patches
diff --git a/tests/models/test_model_common.py b/tests/models/test_model_common.py
index 482505b4a..0f40bce05 100644
--- a/tests/models/test_model_common.py
+++ b/tests/models/test_model_common.py
@@ -230,6 +230,13 @@ class TestWindowPartition:
assert reconstructed.shape == (2, 7, 9, 16)
assert torch.allclose(x, reconstructed)
+ @pytest.mark.parametrize("hw", [(8, 8), (7, 9)], ids=["no-padding", "padding"])
+ def test_roundtrip_empty_batch_6266(self, hw):
+ x = torch.rand(0, *hw, 16)
+ windows, pad_hw = window_partition(x, window_size=4)
+ assert windows.shape == (0, 4, 4, 16)
+ assert window_unpartition(windows, window_size=4, pad_hw=pad_hw, hw=hw).shape == (0, *hw, 16)
+
class TestWindowUnpartition:
def test_no_crop_needed(self):
diff --git a/tests/models/test_sam.py b/tests/models/test_sam.py
index 3c94dbb1b..5258fc8fb 100644
--- a/tests/models/test_sam.py
+++ b/tests/models/test_sam.py
@@ -25,6 +25,8 @@ from torch import nn
from kornia.core.exceptions import BaseError
from kornia.models.sam import Sam, SamConfig
+from kornia.models.sam.architecture.image_encoder import ImageEncoderViT
+from kornia.models.sam.architecture.mask_decoder import MaskDecoder
from kornia.models.sam.architecture.prompt_encoder import PositionEmbeddingRandom, PromptEncoder
from kornia.models.sam.architecture.transformer import TwoWayAttentionBlock, TwoWayTransformer
@@ -156,6 +158,36 @@ class TestSam(BaseTester):
# 89.7M. The whole model is not bounded: ``_build_sam`` always attaches the full mask decoder.
assert sum(p.numel() for p in encoder.parameters()) < 1e6
+ @pytest.mark.parametrize("window_size", [0, 2, 3], ids=["global", "windowed", "padded-windows"])
+ def test_image_encoder_empty_batch_6266(self, device, window_size):
+ """An empty batch goes through windowed, padded-window and global attention with relative positions."""
+ encoder = ImageEncoderViT(
+ img_size=64,
+ patch_size=16,
+ embed_dim=32,
+ depth=2,
+ num_heads=2,
+ out_chans=16,
+ window_size=window_size,
+ global_attn_indexes=(1,),
+ use_rel_pos=True,
+ ).to(device)
+ assert encoder(torch.rand(0, 3, 64, 64, device=device)).shape == (0, 16, 4, 4)
+
+ def test_mask_decoder_no_prompts_6266(self, device):
+ """An image with no prompts gives no masks and no quality scores."""
+ transformer = TwoWayTransformer(depth=1, embedding_dim=32, mlp_dim=64, num_heads=2)
+ decoder = MaskDecoder(transformer_dim=32, transformer=transformer).to(device)
+ masks, iou_pred = decoder(
+ image_embeddings=torch.rand(1, 32, 8, 8, device=device),
+ image_pe=torch.rand(1, 32, 8, 8, device=device),
+ sparse_prompt_embeddings=torch.rand(0, 2, 32, device=device),
+ dense_prompt_embeddings=torch.rand(0, 32, 8, 8, device=device),
+ multimask_output=True,
+ )
+ assert masks.shape == (0, 3, 32, 32)
+ assert iou_pred.shape == (0, 3)
+
@pytest.mark.device_agnostic
def test_pretrained_needs_a_model_type(self):
config = SamConfig(
diff --git a/tests/models/test_sam_onnx.py b/tests/models/test_sam_onnx.py
index 8633b7727..0bed49827 100644
--- a/tests/models/test_sam_onnx.py
+++ b/tests/models/test_sam_onnx.py
@@ -17,12 +17,16 @@
"""ONNX export tests for SAM (image encoder subgraph)."""
+import io
+
import pytest
import torch
from kornia.core._compat import torch_version_ge, torch_version_lt
from kornia.models.sam.architecture.common import LayerNorm
from kornia.models.sam.architecture.image_encoder import ImageEncoderViT
+from kornia.models.sam.architecture.mask_decoder import MaskDecoder
+from kornia.models.sam.architecture.transformer import TwoWayTransformer
from kornia.models.sam.model import Sam, SamConfig
from kornia.models.tiny_vit import TinyViT
@@ -171,3 +175,37 @@ def test_tiny_vit_encoder_legacy_onnx_declares_the_pseudo_batch():
assert _dims(op.graph.input[0]) == [3, 3, 64, 64]
assert _dims(op.graph.output[0])[0] == 3
_check_onnx_against_eager(op, model.image_encoder, (3,), 64)
+
+
+def test_mask_decoder_legacy_onnx_declares_a_dynamic_prompt_batch():
+ """The TorchScript exporter declares the masks' prompt batch dynamic, as the prompt embeddings' batch is."""
+ onnx = pytest.importorskip("onnx")
+ torch.manual_seed(0)
+ transformer = TwoWayTransformer(depth=1, embedding_dim=32, mlp_dim=64, num_heads=2)
+ decoder = MaskDecoder(transformer_dim=32, transformer=transformer).eval()
+ image = (torch.rand(1, 32, 8, 8), torch.rand(1, 32, 8, 8))
+ prompts = (torch.rand(2, 2, 32), torch.rand(2, 32, 8, 8))
+ buffer = io.BytesIO()
+ torch.onnx.export(
+ decoder,
+ (*image, *prompts, True),
+ buffer,
+ input_names=["image_embeddings", "image_pe", "sparse", "dense"],
+ output_names=["masks", "iou_pred"],
+ dynamic_axes={"sparse": {0: "prompts"}, "dense": {0: "prompts"}},
+ dynamo=False,
+ )
+ masks = onnx.load_from_string(buffer.getvalue()).graph.output[0]
+ assert masks.type.tensor_type.shape.dim[0].dim_param
+ # The graph also runs at other prompt batches and matches eager there.
+ ort = pytest.importorskip("onnxruntime")
+ session = ort.InferenceSession(buffer.getvalue(), providers=["CPUExecutionProvider"])
+ for batch in (1, 3):
+ sparse, dense = torch.rand(batch, 2, 32), torch.rand(batch, 32, 8, 8)
+ with torch.no_grad():
+ expected = decoder(*image, sparse, dense, True)
+ names = ["image_embeddings", "image_pe", "sparse", "dense"]
+ actual = session.run(None, {name: t.numpy() for name, t in zip(names, (*image, sparse, dense))})
+ for got, want in zip(actual, expected):
+ torch.testing.assert_close(torch.from_numpy(got), want, rtol=1e-4, atol=1e-5)
diff --git a/tests/models/test_tiny_vit.py b/tests/models/test_tiny_vit.py
index df2c5ccaf..530e93d33 100644
--- a/tests/models/test_tiny_vit.py
+++ b/tests/models/test_tiny_vit.py
@@ -91,6 +91,15 @@ class TestTinyViT(BaseTester):
out = model(data)
assert out.shape == (batch_size, num_classes)
+ @pytest.mark.parametrize("img_size", [128, 224])
+ def test_empty_batch_6266(self, device, dtype, img_size):
+ """An empty batch goes through the padded and the unpadded window attention."""
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ model = TinyViT(img_size=img_size).to(device=device, dtype=dtype).eval()
+ out = model(torch.rand(0, 3, img_size, img_size, device=device, dtype=dtype))
+ assert out.shape == (0, 1000)
+
@pytest.mark.skip("not implemented")
def test_exception(self): ...
diff --git a/tests/models/test_vision_transformer.py b/tests/models/test_vision_transformer.py
index 86b5c4460..9cf763c85 100644
--- a/tests/models/test_vision_transformer.py
+++ b/tests/models/test_vision_transformer.py
@@ -54,6 +54,15 @@ class TestVisionTransformer(BaseTester):
for f in feats:
assert f.shape == (B, T, D)
+ def test_empty_batch_6266(self, device, dtype):
+ """An empty batch keeps the token and embedding dimensions."""
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ vit = VisionTransformer(image_size=32, num_heads=2, embed_dim=64, depth=1).to(device, dtype)
+ out = vit(torch.rand(0, 3, 32, 32, device=device, dtype=dtype))
+ assert out.shape == (0, 5, 64)
+ assert [f.shape for f in vit.encoder_results] == [(0, 5, 64)]
+
@pytest.mark.parametrize("H", [3, 8])
@pytest.mark.parametrize("D", [245, 1001])
@pytest.mark.parametrize("image_size", [32, 224])