π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
perf(feature): ALIKED SDDH convolutions as matrix products (M1 CPU ~3x head, ~1.5x forward); load checkpoints strictly (#6423)
Dmytro Mishkinddcdfef8e33a
diff --git a/changelog.d/6423.added.md b/changelog.d/6423.added.mdnew file mode 100644index 000000000..0a95b5729--- /dev/null+++ b/changelog.d/6423.added.md@@ -0,0 +1 @@+`ALIKED` computes the per-keypoint convolutions of its descriptor head (SDDH) as matrix products with the same weights. At 2048 keypoints on a 480 x 640 image, this makes the head about 3x faster and the whole forward about 1.5x faster on an Apple M1 CPU, where `conv2d` sent these inputs to NNPACK. On an x86 CPU (i7-14700K), which uses oneDNN, the head and the forward are about 5 % faster, and on CUDA the head is about 10 % faster. On CPU the descriptors change at float32 rounding level only. On CUDA they move closer to float64, because the convolutions ran in TF32 by default (`torch.backends.cudnn.allow_tf32`) and the matrix products do not (`torch.backends.cuda.matmul.allow_tf32` is off by default).diff --git a/changelog.d/6423.fixed.md b/changelog.d/6423.fixed.mdnew file mode 100644index 000000000..43de2831c--- /dev/null+++ b/changelog.d/6423.fixed.md@@ -0,0 +1 @@+`ALIKED.from_pretrained` loads the checkpoint strictly, so a checkpoint whose keys do not match the model raises instead of leaving the unmatched layers at their random initialisation. `dkd.hw_grid`, which is rebuilt from `nms_radius`, is no longer part of the state dict; a state dict that contains it still loads.diff --git a/kornia/feature/aliked/aliked.py b/kornia/feature/aliked/aliked.pyindex d10aaac5e..91b1467bc 100644--- a/kornia/feature/aliked/aliked.py+++ b/kornia/feature/aliked/aliked.py@@ -239,7 +239,25 @@ class DKD(nn.Module):self.unfold = nn.Unfold(kernel_size=self.kernel_size, padding=self.radius)x = torch.linspace(-self.radius, self.radius, self.kernel_size)hw_grid = torch.stack(torch.meshgrid([x, x], indexing="ij")).view(2, -1).t()[:, [1, 0]]- self.register_buffer("hw_grid", hw_grid)+ # derived from ``radius``, so not part of the state dict: the pretrained checkpoints do not carry it+ self.register_buffer("hw_grid", hw_grid, persistent=False)++ def _load_from_state_dict(+ self,+ state_dict: dict[str, torch.Tensor],+ prefix: str,+ local_metadata: dict[str, Any],+ strict: bool,+ missing_keys: list[str],+ unexpected_keys: list[str],+ error_msgs: list[str],+ ) -> None:+ # A state dict that carries ``hw_grid`` still loads strictly; the grid is rebuilt from ``radius``.+ # ``load_state_dict`` passes a copy, so the caller's dict keeps the key.+ state_dict.pop(prefix + "hw_grid", None)+ super()._load_from_state_dict(+ state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs+ )def forward(self,@@ -692,6 +710,14 @@ class SDDH(nn.Module):wh = torch.tensor([[w - 1, h - 1]], device=x.device, dtype=torch.promote_types(x.dtype, torch.float32))max_offset = max(h, w) / 4.0+ # Each convolution of the head sees one window (offset_conv) or one sample (sf_conv) per keypoint, so it is a+ # matrix product over the flattened channels and window. F.linear with the convolution's weights computes it+ # as one GEMM; conv2d on these (N, C, k, k) inputs made the head about 3x slower on an Apple M1 CPU.+ conv_offset, gate, conv_offset_1x1 = self.offset_conv+ w_offset = conv_offset.weight.flatten(1)+ w_offset_1x1 = conv_offset_1x1.weight.flatten(1)+ w_sf = self.sf_conv.weight.flatten(1)+offsets_list = []descriptors = []@@ -706,14 +732,16 @@ class SDDH(nn.Module):kptsi_wh_long = kptsi_wh.long()patch = xi[:, kptsi_wh_long[:, 1], kptsi_wh_long[:, 0]].permute(1, 0).reshape(N_kpts, c, 1, 1)- offset = self.offset_conv(patch).clamp(-max_offset, max_offset)+ # the gate gets the (N, C, 1, 1) map the convolution would give it, so a 2-D-only module stays valid+ offset = gate(F.linear(patch.flatten(1), w_offset, conv_offset.bias)[:, :, None, None]).flatten(1)+ offset = F.linear(offset, w_offset_1x1, conv_offset_1x1.bias).clamp(-max_offset, max_offset)if self.mask:- offset = offset[:, :, 0, 0].view(N_kpts, 3, self.n_pos).permute(0, 2, 1)+ offset = offset.view(N_kpts, 3, self.n_pos).permute(0, 2, 1)mask_w = torch.sigmoid(offset[:, :, -1])offset = offset[:, :, :-1]else:- offset = offset[:, :, 0, 0].view(N_kpts, 2, self.n_pos).permute(0, 2, 1)+ offset = offset.view(N_kpts, 2, self.n_pos).permute(0, 2, 1)mask_w = Noneoffsets_list.append(offset)@@ -722,18 +750,18 @@ class SDDH(nn.Module):pos = pos.reshape(1, N_kpts * self.n_pos, 1, 2)features = F.grid_sample(xi.unsqueeze(0), pos.to(xi.dtype), mode="bilinear", align_corners=True)- features = features.reshape(c, N_kpts, self.n_pos, 1).permute(1, 0, 2, 3)+ features = features.reshape(c, N_kpts, self.n_pos).permute(1, 2, 0) # [N_kpts, n_pos, C]if mask_w is not None:- features = torch.einsum("ncpo,np->ncpo", features, mask_w)+ features = features * mask_w[:, :, None]- features = torch.selu_(self.sf_conv(features)).squeeze(-1) # [N_kpts, C, n_pos]+ features = torch.selu_(F.linear(features, w_sf, self.sf_conv.bias)) # [N_kpts, n_pos, C]if not self.conv2d:- descs = torch.einsum("ncp,pcd->nd", features, self.agg_weights) # codespell:ignore+ descs = features.flatten(1) @ self.agg_weights.flatten(0, 1)else:- features = features.reshape(N_kpts, -1)[:, :, None, None]- descs = self.convM(features).squeeze(-1).squeeze(-1)+ features = features.transpose(1, 2).flatten(1) # channel-major, the order of convM's input channels+ descs = F.linear(features, self.convM.weight.flatten(1))descs = F.normalize(descs, p=2.0, dim=1)descriptors.append(descs)@@ -1103,6 +1131,6 @@ class ALIKED(nn.Module):).to(device)urls = [t.format(model_name) for t in _CHECKPOINT_URLS]state_dict = load_state_dict_from_url(urls, map_location=device)- model.load_state_dict(state_dict, strict=False)+ model.load_state_dict(state_dict)model.eval()return modeldiff --git a/tests/feature/test_aliked.py b/tests/feature/test_aliked.pyindex 3bf678967..d4bec1ad4 100644--- a/tests/feature/test_aliked.py+++ b/tests/feature/test_aliked.py@@ -17,14 +17,20 @@import pytestimport torch+import torch.nn.functional as Fimport korniafrom kornia.feature import laf_is_filledfrom kornia.feature.aliked import ALIKED, ALIKEDFeatures-from kornia.feature.aliked.aliked import _affine_from_cov+from kornia.feature.aliked.aliked import SDDH, _affine_from_cov, get_patchesfrom kornia.feature.aliked.deform_conv2d import deform_conv2d-from testing.base import BaseTester, supports_bilinear_2d_grid_sample, supports_replicate_padding+from testing.base import (+ BaseTester,+ supports_bilinear_2d_grid_sample,+ supports_bilinear_2d_grid_sample_backward,+ supports_replicate_padding,+)# ---------------------------------------------------------------------------# deform_conv2d tests@@ -364,6 +370,95 @@ class TestALIKED(BaseTester):assert torch.autograd.gradcheck(fn, [inp], eps=1e-4, atol=1e-3, rtol=1e-3, fast_mode=True, nondet_tol=1e-3)+def _sddh_with_convolutions(sddh: SDDH, x: torch.Tensor, keypoints: list[torch.Tensor]):+ """SDDH as in the reference implementations: its ``Conv2d`` layers applied to the per-keypoint windows."""+ _, c, h, w = x.shape+ wh = torch.tensor([[w - 1, h - 1]], device=x.device, dtype=torch.promote_types(x.dtype, torch.float32))+ max_offset = max(h, w) / 4.0+ descriptors, offsets = [], []+ for xi, kptsi in zip(x, keypoints):+ kptsi_wh = (kptsi / 2 + 0.5) * wh+ n = len(kptsi)+ if sddh.kernel_size > 1:+ patch = get_patches(xi, kptsi_wh.long(), sddh.kernel_size)+ else:+ corner = kptsi_wh.long()+ patch = xi[:, corner[:, 1], corner[:, 0]].permute(1, 0).reshape(n, c, 1, 1)+ offset = sddh.offset_conv(patch).clamp(-max_offset, max_offset)[:, :, 0, 0]+ offset = offset.view(n, 3 if sddh.mask else 2, sddh.n_pos).permute(0, 2, 1)+ offsets.append(offset[:, :, :2])+ pos = (2.0 * (kptsi_wh.unsqueeze(1) + offset[:, :, :2]) / wh[None] - 1).reshape(1, n * sddh.n_pos, 1, 2)+ features = F.grid_sample(xi[None], pos.to(xi.dtype), mode="bilinear", align_corners=True)+ features = features.reshape(c, n, sddh.n_pos, 1).permute(1, 0, 2, 3)+ if sddh.mask:+ features = features * torch.sigmoid(offset[:, :, 2])[:, None, :, None]+ features = torch.selu(sddh.sf_conv(features)).squeeze(-1) # (N, C, n_pos)+ if sddh.conv2d:+ descs = sddh.convM(features.flatten(1)[:, :, None, None])[:, :, 0, 0]+ else:+ descs = torch.einsum("ncp,pcd->nd", features, sddh.agg_weights) # codespell:ignore+ descriptors.append(F.normalize(descs, p=2.0, dim=1))+ return descriptors, offsets+++class TestSDDH(BaseTester):+ """SDDH computes its per-keypoint convolutions as matrix products with the ``Conv2d`` weights."""++ @pytest.fixture(autouse=True)+ def _kernels(self, device, dtype):+ if not supports_bilinear_2d_grid_sample(device, dtype):+ pytest.skip("SDDH samples with bilinear grid_sample, absent for this dtype")++ @staticmethod+ def _inputs(device, dtype):+ x = F.normalize(torch.rand(2, 16, 24, 32, device=device, dtype=dtype), dim=1)+ # the second image has no keypoint+ keypoints = [+ torch.rand(37, 2, device=device, dtype=dtype) * 2 - 1,+ torch.zeros(0, 2, device=device, dtype=dtype),+ ]+ return x, keypoints++ @pytest.mark.parametrize("kernel_size", [1, 3])+ @pytest.mark.parametrize("mask", [False, True])+ @pytest.mark.parametrize("conv2d", [False, True])+ def test_matches_the_convolutions(self, kernel_size, mask, conv2d, device, dtype):+ sddh = SDDH(16, kernel_size, 4, gate=torch.nn.SELU(inplace=True), conv2d=conv2d, mask=mask).to(device, dtype)+ x, keypoints = self._inputs(device, dtype)+ with torch.no_grad():+ descriptors, offsets = sddh(x, keypoints)+ expected_descriptors, expected_offsets = _sddh_with_convolutions(sddh, x, keypoints)+ assert [d.shape for d in descriptors] == [(37, 16), (0, 16)]+ for actual, expected in zip(descriptors + offsets, expected_descriptors + expected_offsets):+ self.assert_close(actual, expected)++ def test_gate_gets_the_convolution_output_map(self, device, dtype):+ # a gate that accepts only (N, C, H, W) input, such as Softmax2d, works as it did on the convolution output+ sddh = SDDH(16, 3, 4, gate=torch.nn.Softmax2d()).to(device, dtype)+ x, keypoints = self._inputs(device, dtype)+ with torch.no_grad():+ descriptors, offsets = sddh(x, keypoints)+ expected_descriptors, expected_offsets = _sddh_with_convolutions(sddh, x, keypoints)+ for actual, expected in zip(descriptors + offsets, expected_descriptors + expected_offsets):+ self.assert_close(actual, expected)++ def test_gradients_match_the_convolutions(self, device, dtype):+ if dtype in (torch.float16, torch.bfloat16):+ # a weight gradient sums over every sample, in another order than conv2d's backward: a few ulps apart+ pytest.skip("half-precision weight gradients differ by their accumulation order")+ if not supports_bilinear_2d_grid_sample_backward(device, dtype):+ pytest.skip("bilinear grid_sample has no backward for this dtype")+ sddh = SDDH(16, 3, 4, gate=torch.nn.SELU(inplace=True)).to(device, dtype)+ x, keypoints = self._inputs(device, dtype)+ x.requires_grad_()+ weights = torch.rand(37, 16, device=device, dtype=dtype)+ inputs = [x, *sddh.parameters()]+ grads = torch.autograd.grad((sddh(x, keypoints)[0][0] * weights).sum(), inputs)+ expected = torch.autograd.grad((_sddh_with_convolutions(sddh, x, keypoints)[0][0] * weights).sum(), inputs)+ for actual, reference in zip(grads, expected):+ self.assert_close(actual, reference)++# ---------------------------------------------------------------------------# forward_laf tests# ---------------------------------------------------------------------------@@ -978,3 +1073,30 @@ class TestALIKEDModes(BaseTester):feat, score = aliked.extract_dense_map(torch.rand(1, 3, 16, 16, device=device, dtype=dtype))assert feat.shape[0] == 1assert score.shape == (1, 1, 16, 16)+++class TestALIKEDStateDict(BaseTester):+ """``dkd.hw_grid`` is rebuilt from ``nms_radius`` and not saved, so the pretrained checkpoints load strictly."""++ def test_hw_grid_is_not_saved_but_follows_the_module(self, device, dtype):+ aliked = ALIKED(model_name="aliked-t16").to(device, dtype)+ assert "dkd.hw_grid" not in aliked.state_dict()+ assert aliked.dkd.hw_grid.device == device+ assert aliked.dkd.hw_grid.dtype == dtype++ def test_from_pretrained_rejects_a_checkpoint_with_a_renamed_key(self, device, dtype, monkeypatch):+ weights = ALIKED(model_name="aliked-t16").state_dict()+ weights["desc_head.sf.weight"] = weights.pop("desc_head.sf_conv.weight")+ monkeypatch.setattr("kornia.feature.aliked.aliked.load_state_dict_from_url", lambda urls, **kwargs: weights)+ with pytest.raises(RuntimeError, match=r"Missing key.*desc_head\.sf_conv\.weight"):+ ALIKED.from_pretrained("aliked-t16", device=device)++ def test_state_dict_with_hw_grid_loads(self, device, dtype, monkeypatch):+ saved = ALIKED(model_name="aliked-t16")+ weights = {**saved.state_dict(), "dkd.hw_grid": saved.dkd.hw_grid.clone()}+ aliked = ALIKED(model_name="aliked-t16")+ aliked.load_state_dict(weights)+ assert "dkd.hw_grid" in weights # the caller's dict is left as it was+ assert all(torch.equal(v, weights[k]) for k, v in aliked.state_dict().items())+ monkeypatch.setattr("kornia.feature.aliked.aliked.load_state_dict_from_url", lambda urls, **kwargs: weights)+ ALIKED.from_pretrained("aliked-t16", device=device)