Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

perf(feature): XFeat* mutual check from a second product, not a column argmax (M1 CPU 1.26x) (#6499)

Dmytro Mishkinfbb106a6b3c9
diff --git a/changelog.d/6499.added.md b/changelog.d/6499.added.md
new file mode 100644
index 000000000..ce32b5405
--- /dev/null
+++ b/changelog.d/6499.added.md
@@ -0,0 +1,6 @@
+`XFeat.match_xfeat_star` finds the nearest neighbour of each second-image descriptor with a second matrix product and a
+row-wise `argmax`, instead of an `argmax` down the columns of the similarity matrix, and frees the first matrix before
+computing the second. XFeat* matching of a 640 x 480 pair is 1.26x faster on an Apple M1 CPU and 1.07-1.25x on an x86
+i7-14700K, with identical matches. On CUDA (RTX 4090) the speed is unchanged, and the call's peak memory falls from 450
+to 66 MiB at top_k=4096, because the column `argmax` allocated about six times the similarity matrix in scratch. MPS is
+unchanged.
diff --git a/kornia/feature/xfeat.py b/kornia/feature/xfeat.py
index 4ad9a9512..95b8dfe4e 100644
--- a/kornia/feature/xfeat.py
+++ b/kornia/feature/xfeat.py
@@ -487,20 +487,20 @@ class XFeat(nn.Module):
B = feats1.shape[0]
cossim = torch.bmm(feats1, feats2.permute(0, 2, 1))
match12 = torch.argmax(cossim, dim=-1)
- match21 = torch.argmax(cossim.permute(0, 2, 1), dim=-1)
+ cossim_max = cossim.amax(dim=-1) if min_cossim > 0 else None
+ # Free the forward matrix before the reverse one, so only one (B, N1, N2) matrix is alive at a time.
+ del cossim
+ # The second product reduces along rows, as _match_mnn does. An argmax down the columns of cossim is a strided
+ # reduction, several times slower on CPU than this matrix product, and on CUDA it allocates about six times
+ # the matrix in scratch memory.
+ match21 = torch.argmax(torch.bmm(feats2, feats1.permute(0, 2, 1)), dim=-1)
idx0 = torch.arange(match12.shape[1], device=match12.device)
batched_matches = []
for b in range(B):
- mutual = match21[b][match12[b]] == idx0
- if min_cossim > 0:
- cossim_max, _ = cossim[b].max(dim=1)
- good = cossim_max > min_cossim
- idx0_b = idx0[mutual & good]
- idx1_b = match12[b][mutual & good]
- else:
- idx0_b = idx0[mutual]
- idx1_b = match12[b][mutual]
- batched_matches.append((idx0_b, idx1_b))
+ keep = match21[b][match12[b]] == idx0
+ if cossim_max is not None:
+ keep = keep & (cossim_max[b] > min_cossim)
+ batched_matches.append((idx0[keep], match12[b][keep]))
return batched_matches
def _extract_dense(self, x: torch.Tensor, top_k: Optional[int] = 8000) -> Tuple[torch.Tensor, torch.Tensor]:
diff --git a/tests/feature/test_xfeat.py b/tests/feature/test_xfeat.py
index bae54a54b..991271b19 100644
--- a/tests/feature/test_xfeat.py
+++ b/tests/feature/test_xfeat.py
@@ -169,6 +169,27 @@ class TestXFeat(BaseTester):
assert dense["scales"].shape == (0, 50)
assert model.match_xfeat_star(x, x, top_k=50) == []
+ @pytest.mark.parametrize("min_cossim", [-1, 0.2])
+ def test_batch_match_is_mutual_nearest_neighbour(self, device, dtype, min_cossim):
+ # Pair b keeps (i, j) when j is the most similar descriptor of the second set to descriptor i of the first,
+ # and i the most similar of the first to j, read off the one similarity matrix.
+ torch.manual_seed(0)
+ feats1 = torch.nn.functional.normalize(torch.randn(2, 70, 64, device=device, dtype=dtype), dim=-1)
+ feats2 = torch.nn.functional.normalize(torch.randn(2, 50, 64, device=device, dtype=dtype), dim=-1)
+ matches = XFeat()._batch_match(feats1, feats2, min_cossim=min_cossim)
+ # The one similarity matrix, from the same batched product that _batch_match starts with.
+ cossims = torch.bmm(feats1, feats2.permute(0, 2, 1))
+ assert len(matches) == 2
+ for b, (idx0, idx1) in enumerate(matches):
+ cossim = cossims[b]
+ best12, best21 = cossim.argmax(dim=1), cossim.argmax(dim=0)
+ keep = best21[best12] == torch.arange(70, device=device)
+ if min_cossim > 0:
+ keep &= cossim.max(dim=1).values > min_cossim
+ assert idx0.shape[0] > 0
+ assert torch.equal(idx0, keep.nonzero()[:, 0])
+ assert torch.equal(idx1, best12[keep])
+
def test_smoke(self, device, dtype):
model = XFeat().to(device).eval()
x = torch.rand(1, 3, 64, 64, device=device)