π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
perf(feature): XFeat* mutual check from a second product, not a column argmax (M1 CPU 1.26x) (#6499)
Dmytro Mishkinfbb106a6b3c9
diff --git a/changelog.d/6499.added.md b/changelog.d/6499.added.mdnew file mode 100644index 000000000..ce32b5405--- /dev/null+++ b/changelog.d/6499.added.md@@ -0,0 +1,6 @@+`XFeat.match_xfeat_star` finds the nearest neighbour of each second-image descriptor with a second matrix product and a+row-wise `argmax`, instead of an `argmax` down the columns of the similarity matrix, and frees the first matrix before+computing the second. XFeat* matching of a 640 x 480 pair is 1.26x faster on an Apple M1 CPU and 1.07-1.25x on an x86+i7-14700K, with identical matches. On CUDA (RTX 4090) the speed is unchanged, and the call's peak memory falls from 450+to 66 MiB at top_k=4096, because the column `argmax` allocated about six times the similarity matrix in scratch. MPS is+unchanged.diff --git a/kornia/feature/xfeat.py b/kornia/feature/xfeat.pyindex 4ad9a9512..95b8dfe4e 100644--- a/kornia/feature/xfeat.py+++ b/kornia/feature/xfeat.py@@ -487,20 +487,20 @@ class XFeat(nn.Module):B = feats1.shape[0]cossim = torch.bmm(feats1, feats2.permute(0, 2, 1))match12 = torch.argmax(cossim, dim=-1)- match21 = torch.argmax(cossim.permute(0, 2, 1), dim=-1)+ cossim_max = cossim.amax(dim=-1) if min_cossim > 0 else None+ # Free the forward matrix before the reverse one, so only one (B, N1, N2) matrix is alive at a time.+ del cossim+ # The second product reduces along rows, as _match_mnn does. An argmax down the columns of cossim is a strided+ # reduction, several times slower on CPU than this matrix product, and on CUDA it allocates about six times+ # the matrix in scratch memory.+ match21 = torch.argmax(torch.bmm(feats2, feats1.permute(0, 2, 1)), dim=-1)idx0 = torch.arange(match12.shape[1], device=match12.device)batched_matches = []for b in range(B):- mutual = match21[b][match12[b]] == idx0- if min_cossim > 0:- cossim_max, _ = cossim[b].max(dim=1)- good = cossim_max > min_cossim- idx0_b = idx0[mutual & good]- idx1_b = match12[b][mutual & good]- else:- idx0_b = idx0[mutual]- idx1_b = match12[b][mutual]- batched_matches.append((idx0_b, idx1_b))+ keep = match21[b][match12[b]] == idx0+ if cossim_max is not None:+ keep = keep & (cossim_max[b] > min_cossim)+ batched_matches.append((idx0[keep], match12[b][keep]))return batched_matchesdef _extract_dense(self, x: torch.Tensor, top_k: Optional[int] = 8000) -> Tuple[torch.Tensor, torch.Tensor]:diff --git a/tests/feature/test_xfeat.py b/tests/feature/test_xfeat.pyindex bae54a54b..991271b19 100644--- a/tests/feature/test_xfeat.py+++ b/tests/feature/test_xfeat.py@@ -169,6 +169,27 @@ class TestXFeat(BaseTester):assert dense["scales"].shape == (0, 50)assert model.match_xfeat_star(x, x, top_k=50) == []+ @pytest.mark.parametrize("min_cossim", [-1, 0.2])+ def test_batch_match_is_mutual_nearest_neighbour(self, device, dtype, min_cossim):+ # Pair b keeps (i, j) when j is the most similar descriptor of the second set to descriptor i of the first,+ # and i the most similar of the first to j, read off the one similarity matrix.+ torch.manual_seed(0)+ feats1 = torch.nn.functional.normalize(torch.randn(2, 70, 64, device=device, dtype=dtype), dim=-1)+ feats2 = torch.nn.functional.normalize(torch.randn(2, 50, 64, device=device, dtype=dtype), dim=-1)+ matches = XFeat()._batch_match(feats1, feats2, min_cossim=min_cossim)+ # The one similarity matrix, from the same batched product that _batch_match starts with.+ cossims = torch.bmm(feats1, feats2.permute(0, 2, 1))+ assert len(matches) == 2+ for b, (idx0, idx1) in enumerate(matches):+ cossim = cossims[b]+ best12, best21 = cossim.argmax(dim=1), cossim.argmax(dim=0)+ keep = best21[best12] == torch.arange(70, device=device)+ if min_cossim > 0:+ keep &= cossim.max(dim=1).values > min_cossim+ assert idx0.shape[0] > 0+ assert torch.equal(idx0, keep.nonzero()[:, 0])+ assert torch.equal(idx1, best12[keep])+def test_smoke(self, device, dtype):model = XFeat().to(device).eval()x = torch.rand(1, 3, 64, 64, device=device)