Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

perf(geometry): nms2d 5x5/7x7 windows take the pooled path on CUDA (2-6x) (#6474)

Dmytro Mishkin47cde5b42f5f
diff --git a/changelog.d/6474.added.md b/changelog.d/6474.added.md
new file mode 100644
index 000000000..15e2f73f2
--- /dev/null
+++ b/changelog.d/6474.added.md
@@ -0,0 +1 @@
+On CUDA, `nms2d` and `NonMaximaSuppression2d` with a `(5, 5)` or `(7, 7)` window take the separable max-pool path instead of their 24- or 48-comparison chains for floating-point maps, with the same masks. On an RTX 4090 the suppression is 2-6x faster for 480 x 640 to 960 x 1280 maps, and the extraction of `XFeat`, `DISK` and `ALIKED`, which suppress with a 5 x 5 window, is 3-5% faster. CPU, MPS and integer maps keep the comparison chains: they are faster on CPU, and before torch 2.16 MPS `max_pool2d` mishandles nan and -inf.
diff --git a/kornia/geometry/subpix/nms.py b/kornia/geometry/subpix/nms.py
index 465547f1c..9da884229 100644
--- a/kornia/geometry/subpix/nms.py
+++ b/kornia/geometry/subpix/nms.py
@@ -38,6 +38,24 @@ def _reduce_max(parts: List[torch.Tensor]) -> torch.Tensor:
return out
+def _pooled_window_is_faster(x: torch.Tensor) -> bool:
+ """Whether the ``(5, 5)`` and ``(7, 7)`` windows should take the pooled path instead of their comparison chains.
+
+ On CUDA the chains are launch-bound: 24 or 48 comparisons and their conjunction cost about 50 or 100 kernels,
+ against about 10 for :func:`_neighbourhood_max2d`, which is 2-6x faster there (RTX 4090, 480 x 640 to
+ 960 x 1280 maps). On CPU the chains are 2-20x faster than the pooled path, so they stay there. MPS keeps them
+ too: before torch 2.16, its stride-1 ``max_pool2d`` drops nan and returns 0 for a window of -inf (pytorch#200910,
+ pytorch#200789), so the pooled path would change the mask there, although it is 1.4-1.9x (5 x 5) and 2.4-3.7x
+ (7 x 7) faster on an M1 with torch 2.14. Integer maps keep the chains everywhere, because CUDA ``max_pool2d`` takes
+ only floating-point input, and so do maps without channels, which ``max_pool2d`` rejects. TorchScript keeps them
+ too: the legacy ONNX exporter has no symbolic for the ``prim::is_cuda`` a scripted check would leave in the graph.
+ Both paths give the same mask.
+ """
+ if torch.jit.is_scripting():
+ return False
+ return x.is_cuda and x.is_floating_point() and x.shape[1] > 0
+
+
def _neighbourhood_max2d(x: torch.Tensor, ky: int, kx: int) -> torch.Tensor:
"""Max over each ``ky x kx`` window with its own centre excluded.
@@ -180,7 +198,7 @@ class NonMaximaSuppression2d(nn.Module):
& (ct > x[..., right, center])
& (ct > x[..., right, right])
)
- elif self.kernel_size == (5, 5):
+ elif self.kernel_size == (5, 5) and not _pooled_window_is_faster(x):
# 24-comparison explicit path for 5x5 neighbourhood.
c2 = slice(0, -4)
c1 = slice(1, -3)
@@ -215,7 +233,7 @@ class NonMaximaSuppression2d(nn.Module):
& (ct > x[..., p2, p1])
& (ct > x[..., p2, p2])
)
- elif self.kernel_size == (7, 7):
+ elif self.kernel_size == (7, 7) and not _pooled_window_is_faster(x):
# 48-comparison explicit path for 7x7 neighbourhood.
c3 = slice(0, -6)
c2 = slice(1, -5)
@@ -277,7 +295,8 @@ class NonMaximaSuppression2d(nn.Module):
& (ct > x[..., p3, p3])
)
else:
- # General path: the same rule as the explicit paths above, for any window size. A
+ # General path: the same rule as the explicit paths above, for any window size, and the (5, 5) and
+ # (7, 7) windows of floating-point CUDA maps (see `_pooled_window_is_faster`). A
# position whose full window does not fit inside the image is not a maximum -- it has
# not been compared against the neighbours that would decide it -- so the border strip
# stays False and only the interior is computed (#4239).
diff --git a/tests/geometry/subpix/test_nms.py b/tests/geometry/subpix/test_nms.py
index 1b7ae37f3..bdccd57a4 100644
--- a/tests/geometry/subpix/test_nms.py
+++ b/tests/geometry/subpix/test_nms.py
@@ -15,6 +15,8 @@
# limitations under the License.
#
+import importlib
+import io
import itertools
import pytest
@@ -225,6 +227,86 @@ class TestNMS2d(BaseTester):
_reference_nms_mask(inp, kernel_size).to(dtype),
)
+ @pytest.mark.parametrize("pooled", [False, True])
+ @pytest.mark.parametrize("kernel_size", [(5, 5), (7, 7)])
+ def test_pooled_and_chained_windows_match_the_reference_rule(self, device, dtype, kernel_size, pooled, monkeypatch):
+ # (5, 5) and (7, 7) take their comparison chains, or the pooled path for floating-point CUDA maps
+ # (`_pooled_window_is_faster`). Force each in turn on ties and non-finite values, which the two handle
+ # differently inside: a chain compares every neighbour, the pooled path compares with their max.
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ if device.type == "mps" and pooled:
+ # before torch 2.16, MPS stride-1 max_pool2d drops nan and turns an all -inf window into 0
+ pytest.skip("MPS maps never take the pooled path (pytorch#200910, pytorch#200789)")
+ module = importlib.import_module("kornia.geometry.subpix.nms")
+ monkeypatch.setattr(module, "_pooled_window_is_faster", lambda x: pooled)
+ torch.manual_seed(0)
+ inp = torch.randint(0, 4, (2, 3, 23, 27), device=device).to(dtype) # small integers: many ties
+ inp[0, 0, 5, 5] = float("nan")
+ inp[0, 1, 7, 9] = float("inf")
+ inp[1, 2, 10, 10] = float("-inf")
+ inp[0, 0, 6, 6] = 9.0 # larger than every finite value, but a nan in its window: not a maximum
+ inp[1, 0, 11, 13] = 9.0 # a strict maximum
+ mask = kornia.geometry.subpix.nms2d(inp, kernel_size, mask_only=True)
+ assert torch.equal(mask, _reference_nms_mask(inp, kernel_size))
+ assert bool(mask[0, 1, 7, 9])
+ assert bool(mask[1, 0, 11, 13])
+ assert not bool(mask[0, 0, 5, 5])
+ assert not bool(mask[0, 0, 6, 6])
+
+ @pytest.mark.parametrize("kernel_size", [(5, 5), (7, 7)])
+ def test_only_floating_point_cuda_maps_take_the_pooled_path(self, device, dtype, kernel_size, monkeypatch):
+ # the production selection, with `_pooled_window_is_faster` left in place: count the pooled calls
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ module = importlib.import_module("kornia.geometry.subpix.nms")
+ calls = []
+ pooled = module._neighbourhood_max2d
+ monkeypatch.setattr(module, "_neighbourhood_max2d", lambda *args: calls.append(1) or pooled(*args))
+ kornia.geometry.subpix.nms2d(torch.rand(1, 2, 11, 13, device=device, dtype=dtype), kernel_size)
+ assert len(calls) == (1 if device.type == "cuda" else 0)
+ kornia.geometry.subpix.nms2d(torch.zeros(1, 2, 11, 13, device=device, dtype=torch.int64), kernel_size)
+ kornia.geometry.subpix.nms2d(torch.rand(1, 0, 11, 13, device=device, dtype=dtype), kernel_size)
+ assert len(calls) == (1 if device.type == "cuda" else 0)
+
+ @pytest.mark.parametrize("mask_only", [True, False])
+ @pytest.mark.parametrize("kernel_size", [(5, 5), (7, 7)])
+ def test_maps_without_channels(self, device, dtype, kernel_size, mask_only):
+ inp = torch.rand(1, 0, 11, 13, device=device, dtype=dtype)
+ out = kornia.geometry.subpix.nms2d(inp, kernel_size, mask_only=mask_only)
+ assert out.shape == inp.shape
+ assert out.dtype == (torch.bool if mask_only else dtype)
+
+ def test_scripted_module_exports_with_the_legacy_onnx_exporter(self):
+ # A scripted device check would leave prim::is_cuda in the graph, which the legacy exporter cannot export.
+ pytest.importorskip("onnx")
+ scripted = torch.jit.script(kornia.geometry.subpix.NonMaximaSuppression2d((5, 5)))
+ torch.onnx.export(scripted, (torch.rand(1, 1, 16, 16),), io.BytesIO(), dynamo=False)
+
+ @pytest.mark.parametrize("kernel_size", [(5, 5), (7, 7)])
+ def test_dynamo_on_ties_and_nonfinite_values(self, device, dtype, kernel_size, torch_optimizer):
+ if device.type == "mps" and dtype == torch.float64:
+ pytest.skip("MPS does not support float64")
+ torch.manual_seed(0)
+ inp = torch.randint(0, 4, (2, 3, 23, 27), device=device).to(dtype)
+ inp[0, 0, 5, 5] = float("nan")
+ inp[0, 1, 7, 9] = float("inf")
+ inp[1, 2, 10, 10] = float("-inf")
+ inp[0, 0, 6, 6] = 9.0
+ op = torch_optimizer(kornia.geometry.subpix.nms2d, fullgraph=True)
+ assert torch.equal(
+ op(inp, kernel_size, mask_only=True), kornia.geometry.subpix.nms2d(inp, kernel_size, mask_only=True)
+ )
+
+ @pytest.mark.parametrize("kernel_size", [(5, 5), (7, 7)])
+ def test_integer_maps_keep_the_comparison_chains(self, device, kernel_size):
+ # CUDA max_pool2d has no integer kernels, so integer maps never take the pooled path.
+ inp = torch.zeros(1, 1, 15, 15, device=device, dtype=torch.int64)
+ inp[0, 0, 7, 7] = 3
+ expected = torch.zeros(1, 1, 15, 15, device=device, dtype=torch.bool)
+ expected[0, 0, 7, 7] = True
+ assert torch.equal(kornia.geometry.subpix.nms2d(inp, kernel_size, mask_only=True), expected)
+
@pytest.mark.parametrize("kernel_size", [(5, 5), (9, 9)])
def test_a_plateau_holds_no_strict_maximum(self, device, dtype, kernel_size):
# The suppression is strict: tied neighbours kill each other. A max-pool written as