Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(feature): detect a float16 image in float32 in ScaleSpaceDetector (#6606)

Dmytro Mishkind8a6850b1684
diff --git a/changelog.d/6606.fixed.md b/changelog.d/6606.fixed.md
new file mode 100644
index 000000000..1fa241e09
--- /dev/null
+++ b/changelog.d/6606.fixed.md
@@ -0,0 +1 @@
+`ScaleSpaceDetector` no longer gives a NaN image gradient for a float16 image with exactly `ScalePyramid`, exactly a `BlobHessian`, `BlobDoG`, `CornerHarris` or `CornerGFTT` response, and a response and refiner holding only scalar parameters and buffers. Such an image used to run through the scale space in float16, where the gradient overflowed, with scores that followed an autocast region in some cases. It is now detected in float32 with autocast off, a candidate whose score is zero in float16 is dropped, and its scores are returned in float16, also in an autocast region. As a correctness fix this ships without a deprecation window ("Escape hatch" on the stability page): the keypoints and scores of such an image change and move toward the float32 ones. A float16 Harris detector still finds fewer keypoints than float32: on 8 half-size HPatches images it fills a mean of 545 slots against float32's 825, and used to fill 1020, almost none of them near a float32 keypoint. With any other pyramid or response function, a subclass or a `CornerHarris` with a non-scalar `k` included, or with a refiner holding non-scalar state, the pyramid and response function get the float16 image as before and can still give a NaN gradient.
diff --git a/kornia/feature/scale_space_detector.py b/kornia/feature/scale_space_detector.py
index 795760d18..7ecb24e98 100644
--- a/kornia/feature/scale_space_detector.py
+++ b/kornia/feature/scale_space_detector.py
@@ -24,6 +24,7 @@ import torch.nn.functional as F
from torch import nn
from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SAME_SIZE, KORNIA_CHECK_SHAPE, KORNIA_CHECK_SIZES
+from kornia.core.utils import _autocast_disabled
from kornia.geometry.subpix import (
AdaptiveQuadInterp3d,
ConvQuadInterp3d,
@@ -35,7 +36,7 @@ from kornia.geometry.transform import ScalePyramid, pyrdown, resize
from .laf import laf_from_center_scale_ori, laf_is_filled
from .orientation import PassLAF
-from .responses import BlobHessian
+from .responses import BlobDoG, BlobHessian, CornerGFTT, CornerHarris
def _scale_index_to_scale(max_coords: torch.Tensor, sigmas: torch.Tensor, num_levels: int) -> torch.Tensor:
@@ -200,6 +201,52 @@ def _subpix_dispatch(module: nn.Module) -> Tuple[bool, bool]:
return is_iterative, batchable
+_FLOAT16_IN_FLOAT32_RESPONSES = (BlobHessian, BlobDoG, CornerHarris, CornerGFTT)
+
+
+def _holds_only_scalars(module: nn.Module) -> bool:
+ """Return whether every parameter and buffer of ``module`` is zero-dimensional, so none rejects an input dtype."""
+ return all(t.dim() == 0 for t in (*module.parameters(), *module.buffers()))
+
+
+def _detects_float16_in_float32(pyramid: nn.Module, response: nn.Module, subpix: nn.Module) -> bool:
+ """Return whether a float16 image can run through these modules in float32, with autocast off.
+
+ The pyramid and the response function get the image, so only their exact built-in classes qualify, read through a
+ :func:`torch.compile` wrapper. These follow their input's dtype, whatever dtype the module was converted to, so a
+ float32 input comes back as float32, which `_detect` can round to float16. A custom module or a subclass may hold
+ float16 weights that reject float32 input, or return a wider response on purpose, and from the outside neither can
+ be told from a module that follows its input, so it gets the float16 image. So does a :class:`CornerHarris` with a
+ non-scalar ``k``, which promotes a float16 response. The refiner gets the response in at least float32 on every
+ path, so its class does not matter; only autocast changes for it, and turning that off would make float16 weights
+ reject the input they now get cast for.
+
+ Measured on a float16 image, CPU and CUDA: with ``isinstance``, a subclass with a float16 weight raises and a
+ subclass returning a wider response has its float32 score of 1.0001 rounded to 1.0; the exact check costs an
+ unmodified subclass the fix (5120 of 5120 non-finite, as on main).
+ """
+ return (
+ type(getattr(pyramid, "_orig_mod", pyramid)) is ScalePyramid
+ and type(getattr(response, "_orig_mod", response)) in _FLOAT16_IN_FLOAT32_RESPONSES
+ and _holds_only_scalars(response)
+ and _holds_only_scalars(subpix)
+ )
+
+
+def _nms_candidates(response: torch.Tensor, float16_scores: bool) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Return the masks of the strict maxima and minima of ``response``, an octave's response in at least float32.
+
+ With ``float16_scores`` it comes from a float16 image, whose quantisation noise makes weak maxima with scores below
+ float16's resolution, so only those are dropped. Rounding the response before NMS would also merge real maxima:
+ Harris fills 252 slots against float32's 825 on HPatches, this 545.
+ """
+ max_mask, min_mask = nms3d_minmax(response)
+ if float16_scores:
+ scored = response.to(torch.float16) != 0
+ return max_mask & scored, min_mask & scored
+ return max_mask, min_mask
+
+
class ScaleSpaceDetector(nn.Module):
r"""Detect LAFs at the sub-pixel scale-space maxima of a pluggable response, for a batch of images.
@@ -238,7 +285,13 @@ class ScaleSpaceDetector(nn.Module):
hold a zero response and a zero LAF.
- The LAFs are float32 for a float16 or bfloat16 image and float64 for a float64 one; the responses keep the
response function's dtype. Non-maximum suppression and sub-pixel refinement run on the response promoted to
- at least float32.
+ at least float32. With :class:`~kornia.geometry.transform.ScalePyramid`, a :class:`BlobHessian`,
+ :class:`BlobDoG`, :class:`CornerHarris` or :class:`CornerGFTT` response, and a response and refiner holding
+ only scalar parameters and buffers, a float16 image is detected in float32 with autocast off and its scores
+ are returned in float16, because the gradient overflows float16 on the way through the scale space. A
+ candidate whose score is zero in float16 is dropped. With any other pyramid or response function, a subclass
+ or a :class:`CornerHarris` with a non-scalar ``k`` included, or a refiner with non-scalar state, the detector
+ passes the float16 image on, in the caller's autocast region.
- With the default modules the responses, centres and scales carry gradients to the image.
- A batch of zero images returns empty outputs of the documented shapes, after the checks a non-empty batch
runs.
@@ -410,6 +463,7 @@ class ScaleSpaceDetector(nn.Module):
is_iterative_subpix: bool,
batchable_subpix: bool,
octave_to_image: Tuple[float, float],
+ float16_scores: bool = False,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.dtype]:
"""Process one scale-space octave: response β†’ NMS/subpix β†’ top-K β†’ LAF.
@@ -460,8 +514,8 @@ class ScaleSpaceDetector(nn.Module):
# The refiners write absolute pixel coordinates into a tensor of the response's dtype, and bfloat16 holds no
# odd integer above 256 (float16: above 2048). Run NMS, refinement and top-K on the response in at least
# float32; promoting is exact, so NMS selects the same candidates. `_detect` casts the responses back.
- resp_dtype = oct_resp.dtype
- oct_resp = oct_resp.to(torch.promote_types(resp_dtype, torch.float32))
+ resp_dtype = torch.float16 if float16_scores else oct_resp.dtype
+ oct_resp = oct_resp.to(torch.promote_types(oct_resp.dtype, torch.float32))
scale_sigmas = sigmas_oct[:, : oct_resp.shape[2]].to(oct_resp.dtype)
# Always precompute NMS masks in one fused pass.
@@ -470,7 +524,7 @@ class ScaleSpaceDetector(nn.Module):
# and drives the sparse top-K below.
max_nms_mask: torch.Tensor
min_nms_mask: torch.Tensor
- max_nms_mask, min_nms_mask = nms3d_minmax(oct_resp)
+ max_nms_mask, min_nms_mask = _nms_candidates(oct_resp, float16_scores)
# The mask is applied to the candidates, not to the response the NMS reads: multiplying the
# response first would carve a hard edge into it, and every response pixel next to a zeroed
@@ -625,6 +679,25 @@ class ScaleSpaceDetector(nn.Module):
"""
if mask is not None:
_check_mask(mask, img)
+ # The backward pass of a float16 scale space overflows: a response's gradient, about 4e4 even when the image's
+ # is about 10, is multiplied by sigma ** 4, and a convolution then turns that inf into NaN at every pixel. So a
+ # float16 image is detected in float32 when the modules allow it (see `_detects_float16_in_float32`), with
+ # autocast off, since its convolutions would otherwise go back to the autocast dtype. The mask weights keep
+ # their float16 values and only the scores are cast back. Every other image takes the path below untouched.
+ if img.dtype == torch.float16 and _detects_float16_in_float32(self.scale_pyr, self.resp, self.subpix):
+ if mask is not None and mask.is_floating_point():
+ mask = mask.to(torch.float16)
+ with _autocast_disabled(img.device.type):
+ return self._detect_in_image_dtype(img.float(), num_feats, mask, True)
+ return self._detect_in_image_dtype(img, num_feats, mask, False)
+
+ def _detect_in_image_dtype(
+ self, img: torch.Tensor, num_feats: int, mask: Optional[torch.Tensor], float16_scores: bool
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Run the scale space, the octaves and the cross-octave ranking on ``img`` in the dtype it has.
+
+ ``float16_scores`` says that ``img`` is the float32 copy of a float16 image: see :meth:`_detect`.
+ """
dev = img.device
dtype: torch.dtype = img.dtype
with self._dynamo_config_patch(("scale_pyr",)):
@@ -671,6 +744,7 @@ class ScaleSpaceDetector(nn.Module):
is_iterative_subpix,
batchable_subpix,
octave_to_image[i],
+ float16_scores,
)
for i in range(n_oct)
]
diff --git a/tests/feature/test_scale_space_detector.py b/tests/feature/test_scale_space_detector.py
index 223766846..f4ceb18be 100644
--- a/tests/feature/test_scale_space_detector.py
+++ b/tests/feature/test_scale_space_detector.py
@@ -22,10 +22,11 @@ from torch.utils._python_dispatch import TorchDispatchMode
import kornia
from kornia.core.check import ShapeError
from kornia.core.exceptions import BaseError
-from kornia.feature import BlobDoG, KeyNetDetector
+from kornia.feature import BlobDoG, BlobHessian, CornerGFTT, CornerHarris, KeyNetDetector
from kornia.feature.scale_space_detector import (
MultiResolutionDetector,
ScaleSpaceDetector,
+ _detects_float16_in_float32,
_resize_mask,
get_default_detector_config,
)
@@ -45,6 +46,12 @@ from testing.base import (
from testing.default_device import default_device
+class _RefinerWithState(ConvQuadInterp3d):
+ def __init__(self) -> None:
+ super().__init__()
+ self.register_buffer("state", torch.ones(1))
+
+
def _skip_without_half_padding(device: torch.device, dtype: torch.dtype) -> None:
# torch 2.5.1 has no float16 CPU reflection_pad2d or replication_pad2d; these pins blur (reflect) and take
# gradients (replicate).
@@ -601,6 +608,167 @@ class TestScaleSpaceDetector(BaseTester):
lafs, _ = det(img, torch.full((1, 1, 96, 128), 1e-8, device=device))
assert bool(kornia.feature.laf_is_filled(lafs).any()) == (dtype != torch.float16)
+ @pytest.mark.parametrize("subpix", [ConvQuadInterp3d, IterativeQuadInterp3d])
+ @pytest.mark.parametrize("convert_module", [False, True])
+ def test_float16_image_gradient_matches_float32_6143(self, device, subpix, convert_module):
+ # The true gradient of about 10 fits float16, but its way through the response does not: 4e4 * sigma ** 4.
+ image = torch.rand(1, 1, 64, 80, generator=torch.Generator().manual_seed(0))
+ gradients = {}
+ for image_dtype in (torch.float32, torch.float16):
+ det = ScaleSpaceDetector(10, subpix_module=subpix()).to(device)
+ if convert_module:
+ det = det.to(image_dtype)
+ x = image.to(device, image_dtype).clone().requires_grad_()
+ lafs, responses = det(x)
+ assert lafs.dtype == torch.float32
+ assert responses.dtype == image_dtype
+ gradients[image_dtype] = torch.autograd.grad(lafs.sum() + responses.sum(), x)[0]
+ assert gradients[torch.float16].isfinite().all()
+ # Rounding the image to float16 moves this gradient by up to 0.015 (CPU, CUDA).
+ self.assert_close(gradients[torch.float16].float(), gradients[torch.float32], rtol=0, atol=0.05)
+
+ def test_float16_image_fills_as_many_slots_as_float32_at_low_contrast(self, device):
+ # Rounding the response to float16 before NMS merges maxima closer than one float16 step: 4 slots here, not 7.
+ image = torch.rand(1, 1, 64, 80, generator=torch.Generator().manual_seed(0))
+ image = (0.5 + (image - 0.5) * 0.01).to(device)
+ filled = {}
+ for image_dtype in (torch.float32, torch.float16):
+ x = image.to(image_dtype).clone().requires_grad_()
+ lafs, responses = ScaleSpaceDetector(10, subpix_module=ConvQuadInterp3d()).to(device)(x)
+ filled[image_dtype] = int(kornia.feature.laf_is_filled(lafs).sum())
+ gradient = torch.autograd.grad(lafs.sum() + responses.sum(), x)[0]
+ assert gradient.isfinite().all()
+ assert filled[torch.float16] == filled[torch.float32]
+
+ def test_float16_custom_response_keeps_its_dtype_and_nms_precision(self, device):
+ # A custom response is not rounded to float16: an isolated peak of 1.0001 on a floor of 1.0 stays a maximum.
+ class WideResponse(torch.nn.Module):
+ def forward(self, x, sigmas=None):
+ out = torch.ones_like(x, dtype=torch.float32)
+ out[out.shape[0] // 2, :, out.shape[-2] // 2, out.shape[-1] // 2] = 1.0001
+ return out
+
+ _skip_without_half_padding(device, torch.float16)
+ det = ScaleSpaceDetector(3, resp_module=WideResponse()).to(device)
+ lafs, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
+ assert responses.dtype == torch.float32
+ assert kornia.feature.laf_is_filled(lafs).any()
+
+ def test_float16_response_with_weights_gets_float16_input(self, device):
+ _skip_without_half_padding(device, torch.float16)
+
+ class LearnedResponse(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.conv = torch.nn.Conv2d(1, 1, 3, padding=1)
+
+ def forward(self, x, sigmas=None):
+ return self.conv(x)
+
+ resp = LearnedResponse().to(device, torch.float16)
+ det = ScaleSpaceDetector(5, resp_module=resp).to(device)
+ _, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
+ assert responses.dtype == torch.float16
+
+ @pytest.mark.parametrize("subpix", [ConvQuadInterp3d, IterativeQuadInterp3d])
+ @pytest.mark.parametrize("region_dtype", [torch.float16, torch.bfloat16])
+ def test_float16_image_gradient_in_a_region_6143(self, device, subpix, region_dtype):
+ if device.type == "cuda" and region_dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
+ pytest.skip("bfloat16 autocast is not supported on this CUDA device")
+ # Autocast would turn the float32 copy's convolutions back to its own dtype, and float16 overflows there too.
+ image = torch.rand(1, 1, 64, 80, generator=torch.Generator().manual_seed(0)).to(device)
+ reference = image.clone().requires_grad_()
+ lafs, responses = ScaleSpaceDetector(10, subpix_module=subpix()).to(device)(reference)
+ expected = torch.autograd.grad(lafs.sum() + responses.sum(), reference)[0]
+
+ x = image.to(torch.float16).clone().requires_grad_()
+ with torch.autocast(device.type, dtype=region_dtype):
+ lafs, responses = ScaleSpaceDetector(10, subpix_module=subpix()).to(device)(x)
+ assert lafs.dtype == torch.float32
+ assert responses.dtype == torch.float16
+ gradient = torch.autograd.grad(lafs.sum() + responses.sum(), x)[0]
+ assert gradient.isfinite().all()
+ # Rounding the image to float16 moves this gradient by up to 0.015 (CPU, CUDA).
+ self.assert_close(gradient.float(), expected, rtol=0, atol=0.05)
+
+ @pytest.mark.parametrize("response_dtype", [torch.float32, torch.float64])
+ def test_float16_image_keeps_a_wider_custom_response(self, device, response_dtype):
+ if response_dtype == torch.float64 and device.type == "mps":
+ pytest.skip("MPS has no float64")
+ _skip_without_half_padding(device, torch.float16)
+ seen = []
+
+ class IsolatedPeak(torch.nn.Module):
+ def forward(self, x, sigmas=None):
+ seen.append(x.dtype)
+ response = torch.ones(x.shape[0], 1, *x.shape[-2:], device=x.device, dtype=response_dtype)
+ response[2, 0, x.shape[-2] // 2, x.shape[-1] // 2] = 1.0001 # one level; float16 rounds it to 1
+ return response
+
+ det = ScaleSpaceDetector(5, resp_module=IsolatedPeak(), subpix_module=ConvQuadInterp3d()).to(device)
+ _, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
+ assert set(seen) == {torch.float16}
+ assert responses.dtype == response_dtype
+ assert responses[0, 0] > 1.0
+
+ def test_float16_image_reaches_a_custom_pyramid_with_float16_weights(self, device):
+ _skip_without_half_padding(device, torch.float16)
+
+ class LearnedPyramid(torch.nn.Module):
+ n_levels = 3
+
+ def __init__(self):
+ super().__init__()
+ self.conv = torch.nn.Conv2d(1, 1, 3, padding=1)
+ self.pyramid = ScalePyramid(3, 1.6, 15, extra_levels=2)
+
+ def forward(self, x):
+ return self.pyramid(self.conv(x))
+
+ det = ScaleSpaceDetector(5, scale_pyr_module=LearnedPyramid().to(device, torch.float16)).to(device)
+ lafs, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
+ assert lafs.dtype == torch.float32
+ assert responses.dtype == torch.float16
+
+ @pytest.mark.parametrize(
+ "modules, scale_pyramid_input",
+ [
+ ({}, torch.float32),
+ ({"subpix_module": ConvSoftArgmax3d()}, torch.float32),
+ ({"subpix_module": type("Subclass", (ConvQuadInterp3d,), {})()}, torch.float32),
+ ({"resp_module": CornerHarris(0.04)}, torch.float32),
+ ({"resp_module": CornerGFTT()}, torch.float32),
+ (
+ {
+ "resp_module": BlobDoG(),
+ "scale_space_response": True,
+ "scale_pyr_module": ScalePyramid(3, 1.6, 15, extra_levels=3),
+ },
+ torch.float32,
+ ),
+ ({"resp_module": type("Subclass", (BlobHessian,), {})()}, torch.float16),
+ ({"resp_module": CornerHarris(torch.full((1, 1, 1, 1), 0.04))}, torch.float16),
+ ({"subpix_module": _RefinerWithState()}, torch.float16),
+ ({"scale_pyr_module": type("Subclass", (ScalePyramid,), {})(3, 1.6, 15, extra_levels=2)}, torch.float16),
+ ],
+ )
+ def test_float16_image_goes_through_float32_only_when_modules_allow(self, device, modules, scale_pyramid_input):
+ if scale_pyramid_input == torch.float16:
+ _skip_without_half_padding(device, torch.float16)
+ seen = []
+ det = ScaleSpaceDetector(5, **modules).to(device)
+ hook = det.scale_pyr.register_forward_pre_hook(lambda _, args: seen.append(args[0].dtype))
+ try:
+ det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
+ finally:
+ hook.remove()
+ assert seen == [scale_pyramid_input]
+
+ @pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
+ def test_float16_detection_dtype_reads_through_a_wrapped_module(self):
+ pyramid, response, subpix = ScalePyramid(), BlobHessian(), ConvQuadInterp3d()
+ assert _detects_float16_in_float32(torch.compile(pyramid), torch.compile(response), torch.compile(subpix))
+
def test_float_weights_above_one_are_clamped(self, device, dtype):
# A float 0/255 mask -- an OpenCV mask after `.astype(np.float32)` -- is weights, and a weight
# above one scaled every score by 255. Weights are clamped to one, so it means what the