π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
import torch
from torch.utils._python_dispatch import TorchDispatchMode
import kornia
from kornia.core.check import ShapeError
from kornia.core.exceptions import BaseError
from kornia.feature import BlobDoG, BlobHessian, CornerGFTT, CornerHarris, KeyNetDetector
from kornia.feature.scale_space_detector import (
MultiResolutionDetector,
ScaleSpaceDetector,
_detects_float16_in_float32,
_resize_mask,
get_default_detector_config,
)
from kornia.geometry.subpix import AdaptiveQuadInterp3d, ConvQuadInterp3d, ConvSoftArgmax3d, IterativeQuadInterp3d
from kornia.geometry.transform import ScalePyramid
from testing.base import (
DYNAMO_UNAVAILABLE_REASON,
BaseTester,
deterministic_algorithms_on_cuda,
dynamo_is_available,
supports_conv2d,
supports_reflect_padding,
supports_replicate_padding,
supports_topk,
)
from testing.default_device import default_device
class _RefinerWithState(ConvQuadInterp3d):
def __init__(self) -> None:
super().__init__()
self.register_buffer("state", torch.ones(1))
def _skip_without_half_padding(device: torch.device, dtype: torch.dtype) -> None:
# torch 2.5.1 has no float16 CPU reflection_pad2d or replication_pad2d; these pins blur (reflect) and take
# gradients (replicate).
if not (supports_reflect_padding(device, dtype) and supports_replicate_padding(device, dtype)):
pytest.skip(f"no reflection or replication padding kernel for {dtype} on {device.type}")
def _require_affine_orientation_kernels(device: torch.device, dtype: torch.dtype) -> None:
if dtype not in (torch.float16, torch.bfloat16):
return
if device.type == "mps":
pytest.skip("MPS autocast changes the effective dtype")
probes = (
("replicate-padding", supports_replicate_padding),
("conv2d", supports_conv2d),
("topk", supports_topk),
)
for name, probe in probes:
if not probe(device, dtype):
pytest.skip(f"no {name} kernel for {dtype} on {device.type}")
def _canonical_order(lafs: torch.Tensor, responses: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
"""Sort one image's detections into a batch-size-independent order.
`topk` breaks equal responses by position, and the positions it ranks over differ between a
batched and a single-image call, so comparing two calls row by row compares an ordering that is
not defined. Half precision makes those ties the norm rather than the exception: bfloat16
quantises this module's candidate responses to a handful of distinct values.
Sorting on the centre first turns the comparison into a set comparison. The remaining keys only
decide between rows that share a centre, and those agree to a ULP anyway, so the order does not
depend on last-bit differences the way a response-first sort would.
"""
# One transfer each, not one per scalar: reading the keys straight off the device tensors costs a
# separate synchronization per element, which is 4500 of them for the default 500 detections and
# measured 1215 ms against 1.81 ms on MPS for an identical result.
laf_rows = lafs.detach().cpu().tolist()
resp_rows = responses.detach().cpu().tolist()
rows = [(laf[0][2], laf[1][2], *laf[0], *laf[1], resp) for laf, resp in zip(laf_rows, resp_rows)]
order = torch.tensor(sorted(range(len(rows)), key=rows.__getitem__), device=lafs.device)
return lafs[order], responses[order]
class TestScaleSpaceDetector(BaseTester):
@pytest.mark.device_agnostic
@pytest.mark.parametrize("constructor", [ScalePyramid, ScaleSpaceDetector, kornia.feature.SIFTFeatureScaleSpace])
@pytest.mark.parametrize("use_set_default_device", [False, True])
def test_constructor_without_non_cpu_float64(self, constructor, use_set_default_device):
# Model MPS's float64 restriction on meta so this regression also runs on CPU-only machines.
class NoDoubleOffCPU(TorchDispatchMode):
def __torch_dispatch__(self, func, types, args=(), kwargs=None):
kwargs = kwargs or {}
if kwargs.get("dtype") == torch.float64:
assert torch.device(kwargs.get("device", "cpu")).type == "cpu"
return func(*args, **kwargs)
# SIFTFeatureScaleSpace explicitly moves to CPU unless its device argument is given.
constructor_kwargs = {"device": "meta"} if constructor is kornia.feature.SIFTFeatureScaleSpace else {}
# ``default_device`` leaves neither a DeviceContext mode nor a broken ``get_default_device`` behind (#5916).
with default_device("meta") if use_set_default_device else torch.device("meta"), NoDoubleOffCPU():
module = constructor(**constructor_kwargs)
pyramids = [child for child in module.modules() if isinstance(child, ScalePyramid)]
assert len(pyramids) == 1
buffers = dict(pyramids[0].named_buffers())
assert buffers
for kernel in buffers.values():
assert kernel.device.type == "meta"
assert kernel.dtype == torch.float32
@pytest.mark.parametrize("constructor", [ScaleSpaceDetector, kornia.feature.SIFTFeatureScaleSpace])
def test_constructor_under_default_device(self, device, constructor):
with device:
module = constructor()
pyramid = next(child for child in module.modules() if isinstance(child, ScalePyramid))
target = "cpu" if constructor is kornia.feature.SIFTFeatureScaleSpace else device
expected_device = torch.empty(0, device=target).device
for kernel in pyramid.buffers():
assert kernel.device == expected_device
assert kernel.dtype == torch.float32
@pytest.mark.parametrize("subpix_type", [AdaptiveQuadInterp3d, ConvQuadInterp3d, IterativeQuadInterp3d])
@pytest.mark.parametrize("batch", [1, 2])
@pytest.mark.parametrize("max_candidates", [None, 3])
def test_joint_extrema_matches_separate_refinement(self, device, dtype, subpix_type, batch, max_candidates):
# A subclass remains on the general extension path, which refines each sign
# independently. Pin the batched built-in path to that reference, including
# signed/weighted responses and gradients through the selected features. The value
# comparison alone cannot fail on a tree without the stacked call, where both detectors
# refine separately, so the refiner call counts pin that the built-in path was taken.
class SeparateSubpix(subpix_type):
pass
if subpix_type is ConvQuadInterp3d and max_candidates is not None:
pytest.skip("ConvQuadInterp3d has no candidate cap")
torch.manual_seed(17)
img = torch.rand(batch, 1, 96, 99, device=device, dtype=dtype, requires_grad=True)
mask = torch.rand(batch, 1, 96, 99, device=device, dtype=dtype).clamp_min(0.05)
# Zero only the top band: a float16 draw also rounds a few scattered weights to exactly 0, and the refined
# centre of a candidate beside one fails the mask re-check, which left this fixture without a detection.
mask[..., :8, :] = 0
def detector(subpix):
return ScaleSpaceDetector(
12,
mr_size=1.0,
scale_pyr_module=ScalePyramid(3, 1.6, 16, extra_levels=3),
resp_module=kornia.feature.BlobDoG(),
subpix_module=subpix,
minima_are_also_good=True,
scale_space_response=True,
).to(device, dtype)
def count_calls(module):
calls = []
forward = module.forward
def counted(*args, **kwargs):
calls.append(1)
return forward(*args, **kwargs)
module.forward = counted
return calls
kwargs = {}
if subpix_type is not ConvQuadInterp3d:
kwargs["max_candidates"] = max_candidates
joint, separate = subpix_type(**kwargs), SeparateSubpix(**kwargs)
joint_calls, separate_calls = count_calls(joint), count_calls(separate)
# On CUDA the packed Cramer solve of `conv_quad_interp3d` selects its system entries with a repeated-index
# `index_select`, whose backward accumulates with atomics in no fixed order. One keypoint of the batch-1
# fixture has an ill-conditioned fit, and the reordering moves the float32 image gradient there by up to
# 4.3e-5 -- within that gradient's own float32 error against float64 (5.5e-5), but over the float32
# tolerance -- so about a third of the runs failed, the capped variant too, where both detectors take the
# same separate path. Deterministic algorithms give both passes one accumulation order: the two detectors
# then agree bit for bit on CUDA, as they do on CPU, and the default tolerance stands.
with deterministic_algorithms_on_cuda(device):
actual = detector(joint)(img, mask)
expected = detector(separate)(img, mask)
assert actual[0].ne(0).any()
for a, e in zip(actual, expected):
self.assert_close(a, e)
# The subclass refines each sign of every octave in its own call. The built-in refiner
# stacks both signs into one call, unless a candidate cap keeps it on the separate path.
assert len(separate_calls) > 0
assert len(separate_calls) % 2 == 0
assert len(joint_calls) == (len(separate_calls) if max_candidates is not None else len(separate_calls) // 2)
actual_grad = torch.autograd.grad(sum(x.sum() for x in actual), img, retain_graph=True)[0]
expected_grad = torch.autograd.grad(sum(x.sum() for x in expected), img)[0]
self.assert_close(actual_grad, expected_grad)
def test_shape(self, device, dtype):
inp = torch.rand(1, 1, 32, 32, device=device, dtype=dtype)
n_feats = 10
det = ScaleSpaceDetector(n_feats).to(device, dtype)
lafs, resps = det(inp)
assert lafs.shape == torch.Size([1, n_feats, 2, 3])
assert resps.shape == torch.Size([1, n_feats])
def test_shape_batch(self, device, dtype):
inp = torch.rand(7, 1, 32, 32, device=device, dtype=dtype)
n_feats = 10
det = ScaleSpaceDetector(n_feats).to(device, dtype)
lafs, resps = det(inp)
assert lafs.shape == torch.Size([7, n_feats, 2, 3])
assert resps.shape == torch.Size([7, n_feats])
@pytest.mark.parametrize("with_mask", [False, True])
@pytest.mark.parametrize("config", ["default", "minima", "scale_space_response", "conv_subpix"])
def test_empty_batch(self, device, dtype, with_mask, config):
# A batch of zero images returns empty outputs of the documented shapes, with the dtypes a one-image
# call returns in the same input dtype (LAFs float32 for a half image, responses in the image dtype),
# still linked to the input for autograd.
kwargs = {
"default": {},
"minima": {"minima_are_also_good": True},
"scale_space_response": {"scale_space_response": True, "resp_module": BlobDoG()},
"conv_subpix": {"subpix_module": ConvQuadInterp3d()},
}[config]
det = ScaleSpaceDetector(50, **kwargs).to(device, dtype)
ones = torch.ones(1, 1, 64, 80, device=device, dtype=dtype)
one_img = torch.rand(1, 1, 64, 80, device=device, dtype=dtype).requires_grad_(True)
one_lafs, one_resps = det(one_img, ones if with_mask else None)
empty = torch.zeros(0, 1, 64, 80, device=device, dtype=dtype).requires_grad_(True)
mask = torch.ones(0, 1, 64, 80, device=device, dtype=dtype) if with_mask else None
lafs, resps = det(empty, mask)
assert lafs.shape == torch.Size([0, 50, 2, 3])
assert resps.shape == torch.Size([0, 50])
assert lafs.dtype == one_lafs.dtype
assert lafs.device == one_lafs.device
assert resps.dtype == one_resps.dtype
assert resps.device == one_resps.device
# Both outputs stay linked to the input, and backward runs.
assert lafs.requires_grad
assert resps.requires_grad
(lafs.sum() + resps.sum()).backward()
assert empty.grad is not None
assert empty.grad.shape == empty.shape
# `detect` with an explicit `num_feats` returns the requested count; a mask of one broadcasts.
one_resps, one_lafs = det.detect(torch.zeros(1, 1, 64, 80, device=device, dtype=dtype), 7)
resps, lafs = det.detect(empty.detach(), 7, ones if with_mask else None)
assert resps.shape == torch.Size([0, 7])
assert lafs.shape == torch.Size([0, 7, 2, 3])
assert lafs.dtype == one_lafs.dtype
assert resps.dtype == one_resps.dtype
def test_an_image_without_candidates_stays_linked_to_the_input(self, device, dtype):
# A DoG response is exactly zero everywhere on an all-zero image, so no octave has a candidate and every slot is
# unfilled; the empty selection must still be a function of the response, so a loss on it can backward.
det = ScaleSpaceDetector(10, scale_space_response=True, resp_module=BlobDoG()).to(device, dtype)
img = torch.zeros(1, 1, 64, 80, device=device, dtype=dtype, requires_grad=True)
lafs, resps = det(img)
assert torch.count_nonzero(resps) == 0
assert lafs.requires_grad
assert resps.requires_grad
(lafs.sum() + resps.sum()).backward()
assert img.grad is not None
# An image without candidates has no gradient to give.
assert torch.count_nonzero(img.grad) == 0
@pytest.mark.parametrize("subpix_type", [IterativeQuadInterp3d, ConvQuadInterp3d])
@pytest.mark.parametrize("image", ["flat", "ramp", "near-flat"])
def test_a_flat_image_has_a_finite_gradient(self, device, dtype, subpix_type, image):
# On a flat float32 image the Hessian entries of the sub-pixel fit are rounding noise of about 1e-15. Their
# determinants reach float32's smallest subnormal, under a relative floor that underflowed to 0, and the
# backward of the Cramer solve divided by them twice: the inf times a later mask's zero gradient made every
# pixel's gradient NaN.
_skip_without_half_padding(device, dtype)
generator = torch.Generator().manual_seed(0)
if image == "flat":
img = torch.full((1, 1, 64, 80), 0.5)
elif image == "ramp":
img = torch.linspace(0, 1, 80).expand(1, 1, 64, 80).clone()
else:
img = 0.5 + 1e-6 * torch.rand(1, 1, 64, 80, generator=generator)
img = img.to(device, dtype).requires_grad_()
det = ScaleSpaceDetector(10, subpix_module=subpix_type()).to(device, dtype)
lafs, resps = det(img)
(lafs.sum() + resps.sum()).backward()
assert bool(torch.isfinite(img.grad).all())
@pytest.mark.parametrize(
"img_tail, mask_shape, error",
[
((3, 64, 80), None, BaseError),
((64, 80), None, ValueError),
((1, 64, 80), (1, 64, 80), BaseError),
((1, 64, 80), (1, 3, 64, 80), BaseError),
((1, 64, 80), (1, 1, 32, 80), BaseError),
((1, 64, 80), (2, 1, 64, 80), BaseError),
],
ids=["3-channels", "3-d-image", "mask-rank-3", "mask-3-channels", "mask-size", "mask-batch"],
)
def test_empty_batch_is_validated_like_a_non_empty_one(self, device, dtype, img_tail, mask_shape, error):
det = ScaleSpaceDetector(5).to(device, dtype)
mask = None if mask_shape is None else torch.ones(mask_shape, device=device, dtype=dtype)
for batch in (0, 1):
img = torch.zeros(batch, *img_tail, device=device, dtype=dtype)
with pytest.raises(error):
det(img, mask)
def test_toy(self, device, dtype):
inp = torch.zeros(1, 1, 33, 33, device=device, dtype=dtype)
inp[:, :, 13:-13, 13:-13] = 1.0
n_feats = 1
det = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0).to(device, dtype)
lafs, resps = det(inp)
expected_laf = torch.tensor(
[[[[8.4260, 0.0000, 16.0], [0.0, 8.4260, 16.0]]]],
device=device,
dtype=torch.promote_types(dtype, torch.float32),
)
expected_resp = torch.tensor([[0.1159]], device=device, dtype=dtype)
# The scale is `sigma * 2 ** (level / num_levels)` after a sub-pixel refinement; float16
# resolves it to ~3e-3 and bfloat16, with its 8-bit mantissa, to ~3e-2.
rtol = {torch.float16: 5e-3, torch.bfloat16: 4e-2}.get(dtype, 1e-3)
self.assert_close(lafs, expected_laf, rtol=rtol, atol=1e-03)
self.assert_close(resps, expected_resp, rtol=rtol, atol=1e-03)
def test_toy_mask(self, device, dtype):
inp = torch.zeros(1, 1, 33, 33, device=device, dtype=dtype)
inp[:, :, 13:-13, 13:-13] = 1.0
mask = torch.zeros(1, 1, 33, 33, device=device, dtype=dtype)
mask[:, :, 1:-1, 3:-3] = 1.0
n_feats = 1
det = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0).to(device, dtype)
lafs, resps = det(inp, mask)
expected_laf = torch.tensor(
[[[[8.4260, 0.0000, 16.0], [0.0, 8.4260, 16.0]]]],
device=device,
dtype=torch.promote_types(dtype, torch.float32),
)
expected_resp = torch.tensor([[0.1159]], device=device, dtype=dtype)
# The scale is `sigma * 2 ** (level / num_levels)` after a sub-pixel refinement; float16
# resolves it to ~3e-3 and bfloat16, with its 8-bit mantissa, to ~3e-2.
rtol = {torch.float16: 5e-3, torch.bfloat16: 4e-2}.get(dtype, 1e-3)
self.assert_close(lafs, expected_laf, rtol=rtol, atol=1e-03)
self.assert_close(resps, expected_resp, rtol=rtol, atol=1e-03)
def test_mask_does_not_promote_the_response_dtype(self, device, dtype):
# The mask is resampled and cast onto the response map, so a mask in a different dtype
# cannot pull the detector's output away from the image dtype.
inp = torch.rand(1, 1, 32, 32, device=device, dtype=dtype)
other = torch.float64 if dtype != torch.float64 else torch.float32
if device.type == "mps" and other == torch.float64: # MPS has no float64
other = torch.float32 if dtype != torch.float32 else torch.float16
assert other != dtype
mask = torch.ones(1, 1, 32, 32, device=device, dtype=other)
det = ScaleSpaceDetector(5).to(device, dtype)
lafs, resps = det(inp, mask)
assert lafs.dtype == torch.promote_types(dtype, torch.float32)
assert resps.dtype == dtype
def test_mask_spatial_size_must_match_the_image(self, device, dtype):
# `_create_octave_mask` resamples the mask onto every octave, so without this check a
# mask of any size is stretched silently onto the wrong geometry.
det = ScaleSpaceDetector(5).to(device, dtype)
inp = torch.rand(1, 1, 32, 32, device=device, dtype=dtype)
with pytest.raises(Exception, match="spatial size"):
det(inp, torch.ones(1, 1, 8, 8, device=device, dtype=dtype))
with pytest.raises(Exception, match="spatial size"):
det(inp, torch.ones(1, 1, 32, 16, device=device, dtype=dtype))
def test_mask_must_be_single_channel(self, device, dtype):
# `_create_octave_mask` broadcasts the mask over the scale levels, so a channel axis would
# land on the level axis: an error when the counts differ, the wrong weighting when they match.
det = ScaleSpaceDetector(5).to(device, dtype)
inp = torch.rand(2, 1, 32, 32, device=device, dtype=dtype)
with pytest.raises(Exception, match="1, H, W"):
det(inp, torch.ones(2, 3, 32, 32, device=device, dtype=dtype))
with pytest.raises(Exception, match="batch"):
det(inp, torch.ones(3, 1, 32, 32, device=device, dtype=dtype))
for b in (1, 2):
lafs, _ = det(inp, torch.ones(b, 1, 32, 32, device=device, dtype=dtype))
assert lafs.shape == torch.Size([2, 5, 2, 3])
def test_unfilled_slots_carry_a_zero_laf(self, device, dtype):
# A slot that no detection filled -- the padding, or a candidate whose frame reaches
# outside the image -- has a zero response and a zero LAF, and every real detection sorts
# ahead of it. On `main` a border-rejected candidate kept its coordinates beside a zero
# response, i.e. a keypoint that was never detected.
torch.manual_seed(0)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = ScaleSpaceDetector(50).to(device, dtype)
resps, lafs = det.detect(inp, 50)
zero_laf = (lafs[0] == 0).all(dim=-1).all(dim=-1)
empty = resps[0] == 0
assert bool(zero_laf.any()), "expected this image to under-fill 50 slots"
assert bool((~zero_laf).any()), "expected this image to yield real detections"
assert torch.equal(zero_laf, empty)
n_real = int((~zero_laf).sum())
assert bool((~zero_laf[:n_real]).all()), "real detections must come first"
lafs_fwd, resps_fwd = det(inp)
assert torch.equal(resps_fwd, resps)
assert torch.equal(lafs_fwd, lafs)
def test_negative_detections_sort_before_the_padding(self, device, dtype):
# A signed response function can have every maximum below zero. The padding must still
# come last: it is ranked with a sentinel and zeroed only afterwards.
class NegatedHessian(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.resp = kornia.feature.BlobHessian()
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
return self.resp(x) - 1.0
if dtype in (torch.float16, torch.bfloat16):
# The shifted response is flat at half precision and yields no maxima at all.
pytest.skip("a Hessian response offset by 1.0 has no resolution left in half precision")
# Gaussian blobs of increasing size give the Hessian one scale-space maximum per blob. Random noise
# left one or two maxima whose survival depended on the exact response scale and on the dtype.
# Built in float64 on the CPU so every device sees the same input: MPS has no float64.
coords = torch.arange(96, dtype=torch.float64)
yy, xx = torch.meshgrid(coords, coords, indexing="ij")
inp = torch.zeros(96, 96, dtype=torch.float64)
for i, sigma in enumerate((1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 5.0, 6.0, 7.0)):
cy, cx = 16 + 32 * (i // 3), 16 + 32 * (i % 3)
inp += torch.exp(-((yy - cy) ** 2 + (xx - cx) ** 2) / (2 * sigma**2))
inp = inp[None, None].to(device=device, dtype=dtype)
det = ScaleSpaceDetector(50, resp_module=NegatedHessian()).to(device, dtype)
lafs, resps = det(inp)
filled = lafs[0].ne(0).any(dim=-1).any(dim=-1)
n_real = int(filled.sum())
assert 0 < n_real < 50, f"expected a partially filled result, got {n_real}"
assert bool(filled[:n_real].all())
assert not bool(filled[n_real:].any())
assert (resps[0, :n_real] < 0).all()
assert (resps[0, n_real:] == 0).all()
def test_float_mask_never_promotes_a_down_weighted_negative_response(self, device, dtype):
# A weight scales a score toward the worst, not toward zero: a negative score is divided
# by the weight, a positive one multiplied. A plain multiply pulled a down-weighted negative
# score toward zero and ranked it above every full-weight one.
class SignedSpikes(torch.nn.Module):
# Four maxima on a -1 floor, all negative: two in the left half, two in the right.
spikes = {(24, 24): -0.20, (24, 72): -0.25, (72, 24): -0.40, (72, 72): -0.30}
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = -torch.ones_like(x)
if out.shape[-1] == 96: # the full-resolution octave only; the coarser ones stay flat
for (y, xx), v in self.spikes.items():
out[:, :, out.shape[2] // 2, y, xx] = v
return out
det = ScaleSpaceDetector(4, resp_module=SignedSpikes(), scale_space_response=True, mr_size=3.0).to(
device, dtype
)
img = torch.zeros(1, 1, 96, 96, device=device, dtype=dtype)
lafs_plain, resps_plain = det(img)
assert bool(lafs_plain.ne(0).any(dim=-1).any(dim=-1).all()), "expected all four spikes"
assert resps_plain[0].tolist() == sorted(resps_plain[0].tolist(), reverse=True)
mask = torch.ones(1, 1, 96, 96, device=device, dtype=dtype)
mask[..., 48:] = 0.5 # the right half is down-weighted
lafs, resps = det(img, mask)
xs = lafs[0, :, 0, 2]
right = xs >= 48
# Same four frames; the full-weight (left, x=24) ones keep their score, the down-weighted
# (right, x=72) ones get worse, and the down-weighted best (-0.25) no longer ranks above the
# full-weight runner-up (-0.40) -- a multiply would have reported it as -0.125 and put it first.
assert int(right.sum()) == 2
self.assert_close(resps[0][~right], torch.tensor([-0.20, -0.40], device=device, dtype=dtype))
self.assert_close(resps[0][right], torch.tensor([-0.50, -0.60], device=device, dtype=dtype))
assert resps[0].tolist() == sorted(resps[0].tolist(), reverse=True)
assert xs[0] < 48, f"a down-weighted candidate outranked a full-weight one: {xs.tolist()}"
assert xs[1] < 48, f"a down-weighted candidate outranked a full-weight one: {xs.tolist()}"
def test_a_weighted_negative_response_never_sorts_below_the_padding(self, device, dtype):
# A weight *divides* a negative score, and a small weight can push the quotient past the
# dtype's range: -inf ties with the `-inf` an unfilled slot carries into the ranking, so a
# non-detection could outrank a real one. The weighted score is kept finite instead.
floor = -float(torch.finfo(dtype).max) / 4
spike = -float(torch.finfo(dtype).max) / 8
class HugeNegativeSpikes(torch.nn.Module):
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = torch.full_like(x, floor)
if out.shape[-1] == 96:
for y, xx in ((24, 24), (24, 72), (72, 24), (72, 72)):
out[:, :, out.shape[2] // 2, y, xx] = spike
return out
det = ScaleSpaceDetector(4, resp_module=HugeNegativeSpikes(), scale_space_response=True, mr_size=3.0).to(
device, dtype
)
img = torch.zeros(1, 1, 96, 96, device=device, dtype=dtype)
mask = torch.full((1, 1, 96, 96), 1e-3, device=device, dtype=dtype)
lafs, resps = det(img, mask)
filled = lafs.ne(0).any(dim=-1).any(dim=-1)
assert int(filled.sum()) == 4, "expected all four spikes"
assert torch.isfinite(resps).all(), f"a weighted score overflowed: {resps.tolist()}"
def test_an_extreme_finite_response_still_outranks_the_padding(self, device, dtype):
# `finfo.min / 2` is not below every finite response: four maxima at `-2e38` (float32) on a
# `finfo.min` floor are real detections, and the single-image path returned all four while
# the batched top-K preferred the sentinel and returned none. Padding is ranked with `-inf`,
# and a finite response, however extreme, sorts ahead of it on both paths.
floor = float(torch.finfo(dtype).min)
spike = -float(torch.finfo(dtype).max) * 0.6
class ExtremeSpikes(torch.nn.Module):
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = torch.full_like(x, floor)
if out.shape[-1] == 96:
for y, xx in ((24, 24), (24, 72), (72, 24), (72, 72)):
out[:, :, out.shape[2] // 2, y, xx] = spike
return out
det = ScaleSpaceDetector(4, resp_module=ExtremeSpikes(), scale_space_response=True, mr_size=3.0).to(
device, dtype
)
img = torch.zeros(2, 1, 96, 96, device=device, dtype=dtype)
lafs1, resps1 = det(img[:1])
lafs2, resps2 = det(img)
filled1 = lafs1.ne(0).any(dim=-1).any(dim=-1)
filled2 = lafs2.ne(0).any(dim=-1).any(dim=-1)
assert int(filled1.sum()) == 4, f"single-image path lost a detection: {resps1.tolist()}"
assert int(filled2[0].sum()) == 4, f"batched path lost one: {resps2.tolist()}"
assert int(filled2[1].sum()) == 4, f"batched path lost one: {resps2.tolist()}"
assert torch.isfinite(resps2).all()
self.assert_close(resps2[0], resps1[0])
def test_subpixel_refinement_cannot_cross_the_mask(self, device, dtype):
# The mask is checked at the integer NMS site, but the sub-pixel step moves the centre. A
# parabola peaked at x = 31.6 has its discrete maximum at x = 32; a mask allowing only
# x >= 32 passed that site and returned a centre of 31.6, inside the zero region. The refined
# centre is re-checked against the resampled mask conservatively (every pixel it touches).
class Parabola(torch.nn.Module):
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
B, L, H, W = x.shape[0], x.shape[2], x.shape[-2], x.shape[-1]
xs = torch.arange(W, device=x.device, dtype=x.dtype).view(1, 1, 1, 1, W)
ys = torch.arange(H, device=x.device, dtype=x.dtype).view(1, 1, 1, H, 1)
ls = torch.arange(L, device=x.device, dtype=x.dtype).view(1, 1, L, 1, 1)
return (10.0 - (xs - 31.6) ** 2 - (ys - 48.0) ** 2 - (ls - L // 2) ** 2).expand(B, 1, L, H, W).clone()
det = ScaleSpaceDetector(1, resp_module=Parabola(), scale_space_response=True, mr_size=1.0).to(device, dtype)
img = torch.zeros(1, 1, 96, 96, device=device, dtype=dtype)
mask = torch.zeros(1, 1, 96, 96, device=device, dtype=torch.bool)
mask[..., 32:] = True
lafs, resps = det(img, mask)
assert bool((lafs == 0).all()), f"a refined centre at {lafs[0, 0, :, 2].tolist()} lies in the masked-out region"
assert bool((resps == 0).all()), (
f"a refined centre at {lafs[0, 0, :, 2].tolist()} lies in the masked-out region"
)
# Control: the same peak is returned, refined, when the mask allows the pixels it touches.
mask[..., 31:] = True
lafs, resps = det(img, mask)
assert bool(resps[0, 0] > 0)
self.assert_close(
lafs[0, 0, :, 2], torch.tensor([31.6, 48.0], device=device, dtype=lafs.dtype), atol=0.2, rtol=0
)
def test_fill_sentinel_follows_the_response_dtype(self, device, dtype):
# The top-K sentinel is written into the *response* tensor, so it must fit that dtype: a
# response module that emits a narrower dtype than the image -- a learned response under
# autocast -- raised "value cannot be converted to type at::Half without overflow".
if not supports_topk(device, torch.float16):
# The top-K ranks the response, which this module narrows to float16; torch 2.1.2 has
# no float16 CPU `topk` kernel.
pytest.skip(f"no topk kernel for torch.float16 on {device.type}")
class NarrowResponse(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.resp = kornia.feature.BlobHessian()
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
return self.resp(x, sigmas).to(torch.float16)
torch.manual_seed(0)
inp = torch.rand(2, 1, 64, 64, device=device, dtype=dtype)
det = ScaleSpaceDetector(20, resp_module=NarrowResponse()).to(device, dtype)
for b in (1, 2):
lafs, resps = det(inp[:b])
assert resps.dtype == torch.float16
assert torch.isfinite(resps).all()
filled = lafs.ne(0).any(dim=-1).any(dim=-1)
assert 0 < int(filled[0].sum()) < 20
assert bool((resps[~filled] == 0).all())
def test_float_weights_are_cast_to_the_response_dtype(self, device, dtype):
# The refinement runs on a float32 copy of a half-precision response, but the weights are still cast to the
# response's dtype first: a weight of 1e-8 is zero in float16, where it suppresses, and positive otherwise.
if device.type == "mps" and dtype == torch.float64:
pytest.skip("MPS has no float64")
_skip_without_half_padding(device, dtype)
g = torch.Generator().manual_seed(0)
img = torch.nn.functional.avg_pool2d(torch.rand(1, 1, 96, 128, generator=g), 5, 1, 2).to(device, dtype)
det = ScaleSpaceDetector(8).to(device, dtype)
assert bool(kornia.feature.laf_is_filled(det(img)[0]).any())
lafs, _ = det(img, torch.full((1, 1, 96, 128), 1e-8, device=device))
assert bool(kornia.feature.laf_is_filled(lafs).any()) == (dtype != torch.float16)
@pytest.mark.parametrize("subpix", [ConvQuadInterp3d, IterativeQuadInterp3d])
@pytest.mark.parametrize("convert_module", [False, True])
def test_float16_image_gradient_matches_float32_6143(self, device, subpix, convert_module):
# The true gradient of about 10 fits float16, but its way through the response does not: 4e4 * sigma ** 4.
image = torch.rand(1, 1, 64, 80, generator=torch.Generator().manual_seed(0))
gradients = {}
for image_dtype in (torch.float32, torch.float16):
det = ScaleSpaceDetector(10, subpix_module=subpix()).to(device)
if convert_module:
det = det.to(image_dtype)
x = image.to(device, image_dtype).clone().requires_grad_()
lafs, responses = det(x)
assert lafs.dtype == torch.float32
assert responses.dtype == image_dtype
gradients[image_dtype] = torch.autograd.grad(lafs.sum() + responses.sum(), x)[0]
assert gradients[torch.float16].isfinite().all()
# Rounding the image to float16 moves this gradient by up to 0.015 (CPU, CUDA).
self.assert_close(gradients[torch.float16].float(), gradients[torch.float32], rtol=0, atol=0.05)
def test_float16_image_fills_as_many_slots_as_float32_at_low_contrast(self, device):
# Rounding the response to float16 before NMS merges maxima closer than one float16 step: 4 slots here, not 7.
image = torch.rand(1, 1, 64, 80, generator=torch.Generator().manual_seed(0))
image = (0.5 + (image - 0.5) * 0.01).to(device)
filled = {}
for image_dtype in (torch.float32, torch.float16):
x = image.to(image_dtype).clone().requires_grad_()
lafs, responses = ScaleSpaceDetector(10, subpix_module=ConvQuadInterp3d()).to(device)(x)
filled[image_dtype] = int(kornia.feature.laf_is_filled(lafs).sum())
gradient = torch.autograd.grad(lafs.sum() + responses.sum(), x)[0]
assert gradient.isfinite().all()
assert filled[torch.float16] == filled[torch.float32]
def test_float16_custom_response_keeps_its_dtype_and_nms_precision(self, device):
# A custom response is not rounded to float16: an isolated peak of 1.0001 on a floor of 1.0 stays a maximum.
class WideResponse(torch.nn.Module):
def forward(self, x, sigmas=None):
out = torch.ones_like(x, dtype=torch.float32)
out[out.shape[0] // 2, :, out.shape[-2] // 2, out.shape[-1] // 2] = 1.0001
return out
_skip_without_half_padding(device, torch.float16)
det = ScaleSpaceDetector(3, resp_module=WideResponse()).to(device)
lafs, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
assert responses.dtype == torch.float32
assert kornia.feature.laf_is_filled(lafs).any()
def test_float16_response_with_weights_gets_float16_input(self, device):
_skip_without_half_padding(device, torch.float16)
class LearnedResponse(torch.nn.Module):
def __init__(self):
super().__init__()
self.conv = torch.nn.Conv2d(1, 1, 3, padding=1)
def forward(self, x, sigmas=None):
return self.conv(x)
resp = LearnedResponse().to(device, torch.float16)
det = ScaleSpaceDetector(5, resp_module=resp).to(device)
_, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
assert responses.dtype == torch.float16
@pytest.mark.parametrize("subpix", [ConvQuadInterp3d, IterativeQuadInterp3d])
@pytest.mark.parametrize("region_dtype", [torch.float16, torch.bfloat16])
def test_float16_image_gradient_in_a_region_6143(self, device, subpix, region_dtype):
if device.type == "cuda" and region_dtype == torch.bfloat16 and not torch.cuda.is_bf16_supported():
pytest.skip("bfloat16 autocast is not supported on this CUDA device")
# Autocast would turn the float32 copy's convolutions back to its own dtype, and float16 overflows there too.
image = torch.rand(1, 1, 64, 80, generator=torch.Generator().manual_seed(0)).to(device)
reference = image.clone().requires_grad_()
lafs, responses = ScaleSpaceDetector(10, subpix_module=subpix()).to(device)(reference)
expected = torch.autograd.grad(lafs.sum() + responses.sum(), reference)[0]
x = image.to(torch.float16).clone().requires_grad_()
with torch.autocast(device.type, dtype=region_dtype):
lafs, responses = ScaleSpaceDetector(10, subpix_module=subpix()).to(device)(x)
assert lafs.dtype == torch.float32
assert responses.dtype == torch.float16
gradient = torch.autograd.grad(lafs.sum() + responses.sum(), x)[0]
assert gradient.isfinite().all()
# Rounding the image to float16 moves this gradient by up to 0.015 (CPU, CUDA).
self.assert_close(gradient.float(), expected, rtol=0, atol=0.05)
@pytest.mark.parametrize("response_dtype", [torch.float32, torch.float64])
def test_float16_image_keeps_a_wider_custom_response(self, device, response_dtype):
if response_dtype == torch.float64 and device.type == "mps":
pytest.skip("MPS has no float64")
_skip_without_half_padding(device, torch.float16)
seen = []
class IsolatedPeak(torch.nn.Module):
def forward(self, x, sigmas=None):
seen.append(x.dtype)
response = torch.ones(x.shape[0], 1, *x.shape[-2:], device=x.device, dtype=response_dtype)
response[2, 0, x.shape[-2] // 2, x.shape[-1] // 2] = 1.0001 # one level; float16 rounds it to 1
return response
det = ScaleSpaceDetector(5, resp_module=IsolatedPeak(), subpix_module=ConvQuadInterp3d()).to(device)
_, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
assert set(seen) == {torch.float16}
assert responses.dtype == response_dtype
assert responses[0, 0] > 1.0
def test_float16_image_reaches_a_custom_pyramid_with_float16_weights(self, device):
_skip_without_half_padding(device, torch.float16)
class LearnedPyramid(torch.nn.Module):
n_levels = 3
def __init__(self):
super().__init__()
self.conv = torch.nn.Conv2d(1, 1, 3, padding=1)
self.pyramid = ScalePyramid(3, 1.6, 15, extra_levels=2)
def forward(self, x):
return self.pyramid(self.conv(x))
det = ScaleSpaceDetector(5, scale_pyr_module=LearnedPyramid().to(device, torch.float16)).to(device)
lafs, responses = det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
assert lafs.dtype == torch.float32
assert responses.dtype == torch.float16
@pytest.mark.parametrize(
"modules, scale_pyramid_input",
[
({}, torch.float32),
({"subpix_module": ConvSoftArgmax3d()}, torch.float32),
({"subpix_module": type("Subclass", (ConvQuadInterp3d,), {})()}, torch.float32),
({"resp_module": CornerHarris(0.04)}, torch.float32),
({"resp_module": CornerGFTT()}, torch.float32),
(
{
"resp_module": BlobDoG(),
"scale_space_response": True,
"scale_pyr_module": ScalePyramid(3, 1.6, 15, extra_levels=3),
},
torch.float32,
),
({"resp_module": type("Subclass", (BlobHessian,), {})()}, torch.float16),
({"resp_module": CornerHarris(torch.full((1, 1, 1, 1), 0.04))}, torch.float16),
({"subpix_module": _RefinerWithState()}, torch.float16),
({"scale_pyr_module": type("Subclass", (ScalePyramid,), {})(3, 1.6, 15, extra_levels=2)}, torch.float16),
],
)
def test_float16_image_goes_through_float32_only_when_modules_allow(self, device, modules, scale_pyramid_input):
if scale_pyramid_input == torch.float16:
_skip_without_half_padding(device, torch.float16)
seen = []
det = ScaleSpaceDetector(5, **modules).to(device)
hook = det.scale_pyr.register_forward_pre_hook(lambda _, args: seen.append(args[0].dtype))
try:
det(torch.rand(1, 1, 64, 80, device=device, dtype=torch.float16))
finally:
hook.remove()
assert seen == [scale_pyramid_input]
@pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
def test_float16_detection_dtype_reads_through_a_wrapped_module(self):
pyramid, response, subpix = ScalePyramid(), BlobHessian(), ConvQuadInterp3d()
assert _detects_float16_in_float32(torch.compile(pyramid), torch.compile(response), torch.compile(subpix))
def test_float_weights_above_one_are_clamped(self, device, dtype):
# A float 0/255 mask -- an OpenCV mask after `.astype(np.float32)` -- is weights, and a weight
# above one scaled every score by 255. Weights are clamped to one, so it means what the
# integer 0/255 mask means.
torch.manual_seed(0)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = ScaleSpaceDetector(50).to(device, dtype)
lafs_b, resps_b = det(inp, torch.ones(1, 1, 64, 64, device=device, dtype=torch.bool))
lafs_f, resps_f = det(inp, torch.full((1, 1, 64, 64), 255.0, device=device, dtype=dtype))
assert torch.equal(resps_f, resps_b)
assert torch.equal(lafs_f, lafs_b)
def test_a_zero_response_maximum_keeps_its_laf(self, device, dtype):
# The response function is pluggable and may be signed, so an exact zero can be a genuine
# maximum rather than an unfilled slot. Classifying the padding by `response == 0` erased
# all three detections here.
class SignedResponse(torch.nn.Module):
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = -torch.ones_like(x)
out[:, :, out.shape[2] // 2, out.shape[-2] // 2, out.shape[-1] // 2] = 0.0
return out
# `mr_size=3` keeps the coarsest octave's frame inside the 96 px image; at the default 6 it
# is 76.8 px wide, and a candidate whose frame leaves the image is not a detection.
det = ScaleSpaceDetector(5, resp_module=SignedResponse(), scale_space_response=True, mr_size=3.0).to(
device, dtype
)
lafs, resps = det(torch.zeros(1, 1, 96, 96, device=device, dtype=dtype))
assert (resps == 0).all()
centers = lafs[0, :, :, 2]
# The maximum sits at octave pixel W_o // 2 of the 96, 48 and 24 px octaves, which ScalePyramid's
# align_corners=True resizes put at 48, 24 * 95 / 47 and 12 * 95 / 23 in the input.
expected = torch.tensor([48.0, 24 * 95 / 47, 12 * 95 / 23], device=device, dtype=centers.dtype)
atol = 1e-9 if dtype == torch.float64 else 1e-4 # float32 centres for float16 and bfloat16 images too
found = torch.isclose(centers[:, None, :], expected[None, :, None], rtol=0.0, atol=atol).all(-1).any(0)
assert bool(found.all()), f"expected the three maxima at {expected.tolist()}, got {centers.tolist()}"
def test_batched_underfill_does_not_leak_the_topk_sentinel(self, device, dtype):
# For B > 1 the octave top-K ranks over the whole volume with non-candidates masked to
# `finfo.min / 2`. An image with fewer maxima than requested gets those sentinels back;
# they are not detections and must not reach the caller as a response or as a LAF.
torch.manual_seed(0)
inp = torch.rand(2, 1, 64, 64, device=device, dtype=dtype)
det = ScaleSpaceDetector().to(device, dtype)
lafs, resps = det(inp)
filled = lafs.ne(0).any(dim=-1).any(dim=-1)
assert 0 < int(filled[0].sum()) < det.num_features, "expected a partially filled result"
assert bool((resps[~filled] == 0).all()), f"sentinel leaked: min response {resps.min().item()}"
assert (resps > torch.finfo(dtype).min / 4).all()
# and the single-image path agrees, which it did not before. As a set, not row by row: the
# two calls rank over differently shaped volumes, so `topk` orders equal responses
# differently, and in bfloat16 this input's responses tie constantly (#4219). Up to
# tolerance, not bit-for-bit either: the response convolutions pick batch-size-dependent
# kernels on some CPUs (ubuntu CI), so the same candidates carry last-ULP-different scores.
lafs1, resps1 = det(inp[:1])
batched_lafs, batched_resps = _canonical_order(lafs[0], resps[0])
single_lafs, single_resps = _canonical_order(lafs1[0], resps1[0])
self.assert_close(single_resps, batched_resps)
self.assert_close(single_lafs, batched_lafs)
def test_padding_survives_the_affine_and_orientation_modules(self, device, dtype):
# Same contract as `MultiResolutionDetector.forward`: the shape and orientation modules
# may transform a zero frame differently by dtype, so the padding is re-applied after them.
_require_affine_orientation_kernels(device, dtype)
det = ScaleSpaceDetector(
10, aff_module=kornia.feature.LAFAffineShapeEstimator(19), ori_module=kornia.feature.LAFOrienter(19)
).to(device, dtype)
lafs, resps = det(torch.zeros(1, 1, 64, 64, device=device, dtype=dtype))
assert (resps == 0).all()
assert (lafs == 0).all()
def test_forward_uses_overridden_detect(self, device, dtype):
class CustomDetector(ScaleSpaceDetector):
def detect(self, img, num_feats, mask=None):
responses = img.new_full((img.shape[0], num_feats), 123.0)
lafs = img.new_full((img.shape[0], num_feats, 2, 3), 7.0)
return responses, lafs
inp = torch.zeros(2, 1, 32, 32, device=device, dtype=dtype)
det = CustomDetector(3).to(device, dtype)
lafs, responses = det(inp)
assert torch.equal(responses, inp.new_full((2, 3), 123.0))
assert torch.equal(lafs, inp.new_full((2, 3, 2, 3), 7.0))
@pytest.mark.parametrize("subpix", ["adaptive", "conv"])
def test_color_input_with_single_channel_response(self, device, dtype, subpix):
class ColorResponse(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.response = kornia.feature.BlobHessian()
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
# Forward the sigmas as the default BlobHessian gets them. Without the sigma**4 normalisation the
# response falls with scale, so a candidate needs a noise feature that beats the level below it,
# and 8-18% of noise images had none; this seed's CUDA float64 draw was one of them.
return self.response(x.mean(dim=1, keepdim=True), sigmas)
torch.manual_seed(3)
inp = torch.rand(1, 3, 96, 96, device=device, dtype=dtype)
kwargs = {} if subpix == "adaptive" else {"subpix_module": ConvQuadInterp3d(10)}
det = ScaleSpaceDetector(20, resp_module=ColorResponse(), **kwargs).to(device, dtype)
lafs, responses = det(inp)
assert lafs.shape == torch.Size([1, 20, 2, 3])
assert responses.shape == torch.Size([1, 20])
valid = lafs.ne(0).any(dim=-1).any(dim=-1)
centers = lafs[..., 2][valid]
assert bool(valid.any())
assert (centers[:, 0] >= 0).all()
assert (centers[:, 0] <= 95).all()
assert (centers[:, 1] >= 0).all()
assert (centers[:, 1] <= 95).all()
def test_multichannel_response_is_rejected(self, device, dtype):
inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype)
detector = ScaleSpaceDetector(20, resp_module=kornia.feature.BlobHessian()).to(device, dtype)
with pytest.raises(Exception, match="one response map"):
detector(inp)
def test_minima_are_also_good(self, device, dtype):
# Image with a bright blob (local max) and dark blob (local min).
# With minima_are_also_good=True both should contribute to detections.
inp = torch.ones(1, 1, 33, 33, device=device, dtype=dtype) * 0.5
inp[:, :, 10:14, 10:14] = 1.0 # bright blob β local maximum
inp[:, :, 10:14, 20:24] = 0.0 # dark blob β local minimum
n_feats = 2
det_max_only = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0).to(
device, dtype
)
det_minmax = ScaleSpaceDetector(
n_feats, resp_module=kornia.feature.BlobHessian(), mr_size=3.0, minima_are_also_good=True
).to(device, dtype)
lafs_max, resps_max = det_max_only(inp)
lafs_minmax, resps_minmax = det_minmax(inp)
assert lafs_max.shape == torch.Size([1, n_feats, 2, 3])
assert lafs_minmax.shape == torch.Size([1, n_feats, 2, 3])
# minmax detector should find a higher total response magnitude (it sees both blobs).
assert resps_minmax.abs().sum() >= resps_max.abs().sum()
def test_scale_space_response_mode(self, device, dtype):
# Smoke test: scale_space_response=True uses a different internal code path.
# BlobDoG operates on the 5D scale-space tensor directly.
inp = torch.rand(1, 1, 32, 32, device=device, dtype=dtype)
n_feats = 5
det = ScaleSpaceDetector(n_feats, resp_module=kornia.feature.BlobDoG(), scale_space_response=True).to(
device, dtype
)
lafs, resps = det(inp)
assert lafs.shape == torch.Size([1, n_feats, 2, 3])
assert resps.shape == torch.Size([1, n_feats])
@pytest.mark.parametrize("sigma", [2.016, 4.032])
def test_octave_boundary_scale_is_detected_once_5670(self, device, dtype, sigma):
# sigma = 1.6 * 2 ** (4 / 3) * 2 ** k is level 4 of one octave and level 1 of the next. With
# extra_levels=3 a per-level response has a level 4 in every octave, but it is only the NMS neighbour of
# level 3: searching it as well found this blob once in each octave.
if dtype == torch.bfloat16:
pytest.skip(
"in bfloat16 the octave-1 copy of this blob does not survive NMS, with or without a spare level"
)
yy, xx = torch.meshgrid(
torch.arange(160, dtype=torch.float64), torch.arange(200, dtype=torch.float64), indexing="ij"
)
blob = torch.exp(-((xx - 150.3) ** 2 + (yy - 110.6) ** 2) / (2 * sigma**2))
img = blob[None, None].to(device, dtype)
det = ScaleSpaceDetector(
16,
resp_module=kornia.feature.BlobHessian(),
scale_pyr_module=ScalePyramid(3, 1.6, 32, double_image=True, extra_levels=3),
).to(device, dtype)
lafs, resps = det(img)
found = kornia.feature.laf_is_filled(lafs)[0] & (resps[0] > 1e-3 * resps[0].max())
assert int(found.sum()) == 1
@pytest.mark.parametrize(
"resp_module, scale_space_response, minima_are_also_good, extra_levels",
[
(kornia.feature.BlobHessian, False, False, (2, 3)),
(kornia.feature.BlobDoG, True, True, (3, 4)),
],
)
def test_spare_pyramid_levels_do_not_change_the_detections_5670(
self, device, dtype, resp_module, scale_space_response, minima_are_also_good, extra_levels
):
# Each octave searches response levels 1..n_levels, and levels 0 and n_levels + 1 are their NMS
# neighbours. A pyramid that builds a level past those detects exactly what the minimal one does.
# Bright and dark blobs from below the first octave boundary to above the second.
yy, xx = torch.meshgrid(
torch.arange(96, device=device, dtype=dtype), torch.arange(128, device=device, dtype=dtype), indexing="ij"
)
img = torch.zeros_like(xx)
for i, sigma in enumerate((1.3, 1.6, 2.016, 2.6, 3.2, 4.032, 5.0)):
cx, cy = 18.0 + 15.0 * i, 30.0 + 36.0 * (i % 2)
img = img + (-1) ** i * torch.exp(-((xx - cx) ** 2 + (yy - cy) ** 2) / (2 * sigma**2))
img = img[None, None]
lafs, resps = [], []
for extra in extra_levels:
det = ScaleSpaceDetector(
30,
mr_size=1.0,
resp_module=resp_module(),
scale_pyr_module=ScalePyramid(3, 1.6, 16, extra_levels=extra),
scale_space_response=scale_space_response,
minima_are_also_good=minima_are_also_good,
).to(device, dtype)
out_lafs, out_resps = det(img)
lafs.append(out_lafs)
resps.append(out_resps)
self.assert_close(lafs[1], lafs[0])
self.assert_close(resps[1], resps[0])
@pytest.mark.parametrize(
"double_image, sigma, octave_div",
[
(True, 1.5, 0.5),
(True, 3.0, 1),
(True, 6.0, 2),
(True, 12.0, 4),
(False, 3.0, 1),
(False, 6.0, 2),
(False, 12.0, 4),
],
)
def test_detection_lands_on_the_blob_centre_in_every_octave_5675(
self, device, dtype, double_image, sigma, octave_div
):
# One blob per octave, far from the origin. ScalePyramid resizes with align_corners=True, so octave pixel u
# is input pixel u * (W - 1) / (W_o - 1); mapping it by 2 ** o instead put the doubled octave ~0.4 px past
# the blob and pulled every coarser octave towards the origin, by ~0.8 px at 1/2 and ~2.3 px at 1/4.
if dtype in (torch.float16, torch.bfloat16):
pytest.skip("the sub-pixel refinement alone moves a half-precision centre by more than the tolerance")
h, w = 241, 320
yy, xx = torch.meshgrid(
torch.arange(h, dtype=torch.float64), torch.arange(w, dtype=torch.float64), indexing="ij"
)
blob = torch.exp(-((xx - 251.3) ** 2 + (yy - 180.6) ** 2) / (2 * sigma**2))
det = ScaleSpaceDetector(
4,
resp_module=kornia.feature.BlobHessian(),
scale_pyr_module=ScalePyramid(3, 1.6, 15, double_image=double_image, extra_levels=2),
).to(device, dtype)
lafs, _ = det(blob[None, None].to(device, dtype))
expected = torch.tensor([[251.3, 180.6]], device=device, dtype=dtype)
self.assert_close(kornia.feature.get_laf_center(lafs)[0, :1], expected, rtol=0.0, atol=0.1)
# The frame follows the same per-axis map. With an odd height the two factors differ (by 0.05 % on the
# doubled octave, 0.5 % at 1/2, 0.7 % at 1/4), so an isotropic frame in octave units fails the ratio.
h_o, w_o = (2 * h, 2 * w) if octave_div == 0.5 else (h // octave_div, w // octave_div)
sx, sy = (w - 1) / (w_o - 1), (h - 1) / (h_o - 1)
frame = lafs[0, 0, :, :2]
assert frame[0, 1] == 0
assert frame[1, 0] == 0
self.assert_close(
frame[1, 1] / frame[0, 0], torch.tensor(sy / sx, device=device, dtype=dtype), rtol=1e-5, atol=0.0
)
def test_few_detections_padding(self, device, dtype):
# Constant image β very few (possibly zero) NMS candidates; output must still
# have the requested shape because the detect() method pads with zeros.
inp = torch.ones(1, 1, 32, 32, device=device, dtype=dtype)
n_feats = 20
det = ScaleSpaceDetector(n_feats, subpix_module=ConvQuadInterp3d(10)).to(device, dtype)
lafs, resps = det(inp)
assert lafs.shape == torch.Size([1, n_feats, 2, 3])
assert resps.shape == torch.Size([1, n_feats])
@pytest.mark.parametrize("x0", [80, 15])
def test_frame_overhanging_the_border_between_boundary_samples_is_rejected_5733(self, device, dtype, x0):
# A candidate is kept when half its frame lies inside the octave less a 5 px border, the rule of
# `laf_is_inside_image(scale_laf(frame, 0.5), octave, 5)`. One response maximum at level 2 of the first
# octave, row 32 and column 80 or 15, gets a half frame of radius 10.3, which overhangs the border at x = 90
# or x = 5 by 0.3 px. Ten points sampled on that circle reach only 10.3 sin(72 deg) = 9.8 px and kept it.
class PeakResponse(torch.nn.Module):
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = -torch.ones_like(x)
if x.shape[-1] == 96: # the first octave only
out[:, :, 2, 32, x0] = 0.0
return out
sigma = 1.6 * 2 ** (2 / 3) # the scale of level 2 of the first octave
img = torch.zeros(1, 1, 64, 96, device=device, dtype=dtype)
for radius, kept in ((10.3, False), (9.7, True)):
det = ScaleSpaceDetector(
1, resp_module=PeakResponse(), scale_space_response=True, mr_size=2 * radius / sigma
).to(device, dtype)
lafs, _ = det(img)
assert bool(kornia.feature.laf_is_filled(lafs)[0, 0]) == kept
half_frame = kornia.feature.laf_from_center_scale_ori(
torch.tensor([[[float(x0), 32.0]]], device=device, dtype=dtype),
torch.full((1, 1, 1, 1), radius, device=device, dtype=dtype),
)
assert bool(kornia.feature.laf_is_inside_image(half_frame, img, 5)[0, 0]) == kept
@pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
def test_scale_space_detector_init_preserves_global_config_4095(self):
torch._dynamo.reset()
try:
with torch._dynamo.config.patch(capture_dynamic_output_shape_ops=False):
assert not torch._dynamo.config.capture_dynamic_output_shape_ops
_ = ScaleSpaceDetector(num_features=5, compile_modules=["resp"])
assert not torch._dynamo.config.capture_dynamic_output_shape_ops
finally:
torch._dynamo.reset()
@pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
def test_compile_modules_forward_restores_dynamo_config_4095(self):
torch._dynamo.reset()
try:
with torch._dynamo.config.patch(capture_dynamic_output_shape_ops=False):
det = ScaleSpaceDetector(
num_features=2,
scale_pyr_module=ScalePyramid(1, 1.6, 15, extra_levels=2),
compile_modules=["subpix"],
)
img = torch.zeros(1, 1, 16, 16)
lafs, resps = det(img)
assert lafs.shape == torch.Size([1, 2, 2, 3])
assert resps.shape == torch.Size([1, 2])
assert not torch._dynamo.config.capture_dynamic_output_shape_ops
finally:
torch._dynamo.reset()
@pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
def test_compile_modules_match_eager_4095(self):
torch._dynamo.reset()
try:
inp = torch.zeros(1, 1, 33, 33)
inp[:, :, 13:-13, 13:-13] = 1.0
with torch._dynamo.config.patch(capture_dynamic_output_shape_ops=False):
eager = ScaleSpaceDetector(
1,
resp_module=kornia.feature.BlobHessian(),
subpix_module=ConvQuadInterp3d(10),
mr_size=3.0,
)
compiled = ScaleSpaceDetector(
1,
resp_module=kornia.feature.BlobHessian(),
subpix_module=ConvQuadInterp3d(10),
mr_size=3.0,
compile_modules=["resp", "subpix"],
)
lafs_eager, resps_eager = eager(inp)
lafs_compiled, resps_compiled = compiled(inp)
assert not torch._dynamo.config.capture_dynamic_output_shape_ops
assert bool((resps_compiled != 0).any())
self.assert_close(lafs_compiled, lafs_eager)
self.assert_close(resps_compiled, resps_eager)
finally:
torch._dynamo.reset()
def test_gradcheck(self, device):
batch_size, channels, height, width = 1, 1, 7, 7
patches = torch.rand(batch_size, channels, height, width, device=device, dtype=torch.float64)
# Use ConvQuadInterp3d for gradcheck β IterativeQuadInterp3d uses non-differentiable
# indexed in-place assignments that are incompatible with torch.autograd.gradcheck.
det = ScaleSpaceDetector(2, subpix_module=ConvQuadInterp3d(10)).to(device)
self.gradcheck(det, patches, nondet_tol=1e-4)
@pytest.mark.parametrize("refiner", ["adaptive", "conv_quad", "iterative", "soft_argmax"])
def test_half_image_refines_coordinates_in_float32(self, refiner, device, dtype):
# R4 / plan review P2: the refiners write absolute coordinates into a tensor of the response's dtype, and
# bfloat16 holds no odd integer above 256 (torch.arange(257, 263, dtype=bfloat16) == [256, 258, 260, 260,
# 260, 262]); float16 holds only multiples of 0.25 in [256, 512). The refiner sees a float32 response, so the
# refined centres past x = 256 lie off that grid; a centre rounded through the half dtype in the image frame
# lies on it, whatever the LAF dtype.
if dtype not in (torch.float16, torch.bfloat16):
pytest.skip("half-precision rule")
_skip_without_half_padding(device, dtype)
refiners = {
"adaptive": AdaptiveQuadInterp3d,
"conv_quad": ConvQuadInterp3d,
"iterative": IterativeQuadInterp3d,
"soft_argmax": ConvSoftArgmax3d,
}
g = torch.Generator().manual_seed(0)
img = torch.nn.functional.avg_pool2d(torch.rand(1, 1, 300, 420, generator=g), 5, 1, 2).to(device, dtype)
det = ScaleSpaceDetector(200, subpix_module=refiners[refiner]()).to(device, dtype)
seen = []
det.subpix.register_forward_pre_hook(lambda mod, args: seen.append(args[0].dtype))
with torch.no_grad():
lafs, responses = det(img)
assert set(seen) == {torch.float32}
assert lafs.dtype == torch.float32
assert responses.dtype == dtype
x = lafs[0, :, 0, 2]
far = x[x > 256]
assert far.numel() > 20
assert (far != far.to(dtype).float()).float().mean() > 0.5
class TestMultiResolutionDetector(BaseTester):
def _make_detector(self, num_features: int = 50, **config_overrides):
cfg = get_default_detector_config()
cfg.update(config_overrides)
return MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=num_features, config=cfg)
def test_multi_resolution_half_image_gives_float32_lafs(self, device, dtype):
if dtype not in (torch.float16, torch.bfloat16):
pytest.skip("half-precision rule")
_skip_without_half_padding(device, dtype)
g = torch.Generator().manual_seed(0)
img = torch.nn.functional.avg_pool2d(torch.rand(1, 1, 300, 420, generator=g), 5, 1, 2).to(device, dtype)
torch.manual_seed(1) # random KeyNet weights; those of seed 0 give no positive response maximum at all
lafs, responses = KeyNetDetector(False, num_features=100).eval().to(device, dtype)(img)
assert lafs.dtype == torch.float32
assert responses.dtype == dtype
x = lafs[0, :, 0, 2]
assert x[x > 256].numel() > 20
assert (x[x > 256] % 2 != 0).any()
def test_shape(self, device, dtype):
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = self._make_detector().to(device, dtype)
lafs, resps = det(inp)
assert lafs.shape == torch.Size([1, 50, 2, 3])
assert resps.shape == torch.Size([1, 50])
def test_shape_non_square(self, device, dtype):
inp = torch.rand(1, 1, 48, 96, device=device, dtype=dtype)
det = self._make_detector().to(device, dtype)
lafs, _ = det(inp)
assert lafs.shape == torch.Size([1, 50, 2, 3])
def test_lafs_inside_image(self, device, dtype):
# All detected LAF centers should lie within the image boundaries.
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = self._make_detector(num_features=20).to(device, dtype)
lafs, _ = det(inp)
cx = lafs[0, :, 0, 2]
cy = lafs[0, :, 1, 2]
assert (cx >= 0).all()
assert (cx <= 64).all()
assert (cy >= 0).all()
assert (cy <= 64).all()
def test_no_upscale_levels(self, device, dtype):
# up_levels=0 disables the upsampling branch; should still produce valid output.
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
cfg = get_default_detector_config()
cfg["up_levels"] = 0
cfg["pyramid_levels"] = 2
det = MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=20, config=cfg).to(device, dtype)
lafs, _ = det(inp)
assert lafs.shape == torch.Size([1, 20, 2, 3])
def test_with_upscale_levels(self, device, dtype):
# up_levels > 0 exercises the upsampling code path.
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
cfg = get_default_detector_config()
cfg["up_levels"] = 2
cfg["pyramid_levels"] = 1
det = MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=20, config=cfg).to(device, dtype)
lafs, _ = det(inp)
assert lafs.shape == torch.Size([1, 20, 2, 3])
def test_score_threshold_reduces_detections(self, device, dtype):
# A very high score_threshold should leave no real detections: every slot is padding,
# which reads as an exactly zero response, while the shape remains fixed.
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det_no_thresh = self._make_detector(num_features=50).to(device, dtype)
det_high_thresh = MultiResolutionDetector(
kornia.feature.BlobHessian(), num_features=50, score_threshold=1e6
).to(device, dtype)
lafs_no_thresh, resps_no_thresh = det_no_thresh(inp)
lafs_high_thresh, resps_high_thresh = det_high_thresh(inp)
assert lafs_high_thresh.shape == lafs_no_thresh.shape
# With an impossibly high threshold every slot is padding: a zero response and a zero
# LAF. Real detections always have positive responses.
assert resps_no_thresh.max().item() > 0
assert (resps_high_thresh == 0).all()
assert (lafs_high_thresh == 0).all()
def test_short_result_is_padded_with_zeros(self, device, dtype):
# A level with fewer above-threshold maxima than its quota must not return the
# `torch.finfo(dtype).min / 2` top-K sentinel, nor the arbitrary border coordinates
# `topk` picks once every remaining candidate is tied at that value.
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = self._make_detector(num_features=100).to(device, dtype)
lafs, resps = det(inp)
assert resps.shape == torch.Size([1, 100])
assert (resps >= 0).all()
padding = resps[0] == 0
assert bool(padding.any()), "expected this image to under-fill 100 slots"
assert (lafs[0][padding] == 0).all()
assert (resps[0][~padding] > 0).all()
def test_mask_suppresses_detections(self, device, dtype):
# Two blobs: a bright one top-left, a dimmer one bottom-right. Unmasked, the bright
# blob wins; with a mask covering only the bottom-right quadrant the dim blob must be
# the best -- and nothing may be detected outside the mask.
inp = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
inp[:, :, 14:19, 14:19] = 1.0
inp[:, :, 46:51, 46:51] = 0.5
mask = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
mask[:, :, 32:, 32:] = 1.0
det = self._make_detector(num_features=10).to(device, dtype)
lafs_free, resps_free = det(inp)
lafs_masked, resps_masked = det(inp, mask)
assert lafs_free[0, int(resps_free[0].argmax()), 0, 2].item() < 32
assert lafs_masked[0, int(resps_masked[0].argmax()), 0, 2].item() > 32
found = resps_masked[0] != 0
assert bool(found.any())
# The mask is resampled conservatively onto every level, so nothing is detected on the
# zero side of its edge at any level.
assert (lafs_masked[0][found][:, :, 2] >= 32).all()
def test_mask_edge_does_not_create_maxima(self, device, dtype):
# A blob wholly inside the zero region, next to the mask edge. Multiplying the response by
# a resampled mask before non-maxima suppression carved an edge into it, and the bilinear
# ramp of that edge was a "maximum" on the suppressed side (kornia#4102).
inp = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
inp[:, :, 30:34, 28:32] = 1.0
mask = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
mask[:, :, :, 32:] = 1.0
det = self._make_detector(num_features=5).to(device, dtype)
lafs, resps = det(inp, mask)
assert (resps == 0).all(), f"detected on the masked side: {lafs[0][resps[0] != 0][:, 0, 2].tolist()}"
assert (lafs == 0).all()
def test_border_does_not_create_maxima_5694(self, device, dtype):
# A cone with its single maximum at x = 8, inside the 15 px border strip of every level. Zeroing
# the strip before non-maxima suppression carved an edge into the response, and on every level the
# pixel beside that edge was reported as a "maximum", 15 level pixels from the image border.
yy, xx = torch.meshgrid(
torch.arange(64, device=device, dtype=dtype), torch.arange(64, device=device, dtype=dtype), indexing="ij"
)
def cone(x0: float) -> torch.Tensor:
return (1 - ((xx - x0) ** 2 + (yy - 32) ** 2).sqrt() / 40).clamp(min=0)[None, None]
det = MultiResolutionDetector(torch.nn.Identity(), num_features=4).to(device, dtype)
lafs, resps = det(cone(8.0))
assert (resps == 0).all(), f"detected beside the border: {lafs[0][resps[0] != 0][:, :, 2].tolist()}"
assert (lafs == 0).all()
# A maximum outside the strip is still detected, and only there. The peak lies at x = 17 at full
# resolution and at x = 24 on the 90 px upscaled level, outside the strip (x < 15), and at x = 12
# or below on every coarser level, inside it, so a one-pixel change in how a level is resampled
# cannot move it across the strip's edge: exactly those two levels report it.
lafs, resps = det(cone(17.0))
found = resps[0] != 0
assert int(found.sum()) == 2, lafs[0][found][:, :, 2].tolist()
centres = lafs[0][found][:, :, 2].float()
target = torch.tensor([17.0, 32.0], device=device)
assert ((centres - target).abs() <= 1.0).all(), centres.tolist()
def test_strip_response_suppresses_like_any_neighbour(self, device, dtype):
# The suppression reads the response inside the strip, so a stronger response there removes a
# weaker maximum within `nms_size // 2` beside it, as a stronger neighbour does anywhere in the
# image. Only the strip's own positions are excluded from the detections.
cfg = get_default_detector_config()
cfg.update(pyramid_levels=0, up_levels=0)
det = MultiResolutionDetector(torch.nn.Identity(), num_features=2, config=cfg).to(device, dtype)
resp = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
resp[0, 0, 32, 16], resp[0, 0, 32, 12], resp[0, 0, 32, 40] = 1.0, 2.0, 0.5
lafs, resps = det(resp)
self.assert_close(resps, torch.tensor([[0.5, 0.0]], device=device, dtype=dtype))
self.assert_close(lafs[0, 0, :, 2], torch.tensor([40.0, 32.0], device=device, dtype=lafs.dtype))
def test_remove_borders_writes_an_exact_zero(self, device, dtype):
# A 0/1 mask multiplied into the scores turns an infinite score inside the strip into NaN, which
# `topk` ranks ahead of every real detection before the validity test empties its slot.
cfg = get_default_detector_config()
cfg.update(pyramid_levels=0, up_levels=0)
det = MultiResolutionDetector(torch.nn.Identity(), num_features=1, config=cfg).to(device, dtype)
score = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
score[0, 0, 32, 10], score[0, 0, 32, 32] = float("inf"), 1.0
out = det.remove_borders(score)
inside = torch.zeros_like(score, dtype=torch.bool)
inside[..., 15:49, 15:49] = True
assert bool((out[~inside] == 0).all()), out[~inside].unique().tolist()
assert torch.equal(out[inside], score[inside])
assert torch.equal(det.remove_borders(score, borders=0), score)
lafs, resps = det(score)
self.assert_close(resps, torch.ones_like(resps))
self.assert_close(lafs[0, 0, :, 2], torch.tensor([32.0, 32.0], device=device, dtype=lafs.dtype))
def test_thin_masked_stripe_survives_downsampling(self, device, dtype):
# A two-pixel zero stripe is narrower than the sampling step of a coarse level, so an
# interpolated mask reads 1.0 there and the stripe is gone; the conservative resample keeps
# every level pixel it touches at zero.
torch.manual_seed(0)
inp = torch.rand(1, 1, 128, 128, device=device, dtype=dtype)
mask = torch.ones(1, 1, 128, 128, device=device, dtype=dtype)
mask[:, :, :, 60:62] = 0.0
det = self._make_detector(num_features=500).to(device, dtype)
lafs, resps = det(inp, mask)
xs = lafs[0][resps[0] != 0][:, 0, 2]
assert xs.numel() > 100
assert not bool(((xs >= 60) & (xs < 62)).any()), sorted(xs[(xs >= 59) & (xs < 63)].tolist())
@pytest.mark.parametrize("src, dst", [((100, 100), (45, 45)), ((38, 38), (90, 90)), ((7, 9), (3, 4))])
def test_mask_resample_is_the_cpu_adaptive_min_pool_on_every_device(self, device, dtype, src, dst):
# `F.adaptive_max_pool2d` on MPS returns the wrong shape when the output is larger than the
# input (the KeyNet up-level, the double-image octave) and pools other windows than CPU for
# a non-integer ratio, so the resampled mask would depend on the device. The hand-rolled
# resample pins the CPU windows on every device.
torch.manual_seed(0)
mask = torch.rand(2, 1, *src, device=device, dtype=dtype)
ref = torch.empty(1, 1, *dst, device=device, dtype=dtype)
expected = -torch.nn.functional.adaptive_max_pool2d(-mask.cpu().float(), dst)
got = _resize_mask(mask, ref)
assert got.shape == (2, 1, *dst)
assert got.dtype == dtype
self.assert_close(got.cpu().float(), expected.to(got.cpu().float().dtype))
def test_float_weights_above_one_are_clamped(self, device, dtype):
# The weighting runs before the `score_threshold` test, so a weight above one lifted
# sub-threshold maxima over it. A float 0/255 mask now means what the integer one means.
torch.manual_seed(0)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = self._make_detector().to(device, dtype)
lafs_b, resps_b = det(inp, torch.ones(1, 1, 64, 64, device=device, dtype=torch.bool))
lafs_f, resps_f = det(inp, torch.full((1, 1, 64, 64), 255.0, device=device, dtype=dtype))
assert torch.equal(resps_f, resps_b)
assert torch.equal(lafs_f, lafs_b)
def test_float_mask_weights_the_score_and_keeps_the_candidates(self, device, dtype):
# A graded mask used to be multiplied into the response before the NMS and the sub-pixel
# step, which moved maxima and their refined positions. It now weights only the score of a
# maximum found in the unweighted response: same detections, same positions, ranked by weight.
# Isolated interior peaks give a known candidate set, independent of random Hessian
# responses or spurious maxima created by removing the border before NMS.
inp = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
inp[0, 0, 20, 20], inp[0, 0, 20, 44] = 4.0, 3.0
inp[0, 0, 44, 20], inp[0, 0, 44, 44] = 2.0, 1.0
# Weighting the response before NMS would move the first peak onto this shoulder.
inp[0, 0, 20, 21] = 3.99
cfg = get_default_detector_config()
cfg.update(pyramid_levels=0, up_levels=0)
det = MultiResolutionDetector(torch.nn.Identity(), num_features=2000, config=cfg).to(device, dtype)
lafs_plain, resps_plain = det(inp)
ramp = torch.linspace(0.2, 1.0, 64, device=device, dtype=dtype).view(1, 1, 1, 64).expand(1, 1, 64, 64)
lafs, resps = det(inp, ramp.contiguous())
keep_plain, keep = resps_plain[0] != 0, resps[0] != 0
assert int(keep.sum()) == int(keep_plain.sum()) == 4
self._check_weighted_like_plain(
lafs[0][keep], resps[0][keep], lafs_plain[0][keep_plain], resps_plain[0][keep_plain]
)
def _check_weighted_like_plain(self, lafs_w, resps_w, lafs_p, resps_p):
# Same set of frames, up to the order the weighted score imposes: pair the detections by
# centre, then by scale, and compare the frames and the score of each pair.
def by_frame(lafs_: torch.Tensor, resps_: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
frames = lafs_.flatten(1).cpu().double().tolist()
idx = sorted(range(len(frames)), key=lambda i: (frames[i][2], frames[i][5], frames[i][0]))
return lafs_[idx], resps_[idx]
lafs_w, resps_w = by_frame(lafs_w, resps_w)
lafs_p, resps_p = by_frame(lafs_p, resps_p)
self.assert_close(lafs_w, lafs_p)
ratio = resps_w / resps_p
assert float(ratio.min()) >= 0.2 - 1e-2
assert float(ratio.max()) <= 1.0 + 1e-2
assert float(ratio.min()) < 0.9, ratio.tolist()
def test_float_mask_weights_the_score_on_every_level(self, device, dtype):
# The same contract on resampled levels: the ramp is min-pooled onto every level, the upscaled
# one included, and still only weights the scores. Three blobs with off-grid centres give
# maxima on the upscaled, the full-resolution and two downsampled levels.
n = 96
yy, xx = torch.meshgrid(
torch.arange(n, dtype=torch.float64), torch.arange(n, dtype=torch.float64), indexing="ij"
)
blobs = [(37.3, 41.6, 1.0, 4.0), (58.7, 52.2, 0.8, 6.0), (45.1, 61.9, 0.6, 3.0)]
inp = sum(a * torch.exp(-((xx - x0) ** 2 + (yy - y0) ** 2) / (2 * s * s)) for x0, y0, a, s in blobs)
inp = inp[None, None].to(device, dtype)
det = MultiResolutionDetector(torch.nn.Identity(), num_features=2000).to(device, dtype)
lafs_plain, resps_plain = det(inp)
ramp = torch.linspace(0.2, 1.0, n, device=device, dtype=dtype).view(1, 1, 1, n).expand(1, 1, n, n)
lafs, resps = det(inp, ramp.contiguous())
keep_plain, keep = resps_plain[0] != 0, resps[0] != 0
assert int(keep.sum()) == int(keep_plain.sum())
# A detection's scale identifies its level.
levels = kornia.feature.get_laf_scale(lafs_plain)[0][keep_plain].view(-1).unique()
assert levels.numel() >= 3, levels.tolist()
self._check_weighted_like_plain(
lafs[0][keep], resps[0][keep], lafs_plain[0][keep_plain], resps_plain[0][keep_plain]
)
def test_response_map_must_match_the_level_size(self, device, dtype):
# A response index is decoded as a level pixel with no offset, so a valid-convolution net
# that returns a smaller map put every keypoint one pixel up and left of its peak and had
# the mask resampled onto the shifted geometry. Nothing can infer the offset from the shape,
# so a map of another size is rejected rather than silently misplaced.
kernel = torch.ones(1, 1, 3, 3, device=device, dtype=dtype) / 9
det = MultiResolutionDetector(lambda x: torch.nn.functional.conv2d(x, kernel), num_features=8).to(device, dtype)
with pytest.raises(Exception, match=r"spatial size"):
det(torch.rand(1, 1, 64, 64, device=device, dtype=dtype))
def test_a_nan_weight_does_not_consume_a_slot(self, device, dtype):
# `s * NaN` sorts first in `topk`, so one NaN pixel in a float mask took a slot on every
# pyramid level and displaced a real maximum (18 filled slots became 13); `ScaleSpaceDetector`
# gates its candidates with `weight > 0`, which drops NaN, and this detector does the same.
torch.manual_seed(0)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = self._make_detector(num_features=20).to(device, dtype)
lafs_ref, _ = det(inp)
mask = torch.ones(1, 1, 64, 64, device=device, dtype=dtype)
mask[0, 0, 30, 30] = float("nan")
lafs, resps = det(inp, mask)
filled_ref = lafs_ref.ne(0).any(dim=-1).any(dim=-1)
filled = lafs.ne(0).any(dim=-1).any(dim=-1)
assert torch.isfinite(resps).all()
# the NaN pixel suppresses its neighbourhood on every level, nothing more
assert int(filled_ref.sum()) - 4 <= int(filled.sum()) <= int(filled_ref.sum())
def test_mask_spatial_size_must_match_the_image(self, device, dtype):
# `KORNIA_CHECK_SHAPE`'s named dims are free per call, so on their own they let a mask
# of any size through to be stretched onto the image; the explicit check is what rejects it.
det = self._make_detector(num_features=10).to(device, dtype)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
with pytest.raises(Exception, match="spatial size"):
det(inp, torch.ones(1, 1, 32, 32, device=device, dtype=dtype))
with pytest.raises(Exception, match="spatial size"):
det(inp, torch.ones(1, 1, 64, 32, device=device, dtype=dtype))
@pytest.mark.parametrize("mask_kind", ["bool", "uint8_255", "int32_100"])
def test_binary_masks_match_float_mask(self, device, dtype, mask_kind):
# A boolean or integer mask is binary: any non-zero value keeps a position, so an OpenCV
# 0/255 mask means the same as a 0/1 one and does not scale the responses by 255.
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
mask = torch.zeros(1, 1, 64, 64, device=device, dtype=torch.bool)
mask[:, :, 32:, 32:] = True
if mask_kind == "uint8_255":
mask = mask.to(torch.uint8) * 255
elif mask_kind == "int32_100":
mask = mask.to(torch.int32) * 100
det = self._make_detector(num_features=10).to(device, dtype)
lafs_bool, resps_bool = det(inp, mask)
lafs_float, resps_float = det(inp, mask.ne(0).to(dtype))
assert bool((resps_bool != 0).any())
assert torch.equal(resps_bool, resps_float)
assert torch.equal(lafs_bool, lafs_float)
# ... and the mask is doing something, so the two arms cannot agree vacuously the way
# they would if the mask were ignored altogether.
_lafs_free, resps_free = det(inp)
assert not torch.equal(resps_bool, resps_free)
def test_padding_survives_the_affine_and_orientation_modules(self, device, dtype):
# `forward` runs `aff` and `ori` after `detect`; they may transform a zero frame differently
# by dtype, so the zero-LAF contract has to be re-applied.
_require_affine_orientation_kernels(device, dtype)
det = MultiResolutionDetector(
kornia.feature.BlobHessian(),
num_features=10,
aff_module=kornia.feature.LAFAffineShapeEstimator(19),
ori_module=kornia.feature.LAFOrienter(19),
).to(device, dtype)
lafs, resps = det(torch.zeros(1, 1, 64, 64, device=device, dtype=dtype))
assert (resps == 0).all()
assert (lafs == 0).all()
def test_zero_mask_detects_nothing(self, device, dtype):
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
mask = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
det = self._make_detector(num_features=20).to(device, dtype)
lafs, resps = det(inp, mask)
assert (resps == 0).all()
assert (lafs == 0).all()
@staticmethod
def _require_half_response(device, half_dtype):
if device.type == "mps":
pytest.skip("MPS autocast changes the effective dtype")
# `BlobHessian` reaches `spatial_gradient`, which pads with mode="replicate".
# `replication_pad2d` has no CPU float16 kernel before torch 2.6.
if not supports_replicate_padding(device, half_dtype):
pytest.skip(f"no replicate-padding kernel for {half_dtype} on {device.type}")
@pytest.mark.parametrize("half_dtype", [torch.float16, torch.bfloat16])
def test_half_precision_gives_float32_lafs_and_half_responses(self, device, half_dtype):
# Pixel coordinates are float32 for a half-precision image: bfloat16 holds no odd pixel above 256. The
# responses keep the response function's dtype.
self._require_half_response(device, half_dtype)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=half_dtype)
det = self._make_detector(num_features=20).to(device, half_dtype)
lafs, resps = det(inp)
assert lafs.dtype == torch.float32
assert resps.dtype == half_dtype
@pytest.mark.parametrize("half_dtype", [torch.float16, torch.bfloat16])
def test_mask_does_not_promote_the_response_dtype(self, device, half_dtype):
# A float32 mask multiplied into a half-precision response map used to promote the
# responses to float32. The LAFs are float32 for a half-precision image either way.
self._require_half_response(device, half_dtype)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=half_dtype)
mask = torch.ones(1, 1, 64, 64, device=device, dtype=torch.float32)
det = self._make_detector(num_features=20).to(device, half_dtype)
lafs, resps = det(inp, mask)
assert lafs.dtype == torch.float32
assert resps.dtype == half_dtype
def test_color_input_with_single_channel_response(self, device, dtype):
class ColorResponse(torch.nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x.mean(dim=1, keepdim=True)
torch.manual_seed(3)
inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype)
det = MultiResolutionDetector(ColorResponse(), num_features=100).to(device, dtype)
lafs, resps = det(inp)
found = resps[0] != 0
assert bool(found.any())
cx = lafs[0, :, 0, 2]
cy = lafs[0, :, 1, 2]
assert (cx >= 0).all()
assert (cx <= 63).all()
assert (cy >= 0).all()
assert (cy <= 63).all()
def test_multichannel_response_is_rejected(self, device, dtype):
inp = torch.rand(1, 3, 64, 64, device=device, dtype=dtype)
detector = self._make_detector(num_features=100).to(device, dtype)
with pytest.raises(Exception, match="one response map"):
detector(inp)
def test_result_is_padded_to_num_features(self, device, dtype):
# An 8x8 image is smaller than the 15px border `remove_borders` strips, so there is
# genuinely nothing to detect and every slot is padding. A level is also capped at its
# own pixel count (64 here), so the concatenated result used to come back short.
cfg = get_default_detector_config()
cfg["pyramid_levels"] = 0
cfg["up_levels"] = 0
tiny = MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=100, config=cfg).to(device, dtype)
lafs, resps = tiny(torch.rand(1, 1, 8, 8, device=device, dtype=dtype))
assert lafs.shape == torch.Size([1, 100, 2, 3])
assert resps.shape == torch.Size([1, 100])
# Say so explicitly, so this test cannot pass by codifying a dummy result elsewhere.
assert (resps == 0).all()
assert (lafs == 0).all()
def test_short_result_is_sorted(self, device, dtype):
# `detect` used to run its final top-K only when the levels had produced *more* slots than
# `num_features`, so a short result came back in level order with each level's own padding
# left in place and the real detections scattered through it.
cfg = get_default_detector_config()
cfg.update(pyramid_levels=2, up_levels=0)
det = MultiResolutionDetector(torch.nn.Identity(), num_features=6000, config=cfg).to(device, dtype)
# A central impulse survives at the original and first downsampled levels, so this
# exercises moving a later level's detection ahead of the first level's padding.
inp = torch.zeros(1, 1, 48, 48, device=device, dtype=dtype)
inp[0, 0, 24, 24] = 1.0
resps, _lafs = det.detect(inp)
found = int((resps[0] != 0).sum())
assert found == 2, f"expected one detection on each of two levels, got {found}"
assert bool((resps[0][:found] != 0).all()), "the detections do not come first"
assert torch.equal(resps[0], torch.sort(resps[0], descending=True).values)
def test_single_feature_request_returns_a_real_detection(self, device, dtype):
# Contract: asking for one feature returns a real detection, and the same one that asking
# for two would rank first. The original defect was an apportionment that gave every level
# a share of `num_features`: at `num_features=1` the default configuration's six shares are
# 0.508 .. 0.016, all of which truncate to zero, so every level was queried for zero
# candidates and the result was a padded dummy on an image full of maxima. The
# apportionment is gone -- every level is now asked for `num_features` -- so this pins the
# contract rather than that mechanism, and it holds under both.
torch.manual_seed(11)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
lafs_one, resps_one = self._make_detector(num_features=1).to(device, dtype)(inp)
lafs_two, resps_two = self._make_detector(num_features=2).to(device, dtype)(inp)
assert resps_one.shape == torch.Size([1, 1])
assert resps_one[0, 0] > 0
# Asking for one feature must return the same best feature as asking for two.
self.assert_close(resps_one[0, 0], resps_two[0].max())
self.assert_close(lafs_one[0, 0], lafs_two[0, int(resps_two[0].argmax())])
def test_small_request_spreads_across_scales(self, device, dtype):
# Contract: a small request still covers more than one of several equally strong features
# rather than returning near-duplicates of the single strongest one. `detect` used to
# apportion `num_features` across pyramid levels by flooring each level's fractional share
# independently (`int(x) for x in shares`); the shares favor the finest level so heavily
# (0.508 .. 0.016 with the default config) that a small `num_features` truncated every other
# level's quota to zero, and `num_features=3` on five well-separated equal blobs returned
# three near-duplicate detections of one blob. Largest-remainder apportionment fixed that,
# and asking every level for `num_features` supersedes it: with no cap the three slots go to
# the three strongest maxima anywhere, which are on different blobs. The pin is on the
# coverage, so it holds under all three.
img = torch.zeros(1, 1, 96, 96, device=device, dtype=dtype)
centers = [(20, 20), (20, 70), (48, 48), (76, 20), (76, 70)]
for y, x in centers:
img[:, :, y - 2 : y + 3, x - 2 : x + 3] = 1.0
det = self._make_detector(num_features=3).to(device, dtype)
_, lafs = det.detect(img)
laf_centers = lafs[0, :, :, 2]
covered = sum(
1 for y, x in centers if bool(((laf_centers - laf_centers.new_tensor([x, y])).abs().amax(dim=1) < 4).any())
)
assert covered >= 2, f"expected the 3-feature request to reach at least 2 of the 5 blobs, got {covered}"
def test_result_for_k_is_a_prefix_of_the_result_for_a_larger_k(self, device, dtype):
# `detect` used to ask each pyramid level only for its own share of the budget, so a level
# could contribute at most `quotas[0]` (the upscaled level) or `sum(quotas[: i + 1 + up])`
# (pyramid level i) -- both well under `num_features` for the finer levels. When an image's
# strongest maxima concentrate on one level, which is the normal case rather than a corner
# case, the excess was never requested and so could not be ranked in by the final global
# `topk`: the result was silently not the top-k, with no signal to the caller. Asking every
# level for `num_features` makes the union a superset of the true top-k, since the m
# detections a level contributes to the global top-k are necessarily that level's own top-m.
#
# The pin is on the sorted response vector rather than on keypoint identities: `detect`
# already returns responses in descending order, and comparing values sidesteps both the
# `topk` tie-break and the fact that one blob can be detected at several scales with the
# same rounded centre.
img = torch.zeros(1, 1, 240, 240, device=device, dtype=dtype)
strength = 0
for y in range(20, 220, 40): # 25 well-separated blobs, all resolved at the same scale
for x in range(20, 220, 40):
img[:, :, y - 1 : y + 2, x - 1 : x + 2] = 1.0 + 0.05 * strength
strength += 1
reference, _ = self._make_detector(num_features=32).to(device, dtype).detect(img)
for k in (4, 8, 16):
responses, _ = self._make_detector(num_features=k).to(device, dtype).detect(img)
self.assert_close(responses[0], reference[0, :k])
def test_the_response_prefix_survives_tied_responses(self, device, dtype):
# The prefix the docstring promises is on the sorted responses, not on which detection carries
# a given response. `topk` breaks a tie by position and the positions it ranks over depend on
# `num_features`, so exactly equal maxima can come back in a different order for a different
# request: with this input `detect(1)` returns the centre that `detect(2)` ranks *second*.
# That permutation is `topk`'s to choose and is deliberately not pinned here -- pinning it
# would codify undefined behaviour that already differs across devices. The response vector
# staying a prefix is the part that is promised, and it holds under ties too.
class Identity(torch.nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
return x
cfg = get_default_detector_config()
cfg.update({"nms_size": 5, "pyramid_levels": 1, "up_levels": 0})
img = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
for y, x in ((20, 20), (20, 30), (40, 40)):
img[:, :, y, x] = 1.0 # three isolated maxima with byte-identical responses
reference, _ = MultiResolutionDetector(Identity(), num_features=3, config=cfg).to(device, dtype).detect(img)
assert int((reference[0] != 0).sum()) == 3, "expected all three tied maxima to be detected"
for k in (1, 2):
responses, _ = MultiResolutionDetector(Identity(), num_features=k, config=cfg).to(device, dtype).detect(img)
self.assert_close(responses[0], reference[0, :k])
def test_negative_score_threshold_is_rejected(self, device, dtype):
# NMS writes an exact zero at every suppressed position, so a negative threshold would
# admit all of them -- and collide with the zero response that marks an unfilled slot.
with pytest.raises(ValueError):
MultiResolutionDetector(kornia.feature.BlobHessian(), num_features=10, score_threshold=-1.0)
def test_detect_rejects_a_batch(self, device, dtype):
# `detect` is public and documents `(1, C, H, W)`; a larger batch would be flattened
# across the batch axis and silently return coordinates for the wrong image.
det = self._make_detector(num_features=10).to(device, dtype)
with pytest.raises(ShapeError):
det.detect(torch.rand(2, 1, 64, 64, device=device, dtype=dtype))
def test_forward_uses_overridden_detect(self, device, dtype):
# `detect` is the public extension point; `forward` reads occupancy off its zero-LAF
# padding rather than off the response, so an override is honoured whole.
class CustomDetector(MultiResolutionDetector):
def detect(self, img, mask=None):
responses = img.new_full((1, self.num_features), 123.0)
lafs = img.new_full((1, self.num_features, 2, 3), 7.0)
return responses, lafs
inp = torch.zeros(1, 1, 32, 32, device=device, dtype=dtype)
det = CustomDetector(kornia.feature.BlobHessian(), num_features=3).to(device, dtype)
lafs, responses = det(inp)
assert torch.equal(responses, inp.new_full((1, 3), 123.0))
assert torch.equal(lafs, inp.new_full((1, 3, 2, 3), 7.0))
def test_three_argument_level_override_still_runs_unmasked(self, device, dtype):
# `detect_features_on_single_level` is public and gained a keyword-only `mask`; a subclass
# written against the historical three-argument signature keeps working without a mask
# and gets a TypeError, not silent misbehaviour, with one.
class LegacyLevel(MultiResolutionDetector):
def detect_features_on_single_level(self, level_img, num_kp, factor):
return super().detect_features_on_single_level(level_img, num_kp, factor)
torch.manual_seed(0)
inp = torch.rand(1, 1, 64, 64, device=device, dtype=dtype)
det = LegacyLevel(kornia.feature.BlobHessian(), num_features=10).to(device, dtype)
ref = self._make_detector(num_features=10).to(device, dtype)
lafs, resps = det(inp)
lafs_ref, resps_ref = ref(inp)
assert torch.equal(lafs, lafs_ref)
assert torch.equal(resps, resps_ref)
with pytest.raises(TypeError):
det(inp, torch.ones(1, 1, 64, 64, device=device, dtype=dtype))
def test_smoke_with_blob_image(self, device, dtype):
# Synthetic image with a bright blob β detector should find it.
inp = torch.zeros(1, 1, 64, 64, device=device, dtype=dtype)
inp[:, :, 28:36, 28:36] = 1.0
det = self._make_detector(num_features=5).to(device, dtype)
lafs, resps = det(inp)
assert lafs.shape == torch.Size([1, 5, 2, 3])
assert resps.abs().max().item() > 0
def _conv_blob(x0: float, y0: float, sigma: float, height: int, width: int, device, dtype) -> torch.Tensor:
"""Gaussian blob of ``sigma`` centred at pixel ``(x0, y0)`` (pixel centres at integers), ``(1, 1, H, W)``."""
yy, xx = torch.meshgrid(
torch.arange(height, dtype=torch.float64), torch.arange(width, dtype=torch.float64), indexing="ij"
)
blob = torch.exp(-((xx - x0) ** 2 + (yy - y0) ** 2) / (2 * sigma**2))
return blob.view(1, 1, height, width).to(device, dtype)
class _TwoMaxima(torch.nn.Module):
"""A scale-space response with two maxima at level 2 of the 64 x 96 octave: A at (40, 50), a weaker B at (30, 20).
``pull`` sets A's right neighbour (41, 50) so that the sub-pixel step moves A's centre towards x = 41.
"""
def __init__(self, pull: float | None = None) -> None:
super().__init__()
self.pull = pull
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = -torch.ones_like(x)
if x.shape[-1] == 96: # the first octave only
out[:, :, 2, 50, 40] = 0.0
if self.pull is not None:
out[:, :, 2, 50, 41] = self.pull
out[:, :, 2, 20, 30] = -0.5
return out
class _Extrema(torch.nn.Module):
"""A scale-space response on a constant background with single-voxel extrema at level 2 of the 64 x 96 octave.
``items`` holds ``((u, v), value, box)``: the voxel's value and, unless ``box`` is None, the value of the rest of
its 3 x 3 x 3 neighbourhood.
"""
def __init__(self, items: list, background: float) -> None:
super().__init__()
self.items = items
self.background = background
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = torch.full_like(x[:, :1], self.background)
if x.shape[-1] == 96: # the first octave only
for (u, v), value, box in self.items:
if box is not None:
out[:, :, 1:4, v - 1 : v + 2, u - 1 : u + 2] = box
out[:, :, 2, v, u] = value
return out
class _QuadraticMinimum(torch.nn.Module):
"""A scale-space response ``(d - 2.375)^2 + (y - 30.375)^2 + (x - 20.375)^2 - 0.3`` on the 64 x 96 octave.
Every other octave is 1. Its minimum, -0.3 at level 2.375 and pixel (20.375, 30.375), lies between samples: every
sampled value is at least 3 * 0.375^2 - 0.3 = 0.121875, at the strict minimum voxel (2, 30, 20). The offsets are
exact in bfloat16.
"""
def forward(self, x: torch.Tensor, sigmas: torch.Tensor) -> torch.Tensor:
out = torch.ones_like(x[:, :1])
if x.shape[-1] == 96: # the first octave only
d, y, xx = torch.meshgrid(
*(torch.arange(n, device=x.device, dtype=x.dtype) for n in out.shape[-3:]), indexing="ij"
)
out[:] = (d - 2.375) ** 2 + (y - 30.375) ** 2 + (xx - 20.375) ** 2 - 0.3
return out
class _LevelPeaks(torch.nn.Module):
"""A response map with one peak per pyramid level, looked up by the level's ``(H, W)``.
Each peak has one lower neighbour, ``0.9`` of it, at the offset ``(du, dv)`` = ``neighbour``. Non-maxima
suppression removes the neighbour, but it makes the peak asymmetric, so any sub-pixel step would move the peak
towards it (a parabolic fit by 0.41 pixel).
"""
def __init__(self, peaks: dict, neighbour: tuple[int, int]) -> None:
super().__init__()
self.peaks = peaks
self.neighbour = neighbour
def forward(self, x: torch.Tensor) -> torch.Tensor:
out = torch.zeros_like(x[:, :1])
peak = self.peaks.get(tuple(x.shape[-2:]))
if peak is not None:
(u, v), value = peak
out[..., v, u] = value
out[..., v + self.neighbour[1], u + self.neighbour[0]] = 0.9 * value
return out
class TestConventionsScaleSpaceDetector(BaseTester):
def test_convention_scale_space_detector_centre_is_sub_pixel_in_the_laf_frame(self, device, dtype):
# A blob of sigma 2.5 centred at x = 25.4, y = 18.6 (pixel centres at integers, LAF frame) on a 48 x 72 image is
# found at its centre to sub-pixel accuracy, and the other 15 slots are padding: zero LAF, zero response.
# Measured on CPU: (25.4001, 18.5999) in float32 and float64. The centres are float32 for a half-precision
# image, and the half-precision response moves them by about 0.05 px in bfloat16 and 0.01 px in float16.
_skip_without_half_padding(device, dtype)
tol = 0.15 if dtype == torch.bfloat16 else 0.05
img = _conv_blob(25.4, 18.6, 2.5, 48, 72, device, dtype)
det = ScaleSpaceDetector(16).to(device, dtype)
# relabelling: the transposed image swaps the centre's coordinates
for image, centre in ((img, (25.4, 18.6)), (img.transpose(-1, -2).contiguous(), (18.6, 25.4))):
lafs, resps = det(image)
expected = torch.tensor(centre, device=device, dtype=lafs.dtype)
self.assert_close(lafs[0, 0, :, 2], expected, rtol=0.0, atol=tol)
assert resps[0, 0] > 0
assert bool((lafs[0, 1:] == 0).all())
assert bool((resps[0, 1:] == 0).all())
@pytest.mark.parametrize(
"sigma, centre, size",
[(2.5, (25.4, 18.6), (48, 72)), (6.0, (70.4, 52.6), (96, 128))],
ids=["octave0", "octave1"],
)
def test_convention_scale_space_detector_laf_scale_is_mr_size_times_the_detection_sigma(
self, device, dtype, sigma, centre, size
):
# The LAF scale is mr_size times the scale the blob is detected at, in input pixels, also from octave 1. The
# detection scale of a blob of sigma s lies within [0.95 s, 1.15 s] (1.035 s and 1.042 s here, CPU float32),
# so the default mr_size = 6 gives a frame of about 6 s; OpenCV's KeyPoint.size is 2 s.
_skip_without_half_padding(device, dtype)
img = _conv_blob(*centre, sigma, *size, device, dtype)
scales = {}
for mr_size in (6.0, 1.0):
lafs, _ = ScaleSpaceDetector(16, mr_size=mr_size).to(device, dtype)(img)
assert int(kornia.feature.laf_is_filled(lafs)[0].sum()) == 1
frame = lafs[0, 0, :, :2]
assert frame[0, 1] == 0
assert frame[1, 0] == 0
scales[mr_size] = kornia.feature.get_laf_scale(lafs)[0, 0, 0, 0]
self.assert_close(scales[6.0], 6.0 * scales[1.0])
assert 0.95 <= scales[1.0].item() / sigma <= 1.15
@pytest.mark.parametrize(
"response, polarity, minima_are_also_good, found",
[
("hessian", "bright", False, True),
("hessian", "dark", False, True),
("dog", "dark", False, True),
("dog", "bright", False, False),
("dog", "bright", True, True),
],
)
def test_convention_scale_space_detector_polarity_follows_the_response(
self, device, dtype, response, polarity, minima_are_also_good, found
):
# The detector keeps response maxima, and minima only with minima_are_also_good. The Hessian determinant is a
# maximum at bright and dark blobs alike; DoG (coarser minus finer blur) is a maximum at a dark blob and a
# minimum at a bright one, which is detected only with minima_are_also_good, and a minimum's slot carries the
# negated response, so it is positive.
_skip_without_half_padding(device, dtype)
if response == "dog" and polarity == "bright" and dtype == torch.bfloat16:
# The same tie would hide the maximum that a reversed DoG makes of the bright blob, so the maxima-only case
# could not fail either.
pytest.skip("bfloat16 rounds the DoG minimum into a tie with a neighbour, and the 3x3x3 NMS is strict")
blob = _conv_blob(25.4, 18.6, 2.5, 48, 72, device, dtype)
img = blob if polarity == "bright" else 1.0 - blob
if response == "hessian":
det = ScaleSpaceDetector(8, minima_are_also_good=minima_are_also_good)
else:
det = ScaleSpaceDetector(
8,
resp_module=kornia.feature.BlobDoG(),
scale_space_response=True,
minima_are_also_good=minima_are_also_good,
)
lafs, resps = det.to(device, dtype)(img)
target = torch.tensor([25.4, 18.6], device=device, dtype=dtype)
near = kornia.feature.laf_is_filled(lafs)[0] & ((lafs[0, :, :, 2] - target).norm(dim=-1) < 1.0)
assert bool(near.any()) == found
if found: # the strongest detection, with a positive response
assert bool(near[0])
assert resps[0, 0] > 0
@pytest.mark.parametrize("rejection", ["border", "mask"])
def test_wart_scale_space_detector_tests_border_and_mask_after_the_octave_top_k_5741(
self, device, dtype, rejection
):
# An octave hands only its `num_features` strongest candidates to the border and mask test, so a rejected
# candidate's slot stays empty while a weaker valid one exists (#5741). A is rejected -- its half frame of
# radius 10.3 overhangs the border y = 63 - 5, or its refined centre x = 40.17 touches the masked pixel
# (41, 50) -- and B at (30, 20) is valid: num_features=1 returns an empty slot, num_features=2 returns B.
_skip_without_half_padding(device, dtype)
sigma = 1.6 * 2 ** (2 / 3) # the scale of level 2 of the first octave
img = torch.zeros(1, 1, 64, 96, device=device, dtype=dtype)
mask = None
if rejection == "border":
response, radius = _TwoMaxima(), 10.3
else:
response, radius = _TwoMaxima(pull=-0.5), 5.0
mask = torch.ones(1, 1, 64, 96, device=device, dtype=torch.bool)
mask[..., 50, 41] = False
filled = {}
for n in (1, 2):
det = ScaleSpaceDetector(n, resp_module=response, scale_space_response=True, mr_size=2 * radius / sigma)
lafs, resps = det.to(device, dtype)(img, mask)
filled[n] = kornia.feature.laf_is_filled(lafs)[0].tolist()
assert filled[1] == [False]
assert filled[2] == [True, False]
self.assert_close(lafs[0, 0, :, 2], torch.tensor([30.0, 20.0], device=device, dtype=lafs.dtype))
self.assert_close(resps[0, 0], torch.tensor(-0.5, device=device, dtype=dtype))
@pytest.mark.parametrize(
"sign, minima_are_also_good, centres, values",
[
(1.0, False, [[40.0, 30.0], [70.0, 45.0]], [0.8, -0.2]),
(1.0, True, [[40.0, 30.0], [25.0, 40.0], [70.0, 45.0]], [0.8, 0.7, -0.2]),
(-1.0, True, [[40.0, 30.0], [25.0, 40.0], [60.0, 25.0]], [0.8, 0.7, -0.3]),
],
ids=["maxima", "minima", "minima-negated-response"],
)
def test_convention_scale_space_detector_takes_single_voxel_minima_below_zero_only(
self, device, dtype, sign, minima_are_also_good, centres, values
):
# On a background of 0.5: maxima of 0.8 at (40, 30) and of -0.2 inside a -0.4 box at (70, 45), minima of -0.7
# at (25, 40) and of 0.3 at (60, 25). A strict maximum is a candidate whatever its sign; with
# minima_are_also_good a strict minimum is one when its refined negated response exceeds the response sampled
# there. Refinement leaves a single-voxel extremum at its sampled value, so such a minimum is one only below
# zero, and returns its negated response. So the minimum of 0.3 is not detected, and negating the response
# detects it (as a maximum of -0.3) instead of the maximum of -0.2 (now a minimum of 0.2).
_skip_without_half_padding(device, dtype)
items = [((40, 30), 0.8, None), ((70, 45), -0.2, -0.4), ((25, 40), -0.7, None), ((60, 25), 0.3, None)]
items = [(site, sign * value, None if box is None else sign * box) for site, value, box in items]
det = ScaleSpaceDetector(
8,
resp_module=_Extrema(items, background=sign * 0.5),
scale_space_response=True,
mr_size=1.0,
minima_are_also_good=minima_are_also_good,
)
lafs, resps = det.to(device, dtype)(torch.zeros(1, 1, 64, 96, device=device, dtype=dtype))
n = len(values)
assert kornia.feature.laf_is_filled(lafs)[0].tolist() == [True] * n + [False] * (8 - n)
self.assert_close(lafs[0, :n, :, 2], torch.tensor(centres, device=device, dtype=lafs.dtype))
self.assert_close(resps[0, :n], torch.tensor(values, device=device, dtype=dtype))
@pytest.mark.parametrize("minima_are_also_good", [False, True])
def test_convention_scale_space_detector_compares_a_refined_minimum_with_the_sampled_response(
self, device, dtype, minima_are_also_good
):
# The selection compares a minimum after refinement: its refined negated response, 0.3, exceeds the 0.121875
# sampled at its voxel, so the minimum is a candidate although no sampled response is below zero. It is
# reported at its refined centre (20.375, 30.375) with response 0.3. The response has no strict maximum, so
# without minima_are_also_good there is no detection.
_skip_without_half_padding(device, dtype)
det = ScaleSpaceDetector(
2,
resp_module=_QuadraticMinimum(),
scale_space_response=True,
mr_size=1.0,
minima_are_also_good=minima_are_also_good,
)
lafs, resps = det.to(device, dtype)(torch.zeros(1, 1, 64, 96, device=device, dtype=dtype))
if not minima_are_also_good:
assert kornia.feature.laf_is_filled(lafs)[0].tolist() == [False, False]
return
assert kornia.feature.laf_is_filled(lafs)[0].tolist() == [True, False]
self.assert_close(lafs[0, 0, :, 2], torch.tensor([20.375, 30.375], device=device, dtype=lafs.dtype))
self.assert_close(resps[0, 0], torch.tensor(0.3, device=device, dtype=dtype))
def test_convention_scale_space_detector_masks_candidates_before_the_top_k(self, device, dtype):
# A mask drops a candidate at its integer site before the octave's top-k; only the check of the refined centre
# comes after it (#5741). A, the strongest maximum, at (40, 50) lies in a masked 5 x 5 region, so the only slot
# of num_features=1 goes to the weaker B at (30, 20); without the mask it goes to A.
_skip_without_half_padding(device, dtype)
img = torch.zeros(1, 1, 64, 96, device=device, dtype=dtype)
mask = torch.ones(1, 1, 64, 96, device=device, dtype=torch.bool)
mask[..., 48:53, 38:43] = False
det = ScaleSpaceDetector(1, resp_module=_TwoMaxima(), scale_space_response=True, mr_size=2.0).to(device, dtype)
lafs, resps = det(img, mask)
assert kornia.feature.laf_is_filled(lafs)[0].tolist() == [True]
self.assert_close(lafs[0, 0, :, 2], torch.tensor([30.0, 20.0], device=device, dtype=lafs.dtype))
self.assert_close(resps[0, 0], torch.tensor(-0.5, device=device, dtype=dtype))
lafs, _ = det(img)
self.assert_close(lafs[0, 0, :, 2], torch.tensor([40.0, 50.0], device=device, dtype=lafs.dtype))
class TestConventionsMultiResolutionDetector(BaseTester):
@pytest.mark.parametrize("detector", ["MultiResolutionDetector", "KeyNetDetector"])
def test_convention_multi_resolution_detector_reports_the_level_pixel_times_the_level_factor(
self, device, dtype, detector
):
# A detection is an integer level pixel (u, v), reported at (u * W / W_l, v * H / H_l) with scale
# s_mult * mean(W / W_l, H / H_l), s_mult = 22 by default; there is no sub-pixel step, and (u, v) is not mapped
# to its pixel centre (u + 0.5) * W / W_l - 0.5 (the Key.Net reference maps it the same way). Level 1 of
# 69 x 128 at factor 2 is 34 x 64: pixel (20, 17) -> (40.0, 17 * 69 / 34 = 34.5), scale 22 * (2 + 69 / 34) / 2.
# Each peak's lower neighbour on its +x side (+y when transposed) would pull a sub-pixel step off the pixel.
_skip_without_half_padding(device, dtype)
cfg = get_default_detector_config()
cfg.update(pyramid_levels=1, up_levels=0, scale_factor_levels=2.0)
tol = 1e-3 # the LAFs are float32 for a half-precision image as well
for transposed in (False, True):
if transposed: # relabelling: the transposed image reports swapped coordinates
size, peaks = (128, 69), {(128, 69): ((31, 45), 1.0), (64, 34): ((17, 20), 0.5)}
centres, neighbour = [[31.0, 45.0], [34.5, 40.0]], (0, 1)
else:
size, peaks = (69, 128), {(69, 128): ((45, 31), 1.0), (34, 64): ((20, 17), 0.5)}
centres, neighbour = [[45.0, 31.0], [40.0, 34.5]], (1, 0)
response = _LevelPeaks(peaks, neighbour)
if detector == "MultiResolutionDetector":
det = MultiResolutionDetector(response, num_features=4, config=cfg)
else: # the KeyNet response is replaced by the same peaks through a forward hook
conf = {**kornia.feature.keynet.keynet_default_config, "Detector_conf": cfg}
det = kornia.feature.KeyNetDetector(False, num_features=4, keynet_conf=conf).eval()
det.model.register_forward_hook(lambda module, args, output, response=response: response(args[0]))
lafs, resps = det.to(device, dtype)(torch.zeros(1, 1, *size, device=device, dtype=dtype))
self.assert_close(resps, torch.tensor([[1.0, 0.5, 0.0, 0.0]], device=device, dtype=dtype))
expected = kornia.feature.laf_from_center_scale_ori(
torch.tensor([centres], device=device, dtype=lafs.dtype),
torch.tensor([22.0, 44.3235294], device=device, dtype=lafs.dtype).view(1, 2, 1, 1),
)
self.assert_close(lafs[:, :2], expected, rtol=0.0, atol=tol)
assert bool((lafs[0, 2:] == 0).all())