π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
import torch
from kornia.core.check import ShapeError
from kornia.feature import HardNet, HardNet8, HyNet, MKDDescriptor, SIFTDescriptor, SOSNet, TFeat
from testing.base import BaseTester, supports_conv2d, supports_replicate_padding
class TestHardNet(BaseTester):
@pytest.mark.slow
def test_shape(self, device):
inp = torch.ones(1, 1, 32, 32, device=device)
hardnet = HardNet(pretrained=False).to(device)
hardnet.eval() # batchnorm with size 1 is not allowed in train mode
out = hardnet(inp)
assert out.shape == (1, 128)
def test_empty_batch(self, device, dtype):
# A batch of zero patches returns an empty descriptor batch, in the dtype a non-empty call returns.
model = HardNet(pretrained=False).to(device, dtype).eval()
one = model(torch.ones(1, 1, 32, 32, device=device, dtype=dtype))
out = model(torch.zeros(0, 1, 32, 32, device=device, dtype=dtype))
assert out.shape == (0, one.shape[1])
assert out.dtype == one.dtype
assert out.device == one.device
@pytest.mark.slow
def test_shape_batch(self, device):
inp = torch.ones(4, 1, 32, 32, device=device)
hardnet = HardNet(pretrained=False).to(device).eval()
out = hardnet(inp)
assert out.shape == (4, 128)
def test_gradcheck(self, device):
patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float64)
hardnet = HardNet(pretrained=False).to(patches.device, patches.dtype).eval()
self.gradcheck(hardnet, (patches,), eps=1e-4, atol=1e-4, nondet_tol=1e-8)
def test_jit(self, device, dtype):
B, C, H, W = 2, 1, 32, 32
patches = torch.ones(B, C, H, W, device=device, dtype=dtype)
model = HardNet(pretrained=False).to(patches.device, patches.dtype).eval()
model_jit = torch.jit.script(HardNet(pretrained=False).to(patches.device, patches.dtype).eval())
self.assert_close(model(patches), model_jit(patches))
class TestHardNet8(BaseTester):
def test_shape(self, device):
inp = torch.ones(1, 1, 32, 32, device=device)
hardnet = HardNet8(pretrained=False).to(device)
hardnet.eval() # batchnorm with size 1 is not allowed in train mode
out = hardnet(inp)
assert out.shape == (1, 128)
def test_empty_batch(self, device, dtype):
# A batch of zero patches returns an empty descriptor batch, in the dtype a non-empty call returns.
model = HardNet8(pretrained=False).to(device, dtype).eval()
one = model(torch.ones(1, 1, 32, 32, device=device, dtype=dtype))
out = model(torch.zeros(0, 1, 32, 32, device=device, dtype=dtype))
assert out.shape == (0, one.shape[1])
assert out.dtype == one.dtype
assert out.device == one.device
def test_shape_batch(self, device):
inp = torch.ones(4, 1, 32, 32, device=device)
hardnet = HardNet8(pretrained=False).to(device).eval()
out = hardnet(inp)
assert out.shape == (4, 128)
@pytest.mark.skip("jacobian not well computed")
def test_gradcheck(self, device):
patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float32)
hardnet = HardNet8(pretrained=False).to(patches.device, patches.dtype).eval()
self.gradcheck(hardnet, (patches,), eps=1e-4, atol=1e-4)
def test_jit(self, device, dtype):
B, C, H, W = 2, 1, 32, 32
patches = torch.ones(B, C, H, W, device=device, dtype=dtype)
model = HardNet8(pretrained=False).to(patches.device, patches.dtype).eval()
model_jit = torch.jit.script(HardNet8(pretrained=False).to(patches.device, patches.dtype).eval())
self.assert_close(model(patches), model_jit(patches))
def test_untrained_descriptors_are_distinct_5692(self, device, dtype):
# An all-ones PCA placeholder mapped every patch to +-(1, ..., 1) / sqrt(128).
if not supports_conv2d(device, dtype):
pytest.skip(f"no conv2d kernel for {dtype} on {device.type}")
torch.manual_seed(0)
patches = torch.rand(8, 1, 32, 32, device=device, dtype=dtype)
out = HardNet8(pretrained=False).eval().to(device, dtype)(patches)
assert out.shape == (8, 128)
self.assert_close(out.norm(dim=1), torch.ones(8, device=device, dtype=dtype))
cos = out @ out.T
off_diagonal = cos[~torch.eye(8, dtype=torch.bool, device=device)]
assert off_diagonal.abs().max() < 0.99
def test_untrained_model_keeps_the_first_features_5692(self, device, dtype):
if not supports_conv2d(device, dtype):
pytest.skip(f"no conv2d kernel for {dtype} on {device.type}")
torch.manual_seed(0)
patches = torch.rand(4, 1, 32, 32, device=device, dtype=dtype)
model = HardNet8(pretrained=False).to(device, dtype).eval()
features = model.features(model._normalize_input(patches)).view(4, -1)
expected = torch.nn.functional.normalize(torch.nn.functional.normalize(features, dim=1)[:, :128], dim=1)
self.assert_close(model(patches), expected)
def test_untrained_model_is_trainable_5692(self, device, dtype):
# The all-ones placeholder made the normalised output constant, so the gradient was zero (~1e-15).
if dtype in (torch.float16, torch.bfloat16):
pytest.skip("half-precision rounding leaves a noise gradient on the degenerate placeholder as well")
torch.manual_seed(0)
patches = torch.rand(4, 1, 32, 32, device=device, dtype=dtype)
model = HardNet8(pretrained=False).to(device, dtype).eval()
target = torch.randn(4, 128, device=device, dtype=dtype)
(model(patches) * target).sum().backward()
grad = model.features[0].weight.grad
assert grad is not None
assert grad.abs().max() > 1e-3
def test_checkpoint_layout_is_unchanged_5692(self, device):
# The pretrained checkpoint loads with strict=True, so the buffer names and shapes must stay.
model = HardNet8(pretrained=False).to(device)
state = model.state_dict()
assert state["components"].shape == (512, 128)
assert state["mean"].shape == (512,)
loaded = torch.rand(512, 128, device=device)
state["components"] = loaded
model.load_state_dict(state, strict=True)
self.assert_close(model.components, loaded)
class TestHardNetConstantPatchIsFinite(BaseTester):
"""A constant patch drives the features to zero; the L2 normalisation must not give NaN.
`F.normalize`'s default eps of 1e-12 rounds to zero in float16, so `0 / 0` reached the
descriptor. A detector that pads a short result feeds exactly this patch through a zero LAF.
"""
@pytest.mark.parametrize("desc_dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("model", [HardNet, HardNet8])
def test_constant_patch(self, device, desc_dtype, model):
if not supports_conv2d(device, desc_dtype):
# The descriptor is a stack of convolutions; torch 2.1.2 has no float16 CPU kernel.
pytest.skip(f"no conv2d kernel for {desc_dtype} on {device.type}")
patches = torch.zeros(2, 1, 32, 32, device=device, dtype=desc_dtype)
out = model(pretrained=False).eval().to(device, desc_dtype)(patches)
assert torch.isfinite(out).all()
# Each patch descriptor with the weights it gets without a download, its patch size, its output width and whether it
# pads with mode="replicate" (its gradient filters). MKD runs unwhitened, which needs no download either and returns
# the full 238-dimensional concat embedding.
_PATCH_DESCRIPTORS = {
"HardNet": (lambda: HardNet(pretrained=False).eval(), 32, 128, False),
"HardNet8": (lambda: HardNet8(pretrained=False).eval(), 32, 128, False),
"HyNet": (lambda: HyNet(pretrained=False).eval(), 32, 128, False),
"SOSNet": (lambda: SOSNet(pretrained=False).eval(), 32, 128, False),
"TFeat": (lambda: TFeat(pretrained=False).eval(), 32, 128, False),
"MKDDescriptor": (lambda: MKDDescriptor(whitening=None), 32, 238, True),
"SIFTDescriptor": (lambda: SIFTDescriptor(32), 32, 128, True),
}
class TestConventionsPatchDescriptors(BaseTester):
"""Pins for the patch-descriptor family: the patch each takes, its output norm and its intensity-range rule."""
@staticmethod
def _model(name, device, dtype):
factory, patch_size, dim, pads = _PATCH_DESCRIPTORS[name]
if not supports_conv2d(device, dtype):
pytest.skip(f"no conv2d kernel for {dtype} on {device.type}")
if pads and not supports_replicate_padding(device, dtype):
pytest.skip(f"no replicate-padding kernel for {dtype} on {device.type}")
torch.manual_seed(0) # random-init weights: no pin downloads
return factory().to(device, dtype), patch_size, dim
@pytest.mark.parametrize("name", ["HardNet", "HardNet8", "HyNet", "SOSNet", "TFeat"])
def test_convention_learned_descriptors_take_one_channel_32x32_patches(self, device, dtype, name):
# The input is validated: another size or channel count raises instead of returning another width (HyNet's
# rejection is also pinned by test_hynet.py::TestHyNet::test_wrong_patch_shape_is_rejected_5724).
model, _, _ = self._model(name, device, dtype)
assert type(model).patch_size == 32
assert model(torch.rand(3, 1, 32, 32, device=device, dtype=dtype)).shape == (3, 128)
for shape in [(3, 1, 40, 40), (3, 1, 64, 64), (3, 3, 32, 32), (3, 1, 32, 31)]:
with pytest.raises(ShapeError):
model(torch.rand(*shape, device=device, dtype=dtype))
@pytest.mark.parametrize("name", ["HardNet", "HardNet8", "HyNet", "SOSNet", "MKDDescriptor", "SIFTDescriptor"])
def test_convention_patch_descriptors_return_unit_norm_rows(self, device, dtype, name):
# TFeat is the exception, a tanh output as in its reference: test_tfeat.py::TestConventionsTFeat.
model, patch_size, dim = self._model(name, device, dtype)
out = model(torch.rand(5, 1, patch_size, patch_size, device=device, dtype=dtype))
assert out.shape == (5, dim)
self.assert_close(out.norm(dim=1), torch.ones(5, device=device, dtype=dtype))
@pytest.mark.parametrize("name", ["HardNet", "HardNet8", "SOSNet", "TFeat", "MKDDescriptor", "SIFTDescriptor"])
def test_convention_patch_descriptors_ignore_an_affine_intensity_change(self, device, dtype, name):
# HardNet and HardNet8 standardise each patch, SOSNet and TFeat start with InstanceNorm2d, MKD and SIFT use
# gradients and an L2 normalisation, so a [0, 1] patch, 2x + 1 and 255x give one descriptor. SOSNet and TFeat
# are invariant only up to the InstanceNorm eps (1e-5 on the variance): it moves TFeat by ~1e-5 here, and with
# pretrained weights a low-contrast patch moves both; random weights hide this for SOSNet, whose bias-free
# layers are positively homogeneous. HyNet is the exception (scale only): test_hynet.py::TestConventionsHyNet.
model, patch_size, _ = self._model(name, device, dtype)
patches = torch.rand(4, 1, patch_size, patch_size, device=device, dtype=dtype)
desc = model(patches)
tol = {"rtol": 0.0, "atol": 1e-4} if dtype in (torch.float32, torch.float64) else {}
# A float16 InstanceNorm that accumulates the squares of a [0, 255] patch in float16 overflows (torch 2.5.1 on
# CPU does), so that leg scales by 16 instead.
scale = 16.0 if dtype == torch.float16 and name in ("SOSNet", "TFeat") else 255.0
self.assert_close(model(2 * patches + 1), desc, **tol)
self.assert_close(model(scale * patches), desc, **tol)
if dtype in (torch.float32, torch.float64):
# each patch is normalised on its own: a different a x + b per patch still gives each patch its descriptor
a = torch.tensor([1.0, 16.0, 0.5, 3.0], device=device, dtype=dtype).view(4, 1, 1, 1)
b = torch.tensor([0.0, 0.0, -2.0, 1.0], device=device, dtype=dtype).view(4, 1, 1, 1)
self.assert_close(model(a * patches + b), desc, **tol)