π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import math
import pytest
import torch
from kornia.core.check import ShapeError
from kornia.feature import TLU, FilterResponseNorm2d, HyNet
from testing.base import DYNAMO_UNAVAILABLE_REASON, BaseTester, dynamo_is_available, supports_conv2d
class TestHyNet(BaseTester):
def test_learnable_eps(self, device):
layer = FilterResponseNorm2d(4, is_eps_leanable=True).to(device)
output = layer(torch.ones(2, 4, 8, 8, device=device))
assert output.shape == (2, 4, 8, 8)
assert layer.eps.requires_grad
def test_shape(self, device):
inp = torch.ones(1, 1, 32, 32, device=device)
hynet = HyNet(pretrained=False).to(device).eval()
out = hynet(inp)
assert out.shape == (1, 128)
@pytest.mark.parametrize("shape", [(2, 1, 40, 40), (2, 1, 32, 31), (2, 3, 32, 32), (1, 32, 32)])
def test_wrong_patch_shape_is_rejected_5724(self, device, shape):
# without the check a 40x40 patch silently produced a (2, 1152) descriptor
with pytest.raises(ShapeError):
HyNet(pretrained=False).to(device).eval()(torch.ones(*shape, device=device))
@pytest.mark.device_agnostic
def test_pretrained_rejects_is_bias_frn_false_6016(self, monkeypatch):
# ``is_bias_FRN=False`` used to load the checkpoint and silently skip its trained FRN biases.
state_dict = HyNet(pretrained=False).state_dict()
downloads = []
def fake_load_state_dict_from_url(*args, **kwargs):
downloads.append(args)
return state_dict
monkeypatch.setattr("kornia.feature.hynet.load_state_dict_from_url", fake_load_state_dict_from_url)
with pytest.raises(ValueError, match="is_bias_FRN"):
HyNet(pretrained=True, is_bias_FRN=False)
assert downloads == []
HyNet(pretrained=True)
assert len(downloads) == 1
HyNet(pretrained=False, is_bias_FRN=False)
def test_empty_batch(self, device, dtype):
# A batch of zero patches returns an empty descriptor batch, in the dtype a non-empty call returns.
model = HyNet(pretrained=False).to(device, dtype).eval()
one = model(torch.ones(1, 1, 32, 32, device=device, dtype=dtype))
out = model(torch.zeros(0, 1, 32, 32, device=device, dtype=dtype))
assert out.shape == (0, one.shape[1])
assert out.dtype == one.dtype
assert out.device == one.device
def test_shape_batch(self, device):
inp = torch.ones(4, 1, 32, 32, device=device)
hynet = HyNet(pretrained=False).to(device).eval()
out = hynet(inp)
assert out.shape == (4, 128)
def test_gradcheck(self, device):
patches = torch.rand(2, 1, 32, 32, device=device, dtype=torch.float64)
hynet = HyNet(pretrained=False).to(patches.device, patches.dtype).eval()
self.gradcheck(hynet, (patches,), eps=1e-4, atol=1e-4, nondet_tol=1e-8)
@pytest.mark.parametrize("patch_value", [0.0, 0.5])
@pytest.mark.parametrize("is_bias", [True, False])
def test_degenerate_patch_gives_finite_descriptors(self, device, dtype, patch_value, is_bias):
# `eps_l2_norm` is what keeps `desc_norm`'s division defined for a patch the network maps to
# exactly zero, and `is_bias=False` is the configuration that gets there: with the TLU bias on,
# the pre-normalisation tensor never lands exactly on zero. 1e-10 flushes to 0.0 in float16, so
# the guard used to be inert and every descriptor came back NaN (kornia#4224). bfloat16 keeps
# float32's exponent range and never had that failure, but on CPU both half dtypes used to
# raise out of `avg_pool3d` before reaching the division at all, so both are pinned here.
patches = torch.full((2, 1, 32, 32), patch_value, device=device, dtype=dtype)
hynet = HyNet(pretrained=False, is_bias=is_bias).to(device, dtype).eval()
descriptors = hynet(patches)
assert torch.isfinite(descriptors).all(), (
f"{int((~torch.isfinite(descriptors)).sum())} non-finite descriptor entries"
)
def test_dynamo(self, device, dtype, torch_optimizer, cudnn_tf32_follows_option):
# The normalisation branches on `dtype` to lift half precision into float32. dtype is static
# metadata, so dynamo resolves that branch at trace time; what this pins is that the compiled
# result still matches eager on every dtype -- including the lifted half path, which returns
# NaN if the branch is dropped. `cudnn_tf32_follows_option` is what makes the float32 leg
# meaningful on CUDA; see its docstring.
patches = torch.rand(2, 1, 32, 32, device=device, dtype=dtype)
model = HyNet(pretrained=False).to(device, dtype).eval()
op = torch_optimizer(model)
self.assert_close(op(patches), model(patches))
@pytest.mark.skipif(not dynamo_is_available(), reason=DYNAMO_UNAVAILABLE_REASON)
def test_dynamo_fullgraph(self, device, dtype, cudnn_tf32_follows_option):
# `export-support.rst` publishes HyNet as capturable with zero graph breaks. The dtype branch
# reads static metadata and must not become the thing that breaks the graph.
patches = torch.rand(2, 1, 32, 32, device=device, dtype=dtype)
model = HyNet(pretrained=False).to(device, dtype).eval()
expected = model(patches)
torch._dynamo.reset()
compiled = torch.compile(model, fullgraph=True)
self.assert_close(compiled(patches), expected)
def test_jit(self, device, dtype):
B, C, H, W = 2, 1, 32, 32
patches = torch.rand(B, C, H, W, device=device, dtype=dtype)
model = HyNet(pretrained=False).to(patches.device, patches.dtype).eval()
model_jit = torch.jit.script(model)
self.assert_close(model(patches), model_jit(patches))
class TestConventionsHyNet(BaseTester):
"""Pins for HyNet's input-range rule and its two normalisation layers, FilterResponseNorm2d and TLU."""
def test_convention_hynet_is_scale_invariant_but_not_offset_invariant(self, device, dtype):
# HyNet's first layer is FilterResponseNorm2d on the raw patch: it divides by the RMS and subtracts no mean, as
# in the reference (yuruntian/HyNet model.py, trained on uint8 patches cast to float). So 255x gives x's
# descriptor, while x + 1 and the mean-subtracted x - 0.5 do not (best row cosine about 0.9 and 0.7 with random
# weights). Feed it non-negative intensities, not standardised patches.
if not supports_conv2d(device, dtype):
pytest.skip(f"no conv2d kernel for {dtype} on {device.type}")
torch.manual_seed(0)
model = HyNet(pretrained=False).to(device, dtype).eval()
patches = torch.rand(4, 1, 32, 32, device=device, dtype=dtype)
desc = model(patches)
self.assert_close(model(255 * patches), desc)
for moved in (patches + 1, patches - 0.5):
cos = torch.nn.functional.cosine_similarity(model(moved).float(), desc.float(), dim=1)
assert cos.max() < 0.95, cos
def test_convention_filter_response_norm_divides_each_map_by_its_rms_with_eps_under_the_root(self, device, dtype):
# y = weight * x / sqrt(mean_HW(x^2) + |eps|) + bias, per sample and channel (weight 1, bias 0 at init).
# eps = 1 makes its place visible: a constant map of 1 gives 1 / sqrt(2), where eps outside the root would give
# 1 / (1 + 1) = 0.5; a constant 3 gives 3 / sqrt(10), not 0.75.
frn = FilterResponseNorm2d(2, eps=1.0).to(device, dtype)
x = torch.tensor(
[[[[1.0, 1.0, 1.0, 1.0]], [[3.0, 3.0, 3.0, 3.0]]], [[[0.0, 1.0, 2.0, 3.0]], [[-2.0, 0.0, 0.0, 2.0]]]],
device=device,
dtype=dtype,
)
# mean squares 1, 9, 3.5 and 2, each plus eps
expected = torch.tensor(
[
[[[1 / math.sqrt(2)] * 4], [[3 / math.sqrt(10)] * 4]],
[
[[0.0, 1 / math.sqrt(4.5), 2 / math.sqrt(4.5), 3 / math.sqrt(4.5)]],
[[-2 / math.sqrt(3), 0, 0, 2 / math.sqrt(3)]],
],
],
device=device,
dtype=dtype,
)
self.assert_close(frn(x), expected)
self.assert_close(frn(x[1:]), expected[1:]) # each sample on its own: no batch statistics
self.assert_close(FilterResponseNorm2d(2, eps=-1.0).to(device, dtype)(x), expected) # |eps|
def test_convention_tlu_floors_each_channel_at_its_tau_initialised_to_minus_one(self, device, dtype):
tlu = TLU(3).to(device, dtype)
assert tlu.tau.shape == (1, 3, 1, 1)
x = torch.tensor([-3.0, -1.0, -0.5, 2.0], device=device, dtype=dtype).view(1, 1, 1, 4).repeat(2, 3, 1, 1)
at_minus_one = torch.tensor([-1.0, -1.0, -0.5, 2.0], device=device, dtype=dtype)
self.assert_close(tlu(x), at_minus_one.expand(2, 3, 1, 4))
with torch.no_grad():
tlu.tau[0, 1] = 0.0
# max(x, tau) per channel: only channel 1 now floors at 0
at_zero = torch.tensor([0.0, 0.0, 0.0, 2.0], device=device, dtype=dtype)
self.assert_close(
tlu(x), torch.stack([at_minus_one, at_zero, at_minus_one]).view(1, 3, 1, 4).expand(2, 3, 1, 4)
)