π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""The mode in which the learned models of ``kornia.feature`` are constructed.
A model built with complete pretrained weights comes back in evaluation mode; a randomly initialised one stays in
PyTorch's default training mode. The classes with a ``pretrained`` flag default to their pretrained weights.
"""
from __future__ import annotations
import inspect
import pytest
import torch
from torch import nn
from kornia.feature import (
ALIKED,
DISK,
SOLD2,
DeDoDe,
DeFMO,
DoGAffNetHardNet,
DoGHardNet,
GFTTAffNetHardNet,
HardNet,
HardNet8,
HesAffNetHardNet,
HyNet,
KeyNet,
KeyNetAffNetHardNet,
KeyNetDetector,
KeyNetHardNet,
LAFAffNetShapeEstimator,
LAFDescriptor,
LightGlue,
LightGlueMatcher,
LoFTR,
OriNet,
SOLD2_detector,
SOSNet,
TFeat,
XFeat,
)
from kornia.feature.sold2 import sold2, sold2_detector
from testing.base import BaseTester
# The classes whose constructor takes a ``pretrained`` flag.
_FLAG_CLASSES = [
HardNet,
HardNet8,
HyNet,
SOSNet,
TFeat,
OriNet,
LAFAffNetShapeEstimator,
KeyNet,
KeyNetDetector,
DeFMO,
]
# Constructions that load no weights, each of which must stay in training mode.
_RANDOM_INIT = [
*[pytest.param(lambda cls=cls: cls(pretrained=False), id=cls.__name__) for cls in _FLAG_CLASSES],
pytest.param(lambda: SOLD2(pretrained=False), id="SOLD2"),
pytest.param(lambda: SOLD2_detector(pretrained=False), id="SOLD2_detector"),
pytest.param(lambda: LoFTR(pretrained=None), id="LoFTR"),
pytest.param(XFeat, id="XFeat"),
pytest.param(lambda: ALIKED(model_name="aliked-n16"), id="ALIKED"),
pytest.param(DISK, id="DISK", marks=pytest.mark.slow),
# The B descriptor: the default G one downloads the DINOv2 backbone in its constructor.
pytest.param(lambda: DeDoDe(descriptor_model="B"), id="DeDoDe", marks=pytest.mark.slow),
]
# Factories that load weights, each of which must return the model in evaluation mode.
_WEIGHTED = [
pytest.param(SOLD2, id="SOLD2"),
pytest.param(SOLD2_detector, id="SOLD2_detector"),
pytest.param(lambda: LoFTR("outdoor"), id="LoFTR"),
pytest.param(XFeat.from_pretrained, id="XFeat"),
pytest.param(lambda: ALIKED.from_pretrained("aliked-n16"), id="ALIKED"),
pytest.param(lambda: GFTTAffNetHardNet(num_features=8), id="GFTTAffNetHardNet"),
pytest.param(lambda: HesAffNetHardNet(num_features=8), id="HesAffNetHardNet"),
pytest.param(lambda: DoGHardNet(num_features=8), id="DoGHardNet"),
pytest.param(lambda: DoGAffNetHardNet(num_features=8), id="DoGAffNetHardNet"),
pytest.param(lambda: KeyNetHardNet(num_features=8), id="KeyNetHardNet"),
pytest.param(lambda: KeyNetAffNetHardNet(num_features=8), id="KeyNetAffNetHardNet"),
pytest.param(LightGlueMatcher, id="LightGlueMatcher"),
pytest.param(DISK.from_pretrained, id="DISK", marks=pytest.mark.slow),
# The pair of the DeDoDe.from_pretrained doctest, which CI prefetches; the default G descriptor is 2.2 GB.
pytest.param(
lambda: DeDoDe.from_pretrained(detector_weights="L-C4-v2", descriptor_weights="B-upright"),
id="DeDoDe",
marks=pytest.mark.slow,
),
]
def _network(model: nn.Module) -> nn.Module:
"""Return the learned network: ``KeyNetDetector`` holds its ``KeyNet`` as ``.model`` and sets no mode itself."""
return model.model if isinstance(model, KeyNetDetector) else model
class TestPretrainedMode(BaseTester):
@pytest.mark.parametrize("cls", _FLAG_CLASSES, ids=lambda cls: cls.__name__)
def test_flag_classes_default_to_pretrained_and_eval(self, cls):
assert inspect.signature(cls).parameters["pretrained"].default is True
default = _network(cls())
assert not default.training
assert not any(m.training for m in default.modules())
expected = _network(cls(pretrained=True)).state_dict()
actual = default.state_dict()
assert actual.keys() == expected.keys()
for key, value in expected.items():
assert torch.equal(actual[key], value), key
@pytest.mark.parametrize("factory", _RANDOM_INIT)
def test_random_init_is_training_mode(self, factory):
model = _network(factory())
assert model.training is True
assert all(m.training for m in model.modules())
@pytest.mark.parametrize("factory", _WEIGHTED)
def test_weighted_factories_are_eval(self, factory, monkeypatch):
# SOLD2_detector fetches the wireframe checkpoint from a URL whose file CI does not prefetch; SOLD2's mirror
# serves the same bytes and is prefetched.
monkeypatch.setitem(sold2_detector.urls, "wireframe", sold2.urls["wireframe"])
model = factory()
assert not model.training
assert not any(m.training for m in model.modules())
def test_non_default_architecture_needs_pretrained_false(self):
# The liberty checkpoint has convolution biases, so it cannot load strictly into bias-free convolutions.
with pytest.raises(RuntimeError):
HyNet(is_bias=False)
model = HyNet(pretrained=False, is_bias=False)
assert model.layer2[0].bias is None
def test_lightglue_mode(self, monkeypatch):
assert LightGlue(features=None).training
# "superpoint" keeps the default architecture, so a fresh default model provides a complete state dict.
weights = LightGlue(features=None).state_dict()
monkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda url, **kwargs: dict(weights))
model = LightGlue("superpoint")
assert not model.training
assert not any(m.training for m in model.modules())
# weights count as loaded only when the state dict holds any
monkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda url, **kwargs: {})
assert LightGlue("superpoint").training
@pytest.mark.device_agnostic
@pytest.mark.parametrize("source", ["url", "file"])
@pytest.mark.parametrize(
"checkpoint_kind", ["complete", "missing_buffer", "missing_parameter", "buffers_only", "empty"]
)
def test_lightglue_checkpoint_completeness_controls_mode(self, monkeypatch, tmp_path, source, checkpoint_kind):
config = {"input_dim": 256, "descriptor_dim": 32, "n_layers": 2, "flash": False}
weights = LightGlue(features=None, **config).state_dict()
if checkpoint_kind == "missing_buffer":
# Reference checkpoints omit this derived buffer, but still contain every learned parameter.
del weights["confidence_thresholds"]
elif checkpoint_kind == "missing_parameter":
del weights["posenc.Wr.weight"]
elif checkpoint_kind == "buffers_only":
weights = {"confidence_thresholds": weights["confidence_thresholds"]}
elif checkpoint_kind == "empty":
weights = {}
if source == "url":
monkeypatch.setattr(
"kornia.feature.lightglue.load_state_dict_from_url", lambda *args, **kwargs: dict(weights)
)
model = LightGlue("superpoint", **config)
else:
# Local checkpoints are resolved relative to lightglue.py's weights directory.
monkeypatch.setattr("kornia.feature.lightglue.__file__", str(tmp_path / "lightglue.py"))
(tmp_path / "weights").mkdir()
torch.save(weights, tmp_path / "weights" / "checkpoint.pth")
model = LightGlue(features=None, weights="checkpoint", **config)
expected_training = checkpoint_kind not in ("complete", "missing_buffer")
assert {module.training for module in model.modules()} == {expected_training}
for name, value in weights.items():
self.assert_close(model.state_dict()[name], value)
if source == "url":
wrapper = LightGlueMatcher("superpoint", params=config)
assert wrapper.training is expected_training
assert {module.training for module in wrapper.modules()} == {expected_training}
for name, value in weights.items():
self.assert_close(wrapper.matcher.state_dict()[name], value)
def test_composites_build_pretrained_eval_modules(self):
# Self-contained pipelines (KeyNetAffNetHardNet and the other presets in _WEIGHTED) return in evaluation mode
# at the root and across all submodules. KeyNetDetector and LAFDescriptor take caller modules, so they put only
# their default pretrained KeyNet / HardNet in evaluation mode, report training=True at the root, and preserve
# the mode of any module passed in.
learned = (HardNet, OriNet, LAFAffNetShapeEstimator, KeyNet)
composite = KeyNetAffNetHardNet(num_features=8)
networks = [m for m in composite.modules() if isinstance(m, learned)]
assert {type(m) for m in networks} == set(learned)
assert not composite.training
assert not any(m.training for m in composite.modules())
detector = KeyNetDetector(num_features=8)
assert detector.training is True
assert not detector.model.training
assert not any(m.training for m in detector.model.modules())
for training in (True, False):
ori = nn.Identity().train(training)
aff = nn.Identity().train(not training)
det = KeyNetDetector(num_features=8, ori_module=ori, aff_module=aff)
assert det.training is True
assert det.ori.training is training
assert det.aff.training is (not training)
descriptor = LAFDescriptor(None)
assert descriptor.training is True
networks = [m for m in descriptor.modules() if isinstance(m, learned)]
assert [type(m) for m in networks] == [HardNet]
assert not any(m.training for m in networks[0].modules())
for training in (True, False):
patch_desc = nn.Identity().train(training)
wrapped = LAFDescriptor(patch_desc)
assert wrapped.training is True
assert wrapped.descriptor.training is training