π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(feature)!: LightGlue keyword arguments win over the features configuration; doghardnet stops early and prunes by default (#6421)
Dmytro Mishkin0d30c4169bf2
diff --git a/changelog.d/6421.breaking.md b/changelog.d/6421.breaking.mdnew file mode 100644index 000000000..0b43644ad--- /dev/null+++ b/changelog.d/6421.breaking.md@@ -0,0 +1 @@+`LightGlue("doghardnet")` and `LightGlue("dog_affnet_hardnet")`, and `LightGlueMatcher` in these modes, now stop early and prune points by default (`depth_confidence=0.95`, `width_confidence=0.99`), as the upstream doghardnet configuration does; they used to run every layer on every point. On 40 Oxford affine pairs with `DoGHardNet(2048)` the matcher runs 1.9x faster on CPU, finds 1.5 % fewer correct matches at 3 px, and estimates homographies as accurately. Pass `depth_confidence=-1, width_confidence=-1` for the previous matches. An architecture keyword argument that contradicts the weights of `features`, such as `LightGlue("disk", input_dim=64)`, used to be ignored and now fails, when the weights load or at the first call, except `n_layers` with `"xfeat"`: more than its 6 layers leaves the extra layers random and the model in training mode, fewer runs a truncated network. A `weights` keyword argument that names another checkpoint than the one of `features`, such as `LightGlue("disk", weights="superpoint_lightglue")`, used to be ignored and now raises: with keyword arguments winning, it would have cached the DISK checkpoint under the SuperPoint file name.diff --git a/changelog.d/6421.fixed.md b/changelog.d/6421.fixed.mdnew file mode 100644index 000000000..e7e35cc0e--- /dev/null+++ b/changelog.d/6421.fixed.md@@ -0,0 +1 @@+Keyword arguments of `LightGlue` (and the `params` of `LightGlueMatcher`) now override the configuration of `features`. `depth_confidence`, `width_confidence` and `filter_threshold` given with `"xfeat"`, `"doghardnet"`, `"dog_affnet_hardnet"` or `"keynet_affnet_hardnet"` used to be replaced silently by the preset's values.diff --git a/kornia/feature/lightglue.py b/kornia/feature/lightglue.pyindex 983583c00..83054ffa3 100644--- a/kornia/feature/lightglue.py+++ b/kornia/feature/lightglue.py@@ -29,6 +29,7 @@ from torch import nnfrom kornia.core._compat import torch_version_ltfrom kornia.core.check import KORNIA_CHECKfrom kornia.core.download import hf_url, load_state_dict_from_url+from kornia.core.exceptions import BaseErrorfrom kornia.core.utils import is_exportingfrom kornia.feature.laf import laf_to_three_points, scale_laf@@ -508,8 +509,13 @@ class LightGlue(nn.Module):features: The type of local features to match (e.g., 'superpoint', 'disk'), whose configuration andpretrained weights are loaded. ``None`` builds the model from ``default_conf`` and the keyword argumentsand downloads no weights.- **conf: keyword arguments that override entries of ``default_conf``, such as ``filter_threshold``; the- configuration of ``features`` is applied after them and wins where both set a key.+ **conf: keyword arguments that override entries of ``default_conf`` and of the configuration of+ ``features``, such as ``filter_threshold`` or ``depth_confidence``. The configuration of ``features`` sets+ the defaults, and an explicit keyword argument always wins, except ``weights``, which must name the+ checkpoint of ``features`` when ``features`` is given; ``weights=None``, the value in ``default_conf``,+ counts as not given. This departs on purpose from cvg/LightGlue, which applies the configuration of+ ``features`` after the keyword arguments, so that the tuning knobs a preset sets, such as ``"xfeat"``'s+ ``width_confidence``, can be changed."""default_conf: ClassVar[Dict[str, Any]] = {@@ -585,16 +591,12 @@ class LightGlue(nn.Module):"dog_affnet_hardnet": {"weights": "doghardnet","input_dim": 128,- "width_confidence": -1,- "depth_confidence": -1,"add_scale_ori": True,"scale_coef": 1.0 / 6.0,},"doghardnet": {"weights": "doghardnet","input_dim": 128,- "width_confidence": -1,- "depth_confidence": -1,"add_scale_ori": True,"scale_coef": 1.0 / 6.0,},@@ -607,13 +609,24 @@ class LightGlue(nn.Module):},}- def __init__(self, features: str = "superpoint", **conf_) -> None:+ def __init__(self, features: Optional[str] = "superpoint", **conf_) -> None:super().__init__()- self.conf = conf = SimpleNamespace(**{**self.default_conf, **conf_})+ if conf_.get("weights", "") is None:+ # `weights=None` is the value of default_conf, so it means "not given", also when default_conf is passed+ # on. It must not reach `conf.weights`, which names the cached file: "None" would be one cache name for+ # every preset.+ conf_ = {k: v for k, v in conf_.items() if k != "weights"}if features is not None:KORNIA_CHECK(features in list(self.features.keys()), "Features keys are wrong")- for k, v in self.features[features].items():- setattr(conf, k, v)+ # The download URL follows `features` and the cached file name `weights`: they must name one checkpoint,+ # or another checkpoint is cached under this name. A plain raise, so that disable_checks() keeps it.+ if conf_.get("weights", self.features[features]["weights"]) != self.features[features]["weights"]:+ raise BaseError(+ "`weights` must name the checkpoint of `features`. To load other weights, build the model with "+ "`features=None` and their architecture keyword arguments, then call `load_state_dict`"+ )+ preset = self.features[features] if features is not None else {}+ self.conf = conf = SimpleNamespace(**{**self.default_conf, **preset, **conf_})KORNIA_CHECK(not (self.conf.add_scale_ori and self.conf.add_laf)) # we use either scale ori, or LAFif conf.input_dim != conf.descriptor_dim:diff --git a/tests/feature/test_lightglue.py b/tests/feature/test_lightglue.pyindex f2fb852ed..8cd70e894 100644--- a/tests/feature/test_lightglue.py+++ b/tests/feature/test_lightglue.py@@ -20,6 +20,8 @@ import unittest.mockimport pytestimport torch+from kornia.core.check import are_checks_enabled, disable_checks, enable_checks+from kornia.core.exceptions import BaseErrorfrom kornia.feature.integrated import LightGlueMatcherfrom kornia.feature.laf import laf_from_center_scale_orifrom kornia.feature.lightglue import (@@ -976,13 +978,46 @@ class TestConventionsLightGlue(BaseTester):with pytest.raises(NotImplementedError, match="orb is not supported"):LightGlueMatcher("orb")- def test_convention_lightglue_features_configuration_wins_over_keyword_arguments(self, device, dtype, monkeypatch):- # "disk" sets input_dim, so the keyword argument loses; filter_threshold is not in its configuration, so the- # keyword argument stays+ def test_convention_lightglue_keyword_arguments_win_over_features_configuration(self, device, dtype, monkeypatch):+ # the configuration of the features gives the defaults, and an explicit keyword argument always wins: "xfeat"+ # sets width_confidence and filter_threshold, "keynet_affnet_hardnet" turns off pruning and early stoppingmonkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda *args, **kwargs: {})- lg = LightGlue("disk", input_dim=64, filter_threshold=0.3, n_layers=1)- assert lg.conf.input_dim == 128- assert lg.conf.filter_threshold == 0.3+ lg = LightGlue("xfeat", width_confidence=0.5, filter_threshold=0.3, n_layers=1)+ assert (lg.conf.depth_confidence, lg.conf.width_confidence, lg.conf.filter_threshold) == (-1, 0.5, 0.3)+ lg = LightGlue("keynet_affnet_hardnet", depth_confidence=0.95, width_confidence=0.99, n_layers=1)+ assert (lg.conf.depth_confidence, lg.conf.width_confidence) == (0.95, 0.99)+ assert LightGlue("keynet_affnet_hardnet", n_layers=1).conf.depth_confidence == -1+ assert LightGlue("disk", n_layers=1).conf.input_dim == 128+ assert LightGlue("disk", input_dim=64, n_layers=1).conf.input_dim == 64++ def test_lightglue_weights_must_name_the_checkpoint_of_features(self, monkeypatch):+ # the URL follows `features` and the cached file name `weights`: another checkpoint is cached under that name+ monkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda *args, **kwargs: {})+ assert LightGlue("disk", weights="disk_lightglue", n_layers=1).conf.weights == "disk_lightglue"+ # weights=None is the value of default_conf and means "not given", also when default_conf is passed on+ assert LightGlue("disk", weights=None, n_layers=1).conf.weights == "disk_lightglue"+ splat = LightGlue("superpoint", **{**LightGlue.default_conf, "n_layers": 1})+ assert splat.conf.weights == "superpoint_lightglue"+ with pytest.raises(BaseError, match="must name the checkpoint of `features`"):+ LightGlue("disk", weights="superpoint_lightglue", n_layers=1)+ # a plain raise: disable_checks() must not let another checkpoint be cached under this name+ were_checks_enabled = are_checks_enabled()+ disable_checks()+ try:+ with pytest.raises(BaseError, match="must name the checkpoint of `features`"):+ LightGlue("disk", weights="superpoint_lightglue", n_layers=1)+ finally:+ if were_checks_enabled:+ enable_checks()++ @pytest.mark.parametrize("features", ["doghardnet", "dog_affnet_hardnet"])+ def test_convention_lightglue_doghardnet_prunes_and_stops_early_by_default(self, features, monkeypatch):+ # like the other LightGlue weights trained with token-confidence heads; -1 turns either off+ monkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda *args, **kwargs: {})+ lg = LightGlue(features, n_layers=1)+ assert (lg.conf.depth_confidence, lg.conf.width_confidence) == (0.95, 0.99)+ lg = LightGlue(features, depth_confidence=-1, width_confidence=-1, n_layers=1)+ assert (lg.conf.depth_confidence, lg.conf.width_confidence) == (-1, -1)class TestLightGlueAllPrunedScoreDtype(BaseTester):