Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(feature)!: LightGlue keyword arguments win over the features configuration; doghardnet stops early and prunes by default (#6421)

Dmytro Mishkin0d30c4169bf2
diff --git a/changelog.d/6421.breaking.md b/changelog.d/6421.breaking.md
new file mode 100644
index 000000000..0b43644ad
--- /dev/null
+++ b/changelog.d/6421.breaking.md
@@ -0,0 +1 @@
+`LightGlue("doghardnet")` and `LightGlue("dog_affnet_hardnet")`, and `LightGlueMatcher` in these modes, now stop early and prune points by default (`depth_confidence=0.95`, `width_confidence=0.99`), as the upstream doghardnet configuration does; they used to run every layer on every point. On 40 Oxford affine pairs with `DoGHardNet(2048)` the matcher runs 1.9x faster on CPU, finds 1.5 % fewer correct matches at 3 px, and estimates homographies as accurately. Pass `depth_confidence=-1, width_confidence=-1` for the previous matches. An architecture keyword argument that contradicts the weights of `features`, such as `LightGlue("disk", input_dim=64)`, used to be ignored and now fails, when the weights load or at the first call, except `n_layers` with `"xfeat"`: more than its 6 layers leaves the extra layers random and the model in training mode, fewer runs a truncated network. A `weights` keyword argument that names another checkpoint than the one of `features`, such as `LightGlue("disk", weights="superpoint_lightglue")`, used to be ignored and now raises: with keyword arguments winning, it would have cached the DISK checkpoint under the SuperPoint file name.
diff --git a/changelog.d/6421.fixed.md b/changelog.d/6421.fixed.md
new file mode 100644
index 000000000..e7e35cc0e
--- /dev/null
+++ b/changelog.d/6421.fixed.md
@@ -0,0 +1 @@
+Keyword arguments of `LightGlue` (and the `params` of `LightGlueMatcher`) now override the configuration of `features`. `depth_confidence`, `width_confidence` and `filter_threshold` given with `"xfeat"`, `"doghardnet"`, `"dog_affnet_hardnet"` or `"keynet_affnet_hardnet"` used to be replaced silently by the preset's values.
diff --git a/kornia/feature/lightglue.py b/kornia/feature/lightglue.py
index 983583c00..83054ffa3 100644
--- a/kornia/feature/lightglue.py
+++ b/kornia/feature/lightglue.py
@@ -29,6 +29,7 @@ from torch import nn
from kornia.core._compat import torch_version_lt
from kornia.core.check import KORNIA_CHECK
from kornia.core.download import hf_url, load_state_dict_from_url
+from kornia.core.exceptions import BaseError
from kornia.core.utils import is_exporting
from kornia.feature.laf import laf_to_three_points, scale_laf
@@ -508,8 +509,13 @@ class LightGlue(nn.Module):
features: The type of local features to match (e.g., 'superpoint', 'disk'), whose configuration and
pretrained weights are loaded. ``None`` builds the model from ``default_conf`` and the keyword arguments
and downloads no weights.
- **conf: keyword arguments that override entries of ``default_conf``, such as ``filter_threshold``; the
- configuration of ``features`` is applied after them and wins where both set a key.
+ **conf: keyword arguments that override entries of ``default_conf`` and of the configuration of
+ ``features``, such as ``filter_threshold`` or ``depth_confidence``. The configuration of ``features`` sets
+ the defaults, and an explicit keyword argument always wins, except ``weights``, which must name the
+ checkpoint of ``features`` when ``features`` is given; ``weights=None``, the value in ``default_conf``,
+ counts as not given. This departs on purpose from cvg/LightGlue, which applies the configuration of
+ ``features`` after the keyword arguments, so that the tuning knobs a preset sets, such as ``"xfeat"``'s
+ ``width_confidence``, can be changed.
"""
default_conf: ClassVar[Dict[str, Any]] = {
@@ -585,16 +591,12 @@ class LightGlue(nn.Module):
"dog_affnet_hardnet": {
"weights": "doghardnet",
"input_dim": 128,
- "width_confidence": -1,
- "depth_confidence": -1,
"add_scale_ori": True,
"scale_coef": 1.0 / 6.0,
},
"doghardnet": {
"weights": "doghardnet",
"input_dim": 128,
- "width_confidence": -1,
- "depth_confidence": -1,
"add_scale_ori": True,
"scale_coef": 1.0 / 6.0,
},
@@ -607,13 +609,24 @@ class LightGlue(nn.Module):
},
}
- def __init__(self, features: str = "superpoint", **conf_) -> None:
+ def __init__(self, features: Optional[str] = "superpoint", **conf_) -> None:
super().__init__()
- self.conf = conf = SimpleNamespace(**{**self.default_conf, **conf_})
+ if conf_.get("weights", "") is None:
+ # `weights=None` is the value of default_conf, so it means "not given", also when default_conf is passed
+ # on. It must not reach `conf.weights`, which names the cached file: "None" would be one cache name for
+ # every preset.
+ conf_ = {k: v for k, v in conf_.items() if k != "weights"}
if features is not None:
KORNIA_CHECK(features in list(self.features.keys()), "Features keys are wrong")
- for k, v in self.features[features].items():
- setattr(conf, k, v)
+ # The download URL follows `features` and the cached file name `weights`: they must name one checkpoint,
+ # or another checkpoint is cached under this name. A plain raise, so that disable_checks() keeps it.
+ if conf_.get("weights", self.features[features]["weights"]) != self.features[features]["weights"]:
+ raise BaseError(
+ "`weights` must name the checkpoint of `features`. To load other weights, build the model with "
+ "`features=None` and their architecture keyword arguments, then call `load_state_dict`"
+ )
+ preset = self.features[features] if features is not None else {}
+ self.conf = conf = SimpleNamespace(**{**self.default_conf, **preset, **conf_})
KORNIA_CHECK(not (self.conf.add_scale_ori and self.conf.add_laf)) # we use either scale ori, or LAF
if conf.input_dim != conf.descriptor_dim:
diff --git a/tests/feature/test_lightglue.py b/tests/feature/test_lightglue.py
index f2fb852ed..8cd70e894 100644
--- a/tests/feature/test_lightglue.py
+++ b/tests/feature/test_lightglue.py
@@ -20,6 +20,8 @@ import unittest.mock
import pytest
import torch
+from kornia.core.check import are_checks_enabled, disable_checks, enable_checks
+from kornia.core.exceptions import BaseError
from kornia.feature.integrated import LightGlueMatcher
from kornia.feature.laf import laf_from_center_scale_ori
from kornia.feature.lightglue import (
@@ -976,13 +978,46 @@ class TestConventionsLightGlue(BaseTester):
with pytest.raises(NotImplementedError, match="orb is not supported"):
LightGlueMatcher("orb")
- def test_convention_lightglue_features_configuration_wins_over_keyword_arguments(self, device, dtype, monkeypatch):
- # "disk" sets input_dim, so the keyword argument loses; filter_threshold is not in its configuration, so the
- # keyword argument stays
+ def test_convention_lightglue_keyword_arguments_win_over_features_configuration(self, device, dtype, monkeypatch):
+ # the configuration of the features gives the defaults, and an explicit keyword argument always wins: "xfeat"
+ # sets width_confidence and filter_threshold, "keynet_affnet_hardnet" turns off pruning and early stopping
monkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda *args, **kwargs: {})
- lg = LightGlue("disk", input_dim=64, filter_threshold=0.3, n_layers=1)
- assert lg.conf.input_dim == 128
- assert lg.conf.filter_threshold == 0.3
+ lg = LightGlue("xfeat", width_confidence=0.5, filter_threshold=0.3, n_layers=1)
+ assert (lg.conf.depth_confidence, lg.conf.width_confidence, lg.conf.filter_threshold) == (-1, 0.5, 0.3)
+ lg = LightGlue("keynet_affnet_hardnet", depth_confidence=0.95, width_confidence=0.99, n_layers=1)
+ assert (lg.conf.depth_confidence, lg.conf.width_confidence) == (0.95, 0.99)
+ assert LightGlue("keynet_affnet_hardnet", n_layers=1).conf.depth_confidence == -1
+ assert LightGlue("disk", n_layers=1).conf.input_dim == 128
+ assert LightGlue("disk", input_dim=64, n_layers=1).conf.input_dim == 64
+
+ def test_lightglue_weights_must_name_the_checkpoint_of_features(self, monkeypatch):
+ # the URL follows `features` and the cached file name `weights`: another checkpoint is cached under that name
+ monkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda *args, **kwargs: {})
+ assert LightGlue("disk", weights="disk_lightglue", n_layers=1).conf.weights == "disk_lightglue"
+ # weights=None is the value of default_conf and means "not given", also when default_conf is passed on
+ assert LightGlue("disk", weights=None, n_layers=1).conf.weights == "disk_lightglue"
+ splat = LightGlue("superpoint", **{**LightGlue.default_conf, "n_layers": 1})
+ assert splat.conf.weights == "superpoint_lightglue"
+ with pytest.raises(BaseError, match="must name the checkpoint of `features`"):
+ LightGlue("disk", weights="superpoint_lightglue", n_layers=1)
+ # a plain raise: disable_checks() must not let another checkpoint be cached under this name
+ were_checks_enabled = are_checks_enabled()
+ disable_checks()
+ try:
+ with pytest.raises(BaseError, match="must name the checkpoint of `features`"):
+ LightGlue("disk", weights="superpoint_lightglue", n_layers=1)
+ finally:
+ if were_checks_enabled:
+ enable_checks()
+
+ @pytest.mark.parametrize("features", ["doghardnet", "dog_affnet_hardnet"])
+ def test_convention_lightglue_doghardnet_prunes_and_stops_early_by_default(self, features, monkeypatch):
+ # like the other LightGlue weights trained with token-confidence heads; -1 turns either off
+ monkeypatch.setattr("kornia.feature.lightglue.load_state_dict_from_url", lambda *args, **kwargs: {})
+ lg = LightGlue(features, n_layers=1)
+ assert (lg.conf.depth_confidence, lg.conf.width_confidence) == (0.95, 0.99)
+ lg = LightGlue(features, depth_confidence=-1, width_confidence=-1, n_layers=1)
+ assert (lg.conf.depth_confidence, lg.conf.width_confidence) == (-1, -1)
class TestLightGlueAllPrunedScoreDtype(BaseTester):