Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

fix(geometry): keep single-vector SVD and eigh gradients finite when other values are equal (#6214)

Saksham Sharma15f12ea3be55
diff --git a/changelog.d/6214.fixed.md b/changelog.d/6214.fixed.md
new file mode 100644
index 000000000..300526efb
--- /dev/null
+++ b/changelog.d/6214.fixed.md
@@ -0,0 +1 @@
+`fit_plane`, `fit_line`, `average_quaternions`, `solve_pnp_dlt`, `find_homography_dlt(solver="svd")` and `find_homography_lines_dlt` return a finite gradient when the singular vector or eigenvector they keep is well defined but two of the other singular values or eigenvalues are equal. torch's SVD and `eigh` backward passes divide by the gap between every pair, so every entry of the gradient was NaN for some exact inputs, depending on the SVD backend: the corners of a square checkerboard (`fit_plane`), points along a coordinate axis (`fit_line`), identical members or rotations about one axis (`average_quaternions`), world points on the corners of a cube (`solve_pnp_dlt`) and the identity homography on an exact grid. The vector's value is unchanged; its gradient now divides only by the gaps to its own value, and equals the SVD's gradient to rounding wherever that is finite. Second derivatives use the same gaps, so they are finite at those inputs too.
diff --git a/kornia/core/utils.py b/kornia/core/utils.py
index 7def1b06e..b4fc2f9f3 100644
--- a/kornia/core/utils.py
+++ b/kornia/core/utils.py
@@ -461,14 +461,20 @@ def _torch_svd_cast(input: torch.Tensor, full_matrices: bool = True) -> Tuple[to
is what keeps the batched minimal solvers behind ``RANSAC`` working on Apple silicon.
``full_matrices`` selects full or reduced factors, as in ``torch.linalg.svd``.
"""
- if is_mps_tensor_safe(input):
- dtype = torch.float32
- elif input.dtype == torch.float32:
- dtype = torch.float64
- else:
- dtype = _normalize_to_float32_or_float64(input.dtype)
+ out1, out2, out3 = _torch_svd_in_work_dtype(input, full_matrices)
+ return (out1.to(input.dtype), out2.to(input.dtype), out3.to(input.dtype))
- x = input.to(dtype)
+
+def _torch_svd_in_work_dtype(
+ input: torch.Tensor, full_matrices: bool = True
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Decompose ``input`` as :func:`_torch_svd_cast` does, returning ``U, S, V`` in :func:`_svd_work_dtype`.
+
+ A caller that needs the factors at the precision they were computed in, such as for a derivative, uses this
+ instead of casting ``input`` to the work dtype first: :func:`_torch_svd_cast` would promote a float32 input
+ to float64 again, so a float16 or bfloat16 input would be decomposed in float64 instead of float32.
+ """
+ x = input.to(_svd_work_dtype(input))
# torch 2.14's MPS ``linalg.svd`` raises "Failed to created pipeline state object" -- a Metal
# shader compilation failure, not memory pressure -- once an input holds 8192 elements
# or more, whatever the per-matrix shape. The bound is inclusive: 8192 is the first failing
@@ -483,8 +489,103 @@ def _torch_svd_cast(input: torch.Tensor, full_matrices: bool = True) -> Tuple[to
else:
out1, out2, out3H = torch.linalg.svd(x, full_matrices=full_matrices)
# Since kornia requires torch>=2.5.1, we can always use .mH
- out3 = out3H.mH
- return (out1.to(input.dtype), out2.to(input.dtype), out3.to(input.dtype))
+ return out1, out2, out3H.mH
+
+
+def _svd_work_dtype(input: torch.Tensor) -> torch.dtype:
+ """Return the dtype :func:`_torch_svd_cast` decomposes ``input`` in: float64 for float32, except on MPS."""
+ if is_mps_tensor_safe(input):
+ return torch.float32
+ if input.dtype == torch.float32:
+ return torch.float64
+ return _normalize_to_float32_or_float64(input.dtype)
+
+
+def _needs_derivatives(input: torch.Tensor) -> bool:
+ """Return whether autograd can ask for a derivative through ``input``.
+
+ Reverse mode needs grad mode and an ``input`` that requires grad; forward mode (``torch.func.jvp``, ``jacfwd``,
+ ``torch.autograd.forward_ad``) gives ``input`` a tangent instead, whatever the grad mode.
+ """
+ if torch.is_grad_enabled() and input.requires_grad:
+ return True
+ return torch.autograd.forward_ad.unpack_dual(input).tangent is not None
+
+
+def _symmetric_eigenvector(
+ input: torch.Tensor, eigenvalues: torch.Tensor, eigenvectors: torch.Tensor, index: Union[int, torch.Tensor]
+) -> torch.Tensor:
+ r"""Return the eigenvector ``eigenvectors[..., :, index]`` of the symmetric ``input``, differentiably.
+
+ The backward passes of ``torch.linalg.eigh`` and ``torch.linalg.svd`` divide by the gap between every pair of
+ eigenvalues, so the gradient of a single eigenvector is NaN where two of the *other* eigenvalues are equal, although
+ the vector is well defined while its own eigenvalue :math:`\lambda_k` is simple. With
+ :math:`R = \sum_{i \ne k} v_i v_i^\top / (\lambda_k - \lambda_i)`, the eigenvector of :math:`C` moves by
+ :math:`R\,dC\,v` to first order and by
+ :math:`R\,dC\,R\,dC\,v - (v^\top dC\,v)\,R^2 dC\,v - \tfrac12 v\,\|R\,dC\,v\|^2` to second. Both divide only by the
+ gaps to :math:`\lambda_k`, and an equal pair among the other eigenvalues enters only through its projector, which
+ does not depend on the basis chosen within it. The first-order term is the derivation of ``_NullSpaceBasis`` in
+ :mod:`kornia.geometry.epipolar.essential`.
+
+ The two terms are added to the detached eigenvector with ``dC = C - C.detach()``, which is zero in value (the
+ detached-value pattern of #4229): the value is the given eigenvector, and only the derivatives come from the terms.
+ A gap that is exactly zero, where the eigenvector is not unique, contributes nothing.
+
+ Args:
+ input: the symmetric matrix ``C``, of shape ``(*, D, D)``.
+ eigenvalues: eigenvalues of ``C``, of shape ``(*, K)``; for a positive semidefinite ``C`` they can be its
+ singular values.
+ eigenvectors: the matching orthonormal eigenvectors of ``C`` as columns, of shape ``(*, D, K)``.
+ index: the column to return, as an ``int`` or a ``(*, 1)`` integer tensor.
+
+ Returns:
+ The eigenvector, of shape ``(*, D)`` in the dtype of ``eigenvectors``, with first and second derivatives
+ with respect to ``input``.
+ """
+ lam = eigenvalues.detach()
+ V = eigenvectors.detach()
+ if isinstance(index, int):
+ v = V[..., :, index]
+ lam_k = lam[..., index, None]
+ else:
+ # gather rather than indexing with a tensor, which torch.compile would turn into a data-dependent shape
+ v = V.gather(-1, index[..., None, :].expand(*V.shape[:-1], 1))[..., 0]
+ lam_k = lam.gather(-1, index)
+ if not _needs_derivatives(input):
+ return v
+ columns = torch.arange(lam.shape[-1], device=lam.device)
+ others = columns != (index % lam.shape[-1] if isinstance(index, int) else index)
+ gap = lam_k - lam
+ # The kept column is excluded by position, not by its gap being zero: CUDA Inductor can evaluate an eigenvalue
+ # computed in the graph, such as (S / scale)^2, differently at its two reads, and a gap of 1e-20 there put 1e20
+ # into R.
+ nonzero = others & (gap.abs() > torch.finfo(gap.dtype).tiny)
+ inv_gap = torch.where(nonzero, 1.0 / torch.where(nonzero, gap, torch.ones_like(gap)), torch.zeros_like(gap))
+ R = (V * inv_gap[..., None, :]) @ V.mT
+ C = input.to(V.dtype)
+ dC = C - C.detach()
+ v = v[..., None]
+ dv = dC @ v
+ first = R @ dv
+ second = R @ (dC @ first) - (v.mT @ dv) * (R @ first) - 0.5 * v * (first.mT @ first)
+ return (v + first + second)[..., 0]
+
+
+def _right_singular_vector(input: torch.Tensor, S: torch.Tensor, V: torch.Tensor, index: int) -> torch.Tensor:
+ """Return the right singular vector ``V[..., :, index]`` of the ``(*, N, D)`` ``input``, differentiably.
+
+ ``S`` and ``V`` are the singular values and the right singular vectors, as columns, of ``input``. The vector is
+ the eigenvector of ``X^T X`` with eigenvalue ``S^2``, with the derivatives of :func:`_symmetric_eigenvector`.
+ ``X`` is ``input`` in the dtype of ``V`` divided by its largest magnitude, held constant, which leaves the vector
+ unchanged and keeps ``X^T X`` in range.
+ """
+ if not _needs_derivatives(input):
+ return V.detach()[..., :, index]
+ x = input.to(V.dtype)
+ scale = x.detach().abs().amax(dim=(-2, -1), keepdim=True)
+ scale = torch.where(scale > 0, scale, torch.ones_like(scale))
+ x = x / scale
+ return _symmetric_eigenvector(x.mT @ x, (S.detach() / scale[..., 0]) ** 2, V, index)
def _torch_linalg_svdvals(input: torch.Tensor) -> torch.Tensor:
diff --git a/kornia/geometry/calibration/pnp.py b/kornia/geometry/calibration/pnp.py
index 683976d85..ee212fe34 100644
--- a/kornia/geometry/calibration/pnp.py
+++ b/kornia/geometry/calibration/pnp.py
@@ -24,7 +24,13 @@ from torch.linalg import qr as linalg_qr
from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SAME_SHAPE, KORNIA_CHECK_SHAPE
from kornia.core.ops import eye_like
-from kornia.core.utils import _autocast_disabled, _torch_linalg_svdvals, _torch_svd_cast
+from kornia.core.utils import (
+ _autocast_disabled,
+ _right_singular_vector,
+ _svd_work_dtype,
+ _torch_linalg_svdvals,
+ _torch_svd_cast,
+)
from kornia.geometry.conversions import convert_points_to_homogeneous, normalize_points_with_intrinsics
from kornia.geometry.linalg import transform_points
@@ -257,8 +263,11 @@ def solve_pnp_dlt(
# Getting the solution vectors.
# Only the right singular vectors are used; reduced SVD avoids saving the unused (2N, 2N) U.
- _, _, v = _torch_svd_cast(system, full_matrices=False)
- solution = v[..., -1]
+ # The SVD is of the detached system: the solution's gradient comes from _right_singular_vector, which stays
+ # finite when other singular values are equal, as for world points on the corners of a cube.
+ work = system.detach().to(_svd_work_dtype(system))
+ _, s, v = _torch_svd_cast(work, full_matrices=False)
+ solution = _right_singular_vector(system, s, v, -1).to(system.dtype)
# Reshaping the solution vectors to the correct shape.
solution = solution.reshape(B, 3, 4)
diff --git a/kornia/geometry/homography.py b/kornia/geometry/homography.py
index a4ef16c20..7dd24b2d7 100644
--- a/kornia/geometry/homography.py
+++ b/kornia/geometry/homography.py
@@ -55,7 +55,8 @@ from kornia.core.exceptions import ShapeError
from kornia.core.utils import (
_autocast_disabled,
_extract_device_dtype,
- _torch_svd_cast,
+ _symmetric_eigenvector,
+ _torch_svd_in_work_dtype,
safe_inverse_with_mask,
safe_solve_with_mask,
)
@@ -755,11 +756,14 @@ def _svd_null_vector(normal: torch.Tensor) -> torch.Tensor:
finite = normal.isfinite().flatten(1).all(-1)
safe = torch.where(finite[:, None, None], normal, torch.zeros_like(normal))
try:
- _, _, V = _torch_svd_cast(safe)
+ # A half input is decomposed in float32, and S and V stay at the precision they were computed in.
+ _, S, V = _torch_svd_in_work_dtype(safe.detach())
except RuntimeError:
warnings.warn("SVD did not converge", RuntimeWarning, stacklevel=1)
return torch.full_like(normal[..., 0], float("nan"))
- null = V[..., -1]
+ # The normal matrix is positive semidefinite, so its singular values are its eigenvalues, and the null vector's
+ # gradient stays finite when other eigenvalues are equal, as for the identity homography on a grid.
+ null = _symmetric_eigenvector(safe, S, V, -1).to(normal.dtype)
return torch.where(finite[:, None], null, torch.full_like(null, float("nan")))
diff --git a/kornia/geometry/line.py b/kornia/geometry/line.py
index e379ceb3a..edd619a96 100644
--- a/kornia/geometry/line.py
+++ b/kornia/geometry/line.py
@@ -43,7 +43,13 @@ from torch import nn
from kornia.core.check import KORNIA_CHECK_IS_TENSOR, KORNIA_CHECK_SHAPE, KORNIA_CHECK_SIZES, are_checks_enabled
from kornia.core.exceptions import ValueCheckError
-from kornia.core.utils import _torch_svd_cast, is_compiling, register_module_state
+from kornia.core.utils import (
+ _svd_work_dtype,
+ _symmetric_eigenvector,
+ _torch_svd_cast,
+ is_compiling,
+ register_module_state,
+)
from kornia.geometry.conversions import _normalize_last_dim
from kornia.geometry.linalg import batched_dot_product
from kornia.geometry.plane import Hyperplane
@@ -530,11 +536,14 @@ def fit_line(points: torch.Tensor, weights: Optional[torch.Tensor] = None) -> Pa
A = (A.transpose(-2, -1) * work_weights[..., None, :]) @ A if weights is not None else A.transpose(-2, -1) @ A
# NOTE: not optimal for 2d points, but for now works for other dimensions
- _, _, V = _torch_svd_cast(A)
- V = V.transpose(-2, -1)
+ # The scatter matrix is positive semidefinite, so its singular values are its eigenvalues. The SVD is of the
+ # detached matrix: the direction's gradient comes from _symmetric_eigenvector, which stays finite when the other
+ # eigenvalues are equal, as for points along a coordinate axis.
+ work = A.detach().to(_svd_work_dtype(A))
+ _, S, V = _torch_svd_cast(work)
# the first left eigenvector is the direction on the fitted line
- direction = V[..., 0, :].to(out_dtype) # BxD
+ direction = _symmetric_eigenvector(A, S, V, 0).to(compute_dtype).to(out_dtype) # BxD
origin = mean[..., 0, :].to(out_dtype) # BxD
return ParametrizedLine(origin, direction)
diff --git a/kornia/geometry/plane.py b/kornia/geometry/plane.py
index 00b726a58..58f2cc47b 100644
--- a/kornia/geometry/plane.py
+++ b/kornia/geometry/plane.py
@@ -44,7 +44,14 @@ from torch import nn
from kornia.core.check import KORNIA_CHECK, KORNIA_CHECK_SHAPE, KORNIA_CHECK_TYPE, are_checks_enabled
from kornia.core.exceptions import BaseError, ValueCheckError
from kornia.core.tensor_wrapper import _unwrap, _wrap
-from kornia.core.utils import _torch_linalg_svdvals, _torch_svd_cast, is_compiling, register_module_state
+from kornia.core.utils import (
+ _right_singular_vector,
+ _svd_work_dtype,
+ _torch_linalg_svdvals,
+ _torch_svd_cast,
+ is_compiling,
+ register_module_state,
+)
from kornia.geometry.linalg import batched_dot_product
from kornia.geometry.vector import Scalar, Vector3
@@ -408,19 +415,22 @@ def fit_plane(points: Vector3) -> Hyperplane:
points_centered = points_work - mean
# With at least three 3D points, reduced SVD retains all right singular vectors while
- # avoiding an unused N-by-N left factor, including when autograd saves it for backward.
- _, S, V = _torch_svd_cast(points_centered, full_matrices=False)
+ # avoiding an unused N-by-N left factor. The SVD is of the detached points: the normal's gradient comes from
+ # _right_singular_vector, which stays finite when the two in-plane singular values are equal.
+ work = points_centered.detach().to(_svd_work_dtype(points_centered))
+ _, S, V = _torch_svd_cast(work, full_matrices=False)
# The plane is determined when the centred points have rank 2: the second singular value
# must be nonzero relative to the first, a scale-invariant test that keeps a small or thin
# valid set working. Keep the input-dtype tolerance even when the fit uses higher precision.
- if checks and not bool((S[..., 1] > S[..., 0] * _rank_tolerance(points_data.dtype)).all()):
+ S_work = S.to(points_work.dtype)
+ if checks and not bool((S_work[..., 1] > S_work[..., 0] * _rank_tolerance(points_data.dtype)).all()):
raise ValueCheckError(
"fit_plane requires points that are not collinear; the given points do not determine a plane."
)
# the first left eigenvector is the direction on the fited line
- direction = V[..., :, -1] # BxD
+ direction = _right_singular_vector(points_centered, S, V, -1).to(points_work.dtype) # BxD
origin = mean[..., 0, :] # BxD
if half:
offset = -batched_dot_product(direction, origin)
diff --git a/kornia/geometry/quaternion.py b/kornia/geometry/quaternion.py
index 187a38f9e..429459e50 100644
--- a/kornia/geometry/quaternion.py
+++ b/kornia/geometry/quaternion.py
@@ -50,6 +50,7 @@ import torch
from torch import nn
from kornia.core.check import KORNIA_CHECK_TYPE
+from kornia.core.utils import _symmetric_eigenvector
from kornia.geometry.conversions import (
axis_angle_to_quaternion,
euler_from_quaternion,
@@ -824,13 +825,13 @@ def average_quaternions(Q: "Quaternion", w: Optional[torch.Tensor] = None) -> "Q
orig_dtype = A.dtype
if A.dtype in (torch.float16, torch.bfloat16):
A = A.float()
- eigenvalues, eigenvectors = torch.linalg.eigh(A)
+ # The decomposition is of the detached matrix: the mean's gradient comes from _symmetric_eigenvector, which stays
+ # finite when the other eigenvalues are equal, as for identical members or rotations about one axis.
+ eigenvalues, eigenvectors = torch.linalg.eigh(A.detach())
# Use float32 eigenvalues for argmax to avoid half-precision rounding
# changing which eigenvector is selected when eigenvalues are close.
max_idx = torch.argmax(eigenvalues)
- eigenvectors = eigenvectors.to(orig_dtype)
- indices = max_idx.reshape(1, 1).expand(eigenvectors.shape[0], 1)
- q_avg = torch.gather(eigenvectors, 1, indices).squeeze(1)
+ q_avg = _symmetric_eigenvector(A, eigenvalues, eigenvectors, max_idx.reshape(1)).to(orig_dtype)
q_avg = q_avg / q_avg.norm()
return Quaternion(q_avg.unsqueeze(0))
diff --git a/tests/core/test_helpers.py b/tests/core/test_helpers.py
index 5b81a6c13..1011536f5 100644
--- a/tests/core/test_helpers.py
+++ b/tests/core/test_helpers.py
@@ -34,6 +34,7 @@ from kornia.core.utils import (
_torch_linalg_svdvals,
_torch_solve_cast,
_torch_svd_cast,
+ _torch_svd_in_work_dtype,
batched_forward,
is_autocast_enabled,
is_compiling,
@@ -611,6 +612,26 @@ class TestSvdCast:
assert a.grad.device == a.device
assert not a.grad.isnan().any()
+ @pytest.mark.parametrize(
+ ("dtype_in", "work"),
+ [(torch.float16, torch.float32), (torch.bfloat16, torch.float32), (torch.float32, torch.float64)],
+ )
+ def test_in_work_dtype_decomposes_once(self, device, dtype_in, work):
+ # The factors come back in the work dtype, and a half input is decomposed in float32, not promoted again to
+ # float64 as casting it first and calling _torch_svd_cast would do (#6214).
+ if device.type == "mps":
+ pytest.skip("MPS decomposes in float32 whatever the input dtype")
+ torch.manual_seed(0)
+ a = torch.randn(3, 9, 9, device=device).to(dtype_in)
+ u, s, v = _torch_svd_in_work_dtype(a)
+ assert all(t.dtype == work for t in (u, s, v))
+ _, s_ref, vh_ref = torch.linalg.svd(a.to(work))
+ assert torch.equal(s, s_ref)
+ assert torch.equal(v, vh_ref.mH)
+ _, s_cast, v_cast = _torch_svd_cast(a)
+ assert torch.equal(s_cast, s.to(dtype_in))
+ assert torch.equal(v_cast, v.to(dtype_in))
+
class TestSolveCast:
def test_rejects_non_tensor_A(self):
diff --git a/tests/geometry/calibration/test_pnp.py b/tests/geometry/calibration/test_pnp.py
index d5728880a..d57af00fe 100644
--- a/tests/geometry/calibration/test_pnp.py
+++ b/tests/geometry/calibration/test_pnp.py
@@ -447,3 +447,75 @@ class TestSolvePnpDltAutocast(BaseTester):
assert actual.dtype == torch.float32
self.assert_close(actual, expected, atol=0, rtol=0)
self.assert_close(actual, expected_pose, atol=1e-4, rtol=0)
+
+
+class TestSolvePnpDltGradient(BaseTester):
+ """``solve_pnp_dlt``'s gradient is finite when other singular values of the system are equal (#6213)."""
+
+ def test_cube_corners_have_the_central_difference_gradient_6213(self, device):
+ # The corners of a cube and its centre make singular values of the DLT system equal in some poses, where
+ # torch's SVD backward made the gradient with respect to the image points NaN. Which poses tie depends on
+ # the SVD backend: 4 to 6 of these 64 on the CPUs measured, and none in float64 on CUDA, so this test pins
+ # the fix on CPU.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ from kornia.geometry.conversions import axis_angle_to_rotation_matrix
+
+ dtype = torch.float64
+ g = torch.Generator().manual_seed(0)
+ corners = [[x, y, z] for x in (-1.0, 1.0) for y in (-1.0, 1.0) for z in (-1.0, 1.0)] + [[0.0, 0.0, 0.0]]
+ cube = torch.tensor(corners, dtype=dtype)
+ batch = 64
+ rotation = axis_angle_to_rotation_matrix(torch.rand(batch, 3, generator=g, dtype=dtype) - 0.5)
+ translation = torch.tensor([0.0, 0.0, 6.0], dtype=dtype) + torch.rand(batch, 1, 3, generator=g, dtype=dtype)
+ intrinsics = torch.tensor([[500.0, 0.0, 320.0], [0.0, 500.0, 240.0], [0.0, 0.0, 1.0]], dtype=dtype)
+ camera = cube @ rotation.mT + translation - 0.5
+ projected = camera @ intrinsics.T
+ img_points = (projected[..., :2] / projected[..., 2:]).to(device)
+ world_points = cube.expand(batch, -1, -1).to(device)
+ intrinsics = intrinsics.expand(batch, -1, -1).to(device)
+
+ def solve(u):
+ return kornia.geometry.solve_pnp_dlt(world_points, u, intrinsics).reshape(batch, -1)
+
+ u = img_points.clone().requires_grad_(True)
+ out = solve(u)
+ # Each pose depends only on its own points, so one backward per output entry gives every pose's Jacobian.
+ jac = torch.stack([torch.autograd.grad(out[:, j].sum(), u, retain_graph=True)[0] for j in range(12)], 1)
+ assert torch.isfinite(jac).all()
+ eps = 1e-5
+ columns = []
+ for i in range(img_points[0].numel()):
+ step = torch.zeros(img_points[0].numel(), device=device, dtype=dtype)
+ step[i] = eps
+ step = step.view_as(img_points[0])
+ columns.append((solve(img_points + step) - solve(img_points - step)) / (2 * eps))
+ self.assert_close(jac.reshape(batch, 12, -1), torch.stack(columns, -1), rtol=1e-5, atol=1e-7)
+
+ def test_dynamo_gradient_matches_eager_6213(self, device, dtype, torch_optimizer):
+ # The kept vector's own gap is excluded by position: CUDA Inductor evaluated (S / scale)^2 differently at its
+ # two reads, a gap of 1e-20 instead of 0, and the compiled gradient of solve_pnp_dlt was up to 1e11 times off.
+ if dtype not in (torch.float32, torch.float64):
+ pytest.skip("solve_pnp_dlt accepts float32 and float64 only")
+ from kornia.geometry.conversions import axis_angle_to_rotation_matrix
+
+ g = torch.Generator().manual_seed(0)
+ batch = 2
+ world = torch.rand(batch, 10, 3, generator=g, dtype=torch.float64) * 2 - 1
+ rotation = axis_angle_to_rotation_matrix(torch.rand(batch, 3, generator=g, dtype=torch.float64) - 0.5)
+ camera = world @ rotation.mT + torch.tensor([0.0, 0.0, 6.0], dtype=torch.float64)
+ intrinsics = torch.tensor([[500.0, 0.0, 320.0], [0.0, 500.0, 240.0], [0.0, 0.0, 1.0]], dtype=torch.float64)
+ projected = camera @ intrinsics.T
+ img_points = (projected[..., :2] / projected[..., 2:]).to(device, dtype)
+ world = world.to(device, dtype)
+ intrinsics = intrinsics.expand(batch, -1, -1).to(device, dtype)
+ weights = torch.randn(batch, 3, 4, generator=g, dtype=torch.float64).to(device, dtype)
+
+ def op(u):
+ return kornia.geometry.solve_pnp_dlt(world, u, intrinsics)
+
+ u_eager = img_points.clone().requires_grad_(True)
+ u_compiled = img_points.clone().requires_grad_(True)
+ (expected,) = torch.autograd.grad((op(u_eager) * weights).sum(), u_eager)
+ (actual,) = torch.autograd.grad((torch_optimizer(op)(u_compiled) * weights).sum(), u_compiled)
+ self.assert_close(actual, expected)
diff --git a/tests/geometry/test_homography.py b/tests/geometry/test_homography.py
index 615845493..8167e4943 100644
--- a/tests/geometry/test_homography.py
+++ b/tests/geometry/test_homography.py
@@ -805,7 +805,7 @@ class TestFindHomographyDLT(BaseTester):
def fail(*args, **kwargs):
raise RuntimeError("linalg.svd: The algorithm failed to converge")
- monkeypatch.setattr("kornia.geometry.homography._torch_svd_cast", fail)
+ monkeypatch.setattr("kornia.geometry.homography._torch_svd_in_work_dtype", fail)
points1 = torch.rand(2, 6, 2, device=device, dtype=dtype)
points2 = torch.rand(2, 6, 2, device=device, dtype=dtype)
with pytest.warns(RuntimeWarning, match="SVD did not converge"):
@@ -1127,7 +1127,7 @@ class TestFindHomographyFromLinesDLT(BaseTester):
def fail(*args, **kwargs):
raise RuntimeError("linalg.svd: The algorithm failed to converge")
- monkeypatch.setattr("kornia.geometry.homography._torch_svd_cast", fail)
+ monkeypatch.setattr("kornia.geometry.homography._torch_svd_in_work_dtype", fail)
ls1 = torch.rand(2, 5, 2, 2, device=device, dtype=dtype)
ls2 = torch.rand(2, 5, 2, 2, device=device, dtype=dtype)
with pytest.warns(RuntimeWarning, match="SVD did not converge"):
@@ -2175,3 +2175,23 @@ class TestHomographySharedKernels(BaseTester):
pts2 = torch.rand(1, 5, 2, device=device, dtype=torch.float64)
H = create_random_homography(torch.zeros(3, 1, device=device, dtype=torch.float64), 3, std_val=0.1)
self.gradcheck(lambda a, b, h: _oneway_transfer_error_shared_impl_(a, b, h, True), (pts1, pts2, H))
+
+
+class TestFindHomographyDLTSvdGradient(BaseTester):
+ """The ``"svd"`` solver's gradient is finite when other eigenvalues of the normal matrix are equal (#6213)."""
+
+ @pytest.mark.parametrize("n", [2, 3, 4, 5])
+ @pytest.mark.parametrize("centred", [False, True])
+ def test_identity_on_an_exact_grid_6213(self, device, n, centred):
+ # H = I on an exact grid gives the normal matrix repeated eigenvalues; on CPU the 3 x 3 grid in [0, 1] made
+ # every entry of the gradient NaN through torch's SVD backward.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ dtype = torch.float64
+ ys, xs = torch.meshgrid(
+ torch.linspace(0, 1, n, dtype=dtype), torch.linspace(0, 1, n, dtype=dtype), indexing="ij"
+ )
+ grid = torch.stack([xs.flatten(), ys.flatten()], -1)[None].to(device)
+ if centred:
+ grid = grid * 2 - 1
+ self.gradcheck(lambda p1, p2: find_homography_dlt(p1, p2, solver="svd"), (grid, grid.clone()), fast_mode=False)
diff --git a/tests/geometry/test_line.py b/tests/geometry/test_line.py
index b82d2304b..86f708393 100644
--- a/tests/geometry/test_line.py
+++ b/tests/geometry/test_line.py
@@ -1244,3 +1244,35 @@ class TestConventionsFitLine(BaseTester):
self.assert_close(torch.linalg.vector_norm(line.direction[i]), one)
single = fit_line(rows[i : i + 1], w[i : i + 1] if weighted else None).direction[0]
self.assert_close(line.direction[i], torch.sign((line.direction[i] * single).sum()) * single)
+
+
+def _direction_with_fixed_sign_6213(points, reference):
+ # The direction's sign is unspecified for D >= 3; fix it against a reference for central differences.
+ direction = fit_line(points).direction
+ return direction * torch.where((direction.detach() * reference).sum(-1, keepdim=True) < 0, -1.0, 1.0)
+
+
+class TestFitLineGradient(BaseTester):
+ """``fit_line``'s gradient is finite when the other eigenvalues of the scatter matrix are equal (#6213)."""
+
+ @pytest.mark.parametrize("dim", [3, 4])
+ def test_points_along_a_coordinate_axis_6213(self, device, dim):
+ # The scatter matrix of points along an axis is diag(s, 0, ..., 0): its other eigenvalues are all zero, and
+ # torch's SVD backward divided by their zero gaps, so every entry of the gradient was NaN.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ points = torch.zeros(1, 5, dim, device=device, dtype=torch.float64)
+ points[..., 0] = torch.linspace(0.0, 1.0, 5, device=device, dtype=torch.float64)
+ reference = fit_line(points).direction.detach()
+ self.gradcheck(lambda p: _direction_with_fixed_sign_6213(p, reference), (points,), fast_mode=False)
+
+ def test_equal_spread_across_the_line_6213(self, device):
+ # A cloud along x with the same spread in y and z: the two smaller eigenvalues are equal.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ x = torch.linspace(-2.0, 2.0, 5, dtype=torch.float64)
+ across = torch.tensor([[0.1, 0.0], [0.0, 0.1], [-0.1, 0.0], [0.0, -0.1]], dtype=torch.float64)
+ points = torch.cat([torch.cat([torch.full((4, 1), float(t), dtype=torch.float64), across], -1) for t in x])
+ points = points[None].to(device)
+ reference = fit_line(points).direction.detach()
+ self.gradcheck(lambda p: _direction_with_fixed_sign_6213(p, reference), (points,), fast_mode=False)
diff --git a/tests/geometry/test_plane.py b/tests/geometry/test_plane.py
index 8ebf592b1..9b60163f6 100644
--- a/tests/geometry/test_plane.py
+++ b/tests/geometry/test_plane.py
@@ -785,3 +785,168 @@ class TestConventionsFitPlane(BaseTester):
for wrong in (points[:, :2], torch.cat([points, points[:, :1]], -1)):
with pytest.raises(TypeError, match=r"vector must be \(\*, 3\)"):
fit_plane(wrong)
+
+
+def _board_6213(rows, cols, device, dtype=torch.float64, posed=True):
+ """Corners of a planar ``rows`` x ``cols`` checkerboard, optionally posed in front of a camera."""
+ ys, xs = torch.meshgrid(torch.arange(rows, dtype=dtype), torch.arange(cols, dtype=dtype), indexing="ij")
+ corners = torch.stack([xs.flatten(), ys.flatten(), torch.zeros(rows * cols, dtype=dtype)], -1) * 0.025
+ if posed:
+ from kornia.geometry.conversions import axis_angle_to_rotation_matrix
+
+ rotation = axis_angle_to_rotation_matrix(torch.tensor([[0.3, -0.4, 0.2]], dtype=dtype))[0]
+ corners = corners @ rotation.T + torch.tensor([0.1, 0.2, 2.0], dtype=dtype)
+ return corners.to(device)
+
+
+def _plane_with_fixed_sign_6213(points, reference):
+ # The normal's sign is unspecified; fix it against a reference so that central differences are meaningful.
+ plane = fit_plane(Vector3(points))
+ normal, offset = plane.normal.data, plane.offset.data.reshape(*plane.normal.data.shape[:-1], 1)
+ sign = torch.where((normal.detach() * reference).sum(-1, keepdim=True) < 0, -1.0, 1.0)
+ return torch.cat([normal * sign, offset * sign], -1)
+
+
+class TestFitPlaneGradient(BaseTester):
+ """``fit_plane``'s gradient is finite when the two in-plane singular values are equal (#6213)."""
+
+ @pytest.mark.parametrize(
+ "rows, cols, posed, noise",
+ [(5, 5, True, 0.0), (7, 7, True, 0.0), (2, 2, False, 0.0), (3, 3, False, 0.0), (4, 6, True, 0.0)]
+ + [(5, 5, True, 1e-3), (7, 7, True, 1e-2)],
+ )
+ def test_square_boards_have_the_central_difference_gradient_6213(self, device, rows, cols, posed, noise):
+ # torch's SVD backward divides by s_i^2 - s_j^2 for every pair, so the bit-equal in-plane singular values of
+ # these square boards made every entry of the gradient NaN, although the normal is well defined. Which boards
+ # tie exactly depends on the SVD backend. With noise, s_min is not zero.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ points = _board_6213(rows, cols, device, posed=posed)
+ if noise:
+ g = torch.Generator().manual_seed(0)
+ points = points + noise * torch.randn(points.shape, generator=g, dtype=points.dtype).to(device)
+ reference = fit_plane(Vector3(points)).normal.data.detach()
+ jac = torch.autograd.functional.jacobian(lambda p: _plane_with_fixed_sign_6213(p, reference), points)
+ jac = jac.reshape(4, -1)
+ assert torch.isfinite(jac).all()
+ eps = 1e-7
+ columns = []
+ for i in range(points.numel()):
+ step = torch.zeros(points.numel(), device=device, dtype=points.dtype)
+ step[i] = eps
+ step = step.view_as(points)
+ columns.append(
+ (
+ _plane_with_fixed_sign_6213(points + step, reference)
+ - _plane_with_fixed_sign_6213(points - step, reference)
+ ).reshape(-1)
+ / (2 * eps)
+ )
+ self.assert_close(jac, torch.stack(columns, -1), atol=1e-6, rtol=1e-6)
+
+ def test_axis_aligned_in_plane_pair_has_first_and_second_derivatives_6213(self, device):
+ # (+-1, 0, 0), (0, +-1, 0) make the in-plane singular values equal by construction, on every SVD backend.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ points = torch.tensor([[1.0, 0.0, 0.0], [-1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, -1.0, 0.0]], device=device)
+ points = points.to(torch.float64)
+ reference = fit_plane(Vector3(points)).normal.data.detach()
+ self.gradcheck(lambda p: _plane_with_fixed_sign_6213(p, reference), (points,), fast_mode=False)
+ p = points.clone().requires_grad_(True)
+ assert torch.autograd.gradgradcheck(lambda p: _plane_with_fixed_sign_6213(p, reference), (p,))
+
+ def test_batched_square_boards_6213(self, device):
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ board = _board_6213(5, 5, device)
+ points = torch.stack([board, board * 2, board + 1.0, board.flip(0)]).requires_grad_(True)
+ plane = fit_plane(Vector3(points))
+ (plane.normal.data.sum() + plane.offset.data.sum()).backward()
+ assert torch.isfinite(points.grad).all()
+
+ @pytest.mark.parametrize(
+ "points_dtype, scale",
+ [(torch.float32, 1e-20), (torch.float32, 1e22), (torch.float64, 1e-160), (torch.float64, 1e160)],
+ )
+ def test_extreme_coordinates_keep_the_normal_and_a_finite_gradient_6213(self, device, points_dtype, scale):
+ # X^T X overflows or underflows near the square root of the dtype's range, where the SVD still works: the
+ # gradient path must not turn the normal into NaN, and its gradient must stay right.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ board = _board_6213(5, 5, device, dtype=points_dtype)
+ points = (board * scale).requires_grad_(True)
+ plane = fit_plane(Vector3(points))
+ expected = fit_plane(Vector3(board)).normal.data
+ assert torch.isfinite(plane.normal.data).all()
+ self.assert_close(
+ (plane.normal.data * expected).sum(-1).abs(),
+ torch.ones((), device=device, dtype=points_dtype),
+ atol=1e-5,
+ rtol=0,
+ )
+ # The normal is scale-invariant, so its gradient at scale s is the unit-scale gradient divided by s.
+ weights = torch.tensor([0.3, -0.7, 1.1], device=device, dtype=points_dtype)
+ (_plane_with_fixed_sign_6213(points, expected)[..., :3] * weights).sum().backward()
+ unit = board.clone().requires_grad_(True)
+ (_plane_with_fixed_sign_6213(unit, expected)[..., :3] * weights).sum().backward()
+ assert torch.isfinite(points.grad).all()
+ self.assert_close(points.grad * scale, unit.grad, rtol=1e-4, atol=1e-4 * unit.grad.abs().max().item())
+
+ def test_value_is_the_svd_normal_6213(self, device, dtype):
+ # The gradient path does not touch the value: the normal is still the SVD's third right singular vector.
+ points = _board_6213(4, 6, device, dtype=torch.float32).to(dtype)
+ work = points.float() if dtype in (torch.float16, torch.bfloat16) else points
+ centered = work - work.mean(-2, keepdim=True)
+ expected = torch.linalg.svd(
+ centered.double() if centered.dtype == torch.float32 and device.type != "mps" else centered
+ )[2][-1]
+ normal = fit_plane(Vector3(points.clone().requires_grad_(True))).normal.data
+ assert (normal.detach().to(expected.dtype) * expected).sum().abs() > 1 - 1e-3
+ # A call that requires grad returns exactly what a call without autograd returns.
+ plane = fit_plane(Vector3(points))
+ plane_grad = fit_plane(Vector3(points.clone().requires_grad_(True)))
+ self.assert_close(plane_grad.normal.data.detach(), plane.normal.data, rtol=0, atol=0)
+ self.assert_close(plane_grad.offset.data.detach(), plane.offset.data, rtol=0, atol=0)
+
+ def test_float32_gradient_of_a_thin_strip_keeps_the_float64_accuracy_6213(self, device):
+ # X^T X squares the condition number of the centred points: the gradient is computed in the SVD's float64,
+ # where float32 would leave this 2 m x 1 mm strip's gradient 2e-1 off instead of 4e-5.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ from kornia.geometry.conversions import axis_angle_to_rotation_matrix
+
+ g = torch.Generator().manual_seed(0)
+ n = 200
+ strip = torch.stack(
+ [
+ torch.linspace(0.0, 2.0, n, dtype=torch.float64),
+ 1e-3 * torch.rand(n, generator=g, dtype=torch.float64),
+ 1e-6 * torch.randn(n, generator=g, dtype=torch.float64),
+ ],
+ -1,
+ )
+ rotation = axis_angle_to_rotation_matrix(torch.tensor([[0.4, 0.1, 0.9]], dtype=torch.float64))[0]
+ strip = strip @ rotation.T + torch.tensor([0.0, 1.0, 4.0], dtype=torch.float64)
+ points = strip.float().to(device) # the float64 reference below sees the same, float32-rounded, points
+ reference = fit_plane(Vector3(points.double())).normal.data.detach()
+ jac64 = torch.autograd.functional.jacobian(lambda p: _plane_with_fixed_sign_6213(p, reference), points.double())
+ jac32 = torch.autograd.functional.jacobian(lambda p: _plane_with_fixed_sign_6213(p, reference.float()), points)
+ self.assert_close(jac32.double(), jac64, rtol=0, atol=1e-3 * jac64.abs().max().item())
+
+ def test_dynamo_gradient_matches_eager_6213(self, device, dtype, torch_optimizer):
+ # The kept vector's own gap is excluded by position: CUDA Inductor evaluated (S / scale)^2 differently at its
+ # two reads, a gap of 1e-20 instead of 0, and the compiled gradient was 1e18 times too large.
+ g = torch.Generator().manual_seed(0)
+ points = torch.randn(2, 30, 3, generator=g, dtype=torch.float64) * torch.tensor([1.0, 0.5, 0.01]).double()
+ points = points.to(device, dtype)
+ weights = torch.randn(2, 4, generator=g, dtype=torch.float64).to(device, dtype)
+
+ def op(p):
+ plane = fit_plane(Vector3(p))
+ return torch.cat([plane.normal.data, plane.offset.data[..., None]], -1)
+
+ p_eager = points.clone().requires_grad_(True)
+ p_compiled = points.clone().requires_grad_(True)
+ (expected,) = torch.autograd.grad((op(p_eager) * weights).sum(), p_eager)
+ (actual,) = torch.autograd.grad((torch_optimizer(op)(p_compiled) * weights).sum(), p_compiled)
+ self.assert_close(actual, expected)
diff --git a/tests/geometry/test_quaternion.py b/tests/geometry/test_quaternion.py
index ce6982f2d..d232324d7 100644
--- a/tests/geometry/test_quaternion.py
+++ b/tests/geometry/test_quaternion.py
@@ -892,3 +892,63 @@ class TestQuaternionConventions(BaseTester):
assert out.shape == (1, 4)
# to_axis_angle is the principal log, so the arbitrary sign of the eigenvector does not matter
self.assert_close(out.to_axis_angle(), expected.to(device=device, dtype=dtype), rtol=0.0, atol=tol)
+
+
+def _average_with_fixed_sign_6213(data, reference, weights=None):
+ # The mean's sign is arbitrary; fix it against a reference for central differences.
+ mean = average_quaternions(Quaternion(data), weights).data
+ return mean * torch.where((mean.detach() * reference).sum(-1, keepdim=True) < 0, -1.0, 1.0)
+
+
+def _yaw_only_6213(device):
+ yaw = torch.tensor([0.1, -0.3, 0.5, 0.2], dtype=torch.float64)
+ zero = torch.zeros_like(yaw)
+ return torch.stack([torch.cos(yaw / 2), zero, zero, torch.sin(yaw / 2)], -1).to(device)
+
+
+class TestQuaternionAverageGradient(BaseTester):
+ """``average_quaternions``' gradient is finite when the three smaller eigenvalues are not distinct (#6213)."""
+
+ def test_identical_members_6213(self, device):
+ # Two identity quaternions: sum q q^T = diag(2, 0, 0, 0), and eigh's backward divided by the zero gaps.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ data = torch.tensor([[1.0, 0.0, 0.0, 0.0], [1.0, 0.0, 0.0, 0.0]], device=device, dtype=torch.float64)
+ reference = average_quaternions(Quaternion(data)).data.detach()
+ self.gradcheck(lambda q: _average_with_fixed_sign_6213(q, reference), (data,), fast_mode=False)
+
+ def test_rotations_about_one_axis_have_first_and_second_derivatives_6213(self, device):
+ # Members about a common coordinate axis span a plane, so two of the eigenvalues are zero.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ data = _yaw_only_6213(device)
+ reference = average_quaternions(Quaternion(data)).data.detach()
+ self.gradcheck(lambda q: _average_with_fixed_sign_6213(q, reference), (data,), fast_mode=False)
+ q = data.clone().requires_grad_(True)
+ assert torch.autograd.gradgradcheck(lambda q: _average_with_fixed_sign_6213(q, reference), (q,))
+
+ def test_weighted_rotations_about_one_axis_6213(self, device):
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ data = _yaw_only_6213(device)
+ weights = torch.tensor([0.5, 1.0, 2.0, 0.25], device=device, dtype=torch.float64)
+ reference = average_quaternions(Quaternion(data), weights).data.detach()
+ self.gradcheck(lambda q, w: _average_with_fixed_sign_6213(q, reference, w), (data, weights), fast_mode=False)
+
+ def test_forward_mode_under_no_grad_6213(self, device):
+ # torch.func.jvp runs under no_grad too, so a call whose input carries a tangent needs the gradient path even
+ # when grad mode is off; skipping it there made the tangent 0.
+ if device.type == "mps":
+ pytest.skip("MPS has no float64")
+ g = torch.Generator().manual_seed(0)
+ q = torch.randn(6, 4, generator=g, dtype=torch.float64).to(device)
+ t = torch.randn(6, 4, generator=g, dtype=torch.float64).to(device)
+
+ def average(x):
+ return average_quaternions(Quaternion(x)).data
+
+ _, expected = torch.func.jvp(average, (q,), (t,))
+ with torch.no_grad():
+ _, actual = torch.func.jvp(average, (q,), (t,))
+ assert expected.abs().max() > 1e-3
+ self.assert_close(actual, expected)