π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pytest
import torch
import kornia
import kornia.geometry.epipolar as epi
from kornia.geometry.conversions import convert_points_to_homogeneous
from kornia.geometry.epipolar.essential import _refine_essential_lm
from testing.base import BaseTester
from testing.geometry.create import generate_two_view_random_scene
from testing.two_view import calibrated_two_view_scene, two_view_scene
def test_generate_two_view_random_scene_is_deterministic():
torch.manual_seed(123)
expected_next = torch.rand(3)
torch.manual_seed(123)
scene1 = generate_two_view_random_scene()
actual_next = torch.rand(3)
for key in scene1:
assert torch.equal(scene1[key], generate_two_view_random_scene()[key]), f"Scene mismatch for {key}"
assert torch.equal(actual_next, expected_next)
class TestScaledPolynomialPowers(BaseTester):
def test_values(self, device, dtype):
# The five-point Newton correction evaluates z**d / max(1, abs(z))**10.
# Both signs, z=0, and the boundary |z|=1 must retain all eleven coefficients.
values = [-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 2.0]
roots = torch.tensor([values], device=device, dtype=dtype)
# Integer powers of these binary fractions are exact; CUDA's general tensor pow can round them.
expected = torch.tensor(
[[[z**d / max(1.0, abs(z)) ** 10 for d in range(11)] for z in values]], device=device, dtype=dtype
)
actual = epi.essential._scaled_polynomial_powers(roots)
self.assert_close(actual, expected, atol=0.0, rtol=0.0)
def test_large_roots_remain_finite(self, device, dtype):
# Direct z**10 would overflow. The highest two powers still have values 1 and 1/z,
# including the sign of the derivative term at a negative root.
large = torch.finfo(dtype).max ** 0.2 / 2
roots = torch.tensor([[-large, large]], device=device, dtype=dtype)
actual = epi.essential._scaled_polynomial_powers(roots)
assert torch.isfinite(actual).all()
assert bool((actual.abs() <= 1).all())
self.assert_close(actual[..., -1], torch.ones_like(roots), atol=0.0, rtol=0.0)
self.assert_close(actual[..., -2], roots.reciprocal(), atol=0.0, rtol=0.0)
class TestFindEssential(BaseTester):
def test_least_squares_basis_backward_storage_scales_with_points(self, device):
dtype = torch.float32 if device.type == "mps" else torch.float64
design = torch.rand(1, 128, 9, device=device, dtype=dtype, requires_grad=True)
saved_sizes = []
def pack(tensor):
saved_sizes.append(tensor.numel())
return tensor
with torch.autograd.graph.saved_tensors_hooks(pack, lambda tensor: tensor):
basis = epi.essential._least_squares_basis(design)
basis.square().sum().backward()
assert torch.isfinite(design.grad).all()
assert max(saved_sizes) <= 4 * design.numel()
def test_smoke(self, device, dtype):
points1 = torch.rand(1, 5, 2, device=device, dtype=dtype)
points2 = torch.rand(1, 5, 2, device=device, dtype=dtype)
weights = torch.ones(1, 5, device=device, dtype=dtype)
E_mat = epi.essential.find_essential(points1, points2, weights)
assert E_mat.shape == (1, 10, 3, 3)
@pytest.mark.parametrize("batch_size, num_points", [(1, 5), (2, 6), (3, 7), (16, 5)])
def test_shape(self, batch_size, num_points, device, dtype):
B, N = batch_size, num_points
points1 = torch.rand(B, N, 2, device=device, dtype=dtype)
points2 = torch.rand(B, N, 2, device=device, dtype=dtype)
weights = torch.ones(B, N, device=device, dtype=dtype)
E_mat = epi.essential.find_essential(points1, points2, weights)
assert E_mat.shape == (B, 10, 3, 3)
@pytest.mark.parametrize("batch_size, num_points", [(1, 5), (2, 6), (3, 7)])
def test_shape_noweights(self, batch_size, num_points, device, dtype):
B, N = batch_size, num_points
points1 = torch.rand(B, N, 2, device=device, dtype=dtype)
points2 = torch.rand(B, N, 2, device=device, dtype=dtype)
weights = None
E_mat = epi.essential.find_essential(points1, points2, weights)
assert E_mat.shape == (B, 10, 3, 3)
@staticmethod
def _true_e_error(candidates, E):
"""Smallest distance, up to sign, of the finite candidates ``(10, 3, 3)`` from ``E``; inf if there are none."""
candidates = candidates.cpu().double()
finite = candidates[torch.isfinite(candidates).flatten(1).all(1)]
if len(finite) == 0:
return float("inf")
E = E.to(finite)
return float(torch.minimum((finite - E).flatten(1).norm(dim=1), (finite + E).flatten(1).norm(dim=1)).min())
@pytest.mark.parametrize("input_dtype", [torch.float32, torch.float64])
def test_exact_five_point_sample_recovers_the_true_e_4884(self, device, input_dtype):
if device.type == "mps" and input_dtype == torch.float64:
pytest.skip("MPS does not support float64")
# #4884: in float32 an exact five-point sample missed the true E; the solve now runs in float64, on the host
# for MPS inputs.
misses = 0
for seed in range(20):
scene = calibrated_two_view_scene(5, 0.0, seed)
points1, points2 = scene["x1"][None].to(device, input_dtype), scene["x2"][None].to(device, input_dtype)
misses += self._true_e_error(epi.find_essential(points1, points2)[0], scene["E"]) > 1e-4
assert misses == 0
@pytest.mark.parametrize("input_dtype", [torch.float32, torch.float64])
def test_no_parallax_backward_is_finite_4903_4952(self, device, input_dtype):
if device.type == "mps" and input_dtype == torch.float64:
pytest.skip("MPS does not support float64")
# With p1 == p2 the essential matrix is not determined. About one such sample in 300 has a multiple root at
# z = 0, where the backward of torch.linalg.eigvals raised for the whole batch (#4903), and in float32 about
# one in ten gave an all-NaN gradient (#4952). The roots now come from a detached solve, differentiated by a
# Newton step that is skipped at a multiple root.
generator = torch.Generator().manual_seed(0)
points = torch.rand(1000, 5, 2, generator=generator, dtype=torch.float64).to(device, input_dtype)
points.requires_grad_()
epi.find_essential(points, points).nan_to_num().sum().backward()
assert torch.isfinite(points.grad).all()
def test_no_parallax_backward_is_finite_with_a_float32_solve(self, monkeypatch):
# In a float32 solve, as MPS ran before its samples moved to the host, a spurious root near a vanishing
# leading coefficient (|z| ~ 5e8) overflows z ** 10, and a backward through the masked powers gave
# 0 * inf = NaN.
monkeypatch.setattr(epi.essential, "_solve_dtype", lambda device: torch.float32)
generator = torch.Generator().manual_seed(0)
points = torch.rand(1000, 5, 2, generator=generator).requires_grad_()
epi.find_essential(points, points).nan_to_num().sum().backward()
assert torch.isfinite(points.grad).all()
def test_true_e_is_recovered_on_random_motions(self, device):
if device.type == "mps":
pytest.skip("MPS does not support float64")
# 2000 exact minimal samples of random scenes, rotations and translation directions. The previous solver
# missed E by more than 1e-3 on two of them, this one on none (worst about 1e-4, depending on the BLAS); with
# the LU null-space basis in place of the Householder one it missed seven.
generator = torch.Generator().manual_seed(0)
f64 = torch.float64
X = torch.randn(2000, 5, 3, generator=generator, dtype=f64) * torch.tensor([1.0, 1.0, 0.5], dtype=f64)
X = X + torch.tensor([0.0, 0.0, 4.0], dtype=f64)
R = kornia.geometry.conversions.axis_angle_to_rotation_matrix(
(torch.rand(2000, 3, generator=generator, dtype=f64) - 0.5) * 0.6
)
t = torch.nn.functional.normalize(torch.randn(2000, 3, generator=generator, dtype=f64), dim=-1)
Y = X @ R.mT + t[:, None]
x1, x2 = X[..., :2] / X[..., 2:], Y[..., :2] / Y[..., 2:]
E = epi.essential_from_Rt(
torch.eye(3, dtype=f64).expand(2000, 3, 3), torch.zeros(2000, 3, 1, dtype=f64), R, t[..., None]
)
E = E / E.flatten(1).norm(dim=1)[:, None, None]
candidates = epi.find_essential(x1.to(device), x2.to(device)).cpu()
distance = torch.minimum(
(candidates - E[:, None]).flatten(2).norm(dim=-1), (candidates + E[:, None]).flatten(2).norm(dim=-1)
)
best = distance.nan_to_num(10.0).amin(1)
assert best.max() < 1e-3
assert torch.quantile(best, 0.99) < 1e-6
def test_degenerate_sample_does_not_affect_the_batch(self, device, dtype):
_skip_find_essential(device, dtype)
scene = calibrated_two_view_scene(5, 0.0, 8)
degenerate1, degenerate2 = scene["x1"].clone(), scene["x2"].clone()
degenerate1[1], degenerate2[1] = degenerate1[0], degenerate2[0] # a repeated correspondence
points1 = torch.stack([scene["x1"], degenerate1]).to(device, dtype)
points2 = torch.stack([scene["x2"], degenerate2]).to(device, dtype)
candidates = epi.find_essential(points1, points2)
assert candidates.shape == (2, 10, 3, 3)
assert candidates.device == points1.device
assert self._true_e_error(candidates[0], scene["E"]) < 1e-3
alone = epi.find_essential(points1[:1], points2[:1])[0]
assert torch.equal(torch.isfinite(candidates[0]), torch.isfinite(alone))
self.assert_close(candidates[0].nan_to_num(7.0), alone.nan_to_num(7.0), atol=1e-6, rtol=0)
@pytest.mark.parametrize("case", ["repeat-1-2", "repeat-2-4", "repeat-3-1", "repeat-0-3", "collinear"])
def test_rank_deficient_sample_has_no_candidates(self, device, case):
if device.type == "mps":
pytest.skip("MPS does not support float64")
# Five constraints of rank four leave a null space of five dimensions: E is not determined. Rounding usually
# leaves a tiny pivot rather than an exact zero, which must not pass for a regular sample.
if case == "collinear": # collinear in both images
t = torch.tensor([-0.4, -0.1, 0.05, 0.2, 0.45], dtype=torch.float64)
points1 = torch.stack([t, 0.3 * t + 0.1], -1)
points2 = torch.stack([t + 0.05, -0.5 * t + 0.2], -1)
else:
_, i, j = case.split("-")
scene = calibrated_two_view_scene(5, 0.0, 9)
points1, points2 = scene["x1"].clone(), scene["x2"].clone()
points1[int(j)], points2[int(j)] = points1[int(i)], points2[int(i)]
points1 = points1[None].to(device).requires_grad_()
candidates = epi.find_essential(points1, points2[None].to(device))
assert torch.isnan(candidates).all()
candidates.nan_to_num().sum().backward()
assert (points1.grad == 0).all()
@pytest.mark.parametrize("num_points", [5, 7])
def test_mps_input_is_solved_on_the_host(self, device, num_points):
if device.type != "mps":
pytest.skip("compares an MPS input with the same input on the CPU")
# MPS has no float64: its samples are solved on the host in float64 and the candidates copied back, so they
# match a CPU solve of the same float32 input and still carry a gradient to the MPS input.
scene = calibrated_two_view_scene(num_points, 0.0, 8)
x1, x2 = scene["x1"][None].float(), scene["x2"][None].float()
on_cpu = epi.find_essential(x1, x2)
points1 = x1.to(device).requires_grad_()
on_mps = epi.find_essential(points1, x2.to(device))
assert on_mps.device == points1.device
assert on_mps.dtype == torch.float32
assert torch.equal(torch.isfinite(on_mps).cpu(), torch.isfinite(on_cpu))
self.assert_close(on_mps.cpu().nan_to_num(7.0), on_cpu.nan_to_num(7.0), atol=1e-6, rtol=0)
assert self._true_e_error(on_mps[0].detach(), scene["E"]) < 1e-4
on_mps.nan_to_num().sum().backward()
assert points1.grad.device == points1.device
assert torch.isfinite(points1.grad).all()
assert bool((points1.grad != 0).any())
def test_repeated_calls_are_bitwise_equal(self, device, dtype):
# No atomic accumulation: CUDA's scatter_add_ summed the determinant polynomial in a varying order.
generator = torch.Generator().manual_seed(0)
points1 = torch.randn(512, 5, 2, generator=generator, dtype=torch.float64).to(device, dtype)
points2 = torch.randn(512, 5, 2, generator=generator, dtype=torch.float64).to(device, dtype)
first, second = epi.find_essential(points1, points2), epi.find_essential(points1, points2)
assert torch.equal(first.nan_to_num(7.0), second.nan_to_num(7.0))
def test_epipolar_constraint(self, device, dtype):
calibrated_x1 = torch.tensor(
[[[0.0640, 0.7799], [-0.2011, 0.2836], [-0.1355, 0.2907], [0.0520, 1.0086], [-0.0361, 0.6533]]],
device=device,
dtype=dtype,
)
calibrated_x2 = torch.tensor(
[[[0.3470, -0.4274], [-0.1818, -0.1281], [-0.1766, -0.1617], [0.4066, -0.0706], [0.1137, 0.0363]]],
device=device,
dtype=dtype,
)
E = epi.essential.find_essential(calibrated_x1, calibrated_x2)
if torch.all(E != 0):
distance = epi.symmetrical_epipolar_distance(calibrated_x1, calibrated_x2, E)
distance = torch.nan_to_num(distance, nan=1e8)
# Note : here we check only the best model, although all solutions are returned
mean_error = distance.mean(-1).min()
self.assert_close(mean_error, torch.tensor(0.0, device=device, dtype=dtype), atol=1e-4, rtol=1e-4)
def test_synthetic_sampson(self, device, dtype, monkeypatch):
calibrated_x1 = torch.tensor(
[[[0.0640, 0.7799], [-0.2011, 0.2836], [-0.1355, 0.2907], [0.0520, 1.0086], [-0.0361, 0.6533]]],
device=device,
dtype=dtype,
)
calibrated_x2 = torch.tensor(
[[[0.3470, -0.4274], [-0.1818, -0.1281], [-0.1766, -0.1617], [0.4066, -0.0706], [0.1137, 0.0363]]],
device=device,
dtype=dtype,
)
weights = torch.ones_like(calibrated_x2)[..., 0]
E_est = epi.essential.find_essential(calibrated_x1, calibrated_x2, weights)
error = epi.sampson_epipolar_distance(calibrated_x1, calibrated_x2, E_est)
error = torch.nan_to_num(error, nan=1e8)
self.assert_close(
error[:, torch.argmin(error.mean(-1))],
torch.zeros((calibrated_x1.shape[:2]), device=device, dtype=dtype),
atol=1e-4,
rtol=1e-4,
)
# Noise-free minimal samples of random scenes: the pose is recovered for nearly all of them. About
# half have a negative leading coefficient in the degree-10 polynomial, which a floor on that
# coefficient such as clamp_min turns into the wrong roots (#4847).
g = torch.Generator().manual_seed(0)
n = 64
points = torch.rand(n, 5, 3, generator=g, dtype=torch.float64) * torch.tensor(
[2.0, 2.0, 4.0], dtype=torch.float64
)
points = points + torch.tensor([-1.0, -1.0, 3.0], dtype=torch.float64)
axis_angle = (torch.rand(n, 3, generator=g, dtype=torch.float64) - 0.5) * 0.4
trans = torch.nn.functional.normalize(torch.rand(n, 3, generator=g, dtype=torch.float64) - 0.5, dim=-1)
points, axis_angle, trans = (v.to(device=device, dtype=dtype) for v in (points, axis_angle, trans))
R = kornia.geometry.conversions.axis_angle_to_rotation_matrix(axis_angle)
points2 = points @ R.transpose(-1, -2) + trans[:, None]
x1, x2 = points[..., :2] / points[..., 2:], points2[..., :2] / points2[..., 2:]
E_gt = epi.essential_from_Rt(
torch.eye(3, device=device, dtype=dtype).expand(n, 3, 3),
torch.zeros(n, 3, 1, device=device, dtype=dtype),
R,
trans[..., None],
)
weights = torch.ones(n, 5, device=device, dtype=dtype)
E_est = epi.essential.find_essential(x1, x2, weights)
def unit(M):
return M / M.flatten(-2).norm(dim=-1)[..., None, None]
err = torch.minimum(
(unit(E_est) - unit(E_gt)[:, None]).flatten(-2).norm(dim=-1),
(unit(E_est) + unit(E_gt)[:, None]).flatten(-2).norm(dim=-1),
)
best = torch.nan_to_num(err, nan=1.0).min(dim=-1).values
tol, at_least = (1e-8, 60) if dtype == torch.float64 else (1e-3, 48)
assert int((best < tol).sum()) >= at_least
# The roots of a polynomial do not depend on its sign or scale, and IEEE division makes that exact
# for a power-of-two scale, so negating every coefficient, or scaling them all by 2**-40, must leave
# the candidates unchanged bit for bit. This holds on every platform, whichever sign the SVD's
# null-space basis gives a sample. The scale takes every leading coefficient here below 1e-8, so it
# also pins that only an exactly zero one is replaced, not a small one.
determinant = epi.essential._determinant_to_polynomial_jit
for factor in (-1.0, 2.0**-40):
monkeypatch.setattr(
epi.essential, "_determinant_to_polynomial_jit", lambda A, *args, f=factor: f * determinant(A, *args)
)
E_scaled = epi.essential.find_essential(x1, x2, weights)
assert torch.equal(torch.isnan(E_scaled), torch.isnan(E_est))
self.assert_close(torch.nan_to_num(E_scaled), torch.nan_to_num(E_est), atol=0.0, rtol=0.0)
@pytest.mark.parametrize("num_points", [5, 6, 8])
def test_gradcheck(self, num_points, device):
# For fewer than 9 points some or all of the four null-space vectors lie past min(N, 9), where
# torch.linalg.svd gives no gradient, so the gradient to the correspondences was dropped: exactly
# zero for 5 points (#4855). Candidates from complex roots are NaN and are zeroed here; they stay
# complex under the small perturbations the check makes.
g = torch.Generator().manual_seed(1)
points1 = torch.rand(1, num_points, 2, generator=g, dtype=torch.float64).to(device)
points2 = torch.rand(1, num_points, 2, generator=g, dtype=torch.float64).to(device)
def proxy(points1, points2):
return epi.essential.find_essential(points1, points2).nan_to_num()
self.gradcheck(proxy, (points1, points2))
def test_null_space_gradient_of_a_discarded_sample(self, device):
# Points all at the origin give a rank-1 design matrix, so the null space the solver uses is not
# unique and its derivative has no gap to divide by. A sample like that has its candidates
# discarded, so no gradient reaches its basis, and it must contribute zero rather than 0 / 0.
if device.type == "mps":
pytest.skip("MPS does not support float64")
design = torch.zeros(1, 5, 9, device=device, dtype=torch.float64)
design[..., 8] = 1.0
design.requires_grad_()
basis = epi.essential._NullSpaceBasis.apply(design)[0]
(basis * 0.0).sum().backward()
assert torch.isfinite(design.grad).all()
assert (design.grad == 0).all()
# A nonzero incoming gradient has no usable derivative there, as for torch.linalg.svd. It must not
# come out as a silent zero, which is what the dropped gradient of #4855 looked like. How it shows
# depends on the SVD: LAPACK returns the four zero singular values as exact zeros, so the backward
# divides by a zero gap and the gradient is not finite, while cuSOLVER's Jacobi SVD, torch's default
# on CUDA, leaves them at rounding level (1.6e-16 down to 1.5e-61) and the gradient is finite but
# near 6e60, as torch.linalg.svd's own backward is on the same input. Past 1 / eps, a gradient can
# only come from a gap at rounding level.
design.grad = None
epi.essential._NullSpaceBasis.apply(design)[0].sum().backward()
usable = torch.isfinite(design.grad) & (design.grad.abs() < 1 / torch.finfo(design.dtype).eps)
assert not usable.all()
def test_torch_func_grad(self, device):
# torch.func transforms accept an autograd.Function only if it defines setup_context. With 5
# points every null-space vector goes through it, so torch.func.grad must agree with autograd.
if device.type == "mps":
pytest.skip("MPS does not support float64")
g = torch.Generator().manual_seed(0)
points1 = torch.rand(1, 5, 2, generator=g, dtype=torch.float64).to(device)
points2 = torch.rand(1, 5, 2, generator=g, dtype=torch.float64).to(device)
def loss(points1):
return (epi.essential.find_essential(points1, points2).nan_to_num() ** 3).sum()
leaf = points1.clone().requires_grad_()
(expected,) = torch.autograd.grad(loss(leaf), leaf)
assert expected.abs().max() > 0
self.assert_close(torch.func.grad(loss)(points1), expected)
@pytest.mark.parametrize("batch_size, num_points", [(5, 5), (10, 5)])
def test_degenerate_case(self, batch_size, num_points, device, dtype, monkeypatch):
B, N = batch_size, num_points
# Points all at the origin give a design matrix whose SVD returns unit vectors for its null
# space, and every basis of four unit vectors makes the 10x10 elimination matrix exactly
# singular. A singular sample has no solution, so all 10 of its slots are NaN (#4883).
zeros = torch.zeros(B, N, 2, device=device, dtype=dtype)
E_zeros = epi.essential.find_essential(zeros, zeros, torch.ones(B, N, device=device, dtype=dtype))
assert E_zeros.shape == (B, 10, 3, 3)
assert torch.isnan(E_zeros).all()
# Its degree-10 polynomial also has an exactly zero leading coefficient. The companion matrix
# built from it had a repeated eigenvalue and a singular eigenvector matrix, so the backward
# raised in torch.linalg.eigvals (#4831). It now completes, and since the sample's candidates are
# discarded its gradient is exactly zero, also through the SVD of its rank-1 design matrix (#4855).
origin = torch.zeros(B, N, 2, device=device, dtype=dtype, requires_grad=True)
weights_bn = torch.ones(B, N, device=device, dtype=dtype)
epi.essential.find_essential(origin, origin, weights_bn).nan_to_num().sum().backward()
assert (origin.grad == 0).all()
# A singular element does not disturb the rest of its batch: next to one, a regular sample
# returns exactly what it returns next to a regular sample, NaN candidates from complex roots
# included. The reference is a batch of the same size, because the batch size alone can change
# the last bits of a result (by 4.8e-15 at float64 on macOS arm64).
x1 = torch.tensor(
[[0.0640, 0.7799], [-0.2011, 0.2836], [-0.1355, 0.2907], [0.0520, 1.0086], [-0.0361, 0.6533]],
device=device,
dtype=dtype,
)
x2 = torch.tensor(
[[0.3470, -0.4274], [-0.1818, -0.1281], [-0.1766, -0.1617], [0.4066, -0.0706], [0.1137, 0.0363]],
device=device,
dtype=dtype,
)
weights = torch.ones(2, 5, device=device, dtype=dtype)
regular = epi.essential.find_essential(torch.stack((x1, x1)), torch.stack((x2, x2)), weights)[1]
mixed = epi.essential.find_essential(
torch.stack((zeros[0, :5], x1)),
torch.stack((zeros[0, :5], x2)),
weights,
)
assert torch.isnan(mixed[0]).all()
assert torch.equal(torch.isnan(mixed[1]), torch.isnan(regular))
self.assert_close(torch.nan_to_num(mixed[1]), torch.nan_to_num(regular), atol=0.0, rtol=0.0)
# Whether any other degenerate set is exactly singular, or ill-conditioned enough to make the
# companion matrix non-finite, depends on the platform's LAPACK. These sets and a random draw
# with the same points in both images exercise those paths where they occur, and must return the
# documented shape either way. The last two give a non-finite companion matrix at float32 on
# Linux x86 and on macOS arm64 respectively.
lshape = torch.tensor([[1.0, 0.0], [0.5, 0.0], [0.0, 1.0], [0.0, 0.0], [0.0, 0.5]], device=device, dtype=dtype)
draw = torch.rand(B, N, 2, generator=torch.Generator().manual_seed(79)).to(device=device, dtype=dtype)
on_axis = torch.tensor([[0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [0.0, 0.0], [1.0, 0.0]], device=device, dtype=dtype)
repeated = torch.tensor(
[[2.0, 0.0], [2.0, 2.0], [2.0, 2.0], [2.0, 2.0], [2.0, 1.0]], device=device, dtype=dtype
)
for points in (lshape.expand(B, 5, 2), draw, on_axis.expand(B, 5, 2), repeated.expand(B, 5, 2)):
weights = torch.ones(points.shape[:2], device=device, dtype=dtype)
assert epi.essential.find_essential(points, points, weights).shape == (B, 10, 3, 3)
# For a design matrix whose rows are unit vectors, the SVD returns a null space of unit vectors,
# which makes the elimination matrix exactly singular. For this one, solving against the identity
# leaves finite candidates that fail the essential-matrix constraints, so they must be dropped.
design = torch.eye(9, device=device, dtype=dtype)[[0, 1, 3, 5, 6]].expand(B, 5, 9)
assert torch.isnan(epi.essential.null_to_Nister_solution(design, B)).all()
# A companion matrix that is not finite has no roots, and torch.linalg.eigvals aborts on one, which
# used to take the whole batch down. Which inputs produce it is platform-dependent, so make the
# determinant polynomial of one element non-finite instead: that element returns NaN slots, and the
# other returns exactly what it returns in the unpatched batch above.
determinant = epi.essential._determinant_to_polynomial_jit
def overflowed(A, *args):
cs = determinant(A, *args).clone()
cs[0] = float("nan")
return cs
monkeypatch.setattr(epi.essential, "_determinant_to_polynomial_jit", overflowed)
patched = epi.essential.find_essential(torch.stack((x1, x1)), torch.stack((x2, x2)), weights)
assert torch.isnan(patched[0]).all()
assert torch.equal(torch.isnan(patched[1]), torch.isnan(regular))
self.assert_close(torch.nan_to_num(patched[1]), torch.nan_to_num(regular), atol=0.0, rtol=0.0)
# A zero leading coefficient means the polynomial has no usable companion matrix either (#4831).
# Which inputs give one exactly is platform-dependent, so zero it for one element: that element
# returns NaN slots, the other returns exactly what it returns unpatched, and the
# backward through both completes with a finite gradient.
def degree_deficient(A, *args):
cs = determinant(A, *args).clone()
cs[0, -1] = 0.0
return cs
monkeypatch.setattr(epi.essential, "_determinant_to_polynomial_jit", degree_deficient)
points1 = torch.stack((x1, x1)).requires_grad_()
patched = epi.essential.find_essential(points1, torch.stack((x2, x2)), weights)
assert torch.isnan(patched[0]).all()
assert torch.equal(torch.isnan(patched[1]), torch.isnan(regular))
self.assert_close(torch.nan_to_num(patched[1].detach()), torch.nan_to_num(regular), atol=0.0, rtol=0.0)
patched.nan_to_num().sum().backward()
assert torch.isfinite(points1.grad).all()
# Check the backward through a zeroed element once more with 9 points, where the null space comes
# from torch.linalg.svd itself instead of the N < 9 path.
g9 = torch.Generator().manual_seed(1)
points9 = torch.rand(2, 9, 2, generator=g9, dtype=torch.float64).to(device=device, dtype=dtype)
points9.requires_grad_()
other9 = torch.rand(2, 9, 2, generator=g9, dtype=torch.float64).to(device=device, dtype=dtype)
E9 = epi.essential.find_essential(points9, other9, torch.ones(2, 9, device=device, dtype=dtype))
assert torch.isnan(E9[0]).all()
E9.nan_to_num().sum().backward()
assert torch.isfinite(points9.grad).all()
class TestEssentialFromFundamental(BaseTester):
def test_smoke(self, device, dtype):
F_mat = torch.rand(1, 3, 3, device=device, dtype=dtype)
K1 = torch.rand(1, 3, 3, device=device, dtype=dtype)
K2 = torch.rand(1, 3, 3, device=device, dtype=dtype)
E_mat = epi.essential_from_fundamental(F_mat, K1, K2)
assert E_mat.shape == (1, 3, 3)
@pytest.mark.parametrize("batch_size", [1, 2, 4, 7])
def test_shape(self, batch_size, device, dtype):
B: int = batch_size
F_mat = torch.rand(B, 3, 3, device=device, dtype=dtype)
K1 = torch.rand(B, 3, 3, device=device, dtype=dtype)
K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check broadcasting
E_mat = epi.essential_from_fundamental(F_mat, K1, K2)
assert E_mat.shape == (B, 3, 3)
def test_from_to_fundamental(self, device, dtype):
# Seeded F and fixed, well-conditioned intrinsics: the round trip F -> K2^T F K1 -> K2^-T E K1^-1 goes
# through the inverses of K1 and K2, so near-singular random intrinsics would miss the tolerance. K1 != K2
# and neither is symmetric, so swapping the cameras or dropping a transpose fails in every dtype.
generator = torch.Generator().manual_seed(0)
F_mat = torch.rand(1, 3, 3, generator=generator, dtype=torch.float64).to(device, dtype)
K1 = torch.tensor([[[1.2, 0.0, 0.1], [0.0, 1.1, -0.2], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
K2 = torch.tensor([[[0.9, 0.0, -0.15], [0.0, 1.3, 0.05], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
E_mat = epi.essential_from_fundamental(F_mat, K1, K2)
F_hat = epi.fundamental_from_essential(E_mat, K1, K2)
self.assert_close(F_mat, F_hat)
def test_shape_large(self, device, dtype):
F_mat = torch.rand(1, 2, 3, 3, device=device, dtype=dtype)
K1 = torch.rand(1, 2, 3, 3, device=device, dtype=dtype)
K2 = torch.rand(1, 1, 3, 3, device=device, dtype=dtype) # check broadcasting
E_mat = epi.essential_from_fundamental(F_mat, K1, K2)
assert E_mat.shape == (1, 2, 3, 3)
def test_from_fundamental(self, device, dtype):
scene = generate_two_view_random_scene(device, dtype)
F_mat = scene["F"]
K1 = scene["K1"]
K2 = scene["K2"]
E_mat = epi.essential_from_fundamental(F_mat, K1, K2)
F_hat = epi.fundamental_from_essential(E_mat, K1, K2)
F_mat_norm = epi.normalize_transformation(F_mat)
F_hat_norm = epi.normalize_transformation(F_hat)
self.assert_close(F_mat_norm, F_hat_norm)
def test_gradcheck(self, device):
F_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True)
K1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64)
K2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64)
self.gradcheck(epi.essential_from_fundamental, (F_mat, K1, K2), requires_grad=(True, False, False))
class TestRelativeCameraMotion(BaseTester):
def test_smoke(self, device, dtype):
R1 = torch.rand(1, 3, 3, device=device, dtype=dtype)
t1 = torch.rand(1, 3, 1, device=device, dtype=dtype)
R2 = torch.rand(1, 3, 3, device=device, dtype=dtype)
t2 = torch.rand(1, 3, 1, device=device, dtype=dtype)
R, t = epi.relative_camera_motion(R1, t1, R2, t2)
assert R.shape == (1, 3, 3)
assert t.shape == (1, 3, 1)
@pytest.mark.parametrize("batch_size", [1, 3, 5, 8])
def test_shape(self, batch_size, device, dtype):
B: int = batch_size
R1 = torch.rand(B, 3, 3, device=device, dtype=dtype)
t1 = torch.rand(B, 3, 1, device=device, dtype=dtype)
R2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check broadcasting
t2 = torch.rand(B, 3, 1, device=device, dtype=dtype)
R, t = epi.relative_camera_motion(R1, t1, R2, t2)
assert R.shape == (B, 3, 3)
assert t.shape == (B, 3, 1)
def test_translation(self, device, dtype):
R1 = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
t1 = torch.tensor([[[10.0], [0.0], [0.0]]]).type_as(R1)
R2 = kornia.core.ops.eye_like(3, R1)
t2 = kornia.core.ops.vec_like(3, t1)
R_expected = R1.clone()
t_expected = -t1
R, t = epi.relative_camera_motion(R1, t1, R2, t2)
self.assert_close(R_expected, R)
self.assert_close(t_expected, t)
def test_rotate_z(self, device, dtype):
R1 = torch.tensor([[[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
R2 = torch.tensor([[[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [0.0, 0.0, 1.0]]], device=device, dtype=dtype)
t1 = kornia.core.ops.vec_like(3, R1)
t2 = kornia.core.ops.vec_like(3, R2)
R_expected = R2.clone()
t_expected = t1
R, t = epi.relative_camera_motion(R1, t1, R2, t2)
self.assert_close(R_expected, R)
self.assert_close(t_expected, t)
def test_gradcheck(self, device):
R1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True)
R2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64)
t1 = torch.rand(1, 3, 1, device=device, dtype=torch.float64)
t2 = torch.rand(1, 3, 1, device=device, dtype=torch.float64)
self.gradcheck(epi.relative_camera_motion, (R1, t1, R2, t2), requires_grad=(True, False, False, False))
class TestEssentalFromRt(BaseTester):
def test_smoke(self, device, dtype):
R1 = torch.rand(1, 3, 3, device=device, dtype=dtype)
t1 = torch.rand(1, 3, 1, device=device, dtype=dtype)
R2 = torch.rand(1, 3, 3, device=device, dtype=dtype)
t2 = torch.rand(1, 3, 1, device=device, dtype=dtype)
E_mat = epi.essential_from_Rt(R1, t1, R2, t2)
assert E_mat.shape == (1, 3, 3)
@pytest.mark.parametrize("batch_size", [1, 3, 5, 8])
def test_shape(self, batch_size, device, dtype):
B: int = batch_size
R1 = torch.rand(B, 3, 3, device=device, dtype=dtype)
t1 = torch.rand(B, 3, 1, device=device, dtype=dtype)
R2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check broadcasting
t2 = torch.rand(B, 3, 1, device=device, dtype=dtype)
E_mat = epi.essential_from_Rt(R1, t1, R2, t2)
assert E_mat.shape == (B, 3, 3)
def test_from_fundamental_Rt(self, device, dtype):
scene = generate_two_view_random_scene(device, dtype)
E_from_Rt = epi.essential_from_Rt(scene["R1"], scene["t1"], scene["R2"], scene["t2"])
E_from_F = epi.essential_from_fundamental(scene["F"], scene["K1"], scene["K2"])
E_from_Rt_norm = epi.normalize_transformation(E_from_Rt)
E_from_F_norm = epi.normalize_transformation(E_from_F)
# The scene is seeded, so this is deterministic. The pixel-unit scene F has entries down to 1e-5 that the
# half dtypes keep with few mantissa bits, and K2^T F K1 with K ~ 30..80 amplifies that: the normalised E
# differ by 1.4e-2 in float16 and 3.1e-2 in bfloat16, against 3.5 for a transposed F or swapped cameras.
tol = {torch.float16: 1e-2, torch.bfloat16: 5e-2}.get(dtype, 1e-3)
self.assert_close(E_from_Rt_norm, E_from_F_norm, rtol=tol, atol=tol)
def test_gradcheck(self, device):
R1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True)
R2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64)
t1 = torch.rand(1, 3, 1, device=device, dtype=torch.float64)
t2 = torch.rand(1, 3, 1, device=device, dtype=torch.float64)
self.gradcheck(epi.essential_from_Rt, (R1, t1, R2, t2), requires_grad=(True, False, False, False))
class TestDecomposeEssentialMatrix(BaseTester):
def test_smoke(self, device, dtype):
E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype)
R1, R2, t = epi.decompose_essential_matrix(E_mat)
assert R1.shape == (1, 3, 3)
assert R2.shape == (1, 3, 3)
assert t.shape == (1, 3, 1)
@pytest.mark.parametrize("batch_shape", [(1, 3, 3), (2, 3, 3), (2, 1, 3, 3), (3, 2, 1, 3, 3)])
def test_shape(self, batch_shape, device, dtype):
E_mat = torch.rand(batch_shape, device=device, dtype=dtype)
R1, R2, t = epi.decompose_essential_matrix(E_mat)
assert R1.shape == batch_shape
assert R2.shape == batch_shape
assert t.shape == batch_shape[:-1] + (1,)
def test_gradcheck(self, device):
E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True)
def eval_rot1(input):
return epi.decompose_essential_matrix(input)[0]
def eval_rot2(input):
return epi.decompose_essential_matrix(input)[1]
def eval_vec(input):
return epi.decompose_essential_matrix(input)[2]
self.gradcheck(eval_rot1, (E_mat,))
self.gradcheck(eval_rot2, (E_mat,))
self.gradcheck(eval_vec, (E_mat,))
class TestDecomposeEssentialMatrixNoSVD(BaseTester):
def test_smoke(self, device, dtype):
E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype)
R1, R2, t = epi.decompose_essential_matrix_no_svd(E_mat)
assert R1.shape == (1, 3, 3)
assert R2.shape == (1, 3, 3)
assert t.shape == (1, 3, 1)
@pytest.mark.parametrize("batch_shape", [(3, 3), (1, 3, 3), (2, 3, 3), (2, 1, 3, 3), (3, 2, 1, 3, 3)])
def test_shape(self, batch_shape, device, dtype):
E_mat = torch.rand(batch_shape, device=device, dtype=dtype)
R1, R2, t = epi.decompose_essential_matrix_no_svd(E_mat)
if len(batch_shape) >= 2:
batch_shape = E_mat.view(-1, 3, 3).shape
assert R1.shape == batch_shape
assert R2.shape == batch_shape
assert t.shape == batch_shape[:-1] + (1,)
def test_gradcheck(self, device):
"""Check both rotation candidates and the translation against finite differences."""
E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True)
def eval_rot1(input):
"""Return the first rotation candidate for its gradient check."""
return epi.decompose_essential_matrix_no_svd(input)[0]
def eval_rot2(input):
"""Return the second rotation candidate for its gradient check."""
return epi.decompose_essential_matrix_no_svd(input)[1]
def eval_vec(input):
"""Return the unit translation candidate for its gradient check."""
return epi.decompose_essential_matrix_no_svd(input)[2]
self.gradcheck(eval_rot1, (E_mat,))
self.gradcheck(eval_rot2, (E_mat,))
self.gradcheck(eval_vec, (E_mat,))
@pytest.mark.parametrize("axis", [0, 1, 2])
@pytest.mark.parametrize("angle", [0.0, 0.3])
def test_axis_aligned_pose_gradients_5867(self, axis, angle, device, dtype):
"""Compare sparse-pose outputs and all angular derivatives with Rodrigues' formula."""
# E(theta) = [t]_x R_t(theta), with t a coordinate axis. One cross product is nonzero;
# the other two vanish. Rodrigues gives R, dR/dtheta, and the second candidate R_pi R.
generators = torch.tensor(
[
[[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]],
[[0.0, 0.0, 1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, 0.0]],
[[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
],
device=device,
dtype=dtype,
)
generator = generators[axis]
squared = generator @ generator
identity = torch.eye(3, device=device, dtype=dtype)
translation = identity[axis]
half_turn = 2 * translation[:, None] * translation[None, :] - identity
theta = torch.tensor(angle, device=device, dtype=dtype, requires_grad=True)
def pose(theta):
"""Decompose the essential matrix of a rotation about the translation axis."""
rotation = identity + theta.sin() * generator + (1 - theta.cos()) * squared
return epi.decompose_essential_matrix_no_svd(generator @ rotation)
rotation = identity + theta.sin() * generator + (1 - theta.cos()) * squared
derivative = theta.cos() * generator + theta.sin() * squared
expected = (rotation[None], (half_turn @ rotation)[None], translation.reshape(1, 3, 1))
expected_jacobians = (derivative[None], (half_turn @ derivative)[None], torch.zeros_like(expected[2]))
for actual, reference in zip(pose(theta), expected):
self.assert_close(actual, reference)
# Check every scalar derivative, including the entries that should remain zero.
for actual, reference in zip(torch.autograd.functional.jacobian(pose, theta), expected_jacobians):
assert torch.isfinite(actual).all()
self.assert_close(actual, reference)
def test_gradcheck_axis_aligned_5867(self, device):
"""Check a sparse batch whose rows select different column cross products."""
# Different rows select different cross products; none should differentiate an unused 0/0.
E_mat = torch.tensor(
[
[[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]],
[[0.0, 0.0, 1.0], [0.0, 0.0, 0.0], [-1.0, 0.0, 0.0]],
[[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 0.0]],
],
device=device,
dtype=torch.float64,
requires_grad=True,
)
self.gradcheck(epi.decompose_essential_matrix_no_svd, (E_mat,))
def test_dynamo_axis_aligned_5867(self, device, dtype, torch_optimizer):
"""Compare fullgraph outputs and finite backward gradients with eager execution."""
E_mat = torch.tensor(
[[[0.0, 0.0, 0.0], [0.0, 0.0, -1.0], [0.0, 1.0, 0.0]]],
device=device,
dtype=dtype,
requires_grad=True,
)
op = epi.decompose_essential_matrix_no_svd
compiled = torch_optimizer(op, fullgraph=True)
expected, actual = op(E_mat), compiled(E_mat)
for result, reference in zip(actual, expected):
self.assert_close(result, reference)
eager_gradient = torch.autograd.grad(sum(value.sum() for value in expected), E_mat)[0]
compiled_gradient = torch.autograd.grad(sum(value.sum() for value in actual), E_mat)[0]
assert torch.isfinite(compiled_gradient).all()
self.assert_close(compiled_gradient, eager_gradient)
def test_correct_decompose(self):
E_mat = torch.tensor([[[0.2057, -3.8266, 3.1615], [4.5417, -1.0707, -2.2023], [-1.0975, 1.6386, -0.6590]]])
R1, R2, t = epi.decompose_essential_matrix(E_mat)
R1_1, R2_1, t_1 = epi.decompose_essential_matrix_no_svd(E_mat)
# As the orders of two R solutions and t solutions might be different from epi.decompose_essential_matrix(),
# we have to check on the correct ones
rtol: float = 1e-4
if (R1 - R1_1).abs().sum() < rtol:
self.assert_close(R1, R1_1)
self.assert_close(R2, R2_1)
else:
self.assert_close(R1, R2_1)
self.assert_close(R2, R1_1)
R1_1, R2_1 = R2_1, R1_1
if (t - t_1).abs().sum() < rtol:
self.assert_close(t, t_1)
else:
self.assert_close(t, -t_1)
t_1 = -t_1.clone()
self.assert_close(
epi.essential_from_Rt(R1_1, t_1, R2_1, -t_1), epi.essential_from_Rt(R1, t, R2, -t), rtol=1e-3, atol=1e-3
)
def test_consistency(self, device, dtype):
_skip_half(dtype, _NO_HALF_LU.format("decompose_essential_matrix"))
scene = generate_two_view_random_scene(device, dtype)
R1, t1 = scene["R1"], scene["t1"]
R2, t2 = scene["R2"], scene["t2"]
E_mat = epi.essential_from_Rt(R1, t1, R2, t2)
# compare the decomposed R and t to the method with svd
R1, R2, t = epi.decompose_essential_matrix(E_mat)
R1_1, R2_1, t_1 = epi.decompose_essential_matrix_no_svd(E_mat)
# As the orders of two R solutions and t solutions might be different from epi.decompose_essential_matrix(),
# we have to check on the correct ones
rtol: float = 1e-4
if (R1 - R1_1).abs().sum() < rtol:
self.assert_close(R1, R1_1)
self.assert_close(R2, R2_1)
else:
self.assert_close(R1, R2_1)
self.assert_close(R2, R1_1)
R1_1, R2_1 = R2_1, R1_1
if (t - t_1).abs().sum() < rtol:
self.assert_close(t, t_1)
else:
self.assert_close(t, -t_1)
t_1 = -t_1.clone()
self.assert_close(epi.essential_from_Rt(R1_1, t_1, R2_1, -t_1), epi.essential_from_Rt(R1, t, R2, -t))
class TestMotionFromEssential(BaseTester):
def test_smoke(self, device, dtype):
E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype)
Rs, Ts = epi.motion_from_essential(E_mat)
assert Rs.shape == (1, 4, 3, 3)
assert Ts.shape == (1, 4, 3, 1)
@pytest.mark.parametrize("batch_shape", [(1, 3, 3), (2, 3, 3), (2, 1, 3, 3), (3, 2, 1, 3, 3)])
def test_shape(self, batch_shape, device, dtype):
E_mat = torch.rand(batch_shape, device=device, dtype=dtype)
Rs, Ts = epi.motion_from_essential(E_mat)
assert Rs.shape == batch_shape[:-2] + (4, 3, 3)
assert Ts.shape == batch_shape[:-2] + (4, 3, 1)
def test_two_view(self, device, dtype):
scene = generate_two_view_random_scene(device, dtype)
R1, t1 = scene["R1"], scene["t1"]
R2, t2 = scene["R2"], scene["t2"]
E_mat = epi.essential_from_Rt(R1, t1, R2, t2)
R, t = epi.relative_camera_motion(R1, t1, R2, t2)
t = torch.nn.functional.normalize(t, dim=1)
Rs, ts = epi.motion_from_essential(E_mat)
rot_error = (Rs - R).abs().sum((-2, -1))
vec_error = (ts - t).abs().sum(-1)
rtol: float = 1e-4
assert (rot_error < rtol).any() & (vec_error < rtol).any()
def test_gradcheck(self, device):
E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True)
def eval_rot(input):
return epi.motion_from_essential(input)[0]
def eval_vec(input):
return epi.motion_from_essential(input)[1]
self.gradcheck(eval_rot, (E_mat,))
self.gradcheck(eval_vec, (E_mat,))
class TestMotionFromEssentialChooseSolution(BaseTester):
def test_smoke(self, device, dtype):
E_mat = torch.rand(1, 3, 3, device=device, dtype=dtype)
K1 = torch.rand(1, 3, 3, device=device, dtype=dtype)
K2 = torch.rand(1, 3, 3, device=device, dtype=dtype)
x1 = torch.rand(1, 1, 2, device=device, dtype=dtype)
x2 = torch.rand(1, 1, 2, device=device, dtype=dtype)
R, t, X, _ = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2)
assert R.shape == (1, 3, 3)
assert t.shape == (1, 3, 1)
assert X.shape == (1, 1, 3)
@pytest.mark.parametrize("batch_size, num_points", [(1, 3), (2, 3), (2, 8), (3, 2)])
def test_shape(self, batch_size, num_points, device, dtype):
B, N = batch_size, num_points
E_mat = torch.rand(B, 3, 3, device=device, dtype=dtype)
K1 = torch.rand(B, 3, 3, device=device, dtype=dtype)
K2 = torch.rand(1, 3, 3, device=device, dtype=dtype) # check for broadcasting
x1 = torch.rand(B, N, 2, device=device, dtype=dtype)
x2 = torch.rand(B, 1, 2, device=device, dtype=dtype) # check for broadcasting
R, t, X, _ = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2)
assert R.shape == (B, 3, 3)
assert t.shape == (B, 3, 1)
assert X.shape == (B, N, 3)
def test_masking(self, device, dtype):
E_mat = torch.rand(2, 3, 3, device=device, dtype=dtype)
K1 = torch.rand(2, 3, 3, device=device, dtype=dtype)
K2 = torch.rand(2, 3, 3, device=device, dtype=dtype)
x1 = torch.rand(2, 10, 2, device=device, dtype=dtype)
x2 = torch.rand(2, 10, 2, device=device, dtype=dtype)
R, t, X, _ = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1[:, 1:-1, :], x2[:, 1:-1, :])
mask = torch.zeros(2, 10, dtype=torch.bool, device=device)
mask[:, 1:-1] = True
Rm, tm, Xm, _ = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2, mask=mask)
self.assert_close(R, Rm)
self.assert_close(t, tm)
self.assert_close(X, Xm[:, 1:-1, :])
@pytest.mark.parametrize("num_points", [10, 15, 20])
def test_unbatched(self, num_points, device, dtype):
N = num_points
E_mat = torch.rand(3, 3, device=device, dtype=dtype)
K1 = torch.rand(3, 3, device=device, dtype=dtype)
K2 = torch.rand(3, 3, device=device, dtype=dtype)
x1 = torch.rand(N, 2, device=device, dtype=dtype)
x2 = torch.rand(N, 2, device=device, dtype=dtype)
R, t, X, n = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1[1:-1, :], x2[1:-1, :])
assert R.shape == (3, 3)
assert t.shape == (3, 1)
assert X.shape == (N - 2, 3)
assert n.shape == ()
mask = torch.zeros(N, dtype=torch.bool, device=device)
mask[1:-1] = True
Rm, tm, Xm, nm = epi.motion_from_essential_choose_solution(E_mat, K1, K2, x1, x2, mask=mask)
self.assert_close(R, Rm)
self.assert_close(t, tm)
self.assert_close(X, Xm[1:-1, :])
# the count covers the unmasked points only
assert torch.equal(n, nm)
def test_two_view(self, device, dtype):
scene = generate_two_view_random_scene(device, dtype)
E_mat = epi.essential_from_Rt(scene["R1"], scene["t1"], scene["R2"], scene["t2"])
R, t = epi.relative_camera_motion(scene["R1"], scene["t1"], scene["R2"], scene["t2"])
t = torch.nn.functional.normalize(t, dim=1)
R_hat, t_hat, _, _ = epi.motion_from_essential_choose_solution(
E_mat, scene["K1"], scene["K2"], scene["x1"], scene["x2"]
)
self.assert_close(t, t_hat)
self.assert_close(R, R_hat, rtol=1e-4, atol=1e-4)
def test_gradcheck(self, device):
E_mat = torch.rand(1, 3, 3, device=device, dtype=torch.float64, requires_grad=True)
K1 = torch.rand(1, 3, 3, device=device, dtype=torch.float64)
K2 = torch.rand(1, 3, 3, device=device, dtype=torch.float64)
x1 = torch.rand(1, 2, 2, device=device, dtype=torch.float64)
x2 = torch.rand(1, 2, 2, device=device, dtype=torch.float64)
self.gradcheck(
epi.motion_from_essential_choose_solution,
(E_mat, K1, K2, x1, x2),
requires_grad=(True, False, False, False, False),
)
_NO_HALF_LU = "{} calls torch.det (LU), which has no float16/bfloat16 kernel"
_NO_HALF_FIND_ESSENTIAL = (
"find_essential solves in float64, but float16/bfloat16 correspondences are too coarse for these tolerances and "
"the reference helpers call torch.linalg.inv, which has no half kernel"
)
_HALF_PIXEL_F = (
"a pixel-unit F spans eight decades (entries down to ~1e-8): float16 flushes the small entries to zero and "
"bfloat16's 8-bit mantissa cannot resolve the epipolar residual, which kornia evaluates in the input dtype"
)
# A five-point sample of normalised coordinates with no real root (recipe in the #4883 pin).
_NO_REAL_ROOT_P1 = [
[-0.094936139146062, 0.3761027702259922],
[-0.13508466816570844, -0.15520684765969847],
[0.4518000060718804, 0.7491498583169849],
[-0.021505790125727963, 0.25308016814257595],
[-0.021008959788924204, -0.29327032505050715],
]
_NO_REAL_ROOT_P2 = [
[0.5908350760196772, -0.024407062108416273],
[0.4805667988287366, -0.025019944402247606],
[-0.5701912982679379, 0.4911702947428827],
[-0.8380815108891836, 0.2981499818513886],
[-0.8246696433582323, 0.08361235498654443],
]
def _skip_half(dtype: torch.dtype, reason: str) -> None:
if dtype in (torch.float16, torch.bfloat16):
pytest.skip(reason)
def _skip_find_essential(device: torch.device, dtype: torch.dtype) -> None:
_skip_half(dtype, _NO_HALF_FIND_ESSENTIAL)
def _hom(p: torch.Tensor) -> torch.Tensor:
return torch.cat([p, torch.ones_like(p[..., :1])], -1)
def _epipolar_residual(F: torch.Tensor, pts1: torch.Tensor, pts2: torch.Tensor) -> torch.Tensor:
"""|pts2^T F pts1| per correspondence."""
return (_hom(pts2) * (_hom(pts1) @ F.transpose(-2, -1))).sum(-1).abs()
def _normalized(K: torch.Tensor, pts: torch.Tensor) -> torch.Tensor:
"""Normalised camera coordinates K^-1 [u, v, 1]^T, dehomogenised."""
return (_hom(pts) @ torch.linalg.inv(K).transpose(-2, -1))[..., :2]
def _gt_essential(scene):
eye = torch.eye(3, device=scene["R"].device, dtype=scene["R"].dtype)[None]
return epi.essential_from_Rt(eye, torch.zeros_like(scene["t"]), scene["R"], scene["t"])
def _first_camera(device, dtype):
"""A non-identity first camera, as world-to-camera extrinsics (R, t)."""
Ra = kornia.geometry.conversions.axis_angle_to_rotation_matrix(
torch.tensor([[-0.3, 0.15, 0.2]], device=device, dtype=dtype)
)
ta = torch.tensor([[[0.2], [-0.4], [1.1]]], device=device, dtype=dtype)
return Ra, ta
class TestConventionEssential(BaseTester):
def test_convention_find_essential_returns_ten_candidates(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_find_essential(device, dtype)
n1, n2 = _normalized(two_view["K1"], two_view["x1"]), _normalized(two_view["K2"], two_view["x2"])
real = {}
for num_points in (5, 12):
E = epi.find_essential(n1[:, :num_points], n2[:, :num_points])
# Always ten slots; each is either a real solution of unit Frobenius norm or all-NaN (complex root).
assert E.shape == (1, 10, 3, 3)
finite = torch.isfinite(E).all(dim=-1).all(dim=-1)[0]
assert (finite | torch.isnan(E).all(dim=-1).all(dim=-1)[0]).all()
real[num_points] = E[0, finite]
assert real[num_points].shape[0] >= 1
self.assert_close(real[num_points].norm(dim=(-2, -1)), torch.ones_like(real[num_points][:, 0, 0]))
# Minimal sample: every real candidate satisfies x2^T E x1 = 0 on the normalised coordinates of points1
# (first image) and points2 (second image); the swapped product is the control.
p1, p2 = n1[:, :5], n2[:, :5]
for cand in real[5]:
assert _epipolar_residual(cand[None], p1, p2).max() < 1e-3 * _epipolar_residual(cand[None], p2, p1).max()
# The minimal sample and all twelve points: one candidate is the true E up to sign, on the same side. The
# minimal sample used to miss it in float32 (#4884); the solve now runs in float64, on the host for MPS.
E_gt = _gt_essential(two_view)
E_gt = E_gt / E_gt.norm()
for num_points in (5, 12):
candidates = real[num_points]
dist = torch.minimum(
(candidates - E_gt).abs().amax(dim=(-2, -1)), (candidates + E_gt).abs().amax(dim=(-2, -1))
)
assert dist.min() < 1e-4
best = candidates[dist.argmin()][None]
assert _epipolar_residual(best, n1, n2).max() < 1e-3 * _epipolar_residual(best, n2, n1).max()
def test_convention_essential_from_Rt_is_tx_R_of_relative_motion(self, device, dtype):
two_view = two_view_scene(device, dtype)
R, t = two_view["R"], two_view["t"]
Ra, ta = _first_camera(device, dtype)
# E = [t]x R of relative_camera_motion(R1, t1, R2, t2), here with a non-identity first camera.
E = epi.essential_from_Rt(Ra, ta, R @ Ra, R @ ta + t)
self.assert_close(E, epi.cross_product_matrix(t[..., 0]) @ R, low_tolerance=True)
# Not normalised: ||E||_F = sqrt(2) ||t||.
self.assert_close(E.norm(dim=(-2, -1)), (2.0**0.5) * t.norm(dim=(-2, -1)), low_tolerance=True)
# Swapping the cameras gives E^T, which is far from E on this fixture.
E_swapped = epi.essential_from_Rt(R @ Ra, R @ ta + t, Ra, ta)
self.assert_close(E_swapped, E.transpose(-2, -1), low_tolerance=True)
assert (E - E.transpose(-2, -1)).abs().max() > 0.5
def test_convention_essential_from_fundamental_K_sides(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_half(dtype, _HALF_PIXEL_F)
K1, K2 = two_view["K1"], two_view["K2"]
E_gt = _gt_essential(two_view)
F = epi.fundamental_from_essential(E_gt, K1, K2)
F = F / F[..., 2:, 2:] # the scale an estimator returns
# E = K2^T F K1: K1 is the camera of points1.
E = epi.essential_from_fundamental(F, K1, K2)
self.assert_close(E, K2.transpose(-2, -1) @ F @ K1)
n1, n2 = _normalized(K1, two_view["x1"]), _normalized(K2, two_view["x2"])
E_swapped = epi.essential_from_fundamental(F, K2, K1)
resid = _epipolar_residual(E / E.norm(), n1, n2).max()
assert resid < 1e-3 * _epipolar_residual(E_swapped / E_swapped.norm(), n1, n2).max()
def test_convention_motion_from_essential_candidate_order(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_half(dtype, _NO_HALF_LU.format("motion_from_essential"))
R, t = two_view["R"], two_view["t"]
t_unit = t / t.norm(dim=-2, keepdim=True)
E = _gt_essential(two_view)
# project_to_essential keeps the input's scale: it is not normalised to unit Frobenius norm.
E_unit = E / E.norm(dim=(-2, -1), keepdim=True)
projected_norm = epi.project_to_essential(5.0 * E_unit).norm(dim=(-2, -1))
self.assert_close(projected_norm, torch.full_like(projected_norm, 5.0))
candidate_sets = []
for E_in in (E, -E, 3.0 * E):
Rs, ts = epi.motion_from_essential(E_in)
R1, R2, t_dec = epi.decompose_essential_matrix(E_in)
assert Rs.shape == (1, 4, 3, 3)
assert ts.shape == (1, 4, 3, 1)
# Order [(R1, t), (R1, -t), (R2, t), (R2, -t)] of decompose_essential_matrix, with a unit t.
self.assert_close(Rs[:, 0], R1)
self.assert_close(Rs[:, 1], R1)
self.assert_close(Rs[:, 2], R2)
self.assert_close(Rs[:, 3], R2)
self.assert_close(ts[:, 0], t_dec)
self.assert_close(ts[:, 1], -t_dec)
self.assert_close(ts[:, 2], t_dec)
self.assert_close(ts[:, 3], -t_dec)
self.assert_close(ts.norm(dim=(-2, -1)), torch.ones(1, 4, device=device, dtype=dtype))
# Candidates 1 and 2 rebuild E_in itself as [t]x R (up to a positive scale), candidates 0 and 3 its
# negative. Which of them is the true pose follows the sign and scale of E_in, so no index is fixed.
# With det U = +1, [u3]x U W V^T = -U diag(1, 1, 0) V^T, so this pairing does not depend on the SVD's
# sign choices.
unit = E_in * (2.0**0.5) / E_in.norm()
for i, sign in enumerate((-1.0, 1.0, 1.0, -1.0)):
self.assert_close(epi.cross_product_matrix(ts[:, i, :, 0]) @ Rs[:, i], sign * unit)
hits = [
bool((Rs[:, i] - R).abs().max() < 1e-4 and (ts[:, i] - t_unit).abs().max() < 1e-4) for i in range(4)
]
assert sum(hits) == 1
candidate_sets.append((Rs[0], ts[0]))
# E, -E and 3E give the same candidate set.
Rs0, ts0 = candidate_sets[0]
for Rs_k, ts_k in candidate_sets[1:]:
for i in range(4):
match = (Rs_k - Rs0[i]).abs().amax(dim=(-2, -1)) + (ts_k - ts0[i]).abs().amax(dim=(-2, -1))
assert match.min() < 1e-4
def test_convention_choose_solution_recovers_relative_motion(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_half(dtype, _NO_HALF_LU.format("motion_from_essential_choose_solution"))
R, t, X = two_view["R"], two_view["t"], two_view["X"]
K1, K2, x1, x2 = two_view["K1"], two_view["K2"], two_view["x1"], two_view["x2"]
t_norm = t.norm(dim=-2, keepdim=True)
E = _gt_essential(two_view)
# Pixel coordinates in, K1 and K2 applied inside: the pose of camera 2 relative to camera 1 with a unit t,
# and the points triangulated in the first camera's frame at that scale, all at positive depth.
for E_in in (E, -E):
R_out, t_out, X_out, _ = epi.motion_from_essential_choose_solution(E_in, K1, K2, x1, x2)
self.assert_close(R_out, R, rtol=1e-4, atol=1e-4)
self.assert_close(t_out, t / t_norm, rtol=1e-4, atol=1e-4)
self.assert_close(X_out, X / t_norm, rtol=1e-4, atol=1e-3)
assert (X_out[..., 2] > 0).all()
# Swapping the images (E^T, K2, K1, x2, x1) returns the inverse motion.
R_sw, t_sw, _, _ = epi.motion_from_essential_choose_solution(E.transpose(-2, -1), K2, K1, x2, x1)
self.assert_close(R_sw, R.transpose(-2, -1), rtol=1e-4, atol=1e-4)
self.assert_close(t_sw, -R.transpose(-2, -1) @ t / t_norm, rtol=1e-4, atol=1e-4)
# Control: with K1 and K2 swapped the triangulated points are wrong.
_, _, X_swapped_k, _ = epi.motion_from_essential_choose_solution(E, K2, K1, x1, x2)
assert (X_swapped_k - X / t_norm).abs().max() > 1.0
def test_convention_relative_camera_motion_world_to_camera(self, device, dtype):
two_view = two_view_scene(device, dtype)
R, t = two_view["R"], two_view["t"]
Ra, ta = _first_camera(device, dtype)
# World-to-camera extrinsics: camera 2 = (R Ra, R ta + t) sits at (R, t) from camera 1 = (Ra, ta), and the
# result is (R2 R1^T, t2 - R2 R1^T t1).
R_rel, t_rel = epi.relative_camera_motion(Ra, ta, R @ Ra, R @ ta + t)
self.assert_close(R_rel, R, low_tolerance=True)
self.assert_close(t_rel, t, low_tolerance=True)
# Swapping the cameras gives the inverse motion, far from (R, t) on this fixture.
R_sw, t_sw = epi.relative_camera_motion(R @ Ra, R @ ta + t, Ra, ta)
self.assert_close(R_sw, R.transpose(-2, -1), low_tolerance=True)
self.assert_close(t_sw, -R.transpose(-2, -1) @ t, low_tolerance=True)
assert (R_sw - R).abs().max() > 0.1
assert (t_sw - t).abs().max() > 0.5
def test_convention_find_essential_ignores_weights_4876(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_find_essential(device, dtype)
# #4876: weights are intentionally ignored for compatibility, including for this non-minimal sample:
# an outlier with weight 0, all-zero weights and all-one weights give the same output (NaN slots compared as 0).
n1, n2 = _normalized(two_view["K1"], two_view["x1"]), _normalized(two_view["K2"], two_view["x2"])
p1 = torch.cat([n1, torch.tensor([[[0.4, -0.3]]], device=device, dtype=dtype)], 1)
p2 = torch.cat([n2, torch.tensor([[[-0.35, 0.25]]], device=device, dtype=dtype)], 1)
ones = torch.ones(1, 13, device=device, dtype=dtype)
outlier_off = ones.clone()
outlier_off[0, 12] = 0.0
E_ones = epi.find_essential(p1, p2, ones)
assert torch.equal(epi.find_essential(p1, p2, outlier_off).nan_to_num(0.0), E_ones.nan_to_num(0.0))
assert torch.equal(epi.find_essential(p1, p2, torch.zeros_like(ones)).nan_to_num(0.0), E_ones.nan_to_num(0.0))
# Control: the outlier does move the estimate, so a working weight would change the result.
E_clean = epi.find_essential(n1, n2)
def best(E):
real = E[0, torch.isfinite(E[0]).all(dim=-1).all(dim=-1)]
return min(_epipolar_residual(e[None], n1, n2).max() for e in real)
assert best(E_ones) > 1e3 * best(E_clean)
def test_decompose_unbatched_keeps_input_dims_4878(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_half(dtype, _NO_HALF_LU.format("decompose_essential_matrix"))
# #4878 (fixed): the constant W was built with a batch dim, so a (3, 3) input returned rotations of shape
# (1, 3, 3) next to t of shape (3, 1). All outputs now keep the leading dims of the input.
E = _gt_essential(two_view)
R1, R2, t = epi.decompose_essential_matrix(E[0])
assert R1.shape == (3, 3)
assert R2.shape == (3, 3)
assert t.shape == (3, 1)
Rs, ts = epi.motion_from_essential(E[0])
assert Rs.shape == (4, 3, 3)
assert ts.shape == (4, 3, 1)
R1b, R2b, tb = epi.decompose_essential_matrix(E)
self.assert_close(R1, R1b[0])
self.assert_close(R2, R2b[0])
self.assert_close(t, tb[0])
Rsb, tsb = epi.motion_from_essential(E.expand(2, 1, 3, 3))
assert Rsb.shape == (2, 1, 4, 3, 3)
assert tsb.shape == (2, 1, 4, 3, 1)
def test_convention_choose_solution_reports_valid_count_4879(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_half(dtype, _NO_HALF_LU.format("motion_from_essential_choose_solution"))
K1, K2, x1, x2 = two_view["K1"], two_view["K2"], two_view["x1"], two_view["x2"]
R, t = two_view["R"], two_view["t"]
E = _gt_essential(two_view)
Rs, ts = epi.motion_from_essential(E)
t_unit = t / t.norm(dim=-2, keepdim=True)
# On this E candidate 0 is not the true pose, so returning it is visibly wrong.
assert (Rs[:, 0] - R).abs().max() > 0.1 or (ts[:, 0] - t_unit).abs().max() > 0.1
# #4879: expose how many correspondences passed the positive-depth test.
R_all, t_all, _, valid_all = epi.motion_from_essential_choose_solution(E, K1, K2, x1, x2)
self.assert_close(R_all, R, rtol=1e-4, atol=1e-4)
self.assert_close(t_all, t_unit, rtol=1e-4, atol=1e-4)
assert torch.equal(valid_all, torch.tensor([12], device=device))
# With every point masked out no candidate passes the depth test.
# The caller must be able to detect this instead of silently receiving candidate 0.
mask = torch.zeros(1, 12, dtype=torch.bool, device=device)
_, _, _, valid_none = epi.motion_from_essential_choose_solution(E, K1, K2, x1, x2, mask=mask)
assert torch.equal(valid_none, torch.tensor([0], device=device))
# Control: one unmasked point is enough to select the true pose and reports one valid point.
mask[0, 5] = True
R_one, t_one, _, valid_one = epi.motion_from_essential_choose_solution(E, K1, K2, x1, x2, mask=mask)
self.assert_close(R_one, R, rtol=1e-4, atol=1e-4)
self.assert_close(t_one, t_unit, rtol=1e-4, atol=1e-4)
assert torch.equal(valid_one, torch.tensor([1], device=device))
def test_decompose_no_svd_batch_matches_single_4880(self, device, dtype):
two_view = two_view_scene(device, dtype)
# #4880 (fixed): the rotation normaliser used to sum over the whole batch, so a batch of two copies of E
# returned non-rotations. Each element is now normalised on its own and matches the unbatched result.
E = _gt_essential(two_view)
eye = torch.eye(3, device=device, dtype=torch.float32)
def orthogonality_error(Rm):
Rm = Rm.float()
return (Rm @ Rm.transpose(-2, -1) - eye).norm(dim=(-2, -1))
R1, R2, t = epi.decompose_essential_matrix_no_svd(E)
assert orthogonality_error(R1).max() < 0.25
assert orthogonality_error(R2).max() < 0.25
for B in (2, 3):
R1b, R2b, tb = epi.decompose_essential_matrix_no_svd(E.repeat(B, 1, 1))
assert orthogonality_error(R1b).max() < 0.25
assert orthogonality_error(R2b).max() < 0.25
self.assert_close(R1b, R1.expand_as(R1b))
self.assert_close(R2b, R2.expand_as(R2b))
self.assert_close(tb, t.expand_as(tb))
def test_convention_choose_solution_batch_independent_2198(self, device, dtype):
two_view = two_view_scene(device, dtype)
_skip_half(dtype, _NO_HALF_LU.format("motion_from_essential_choose_solution"))
K1, K2, x1, x2 = two_view["K1"], two_view["K2"], two_view["x1"], two_view["x2"]
R, t = two_view["R"], two_view["t"]
t_unit = t / t.norm(dim=-2, keepdim=True)
E = _gt_essential(two_view)
def is_truth(R_out, t_out):
return bool((R_out - R[0]).abs().max() < 1e-4 and (t_out - t_unit[0]).abs().max() < 1e-4)
# E and -E have their true pose at different candidate indices; each alone is recovered.
for E_in in (E, -E):
R_out, t_out, _, _ = epi.motion_from_essential_choose_solution(E_in, K1, K2, x1, x2)
assert is_truth(R_out[0], t_out[0])
# #2198: each batch element must select its own candidate, independently of the others.
R_b, t_b, X_b, _ = epi.motion_from_essential_choose_solution(
torch.cat([E, -E]), K1.expand(2, 3, 3), K2.expand(2, 3, 3), x1.expand(2, 12, 2), x2.expand(2, 12, 2)
)
assert is_truth(R_b[0], t_b[0])
assert is_truth(R_b[1], t_b[1])
assert (X_b[1, :, 2] > 0).all()
def test_convention_find_essential_no_real_root_nan_4883(self, device, dtype):
# #4883: a five-point sample with no real root returns ten NaN slots, like the complex slots of any other
# sample, so an isfinite filter drops it. Sample:
# g = torch.Generator().manual_seed(0)
# p1 = torch.randn(4000, 5, 2, generator=g, dtype=torch.float64) * 0.5 # then p2 from the same g
# p1[1371], p2[1371]
# Batched with an ordinary sample, which keeps its real candidates and finite gradients.
g = torch.Generator().manual_seed(0)
q1, q2 = (torch.randn(1, 5, 2, generator=g, dtype=torch.float64) * 0.5 for _ in range(2))
p1 = torch.cat([torch.tensor(_NO_REAL_ROOT_P1, dtype=torch.float64)[None], q1]).to(device, dtype)
p2 = torch.cat([torch.tensor(_NO_REAL_ROOT_P2, dtype=torch.float64)[None], q2]).to(device, dtype)
p1.requires_grad_()
E = epi.find_essential(p1, p2)
assert torch.isnan(E[0]).all()
real = torch.isfinite(E[1]).flatten(1).all(1)
assert real.any()
(grad,) = torch.autograd.grad(E[1][real].sum(), p1)
assert torch.isfinite(grad).all()
assert (grad[0] == 0).all()
class TestRefineEssentialLM(BaseTester):
"""``_refine_essential_lm``, RANSAC's Levenberg-Marquardt refinement of essential matrices (float64, host)."""
@staticmethod
def _start(E, scale, seed):
noise = torch.randn(3, 3, generator=torch.Generator().manual_seed(seed), dtype=E.dtype)
start = epi.project_to_essential(E + scale * noise)
return (start / start.norm())[None]
@staticmethod
def _aligned(E, reference):
return E * torch.sign((E * reference).sum())
@pytest.mark.parametrize("loss", ["truncated", "cauchy"])
def test_stays_on_the_manifold_and_lowers_the_cost(self, loss):
scene = calibrated_two_view_scene(200, 1e-3, 0)
x1, x2 = scene["x1"], scene["x2"]
start = self._start(scene["E"], 0.02, 1)
refined = _refine_essential_lm(
start, convert_points_to_homogeneous(x1), convert_points_to_homogeneous(x2), None, loss, 3e-3**2, 10
)
singular_values = torch.linalg.svdvals(refined[0])
self.assert_close(
singular_values, torch.tensor([2**-0.5, 2**-0.5, 0.0], dtype=torch.float64), atol=1e-12, rtol=0
)
before = epi.sampson_epipolar_distance(x1[None], x2[None], start).sum()
after = epi.sampson_epipolar_distance(x1[None], x2[None], refined).sum()
assert after < 0.5 * before
def test_reaches_the_noise_free_model(self):
scene = calibrated_two_view_scene(100, 0.0, 2)
h1, h2 = convert_points_to_homogeneous(scene["x1"]), convert_points_to_homogeneous(scene["x2"])
refined = _refine_essential_lm(self._start(scene["E"], 0.01, 3), h1, h2, None, "cauchy", 1e-6, 20)[0]
self.assert_close(self._aligned(refined, scene["E"]), scene["E"], atol=1e-9, rtol=0)
def test_mask_restricts_each_model_to_its_correspondences(self):
scene = calibrated_two_view_scene(60, 0.0, 4)
x2 = scene["x2"].clone()
x2[30:] = x2[30:] + 0.3 # gross outliers, excluded by the mask
h1, h2 = convert_points_to_homogeneous(scene["x1"]), convert_points_to_homogeneous(x2)
mask = torch.zeros(1, 60, dtype=torch.float64)
mask[0, :30] = 1.0
refined = _refine_essential_lm(self._start(scene["E"], 0.01, 5), h1, h2, mask, "cauchy", 1e-6, 20)[0]
self.assert_close(self._aligned(refined, scene["E"]), scene["E"], atol=1e-9, rtol=0)
def test_rejects_steps_that_raise_the_cost(self):
# A start 0.3 off and 30 of 100 correspondences moved off their epipolar lines: an undamped Gauss-Newton step
# raises the Cauchy cost at the third iteration here. The refiner keeps only steps that lower it, so its cost
# after each iteration is never above the one before (the loop is deterministic, so k iterations are the
# first k of k + 1).
scene = calibrated_two_view_scene(100, 1e-3, 8)
x1, x2 = scene["x1"], scene["x2"].clone()
x2[70:] = x2[70:] + 0.2 * torch.randn(30, 2, generator=torch.Generator().manual_seed(8), dtype=x2.dtype)
h1, h2 = convert_points_to_homogeneous(x1), convert_points_to_homogeneous(x2)
start, scale2 = self._start(scene["E"], 0.3, 108), 3e-3**2
costs = []
for iters in range(11):
refined = _refine_essential_lm(start, h1, h2, None, "cauchy", scale2, iters)
costs.append(float(torch.log1p(epi.sampson_epipolar_distance(x1[None], x2[None], refined) / scale2).sum()))
assert all(later <= earlier * (1 + 1e-9) for earlier, later in zip(costs, costs[1:]))
assert costs[-1] < 0.5 * costs[0]
def test_batched_models_are_refined_independently(self):
scene = calibrated_two_view_scene(80, 0.0, 6)
h1, h2 = convert_points_to_homogeneous(scene["x1"]), convert_points_to_homogeneous(scene["x2"])
starts = torch.cat([self._start(scene["E"], 0.01, 7), self._start(scene["E"], 0.03, 8)])
together = _refine_essential_lm(starts, h1, h2, None, "truncated", 1e-4, 15)
alone = torch.cat([_refine_essential_lm(start[None], h1, h2, None, "truncated", 1e-4, 15) for start in starts])
self.assert_close(together, alone, atol=1e-12, rtol=0)
def _skip_without_bfloat16_autocast(device):
# A bfloat16 autocast region raises on a CUDA device without bfloat16 support, and other devices are not covered.
if device.type == "cuda" and not torch.cuda.is_bf16_supported():
pytest.skip("this CUDA device does not support bfloat16")
if device.type not in ("cpu", "cuda"):
pytest.skip("CPU and CUDA autocast")
def _autocast_scene(device):
# A float32 two-view scene built outside any autocast region: K, a small motion, and 30 points in front of both
# cameras.
generator = torch.Generator().manual_seed(6146)
K = torch.tensor([[[300.0, 0.0, 50.0], [0.0, 300.0, 50.0], [0.0, 0.0, 1.0]]])
R = kornia.geometry.conversions.axis_angle_to_rotation_matrix(torch.tensor([[0.05, -0.1, 0.02]]))
t = torch.tensor([[[0.5], [0.1], [0.0]]])
E_mat = epi.essential_from_Rt(torch.eye(3)[None], torch.zeros(1, 3, 1), R, t)
X = torch.rand(1, 30, 3, generator=generator) * 4 + torch.tensor([-2.0, -2.0, 5.0])
x1 = kornia.geometry.conversions.convert_points_from_homogeneous(X @ K.mT)
x2 = kornia.geometry.conversions.convert_points_from_homogeneous((X @ R.mT + t.mT) @ K.mT)
return [v.to(device) for v in (E_mat, K, x1, x2, R)]
_AUTOCAST_CALLS = {
"decompose_essential_matrix": lambda E_mat, K, x1, x2: epi.decompose_essential_matrix(E_mat),
"decompose_essential_matrix_no_svd": lambda E_mat, K, x1, x2: epi.decompose_essential_matrix_no_svd(E_mat),
"motion_from_essential": lambda E_mat, K, x1, x2: epi.motion_from_essential(E_mat),
"motion_from_essential_choose_solution": lambda E_mat, K, x1, x2: epi.motion_from_essential_choose_solution(
E_mat, K, K, x1, x2
),
"project_to_essential": lambda E_mat, K, x1, x2: epi.project_to_essential(E_mat),
}
class TestEssentialDecompositionAutocast(BaseTester):
"""A caller's autocast region does not reach the essential-matrix decompositions (#6146), as for #6072's solvers."""
@pytest.mark.parametrize("name", list(_AUTOCAST_CALLS))
def test_autocast_does_not_change_the_result_6146(self, device, name):
# Under bfloat16 autocast, a float32 essential matrix was decomposed with bfloat16 matrix products: bfloat16
# rotations about 1.6e-3 off and no longer orthogonal, next to a float32 translation.
_skip_without_bfloat16_autocast(device)
E_mat, K, x1, x2, _ = _autocast_scene(device)
call = _AUTOCAST_CALLS[name]
expected = call(E_mat, K, x1, x2)
with torch.autocast(device.type, dtype=torch.bfloat16):
actual = call(E_mat, K, x1, x2)
expected = expected if isinstance(expected, tuple) else (expected,)
actual = actual if isinstance(actual, tuple) else (actual,)
for out, ref in zip(actual, expected):
assert out.dtype == ref.dtype
self.assert_close(out, ref, atol=0, rtol=0)
def test_chosen_rotation_stays_a_rotation_under_autocast_6146(self, device):
_skip_without_bfloat16_autocast(device)
E_mat, K, x1, x2, R = _autocast_scene(device)
with torch.autocast(device.type, dtype=torch.bfloat16):
R_out, t_out, _, count = epi.motion_from_essential_choose_solution(E_mat, K, K, x1, x2)
assert R_out.dtype == t_out.dtype == torch.float32
assert int(count) == 30
self.assert_close(R_out, R, atol=1e-5, rtol=0)
self.assert_close(R_out @ R_out.mT, torch.eye(3, device=device).expand_as(R_out), atol=1e-5, rtol=0)
def test_half_precision_intrinsics_under_autocast_6146(self, device):
# Autocast used to cast bfloat16 intrinsics and a float32 essential matrix to one dtype. With autocast off
# inside, the intrinsics are cast to the matrix's dtype, so the call still runs and gives the float32 result.
_skip_without_bfloat16_autocast(device)
E_mat, K, x1, x2, _ = _autocast_scene(device)
K_half = K.to(torch.bfloat16)
expected = epi.motion_from_essential_choose_solution(E_mat, K_half.float(), K_half.float(), x1, x2)
with torch.autocast(device.type, dtype=torch.bfloat16):
actual = epi.motion_from_essential_choose_solution(E_mat, K_half, K_half, x1, x2)
for out, ref in zip(actual, expected):
assert out.dtype == ref.dtype
self.assert_close(out, ref, atol=0, rtol=0)
@pytest.mark.parametrize(
"fn",
[
epi.decompose_essential_matrix,
epi.decompose_essential_matrix_no_svd,
epi.motion_from_essential,
epi.project_to_essential,
],
ids=lambda fn: fn.__name__,
)
def test_still_scriptable_6146(self, device, dtype, fn):
# TorchScript has no context managers, so a scripted call skips the autocast guard; it must keep scripting.
E_mat = epi.essential_from_Rt(
torch.eye(3, device=device, dtype=dtype)[None],
torch.zeros(1, 3, 1, device=device, dtype=dtype),
kornia.geometry.conversions.axis_angle_to_rotation_matrix(
torch.tensor([[0.05, -0.1, 0.02]], device=device, dtype=dtype)
),
torch.tensor([[[0.5], [0.1], [0.0]]], device=device, dtype=dtype),
)
if dtype in (torch.float16, torch.bfloat16) and fn is not epi.decompose_essential_matrix_no_svd:
pytest.skip("calls torch.det or torch.linalg.svd, which have no half-precision CPU kernel")
scripted = torch.jit.script(fn)
expected = fn(E_mat)
actual = scripted(E_mat)
expected = expected if isinstance(expected, tuple) else (expected,)
actual = actual if isinstance(actual, tuple) else (actual,)
for out, ref in zip(actual, expected):
self.assert_close(out, ref)