π Geometric Computer Vision Library for Spatial AI
Python artificial-intelligencecomputer-visiondeep-learninghacktoberfestimage-processingmachine-learningneural-networkpythonpytorchroboticsspatial-ai
fix(metrics): reject mismatched batches and validate inputs in accuracy (#6220)
Ali Tharwata502c4f2110d
diff --git a/changelog.d/6220.fixed.md b/changelog.d/6220.fixed.mdnew file mode 100644index 000000000..18454987f--- /dev/null+++ b/changelog.d/6220.fixed.md@@ -0,0 +1 @@+`kornia.metrics.accuracy` no longer reports accuracies above 100 % when `target` holds a different number of samples than `pred`: a one-entry `target` was compared with every row of `pred` and the count divided by 1, so `accuracy(torch.tensor([[0.0, 1.0], [0.0, 1.0]]), torch.tensor([1]))` returned `[tensor(200.)]`. Such calls now raise `ShapeError`, as do other malformed `pred` and `target` shapes. `topk` entries must be positive integers (Python or NumPy integers or integer tensors, not bools): a `k` of 0 used to report 0 %, and a negative `k` counted from the other end. A `pred` or `target` that is not a tensor now raises `TypeCheckError`, and tensors on different devices raise `DeviceError`.diff --git a/kornia/metrics/accuracy.py b/kornia/metrics/accuracy.pyindex 08ce90bda..6a3ddc205 100644--- a/kornia/metrics/accuracy.py+++ b/kornia/metrics/accuracy.py@@ -15,17 +15,39 @@# limitations under the License.#-from typing import List, Tuple+from __future__ import annotations++import operator+from typing import Any, Tupleimport torch+from kornia.core.check import (+ KORNIA_CHECK,+ KORNIA_CHECK_IS_TENSOR,+ KORNIA_CHECK_SAME_DEVICE,+ KORNIA_CHECK_SHAPE,+ KORNIA_CHECK_SIZES,+)+++def _is_positive_index(k: Any) -> bool:+ """Return whether ``k`` is a positive integer; NumPy integers and 0-d integer tensors count, bools do not."""+ if isinstance(k, bool) or (isinstance(k, torch.Tensor) and k.dtype == torch.bool):+ return False+ try:+ return operator.index(k) > 0+ except (TypeError, ValueError):+ return False-def accuracy(pred: torch.Tensor, target: torch.Tensor, topk: Tuple[int, ...] = (1,)) -> List[torch.Tensor]:++def accuracy(pred: torch.Tensor, target: torch.Tensor, topk: Tuple[int, ...] = (1,)) -> list[torch.Tensor]:"""Compute the accuracy over the k top predictions for the specified values of k.Convention:- ``pred`` holds one score per class, :math:`(B, C)`; only the order of the scores in a row matters.- ``target`` holds class indices, :math:`(B,)` or :math:`(B, 1)`.+ ``target`` holds class indices, :math:`(B,)` or :math:`(B, 1)`. The batch sizes of ``pred`` and+ ``target`` must match.- The result is a list with one 0-d float32 tensor per entry of ``topk``, in that order. Each is a**percentage** in :math:`[0, 100]`, not a fraction, taken over the whole batch: a sample counts for ``k``when its target is among its ``k`` highest scores, and a ``k`` above :math:`C` counts as :math:`C`.@@ -43,6 +65,17 @@ def accuracy(pred: torch.Tensor, target: torch.Tensor, topk: Tuple[int, ...] = ([tensor(100.)]"""+ KORNIA_CHECK_IS_TENSOR(pred)+ KORNIA_CHECK_IS_TENSOR(target)+ KORNIA_CHECK_SAME_DEVICE(pred, target)+ KORNIA_CHECK_SHAPE(pred, ["B", "C"])+ # one class index per sample; size-1 dimensions after the first, as in (B, 1), are read the same way+ KORNIA_CHECK_SIZES(target, [pred.shape[0], *([1] * (target.ndim - 1))])+ KORNIA_CHECK(+ len(topk) > 0 and all(_is_positive_index(k) for k in topk),+ "topk must be a non-empty sequence of positive integers",+ )+maxk = min(max(topk), pred.size()[1])batch_size = target.size(0)_, pred = pred.topk(maxk, 1, True, True)diff --git a/tests/metrics/test_accuracy.py b/tests/metrics/test_accuracy.pyindex ce2cdd50e..226df0953 100644--- a/tests/metrics/test_accuracy.py+++ b/tests/metrics/test_accuracy.py@@ -17,40 +17,60 @@from __future__ import annotations+import numpy as npimport pytestimport torchimport kornia+from kornia.core.exceptions import BaseError, DeviceError, ShapeError, TypeCheckErrorfrom kornia.metrics import accuracyfrom testing.base import BaseTester, supports_topkclass TestAccuracy:- def test_top1_perfect(self):- logits = torch.tensor([[0.0, 1.0, 0.0], [1.0, 0.0, 0.0]])- target = torch.tensor([[1], [0]])- result = accuracy(logits, target, topk=(1,))- assert len(result) == 1- assert result[0].item() == 100.0+ def test_smoke(self, device):+ logits = torch.tensor([[0.0, 1.0, 2.0]], device=device)+ target = torch.tensor([2], device=device)+ assert accuracy(logits, target)[0].item() == 100.0++ def test_smoke_with_batch(self, device):+ logits = torch.tensor([[0.0, 1.0, 2.0], [2.0, 1.0, 0.0]], device=device)+ target = torch.tensor([2, 0], device=device)+ assert accuracy(logits, target)[0].item() == 100.0- def test_top1_zero(self):- logits = torch.tensor([[1.0, 0.0, 0.0]])- target = torch.tensor([[2]])- result = accuracy(logits, target, topk=(1,))- assert result[0].item() == 0.0+ def test_smoke_with_target_shape_b_1(self, device):+ logits = torch.tensor([[0.0, 1.0, 2.0], [2.0, 1.0, 0.0]], device=device)+ target = torch.tensor([[2], [0]], device=device)+ assert accuracy(logits, target)[0].item() == 100.0++ def test_topk(self, device):+ logits = torch.tensor([[0.0, 3.0, 1.0, 2.0]], device=device)+ target = torch.tensor([1], device=device)+ top1, top2 = accuracy(logits, target, topk=(1, 2))+ assert top1.item() == 100.0+ assert top2.item() == 100.0- def test_topk(self):- # 3 classes; true class is index 2 (lowest logit) β wrong for both top-1 and top-2- logits = torch.tensor([[0.3, 0.5, 0.2]])- target = torch.tensor([[2]]) # true class is index 2 (third highest)+ logits = torch.tensor([[0.0, 1.0, 3.0, 2.0]], device=device)+ target = torch.tensor([3], device=device)top1, top2 = accuracy(logits, target, topk=(1, 2))assert top1.item() == 0.0- assert top2.item() == 0.0+ assert top2.item() == 100.0- logits2 = torch.tensor([[0.3, 0.2, 0.5]])- target2 = torch.tensor([[2]])- top1b, _ = accuracy(logits2, target2, topk=(1, 2))+ def test_topk_with_target_shape_b_1(self, device):+ logits = torch.tensor([[0.0, 3.0, 1.0, 2.0], [0.0, 1.0, 3.0, 2.0]], device=device)+ target = torch.tensor([[1], [3]], device=device)+ top1, top2 = accuracy(logits, target, topk=(1, 2))+ assert top1.item() == 50.0+ assert top2.item() == 100.0++ top1a, top1b = accuracy(logits, target, topk=(1, 1))+ assert top1a.item() == 50.0+ assert top1b.item() == 50.0++ target = torch.tensor([[1], [2]], device=device)+ top1a, top1b = accuracy(logits, target, topk=(1, 2))+ assert top1a.item() == 100.0assert top1b.item() == 100.0def test_topk_exceeds_num_classes_is_clipped(self):@@ -60,6 +80,78 @@ class TestAccuracy:result = accuracy(logits, target, topk=(5,))assert result[0].item() == 100.0+ @pytest.mark.parametrize(("pred_batch", "target_batch"), [(1, 2), (2, 1), (2, 3), (3, 2)])+ def test_mismatched_batch(self, pred_batch, target_batch):+ """Reject mismatched batch sizes between pred and target."""+ logits = torch.zeros(pred_batch, 3)+ target = torch.zeros(target_batch, dtype=torch.int64)+ with pytest.raises(ShapeError):+ accuracy(logits, target)++ def test_singleton_target_batch_mismatch_does_not_broadcast(self):+ """Ensure singleton target batch size is not silently broadcast across pred batch."""+ logits = torch.tensor([[0.0, 1.0], [0.0, 1.0]])+ target = torch.tensor([1])+ with pytest.raises(ShapeError):+ accuracy(logits, target)++ @pytest.mark.parametrize("pred_shape", [(3,), (2, 3, 4)])+ def test_invalid_pred_shape(self, pred_shape):+ """Reject pred tensors that are not 2-dimensional (B, C)."""+ logits = torch.zeros(pred_shape)+ target = torch.zeros(logits.shape[0] if logits.ndim > 0 else 1, dtype=torch.int64)+ with pytest.raises(ShapeError):+ accuracy(logits, target)++ @pytest.mark.parametrize("target_shape", [(2, 2), (1, 2), (2, 1, 2), ()])+ def test_invalid_target_shape(self, target_shape):+ """Reject target tensors that do not hold one class index per sample."""+ logits = torch.zeros(2, 3)+ target = torch.zeros(target_shape, dtype=torch.int64)+ with pytest.raises(ShapeError):+ accuracy(logits, target)++ def test_target_with_more_size_one_dims(self):+ """A (B, 1, 1) target is read like a (B,) one."""+ logits = torch.tensor([[0.0, 3.0, 1.0, 2.0], [0.0, 1.0, 3.0, 2.0]])+ target = torch.tensor([[[1]], [[3]]])+ assert torch.equal(torch.stack(accuracy(logits, target, topk=(1, 2))), torch.tensor([50.0, 100.0]))++ def test_non_tensor_inputs(self):+ """Reject non-tensor pred and target inputs."""+ with pytest.raises(TypeCheckError):+ accuracy([[0.0, 1.0]], torch.tensor([1])) # type: ignore[arg-type]+ with pytest.raises(TypeCheckError):+ accuracy(torch.tensor([[0.0, 1.0]]), [1]) # type: ignore[arg-type]++ @pytest.mark.parametrize(+ "invalid_topk",+ [(), (0,), (-1,), (1, 0), (1.5,), (True,), (2, True), (torch.tensor(True),), (2, torch.tensor(True))],+ )+ def test_invalid_topk(self, invalid_topk):+ """Reject empty, non-positive, or non-integer topk sequences."""+ logits = torch.tensor([[0.0, 1.0]])+ target = torch.tensor([1])+ with pytest.raises(BaseError, match="topk must be a non-empty sequence of positive integers"):+ accuracy(logits, target, topk=invalid_topk)++ @pytest.mark.parametrize(+ "topk",+ [[1, 2], (np.int64(1), np.int64(2)), (torch.tensor(1), torch.tensor(2))],+ ids=["list", "numpy", "tensor"],+ )+ def test_integer_like_topk(self, topk):+ logits = torch.tensor([[0.0, 1.0, 2.0], [2.0, 1.0, 0.0]])+ target = torch.tensor([1, 0])+ assert torch.equal(torch.stack(accuracy(logits, target, topk=topk)), torch.tensor([50.0, 100.0]))++ def test_device_mismatch(self):+ """Reject pred and target tensors located on different devices."""+ logits = torch.zeros(2, 3, device="cpu")+ target = torch.zeros(2, dtype=torch.int64, device="meta")+ with pytest.raises(DeviceError):+ accuracy(logits, target)+class TestConventionsAccuracy(BaseTester):def test_convention_accuracy_is_a_percentage_per_topk(self, device, dtype):