Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

7.0 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import annotations

from pathlib import Path

import pytest

from testing.known_failures import load_known_failures, mark_known_failures

pytest_plugins = ["pytester"]


def _write_manifest(directory: Path, contents: str) -> None:
    (directory / "mps_float32.txt").write_text(contents)


class TestLoadKnownFailures:
    def test_loads_nodeids_and_exception_types(self, tmp_path: Path) -> None:
        _write_manifest(
            tmp_path,
            "# tracked in kornia#1234\n"
            "AssertionError\ttests/a.py::test_a[mps-float32]\n"
            "RuntimeError\ttests/b.py::TestB::test_b[mps]\n",
        )

        failures, path = load_known_failures("mps", "float32", tmp_path)

        assert failures == {
            "tests/a.py::test_a[mps-float32]": AssertionError,
            "tests/b.py::TestB::test_b[mps]": RuntimeError,
        }
        assert path == tmp_path / "mps_float32.txt"

    def test_rejects_duplicate_nodeids(self, tmp_path: Path) -> None:
        _write_manifest(
            tmp_path,
            "AssertionError\ttests/a.py::test_a[mps-float32]\nRuntimeError\ttests/a.py::test_a[mps-float32]\n",
        )

        with pytest.raises(ValueError, match="duplicate node ID"):
            load_known_failures("mps", "float32", tmp_path)

    def test_rejects_unknown_exception_types(self, tmp_path: Path) -> None:
        _write_manifest(tmp_path, "UnknownError\ttests/a.py::test_a[mps-float32]\n")

        with pytest.raises(ValueError, match="unknown exception type"):
            load_known_failures("mps", "float32", tmp_path)

    def test_rejects_entries_for_the_wrong_device(self, tmp_path: Path) -> None:
        _write_manifest(tmp_path, "AssertionError\ttests/a.py::test_a[cpu-float32]\n")

        with pytest.raises(ValueError, match="does not select mps"):
            load_known_failures("mps", "float32", tmp_path)


class _Item:
    def __init__(self, nodeid: str) -> None:
        self.nodeid = nodeid
        self.markers: list[pytest.MarkDecorator] = []

    def add_marker(self, marker: pytest.MarkDecorator) -> None:
        self.markers.append(marker)


class TestMarkKnownFailures:
    def test_marks_exact_nodeid_strictly_with_expected_exception(self, tmp_path: Path) -> None:
        nodeid = "tests/a.py::test_a[mps-float32]"
        _write_manifest(tmp_path, f"AssertionError\t{nodeid}\n")
        item = _Item(nodeid)

        tracker = mark_known_failures([item], "mps", "float32", tmp_path)

        assert len(item.markers) == 1
        assert item.markers[0].mark.kwargs == {
            "raises": AssertionError,
            "reason": tracker.reason,
            "strict": True,
        }
        assert tracker.reason == "Known mps/float32 failure recorded in mps_float32.txt; see its tracking issue there"

    def test_rejects_manifest_entries_that_were_not_collected(self, tmp_path: Path) -> None:
        _write_manifest(tmp_path, "AssertionError\ttests/missing.py::test_missing[mps-float32]\n")

        with pytest.raises(ValueError, match=r"1 failures .* were not collected"):
            mark_known_failures([], "mps", "float32", tmp_path)


def _run_manifest_case(pytester: pytest.Pytester, test_body: str, extra_marker: str = "") -> pytest.RunResult:
    nodeid = "test_sample.py::test_known_failure[mps-float32]"
    _write_manifest(pytester.path, f"RuntimeError\t{nodeid}\n")
    pytester.makeconftest(
        """
        from pathlib import Path

        from testing.known_failures import mark_known_failures


        def pytest_collection_modifyitems(config, items):
            tracker = mark_known_failures(items, "mps", "float32", Path(__file__).parent)
            config.pluginmanager.register(tracker, "known-failure-tracker")
        """
    )
    pytester.makepyfile(
        test_sample=f"""
        import pytest


        {extra_marker}
        @pytest.mark.parametrize("unused", [None], ids=["mps-float32"])
        def test_known_failure(unused, request):
            {test_body}
        """
    )
    return pytester.runpytest("-q")


class TestKnownFailureOutcomes:
    def test_accepts_exact_manifest_xfail(self, pytester: pytest.Pytester) -> None:
        result = _run_manifest_case(pytester, "raise RuntimeError('known failure')")

        result.assert_outcomes(xfailed=1)
        assert result.ret == pytest.ExitCode.OK

    @pytest.mark.parametrize(
        ("test_body", "extra_marker"),
        [
            ("pass", ""),
            ("raise AssertionError('wrong failure type')", ""),
            ("raise NotImplementedError('subclass of RuntimeError')", ""),
            ("pytest.skip('disabled test')", ""),
            (
                "raise RuntimeError('known failure')",
                "@pytest.mark.skip(reason='disabled test')",
            ),
            (
                "raise RuntimeError('known failure')",
                "@pytest.mark.xfail(reason='different xfail marker', strict=False)",
            ),
            (
                (
                    "request.addfinalizer(lambda: pytest.skip('disabled in teardown')); "
                    "raise RuntimeError('known failure')"
                ),
                "",
            ),
            (
                "raise RuntimeError('known failure')",
                (
                    "@pytest.fixture(autouse=True)\n"
                    "        def fail_setup():\n"
                    "            raise RuntimeError('setup failure')"
                ),
            ),
        ],
        ids=[
            "fixed",
            "wrong-exception",
            "exception-subclass",
            "runtime-skip",
            "static-skip",
            "prior-xfail",
            "teardown-skip",
            "setup-failure",
        ],
    )
    def test_rejects_changed_or_bypassed_outcome(
        self, pytester: pytest.Pytester, test_body: str, extra_marker: str
    ) -> None:
        result = _run_manifest_case(pytester, test_body, extra_marker)

        assert result.ret == pytest.ExitCode.TESTS_FAILED
        output = result.stdout.str()
        assert "ERROR: 1 known failure did not reproduce exactly" in output
        assert "test_sample.py::test_known_failure[mps-float32]" in output


def test_recorded_mps_float32_baseline() -> None:
    failures, path = load_known_failures("mps", "float32")

    # The exact count changes with every fix that removes a manifest line, down to none; the contract is the file,
    # not a number.
    assert path.name == "mps_float32.txt"
    assert all(issubclass(exception, BaseException) for exception in failures.values())