Skip to content

🐍 Geometric Computer Vision Library for Spatial AI

24.9 KB
Raw Blame
# LICENSE HEADER MANAGED BY add-license-header
#
# Copyright 2018 Kornia Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

from __future__ import annotations

import contextlib
import hashlib
import os
import platform
import shutil
import subprocess
import sys
import tempfile
import warnings
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal, Mapping, Sequence, cast

import numpy as np
import pytest
import torch

from testing.half_precision_eager_rng import AuditedEagerRngCall, eager_rng_calls_for_node

ISSUE_URL = "https://github.com/kornia/kornia/issues/4153"
_MANIFEST_DIR = Path(__file__).with_name("half_precision_xfails")
_XFAIL_REASON_PREFIX = f"Known Linux CPU half-precision failure tracked in {ISSUE_URL}"
_SCHEMA_VERSION = "1"
_SEED_SCHEME = "sha256-nodeid-v1"
_ALLOWED_PHASES = {"setup", "call"}


@dataclass(frozen=True)
class ManifestProfile:
    """Stable configuration for one Linux CPU half-precision baseline."""

    name: str
    dtype: str
    manifest_path: Path


@dataclass(frozen=True)
class ManifestEntry:
    """One exact failure outcome accepted by a manifest."""

    phase: Literal["setup", "call"]
    exception: str
    nodeid: str


@dataclass(frozen=True)
class ManifestEnvironment:
    """Compatibility and provenance fields recorded in a manifest header."""

    python: str
    pytorch: str
    os: str
    arch: str
    pytest: str
    numpy: str
    pillow: str


_PROFILES = {
    name: ManifestProfile(name, dtype, _MANIFEST_DIR / f"cpu_{dtype}.txt")
    for name, dtype in (("cpu-float16", "float16"), ("cpu-bfloat16", "bfloat16"))
}


def get_profile(name: str) -> ManifestProfile:
    """Return a supported CPU-half manifest profile."""
    try:
        return _PROFILES[name]
    except KeyError as error:
        known = ", ".join(sorted(_PROFILES))
        raise ValueError(f"unknown profile {name!r}; known profiles are: {known}") from error


def current_environment() -> ManifestEnvironment:
    """Capture enforced compatibility and warning-only provenance metadata."""
    try:
        from PIL import __version__ as pillow_version
    except ImportError:
        pillow_version = "not-installed"
    return ManifestEnvironment(
        python=f"{sys.version_info.major}.{sys.version_info.minor}",
        pytorch=torch.__version__.partition("+")[0],
        os=platform.system(),
        arch=platform.machine(),
        pytest=pytest.__version__,
        numpy=np.__version__,
        pillow=pillow_version,
    )


def serialize_manifest(
    profile: ManifestProfile,
    entries: Sequence[ManifestEntry],
    environment: ManifestEnvironment,
    generation_command: str,
) -> str:
    """Serialize a canonical profile manifest without resolving exception classes."""
    header = {
        "known-failure-schema": _SCHEMA_VERSION,
        "profile": profile.name,
        "python": environment.python,
        "pytorch": environment.pytorch,
        "os": environment.os,
        "arch": environment.arch,
        "seed-scheme": _SEED_SCHEME,
        "pytest": environment.pytest,
        "numpy": environment.numpy,
        "pillow": environment.pillow,
        "generated-by": generation_command,
    }
    lines = [*(f"# {key}: {value}" for key, value in header.items()), ""]
    lines.extend(
        f"{entry.phase}\t{entry.exception}\t{entry.nodeid}" for entry in sorted(entries, key=lambda x: x.nodeid)
    )
    return "\n".join([*lines, ""])


def parse_manifest(
    profile: ManifestProfile,
    text: str,
    environment: ManifestEnvironment,
    *,
    source: Path | None = None,
    enforce_compatibility: bool = True,
) -> dict[str, ManifestEntry]:
    """Parse one manifest without importing exceptions, optionally warning on environment drift."""
    label = str(source) if source is not None else "manifest"
    header: dict[str, str] = {}
    entries: dict[str, ManifestEntry] = {}
    for line_number, line in enumerate(text.splitlines(), start=1):
        if not line:
            continue
        if line.startswith("# ") and ": " in line:
            key, value = line[2:].split(": ", maxsplit=1)
            header[key] = value
            continue
        if line.startswith("#"):
            continue
        fields = line.split("\t")
        if len(fields) != 3:
            raise ValueError(f"{label}:{line_number}: expected '<phase>\\t<exception>\\t<node ID>'")
        phase, exception, nodeid = fields
        if phase not in _ALLOWED_PHASES:
            raise ValueError(f"{label}:{line_number}: unsupported phase: {phase!r}")
        if not exception or not nodeid:
            raise ValueError(f"{label}:{line_number}: exception identity and node ID must be non-empty")
        if nodeid in entries:
            raise ValueError(f"{label}:{line_number}: duplicate node ID: {nodeid}")
        entries[nodeid] = ManifestEntry(cast(Literal["setup", "call"], phase), exception, nodeid)

    identity = {
        "known-failure-schema": _SCHEMA_VERSION,
        "profile": profile.name,
        "seed-scheme": _SEED_SCHEME,
    }
    for key, expected in identity.items():
        observed = header.get(key)
        if observed != expected:
            raise ValueError(f"{label}: {key} mismatch: expected {expected!r}, found {observed!r}")

    for key, expected in {
        "python": environment.python,
        "pytorch": environment.pytorch,
        "os": environment.os,
        "arch": environment.arch,
    }.items():
        observed = header.get(key)
        if observed == expected:
            continue
        display = "Python" if key == "python" else key
        message = f"{label}: {display} mismatch: expected {expected!r}, found {observed!r}"
        if enforce_compatibility:
            raise ValueError(message)
        warnings.warn(f"{display} compatibility differs; {message}", stacklevel=2)

    for key, expected in {
        "pytest": environment.pytest,
        "numpy": environment.numpy,
        "pillow": environment.pillow,
    }.items():
        observed = header.get(key)
        if observed != expected:
            warnings.warn(
                f"{label}: {key} provenance differs: recorded {observed!r}, running {expected!r}", stacklevel=2
            )
    return entries


def seed_test_rng(nodeid: str) -> int:
    """Derive a stable per-node seed without mutating any process-global RNG."""
    return int.from_bytes(hashlib.sha256(f"v1\0{nodeid}".encode()).digest()[:4], "big")


def _canonical_exception_identity(exception_type: type[BaseException]) -> str:
    return f"{exception_type.__module__}.{exception_type.__qualname__}"


def validate_record_destination(output_path: Path, rootpath: Path) -> Path:
    """Resolve a candidate destination and reject checked-in or Git-tracked aliases."""
    root = rootpath.resolve()
    unresolved = output_path if output_path.is_absolute() else root / output_path
    destination = unresolved.resolve()
    manifest_dir = _MANIFEST_DIR.resolve()
    if destination == manifest_dir or destination.is_relative_to(manifest_dir):
        raise ValueError(f"record destination resolves inside the checked-in manifest directory: {destination}")
    if destination.is_relative_to(root):
        git = shutil.which("git")
        if git is None:
            raise OSError("cannot validate the candidate destination because Git is unavailable")
        repository = subprocess.run(  # noqa: S603 - all arguments are fixed or resolved local paths.
            [git, "-C", str(root), "rev-parse", "--show-toplevel"],
            check=False,
            stdout=subprocess.PIPE,
            stderr=subprocess.DEVNULL,
            text=True,
        )
        if repository.returncode == 0 and Path(repository.stdout.strip()).resolve() == root:
            relative = destination.relative_to(root)
            tracked = subprocess.run(  # noqa: S603 - -- terminates options before the resolved relative path.
                [git, "-C", str(root), "ls-files", "--error-unmatch", "--", relative.as_posix()],
                check=False,
                stdout=subprocess.DEVNULL,
                stderr=subprocess.DEVNULL,
            )
            if tracked.returncode == 0:
                raise ValueError(f"record destination is Git-tracked: {destination}")
            if tracked.returncode != 1:
                raise OSError(f"could not determine whether record destination is Git-tracked: {destination}")
    if not destination.parent.is_dir():
        raise ValueError(f"record destination parent does not exist: {destination.parent}")
    return destination


def _previous_manifest_entries(profile: ManifestProfile) -> dict[str, ManifestEntry]:
    """Load the checked-in v1 entries used to categorize a candidate delta."""
    text = profile.manifest_path.read_text(encoding="utf-8")
    return parse_manifest(
        profile, text, current_environment(), source=profile.manifest_path, enforce_compatibility=False
    )


class FailureRecorder:
    """Write a candidate only after proving that the full pytest session completed."""

    def __init__(
        self,
        profile: ManifestProfile,
        output_path: Path,
        *,
        rootpath: Path,
        previous_entries: Mapping[str, ManifestEntry] | None = None,
    ) -> None:
        self.profile = profile
        self.output_path = validate_record_destination(output_path, rootpath)
        self.previous_entries = (
            dict(previous_entries) if previous_entries is not None else _previous_manifest_entries(profile)
        )
        self.entries: dict[str, ManifestEntry] = {}
        self.collected: set[str] = set()
        self.finished: set[str] = set()
        self.reports: dict[str, list[tuple[str, str, str | None]]] = defaultdict(list)
        self.collection_finished = False
        self.collection_failed = False
        self.interrupted = False
        self.abort_reasons: list[str] = []
        self.eager_blockers: dict[str, tuple[AuditedEagerRngCall, ...]] = {}
        self.categories: dict[str, str] = {}
        self.written = False

    @pytest.hookimpl(tryfirst=True)
    def pytest_collectreport(self, report: Any) -> None:
        if report.failed:
            self.collection_failed = True

    def pytest_collection_finish(self, session: pytest.Session) -> None:
        self.collection_finished = True
        self.collected = {item.nodeid for item in session.items}

    def pytest_runtest_logfinish(self, nodeid: str) -> None:
        self.finished.add(nodeid)

    def pytest_keyboard_interrupt(self) -> None:
        self.interrupted = True

    @pytest.hookimpl(hookwrapper=True, tryfirst=True)
    def pytest_runtest_makereport(self, item: Any, call: Any) -> Any:
        """Capture exact unhandled setup/call failures and all lifecycle outcomes."""
        outcome = yield
        report = outcome.get_result()
        wasxfail = getattr(report, "wasxfail", None)
        self.reports[report.nodeid].append((report.when, report.outcome, wasxfail))
        if (
            report.when in _ALLOWED_PHASES
            and report.outcome == "failed"
            and wasxfail is None
            and call.excinfo is not None
        ):
            entry = ManifestEntry(report.when, _canonical_exception_identity(call.excinfo.type), report.nodeid)
            existing = self.entries.setdefault(report.nodeid, entry)
            if existing != entry:
                self.abort_reasons.append(f"multiple representable failures for {report.nodeid}")
            blockers = eager_rng_calls_for_node(report.nodeid)
            if blockers:
                self.eager_blockers[report.nodeid] = blockers
        if (
            report.when == "teardown"
            and report.outcome != "passed"
            and (report.nodeid in self.entries or report.nodeid in self.previous_entries)
        ):
            self.abort_reasons.append(f"unrepresentable teardown failure for {report.nodeid}")

    def _check_completeness(self, session: pytest.Session) -> None:
        if session.config.option.collectonly:
            self.abort_reasons.append("collect-only execution cannot produce a candidate")
        if not self.collection_finished or self.collection_failed:
            self.abort_reasons.append("collection did not finish successfully")
        if not self.collected:
            self.abort_reasons.append("the canonical suite collected no tests")
        unfinished = self.collected - self.finished
        if unfinished:
            self.abort_reasons.append(f"{len(unfinished)} collected test(s) never reached log finish")
        if self.interrupted:
            self.abort_reasons.append("execution was interrupted")
        if session.exitstatus not in {pytest.ExitCode.OK, pytest.ExitCode.TESTS_FAILED}:
            self.abort_reasons.append(f"pytest stopped with incomplete exit status {session.exitstatus}")
        for nodeid, blockers in sorted(self.eager_blockers.items()):
            sites = ", ".join(f"{entry.call.path}:{entry.call.line} ({entry.call.name})" for entry in blockers)
            self.abort_reasons.append(
                f"eager RNG feeds candidate node {nodeid}: {sites}; move the draw into the test or a seeded fixture"
            )

    def _categorize_previous_entries(self) -> None:
        for nodeid, previous in self.previous_entries.items():
            candidate = self.entries.get(nodeid)
            if candidate is not None:
                self.categories[nodeid] = "reproduced" if candidate == previous else "changed-outcome"
            elif nodeid not in self.collected:
                self.categories[nodeid] = "deleted-or-renamed"
            elif nodeid not in self.finished:
                self.categories[nodeid] = "incomplete"
            elif any(wasxfail is not None for _, _, wasxfail in self.reports[nodeid]):
                self.categories[nodeid] = "competing-xfail"
            elif any(outcome == "skipped" for _, outcome, _ in self.reports[nodeid]):
                self.categories[nodeid] = "skipped"
            else:
                self.categories[nodeid] = "passed"

    def _write_candidate(self) -> None:
        generation_command = (
            f"pytest tests/ --record-known-failures=<candidate> --known-failure-profile={self.profile.name}"
        )
        contents = serialize_manifest(
            self.profile, tuple(self.entries.values()), current_environment(), generation_command
        )
        parse_manifest(self.profile, contents, current_environment(), source=self.output_path)
        temporary_path: Path | None = None
        try:
            with tempfile.NamedTemporaryFile(
                mode="w",
                encoding="utf-8",
                dir=self.output_path.parent,
                prefix=f".{self.output_path.name}.",
                delete=False,
            ) as temporary:
                temporary.write(contents)
                temporary.flush()
                os.fsync(temporary.fileno())
                temporary_path = Path(temporary.name)
            os.replace(temporary_path, self.output_path)
            self.written = True
        finally:
            if temporary_path is not None and temporary_path.exists():
                temporary_path.unlink()

    def pytest_sessionfinish(self, session: pytest.Session) -> None:
        """Serialize atomically only when completeness and lifecycle checks succeed."""
        self._check_completeness(session)
        self._categorize_previous_entries()
        if not self.abort_reasons:
            try:
                self._write_candidate()
            except Exception as error:  # noqa: BLE001 - every writer failure must preserve the previous candidate.
                self.abort_reasons.append(f"candidate serialization failed: {error}")
        if self.abort_reasons and session.exitstatus == pytest.ExitCode.OK:
            session.exitstatus = pytest.ExitCode.TESTS_FAILED

    def pytest_terminal_summary(self, terminalreporter: Any) -> None:
        """Print a receipt or an actionable abort report independently of pytest's test count."""
        if self.abort_reasons:
            terminalreporter.write_line(
                f"known-failure candidate ABORTED for {self.profile.name}; preserved {self.output_path}", red=True
            )
            for reason in dict.fromkeys(self.abort_reasons):
                terminalreporter.write_line(f"  {reason}")
            return
        if not self.written:
            return
        terminalreporter.write_line(
            f"known-failure candidate complete for {self.profile.name}: "
            f"{len(self.entries)} entries at {self.output_path}"
        )
        for nodeid, category in sorted(self.categories.items()):
            if category != "reproduced":
                terminalreporter.write_line(f"OLD {category}: {nodeid}")
        for nodeid in sorted(set(self.entries) - set(self.previous_entries)):
            entry = self.entries[nodeid]
            terminalreporter.write_line(f"NEW: {entry.phase}\t{entry.exception}\t{entry.nodeid}")


class KnownFailureTracker:
    """Require every manifest entry to reproduce its exact phase and exception identity."""

    def __init__(
        self,
        profile: ManifestProfile,
        manifest_path: Path,
        *,
        mode: Literal["focus", "complete"] = "focus",
        selectors: Sequence[str] | None = None,
        rootpath: Path | None = None,
    ) -> None:
        self.mode = mode
        self.selectors = tuple(selectors or ())
        self.rootpath = rootpath
        self.reason = _XFAIL_REASON_PREFIX
        self._collect_only = False
        self._collection_failed = False
        self._matched: set[str] = set()
        self._invalid: set[str] = set()
        self._finished_with_unexpected_outcome: set[str] = set()
        self._observed: dict[str, list[str]] = {}
        self.profile = profile
        self.manifest_paths = (manifest_path,)
        entries = parse_manifest(
            profile,
            manifest_path.read_text(encoding="utf-8"),
            current_environment(),
            source=manifest_path,
            enforce_compatibility=self.mode == "complete",
        )
        if self.mode == "focus" and self.selectors:
            entries = {
                nodeid: entry
                for nodeid, entry in entries.items()
                if any(_matches_selector(nodeid, selector, self.rootpath) for selector in self.selectors)
            }
        self._expected_entries = entries
        self.pending = set(self._expected_entries)

    def pytest_sessionstart(self, session: pytest.Session) -> None:
        """Disable runtime-outcome validation during collection-only runs."""
        self._collect_only = session.config.getoption("collectonly")

    @pytest.hookimpl(tryfirst=True)
    def pytest_collectreport(self, report: Any) -> None:
        """Remember collection failure so stale-manifest advice cannot mask it."""
        if report.failed:
            self._collection_failed = True

    @pytest.hookimpl(tryfirst=True)
    def pytest_collection_modifyitems(self, items: Sequence[Any]) -> None:
        """Apply the manifest marker after collection without inferring configuration from node IDs."""
        for item in items:
            if item.nodeid in self._expected_entries:
                item.add_marker(pytest.mark.xfail(reason=self.reason, strict=True))

    @pytest.hookimpl(hookwrapper=True, tryfirst=True)
    def pytest_runtest_makereport(self, item: Any, call: Any) -> Any:
        """Observe all phases; pytest's xfail result alone is not the exact-outcome authority."""
        outcome = yield
        report = outcome.get_result()
        if report.nodeid not in self.pending:
            return
        wasxfail = getattr(report, "wasxfail", None)
        expected = self._expected_entries[report.nodeid]
        exception_type = call.excinfo.type if call.excinfo is not None else None

        if report.when in {"setup", "call"} and report.outcome == "skipped" and wasxfail == self.reason:
            if exception_type is None:
                observed = f"{report.when} xfail without an exception"
            else:
                identity = _canonical_exception_identity(exception_type)
                observed = f"{report.when} {identity}"
                exact_match = report.when == expected.phase and identity == expected.exception
                if exact_match:
                    self._matched.add(report.nodeid)
                    return
            self._observed.setdefault(report.nodeid, []).append(observed)
            self._invalid.add(report.nodeid)
        elif report.when == "teardown" and report.outcome != "passed":
            self._observed.setdefault(report.nodeid, []).append(f"teardown {report.outcome}")
            self._invalid.add(report.nodeid)
        elif report.outcome == "skipped":
            category = "competing xfail" if wasxfail is not None else "skip"
            self._observed.setdefault(report.nodeid, []).append(f"{report.when} {category}")
            self._invalid.add(report.nodeid)
        elif report.outcome == "failed" or (wasxfail is not None and report.outcome != "passed"):
            identity = _canonical_exception_identity(exception_type) if exception_type is not None else "strict XPASS"
            self._observed.setdefault(report.nodeid, []).append(f"{report.when} {identity}")
            self._invalid.add(report.nodeid)

    def pytest_runtest_logfinish(self, nodeid: str) -> None:
        """Accept an entry only after all setup, call, and teardown reports are known."""
        if nodeid in self._matched and nodeid not in self._invalid:
            self.pending.remove(nodeid)
        elif nodeid in self.pending:
            self._finished_with_unexpected_outcome.add(nodeid)
        self._matched.discard(nodeid)
        self._invalid.discard(nodeid)

    def pytest_deselected(self, items: Sequence[Any]) -> None:
        """Exclude locally deselected entries from outcome validation."""
        if self.mode == "focus":
            for item in items:
                self.pending.discard(item.nodeid)

    def pytest_sessionfinish(self, session: pytest.Session) -> None:
        """Fail the session when a manifest entry was skipped or handled by another xfail."""
        if (
            not self._collect_only
            and self.pending
            and not self._collection_failed
            and session.exitstatus == pytest.ExitCode.OK
        ):
            session.exitstatus = pytest.ExitCode.TESTS_FAILED

    def pytest_terminal_summary(self, terminalreporter: Any) -> None:
        """Report manifest entries that did not produce their required xfail outcome."""
        if self._collect_only or not self.pending or self._collection_failed:
            return
        count = len(self.pending)
        noun = "failure" if count == 1 else "failures"
        manifests = ", ".join(str(path) for path in self.manifest_paths)
        verb = "needs" if count == 1 else "need"
        terminalreporter.write_line(f"ERROR: {count} known half-precision {noun} {verb} updates in {manifests}:")
        unexpected = self.pending & self._finished_with_unexpected_outcome
        not_run = self.pending - unexpected
        if unexpected:
            terminalreporter.write_line("Ran with an unexpected outcome:")
            for nodeid in sorted(unexpected):
                expected = self._expected_entries[nodeid]
                terminalreporter.write_line(f"{nodeid}: expected {expected.phase} {expected.exception}")
                for observed in self._observed.get(nodeid, ["pass or incomplete lifecycle"]):
                    terminalreporter.write_line(f"  observed {observed}")
                terminalreporter.write_line(
                    f"  remove or update: {expected.phase}\t{expected.exception}\t{expected.nodeid}"
                )
        if not_run:
            terminalreporter.write_line("Did not run; rerun without selection or remove/update a stale manifest line:")
            for nodeid in sorted(not_run):
                terminalreporter.write_line(nodeid)


def _normalize_selector(selector: str, rootpath: Path | None = None) -> str:
    path, separator, node = selector.partition("::")
    selector_path = Path(path)
    if selector_path.is_absolute() and rootpath is not None:
        with contextlib.suppress(ValueError):
            selector_path = selector_path.relative_to(rootpath)
    normalized_path = selector_path.as_posix().removeprefix("./").rstrip("/")
    return f"{normalized_path}::{node}" if separator else normalized_path


def _matches_selector(nodeid: str, selector: str, rootpath: Path | None = None) -> bool:
    selector = _normalize_selector(selector, rootpath)
    node_path = nodeid.partition("::")[0]
    if "::" in selector:
        return nodeid == selector or nodeid.startswith((f"{selector}[", f"{selector}::"))
    return node_path == selector or node_path.startswith(f"{selector}/")