Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

10.0 KB
Raw Blame
"""GPU-free tests for AcceleratorError-as-OOM classification (issue #1002).

#989 classified an OOM arriving as AcceleratorError as a result (oom=True) rather
than an instrument failure (failed=True). However, that coverage was gated behind
@requires_cuda in tests/test_issue901_failed_page_lock_recovery.py, causing it to
skip across all CI matrix cells without a GPU runner.

The classifier (_is_out_of_memory, delegating to batch_probe._is_cuda_oom) and
the outcome mapping (_classify_probe_exception) are pure functions of the exception
type and message. These tests exercise the classifier, the three-outcome StepPeak
mapping, and the anti-forking predicate delegation directly without requiring a GPU.
"""

from __future__ import annotations

import pytest

from soup_cli.utils.layer_stream_runtime import (
    StepPeak,
    _classify_probe_exception,
    _is_out_of_memory,
    measure_step_peak_bytes,
)


def _get_torch_and_accelerator_error():
    """Import torch and return (torch, AcceleratorError) or skip if AcceleratorError is absent.

    torch.AcceleratorError was introduced in torch 2.8; on older versions (declared
    floor is 2.6) this skips with a version note, not a GPU note.
    """
    torch = pytest.importorskip("torch", reason="torch is not installed in this environment")
    acc = getattr(torch, "AcceleratorError", None)
    if acc is None:
        pytest.skip(
            f"torch {torch.__version__} predates torch.AcceleratorError "
            "(added in 2.8); nothing to classify here"
        )
    return torch, acc


class TestIsOutOfMemoryClassification:
    """Direct GPU-free classification tests for _is_out_of_memory."""

    def test_torch_cuda_out_of_memory_error_is_oom(self):
        torch = pytest.importorskip("torch", reason="torch is not installed in this environment")
        oom_cls = getattr(torch.cuda, "OutOfMemoryError", None)
        if oom_cls is None:
            pytest.skip("torch.cuda.OutOfMemoryError is not available in this torch")
        exc = oom_cls("CUDA out of memory. Tried to allocate 2.00 GiB")
        assert _is_out_of_memory(exc) is True

    def test_accelerator_error_out_of_memory_is_oom(self):
        _, accelerator_error = _get_torch_and_accelerator_error()
        exc = accelerator_error("CUDA error: out of memory")
        assert _is_out_of_memory(exc) is True

    def test_accelerator_error_illegal_access_is_not_oom(self):
        _, accelerator_error = _get_torch_and_accelerator_error()
        exc = accelerator_error("CUDA error: an illegal memory access was encountered")
        assert _is_out_of_memory(exc) is False

    def test_bare_runtime_error_out_of_memory_is_oom(self):
        exc = RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB (GPU 0; ...)")
        assert _is_out_of_memory(exc) is True

    def test_bare_runtime_error_non_oom_is_not_oom(self):
        assert _is_out_of_memory(RuntimeError("device-side assert triggered")) is False
        assert _is_out_of_memory(RuntimeError("an illegal memory access was encountered")) is False

    def test_non_runtime_error_is_not_oom(self):
        """Exceptions outside RuntimeError/OutOfMemoryError must never classify as OOM."""
        assert _is_out_of_memory(ValueError("CUDA out of memory")) is False
        assert _is_out_of_memory(KeyError("out of memory")) is False
        assert _is_out_of_memory(TypeError("out of memory")) is False
        assert _is_out_of_memory(Exception("out of memory")) is False


class TestClassifyProbeExceptionOutcomes:
    """GPU-free tests for the three-outcome mapping in _classify_probe_exception."""

    def test_oom_accelerator_error_yields_oom_result(self):
        _, accelerator_error = _get_torch_and_accelerator_error()
        exc = accelerator_error("CUDA error: out of memory")
        peak = _classify_probe_exception(exc, rows=2, seq_len=16, seconds=0.42)

        assert isinstance(peak, StepPeak)
        assert peak.oom is True
        assert peak.failed is False
        assert peak.error is None
        assert peak.peak_bytes == 0
        assert peak.reserved_bytes == 0
        assert peak.rows == 2
        assert peak.seq_len == 16
        assert peak.seconds == 0.42

    def test_torch_cuda_oom_error_yields_oom_result(self):
        torch = pytest.importorskip("torch", reason="torch is not installed in this environment")
        oom_cls = getattr(torch.cuda, "OutOfMemoryError", None)
        if oom_cls is None:
            pytest.skip("torch.cuda.OutOfMemoryError is not available in this torch")
        exc = oom_cls("CUDA out of memory")
        peak = _classify_probe_exception(exc, rows=1, seq_len=8, seconds=0.1)

        assert peak.oom is True
        assert peak.failed is False
        assert peak.error is None
        assert peak.peak_bytes == 0
        assert peak.reserved_bytes == 0
        assert peak.rows == 1
        assert peak.seq_len == 8

    def test_non_oom_accelerator_error_yields_instrument_failure(self):
        _, accelerator_error = _get_torch_and_accelerator_error()
        exc = accelerator_error("CUDA error: an illegal memory access was encountered")
        peak = _classify_probe_exception(exc, rows=2, seq_len=16, seconds=0.25)

        assert isinstance(peak, StepPeak)
        assert peak.oom is False
        assert peak.failed is True
        assert peak.error == "AcceleratorError"
        assert peak.peak_bytes == 0
        assert peak.reserved_bytes == 0
        assert peak.rows == 2
        assert peak.seq_len == 16

    def test_generic_runtime_error_yields_instrument_failure(self):
        exc = RuntimeError("CUDA error: device-side assert triggered")
        peak = _classify_probe_exception(exc, rows=1, seq_len=4, seconds=0.05)

        assert peak.oom is False
        assert peak.failed is True
        assert peak.error == "RuntimeError"
        assert peak.peak_bytes == 0
        assert peak.reserved_bytes == 0


class TestPredicateIdentityAndAntiForking:
    """Pin that _is_out_of_memory delegates to batch_probe._is_cuda_oom."""

    def test_is_out_of_memory_delegates_to_batch_probe_predicate(self, monkeypatch):
        """Pin that _is_out_of_memory calls batch_probe._is_cuda_oom directly.

        A third spelling cannot appear: _is_out_of_memory must delegate to
        batch_probe._is_cuda_oom rather than forking a divergent copy (precedent:
        test_tracker_and_execution_share_one_primitive in
        test_issue424_windows_liveness.py).
        """
        torch = pytest.importorskip("torch", reason="torch is not installed in this environment")

        calls: list[tuple[BaseException, object]] = []
        sentinel = object()

        def spy_is_cuda_oom(exc: BaseException, torch_arg: object) -> object:
            calls.append((exc, torch_arg))
            return sentinel

        monkeypatch.setattr("soup_cli.utils.batch_probe._is_cuda_oom", spy_is_cuda_oom)

        exc = RuntimeError("test exception")
        result = _is_out_of_memory(exc)

        assert result is sentinel, "_is_out_of_memory must return the verdict from _is_cuda_oom"
        assert len(calls) == 1, "_is_out_of_memory must call batch_probe._is_cuda_oom exactly once"
        assert calls[0][0] is exc
        assert calls[0][1] is torch

    @pytest.mark.parametrize(
        "exc",
        [
            RuntimeError("CUDA out of memory. Tried to allocate 1024 bytes"),
            RuntimeError("CUDA error: out of memory"),
            RuntimeError("CUDA error: an illegal memory access was encountered"),
            RuntimeError("device-side assert triggered"),
            ValueError("CUDA out of memory"),
            TypeError("invalid argument"),
        ],
    )
    def test_is_out_of_memory_agrees_with_batch_probe_predicate(self, exc):
        """Verify output identity between _is_out_of_memory and batch_probe._is_cuda_oom."""
        torch = pytest.importorskip("torch", reason="torch is not installed in this environment")
        from soup_cli.utils.batch_probe import _is_cuda_oom

        assert _is_out_of_memory(exc) == _is_cuda_oom(exc, torch)


class TestMeasureStepPeakBytesOutcomeMappingGpuFree:
    """Pin that measure_step_peak_bytes delegates to the classifier without a GPU.

    Pins the call site at measure_step_peak_bytes:1792 so a mutation bypassing
    _classify_probe_exception fails in GPU-free CI (#1002 criterion 2).
    """

    def test_measure_step_peak_bytes_oom_accelerator_error(self, monkeypatch):
        torch, accelerator_error = _get_torch_and_accelerator_error()

        monkeypatch.setattr(torch.cuda, "is_available", lambda: True)

        def raise_oom(*args, **kwargs):
            raise accelerator_error("CUDA error: out of memory")

        monkeypatch.setattr(torch.cuda, "synchronize", raise_oom)
        monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream_runtime._zero_probe_grads", lambda *_: None
        )

        peak = measure_step_peak_bytes(object(), rows=1, seq_len=4, vocab_size=8)
        assert peak is not None
        assert peak.oom is True
        assert peak.failed is False
        assert peak.error is None
        assert peak.rows == 1
        assert peak.seq_len == 4

    def test_measure_step_peak_bytes_illegal_access_accelerator_error(self, monkeypatch):
        torch, accelerator_error = _get_torch_and_accelerator_error()

        monkeypatch.setattr(torch.cuda, "is_available", lambda: True)

        def raise_illegal_access(*args, **kwargs):
            raise accelerator_error("CUDA error: an illegal memory access was encountered")

        monkeypatch.setattr(torch.cuda, "synchronize", raise_illegal_access)
        monkeypatch.setattr(torch.cuda, "empty_cache", lambda: None)
        monkeypatch.setattr(
            "soup_cli.utils.layer_stream_runtime._zero_probe_grads", lambda *_: None
        )

        peak = measure_step_peak_bytes(object(), rows=1, seq_len=4, vocab_size=8)
        assert peak is not None
        assert peak.oom is False
        assert peak.failed is True
        assert peak.error == "AcceleratorError"
        assert peak.rows == 1
        assert peak.seq_len == 4