Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

11.0 KB
Raw Blame
"""Tests for friendly error handling and --verbose flag."""

from io import StringIO
from unittest.mock import patch

import pytest
from rich.console import Console
from typer.testing import CliRunner

from soup_cli.cli import app
from soup_cli.utils.errors import format_friendly_error
from tests.conftest import strip_ansi

runner = CliRunner()


# --- format_friendly_error tests ---


def test_cuda_oom_error():
    """CUDA OOM gets a friendly message."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "--batch-size" in output
    assert "GPU ran out of memory" in output
    # v0.53.4 #11 — message now suggests --batch-size / --grad-accum CLI flags.
    assert "gradient_checkpointing" in output
    assert "4bit" in output
    assert "data.max_length" in output


def test_missing_fastapi_error():
    """Missing fastapi gives install hint."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = ModuleNotFoundError("No module named 'fastapi'")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "soup-cli[serve]" in output


def test_missing_datasketch_error():
    """Missing datasketch gives install hint."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = ModuleNotFoundError("No module named 'datasketch'")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "soup-cli[data]" in output


def test_missing_wandb_error():
    """Missing wandb gives install hint."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = ModuleNotFoundError("No module named 'wandb'")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "pip install wandb" in output


def test_missing_deepspeed_error():
    """Missing deepspeed gives install hint."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = ModuleNotFoundError("No module named 'deepspeed'")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "soup-cli[deepspeed]" in output


@pytest.mark.parametrize(
    "module",
    ["torch", "transformers", "peft", "trl", "datasets", "bitsandbytes", "accelerate"],
)
def test_missing_train_dep_points_at_train_extra(module):
    """v0.71.0 — any missing heavy training dep points at the [train] extra."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = ModuleNotFoundError(f"No module named '{module}'")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    # The `\[train]` escape renders to a literal `[train]` in the output.
    assert "soup-cli[train]" in output
    assert "[train] extra" in output


def test_connection_error():
    """Connection error gets friendly message."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = ConnectionError("Failed to connect")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "Network connection failed" in output


def test_unknown_error_shows_type():
    """Unknown errors show the exception type and message."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = ZeroDivisionError("division by zero")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "ZeroDivisionError" in output
    assert "division by zero" in output
    assert "--verbose" in output


def test_verbose_shows_traceback():
    """Verbose mode shows full traceback."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        try:
            raise RuntimeError("CUDA out of memory. Tried to allocate")
        except RuntimeError as exc:
            format_friendly_error(exc, verbose=True)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "GPU ran out of memory" in output
    assert "Traceback" in output or "RuntimeError" in output


def test_verbose_unknown_error():
    """Verbose mode for unknown errors shows traceback."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        try:
            raise ValueError("something weird")
        except ValueError as exc:
            format_friendly_error(exc, verbose=True)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "ValueError" in output
    assert "something weird" in output


def test_yaml_error():
    """YAML syntax error gets friendly message."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = Exception("yaml.scanner.ScannerError: mapping values are not allowed here")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "YAML syntax" in output


def test_validation_error():
    """Pydantic validation error gets friendly message."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = Exception("2 validation error for SoupConfig")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "Config validation failed" in output


def test_auth_401_error():
    """401 error gets auth hint."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = Exception("401 Client Error: Unauthorized")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "Authentication failed" in output


def test_file_not_found_error():
    """File not found gets friendly message."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)
    with patch("soup_cli.utils.errors.console", test_console):
        exc = FileNotFoundError("No such file or directory: 'model.bin'")
        format_friendly_error(exc, verbose=False)
    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "No such file or directory" in output
    assert "soup init" in output


# --- CLI --verbose flag tests ---


def test_verbose_flag_in_help():
    """--verbose flag is shown in help."""
    result = runner.invoke(app, ["--help"])
    assert result.exit_code == 0
    assert "verbose" in result.output


def test_help_shows_doctor_and_quickstart():
    """New commands are visible in help."""
    result = runner.invoke(app, ["--help"])
    assert result.exit_code == 0
    assert "doctor" in result.output
    assert "quickstart" in result.output

def test_cuda_oom_mentions_gradient_checkpointing():
    """CUDA OOM hint should mention gradient_checkpointing and 4bit."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)

    with patch("soup_cli.utils.errors.console", test_console):
        exc = RuntimeError("CUDA out of memory")
        format_friendly_error(exc, verbose=False)

    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "gradient_checkpointing" in output
    assert "4bit" in output


def test_gated_repo_error():
    """HF gated repos should suggest login and HF_TOKEN."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)

    with patch("soup_cli.utils.errors.console", test_console):
        exc = Exception("GatedRepoError: Cannot access gated repo")
        format_friendly_error(exc, verbose=False)

    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "huggingface-cli login" in output
    assert "HF_TOKEN" in output


def test_gated_repo_hyphenated_error():
    """Hyphenated gated-repo errors should match."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)

    with patch("soup_cli.utils.errors.console", test_console):
        exc = Exception("Access denied to gated-repo")
        format_friendly_error(exc, verbose=False)

    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "huggingface-cli login" in output


def test_trust_remote_code_error():
    """trust_remote_code errors should provide actionable hint."""
    buf = StringIO()
    test_console = Console(file=buf, stderr=False)

    with patch("soup_cli.utils.errors.console", test_console):
        exc = Exception(
            "This repository requires you to execute the configuration file"
        )
        format_friendly_error(exc, verbose=False)

    output = " ".join(strip_ansi(buf.getvalue()).split())
    assert "trust_remote_code=True" in output


_HINT = 'pip install "soup-cli[audio]"'


@pytest.mark.parametrize("verbose", [False, True], ids=["plain", "verbose"])
@pytest.mark.parametrize(
    "exc",
    [
        FileNotFoundError(f"No such file or directory: \x1b]0;pwned\x07 {_HINT}"),
        ImportError(f"XCodec2 needs torchaudio: \x1b]0;pwned\x07 {_HINT}"),
        # Message contains Rich markup close tag: must not be parsed as markup
        # (on main this raises rich.errors.MarkupError, and under --verbose the
        # last traceback line reintroduces it one panel lower).
        RuntimeError(f"broken [/] pipe {_HINT}"),
    ],
    ids=["known-pattern", "unknown-error", "markup-close-tag"],
)
def test_exception_text_is_not_parsed_as_markup(exc, verbose):
    """#1200: `[extra]` in exception text survives, control bytes are stripped,
    and a `[/]` in the message never raises MarkupError — in both the plain and
    the --verbose panel, whose traceback ends with the same exception text."""
    buf = StringIO()
    test_console = Console(file=buf, force_terminal=True, color_system="truecolor", width=300)
    # Raise-and-catch so traceback.format_exc() carries the exception text into
    # the verbose panel, matching how cli.py calls the handler from an except block.
    with patch("soup_cli.utils.errors.console", test_console):
        try:
            raise exc
        except Exception as caught:
            format_friendly_error(caught, verbose=verbose)
    output = strip_ansi(buf.getvalue())
    assert "soup-cli[audio]" in output
    assert "\x1b" not in output
    assert "\x07" not in output