Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

13.3 KB
Raw Blame
"""Friendly error handling — maps raw exceptions to actionable messages."""

import re
import traceback

from rich.console import Console
from rich.panel import Panel

from soup_cli.utils.terminal import for_terminal

console = Console(stderr=True)

# v0.71.0 — the heavy training stack (torch / transformers / peft / trl /
# datasets / bitsandbytes / accelerate) moved out of the core install into the
# `[train]` extra. A missing one of these surfaces this single, actionable fix.
# The `\\[` escapes the literal `[` for Rich markup (it renders as `[train]`).
_TRAIN_FIX = "Training needs the \\[train] extra. Run: pip install \"soup-cli\\[train]\""

# Map known error patterns to (short message, fix suggestion)
ERROR_MAP = [
    # CUDA OOM
    (
        "CUDA out of memory",
        "GPU ran out of memory during training.",
        (
            "Try --batch-size <half> or --grad-accum <double> "
            "(keeps effective batch size); lower data.max_length; "
            "enable gradient_checkpointing, use 4bit quantization, or use a smaller model."
        ),
    ),
    (
        "OutOfMemoryError",
        "GPU ran out of memory.",
        (
            "Try --batch-size <half> or --grad-accum <double> "
            "(keeps effective batch size); lower data.max_length; "
            "enable gradient_checkpointing, use 4bit quantization, or use a smaller model."
        ),
    ),
    # Missing optional deps
    (
        "No module named 'fastapi'",
        "FastAPI is not installed (needed for soup serve).",
        "Run: pip install \"soup-cli\\[serve]\"",
    ),
    (
        "No module named 'uvicorn'",
        "Uvicorn is not installed (needed for soup serve).",
        "Run: pip install \"soup-cli\\[serve]\"",
    ),
    (
        "No module named 'datasketch'",
        "Datasketch is not installed (needed for dedup).",
        "Run: pip install \"soup-cli\\[data]\"",
    ),
    (
        "No module named 'lm_eval'",
        "lm-evaluation-harness is not installed (needed for eval).",
        "Run: pip install \"soup-cli\\[eval]\"",
    ),
    (
        "No module named 'wandb'",
        "Weights & Biases is not installed.",
        "Run: pip install wandb",
    ),
    (
        "No module named 'deepspeed'",
        "DeepSpeed is not installed.",
        "Run: pip install \"soup-cli\\[deepspeed]\"",
    ),
    (
        "No module named 'httpx'",
        "httpx is not installed (needed for data generate).",
        "Run: pip install \"soup-cli\\[generate]\"",
    ),
    # Heavy training stack — all moved to the [train] extra in v0.71.0.
    (
        "No module named 'torch'",
        "PyTorch is not installed (needed for training).",
        _TRAIN_FIX,
    ),
    (
        "No module named 'transformers'",
        "Transformers is not installed (needed for training).",
        _TRAIN_FIX,
    ),
    (
        "No module named 'peft'",
        "PEFT is not installed (needed for LoRA training).",
        _TRAIN_FIX,
    ),
    (
        "No module named 'trl'",
        "TRL is not installed (needed for training).",
        _TRAIN_FIX,
    ),
    (
        "No module named 'datasets'",
        "Datasets is not installed (needed for training).",
        _TRAIN_FIX,
    ),
    (
        "No module named 'bitsandbytes'",
        "BitsAndBytes is not installed (needed for quantization).",
        _TRAIN_FIX,
    ),
    (
        "No module named 'accelerate'",
        "Accelerate is not installed (needed for training).",
        _TRAIN_FIX,
    ),
    # CPU / quantization issues
    (
        "expanded size of the tensor",
        "Model generation failed (empty tensors, likely GRPO/PPO on CPU).",
        "GRPO requires a CUDA GPU. For CPU training, use SFT or DPO instead.",
    ),
    (
        "expected m1 and m2 to have the same dtype",
        "Dtype mismatch (likely 4bit quantization on CPU).",
        "Use a GPU, or set quantization: none in your config for CPU training.",
    ),
    (
        "Your setup doesn't support bf16",
        "This training task requires GPU with bf16 support.",
        "Use a CUDA GPU, or try a simpler task (SFT/DPO work on CPU).",
    ),
    (
        "use_cpu",
        "This training task requires use_cpu flag on CPU-only systems.",
        "Use a CUDA GPU for best results, or upgrade trl: pip install -U trl",
    ),
    (
        "nms does not exist",
        "torchvision version is incompatible with torch.",
        "Run: pip install torchvision --force-reinstall (or check soup doctor).",
    ),
    # peft's is_torchao_available() RAISES ImportError instead of returning
    # False when a too-old torchao is installed (peft/import_utils.py) — hit
    # nine frames inside get_peft_model on hosted notebooks (Colab, Kaggle)
    # that preinstall an old torchao. v0.73.x #389.
    (
        "Found an incompatible version of torchao",
        "Your installed peft refuses to load with the preinstalled torchao version.",
        (
            "Run: pip uninstall -y torchao (or upgrade it past the version peft "
            "demands). Soup does not need torchao unless you set "
            "training.quantization_aware. This usually comes from a hosted "
            "notebook (Colab, Kaggle) that preinstalls an old torchao."
        ),
    ),
    # Connection errors
    (
        "ConnectionError",
        "Network connection failed.",
        "Check your internet connection. If downloading from HuggingFace, check HF_TOKEN.",
    ),
    (
        "HTTPError",
        "HTTP request failed.",
        "Check your internet connection and API keys (OPENAI_API_KEY, HF_TOKEN).",
    ),
    (
        "ConnectTimeout",
        "Connection timed out.",
        "Check your internet connection and try again.",
    ),
    # File not found
    (
        "No such file or directory",
        None,  # Will use the original message
        "Check the file path. Run 'soup init' to create a config.",
    ),
    # YAML errors
    (
        "yaml.scanner.ScannerError",
        "Invalid YAML syntax in config file.",
        "Check your soup.yaml for syntax errors (indentation, colons, quotes).",
    ),
    # Pydantic validation
    (
        "validation error",
        "Config validation failed.",
        "Check your soup.yaml values. Run 'soup init' to generate a valid config.",
    ),
    # Hugging Face gated/private model access
    (
        "gated repo",
        "This Hugging Face model requires authentication or license acceptance.",
        (
            "Run: huggingface-cli login or set HF_TOKEN. "
            "Also accept the model license on Hugging Face."
        ),
    ),
    (
        "gated-repo",
        "This Hugging Face model requires authentication or license acceptance.",
        (
        "Run: huggingface-cli login or set HF_TOKEN. "
        "Also accept the model license on Hugging Face."
        ),
    ),
    (
        "GatedRepoError",
        "This Hugging Face model requires authentication or license acceptance.",
        (
            "Run: huggingface-cli login or set HF_TOKEN. "
            "Also accept the model license on Hugging Face."
        ),
    ),

    # trust_remote_code errors
    (
        "trust_remote_code",
        "This model requires remote code execution approval.",
        (
            "Set trust_remote_code=True in your config or command "
            "if you trust the model source."
        ),
    ),
    (
        "requires you to execute the configuration file",
        "This model requires remote code execution approval.",
        (
            "Set trust_remote_code=True in your config or command "
            "if you trust the model source."
        ),
    ),
    # Layer streaming stripe roots
    (
        "StripeRootError",
        None,
        (
            "Fix or remove the invalid entry in SOUP_LAYER_STREAM_STRIPE_DIRS, "
            "reconnect the drive, or unset the variable to train using the primary "
            "cache root only."
        ),
    ),
]


# 401/403 must be decided from the exception OBJECT (or explicit HTTP
# phrasing), never from bare digits — "train row 401:", a checkpoint path
# holding "4010", or a hash slice were all reported as auth failures and the
# real message was hidden (#1267). The old bare "401"/"403" ERROR_MAP entries
# are gone; this rule runs before the map so a genuine status also beats the
# generic "HTTPError" entry.
_AUTH_HINTS = {
    401: (
        "Authentication failed.",
        "Check your API key or token (HF_TOKEN, OPENAI_API_KEY, WANDB_API_KEY).",
    ),
    403: (
        "Access denied.",
        "Check your permissions. Some models require accepting a license on HuggingFace.",
    ),
}

# Text fallback for stacks that only put the status in the message. Every
# pattern requires HTTP context around the digits, so a row number, tensor
# size, path component, or hash slice cannot match.
_AUTH_STATUS_TEXT = (
    (re.compile(r"401\s+(?:Unauthorized|Client Error)"), 401),
    (re.compile(r"HTTP\s+(?:Error\s+)?(?:[:=]\s*)?401\b"), 401),
    (re.compile(r"Error code:\s*401\b"), 401),
    (re.compile(r"403\s+(?:Forbidden|Client Error)"), 403),
    (re.compile(r"HTTP\s+(?:Error\s+)?(?:[:=]\s*)?403\b"), 403),
    (re.compile(r"Error code:\s*403\b"), 403),
)

# These carry 401/403 but must not become the generic auth hint: GatedRepoError
# has its own ERROR_MAP entry, and a top-level RepositoryNotFoundError reaches
# the "gated repo" text entry. The rule declines both outright, so a wrapped
# one keeps the wrapper's own message.
_STATUS_DECLINE_TYPES = ("GatedRepoError", "RepositoryNotFoundError")


def _exception_chain(exc: Exception):
    """Yield exc and the exceptions it wraps, without looping."""
    seen = set()
    current = exc
    while current is not None and id(current) not in seen:
        seen.add(id(current))
        yield current
        current = current.__cause__ or current.__context__


def _safe_str(exc: BaseException) -> str:
    """`str(exc)`, or an empty string when the exception's own `__str__` raises.

    The text fallback below runs over every exception in the chain, so one class
    with a broken `__str__` would otherwise turn the friendly error into a second
    traceback. Same guard CPython's `traceback` module applies.
    """
    try:
        return str(exc)
    except Exception:
        return ""


def _auth_status(exc: Exception) -> int | None:
    """Return 401/403 when the exception is genuinely an auth failure.

    The status is read from the exception object first — `response.status_code`
    (requests / httpx / huggingface_hub), `.code` (urllib), or a direct
    `.status_code` — walking `__cause__`/`__context__` because peft and
    transformers wrap hub errors. Only when no object in the chain carries a
    401/403 status does the HTTP-phrasing text fallback apply; an object
    carrying some other status (a 404, say) still falls through to the text
    check.

    The fallback walks the chain the same way, exception-major: the outermost
    exception that carries a status phrase decides, so a wrapper that says
    nothing about status does not hide the failure it wraps, and a 403 wrapper
    of a 401 is still read as the 403 it says. Pattern-major (every pattern
    across the chain before the next pattern) would let the 401 entries, which
    come first in `_AUTH_STATUS_TEXT`, beat an outer 403.
    """
    chain = list(_exception_chain(exc))
    if any(type(e).__name__ in _STATUS_DECLINE_TYPES for e in chain):
        return None
    for e in chain:
        status = getattr(getattr(e, "response", None), "status_code", None)
        if isinstance(status, int) and status in _AUTH_HINTS:
            return status
        for attr in ("code", "status_code"):
            status = getattr(e, attr, None)
            if isinstance(status, int) and status in _AUTH_HINTS:
                return status
    for e in chain:
        for pattern, status in _AUTH_STATUS_TEXT:
            if pattern.search(_safe_str(e)):
                return status
    return None


def format_friendly_error(exc: Exception, verbose: bool = False) -> None:
    """Display a friendly error message for the given exception.

    In normal mode: 2-3 lines with error + fix suggestion.
    In verbose mode: full traceback.
    """
    exc_str = str(exc)
    exc_type = type(exc).__name__

    matched = None
    status = _auth_status(exc)
    if status is not None:
        matched = _AUTH_HINTS[status]

    # Search for known error patterns
    if matched is None:
        for pattern, short_msg, fix in ERROR_MAP:
            if pattern in exc_str or pattern in exc_type:
                matched = (short_msg or exc_str, fix)
                break

    if matched is not None:
        error_msg, fix = matched
        console.print(f"\n[bold red]Error:[/] {for_terminal(error_msg)}")
        console.print(f"[green]Fix:[/] {fix}")
        if verbose:
            console.print()
            console.print(
                Panel(
                    for_terminal(traceback.format_exc()),
                    title="[dim]Full Traceback[/]",
                    border_style="dim",
                )
            )
        return

    # Unknown error — show type + message
    console.print(f"\n[bold red]Error:[/] {for_terminal(exc_type)}: {for_terminal(exc_str)}")
    console.print("[dim]Run with --verbose for the full traceback.[/]")
    if verbose:
        console.print()
        console.print(
            Panel(
                for_terminal(traceback.format_exc()),
                title="[dim]Full Traceback[/]",
                border_style="dim",
            )
        )