Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

15.9 KB
Raw Blame
"""Accelerate command construction for automatic and advisory launches."""

from __future__ import annotations

import errno
import ipaddress
import os
import re
import shlex
import sys
from typing import Sequence

VALID_MIXED_PRECISION = ("no", "fp16", "bf16", "fp8")
MAX_NUM_MACHINES = 256  # sanity cap, consistent with --gpus MAX_GPU_COUNT=128
DEFAULT_MAIN_PROCESS_PORT = 29500
MAX_HOSTNAME_LEN = 253  # RFC 1035: 255 wire octets less the leading length and root bytes
_HOSTNAME_LABEL = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?")
# Windows cannot replace a process image; see run_launcher.
_IS_WINDOWS = os.name == "nt"


def _is_hostname_or_ip(value: object) -> bool:
    """True for an IPv4/IPv6 literal or an RFC 1123 hostname.

    The value is interpolated into an ``accelerate launch`` argv and exported
    as ``MASTER_ADDR``, so anything that is not actually addressable is a
    typo at best. A character blacklist cannot do this job: IPv6 literals are
    full of colons, while ``head:29500`` -- a host with the port glued on, the
    most common way to get this flag wrong -- must be rejected.
    """
    if not isinstance(value, str) or not value:
        return False
    try:
        ipaddress.ip_address(value)
        return True
    except ValueError:
        pass
    host = value[:-1] if value.endswith(".") else value  # trailing root dot
    if not host or len(host) > MAX_HOSTNAME_LEN:
        return False
    return all(_HOSTNAME_LABEL.fullmatch(label) for label in host.split("."))


def validate_multi_node_options(
    num_machines: int,
    machine_rank: int | None = None,
    main_process_ip: str | None = None,
    main_process_port: int | None = None,
) -> None:
    """Validate node settings without loading a model or probing the network."""
    if (
        isinstance(num_machines, bool) or not isinstance(num_machines, int)
        or not 1 <= num_machines <= MAX_NUM_MACHINES
    ):
        raise ValueError(f"num_machines must be an integer in [1, {MAX_NUM_MACHINES}]")
    if machine_rank is not None and (
        isinstance(machine_rank, bool) or not isinstance(machine_rank, int)
        or not 0 <= machine_rank < num_machines
    ):
        raise ValueError("machine_rank must be an integer from 0 to num_machines - 1")
    if main_process_port is not None and (
        isinstance(main_process_port, bool) or not isinstance(main_process_port, int)
        or not 1 <= main_process_port <= 65535
    ):
        raise ValueError("main_process_port must be an integer from 1 to 65535")
    if num_machines == 1:
        if any(value is not None for value in (machine_rank, main_process_ip, main_process_port)):
            raise ValueError("node rank and coordinator options require num_machines > 1")
        return
    if not _is_hostname_or_ip(main_process_ip):
        raise ValueError("main_process_ip is required and must be a hostname or IP address")


def is_in_distributed() -> bool:
    """Return True if the current process was launched by torchrun / accelerate.

    Detection is based on the presence of either the torch.distributed standard
    variables (RANK + WORLD_SIZE) or any Accelerate-specific marker.
    """
    if "RANK" in os.environ and "WORLD_SIZE" in os.environ:
        return True
    return any(
        key in os.environ
        for key in (
            "ACCELERATE_MIXED_PRECISION",
            "ACCELERATE_USE_DEEPSPEED",
            "ACCELERATE_USE_FSDP",
        )
    )


def build_accelerate_argv(
    num_processes: int,
    script_args: Sequence[str],
    mixed_precision: str | None = None,
    num_machines: int = 1,
    machine_rank: int | None = None,
    main_process_ip: str | None = None,
    main_process_port: int | None = None,
) -> list[str]:
    """Build argv that wraps ``script_args`` with ``accelerate launch``.

    When ``num_processes == 1`` the launcher wrapper is skipped and
    ``script_args`` is returned unchanged.

    Args:
        num_processes: Total number of processes. Must be >= 1.
        script_args: The command to run (e.g. ``["soup", "train"]``).
        mixed_precision: One of ``no``, ``fp16``, ``bf16``, ``fp8``.
        num_machines: Number of nodes. Defaults to 1.
        machine_rank: This node's zero-based rank. Defaults to 0 for multiple nodes.
        main_process_ip: Coordinator hostname or IP, required for multiple nodes.
        main_process_port: Coordinator port. Defaults to 29500 for multiple nodes.

    Raises:
        ValueError: On invalid launch settings or an uneven process count per node.
    """
    if isinstance(num_processes, bool) or not isinstance(num_processes, int) or num_processes < 1:
        raise ValueError(
            f"num_processes must be a positive integer (got {num_processes!r})."
        )
    if mixed_precision is not None and mixed_precision not in VALID_MIXED_PRECISION:
        raise ValueError(
            f"Invalid mixed_precision: {mixed_precision!r}. "
            f"Options: {', '.join(VALID_MIXED_PRECISION)}."
        )
    validate_multi_node_options(num_machines, machine_rank, main_process_ip, main_process_port)
    if num_processes % num_machines:
        raise ValueError("num_processes must be divisible by num_machines")

    script_list = list(script_args)
    if num_processes == 1:
        return script_list

    argv: list[str] = ["accelerate", "launch", "--num_processes", str(num_processes)]
    if num_machines > 1:
        # Select the distributed path even on a node with only one local GPU.
        # Pin rendezvous settings instead of inheriting a cached Accelerate config.
        assert main_process_ip is not None  # validated above
        argv.extend([
            "--multi_gpu", "--num_machines", str(num_machines),
            "--machine_rank", str(0 if machine_rank is None else machine_rank),
            "--main_process_ip", main_process_ip,
            "--main_process_port", str(
                DEFAULT_MAIN_PROCESS_PORT if main_process_port is None else main_process_port
            ),
            "--rdzv_backend", "static",
        ])
    if mixed_precision is not None:
        argv.extend(["--mixed_precision", mixed_precision])
    argv.extend(_as_accelerate_target(script_list))
    return argv


def _is_python_interpreter(path: str) -> bool:
    name = os.path.basename(path).lower()
    if name.endswith(".exe"):
        name = name[: -len(".exe")]
    return path == sys.executable or name.startswith("python") or name.startswith("pypy")


def _as_accelerate_target(script_list: list[str]) -> list[str]:
    """Turn ``[python, "-m", "pkg.mod", ...]`` into ``["--module", "pkg.mod", ...]``.

    #77 — ``accelerate launch`` takes a **script path** positionally, or a module
    behind ``--module``. Soup passed ``sys.executable``, so accelerate opened the
    Python ELF binary and parsed it as source::

        File "/root/venv/bin/python", line 1
          ELF
        SyntaxError: source code cannot contain null bytes

    Every rank died before the trainer existed, i.e. ``soup train --gpus N`` — the
    documented multi-GPU entry point — never ran at all. Measured on 4xH100; every
    arm of the #77 matrix had to be launched by hand.

    A real script path is returned untouched, because that form is valid and
    translating it would break it.
    """
    if (
        len(script_list) >= 3
        and script_list[1] == "-m"
        and _is_python_interpreter(script_list[0])
    ):
        return ["--module", script_list[2], *script_list[3:]]
    return list(script_list)


def collect_reexec_passthrough(
    *,
    name: str | None = None,
    fsdp: str | None = None,
    deepspeed: str | None = None,
    resume: str | None = None,
    wandb: bool = False,
    tensorboard: bool = False,
    echo_trap_tokenizer_aware: bool = False,
    reward_hack_detector: str | None = None,
    reward_hack_halt: bool = False,
    reward_hack_mitigation: str | None = None,
    gate: str | None = None,
    push_as: str | None = None,
    hf_resume: bool = False,
    trust_remote_code: bool = False,
    tracker: str | None = None,
    diagnose_gate: str | None = None,
    annex_xi: str | None = None,
    repro_receipt: str | None = None,
    profile_run: bool = False,
    allow_oom_attempt: bool = False,
    track_energy: bool = False,
    energy_country: str | None = None,
    energy_out: str | None = None,
    yes: bool = False,
    minillm_on_policy: bool = False,
    capture_activations: str | None = None,
    capture_prompts: str | None = None,
    replay: str | None = None,
    replay_ratio: float | None = None,
    replay_seed: int | None = None,
) -> list[str]:
    """User-flag tail shared by auto-reexec and the ``--no-reexec`` hint (#372).

    Keep this as the only place that decides which ``soup train`` flags survive
    a multi-GPU launch. ``build_train_reexec_argv`` wraps the result; the hint
    is derived from that same argv, so the two cannot drift.

    Flags that are deliberately *not* parameters here, because they must not
    survive onto the launched command:

    * ``config`` / ``no_reexec`` — injected by ``build_train_reexec_argv``
      (the child is already under a launcher and must not re-exec).
    * ``gpus`` — becomes ``accelerate launch --num_processes``; repeating
      ``--gpus`` under the launcher would double-count.
    * ``nodes`` / ``node_rank`` / ``master_addr`` / ``master_port`` configure
      Accelerate itself, not the training process it launches.
    * ``dry_run`` — plan-only; the multi-GPU path skips re-exec entirely.
    * ``find_lr`` and its range/output knobs — early-return before launch.
    * ``cloud`` / ``gpu`` / ``cloud_submit`` — a different launch path
      (Modal stub / submit). They return before the accelerate re-exec, and
      ``--cloud`` with ``--gpus`` is two launchers; they are incompatible.
    """
    extra: list[str] = []

    def _opt(flag: str, value: object) -> None:
        if value is not None and value != "":
            extra.extend([flag, str(value)])

    def _switch(flag: str, on: bool) -> None:
        if on:
            extra.append(flag)

    _opt("--name", name)
    _opt("--fsdp", fsdp)
    _opt("--deepspeed", deepspeed)
    _opt("--resume", resume)
    _switch("--wandb", wandb)
    _switch("--tensorboard", tensorboard)
    _switch("--echo-trap-tokenizer-aware", echo_trap_tokenizer_aware)
    _opt("--reward-hack-detector", reward_hack_detector)
    _switch("--reward-hack-halt", reward_hack_halt)
    _opt("--reward-hack-mitigation", reward_hack_mitigation)
    _opt("--gate", gate)
    _opt("--push-as", push_as)
    _switch("--hf-resume", hf_resume)
    _switch("--trust-remote-code", trust_remote_code)
    _opt("--tracker", tracker)
    _opt("--diagnose-gate", diagnose_gate)
    _opt("--annex-xi", annex_xi)
    _opt("--repro-receipt", repro_receipt)
    _switch("--profile", profile_run)
    _switch("--allow-oom-attempt", allow_oom_attempt)
    if track_energy:
        extra.append("--track-energy")
        _opt("--energy-country", energy_country)
        _opt("--energy-out", energy_out)
    _switch("--yes", yes)
    _switch("--minillm-on-policy", minillm_on_policy)
    _opt("--capture-activations", capture_activations)
    _opt("--capture-prompts", capture_prompts)
    _opt("--replay", replay)
    _opt("--replay-ratio", replay_ratio)
    _opt("--replay-seed", replay_seed)
    return extra


def build_train_reexec_argv(
    config: str,
    extra_flags: Sequence[str] | None = None,
) -> list[str]:
    """Argv soup would pass through ``accelerate launch`` on auto-reexec.

    This is the single source for "what the user typed" (#372). Both the
    hand-over (``run_launcher``) and the ``--no-reexec`` printed hint derive from this list, so
    they cannot drift. ``--no-reexec`` is always present because the child is
    already under a launcher and must not re-exec again.
    """
    if not isinstance(config, str) or not config:
        raise ValueError("config must be a non-empty str")
    extra = list(extra_flags or ())
    return [
        sys.executable,
        "-m",
        "soup_cli.cli",
        "train",
        "--config",
        config,
        "--no-reexec",
        *extra,
    ]


def run_launcher(argv: Sequence[str]) -> int:
    """Hand the run over to the launcher ``argv`` and report ITS exit code.

    POSIX: ``os.execvp`` replaces this process with the launcher, so the
    launcher's exit status is the command's and this function does not return.

    Windows has no exec. ``os.execvp`` there starts the launcher as a NEW
    process and ends the calling one with status 0 at once: the shell gets its
    prompt back while the run is still going, and ``soup train --gpus N &&
    next-step`` runs ``next-step`` straight away, whatever the launcher returns
    later. So on Windows the launcher runs as a child (list argv, no shell,
    inherited stdio and environment), this process waits for it, and its exit
    code is returned for the caller to exit with.

    Raises ``OSError`` when the launcher cannot be started.
    """
    args = list(argv)
    if not args:
        raise ValueError("launcher argv must not be empty")
    if _IS_WINDOWS:
        import subprocess

        resolved = _find_on_path(args[0])
        if resolved is None:
            raise FileNotFoundError(errno.ENOENT, "launcher not found on PATH", args[0])
        process = subprocess.Popen([resolved, *args[1:]])
        while True:
            try:
                return process.wait()
            except KeyboardInterrupt:
                # The console delivers Ctrl+C to the launcher as well. Wait for
                # it to finish its own shutdown and report the code it ends with.
                continue
    os.execvp(args[0], args)


def _find_on_path(name: str) -> str | None:
    """Full path of the program ``name`` in a ``PATH`` directory, or ``None``.

    Tries the ``PATHEXT`` suffixes, so ``accelerate`` finds ``accelerate.exe``.
    ``PATH`` directories only: ``shutil.which`` on Windows also looks in the
    current directory first, which would run a file named like the launcher
    out of the project directory. ``os.execvp`` never searched it either.
    """
    if os.path.dirname(name):
        return name if os.path.isfile(name) else None
    if os.path.splitext(name)[1]:
        candidates = [name]
    else:
        suffixes = os.environ.get("PATHEXT", ".COM;.EXE;.BAT;.CMD").split(";")
        candidates = [name + suffix for suffix in suffixes if suffix]
    for directory in os.get_exec_path():
        if not directory:
            continue
        for candidate in candidates:
            full = os.path.join(directory, candidate)
            if os.path.isfile(full):
                return full
    return None


def hint_argv_from_reexec(script_args: Sequence[str]) -> list[str]:
    """``soup train ...`` form of a re-exec argv, for the ``--no-reexec`` hint.

    Drops ``--no-reexec``: under ``accelerate launch`` the run is already
    distributed and never re-execs, so repeating the flag would be noise.
    """
    args = list(script_args)
    if (
        len(args) < 4
        or args[1] != "-m"
        or args[3] != "train"
    ):
        raise ValueError(
            "re-exec argv must look like "
            "[python, '-m', 'soup_cli.cli', 'train', ...]"
        )
    rest = [item for item in args[4:] if item != "--no-reexec"]
    return ["soup", "train", *rest]


def format_advice(
    num_processes: int,
    script_args: Sequence[str],
    *,
    num_machines: int = 1,
    machine_rank: int | None = None,
    main_process_ip: str | None = None,
    main_process_port: int | None = None,
) -> str:
    """Human-readable hint telling the user the exact command to re-run."""
    cmd = build_accelerate_argv(
        num_processes, script_args, num_machines=num_machines,
        machine_rank=machine_rank, main_process_ip=main_process_ip,
        main_process_port=main_process_port,
    )
    quoted = " ".join(shlex.quote(arg) for arg in cmd)
    footer = (
        "Run this command on each node with that node's rank."
        if num_machines > 1 else "Run this command to start training."
    )
    return (
        f"To train on {num_processes} GPUs, re-run under accelerate:\n\n"
        f"    {quoted}\n\n"
        f"{footer}"
    )