Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

9.2 KB
Raw Blame
"""Shared LoRA -> dense merge (v0.71.33).

Extracted from ``commands/shrink.py`` (v0.71.29) so both ``soup shrink`` (fuse
the distill-heal adapter back into the pruned model) and ``soup draft`` (a
speculative-decoding draft must be loadable standalone as ``assistant_model=``,
so the distilled adapter has to be merged into a dense checkpoint) go through
ONE implementation instead of two copies that can drift.

:func:`merge_adapter_to_dense` is the general form — base model + adapter ->
dense model at an arbitrary destination. :func:`fuse_adapter_into` is the
in-place special case (destination == the base directory) that ``soup shrink``
uses. Heavy imports (torch / transformers / peft) stay inside the functions.
"""

from __future__ import annotations

import os

from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink


def release_cuda() -> None:
    """Return the CUDA caching-allocator pool to the driver (best-effort)."""
    import gc

    gc.collect()
    try:
        import torch

        if torch.cuda.is_available():
            torch.cuda.empty_cache()
    except ImportError:
        pass


def merge_adapter_to_dense(
    *, base_model: str, adapter_dir: str, out_dir: str, trc: bool = False
) -> None:
    """Merge a LoRA adapter into ``base_model`` and write a DENSE model to ``out_dir``.

    ``base_model`` may be a local directory OR a hub id — PEFT only ever writes
    ``adapter_config.json`` + ``adapter_model.safetensors``, so the base weights
    always have to come from somewhere else.

    The merged model is written to a sibling temp dir and only then swapped in
    for ``out_dir``. An in-place ``save_pretrained`` over a just-loaded model
    directory fails on Windows (error 1224 — the source ``.safetensors`` is
    still memory-mapped by the loaded weights), so the temp-dir swap is the
    cross-platform-safe path. A pre-existing ``out_dir`` is renamed aside
    rather than deleted, and only removed once the staged model has taken
    its place; if the final swap itself then fails (disk full, a lock held
    on the staging dir), the renamed-aside copy is put back before the error
    propagates. If that restore also fails, neither artifact is deleted:
    both the previous model (at the renamed-aside path) and the newly merged
    model (at the staging path) are left on disk and named in the raised
    error, so a failure here can never destroy the last complete copy —
    worst case it leaves two, at paths the caller can recover by hand.
    If renaming the previous model aside fails in the first place, ``out_dir``
    is left unchanged, the staging dir is kept, and the raised error names it.

    Guarantee: whichever way the swap ends, ``out_dir`` is never left holding
    a partial model, and a complete copy of the previous model and/or of the
    newly merged model always survives on disk at a path the caller is told.

    ``out_dir`` is re-validated immediately before the swap: the training
    subprocess that produced ``adapter_dir`` may have run for hours, so the
    containment check done at command entry is stale by now (TOCTOU).
    """
    import gc
    import shutil
    import tempfile

    # Cheap containment check BEFORE the heavy (and, on a core-only install,
    # ImportError-raising) transformers/peft imports, so a bad path reports the
    # actual problem instead of a confusing missing-dependency error.
    enforce_under_cwd_and_no_symlink(out_dir, "fused output dir")

    # Refuse pickle / PyTorch-classic weights in the adapter dir before PEFT
    # torch.load's them — the adapter dir may have been produced (or swapped)
    # by an untrusted process. Shallow scan: PEFT loads adapter_model.* from
    # the TOP LEVEL, while a training output dir also holds the HF Trainer's
    # own checkpoint-N/optimizer.pt pickles, which are not the threat (a
    # recursive scan would make Soup refuse its own trainer's output).
    from soup_cli.utils.strict_safetensors import assert_safe_top_level_weights

    assert_safe_top_level_weights(adapter_dir)

    from peft import PeftModel
    from transformers import AutoModelForCausalLM, AutoTokenizer

    base = AutoModelForCausalLM.from_pretrained(
        base_model, trust_remote_code=trc, torch_dtype="auto"
    )
    merged = PeftModel.from_pretrained(base, adapter_dir).merge_and_unload()

    # The trainer saves the tokenizer alongside the adapter; fall back to the
    # base model's own tokenizer if it didn't.
    try:
        tokenizer = AutoTokenizer.from_pretrained(adapter_dir, trust_remote_code=trc)
    except (OSError, ValueError):
        tokenizer = AutoTokenizer.from_pretrained(base_model, trust_remote_code=trc)

    parent = os.path.dirname(os.path.abspath(out_dir)) or "."
    os.makedirs(parent, exist_ok=True)
    staging = tempfile.mkdtemp(prefix=".fuse_", dir=parent)
    backup: str | None = None
    keep_staging = False
    try:
        try:
            merged.save_pretrained(staging)
            tokenizer.save_pretrained(staging)
        finally:
            # Drop every reference so Windows releases the out_dir mmap before
            # we rename it aside; otherwise the rename also hits error 1224.
            del merged, base, tokenizer
            gc.collect()
            release_cuda()
        # Re-validate the swap target IMMEDIATELY before the destructive
        # rename/replace — the model load + merge + save above took minutes, a
        # real window in which a junction/symlink could be planted at out_dir
        # (the entry-time check is now stale). enforce_* refuses symlinks AND
        # Windows reparse points, so the rename cannot be redirected outside cwd.
        enforce_under_cwd_and_no_symlink(out_dir, "fused output dir")
        if os.path.isdir(out_dir):
            # Rename the previous model ASIDE instead of deleting it. The
            # actual swap below can still fail (AV/indexer holding the
            # staging dir, disk full), and until it succeeds the old model
            # is the only good copy there is.
            backup = os.path.join(
                parent, f".{os.path.basename(os.path.normpath(out_dir))}.old-{os.getpid()}"
            )
            try:
                os.replace(out_dir, backup)
            except OSError as backup_exc:
                # Renaming the previous model aside failed (the same AV /
                # indexer lock the swap below guards against), so out_dir was
                # never touched and still holds the old model. Staging is now
                # the only copy of the new merge: keep it and name it.
                keep_staging = True
                raise RuntimeError(
                    f"failed to move the previous model at '{out_dir}' aside "
                    f"({backup_exc!r}); it is unchanged, and the newly merged "
                    f"model is intact at '{staging}' — move it into place "
                    f"manually"
                ) from backup_exc
        try:
            os.replace(staging, out_dir)
        except BaseException as swap_exc:
            # The swap itself failed: put the previous model straight back
            # before anything else runs, so this failure never leaves
            # neither copy in place.
            if backup is not None:
                try:
                    os.replace(backup, out_dir)
                except OSError as restore_exc:
                    # The restore ALSO failed: out_dir now holds neither
                    # copy. Do not guess which of staging/backup to keep —
                    # keep both on disk and name them in the error, so the
                    # outer handler below must not rmtree(staging) either.
                    keep_staging = True
                    raise RuntimeError(
                        f"failed to swap the merged model into '{out_dir}' "
                        f"({swap_exc!r}), and restoring the previous model "
                        f"from '{backup}' also failed ({restore_exc!r}); "
                        f"the previous model is intact at '{backup}' and the "
                        f"newly merged model is intact at '{staging}' — move "
                        f"one of them into place manually"
                    ) from restore_exc
                backup = None
            raise
        if backup is not None:
            shutil.rmtree(backup, ignore_errors=True)
    except BaseException:
        # A half-written staging dir (disk full, interrupted save) must not be
        # orphaned next to the model — it would silently accumulate a full
        # model's worth of bytes per failed run. But if the swap AND the
        # restore both failed above, staging is the only copy of the new
        # model left anywhere: keep it, and let the RuntimeError above name
        # it instead of deleting it here.
        if not keep_staging:
            shutil.rmtree(staging, ignore_errors=True)
        raise


def fuse_adapter_into(*, base_dir: str, adapter_dir: str, trc: bool = False) -> None:
    """Merge a LoRA adapter into ``base_dir`` in place (``soup shrink``'s case).

    ``base_dir`` is BOTH the base weights and the destination: the healed model
    replaces the pruned one. Thin wrapper over :func:`merge_adapter_to_dense`.
    """
    merge_adapter_to_dense(
        base_model=base_dir, adapter_dir=adapter_dir, out_dir=base_dir, trc=trc
    )