Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

14.9 KB
Raw Blame
"""#1393 — DPO loads a vision-language wrapper checkpoint when ``modality: vision``.

``DPOTrainerWrapper`` always built ``AutoModelForCausalLM``, whatever ``modality``
said, so a checkpoint published as a vision-language wrapper (MiniMax-M3,
``model_type: minimax_m3_vl``) could not load at all, while SFT trained its language
tower through ``AutoModelForImageTextToText``. DPO now picks the image-text class and
the processor for ``modality: vision``, the attach preflight mirrors it (the class
``soup recipes verify`` builds is the class the trainer loads), and the one backend
whose DPO setup ignores ``modality`` is refused at config load. A tiny LLaVA stands in
for MiniMax-M3: the same wrapper shape, built locally, no weights downloaded.
"""

from __future__ import annotations

import json

import pytest

torch = pytest.importorskip("torch")
transformers = pytest.importorskip("transformers")
pytest.importorskip("peft")

from soup_cli.config.loader import load_config_from_string  # noqa: E402
from soup_cli.utils.attach_preflight import (  # noqa: E402
    Verdict,
    build_on_meta,
    check_attach,
    loader_for,
)

_LANGUAGE_TOWER = r".*language_model\..*\.self_attn\.(q_proj|v_proj)"


def _llava_config():
    from transformers import CLIPVisionConfig, LlamaConfig, LlavaConfig

    return LlavaConfig(
        text_config=LlamaConfig(
            vocab_size=64, hidden_size=16, intermediate_size=32,
            num_hidden_layers=2, num_attention_heads=2, num_key_value_heads=2,
            pad_token_id=3, bos_token_id=1, eos_token_id=2,
        ),
        vision_config=CLIPVisionConfig(
            hidden_size=16, intermediate_size=32, num_hidden_layers=2,
            num_attention_heads=2, image_size=32, patch_size=16,
        ),
        image_token_index=63,
    )


def _tokenizer():
    from tokenizers import Tokenizer, models, pre_tokenizers
    from transformers import PreTrainedTokenizerFast

    names = ["<unk>", "<s>", "</s>", "<pad>", "say", "hi", "hello", "world", "yes", "no"]
    names += [f"w{i}" for i in range(len(names), 63)] + ["<image>"]
    tok = Tokenizer(models.WordLevel(vocab={w: i for i, w in enumerate(names)}, unk_token="<unk>"))
    tok.pre_tokenizer = pre_tokenizers.Whitespace()
    # ``<image>`` is registered as a special token so the processor resolves its id
    # (63) after a save/load round trip instead of falling back to the unknown id.
    return PreTrainedTokenizerFast(
        tokenizer_object=tok, unk_token="<unk>", bos_token="<s>", eos_token="</s>",
        pad_token="<pad>", additional_special_tokens=["<image>"],
    )


def _save_tiny_llava(root):
    """A LLaVA checkpoint on disk, the shape MiniMax-M3 is published in."""
    from transformers import CLIPImageProcessor, LlavaForConditionalGeneration, LlavaProcessor

    torch.manual_seed(7)
    LlavaForConditionalGeneration(_llava_config()).save_pretrained(root)
    LlavaProcessor(
        image_processor=CLIPImageProcessor(
            size={"height": 32, "width": 32}, crop_size={"height": 32, "width": 32}
        ),
        tokenizer=_tokenizer(),
        patch_size=16,
    ).save_pretrained(root)
    return root.as_posix()


def _save_tiny_llama(root):
    from transformers import LlamaConfig, LlamaForCausalLM

    torch.manual_seed(7)
    LlamaForCausalLM(LlamaConfig(
        vocab_size=64, hidden_size=16, intermediate_size=32, num_hidden_layers=2,
        num_attention_heads=2, num_key_value_heads=2, pad_token_id=3, bos_token_id=1,
        eos_token_id=2,
    )).save_pretrained(root)
    _tokenizer().save_pretrained(root)
    return root.as_posix()


def _yaml(base="org/m", task="dpo", modality=None, backend="transformers", targets=_LANGUAGE_TOWER,
          output=None, **extra):
    lines = [f"base: {base}", f"task: {task}", f"backend: {backend}"]
    if modality is not None:
        lines.append(f"modality: {modality}")
    lines += ["data:", "  train: x.jsonl", "  format: dpo", "  max_length: 64",
              "training:", "  quantization: none", "  epochs: 1", "  batch_size: 2"]
    lines += [f"  {k}: {v}" for k, v in extra.items()]  # training.* knobs such as preference_loss
    lines += ["  lora:", "    r: 4", "    alpha: 8", "    dropout: 0.0",
              f"    target_modules: {json.dumps(targets)}"]
    if output is not None:
        lines.append(f"output: {output}")
    return "\n".join(lines) + "\n"


def _cfg(**kw):
    return load_config_from_string(_yaml(**kw))


class TestTheLoaderMapFollowsTheTrainer:
    """``loader_for`` must build the class ``trainer/dpo.py`` now loads."""

    def test_vision_dpo_uses_the_image_text_class(self):
        assert loader_for(_cfg(modality="vision")) == ("AutoModelForImageTextToText",)

    def test_text_dpo_still_uses_a_causal_lm(self):
        assert loader_for(_cfg()) == ("AutoModelForCausalLM",)
        assert loader_for(_cfg(modality="text")) == ("AutoModelForCausalLM",)

    def test_only_vision_changes_the_dpo_class(self):
        """DPO reads ``vision`` only; its audio config still loads a causal LM, so the
        preflight must say so too (review of #1707)."""
        assert loader_for(_cfg(modality="audio")) == ("AutoModelForCausalLM",)

    def test_a_blend_follows_its_highest_weighted_loss(self):
        """``preference_loss_weights`` builds the wrapper of the largest weight."""
        dpo_led = _cfg(task="preference", modality="vision",
                       preference_loss_weights="{dpo: 0.7, ipo: 0.3}")
        ipo_led = _cfg(task="preference", modality="vision",
                       preference_loss_weights="{dpo: 0.3, ipo: 0.7}")
        assert loader_for(dpo_led) == ("AutoModelForImageTextToText",)
        assert loader_for(ipo_led) == ("AutoModelForCausalLM",)

    def test_the_preference_alias_with_the_dpo_loss_follows_dpo(self):
        """``task: preference`` with ``preference_loss: dpo`` runs the DPO wrapper, so
        the preflight must answer for it the same way; the other losses keep their
        own loaders and stay a causal LM."""
        vision_dpo = _cfg(task="preference", modality="vision", preference_loss="dpo")
        vision_simpo = _cfg(task="preference", modality="vision", preference_loss="simpo")
        assert loader_for(vision_dpo) == ("AutoModelForImageTextToText",)
        assert loader_for(vision_simpo) == ("AutoModelForCausalLM",)


    def test_a_tied_blend_follows_the_first_listed_loss(self):
        """Equal weights pick the first-listed loss, as ``trainer/preference.py`` does
        (review of #1707, round 2)."""
        dpo_first = _cfg(task="preference", modality="vision",
                         preference_loss_weights="{dpo: 0.5, ipo: 0.5}")
        ipo_first = _cfg(task="preference", modality="vision",
                         preference_loss_weights="{ipo: 0.5, dpo: 0.5}")
        assert loader_for(dpo_first) == ("AutoModelForImageTextToText",)
        assert loader_for(ipo_first) == ("AutoModelForCausalLM",)


class TestTheRealAttach:
    """The acceptance row of the issue, on the real resolution and the real
    ``get_peft_model``: a DPO config with ``modality: vision`` attaches to the
    language tower of a vision-language wrapper, and without the modality the
    causal-LM class refuses the checkpoint, exactly what #1145 measured."""

    @staticmethod
    def _check(cfg):
        return check_attach(
            "vl-dpo", cfg, load_hf_config=lambda _base: _llava_config(), build_model=build_on_meta
        )

    def test_a_vision_dpo_config_attaches_to_the_language_tower(self):
        check = self._check(_cfg(modality="vision"))
        assert check.verdict is Verdict.ATTACHES, check.detail
        assert check.adapted > 0
        assert check.vision_modules == 0

    def test_the_same_config_without_modality_still_cannot_load(self):
        check = self._check(_cfg())
        assert check.verdict is Verdict.CANNOT_ATTACH, check.detail

    def test_a_dpo_audio_config_still_cannot_load(self):
        check = self._check(_cfg(modality="audio"))
        assert check.verdict is Verdict.CANNOT_ATTACH, check.detail


@pytest.mark.filterwarnings("ignore::UserWarning")
class TestTheWrapperLoadsTheRightClass:
    """Through the real loaders on checkpoints saved to disk, nothing stubbed."""

    @staticmethod
    def _wrapper(tmp_path, monkeypatch, base, modality, targets=_LANGUAGE_TOWER):
        from soup_cli.trainer.dpo import DPOTrainerWrapper

        monkeypatch.setenv("WANDB_DISABLED", "true")
        monkeypatch.chdir(tmp_path)
        cfg = _cfg(base=base, modality=modality, targets=targets,
                   output=(tmp_path / "out").as_posix())
        wrapper = DPOTrainerWrapper(cfg, device="cpu")
        wrapper._setup_transformers(cfg, cfg.training)
        return wrapper

    def test_vision_loads_the_image_text_model_and_its_processor(self, tmp_path, monkeypatch):
        from transformers import LlavaForConditionalGeneration, ProcessorMixin

        base = _save_tiny_llava(tmp_path / "vl")
        wrapper = self._wrapper(tmp_path, monkeypatch, base, "vision")
        assert isinstance(wrapper.model.get_base_model(), LlavaForConditionalGeneration)
        assert isinstance(wrapper.tokenizer, ProcessorMixin)
        assert wrapper.tokenizer.pad_token is not None
        adapted = [n for n, m in wrapper.model.named_modules() if hasattr(m, "lora_A")]
        assert adapted and all("language_model" in n for n in adapted), adapted
        assert not any("vision_tower" in n for n in adapted)

    def test_vision_vocab_expansion_reaches_the_processors_tokenizer(self, tmp_path, monkeypatch):
        """``data.add_new_tokens`` is applied by DPO (#1358), so on the vision path it
        must go through the processor's nested tokenizer and resize the embeddings;
        handing the processor itself to the expansion would skip both."""
        from soup_cli.trainer.dpo import DPOTrainerWrapper

        monkeypatch.setenv("WANDB_DISABLED", "true")
        monkeypatch.chdir(tmp_path)
        base = _save_tiny_llava(tmp_path / "vl")
        text = _yaml(base=base, modality="vision", output=(tmp_path / "out").as_posix())
        text = text.replace("  format: dpo\n", "  format: dpo\n  add_new_tokens: ['<zz>']\n")
        cfg = load_config_from_string(text)
        wrapper = DPOTrainerWrapper(cfg, device="cpu")
        wrapper._setup_transformers(cfg, cfg.training)
        tokenizer = wrapper.tokenizer.tokenizer
        new_id = tokenizer.convert_tokens_to_ids("<zz>")
        assert new_id not in (None, tokenizer.unk_token_id)
        assert wrapper.model.get_input_embeddings().num_embeddings >= len(tokenizer)

    def test_text_still_loads_a_causal_lm_with_a_tokenizer(self, tmp_path, monkeypatch):
        from transformers import LlamaForCausalLM, PreTrainedTokenizerBase

        base = _save_tiny_llama(tmp_path / "lm")
        wrapper = self._wrapper(tmp_path, monkeypatch, base, None, targets=["q_proj", "v_proj"])
        assert isinstance(wrapper.model.get_base_model(), LlamaForCausalLM)
        assert isinstance(wrapper.tokenizer, PreTrainedTokenizerBase)


@pytest.mark.filterwarnings("ignore::UserWarning")
class TestARealDpoStepOnAVisionWrapper:
    """The question the issue left open: what trl's ``DPOTrainer`` does with a
    processor and preference rows that carry no images. Measured: it tokenizes the
    text through the processor and trains. Nothing about trl is mocked."""

    def test_setup_and_one_step(self, tmp_path, monkeypatch):
        pytest.importorskip("trl")
        pytest.importorskip("datasets")
        from soup_cli.trainer.dpo import DPOTrainerWrapper

        monkeypatch.setenv("WANDB_DISABLED", "true")
        monkeypatch.delenv("WORLD_SIZE", raising=False)
        monkeypatch.chdir(tmp_path)
        base = _save_tiny_llava(tmp_path / "vl")
        cfg = _cfg(base=base, modality="vision", output=(tmp_path / "out").as_posix())
        # trl concatenates prompt + completion as strings, so the completions lead
        # with a space to stay on the tiny word-level vocabulary.
        rows = [
            {"prompt": "say hi", "chosen": " hi", "rejected": " no"},
            {"prompt": "say hello", "chosen": " hello world", "rejected": " no"},
            {"prompt": "hello", "chosen": " hi", "rejected": " world"},
            {"prompt": "hi", "chosen": " hello", "rejected": " no no"},
        ]
        wrapper = DPOTrainerWrapper(cfg, device="cpu")
        wrapper.setup({"train": rows})
        assert wrapper.trainer.processing_class is wrapper.tokenizer
        result = wrapper.train()
        assert result["total_steps"] >= 1, result
        assert (tmp_path / "out" / "adapter_config.json").is_file()


class TestABackendWhoseDpoIgnoresModalityIsRefused:
    def test_unsloth_vision_dpo_is_refused_at_config_load(self):
        with pytest.raises(ValueError) as excinfo:
            _cfg(modality="vision", backend="unsloth")
        message = " ".join(str(excinfo.value).split())
        assert "modality='vision'" in message and "backend='transformers'" in message
        assert "unsloth" in message

    def test_the_preference_alias_with_the_dpo_loss_is_refused_the_same_way(self):
        """``task: preference`` + ``preference_loss: dpo`` runs the same unsloth setup."""
        with pytest.raises(ValueError) as excinfo:
            _cfg(task="preference", modality="vision", backend="unsloth", preference_loss="dpo")
        message = " ".join(str(excinfo.value).split())
        assert "task='preference'" in message and "backend='transformers'" in message
        alias = _cfg(task="preference", modality="vision", preference_loss="dpo")
        assert alias.backend == "transformers"

    def test_a_blend_led_by_dpo_is_refused_the_same_way(self):
        with pytest.raises(ValueError) as excinfo:
            _cfg(task="preference", modality="vision", backend="unsloth",
                 preference_loss_weights="{dpo: 0.7, ipo: 0.3}")
        assert "backend='transformers'" in " ".join(str(excinfo.value).split())
        led_by_ipo = _cfg(task="preference", modality="vision", backend="unsloth",
                          preference_loss_weights="{dpo: 0.3, ipo: 0.7}")
        assert led_by_ipo.backend == "unsloth"

    def test_unsloth_audio_dpo_is_not_refused(self):
        """The refusal is about vision; widening it to every non-text modality would
        survive every other test here."""
        assert _cfg(modality="audio", backend="unsloth").backend == "unsloth"

    def test_unsloth_text_dpo_and_transformers_vision_dpo_still_load(self):
        assert _cfg(backend="unsloth").backend == "unsloth"
        assert _cfg(modality="vision").modality == "vision"

    def test_a_tied_blend_is_refused_only_when_dpo_is_listed_first(self):
        """The refusal breaks a tie the same way the preflight and the trainer do."""
        with pytest.raises(ValueError) as excinfo:
            _cfg(task="preference", modality="vision", backend="unsloth",
                 preference_loss_weights="{dpo: 0.5, ipo: 0.5}")
        assert "requires backend='transformers'" in str(excinfo.value)
        ipo_first = _cfg(task="preference", modality="vision", backend="unsloth",
                         preference_loss_weights="{ipo: 0.5, dpo: 0.5}")
        assert ipo_first.backend == "unsloth"