Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

21.9 KB
Raw Blame
"""#785 — train_on_responses_only=false trained on a doubled BOS.

The default SFT path (``train_on_responses_only``) builds ids with
``add_special_tokens=False`` in ``data/loss_mask.py``, so it never doubled the
BOS. The opt-out text path did not: ``data/sft_format.py`` handed TRL a
``{"text"}`` column, and TRL 0.29.1's language-modeling ``tokenize_fn``
(``sft_trainer.py`` ``processing_class(text=...)``) re-tokenised it with the
tokenizer's default ``add_special_tokens=True``. A template that renders
``{{ bos_token }}`` (Llama-3 / Gemma / Mistral) on a tokenizer whose
post-processor also prepends BOS therefore trained on ``[bos, bos, ...]``.
After #782 fixed inference to send exactly one BOS, those adapters are one
token off from how they are prompted.

The invariant pinned here mirrors #781/#782 on the training side: when a
template renders the sequence, the tokenizer adds nothing on top. That is what
``apply_chat_template(tokenize=True)`` does and what inference now does. It is
deliberately NOT "drop a duplicate BOS": Soup's ``data.chat_template`` presets
render no BOS at all, so their adapters must train on none.

Every tokenizer below is a real ``transformers`` fast tokenizer built offline
from an in-memory vocab, so ``apply_chat_template`` is the genuine Jinja
renderer and the BOS/EOS come from a genuine ``tokenizers`` post-processor.
"""


import pytest

_SPECIALS = [
    "<unk>", "<s>", "</s>",
    "<|system|>", "<|user|>", "<|assistant|>", "<|end|>",
    "<|im_start|>", "<|im_end|>",
]
_WORDS = [
    "You", "are", "terse", ".", "What", "is", "the", "capital", "of", "France",
    "?", "Paris", "system", "user", "assistant",
]
_BOS_ID = _SPECIALS.index("<s>")
_EOS_ID = _SPECIALS.index("</s>")

_BOS_TEMPLATE = (
    "{{ bos_token }}"
    "{% for m in messages %}<|{{ m['role'] }}|> {{ m['content'] }} <|end|> {% endfor %}"
    "{% if add_generation_prompt %}<|assistant|>{% endif %}"
)
# Renders the tokenizer's EOS itself, so the template is the source of the stop token.
_BOS_EOS_TEMPLATE = (
    "{{ bos_token }}"
    "{% for m in messages %}<|{{ m['role'] }}|> {{ m['content'] }} {{ eos_token }} {% endfor %}"
)

_USER = {"role": "user", "content": "What is the capital of France?"}
_ANSWER = {"role": "assistant", "content": "Paris ."}
_MESSAGES = [_USER, _ANSWER]


def _tokenizer(chat_template, *, post_processor="bos"):
    """A real fast tokenizer whose post-processor adds ``post_processor``."""
    transformers = pytest.importorskip("transformers")
    tokenizers = pytest.importorskip("tokenizers")
    from tokenizers import models, pre_tokenizers, processors

    vocab = {token: index for index, token in enumerate(_SPECIALS + _WORDS)}
    backend = tokenizers.Tokenizer(models.WordLevel(vocab=vocab, unk_token="<unk>"))
    backend.pre_tokenizer = pre_tokenizers.Whitespace()
    backend.add_special_tokens(_SPECIALS)
    single = {"bos": "<s> $A", "bos_eos": "<s> $A </s>", None: None}[post_processor]
    if single is not None:
        backend.post_processor = processors.TemplateProcessing(
            single=single,
            special_tokens=[("<s>", _BOS_ID), ("</s>", _SPECIALS.index("</s>"))],
        )
    tok = transformers.PreTrainedTokenizerFast(
        tokenizer_object=backend, unk_token="<unk>", bos_token="<s>", eos_token="</s>"
    )
    tok.chat_template = chat_template
    return tok


def _legacy_format_row(tok):
    """The ``train_on_responses_only=false`` row builder."""
    from soup_cli.config.schema import DataConfig
    from soup_cli.data.sft_format import build_format_row

    dcfg = DataConfig(
        train="train.jsonl",
        train_on_responses_only=False,
        train_on_messages_with_train_field=False,
        max_length=2048,
    )
    return build_format_row(tokenizer=tok, data_cfg=dcfg)


def _trl_main_lm_ids(tok, messages):
    """The ids the SFT language-modeling text path trained on for ``messages`` on
    ``main`` (the baseline #788 must be measured against).

    Reproduces TRL 0.29.1's two real operations on a legacy ``{"text"}`` row using
    the real tokenizer: ``add_eos`` appends ``eos_token`` as a string when the
    rendered text does not already end with it (``sft_trainer.py`` ``add_eos``,
    lines 1026-1031), then TRL tokenizes with the tokenizer's default
    ``add_special_tokens=True`` (line 1131). ``tok(text=text)`` alone omits
    ``add_eos`` and under-counts the trained EOS — the mistake the first #788
    attempt made and the reason these tests are baselined against this instead.
    """
    text = tok.apply_chat_template(
        messages, tokenize=False, add_generation_prompt=False
    )
    if not text.endswith(tok.eos_token):
        text = text + tok.eos_token
    return tok(text)["input_ids"]


class TestLegacyTextPathBOS:
    def test_bos_template_trains_on_one_bos_and_keeps_the_eos(self):
        """A template that renders {{ bos_token }} must train on one BOS (not the
        two ``main`` produced) while keeping the trailing EOS ``main`` trained on.

        The body equals inference's one-step encode (add_special_tokens=False,
        what #782 sends) plus the training stop token inference does not send.

        Fails on pre-fix main: that path returned a ``{"text"}`` row, so there is
        no ``input_ids`` to read, and TRL then tokenized the text to two BOS.
        """
        tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos")
        row = _legacy_format_row(tok)({"messages": _MESSAGES})

        assert "input_ids" in row, (
            "legacy path must pre-tokenize so TRL never re-adds special tokens"
        )
        ids = row["input_ids"]
        assert ids.count(_BOS_ID) == 1, f"expected one BOS, got ids={ids}"

        main_ids = _trl_main_lm_ids(tok, _MESSAGES)
        assert main_ids.count(_BOS_ID) == 2, "sanity: main doubled the BOS here"
        # The stop token main trained on (TRL add_eos) is preserved, not dropped.
        assert ids[-1] == _EOS_ID and main_ids[-1] == _EOS_ID
        assert ids.count(_EOS_ID) == main_ids.count(_EOS_ID) == 1

        inference = tok.apply_chat_template(
            _MESSAGES, tokenize=True, add_generation_prompt=False,
            add_special_tokens=False, return_dict=True,
        )["input_ids"]
        assert ids == list(inference) + [_EOS_ID]
        # Full-sequence training: every token is a target, none masked.
        assert row["labels"] == ids
        assert set(row["attention_mask"]) == {1}

    def test_the_prefix_encoding_would_have_doubled_the_bos(self):
        """Document the defect: the pre-fix mechanism (render to text, then let
        the tokenizer add its defaults) yields two BOS on the same tokenizer."""
        tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos")
        text = tok.apply_chat_template(
            _MESSAGES, tokenize=False, add_generation_prompt=False
        )
        pre_fix_ids = tok(text=text)["input_ids"]  # TRL's default add_special_tokens=True
        assert pre_fix_ids.count(_BOS_ID) == 2

        fixed_ids = _legacy_format_row(tok)({"messages": _MESSAGES})["input_ids"]
        assert fixed_ids.count(_BOS_ID) == 1
        assert fixed_ids != pre_fix_ids

    def test_preset_template_trains_on_zero_bos(self):
        """Soup's data.chat_template presets render no BOS; the path must add
        none, even on a tokenizer whose post-processor prepends one."""
        from soup_cli.data.chat_templates import apply_chat_template_override

        tok = _tokenizer(None, post_processor="bos")
        apply_chat_template_override(tok, "chatml")
        ids = _legacy_format_row(tok)({"messages": _MESSAGES})["input_ids"]
        assert ids.count(_BOS_ID) == 0, f"chatml preset must add no BOS, got {ids}"


class TestTrainingEOSPreserved:
    """The BOS fix must not cost the stop token (maintainer req 1 on #785).

    ``main``'s live text path trained on the EOS TRL's ``add_eos`` appends as a
    string, independent of the tokenizer's post-processor. Dropping it teaches
    run-on generation. Every case is baselined against ``_trl_main_lm_ids`` (what
    ``main`` trained on), not ``tok(text=...)`` which omits ``add_eos``. The fix
    reproduces TRL's rule: end on exactly one EOS, removing ``main``'s duplicates
    but never dropping the stop token.
    """

    @pytest.mark.parametrize(
        "post_processor",
        ["bos", None],
        ids=["bos-only (Gemma/Llama-2)", "no-post-processor (Qwen)"],
    )
    def test_eos_kept_when_post_processor_appends_none(self, post_processor):
        """The regression the #788 review caught. Template renders no EOS on a
        tokenizer whose post-processor appends none. ``main`` still trained on one
        EOS because TRL's ``add_eos`` appends it as a string; probing the
        post-processor (the first #788 attempt) added none and dropped the stop
        token 1 -> 0. The fix keeps it."""
        tok = _tokenizer(_BOS_TEMPLATE, post_processor=post_processor)
        main_ids = _trl_main_lm_ids(tok, _MESSAGES)
        assert main_ids.count(_EOS_ID) == 1, "main trained on one EOS via add_eos"

        ids = _legacy_format_row(tok)({"messages": _MESSAGES})["input_ids"]
        assert ids[-1] == _EOS_ID and ids.count(_EOS_ID) == 1, (
            f"fix must keep the EOS main trained on, not drop it; got {ids}"
        )

    def test_eos_not_duplicated_when_post_processor_also_appends(self):
        """Template renders no EOS; the post-processor appends one AND TRL's
        add_eos appended another, so ``main`` trained on a duplicated EOS. The fix
        ends on exactly one stop token — main's duplicate removed, like the BOS."""
        tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos_eos")
        main_ids = _trl_main_lm_ids(tok, _MESSAGES)
        assert main_ids.count(_EOS_ID) == 2, "sanity: main duplicated the EOS here"

        ids = _legacy_format_row(tok)({"messages": _MESSAGES})["input_ids"]
        assert ids[-1] == _EOS_ID, "row ends on the stop token"
        assert ids.count(_EOS_ID) == 1, "exactly one, the duplicate removed"
        assert ids.count(_BOS_ID) == 1

    def test_template_rendered_eos_is_kept_and_not_re_appended(self):
        """Template renders the EOS itself; the fix adds none because the sequence
        already ends on it (TRL's rule: append only if not already ending on EOS).
        The body equals inference's encode; main's add_eos duplicate is gone."""
        tok = _tokenizer(_BOS_EOS_TEMPLATE, post_processor="bos")
        ids = _legacy_format_row(tok)({"messages": _MESSAGES})["input_ids"]

        assert ids[-1] == _EOS_ID, "row ends on the template's stop token"
        inference = tok.apply_chat_template(
            _MESSAGES, tokenize=True, add_generation_prompt=False,
            add_special_tokens=False, return_dict=True,
        )["input_ids"]
        assert ids == list(inference), "nothing re-appended; already ends on EOS"
        assert ids.count(_BOS_ID) == 1

        main_ids = _trl_main_lm_ids(tok, _MESSAGES)
        assert main_ids.count(_EOS_ID) == ids.count(_EOS_ID) + 1, (
            "main's add_eos duplicated the trailing EOS; the fix does not"
        )


class TestTemplatedOnly:
    """The rule applies only to text a chat template rendered (maintainer req 2)."""

    def test_no_template_row_raises_rather_than_being_silently_pretokenized(self):
        """A tokenizer with no chat_template must raise, not be silently stripped
        into wrong training data. Routed through ``build_format_row`` — the real
        dispatch, present on ``main`` too — so it discriminates the raise
        behaviour rather than the mere absence of a #788 import."""
        from soup_cli.config.schema import DataConfig
        from soup_cli.data.sft_format import build_format_row

        tok = _tokenizer(None, post_processor="bos")  # no chat_template
        tok.chat_template = None
        dcfg = DataConfig(
            train="t.jsonl",
            train_on_responses_only=False,
            train_on_messages_with_train_field=False,
            max_length=2048,
        )
        format_row = build_format_row(tokenizer=tok, data_cfg=dcfg)
        with pytest.raises(ValueError, match="chat_template"):
            format_row({"messages": _MESSAGES})


class TestPreprocessCachePathEOS:
    """`soup data preprocess` EOS behaviour.

    #788 de-duplicated the BOS on the preprocess cache path. It deliberately left
    the EOS as ``main`` had it (post-processor only), which diverged from the live
    path's TRL ``add_eos`` rule on the Qwen shape — tracked as #791. #791 closes
    that: the cache now applies the same ``add_eos`` rule, so both paths train on
    the same EOS count. The BOS still differs between the two paths by design
    (#876); these tests pin that the EOS agrees while the BOS stays as ``main``.
    """

    def _run_preprocess(
        self, tmp_path, monkeypatch, tok, *, task="sft", rows=None, max_length=2048
    ):
        transformers = pytest.importorskip("transformers")
        datasets = pytest.importorskip("datasets")
        from typer.testing import CliRunner

        from soup_cli.cli import app

        rows = rows if rows is not None else [{"messages": _MESSAGES}]
        monkeypatch.chdir(tmp_path)
        (tmp_path / "soup.yaml").write_text(
            f"base: x/y\ntask: {task}\n"
            "data:\n  train: ./d.jsonl\n  format: chatml\n"
            f"  max_length: {max_length}\n",
            encoding="utf-8",
        )
        (tmp_path / "d.jsonl").write_text("{}\n", encoding="utf-8")
        monkeypatch.setattr(
            transformers.AutoTokenizer, "from_pretrained", lambda *a, **k: tok
        )
        # Control the rows directly so the assertion is about tokenization, not
        # format conversion. preprocess_dataset local-imports this name.
        monkeypatch.setattr(
            "soup_cli.data.loader.load_dataset", lambda *a, **k: {"train": rows}
        )
        result = CliRunner().invoke(app, ["data", "preprocess", "soup.yaml", "--yes"])
        assert result.exit_code == 0, result.output
        cache_dirs = [p for p in (tmp_path / ".soup-tokenized").iterdir() if p.is_dir()]
        assert len(cache_dirs) == 1, cache_dirs
        ds = datasets.load_from_disk(str(cache_dirs[0]))
        return list(ds[0]["input_ids"])

    def _main_preprocess_ids(self, tok, *, messages=None, max_length=2048):
        """What ``main``'s preprocess chat path produced: ``tokenizer(text)`` with
        the default ``add_special_tokens=True`` and the same truncation budget (no
        TRL ``add_eos`` on this path). HF truncation reserves room for the
        post-processor's specials, so a truncated ``main`` row still ends on EOS."""
        messages = messages if messages is not None else _MESSAGES
        text = tok.apply_chat_template(
            messages, tokenize=False, add_generation_prompt=False
        )
        return tok(
            text, max_length=max_length, truncation=True, add_special_tokens=True
        )["input_ids"]

    def test_cache_eos_matches_main_when_post_processor_appends(self, tmp_path, monkeypatch):
        """bos_eos post-processor: ``main``'s preprocess kept one EOS (from the
        post-processor). The cache keeps exactly that and drops the doubled BOS."""
        tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos_eos")
        ids = self._run_preprocess(tmp_path, monkeypatch, tok)
        main_pp = self._main_preprocess_ids(tok)

        assert ids.count(_EOS_ID) == main_pp.count(_EOS_ID) == 1, "EOS unchanged vs main"
        assert main_pp.count(_BOS_ID) == 2 and ids.count(_BOS_ID) == 1, "BOS de-duplicated"

    def test_cache_matches_live_eos_for_no_eos_post_processor(self, tmp_path, monkeypatch):
        """Qwen/BOS-only shape: ``main``'s preprocess added no EOS, so the cache
        trained on zero stop tokens while the live path trained on one. #791: the
        cache now applies TRL's ``add_eos`` rule and trains on the same single EOS
        as the live path. Fails on pre-#791 ``main`` (which cached zero)."""
        from soup_cli.data.loss_mask import build_full_sequence_labels

        tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos")
        ids = self._run_preprocess(tmp_path, monkeypatch, tok)
        main_pp = self._main_preprocess_ids(tok)

        live = build_full_sequence_labels(_MESSAGES, tok, max_length=2048)["input_ids"]
        assert main_pp.count(_EOS_ID) == 0, "sanity: main's cache trained on no EOS"
        assert live.count(_EOS_ID) == 1, "sanity: the live path trains on one EOS"
        assert ids.count(_EOS_ID) == live.count(_EOS_ID) == 1, (
            "cache now trains on the same EOS as the live path (#791)"
        )
        assert ids[-1] == _EOS_ID, "the stop token is the trailing token"
        assert ids.count(_BOS_ID) == 1, "BOS still de-duplicated (unchanged by #791)"

    def test_pretrain_cache_is_byte_identical_to_main(self, tmp_path, monkeypatch):
        """Pretrain rows never went through a template, so #788 must leave them
        exactly as ``main``: ``add_special_tokens=True`` and no EOS re-append.
        Kills the two surviving mutations (add_special_tokens=False on pretrain,
        and the EOS re-append applied to pretrain)."""
        tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos_eos")
        raw = "You are terse ."
        ids = self._run_preprocess(
            tmp_path, monkeypatch, tok, task="pretrain", rows=[{"text": raw}]
        )
        assert ids == tok(raw, add_special_tokens=True)["input_ids"]

    def test_cache_keeps_eos_on_truncated_row(self, tmp_path, monkeypatch):
        """The #788 round-2 blocker. A row long enough to truncate must still end
        on the post-processor's EOS, exactly as ``main``. The first round-2 attempt
        (``add_special_tokens=False``, re-append the EOS, slice ``[:max_length]``)
        filled the whole budget with content and sliced the re-appended EOS back
        off (1 -> 0). Tokenising as ``main`` reserves truncation room for the EOS,
        so it survives; we only drop the one duplicated leading BOS."""
        tok = _tokenizer(_BOS_TEMPLATE, post_processor="bos_eos")
        long_msgs = [
            {"role": "user", "content": "What is the capital of France ? " * 40},
            {"role": "assistant", "content": "Paris . " * 40},
        ]
        ids = self._run_preprocess(
            tmp_path, monkeypatch, tok, rows=[{"messages": long_msgs}], max_length=64
        )
        main_pp = self._main_preprocess_ids(tok, messages=long_msgs, max_length=64)
        assert len(main_pp) == 64, "sanity: main filled the truncation budget"
        assert main_pp[0] == main_pp[1] == _BOS_ID, "sanity: main doubled the BOS"
        assert ids == main_pp[1:], "cache == main minus the one duplicated leading BOS"
        assert ids[-1] == _EOS_ID and ids.count(_EOS_ID) == 1, (
            f"truncated row must still end on the stop token; got {ids}"
        )
        assert ids.count(_BOS_ID) == 1, "doubled BOS removed even under truncation"

    def test_cache_keeps_mains_eos_when_template_renders_it(self, tmp_path, monkeypatch):
        """Template renders the EOS itself AND the post-processor appends one, so
        ``main`` trained on several trailing EOS. The cache reproduces ``main``'s
        EOS count (it does NOT collapse to one like the live path — that divergence
        is #791), minus only the duplicated leading BOS."""
        tok = _tokenizer(_BOS_EOS_TEMPLATE, post_processor="bos_eos")
        ids = self._run_preprocess(tmp_path, monkeypatch, tok)
        main_pp = self._main_preprocess_ids(tok)

        assert main_pp[0] == main_pp[1] == _BOS_ID, "sanity: main doubled the BOS"
        assert ids == main_pp[1:], "cache == main minus the duplicated BOS"
        assert ids.count(_EOS_ID) == main_pp.count(_EOS_ID) > 1, "EOS count pinned to main"
        assert ids[-1] == _EOS_ID and ids.count(_BOS_ID) == 1


class TestPreprocessCacheKey:
    def test_cache_key_changed_from_pre_fix_blob(self):
        """The #785 tokenization change must invalidate old preprocess caches:
        the key includes a schema token, so it differs from the old blob's hash.
        Fails if someone drops the schema token and reverts to the old format.
        """
        from soup_cli.utils.data_pipeline import make_preprocess_cache_key
        from tests.preprocess_cache_blob import blob_key

        args = dict(
            dataset_path="data/train.jsonl",
            tokenizer_name="meta-llama/Llama-3.1-8B",
            max_length=2048,
            format_name="chatml",
            mask_mode="responses_only",
            task="sft",
        )
        assert make_preprocess_cache_key(**args) != blob_key("", **args)


class TestDataDoctorLegacyMatchesTraining:
    """``soup data doctor --show-mask`` must X-ray what training actually
    consumes. Its legacy (``train_on_responses_only=false``) branch therefore
    calls the SAME builder as the trainer, :func:`build_full_sequence_labels`.

    #788 regression guard: ``main`` re-rendered here with
    ``apply_chat_template(tokenize=True)`` and the tokenizer's default specials,
    which doubled the BOS and skipped TRL's ``add_eos`` — so the X-ray diverged
    from training. Reverting ``_build_row_labels`` to that now fails here rather
    than silently. (Without this test the revert passes the whole suite.)
    """

    @pytest.mark.parametrize(
        "post_processor",
        ["bos_eos", "bos", None],
        ids=["bos+eos", "bos-only", "no-post-processor (Qwen)"],
    )
    def test_legacy_branch_matches_the_training_builder(self, post_processor):
        from soup_cli.data.loss_mask import build_full_sequence_labels
        from soup_cli.utils.data_doctor import _build_row_labels

        tok = _tokenizer(_BOS_TEMPLATE, post_processor=post_processor)
        doctor = _build_row_labels(
            tok,
            _MESSAGES,
            max_length=2048,
            train_on_responses_only=False,
            train_on_messages_with_train_field=False,
            include_eot=True,
        )
        training = build_full_sequence_labels(_MESSAGES, tok, max_length=2048)
        assert doctor["input_ids"] == training["input_ids"], (
            "the mask X-ray must tokenize identically to training"
        )
        assert doctor["labels"] == training["labels"]
        # and it is genuinely the post-#785 shape, not main's doubled BOS
        assert doctor["input_ids"].count(_BOS_ID) == 1