Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

9.8 KB
Raw Blame
"""SFT row-formatter factory (v0.36.0 Part A).

Builds the ``format_row`` function used by ``SFTTrainerWrapper`` based on
``DataConfig`` flags. Three modes:

- ``train_on_responses_only=True`` (default): pre-tokenise to
  ``{input_ids, labels, attention_mask}`` with non-assistant tokens masked
  to ``IGNORE_INDEX``. SFTTrainer detects pre-tokenised columns and skips
  its own tokenization.
- ``train_on_messages_with_train_field=True``: like above but uses the
  per-message ``train: bool`` field.
- both False: legacy ``{text}`` path. SFTTrainer tokenizes on its own (TRL
  heuristic — known to be wrong for multi-turn chat data; left as opt-out
  for backwards compat).

Tokenizer-without-chat-template degrades to the legacy text path and emits
a single warning. v0.36.0 Part C will harden this into a hard error once
``chat_template`` is a first-class config field.
"""

from __future__ import annotations

from typing import Any, Callable

from soup_cli.config.schema import DataConfig
from soup_cli.data.chat_templates import apply_chat_template_override
from soup_cli.data.loss_mask import (
    build_assistant_only_labels,
    build_full_sequence_labels,
    build_per_message_train_labels,
)


def build_format_row(
    tokenizer: Any,
    data_cfg: DataConfig,
    console: Any | None = None,
    training_cfg: Any | None = None,
) -> Callable[[dict], dict]:
    """Factory: return the ``format_row`` function appropriate for ``data_cfg``.

    v0.53.2 #137: when ``training_cfg.reasoning_effort`` is set the gpt-oss
    ``<|reasoning_effort|>...<|/reasoning_effort|>`` control tag is injected
    into the system message before formatting. When ``training_cfg.train_on_eot``
    is true the loss mask is extended to include the trailing EOT/EOS token
    after each assistant span (axolotl ``train_on_eot``).
    """
    # v0.36.0 Part C: apply chat-template override BEFORE deciding on path.
    # Override may turn a templateless tokenizer into a usable one. The
    # override warning surfaces here (not at sft.py call site) so it fires
    # exactly once per setup, regardless of whether the legacy text path or
    # the loss-mask path is selected.
    apply_chat_template_override(tokenizer, data_cfg.chat_template, console=console)

    has_template = bool(getattr(tokenizer, "chat_template", None))
    use_responses_only = bool(data_cfg.train_on_responses_only)
    # #761: getattr, not attribute access. Several suites hand this factory a
    # duck-typed stand-in (SimpleNamespace, a local DataCfg) carrying only the
    # fields that existed when they were written, and a new required attribute
    # turns them into AttributeError at setup. The schema default is False.
    mask_history = bool(getattr(data_cfg, "mask_history", False))
    use_train_field = bool(data_cfg.train_on_messages_with_train_field)
    max_length = int(data_cfg.max_length)

    reasoning_effort = (
        getattr(training_cfg, "reasoning_effort", None) if training_cfg else None
    )
    include_eot = bool(
        getattr(training_cfg, "train_on_eot", False) if training_cfg else False
    )

    # v0.53.7 #87: custom prompt_strategy live runtime. Resolves the
    # ``module.path:fn_name`` spec once at setup time (fail-fast on bad import)
    # and applies the transform to each row BEFORE template rendering.
    prompt_strategy_spec = getattr(data_cfg, "prompt_strategy", None)

    if (use_responses_only or use_train_field) and not has_template:
        if console is not None:
            console.print(
                "[yellow]train_on_responses_only requested but tokenizer "
                "has no chat_template — falling back to text path. Pass "
                "data.chat_template explicitly to enable masking.[/]"
            )
        return _wrap_dropping_arrow_nulls(
            _wrap_with_prompt_strategy(
                _wrap_with_reasoning_effort(
                    _legacy_text_format_row(tokenizer), reasoning_effort
                ),
                prompt_strategy_spec,
            )
        )

    if use_train_field:
        inner = _build_per_message_format_row(tokenizer, max_length)
    elif use_responses_only:
        inner = _build_assistant_only_format_row(
            tokenizer,
            max_length,
            include_eot=include_eot,
            mask_history=mask_history,
        )
    else:
        inner = _build_full_sequence_format_row(tokenizer, max_length)
    return _wrap_dropping_arrow_nulls(
        _wrap_with_prompt_strategy(
            _wrap_with_reasoning_effort(inner, reasoning_effort),
            prompt_strategy_spec,
        )
    )


# Per-message keys only some turns carry (#1217). ``Dataset.from_list`` gives
# every message every key seen in the column, so a plain user turn reaches the
# formatter as ``{"tool_calls": None, "tool_call_id": None, ...}``, and a
# template that branches on ``'tool_calls' in message`` (Llama 3.x) renders
# it as a call.
_SPARSE_MESSAGE_KEYS = ("tool_calls", "tool_call_id")


def _wrap_dropping_arrow_nulls(inner: Callable[[dict], dict]) -> Callable[[dict], dict]:
    """Decorate ``inner`` so the ``None`` placeholders Arrow adds are removed."""

    def wrapped(example: dict) -> dict:
        msgs = example.get("messages")
        if not isinstance(msgs, list):
            return inner(example)
        cleaned = [
            {
                key: value for key, value in msg.items()
                if not (key in _SPARSE_MESSAGE_KEYS and value is None)
            }
            if isinstance(msg, dict) else msg
            for msg in msgs
        ]
        return inner({**example, "messages": cleaned})

    return wrapped


def _wrap_with_prompt_strategy(
    inner: Callable[[dict], dict], spec: Any | None
) -> Callable[[dict], dict]:
    """v0.53.7 #87 — apply ``data.prompt_strategy`` transform per-row.

    Resolves the spec eagerly at wrap-time so a bad import fails at setup
    rather than mid-training. Per-row callable exceptions are logged at
    DEBUG and the original row falls through (matches v0.33.0 #47
    CrossDocCollator silent-degrade policy).
    """
    if spec is None or not isinstance(spec, str):
        return inner
    # Resolve eagerly so trainer setup surfaces a bad spec loudly.
    from soup_cli.utils.data_pipeline import (
        apply_prompt_strategy,
        resolve_prompt_strategy,
    )

    resolve_prompt_strategy(spec)  # fail fast on import / signature errors

    def wrapped(example: dict) -> dict:
        transformed = apply_prompt_strategy(spec, example)
        if isinstance(transformed, dict):
            return inner(transformed)
        # apply_prompt_strategy may return a non-dict Mapping (e.g.
        # MappingProxyType); coerce so the existing format_row helpers see
        # an ordinary mutable dict.
        return inner(dict(transformed))

    return wrapped


def _wrap_with_reasoning_effort(
    inner: Callable[[dict], dict], level: Any | None
) -> Callable[[dict], dict]:
    """Decorate ``inner`` so each example's messages get a reasoning-effort prefix."""
    if level is None:
        return inner
    from soup_cli.utils.reasoning_effort import apply_reasoning_effort_prefix

    def wrapped(example: dict) -> dict:
        msgs = example.get("messages")
        if isinstance(msgs, list) and msgs:
            new_example = {
                **example,
                "messages": apply_reasoning_effort_prefix(msgs, level),
            }
            return inner(new_example)
        return inner(example)

    return wrapped


def _build_assistant_only_format_row(
    tokenizer: Any,
    max_length: int,
    include_eot: bool = False,
    mask_history: bool = False,
) -> Callable[[dict], dict]:
    def format_row(example: dict) -> dict:
        return build_assistant_only_labels(
            example["messages"],
            tokenizer,
            max_length=max_length,
            include_eot=include_eot,
            mask_history=mask_history,
        )

    return format_row


def _build_per_message_format_row(
    tokenizer: Any, max_length: int
) -> Callable[[dict], dict]:
    def format_row(example: dict) -> dict:
        return build_per_message_train_labels(
            example["messages"], tokenizer, max_length=max_length
        )

    return format_row


def _build_full_sequence_format_row(
    tokenizer: Any, max_length: int
) -> Callable[[dict], dict]:
    """Legacy (both flags False) path: train on every token, no masking.

    #785: pre-tokenise with ``add_special_tokens=False`` so TRL skips its own
    language-modeling ``tokenize_fn`` (which re-adds the tokenizer's default
    special tokens and doubled the BOS on ``{{ bos_token }}`` templates). The
    rendered chat template is the single source of special tokens, matching
    inference after #782.
    """

    def format_row(example: dict) -> dict:
        return build_full_sequence_labels(
            example["messages"], tokenizer, max_length=max_length
        )

    return format_row


def _legacy_text_format_row(tokenizer: Any) -> Callable[[dict], dict]:
    """No-template fallback for a masking config: reached from
    :func:`build_format_row` only when a masking flag is set but the tokenizer has
    no ``chat_template``, so it always raises.

    #788: the ``{"text": rendered}`` return this used to produce is dead now that
    the full-sequence path pre-tokenises (returning ``input_ids``), so it is gone;
    what remains is the hard error that (v0.36.0 Part C) replaced the silent
    ``f"{role}: {content}"`` fallback which produced garbage training data.
    """

    def format_row(example: dict) -> dict:
        raise ValueError(
            "Tokenizer has no chat_template. Pass "
            "data.chat_template: chatml (or llama3/qwen2.5/mistral/"
            "gemma3/phi4/deepseek-r1) in soup.yaml, or supply a raw "
            "Jinja string. The previous silent f-string fallback "
            "produced wrong loss labels and was removed in v0.36.0."
        )

    return format_row