Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

6.9 KB
Raw Blame
"""#1217: a load that ends with zero training rows stops `soup train`.

The loader drops rows its format cannot convert, so a format mismatch loads
nothing and still returns cleanly. `soup train --dry-run` then printed
"Data OK: 0 train samples" and "Config valid. Ready to train!", and the real
run loaded the model before failing on an empty dataset. Both now exit 1
with the format and the first drop reason.
"""

from __future__ import annotations

import dataclasses
import json
import re
from pathlib import Path
from types import SimpleNamespace

import pytest
from typer.testing import CliRunner

from soup_cli.cli import app
from soup_cli.config.schema import DataConfig
from soup_cli.data.loader import last_load_outcome, load_dataset
from tests.conftest import strip_ansi

_BASE = "hf-internal-testing/tiny-random-LlamaForCausalLM"
_CHAT = [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"}]


def _plain(text: str) -> str:
    return re.sub(r"\s+", " ", strip_ansi(text)).strip()


def _write_jsonl(path: Path, rows: list[dict]) -> None:
    path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")


def _loaded_rows(path: str, data_format: str) -> list[dict]:
    config = DataConfig(train=path, format=data_format, val_split=0.0)
    return list(load_dataset(config)["train"])



def _write_config(path: Path, data_file: str, data_format: str) -> None:
    path.write_text(
        f"base: {_BASE}\ntask: sft\ndata:\n  train: {data_file}\n"
        f"  format: {data_format}\n  val_split: 0.0\noutput: ./out\n",
        encoding="utf-8",
    )


def test_dry_run_with_zero_rows_exits_non_zero_naming_format_and_reason(
    tmp_path, monkeypatch
):
    monkeypatch.chdir(tmp_path)
    _write_jsonl(Path("data.jsonl"), [{"instruction": "q", "output": "a"}] * 2)
    _write_config(Path("soup.yaml"), "data.jsonl", "chatml")

    result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--dry-run"])

    output = _plain(result.output)
    assert result.exit_code == 1, output
    assert "Ready to train" not in output
    assert "Data OK" not in output
    assert (
        "No training rows: data.jsonl loaded 0 train samples as format 'chatml'."
    ) in output
    assert "First: row 0 (as 'chatml'): 'messages'." in output


def test_the_real_run_with_zero_rows_stops_before_the_model_loads(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    _write_jsonl(Path("data.jsonl"), [{"instruction": "q", "output": "a"}])
    _write_config(Path("soup.yaml"), "data.jsonl", "chatml")

    import soup_cli.trainer.sft as sft

    def _must_not_run(*args, **kwargs):
        raise AssertionError("the trainer was set up for a run with no rows")

    monkeypatch.setattr(sft.SFTTrainerWrapper, "setup", _must_not_run)

    result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--yes"])

    output = _plain(result.output)
    assert result.exit_code == 1, output
    assert "No training rows: data.jsonl loaded 0 train samples as format 'chatml'." in output
    assert "Loaded:" not in output


def test_zero_rows_without_a_drop_says_no_row_failed_to_convert(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    # A drop in an earlier load must not leak into this one's message.
    _write_jsonl(Path("bad.jsonl"), [{"instruction": "q", "output": "a"}])
    _loaded_rows("bad.jsonl", "chatml")
    assert last_load_outcome().first_drop is not None
    Path("empty.jsonl").write_text("", encoding="utf-8")
    _write_config(Path("soup.yaml"), "empty.jsonl", "chatml")

    result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--dry-run"])

    output = _plain(result.output)
    assert result.exit_code == 1, output
    assert "loaded 0 train samples as format 'chatml'." in output
    assert "No row failed to convert" in output
    assert "First: row" not in output


def test_dry_run_with_rows_still_says_ready_to_train(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    _write_jsonl(Path("data.jsonl"), [{"messages": _CHAT}])
    _write_config(Path("soup.yaml"), "data.jsonl", "chatml")

    result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--dry-run"])

    output = _plain(result.output)
    assert result.exit_code == 0, output
    assert "Data OK: 1 train samples" in output
    assert "Config valid. Ready to train!" in output


def test_a_pre_tokenized_run_counts_its_cache_not_the_loaders_view(tmp_path):
    # #1054: under `pre_tokenized` the loader sees the original file and keeps
    # nothing, while training reads the Arrow cache. The stop must not fire.
    import typer

    from soup_cli.commands.train import _refuse_empty_train

    cache = tmp_path / "cache"
    cache.mkdir()
    (cache / "metadata.json").write_text(json.dumps({"row_count": 6}), encoding="utf-8")
    dcfg = SimpleNamespace(format="pre_tokenized", tokenized_path=str(cache), train="d.jsonl")

    _refuse_empty_train(dcfg, {"train": []})

    (cache / "metadata.json").unlink()
    with pytest.raises(typer.Exit):
        _refuse_empty_train(dcfg, {"train": []})


def test_a_pre_tokenized_dry_run_reports_the_caches_row_count(tmp_path, monkeypatch):
    # The loader sees the original file and keeps nothing; the run trains on
    # the cache's rows, so that is the count the dry run must print.
    monkeypatch.chdir(tmp_path)
    Path("cache").mkdir()
    Path("cache/metadata.json").write_text(json.dumps({"row_count": 6}), encoding="utf-8")
    _write_jsonl(Path("data.jsonl"), [{"messages": _CHAT}])
    Path("soup.yaml").write_text(
        f"base: {_BASE}\ntask: sft\ndata:\n  train: data.jsonl\n"
        "  format: pre_tokenized\n  tokenized_path: ./cache\n  val_split: 0.0\n"
        "output: ./out\n",
        encoding="utf-8",
    )

    result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--dry-run"])

    output = _plain(result.output)
    assert result.exit_code == 0, output
    assert "Data OK: 6 train samples" in output
    assert "Config valid. Ready to train!" in output


def test_the_outcome_is_a_snapshot_no_caller_or_later_load_can_change(
    tmp_path, monkeypatch
):
    monkeypatch.chdir(tmp_path)
    _write_jsonl(Path("bad.jsonl"), [{"instruction": "q", "output": "a"}])
    _write_jsonl(Path("good.jsonl"), [{"messages": _CHAT}])

    _loaded_rows("bad.jsonl", "chatml")
    held = last_load_outcome()
    with pytest.raises(dataclasses.FrozenInstanceError):
        held.first_drop = None

    _loaded_rows("good.jsonl", "chatml")

    assert held.first_drop == ("chatml", 0, "'messages'", "bad.jsonl")
    assert last_load_outcome().first_drop is None


def test_the_outcome_names_the_resolved_format_not_auto(tmp_path, monkeypatch):
    monkeypatch.chdir(tmp_path)
    _write_jsonl(Path("data.jsonl"), [{"messages": _CHAT}, {"instruction": "q"}])

    assert len(_loaded_rows("data.jsonl", "auto")) == 1

    outcome = last_load_outcome()
    assert outcome.fmt == "chatml"
    assert outcome.first_drop == ("chatml", 1, "'messages'", "data.jsonl")