Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

10.5 KB
Raw Blame
"""`soup data mix --optimize` writes a recipe `soup train` cannot load (issue #330).

`render_mix_recipe_yaml` emitted `data.train` as a YAML list, but `DataConfig.train`
was typed `str` — so the recipe it writes for a human to splice into `soup.yaml`
failed `load_config_from_string` with `data -> train: Input should be a valid
string`. #330's fix collapsed `data.train` to the single highest-weighted dataset
from the search, since there was no training-time reader for a real mixture yet.

#443 wired `data.interleave` into `load_dataset()` and widened `DataConfig.train`
to accept a list, so this module's renderer went back to emitting the full
dataset list (the shape #330 originally had to move away from) — it is now a
config `soup train` can actually load. The tests below were updated in place
(renamed where the assertion itself flipped) rather than deleted, since the
schema-loadability guarantee they pin still matters, just against the new shape.
"""

from __future__ import annotations

from soup_cli.config.loader import load_config_from_string
from soup_cli.utils.data_mix import MixOptimizationReport, render_mix_recipe_yaml

from .conftest import strip_ansi


def _make_files(tmp_path, names):
    for n in names:
        (tmp_path / n).write_text("{}\n")


def _report(datasets, weights, tmp_path) -> MixOptimizationReport:
    return MixOptimizationReport(
        datasets=tuple(str(tmp_path / d) for d in datasets),
        candidates=(),
        best_weights=tuple(weights),
        best_eval_loss=0.123,
        partial=False,
        elapsed_seconds=10.0,
    )


def _splice_into_config(data_block_text: str) -> str:
    return "base: test-base\ntask: sft\n" + data_block_text


def test_rendered_recipe_loads_through_config_schema(tmp_path):
    report = _report(["a.jsonl", "b.jsonl"], [0.7, 0.3], tmp_path)
    text = render_mix_recipe_yaml(report)
    # Strip the leading comment lines — only the `data:` block onward is
    # spliced into a real soup.yaml, matching how a human would use it.
    data_block = text[text.index("data:"):]
    cfg = load_config_from_string(_splice_into_config(data_block))
    assert cfg.data.train == [str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")]


def test_train_contains_all_datasets_in_report_order(tmp_path):
    # #443 superseded #330/#442's single-best-dataset collapse now that the
    # trainer can consume a real mixture — this test's old name/assertion
    # ("highest weighted dataset only") is intentionally reversed, not
    # deleted: the full dataset list must survive, in report.datasets order,
    # index-aligned with data.interleave.probs.
    report = _report(
        ["a.jsonl", "b.jsonl", "c.jsonl"], [0.2, 0.55, 0.25], tmp_path
    )
    text = render_mix_recipe_yaml(report)
    data_block = text[text.index("data:"):]
    cfg = load_config_from_string(_splice_into_config(data_block))
    assert cfg.data.train == [
        str(tmp_path / "a.jsonl"),
        str(tmp_path / "b.jsonl"),
        str(tmp_path / "c.jsonl"),
    ]
    assert cfg.data.interleave == {"strategy": "probs", "probs": [0.2, 0.55, 0.25]}


def test_full_ranked_breakdown_still_in_comments(tmp_path):
    # The human-review value of the original recipe (every dataset + its
    # weight) must survive collapsing `data.train` to one path.
    report = _report(["a.jsonl", "b.jsonl"], [0.7, 0.3], tmp_path)
    text = render_mix_recipe_yaml(report)
    assert "0.700000" in text
    assert "0.300000" in text
    assert str(tmp_path / "a.jsonl") in text
    assert str(tmp_path / "b.jsonl") in text


def test_apply_cli_prints_dataset_list_shape_recipe(tmp_path, monkeypatch):
    # #443 flipped the canonical `--optimize` output shape: `train:` is now
    # followed by a YAML list ("-" markers), not a single quoted string —
    # the reverse of what this test asserted pre-#443.
    _make_files(tmp_path, ["a.jsonl", "b.jsonl"])
    monkeypatch.chdir(tmp_path)
    from typer.testing import CliRunner

    from soup_cli.cli import app

    # Force a wide terminal — the datasets resolve to absolute paths under
    # tmp_path, long enough that Rich's default wrapping would otherwise
    # break a path across the 200-char window this test inspects.
    runner = CliRunner(env={"COLUMNS": "300"})
    result = runner.invoke(
        app,
        [
            "data", "mix",
            "--optimize",
            "--datasets", "a.jsonl,b.jsonl",
            "--budget", "60s",
            "--num-probes", "2",
            "--output", "rec.yaml",
        ],
    )
    assert result.exit_code == 0, (result.output, repr(result.exception))
    result = runner.invoke(app, ["data", "mix", "--apply", "rec.yaml"])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    # Rich wraps long lines to the console width, so compare with whitespace
    # collapsed rather than by exact line — "train:" must be followed by a
    # "-" list marker (the new shape), not directly by a path.
    # #633: collapse whitespace AND strip ANSI. Rich splits "train:" across
    # Pygments tokens when colour is enabled, so whitespace collapse alone --
    # the fix that shipped here originally -- leaves escapes mid-token.
    compact = "".join(strip_ansi(result.output).split())
    assert "train:" in compact, result.output
    after = compact[compact.index("train:") + len("train:"):]
    assert after.startswith("-"), result.output
    assert "a.jsonl" in after[:400], result.output
    assert "b.jsonl" in after[:400], result.output


def test_render_recipe_rejects_empty_weights(tmp_path):
    report = _report([], [], tmp_path)
    try:
        render_mix_recipe_yaml(report)
    except ValueError as exc:
        assert "best_weights" in str(exc)
    else:
        raise AssertionError("expected ValueError for empty best_weights")


def test_render_recipe_rejects_mismatched_weights_length(tmp_path):
    report = _report(["a.jsonl", "b.jsonl"], [0.5, 0.3, 0.2], tmp_path)
    try:
        render_mix_recipe_yaml(report)
    except ValueError as exc:
        assert "best_weights" in str(exc)
    else:
        raise AssertionError("expected ValueError for mismatched lengths")


def test_apply_cli_quotes_path_needing_quoting(tmp_path, monkeypatch):
    # Maintainer's repro: an unquoted `train: odd: name.jsonl` line is not
    # valid YAML when pasted back — the --apply echo must quote it the same
    # way the renderer does. Exercises the legacy single-string on-disk
    # shape (pre-#443, or a search over exactly one dataset) — #443's
    # renderer no longer emits single-string form for >= 2 datasets, but
    # --apply must still round-trip old recipe files correctly.
    import yaml

    monkeypatch.chdir(tmp_path)
    (tmp_path / "odd.yaml").write_text(
        'data:\n'
        '  interleave:\n'
        '    strategy: probs\n'
        '    probs:\n'
        '      - 1.000000\n'
        '  train: "odd: name.jsonl"\n'
    )
    from typer.testing import CliRunner

    from soup_cli.cli import app

    runner = CliRunner()
    result = runner.invoke(app, ["data", "mix", "--apply", "odd.yaml"])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    plain = strip_ansi(result.output)  # #633 -- \x1b is not valid YAML
    data_block = plain[plain.index("data:"):]
    loaded = yaml.safe_load(data_block)
    assert loaded["data"]["train"] == "odd: name.jsonl"


def test_apply_handles_pre_fix_list_shaped_recipe(tmp_path, monkeypatch):
    # A recipe written by the pre-#330-fix code (data.train as a YAML list,
    # by accident — there was no training-time reader yet) must still print
    # via --apply rather than crash. This happens to be byte-identical to
    # #443's new intentional list shape from the CLI's point of view (both
    # go through the "else" branch in commands/data_mix.py), but the two
    # are conceptually distinct on-disk provenances, hence keeping this
    # test separate from test_apply_handles_new_multi_dataset_recipe below.
    monkeypatch.chdir(tmp_path)
    (tmp_path / "old.yaml").write_text(
        "data:\n"
        "  interleave:\n"
        "    strategy: probs\n"
        "    probs:\n"
        "      - 0.6\n"
        "      - 0.4\n"
        "  train:\n"
        "    - a.jsonl\n"
        "    - b.jsonl\n"
    )
    from typer.testing import CliRunner

    from soup_cli.cli import app

    runner = CliRunner()
    result = runner.invoke(app, ["data", "mix", "--apply", "old.yaml"])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    assert "a.jsonl" in result.output
    assert "b.jsonl" in result.output


def test_apply_handles_new_multi_dataset_recipe(tmp_path, monkeypatch):
    # #443's canonical current shape: write a real recipe via --optimize
    # (so it goes through render_mix_recipe_yaml, not a hand-written
    # fixture), then confirm --apply round-trips data.train as a list and
    # data.interleave.probs stays index-aligned with it.
    _make_files(tmp_path, ["a.jsonl", "b.jsonl", "c.jsonl"])
    monkeypatch.chdir(tmp_path)
    from rich.console import Console
    from typer.testing import CliRunner

    from soup_cli.cli import app
    from soup_cli.commands import data_mix as data_mix_cmd

    # Force a wide console — tmp_path is long enough on this OS that Rich's
    # default-width line wrapping would otherwise break the path across
    # lines mid-token, which yaml.safe_load below can't parse. COLUMNS in the
    # runner's env is not enough: the module built its console at import, and
    # Rich keeps the width a console was built with when COLUMNS was set at
    # that moment (80 in a pytest-xdist worker on Linux).
    monkeypatch.setattr(data_mix_cmd, "console", Console(width=300))
    runner = CliRunner()
    result = runner.invoke(
        app,
        [
            "data", "mix",
            "--optimize",
            "--datasets", "a.jsonl,b.jsonl,c.jsonl",
            "--budget", "60s",
            "--num-probes", "2",
            "--output", "rec3.yaml",
        ],
    )
    assert result.exit_code == 0, (result.output, repr(result.exception))
    result = runner.invoke(app, ["data", "mix", "--apply", "rec3.yaml"])
    assert result.exit_code == 0, (result.output, repr(result.exception))
    plain = strip_ansi(result.output)  # #633 -- \x1b is not valid YAML
    data_block = plain[plain.index("data:"):]
    from pathlib import Path

    import yaml

    loaded = yaml.safe_load(data_block)
    assert isinstance(loaded["data"]["train"], list)
    # --optimize resolves --datasets to absolute paths under tmp_path.
    basenames = {Path(p).name for p in loaded["data"]["train"]}
    assert basenames == {"a.jsonl", "b.jsonl", "c.jsonl"}
    assert len(loaded["data"]["interleave"]["probs"]) == 3