Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

18.0 KB
Raw Blame
"""Tests for Issue #1219: classifier/reranker/cross_encoder data loading and validation.

Verifies:
1. Source column retention via shared helper `task_preserves_source_columns`.
2. Format detection and conversion for `cross_encoder` (both text_a/text_b and question/answer).
3. Upfront label and shape validation with specific row indices on --dry-run and at load.
4. All variants: string labels, integer labels, multi-label lists, chatml with label,
   reranker, cross_encoder (with both auto and explicit formats).
5. Documented example end-to-end training through `soup train`.
"""

from __future__ import annotations

import ast
import json
from pathlib import Path
from typing import Any

import pytest
from typer.testing import CliRunner

from soup_cli.cli import app
from soup_cli.config.loader import load_config_from_string
from soup_cli.data.formats import (
    FORMAT_SIGNATURES,
    VALID_FORMATS,
    detect_format,
    format_to_messages,
)
from soup_cli.data.loader import (
    PRESERVE_SOURCE_TASKS,
    data_config_for_task,
    load_dataset,
    task_preserves_source_columns,
)
from soup_cli.trainer.classifier import (
    ClassifierTrainerWrapper,
    validate_classification_dataset,
)
from tests._windows_ci import skip_on_windows_ci
from tests.conftest import strip_ansi


def _strip_ansi(text: str) -> str:
    """Strip ANSI color codes and collapse whitespace for resilient assertions."""
    plain = strip_ansi(text)
    return " ".join(plain.split())


def _local_base_model_dir() -> str:
    """A tiny Llama causal LM + tokenizer built locally (no download),
    standing in for hf-internal-testing/tiny-random-LlamaForCausalLM -- #1356
    found these tests pulling it from the Hub just to initialise a
    classification head on top, which a randomly-initialised base serves
    identically. Shared and cached across every file that needs one
    (tests/_tiny_hf_models.py)."""
    from tests._tiny_hf_models import tiny_model_dir

    return tiny_model_dir("llama")


class TestTaskPreservesSourceColumns:
    """One helper decides which tasks keep source columns; train and sweep must agree."""

    def test_expected_tasks_in_preserve_set(self) -> None:
        assert PRESERVE_SOURCE_TASKS == {
            "grpo",
            "classifier",
            "reranker",
            "cross_encoder",
        }

    @pytest.mark.parametrize(
        ("task", "expected"),
        [
            ("grpo", True),
            ("classifier", True),
            ("reranker", True),
            ("cross_encoder", True),
            ("sft", False),
            ("dpo", False),
            ("kto", False),
            ("orpo", False),
            ("reward_model", False),
            ("pretrain", False),
            ("embedding", False),
            ("distill", False),
            ("unlearn", False),
        ],
    )
    def test_helper_predicate_values(self, task: str, expected: bool) -> None:
        assert task_preserves_source_columns(task) is expected

    def test_train_and_sweep_use_same_helper_ast(self) -> None:
        """Audit AST of train.py and sweep.py: both must call task_preserves_source_columns."""
        repo_root = Path(__file__).resolve().parents[1]
        train_py = (repo_root / "src/soup_cli/commands/train.py").read_text(encoding="utf-8")
        sweep_py = (repo_root / "src/soup_cli/commands/sweep.py").read_text(encoding="utf-8")

        # Neither file should have the old hardcoded cfg.task == "grpo"
        assert 'preserve_source_columns=cfg.task == "grpo"' not in train_py
        assert 'preserve_source_columns=cfg.task == "grpo"' not in sweep_py

        # Both must call task_preserves_source_columns
        assert "task_preserves_source_columns(cfg.task)" in train_py
        assert "task_preserves_source_columns(cfg.task)" in sweep_py

        # Both must import task_preserves_source_columns from soup_cli.data.loader
        train_ast = ast.parse(train_py)
        sweep_ast = ast.parse(sweep_py)

        def finds_import(tree: ast.AST, target_name: str) -> bool:
            for node in ast.walk(tree):
                if isinstance(node, ast.ImportFrom) and node.module == "soup_cli.data.loader":
                    for name in node.names:
                        if name.name == target_name:
                            return True
            return False

        assert finds_import(train_ast, "task_preserves_source_columns"), (
            "train.py must import task_preserves_source_columns"
        )
        assert finds_import(sweep_ast, "task_preserves_source_columns"), (
            "sweep.py must import task_preserves_source_columns"
        )


class TestCrossEncoderFormat:
    """Format detection and conversion for paired sequence classification."""

    def test_signatures_and_valid_formats(self) -> None:
        assert "cross_encoder" not in FORMAT_SIGNATURES
        assert "cross_encoder" in VALID_FORMATS

    def test_detect_format_text_a_text_b(self) -> None:
        rows = [{"text_a": "query", "text_b": "passage", "label": 1}]
        with pytest.raises(ValueError, match="Cannot detect format"):
            detect_format(rows)

    def test_detect_format_question_answer(self) -> None:
        rows = [{"question": "what is this?", "answer": "it is a cat", "label": 0}]
        with pytest.raises(ValueError, match="Cannot detect format"):
            detect_format(rows)

    def test_conversion_preserves_pairs_and_label(self) -> None:
        row_ab = {"text_a": "first", "text_b": "second", "label": "relevant", "meta": 123}
        out_ab = format_to_messages(row_ab, "cross_encoder")
        assert out_ab is not None
        assert out_ab["text_a"] == "first"
        assert out_ab["text_b"] == "second"
        assert out_ab["label"] == "relevant"

        row_qa = {"question": "q?", "answer": "a!", "label": 1}
        out_qa = format_to_messages(row_qa, "cross_encoder")
        assert out_qa is not None
        assert out_qa["question"] == "q?"
        assert out_qa["answer"] == "a!"
        assert out_qa["label"] == 1

    def test_conversion_rejects_non_string_pairs(self) -> None:
        assert format_to_messages({"text_a": 123, "text_b": "good"}, "cross_encoder") is None
        bad_qa = {"question": "q", "answer": ["not", "string"]}
        assert format_to_messages(bad_qa, "cross_encoder") is None


class TestLabelValidationUpfront:
    """Missing or invalid labels must fail at load and on --dry-run with row index."""

    def test_validate_missing_label_names_row_index(self) -> None:
        cfg = load_config_from_string(
            """
            base: sshleifer/tiny-gpt2
            task: classifier
            training:
              num_labels: 2
            data:
              train: ./fake.jsonl
            """
        )
        dataset = {
            "train": [
                {"text": "good message", "label": 1},
                {"text": "bad message missing label"},
                {"text": "another good message", "label": 0},
            ]
        }
        with pytest.raises(ValueError, match=r"train row 1: missing required 'label' field"):
            validate_classification_dataset(cfg, dataset)

    def test_validate_invalid_label_type_names_row_index(self) -> None:
        cfg = load_config_from_string(
            """
            base: sshleifer/tiny-gpt2
            task: classifier
            training:
              num_labels: 2
            data:
              train: ./fake.jsonl
            """
        )
        dataset = {
            "train": [
                {"text": "message", "label": True},
            ]
        }
        with pytest.raises(ValueError, match=r"train row 0: label must not be bool"):
            validate_classification_dataset(cfg, dataset)

    def test_dry_run_refuses_missing_label_with_row_index(self, tmp_path: Path) -> None:
        data_file = tmp_path / "data.jsonl"
        data_file.write_text(
            json.dumps({"text": "sample 0", "label": "ham"}) + "\n"
            + json.dumps({"text": "sample 1"}) + "\n"
        )
        cfg_file = tmp_path / "soup.yaml"
        cfg_file.write_text(
            f"""
            base: sshleifer/tiny-gpt2
            task: classifier
            data:
              train: {data_file.as_posix()}
              max_length: 64
              val_split: 0.0
            training:
              num_labels: 2
              label_names: [ham, spam]
            """
        )
        runner = CliRunner()
        res = runner.invoke(app, ["train", "--config", str(cfg_file), "--dry-run"])
        assert res.exit_code == 1
        clean = _strip_ansi(res.output)
        assert "train row 1" in clean
        assert "missing required 'label' field" in clean

    def test_train_load_refuses_missing_label_before_model_loads(self, tmp_path: Path) -> None:
        data_file = tmp_path / "data.jsonl"
        data_file.write_text(
            json.dumps({"text": "sample 0"}) + "\n"
        )
        cfg_file = tmp_path / "soup.yaml"
        cfg_file.write_text(
            f"""
            base: invalid-model-that-would-fail-if-loaded
            task: classifier
            data:
              train: {data_file.as_posix()}
              max_length: 64
              val_split: 0.0
            training:
              num_labels: 2
              label_names: [ham, spam]
            """
        )
        runner = CliRunner()
        res = runner.invoke(app, ["train", "--config", str(cfg_file), "--yes"])
        assert res.exit_code == 1
        clean = _strip_ansi(res.output)
        assert "train row 0" in clean
        assert "missing required 'label' field" in clean
        assert "Setting up model + trainer" not in clean


class TestDatasetVariantsLoading:
    """Verify all variants preserve source columns and setup succeeds."""

    @pytest.mark.parametrize(
        ("fmt", "rows", "task", "training_dict", "expected_labels"),
        [
            # String labels with label_names (auto format)
            (
                "auto",
                [{"text": f"text {i}", "label": "spam" if i % 2 else "ham"} for i in range(4)],
                "classifier",
                {"num_labels": 2, "label_names": ["ham", "spam"]},
                [0, 1, 0, 1],
            ),
            # String labels with explicit plaintext format
            (
                "plaintext",
                [{"text": f"text {i}", "label": "spam" if i % 2 else "ham"} for i in range(4)],
                "classifier",
                {"num_labels": 2, "label_names": ["ham", "spam"]},
                [0, 1, 0, 1],
            ),
            # Integer labels (auto format)
            (
                "auto",
                [{"text": f"text {i}", "label": i % 3} for i in range(6)],
                "classifier",
                {"num_labels": 3},
                [0, 1, 2, 0, 1, 2],
            ),
            # Integer labels with explicit plaintext format
            (
                "plaintext",
                [{"text": f"text {i}", "label": i % 3} for i in range(6)],
                "classifier",
                {"num_labels": 3},
                [0, 1, 2, 0, 1, 2],
            ),
            # Multi-label lists (auto format)
            (
                "auto",
                [{"text": f"text {i}", "label": [0, 2]} for i in range(3)],
                "classifier",
                {"num_labels": 3, "classifier_kind": "multi_label"},
                [[1.0, 0.0, 1.0], [1.0, 0.0, 1.0], [1.0, 0.0, 1.0]],
            ),
            # Multi-label lists with label names
            (
                "plaintext",
                [{"text": f"text {i}", "label": ["ham", "promo"]} for i in range(3)],
                "classifier",
                {
                    "num_labels": 3,
                    "classifier_kind": "multi_label",
                    "label_names": ["ham", "spam", "promo"],
                },
                [[1.0, 0.0, 1.0], [1.0, 0.0, 1.0], [1.0, 0.0, 1.0]],
            ),
            # ChatML rows carrying label (auto format)
            (
                "auto",
                [
                    {
                        "messages": [{"role": "user", "content": f"hello {i}"}],
                        "label": "spam" if i % 2 else "ham",
                    }
                    for i in range(4)
                ],
                "classifier",
                {"num_labels": 2, "label_names": ["ham", "spam"]},
                [0, 1, 0, 1],
            ),
            # ChatML rows carrying label (explicit chatml format)
            (
                "chatml",
                [
                    {
                        "messages": [{"role": "user", "content": f"hello {i}"}],
                        "label": "spam" if i % 2 else "ham",
                    }
                    for i in range(4)
                ],
                "classifier",
                {"num_labels": 2, "label_names": ["ham", "spam"]},
                [0, 1, 0, 1],
            ),
            # task: reranker (auto format)
            (
                "auto",
                [{"text": f"query doc {i}", "label": "relevant"} for i in range(3)],
                "reranker",
                {"num_labels": 1, "label_names": ["relevant"]},
                [0, 0, 0],
            ),
            # task: reranker (explicit plaintext format)
            (
                "plaintext",
                [{"text": f"query doc {i}", "label": "relevant"} for i in range(3)],
                "reranker",
                {"num_labels": 1, "label_names": ["relevant"]},
                [0, 0, 0],
            ),
            # task: cross_encoder with text_a and text_b (auto format)
            (
                "auto",
                [{"text_a": f"q {i}", "text_b": f"doc {i}", "label": i % 2} for i in range(4)],
                "cross_encoder",
                {"num_labels": 2},
                [0, 1, 0, 1],
            ),
            # task: cross_encoder with text_a and text_b (explicit cross_encoder format)
            (
                "cross_encoder",
                [{"text_a": f"q {i}", "text_b": f"doc {i}", "label": i % 2} for i in range(4)],
                "cross_encoder",
                {"num_labels": 2},
                [0, 1, 0, 1],
            ),
            # task: cross_encoder with question and answer (auto format)
            (
                "auto",
                [{"question": f"q {i}", "answer": f"ans {i}", "label": 1} for i in range(3)],
                "cross_encoder",
                {"num_labels": 2},
                [1, 1, 1],
            ),
            # task: cross_encoder with question and answer (explicit cross_encoder format)
            (
                "cross_encoder",
                [{"question": f"q {i}", "answer": f"ans {i}", "label": 1} for i in range(3)],
                "cross_encoder",
                {"num_labels": 2},
                [1, 1, 1],
            ),
        ],
    )
    def test_load_dataset_retains_columns_and_validates(
        self,
        tmp_path: Path,
        fmt: str,
        rows: list[dict[str, Any]],
        task: str,
        training_dict: dict[str, Any],
        expected_labels: list[Any],
    ) -> None:
        import yaml

        data_file = tmp_path / f"test_{task}_{fmt}.jsonl"
        data_file.write_text("\n".join(json.dumps(r) for r in rows) + "\n")

        cfg_obj = {
            "base": _local_base_model_dir(),
            "task": task,
            "data": {
                "train": data_file.as_posix(),
                "format": fmt,
                "val_split": 0.0,
                "max_length": 64,
            },
            "training": training_dict,
        }
        cfg = load_config_from_string(yaml.safe_dump(cfg_obj))

        # 1. load_dataset with task_preserves_source_columns and data_config_for_task
        dataset = load_dataset(
            data_config_for_task(cfg.data, cfg.task),
            preserve_source_columns=task_preserves_source_columns(cfg.task),
        )
        assert len(dataset["train"]) == len(rows)
        for row in dataset["train"]:
            assert "label" in row

        # 2. validate_classification_dataset passes
        validate_classification_dataset(cfg, dataset)

        # 3. Setup wrapper
        wrapper = ClassifierTrainerWrapper(cfg, device="cpu")
        wrapper.setup(dataset)
        assert wrapper.train_dataset is not None
        assert wrapper.train_dataset["labels"] == expected_labels


class TestDocumentedExampleTraining:
    """End-to-end training of the documented classifier example."""

    @skip_on_windows_ci
    def test_documented_classifier_example_trains_and_records_labels(
        self, tmp_path: Path
    ) -> None:
        labels = ["ham", "spam", "promo"]
        data_path = tmp_path / "labelled.jsonl"
        data_path.write_text(
            "\n".join(
                json.dumps({"text": f"message {i}", "label": labels[i % 3]})
                for i in range(6)
            )
            + "\n"
        )
        out_dir = tmp_path / "out"
        cfg_file = tmp_path / "cls.yaml"
        cfg_file.write_text(
            f"""
            base: {Path(_local_base_model_dir()).as_posix()}
            task: classifier
            data:
              train: {data_path.as_posix()}
              max_length: 64
              val_split: 0.0
            training:
              num_labels: 3
              label_names: [ham, spam, promo]
              quantization: none
              epochs: 1
              batch_size: 2
            output: {out_dir.as_posix()}
            """
        )
        runner = CliRunner()
        res = runner.invoke(app, ["train", "--config", str(cfg_file), "--yes"])
        clean = _strip_ansi(res.output)
        assert res.exit_code == 0, f"Command failed: {res.output}"
        assert "Auto-detected format: plaintext" in clean
        assert "Loaded: 6 train samples" in clean
        assert "Training Complete!" in clean

        # Verify saved model config recorded the labels
        saved_config_path = out_dir / "config.json"
        assert saved_config_path.exists(), "config.json should be saved in output directory"
        saved_cfg = json.loads(saved_config_path.read_text(encoding="utf-8"))
        assert saved_cfg.get("id2label") == {"0": "ham", "1": "spam", "2": "promo"}
        assert saved_cfg.get("label2id") == {"ham": 0, "spam": 1, "promo": 2}