Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

9.5 KB
Raw Blame
"""Tests for soup runs CLI commands."""

from pathlib import Path

import pytest
from typer.testing import CliRunner

from soup_cli.cli import app
from soup_cli.experiment.tracker import ExperimentTracker

runner = CliRunner()


@pytest.fixture(autouse=True)
def _use_temp_db(tmp_path: Path, monkeypatch):
    """Redirect experiment DB to a temp directory for all tests."""
    db_path = str(tmp_path / "test_experiments.db")
    monkeypatch.setenv("SOUP_DB_PATH", db_path)


@pytest.fixture
def tracker(tmp_path: Path) -> ExperimentTracker:
    """Create tracker using the same temp DB."""
    import os

    db_path = Path(os.environ["SOUP_DB_PATH"])
    return ExperimentTracker(db_path=db_path)


def test_runs_empty():
    """soup runs with no runs should show a message, not crash."""
    result = runner.invoke(app, ["runs"])
    assert result.exit_code == 0
    assert "No runs found" in result.output


def test_runs_list_with_data(tracker):
    """soup runs should show a table when runs exist."""
    run_id = tracker.start_run(
        config_dict={"base": "test-model", "task": "sft"},
        device="cpu",
        device_name="CPU",
        gpu_info={"memory_total": "N/A"},
        experiment_name="test-exp",
    )
    result = runner.invoke(app, ["runs"])
    assert result.exit_code == 0
    assert "Training Runs" in result.output
    # Run ID column has no_wrap so it should always be visible
    assert run_id in result.output


def test_runs_show_not_found():
    """soup runs show nonexistent should fail."""
    result = runner.invoke(app, ["runs", "show", "nonexistent_run_id"])
    assert result.exit_code == 1
    assert "not found" in result.output.lower()


def test_runs_show_with_data(tracker):
    """soup runs show should display run details."""
    run_id = tracker.start_run(
        config_dict={"base": "test-model", "task": "sft"},
        device="cpu",
        device_name="CPU",
        gpu_info={"memory_total": "16 GB"},
    )
    tracker.finish_run(
        run_id=run_id,
        initial_loss=2.5,
        final_loss=0.8,
        total_steps=100,
        duration_secs=600.0,
        output_dir="/tmp/output",
    )
    result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
    assert result.exit_code == 0
    assert run_id in result.output
    assert "test-model" in result.output


def test_runs_show_a_terminated_run_reads_terminated_not_running(tracker):
    """#764/#767 review: the status branch had a silent 'else: running'
    catch-all, so a row _reconcile_orphaned_run (#401) had already
    rewritten to 'terminated' still displayed as 'running' in `soup runs
    show` — the database and the CLI disagreed about the same run."""
    import subprocess
    import sys

    run_id = tracker.start_run(
        config_dict={"base": "test-model", "task": "sft"},
        device="cpu",
        device_name="CPU",
        gpu_info={"memory_total": "16 GB"},
    )
    proc = subprocess.Popen([sys.executable, "-c", "pass"])
    proc.wait()
    tracker.mark_running(run_id, pid=proc.pid)

    assert tracker.get_run(run_id)["status"] == "terminated"

    result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
    assert result.exit_code == 0
    assert "terminated" in result.output.lower()
    assert "running" not in result.output.lower()


def test_runs_show_surfaces_the_stored_error_message(tracker):
    """error_message was written by fail_run since #764, but nothing read
    it — a failed run's recorded reason was sitting in the database with
    no way to see it from the CLI."""
    run_id = tracker.start_run(
        config_dict={"base": "test-model", "task": "sft"},
        device="cpu",
        device_name="CPU",
        gpu_info={"memory_total": "16 GB"},
    )
    tracker.fail_run(run_id, error="RuntimeError: simulated tokenizer load failure")

    result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
    assert result.exit_code == 0
    assert "RuntimeError" in result.output
    assert "simulated tokenizer load failure" in result.output


def test_runs_show_strips_control_bytes_from_the_error_message(tracker, monkeypatch):
    """#767 follow-up review: error_message is exception text, not a
    config-derived string, so it can carry a remote error body verbatim.
    markup_escape alone only neutralises '[...]' -- a raw ESC (window-title
    or SGR color) reaches the terminal untouched. for_terminal strips C0
    control bytes first, so neither escape sequence should survive."""
    from rich.console import Console

    from soup_cli.commands import runs as runs_command

    # Pin tty detection so this asserts that for_terminal sanitised the
    # error_message, not that the ambient shell happens not to force colour
    # on the surrounding panel markup.
    monkeypatch.setattr(
        runs_command, "console", Console(force_terminal=False)
    )

    run_id = tracker.start_run(
        config_dict={"base": "test-model", "task": "sft"},
        device="cpu",
        device_name="CPU",
        gpu_info={"memory_total": "16 GB"},
    )
    tracker.fail_run(
        run_id,
        error="boom \x1b]0;HIJACKED\x07 and \x1b[31mRED\x1b[0m tail",
    )

    result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
    assert result.exit_code == 0
    assert "\x1b" not in result.output
    assert "boom" in result.output
    assert "HIJACKED" in result.output
    assert "tail" in result.output


def test_runs_show_omits_the_error_line_when_there_is_none(tracker):
    """Control: a run with no error_message must not print a stray 'Error:'
    line — the field is genuinely absent for completed/running runs, not
    an empty string worth displaying."""
    run_id = tracker.start_run(
        config_dict={"base": "test-model", "task": "sft"},
        device="cpu",
        device_name="CPU",
        gpu_info={"memory_total": "16 GB"},
    )

    result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
    assert result.exit_code == 0
    assert "Error:" not in result.output


def test_runs_compare_not_found():
    """soup runs compare with bad IDs should fail."""
    result = runner.invoke(app, ["runs", "compare", "run_1", "run_2"])
    assert result.exit_code == 1


def test_runs_compare_with_data(tracker):
    """soup runs compare should show side-by-side table."""
    id1 = tracker.start_run(
        config_dict={"base": "model-a", "task": "sft", "training": {"epochs": 3, "lr": 2e-5}},
        device="cpu", device_name="CPU", gpu_info={},
        experiment_name="exp-a",
    )
    id2 = tracker.start_run(
        config_dict={"base": "model-b", "task": "dpo", "training": {"epochs": 5, "lr": 1e-5}},
        device="cuda", device_name="RTX 4090", gpu_info={},
        experiment_name="exp-b",
    )
    result = runner.invoke(app, ["runs", "compare", id1, id2])
    assert result.exit_code == 0
    assert "model-a" in result.output
    assert "model-b" in result.output


def test_runs_delete_not_found():
    """soup runs delete nonexistent should fail."""
    result = runner.invoke(app, ["runs", "delete", "nonexistent", "--force"])
    assert result.exit_code == 1


def test_runs_delete_with_data(tracker):
    """soup runs delete should remove the run."""
    run_id = tracker.start_run(
        config_dict={}, device="cpu", device_name="CPU", gpu_info={},
    )
    result = runner.invoke(app, ["runs", "delete", run_id, "--force"])
    assert result.exit_code == 0
    assert "Deleted" in result.output
    # Verify it's gone
    assert tracker.get_run(run_id) is None


def test_runs_clean_not_found():
    """soup runs clean with bad IDs should fail."""
    result = runner.invoke(app, ["runs", "clean", "nonexistent"])
    assert result.exit_code == 1


def test_runs_clean_with_data(tracker, tmp_path):
    """soup runs clean should reclaim space natively."""
    # Note: Using Path.cwd() / "test_output" to satisfy security check that paths stay under CWD
    out_dir = Path.cwd() / "test_output_clean"
    if out_dir.exists():
        import shutil
        shutil.rmtree(out_dir)
    out_dir.mkdir(parents=True)

    try:
        # create some dummy checkpoints
        ckpt1 = out_dir / "checkpoint-100"
        ckpt1.mkdir()
        (ckpt1 / "optimizer.pt").write_text("dummy")
        (ckpt1 / "adapter_model.bin").write_text("model")

        ckpt2 = out_dir / "checkpoint-200"
        ckpt2.mkdir()
        (ckpt2 / "optimizer.pt").write_text("dummy")
        (ckpt2 / "adapter_model.bin").write_text("model")

        run_id = tracker.start_run(
            config_dict={}, device="cpu", device_name="CPU", gpu_info={},
        )
        tracker.finish_run(
            run_id=run_id,
            initial_loss=2.0,
            final_loss=1.0, # let's say step 200 has lower loss
            total_steps=200,
            duration_secs=100.0,
            output_dir=str(out_dir),
        )
        # mock metrics: step 200 is best
        tracker.log_metrics(run_id, step=100, loss=2.0)
        tracker.log_metrics(run_id, step=200, loss=1.0)

        result = runner.invoke(app, ["runs", "clean", run_id, "--force"])
        assert result.exit_code == 0
        assert "Successfully reclaimed" in result.output

        # verify optimizer in ckpt1 is gone, but in ckpt2 it's potentially kept or not
        # based on keep-weights
        assert not (ckpt1 / "optimizer.pt").exists()
        assert (ckpt2 / "optimizer.pt").exists()

        # test --all
        result = runner.invoke(app, ["runs", "clean", "--all", "--force"])
        assert "No disposable checkpoint files found" in result.output
    finally:
        import shutil
        if out_dir.exists():
            shutil.rmtree(out_dir)