Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for soup runs CLI commands."""
from pathlib import Path
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.experiment.tracker import ExperimentTracker
runner = CliRunner()
@pytest.fixture(autouse=True)
def _use_temp_db(tmp_path: Path, monkeypatch):
"""Redirect experiment DB to a temp directory for all tests."""
db_path = str(tmp_path / "test_experiments.db")
monkeypatch.setenv("SOUP_DB_PATH", db_path)
@pytest.fixture
def tracker(tmp_path: Path) -> ExperimentTracker:
"""Create tracker using the same temp DB."""
import os
db_path = Path(os.environ["SOUP_DB_PATH"])
return ExperimentTracker(db_path=db_path)
def test_runs_empty():
"""soup runs with no runs should show a message, not crash."""
result = runner.invoke(app, ["runs"])
assert result.exit_code == 0
assert "No runs found" in result.output
def test_runs_list_with_data(tracker):
"""soup runs should show a table when runs exist."""
run_id = tracker.start_run(
config_dict={"base": "test-model", "task": "sft"},
device="cpu",
device_name="CPU",
gpu_info={"memory_total": "N/A"},
experiment_name="test-exp",
)
result = runner.invoke(app, ["runs"])
assert result.exit_code == 0
assert "Training Runs" in result.output
# Run ID column has no_wrap so it should always be visible
assert run_id in result.output
def test_runs_show_not_found():
"""soup runs show nonexistent should fail."""
result = runner.invoke(app, ["runs", "show", "nonexistent_run_id"])
assert result.exit_code == 1
assert "not found" in result.output.lower()
def test_runs_show_with_data(tracker):
"""soup runs show should display run details."""
run_id = tracker.start_run(
config_dict={"base": "test-model", "task": "sft"},
device="cpu",
device_name="CPU",
gpu_info={"memory_total": "16 GB"},
)
tracker.finish_run(
run_id=run_id,
initial_loss=2.5,
final_loss=0.8,
total_steps=100,
duration_secs=600.0,
output_dir="/tmp/output",
)
result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
assert result.exit_code == 0
assert run_id in result.output
assert "test-model" in result.output
def test_runs_show_a_terminated_run_reads_terminated_not_running(tracker):
"""#764/#767 review: the status branch had a silent 'else: running'
catch-all, so a row _reconcile_orphaned_run (#401) had already
rewritten to 'terminated' still displayed as 'running' in `soup runs
show` — the database and the CLI disagreed about the same run."""
import subprocess
import sys
run_id = tracker.start_run(
config_dict={"base": "test-model", "task": "sft"},
device="cpu",
device_name="CPU",
gpu_info={"memory_total": "16 GB"},
)
proc = subprocess.Popen([sys.executable, "-c", "pass"])
proc.wait()
tracker.mark_running(run_id, pid=proc.pid)
assert tracker.get_run(run_id)["status"] == "terminated"
result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
assert result.exit_code == 0
assert "terminated" in result.output.lower()
assert "running" not in result.output.lower()
def test_runs_show_surfaces_the_stored_error_message(tracker):
"""error_message was written by fail_run since #764, but nothing read
it — a failed run's recorded reason was sitting in the database with
no way to see it from the CLI."""
run_id = tracker.start_run(
config_dict={"base": "test-model", "task": "sft"},
device="cpu",
device_name="CPU",
gpu_info={"memory_total": "16 GB"},
)
tracker.fail_run(run_id, error="RuntimeError: simulated tokenizer load failure")
result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
assert result.exit_code == 0
assert "RuntimeError" in result.output
assert "simulated tokenizer load failure" in result.output
def test_runs_show_strips_control_bytes_from_the_error_message(tracker, monkeypatch):
"""#767 follow-up review: error_message is exception text, not a
config-derived string, so it can carry a remote error body verbatim.
markup_escape alone only neutralises '[...]' -- a raw ESC (window-title
or SGR color) reaches the terminal untouched. for_terminal strips C0
control bytes first, so neither escape sequence should survive."""
from rich.console import Console
from soup_cli.commands import runs as runs_command
# Pin tty detection so this asserts that for_terminal sanitised the
# error_message, not that the ambient shell happens not to force colour
# on the surrounding panel markup.
monkeypatch.setattr(
runs_command, "console", Console(force_terminal=False)
)
run_id = tracker.start_run(
config_dict={"base": "test-model", "task": "sft"},
device="cpu",
device_name="CPU",
gpu_info={"memory_total": "16 GB"},
)
tracker.fail_run(
run_id,
error="boom \x1b]0;HIJACKED\x07 and \x1b[31mRED\x1b[0m tail",
)
result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
assert result.exit_code == 0
assert "\x1b" not in result.output
assert "boom" in result.output
assert "HIJACKED" in result.output
assert "tail" in result.output
def test_runs_show_omits_the_error_line_when_there_is_none(tracker):
"""Control: a run with no error_message must not print a stray 'Error:'
line — the field is genuinely absent for completed/running runs, not
an empty string worth displaying."""
run_id = tracker.start_run(
config_dict={"base": "test-model", "task": "sft"},
device="cpu",
device_name="CPU",
gpu_info={"memory_total": "16 GB"},
)
result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
assert result.exit_code == 0
assert "Error:" not in result.output
def test_runs_compare_not_found():
"""soup runs compare with bad IDs should fail."""
result = runner.invoke(app, ["runs", "compare", "run_1", "run_2"])
assert result.exit_code == 1
def test_runs_compare_with_data(tracker):
"""soup runs compare should show side-by-side table."""
id1 = tracker.start_run(
config_dict={"base": "model-a", "task": "sft", "training": {"epochs": 3, "lr": 2e-5}},
device="cpu", device_name="CPU", gpu_info={},
experiment_name="exp-a",
)
id2 = tracker.start_run(
config_dict={"base": "model-b", "task": "dpo", "training": {"epochs": 5, "lr": 1e-5}},
device="cuda", device_name="RTX 4090", gpu_info={},
experiment_name="exp-b",
)
result = runner.invoke(app, ["runs", "compare", id1, id2])
assert result.exit_code == 0
assert "model-a" in result.output
assert "model-b" in result.output
def test_runs_delete_not_found():
"""soup runs delete nonexistent should fail."""
result = runner.invoke(app, ["runs", "delete", "nonexistent", "--force"])
assert result.exit_code == 1
def test_runs_delete_with_data(tracker):
"""soup runs delete should remove the run."""
run_id = tracker.start_run(
config_dict={}, device="cpu", device_name="CPU", gpu_info={},
)
result = runner.invoke(app, ["runs", "delete", run_id, "--force"])
assert result.exit_code == 0
assert "Deleted" in result.output
# Verify it's gone
assert tracker.get_run(run_id) is None
def test_runs_clean_not_found():
"""soup runs clean with bad IDs should fail."""
result = runner.invoke(app, ["runs", "clean", "nonexistent"])
assert result.exit_code == 1
def test_runs_clean_with_data(tracker, tmp_path):
"""soup runs clean should reclaim space natively."""
# Note: Using Path.cwd() / "test_output" to satisfy security check that paths stay under CWD
out_dir = Path.cwd() / "test_output_clean"
if out_dir.exists():
import shutil
shutil.rmtree(out_dir)
out_dir.mkdir(parents=True)
try:
# create some dummy checkpoints
ckpt1 = out_dir / "checkpoint-100"
ckpt1.mkdir()
(ckpt1 / "optimizer.pt").write_text("dummy")
(ckpt1 / "adapter_model.bin").write_text("model")
ckpt2 = out_dir / "checkpoint-200"
ckpt2.mkdir()
(ckpt2 / "optimizer.pt").write_text("dummy")
(ckpt2 / "adapter_model.bin").write_text("model")
run_id = tracker.start_run(
config_dict={}, device="cpu", device_name="CPU", gpu_info={},
)
tracker.finish_run(
run_id=run_id,
initial_loss=2.0,
final_loss=1.0, # let's say step 200 has lower loss
total_steps=200,
duration_secs=100.0,
output_dir=str(out_dir),
)
# mock metrics: step 200 is best
tracker.log_metrics(run_id, step=100, loss=2.0)
tracker.log_metrics(run_id, step=200, loss=1.0)
result = runner.invoke(app, ["runs", "clean", run_id, "--force"])
assert result.exit_code == 0
assert "Successfully reclaimed" in result.output
# verify optimizer in ckpt1 is gone, but in ckpt2 it's potentially kept or not
# based on keep-weights
assert not (ckpt1 / "optimizer.pt").exists()
assert (ckpt2 / "optimizer.pt").exists()
# test --all
result = runner.invoke(app, ["runs", "clean", "--all", "--force"])
assert "No disposable checkpoint files found" in result.output
finally:
import shutil
if out_dir.exists():
shutil.rmtree(out_dir)