Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Regression tests for ``soup data validate`` exit codes (#811)."""
from __future__ import annotations
import json
from pathlib import Path
import pytest
from click.testing import Result
from typer.testing import CliRunner
from soup_cli.cli import app
from soup_cli.data.formats import VALID_FORMATS
from .conftest import strip_ansi
def _write_alpaca(path: Path, rows: list[dict]) -> None:
path.write_text(
"".join(json.dumps(row) + "\n" for row in rows),
encoding="utf-8",
)
def _invoke_validate(path: Path, *extra_args: str) -> Result:
return CliRunner().invoke(
app,
["data", "validate", str(path), "--format", "alpaca", *extra_args],
)
def test_validate_exits_two_when_all_rows_are_unusable(tmp_path: Path) -> None:
path = tmp_path / "invalid.jsonl"
_write_alpaca(path, [{"instruction": None, "output": None}] * 5)
result = _invoke_validate(path)
output = strip_ansi(result.output)
assert result.exit_code == 2
assert "0/5 rows valid" in output
assert "no usable rows remain" in output
def test_validate_allows_partially_valid_data_by_default(tmp_path: Path) -> None:
path = tmp_path / "partial.jsonl"
_write_alpaca(
path,
[
{"instruction": "Summarize this", "output": "A summary"},
{"instruction": None, "output": None},
],
)
result = _invoke_validate(path)
assert result.exit_code == 0, result.output
assert "1/2 rows valid" in strip_ansi(result.output)
@pytest.mark.parametrize(
("minimum", "expected_exit_code"),
[("0.75", 2), ("0.5", 0)],
)
def test_validate_applies_minimum_fraction(
tmp_path: Path,
minimum: str,
expected_exit_code: int,
) -> None:
path = tmp_path / "partial.jsonl"
_write_alpaca(
path,
[
{"instruction": "Summarize this", "output": "A summary"},
{"instruction": None, "output": None},
],
)
result = _invoke_validate(path, "--min-valid-fraction", minimum)
assert result.exit_code == expected_exit_code, result.output
if expected_exit_code == 2:
assert "valid fraction 0.500 is below --min-valid-fraction 0.750" in strip_ansi(
result.output
)
def test_validate_keeps_input_errors_at_exit_three(tmp_path: Path) -> None:
missing = _invoke_validate(tmp_path / "missing.jsonl")
assert missing.exit_code == 3
assert "File not found" in strip_ansi(missing.output)
unknown = tmp_path / "unknown.jsonl"
unknown.write_text('{"unrecognized": "shape"}\n', encoding="utf-8")
undetectable = CliRunner().invoke(app, ["data", "validate", str(unknown)])
assert undetectable.exit_code == 3
assert "Cannot detect format" in strip_ansi(undetectable.output)
def test_validate_rejects_an_unknown_format(tmp_path: Path) -> None:
"""#866: --format accepted any string and reported every row valid for
it. An unknown format is an input error (exit 3), not the failed-gate
exit code (2) #858 settled on for '0 usable rows' -- the two must stay
distinguishable so a CI gate can tell a typo'd flag from real bad data."""
path = tmp_path / "two_alpaca_rows.jsonl"
_write_alpaca(
path,
[
{"instruction": "Summarize this", "output": "A summary long enough"},
{"instruction": "Another one", "output": "Another summary here"},
],
)
result = CliRunner().invoke(
app, ["data", "validate", str(path), "--format", "bogus"]
)
output = strip_ansi(result.output)
assert result.exit_code == 3, output
assert "bogus" in output
assert "alpaca" in output # names at least one accepted format
@pytest.mark.parametrize("fmt", VALID_FORMATS)
def test_validate_still_accepts_every_valid_format(tmp_path: Path, fmt: str) -> None:
"""The guard above must reject unknown values without narrowing what
was already accepted. Rows need not be semantically valid for every
format here -- only that --format itself is not rejected before
validate_and_stats ever runs."""
path = tmp_path / "rows.jsonl"
path.write_text('{"anything": "goes"}\n', encoding="utf-8")
result = CliRunner().invoke(
app, ["data", "validate", str(path), "--format", fmt]
)
assert "Unknown --format" not in strip_ansi(result.output), result.output