Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""Tests for Issue #1434: ``soup expect`` fails closed on unreadable JSONL input.
A UTF-8 BOM must be stripped like the training loader does, malformed JSON
lines and non-object rows must be reported (not silently dropped), and a run
that checked zero rows must not pass.
"""
from __future__ import annotations
from pathlib import Path
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
from tests.conftest import strip_ansi
PII_LINE = b'{"text": "mail [email protected]"}\n'
SUITE_YAML = "expectations:\n - name: expect_no_pii\n"
FILES = {
"bom.jsonl": b"\xef\xbb\xbf" + PII_LINE + b'{"text": "hi"}\n',
"nobom.jsonl": PII_LINE + b'{"text": "hi"}\n',
"array_per_line.jsonl": b'[{"text": "mail [email protected]"}]\n',
"non_object_rows.jsonl": b'"mail [email protected]"\n42\nnull\n',
"all_malformed.jsonl": b"{not json\n{still: not}\n",
"empty.jsonl": b"",
}
def _run(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str, content: bytes):
(tmp_path / "suite.yaml").write_text(SUITE_YAML, encoding="utf-8")
(tmp_path / name).write_bytes(content)
monkeypatch.chdir(tmp_path)
return CliRunner().invoke(app, ["expect", name, "suite.yaml"])
def test_bom_file_matches_no_bom_control(tmp_path, monkeypatch):
"""The BOM-prefixed file reports the PII in row 1 and exits 2, like the control."""
with_bom = _run(tmp_path, monkeypatch, "bom.jsonl", FILES["bom.jsonl"])
assert with_bom.exit_code == 2
clean = " ".join(strip_ansi(with_bom.output).split())
assert "[email protected]" in clean or "FAIL" in clean
control = _run(tmp_path, monkeypatch, "nobom.jsonl", FILES["nobom.jsonl"])
assert control.exit_code == 2
clean = " ".join(strip_ansi(control.output).split())
assert "[email protected]" in clean or "FAIL" in clean
@pytest.mark.parametrize(
"name, expected_fragment",
[
("empty.jsonl", "no checkable rows"),
("all_malformed.jsonl", "malformed JSON"),
("array_per_line.jsonl", "not a JSON object"),
("non_object_rows.jsonl", "not a JSON object"),
],
)
def test_uncheckable_files_fail_nonzero(tmp_path, monkeypatch, name, expected_fragment):
"""Files with no checkable row exit non-zero and say why lines could not be checked."""
result = _run(tmp_path, monkeypatch, name, FILES[name])
assert result.exit_code == 3
clean = " ".join(strip_ansi(result.output).split())
assert expected_fragment in clean
assert "PASS" not in clean
def test_single_malformed_line_reported_with_line_number(tmp_path, monkeypatch):
"""One malformed line among valid rows is reported with its line number."""
content = b'{"text": "hi"}\n{broken\n{"text": "ok"}\n'
result = _run(tmp_path, monkeypatch, "mixed.jsonl", content)
assert result.exit_code == 3
clean = " ".join(strip_ansi(result.output).split())
assert "line 2" in clean
assert "malformed JSON" in clean
def test_refusal_is_exit_3_not_a_suite_failure(tmp_path, monkeypatch):
result = _run(tmp_path, monkeypatch, "bad.jsonl", b'{"text": "hi"}\n{broken\n')
assert result.exit_code == 3, result.output
def test_refusal_message_counts_the_uncheckable_lines(tmp_path, monkeypatch):
result = _run(tmp_path, monkeypatch, "bad.jsonl", b"{not json\n{still: not}\n")
assert result.exit_code == 3
clean = " ".join(strip_ansi(result.output).split())
assert "2 line(s) could not be checked" in clean
def test_refusal_lists_at_most_ten_lines_and_counts_the_rest(tmp_path, monkeypatch):
content = b'{"text": "ok"}\n' + b"".join(b"{bad%d\n" % i for i in range(12))
result = _run(tmp_path, monkeypatch, "twelve.jsonl", content)
assert result.exit_code == 3
clean = " ".join(strip_ansi(result.output).split())
assert "12 line(s) could not be checked" in clean
assert "line 11: malformed JSON" in clean
assert "line 12" not in clean
assert "(+2 more)" in clean