Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
fix(data): refuse malformed KTO labels (#2126)
Nurkhan Esenbeke5a22c28c9eb
diff --git a/changelog.d/0.76.0/2126.fixed.md b/changelog.d/0.76.0/2126.fixed.mdnew file mode 100644index 00000000..55eb2d4d--- /dev/null+++ b/changelog.d/0.76.0/2126.fixed.md@@ -0,0 +1,4 @@+- Refuse malformed KTO labels instead of silently training nulls, floats or containers+as desirable or undesirable rows. Accept booleans, the documented normalized+true/1/yes and false/0/no strings, and exactly the integers -1, 0 and 1. Validation+and training report the offending label through the shared converter (#2126; issue #2119).diff --git a/docs/data.md b/docs/data.mdindex 23a4b54b..04f643f5 100644--- a/docs/data.md+++ b/docs/data.md@@ -738,6 +738,15 @@ columns is refused before loading a model, including with `soup train --dry-run`{"prompt": "Explain gravity", "completion": "Gravity is a force...", "label": true}```+`label` accepts booleans; strings `true`, `1`, `yes`, `false`, `0`, `no`+(ignoring case and surrounding spaces); and integers `1`, `0`, `-1`.+`1` is desirable; `0` and `-1` are undesirable. Nulls, floats (including+`1.0` and non-finite numbers), containers and other integers are refused instead+of coerced. `soup data validate --format kto` reports the offending label;+training drops malformed rows with the same reason and refuses an empty result,+including with `--dry-run`. Use `--min-valid-fraction 1.0` to reject any bad row+in standalone validation.+**LLaVA (vision):**```json{"image": "photo.jpg", "conversations": [{"from": "human", "value": "<image>\nDescribe this."}, {"from": "gpt", "value": "A cat."}]}diff --git a/src/soup_cli/data/formats.py b/src/soup_cli/data/formats.pyindex 99e74b95..68619dcf 100644--- a/src/soup_cli/data/formats.py+++ b/src/soup_cli/data/formats.py@@ -293,7 +293,9 @@ def format_to_messages(row: dict, fmt: str) -> Optional[dict]:- Vision formats: {"messages": [...], "image": "path"}- DPO format: {"prompt": ..., "chosen": ..., "rejected": ...}, the three allstrings or all message lists (see :func:`_convert_dpo`)- - KTO format: {"prompt": ..., "completion": ..., "label": bool}, likewise+ - KTO format: {"prompt": ..., "completion": ..., "label": bool}, likewise.+ Labels accept bool, true/1/yes or false/0/no strings (case/space ignored),+ and integers 1, 0, -1 only; -1 means undesirable. Other values are refused.A malformed row is dropped (returns ``None``) rather than raising, so onebad line does not abort a whole dataset load.@@ -618,7 +620,9 @@ def _convert_kto(row: dict) -> dict:(the row a paired file with a string ``prompt`` column gives when unpaired)."""raw_label = row["label"]- if isinstance(raw_label, str):+ if isinstance(raw_label, bool):+ label = raw_label+ elif isinstance(raw_label, str):low = raw_label.strip().lower()if low in ("true", "1", "yes"):label = True@@ -628,15 +632,15 @@ def _convert_kto(row: dict) -> dict:raise ValueError(f"KTO label must be true/false, got string: {raw_label!r}")- elif isinstance(raw_label, bool):- label = raw_label- elif isinstance(raw_label, (int, float)):+ elif isinstance(raw_label, int) and raw_label in (-1, 0, 1):# Both the ±1 convention (+1 desirable / -1 undesirable) and the 0/1# convention map "positive == desirable". `bool(-1)` is True, which# would silently INVERT a -1 "bad" label — flip it to False here.label = raw_label > 0else:- label = bool(raw_label)+ raise ValueError(+ f"KTO label must be a bool, true/false string or -1/0/1 integer, got: {raw_label!r}"+ )prompt = _preference_field(row.get("prompt"), "kto.prompt")completion = _preference_field(row.get("completion"), "kto.completion")diff --git a/tests/test_kto.py b/tests/test_kto.pyindex 63f8d42d..a77fe13e 100644--- a/tests/test_kto.py+++ b/tests/test_kto.py@@ -467,13 +467,13 @@ class TestKTODataFormatEdgeCases:result = format_to_messages(row, "kto")assert set(result.keys()) == {"prompt", "completion", "label"}- def test_convert_kto_none_label_coerced_to_false(self):- """None label should be coerced to False via bool()."""+ def test_convert_kto_none_label_is_refused(self):+ """A missing-value label must not train as an undesirable row."""from soup_cli.data.formats import format_to_messagesrow = {"prompt": "Q", "completion": "A", "label": None}result = format_to_messages(row, "kto")- assert result["label"] is False+ assert result is None# ─── Train Method Guard Test ──────────────────────────────────────────────────diff --git a/tests/test_kto_labels.py b/tests/test_kto_labels.pynew file mode 100644index 00000000..c5f342f3--- /dev/null+++ b/tests/test_kto_labels.py@@ -0,0 +1,87 @@+"""KTO labels have one strict meaning in conversion, validation and training."""++import json+from unittest.mock import Mock++import pytest+from typer.testing import CliRunner++from soup_cli.data.formats import _convert_kto, format_to_messages_with_reason+from tests.conftest import strip_ansi++ACCEPTED = [+ (True, True), (False, False), (1, True), (0, False), (-1, False),+ ("true", True), ("1", True), ("yes", True),+ ("false", False), ("0", False), ("no", False),+ (" TRUE ", True), (" YeS ", True), (" FALSE ", False), (" No ", False),+]+REFUSED = [None, 0.5, 1.0, 0.0, -1.0, float("nan"), float("inf"),+ float("-inf"), 2, -2, [], ["no"], {}, {"label": True}, "maybe", "-1"]++[email protected]("value,expected", ACCEPTED)+def test_accepted_kto_label_spellings(value, expected):+ converted = _convert_kto({"prompt": "Q", "completion": "A", "label": value})+ assert converted == {+ "prompt": "Q", "completion": "A", "label": expected,+ }+ assert converted["label"] is expected++[email protected]("value", REFUSED)+def test_kto_refuses_non_labels_with_the_value(value):+ row = {"prompt": "Q", "completion": "A", "label": value}+ with pytest.raises(ValueError, match="label") as error:+ _convert_kto(row)+ assert repr(value) in str(error.value)+ converted, reason = format_to_messages_with_reason(row, "kto")+ assert converted is None+ assert "label" in reason and repr(value) in reason++[email protected]("value", REFUSED)[email protected]("command", ["validate", "dry-run"])+def test_cli_refuses_invalid_kto_label_before_model_load(tmp_path, monkeypatch, value, command):+ from soup_cli.cli import app+ from soup_cli.trainer import dispatch++ monkeypatch.chdir(tmp_path)+ (tmp_path / "data.jsonl").write_text(+ json.dumps({"prompt": "Q", "completion": "A", "label": value}) + "\n",+ encoding="utf-8",+ )+ (tmp_path / "soup.yaml").write_text(+ "base: public/model\ntask: kto\n"+ "data: {train: data.jsonl, format: kto, val_split: 0}\n"+ "training: {quantization: none}\noutput: out\n", encoding="utf-8",+ )+ build = Mock(side_effect=AssertionError("invalid label reached a trainer"))+ monkeypatch.setattr(dispatch, "build_trainer", build)+ args = (["data", "validate", "data.jsonl", "--format", "kto"] if command == "validate"+ else ["train", "--config", "soup.yaml", "--dry-run"])+ result = CliRunner().invoke(app, args)+ output = " ".join(strip_ansi(result.output).split())+ assert result.exit_code == (2 if command == "validate" else 1), (output, result.exception)+ assert "label" in output and repr(value) in output, output+ build.assert_not_called()++[email protected]("value", [False, -1, " No "])+def test_valid_undesirable_label_passes_cli_preflight(tmp_path, monkeypatch, value):+ from soup_cli.cli import app++ monkeypatch.chdir(tmp_path)+ (tmp_path / "data.jsonl").write_text(+ json.dumps({"prompt": "Q", "completion": "A", "label": value}) + "\n",+ encoding="utf-8",+ )+ (tmp_path / "soup.yaml").write_text(+ "base: public/model\ntask: kto\n"+ "data: {train: data.jsonl, format: kto, val_split: 0}\n"+ "training: {quantization: none}\noutput: out\n", encoding="utf-8",+ )+ result = CliRunner().invoke(app, ["data", "validate", "data.jsonl", "--format", "kto"])+ assert result.exit_code == 0, strip_ansi(result.output)+ result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--dry-run"])+ assert result.exit_code == 0, strip_ansi(result.output)+ assert "Ready to train" in strip_ansi(result.output)