Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

fix(data): refuse malformed KTO labels (#2126)

Nurkhan Esenbeke5a22c28c9eb
diff --git a/changelog.d/0.76.0/2126.fixed.md b/changelog.d/0.76.0/2126.fixed.md
new file mode 100644
index 00000000..55eb2d4d
--- /dev/null
+++ b/changelog.d/0.76.0/2126.fixed.md
@@ -0,0 +1,4 @@
+- Refuse malformed KTO labels instead of silently training nulls, floats or containers
+as desirable or undesirable rows. Accept booleans, the documented normalized
+true/1/yes and false/0/no strings, and exactly the integers -1, 0 and 1. Validation
+and training report the offending label through the shared converter (#2126; issue #2119).
diff --git a/docs/data.md b/docs/data.md
index 23a4b54b..04f643f5 100644
--- a/docs/data.md
+++ b/docs/data.md
@@ -738,6 +738,15 @@ columns is refused before loading a model, including with `soup train --dry-run`
{"prompt": "Explain gravity", "completion": "Gravity is a force...", "label": true}
```
+`label` accepts booleans; strings `true`, `1`, `yes`, `false`, `0`, `no`
+(ignoring case and surrounding spaces); and integers `1`, `0`, `-1`.
+`1` is desirable; `0` and `-1` are undesirable. Nulls, floats (including
+`1.0` and non-finite numbers), containers and other integers are refused instead
+of coerced. `soup data validate --format kto` reports the offending label;
+training drops malformed rows with the same reason and refuses an empty result,
+including with `--dry-run`. Use `--min-valid-fraction 1.0` to reject any bad row
+in standalone validation.
+
**LLaVA (vision):**
```json
{"image": "photo.jpg", "conversations": [{"from": "human", "value": "<image>\nDescribe this."}, {"from": "gpt", "value": "A cat."}]}
diff --git a/src/soup_cli/data/formats.py b/src/soup_cli/data/formats.py
index 99e74b95..68619dcf 100644
--- a/src/soup_cli/data/formats.py
+++ b/src/soup_cli/data/formats.py
@@ -293,7 +293,9 @@ def format_to_messages(row: dict, fmt: str) -> Optional[dict]:
- Vision formats: {"messages": [...], "image": "path"}
- DPO format: {"prompt": ..., "chosen": ..., "rejected": ...}, the three all
strings or all message lists (see :func:`_convert_dpo`)
- - KTO format: {"prompt": ..., "completion": ..., "label": bool}, likewise
+ - KTO format: {"prompt": ..., "completion": ..., "label": bool}, likewise.
+ Labels accept bool, true/1/yes or false/0/no strings (case/space ignored),
+ and integers 1, 0, -1 only; -1 means undesirable. Other values are refused.
A malformed row is dropped (returns ``None``) rather than raising, so one
bad line does not abort a whole dataset load.
@@ -618,7 +620,9 @@ def _convert_kto(row: dict) -> dict:
(the row a paired file with a string ``prompt`` column gives when unpaired).
"""
raw_label = row["label"]
- if isinstance(raw_label, str):
+ if isinstance(raw_label, bool):
+ label = raw_label
+ elif isinstance(raw_label, str):
low = raw_label.strip().lower()
if low in ("true", "1", "yes"):
label = True
@@ -628,15 +632,15 @@ def _convert_kto(row: dict) -> dict:
raise ValueError(
f"KTO label must be true/false, got string: {raw_label!r}"
)
- elif isinstance(raw_label, bool):
- label = raw_label
- elif isinstance(raw_label, (int, float)):
+ elif isinstance(raw_label, int) and raw_label in (-1, 0, 1):
# Both the ±1 convention (+1 desirable / -1 undesirable) and the 0/1
# convention map "positive == desirable". `bool(-1)` is True, which
# would silently INVERT a -1 "bad" label — flip it to False here.
label = raw_label > 0
else:
- label = bool(raw_label)
+ raise ValueError(
+ f"KTO label must be a bool, true/false string or -1/0/1 integer, got: {raw_label!r}"
+ )
prompt = _preference_field(row.get("prompt"), "kto.prompt")
completion = _preference_field(row.get("completion"), "kto.completion")
diff --git a/tests/test_kto.py b/tests/test_kto.py
index 63f8d42d..a77fe13e 100644
--- a/tests/test_kto.py
+++ b/tests/test_kto.py
@@ -467,13 +467,13 @@ class TestKTODataFormatEdgeCases:
result = format_to_messages(row, "kto")
assert set(result.keys()) == {"prompt", "completion", "label"}
- def test_convert_kto_none_label_coerced_to_false(self):
- """None label should be coerced to False via bool()."""
+ def test_convert_kto_none_label_is_refused(self):
+ """A missing-value label must not train as an undesirable row."""
from soup_cli.data.formats import format_to_messages
row = {"prompt": "Q", "completion": "A", "label": None}
result = format_to_messages(row, "kto")
- assert result["label"] is False
+ assert result is None
# ─── Train Method Guard Test ──────────────────────────────────────────────────
diff --git a/tests/test_kto_labels.py b/tests/test_kto_labels.py
new file mode 100644
index 00000000..c5f342f3
--- /dev/null
+++ b/tests/test_kto_labels.py
@@ -0,0 +1,87 @@
+"""KTO labels have one strict meaning in conversion, validation and training."""
+
+import json
+from unittest.mock import Mock
+
+import pytest
+from typer.testing import CliRunner
+
+from soup_cli.data.formats import _convert_kto, format_to_messages_with_reason
+from tests.conftest import strip_ansi
+
+ACCEPTED = [
+ (True, True), (False, False), (1, True), (0, False), (-1, False),
+ ("true", True), ("1", True), ("yes", True),
+ ("false", False), ("0", False), ("no", False),
+ (" TRUE ", True), (" YeS ", True), (" FALSE ", False), (" No ", False),
+]
+REFUSED = [None, 0.5, 1.0, 0.0, -1.0, float("nan"), float("inf"),
+ float("-inf"), 2, -2, [], ["no"], {}, {"label": True}, "maybe", "-1"]
+
+
[email protected]("value,expected", ACCEPTED)
+def test_accepted_kto_label_spellings(value, expected):
+ converted = _convert_kto({"prompt": "Q", "completion": "A", "label": value})
+ assert converted == {
+ "prompt": "Q", "completion": "A", "label": expected,
+ }
+ assert converted["label"] is expected
+
+
[email protected]("value", REFUSED)
+def test_kto_refuses_non_labels_with_the_value(value):
+ row = {"prompt": "Q", "completion": "A", "label": value}
+ with pytest.raises(ValueError, match="label") as error:
+ _convert_kto(row)
+ assert repr(value) in str(error.value)
+ converted, reason = format_to_messages_with_reason(row, "kto")
+ assert converted is None
+ assert "label" in reason and repr(value) in reason
+
+
[email protected]("value", REFUSED)
[email protected]("command", ["validate", "dry-run"])
+def test_cli_refuses_invalid_kto_label_before_model_load(tmp_path, monkeypatch, value, command):
+ from soup_cli.cli import app
+ from soup_cli.trainer import dispatch
+
+ monkeypatch.chdir(tmp_path)
+ (tmp_path / "data.jsonl").write_text(
+ json.dumps({"prompt": "Q", "completion": "A", "label": value}) + "\n",
+ encoding="utf-8",
+ )
+ (tmp_path / "soup.yaml").write_text(
+ "base: public/model\ntask: kto\n"
+ "data: {train: data.jsonl, format: kto, val_split: 0}\n"
+ "training: {quantization: none}\noutput: out\n", encoding="utf-8",
+ )
+ build = Mock(side_effect=AssertionError("invalid label reached a trainer"))
+ monkeypatch.setattr(dispatch, "build_trainer", build)
+ args = (["data", "validate", "data.jsonl", "--format", "kto"] if command == "validate"
+ else ["train", "--config", "soup.yaml", "--dry-run"])
+ result = CliRunner().invoke(app, args)
+ output = " ".join(strip_ansi(result.output).split())
+ assert result.exit_code == (2 if command == "validate" else 1), (output, result.exception)
+ assert "label" in output and repr(value) in output, output
+ build.assert_not_called()
+
+
[email protected]("value", [False, -1, " No "])
+def test_valid_undesirable_label_passes_cli_preflight(tmp_path, monkeypatch, value):
+ from soup_cli.cli import app
+
+ monkeypatch.chdir(tmp_path)
+ (tmp_path / "data.jsonl").write_text(
+ json.dumps({"prompt": "Q", "completion": "A", "label": value}) + "\n",
+ encoding="utf-8",
+ )
+ (tmp_path / "soup.yaml").write_text(
+ "base: public/model\ntask: kto\n"
+ "data: {train: data.jsonl, format: kto, val_split: 0}\n"
+ "training: {quantization: none}\noutput: out\n", encoding="utf-8",
+ )
+ result = CliRunner().invoke(app, ["data", "validate", "data.jsonl", "--format", "kto"])
+ assert result.exit_code == 0, strip_ansi(result.output)
+ result = CliRunner().invoke(app, ["train", "--config", "soup.yaml", "--dry-run"])
+ assert result.exit_code == 0, strip_ansi(result.output)
+ assert "Ready to train" in strip_ansi(result.output)