Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MLX decision training: Clef rows and predict take images (#13127)

Long Yixingba5239aeaeb4
diff --git a/tests/test_decision_model_mlx.py b/tests/test_decision_model_mlx.py
index c709f846962..48ff1de000c 100644
--- a/tests/test_decision_model_mlx.py
+++ b/tests/test_decision_model_mlx.py
@@ -405,6 +405,92 @@ def test_predict_answers_at_the_calibrated_temperatures(checkpoint, monkeypatch)
FastDecisionModel.predict(model, tokenizer, "s", {})
+def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(
+ clef_checkpoint, checkpoint, monkeypatch, tmp_path
+):
+ import base64
+ import io
+
+ from PIL import Image
+
+ model, tokenizer = FastDecisionModel.from_pretrained(str(clef_checkpoint), max_seq_length = 2048)
+ zoo = unsloth._decision_mlx._decision_zoo()
+ buffer = io.BytesIO()
+ Image.new("RGB", (5, 4)).save(buffer, "PNG")
+ url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+ rows = [
+ {**_row(0), "images": [url, url]},
+ {**_row(1), "images": json.dumps([url])},
+ {**_row(2), "images": [None, {"bytes": None, "path": None}]},
+ ]
+ (tmp_path / "i.png").write_bytes(buffer.getvalue())
+ rows += [
+ {**_row(3), "images": {"bytes": buffer.getvalue()}},
+ {**_row(4), "images": [{"bytes": b"cut"}]},
+ {**_row(5), "images": [{"bytes": None, "path": str(tmp_path / "i.png")}]},
+ ]
+ # This decoder has no vision tower: a row with images is skipped, and says why.
+ items, report = FastDecisionModel.build_dataset(rows, tokenizer, model)
+ assert [item["row"] for item in items] == [2] and "does not support images" in report["reason"]
+
+ seen, real, keep = [], zoo.clef_training_item, [True]
+
+ def item(pipeline, state, questions, max_length, *images):
+ # A row without images is asked as before, with no fifth argument.
+ seen.append([getattr(image, "size", image) for image in images[0]] if images else None)
+ return {
+ **real(pipeline, state, questions, max_length),
+ **({"images": images[0]} if keep and images else {}),
+ }
+
+ monkeypatch.setattr(zoo, "clef_training_item", item)
+ items, report = FastDecisionModel.build_dataset(rows, tokenizer, model)
+ assert [item["row"] for item in items] == [0, 1, 2, 3, 5] and seen == [
+ [url, url],
+ [url],
+ None,
+ [(5, 4)],
+ [(5, 4)],
+ ]
+ assert items[0]["images"] == [url, url]
+ keep.clear()
+ assert "cannot identify image" in report["reason"]
+ bomb = io.BytesIO()
+ Image.new("1", (14000, 14000)).save(bomb, "PNG")
+ items, report = FastDecisionModel.build_dataset(
+ [_row(0), {**_row(1), "images": [{"bytes": bomb.getvalue()}]}], tokenizer, model
+ )
+ assert [item["row"] for item in items] == [0] and "decompression bomb" in report["reason"]
+ FastDecisionModel.predict(
+ model, tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = url
+ )
+ assert seen[-1] == [url]
+ # An unsloth-zoo that takes no images says so, where skipping the rows would train without them.
+ monkeypatch.setattr(
+ zoo,
+ "clef_training_item",
+ lambda pipeline, state, questions, max_length = None: real(
+ pipeline, state, questions, max_length
+ ),
+ )
+ assert len(FastDecisionModel.build_dataset(rows[2:3], tokenizer, model)[0]) == 1
+ question = {"q": {"type": "noul", "instructions": "i"}}
+ for call in (
+ lambda: FastDecisionModel.build_dataset(rows, tokenizer, model),
+ lambda: FastDecisionModel.predict(model, tokenizer, "s", question, images = url),
+ ):
+ with pytest.raises(ImportError, match = "newer unsloth-zoo"):
+ call()
+ laya, laya_tokenizer = FastDecisionModel.from_pretrained(str(checkpoint))
+ with pytest.raises(unsloth._decision_mlx.DecisionDataError, match = "Laya reads text only"):
+ FastDecisionModel.predict(
+ laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = [url]
+ )
+ assert FastDecisionModel.predict(
+ laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = "[]"
+ )
+
+
@pytest.mark.parametrize("four_bit", [False, True])
def test_a_plain_language_model_trains_as_a_clef(clef_checkpoint, tmp_path, four_bit):
for name in ("joint_head.safetensors", "joint_head_config.json"):
diff --git a/unsloth/models/decision_mlx.py b/unsloth/models/decision_mlx.py
index f1b380180db..ecb950c9680 100644
--- a/unsloth/models/decision_mlx.py
+++ b/unsloth/models/decision_mlx.py
@@ -12,6 +12,7 @@ __all__ = [
import copy
import dataclasses
import functools
+import inspect
import importlib.util
import json
import random
@@ -56,9 +57,37 @@ from unsloth._decision_common import (
)
+def _row_images(row: dict) -> list:
+ # A row's "images": PIL images, data URLs, or the {"bytes": ..., "path": ...} an undecoded image column holds.
+ images = _parsed(row.get("images"))
+ if images is None:
+ return []
+ found = []
+ for image in images if isinstance(images, (list, tuple)) else [images]:
+ if isinstance(image, dict) and set(image) <= {"bytes", "path"}:
+ import io
+
+ from PIL import Image
+
+ data = image.get("bytes")
+ # A nullable image column holds an empty struct where a row has no image.
+ try:
+ image = (
+ Image.open(io.BytesIO(data) if data is not None else image["path"])
+ if data is not None or image.get("path")
+ else None
+ )
+ except Image.DecompressionBombError as exc:
+ # Not an OSError: without this one oversized image aborts the whole dataset.
+ raise ValueError(str(exc)) from None
+ if image is not None:
+ found.append(image)
+ return found
+
+
def _decision_clef_items(rows, options: Callable, encode: Callable, validate, report, skip) -> list:
# One item per row. `options(question)` names a question's options in the head's order and
- # `encode(state, questions)` returns the backend's tokenized fields, raising when they do not fit.
+ # `encode(state, questions, images)` returns the backend's tokenized fields, raising when they do not fit.
items = []
for index, row in enumerate(rows):
row = row if isinstance(row, dict) else {}
@@ -92,8 +121,8 @@ def _decision_clef_items(rows, options: Callable, encode: Callable, validate, re
if not kept:
continue
try:
- encoded = encode(state, kept)
- except (TypeError, ValueError) as exc:
+ encoded = encode(state, kept, _row_images(row))
+ except (TypeError, ValueError, OSError) as exc:
for name in kept:
skip(index, f"does not fit: {exc}", name)
continue
@@ -413,12 +442,26 @@ def _decision_pad_token_id(tokenizer) -> int:
return getattr(tokenizer, "tokenizer", tokenizer).pad_token_id
+def _clef_item(zoo, pipeline, state, questions, max_length, images):
+ if not images:
+ return zoo.clef_training_item(pipeline, state, questions, max_length)
+ if "images" not in inspect.signature(zoo.clef_training_item).parameters:
+ # Left to fail as a row that does not fit, a dataset would train without its image rows.
+ raise ImportError(
+ "Unsloth: images in decision rows need a newer unsloth-zoo. "
+ "Upgrade with `pip install -U unsloth-zoo`."
+ )
+ return zoo.clef_training_item(pipeline, state, questions, max_length, images)
+
+
def _clef_items(pipeline, rows, tokenizer, max_len, validate, report, skip) -> list:
zoo = _decision_zoo()
return _decision_clef_items(
rows,
functools.partial(zoo.clef_option_keys, pipeline),
- lambda state, questions: zoo.clef_training_item(pipeline, state, questions, max_len),
+ lambda state, questions, images: _clef_item(
+ zoo, pipeline, state, questions, max_len, images
+ ),
validate,
report,
skip,
@@ -821,8 +864,15 @@ class FastDecisionModel:
split_holdout = staticmethod(_decision_holdout)
@staticmethod
- def predict(model, tokenizer, state, questions: dict) -> dict:
+ def predict(
+ model,
+ tokenizer,
+ state,
+ questions: dict,
+ images = None,
+ ) -> dict:
"""Answers one Decision API request: {name: answer} at the calibrated temperatures.
+ `images` (as in a row: PIL images, data URLs, or bytes / path dicts) are read by a Clef whose base model has a vision tower.
Each answer is the Decision API's (choice / confidence, score / legend, or noul) plus
"answer" (the option, True / False for noul, the level number for score) and
"probabilities" over every option."""
@@ -833,16 +883,19 @@ class FastDecisionModel:
if not isinstance(questions, dict) or not questions:
raise DecisionDataError("questions must be a non-empty dict of name to question")
state, config, zoo = _parsed(state), model.decision_config, _decision_zoo()
+ images = _row_images({"images": images})
if _is_clef(model):
pipeline = model._unsloth_pipeline
questions = {str(name): _clef_question(q) for name, q in questions.items()}
# Read up to CLEF_SERVE_MAX_LEN tokens, like serving, even past the training cut.
max_length = max(int(config.get("max_len", CLEF_MAX_LEN)), CLEF_SERVE_MAX_LEN)
- item = zoo.clef_training_item(pipeline, state, questions, max_length)
+ item = _clef_item(zoo, pipeline, state, questions, max_length, images)
logits = zoo.clef_logits(model, [item])[0]
keys = [zoo.clef_option_keys(pipeline, q) for q in questions.values()]
kinds = [QUESTION_TYPES.index(q["type"]) for q in questions.values()]
else:
+ if images:
+ raise DecisionDataError("Laya reads text only; images need a Clef model.")
common, max_len = _laya().common, int(config.get("max_len", 512))
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
items, keys = [], []