Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MLX decision training: Clef rows and predict take images (#13127)
Long Yixingba5239aeaeb4
diff --git a/tests/test_decision_model_mlx.py b/tests/test_decision_model_mlx.pyindex c709f846962..48ff1de000c 100644--- a/tests/test_decision_model_mlx.py+++ b/tests/test_decision_model_mlx.py@@ -405,6 +405,92 @@ def test_predict_answers_at_the_calibrated_temperatures(checkpoint, monkeypatch)FastDecisionModel.predict(model, tokenizer, "s", {})+def test_the_images_of_a_row_reach_the_clef_trainer_and_predict(+ clef_checkpoint, checkpoint, monkeypatch, tmp_path+):+ import base64+ import io++ from PIL import Image++ model, tokenizer = FastDecisionModel.from_pretrained(str(clef_checkpoint), max_seq_length = 2048)+ zoo = unsloth._decision_mlx._decision_zoo()+ buffer = io.BytesIO()+ Image.new("RGB", (5, 4)).save(buffer, "PNG")+ url = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()+ rows = [+ {**_row(0), "images": [url, url]},+ {**_row(1), "images": json.dumps([url])},+ {**_row(2), "images": [None, {"bytes": None, "path": None}]},+ ]+ (tmp_path / "i.png").write_bytes(buffer.getvalue())+ rows += [+ {**_row(3), "images": {"bytes": buffer.getvalue()}},+ {**_row(4), "images": [{"bytes": b"cut"}]},+ {**_row(5), "images": [{"bytes": None, "path": str(tmp_path / "i.png")}]},+ ]+ # This decoder has no vision tower: a row with images is skipped, and says why.+ items, report = FastDecisionModel.build_dataset(rows, tokenizer, model)+ assert [item["row"] for item in items] == [2] and "does not support images" in report["reason"]++ seen, real, keep = [], zoo.clef_training_item, [True]++ def item(pipeline, state, questions, max_length, *images):+ # A row without images is asked as before, with no fifth argument.+ seen.append([getattr(image, "size", image) for image in images[0]] if images else None)+ return {+ **real(pipeline, state, questions, max_length),+ **({"images": images[0]} if keep and images else {}),+ }++ monkeypatch.setattr(zoo, "clef_training_item", item)+ items, report = FastDecisionModel.build_dataset(rows, tokenizer, model)+ assert [item["row"] for item in items] == [0, 1, 2, 3, 5] and seen == [+ [url, url],+ [url],+ None,+ [(5, 4)],+ [(5, 4)],+ ]+ assert items[0]["images"] == [url, url]+ keep.clear()+ assert "cannot identify image" in report["reason"]+ bomb = io.BytesIO()+ Image.new("1", (14000, 14000)).save(bomb, "PNG")+ items, report = FastDecisionModel.build_dataset(+ [_row(0), {**_row(1), "images": [{"bytes": bomb.getvalue()}]}], tokenizer, model+ )+ assert [item["row"] for item in items] == [0] and "decompression bomb" in report["reason"]+ FastDecisionModel.predict(+ model, tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = url+ )+ assert seen[-1] == [url]+ # An unsloth-zoo that takes no images says so, where skipping the rows would train without them.+ monkeypatch.setattr(+ zoo,+ "clef_training_item",+ lambda pipeline, state, questions, max_length = None: real(+ pipeline, state, questions, max_length+ ),+ )+ assert len(FastDecisionModel.build_dataset(rows[2:3], tokenizer, model)[0]) == 1+ question = {"q": {"type": "noul", "instructions": "i"}}+ for call in (+ lambda: FastDecisionModel.build_dataset(rows, tokenizer, model),+ lambda: FastDecisionModel.predict(model, tokenizer, "s", question, images = url),+ ):+ with pytest.raises(ImportError, match = "newer unsloth-zoo"):+ call()+ laya, laya_tokenizer = FastDecisionModel.from_pretrained(str(checkpoint))+ with pytest.raises(unsloth._decision_mlx.DecisionDataError, match = "Laya reads text only"):+ FastDecisionModel.predict(+ laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = [url]+ )+ assert FastDecisionModel.predict(+ laya, laya_tokenizer, "s", {"q": {"type": "noul", "instructions": "i"}}, images = "[]"+ )++@pytest.mark.parametrize("four_bit", [False, True])def test_a_plain_language_model_trains_as_a_clef(clef_checkpoint, tmp_path, four_bit):for name in ("joint_head.safetensors", "joint_head_config.json"):diff --git a/unsloth/models/decision_mlx.py b/unsloth/models/decision_mlx.pyindex f1b380180db..ecb950c9680 100644--- a/unsloth/models/decision_mlx.py+++ b/unsloth/models/decision_mlx.py@@ -12,6 +12,7 @@ __all__ = [import copyimport dataclassesimport functools+import inspectimport importlib.utilimport jsonimport random@@ -56,9 +57,37 @@ from unsloth._decision_common import ()+def _row_images(row: dict) -> list:+ # A row's "images": PIL images, data URLs, or the {"bytes": ..., "path": ...} an undecoded image column holds.+ images = _parsed(row.get("images"))+ if images is None:+ return []+ found = []+ for image in images if isinstance(images, (list, tuple)) else [images]:+ if isinstance(image, dict) and set(image) <= {"bytes", "path"}:+ import io++ from PIL import Image++ data = image.get("bytes")+ # A nullable image column holds an empty struct where a row has no image.+ try:+ image = (+ Image.open(io.BytesIO(data) if data is not None else image["path"])+ if data is not None or image.get("path")+ else None+ )+ except Image.DecompressionBombError as exc:+ # Not an OSError: without this one oversized image aborts the whole dataset.+ raise ValueError(str(exc)) from None+ if image is not None:+ found.append(image)+ return found++def _decision_clef_items(rows, options: Callable, encode: Callable, validate, report, skip) -> list:# One item per row. `options(question)` names a question's options in the head's order and- # `encode(state, questions)` returns the backend's tokenized fields, raising when they do not fit.+ # `encode(state, questions, images)` returns the backend's tokenized fields, raising when they do not fit.items = []for index, row in enumerate(rows):row = row if isinstance(row, dict) else {}@@ -92,8 +121,8 @@ def _decision_clef_items(rows, options: Callable, encode: Callable, validate, reif not kept:continuetry:- encoded = encode(state, kept)- except (TypeError, ValueError) as exc:+ encoded = encode(state, kept, _row_images(row))+ except (TypeError, ValueError, OSError) as exc:for name in kept:skip(index, f"does not fit: {exc}", name)continue@@ -413,12 +442,26 @@ def _decision_pad_token_id(tokenizer) -> int:return getattr(tokenizer, "tokenizer", tokenizer).pad_token_id+def _clef_item(zoo, pipeline, state, questions, max_length, images):+ if not images:+ return zoo.clef_training_item(pipeline, state, questions, max_length)+ if "images" not in inspect.signature(zoo.clef_training_item).parameters:+ # Left to fail as a row that does not fit, a dataset would train without its image rows.+ raise ImportError(+ "Unsloth: images in decision rows need a newer unsloth-zoo. "+ "Upgrade with `pip install -U unsloth-zoo`."+ )+ return zoo.clef_training_item(pipeline, state, questions, max_length, images)++def _clef_items(pipeline, rows, tokenizer, max_len, validate, report, skip) -> list:zoo = _decision_zoo()return _decision_clef_items(rows,functools.partial(zoo.clef_option_keys, pipeline),- lambda state, questions: zoo.clef_training_item(pipeline, state, questions, max_len),+ lambda state, questions, images: _clef_item(+ zoo, pipeline, state, questions, max_len, images+ ),validate,report,skip,@@ -821,8 +864,15 @@ class FastDecisionModel:split_holdout = staticmethod(_decision_holdout)@staticmethod- def predict(model, tokenizer, state, questions: dict) -> dict:+ def predict(+ model,+ tokenizer,+ state,+ questions: dict,+ images = None,+ ) -> dict:"""Answers one Decision API request: {name: answer} at the calibrated temperatures.+ `images` (as in a row: PIL images, data URLs, or bytes / path dicts) are read by a Clef whose base model has a vision tower.Each answer is the Decision API's (choice / confidence, score / legend, or noul) plus"answer" (the option, True / False for noul, the level number for score) and"probabilities" over every option."""@@ -833,16 +883,19 @@ class FastDecisionModel:if not isinstance(questions, dict) or not questions:raise DecisionDataError("questions must be a non-empty dict of name to question")state, config, zoo = _parsed(state), model.decision_config, _decision_zoo()+ images = _row_images({"images": images})if _is_clef(model):pipeline = model._unsloth_pipelinequestions = {str(name): _clef_question(q) for name, q in questions.items()}# Read up to CLEF_SERVE_MAX_LEN tokens, like serving, even past the training cut.max_length = max(int(config.get("max_len", CLEF_MAX_LEN)), CLEF_SERVE_MAX_LEN)- item = zoo.clef_training_item(pipeline, state, questions, max_length)+ item = _clef_item(zoo, pipeline, state, questions, max_length, images)logits = zoo.clef_logits(model, [item])[0]keys = [zoo.clef_option_keys(pipeline, q) for q in questions.values()]kinds = [QUESTION_TYPES.index(q["type"]) for q in questions.values()]else:+ if images:+ raise DecisionDataError("Laya reads text only; images need a Clef model.")common, max_len = _laya().common, int(config.get("max_len", 512))tokenizer = getattr(tokenizer, "tokenizer", tokenizer)items, keys = [], []