Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO: forward every vision kwarg, and accept an image column holding a list (#11031)

Daniel Han448bf6b05667
diff --git a/pyproject.toml b/pyproject.toml
index 729faf125bc..e4d299e9794 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -210,6 +210,10 @@ huggingfacenotorch = [
# gate on the INSTALLED zoo rather than on this pin, so they start running by themselves
# wherever 2026.9.5 or newer is the zoo actually present.
#
+ # This branch needs that same floor for a second, independent reason: 2026.9.5 is also the
+ # first release to export `grpo_vision_chunks`, which both vision GRPO logprob paths in
+ # unsloth/models/rl_replacements.py refuse to run without.
+ #
# This does NOT lift the transformers ceiling below. 2026.9.5 still declares
# `transformers<=5.5.0` in its own metadata, so the resolver intersects our transformers
# range down to 5.5.0 whatever this file says. That one waits on
diff --git a/tests/test_grpo_hidden_states_signal.py b/tests/test_grpo_hidden_states_signal.py
index 0eba609444e..4f21950566c 100644
--- a/tests/test_grpo_hidden_states_signal.py
+++ b/tests/test_grpo_hidden_states_signal.py
@@ -323,8 +323,7 @@ def _run_padded_loop(*, returns_hidden_states, signal):
"pixel_values": None,
"lm_head": lm_head,
"zipped_inputs": [
- (input_ids[i : i + 1], torch.ones(1, SEQ, dtype = torch.long)) + (None,) * 6
- for i in range(BATCH)
+ (input_ids[i : i + 1], torch.ones(1, SEQ, dtype = torch.long), {}) for i in range(BATCH)
],
"logits_to_keep": KEEP,
"max_left_pad": MAX_LEFT_PAD,
diff --git a/tests/test_grpo_image_column_normalisation.py b/tests/test_grpo_image_column_normalisation.py
new file mode 100644
index 00000000000..b226f1f7238
--- /dev/null
+++ b/tests/test_grpo_image_column_normalisation.py
@@ -0,0 +1,394 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A singular `image` column holding a list takes the plural path: TRL wraps the cell
+again, so two images reach the processor as [[[img, img]]]. unslothai/unsloth#3605."""
+
+import textwrap
+
+import pytest
+
+from unsloth.models.rl_replacements import (
+ _unsloth_grpo_image_cell,
+ _unsloth_reject_grpo_image_list,
+ grpo_trainer__generate_and_score_completions,
+)
+
+
+class _Img:
+ def __init__(self, name):
+ self.name = name
+
+ def __repr__(self):
+ return f"<{self.name}>"
+
+
+def test_cell_helper_keeps_a_list_and_wraps_a_single_image():
+ a, b = _Img("a"), _Img("b")
+ assert _unsloth_grpo_image_cell(None) is None
+ assert _unsloth_grpo_image_cell(a) == [a]
+ assert _unsloth_grpo_image_cell([a, b]) == [a, b]
+ assert _unsloth_grpo_image_cell((a, b)) == [a, b]
+ assert _unsloth_grpo_image_cell([a, b]) != [[a, b]]
+
+
+def test_the_reporter_shape_is_flattened_not_nested():
+ a, b = _Img("a"), _Img("b")
+ inputs = [{"image": [a, b]}, {"image": [a, b]}]
+ images = [_unsloth_grpo_image_cell(example.get("image")) for example in inputs]
+ assert images == [[a, b], [a, b]]
+ before = [[example.get("image")] for example in inputs]
+ assert before == [[[a, b]], [[a, b]]]
+
+
+def test_single_image_cells_are_unchanged():
+ a = _Img("a")
+ inputs = [{"image": a}, {"image": a}]
+ assert [_unsloth_grpo_image_cell(x.get("image")) for x in inputs] == [[a], [a]]
+
+
+def test_patcher_rewrites_the_installed_trl_extraction():
+ """Read TRL's file, not the class: importing unsloth already replaced the method."""
+ trl_grpo = pytest.importorskip("trl.trainer.grpo_trainer")
+ with open(trl_grpo.__file__, "r", encoding = "utf-8") as fh:
+ module_source = fh.read()
+ start = module_source.find(" def _generate_and_score_completions(")
+ assert start != -1, "TRL renamed _generate_and_score_completions"
+ source = module_source[start:]
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert "_unsloth_grpo_image_cell" in patched or "_unsloth_reject_grpo_image_list" in patched
+ if "_unsloth_grpo_image_cell" in patched:
+ assert '[[example.get("image")] if example.get("image") is not None' not in patched
+ assert 'kwargs = {"images": [[img] for img in images]}' not in patched
+
+
+def test_trl_0_22_placeholder_count_follows_the_cell():
+ source = (
+ ' has_images = "image" in inputs[0]\n'
+ " if has_images:\n"
+ ' images = [example.get("image") for example in inputs]\n'
+ ' kwargs = {"images": [[img] for img in images]}\n'
+ " for prompt in prompts:\n"
+ " if isinstance(prompt, list): # i.e., when using conversational data\n"
+ " prepare_multimodal_messages(prompt, num_images=1)\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert "_unsloth_grpo_image_cell(img)" in patched
+ assert "num_images=1)" not in patched
+ assert "len(_unsloth_cell)" in patched
+
+ a, b = _Img("a"), _Img("b")
+ calls = []
+
+ def prepare_multimodal_messages(prompt, num_images):
+ calls.append(num_images)
+
+ namespace = {
+ "inputs": [{"image": [a, b]}],
+ "prompts": [[{"role": "user", "content": "x"}]],
+ "prepare_multimodal_messages": prepare_multimodal_messages,
+ "_unsloth_grpo_image_cell": _unsloth_grpo_image_cell,
+ }
+ exec(textwrap.dedent(patched), namespace)
+ assert namespace["kwargs"]["images"] == [[a, b]]
+ assert calls == [2]
+
+
+def test_guard_names_the_column_and_the_issue():
+ a, b = _Img("a"), _Img("b")
+ _unsloth_reject_grpo_image_list([{"image": a}])
+ _unsloth_reject_grpo_image_list([{"image": [a]}])
+ _unsloth_reject_grpo_image_list([{"prompt": "x"}])
+ _unsloth_reject_grpo_image_list([])
+ with pytest.raises(ValueError) as excinfo:
+ _unsloth_reject_grpo_image_list([{"image": [a, b]}])
+ message = str(excinfo.value)
+ assert "`images`" in message
+ assert "3605" in message
+
+
+def test_the_guard_reads_every_row_not_only_the_first():
+ """One list cell anywhere puts that row's images and placeholders out of step, and a
+ dataset mixing a bare image with a list is exactly the shape that puts the list somewhere
+ other than row 0. Reading only `inputs[0]` would let it through to the processor, which is
+ the outcome this guard exists to replace."""
+ a, b = _Img("a"), _Img("b")
+ for rows in (
+ [{"image": [a, b]}, {"image": a}],
+ [{"image": a}, {"image": [a, b]}],
+ [{"image": a}, {"image": a}, {"image": [a, b]}],
+ [{"image": None}, {"image": [a, b]}],
+ ):
+ with pytest.raises(ValueError) as excinfo:
+ _unsloth_reject_grpo_image_list(rows)
+ assert "3605" in str(excinfo.value)
+
+ # Nothing that works today starts failing, including shapes the guard must not choke on.
+ for rows in (
+ [{"image": a}, {"image": a}],
+ [{"image": [a]}, {"image": [a]}],
+ [{"prompt": "x"}, {"prompt": "y"}],
+ ["not a dict", 7],
+ [],
+ None,
+ ):
+ _unsloth_reject_grpo_image_list(rows)
+
+
+def test_guard_is_injected_when_no_anchor_matches():
+ source = " def _generate_and_score_completions(self, inputs):\n return inputs\n"
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert "_unsloth_reject_grpo_image_list(inputs)" in patched
+ lines = patched.splitlines()
+ assert lines[1].strip() == "_unsloth_reject_grpo_image_list(inputs)"
+
+
+def test_guard_is_injected_when_the_legacy_reference_calls_drift():
+ """Rewriting the cell is half the job. A legacy TRL also has to carry the image counts
+ into the reference logprob calls; without them the images are sliced by sample index and
+ all but the first are dropped, which the model reports as a token/feature mismatch."""
+ source = (
+ " def _generate_and_score_completions(self, inputs):\n"
+ ' has_images = "image" in inputs[0]\n'
+ " if has_images:\n"
+ ' images = [example.get("image") for example in inputs]\n'
+ ' kwargs = {"images": [[img] for img in images]}\n'
+ " ref = self._get_per_token_logps_and_entropies(\n"
+ " self.model,\n"
+ ' pixel_values=prompt_inputs.get("pixel_values"), image_grid_thw=None,\n'
+ " )\n"
+ " return inputs\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert "_unsloth_grpo_image_cell(img)" in patched
+ assert "_unsloth_reject_grpo_image_list(inputs)" in patched
+ lines = patched.splitlines()
+ assert lines[1].strip() == "_unsloth_reject_grpo_image_list(inputs)"
+
+
+def test_guard_stays_off_a_trl_whose_reference_calls_do_take_the_counts():
+ """The installed TRL is fully plumbed, so a multi image row works and must not be refused."""
+ trl_grpo = pytest.importorskip("trl.trainer.grpo_trainer")
+ with open(trl_grpo.__file__, "r", encoding = "utf-8") as fh:
+ module_source = fh.read()
+ start = module_source.find(" def _generate_and_score_completions(")
+ assert start != -1, "TRL renamed _generate_and_score_completions"
+ source = module_source[start:]
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert "_unsloth_reject_grpo_image_list(inputs)" not in patched
+ if 'pixel_values=prompt_inputs.get("pixel_values")' in source:
+ # A legacy TRL: every reference call site must have taken the counts.
+ assert 'pixel_values=prompt_inputs.get("pixel_values")' not in patched
+ assert "**_unsloth_legacy_vision," in patched
+
+
+def _legacy_source(*, with_placeholder_helper):
+ """TRL 0.20.0/0.21.0 against TRL 0.22.x. Both take the legacy `[[img] for img in images]`
+ cell spelling; only 0.22.x factored the placeholders out into a helper that takes a count.
+ 0.20.0 and 0.21.0 inline one {"type": "image"} per user message, with no count at all."""
+ head = (
+ " def _generate_and_score_completions(self, inputs):\n"
+ " prompts = [x['prompt'] for x in inputs]\n"
+ " kwargs = {}\n"
+ ' has_images = "image" in inputs[0]\n'
+ " if has_images:\n"
+ ' images = [example.get("image") for example in inputs]\n'
+ ' kwargs = {"images": [[img] for img in images]}\n'
+ )
+ if with_placeholder_helper:
+ placeholders = (
+ " for prompt in prompts:\n"
+ " if isinstance(prompt, list): # i.e., when using conversational data\n"
+ " prepare_multimodal_messages(prompt, num_images=1)\n"
+ )
+ else:
+ placeholders = (
+ " for prompt in prompts:\n"
+ " if isinstance(prompt, list):\n"
+ " for message in prompt:\n"
+ " if message.get('role') == 'user':\n"
+ " message['content'] = [{'type': 'image'}, message['content']]\n"
+ )
+ tail = (
+ " ref = self._get_per_token_logps_and_entropies(\n"
+ " self.model,\n"
+ ' pixel_values=prompt_inputs.get("pixel_values"),\n'
+ ' image_grid_thw=prompt_inputs.get("image_grid_thw"),\n'
+ ' pixel_attention_mask=prompt_inputs.get("pixel_attention_mask"),\n'
+ ' image_sizes=prompt_inputs.get("image_sizes"),\n'
+ " )\n"
+ " return inputs\n"
+ )
+ return head + placeholders + tail
+
+
+def test_a_trl_that_cannot_size_its_placeholders_refuses_the_multi_image_row():
+ """The cell rewrite alone would hand the processor two images while the prompt still
+ carries one placeholder, and nothing else in the batch disagrees, because the prologue
+ counts the same cells. That surfaces inside the processor naming neither the column nor
+ the fix, which is exactly what the guard exists to replace."""
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", _legacy_source(with_placeholder_helper = False)
+ )
+ assert "_unsloth_grpo_image_cell(img)" in patched
+ assert "_unsloth_reject_grpo_image_list(inputs)" in patched
+ assert patched.splitlines()[1].strip() == "_unsloth_reject_grpo_image_list(inputs)"
+
+
+def test_a_trl_that_can_size_its_placeholders_is_not_refused():
+ """The control, and the reason the guard cannot simply key on the legacy cell spelling:
+ 0.22.x takes the same spelling and does size its placeholders, so it must keep working."""
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", _legacy_source(with_placeholder_helper = True)
+ )
+ assert "_unsloth_grpo_image_cell(img)" in patched
+ assert "len(_unsloth_cell) if _unsloth_cell else 0" in patched
+ assert "_unsloth_reject_grpo_image_list(inputs)" not in patched
+
+
+def _run_legacy_placeholders(cells):
+ """Execute the rewritten TRL 0.22.x block and report what the processor would be handed."""
+ source = (
+ ' has_images = "image" in inputs[0]\n'
+ " if has_images:\n"
+ ' images = [example.get("image") for example in inputs]\n'
+ ' kwargs = {"images": [[img] for img in images]}\n'
+ " for prompt in prompts:\n"
+ " if isinstance(prompt, list): # i.e., when using conversational data\n"
+ " prepare_multimodal_messages(prompt, num_images=1)\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ calls = []
+ namespace = {
+ "inputs": [{"image": cell} for cell in cells],
+ "prompts": [[{"role": "user", "content": "x"}] for _ in cells],
+ "prepare_multimodal_messages": lambda prompt, num_images: calls.append(num_images),
+ "_unsloth_grpo_image_cell": _unsloth_grpo_image_cell,
+ }
+ exec(textwrap.dedent(patched), namespace)
+ return namespace["kwargs"].get("images", None), calls
+
+
+def test_an_empty_cell_takes_no_placeholder_so_a_mixed_batch_stays_in_step():
+ """One placeholder for a row with no image leaves the prompt a token ahead of the pixels,
+ which the processor rejects. Measured against a real Gemma 3 processor: this batch fails
+ with one placeholder for the empty row and succeeds with none."""
+ a, b = _Img("a"), _Img("b")
+ assert _run_legacy_placeholders([a, []]) == ([[a], []], [1, 0])
+ assert _run_legacy_placeholders([[a, b], []]) == ([[a, b], []], [2, 0])
+ assert _run_legacy_placeholders([[], a]) == ([[], [a]], [0, 1])
+
+
+def test_an_all_empty_image_column_is_demoted_to_a_text_batch():
+ """TRL does this itself from 0.24.0. Without it the processor is handed [[], []] against
+ zero placeholders, which is an IndexError inside the processor rather than a text run."""
+ images, calls = _run_legacy_placeholders([[], []])
+ assert images is None, images
+ assert calls == []
+
+
+def test_a_none_cell_is_left_to_fail_exactly_as_it_does_on_main():
+ """The guard mirrors TRL's own `all(img_list == [])`, which never fires for a None entry.
+ Diverging from TRL's extraction here is a bigger change than the bug is worth."""
+ images, calls = _run_legacy_placeholders([None, None])
+ assert images == [None, None], images
+ assert calls == [0, 0]
+
+
+class _VllmTrainer:
+ def __init__(self, use_vllm, vllm_mode):
+ self.use_vllm = use_vllm
+ self.vllm_mode = vllm_mode
+
+
+def test_the_guard_refuses_a_multi_image_row_only_in_legacy_vllm_server_mode():
+ """A legacy TRL's vLLM server path hands the raw cells to VLLMClient.generate, which does
+ `[pil_to_base64(img) for img in images]` over the top level entries, so a cell holding two
+ images reaches `list.save(...)`. Colocate mode and the no vLLM path both go through the
+ processor, which this change fixed, so they must keep working."""
+ a, b = _Img("a"), _Img("b")
+ rows = [{"image": [a, b]}, {"image": a}]
+
+ with pytest.raises(ValueError) as excinfo:
+ _unsloth_reject_grpo_image_list(rows, _VllmTrainer(True, "server"))
+ assert "3605" in str(excinfo.value)
+
+ # The modes that do carry it: no refusal.
+ _unsloth_reject_grpo_image_list(rows, _VllmTrainer(True, "colocate"))
+ _unsloth_reject_grpo_image_list(rows, _VllmTrainer(False, "server"))
+ _unsloth_reject_grpo_image_list(rows, _VllmTrainer(False, None))
+ _unsloth_reject_grpo_image_list(rows, object())
+
+ # A single image row is carried by every mode, server included.
+ _unsloth_reject_grpo_image_list([{"image": a}], _VllmTrainer(True, "server"))
+
+ # No trainer means the anchors did not land at all, so nothing is known to carry it.
+ with pytest.raises(ValueError):
+ _unsloth_reject_grpo_image_list(rows)
+
+
+def test_the_legacy_guard_is_installed_with_the_trainer_and_the_modern_one_is_not():
+ trl_grpo = pytest.importorskip("trl.trainer.grpo_trainer")
+ with open(trl_grpo.__file__, "r", encoding = "utf-8") as fh:
+ module_source = fh.read()
+ start = module_source.find(" def _generate_and_score_completions(")
+ source = module_source[start:]
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ if 'kwargs = {"images": [[img] for img in images]}' in source:
+ assert "_unsloth_reject_grpo_image_list(inputs, self)" in patched
+ else:
+ assert "_unsloth_reject_grpo_image_list(inputs" not in patched
+
+
+def test_demoting_an_all_empty_column_also_clears_the_image_flag():
+ """Emptying `kwargs` is not enough: `has_images` gates the legacy vLLM paths, which read
+ the raw cells rather than `kwargs`, so a still-true flag submits `[[], ...]` as an image
+ payload and server mode calls `.save()` on an empty list."""
+ source = (
+ ' has_images = "image" in inputs[0]\n'
+ " if has_images:\n"
+ ' images = [example.get("image") for example in inputs]\n'
+ ' kwargs = {"images": [[img] for img in images]}\n'
+ " for prompt in prompts:\n"
+ " if isinstance(prompt, list): # i.e., when using conversational data\n"
+ " prepare_multimodal_messages(prompt, num_images=1)\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert "has_images = False" in patched
+
+ namespace = {
+ "inputs": [{"image": []}, {"image": []}],
+ "prompts": [[{"role": "user", "content": "x"}] for _ in range(2)],
+ "prepare_multimodal_messages": lambda prompt, num_images: None,
+ "_unsloth_grpo_image_cell": _unsloth_grpo_image_cell,
+ }
+ exec(textwrap.dedent(patched), namespace)
+ assert namespace["has_images"] is False
+ assert namespace["kwargs"] == {}
+
+ # An image bearing batch keeps the flag, so the vLLM paths still run for it.
+ namespace = {
+ "inputs": [{"image": _Img("a")}, {"image": _Img("b")}],
+ "prompts": [[{"role": "user", "content": "x"}] for _ in range(2)],
+ "prepare_multimodal_messages": lambda prompt, num_images: None,
+ "_unsloth_grpo_image_cell": _unsloth_grpo_image_cell,
+ }
+ exec(textwrap.dedent(patched), namespace)
+ assert namespace["has_images"] is True
+ assert len(namespace["kwargs"]["images"]) == 2
diff --git a/tests/test_grpo_legacy_trl_vision.py b/tests/test_grpo_legacy_trl_vision.py
new file mode 100644
index 00000000000..d6df7ce2206
--- /dev/null
+++ b/tests/test_grpo_legacy_trl_vision.py
@@ -0,0 +1,534 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""TRL 0.22.x-0.23.x carries the whole multimodal batch through the GRPO step.
+
+Those versions have neither `forward_kwargs` nor `num_images`: `_generate_and_score_completions`
+hands the no-grad old/reference logprob pass four hand named processor tensors and saves the same
+four into the training batch. Once a singular `image` cell may hold several images, the counts
+decide how every vision tensor is sliced, and a processor that emits `spatial_shapes` or
+`num_tiles` loses them entirely. unslothai/unsloth#6960.
+
+The sources below are verbatim from trl 0.22.2 / 0.23.1 `grpo_trainer.py`, trimmed to the lines
+the rewrite anchors on, so the patcher is exercised against the text it will actually meet.
+"""
+
+import textwrap
+
+import pytest
+
+
+# trl 0.22.2 grpo_trainer.py lines 1057-1084, 1321-1371 and 1471-1485, with generation replaced by
+# a fixed completion. 0.23.0 and 0.23.1 are identical over this range.
+_LEGACY_SOURCE = """
+ def _generate_and_score_completions(self, inputs):
+ mode = "train"
+ prompts = [x["prompt"] for x in inputs]
+ kwargs = {}
+ has_images = "image" in inputs[0]
+ if has_images:
+ images = [example.get("image") for example in inputs]
+ kwargs = {"images": [[img] for img in images]}
+ for prompt in prompts:
+ if isinstance(prompt, list): # i.e., when using conversational data
+ prepare_multimodal_messages(prompt, num_images=1)
+
+ prompt_inputs = self.processing_class(text=prompts, **kwargs)
+ prompt_ids, prompt_mask = prompt_inputs["input_ids"], prompt_inputs["attention_mask"]
+ completion_ids = self._completion_ids
+ prompt_completion_ids = torch.cat([prompt_ids, completion_ids], dim=1)
+ attention_mask = torch.cat([prompt_mask, torch.ones_like(completion_ids)], dim=1)
+
+ logits_to_keep = completion_ids.size(1) # we only need to compute the logits for the completion tokens
+ batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size
+
+ with torch.no_grad():
+ old_per_token_logps, _ = self._get_per_token_logps_and_entropies(
+ self.model,
+ prompt_completion_ids,
+ attention_mask,
+ logits_to_keep,
+ batch_size,
+ pixel_values=prompt_inputs.get("pixel_values"),
+ image_grid_thw=prompt_inputs.get("image_grid_thw"),
+ pixel_attention_mask=prompt_inputs.get("pixel_attention_mask"),
+ image_sizes=prompt_inputs.get("image_sizes"),
+ )
+ if self.beta != 0.0:
+ ref_per_token_logps, _ = self._get_per_token_logps_and_entropies(
+ self.ref_model,
+ prompt_completion_ids,
+ attention_mask,
+ logits_to_keep,
+ batch_size=batch_size,
+ pixel_values=prompt_inputs.get("pixel_values"),
+ image_grid_thw=prompt_inputs.get("image_grid_thw"),
+ pixel_attention_mask=prompt_inputs.get("pixel_attention_mask"),
+ image_sizes=prompt_inputs.get("image_sizes"),
+ )
+
+ output = {
+ "prompt_ids": prompt_ids,
+ "prompt_mask": prompt_mask,
+ "completion_ids": completion_ids,
+ }
+ if "pixel_values" in prompt_inputs:
+ output["pixel_values"] = prompt_inputs["pixel_values"]
+ if "image_grid_thw" in prompt_inputs:
+ output["image_grid_thw"] = prompt_inputs["image_grid_thw"]
+ if "pixel_attention_mask" in prompt_inputs:
+ output["pixel_attention_mask"] = prompt_inputs["pixel_attention_mask"]
+ if "image_sizes" in prompt_inputs:
+ output["image_sizes"] = prompt_inputs["image_sizes"]
+ return output
+"""
+
+
+class _Model:
+ def for_training(self, use_gradient_checkpointing = True):
+ pass
+
+
+class _Processor:
+ """Callable like a processor, and carrying the chat_template the rewrite reads."""
+
+ chat_template = ""
+ pad_token_id = 0
+
+ def __init__(self, processor_output):
+ self._processor_output = processor_output
+ self.seen_image_kwarg = None
+
+ def __call__(
+ self,
+ text = None,
+ **kwargs,
+ ):
+ self.seen_image_kwarg = kwargs.get("images", None)
+ return self._processor_output
+
+
+class _Trainer:
+ """Only what the patched legacy body touches."""
+
+ beta = 0.1
+ use_vllm = False
+
+ def __init__(self, processor_output, completion_ids):
+ import types
+
+ self.model = _Model()
+ self.ref_model = _Model()
+ self.args = types.SimpleNamespace(
+ per_device_train_batch_size = 1, per_device_eval_batch_size = 1
+ )
+ self.processing_class = _Processor(processor_output)
+ self._completion_ids = completion_ids
+ self.calls = []
+
+ @property
+ def seen_image_kwarg(self):
+ return self.processing_class.seen_image_kwarg
+
+ def _get_per_token_logps_and_entropies(self, model, *args, **kwargs):
+ self.calls.append(kwargs)
+ return None, None
+
+
+def _run_legacy(processor_output, inputs, completion_ids):
+ """Patch the verbatim legacy source, run it, and return (trainer, output)."""
+ import torch
+
+ from unsloth.models.rl_replacements import (
+ _unsloth_grpo_image_cell,
+ _unsloth_grpo_vision_inputs,
+ _unsloth_reject_grpo_image_list,
+ grpo_trainer__generate_and_score_completions,
+ )
+
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", _LEGACY_SOURCE
+ )
+ namespace = {
+ "torch": torch,
+ "_unsloth_grpo_vision_inputs": _unsloth_grpo_vision_inputs,
+ "_unsloth_grpo_image_cell": _unsloth_grpo_image_cell,
+ # Injected on this TRL too now: its vLLM server path cannot carry a multi image row,
+ # so the guard is handed the trainer and decides at runtime.
+ "_unsloth_reject_grpo_image_list": _unsloth_reject_grpo_image_list,
+ "prepare_multimodal_messages": lambda prompt, num_images = 1: prompt.append(num_images),
+ # Injected by the same rewrite for the text-only branch; TRL never sees it.
+ "calculate_pad_tokens_in_prompt": lambda ids, keep, pad: torch.zeros(
+ ids.shape[0], dtype = torch.long
+ ),
+ }
+ exec(compile(textwrap.dedent(patched), "<legacy-trl>", "exec"), namespace)
+ trainer = _Trainer(processor_output, completion_ids)
+ output = namespace["_generate_and_score_completions"](trainer, inputs)
+ return trainer, output
+
+
+def _grid_batch():
+ """Two samples, the first holding two images. Qwen2-VL style, four patch rows per image."""
+ import torch
+
+ processor_output = {
+ "input_ids": torch.zeros(2, 5, dtype = torch.long),
+ "attention_mask": torch.ones(2, 5, dtype = torch.long),
+ "pixel_values": torch.arange(12).reshape(12, 1).float(),
+ "image_grid_thw": torch.tensor([[1, 2, 2], [1, 2, 2], [1, 2, 2]]),
+ }
+ inputs = [
+ {"prompt": "a", "image": ["one", "two"]},
+ {"prompt": "b", "image": "three"},
+ ]
+ return processor_output, inputs, torch.zeros(2, 3, dtype = torch.long)
+
+
+def test_the_legacy_no_grad_calls_are_told_how_many_images_each_sample_has():
+ """Without the counts the shared chunker slices an image indexed grid by sample index, so
+ the old and reference logprobs of a two image row come from another sample's pixels."""
+ trainer, _ = _run_legacy(*_grid_batch())
+ assert len(trainer.calls) == 2, "the old-policy and the reference call"
+ for call in trainer.calls:
+ assert call.get("num_images") == [2, 1], call.get("num_images")
+ assert call["image_grid_thw"].shape[0] == 3
+ assert call["pixel_values"].shape[0] == 12
+
+
+def test_the_legacy_training_batch_carries_the_counts_too():
+ _, output = _run_legacy(*_grid_batch())
+ assert output["num_images"] == [2, 1]
+
+
+def test_a_legacy_row_with_no_image_of_its_own_counts_zero():
+ import torch
+
+ processor_output, inputs, completion_ids = _grid_batch()
+ inputs[1]["image"] = None
+ _, output = _run_legacy(processor_output, inputs, completion_ids)
+ assert output["num_images"] == [2, 0]
+
+
+def test_a_singular_cell_is_counted_the_way_the_processor_was_given_it():
+ processor_output, inputs, completion_ids = _grid_batch()
+ inputs[0]["image"] = "one"
+ trainer, output = _run_legacy(processor_output, inputs, completion_ids)
+ assert output["num_images"] == [1, 1]
+ assert trainer.seen_image_kwarg == [["one"], ["three"]]
+
+
+def _lfm2vl_batch():
+ """LFM2-VL style: tile indexed pixels plus spatial_shapes, and no image_grid_thw."""
+ import torch
+
+ processor_output = {
+ "input_ids": torch.zeros(2, 5, dtype = torch.long),
+ "attention_mask": torch.ones(2, 5, dtype = torch.long),
+ "pixel_values": torch.arange(5).reshape(5, 1).float(),
+ "spatial_shapes": torch.arange(5).reshape(5, 1),
+ "pixel_attention_mask": torch.ones(5, 1),
+ "num_tiles": [2, 3],
+ }
+ inputs = [{"prompt": "a", "image": "one"}, {"prompt": "b", "image": "two"}]
+ return processor_output, inputs, torch.zeros(2, 3, dtype = torch.long)
+
+
+def test_a_legacy_processors_extra_keys_reach_the_no_grad_calls():
+ """0.22.x-0.23.x names four tensors by hand, so spatial_shapes and num_tiles never left
+ the processor and the reference forward ran without arguments the model requires."""
+ trainer, _ = _run_legacy(*_lfm2vl_batch())
+ for call in trainer.calls:
+ assert call["spatial_shapes"].shape[0] == 5, call.keys()
+ assert call["num_tiles"] == [2, 3], call.keys()
+ assert call["num_images"] == [1, 1]
+
+
+def test_a_legacy_processors_extra_keys_reach_the_training_batch():
+ """The gradient pass reads them off the batch the step returns, not off the processor."""
+ _, output = _run_legacy(*_lfm2vl_batch())
+ assert output["spatial_shapes"].shape[0] == 5, sorted(output)
+ assert output["num_tiles"] == [2, 3], sorted(output)
+ assert output["num_images"] == [1, 1], sorted(output)
+
+
+def test_a_key_the_legacy_processor_did_not_produce_is_not_invented():
+ _, output = _run_legacy(*_lfm2vl_batch())
+ assert "image_grid_thw" not in output, sorted(output)
+ assert "image_position_ids" not in output, sorted(output)
+
+
+def test_a_text_only_legacy_batch_forwards_nothing_and_saves_nothing():
+ import torch
+
+ processor_output = {
+ "input_ids": torch.zeros(2, 5, dtype = torch.long),
+ "attention_mask": torch.ones(2, 5, dtype = torch.long),
+ }
+ inputs = [{"prompt": "a"}, {"prompt": "b"}]
+ trainer, output = _run_legacy(processor_output, inputs, torch.zeros(2, 3, dtype = torch.long))
+ for call in trainer.calls:
+ assert set(call) <= {"batch_size"}, call
+ assert "num_images" not in output, sorted(output)
+
+
+def test_the_legacy_token_type_ids_are_widened_over_the_completion():
+ """TRL 0.24.0 does this itself right before the same call ("If token_type_ids are used,
+ extend them with zeros for the completion part"); the legacy versions never forwarded them,
+ so the processor's prompt length copy would not fit a prompt+completion forward."""
+ import torch
+
+ processor_output, inputs, completion_ids = _grid_batch()
+ processor_output["token_type_ids"] = torch.ones(2, 5, dtype = torch.long)
+ trainer, output = _run_legacy(processor_output, inputs, completion_ids)
+ for call in trainer.calls:
+ assert call["token_type_ids"].shape == (2, 8), call["token_type_ids"].shape
+ assert call["token_type_ids"][:, 5:].sum().item() == 0
+ assert output["token_type_ids"].shape == (2, 8)
+
+
+def test_a_legacy_token_type_ids_of_the_wrong_width_is_dropped_not_forwarded():
+ """max_prompt_length truncates prompt_ids after the processor ran, so the processor's own
+ copy can be wider than the prompt. Sending it would be a shape error in the forward."""
+ import torch
+
+ processor_output, inputs, completion_ids = _grid_batch()
+ processor_output["token_type_ids"] = torch.ones(2, 9, dtype = torch.long)
+ trainer, output = _run_legacy(processor_output, inputs, completion_ids)
+ for call in trainer.calls:
+ assert "token_type_ids" not in call, call.keys()
+ assert "token_type_ids" not in output, sorted(output)
+
+
+def test_a_modern_trl_is_untouched_by_the_legacy_rewrite():
+ """0.24.0 and up keep their own forward_kwargs path; the legacy anchors must not fire."""
+ from unsloth.models.rl_replacements import grpo_trainer__generate_and_score_completions
+
+ source = (
+ " def _generate_and_score_completions(self, inputs):\n"
+ ' if "image_sizes" in forward_kwargs:\n'
+ ' output["image_sizes"] = forward_kwargs["image_sizes"]\n'
+ " if images is not None:\n"
+ ' output["num_images"] = num_images\n'
+ " return output\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert "**_unsloth_legacy_vision," not in patched
+ assert patched.count('output["num_images"]') == 1, patched
+
+
+# ---------------------------------------------------------------------------------------------
+# The shuffle and the slice, on the layout 0.22.x-0.23.x leaves behind.
+# ---------------------------------------------------------------------------------------------
+
+
+def _legacy_split_pixel_values_by_grid(batch):
+ """Verbatim from trl 0.22.2 / 0.23.1 trainer/utils.py: one list element per GRID ROW, which
+ is one per image, while everything else in the batch stays indexed by sample."""
+ import torch
+
+ if "image_grid_thw" not in batch or "pixel_values" not in batch:
+ return batch
+ lengths = batch["image_grid_thw"].prod(dim = 1).tolist()
+ pixel_values = batch["pixel_values"]
+ if sum(lengths) != pixel_values.size(0):
+ raise ValueError("Mismatch")
+ return {**batch, "pixel_values": list(torch.split(batch["pixel_values"], lengths, dim = 0))}
+
+
+def _legacy_unsplit_pixel_values_by_grid(batch):
+ """Verbatim from trl 0.22.2 / 0.23.1: merges pixel_values and nothing else."""
+ import torch
+
+ pixel_values = batch.get("pixel_values")
+ if isinstance(pixel_values, list):
+ return {**batch, "pixel_values": torch.cat(pixel_values, dim = 0)}
+ return batch
+
+
+def test_a_multi_image_grid_batch_keeps_its_images_with_their_samples():
+ """0.22.x-0.23.x splits pixel_values by grid row, so with two images on one sample the list
+ is indexed by image while prompt_ids is indexed by sample. shuffle_sequence_dict then takes
+ its length from prompt_ids and drops the tail of the list."""
+ import torch
+
+ trl_utils = pytest.importorskip("trl.trainer.utils")
+ from unsloth.models.rl_replacements import (
+ _unsloth_grpo_split_vision_by_sample,
+ _unsloth_grpo_unsplit_vision,
+ )
+
+ num_images = [2, 1, 1]
+ owners = torch.tensor([0, 0, 1, 2]) # one grid row per image, one patch row per image
+ batch = {
+ "prompt_ids": torch.arange(3).unsqueeze(-1),
+ "advantages": torch.zeros(3),
+ "num_images": num_images,
+ "image_grid_thw": torch.tensor([[1, 1, 1]] * 4),
+ "pixel_values": owners.reshape(-1, 1).float(),
+ "image_sizes": owners.reshape(-1, 1).clone(),
+ }
+ split = _unsloth_grpo_split_vision_by_sample(_legacy_split_pixel_values_by_grid(batch))
+ assert [t.shape[0] for t in split["pixel_values"]] == num_images, "still indexed by image"
+ assert [t.shape[0] for t in split["image_grid_thw"]] == num_images
+
+ torch.manual_seed(0)
+ shuffled = trl_utils.shuffle_sequence_dict(split)
+ chunk = trl_utils.split_tensor_dict(shuffled, 1)[0]
+ restored = _unsloth_grpo_unsplit_vision(_legacy_unsplit_pixel_values_by_grid(chunk))
+
+ assert restored["pixel_values"].shape[0] == len(owners), "an image was dropped"
+ assert restored["image_grid_thw"].shape[0] == len(owners)
+ at = 0
+ seen = []
+ for prompt, count in zip(restored["prompt_ids"].tolist(), restored["num_images"]):
+ sample = prompt[0]
+ seen.append(sample)
+ assert count == num_images[sample], (sample, count)
+ for key in ("pixel_values", "image_sizes"):
+ rows = restored[key][at : at + count].flatten().tolist()
+ assert rows == [sample] * count, (key, sample, rows)
+ at += count
+ assert sorted(seen) == [0, 1, 2], seen
+
+
+def test_one_image_per_sample_on_a_legacy_trl_is_left_exactly_as_trl_left_it():
+ """The grid row axis and the sample axis coincide there, and re-splitting would be churn."""
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {
+ "num_images": [1, 1],
+ "image_grid_thw": torch.tensor([[1, 2, 2], [1, 2, 2]]),
+ "pixel_values": torch.zeros(8, 1),
+ }
+ already = _legacy_split_pixel_values_by_grid(batch)
+ assert _unsloth_grpo_split_vision_by_sample(already) is already
+
+
+def test_a_grid_batch_trl_1_1_0_already_split_by_sample_is_left_alone():
+ """From 1.1.0 split_pixel_values_by_grid splits both by sample and returns the grid as a
+ list, so a second regroup would nest the lists."""
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {
+ "num_images": [2, 1],
+ "image_grid_thw": [torch.tensor([[1, 1, 1]] * 2), torch.tensor([[1, 1, 1]])],
+ "pixel_values": [torch.zeros(2, 1), torch.zeros(1, 1)],
+ }
+ assert _unsloth_grpo_split_vision_by_sample(batch) is batch
+
+
+def test_a_legacy_grid_row_with_no_image_still_gets_its_neighbours_images_apart():
+ """num_images = [0, 2] over two samples makes the image count and the sample count agree
+ while the axes still differ, so a bare length comparison would leave the list alone and the
+ shuffle would hand both of the second sample's images to the first."""
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {
+ "prompt_ids": torch.arange(2).unsqueeze(-1),
+ "num_images": [0, 2],
+ "image_grid_thw": torch.tensor([[1, 1, 1], [1, 1, 1]]),
+ "pixel_values": torch.tensor([[7.0], [8.0]]),
+ }
+ split = _unsloth_grpo_split_vision_by_sample(_legacy_split_pixel_values_by_grid(batch))
+ assert [t.shape[0] for t in split["pixel_values"]] == [0, 2]
+ assert split["pixel_values"][1].flatten().tolist() == [7.0, 8.0]
+ assert [t.shape[0] for t in split["image_grid_thw"]] == [0, 2]
+
+
+def test_a_legacy_grid_batch_whose_counts_do_not_span_the_samples_is_left_alone():
+ """A count list that is not one entry per row cannot index the batch, and regrouping on it
+ would silently shorten the list."""
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {
+ "prompt_ids": torch.arange(3).unsqueeze(-1),
+ "num_images": [2, 1],
+ "image_grid_thw": torch.tensor([[1, 1, 1]] * 3),
+ "pixel_values": torch.arange(3).reshape(3, 1).float(),
+ }
+ already = _legacy_split_pixel_values_by_grid(batch)
+ assert _unsloth_grpo_split_vision_by_sample(already) is already
+
+
+def test_a_text_only_legacy_batch_does_not_pick_up_a_tokenizers_token_type_ids():
+ """Gemma 3's processor returns token_type_ids with no images at all
+ (transformers/models/gemma3/processing_gemma3.py, return_mm_token_type_ids defaults True),
+ and both PrefixGrouper and sequence packing are keyed on that name being absent. TRL 0.24.0
+ builds forward_kwargs only inside `if images is not None`, so it never sees them either."""
+ import torch
+
+ processor_output = {
+ "input_ids": torch.zeros(2, 5, dtype = torch.long),
+ "attention_mask": torch.ones(2, 5, dtype = torch.long),
+ "token_type_ids": torch.zeros(2, 5, dtype = torch.long),
+ }
+ inputs = [{"prompt": "a"}, {"prompt": "b"}]
+ trainer, output = _run_legacy(processor_output, inputs, torch.zeros(2, 3, dtype = torch.long))
+ for call in trainer.calls:
+ assert "token_type_ids" not in call, call.keys()
+ assert "token_type_ids" not in output, sorted(output)
+
+
+def test_an_image_column_with_no_images_in_it_is_a_text_batch():
+ """`has_images` is only "the column exists". Every cell empty means the processor produced no
+ pixels, so the run is a text run and must keep its fast paths."""
+ import torch
+
+ processor_output = {
+ "input_ids": torch.zeros(2, 5, dtype = torch.long),
+ "attention_mask": torch.ones(2, 5, dtype = torch.long),
+ "token_type_ids": torch.zeros(2, 5, dtype = torch.long),
+ }
+ inputs = [{"prompt": "a", "image": None}, {"prompt": "b", "image": None}]
+ trainer, output = _run_legacy(processor_output, inputs, torch.zeros(2, 3, dtype = torch.long))
+ for call in trainer.calls:
+ assert "token_type_ids" not in call, call.keys()
+ assert "token_type_ids" not in output, sorted(output)
+
+
+def test_an_untruncated_token_type_ids_as_wide_as_prompt_plus_completion_is_dropped():
+ """max_prompt_length runs truncate_with_protected_tokens after the processor, which keeps the
+ image tokens but MOVES them. A processor copy whose width happens to equal prompt+completion
+ is still the untruncated one, so its marks sit at the wrong offsets."""
+ import torch
+
+ processor_output, inputs, completion_ids = _grid_batch()
+ # prompt_ids 5 wide, completion 3 wide: an 8 wide untruncated processor copy.
+ processor_output["token_type_ids"] = torch.ones(2, 8, dtype = torch.long)
+ trainer, output = _run_legacy(processor_output, inputs, completion_ids)
+ for call in trainer.calls:
+ assert "token_type_ids" not in call, call["token_type_ids"].shape
+ assert "token_type_ids" not in output, sorted(output)
+
+
+def test_a_modern_trl_gets_no_legacy_prologue_at_all():
+ """0.24.0 and up leave prompt_inputs unbound at the anchor, so the block would fall through
+ anyway; not emitting it keeps their generated source exactly what it was."""
+ from unsloth.models.rl_replacements import grpo_trainer__generate_and_score_completions
+
+ modern = (
+ " def _generate_and_score_completions(self, inputs):\n"
+ ' batch_size = self.args.per_device_train_batch_size if mode == "train" '
+ "else self.args.per_device_eval_batch_size\n"
+ ' if "image_sizes" in forward_kwargs:\n'
+ ' output["image_sizes"] = forward_kwargs["image_sizes"]\n'
+ " if images is not None:\n"
+ ' output["num_images"] = num_images\n'
+ " return output\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", modern
+ )
+ assert "_unsloth_legacy_vision" not in patched, patched
+ assert "max_left_pad = None" in patched, "the rest of the anchor rewrite still applies"
diff --git a/tests/test_grpo_padded_raw_logits.py b/tests/test_grpo_padded_raw_logits.py
index 6383e86b6ad..5bda19dd2de 100644
--- a/tests/test_grpo_padded_raw_logits.py
+++ b/tests/test_grpo_padded_raw_logits.py
@@ -285,12 +285,7 @@ def _build_namespace(
(
data.input_ids[i : i + 1],
data.attention_mask[i : i + 1],
- torch.zeros(1, 3) if is_vlm else None, # pixel_values_chunk (the stub ignores it)
- None, # image_grid_thw_chunk
- None, # pixel_attention_mask_chunk
- None, # image_sizes_chunk
- None, # token_type_ids_chunk
- None, # mm_token_type_ids_chunk
+ {"pixel_values": torch.zeros(1, 3)} if is_vlm else {},
)
for i in range(_BATCH)
]
diff --git a/tests/test_grpo_vision_kwargs_forwarded.py b/tests/test_grpo_vision_kwargs_forwarded.py
new file mode 100644
index 00000000000..0442aef61ce
--- /dev/null
+++ b/tests/test_grpo_vision_kwargs_forwarded.py
@@ -0,0 +1,728 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Both GRPO logprob paths forward every multimodal key TRL produced: two hand written
+four-key lists dropped spatial_shapes, num_tiles, image_position_ids, and pixel_values for
+models without image_grid_thw. unslothai/unsloth#6960."""
+
+import inspect
+import os
+
+import pytest
+
+REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
+SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py")
+
+
+def _zoo_vision_helpers(*names):
+ """The multi image helpers ship with unsloth_zoo. An unsloth_zoo installed from before
+ they landed has none of them, and the static gates in this file already prove this repo
+ asks for them and fails loudly without them, so behaviour that can only be driven
+ through the zoo is skipped rather than reported as this repo being broken."""
+ zoo = pytest.importorskip("unsloth_zoo.rl_replacements")
+ missing = [name for name in names if not hasattr(zoo, name)]
+ if missing:
+ pytest.skip(f"the installed unsloth_zoo has no {', '.join(missing)}")
+ return tuple(getattr(zoo, name) for name in names)
+
+
+def _read_source() -> str:
+ with open(SOURCE_PATH, "r", encoding = "utf-8") as fh:
+ return fh.read()
+
+
+def test_the_key_tuple_lives_in_one_place():
+ src = _read_source()
+ assert "grpo_get_vision_inputs" in src
+ assert src.count("vision_inputs.get(") >= 10
+ assert src.count("def _unsloth_grpo_vision_inputs(") == 1
+ assert src.count("_unsloth_grpo_vision_inputs(kwargs)") == 1
+ assert src.count("_unsloth_grpo_vision_inputs(inputs)") == 1
+ for gone in (
+ 'kwargs.get("image_grid_thw", None)',
+ 'kwargs.get("num_images", None)',
+ 'kwargs.get("pixel_values", None)',
+ ):
+ assert gone not in src, gone
+
+
+def test_no_grad_pass_uses_the_shared_chunker():
+ from unsloth.models.rl_replacements import grpo_trainer__get_per_token_logps_and_entropies
+
+ patched = grpo_trainer__get_per_token_logps_and_entropies(
+ "_get_per_token_logps_and_entropies", ""
+ )
+ assert "grpo_vision_chunks" in patched
+ assert "**vision_chunk" in patched
+ for gone in (
+ "pixel_values_chunks",
+ "image_grid_thw_chunks",
+ "pixel_attention_mask_chunks",
+ "image_sizes_chunks",
+ "_extra_vision_kwargs",
+ ):
+ assert gone not in patched, gone
+
+
+def test_compute_loss_hands_the_whole_set_to_the_gradient_pass():
+ from unsloth.models.rl_replacements import grpo_trainer_compute_loss
+
+ patched = grpo_trainer_compute_loss("compute_loss", "")
+ assert "**_vision_inputs" in patched
+ assert patched.count("**_vision_inputs") == 2, "both the modern and the legacy call"
+ # A hand named kwarg here would reach the logprob pass but not the loss.
+ for call in patched.split("grpo_accumulated_loss(")[1:]:
+ body = call.split("\n )")[0]
+ for gone in (
+ "pixel_values =",
+ "image_grid_thw =",
+ "pixel_attention_mask =",
+ "image_sizes =",
+ "num_images =",
+ "token_type_ids =",
+ ):
+ assert gone not in body, gone
+
+
+def test_both_paths_call_the_same_helper():
+ grpo_accumulated_loss, grpo_vision_chunks = _zoo_vision_helpers(
+ "grpo_accumulated_loss", "grpo_vision_chunks"
+ )
+
+ grad_source = inspect.getsource(grpo_accumulated_loss)
+ assert "grpo_vision_chunks" in grad_source
+ from unsloth_zoo import rl_replacements as zoo_rl
+
+ assert zoo_rl.RL_REPLACEMENTS["grpo_vision_chunks"] is grpo_vision_chunks
+
+
+def test_old_zoo_without_the_helper_fails_loudly_for_vision_runs():
+ from unsloth.models.rl_replacements import grpo_trainer__get_per_token_logps_and_entropies
+
+ patched = grpo_trainer__get_per_token_logps_and_entropies(
+ "_get_per_token_logps_and_entropies", ""
+ )
+ assert "Please upgrade unsloth_zoo" in patched
+ assert "grpo_vision_chunks" in patched
+
+
+def test_lfm2vl_keys_reach_the_forward_kwargs():
+ import torch
+
+ grpo_get_vision_inputs, grpo_vision_chunks = _zoo_vision_helpers(
+ "grpo_get_vision_inputs", "grpo_vision_chunks"
+ )
+
+ num_tiles = [2, 3]
+ total_tiles = sum(num_tiles)
+ inputs = {
+ "advantages": torch.zeros(2),
+ "pixel_values": torch.randn(total_tiles, 3, 16, 16),
+ "pixel_attention_mask": torch.ones(total_tiles, 16, 16),
+ "spatial_shapes": torch.tensor([[2, 2]] * total_tiles),
+ "num_tiles": num_tiles,
+ "num_images": [1, 1],
+ }
+ chunks = grpo_vision_chunks(grpo_get_vision_inputs(inputs), 2, 1)
+ assert [c["pixel_values"].shape[0] for c in chunks] == [2, 3]
+ assert [c["spatial_shapes"].shape[0] for c in chunks] == [2, 3]
+ assert [c["pixel_attention_mask"].shape[0] for c in chunks] == [2, 3]
+
+
+def test_legacy_fallback_list_matches_the_zoo_tuple():
+ import ast
+ import re
+
+ (GRPO_VISION_KEYS,) = _zoo_vision_helpers("GRPO_VISION_KEYS")
+
+ src = _read_source()
+ match = re.search(
+ r"key: get\(key, None\)\s*for key in (\([^)]*\))",
+ src,
+ re.DOTALL,
+ )
+ assert match, "the old-zoo fallback key list moved; update this gate"
+ fallback = ast.literal_eval(match.group(1))
+ assert set(fallback) == set(GRPO_VISION_KEYS), (
+ "the fallback list and unsloth_zoo.GRPO_VISION_KEYS have drifted: "
+ f"{sorted(set(GRPO_VISION_KEYS) - set(fallback))} missing"
+ )
+
+
+def test_generate_forward_wrapper_keeps_the_real_signature():
+ """generate() validates kwargs against inspect.signature(self.forward)."""
+ import inspect as _inspect
+
+ import torch
+
+ from unsloth.models.rl import _install_grpo_hidden_states_forward_wrapper
+
+ class _Toy(torch.nn.Module):
+ config = type("cfg", (), {"is_encoder_decoder": False})()
+
+ def forward(
+ self,
+ input_ids = None,
+ pixel_values = None,
+ spatial_shapes = None,
+ **kwargs,
+ ):
+ return input_ids
+
+ model = _Toy()
+ before = list(_inspect.signature(model.forward).parameters)
+ installed = _install_grpo_hidden_states_forward_wrapper(model)
+ assert installed, "the wrapper did not install on a model without hidden-state support"
+ after = list(_inspect.signature(model.forward).parameters)
+ assert after == before, f"signature lost: {before} became {after}"
+ assert "pixel_values" in after and "spatial_shapes" in after
+
+
+def test_the_wrapper_survives_accelerates_fp32_unwrap():
+ """The signature must not be bought by making the wrapper removable: functools.wraps
+ sets __wrapped__, which accelerate's extract_model_from_parallel(keep_fp32_wrapper =
+ False) walks straight past this wrapper on every GRPO step."""
+ import inspect as _inspect
+
+ import torch
+ from accelerate.utils.operations import convert_outputs_to_fp32
+ from accelerate.utils.other import extract_model_from_parallel
+
+ from unsloth.models.rl import _install_grpo_hidden_states_forward_wrapper
+
+ class _Toy(torch.nn.Module):
+ config = type("cfg", (), {"is_encoder_decoder": False})()
+
+ def forward(
+ self,
+ input_ids = None,
+ pixel_values = None,
+ spatial_shapes = None,
+ **kwargs,
+ ):
+ return input_ids
+
+ model = _Toy()
+ real_forward = model.forward
+ model._original_forward = real_forward
+ model.forward = convert_outputs_to_fp32(
+ torch.autocast(device_type = "cuda", dtype = torch.bfloat16)(real_forward)
+ )
+
+ assert _install_grpo_hidden_states_forward_wrapper(model)
+ assert getattr(model.forward, "_unsloth_grpo_hidden_states_forward_wrapped", False)
+ assert not hasattr(
+ model.forward, "__wrapped__"
+ ), "the wrapper carries __wrapped__, so accelerate will unwrap straight past it"
+
+ unwrapped = extract_model_from_parallel(model, keep_fp32_wrapper = False)
+
+ assert getattr(
+ unwrapped.forward, "_unsloth_grpo_hidden_states_forward_wrapped", False
+ ) or getattr(
+ getattr(unwrapped.forward, "__func__", None),
+ "_unsloth_grpo_hidden_states_forward_wrapped",
+ False,
+ ), "accelerate removed the hidden-state wrapper"
+ after = list(_inspect.signature(unwrapped.forward).parameters)
+ assert "pixel_values" in after and "spatial_shapes" in after, after
+
+
+def test_num_tiles_survives_the_output_dict_rewrite():
+ """TRL 1.7.0 nested num_tiles inside the num_images block; the insert must go after it."""
+ from unsloth.models.rl_replacements import grpo_trainer__generate_and_score_completions
+
+ source = (
+ ' if "image_sizes" in forward_kwargs:\n'
+ ' output["image_sizes"] = forward_kwargs["image_sizes"]\n'
+ " if images is not None:\n"
+ ' output["num_images"] = num_images\n'
+ " if num_tiles is not None:\n"
+ ' output["num_tiles"] = num_tiles\n'
+ " return output\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ lines = patched.splitlines()
+ num_images_at = next(i for i, l in enumerate(lines) if 'output["num_images"]' in l)
+ num_tiles_at = next(i for i, l in enumerate(lines) if 'output["num_tiles"]' in l)
+ except_at = next((i for i, l in enumerate(lines) if l.strip() == "except NameError:"), None)
+ assert except_at is not None, "the sampling logprob block was not inserted"
+ assert num_images_at < num_tiles_at < except_at
+ # still nested inside `if images is not None:`
+ assert lines[num_tiles_at].startswith(" " * 16)
+
+
+def test_the_trl_1_0_spelling_of_the_gemma_4_position_ids_is_forwarded():
+ """TRL 1.0.x emits `pixel_position_ids`, 1.1.0 `image_position_ids`; both are live."""
+ import torch
+
+ GRPO_VISION_KEYS, grpo_get_vision_inputs, grpo_vision_chunks = _zoo_vision_helpers(
+ "GRPO_VISION_KEYS", "grpo_get_vision_inputs", "grpo_vision_chunks"
+ )
+
+ assert "pixel_position_ids" in GRPO_VISION_KEYS
+ assert "image_position_ids" in GRPO_VISION_KEYS
+
+ for key in ("pixel_position_ids", "image_position_ids"):
+ inputs = {
+ "pixel_values": torch.randn(4, 3, 16, 16),
+ key: torch.arange(4).unsqueeze(-1),
+ "num_images": [1, 1, 1, 1],
+ }
+ chunks = grpo_vision_chunks(grpo_get_vision_inputs(inputs), 4, 2)
+ assert [tuple(c) for c in chunks] == [("pixel_values", key)] * 2, chunks
+ assert [c["pixel_values"].shape[0] for c in chunks] == [2, 2]
+ assert [c[key].shape[0] for c in chunks] == [2, 2]
+ other = "image_position_ids" if key == "pixel_position_ids" else "pixel_position_ids"
+ assert all(other not in c for c in chunks)
+
+
+def test_a_multi_image_row_indexes_the_position_ids_by_image():
+ import torch
+
+ grpo_get_vision_inputs, grpo_vision_chunks = _zoo_vision_helpers(
+ "grpo_get_vision_inputs", "grpo_vision_chunks"
+ )
+
+ inputs = {
+ "pixel_values": torch.randn(3, 3, 16, 16),
+ "image_position_ids": torch.arange(3).unsqueeze(-1),
+ "num_images": [2, 1],
+ }
+ chunks = grpo_vision_chunks(grpo_get_vision_inputs(inputs), 2, 1)
+ assert [c["pixel_values"].shape[0] for c in chunks] == [2, 1]
+ assert [c["image_position_ids"].tolist() for c in chunks] == [[[0], [1]], [[2]]]
+
+
+def test_an_old_zoo_still_forwards_the_sample_indexed_keys():
+ """An older unsloth_zoo must cost only image indexed slicing, not token_type_ids."""
+ from unsloth.models.rl_replacements import grpo_trainer__get_per_token_logps_and_entropies
+
+ patched = grpo_trainer__get_per_token_logps_and_entropies(
+ "_get_per_token_logps_and_entropies", ""
+ )
+ assert "_unsloth_grpo_vision_inputs(kwargs)" in patched
+ fallback = patched.split("if _grpo_vision_chunks is None:")[-1]
+ assert '"token_type_ids", "mm_token_type_ids"' in fallback
+ assert "vision_chunks = [{} for _ in input_ids_chunks]" not in patched
+
+
+def test_the_wrapper_keeps_trls_logits_to_keep_probe_answerable():
+ """GRPOTrainer.__init__ probes the forward signature for `logits_to_keep` once."""
+ import inspect as _inspect
+
+ import torch
+
+ from unsloth.models.rl import _install_grpo_hidden_states_forward_wrapper
+
+ class _Takes(torch.nn.Module):
+ config = type("cfg", (), {"is_encoder_decoder": False})()
+
+ def forward(
+ self,
+ input_ids = None,
+ logits_to_keep = 0,
+ **kwargs,
+ ):
+ return input_ids
+
+ class _Refuses(torch.nn.Module):
+ config = type("cfg", (), {"is_encoder_decoder": False})()
+
+ def forward(
+ self,
+ input_ids = None,
+ pixel_values = None,
+ **kwargs,
+ ):
+ return input_ids
+
+ takes, refuses = _Takes(), _Refuses()
+ assert _install_grpo_hidden_states_forward_wrapper(takes)
+ assert _install_grpo_hidden_states_forward_wrapper(refuses)
+ assert "logits_to_keep" in _inspect.signature(takes.forward).parameters
+ assert "logits_to_keep" not in _inspect.signature(refuses.forward).parameters
+
+
+def test_the_multi_image_zoo_probe_does_not_rest_on_a_local_variable():
+ """Grepping grpo_accumulated_loss for "num_images", now an unread local, faked it."""
+ from unsloth.models.rl_replacements import grpo_trainer_compute_loss
+
+ patched = grpo_trainer_compute_loss("compute_loss", "")
+ gate = patched.split("_unsloth_requires_multi_image_zoo(num_images)")[1]
+ gate = gate.split("Please upgrade ")[0]
+ assert "from unsloth_zoo.rl_replacements import grpo_vision_chunks" in gate
+ # probe before grep, or the grep still decides on a current zoo
+ assert gate.index("grpo_vision_chunks") < gate.index("inspect.getsource")
+
+
+def _gradient_zoo_gate():
+ """The gradient path's zoo gate, as a runnable block taken out of the patched source.
+
+ Extracted by AST rather than by string slicing so the case below RUNS the real check
+ instead of restating it: the block is self contained (it reads `self` and `pixel_values`
+ and imports the zoo itself), so executing it is the behaviour and not a description.
+ """
+ import ast
+ import textwrap
+
+ from unsloth.models.rl_replacements import grpo_trainer_compute_loss
+
+ source = textwrap.dedent(grpo_trainer_compute_loss("compute_loss", ""))
+ tree = ast.parse(source)
+ blocks = [
+ segment
+ for node in ast.walk(tree)
+ if isinstance(node, ast.If)
+ for segment in [ast.get_source_segment(source, node) or ""]
+ if "grpo_vision_chunks" in segment and "_unsloth_grpo_vision_zoo_checked" in segment
+ ]
+ assert blocks, "the gradient path has no zoo gate at all"
+ # The innermost match: every enclosing `if` contains the gate's text too.
+ return textwrap.dedent(min(blocks, key = len))
+
+
+class _Trainer:
+ pass
+
+
+def _run_gate(block, *, pixel_values, zoo_module):
+ import sys
+ import types
+
+ name = "unsloth_zoo.rl_replacements"
+ previous = sys.modules.get(name)
+ sys.modules[name] = zoo_module
+ trainer = _Trainer()
+ try:
+ exec(compile(block, "<gate>", "exec"), {"self": trainer, "pixel_values": pixel_values})
+ finally:
+ if previous is None:
+ del sys.modules[name]
+ else:
+ sys.modules[name] = previous
+ return trainer
+
+
+def test_the_gradient_path_refuses_an_old_zoo_without_the_chunker():
+ """The no-grad gate does not cover the gradient call, and the default run never reaches it.
+
+ With beta = 0 and num_iterations = 1 there are no reference or old logprobs to compute, so
+ `_get_per_token_logps_and_entropies` -- where the other gate lives -- is never called. An
+ older `grpo_accumulated_loss` takes arbitrary kwargs, ignores the ones it does not know and
+ replaces `pixel_values` with None for a model that carries no `image_grid_thw`, so a vision
+ run would have trained on the text alone and said nothing.
+ """
+ import types
+
+ block = _gradient_zoo_gate()
+ old_zoo = types.ModuleType("unsloth_zoo.rl_replacements")
+ with pytest.raises(RuntimeError) as raised:
+ _run_gate(block, pixel_values = object(), zoo_module = old_zoo)
+ assert "upgrade unsloth_zoo" in str(raised.value)
+
+ # A text-only run is untouched: there are no pixels to drop.
+ _run_gate(block, pixel_values = None, zoo_module = old_zoo)
+
+ # And a zoo that exports the chunker passes, once, and remembers it.
+ new_zoo = types.ModuleType("unsloth_zoo.rl_replacements")
+ new_zoo.grpo_vision_chunks = lambda *_a, **_k: None
+ trainer = _run_gate(block, pixel_values = object(), zoo_module = new_zoo)
+ assert trainer._unsloth_grpo_vision_zoo_checked is True
+ # Checked once per trainer: the flag short-circuits a later step even on a broken zoo,
+ # so the probe is not paid on every accumulation step.
+ already = _Trainer()
+ already._unsloth_grpo_vision_zoo_checked = True
+ import sys
+
+ sys.modules["unsloth_zoo.rl_replacements"] = old_zoo
+ try:
+ exec(compile(block, "<gate>", "exec"), {"self": already, "pixel_values": object()})
+ finally:
+ del sys.modules["unsloth_zoo.rl_replacements"]
+
+
+def test_the_gradient_gate_runs_before_every_accumulated_loss_call():
+ """Placement, since the block above proves only what it does once reached."""
+ from unsloth.models.rl_replacements import grpo_trainer_compute_loss
+
+ patched = grpo_trainer_compute_loss("compute_loss", "")
+ gate = patched.index("_unsloth_grpo_vision_zoo_checked")
+ calls = [
+ index
+ for index in range(len(patched))
+ if patched.startswith("grpo_accumulated_loss(", index)
+ ]
+ assert calls, "no accumulated-loss call to gate"
+ assert all(gate < index for index in calls), (gate, calls)
+
+
+def _persisted_vision_output(forward_kwargs, output):
+ """Run the block the output rewrite injects, on the batch dict it is given."""
+ import textwrap
+
+ from unsloth.models.rl_replacements import (
+ _unsloth_grpo_vision_inputs,
+ grpo_trainer__generate_and_score_completions,
+ )
+
+ source = (
+ ' if "image_sizes" in forward_kwargs:\n'
+ ' output["image_sizes"] = forward_kwargs["image_sizes"]\n'
+ " if images is not None:\n"
+ ' output["num_images"] = num_images\n'
+ " return output\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ lines = patched.splitlines()
+ start = next(i for i, line in enumerate(lines) if "_unsloth_vision_output = _unsloth" in line)
+ stop = next(i for i, line in enumerate(lines) if "output[_vision_key] = _vision_value" in line)
+ block = textwrap.dedent("\n".join(lines[start - 1 : stop + 1]))
+ namespace = {
+ "_unsloth_grpo_vision_inputs": _unsloth_grpo_vision_inputs,
+ "forward_kwargs": forward_kwargs,
+ "output": output,
+ }
+ exec(compile(block, "<output-extras>", "exec"), namespace)
+ return output
+
+
+def test_every_vision_kwarg_reaches_the_training_batch():
+ """TRL 0.24.0's output block copies five vision keys, so spatial_shapes, num_tiles and
+ the position ids never reach compute_loss and the gradient forward runs without them."""
+ already = object()
+ forward_kwargs = {
+ "pixel_values": already,
+ "spatial_shapes": "shapes",
+ "num_tiles": [2, 3],
+ "image_position_ids": "positions",
+ "token_type_ids": "types",
+ }
+ output = {"pixel_values": already, "num_images": [1, 1]}
+ persisted = _persisted_vision_output(forward_kwargs, output)
+ for key, value in forward_kwargs.items():
+ assert persisted[key] == value, key
+ assert persisted["pixel_values"] is already, "an already saved key was overwritten"
+ assert persisted["num_images"] == [1, 1]
+
+
+def test_a_key_the_processor_did_not_produce_is_not_invented():
+ persisted = _persisted_vision_output({"pixel_values": "pixels"}, {})
+ assert persisted == {"pixel_values": "pixels"}, persisted
+
+
+def test_the_tiles_of_a_sample_survive_the_shuffle_and_the_slice():
+ """_prepare_inputs shuffles and slices the batch by sample index, so a tile indexed
+ tensor has to be a list per sample while that happens."""
+ import torch
+
+ trl_utils = pytest.importorskip("trl.trainer.utils")
+ from unsloth.models.rl_replacements import (
+ _unsloth_grpo_split_vision_by_sample,
+ _unsloth_grpo_unsplit_vision,
+ )
+
+ num_tiles = [2, 3, 1]
+ owners = torch.tensor([0, 0, 1, 1, 1, 2])
+ batch = {
+ "prompt_ids": torch.arange(3).unsqueeze(-1),
+ "advantages": torch.zeros(3),
+ "num_images": [1, 1, 1],
+ "num_tiles": num_tiles,
+ "pixel_values": owners.reshape(-1, 1).float(),
+ "spatial_shapes": owners.reshape(-1, 1).clone(),
+ "pixel_attention_mask": owners.reshape(-1, 1).clone(),
+ }
+ torch.manual_seed(0)
+ split = _unsloth_grpo_split_vision_by_sample(batch)
+ assert isinstance(split["pixel_values"], list), "the tile axis was left flat"
+ shuffled = trl_utils.shuffle_sequence_dict(split)
+ chunks = trl_utils.split_tensor_dict(shuffled, 1)
+ restored = _unsloth_grpo_unsplit_vision(trl_utils.unsplit_pixel_values_by_grid(chunks[0]))
+
+ assert restored["pixel_values"].shape[0] == len(owners)
+ at = 0
+ seen = []
+ for prompt, tiles in zip(restored["prompt_ids"].tolist(), restored["num_tiles"]):
+ sample = prompt[0]
+ seen.append(sample)
+ assert tiles == num_tiles[sample], (sample, tiles)
+ for key in ("pixel_values", "spatial_shapes", "pixel_attention_mask"):
+ rows = restored[key][at : at + tiles].flatten().tolist()
+ assert rows == [sample] * tiles, (key, sample, rows)
+ at += tiles
+ assert sorted(seen) == [0, 1, 2], seen
+
+
+def test_the_gemma_position_ids_are_split_by_image_not_by_tile():
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {
+ "num_images": [2, 1],
+ "pixel_values": torch.arange(3).reshape(3, 1).float(),
+ "image_position_ids": torch.arange(3).reshape(3, 1),
+ }
+ split = _unsloth_grpo_split_vision_by_sample(batch)
+ assert [tensor.shape[0] for tensor in split["pixel_values"]] == [2, 1]
+ assert [tensor.flatten().tolist() for tensor in split["image_position_ids"]] == [[0, 1], [2]]
+
+
+def test_the_grid_layout_is_left_to_trl():
+ """TRL splits image_grid_thw itself in every version that persists it; splitting it
+ twice would hand the forward a list of lists."""
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {
+ "num_images": [1, 1],
+ "image_grid_thw": torch.tensor([[1, 2, 2], [1, 2, 2]]),
+ "pixel_values": torch.zeros(8, 1),
+ }
+ assert _unsloth_grpo_split_vision_by_sample(batch) is batch
+
+
+def test_a_padded_row_per_sample_is_left_alone():
+ """Idefics and SmolVLM pad to one row per sample, which TRL slices correctly already."""
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {
+ "num_images": [2, 2],
+ "pixel_values": torch.zeros(2, 2, 3, 4),
+ }
+ assert _unsloth_grpo_split_vision_by_sample(batch) is batch
+
+
+def test_a_padded_row_per_sample_survives_counts_that_sum_to_the_batch_size():
+ """The case above only holds because [2, 2] over two rows does not sum to two. A mixed
+ batch like [2, 0] does, and a padded sample axis then looks exactly like a flat image
+ axis: splitting it hands both rows to the first sample and the second an empty tensor,
+ so the images train against the wrong prompt with no error anywhere."""
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ for num_images in ([2, 0], [0, 2], [1, 2, 0], [1, 0, 2, 1]):
+ samples = len(num_images)
+ # Every value in row s is s, so a misattributed row is visible rather than inferred.
+ pixel_values = torch.stack(
+ [torch.full((2, 3, 4, 4), float(sample)) for sample in range(samples)]
+ )
+ batch = {
+ "num_images": num_images,
+ "pixel_values": pixel_values,
+ "prompt_ids": torch.zeros(samples, 5, dtype = torch.long),
+ }
+ split = _unsloth_grpo_split_vision_by_sample(batch)
+ assert split is batch, num_images
+ assert split["pixel_values"] is pixel_values, num_images
+
+ # All ones is not ambiguous: both readings put row s with sample s, so it still splits.
+ pixel_values = torch.stack([torch.full((3, 4, 4), float(s)) for s in range(2)])
+ split = _unsloth_grpo_split_vision_by_sample(
+ {"num_images": [1, 1], "pixel_values": pixel_values}
+ )
+ owners = [tensor.flatten().unique().tolist() for tensor in split["pixel_values"]]
+ assert owners == [[0.0], [1.0]], owners
+
+
+def test_a_batch_trl_already_split_is_not_split_again():
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_split_vision_by_sample
+
+ batch = {"num_images": [1, 1], "pixel_values": [torch.zeros(2, 1), torch.zeros(3, 1)]}
+ assert _unsloth_grpo_split_vision_by_sample(batch) is batch
+
+
+def test_the_counts_are_not_merged_as_if_they_were_tensors():
+ import torch
+
+ from unsloth.models.rl_replacements import _unsloth_grpo_unsplit_vision
+
+ batch = {"num_images": [1, 1], "num_tiles": [2, 3], "pixel_values": [torch.zeros(2, 1)]}
+ restored = _unsloth_grpo_unsplit_vision(batch)
+ assert restored["num_images"] == [1, 1]
+ assert restored["num_tiles"] == [2, 3]
+ assert restored["pixel_values"].shape == (2, 1)
+
+
+def test_prepare_inputs_splits_before_the_shuffle_and_merges_after_the_slice():
+ from unsloth.models.rl_replacements import grpo_trainer__prepare_inputs
+
+ source = (
+ " def _prepare_inputs(self, generation_batch):\n"
+ " if self._step % generate_every == 0:\n"
+ " generation_batch = self._generate_and_score_completions(generation_batch)\n"
+ " generation_batch = split_pixel_values_by_grid(generation_batch)\n"
+ " generation_batch = shuffle_sequence_dict(generation_batch)\n"
+ " generation_batches = split_tensor_dict(generation_batch, 2)\n"
+ " self._buffered_inputs = ["
+ "unsplit_pixel_values_by_grid(batch) for batch in generation_batches]\n"
+ )
+ patched = grpo_trainer__prepare_inputs("_prepare_inputs", source)
+ lines = patched.splitlines()
+ split_at = next(i for i, l in enumerate(lines) if "split_pixel_values_by_grid(" in l)
+ ours_at = next(i for i, l in enumerate(lines) if "_unsloth_grpo_split_vision_by_sample" in l)
+ shuffle_at = next(i for i, l in enumerate(lines) if "shuffle_sequence_dict(" in l)
+ assert split_at < ours_at < shuffle_at, (split_at, ours_at, shuffle_at)
+ assert lines[ours_at].startswith(" " * 12), lines[ours_at]
+ assert (
+ "_unsloth_grpo_unsplit_vision(unsplit_pixel_values_by_grid(batch))" in patched
+ ), "the slice is handed back still split"
+
+
+def test_a_prepare_inputs_that_changed_shape_says_so():
+ from unsloth.models import rl_replacements
+
+ said = []
+ original = rl_replacements._warn_once
+ rl_replacements._warn_once = lambda where, message: said.append(where)
+ try:
+ patched = rl_replacements.grpo_trainer__prepare_inputs(
+ "_prepare_inputs",
+ " generation_batch = split_pixel_values_by_grid(generation_batch)\n",
+ )
+ finally:
+ rl_replacements._warn_once = original
+ assert "_unsloth_grpo_split_vision_by_sample" not in patched
+ assert said == ["grpo_split_vision_by_sample"], said
+
+
+def test_both_halves_travel_with_the_generated_module():
+ from unsloth.models.rl_replacements import RL_PRE_ITEMS
+
+ pre = "\n".join(RL_PRE_ITEMS["grpo_trainer"])
+ assert "def _unsloth_grpo_split_vision_by_sample(" in pre
+ assert "def _unsloth_grpo_unsplit_vision(" in pre
+
+
+# The legacy 0.22.x-0.23.x rewrite is no longer a block that can be exec'd on its own: the
+# counts are built before the no-grad logprob calls and the output block reads them back.
+# tests/test_grpo_legacy_trl_vision.py runs the whole patched function instead.
+
+
+def test_the_counts_are_not_added_twice_on_a_trl_that_has_them():
+ """0.24.0 saves `num_images` itself, and a second assignment off a different variable is
+ two sources for one fact."""
+ from unsloth.models.rl_replacements import grpo_trainer__generate_and_score_completions
+
+ source = (
+ ' if "image_sizes" in forward_kwargs:\n'
+ ' output["image_sizes"] = forward_kwargs["image_sizes"]\n'
+ " if images is not None:\n"
+ ' output["num_images"] = num_images\n'
+ " return output\n"
+ )
+ patched = grpo_trainer__generate_and_score_completions(
+ "_generate_and_score_completions", source
+ )
+ assert patched.count('output["num_images"]') == 1, patched
diff --git a/tests/test_grpo_vision_zoo_floor.py b/tests/test_grpo_vision_zoo_floor.py
new file mode 100644
index 00000000000..53f7de6c56a
--- /dev/null
+++ b/tests/test_grpo_vision_zoo_floor.py
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""The declared unsloth_zoo floor has to supply the helper vision GRPO refuses to run without.
+
+Both GRPO logprob paths raise when `unsloth_zoo.rl_replacements.grpo_vision_chunks` is missing,
+because a local fallback could only cover the no-grad half: the gradient half is inside the zoo's
+own `grpo_accumulated_loss`, an older copy of which ignores the keys it does not know and, for a
+model with no `image_grid_thw` to slice by, replaces `pixel_values` with `None` outright. Half a
+fallback makes the two policies disagree, which is the defect this whole path exists to remove.
+
+So the guard is only honest while the floor in pyproject.toml names a release that carries the
+helper. With the floor left at 2026.9.4 (the last release that does not export it) an otherwise
+dependency-compliant install raises for every vision batch, including the single image grid
+models that worked before. unslothai/unsloth#6960, unslothai/unsloth-zoo#1233.
+
+2026.9.5 is published, so the floor is declared rather than deferred and the assertion below is
+live again. The message assertion is live too: whatever the metadata says, the raise has to name
+the release that fixes it.
+
+Reads files only, so it runs on the Windows and macOS runners too.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+import pytest
+from packaging.requirements import InvalidRequirement, Requirement
+from packaging.version import Version
+
+
+REPO = Path(__file__).resolve().parents[1]
+PYPROJECT = REPO / "pyproject.toml"
+RL_REPLACEMENTS = REPO / "unsloth" / "models" / "rl_replacements.py"
+
+# The first unsloth_zoo release that carries grpo_vision_chunks. The same number
+# unslothai/unsloth#11137 moves the floor to for the TRL ceiling, deliberately: one floor, not
+# two, or whichever is higher silently decides what a user resolves.
+ZOO_FLOOR_WITH_THE_VISION_CHUNKER = Version("2026.9.5")
+
+
+def _zoo_requirements() -> list[Requirement]:
+ import tomllib
+
+ data = tomllib.loads(PYPROJECT.read_text(encoding = "utf-8"))
+ project = data.get("project") or {}
+ raws: list[str] = list(project.get("dependencies") or [])
+ for extra in (project.get("optional-dependencies") or {}).values():
+ raws.extend(extra)
+ out = []
+ for raw in raws:
+ try:
+ req = Requirement(raw)
+ except InvalidRequirement:
+ continue
+ if req.name.lower().replace("_", "-") == "unsloth-zoo" and req.specifier:
+ out.append(req)
+ return out
+
+
+def _the_chunker_is_required() -> bool:
+ source = RL_REPLACEMENTS.read_text(encoding = "utf-8")
+ return "grpo_vision_chunks" in source and "needs an unsloth_zoo build that exports" in source
+
+
+def test_the_declared_zoo_floor_carries_the_chunker_the_vision_paths_require() -> None:
+ if not _the_chunker_is_required():
+ pytest.skip("rl_replacements.py no longer requires grpo_vision_chunks outright")
+ reqs = _zoo_requirements()
+ assert reqs, (
+ "vision GRPO raises without unsloth_zoo.grpo_vision_chunks and pyproject.toml names "
+ "no versioned unsloth_zoo requirement at all, so pip may resolve a zoo without it"
+ )
+ floors = {}
+ for req in reqs:
+ lower = [
+ Version(str(spec.version))
+ for spec in req.specifier
+ if spec.operator in (">=", "==", "~=")
+ ]
+ assert lower, f"unsloth_zoo requirement {req} has no lower bound"
+ floors[str(req)] = max(lower)
+ stale = {
+ raw: str(floor)
+ for raw, floor in floors.items()
+ if floor < ZOO_FLOOR_WITH_THE_VISION_CHUNKER
+ }
+ assert not stale, (
+ f"pyproject.toml still accepts unsloth_zoo {stale}, which predates "
+ f"grpo_vision_chunks. Every vision GRPO batch raises on such an install, including "
+ f"the single image grid models that used to work. Move the floor to "
+ f"{ZOO_FLOOR_WITH_THE_VISION_CHUNKER} in the same commit that requires the helper."
+ )
+ assert len(set(floors.values())) == 1, (
+ f"pyproject.toml declares more than one unsloth_zoo floor: {floors}. One of them is "
+ f"the one users actually resolve."
+ )
+
+
+def test_the_upgrade_message_names_the_release_the_floor_names() -> None:
+ """A message that says only "upgrade unsloth_zoo" leaves the user guessing which one."""
+ source = RL_REPLACEMENTS.read_text(encoding = "utf-8")
+ lines = source.splitlines()
+ raised = [
+ "\n".join(lines[i : i + 5])
+ for i, line in enumerate(lines)
+ if "Unsloth: vision GRPO needs an unsloth_zoo build that exports" in line
+ ]
+ assert len(raised) == 2, f"expected the no-grad and the gradient gate, found {len(raised)}"
+ for block in raised:
+ assert str(ZOO_FLOOR_WITH_THE_VISION_CHUNKER) in block, block
diff --git a/tests/test_multi_image_grpo_chunking.py b/tests/test_multi_image_grpo_chunking.py
index 350dc403cd5..7774719e85b 100644
--- a/tests/test_multi_image_grpo_chunking.py
+++ b/tests/test_multi_image_grpo_chunking.py
@@ -6,58 +6,94 @@ from __future__ import annotations
import math
import os
import re
+import pytest
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
SOURCE_PATH = os.path.join(REPO_ROOT, "unsloth", "models", "rl_replacements.py")
+def _zoo_vision_helpers(*names):
+ """The multi image helpers ship with unsloth_zoo. An unsloth_zoo installed from before
+ they landed has none of them, and the static gates in this file already prove this repo
+ asks for them and fails loudly without them, so behaviour that can only be driven
+ through the zoo is skipped rather than reported as this repo being broken."""
+ zoo = pytest.importorskip("unsloth_zoo.rl_replacements")
+ missing = [name for name in names if not hasattr(zoo, name)]
+ if missing:
+ pytest.skip(f"the installed unsloth_zoo has no {', '.join(missing)}")
+ return tuple(getattr(zoo, name) for name in names)
+
+
def _read_source() -> str:
with open(SOURCE_PATH, "r", encoding = "utf-8") as fh:
return fh.read()
-# Per-chunk slicing fixes (cum_rows, cum_imgs, axes)
-
-
-def test_cum_rows_materialized_on_cpu():
- src = _read_source()
- idx = src.find("cum_rows = torch.cat")
- assert idx != -1, "cum_rows assignment must exist"
- window = src[idx : idx + 400]
- assert "rows_per_sample.cumsum(0)" in window
- assert ").cpu()" in window, "cum_rows must be moved to CPU once via .cpu() after construction"
-
-
-def test_cum_imgs_slice_indices_use_item():
- src = _read_source()
- assert "cum_imgs[start].item()" in src
- assert "cum_imgs[end].item()" in src
-
-
-def test_image_sizes_image_axis_branch_present():
+def test_source_reads_the_shared_key_tuple():
src = _read_source()
- assert "image_sizes[img_start:img_end]" in src
- assert "_image_sizes_n" in src and "total_images" in src
-
-
-def test_pixel_attention_mask_three_way_check_present():
- src = _read_source()
- assert "pixel_attention_mask[img_start:img_end]" in src
- assert "pixel_attention_mask[start_pixel_idx:end_pixel_idx]" in src
- assert "pixel_attention_mask[start:end]" in src
- assert "image_grid_thw.shape[0]" in src
-
-
-def test_image_sizes_chunked_after_branch_decision():
- src = _read_source()
- pattern = re.compile(
- r"attention_mask_chunks\.append\(attention_mask\[start:end\]\)\s*\n\s*"
- r"image_sizes_chunks\.append\(slice_sample_axis\(image_sizes,\s*start,\s*end\)\)",
- )
- assert pattern.search(src) is None, (
- "image_sizes_chunks must not be appended unconditionally on the "
- "sample axis above the if/else; the axis is chosen per branch"
- )
+ assert "grpo_get_vision_inputs" in src
+ assert "grpo_vision_chunks" in src
+ assert "pixel_values_chunks" not in src
+ assert "image_grid_thw_chunks" not in src
+
+
+def test_grid_model_slices_rows_by_patch_and_grid_by_image():
+ import torch
+
+ (grpo_vision_chunks,) = _zoo_vision_helpers("grpo_vision_chunks")
+
+ num_images = [2, 1, 3, 1]
+ grid = torch.tensor([[1, 2, 2]] * sum(num_images)) # 4 patch rows per image
+ rows = int(grid.prod(dim = -1).sum())
+ vision = {
+ "pixel_values": torch.arange(rows).reshape(rows, 1).float(),
+ "image_grid_thw": grid,
+ "num_images": num_images,
+ }
+ chunks = grpo_vision_chunks(vision, total_samples = 4, batch_size = 2)
+ assert len(chunks) == 2
+ # samples 0 and 1 hold images 0 to 2, so patch rows 0 to 11
+ assert chunks[0]["pixel_values"].shape[0] == 12
+ assert chunks[0]["image_grid_thw"].shape[0] == 3
+ assert chunks[1]["pixel_values"].shape[0] == 16
+ assert chunks[1]["image_grid_thw"].shape[0] == 4
+ assert torch.equal(chunks[1]["pixel_values"], vision["pixel_values"][12:])
+
+
+def test_image_sizes_follows_the_image_axis_when_it_is_per_image():
+ import torch
+
+ (grpo_vision_chunks,) = _zoo_vision_helpers("grpo_vision_chunks")
+
+ vision = {
+ "pixel_values": torch.zeros(12, 1),
+ "image_grid_thw": torch.tensor([[1, 2, 2]] * 3),
+ "image_sizes": torch.tensor([[10, 10], [20, 20], [30, 30]]),
+ "num_images": [2, 1],
+ }
+ chunks = grpo_vision_chunks(vision, total_samples = 2, batch_size = 1)
+ assert chunks[0]["image_sizes"].tolist() == [[10, 10], [20, 20]]
+ assert chunks[1]["image_sizes"].tolist() == [[30, 30]]
+
+
+def test_pixel_attention_mask_axis_is_chosen_per_shape():
+ import torch
+
+ (grpo_vision_chunks,) = _zoo_vision_helpers("grpo_vision_chunks")
+
+ base = {
+ "pixel_values": torch.zeros(12, 1),
+ "image_grid_thw": torch.tensor([[1, 2, 2]] * 3),
+ "num_images": [2, 1],
+ }
+ # one mask row per image: image axis
+ per_image = grpo_vision_chunks({**base, "pixel_attention_mask": torch.zeros(3, 4)}, 2, 1)
+ assert per_image[0]["pixel_attention_mask"].shape[0] == 2
+ assert per_image[1]["pixel_attention_mask"].shape[0] == 1
+ # one mask row per patch row: patch axis
+ per_row = grpo_vision_chunks({**base, "pixel_attention_mask": torch.zeros(12, 4)}, 2, 1)
+ assert per_row[0]["pixel_attention_mask"].shape[0] == 8
+ assert per_row[1]["pixel_attention_mask"].shape[0] == 4
# Behavioral simulation of chunk math
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index bc1b3ac4c9f..aeb143d1c24 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -1529,6 +1529,17 @@ def _install_grpo_hidden_states_forward_wrapper(model):
_note_grpo_hidden_states_success(target_model)
return _replace_outputs_logits(outputs, hidden_states)
+ # generate() validates its kwargs against inspect.signature(self.forward), so a bare
+ # (*args, **kwargs) wrapper makes every forward-only vision kwarg look unused.
+ # __signature__ and NOT functools.wraps: wraps sets __wrapped__, and accelerate's
+ # extract_model_from_parallel(keep_fp32_wrapper = False) walks that chain every GRPO
+ # step and rebinds past this wrapper, losing it. inspect.unwrap() stops at __signature__.
+ wrapped_forward.__signature__ = forward_signature
+ for _attribute in ("__name__", "__qualname__", "__doc__", "__module__"):
+ try:
+ setattr(wrapped_forward, _attribute, getattr(original_forward, _attribute))
+ except Exception:
+ pass
wrapped_forward._unsloth_grpo_hidden_states_forward_wrapped = True
target_model.forward = wrapped_forward
setattr(target_model, _UNSLOTH_GRPO_HIDDEN_STATES_WRAPPED_ATTR, True)
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 73ee531c422..a5dc834a8ff 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -920,6 +920,224 @@ def _unsloth_grpo_autocast_kwargs(self, device_type = "cuda"):
return {"enabled": enabled, "dtype": dtype}
+def _unsloth_grpo_vision_inputs(source):
+ """unsloth_zoo owns this key tuple; the copy below is the fallback for a zoo predating
+ GRPO_VISION_KEYS, and test_grpo_vision_kwargs_forwarded.py fails if the two diverge."""
+ try:
+ from unsloth_zoo.rl_replacements import grpo_get_vision_inputs
+ return grpo_get_vision_inputs(source)
+ except Exception:
+ pass
+ if source is None:
+ return {}
+ get = getattr(source, "get", None)
+ if get is None:
+ return {}
+ return {
+ key: get(key, None)
+ for key in (
+ "pixel_values",
+ "image_grid_thw",
+ "pixel_attention_mask",
+ "image_sizes",
+ "spatial_shapes",
+ "num_tiles",
+ # both Gemma 4 spellings: image_position_ids is TRL >= 1.1.0
+ "image_position_ids",
+ "pixel_position_ids",
+ "num_images",
+ "token_type_ids",
+ "mm_token_type_ids",
+ )
+ }
+
+
+def _unsloth_grpo_split_vision_by_sample(batch):
+ """TRL only splits the tile and image indexed vision tensors per sample from its own
+ split_pixel_values_by_grid, and before TRL 1.1.0 that function knows only the
+ image_grid_thw layout. Every other layout is left flat, and _prepare_inputs then
+ shuffles and slices the batch by sample index, so an LFM2-VL or Gemma batch has its
+ tiles reordered away from the samples they belong to. This mirrors the current
+ trl.trainer.utils.split_pixel_values_by_grid so an older TRL keeps them together."""
+
+ def _counts(value):
+ if value is None:
+ return None
+ if hasattr(value, "tolist"):
+ value = value.tolist()
+ try:
+ return [int(count) for count in value]
+ except TypeError:
+ return None
+
+ pixel_values = batch.get("pixel_values", None)
+ if pixel_values is None:
+ return batch
+ image_grid_thw = batch.get("image_grid_thw", None)
+ num_images = _counts(batch.get("num_images", None))
+
+ if isinstance(pixel_values, list):
+ # TRL split it. From 1.1.0 split_pixel_values_by_grid splits by SAMPLE, off num_images;
+ # 0.22.x-0.23.x splits by GRID ROW -- "lengths = batch["image_grid_thw"].prod(dim=1)",
+ # one element per image -- and leaves image_grid_thw itself flat. The shuffle right
+ # after takes its length from the first entry of the batch and indexes everything by
+ # sample, so on a row holding two images that list is both permuted wrongly and
+ # truncated to the sample count. Regroup it, and split the grid the way 1.1.0 does.
+ # all-ones, not len(pixel_values) == len(num_images): over two samples num_images = [0, 2]
+ # makes those two numbers agree while the axes still differ, and the early return would
+ # hand both of the second sample's images to the first.
+ _prompt_ids = batch.get("prompt_ids", None)
+ if (
+ image_grid_thw is None
+ or isinstance(image_grid_thw, list)
+ or not num_images
+ or not pixel_values
+ or len(pixel_values) != sum(num_images)
+ or all(_count == 1 for _count in num_images)
+ or (_prompt_ids is not None and len(num_images) != _prompt_ids.shape[0])
+ ):
+ return batch
+ split = dict(batch)
+ _empty = pixel_values[0][:0]
+ _grouped = []
+ _offset = 0
+ for _count in num_images:
+ _group = pixel_values[_offset : _offset + _count]
+ _offset += _count
+ _grouped.append(torch.cat(_group, dim = 0) if _group else _empty)
+ split["pixel_values"] = _grouped
+ split["image_grid_thw"] = list(torch.split(image_grid_thw, num_images, dim = 0))
+ for _image_key in ("pixel_attention_mask", "image_sizes"):
+ _per_image = batch.get(_image_key, None)
+ if (
+ _per_image is not None
+ and not isinstance(_per_image, list)
+ and _per_image.shape[0] == sum(num_images)
+ and _per_image.shape[0] != len(num_images)
+ ):
+ split[_image_key] = list(torch.split(_per_image, num_images, dim = 0))
+ return split
+
+ if image_grid_thw is not None:
+ # An unsplit grid batch: TRL owns this layout in every version that persists it.
+ return batch
+ if not num_images:
+ return batch
+ rows = pixel_values.shape[0]
+ num_tiles = _counts(batch.get("num_tiles", None))
+ split = dict(batch)
+ for _position_key in ("image_position_ids", "pixel_position_ids"):
+ _position_ids = batch.get(_position_key, None)
+ if _position_ids is None or isinstance(_position_ids, list):
+ continue
+ if rows != sum(num_images) or _position_ids.shape[0] != sum(num_images):
+ continue
+ split["pixel_values"] = list(torch.split(pixel_values, num_images, dim = 0))
+ split[_position_key] = list(torch.split(_position_ids, num_images, dim = 0))
+ return split
+ if num_tiles and rows == sum(num_tiles):
+ split["pixel_values"] = list(torch.split(pixel_values, num_tiles, dim = 0))
+ for _tile_key in ("pixel_attention_mask", "spatial_shapes"):
+ _tiled = batch.get(_tile_key, None)
+ if (
+ _tiled is not None
+ and not isinstance(_tiled, list)
+ and _tiled.shape[0] == sum(num_tiles)
+ ):
+ split[_tile_key] = list(torch.split(_tiled, num_tiles, dim = 0))
+ return split
+ if rows != sum(num_images):
+ # One padded row per sample already (Idefics, SmolVLM): TRL leaves this alone.
+ return batch
+ if rows == len(num_images) and any(_count != 1 for _count in num_images):
+ # A padded sample axis and a flat image axis are the same length here, and only the
+ # padded reading keeps each row with the sample it came from: num_images = [2, 0]
+ # over two padded rows would hand both of them to the first sample and the second an
+ # empty tensor. Nothing in the batch tells the two apart, so keep TRL's layout, which
+ # is what every version does today. The all-ones case is excluded because the two
+ # readings agree there. Same hazard the list branch above guards against.
+ return batch
+ split["pixel_values"] = list(torch.split(pixel_values, num_images, dim = 0))
+ _image_sizes = batch.get("image_sizes", None)
+ if (
+ _image_sizes is not None
+ and not isinstance(_image_sizes, list)
+ and _image_sizes.shape[0] == sum(num_images)
+ ):
+ split["image_sizes"] = list(torch.split(_image_sizes, num_images, dim = 0))
+ return split
+
+
+def _unsloth_grpo_unsplit_vision(batch):
+ """Undo _unsloth_grpo_split_vision_by_sample once this step's slice has been taken, so
+ the forward sees the layout the processor produced. TRL's own unsplit merges only
+ pixel_values before 1.1.0, and pixel_values plus the grid and position ids from 1.1.0."""
+ merged = None
+ for key in (
+ "pixel_values",
+ "image_grid_thw",
+ "pixel_attention_mask",
+ "spatial_shapes",
+ "image_sizes",
+ "image_position_ids",
+ "pixel_position_ids",
+ ):
+ value = batch.get(key, None)
+ if not isinstance(value, list) or len(value) == 0:
+ continue
+ if not hasattr(value[0], "shape"):
+ # num_images and num_tiles are plain counts, not tensors to merge.
+ continue
+ if merged is None:
+ merged = dict(batch)
+ merged[key] = torch.cat(value, dim = 0)
+ return batch if merged is None else merged
+
+
+# A list cell reaches the processor as [[[img, img]]], which it rejects. unsloth#3605.
+def _unsloth_grpo_image_cell(value):
+ if value is None:
+ return None
+ if isinstance(value, (list, tuple)):
+ return list(value)
+ return [value]
+
+
+def _unsloth_reject_grpo_image_list(inputs, trainer = None):
+ """Refuse it where the rewrite above missed TRL's spelling: the processor's own error
+ names neither the column nor the fix.
+
+ `trainer` narrows the refusal to the one runtime mode a legacy TRL cannot carry a multi
+ image row through. Its vLLM server path sends the raw cells to `VLLMClient.generate`,
+ which does `[pil_to_base64(img) for img in images]` over the top level entries, so a cell
+ holding two images reaches `list.save(...)` and dies with an AttributeError naming neither.
+ Colocate mode and the no vLLM path both go through the processor, which this change fixed,
+ so they keep working and must not be refused."""
+ if trainer is not None:
+ if not getattr(trainer, "use_vllm", False):
+ return
+ if getattr(trainer, "vllm_mode", None) != "server":
+ return
+ # Every row, not just the first: one list cell anywhere in the batch is enough to put the
+ # images and the placeholders out of step, and a dataset that mixes a bare image with a
+ # list is exactly the shape that puts the list somewhere other than row 0.
+ try:
+ rows = list(inputs)
+ except Exception:
+ return
+ for _row_index, _row in enumerate(rows):
+ value = _row.get("image", None) if isinstance(_row, dict) else None
+ if isinstance(value, (list, tuple)) and len(value) > 1:
+ raise ValueError(
+ f"Unsloth: GRPO received a singular `image` column holding {len(value)} "
+ f"images in row {_row_index}, and this TRL version cannot carry more than "
+ "one image per row through to the model. "
+ "Rename the column to `images`, which TRL reads as the per example list of "
+ "images, or keep one image per row. "
+ "See https://github.com/unslothai/unsloth/issues/3605"
+ )
+
+
def grpo_trainer__prepare_inputs(function_name, function):
if function_name != "_prepare_inputs":
return function
@@ -934,6 +1152,31 @@ def grpo_trainer__prepare_inputs(function_name, function):
"self.accelerator.unwrap_model(self.model)",
"self.accelerator.unwrap_model(self.model, keep_fp32_wrapper = False)",
)
+
+ # Keep every tile with its own sample across the shuffle and the slice. unsloth#6960.
+ _split_anchor = re.search(
+ r"\n([ \t]+)generation_batch = split_pixel_values_by_grid\(generation_batch\)\n",
+ function,
+ )
+ _unsplit_anchor = "unsplit_pixel_values_by_grid(batch) for batch in generation_batches"
+ if _split_anchor is not None and _unsplit_anchor in function:
+ _spacing = _split_anchor.group(1)
+ function = function.replace(
+ _split_anchor.group(0),
+ _split_anchor.group(0)
+ + f"{_spacing}generation_batch = _unsloth_grpo_split_vision_by_sample(generation_batch)\n",
+ )
+ function = function.replace(
+ _unsplit_anchor,
+ "_unsloth_grpo_unsplit_vision(unsplit_pixel_values_by_grid(batch)) "
+ "for batch in generation_batches",
+ )
+ elif "split_pixel_values_by_grid" in function:
+ _warn_once(
+ "grpo_split_vision_by_sample",
+ "Unsloth: the GRPO batch splitting changed shape, so a multi tile vision "
+ "batch is being shuffled by sample index without its tiles.",
+ )
return function
@@ -1031,7 +1274,7 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
line_to_replace = 'batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size'
- replacement_lines = """
+ replacement_lines_head = """
max_left_pad = None
batch_size = self.args.per_device_train_batch_size if mode == "train" else self.args.per_device_eval_batch_size
try:
@@ -1049,8 +1292,100 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
_use_gc = self.model._unsloth_gradient_checkpointing if hasattr(self.model, '_unsloth_gradient_checkpointing') else getattr(self.args, 'gradient_checkpointing', True)
self.model.for_training(use_gradient_checkpointing=_use_gc)"""
+ # TRL 0.22.x-0.23.x has neither `forward_kwargs` nor `num_images`: it hardcoded one image per
+ # example, counted nothing, and saved only four processor keys. The singular `image` column
+ # normalisation below lets a cell hold several, and every consumer of a multi image batch --
+ # the shared chunker, and the per sample split in _prepare_inputs -- reads those counts, so
+ # without them a two image row is sliced as if it were two rows and the images are handed to
+ # the wrong samples. Built at this anchor rather than in the output block, because the no-grad
+ # old/reference logprob calls a few lines below take it too and the output block only runs once
+ # they have already returned. Counted off the same normalised cells the processor was given, so
+ # the two can never disagree; a row with no image counts zero.
+ #
+ # Inserted only for the versions whose call sites it feeds. 0.24.0 and up leave prompt_inputs
+ # unbound here, so the try below would fall straight through, but not emitting it at all is
+ # what keeps their generated source byte for byte what it was.
+ _legacy_vision_prologue = """
+ _unsloth_legacy_vision = {}
+ try:
+ # Gated the way TRL 0.24.0 gates forward_kwargs ("if images is not None"): a text only
+ # batch must forward nothing. Gemma 3's processor returns token_type_ids even with no
+ # images, and both PrefixGrouper and sequence packing are keyed on that name being
+ # absent, so copying it across would quietly cost a text run both fast paths.
+ if has_images and images is not None:
+ _unsloth_legacy_vision = _unsloth_grpo_vision_inputs(prompt_inputs)
+ _unsloth_legacy_vision["num_images"] = [
+ len(_unsloth_cell) if _unsloth_cell else 0
+ for _unsloth_cell in (
+ _unsloth_grpo_image_cell(_unsloth_image) for _unsloth_image in images
+ )
+ ]
+ _unsloth_legacy_vision = {
+ _unsloth_key: _unsloth_value
+ for _unsloth_key, _unsloth_value in _unsloth_legacy_vision.items()
+ if _unsloth_value is not None
+ }
+ if _unsloth_legacy_vision.get("pixel_values", None) is None:
+ # An image column whose cells are all empty: a text batch wearing one.
+ _unsloth_legacy_vision.pop("token_type_ids", None)
+ _unsloth_legacy_vision.pop("mm_token_type_ids", None)
+ # TRL 0.24.0 extends these with zeros over the completion before the very same call
+ # (grpo_trainer.py, "If token_type_ids are used, extend them with zeros"); 0.22.x-0.23.x
+ # never forwarded them at all, so the processor's prompt length tensor would not fit a
+ # prompt+completion forward. Only the prompt width is widened; every other width is
+ # dropped rather than sent, including one that happens to equal prompt+completion, which
+ # on these versions can only be an untruncated prompt whose tokens sit at the wrong
+ # offsets after max_prompt_length moved them.
+ for _unsloth_key in ("token_type_ids", "mm_token_type_ids"):
+ _unsloth_ids = _unsloth_legacy_vision.get(_unsloth_key, None)
+ if _unsloth_ids is None:
+ continue
+ if (
+ getattr(_unsloth_ids, "ndim", 0) == 2
+ and _unsloth_ids.shape[0] == prompt_completion_ids.shape[0]
+ and _unsloth_ids.shape[1] == prompt_ids.shape[1]
+ ):
+ _unsloth_legacy_vision[_unsloth_key] = torch.cat(
+ [_unsloth_ids, _unsloth_ids.new_zeros(completion_ids.shape)], dim = 1
+ )
+ else:
+ _unsloth_legacy_vision.pop(_unsloth_key, None)
+ except NameError:
+ _unsloth_legacy_vision = {}"""
+
+ _legacy_vision_trl = 'pixel_values=prompt_inputs.get("pixel_values")' in function
+ replacement_lines = replacement_lines_head + (
+ _legacy_vision_prologue if _legacy_vision_trl else ""
+ )
+
function = function.replace(line_to_replace, replacement_lines)
+ # TRL 0.22.x-0.23.x calls the no-grad old/reference logprob pass with four hand named
+ # processor tensors and no counts, so the shared chunker cannot tell one sample's images
+ # from the next and slices an image indexed grid by sample index.
+ _legacy_vision_call = re.compile(
+ r"(^[ \t]+)pixel_values=prompt_inputs\.get\(\"pixel_values\"\),\n"
+ r"[ \t]+image_grid_thw=prompt_inputs\.get\(\"image_grid_thw\"\),\n"
+ r"[ \t]+pixel_attention_mask=prompt_inputs\.get\(\"pixel_attention_mask\"\),\n"
+ r"[ \t]+image_sizes=prompt_inputs\.get\(\"image_sizes\"\),\n",
+ re.MULTILINE,
+ )
+ function, _legacy_vision_calls = _legacy_vision_call.subn(
+ r"\1**_unsloth_legacy_vision,\n",
+ function,
+ )
+ # Every TRL from 0.22.2 to 0.23.1 takes all three rewrites, and 0.24.0 and up pass
+ # forward_kwargs instead and need none, so this is a guard against a future drift.
+ _legacy_vision_unplumbed = (
+ _legacy_vision_calls == 0 and 'prompt_inputs.get("pixel_values")' in function
+ )
+ if _legacy_vision_unplumbed:
+ _warn_once(
+ "grpo_legacy_vision_calls",
+ "Unsloth: the GRPO reference logprob call sites changed shape, so a multi "
+ "image batch is being sliced there without its image counts.",
+ )
+
pattern_to_find = re.compile(
r"^\s*if self\.args\.gradient_accumulation_steps % generate_every != 0 or \(\s*"
r"self\.use_vllm and self\.vllm_importance_sampling_correction\s*"
@@ -1142,8 +1477,18 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
string_to_find = """ if "image_sizes" in prompt_inputs:
output["image_sizes"] = prompt_inputs["image_sizes"]"""
+ # 0.22.x-0.23.x saves the four keys above and nothing else, so a processor that emits
+ # spatial_shapes, num_tiles or the Gemma position ids loses them between generation and the
+ # gradient forward, and there is no `if images is not None` anchor here to hang the generic
+ # copy on. Same mapping the no-grad calls above were given, so both policies see one batch.
replacement_string = """ if "image_sizes" in prompt_inputs:
output["image_sizes"] = prompt_inputs["image_sizes"]
+ try:
+ for _vision_key, _vision_value in _unsloth_legacy_vision.items():
+ if _vision_value is not None and _vision_key not in output:
+ output[_vision_key] = _vision_value
+ except NameError:
+ pass
if max_left_pad is not None:
output["max_left_pad"] = torch.tensor(prompt_ids.shape[0] * [max_left_pad]).unsqueeze(-1)
try:
@@ -1177,12 +1522,14 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
_target_line + _metadata_extraction,
)
- # This path is for TRL 0.24.0: `images` is a variable exclusive to that version.
- string_to_find = """ if images is not None:
- output["num_images"] = num_images"""
-
- replacement_string = """ if images is not None:
- output["num_images"] = num_images
+ _output_extras = """
+ try:
+ _unsloth_vision_output = _unsloth_grpo_vision_inputs(forward_kwargs)
+ except NameError:
+ _unsloth_vision_output = {}
+ for _vision_key, _vision_value in _unsloth_vision_output.items():
+ if _vision_value is not None and _vision_key not in output:
+ output[_vision_key] = _vision_value
if max_left_pad is not None:
output["max_left_pad"] = torch.tensor(prompt_ids.shape[0] * [max_left_pad]).unsqueeze(-1)
try:
@@ -1191,7 +1538,26 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
except NameError:
output["sampling_per_token_logps"] = None"""
- function = function.replace(string_to_find, replacement_string)
+ # Longest first: the short anchor strands TRL 1.7.0's num_tiles line. unsloth#6960.
+ _num_images_anchors = (
+ """ if images is not None:
+ output["num_images"] = num_images
+ if num_tiles is not None:
+ output["num_tiles"] = num_tiles""",
+ """ if images is not None:
+ output["num_images"] = num_images""",
+ )
+ for _anchor in _num_images_anchors:
+ if _anchor in function:
+ function = function.replace(_anchor, _anchor + _output_extras)
+ break
+ else:
+ if 'output["num_images"]' in function:
+ _warn_once(
+ "grpo_output_num_images",
+ "Unsloth: the GRPO num_images output block changed shape, so "
+ "max_left_pad and the vLLM sampling logprobs are not being saved.",
+ )
if trl_version >= Version("0.24.0"):
string_to_find = " rewards_per_func = self._calculate_rewards(inputs, prompts, completions, completion_ids_list)"
@@ -1287,6 +1653,99 @@ def grpo_trainer__generate_and_score_completions(function_name, function):
" return output",
)
+ # Anchors for TRL >= 1.0.0, then 0.22.x-0.23.x.
+ _image_cell_anchors = (
+ (
+ 'images = [[example.get("image")] if example.get("image") is not None else None for example in inputs]',
+ 'images = [_unsloth_grpo_image_cell(example.get("image")) for example in inputs]',
+ ),
+ (
+ 'kwargs = {"images": [[img] for img in images]}',
+ 'kwargs = {"images": [_unsloth_grpo_image_cell(img) for img in images]}',
+ ),
+ )
+ _image_cell_normalised = False
+ _legacy_image_cell = False
+ for _index, (_old_cell, _new_cell) in enumerate(_image_cell_anchors):
+ if _old_cell in function:
+ function = function.replace(_old_cell, _new_cell)
+ _image_cell_normalised = True
+ # The legacy spelling is the only one whose placeholders are sized elsewhere.
+ _legacy_image_cell = _index == 1
+
+ # TRL 0.22.x-0.23.x hardcodes one image placeholder per example.
+ _placeholder_old = (
+ " for prompt in prompts:\n"
+ " if isinstance(prompt, list): # i.e., when using conversational data\n"
+ " prepare_multimodal_messages(prompt, num_images=1)"
+ )
+ # An empty cell takes zero placeholders, not one: the prologue already counts it as zero,
+ # and a batch mixing an image bearing row with an empty one otherwise reaches the processor
+ # a placeholder short of its images. An all empty column is a text batch wearing an image
+ # one, which TRL demotes itself from 0.24.0; ported here, because zero placeholders against
+ # `[[], []]` is an IndexError inside the processor rather than a text run. The guard mirrors
+ # TRL's `all(img_list == [] ...)`, so a `None` cell still fails exactly as it does on main.
+ _placeholder_new = (
+ ' if kwargs.get("images") and all(\n'
+ ' _unsloth_cell == [] for _unsloth_cell in kwargs["images"]\n'
+ " ):\n"
+ " kwargs = {}\n"
+ " has_images = False\n"
+ ' for prompt, _unsloth_cell in zip(prompts, kwargs.get("images", [])):\n'
+ " if isinstance(prompt, list): # i.e., when using conversational data\n"
+ " prepare_multimodal_messages(\n"
+ " prompt, num_images=len(_unsloth_cell) if _unsloth_cell else 0\n"
+ " )"
+ )
+ # TRL 0.24.0 and up size the placeholders themselves, off the cell they were handed, so the
+ # rewrite above is the whole job there. TRL 0.20.0 and 0.21.0 are the gap: they take the
+ # legacy cell spelling but predate `prepare_multimodal_messages` entirely and inline one
+ # `{"type": "image"}` per user message with no count, so a normalised two image cell would
+ # reach the processor with one placeholder. Nothing else in the batch disagrees, because the
+ # prologue counts the same cells, so it surfaces inside the processor.
+ _placeholder_sized = not _legacy_image_cell
+ if _placeholder_old in function:
+ function = function.replace(_placeholder_old, _placeholder_new)
+ _placeholder_sized = True
+
+ # Rewriting the cell is only half of a multi image row: on a legacy TRL the reference
+ # logprob call sites have to carry the counts too, or the images are sliced by sample
+ # index and all but the first are dropped. That surfaces as a token/feature mismatch
+ # raised inside the model, naming neither the column nor the fix, so refuse it here
+ # instead. A single image cell is unaffected either way and keeps working.
+ # `self` and not nothing: on a legacy TRL every other path now carries a multi image row,
+ # so the refusal has to narrow to the vLLM server mode that still cannot. Passing the
+ # trainer moves that decision to runtime, where `vllm_mode` is actually known.
+ _guard_argument = ""
+ if not _image_cell_normalised or _legacy_vision_unplumbed or not _placeholder_sized:
+ pass
+ elif _legacy_image_cell:
+ _guard_argument = ", self"
+ if (
+ _guard_argument
+ or not _image_cell_normalised
+ or _legacy_vision_unplumbed
+ or (not _placeholder_sized)
+ ):
+ _signature = re.search(
+ r"([ \t]*)def _generate_and_score_completions\((?:[^()]|\([^()]*\))*\)[^\n]*:\n",
+ function,
+ )
+ if _signature is None:
+ _warn_once(
+ "grpo_image_column",
+ "Unsloth: could not install the GRPO singular `image` column "
+ "normalisation; a column holding a list of images may fail inside the "
+ "processor. See https://github.com/unslothai/unsloth/issues/3605",
+ )
+ else:
+ _indent = _signature.group(1) + " "
+ function = (
+ function[: _signature.end()]
+ + f"{_indent}_unsloth_reject_grpo_image_list(inputs{_guard_argument})\n"
+ + function[_signature.end() :]
+ )
+
return function
@@ -1423,22 +1882,32 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
compute_aux_loss = kwargs.get("compute_aux_loss", None)
- pixel_values, image_grid_thw = (
- kwargs.get("pixel_values", None),
- kwargs.get("image_grid_thw", None),
- )
- pixel_attention_mask, image_sizes = (
- kwargs.get("pixel_attention_mask", None),
- kwargs.get("image_sizes", None),
- )
- num_images = kwargs.get("num_images", None)
+ # Body-local: this source is copied out without this module's imports. #6960.
+ _grpo_vision_chunks = None
+ try:
+ from unsloth_zoo.rl_replacements import grpo_vision_chunks as _grpo_vision_chunks
+ except Exception:
+ pass
+ # Collected even without the zoo: an older one must cost only image slicing.
+ vision_inputs = _unsloth_grpo_vision_inputs(kwargs)
+ if _grpo_vision_chunks is None and vision_inputs.get("pixel_values", None) is not None:
+ raise RuntimeError(
+ "Unsloth: vision GRPO needs an unsloth_zoo build that exports "
+ "grpo_vision_chunks, the shared multimodal key tuple and chunker "
+ "used by both GRPO logprob paths. Please upgrade unsloth_zoo to "
+ "2026.9.5 or newer: pip install -U unsloth_zoo"
+ )
+ pixel_values = vision_inputs.get("pixel_values", None)
+ image_grid_thw = vision_inputs.get("image_grid_thw", None)
+ num_images = vision_inputs.get("num_images", None)
# Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models.
- token_type_ids = kwargs.get("token_type_ids", None)
- mm_token_type_ids = kwargs.get("mm_token_type_ids", None)
+ token_type_ids = vision_inputs.get("token_type_ids", None)
+ mm_token_type_ids = vision_inputs.get("mm_token_type_ids", None)
if mm_token_type_ids is not None or image_grid_thw is not None:
mm_token_type_ids = _unsloth_fix_mm_token_type_ids(
self.processing_class, input_ids, mm_token_type_ids
)
+ vision_inputs["mm_token_type_ids"] = mm_token_type_ids
unwrapped_model = self.accelerator.unwrap_model(model, keep_fp32_wrapper = False)
@@ -1484,116 +1953,33 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
else:
max_left_pad = 0
- def slice_sample_axis(value, start, end):
- if value is None:
- return None
- return value[start:end]
-
import math
total_samples = input_ids.shape[0]
batch_size = math.ceil(total_samples / B)
- if isinstance(num_images, torch.Tensor):
- num_images = num_images.detach().cpu().reshape(-1).tolist()
- if image_grid_thw is not None and pixel_values is not None and num_images is not None:
- rows_per_image = image_grid_thw.prod(dim = -1)
- rows_per_sample = torch.split(rows_per_image, num_images)
- rows_per_sample = torch.stack([s.sum() for s in rows_per_sample])
- # cum_rows is indexed via .item() inside the per-chunk loop, so keeping it on CPU avoids a per-iteration GPU->CPU sync.
- cum_rows = torch.cat(
- [
- torch.tensor([0], device = rows_per_sample.device),
- rows_per_sample.cumsum(0),
- ]
- ).cpu()
- cum_imgs = torch.tensor([0] + num_images).cumsum(0)
- else:
- cum_rows = None
- cum_imgs = None
-
- def _first_dim_len(value):
- if value is None:
- return None
- if hasattr(value, "shape"):
- return value.shape[0]
- try:
- return len(value)
- except TypeError:
- return None
-
- total_images = sum(num_images) if num_images is not None else None
- _image_sizes_n = _first_dim_len(image_sizes)
input_ids_chunks = []
attention_mask_chunks = []
- pixel_values_chunks = []
- image_grid_thw_chunks = []
- pixel_attention_mask_chunks = []
- image_sizes_chunks = []
- token_type_ids_chunks = []
- mm_token_type_ids_chunks = []
-
- current_pixel_idx = 0
for start in range(0, total_samples, batch_size):
end = min(start + batch_size, total_samples)
-
input_ids_chunks.append(input_ids[start:end])
attention_mask_chunks.append(attention_mask[start:end])
- token_type_ids_chunks.append(slice_sample_axis(token_type_ids, start, end))
- mm_token_type_ids_chunks.append(slice_sample_axis(mm_token_type_ids, start, end))
-
- if image_grid_thw is not None and pixel_values is not None:
- if num_images is None:
- grid_slice = image_grid_thw[start:end]
- batch_pixel_count = grid_slice.prod(dim = -1).sum().item()
- start_pixel_idx = current_pixel_idx
- end_pixel_idx = current_pixel_idx + batch_pixel_count
- current_pixel_idx = end_pixel_idx
- img_start = img_end = None
- else:
- start_pixel_idx = cum_rows[start].item()
- end_pixel_idx = cum_rows[end].item()
- img_start = cum_imgs[start].item()
- img_end = cum_imgs[end].item()
- grid_slice = image_grid_thw[img_start:img_end]
- image_grid_thw_chunks.append(grid_slice)
-
- pixel_values_chunks.append(pixel_values[start_pixel_idx:end_pixel_idx])
-
- if image_sizes is None:
- image_sizes_chunks.append(None)
- elif (
- num_images is not None
- and _image_sizes_n == total_images
- and img_start is not None
- ):
- image_sizes_chunks.append(image_sizes[img_start:img_end])
- else:
- image_sizes_chunks.append(slice_sample_axis(image_sizes, start, end))
-
- if pixel_attention_mask is None:
- pixel_attention_mask_chunks.append(None)
- elif (
- num_images is not None
- and img_start is not None
- and pixel_attention_mask.shape[0] == image_grid_thw.shape[0]
- ):
- pixel_attention_mask_chunks.append(pixel_attention_mask[img_start:img_end])
- elif (
- pixel_attention_mask.shape[0] == pixel_values.shape[0]
- and pixel_attention_mask.shape[0] != input_ids.shape[0]
- ):
- pixel_attention_mask_chunks.append(
- pixel_attention_mask[start_pixel_idx:end_pixel_idx]
- )
- else:
- pixel_attention_mask_chunks.append(pixel_attention_mask[start:end])
- else:
- pixel_values_chunks.append(None)
- image_grid_thw_chunks.append(None)
- pixel_attention_mask_chunks.append(None)
- image_sizes_chunks.append(slice_sample_axis(image_sizes, start, end))
+ # One chunker shared with the gradient pass, so the two cannot disagree.
+ if _grpo_vision_chunks is None:
+ # Image-indexed keys already raised above, so only per-sample ones are left.
+ vision_chunks = []
+ for _start in range(0, total_samples, batch_size):
+ _end = min(_start + batch_size, total_samples)
+ vision_chunks.append(
+ {
+ _key: vision_inputs[_key][_start:_end]
+ for _key in ("token_type_ids", "mm_token_type_ids")
+ if vision_inputs.get(_key, None) is not None
+ }
+ )
+ else:
+ vision_chunks = _grpo_vision_chunks(vision_inputs, total_samples, batch_size)
temperature = self.temperature
model_config = _unsloth_get_model_config(model)
@@ -1615,12 +2001,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
zipped_inputs = zip(
input_ids_chunks,
attention_mask_chunks,
- pixel_values_chunks,
- image_grid_thw_chunks,
- pixel_attention_mask_chunks,
- image_sizes_chunks,
- token_type_ids_chunks,
- mm_token_type_ids_chunks,
+ vision_chunks,
)
os.environ["UNSLOTH_RETURN_HIDDEN_STATES"] = "1"
@@ -2061,18 +2442,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
for (
input_ids_chunk,
attention_mask_chunk,
- pixel_values_chunk,
- image_grid_thw_chunk,
- pixel_attention_mask_chunk,
- image_sizes_chunk,
- token_type_ids_chunk,
- mm_token_type_ids_chunk,
+ vision_chunk,
) in zipped_inputs:
- _extra_vision_kwargs = {}
- if token_type_ids_chunk is not None:
- _extra_vision_kwargs["token_type_ids"] = token_type_ids_chunk
- if mm_token_type_ids_chunk is not None:
- _extra_vision_kwargs["mm_token_type_ids"] = mm_token_type_ids_chunk
with torch.amp.autocast(
device_type = "cuda",
dtype = self._autocast_dtype,
@@ -2082,11 +2453,7 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
outputs = unwrapped_model(
input_ids = input_ids_chunk,
attention_mask = attention_mask_chunk,
- pixel_values = pixel_values_chunk,
- image_grid_thw = image_grid_thw_chunk,
- pixel_attention_mask = pixel_attention_mask_chunk,
- image_sizes = image_sizes_chunk,
- **_extra_vision_kwargs,
+ **vision_chunk,
)
logits_chunk = outputs.logits
@@ -2126,12 +2493,8 @@ def grpo_trainer__get_per_token_logps_and_entropies(function_name, function):
outputs = unwrapped_model(
input_ids = input_ids_chunk,
attention_mask = attention_mask_chunk,
- pixel_values = pixel_values_chunk,
- image_grid_thw = image_grid_thw_chunk,
- pixel_attention_mask = pixel_attention_mask_chunk,
- image_sizes = image_sizes_chunk,
logits_to_keep = logits_to_keep + 1,
- **_extra_vision_kwargs,
+ **vision_chunk,
)
logits_chunk = outputs.logits
@@ -2344,6 +2707,11 @@ RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_returns_hidd
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_hidden_states_signal))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_get_mm_token_id))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_fix_mm_token_type_ids))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_vision_inputs))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_split_vision_by_sample))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_unsplit_vision))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_grpo_image_cell))
+RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_reject_grpo_image_list))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(_unsloth_clear_stateful_mrope))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(grpo_compute_loss))
RL_PRE_ITEMS["grpo_trainer"].append(inspect.getsource(UnslothEfficientGRPO))
@@ -2406,18 +2774,15 @@ def grpo_trainer_compute_loss(function_name, function):
inputs["completion_ids"],
inputs["completion_mask"],
)
- pixel_values, image_grid_thw = (
- inputs.get("pixel_values", None),
- inputs.get("image_grid_thw", None),
- )
- pixel_attention_mask, image_sizes = (
- inputs.get("pixel_attention_mask", None),
- inputs.get("image_sizes", None),
- )
- num_images = inputs.get("num_images", None)
+ _vision_inputs = _unsloth_grpo_vision_inputs(inputs)
+ pixel_values = _vision_inputs.get("pixel_values", None)
+ image_grid_thw = _vision_inputs.get("image_grid_thw", None)
+ pixel_attention_mask = _vision_inputs.get("pixel_attention_mask", None)
+ image_sizes = _vision_inputs.get("image_sizes", None)
+ num_images = _vision_inputs.get("num_images", None)
# Transformers 5.x needs token_type_ids/mm_token_type_ids for some vision models.
- token_type_ids = inputs.get("token_type_ids", None)
- mm_token_type_ids = inputs.get("mm_token_type_ids", None)
+ token_type_ids = _vision_inputs.get("token_type_ids", None)
+ mm_token_type_ids = _vision_inputs.get("mm_token_type_ids", None)
num_items_in_batch = inputs.get("num_items_in_batch", None)
sampling_per_token_logps = inputs.get("sampling_per_token_logps", None)
tool_mask = inputs.get("tool_mask", None)
@@ -2437,6 +2802,9 @@ def grpo_trainer_compute_loss(function_name, function):
mm_token_type_ids,
completion_ids = completion_ids,
)
+ _vision_inputs["mm_token_type_ids"] = mm_token_type_ids
+ # Only the keys the processor produced: an older zoo must not see unknown kwargs.
+ _vision_inputs = {k: v for k, v in _vision_inputs.items() if v is not None}
logits_to_keep = completion_ids.size(
1
) # we only need to compute the logits for the completion tokens
@@ -2542,6 +2910,34 @@ def grpo_trainer_compute_loss(function_name, function):
num_processes = num_processes,
)
else:
+ # The gradient path needs the same zoo the no-grad path checks for, and nothing
+ # has checked it here: with beta = 0 and num_iterations = 1 there are no reference
+ # or old logprobs to compute, so _get_per_token_logps_and_entropies -- where that
+ # gate lives -- never runs at all. An older grpo_accumulated_loss accepts
+ # arbitrary kwargs, ignores the keys it does not know (spatial_shapes, num_tiles,
+ # the position ids) and, for a model that carries no image_grid_thw to slice by,
+ # replaces pixel_values with None outright, so training would compute its gradient
+ # logprobs from the text alone and report nothing. Import-probed rather than
+ # signature-probed, and body-local like the one in the no-grad path: this source is
+ # copied out without this module's imports (#6960).
+ if pixel_values is not None and not getattr(
+ self, "_unsloth_grpo_vision_zoo_checked", False
+ ):
+ _grpo_vision_chunks = None
+ try:
+ from unsloth_zoo.rl_replacements import (
+ grpo_vision_chunks as _grpo_vision_chunks,
+ )
+ except Exception:
+ pass
+ if _grpo_vision_chunks is None:
+ raise RuntimeError(
+ "Unsloth: vision GRPO needs an unsloth_zoo build that exports "
+ "grpo_vision_chunks, the shared multimodal key tuple and chunker "
+ "used by both GRPO logprob paths. Please upgrade unsloth_zoo to "
+ "2026.9.5 or newer: pip install -U unsloth_zoo"
+ )
+ self._unsloth_grpo_vision_zoo_checked = True
def _unsloth_requires_multi_image_zoo(value):
if value is None:
@@ -2558,6 +2954,13 @@ def grpo_trainer_compute_loss(function_name, function):
_supports_num_images = (
"num_images" in inspect.signature(grpo_accumulated_loss).parameters
)
+ if not _supports_num_images:
+ # Probe by import: the grep below cries "upgrade unsloth_zoo" falsely.
+ try:
+ from unsloth_zoo.rl_replacements import grpo_vision_chunks
+ _supports_num_images = grpo_vision_chunks is not None
+ except Exception:
+ pass
if not _supports_num_images:
try:
_zoo_src = inspect.getsource(grpo_accumulated_loss)
@@ -2605,11 +3008,6 @@ def grpo_trainer_compute_loss(function_name, function):
) = grpo_accumulated_loss(
trainer = self,
input_ids = _input_ids,
- pixel_values = pixel_values,
- image_grid_thw = image_grid_thw,
- pixel_attention_mask = pixel_attention_mask,
- image_sizes = image_sizes,
- num_images = num_images,
logits_to_keep = logits_to_keep,
completion_mask = completion_mask,
advantages = advantages,
@@ -2632,8 +3030,7 @@ def grpo_trainer_compute_loss(function_name, function):
current_gradient_accumulation_steps = current_gradient_accumulation_steps,
num_processes = num_processes,
sampling_per_token_logps = sampling_per_token_logps,
- token_type_ids = token_type_ids,
- mm_token_type_ids = mm_token_type_ids,
+ **_vision_inputs,
**_grpo_accumulated_loss_kwargs,
)
else:
@@ -2641,11 +3038,6 @@ def grpo_trainer_compute_loss(function_name, function):
loss, completion_length, mean_kl, coef_1, completion_mask = grpo_accumulated_loss(
trainer = self,
input_ids = _input_ids,
- pixel_values = pixel_values,
- image_grid_thw = image_grid_thw,
- pixel_attention_mask = pixel_attention_mask,
- image_sizes = image_sizes,
- num_images = num_images,
logits_to_keep = logits_to_keep,
completion_mask = completion_mask,
advantages = advantages,
@@ -2657,8 +3049,7 @@ def grpo_trainer_compute_loss(function_name, function):
logit_scale_multiply = logit_scale_multiply,
logit_scale_divide = logit_scale_divide,
attention_mask = attention_mask,
- token_type_ids = token_type_ids,
- mm_token_type_ids = mm_token_type_ids,
+ **_vision_inputs,
**_grpo_accumulated_loss_kwargs,
)
if "train" in self._metrics: