Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio FLUX.3 / Qwen-Image-2.1: prompt encodes keep only the hidden states they read (#17182)
Daniel Hancd95bc8544f7
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex dbd52741476..a41c88ce8f8 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -11139,9 +11139,13 @@ class DiffusionBackend:from .diffusion_qwenimage21_vision import configure_vision_attentionconfigure_vision_attention(pipe, family = fam.name, target = target, logger = logger)- from .diffusion_qwenimage21_te import skip_text_encoder_kv_cache+ from .diffusion_qwenimage21_te import (+ select_final_hidden_state,+ skip_text_encoder_kv_cache,+ )skip_text_encoder_kv_cache(pipe, logger = logger)+ select_final_hidden_state(pipe, logger = logger)# After the TE quant, so the cache key reflects the encoders that run.cond_cache.install(diff --git a/studio/backend/core/inference/diffusion_flux3.py b/studio/backend/core/inference/diffusion_flux3.pyindex 175463757f6..2816ac8a6c1 100644--- a/studio/backend/core/inference/diffusion_flux3.py+++ b/studio/backend/core/inference/diffusion_flux3.py@@ -392,7 +392,11 @@ class _Flux3KleinPipelineMixin:raise RuntimeError("this diffusers build's FLUX.3 text step names no default layers")- with torch.no_grad():+ from .diffusion_te_select import run_selected++ layers = tuple(hidden_states_layers)++ def call() -> Any:return helper(text_encoder = self.text_encoder,tokenizer = self.tokenizer,@@ -400,9 +404,12 @@ class _Flux3KleinPipelineMixin:device = device,max_sequence_length = int(max_sequence_length),pad_multiple = int(pad_multiple),- hidden_states_layers = tuple(hidden_states_layers),+ hidden_states_layers = layers,)+ with torch.no_grad():+ return run_selected(self.text_encoder, layers, call)+def _set_guidance(self, mp, guidance_scale: Optional[float]) -> None:guider = getattr(mp, "guider", None)if guider is None or guidance_scale is None:diff --git a/studio/backend/core/inference/diffusion_qwenimage21_te.py b/studio/backend/core/inference/diffusion_qwenimage21_te.pyindex b826be4875d..35147b15cda 100644--- a/studio/backend/core/inference/diffusion_qwenimage21_te.py+++ b/studio/backend/core/inference/diffusion_qwenimage21_te.py@@ -2,7 +2,9 @@# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0"""Qwen-Image-2.1 prompt encoding without the text encoder's K/V cache (UNSLOTH_QWEN21_TE_NO_KV=0 disables): the-pipeline's one hidden-state forward never passes ``use_cache``, so the config default built a cache nobody reads."""+pipeline's one hidden-state forward never passes ``use_cache``, so the config default built a cache nobody reads.+It also keeps only the one hidden state it reads (``hidden_states[-1]``) instead of every decoder layer's output+(diffusion_te_select; UNSLOTH_DIFFUSION_TE_SELECT_HIDDEN=0 disables)."""from __future__ import annotations@@ -12,6 +14,10 @@ from typing import Any_ENV = "UNSLOTH_QWEN21_TE_NO_KV"_MARK = "_unsloth_te_no_kv"+_SELECT_MARK = "_unsloth_te_select_final"+# The pipeline's own norm-neutralizing hook makes its ``hidden_states[-1]`` the final norm's input on every+# transformers version; the selective encode records that input, so it is used only when the source still has it.+_NEUTRALIZED_NORM = ("norm.register_forward_hook", "outputs.hidden_states[-1]")def _enabled() -> bool:@@ -59,3 +65,44 @@ def skip_text_encoder_kv_cache(pipe: Any, logger: Any = None) -> bool:"diffusion.qwen_image_21: prompt encoding skips the text encoder's unused K/V cache")return True+++def select_final_hidden_state(pipe: Any, logger: Any = None) -> bool:+ """Wrap ``pipe._get_qwen_prompt_embeds`` so its encoder forward records only the final norm's input, the one+ entry the pipeline reads. Idempotent; False when the pipeline is not the expected one."""+ from .diffusion_te_select import enabled as select_enabled+ from .diffusion_te_select import run_selected++ if not select_enabled():+ return False+ encoder = getattr(pipe, "text_encoder", None)+ original = getattr(pipe, "_get_qwen_prompt_embeds", None)+ if (+ encoder is None+ or not callable(original)+ or not type(encoder).__name__.startswith("Qwen3VL")+ ):+ return False+ if getattr(original, _SELECT_MARK, False):+ return True+ try:+ import inspect+ source = inspect.getsource(getattr(type(pipe), "_get_qwen_prompt_embeds"))+ except Exception: # noqa: BLE001+ return False+ if not all(needle in source for needle in _NEUTRALIZED_NORM):+ return False++ @functools.wraps(original)+ def _get_qwen_prompt_embeds(*args: Any, **kwargs: Any) -> Any:+ current = getattr(pipe, "text_encoder", None)+ return run_selected(current, (-1,), lambda: original(*args, **kwargs), last_pre_norm = True)++ setattr(_get_qwen_prompt_embeds, _SELECT_MARK, True)+ setattr(_get_qwen_prompt_embeds, _MARK, getattr(original, _MARK, False))+ pipe._get_qwen_prompt_embeds = _get_qwen_prompt_embeds+ if logger is not None:+ logger.info(+ "diffusion.qwen_image_21: prompt encoding keeps only the final hidden state it reads"+ )+ return Truediff --git a/studio/backend/core/inference/diffusion_te_select.py b/studio/backend/core/inference/diffusion_te_select.pynew file mode 100644index 00000000000..084f529b92e--- /dev/null+++ b/studio/backend/core/inference/diffusion_te_select.py@@ -0,0 +1,225 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Prompt encodes that read a few hidden states keep only those (UNSLOTH_DIFFUSION_TE_SELECT_HIDDEN=0 disables).++The encoder runs with ``output_hidden_states=False`` and hooks record only the requested layers; reading any other+index raises ``SelectionError`` so the caller re-runs stock."""++from __future__ import annotations++import contextlib+import os+from typing import Any, Iterator, Optional++SELECT_ENV = "UNSLOTH_DIFFUSION_TE_SELECT_HIDDEN"+_CALLS = [0]+++class SelectionError(RuntimeError):+ """The selective encode could not answer what the caller read: re-run stock."""+++def enabled() -> bool:+ return (os.environ.get(SELECT_ENV) or "1").strip().lower() not in ("0", "off", "false", "no")+++def call_count() -> int:+ return _CALLS[0]+++class SelectedHiddenStates:+ """The requested entries of the stock ``hidden_states`` tuple (length ``n + 1``: embeddings, then one per+ decoder layer, the last per the caller's contract); reading any other entry raises ``SelectionError``."""++ __slots__ = ("_length", "_states")++ def __init__(self, length: int, states: dict) -> None:+ self._length = length+ self._states = states++ def __len__(self) -> int:+ return self._length++ def __getitem__(self, index: Any) -> Any:+ if not isinstance(index, int) or isinstance(index, bool):+ raise SelectionError(f"hidden_states[{index!r}] is not a plain index")+ k = index + self._length if index < 0 else index+ if k not in self._states:+ raise SelectionError(f"hidden_states[{index}] was not recorded")+ return self._states[k]++ def __iter__(self) -> Iterator:+ raise SelectionError("the selective hidden states cannot be iterated")+++def _text_config(encoder: Any) -> Any:+ config = getattr(encoder, "config", None)+ getter = getattr(config, "get_text_config", None)+ if callable(getter):+ try:+ return getter()+ except Exception: # noqa: BLE001+ pass+ return getattr(config, "text_config", None) or config+++def _stack(encoder: Any) -> Optional[tuple[str, Any]]:+ from .diffusion_te_truncate import _find_stack+ return _find_stack(encoder)+++def _parent(root: Any, dotted: str) -> Any:+ parent = root+ for part in dotted.split(".")[:-1]:+ parent = getattr(parent, part, None)+ if parent is None:+ return None+ return parent+++def plan(+ encoder: Any,+ indices: Any,+ *,+ last_pre_norm: bool = False,+) -> Optional[dict]:+ """``{"length", "layers": {k: stack slot}, "norm"}`` for the requested indices, or None (run stock).++ Entry ``k`` (1 <= k < n) is decoder layer ``k - 1``'s output; entry ``n`` (or -1) only with ``last_pre_norm``,+ the caller's guarantee that its stock reading is the final norm's input. Entry 0 (embeddings) is not handled."""+ import torch++ if not enabled() or not isinstance(encoder, torch.nn.Module):+ return None+ if type(encoder).__name__ == "OptimizedModule":+ return None+ found = _stack(encoder)+ if found is None:+ return None+ name, stack = found+ n = len(stack)+ expected = getattr(_text_config(encoder), "num_hidden_layers", None)+ if not isinstance(expected, int) or expected != n:+ return None+ try:+ wanted = [int(k) for k in indices]+ except (TypeError, ValueError):+ return None+ if not wanted:+ return None+ layers: dict[int, int] = {}+ norm = None+ for raw in wanted:+ k = raw + n + 1 if raw < 0 else raw+ if 1 <= k < n:+ layers[k] = k - 1+ elif k == n and last_pre_norm:+ norm = getattr(_parent(encoder, name), "norm", None)+ if not isinstance(norm, torch.nn.Module):+ return None+ else:+ return None+ return {"length": n + 1, "layers": layers, "norm": norm, "stack": stack, "n": n}+++def selected_hidden_states(+ encoder: Any,+ indices: Any,+ *,+ last_pre_norm: bool = False,+) -> Iterator[bool]:+ """Yields True when armed (the next ``output_hidden_states=True`` forward of ``encoder`` records only+ ``indices``), False when the caller runs stock. Hooks are removed on exit."""+ p = plan(encoder, indices, last_pre_norm = last_pre_norm)+ if p is None:+ yield False+ return+ state: dict = {"active": False, "states": {}}+ handles = []++ def pre(_module: Any, args: tuple, kwargs: dict) -> Any:+ if state["active"] or not kwargs.get("output_hidden_states"):+ return None+ state["active"] = True+ state["states"] = {}+ kwargs = dict(kwargs)+ kwargs["output_hidden_states"] = False+ return args, kwargs++ def layer_hook(k: int) -> Any:+ def hook(_module: Any, _args: Any, output: Any) -> None:+ if state["active"]:+ state["states"][k] = output[0] if isinstance(output, tuple) else output++ return hook++ def norm_hook(_module: Any, args: Any, _output: Any) -> None:+ if state["active"] and args:+ state["states"][p["n"]] = args[0]++ def post(_module: Any, _args: Any, _kwargs: dict, output: Any) -> Any:+ if not state["active"]:+ return None+ state["active"] = False+ states, state["states"] = state["states"], {}+ want = set(p["layers"]) | ({p["n"]} if p["norm"] is not None else set())+ if set(states) != want or getattr(output, "hidden_states", None) is not None:+ # a recorder that did not fire, or a forward that still built the full tuple: refuse the result+ raise SelectionError("the selective encode did not record exactly the requested states")+ selected = SelectedHiddenStates(p["length"], states)+ try:+ output["hidden_states"] = selected+ except Exception: # noqa: BLE001 - not a ModelOutput+ try:+ output.hidden_states = selected+ except Exception as exc: # noqa: BLE001+ raise SelectionError(+ f"cannot attach hidden states to {type(output).__name__}"+ ) from exc+ _CALLS[0] += 1+ return output++ try:+ handles.append(encoder.register_forward_pre_hook(pre, with_kwargs = True))+ for k, slot in p["layers"].items():+ handles.append(p["stack"][slot].register_forward_hook(layer_hook(k)))+ if p["norm"] is not None:+ handles.append(p["norm"].register_forward_hook(norm_hook))+ handles.append(encoder.register_forward_hook(post, with_kwargs = True))+ yield True+ finally:+ state["active"] = False+ state["states"] = {}+ for handle in handles:+ handle.remove()+++def run_selected(+ encoder: Any,+ indices: Any,+ call: Any,+ *,+ last_pre_norm: bool = False,+) -> Any:+ """``call()`` under ``selected_hidden_states``; a ``SelectionError`` re-runs ``call()`` stock."""+ try:+ with selected_hidden_states(encoder, indices, last_pre_norm = last_pre_norm) as armed:+ if armed:+ return call()+ except SelectionError:+ pass+ return call()+++__all__ = [+ "SELECT_ENV",+ "SelectedHiddenStates",+ "SelectionError",+ "call_count",+ "enabled",+ "plan",+ "run_selected",+ "selected_hidden_states",+]diff --git a/studio/backend/tests/test_diffusion_qwenimage21_te_no_kv.py b/studio/backend/tests/test_diffusion_qwenimage21_te_no_kv.pyindex fbba2690f36..4f525b55d34 100644--- a/studio/backend/tests/test_diffusion_qwenimage21_te_no_kv.py+++ b/studio/backend/tests/test_diffusion_qwenimage21_te_no_kv.py@@ -143,3 +143,31 @@ def test_other_pipelines_are_left_alone():other = _Other()assert te.skip_text_encoder_kv_cache(other) is Falseassert te.skip_text_encoder_kv_cache(object()) is False+++def test_final_state_selection_keeps_the_embeddings_bitwise(monkeypatch):+ from core.inference import diffusion_te_select as sel++ monkeypatch.delenv(sel.SELECT_ENV, raising = False)+ pipe = _Pipe()+ (embeds, mask, pad), _ = _encode(pipe)+ assert te.skip_text_encoder_kv_cache(pipe) is True+ assert te.select_final_hidden_state(pipe) is True+ wrapped = pipe._get_qwen_prompt_embeds+ assert te.select_final_hidden_state(pipe) is True and pipe._get_qwen_prompt_embeds is wrapped+ assert te.skip_text_encoder_kv_cache(pipe) is True and pipe._get_qwen_prompt_embeds is wrapped+ before = sel.call_count()+ (embeds2, mask2, pad2), caches = _encode(pipe)+ assert sel.call_count() == before + 1 # the selective encode ran (no stock fallback)+ assert caches == [None]+ assert torch.equal(embeds2.view(torch.int32), embeds.view(torch.int32))+ assert torch.equal(mask2, mask) and torch.equal(pad2, pad)+++def test_final_state_selection_kill_switch(monkeypatch):+ from core.inference import diffusion_te_select as sel++ monkeypatch.setenv(sel.SELECT_ENV, "0")+ pipe = _Pipe()+ assert te.select_final_hidden_state(pipe) is False+ assert te.select_final_hidden_state(object()) is Falsediff --git a/studio/backend/tests/test_diffusion_te_select.py b/studio/backend/tests/test_diffusion_te_select.pynew file mode 100644index 00000000000..33939eafda8--- /dev/null+++ b/studio/backend/tests/test_diffusion_te_select.py@@ -0,0 +1,269 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Selective prompt-encoder hidden states: the requested entries are the stock tensors bitwise, nothing else is kept,+and anything unexpected falls back to the stock call. Runs on CPU with a random tiny Qwen3-VL."""++from __future__ import annotations++import gc+import weakref++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("transformers.models.qwen3_vl")++from core.inference import diffusion_te_select as sel # noqa: E402++VOCAB, HIDDEN, LAYERS = 512, 64, 9+FLUX3_LAYERS = (+ 2,+ 4,+ 6,+ 8,+) # FLUX.3's (4, 8, ..., 32) pattern on a 9-layer stack: every one an intermediate state+++def _encoder(layers: int = LAYERS):+ from transformers import Qwen3VLForConditionalGeneration+ from transformers.models.qwen3_vl.configuration_qwen3_vl import (+ Qwen3VLConfig,+ Qwen3VLTextConfig,+ Qwen3VLVisionConfig,+ )++ config = Qwen3VLConfig(+ vision_config = Qwen3VLVisionConfig(+ depth = 1,+ hidden_size = 32,+ intermediate_size = 64,+ num_heads = 4,+ out_hidden_size = HIDDEN,+ deepstack_visual_indexes = [],+ ).to_dict(),+ text_config = Qwen3VLTextConfig(+ hidden_size = HIDDEN,+ intermediate_size = 128,+ num_hidden_layers = layers,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 16,+ vocab_size = VOCAB,+ use_cache = False,+ rope_scaling = {+ "rope_type": "default",+ "mrope_section": [4, 2, 2],+ "mrope_interleaved": True,+ },+ ).to_dict(),+ )+ config._attn_implementation = "sdpa"+ torch.manual_seed(0)+ return Qwen3VLForConditionalGeneration(config).eval()+++def _inputs(length: int = 37):+ ids = (torch.arange(length)[None] * 13 % (VOCAB - 10)) + 10+ mask = torch.ones_like(ids)+ mask[:, -5:] = 0 # right padding, as FLUX.3 pads its window+ pos = torch.arange(length)[None].masked_fill(mask == 0, 1)+ return {"input_ids": ids, "attention_mask": mask, "position_ids": pos}+++def _flux3_embeds(encoder, layers = FLUX3_LAYERS):+ """diffusers' ``_get_qwen3_vl_prompt_embeds`` read of the encoder output, verbatim."""+ output = encoder(**_inputs(), output_hidden_states = True, use_cache = False)+ out = torch.stack([output.hidden_states[k] for k in layers], dim = 1)+ b, c, s, h = out.shape+ return out.permute(0, 2, 1, 3).reshape(b, s, c * h)+++def _qwen21_final(encoder):+ """The Qwen-Image-2.1 pipeline's read: ``hidden_states[-1]`` with the final norm neutralized for the call."""+ text_model = getattr(encoder.model, "language_model", encoder.model)+ handle = text_model.norm.register_forward_hook(lambda module, args, output: args[0])+ try:+ outputs = encoder(**_inputs(), output_hidden_states = True)+ finally:+ handle.remove()+ return outputs.hidden_states[-1]++[email protected](autouse = True)+def _env(monkeypatch):+ monkeypatch.delenv(sel.SELECT_ENV, raising = False)+++def test_flux3_layers_are_the_stock_tensors_bitwise():+ encoder = _encoder()+ with torch.no_grad():+ stock = _flux3_embeds(encoder)+ before = sel.call_count()+ got = sel.run_selected(encoder, FLUX3_LAYERS, lambda: _flux3_embeds(encoder))+ assert sel.call_count() == before + 1 # the selective path ran, not the fallback+ assert torch.equal(got.view(torch.int32), stock.view(torch.int32))+++def test_qwen21_final_pre_norm_state_is_the_stock_tensor_bitwise():+ encoder = _encoder()+ with torch.no_grad():+ stock = _qwen21_final(encoder)+ before = sel.call_count()+ got = sel.run_selected(encoder, (-1,), lambda: _qwen21_final(encoder), last_pre_norm = True)+ assert sel.call_count() == before + 1+ assert torch.equal(got.view(torch.int32), stock.view(torch.int32))+++def test_only_the_requested_states_stay_alive():+ """Every unrequested layer output is freed during the forward: the recorded states are the only survivors."""+ encoder = _encoder()+ alive: list = []+ hooks = [+ layer.register_forward_hook(+ lambda m, a, out: alive.append(weakref.ref(out[0] if isinstance(out, tuple) else out))+ )+ for layer in encoder.model.language_model.layers+ ]+ seen = {}+ try:+ with torch.no_grad(), sel.selected_hidden_states(encoder, FLUX3_LAYERS) as armed:+ assert armed+ output = encoder(**_inputs(), output_hidden_states = True, use_cache = False)+ gc.collect()+ seen = {i for i, ref in enumerate(alive) if ref() is not None}+ kept = [output.hidden_states[k] for k in FLUX3_LAYERS]+ finally:+ for h in hooks:+ h.remove()+ assert seen == {k - 1 for k in FLUX3_LAYERS}+ assert len(kept) == len(FLUX3_LAYERS)+ with pytest.raises(sel.SelectionError):+ output.hidden_states[1]+++def test_an_unrecorded_read_falls_back_to_the_stock_call():+ encoder = _encoder()+ calls = []++ def read():+ calls.append(1)+ output = encoder(**_inputs(), output_hidden_states = True, use_cache = False)+ return output.hidden_states[3] # not requested++ with torch.no_grad():+ stock = encoder(**_inputs(), output_hidden_states = True, use_cache = False).hidden_states[3]+ got = sel.run_selected(encoder, FLUX3_LAYERS, read)+ assert len(calls) == 2 # selective attempt, then the stock re-run+ assert torch.equal(got, stock)+++def test_unsupported_requests_and_the_kill_switch_run_stock(monkeypatch):+ encoder = _encoder()+ assert sel.plan(encoder, (0, 2)) is None # embeddings: not handled+ assert (+ sel.plan(encoder, (LAYERS,)) is None+ ) # the final entry without the caller's pre-norm contract+ assert sel.plan(encoder, (-1,), last_pre_norm = True) is not None+ assert sel.plan(torch.nn.Linear(2, 2), (1,)) is None+ monkeypatch.setenv(sel.SELECT_ENV, "0")+ assert sel.plan(encoder, FLUX3_LAYERS) is None+ with torch.no_grad():+ before = sel.call_count()+ stock = _flux3_embeds(encoder)+ got = sel.run_selected(encoder, FLUX3_LAYERS, lambda: _flux3_embeds(encoder))+ assert sel.call_count() == before+ assert torch.equal(got, stock)+++def test_dead_tail_passthroughs_keep_the_selection_exact():+ """After Studio's dead-tail truncation the trailing slots are passthroughs: the selection still matches."""+ from core.inference.diffusion_te_truncate import _make_passthrough++ encoder = _encoder()+ layers = encoder.model.language_model.layers+ fmt = {"tuple_len": None, "seen": True}++ def record(_m, _a, out):+ fmt["tuple_len"] = len(out) if isinstance(out, tuple) else None++ handle = layers[0].register_forward_hook(record)+ with torch.no_grad():+ _flux3_embeds(encoder)+ handle.remove()+ for i in range(max(FLUX3_LAYERS), LAYERS):+ layers[i] = _make_passthrough(layers[i], fmt)+ with torch.no_grad():+ stock = _flux3_embeds(encoder)+ got = sel.run_selected(encoder, FLUX3_LAYERS, lambda: _flux3_embeds(encoder))+ assert torch.equal(got.view(torch.int32), stock.view(torch.int32))+++def test_hooks_are_removed_after_the_call():+ encoder = _encoder()+ counts = (+ len(encoder._forward_pre_hooks),+ len(encoder._forward_hooks),+ sum(len(layer._forward_hooks) for layer in encoder.model.language_model.layers),+ )+ with torch.no_grad():+ sel.run_selected(encoder, FLUX3_LAYERS, lambda: _flux3_embeds(encoder))+ assert counts == (+ len(encoder._forward_pre_hooks),+ len(encoder._forward_hooks),+ sum(len(layer._forward_hooks) for layer in encoder.model.language_model.layers),+ )+ with torch.no_grad():+ full = encoder(**_inputs(), output_hidden_states = True, use_cache = False)+ assert len(full.hidden_states) == LAYERS + 1+++def test_flux3_prompt_helper_matches_bitwise():+ """diffusers' own FLUX.3 prompt helper (when this diffusers has it), stock vs selective."""+ encoders = pytest.importorskip("diffusers.modular_pipelines.flux3.encoders")+ helper = encoders.Flux3TextEncoderStep._get_qwen3_vl_prompt_embeds++ class _Tok:+ def apply_chat_template(+ self,+ messages,+ tokenize = False,+ add_generation_prompt = True,+ ):+ return messages[0]["content"]++ def tokenizer(+ self,+ text,+ return_tensors = "pt",+ padding = None,+ padding_side = None,+ truncation = True,+ max_length = None,+ ):+ ids = (torch.arange(len(text) + 3)[None] * 11 % (VOCAB - 10)) + 10+ mask = torch.ones_like(ids)+ if padding == "max_length" and max_length > ids.shape[1]:+ pad = max_length - ids.shape[1]+ ids = torch.cat([ids, torch.zeros(1, pad, dtype = ids.dtype)], 1)+ mask = torch.cat([mask, torch.zeros(1, pad, dtype = mask.dtype)], 1)+ return {"input_ids": ids[:, :max_length], "attention_mask": mask[:, :max_length]}++ tok = _Tok()+ tok.tokenizer = tok.tokenizer+ encoder = _encoder()+ kwargs = dict(+ text_encoder = encoder,+ tokenizer = tok,+ prompt = "a red fox in snow",+ device = "cpu",+ max_sequence_length = 64,+ pad_multiple = 8,+ hidden_states_layers = FLUX3_LAYERS,+ )+ with torch.no_grad():+ stock = helper(**kwargs)+ before = sel.call_count()+ got = sel.run_selected(encoder, FLUX3_LAYERS, lambda: helper(**kwargs))+ assert sel.call_count() == before + 1+ assert torch.equal(got.view(torch.int32), stock.view(torch.int32))