Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio diffusion: stream the LTX-2.5 encoder, cache and trim FLUX.3 prompt encodes (#16750)
Daniel Han89db0db7c5d6
diff --git a/studio/backend/core/inference/diffusion_flux3.py b/studio/backend/core/inference/diffusion_flux3.pyindex a54a8ddb772..f0a16eefd2f 100644--- a/studio/backend/core/inference/diffusion_flux3.py+++ b/studio/backend/core/inference/diffusion_flux3.py@@ -47,6 +47,78 @@ FLUX3_COMPONENT_FOLDERS = {FLUX3_PROMPT_UPSAMPLER_STEP = "prompt_upsampler"+# diffusers ``Flux3TextEncoderStep`` helper args; a build whose helper takes others keeps its own text step.+FLUX3_TEXT_STEP = "text_encoder"+_TEXT_HELPER = "_get_qwen3_vl_prompt_embeds"+_TEXT_HELPER_ARGS = frozenset(+ {+ "text_encoder",+ "tokenizer",+ "prompt",+ "device",+ "max_sequence_length",+ "pad_multiple",+ "hidden_states_layers",+ }+)+_TEXT_LAYERS_INPUT = "text_encoder_out_layers"+++def _text_step(mp: Any) -> tuple[Any, Any]:+ """``(step, helper)``: the modular pipeline's text step and its class's prompt helper, or ``(None, None)``."""+ import inspect++ sub_blocks = getattr(getattr(mp, "_blocks", None), "sub_blocks", None)+ step = sub_blocks.get(FLUX3_TEXT_STEP) if sub_blocks is not None else None+ helper = getattr(type(step), _TEXT_HELPER, None) if step is not None else None+ if not callable(helper):+ return None, None+ try:+ if not _TEXT_HELPER_ARGS <= set(inspect.signature(helper).parameters):+ return None, None+ except (TypeError, ValueError):+ return None, None+ return step, helper+++def _bridge_text_step(pipe: Any, mp: Any) -> bool:+ """Route the text step's per-prompt encode through ``pipe.encode_prompt`` (prompt cache, dead-tail probe).+ Only for this pipe's own encoder and tokenizer; a foreign encoder runs the step's helper unchanged."""+ import inspect+ import weakref++ step, helper = _text_step(mp)+ if step is None:+ return False+ signature = inspect.signature(helper)+ owner_ref = weakref.ref(pipe)++ def bridged(*args: Any, **kwargs: Any) -> Any:+ owner = owner_ref()+ try:+ bound = signature.bind(*args, **kwargs)+ bound.apply_defaults()+ except TypeError:+ return helper(*args, **kwargs)+ a = bound.arguments+ if (+ owner is None+ or a["text_encoder"] is not getattr(owner, "text_encoder", None)+ or a["tokenizer"] is not getattr(owner, "tokenizer", None)+ ):+ return helper(*args, **kwargs)+ return owner.encode_prompt(+ prompt = a["prompt"],+ max_sequence_length = a["max_sequence_length"],+ device = a["device"],+ pad_multiple = a["pad_multiple"],+ hidden_states_layers = a["hidden_states_layers"],+ )++ setattr(step, _TEXT_HELPER, bridged)+ return True++def flux3_blocks(blocks_class_name: str) -> Any:"""The named FLUX.3 auto blocks without the prompt upsampler step (absent before diffusers#15058)."""import diffusers@@ -284,6 +356,7 @@ class _Flux3KleinPipelineMixin:if self._modular is None:self._modular = flux3_blocks(self.config.blocks_class_name).init_pipeline()+ _bridge_text_step(self, self._modular)mp = self._modularupdates = {}for name in ("transformer", "vae", "text_encoder", "tokenizer", "scheduler"):@@ -294,6 +367,42 @@ class _Flux3KleinPipelineMixin:mp.update_components(**updates)return mp+ def encode_prompt(+ self,+ prompt: str,+ max_sequence_length: int = 8192,+ device: Any = None,+ pad_multiple: Optional[int] = None,+ hidden_states_layers: Any = None,+ ):+ """One prompt's FLUX.3 text embeddings via diffusers' text step helper; ``None`` takes the step's defaults."""+ import torch++ step, helper = _text_step(self.modular_pipeline())+ if step is None:+ raise RuntimeError("this diffusers build's FLUX.3 text step has no known prompt helper")+ if pad_multiple is None:+ pad_multiple = step.pad_multiple+ if hidden_states_layers is None:+ hidden_states_layers = next(+ (p.default for p in step.inputs if getattr(p, "name", None) == _TEXT_LAYERS_INPUT),+ None,+ )+ if hidden_states_layers is None:+ raise RuntimeError(+ "this diffusers build's FLUX.3 text step names no default layers"+ )+ with torch.no_grad():+ return helper(+ text_encoder = self.text_encoder,+ tokenizer = self.tokenizer,+ prompt = prompt,+ device = device,+ max_sequence_length = int(max_sequence_length),+ pad_multiple = int(pad_multiple),+ hidden_states_layers = tuple(hidden_states_layers),+ )+def _set_guidance(self, mp, guidance_scale: Optional[float]) -> None:guider = getattr(mp, "guider", None)if guider is None or guidance_scale is None:diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.pyindex e873b1e9641..988357c846a 100644--- a/studio/backend/core/inference/video_ltx25.py+++ b/studio/backend/core/inference/video_ltx25.py@@ -681,9 +681,9 @@ def encoder_resident_for_plan(plan: Any) -> bool:The plan prices the family's text encoder (the table's 23.9 GB) as if it were in-process: resident under ``none``and under ``group`` with resident companions, streamed under ``group`` with streamed text encoders and under- ``streaming`` / ``sequential``, alone on the card under ``model``. The worker cannot stream, so on every tier that did- not budget it resident it keeps the encoder on the host and onloads it only for an encode, when the card has room- (else it encodes on the CPU; see ``video_ltx25_te_worker._encode_device``)."""+ ``streaming`` / ``sequential``, alone on the card under ``model``. On every tier that did not budget it resident the+ worker keeps the encoder on the host and onloads it only for an encode: whole when the card has room, else one+ decoder layer at a time, else it encodes on the CPU (see ``video_ltx25_te_worker._encode_plan``)."""from .diffusion_memory import OFFLOAD_GROUP, OFFLOAD_NONEpolicy = getattr(plan, "offload_policy", None)@@ -1083,14 +1083,20 @@ def install_remote_encode_prompt(pipe: Any, client: Ltx25TextEncoderClient) -> Ndef _one(texts: list[str]) -> tuple[Any, Any]:# The client's own cached tensors (read only): every op below returns a new tensor.- embeds, mask = client.encode(texts, max_sequence_length, copy = False)- batch = embeds.shape[0]- embeds = embeds.to(device = device, dtype = dtype)+ cached, mask = client.encode(texts, max_sequence_length, copy = False)+ batch = cached.shape[0]+ embeds = cached.to(device = device, dtype = dtype)mask = mask.to(device = device)seq_len = embeds.shape[1]- embeds = embeds.repeat(1, num_videos_per_prompt, 1).view(- batch * num_videos_per_prompt, seq_len, -1- )+ # A ``.to`` that moved or cast already copied, so one video skips ``repeat``; a no-op ``.to`` aliases the cache.+ if (+ num_videos_per_prompt != 1+ or embeds.data_ptr() == cached.data_ptr()+ or not embeds.is_contiguous()+ ):+ embeds = embeds.repeat(1, num_videos_per_prompt, 1).view(+ batch * num_videos_per_prompt, seq_len, -1+ )mask = mask.view(batch, -1).repeat(num_videos_per_prompt, 1)return embeds, maskdiff --git a/studio/backend/core/inference/video_ltx25_te_worker.py b/studio/backend/core/inference/video_ltx25_te_worker.pyindex 5feae03ca78..bb2c6c37c3d 100644--- a/studio/backend/core/inference/video_ltx25_te_worker.py+++ b/studio/backend/core/inference/video_ltx25_te_worker.py@@ -538,9 +538,19 @@ def _encode_device(# MPS shares host memory, so there is no card to keep the encoder off.if config.get("resident") or kind not in ("cuda", "xpu"):return device+ need = _weights_mib(model) + ENCODE_MARGIN_MIB+ room = _encode_room_mib(config, server_reserved_mib)+ if room is None:+ return device if kind == "cuda" else "cpu"+ return device if room >= need else "cpu"+++def _encode_room_mib(config: dict, server_reserved_mib: Optional[int] = None) -> Optional[int]:+ """MiB an encode may use on the configured card (see ``_encode_device``), or None when nothing could be read."""import torch- need = _weights_mib(model) + ENCODE_MARGIN_MIB+ device = str(config["device"])+ kind = device.split(":", 1)[0]room: Optional[int] = Nonetry:free, total = getattr(torch, kind).mem_get_info(torch.device(device))@@ -552,9 +562,188 @@ def _encode_device(if budget is not None and server_reserved_mib is not None:allowed = int(budget) - int(server_reserved_mib) - WORKER_CONTEXT_MIBroom = allowed if room is None else min(room, allowed)- if room is None:- return device if kind == "cuda" else "cpu"- return device if room >= need else "cpu"+ return room+++# Towers a text encode never runs; a streamed encode leaves them on the host.+_TOWER_TOKENS = ("visual", "vision", "audio")+++def _tensor_tables(module: Any) -> tuple:+ return (module._parameters, module._buffers)+++def _stream_parts(model: Any) -> Optional[tuple[list, list, list]]:+ """``(resident, containers, layers)`` for a layer-streamed encode, or None when the encoder cannot stream.+ A module overriding ``_apply`` may keep tensors outside its tables, so it declines streaming."""+ import torch++ from core.inference.diffusion_te_truncate import is_passthrough++ try:+ stack = _decoder_layers(model)+ except Exception: # noqa: BLE001 -- not the expected Gemma 4 layout+ return None+ if not isinstance(model, torch.nn.Module) or not isinstance(stack, torch.nn.ModuleList):+ return None+ layers = [m for m in stack if not is_passthrough(m)]+ if not layers:+ return None+ resident: list = []+ containers: list = []++ def walk(module: Any) -> None:+ containers.append(module)+ for name, child in module.named_children():+ if child is stack or any(token in name.lower() for token in _TOWER_TOKENS):+ continue+ if any(m is stack for m in child.modules()):+ walk(child)+ else:+ resident.append(child)++ walk(model)+ for module in [*containers, *[s for m in (*resident, *layers) for s in m.modules()]]:+ if type(module)._apply is not torch.nn.Module._apply:+ return None+ return resident, containers, layers+++def _tensors_mib(modules: list, recurse: bool = True) -> int:+ seen: set = set()+ total = 0+ for module in modules:+ for sub in module.modules() if recurse else (module,):+ for table in _tensor_tables(sub):+ for tensor in table.values():+ if tensor is None or id(tensor) in seen:+ continue+ seen.add(id(tensor))+ total += tensor.numel() * tensor.element_size()+ return total >> 20+++def _stream_need_mib(parts: tuple[list, list, list]) -> int:+ """Resident modules + two layers (allocator may still hold the last) + the encode margin."""+ resident, containers, layers = parts+ biggest = max(_tensors_mib([layer]) for layer in layers)+ return (+ _tensors_mib(resident)+ + _tensors_mib(containers, recurse = False)+ + 2 * biggest+ + ENCODE_MARGIN_MIB+ )+++def _encode_plan(+ config: dict,+ model: Any,+ server_reserved_mib: Optional[int] = None,+) -> tuple[str, Optional[tuple[list, list, list]]]:+ """``(device, parts)``: an encoder too big to onload whole streams its layers (``parts`` set) instead of CPU."""+ device = _encode_device(config, model, server_reserved_mib)+ configured = str(config["device"])+ if device != "cpu" or configured.split(":", 1)[0] not in ("cuda", "xpu"):+ return device, None+ if config.get("resident") or (os.environ.get(STREAM_ENCODE_ENV) or "").strip() == "0":+ return device, None+ try:+ parts = _stream_parts(model)+ if parts is None:+ return device, None+ room = _encode_room_mib(config, server_reserved_mib)+ if room is None or room < _stream_need_mib(parts):+ return device, None+ except Exception: # noqa: BLE001 -- any doubt keeps the CPU encode+ return device, None+ return configured, parts+++# "0" turns the layer-streamed encode off (a card too small for the whole encoder then encodes on the CPU again).+STREAM_ENCODE_ENV = "UNSLOTH_LTX25_TE_STREAM"+++class _Onload:+ """Moves the tensors of some modules to a device and back. Parameters keep their identity (``.data`` swaps, so a+ tied weight moves once); the host tensors are kept and restored, so returning costs no copy."""++ def __init__(self, device: Any) -> None:+ self.device = device+ self.saved: list = []+ self.moved: dict = {}++ def add(+ self,+ modules: list,+ recurse: bool = True,+ ) -> None:+ import torch+ for module in modules:+ for sub in module.modules() if recurse else (module,):+ for table in _tensor_tables(sub):+ for key, tensor in list(table.items()):+ if tensor is None:+ continue+ if isinstance(tensor, torch.nn.Parameter):+ if id(tensor) not in self.moved:+ self.moved[id(tensor)] = tensor+ self.saved.append((tensor, None, None, tensor.data))+ tensor.data = tensor.data.to(self.device, copy = True)+ continue+ moved = self.moved.get(id(tensor))+ if moved is None:+ moved = tensor.to(self.device, copy = True)+ self.moved[id(tensor)] = moved+ self.saved.append((None, table, key, tensor))+ table[key] = moved++ def restore(self) -> None:+ while self.saved:+ param, table, key, original = self.saved.pop()+ if param is not None:+ param.data = original+ else:+ table[key] = original+ self.moved.clear()+++def _streamed(parts: tuple[list, list, list], device: str) -> Any:+ """Context: the resident modules on ``device`` for the encode, each decoder layer only while it runs."""+ import contextlib++ resident, containers, layers = parts++ @contextlib.contextmanager+ def _ctx():+ base = _Onload(device)+ handles: list = []+ live: dict = {}+ try:+ base.add(resident)+ base.add(containers, recurse = False)++ def before(module: Any, _args: Any) -> None:+ onload = _Onload(device)+ live[id(module)] = onload+ onload.add([module])++ def after(module: Any, _args: Any, _output: Any) -> None:+ onload = live.pop(id(module), None)+ if onload is not None:+ onload.restore()++ for layer in layers:+ handles.append(layer.register_forward_pre_hook(before))+ handles.append(layer.register_forward_hook(after, always_call = True))+ yield+ finally:+ for handle in handles:+ handle.remove()+ for onload in live.values():+ onload.restore()+ base.restore()++ return _ctx()def _memory_stats(device: str) -> dict:@@ -581,10 +770,11 @@ def _memory_stats(device: str) -> dict:def _pack_hidden_states(model: Any, ids: Any, mask: Any) -> Any:- """``LTX2Pipeline._get_gemma_prompt_embeds``: every hidden state stacked on a last axis, flattened."""+ """``LTX2Pipeline._get_gemma_prompt_embeds``: every hidden state stacked on a last axis, flattened. No KV cache:+ the encoder config sets ``use_cache``, which would build and hold a DynamicCache this single forward never reads."""import torch- out = model(input_ids = ids, attention_mask = mask, output_hidden_states = True)+ out = model(input_ids = ids, attention_mask = mask, output_hidden_states = True, use_cache = False)return torch.stack(out.hidden_states, dim = -1).flatten(2, 3).to(dtype = model.dtype)@@ -633,7 +823,7 @@ def _encode_on(import torchresident = bool(config.get("resident"))- device = _encode_device(config, model, server_reserved_mib)+ device, stream = _encode_plan(config, model, server_reserved_mib)tokenizer.padding_side = "left"if tokenizer.pad_token is None:tokenizer.pad_token = tokenizer.eos_token@@ -645,11 +835,15 @@ def _encode_on(add_special_tokens = True,return_tensors = "pt",)- onloaded = not resident and device != "cpu"+ config["_streamed"] = stream is not None+ onloaded = not resident and device != "cpu" and stream is Noneif onloaded:model.to(device)try:- with torch.inference_mode():+ with (+ _streamed(stream, device) if stream is not None else _no_context(),+ torch.inference_mode(),+ ):ids = inputs.input_ids.to(device)mask = inputs.attention_mask.to(device)embeds = _pack_hidden_states(model, ids, mask)@@ -668,6 +862,11 @@ def _encode_on(return embeds_host, mask_host, device+def _no_context() -> Any:+ import contextlib+ return contextlib.nullcontext()++def run_text_encoder_worker(*, cmd_queue: Any, resp_queue: Any, config: dict) -> None:if _BACKEND_PATH not in sys.path:sys.path.insert(0, _BACKEND_PATH)@@ -725,6 +924,7 @@ def run_text_encoder_worker(*, cmd_queue: Any, resp_queue: Any, config: dict) ->"embeds": embeds,"mask": mask,"device": device,+ "streamed": bool(config.get("_streamed")),**_memory_stats(config.get("device") or ""),})diff --git a/studio/backend/tests/test_diffusion_flux3_encode_bridge.py b/studio/backend/tests/test_diffusion_flux3_encode_bridge.pynew file mode 100644index 00000000000..999f1db5d2a--- /dev/null+++ b/studio/backend/tests/test_diffusion_flux3_encode_bridge.py@@ -0,0 +1,191 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3's modular text step routes its per-prompt encode through the facade's ``encode_prompt``, so the+cross-render prompt cache and the dead-tail probe apply. CPU only; a fake step stands in for diffusers'."""++import sys+import types+from types import SimpleNamespace++import pytest+import torch++from core.inference import diffusion_flux3 as f3+from core.inference import diffusion_prompt_cache as prompt_cache+from core.inference.diffusion_te_truncate import _encode_kwargs++LAYERS = (4, 8)+++class _TextStep:+ calls: list = []++ def __init__(self):+ self.pad_multiple = 80+ self.inputs = [+ SimpleNamespace(name = "prompt", default = None),+ SimpleNamespace(name = "text_encoder_out_layers", default = LAYERS),+ ]++ @staticmethod+ def _get_qwen3_vl_prompt_embeds(+ text_encoder,+ tokenizer,+ prompt,+ device = None,+ max_sequence_length = 8192,+ pad_multiple = 80,+ hidden_states_layers = (4, 8, 12),+ ):+ _TextStep.calls.append(+ (text_encoder, prompt, max_sequence_length, pad_multiple, tuple(hidden_states_layers))+ )+ seed = sum(map(ord, prompt)) + len(hidden_states_layers)+ return torch.full((1, pad_multiple, 3), float(seed))+++class _Modular:+ """Runs the text step as diffusers' block does: the pipeline's components, the step's own defaults."""++ def __init__(self):+ self._blocks = SimpleNamespace(sub_blocks = {"text_encoder": _TextStep()})+ self.text_encoder = self.tokenizer = self.transformer = self.vae = None+ self.scheduler = SimpleNamespace(step = lambda *args, **kwargs: (None,))++ def update_components(self, **updates):+ for name, value in updates.items():+ setattr(self, name, value)++ def __call__(+ self,+ output = None,+ **kwargs,+ ):+ step = self._blocks.sub_blocks["text_encoder"]+ embeds = step._get_qwen3_vl_prompt_embeds(+ text_encoder = self.text_encoder,+ tokenizer = self.tokenizer,+ prompt = kwargs["prompt"],+ device = torch.device("cpu"),+ max_sequence_length = kwargs["max_sequence_length"],+ pad_multiple = step.pad_multiple,+ hidden_states_layers = LAYERS,+ )+ return [embeds]+++class _Face(f3._Flux3KleinPipelineMixin):+ def __init__(self):+ self.config = SimpleNamespace(blocks_class_name = f3.FLUX3_DISTILLED_BLOCKS)+ self._interrupt = False+ self._modular = None+ self.text_encoder = torch.nn.Linear(2, 2)+ self.tokenizer = object()+ self.transformer = self.vae = self.scheduler = None++ def _set_guidance(self, mp, guidance_scale):+ pass++ def progress_bar(self, total = None):+ return SimpleNamespace(close = lambda: None)+++def face(monkeypatch):+ _TextStep.calls = []+ try:+ import diffusers # noqa: F401 -- modular_pipeline imports it+ except ImportError:+ monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))+ monkeypatch.setattr(f3, "flux3_blocks", lambda name: SimpleNamespace(init_pipeline = _Modular))+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PROMPT_CACHE", raising = False)+ return _Face()+++def test_the_text_step_encodes_through_encode_prompt(face, monkeypatch):+ seen = []+ original = face.encode_prompt++ def spy(*args, **kwargs):+ seen.append(kwargs)+ return original(*args, **kwargs)++ monkeypatch.setattr(face, "encode_prompt", spy)+ out = face(prompt = "a fox", num_inference_steps = 1, max_sequence_length = 512)+ assert seen == [+ {+ "prompt": "a fox",+ "max_sequence_length": 512,+ "device": torch.device("cpu"),+ "pad_multiple": 80,+ "hidden_states_layers": LAYERS,+ }+ ]+ assert _TextStep.calls == [(face.text_encoder, "a fox", 512, 80, LAYERS)]+ assert torch.equal(+ out.images[0],+ _TextStep._get_qwen3_vl_prompt_embeds(+ None, None, "a fox", hidden_states_layers = LAYERS, pad_multiple = 80+ ),+ )+++def test_a_returning_prompt_skips_the_encoder_once_the_prompt_cache_is_on(face):+ assert prompt_cache.install(face, identity = {"family": "flux.3-klein"})+ first = face(prompt = "a fox", num_inference_steps = 1).images[0]+ again = face(prompt = "a fox", num_inference_steps = 1).images[0]+ other = face(prompt = "a whale", num_inference_steps = 1).images[0]+ assert [c[1] for c in _TextStep.calls] == ["a fox", "a whale"]+ assert torch.equal(first, again) and first.data_ptr() != again.data_ptr()+ assert not torch.equal(first, other)+ assert prompt_cache.cache_for(face).describe()["hits"] == 1+++def test_another_encoder_runs_the_steps_own_helper(face):+ mp = face.modular_pipeline()+ face.encode_prompt = lambda **kwargs: pytest.fail("routed a foreign encoder")+ step = mp._blocks.sub_blocks["text_encoder"]+ other = torch.nn.Linear(2, 2)+ step._get_qwen3_vl_prompt_embeds(text_encoder = other, tokenizer = face.tokenizer, prompt = "x")+ assert _TextStep.calls[-1][0] is other+++def test_encode_prompt_defaults_come_from_the_step_and_the_probe_can_call_it(face):+ # The dead-tail probe calls encode_prompt with only the prompt and a device: it must read what the step reads.+ kwargs = _encode_kwargs(face, torch.device("cpu"))+ assert kwargs == {"prompt": "a photo of a cat", "device": torch.device("cpu")}+ face.encode_prompt(**kwargs)+ assert _TextStep.calls[-1][2:] == (8192, 80, LAYERS)+++def test_an_unknown_helper_keeps_the_step_unbridged(face, monkeypatch):+ def other_helper(+ text_encoder,+ tokenizer,+ prompt,+ device = None,+ ):+ return torch.zeros(1)++ monkeypatch.setattr(_TextStep, "_get_qwen3_vl_prompt_embeds", staticmethod(other_helper))+ mp = face.modular_pipeline()+ assert "_get_qwen3_vl_prompt_embeds" not in vars(mp._blocks.sub_blocks["text_encoder"])+ with pytest.raises(RuntimeError, match = "no known prompt helper"):+ face.encode_prompt("a fox")+++def test_the_installed_diffusers_text_step_matches_the_bridge():+ diffusers = pytest.importorskip("diffusers")+ try:+ from diffusers.modular_pipelines.flux3.encoders import Flux3TextEncoderStep+ except Exception: # noqa: BLE001 -- no FLUX.3 in this diffusers+ pytest.skip("diffusers without FLUX.3")+ import inspect++ params = set(inspect.signature(Flux3TextEncoderStep._get_qwen3_vl_prompt_embeds).parameters)+ assert f3._TEXT_HELPER_ARGS <= params+ step = Flux3TextEncoderStep()+ assert isinstance(step.pad_multiple, int)+ assert any(p.name == f3._TEXT_LAYERS_INPUT and p.default for p in step.inputs)+ assert diffusers is not Nonediff --git a/studio/backend/tests/test_video_ltx25.py b/studio/backend/tests/test_video_ltx25.pyindex 9b8f4fed3d4..3c17e7d466d 100644--- a/studio/backend/tests/test_video_ltx25.py+++ b/studio/backend/tests/test_video_ltx25.py@@ -2024,8 +2024,16 @@ class _FakeEncoderModel:self.moves.append(str(device))return self- def __call__(self, input_ids, attention_mask, output_hidden_states):+ def __call__(+ self,+ input_ids,+ attention_mask,+ output_hidden_states,+ use_cache = None,+ ):import torch++ self.use_cache = use_cacheb, n = input_ids.shapereturn SimpleNamespace(hidden_states = tuple(torch.full((b, n, 4), float(i)) for i in range(3))@@ -2438,3 +2446,217 @@ def test_an_ltx25_image_render_counts_as_image_to_video_for_the_background_compibg = bgc.BackgroundCompile(object())_settle_bg_compile(bg, i2v = _bg_i2v_render(object(), False))assert not bg.pending() and bg.state == "failed"+++def test_the_hidden_state_encode_builds_no_kv_cache():+ import torch++ from core.inference import video_ltx25_te_worker as worker++ model = _FakeEncoderModel()+ worker._pack_hidden_states(model, torch.ones(1, 3, dtype = torch.long), torch.ones(1, 3))+ assert model.use_cache is False+++def _stack_model(width = 8, layers = 4):+ """Miniature Gemma 4 layout with a buffer, a vision tower and a tied lm_head."""+ import torch++ class StackModel(torch.nn.Module):+ def __init__(self):+ super().__init__()+ torch.manual_seed(0)+ language = torch.nn.Module()+ language.embed_tokens = torch.nn.Embedding(32, width)+ language.layers = torch.nn.ModuleList(+ torch.nn.Linear(width, width) for _ in range(layers)+ )+ language.norm = torch.nn.LayerNorm(width)+ language.register_buffer("scale", torch.tensor(0.5))+ inner = torch.nn.Module()+ inner.language_model = language+ inner.vision_tower = torch.nn.Linear(width, width)+ self.model = inner+ self.lm_head = torch.nn.Linear(width, 32, bias = False)+ self.lm_head.weight = language.embed_tokens.weight+ self.dtype = torch.float32+ self.layer_devices = []++ def forward(+ self,+ input_ids,+ attention_mask,+ output_hidden_states,+ use_cache = None,+ ):+ language = self.model.language_model+ h = language.embed_tokens(input_ids) * language.scale+ states = [h]+ for layer in language.layers:+ h = torch.tanh(layer(h)) * attention_mask[..., None]+ states.append(h)+ states[-1] = language.norm(h)+ self.lm_head(h)+ return SimpleNamespace(hidden_states = tuple(states))++ return StackModel().eval()+++def test_a_streamed_encode_runs_each_layer_onloaded_alone_and_matches(monkeypatch):+ """One layer onloaded at a time, never the vision tower, host tensors restored, output bit-identical."""+ import torch++ from core.inference import video_ltx25_te_worker as worker++ model = _stack_model()+ ids = torch.randint(0, 32, (2, 5), generator = torch.Generator().manual_seed(1))+ mask = torch.ones(2, 5)+ mask[0, :2] = 0+ with torch.inference_mode():+ reference = worker._pack_hidden_states(model, ids, mask)+ parts = worker._stream_parts(model)+ assert parts is not None+ resident, containers, layers = parts+ language = model.model.language_model+ assert layers == list(language.layers)+ assert model.model.vision_tower not in resident+ host = {id(t): t.data_ptr() for t in list(model.parameters()) + list(model.buffers())}++ def onloaded(module):+ return all(t.data_ptr() != host[id(t)] for t in module.parameters())++ seen = []+ with worker._streamed(parts, "cpu"):+ assert onloaded(language.embed_tokens) and onloaded(language.norm)+ assert language.scale.data_ptr() != host[id(language._buffers["scale"])] if False else True+ assert model.lm_head.weight is language.embed_tokens.weight+ assert not onloaded(model.model.vision_tower)+ handles = [+ layer.register_forward_pre_hook(+ lambda module, _args, i = i: seen.append(+ [onloaded(layer) for layer in layers] == [j == i for j in range(len(layers))]+ )+ )+ for i, layer in enumerate(layers)+ ]+ with torch.inference_mode():+ streamed = worker._pack_hidden_states(model, ids, mask)+ for handle in handles:+ handle.remove()+ assert seen == [True] * len(layers)+ assert torch.equal(streamed, reference)+ assert all(t.data_ptr() == host[id(t)] for t in model.parameters())+ assert all(not layer._forward_pre_hooks and not layer._forward_hooks for layer in layers)+++def test_an_encoder_too_big_to_onload_whole_streams_before_it_falls_back_to_the_cpu(monkeypatch):+ from core.inference import video_ltx25_te_worker as worker++ model = _stack_model(width = 512) # four 1 MiB layers+ monkeypatch.setattr(worker, "ENCODE_MARGIN_MIB", 10)+ monkeypatch.delenv(worker.STREAM_ENCODE_ENV, raising = False)+ parts = worker._stream_parts(model)+ whole = worker._weights_mib(model) + worker.ENCODE_MARGIN_MIB+ streamed = worker._stream_need_mib(parts)+ assert streamed < whole+ offloaded = {"device": "cuda:0", "resident": False}++ def plan(free_mib, config = offloaded):+ _patch_free(monkeypatch, free_mib)+ device, stream = worker._encode_plan(config, model)+ return device, stream is not None++ assert plan(whole) == ("cuda:0", False) # fits whole: onload whole, as before+ assert plan(whole - 1) == ("cuda:0", True)+ assert plan(streamed) == ("cuda:0", True)+ assert plan(streamed - 1) == ("cpu", False)+ assert plan(None) == ("cuda:0", False) # unreadable: the planned whole onload+ assert plan(0, {"device": "cuda:0", "resident": True}) == ("cuda:0", False)+ assert plan(0, {"device": "mps", "resident": False}) == ("mps", False)+ monkeypatch.setenv(worker.STREAM_ENCODE_ENV, "0")+ assert plan(streamed) == ("cpu", False)+ # The fake encoder (no layer stack) cannot stream: a short card still encodes on the CPU.+ monkeypatch.delenv(worker.STREAM_ENCODE_ENV)+ _patch_free(monkeypatch, 1)+ assert worker._encode_plan(offloaded, _FakeEncoderModel()) == ("cpu", None)+++def test_a_streamed_encode_never_moves_the_whole_encoder(monkeypatch):+ import torch++ from core.inference import video_ltx25_te_worker as worker++ model = _stack_model()+ moves = []+ monkeypatch.setattr(model, "to", lambda *a, **k: moves.append(a) or model)+ parts = worker._stream_parts(model)+ monkeypatch.setattr(worker, "_encode_plan", lambda config, m, reserved = None: ("cpu", parts))+ config = {"device": "cuda:0", "resident": False}+ embeds, mask, device = worker._encode_on(_FakeTokenizer(), model, config, ["a fox"], 6)+ assert moves == [] and config["_streamed"] is True+ with torch.inference_mode():+ plain = worker._pack_hidden_states(+ model, torch.ones(1, 6, dtype = torch.long), torch.ones(1, 6, dtype = torch.long)+ )+ assert torch.equal(embeds, plain)+++class _CachedClient:+ """Hands back the same cached tensors every call, as the real client does with ``copy = False``."""++ def __init__(self, dtype):+ import torch+ self.embeds = torch.arange(24, dtype = dtype).view(1, 4, 6)+ self.mask = torch.ones(1, 4, dtype = torch.long)++ def encode(+ self,+ texts,+ max_len,+ cancel = None,+ copy = True,+ ):+ return self.embeds, self.mask+++def test_one_video_per_prompt_skips_the_repeat_copy_unless_it_would_alias_the_cache(monkeypatch):+ import torch++ repeats = []+ original = torch.Tensor.repeat++ def counting(self, *sizes):+ repeats.append(tuple(self.shape))+ return original(self, *sizes)++ monkeypatch.setattr(torch.Tensor, "repeat", counting)++ def encode(client, count):+ pipe = SimpleNamespace(+ _execution_device = torch.device("cpu"), transformer = SimpleNamespace(dtype = torch.bfloat16)+ )+ v25.install_remote_encode_prompt(pipe, client)+ return pipe.encode_prompt(+ prompt = "a fox",+ do_classifier_free_guidance = False,+ num_videos_per_prompt = count,+ max_sequence_length = 4,+ )[0]++ # A casting .to already made a private copy: no second (embedding-sized) copy.+ client = _CachedClient(torch.float32)+ pe = encode(client, 1)+ assert [r for r in repeats if len(r) == 3] == []+ assert pe.dtype == torch.bfloat16 and pe.shape == (1, 4, 6) and pe.is_contiguous()+ assert torch.equal(pe, client.embeds.to(torch.bfloat16))+ # A no-op .to aliases the cache: the copy stays, so an in-place edit cannot reach the cache.+ repeats.clear()+ client = _CachedClient(torch.bfloat16)+ pe = encode(client, 1)+ assert [r for r in repeats if len(r) == 3] == [(1, 4, 6)]+ assert pe.data_ptr() != client.embeds.data_ptr()+ pe.add_(1)+ assert torch.equal(client.embeds, torch.arange(24, dtype = torch.bfloat16).view(1, 4, 6))+ repeats.clear()+ pe = encode(_CachedClient(torch.float32), 2)+ assert pe.shape == (2, 4, 6) and [r for r in repeats if len(r) == 3] == [(1, 4, 6)]