Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio diffusion: stream the LTX-2.5 encoder, cache and trim FLUX.3 prompt encodes (#16750)

Daniel Han89db0db7c5d6
diff --git a/studio/backend/core/inference/diffusion_flux3.py b/studio/backend/core/inference/diffusion_flux3.py
index a54a8ddb772..f0a16eefd2f 100644
--- a/studio/backend/core/inference/diffusion_flux3.py
+++ b/studio/backend/core/inference/diffusion_flux3.py
@@ -47,6 +47,78 @@ FLUX3_COMPONENT_FOLDERS = {
FLUX3_PROMPT_UPSAMPLER_STEP = "prompt_upsampler"
+# diffusers ``Flux3TextEncoderStep`` helper args; a build whose helper takes others keeps its own text step.
+FLUX3_TEXT_STEP = "text_encoder"
+_TEXT_HELPER = "_get_qwen3_vl_prompt_embeds"
+_TEXT_HELPER_ARGS = frozenset(
+ {
+ "text_encoder",
+ "tokenizer",
+ "prompt",
+ "device",
+ "max_sequence_length",
+ "pad_multiple",
+ "hidden_states_layers",
+ }
+)
+_TEXT_LAYERS_INPUT = "text_encoder_out_layers"
+
+
+def _text_step(mp: Any) -> tuple[Any, Any]:
+ """``(step, helper)``: the modular pipeline's text step and its class's prompt helper, or ``(None, None)``."""
+ import inspect
+
+ sub_blocks = getattr(getattr(mp, "_blocks", None), "sub_blocks", None)
+ step = sub_blocks.get(FLUX3_TEXT_STEP) if sub_blocks is not None else None
+ helper = getattr(type(step), _TEXT_HELPER, None) if step is not None else None
+ if not callable(helper):
+ return None, None
+ try:
+ if not _TEXT_HELPER_ARGS <= set(inspect.signature(helper).parameters):
+ return None, None
+ except (TypeError, ValueError):
+ return None, None
+ return step, helper
+
+
+def _bridge_text_step(pipe: Any, mp: Any) -> bool:
+ """Route the text step's per-prompt encode through ``pipe.encode_prompt`` (prompt cache, dead-tail probe).
+ Only for this pipe's own encoder and tokenizer; a foreign encoder runs the step's helper unchanged."""
+ import inspect
+ import weakref
+
+ step, helper = _text_step(mp)
+ if step is None:
+ return False
+ signature = inspect.signature(helper)
+ owner_ref = weakref.ref(pipe)
+
+ def bridged(*args: Any, **kwargs: Any) -> Any:
+ owner = owner_ref()
+ try:
+ bound = signature.bind(*args, **kwargs)
+ bound.apply_defaults()
+ except TypeError:
+ return helper(*args, **kwargs)
+ a = bound.arguments
+ if (
+ owner is None
+ or a["text_encoder"] is not getattr(owner, "text_encoder", None)
+ or a["tokenizer"] is not getattr(owner, "tokenizer", None)
+ ):
+ return helper(*args, **kwargs)
+ return owner.encode_prompt(
+ prompt = a["prompt"],
+ max_sequence_length = a["max_sequence_length"],
+ device = a["device"],
+ pad_multiple = a["pad_multiple"],
+ hidden_states_layers = a["hidden_states_layers"],
+ )
+
+ setattr(step, _TEXT_HELPER, bridged)
+ return True
+
+
def flux3_blocks(blocks_class_name: str) -> Any:
"""The named FLUX.3 auto blocks without the prompt upsampler step (absent before diffusers#15058)."""
import diffusers
@@ -284,6 +356,7 @@ class _Flux3KleinPipelineMixin:
if self._modular is None:
self._modular = flux3_blocks(self.config.blocks_class_name).init_pipeline()
+ _bridge_text_step(self, self._modular)
mp = self._modular
updates = {}
for name in ("transformer", "vae", "text_encoder", "tokenizer", "scheduler"):
@@ -294,6 +367,42 @@ class _Flux3KleinPipelineMixin:
mp.update_components(**updates)
return mp
+ def encode_prompt(
+ self,
+ prompt: str,
+ max_sequence_length: int = 8192,
+ device: Any = None,
+ pad_multiple: Optional[int] = None,
+ hidden_states_layers: Any = None,
+ ):
+ """One prompt's FLUX.3 text embeddings via diffusers' text step helper; ``None`` takes the step's defaults."""
+ import torch
+
+ step, helper = _text_step(self.modular_pipeline())
+ if step is None:
+ raise RuntimeError("this diffusers build's FLUX.3 text step has no known prompt helper")
+ if pad_multiple is None:
+ pad_multiple = step.pad_multiple
+ if hidden_states_layers is None:
+ hidden_states_layers = next(
+ (p.default for p in step.inputs if getattr(p, "name", None) == _TEXT_LAYERS_INPUT),
+ None,
+ )
+ if hidden_states_layers is None:
+ raise RuntimeError(
+ "this diffusers build's FLUX.3 text step names no default layers"
+ )
+ with torch.no_grad():
+ return helper(
+ text_encoder = self.text_encoder,
+ tokenizer = self.tokenizer,
+ prompt = prompt,
+ device = device,
+ max_sequence_length = int(max_sequence_length),
+ pad_multiple = int(pad_multiple),
+ hidden_states_layers = tuple(hidden_states_layers),
+ )
+
def _set_guidance(self, mp, guidance_scale: Optional[float]) -> None:
guider = getattr(mp, "guider", None)
if guider is None or guidance_scale is None:
diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.py
index e873b1e9641..988357c846a 100644
--- a/studio/backend/core/inference/video_ltx25.py
+++ b/studio/backend/core/inference/video_ltx25.py
@@ -681,9 +681,9 @@ def encoder_resident_for_plan(plan: Any) -> bool:
The plan prices the family's text encoder (the table's 23.9 GB) as if it were in-process: resident under ``none``
and under ``group`` with resident companions, streamed under ``group`` with streamed text encoders and under
- ``streaming`` / ``sequential``, alone on the card under ``model``. The worker cannot stream, so on every tier that did
- not budget it resident it keeps the encoder on the host and onloads it only for an encode, when the card has room
- (else it encodes on the CPU; see ``video_ltx25_te_worker._encode_device``)."""
+ ``streaming`` / ``sequential``, alone on the card under ``model``. On every tier that did not budget it resident the
+ worker keeps the encoder on the host and onloads it only for an encode: whole when the card has room, else one
+ decoder layer at a time, else it encodes on the CPU (see ``video_ltx25_te_worker._encode_plan``)."""
from .diffusion_memory import OFFLOAD_GROUP, OFFLOAD_NONE
policy = getattr(plan, "offload_policy", None)
@@ -1083,14 +1083,20 @@ def install_remote_encode_prompt(pipe: Any, client: Ltx25TextEncoderClient) -> N
def _one(texts: list[str]) -> tuple[Any, Any]:
# The client's own cached tensors (read only): every op below returns a new tensor.
- embeds, mask = client.encode(texts, max_sequence_length, copy = False)
- batch = embeds.shape[0]
- embeds = embeds.to(device = device, dtype = dtype)
+ cached, mask = client.encode(texts, max_sequence_length, copy = False)
+ batch = cached.shape[0]
+ embeds = cached.to(device = device, dtype = dtype)
mask = mask.to(device = device)
seq_len = embeds.shape[1]
- embeds = embeds.repeat(1, num_videos_per_prompt, 1).view(
- batch * num_videos_per_prompt, seq_len, -1
- )
+ # A ``.to`` that moved or cast already copied, so one video skips ``repeat``; a no-op ``.to`` aliases the cache.
+ if (
+ num_videos_per_prompt != 1
+ or embeds.data_ptr() == cached.data_ptr()
+ or not embeds.is_contiguous()
+ ):
+ embeds = embeds.repeat(1, num_videos_per_prompt, 1).view(
+ batch * num_videos_per_prompt, seq_len, -1
+ )
mask = mask.view(batch, -1).repeat(num_videos_per_prompt, 1)
return embeds, mask
diff --git a/studio/backend/core/inference/video_ltx25_te_worker.py b/studio/backend/core/inference/video_ltx25_te_worker.py
index 5feae03ca78..bb2c6c37c3d 100644
--- a/studio/backend/core/inference/video_ltx25_te_worker.py
+++ b/studio/backend/core/inference/video_ltx25_te_worker.py
@@ -538,9 +538,19 @@ def _encode_device(
# MPS shares host memory, so there is no card to keep the encoder off.
if config.get("resident") or kind not in ("cuda", "xpu"):
return device
+ need = _weights_mib(model) + ENCODE_MARGIN_MIB
+ room = _encode_room_mib(config, server_reserved_mib)
+ if room is None:
+ return device if kind == "cuda" else "cpu"
+ return device if room >= need else "cpu"
+
+
+def _encode_room_mib(config: dict, server_reserved_mib: Optional[int] = None) -> Optional[int]:
+ """MiB an encode may use on the configured card (see ``_encode_device``), or None when nothing could be read."""
import torch
- need = _weights_mib(model) + ENCODE_MARGIN_MIB
+ device = str(config["device"])
+ kind = device.split(":", 1)[0]
room: Optional[int] = None
try:
free, total = getattr(torch, kind).mem_get_info(torch.device(device))
@@ -552,9 +562,188 @@ def _encode_device(
if budget is not None and server_reserved_mib is not None:
allowed = int(budget) - int(server_reserved_mib) - WORKER_CONTEXT_MIB
room = allowed if room is None else min(room, allowed)
- if room is None:
- return device if kind == "cuda" else "cpu"
- return device if room >= need else "cpu"
+ return room
+
+
+# Towers a text encode never runs; a streamed encode leaves them on the host.
+_TOWER_TOKENS = ("visual", "vision", "audio")
+
+
+def _tensor_tables(module: Any) -> tuple:
+ return (module._parameters, module._buffers)
+
+
+def _stream_parts(model: Any) -> Optional[tuple[list, list, list]]:
+ """``(resident, containers, layers)`` for a layer-streamed encode, or None when the encoder cannot stream.
+ A module overriding ``_apply`` may keep tensors outside its tables, so it declines streaming."""
+ import torch
+
+ from core.inference.diffusion_te_truncate import is_passthrough
+
+ try:
+ stack = _decoder_layers(model)
+ except Exception: # noqa: BLE001 -- not the expected Gemma 4 layout
+ return None
+ if not isinstance(model, torch.nn.Module) or not isinstance(stack, torch.nn.ModuleList):
+ return None
+ layers = [m for m in stack if not is_passthrough(m)]
+ if not layers:
+ return None
+ resident: list = []
+ containers: list = []
+
+ def walk(module: Any) -> None:
+ containers.append(module)
+ for name, child in module.named_children():
+ if child is stack or any(token in name.lower() for token in _TOWER_TOKENS):
+ continue
+ if any(m is stack for m in child.modules()):
+ walk(child)
+ else:
+ resident.append(child)
+
+ walk(model)
+ for module in [*containers, *[s for m in (*resident, *layers) for s in m.modules()]]:
+ if type(module)._apply is not torch.nn.Module._apply:
+ return None
+ return resident, containers, layers
+
+
+def _tensors_mib(modules: list, recurse: bool = True) -> int:
+ seen: set = set()
+ total = 0
+ for module in modules:
+ for sub in module.modules() if recurse else (module,):
+ for table in _tensor_tables(sub):
+ for tensor in table.values():
+ if tensor is None or id(tensor) in seen:
+ continue
+ seen.add(id(tensor))
+ total += tensor.numel() * tensor.element_size()
+ return total >> 20
+
+
+def _stream_need_mib(parts: tuple[list, list, list]) -> int:
+ """Resident modules + two layers (allocator may still hold the last) + the encode margin."""
+ resident, containers, layers = parts
+ biggest = max(_tensors_mib([layer]) for layer in layers)
+ return (
+ _tensors_mib(resident)
+ + _tensors_mib(containers, recurse = False)
+ + 2 * biggest
+ + ENCODE_MARGIN_MIB
+ )
+
+
+def _encode_plan(
+ config: dict,
+ model: Any,
+ server_reserved_mib: Optional[int] = None,
+) -> tuple[str, Optional[tuple[list, list, list]]]:
+ """``(device, parts)``: an encoder too big to onload whole streams its layers (``parts`` set) instead of CPU."""
+ device = _encode_device(config, model, server_reserved_mib)
+ configured = str(config["device"])
+ if device != "cpu" or configured.split(":", 1)[0] not in ("cuda", "xpu"):
+ return device, None
+ if config.get("resident") or (os.environ.get(STREAM_ENCODE_ENV) or "").strip() == "0":
+ return device, None
+ try:
+ parts = _stream_parts(model)
+ if parts is None:
+ return device, None
+ room = _encode_room_mib(config, server_reserved_mib)
+ if room is None or room < _stream_need_mib(parts):
+ return device, None
+ except Exception: # noqa: BLE001 -- any doubt keeps the CPU encode
+ return device, None
+ return configured, parts
+
+
+# "0" turns the layer-streamed encode off (a card too small for the whole encoder then encodes on the CPU again).
+STREAM_ENCODE_ENV = "UNSLOTH_LTX25_TE_STREAM"
+
+
+class _Onload:
+ """Moves the tensors of some modules to a device and back. Parameters keep their identity (``.data`` swaps, so a
+ tied weight moves once); the host tensors are kept and restored, so returning costs no copy."""
+
+ def __init__(self, device: Any) -> None:
+ self.device = device
+ self.saved: list = []
+ self.moved: dict = {}
+
+ def add(
+ self,
+ modules: list,
+ recurse: bool = True,
+ ) -> None:
+ import torch
+ for module in modules:
+ for sub in module.modules() if recurse else (module,):
+ for table in _tensor_tables(sub):
+ for key, tensor in list(table.items()):
+ if tensor is None:
+ continue
+ if isinstance(tensor, torch.nn.Parameter):
+ if id(tensor) not in self.moved:
+ self.moved[id(tensor)] = tensor
+ self.saved.append((tensor, None, None, tensor.data))
+ tensor.data = tensor.data.to(self.device, copy = True)
+ continue
+ moved = self.moved.get(id(tensor))
+ if moved is None:
+ moved = tensor.to(self.device, copy = True)
+ self.moved[id(tensor)] = moved
+ self.saved.append((None, table, key, tensor))
+ table[key] = moved
+
+ def restore(self) -> None:
+ while self.saved:
+ param, table, key, original = self.saved.pop()
+ if param is not None:
+ param.data = original
+ else:
+ table[key] = original
+ self.moved.clear()
+
+
+def _streamed(parts: tuple[list, list, list], device: str) -> Any:
+ """Context: the resident modules on ``device`` for the encode, each decoder layer only while it runs."""
+ import contextlib
+
+ resident, containers, layers = parts
+
+ @contextlib.contextmanager
+ def _ctx():
+ base = _Onload(device)
+ handles: list = []
+ live: dict = {}
+ try:
+ base.add(resident)
+ base.add(containers, recurse = False)
+
+ def before(module: Any, _args: Any) -> None:
+ onload = _Onload(device)
+ live[id(module)] = onload
+ onload.add([module])
+
+ def after(module: Any, _args: Any, _output: Any) -> None:
+ onload = live.pop(id(module), None)
+ if onload is not None:
+ onload.restore()
+
+ for layer in layers:
+ handles.append(layer.register_forward_pre_hook(before))
+ handles.append(layer.register_forward_hook(after, always_call = True))
+ yield
+ finally:
+ for handle in handles:
+ handle.remove()
+ for onload in live.values():
+ onload.restore()
+ base.restore()
+
+ return _ctx()
def _memory_stats(device: str) -> dict:
@@ -581,10 +770,11 @@ def _memory_stats(device: str) -> dict:
def _pack_hidden_states(model: Any, ids: Any, mask: Any) -> Any:
- """``LTX2Pipeline._get_gemma_prompt_embeds``: every hidden state stacked on a last axis, flattened."""
+ """``LTX2Pipeline._get_gemma_prompt_embeds``: every hidden state stacked on a last axis, flattened. No KV cache:
+ the encoder config sets ``use_cache``, which would build and hold a DynamicCache this single forward never reads."""
import torch
- out = model(input_ids = ids, attention_mask = mask, output_hidden_states = True)
+ out = model(input_ids = ids, attention_mask = mask, output_hidden_states = True, use_cache = False)
return torch.stack(out.hidden_states, dim = -1).flatten(2, 3).to(dtype = model.dtype)
@@ -633,7 +823,7 @@ def _encode_on(
import torch
resident = bool(config.get("resident"))
- device = _encode_device(config, model, server_reserved_mib)
+ device, stream = _encode_plan(config, model, server_reserved_mib)
tokenizer.padding_side = "left"
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
@@ -645,11 +835,15 @@ def _encode_on(
add_special_tokens = True,
return_tensors = "pt",
)
- onloaded = not resident and device != "cpu"
+ config["_streamed"] = stream is not None
+ onloaded = not resident and device != "cpu" and stream is None
if onloaded:
model.to(device)
try:
- with torch.inference_mode():
+ with (
+ _streamed(stream, device) if stream is not None else _no_context(),
+ torch.inference_mode(),
+ ):
ids = inputs.input_ids.to(device)
mask = inputs.attention_mask.to(device)
embeds = _pack_hidden_states(model, ids, mask)
@@ -668,6 +862,11 @@ def _encode_on(
return embeds_host, mask_host, device
+def _no_context() -> Any:
+ import contextlib
+ return contextlib.nullcontext()
+
+
def run_text_encoder_worker(*, cmd_queue: Any, resp_queue: Any, config: dict) -> None:
if _BACKEND_PATH not in sys.path:
sys.path.insert(0, _BACKEND_PATH)
@@ -725,6 +924,7 @@ def run_text_encoder_worker(*, cmd_queue: Any, resp_queue: Any, config: dict) ->
"embeds": embeds,
"mask": mask,
"device": device,
+ "streamed": bool(config.get("_streamed")),
**_memory_stats(config.get("device") or ""),
}
)
diff --git a/studio/backend/tests/test_diffusion_flux3_encode_bridge.py b/studio/backend/tests/test_diffusion_flux3_encode_bridge.py
new file mode 100644
index 00000000000..999f1db5d2a
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux3_encode_bridge.py
@@ -0,0 +1,191 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3's modular text step routes its per-prompt encode through the facade's ``encode_prompt``, so the
+cross-render prompt cache and the dead-tail probe apply. CPU only; a fake step stands in for diffusers'."""
+
+import sys
+import types
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from core.inference import diffusion_flux3 as f3
+from core.inference import diffusion_prompt_cache as prompt_cache
+from core.inference.diffusion_te_truncate import _encode_kwargs
+
+LAYERS = (4, 8)
+
+
+class _TextStep:
+ calls: list = []
+
+ def __init__(self):
+ self.pad_multiple = 80
+ self.inputs = [
+ SimpleNamespace(name = "prompt", default = None),
+ SimpleNamespace(name = "text_encoder_out_layers", default = LAYERS),
+ ]
+
+ @staticmethod
+ def _get_qwen3_vl_prompt_embeds(
+ text_encoder,
+ tokenizer,
+ prompt,
+ device = None,
+ max_sequence_length = 8192,
+ pad_multiple = 80,
+ hidden_states_layers = (4, 8, 12),
+ ):
+ _TextStep.calls.append(
+ (text_encoder, prompt, max_sequence_length, pad_multiple, tuple(hidden_states_layers))
+ )
+ seed = sum(map(ord, prompt)) + len(hidden_states_layers)
+ return torch.full((1, pad_multiple, 3), float(seed))
+
+
+class _Modular:
+ """Runs the text step as diffusers' block does: the pipeline's components, the step's own defaults."""
+
+ def __init__(self):
+ self._blocks = SimpleNamespace(sub_blocks = {"text_encoder": _TextStep()})
+ self.text_encoder = self.tokenizer = self.transformer = self.vae = None
+ self.scheduler = SimpleNamespace(step = lambda *args, **kwargs: (None,))
+
+ def update_components(self, **updates):
+ for name, value in updates.items():
+ setattr(self, name, value)
+
+ def __call__(
+ self,
+ output = None,
+ **kwargs,
+ ):
+ step = self._blocks.sub_blocks["text_encoder"]
+ embeds = step._get_qwen3_vl_prompt_embeds(
+ text_encoder = self.text_encoder,
+ tokenizer = self.tokenizer,
+ prompt = kwargs["prompt"],
+ device = torch.device("cpu"),
+ max_sequence_length = kwargs["max_sequence_length"],
+ pad_multiple = step.pad_multiple,
+ hidden_states_layers = LAYERS,
+ )
+ return [embeds]
+
+
+class _Face(f3._Flux3KleinPipelineMixin):
+ def __init__(self):
+ self.config = SimpleNamespace(blocks_class_name = f3.FLUX3_DISTILLED_BLOCKS)
+ self._interrupt = False
+ self._modular = None
+ self.text_encoder = torch.nn.Linear(2, 2)
+ self.tokenizer = object()
+ self.transformer = self.vae = self.scheduler = None
+
+ def _set_guidance(self, mp, guidance_scale):
+ pass
+
+ def progress_bar(self, total = None):
+ return SimpleNamespace(close = lambda: None)
+
+
+def face(monkeypatch):
+ _TextStep.calls = []
+ try:
+ import diffusers # noqa: F401 -- modular_pipeline imports it
+ except ImportError:
+ monkeypatch.setitem(sys.modules, "diffusers", types.ModuleType("diffusers"))
+ monkeypatch.setattr(f3, "flux3_blocks", lambda name: SimpleNamespace(init_pipeline = _Modular))
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_PROMPT_CACHE", raising = False)
+ return _Face()
+
+
+def test_the_text_step_encodes_through_encode_prompt(face, monkeypatch):
+ seen = []
+ original = face.encode_prompt
+
+ def spy(*args, **kwargs):
+ seen.append(kwargs)
+ return original(*args, **kwargs)
+
+ monkeypatch.setattr(face, "encode_prompt", spy)
+ out = face(prompt = "a fox", num_inference_steps = 1, max_sequence_length = 512)
+ assert seen == [
+ {
+ "prompt": "a fox",
+ "max_sequence_length": 512,
+ "device": torch.device("cpu"),
+ "pad_multiple": 80,
+ "hidden_states_layers": LAYERS,
+ }
+ ]
+ assert _TextStep.calls == [(face.text_encoder, "a fox", 512, 80, LAYERS)]
+ assert torch.equal(
+ out.images[0],
+ _TextStep._get_qwen3_vl_prompt_embeds(
+ None, None, "a fox", hidden_states_layers = LAYERS, pad_multiple = 80
+ ),
+ )
+
+
+def test_a_returning_prompt_skips_the_encoder_once_the_prompt_cache_is_on(face):
+ assert prompt_cache.install(face, identity = {"family": "flux.3-klein"})
+ first = face(prompt = "a fox", num_inference_steps = 1).images[0]
+ again = face(prompt = "a fox", num_inference_steps = 1).images[0]
+ other = face(prompt = "a whale", num_inference_steps = 1).images[0]
+ assert [c[1] for c in _TextStep.calls] == ["a fox", "a whale"]
+ assert torch.equal(first, again) and first.data_ptr() != again.data_ptr()
+ assert not torch.equal(first, other)
+ assert prompt_cache.cache_for(face).describe()["hits"] == 1
+
+
+def test_another_encoder_runs_the_steps_own_helper(face):
+ mp = face.modular_pipeline()
+ face.encode_prompt = lambda **kwargs: pytest.fail("routed a foreign encoder")
+ step = mp._blocks.sub_blocks["text_encoder"]
+ other = torch.nn.Linear(2, 2)
+ step._get_qwen3_vl_prompt_embeds(text_encoder = other, tokenizer = face.tokenizer, prompt = "x")
+ assert _TextStep.calls[-1][0] is other
+
+
+def test_encode_prompt_defaults_come_from_the_step_and_the_probe_can_call_it(face):
+ # The dead-tail probe calls encode_prompt with only the prompt and a device: it must read what the step reads.
+ kwargs = _encode_kwargs(face, torch.device("cpu"))
+ assert kwargs == {"prompt": "a photo of a cat", "device": torch.device("cpu")}
+ face.encode_prompt(**kwargs)
+ assert _TextStep.calls[-1][2:] == (8192, 80, LAYERS)
+
+
+def test_an_unknown_helper_keeps_the_step_unbridged(face, monkeypatch):
+ def other_helper(
+ text_encoder,
+ tokenizer,
+ prompt,
+ device = None,
+ ):
+ return torch.zeros(1)
+
+ monkeypatch.setattr(_TextStep, "_get_qwen3_vl_prompt_embeds", staticmethod(other_helper))
+ mp = face.modular_pipeline()
+ assert "_get_qwen3_vl_prompt_embeds" not in vars(mp._blocks.sub_blocks["text_encoder"])
+ with pytest.raises(RuntimeError, match = "no known prompt helper"):
+ face.encode_prompt("a fox")
+
+
+def test_the_installed_diffusers_text_step_matches_the_bridge():
+ diffusers = pytest.importorskip("diffusers")
+ try:
+ from diffusers.modular_pipelines.flux3.encoders import Flux3TextEncoderStep
+ except Exception: # noqa: BLE001 -- no FLUX.3 in this diffusers
+ pytest.skip("diffusers without FLUX.3")
+ import inspect
+
+ params = set(inspect.signature(Flux3TextEncoderStep._get_qwen3_vl_prompt_embeds).parameters)
+ assert f3._TEXT_HELPER_ARGS <= params
+ step = Flux3TextEncoderStep()
+ assert isinstance(step.pad_multiple, int)
+ assert any(p.name == f3._TEXT_LAYERS_INPUT and p.default for p in step.inputs)
+ assert diffusers is not None
diff --git a/studio/backend/tests/test_video_ltx25.py b/studio/backend/tests/test_video_ltx25.py
index 9b8f4fed3d4..3c17e7d466d 100644
--- a/studio/backend/tests/test_video_ltx25.py
+++ b/studio/backend/tests/test_video_ltx25.py
@@ -2024,8 +2024,16 @@ class _FakeEncoderModel:
self.moves.append(str(device))
return self
- def __call__(self, input_ids, attention_mask, output_hidden_states):
+ def __call__(
+ self,
+ input_ids,
+ attention_mask,
+ output_hidden_states,
+ use_cache = None,
+ ):
import torch
+
+ self.use_cache = use_cache
b, n = input_ids.shape
return SimpleNamespace(
hidden_states = tuple(torch.full((b, n, 4), float(i)) for i in range(3))
@@ -2438,3 +2446,217 @@ def test_an_ltx25_image_render_counts_as_image_to_video_for_the_background_compi
bg = bgc.BackgroundCompile(object())
_settle_bg_compile(bg, i2v = _bg_i2v_render(object(), False))
assert not bg.pending() and bg.state == "failed"
+
+
+def test_the_hidden_state_encode_builds_no_kv_cache():
+ import torch
+
+ from core.inference import video_ltx25_te_worker as worker
+
+ model = _FakeEncoderModel()
+ worker._pack_hidden_states(model, torch.ones(1, 3, dtype = torch.long), torch.ones(1, 3))
+ assert model.use_cache is False
+
+
+def _stack_model(width = 8, layers = 4):
+ """Miniature Gemma 4 layout with a buffer, a vision tower and a tied lm_head."""
+ import torch
+
+ class StackModel(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ torch.manual_seed(0)
+ language = torch.nn.Module()
+ language.embed_tokens = torch.nn.Embedding(32, width)
+ language.layers = torch.nn.ModuleList(
+ torch.nn.Linear(width, width) for _ in range(layers)
+ )
+ language.norm = torch.nn.LayerNorm(width)
+ language.register_buffer("scale", torch.tensor(0.5))
+ inner = torch.nn.Module()
+ inner.language_model = language
+ inner.vision_tower = torch.nn.Linear(width, width)
+ self.model = inner
+ self.lm_head = torch.nn.Linear(width, 32, bias = False)
+ self.lm_head.weight = language.embed_tokens.weight
+ self.dtype = torch.float32
+ self.layer_devices = []
+
+ def forward(
+ self,
+ input_ids,
+ attention_mask,
+ output_hidden_states,
+ use_cache = None,
+ ):
+ language = self.model.language_model
+ h = language.embed_tokens(input_ids) * language.scale
+ states = [h]
+ for layer in language.layers:
+ h = torch.tanh(layer(h)) * attention_mask[..., None]
+ states.append(h)
+ states[-1] = language.norm(h)
+ self.lm_head(h)
+ return SimpleNamespace(hidden_states = tuple(states))
+
+ return StackModel().eval()
+
+
+def test_a_streamed_encode_runs_each_layer_onloaded_alone_and_matches(monkeypatch):
+ """One layer onloaded at a time, never the vision tower, host tensors restored, output bit-identical."""
+ import torch
+
+ from core.inference import video_ltx25_te_worker as worker
+
+ model = _stack_model()
+ ids = torch.randint(0, 32, (2, 5), generator = torch.Generator().manual_seed(1))
+ mask = torch.ones(2, 5)
+ mask[0, :2] = 0
+ with torch.inference_mode():
+ reference = worker._pack_hidden_states(model, ids, mask)
+ parts = worker._stream_parts(model)
+ assert parts is not None
+ resident, containers, layers = parts
+ language = model.model.language_model
+ assert layers == list(language.layers)
+ assert model.model.vision_tower not in resident
+ host = {id(t): t.data_ptr() for t in list(model.parameters()) + list(model.buffers())}
+
+ def onloaded(module):
+ return all(t.data_ptr() != host[id(t)] for t in module.parameters())
+
+ seen = []
+ with worker._streamed(parts, "cpu"):
+ assert onloaded(language.embed_tokens) and onloaded(language.norm)
+ assert language.scale.data_ptr() != host[id(language._buffers["scale"])] if False else True
+ assert model.lm_head.weight is language.embed_tokens.weight
+ assert not onloaded(model.model.vision_tower)
+ handles = [
+ layer.register_forward_pre_hook(
+ lambda module, _args, i = i: seen.append(
+ [onloaded(layer) for layer in layers] == [j == i for j in range(len(layers))]
+ )
+ )
+ for i, layer in enumerate(layers)
+ ]
+ with torch.inference_mode():
+ streamed = worker._pack_hidden_states(model, ids, mask)
+ for handle in handles:
+ handle.remove()
+ assert seen == [True] * len(layers)
+ assert torch.equal(streamed, reference)
+ assert all(t.data_ptr() == host[id(t)] for t in model.parameters())
+ assert all(not layer._forward_pre_hooks and not layer._forward_hooks for layer in layers)
+
+
+def test_an_encoder_too_big_to_onload_whole_streams_before_it_falls_back_to_the_cpu(monkeypatch):
+ from core.inference import video_ltx25_te_worker as worker
+
+ model = _stack_model(width = 512) # four 1 MiB layers
+ monkeypatch.setattr(worker, "ENCODE_MARGIN_MIB", 10)
+ monkeypatch.delenv(worker.STREAM_ENCODE_ENV, raising = False)
+ parts = worker._stream_parts(model)
+ whole = worker._weights_mib(model) + worker.ENCODE_MARGIN_MIB
+ streamed = worker._stream_need_mib(parts)
+ assert streamed < whole
+ offloaded = {"device": "cuda:0", "resident": False}
+
+ def plan(free_mib, config = offloaded):
+ _patch_free(monkeypatch, free_mib)
+ device, stream = worker._encode_plan(config, model)
+ return device, stream is not None
+
+ assert plan(whole) == ("cuda:0", False) # fits whole: onload whole, as before
+ assert plan(whole - 1) == ("cuda:0", True)
+ assert plan(streamed) == ("cuda:0", True)
+ assert plan(streamed - 1) == ("cpu", False)
+ assert plan(None) == ("cuda:0", False) # unreadable: the planned whole onload
+ assert plan(0, {"device": "cuda:0", "resident": True}) == ("cuda:0", False)
+ assert plan(0, {"device": "mps", "resident": False}) == ("mps", False)
+ monkeypatch.setenv(worker.STREAM_ENCODE_ENV, "0")
+ assert plan(streamed) == ("cpu", False)
+ # The fake encoder (no layer stack) cannot stream: a short card still encodes on the CPU.
+ monkeypatch.delenv(worker.STREAM_ENCODE_ENV)
+ _patch_free(monkeypatch, 1)
+ assert worker._encode_plan(offloaded, _FakeEncoderModel()) == ("cpu", None)
+
+
+def test_a_streamed_encode_never_moves_the_whole_encoder(monkeypatch):
+ import torch
+
+ from core.inference import video_ltx25_te_worker as worker
+
+ model = _stack_model()
+ moves = []
+ monkeypatch.setattr(model, "to", lambda *a, **k: moves.append(a) or model)
+ parts = worker._stream_parts(model)
+ monkeypatch.setattr(worker, "_encode_plan", lambda config, m, reserved = None: ("cpu", parts))
+ config = {"device": "cuda:0", "resident": False}
+ embeds, mask, device = worker._encode_on(_FakeTokenizer(), model, config, ["a fox"], 6)
+ assert moves == [] and config["_streamed"] is True
+ with torch.inference_mode():
+ plain = worker._pack_hidden_states(
+ model, torch.ones(1, 6, dtype = torch.long), torch.ones(1, 6, dtype = torch.long)
+ )
+ assert torch.equal(embeds, plain)
+
+
+class _CachedClient:
+ """Hands back the same cached tensors every call, as the real client does with ``copy = False``."""
+
+ def __init__(self, dtype):
+ import torch
+ self.embeds = torch.arange(24, dtype = dtype).view(1, 4, 6)
+ self.mask = torch.ones(1, 4, dtype = torch.long)
+
+ def encode(
+ self,
+ texts,
+ max_len,
+ cancel = None,
+ copy = True,
+ ):
+ return self.embeds, self.mask
+
+
+def test_one_video_per_prompt_skips_the_repeat_copy_unless_it_would_alias_the_cache(monkeypatch):
+ import torch
+
+ repeats = []
+ original = torch.Tensor.repeat
+
+ def counting(self, *sizes):
+ repeats.append(tuple(self.shape))
+ return original(self, *sizes)
+
+ monkeypatch.setattr(torch.Tensor, "repeat", counting)
+
+ def encode(client, count):
+ pipe = SimpleNamespace(
+ _execution_device = torch.device("cpu"), transformer = SimpleNamespace(dtype = torch.bfloat16)
+ )
+ v25.install_remote_encode_prompt(pipe, client)
+ return pipe.encode_prompt(
+ prompt = "a fox",
+ do_classifier_free_guidance = False,
+ num_videos_per_prompt = count,
+ max_sequence_length = 4,
+ )[0]
+
+ # A casting .to already made a private copy: no second (embedding-sized) copy.
+ client = _CachedClient(torch.float32)
+ pe = encode(client, 1)
+ assert [r for r in repeats if len(r) == 3] == []
+ assert pe.dtype == torch.bfloat16 and pe.shape == (1, 4, 6) and pe.is_contiguous()
+ assert torch.equal(pe, client.embeds.to(torch.bfloat16))
+ # A no-op .to aliases the cache: the copy stays, so an in-place edit cannot reach the cache.
+ repeats.clear()
+ client = _CachedClient(torch.bfloat16)
+ pe = encode(client, 1)
+ assert [r for r in repeats if len(r) == 3] == [(1, 4, 6)]
+ assert pe.data_ptr() != client.embeds.data_ptr()
+ pe.add_(1)
+ assert torch.equal(client.embeds, torch.arange(24, dtype = torch.bfloat16).view(1, 4, 6))
+ repeats.clear()
+ pe = encode(_CachedClient(torch.float32), 2)
+ assert pe.shape == (2, 4, 6) and [r for r in repeats if len(r) == 3] == [(1, 4, 6)]