Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio LTX-2.5: release the server's cached GPU blocks before an encode, so later prompts stop encoding on the CPU (#16804)

Daniel Han2821cbb464c4
diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.py
index 988357c846a..4cdb72636cf 100644
--- a/studio/backend/core/inference/video_ltx25.py
+++ b/studio/backend/core/inference/video_ltx25.py
@@ -791,6 +791,19 @@ def ensure_text_encoder_sidecar() -> str:
return _VENV_T5_510_DIR
+def _release_cached_device_memory(device: Optional[str]) -> None:
+ kind = str(device or "").split(":", 1)[0]
+ if kind not in ("cuda", "xpu"):
+ return
+ try:
+ import torch
+ accel = getattr(torch, kind)
+ if accel.is_available():
+ accel.empty_cache()
+ except Exception: # noqa: BLE001 -- best effort; the worker re-measures
+ pass
+
+
def _server_reserved_mib(device: Optional[str]) -> Optional[int]:
"""What this (server) process holds on ``device`` right now, for the worker's budget check."""
kind = str(device or "").split(":", 1)[0]
@@ -1033,6 +1046,9 @@ class Ltx25TextEncoderClient:
self.wait_ready(cancel)
self._seq += 1
seq = self._seq
+ if not self.config.get("resident"):
+ # Cached-only blocks must not count against the worker's room (it subtracts what we reserve).
+ _release_cached_device_memory(self.config.get("device"))
self._cmd.put(
{
"type": "encode",
diff --git a/studio/backend/tests/test_video_ltx25.py b/studio/backend/tests/test_video_ltx25.py
index 3c17e7d466d..0f73fea1033 100644
--- a/studio/backend/tests/test_video_ltx25.py
+++ b/studio/backend/tests/test_video_ltx25.py
@@ -2349,6 +2349,72 @@ def test_client_sends_the_budget_and_the_servers_share():
assert v25._server_reserved_mib("cpu") is None
[email protected]("resident", [False, True])
+def test_client_frees_the_servers_cached_blocks_before_a_non_resident_encode(monkeypatch, resident):
+ """Only a non-resident cache miss releases the server's cached blocks, before its reservation is sent."""
+ import torch
+
+ events = []
+ state = {"reserved": 9000 << 20}
+
+ def empty_cache():
+ events.append("empty_cache")
+ state["reserved"] = 6000 << 20
+
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "empty_cache", empty_cache)
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: state["reserved"])
+ client = v25.Ltx25TextEncoderClient(
+ repo = "r",
+ hf_token = None,
+ cache_dir = None,
+ local_files_only = True,
+ device = "cuda:0",
+ dtype = "bfloat16",
+ resident = resident,
+ venv_dir = "/x",
+ )
+ client._cmd, client._resp = queue.Queue(), queue.Queue()
+ client._proc = SimpleNamespace(is_alive = lambda: True, exitcode = None)
+ client._ready = True
+ sent = []
+
+ def fake_put(msg):
+ events.append("put")
+ sent.append(msg)
+ client._resp.put(
+ {
+ "type": "embeds",
+ "id": msg["id"],
+ "embeds": torch.zeros(1, 2, 3),
+ "mask": torch.ones(1, 2),
+ }
+ )
+
+ client._cmd.put = fake_put
+ client.encode(["p"], 2)
+ client.encode(["p"], 2)
+ if resident:
+ assert events == ["put"] and sent[0]["server_reserved_mib"] == 9000
+ else:
+ assert events == ["empty_cache", "put"] and sent[0]["server_reserved_mib"] == 6000
+
+
+def test_release_cached_device_memory_skips_hosts_without_a_card(monkeypatch):
+ import torch
+
+ calls = []
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: calls.append("cuda"))
+ for device in ("cpu", "mps", None, ""):
+ v25._release_cached_device_memory(device)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ v25._release_cached_device_memory("cuda:0")
+ assert calls == []
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ v25._release_cached_device_memory("cuda:0")
+ assert calls == ["cuda"]
+
+
# ── the encoder worker spawn consults the shutdown latch ─────────────────────
class _FakeEncoderProc:
def __init__(self, *args, **kwargs):