Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio LTX-2.5: release the server's cached GPU blocks before an encode, so later prompts stop encoding on the CPU (#16804)
Daniel Han2821cbb464c4
diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.pyindex 988357c846a..4cdb72636cf 100644--- a/studio/backend/core/inference/video_ltx25.py+++ b/studio/backend/core/inference/video_ltx25.py@@ -791,6 +791,19 @@ def ensure_text_encoder_sidecar() -> str:return _VENV_T5_510_DIR+def _release_cached_device_memory(device: Optional[str]) -> None:+ kind = str(device or "").split(":", 1)[0]+ if kind not in ("cuda", "xpu"):+ return+ try:+ import torch+ accel = getattr(torch, kind)+ if accel.is_available():+ accel.empty_cache()+ except Exception: # noqa: BLE001 -- best effort; the worker re-measures+ pass++def _server_reserved_mib(device: Optional[str]) -> Optional[int]:"""What this (server) process holds on ``device`` right now, for the worker's budget check."""kind = str(device or "").split(":", 1)[0]@@ -1033,6 +1046,9 @@ class Ltx25TextEncoderClient:self.wait_ready(cancel)self._seq += 1seq = self._seq+ if not self.config.get("resident"):+ # Cached-only blocks must not count against the worker's room (it subtracts what we reserve).+ _release_cached_device_memory(self.config.get("device"))self._cmd.put({"type": "encode",diff --git a/studio/backend/tests/test_video_ltx25.py b/studio/backend/tests/test_video_ltx25.pyindex 3c17e7d466d..0f73fea1033 100644--- a/studio/backend/tests/test_video_ltx25.py+++ b/studio/backend/tests/test_video_ltx25.py@@ -2349,6 +2349,72 @@ def test_client_sends_the_budget_and_the_servers_share():assert v25._server_reserved_mib("cpu") is None[email protected]("resident", [False, True])+def test_client_frees_the_servers_cached_blocks_before_a_non_resident_encode(monkeypatch, resident):+ """Only a non-resident cache miss releases the server's cached blocks, before its reservation is sent."""+ import torch++ events = []+ state = {"reserved": 9000 << 20}++ def empty_cache():+ events.append("empty_cache")+ state["reserved"] = 6000 << 20++ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "empty_cache", empty_cache)+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda device = None: state["reserved"])+ client = v25.Ltx25TextEncoderClient(+ repo = "r",+ hf_token = None,+ cache_dir = None,+ local_files_only = True,+ device = "cuda:0",+ dtype = "bfloat16",+ resident = resident,+ venv_dir = "/x",+ )+ client._cmd, client._resp = queue.Queue(), queue.Queue()+ client._proc = SimpleNamespace(is_alive = lambda: True, exitcode = None)+ client._ready = True+ sent = []++ def fake_put(msg):+ events.append("put")+ sent.append(msg)+ client._resp.put(+ {+ "type": "embeds",+ "id": msg["id"],+ "embeds": torch.zeros(1, 2, 3),+ "mask": torch.ones(1, 2),+ }+ )++ client._cmd.put = fake_put+ client.encode(["p"], 2)+ client.encode(["p"], 2)+ if resident:+ assert events == ["put"] and sent[0]["server_reserved_mib"] == 9000+ else:+ assert events == ["empty_cache", "put"] and sent[0]["server_reserved_mib"] == 6000+++def test_release_cached_device_memory_skips_hosts_without_a_card(monkeypatch):+ import torch++ calls = []+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda: calls.append("cuda"))+ for device in ("cpu", "mps", None, ""):+ v25._release_cached_device_memory(device)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ v25._release_cached_device_memory("cuda:0")+ assert calls == []+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ v25._release_cached_device_memory("cuda:0")+ assert calls == ["cuda"]++# ── the encoder worker spawn consults the shutdown latch ─────────────────────class _FakeEncoderProc:def __init__(self, *args, **kwargs):