Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix the three Backend CI reds LTX-2.5 left on main (#13496)

Daniel Han62d2852eff11
diff --git a/studio/backend/core/inference/video_ltx25.py b/studio/backend/core/inference/video_ltx25.py
index 1866c06e0cd..050d831941d 100644
--- a/studio/backend/core/inference/video_ltx25.py
+++ b/studio/backend/core/inference/video_ltx25.py
@@ -767,6 +767,13 @@ class Ltx25TextEncoderClient:
run_without_native_path_secret,
)
+ from utils.process_lifetime import is_process_shutting_down
+
+ # A quit during an LTX-2.5 load must not start an encoder worker the shutdown sweep has already passed.
+ if is_process_shutting_down():
+ raise TextEncoderWorkerError(
+ "Unsloth is shutting down; not starting the LTX-2.5 text encoder worker."
+ )
ctx = mp.get_context("spawn")
cache_env = get_hf_cache_paths().child_env({})
with child_environment_for_spawn(cache_env), native_path_secret_removed_for_child_start():
@@ -790,6 +797,24 @@ class Ltx25TextEncoderClient:
adopt_pid(proc.pid)
except Exception: # noqa: BLE001 -- the daemon flag still ties it to this process
pass
+ # Recheck once the pid is recorded: the latch can be set between the gate above and the record.
+ if is_process_shutting_down():
+ logger.info(
+ "video.ltx25_te: shutdown began during the spawn; reaping the encoder worker"
+ )
+ self._proc = None
+ try:
+ if proc.is_alive():
+ proc.terminate()
+ proc.join(5)
+ if proc.is_alive():
+ proc.kill()
+ proc.join(5)
+ except Exception: # noqa: BLE001 -- the reap is best-effort
+ pass
+ raise TextEncoderWorkerError(
+ "Unsloth is shutting down; stopped the LTX-2.5 text encoder worker it had just started."
+ )
logger.info("video.ltx25_te: started the Gemma 4 encoder worker (pid %s)", proc.pid)
def alive(self) -> bool:
diff --git a/studio/backend/tests/fixtures/vae_tile_configs.json b/studio/backend/tests/fixtures/vae_tile_configs.json
index 9336ed64a24..ef2d0c7f4f8 100644
--- a/studio/backend/tests/fixtures/vae_tile_configs.json
+++ b/studio/backend/tests/fixtures/vae_tile_configs.json
@@ -928,6 +928,104 @@
]
}
},
+ "ltx-2.5": {
+ "kind": "video",
+ "repo": "Lightricks/LTX-2.5-Diffusers",
+ "revision": "da0b690960026eab794b8ceaf841233cf7064478",
+ "config": {
+ "_class_name": "AutoencoderKLLTX2Video",
+ "_diffusers_version": "0.40.0.dev0",
+ "block_out_channels": [
+ 256,
+ 512,
+ 1024,
+ 1024
+ ],
+ "decoder_block_out_channels": [
+ 256,
+ 512,
+ 512,
+ 1024
+ ],
+ "decoder_causal": false,
+ "decoder_inject_noise": [
+ false,
+ false,
+ false,
+ false,
+ false
+ ],
+ "decoder_layers_per_block": [
+ 4,
+ 6,
+ 4,
+ 2,
+ 2
+ ],
+ "decoder_spatial_padding_mode": "zeros",
+ "decoder_spatio_temporal_scaling": [
+ true,
+ true,
+ true,
+ true
+ ],
+ "down_block_types": [
+ "LTX2VideoDownBlock3D",
+ "LTX2VideoDownBlock3D",
+ "LTX2VideoDownBlock3D",
+ "LTX2VideoDownBlock3D"
+ ],
+ "downsample_type": [
+ "spatial",
+ "temporal",
+ "spatiotemporal",
+ "spatiotemporal"
+ ],
+ "encoder_causal": true,
+ "encoder_spatial_padding_mode": "zeros",
+ "in_channels": 3,
+ "latent_channels": 128,
+ "layers_per_block": [
+ 4,
+ 6,
+ 4,
+ 2,
+ 2
+ ],
+ "out_channels": 3,
+ "patch_size": 4,
+ "patch_size_t": 1,
+ "resnet_norm_eps": 1e-06,
+ "scaling_factor": 1.0,
+ "spatial_compression_ratio": 32,
+ "spatio_temporal_scaling": [
+ true,
+ true,
+ true,
+ true
+ ],
+ "temporal_compression_ratio": 8,
+ "timestep_conditioning": false,
+ "upsample_factor": [
+ 2,
+ 2,
+ 1,
+ 2
+ ],
+ "upsample_residual": [
+ false,
+ false,
+ false,
+ false
+ ],
+ "upsample_type": [
+ "spatiotemporal",
+ "spatiotemporal",
+ "temporal",
+ "spatial"
+ ]
+ }
+ },
"ltx-2": {
"kind": "video",
"repo": "Lightricks/LTX-2",
diff --git a/studio/backend/tests/test_orchestrator_unload_cancel.py b/studio/backend/tests/test_orchestrator_unload_cancel.py
index 75f44e5fcf7..6fc4e7890e5 100644
--- a/studio/backend/tests/test_orchestrator_unload_cancel.py
+++ b/studio/backend/tests/test_orchestrator_unload_cancel.py
@@ -3881,6 +3881,7 @@ def test_every_long_lived_spawner_consults_the_shutdown_latch():
"core/inference/stt_ggml_sidecar.py",
"core/inference/stt_mtmd_sidecar.py",
"core/inference/stt_transformers_worker.py",
+ "core/inference/video_ltx25.py",
"core/rag/embed_llama_server.py",
"core/systemone/native_worker.py",
"core/training/training.py",
diff --git a/studio/backend/tests/test_video_ltx25.py b/studio/backend/tests/test_video_ltx25.py
index 8b52c3c9a30..81df117f75c 100644
--- a/studio/backend/tests/test_video_ltx25.py
+++ b/studio/backend/tests/test_video_ltx25.py
@@ -1037,6 +1037,7 @@ def test_an_abandoned_encodes_error_does_not_fail_the_next_one():
def test_ancestral_step_matches_diffusers_rf_reference():
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
import torch
from diffusers import FlowMatchEulerDiscreteScheduler
from diffusers.schedulers.scheduling_ltx_euler_ancestral_rf import LTXEulerAncestralRFScheduler
@@ -1062,6 +1063,7 @@ def test_ancestral_step_matches_diffusers_rf_reference():
def test_keep_tokens_step_deterministically_and_copies_share_the_generator():
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
import torch
from diffusers import FlowMatchEulerDiscreteScheduler
@@ -1081,6 +1083,7 @@ def test_keep_tokens_step_deterministically_and_copies_share_the_generator():
def test_prepare_scheduler_per_variant(monkeypatch):
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
from diffusers import FlowMatchEulerDiscreteScheduler
pipe = SimpleNamespace(scheduler = FlowMatchEulerDiscreteScheduler(use_dynamic_shifting = False))
@@ -1217,6 +1220,7 @@ def test_two_stage_t2v_follows_the_reference_recipe():
def test_two_stage_i2v_conditions_both_stages(monkeypatch):
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
pipe = _FakePipe(SimpleNamespace(keep_tokens = None))
cond = _FakePipe(pipe.scheduler)
pipe._unsloth_ltx25_condition_pipe = cond
@@ -1233,6 +1237,7 @@ def test_two_stage_i2v_conditions_both_stages(monkeypatch):
def test_every_image_path_resets_the_offload_hooks(monkeypatch):
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
import threading
image = object()
@@ -1400,6 +1405,7 @@ def test_real_ltx25_header_converts_onto_the_ltx25_transformer(key):
shape; range-read headers) lands on a parameter of
the same shape after load_ltx25_transformer's key handling (2.3 pre-rename + diffusers' converter), with nothing
missing; the connector stacks in the file are dropped (they come from the pack)."""
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
import torch
from accelerate import init_empty_weights
from diffusers import LTX2VideoTransformer3DModel
@@ -1563,6 +1569,7 @@ def test_embedded_transformer_config_matches_the_tested_layout():
def test_single_file_assembly_refuses_quantized_connectors(tmp_path):
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
import torch
from safetensors.torch import save_file
@@ -2015,6 +2022,7 @@ def _ltx25_pack_plan(monkeypatch, total_gib):
def test_plan_counts_the_encoder_worker_per_card(
monkeypatch, total_gib, policy, stream_te, stream_dit, resident
):
+ pytest.importorskip("diffusers") # Backend CI has no diffusers; this path runs on it
from core.inference.video_ltx25 import LTX25_WORKER_RESIDENT_GB
plan, inputs, placed_resident = _ltx25_pack_plan(monkeypatch, total_gib)
@@ -2117,3 +2125,89 @@ def test_client_sends_the_budget_and_the_servers_share():
)
assert client.config["budget_mib"] == 43622
assert v25._server_reserved_mib("cpu") is None
+
+
+# ── the encoder worker spawn consults the shutdown latch ─────────────────────
+class _FakeEncoderProc:
+ def __init__(self, *args, **kwargs):
+ self.pid = 424242
+ self.started = False
+ self.alive = False
+ self.terminated = False
+
+ def start(self):
+ self.started = True
+ self.alive = True
+
+ def is_alive(self):
+ return self.alive
+
+ def terminate(self):
+ self.terminated = True
+ self.alive = False
+
+ def join(self, timeout = None):
+ pass
+
+ def kill(self):
+ self.alive = False
+
+
+def _gated_encoder_client(monkeypatch, latch_answers):
+ import multiprocessing as mp
+
+ import utils.process_lifetime as lifetime
+
+ answers = iter(latch_answers)
+ spawned: list[_FakeEncoderProc] = []
+ adopted: list[int] = []
+
+ class _Ctx:
+ def Queue(self):
+ return queue.Queue()
+
+ def Process(self, *args, **kwargs):
+ proc = _FakeEncoderProc()
+ spawned.append(proc)
+ return proc
+
+ monkeypatch.setattr(mp, "get_context", lambda method = None: _Ctx())
+ monkeypatch.setattr(lifetime, "is_process_shutting_down", lambda: next(answers))
+ monkeypatch.setattr(lifetime, "adopt_pid", adopted.append)
+ client = v25.Ltx25TextEncoderClient(
+ repo = "Lightricks/LTX-2.5-Diffusers",
+ hf_token = None,
+ cache_dir = None,
+ local_files_only = True,
+ device = "cpu",
+ dtype = "bfloat16",
+ resident = False,
+ venv_dir = "/nonexistent",
+ )
+ return client, spawned, adopted
+
+
+def test_encoder_worker_is_not_started_once_shutdown_has_begun(monkeypatch):
+ client, spawned, adopted = _gated_encoder_client(monkeypatch, [True])
+ with pytest.raises(v25.TextEncoderWorkerError, match = "shutting down"):
+ client.start()
+ assert spawned == [] and adopted == []
+ assert not client.alive()
+
+
+def test_encoder_worker_started_as_shutdown_begins_is_reaped(monkeypatch):
+ client, spawned, adopted = _gated_encoder_client(monkeypatch, [False, True])
+ with pytest.raises(v25.TextEncoderWorkerError, match = "shutting down"):
+ client.start()
+ (proc,) = spawned
+ # Adopted first, so the sweep still holds it, then reaped here.
+ assert adopted == [proc.pid]
+ assert proc.started and proc.terminated and not proc.is_alive()
+ assert not client.alive()
+
+
+def test_encoder_worker_starts_while_the_latch_is_clear(monkeypatch):
+ client, spawned, adopted = _gated_encoder_client(monkeypatch, [False, False])
+ client.start()
+ (proc,) = spawned
+ assert adopted == [proc.pid] and client.alive() and not proc.terminated