Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: install the pinned Diffusers build before the backend imports what it replaces (#11635)
oobabooga94ca6f4a62aa
diff --git a/studio/backend/core/inference/diffusion_families.py b/studio/backend/core/inference/diffusion_families.pyindex bd3ff98f0ad..66d1ec64f3d 100644--- a/studio/backend/core/inference/diffusion_families.py+++ b/studio/backend/core/inference/diffusion_families.py@@ -1380,17 +1380,6 @@ def _too_old_message(pipeline_class: str, family_name: str, installed: str) -> sf"diffusers {installed}. Upgrade with: pip install -U diffusers.")if minimum in _UNRELEASED_MIN_DIFFUSERS:- try:- from utils.diffusers_repair import diffusers_repair_installed- installed_since = diffusers_repair_installed()- except Exception: # noqa: BLE001 - no self-heal module is no repair- installed_since = False- if installed_since:- return (- f"'{family_name}' needs diffusers >= {minimum} ({pipeline_class}). Unsloth has just "- f"installed it, but this session already loaded diffusers {installed}. Restart "- "Unsloth Studio to use it."- )remedy = _diffusers_main_archive_remedy()return (f"'{family_name}' needs diffusers >= {minimum} ({pipeline_class}), which has not been "@@ -1475,16 +1464,6 @@ def assert_pipeline_class_available(except Exception: # noqa: BLE001, S110 - optimisation only, and this module has no loggerpass- if "diffusers" not in sys.modules:- try:- from utils.diffusers_repair import IN_FLIGHT_MESSAGE, diffusers_repair_in_flight- repairing = diffusers_repair_in_flight()- except Exception: # noqa: BLE001 - no self-heal module is no repair- repairing = False- # Importing now would read files the install is replacing, and pin the release for the session.- if repairing:- raise ValueError(IN_FLIGHT_MESSAGE)-try:import diffuserspresent = hasattr(diffusers, pipeline_class)diff --git a/studio/backend/main.py b/studio/backend/main.pyindex c515d60e150..997a7991583 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -662,11 +662,7 @@ def _post_warm_background_work(generation: Optional[int] = None) -> None:if _post_warm_retired(generation):returntry:- from utils.diffusers_repair import diffusers_repair_in_flight-- # Importing the release now would pin it in this process for the whole session.- if not diffusers_repair_in_flight():- prewarm_diffusers_if_image_models_exist()+ prewarm_diffusers_if_image_models_exist()except Exception as _prewarm_exc: # noqa: BLE001 -- latency work must never end the workerimport structlog as _structlog_structlog.get_logger(__name__).debug("diffusers prewarm skipped: %s", _prewarm_exc)@@ -882,14 +878,6 @@ async def lifespan(app: FastAPI):+ "\n")- # Before the socket binds, or a first diffusion load can import the release the repair is replacing.- # A metadata read and a thread start; the install itself runs on that thread.- try:- from utils.diffusers_repair import start_diffusers_autorepair_if_needed- start_diffusers_autorepair_if_needed()- except Exception as _diffusers_exc: # noqa: BLE001 -- a self-heal must never block startup- _lifespan_log.warning("diffusers autorepair skipped: %s", _diffusers_exc)-# Last, so it never contends for the GIL: the socket binds as soon as this returns, so the login# screen is up while torch/transformers/datasets load.start_background_warm()diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex 737184b51ab..f088df8297f 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -14230,11 +14230,6 @@ async def _preflight_native_audio_placement("Load a merged checkpoint instead."),)- if audio_type == "minimax_music3":- # Its worker imports diffusers, and cannot see this process's repair.- from utils.diffusers_repair import IN_FLIGHT_MESSAGE, diffusers_repair_in_flight- if diffusers_repair_in_flight():- raise HTTPException(status_code = 400, detail = IN_FLIGHT_MESSAGE)if audio_type in ("higgs_tts2", "higgs_tts3") and sys.version_info < (3, 10):raise HTTPException(status_code = 400,diff --git a/studio/backend/run.py b/studio/backend/run.pyindex e5a3136cbfe..852fe698397 100644--- a/studio/backend/run.py+++ b/studio/backend/run.py@@ -2310,6 +2310,27 @@ def _drops_its_marker_on_failure(start):return started+def _repair_pinned_diffusers(silent: bool) -> None:+ """Repair before importing the app; exit if packages may still be half replaced at timeout."""+ echo = (lambda _line: None) if silent else (lambda line: print(line, flush = True))+ try:+ from utils.diffusers_repair import (+ InstallInterrupted,+ PeerInstallInProgress,+ repair_diffusers_before_imports,+ )+ except Exception as exc: # noqa: BLE001 -- a self-heal must never block startup+ echo(f" - diffusers self-heal skipped: {exc}")+ return+ try:+ repair_diffusers_before_imports(echo)+ except (PeerInstallInProgress, InstallInterrupted) as exc:+ print(f"Error: {exc}", file = sys.stderr, flush = True)+ sys.exit(1)+ except Exception as exc: # noqa: BLE001 -- a self-heal must never block startup+ echo(f" - diffusers self-heal skipped: {exc}")++@_drops_its_marker_on_failuredef run_server(host: str = "127.0.0.1",@@ -2451,6 +2472,10 @@ def run_server("Loading Unsloth Studio, please wait... (this can take a few minutes)",flush = True,)++ _repair_pinned_diffusers(silent)++ if not silent:print(" - loading PyTorch, Unsloth and Transformers...", flush = True)import_started = time.perf_counter()diff --git a/studio/backend/tests/test_diffusers_repair.py b/studio/backend/tests/test_diffusers_repair.pyindex 50817d1ffdd..21da8c65b75 100644--- a/studio/backend/tests/test_diffusers_repair.py+++ b/studio/backend/tests/test_diffusers_repair.py@@ -1,15 +1,14 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0-"""Diffusers self-heal: an install updated by an installer without the Diffusers main step gets the-pinned build from the backend on its next start, instead of needing a second update."""+"""Test startup repair of the pinned Diffusers build before app imports."""from __future__ import annotationsimport jsonimport os+import subprocessimport sys-import threadingimport timefrom pathlib import Path@@ -22,6 +21,8 @@ if str(_BACKEND) not in sys.path:import utils.diffusers_repair as dr # noqa: E402_REAL_PEER_HOLDS_PASS = dr._peer_holds_pass+_REAL_INSTALLER_WOULD_SKIP = dr._installer_would_skip+_REAL_LOADED_REPLACEABLE = dr._loaded_replaceable_modulesclass _Dist:@@ -36,11 +37,11 @@ class _Dist:@pytest.fixture(autouse = True)def _reset(monkeypatch):- monkeypatch.setattr(dr, "_thread", None)- monkeypatch.setattr(dr, "_installed", False)monkeypatch.delenv(dr.DISABLE_ENV_VAR, raising = False)monkeypatch.delenv("UNSLOTH_DIFFUSERS_MAIN", raising = False)monkeypatch.setattr(dr, "_peer_holds_pass", lambda: False)+ monkeypatch.setattr(dr, "_installer_would_skip", lambda: False)+ monkeypatch.setattr(dr, "_loaded_replaceable_modules", lambda: [])def _installed_diffusers(monkeypatch, direct_url):@@ -69,60 +70,53 @@ def test_only_an_index_release_is_a_candidate(monkeypatch, direct_url, expected)class _Proc:"""A finished installer, for the Popen the repair starts."""- def __init__(- self,- returncode,- wait = None,- ):- self.pid, self.returncode, self._wait = 0, returncode, wait+ def __init__(self, returncode):+ self.pid, self.returncode = 0, returncodedef communicate(self, timeout = None):- if self._wait is not None:- self._wait.wait(5)- return "", None+ return "installer output", Nonedef poll(self):return self.returncode-def test_a_release_install_starts_one_repair_and_records_success(monkeypatch):+def test_a_release_install_runs_the_installer_step_and_reports_it(monkeypatch):_installed_diffusers(monkeypatch, None)- calls = []- release = threading.Event()+ calls, lines = [], []def fake_popen(argv, **kwargs):calls.append((argv, kwargs["env"]))- return _Proc(dr._INSTALLED, wait = release)+ return _Proc(dr._INSTALLED)monkeypatch.setattr(dr.subprocess, "Popen", fake_popen)- assert dr.start_diffusers_autorepair_if_needed() is True- assert dr.diffusers_repair_in_flight() is True- assert dr.start_diffusers_autorepair_if_needed() is False, "at most once per process"- release.set()- dr._thread.join(5)- assert dr.diffusers_repair_in_flight() is False- assert dr.diffusers_repair_installed() is True- argv, env = calls[0]- assert argv == [sys.executable, str(dr._INSTALLER), "--repair-diffusers-main"]- assert env["VIRTUAL_ENV"] == sys.prefix+ assert dr.repair_diffusers_before_imports(lines.append) is True+ # The slow fetch first, into uv's cache only, then the install from it.+ assert [argv for argv, _env in calls] == [+ [sys.executable, str(dr._INSTALLER), "--prefetch-diffusers-main"],+ [sys.executable, str(dr._INSTALLER), "--repair-diffusers-main"],+ ]+ assert all(env["VIRTUAL_ENV"] == sys.prefix for _argv, env in calls)+ assert lines == [+ " - installing the pinned Diffusers build (first start after an update)...",+ " - installed the pinned Diffusers build",+ ]def test_nothing_to_do_and_failure_do_not_report_an_install(monkeypatch):_installed_diffusers(monkeypatch, None)- for code in (dr._NOTHING_TO_DO, 2):- monkeypatch.setattr(dr, "_thread", None)+ for code, says_retry in ((dr._NOTHING_TO_DO, False), (2, True)):+ lines = []monkeypatch.setattr(dr.subprocess, "Popen", lambda argv, code = code, **kw: _Proc(code))- assert dr.start_diffusers_autorepair_if_needed() is True- dr._thread.join(5)- assert dr.diffusers_repair_installed() is False+ assert dr.repair_diffusers_before_imports(lines.append) is False+ assert any("unsloth studio update" in line for line in lines) is says_retry[email protected](sys.platform == "win32", reason = "POSIX process tree")-def test_a_timed_out_repair_stops_the_installers_children_too(monkeypatch, tmp_path):- """uv, not the installer, rewrites diffusers, so killing only the installer on timeout would- reopen the load gate while the files are still being replaced."""- from utils.process_lifetime import _pid_alive, _pid_is_zombie-+def _slow_installer(+ monkeypatch,+ tmp_path,+ timeout_s = 3,+):+ """An installer that outlives the budget, with a child standing in for uv."""pid_file = tmp_path / "child.pid"installer = tmp_path / "installer.py"installer.write_text(@@ -133,20 +127,162 @@ def test_a_timed_out_repair_stops_the_installers_children_too(monkeypatch, tmp_pencoding = "utf-8",)monkeypatch.setattr(dr, "_INSTALLER", installer)- monkeypatch.setattr(dr, "_REPAIR_TIMEOUT_S", 3)- waited = []- monkeypatch.setattr(dr, "_wait_for_peer_pass", lambda: waited.append(True))- dr._run_repair()- assert waited == [True], "a timed-out repair must hold the gate for a peer still in the pass"+ monkeypatch.setattr(dr, "_REPAIR_TIMEOUT_S", timeout_s)+ monkeypatch.setattr(dr, "_INSTALL_MIN_TIMEOUT_S", timeout_s)+ # The tree kill names descendants before it signals, so a child forked in between is missed.+ # A slow runner's installer can still be starting at the deadline: let it spawn first.+ import utils.process_lifetime as pl++ real_terminate = pl.terminate_pid++ def terminate_after_spawn(pid, *args, **kwargs):+ for _ in range(300):+ if pid_file.exists() and pid_file.read_text():+ break+ time.sleep(0.1)+ return real_terminate(pid, *args, **kwargs)++ monkeypatch.setattr(pl, "terminate_pid", terminate_after_spawn)+ started = []+ real_popen = dr.subprocess.Popen++ def popen(argv, *args, **kwargs):+ # Global: on macOS the tree kill runs `ps` through it too.+ if str(installer) in argv:+ started.append(argv[-1])+ return real_popen(argv, *args, **kwargs)++ monkeypatch.setattr(dr.subprocess, "Popen", popen)+ return pid_file, started+++def _assert_child_stopped(pid_file):+ from utils.process_lifetime import _pid_alive, _pid_is_zombie+child = int(pid_file.read_text())for _ in range(50):if not _pid_alive(child) or _pid_is_zombie(child):- break+ returntime.sleep(0.1)- else:- os.kill(child, 9)- pytest.fail("the installer's child outlived the timed-out repair")- assert dr.diffusers_repair_installed() is False+ os.kill(child, 9)+ pytest.fail("the installer's child outlived the timed-out repair")++[email protected](sys.platform == "win32", reason = "POSIX process tree")+def test_a_timed_out_repair_stops_the_installers_children_and_records_it(monkeypatch, tmp_path):+ """Stop uv children before app imports and record the timeout to prevent repeated retries."""+ pid_file, started = _slow_installer(monkeypatch, tmp_path)+ recorded = []+ monkeypatch.setattr(dr, "_record_failure", lambda: recorded.append(True))+ lines = []+ assert dr._run_repair(lines.append) is False+ assert recorded == [True]+ assert "unsloth studio update" in lines[-1]+ assert started == ["--prefetch-diffusers-main"], "a slow fetch must never reach the install"+ _assert_child_stopped(pid_file)++[email protected](sys.platform == "win32", reason = "POSIX process tree")+def test_a_peer_that_started_during_a_slow_prefetch_stops_startup(monkeypatch, tmp_path):+ """The prefetch holds no lock, so a pass can begin under it; importing then mixes versions."""+ pid_file, started = _slow_installer(monkeypatch, tmp_path, timeout_s = 2)+ recorded = []+ monkeypatch.setattr(dr, "_record_failure", lambda: recorded.append(True))+ monkeypatch.setattr(dr, "_peer_holds_pass", lambda: True)+ with pytest.raises(dr.PeerInstallInProgress, match = "Start Unsloth Studio again"):+ dr._run_repair(lambda _line: None)+ assert started == ["--prefetch-diffusers-main"]+ assert recorded == [], "the peer's manifest is not ours to mark"+ _assert_child_stopped(pid_file)++[email protected](sys.platform == "win32", reason = "POSIX process tree")+def test_our_own_install_stopped_at_the_deadline_stops_startup(monkeypatch, tmp_path):+ """Stopped mid-install, packages may be half replaced: never import them, never record."""+ pid_file, started = _slow_installer(monkeypatch, tmp_path, timeout_s = 2)+ recorded = []+ monkeypatch.setattr(dr, "_record_failure", lambda: recorded.append(True))+ with pytest.raises(dr.InstallInterrupted, match = "Start Unsloth Studio again"):+ dr._run_repair(lambda _line: None, prefetch = False)+ assert started == ["--repair-diffusers-main"]+ assert recorded == [], "the next start has to retry, not skip, a half-finished install"+ _assert_child_stopped(pid_file)++[email protected](sys.platform == "win32", reason = "POSIX process tree")+def test_a_peer_still_in_the_pass_at_the_deadline_stops_startup_in_time(monkeypatch, tmp_path):+ """Abort at timeout if a peer is still installing, without changing its manifest."""+ pid_file, _started = _slow_installer(monkeypatch, tmp_path, timeout_s = 2)+ recorded = []+ monkeypatch.setattr(dr, "_record_failure", lambda: recorded.append(True))+ monkeypatch.setattr(dr, "_peer_holds_pass", lambda: True)+ started = time.monotonic()+ with pytest.raises(dr.PeerInstallInProgress, match = "Start Unsloth Studio again"):+ dr._run_repair(lambda _line: None, prefetch = False)+ assert time.monotonic() - started < 2 + 40+ assert recorded == []+ _assert_child_stopped(pid_file)+++ "error, message",+ [+ (dr.PeerInstallInProgress, dr.PEER_INSTALL_MESSAGE),+ (dr.InstallInterrupted, dr.INTERRUPTED_MESSAGE),+ ],+)+def test_run_server_exits_with_the_message_when_the_environment_is_unsafe(+ monkeypatch, capsys, error, message+):+ import run++ def blocked(echo):+ raise error(message)++ monkeypatch.setattr(dr, "repair_diffusers_before_imports", blocked)+ with pytest.raises(SystemExit) as excinfo:+ run._repair_pinned_diffusers(silent = True)+ assert excinfo.value.code == 1+ assert message in capsys.readouterr().err, "shown even under --silent"++[email protected]("loaded", ["huggingface_hub", "diffusers"])+def test_an_embedding_host_that_imported_the_packages_is_not_repaired_under_them(+ monkeypatch, loaded+):+ """A notebook kernel calling run_server() after importing them would mix old and new files."""+ _installed_diffusers(monkeypatch, None)+ monkeypatch.setattr(dr, "_loaded_replaceable_modules", _REAL_LOADED_REPLACEABLE)+ for name in ("huggingface_hub", "diffusers"):+ monkeypatch.delitem(sys.modules, name, raising = False)+ monkeypatch.setitem(sys.modules, loaded, object())+ monkeypatch.setattr(dr.subprocess, "Popen", lambda *a, **k: pytest.fail("started a repair"))+ lines = []+ assert dr.repair_diffusers_before_imports(lines.append) is False+ assert len(lines) == 1 and loaded in lines[0] and "unsloth studio update" in lines[0]+++def test_any_other_repair_error_lets_startup_continue(monkeypatch, capsys):+ import run++ def broken(echo):+ raise OSError("disk full")++ monkeypatch.setattr(dr, "repair_diffusers_before_imports", broken)+ run._repair_pinned_diffusers(silent = False)+ assert "diffusers self-heal skipped: disk full" in capsys.readouterr().out+++def test_a_recorded_failure_uses_the_key_the_installer_reads(monkeypatch):+ import types++ recorded = {}+ fake = types.SimpleNamespace(update_manifest = lambda **extra: recorded.update(extra))+ monkeypatch.setitem(sys.modules, "studio.install_manifest", fake)+ dr._record_failure()+ source = dr._INSTALLER.read_text(encoding = "utf-8")+ assert recorded == {"diffusers_main_repair": "failed"}+ assert '_DIFFUSERS_MAIN_REPAIR_KEY = "diffusers_main_repair"' in sourcedef _peer_pass(monkeypatch, *uncontended):@@ -159,27 +295,60 @@ def _peer_pass(monkeypatch, *uncontended):monkeypatch.setitem(sys.modules, "studio.install_manifest", fake)-def test_the_gate_waits_for_a_peer_to_leave_the_pass(monkeypatch):- """The repair's installer may have been waiting behind a sibling's repair or an update when it- timed out; that peer can still be rewriting diffusers."""- _peer_pass(monkeypatch, False, False, True)- polls = []- monkeypatch.setattr(dr.time, "sleep", lambda seconds: polls.append(seconds))- dr._wait_for_peer_pass()- assert polls == [dr._PEER_POLL_S] * 2--def test_a_start_during_a_peers_pass_waits_it_out(monkeypatch):"""A peer swapping diffusers can have removed its metadata, which reads as no index release."""_peer_pass(monkeypatch, False)_installed_diffusers(monkeypatch, {"url": "https://github.com/huggingface/diffusers", "vcs_info": {}})- started = []+ started, lines = [], []monkeypatch.setattr(dr.subprocess, "Popen", lambda argv, **kw: started.append(argv) or _Proc(1))- assert dr.start_diffusers_autorepair_if_needed() is True- dr._thread.join(5)+ assert dr.repair_diffusers_before_imports(lines.append) is Falseassert started and started[0][-1] == "--repair-diffusers-main"+ assert lines == [" - waiting for another Unsloth install or update to finish..."]+++def _manifest(monkeypatch, manifest):+ import types++ monkeypatch.setattr(dr, "_installer_would_skip", _REAL_INSTALLER_WOULD_SKIP)+ fake = types.SimpleNamespace(read_manifest = lambda: manifest)+ monkeypatch.setitem(sys.modules, "studio.install_manifest", fake)+++ "min_python, manifest",+ [+ ((99, 0), {}), # Python 3.9 against the real (3, 10) floor+ ((3, 0), {"diffusers_main_repair": "failed"}),+ ((3, 0), {"step_results": {"diffusers-main.txt": "failed"}}),+ ],+)+def test_a_start_the_installer_would_skip_spawns_nothing(monkeypatch, min_python, manifest):+ """Every start would otherwise block on a no-op installer and claim it is installing."""+ _installed_diffusers(monkeypatch, None)+ _manifest(monkeypatch, manifest)+ monkeypatch.setattr(dr, "_MAIN_MIN_PYTHON", min_python)+ monkeypatch.setattr(dr.subprocess, "Popen", lambda *a, **k: pytest.fail("started a repair"))+ lines = []+ assert dr.repair_diffusers_before_imports(lines.append) is False+ assert lines == []+++def test_a_skipped_start_still_waits_out_a_peers_pass(monkeypatch):+ _installed_diffusers(monkeypatch, None)+ _manifest(monkeypatch, {"diffusers_main_repair": "failed"})+ monkeypatch.setattr(dr, "_peer_holds_pass", lambda: True)+ started, lines = [], []+ monkeypatch.setattr(dr.subprocess, "Popen", lambda argv, **kw: started.append(argv) or _Proc(1))+ assert dr.repair_diffusers_before_imports(lines.append) is False+ assert started+ assert lines == [" - waiting for another Unsloth install or update to finish..."]+++def test_the_python_floor_matches_the_installer():+ source = dr._INSTALLER.read_text(encoding = "utf-8")+ assert f"DIFFUSERS_MAIN_MIN_PYTHON = {dr._MAIN_MIN_PYTHON!r}" in source@pytest.mark.parametrize(@@ -191,7 +360,7 @@ def test_opt_outs_start_nothing(monkeypatch, env):for key, value in env.items():monkeypatch.setenv(key, value)monkeypatch.setattr(dr.subprocess, "Popen", lambda *a, **k: pytest.fail("started a repair"))- assert dr.start_diffusers_autorepair_if_needed() is False+ assert dr.repair_diffusers_before_imports() is Falsedef test_the_pinned_build_starts_nothing(monkeypatch):@@ -199,7 +368,7 @@ def test_the_pinned_build_starts_nothing(monkeypatch):monkeypatch, {"url": "https://github.com/huggingface/diffusers", "vcs_info": {}})monkeypatch.setattr(dr.subprocess, "Popen", lambda *a, **k: pytest.fail("started a repair"))- assert dr.start_diffusers_autorepair_if_needed() is False+ assert dr.repair_diffusers_before_imports() is Falsedef test_secrets_and_index_redirects_stay_out_of_the_installer_env(monkeypatch):@@ -213,78 +382,58 @@ def test_secrets_and_index_redirects_stay_out_of_the_installer_env(monkeypatch):assert env["UV_OFFLINE"] == "1", "an offline install must not reach GitHub from the repair"-def test_the_load_gate_waits_for_a_running_repair(monkeypatch):- """Importing diffusers mid-install reads half-replaced files and pins the release for the- session, so a load during the repair is refused with a retry message instead."""- from core.inference import diffusion_families as fam-- monkeypatch.setattr(dr, "diffusers_repair_in_flight", lambda: True)- monkeypatch.delitem(sys.modules, "diffusers", raising = False)- with pytest.raises(ValueError, match = "installing the pinned diffusers build"):- fam.assert_pipeline_class_available("QwenImage21Pipeline", "qwen-image-2.1")---def test_minimax_music3_is_refused_before_eviction_while_the_repair_runs(monkeypatch):- """Its worker is a separate process that imports diffusers and never sees this one's repair."""- import asyncio- import types-- from fastapi import HTTPException-- import routes.inference as ri-- monkeypatch.setattr(dr, "diffusers_repair_in_flight", lambda: True)- config = types.SimpleNamespace(audio_type = "minimax_music3", is_lora = False, identifier = "x/y")- with pytest.raises(HTTPException) as excinfo:- asyncio.run(- ri._preflight_native_audio_placement(- config,- types.SimpleNamespace(audio_device = None),- types.SimpleNamespace(requested_gpu_ids = None),- )- )- assert excinfo.value.status_code == 400- assert excinfo.value.detail == dr.IN_FLIGHT_MESSAGE---def test_the_repair_is_decided_before_the_socket_binds():- """The lifespan yields, and the server accepts requests, before the post-warm thread runs, so a- repair started there leaves a window where a load imports the release being replaced. Checked on- the source because running the real lifespan brings up the whole backend."""+def test_run_server_repairs_before_it_imports_the_app():+ """Check import ordering in the AST to avoid starting the full backend."""import ast- tree = ast.parse((_BACKEND / "main.py").read_text(encoding = "utf-8"))- functions = {- node.name: node+ tree = ast.parse((_BACKEND / "run.py").read_text(encoding = "utf-8"))+ run_server = next(+ nodefor node in ast.walk(tree)- if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))- }-- def first_call(function, name):- lines = [- node.lineno- for node in ast.walk(function)- if isinstance(node, ast.Call) and getattr(node.func, "id", None) == name- ]- return min(lines) if lines else None-- lifespan = functions["lifespan"]- start = first_call(lifespan, "start_diffusers_autorepair_if_needed")- assert start is not None- assert start < first_call(lifespan, "_start_post_warm_thread")- assert start < min(n.lineno for n in ast.walk(lifespan) if isinstance(n, ast.Yield))- post_warm = functions["_post_warm_background_work"]- assert first_call(post_warm, "start_diffusers_autorepair_if_needed") is None---def test_a_finished_repair_behind_a_loaded_release_asks_for_a_restart(monkeypatch):- from core.inference import diffusion_families as fam-- monkeypatch.setattr(dr, "_installed", True)- message = fam._too_old_message("QwenImage21Pipeline", "qwen-image-2.1", "0.40.0")- assert "Restart Unsloth Studio" in message+ if isinstance(node, ast.FunctionDef) and node.name == "run_server"+ )+ repair = min(+ node.lineno+ for node in ast.walk(run_server)+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None) == "_repair_pinned_diffusers"+ )+ app_import = min(+ node.lineno+ for node in ast.walk(run_server)+ if isinstance(node, ast.ImportFrom) and node.module == "main"+ )+ assert repair < app_import+ main_source = (_BACKEND / "main.py").read_text(encoding = "utf-8")+ assert "diffusers_repair" not in main_source, "the app itself must never install under itself"+++def test_the_repair_path_imports_nothing_it_can_replace():+ """Use a fresh interpreter to exclude imports made by the test suite."""+ code = (+ "import sys\n"+ "sys.argv = ['run.py']\n"+ "import run\n"+ "import utils.diffusers_repair as dr\n"+ "from utils.child_stdio import utf8_child_env\n"+ "from utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid, terminate_pid\n"+ "dr._repair_env()\n"+ "dr._diffusers_is_an_index_install()\n"+ "names = ('huggingface_hub', 'diffusers', 'torch', 'transformers', 'safetensors', 'numpy')\n"+ "print('LOADED=' + ','.join(n for n in names if n in sys.modules))\n"+ )+ result = subprocess.run(+ [sys.executable, "-c", code],+ cwd = _BACKEND,+ capture_output = True,+ text = True,+ timeout = 120,+ )+ assert result.returncode == 0, result.stderr+ assert "LOADED=\n" in result.stdout, result.stdoutdef test_the_installer_exposes_the_repair_flag():source = dr._INSTALLER.read_text(encoding = "utf-8")assert '["--repair-diffusers-main"]' in source+ assert '["--prefetch-diffusers-main"]' in sourcediff --git a/studio/backend/tests/test_orphaned_children.py b/studio/backend/tests/test_orphaned_children.pyindex 17d1476c0f3..6d86bbb4960 100644--- a/studio/backend/tests/test_orphaned_children.py+++ b/studio/backend/tests/test_orphaned_children.py@@ -342,6 +342,20 @@ def test_a_record_written_with_a_process_name_still_matches():assert pl._same_identity("196794665:python3", "196794999") is False+def test_a_macos_child_that_re_execs_is_still_the_same_process(monkeypatch):+ """A framework python re-execs into Python.app, so comm read at adopt time differs later."""+ from utils import process_lifetime as pl++ monkeypatch.setattr(pl, "_is_linux", lambda: False)+ monkeypatch.setattr(pl.sys, "platform", "darwin")+ adopted = "Wed Sep 23 06:37:18 2026 /Library/Frameworks/Python.framework/Versions/3.12/bin/python3"+ later = "Wed Sep 23 06:37:18 2026 /Library/Frameworks/Python.framework/Versions/3.12/Resources/Python.app/Contents/MacOS/Python"+ assert pl._same_identity(adopted, later) is True+ assert pl._same_identity(adopted, "Wed Sep 23 06:37:19 2026 python3") is False+ assert pl._same_identity("garbage", "garbage") is True+ assert pl._same_identity("garbage", "other") is False++def test_a_group_outliving_its_leader_is_still_reaped(tmp_path, monkeypatch):"""The shim can crash while the visual server holds the GPU; the record'spid is then gone and the group is the only handle left."""diff --git a/studio/backend/tests/test_studio_pid_files.py b/studio/backend/tests/test_studio_pid_files.pyindex 84e51bc51e5..80b698eee36 100644--- a/studio/backend/tests/test_studio_pid_files.py+++ b/studio/backend/tests/test_studio_pid_files.py@@ -1106,6 +1106,18 @@ def test_a_startup_that_raises_takes_its_marker_back(tmp_path, monkeypatch):assert run._OWN_STARTUP_MARKERS == []+def test_the_real_run_server_takes_its_marker_back(tmp_path):+ # Exercise the real decorator via run_server's empty-host rejection.+ run.write_startup_marker()+ assert list(tmp_path.glob(run.STARTUP_MARKER_GLOB)) != []++ with pytest.raises(SystemExit):+ run.run_server(host = "")++ assert list(tmp_path.glob(run.STARTUP_MARKER_GLOB)) == []+ assert run._OWN_STARTUP_MARKERS == []++def test_a_startup_that_exits_takes_its_marker_back(tmp_path, monkeypatch):# SystemExit is a BaseException, and it is the one colab.py catches.run.write_startup_marker()diff --git a/studio/backend/utils/diffusers_repair.py b/studio/backend/utils/diffusers_repair.pyindex 1b04c779c6d..b8fd0d04a57 100644--- a/studio/backend/utils/diffusers_repair.py+++ b/studio/backend/utils/diffusers_repair.py@@ -3,11 +3,9 @@"""Startup self-heal for the pinned Diffusers main build.-An update from a release that predates the installer's Diffusers main step runs that release's-installer, so the build never goes in and Qwen-Image-2.1 refuses to load until a second update. The-backend is the first new code such a host runs, so it runs the installer's own step here, on a-background thread (git, or the zip route without git). Opt out with-UNSLOTH_DISABLE_DIFFUSERS_AUTOREPAIR=1.+Older installers can miss the pinned build during an update. Repair before importing the app:+the build also upgrades huggingface_hub, which would otherwise mix new files with cached modules.+Opt out with UNSLOTH_DISABLE_DIFFUSERS_AUTOREPAIR=1."""from __future__ import annotations@@ -16,10 +14,9 @@ import jsonimport osimport subprocessimport sys-import threadingimport timefrom pathlib import Path-from typing import Optional+from typing import Callableimport structlog@@ -29,10 +26,14 @@ DISABLE_ENV_VAR = "UNSLOTH_DISABLE_DIFFUSERS_AUTOREPAIR"_STUDIO_DIR = Path(__file__).resolve().parents[2]_INSTALLER = _STUDIO_DIR / "install_python_stack.py"_MAIN_PIN = _STUDIO_DIR / "backend" / "requirements" / "diffusers-main.txt"-_REPAIR_TIMEOUT_S = 900-_PEER_POLL_S = 5+# Inside the desktop app's 300 s start deadline, with room left for the app import after it.+_REPAIR_TIMEOUT_S = 120+# The install from uv's cache takes about a second, so this only matters behind a slow prefetch.+_INSTALL_MIN_TIMEOUT_S = 30# The installer's exit codes for --repair-diffusers-main._INSTALLED, _NOTHING_TO_DO = 0, 1+# The installer's DIFFUSERS_MAIN_MIN_PYTHON: diffusers main needs 3.10, and 3.9 stays supported.+_MAIN_MIN_PYTHON = (3, 10)# Unattended, so secrets and index redirects stay out, as in mlx_repair. The Windows names are what# Python, git and uv need to start at all there; UV_OFFLINE is the operator's no-network switch.@@ -76,10 +77,6 @@ _ENV_ALLOWLIST = frozenset(})-_lock = threading.Lock()-_thread: Optional[threading.Thread] = None-_installed = False-def _opted_out() -> bool:if os.environ.get(DISABLE_ENV_VAR) == "1":@@ -133,93 +130,172 @@ def _peer_holds_pass() -> bool:return not uncontended-def _wait_for_peer_pass() -> None:- """Hold the gate until no process holds the dependency pass. Killing our installer released it,- but the pass it was waiting behind may still be writing."""- while _peer_holds_pass():- time.sleep(_PEER_POLL_S)+class PeerInstallInProgress(RuntimeError):+ """A peer still holds the install lock at timeout, so importing packages is unsafe."""+++class InstallInterrupted(RuntimeError):+ """Our own install was stopped at its deadline, so the environment may be half replaced."""+++PEER_INSTALL_MESSAGE = (+ "Another Unsloth install or update is still changing this environment after "+ f"{_REPAIR_TIMEOUT_S}s. Start Unsloth Studio again once it has finished."+)+INTERRUPTED_MESSAGE = (+ "Installing the pinned Diffusers build was stopped part way, so this environment may be "+ "half updated. Start Unsloth Studio again to finish it, or run `unsloth studio update`."+)+++def _record_failure() -> None:+ """Suppress startup retries until an explicit update clears the failure key."""+ try:+ from studio.install_manifest import update_manifest+ update_manifest(diffusers_main_repair = "failed")+ except Exception as exc: # noqa: BLE001 - unrecorded means the next start tries again+ logger.warning("diffusers self-heal could not record its failure: %s", exc)+++def _installer_would_skip() -> bool:+ """The installer's no-op gates: a start it would skip must not block or claim to install."""+ if sys.version_info[:2] < _MAIN_MIN_PYTHON:+ return True+ try:+ from studio.install_manifest import read_manifest+ manifest = read_manifest() or {}+ except Exception: # noqa: BLE001 - an unreadable manifest is no record of a failed try+ return False+ step_results = manifest.get("step_results")+ update_failed = (+ isinstance(step_results, dict) and step_results.get("diffusers-main.txt") == "failed"+ )+ return manifest.get("diffusers_main_repair") == "failed" or update_failed++def _run_installer(flag: str, timeout: float) -> "tuple[int | None, str]":+ """Run one installer mode; None as the code when it was stopped at ``timeout``.-def _run_repair() -> None:- global _installed+ Raises OSError when the installer cannot start.+ """from utils.child_stdio import utf8_child_envfrom utils.process_lifetime import adopt_pid, child_popen_kwargs, forget_pid, terminate_pidkwargs = child_popen_kwargs()if os.name == "nt":kwargs["creationflags"] = getattr(subprocess, "CREATE_NO_WINDOW", 0)- try:- proc = subprocess.Popen(- [sys.executable, str(_INSTALLER), "--repair-diffusers-main"],- env = utf8_child_env(_repair_env()),- stdout = subprocess.PIPE,- stderr = subprocess.STDOUT,- text = True,- encoding = "utf-8",- errors = "replace",- **kwargs,- )- except OSError as exc:- logger.warning("diffusers self-heal could not start the installer: %s", exc)- return+ proc = subprocess.Popen(+ [sys.executable, str(_INSTALLER), flag],+ env = utf8_child_env(_repair_env()),+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ text = True,+ encoding = "utf-8",+ errors = "replace",+ **kwargs,+ )# Tracked, so a backend that exits mid-install takes the installer and its uv/git children down.adopt_pid(proc.pid)try:- output, _ = proc.communicate(timeout = _REPAIR_TIMEOUT_S)+ output, _ = proc.communicate(timeout = timeout)except subprocess.TimeoutExpired:- # The whole tree, before the gate reopens: uv keeps rewriting diffusers after its parent dies.terminate_pid(proc.pid, owner_verified = True)proc.kill()proc.wait()- logger.warning("diffusers self-heal timed out after %ss", _REPAIR_TIMEOUT_S)- _wait_for_peer_pass()- return+ return None, ""finally:if proc.poll() is not None:forget_pid(proc.pid)- if proc.returncode == _INSTALLED:- _installed = True+ return proc.returncode, output or ""+++def _run_repair(echo: Callable[[str], None], prefetch: bool = True) -> bool:+ started = time.monotonic()+ if prefetch:+ # The slow part (clone or download, then build) goes into uv's cache only, so stopping it+ # at the deadline leaves site-packages untouched and the app can import as it is.+ try:+ code, output = _run_installer("--prefetch-diffusers-main", _REPAIR_TIMEOUT_S)+ except OSError as exc:+ logger.warning("diffusers self-heal could not start the installer: %s", exc)+ return False+ if code is None:+ # The prefetch takes no lock, so a peer may have started replacing packages meanwhile.+ if _peer_holds_pass():+ logger.warning("diffusers self-heal timed out while a peer's dependency pass ran")+ raise PeerInstallInProgress(PEER_INSTALL_MESSAGE)+ _record_failure()+ echo(+ f" - the pinned Diffusers build took over {_REPAIR_TIMEOUT_S}s to download and "+ "was stopped; run `unsloth studio update` to install it"+ )+ logger.warning("diffusers self-heal timed out after %ss", _REPAIR_TIMEOUT_S)+ return False+ if code not in (0, _NOTHING_TO_DO):+ # The install below is authoritative: it fetches for itself and records its own failure.+ logger.warning("diffusers prefetch failed. Installer output:\n%s", output[-4000:])+ budget = max(_INSTALL_MIN_TIMEOUT_S, _REPAIR_TIMEOUT_S - (time.monotonic() - started))+ try:+ code, output = _run_installer("--repair-diffusers-main", budget)+ except OSError as exc:+ logger.warning("diffusers self-heal could not start the installer: %s", exc)+ return False+ if code is None:+ # A remaining lock belongs to a peer. Abort without changing its manifest.+ if _peer_holds_pass():+ logger.warning("diffusers self-heal timed out behind a peer's dependency pass")+ raise PeerInstallInProgress(PEER_INSTALL_MESSAGE)+ # Ours, and possibly stopped part way through replacing packages: importing them is unsafe.+ # Not recorded as failed, so the next start retries from uv's cache.+ logger.warning("diffusers self-heal install stopped after %.0fs", budget)+ raise InstallInterrupted(INTERRUPTED_MESSAGE)+ if code == _INSTALLED:+ echo(" - installed the pinned Diffusers build")logger.info("diffusers self-heal installed the pinned Diffusers main build")- elif proc.returncode != _NOTHING_TO_DO:+ return True+ if code != _NOTHING_TO_DO:+ echo(+ " - could not install the pinned Diffusers build; run `unsloth studio update` to retry"+ )logger.warning("diffusers self-heal could not install the pinned build; run `unsloth studio ""update` to retry. Installer output:\n%s",- (output or "")[-4000:],+ output[-4000:],)+ return False-def start_diffusers_autorepair_if_needed() -> bool:- """Start the background install when diffusers is an index release and the pin wants main.- True iff a repair thread was started; at most once per process."""- global _thread- if _opted_out() or not _MAIN_PIN.is_file() or not _INSTALLER.is_file():- return False- # A peer mid-install can have removed the metadata, so its pass also starts the installer, which- # waits the pass out and keeps loads refused meanwhile.- if not _diffusers_is_an_index_install() and not _peer_holds_pass():- return False- with _lock:- if _thread is not None:- return False- _thread = threading.Thread(target = _run_repair, daemon = True, name = "diffusers-autorepair")- _thread.start()- logger.info(- "checking for the pinned Diffusers main build in the background. Set %s=1 to disable.",- DISABLE_ENV_VAR,- )- return True---IN_FLIGHT_MESSAGE = (- "Unsloth is installing the pinned diffusers build in the background, which takes a few "- "minutes on the first start after an update. Try again shortly."-)-+def _loaded_replaceable_modules() -> "list[str]":+ """Packages the repair replaces that this process has already imported."""+ return [name for name in ("huggingface_hub", "diffusers") if name in sys.modules]-def diffusers_repair_in_flight() -> bool:- thread = _thread- return thread is not None and thread.is_alive()+def repair_diffusers_before_imports(echo: Callable[[str], None] = lambda _line: None) -> bool:+ """Repair an index install before app imports. Return True if the build was installed.-def diffusers_repair_installed() -> bool:- return _installed+ Raise PeerInstallInProgress if a peer holds the install lock at timeout, and+ InstallInterrupted if our own install had to be stopped.+ """+ if _opted_out() or not _MAIN_PIN.is_file() or not _INSTALLER.is_file():+ return False+ # Check the lock too: an active install may have temporarily removed the metadata.+ if _diffusers_is_an_index_install() and not _installer_would_skip():+ loaded = _loaded_replaceable_modules()+ if loaded:+ # An embedding host (a notebook kernel) imported them already; replacing the files+ # under those modules is what this repair exists to avoid.+ echo(+ f" - skipped the pinned Diffusers build: {', '.join(loaded)} is already imported "+ "in this process; run `unsloth studio update`, then restart it"+ )+ logger.warning("diffusers self-heal skipped: %s already imported", ", ".join(loaded))+ return False+ echo(" - installing the pinned Diffusers build (first start after an update)...")+ prefetch = True+ elif _peer_holds_pass():+ echo(" - waiting for another Unsloth install or update to finish...")+ prefetch = False+ else:+ return False+ logger.info("installing the pinned Diffusers main build. Set %s=1 to disable.", DISABLE_ENV_VAR)+ return _run_repair(echo, prefetch = prefetch)diff --git a/studio/backend/utils/process_lifetime.py b/studio/backend/utils/process_lifetime.pyindex 13af06ddfa2..00d447f422e 100644--- a/studio/backend/utils/process_lifetime.py+++ b/studio/backend/utils/process_lifetime.py@@ -494,6 +494,11 @@ def _same_identity(recorded: str, current: str) -> bool:return Falseif _is_linux():return recorded.split(":", 1)[0] == current.split(":", 1)[0]+ if sys.platform == "darwin":+ # lstart alone, as on Linux: a framework python re-execs into Python.app, changing comm.+ recorded_start, current_start = recorded.split()[:5], current.split()[:5]+ if len(recorded_start) == 5 and len(current_start) == 5:+ return recorded_start == current_startreturn recorded == currentdiff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex aada7778eb1..8867c562852 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -10691,6 +10691,68 @@ def _repair_diffusers_main() -> int:time.sleep(_REPAIR_LOCK_POLL_S)+_PREFETCH_SCRATCH_PREFIX = "unsloth-diffusers-prefetch-"+++def _prefetch_diffusers_main() -> int:+ """For the startup repair: fetch and build the pinned build into uv's cache, installing nothing.++ The backend stops this at its deadline, which is safe only because ``--target`` points uv at a+ scratch directory, so site-packages is never touched; the ``--repair-diffusers-main`` that+ follows then installs from the cache in about a second. 0 fetched, 1 nothing to fetch (pip keeps+ no cache here, so it fetches during the install), 2 failed.+ """+ import time++ global USE_UV+ req = REQ_ROOT / "diffusers-main.txt"+ if (+ not req.is_file()+ or not _diffusers_main_requested()+ or not _diffusers_main_needs_dependency_pass()+ or _startup_repair_failed()+ ):+ return 1+ USE_UV = _bootstrap_uv()+ if not USE_UV:+ return 1+ # A prefetch stopped at the deadline cannot clean up after itself.+ for stale in Path(tempfile.gettempdir()).glob(f"{_PREFETCH_SCRATCH_PREFIX}*"):+ try:+ if time.time() - stale.stat().st_mtime > 3600:+ shutil.rmtree(stale, ignore_errors = True)+ except OSError:+ pass+ # The same source _diffusers_main_step will install from, so the install hits this cache entry.+ archive = None if _has_working_git() else _diffusers_main_archive(req)+ scratch = Path(tempfile.mkdtemp(prefix = _PREFETCH_SCRATCH_PREFIX))+ temp_reqs: list[Path] = []+ try:+ args = ("--no-deps", "--target", str(scratch))+ if archive is not None:+ cmd = _build_uv_cmd((*args, f"diffusers @ {archive}"))+ else:+ actual_req, temp_reqs = _effective_requirements(req)+ cmd = _build_uv_cmd(args) + ["-r", _uv_safe_path(actual_req)]+ cmd, env = _pinned_cmd_and_env(cmd)+ result = subprocess.run(+ cmd,+ stdout = subprocess.PIPE,+ stderr = subprocess.STDOUT,+ env = env,+ **_windows_hidden_subprocess_kwargs(),+ )+ finally:+ for temp_req in temp_reqs:+ temp_req.unlink(missing_ok = True)+ shutil.rmtree(scratch, ignore_errors = True)+ if result.returncode != 0:+ if result.stdout:+ _safe_print(_redact_install_output(result.stdout))+ return 2+ return 0++def _diffusers_main_step() -> None:"""Install the pinned Diffusers commit, or leave the release pin alone.@@ -11978,6 +12040,8 @@ if __name__ == "__main__":sys.exit(0 if _missing_torch_needs_dependency_pass() else 1)if sys.argv[1:] == ["--repair-diffusers-main"]:sys.exit(_repair_diffusers_main())+ if sys.argv[1:] == ["--prefetch-diffusers-main"]:+ sys.exit(_prefetch_diffusers_main())if sys.argv[1:] == ["--diffusers-main-needs-dependency-pass"]:# Exit 0 forces the dependency pass; exit 1 keeps the fast path.sys.exit(0 if _diffusers_main_needs_dependency_pass() else 1)diff --git a/tests/studio/install/test_diffusers_pin.py b/tests/studio/install/test_diffusers_pin.pyindex e6c33b3413d..c0605b8735e 100644--- a/tests/studio/install/test_diffusers_pin.py+++ b/tests/studio/install/test_diffusers_pin.py@@ -890,3 +890,42 @@ def test_a_recorded_repair_failure_still_waits_for_a_peer(monkeypatch):)assert module._repair_diffusers_main() == 1 and ran == []assert module.lock_polls() == 2++[email protected]("git", [True, False])+def test_the_startup_prefetch_fills_the_cache_and_never_the_environment(monkeypatch, git):+ """The backend stops this at its deadline, which is only safe while it installs nothing here."""+ import types++ module = _probe_module("install_python_stack_prefetch_probe")+ monkeypatch.delenv("UNSLOTH_DIFFUSERS_MAIN", raising = False)+ monkeypatch.setattr(module, "_diffusers_main_needs_dependency_pass", lambda: True)+ monkeypatch.setattr(module, "_startup_repair_failed", lambda: False)+ monkeypatch.setattr(module, "_bootstrap_uv", lambda: True)+ monkeypatch.setattr(module, "_has_working_git", lambda: git)+ for forbidden in ("_diffusers_main_step", "pip_install_try", "pip_install"):+ monkeypatch.setattr(module, forbidden, lambda *a, **k: pytest.fail("installed"))+ monkeypatch.setattr(+ module.install_manifest, "pass_lock", lambda *a, **k: pytest.fail("took the pass lock")+ )+ runs = []++ def fake_run(cmd, **kwargs):+ runs.append(cmd)+ target = pathlib.Path(cmd[cmd.index("--target") + 1])+ assert target.is_dir()+ return types.SimpleNamespace(returncode = 0 if len(runs) == 1 else 1, stdout = b"")++ monkeypatch.setattr(module.subprocess, "run", fake_run)+ assert module._prefetch_diffusers_main() == 0+ cmd = runs[0]+ assert cmd[:3] == ["uv", "pip", "install"] and "--no-deps" in cmd+ target = pathlib.Path(cmd[cmd.index("--target") + 1])+ assert not target.exists(), "the scratch install is thrown away"+ if git:+ assert "-r" in cmd+ else:+ assert any(arg.startswith("diffusers @ https://") for arg in cmd)+ assert module._prefetch_diffusers_main() == 2+ monkeypatch.setattr(module, "_bootstrap_uv", lambda: False)+ assert module._prefetch_diffusers_main() == 1 and len(runs) == 2, "pip has no cache to fill"