Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: do not start a queued fine-tuning job for a disabled or deleted account (#17392)
Daniel Hanac8c5cd3b29b
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.pyindex b775741000c..d71185adeb3 100644--- a/studio/backend/routes/openai_fine_tuning.py+++ b/studio/backend/routes/openai_fine_tuning.py@@ -861,6 +861,11 @@ def _finish(_ALIASES.pop(old.alias, None)+def _account_still_active(account: AccountContext) -> bool:+ from core.training.account_jobs import job_accounts+ return account.is_owner or any(live.account_id == account.account_id for live in job_accounts())++async def _start_queued(entry: _Queued) -> bool:from routes import training as training_routes@@ -870,6 +875,13 @@ async def _start_queued(entry: _Queued) -> bool:if _training_busy():return Falsegroup = _batch_for(entry)+ # A queued start outlives its request: an account disabled or deleted since then starts nothing.+ for member in group:+ if not await asyncio.to_thread(_account_still_active, member.account):+ _finish(member, "cancelled")+ group = [member for member in group if member.status == "queued"]+ if entry.status != "queued":+ return Trueif len(group) > 1 and await arun_as(entry.account, _trains_alone(entry.request)):group = [entry]# A cancel that landed while the model probe ran has already returned "cancelled": never start it.diff --git a/studio/backend/tests/test_fine_tuning_queue_account_liveness.py b/studio/backend/tests/test_fine_tuning_queue_account_liveness.pynew file mode 100644index 00000000000..8066e5e759d--- /dev/null+++ b/studio/backend/tests/test_fine_tuning_queue_account_liveness.py@@ -0,0 +1,75 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A queued /v1/fine_tuning job outlives its request: it must not start for a disabled or+deleted account."""++import asyncio++import pytest++from auth import storage+from models.training import TrainingJobResponse, TrainingStartRequest+from routes import openai_fine_tuning as ft, training as training_routes+from studio.backend.tests.test_account_lifecycle import auth_env, headers, matrix # noqa: F401+from utils.account_context import OWNER, current_account, run_as+++def _request(model = "unsloth/Qwen3-0.6B"):+ return TrainingStartRequest.model_validate(+ {"model_name": model, "format_type": "auto", "training_type": "LoRA/QLoRA"}+ )+++def queue(matrix, monkeypatch):+ for name, value in (("_QUEUE", []), ("_ALIASES", {}), ("_STARTED", {}), ("_RUN_ALIAS", {})):+ monkeypatch.setattr(ft, name, value)+ monkeypatch.setattr(ft, "_ensure_drainer", lambda: None)+ monkeypatch.setattr(ft, "_save_alias", lambda *a: None)+ monkeypatch.setattr(ft, "_remember_start", lambda *a: None)+ monkeypatch.setattr(ft, "_training_busy", lambda: False)+ started = []++ async def start_training(request, current_subject, via_api_key):+ started.append(current_account().account_id)+ return TrainingJobResponse(job_id = f"job_{len(started)}", status = "queued", message = "queued")++ monkeypatch.setattr(training_routes, "start_training", start_training)+ client, _, _ = matrix+ return client, started+++def test_disabled_account_queued_job_never_starts(queue):+ client, started = queue+ bob = storage.get_account("bob")+ entry = run_as(bob, lambda: ft._enqueue(_request(), "bob", True))+ response = client.patch(+ f"/api/accounts/{bob.account_id}", headers = headers(), json = {"is_active": False}+ )+ assert response.status_code == 200+ asyncio.run(ft._drain_once())+ assert started == []+ assert entry.status == "cancelled" and entry not in ft._QUEUE+++def test_deleted_account_queued_job_never_starts(queue):+ client, started = queue+ bob = storage.get_account("bob")+ entry = run_as(bob, lambda: ft._enqueue(_request(), "bob", True))+ assert client.delete(f"/api/accounts/{bob.account_id}", headers = headers()).status_code == 204+ asyncio.run(ft._drain_once())+ assert started == []+ assert entry.status == "cancelled"+++def test_disabled_member_is_dropped_from_a_batch(queue, monkeypatch):+ client, started = queue+ bob = storage.get_account("bob")+ head = ft._enqueue(_request(), "unsloth", False)+ member = run_as(bob, lambda: ft._enqueue(_request(), "bob", True))+ monkeypatch.setattr(ft, "_batch_for", lambda entry: [entry, member])+ client.patch(f"/api/accounts/{bob.account_id}", headers = headers(), json = {"is_active": False})+ asyncio.run(ft._drain_once())+ assert started == [OWNER.account_id]+ assert head.status == "started" and member.status == "cancelled"diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.pyindex 84ff8984dcb..129b4dcbea7 100644--- a/studio/backend/tests/test_openai_fine_tuning.py+++ b/studio/backend/tests/test_openai_fine_tuning.py@@ -935,6 +935,8 @@ def test_queue_ids_are_stored_and_loaded_per_account(queue, monkeypatch):monkeypatch.setattr(openai_fine_tuning, "_alias_db", spy)monkeypatch.setattr(openai_fine_tuning, "_save_alias", save_spy)+ # The drainer only starts jobs of live accounts.+ monkeypatch.setattr("core.training.account_jobs.job_accounts", lambda: [bob])async def flow():uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")