Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: do not start a queued fine-tuning job for a disabled or deleted account (#17392)

Daniel Hanac8c5cd3b29b
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.py
index b775741000c..d71185adeb3 100644
--- a/studio/backend/routes/openai_fine_tuning.py
+++ b/studio/backend/routes/openai_fine_tuning.py
@@ -861,6 +861,11 @@ def _finish(
_ALIASES.pop(old.alias, None)
+def _account_still_active(account: AccountContext) -> bool:
+ from core.training.account_jobs import job_accounts
+ return account.is_owner or any(live.account_id == account.account_id for live in job_accounts())
+
+
async def _start_queued(entry: _Queued) -> bool:
from routes import training as training_routes
@@ -870,6 +875,13 @@ async def _start_queued(entry: _Queued) -> bool:
if _training_busy():
return False
group = _batch_for(entry)
+ # A queued start outlives its request: an account disabled or deleted since then starts nothing.
+ for member in group:
+ if not await asyncio.to_thread(_account_still_active, member.account):
+ _finish(member, "cancelled")
+ group = [member for member in group if member.status == "queued"]
+ if entry.status != "queued":
+ return True
if len(group) > 1 and await arun_as(entry.account, _trains_alone(entry.request)):
group = [entry]
# A cancel that landed while the model probe ran has already returned "cancelled": never start it.
diff --git a/studio/backend/tests/test_fine_tuning_queue_account_liveness.py b/studio/backend/tests/test_fine_tuning_queue_account_liveness.py
new file mode 100644
index 00000000000..8066e5e759d
--- /dev/null
+++ b/studio/backend/tests/test_fine_tuning_queue_account_liveness.py
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A queued /v1/fine_tuning job outlives its request: it must not start for a disabled or
+deleted account."""
+
+import asyncio
+
+import pytest
+
+from auth import storage
+from models.training import TrainingJobResponse, TrainingStartRequest
+from routes import openai_fine_tuning as ft, training as training_routes
+from studio.backend.tests.test_account_lifecycle import auth_env, headers, matrix # noqa: F401
+from utils.account_context import OWNER, current_account, run_as
+
+
+def _request(model = "unsloth/Qwen3-0.6B"):
+ return TrainingStartRequest.model_validate(
+ {"model_name": model, "format_type": "auto", "training_type": "LoRA/QLoRA"}
+ )
+
+
+def queue(matrix, monkeypatch):
+ for name, value in (("_QUEUE", []), ("_ALIASES", {}), ("_STARTED", {}), ("_RUN_ALIAS", {})):
+ monkeypatch.setattr(ft, name, value)
+ monkeypatch.setattr(ft, "_ensure_drainer", lambda: None)
+ monkeypatch.setattr(ft, "_save_alias", lambda *a: None)
+ monkeypatch.setattr(ft, "_remember_start", lambda *a: None)
+ monkeypatch.setattr(ft, "_training_busy", lambda: False)
+ started = []
+
+ async def start_training(request, current_subject, via_api_key):
+ started.append(current_account().account_id)
+ return TrainingJobResponse(job_id = f"job_{len(started)}", status = "queued", message = "queued")
+
+ monkeypatch.setattr(training_routes, "start_training", start_training)
+ client, _, _ = matrix
+ return client, started
+
+
+def test_disabled_account_queued_job_never_starts(queue):
+ client, started = queue
+ bob = storage.get_account("bob")
+ entry = run_as(bob, lambda: ft._enqueue(_request(), "bob", True))
+ response = client.patch(
+ f"/api/accounts/{bob.account_id}", headers = headers(), json = {"is_active": False}
+ )
+ assert response.status_code == 200
+ asyncio.run(ft._drain_once())
+ assert started == []
+ assert entry.status == "cancelled" and entry not in ft._QUEUE
+
+
+def test_deleted_account_queued_job_never_starts(queue):
+ client, started = queue
+ bob = storage.get_account("bob")
+ entry = run_as(bob, lambda: ft._enqueue(_request(), "bob", True))
+ assert client.delete(f"/api/accounts/{bob.account_id}", headers = headers()).status_code == 204
+ asyncio.run(ft._drain_once())
+ assert started == []
+ assert entry.status == "cancelled"
+
+
+def test_disabled_member_is_dropped_from_a_batch(queue, monkeypatch):
+ client, started = queue
+ bob = storage.get_account("bob")
+ head = ft._enqueue(_request(), "unsloth", False)
+ member = run_as(bob, lambda: ft._enqueue(_request(), "bob", True))
+ monkeypatch.setattr(ft, "_batch_for", lambda entry: [entry, member])
+ client.patch(f"/api/accounts/{bob.account_id}", headers = headers(), json = {"is_active": False})
+ asyncio.run(ft._drain_once())
+ assert started == [OWNER.account_id]
+ assert head.status == "started" and member.status == "cancelled"
diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.py
index 84ff8984dcb..129b4dcbea7 100644
--- a/studio/backend/tests/test_openai_fine_tuning.py
+++ b/studio/backend/tests/test_openai_fine_tuning.py
@@ -935,6 +935,8 @@ def test_queue_ids_are_stored_and_loaded_per_account(queue, monkeypatch):
monkeypatch.setattr(openai_fine_tuning, "_alias_db", spy)
monkeypatch.setattr(openai_fine_tuning, "_save_alias", save_spy)
+ # The drainer only starts jobs of live accounts.
+ monkeypatch.setattr("core.training.account_jobs.job_accounts", lambda: [bob])
async def flow():
uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")