Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: queue OpenAI fine-tuning jobs while the GPU is busy (#14645)

Daniel Han4d5e78ceed3d
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.py
index 78a104b6799..db854385283 100644
--- a/studio/backend/routes/openai_fine_tuning.py
+++ b/studio/backend/routes/openai_fine_tuning.py
@@ -7,6 +7,9 @@ A thin translation layer: uploads land where ``/api/hub/datasets/upload`` puts t
same ``start_training`` handler as ``/api/train/start`` (every guard and refusal included), and job state is
read back from the training history, so a job is just a Studio training run under another name. Nothing here
grants a capability an API key does not already have through ``/api/train``.
+
+A job created while the GPU is busy waits in a first-in-first-out queue and starts, as its own account, once
+training is idle. The queue lives in this process: queued jobs do not survive a Studio restart.
"""
from __future__ import annotations
@@ -14,6 +17,9 @@ from __future__ import annotations
import asyncio
import json
import re
+import secrets
+import weakref
+from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Literal, Optional
@@ -27,6 +33,7 @@ from hub.utils.host_paths import host_paths_visible, redact_host_paths, redact_p
from loggers import get_logger
from models.training import TrainingStartRequest
from storage.studio_db import get_connection, get_run, list_runs
+from utils.account_context import AccountContext, arun_as, current_account, current_account_id
from utils.training_runs import drop_non_finite
logger = get_logger(__name__)
@@ -203,10 +210,13 @@ def _job_object(
hyperparameters = _hyperparameters(config)
datasets = config.get("local_datasets") or []
eval_datasets = config.get("local_eval_datasets") or []
+ alias = _RUN_ALIAS.get(run_id)
return {
- "id": _job_id(run_id),
+ "id": alias or _job_id(run_id),
"object": "fine_tuning.job",
- "created_at": created_at or int(datetime.now(timezone.utc).timestamp()),
+ "created_at": _ALIAS_CREATED.get(alias)
+ or created_at
+ or int(datetime.now(timezone.utc).timestamp()),
"error": {"code": "training_failed", "message": error, "param": None} if error else None,
"fine_tuned_model": fine_tuned_model,
"finished_at": finished_at,
@@ -403,6 +413,277 @@ def _training_config(body: FineTuningJobCreate) -> dict[str, Any]:
return config
+QUEUE_POLL_S = 5.0
+# A 409 with training idle is a conflict that will not clear by waiting (resume, offline state): give up.
+QUEUE_IDLE_CONFLICTS = 12
+_TERMINAL_KEEP = 256
+_BUSY_ERRORS = ("Training already active", "Diffusion training already active")
+# 409s that clear by themselves (a chat request in flight, a transformers install): never give up on these.
+_TRANSIENT_CONFLICTS = (
+ "inference request is in progress",
+ "installation is in progress",
+ "is replacing the latest sidecar",
+)
+_ALIAS_TABLE = "openai_fine_tuning_aliases"
+
+
+@dataclass
+class _Queued:
+ alias: str
+ account: AccountContext
+ subject: str
+ via_api_key: bool
+ request: TrainingStartRequest
+ created_at: int
+ status: str = "queued" # queued, starting, started, failed, cancelled
+ error: Optional[str] = None
+ finished_at: Optional[int] = None
+ idle_conflicts: int = 0
+ cancel_requested: bool = False
+
+
+_QUEUE: list[_Queued] = []
+# Jobs that have not started (waiting, or ended before starting). A started job's id lives in the database.
+_ALIASES: dict[str, _Queued] = {}
+_ALIAS_RUN: dict[str, tuple[str, Optional[str]]] = {}
+_RUN_ALIAS: dict[str, str] = {}
+# A queued job keeps the creation time it was given, not the later time its run started.
+_ALIAS_CREATED: dict[str, int] = {}
+# Each account has its own database, so its ids load the first time that account asks.
+_aliases_loaded: set[Optional[str]] = set()
+_drainer: Optional[asyncio.Task] = None
+_start_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = (
+ weakref.WeakKeyDictionary()
+)
+
+
+def _now() -> int:
+ return int(datetime.now(timezone.utc).timestamp())
+
+
+def _start_lock() -> asyncio.Lock:
+ # One start decision at a time, so two idle creates (or a create and the drainer) never race.
+ loop = asyncio.get_running_loop()
+ lock = _start_locks.get(loop)
+ if lock is None:
+ lock = _start_locks[loop] = asyncio.Lock()
+ return lock
+
+
+def _alias_db():
+ conn = get_connection()
+ conn.execute(
+ f"CREATE TABLE IF NOT EXISTS {_ALIAS_TABLE} "
+ "(alias TEXT PRIMARY KEY, run_id TEXT NOT NULL, account_id TEXT, created_at INTEGER)"
+ )
+ return conn
+
+
+def _load_aliases() -> None:
+ account_id = current_account_id()
+ if account_id in _aliases_loaded:
+ return
+ conn = _alias_db()
+ try:
+ rows = conn.execute(
+ f"SELECT alias, run_id, account_id, created_at FROM {_ALIAS_TABLE}"
+ ).fetchall()
+ finally:
+ conn.close()
+ for alias, run_id, account_id, created_at in rows:
+ _ALIAS_RUN[alias] = (run_id, account_id)
+ _RUN_ALIAS[run_id] = alias
+ if created_at:
+ _ALIAS_CREATED[alias] = created_at
+ _aliases_loaded.add(account_id)
+
+
+def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:
+ _ALIAS_RUN[alias] = (run_id, account_id)
+ _RUN_ALIAS[run_id] = alias
+ _ALIAS_CREATED[alias] = created_at
+ try:
+ conn = _alias_db()
+ try:
+ conn.execute(
+ f"INSERT OR REPLACE INTO {_ALIAS_TABLE} (alias, run_id, account_id, created_at) "
+ "VALUES (?, ?, ?, ?)",
+ (alias, run_id, account_id, created_at),
+ )
+ conn.commit()
+ finally:
+ conn.close()
+ except Exception: # noqa: BLE001
+ # The run is already training: keep it reachable by its id in this process rather than fail it.
+ logger.exception("Could not persist fine-tuning job id %s", alias)
+
+
+def _transient(exc: HTTPException) -> bool:
+ return exc.status_code == 409 and any(text in str(exc.detail) for text in _TRANSIENT_CONFLICTS)
+
+
+def _training_busy() -> bool:
+ from core.training import get_training_backend
+ from routes import training as training_routes
+ return bool(
+ get_training_backend().is_training_active() or training_routes._diffusion_training_active()
+ )
+
+
+def _queued_job(entry: _Queued) -> dict[str, Any]:
+ job = _job_object(
+ entry.alias,
+ status = "cancelled"
+ if entry.cancel_requested
+ else entry.status.replace("starting", "queued"),
+ config = entry.request.model_dump(),
+ created_at = entry.created_at,
+ finished_at = entry.finished_at,
+ error = entry.error,
+ )
+ return {**job, "id": entry.alias}
+
+
+def _finish(
+ entry: _Queued,
+ status: str,
+ error: Optional[str] = None,
+) -> None:
+ entry.status, entry.error, entry.finished_at = status, error, _now()
+ if entry in _QUEUE:
+ _QUEUE.remove(entry)
+ if status == "started":
+ _ALIASES.pop(entry.alias, None)
+ done = [e for e in _ALIASES.values() if e.status in ("failed", "cancelled")]
+ for old in done[: max(0, len(done) - _TERMINAL_KEEP)]:
+ _ALIASES.pop(old.alias, None)
+
+
+async def _start_queued(entry: _Queued) -> bool:
+ """Start the queue head as its own account; False while the GPU is still taken."""
+ from routes import training as training_routes
+
+ async with _start_lock():
+ if entry.status != "queued":
+ return True
+ if _training_busy():
+ return False
+ entry.status = "starting"
+ try:
+ started = await arun_as(
+ entry.account,
+ training_routes.start_training(
+ entry.request, current_subject = entry.subject, via_api_key = entry.via_api_key
+ ),
+ )
+ except HTTPException as exc:
+ entry.status = "queued"
+ if entry.cancel_requested:
+ _finish(entry, "cancelled")
+ return True
+ transient = _transient(exc) or _training_busy()
+ entry.idle_conflicts += 0 if transient else 1
+ if transient or (
+ exc.status_code == 409 and entry.idle_conflicts < QUEUE_IDLE_CONFLICTS
+ ):
+ return False
+ _finish(entry, "failed", str(exc.detail))
+ return True
+ if started.status == "error" or not started.job_id:
+ entry.status = "queued"
+ if entry.cancel_requested:
+ _finish(entry, "cancelled")
+ return True
+ # Busy, or another start (a UI run) took the GPU while this one was admitted: keep waiting.
+ if started.error in _BUSY_ERRORS or _training_busy():
+ return False
+ _finish(entry, "failed", started.message or started.error or "Training did not start")
+ return True
+ # In the job's own account database: the drainer may carry whichever account started it.
+ await arun_as(
+ entry.account,
+ asyncio.to_thread(
+ _save_alias, entry.alias, started.job_id, entry.account.account_id, entry.created_at
+ ),
+ )
+ _remember_start(started.job_id, entry.request.model_dump(), entry.created_at)
+ _finish(entry, "started")
+ if entry.cancel_requested:
+ # Cancelled while its start was in flight: stop it as a cancel of the running job would.
+ await arun_as(entry.account, _stop_run(started.job_id, entry.subject))
+ return True
+
+
+async def _drain_once() -> None:
+ if not _QUEUE:
+ return
+ entry = _QUEUE[0]
+ try:
+ await _start_queued(entry)
+ except Exception as exc: # noqa: BLE001
+ logger.exception("Could not start queued fine-tuning job %s", entry.alias)
+ if entry.cancel_requested:
+ _finish(entry, "cancelled")
+ else:
+ _finish(entry, "failed", f"Training did not start: {type(exc).__name__}")
+
+
+async def _drain() -> None:
+ while _QUEUE:
+ head = _QUEUE[0]
+ await _drain_once()
+ if _QUEUE and _QUEUE[0] is head:
+ await asyncio.sleep(QUEUE_POLL_S)
+
+
+def _ensure_drainer() -> None:
+ global _drainer
+ if _drainer is None or _drainer.done():
+ _drainer = asyncio.get_running_loop().create_task(_drain())
+
+
+def _enqueue(request: TrainingStartRequest, current_subject: str, via_api_key: bool) -> _Queued:
+ entry = _Queued(
+ alias = f"ftjob_q{secrets.token_hex(8)}",
+ account = current_account(),
+ subject = current_subject,
+ via_api_key = via_api_key,
+ request = request,
+ created_at = _now(),
+ )
+ _QUEUE.append(entry)
+ _ALIASES[entry.alias] = entry
+ _ensure_drainer()
+ return entry
+
+
+def _own_alias(job_id: str) -> Optional[_Queued]:
+ entry = _ALIASES.get(job_id)
+ return entry if entry is not None and entry.account.account_id == current_account_id() else None
+
+
+async def _resolve(job_id: str) -> str:
+ """The run behind a job id: a queue id once its job started, else the id itself."""
+ await asyncio.to_thread(_load_aliases)
+ record = _ALIAS_RUN.get(job_id)
+ if record is not None and record[1] == current_account_id():
+ return record[0]
+ return _run_id(job_id)
+
+
+async def _stop_run(run_id: str, current_subject: str) -> Any:
+ from routes import training as training_routes
+
+ stopped = await training_routes.stop_training(
+ training_routes.TrainingStopRequest(save = True, expected_job_id = run_id),
+ current_subject = current_subject,
+ )
+ if stopped.status != "idle":
+ # The worker stops at its next safe step; the job reads "cancelled" from then on.
+ _CANCELLING.add(run_id)
+ return stopped
+
+
@router.post("/fine_tuning/jobs")
async def create_fine_tuning_job(
body: FineTuningJobCreate,
@@ -420,9 +701,26 @@ async def create_fine_tuning_job(
first = exc.errors()[0]
field = ".".join(str(part) for part in first.get("loc", ()))
raise _bad_request(f"Invalid parameter {field}: {first.get('msg')}") from None
- started = await training_routes.start_training(
- request, current_subject = current_subject, via_api_key = via_api_key
- )
+ async with _start_lock():
+ # First come, first served: with jobs already waiting, a new one joins the back of the queue.
+ if _QUEUE or _training_busy():
+ return _for_caller(
+ _queued_job(_enqueue(request, current_subject, via_api_key)), via_api_key
+ )
+ try:
+ started = await training_routes.start_training(
+ request, current_subject = current_subject, via_api_key = via_api_key
+ )
+ except HTTPException as exc:
+ if not (_transient(exc) or _training_busy()):
+ raise
+ return _for_caller(
+ _queued_job(_enqueue(request, current_subject, via_api_key)), via_api_key
+ )
+ if started.error in _BUSY_ERRORS or (started.status == "error" and _training_busy()):
+ return _for_caller(
+ _queued_job(_enqueue(request, current_subject, via_api_key)), via_api_key
+ )
if started.status == "error" or not started.job_id:
message = started.message or started.error or "Training did not start"
raise _bad_request(
@@ -471,7 +769,16 @@ async def list_fine_tuning_jobs(
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
):
- after_run = _run_id(after) if after else None
+ await asyncio.to_thread(_load_aliases)
+ own = [e for e in _ALIASES.values() if _own_alias(e.alias)]
+ # Newest first, as the history lists runs; insertion order breaks same-second ties.
+ pending = [(e, _for_caller(_queued_job(e), via_api_key)) for e in reversed(own)]
+ pending_ids = [job["id"] for _, job in pending]
+ if after in pending_ids:
+ pending, after = pending[pending_ids.index(after) + 1 :], None
+ elif after:
+ pending = []
+ after_run = await _resolve(after) if after else None
runs = await asyncio.to_thread(_runs_after, after_run, limit + 1)
if after_run and not runs:
# The live job has no history row yet and is listed first, so the history follows it.
@@ -485,6 +792,13 @@ async def list_fine_tuning_jobs(
live_phase = _status_field(status, "phase") if status is not None else None
if live_id and live_phase != "idle" and all(run["id"] != live_id for run in runs):
jobs.insert(0, await _job(live_id, current_subject, via_api_key))
+ # A job that ended before it started sorts by its creation; once a newer job ran, it is only retrievable.
+ newest = max((job["created_at"] for job in jobs), default = 0)
+ jobs = [
+ job
+ for e, job in pending
+ if e.status in ("queued", "starting") or job["created_at"] > newest
+ ] + jobs
return {"object": "list", "data": jobs[:limit], "has_more": len(jobs) > limit}
@@ -494,7 +808,10 @@ async def retrieve_fine_tuning_job(
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
):
- return await _job(_run_id(job_id), current_subject, via_api_key)
+ entry = _own_alias(job_id)
+ if entry is not None:
+ return _for_caller(_queued_job(entry), via_api_key)
+ return await _job(await _resolve(job_id), current_subject, via_api_key)
def _events(run: dict[str, Any], via_api_key: bool) -> list[dict[str, Any]]:
@@ -567,6 +884,26 @@ def _events(run: dict[str, Any], via_api_key: bool) -> list[dict[str, Any]]:
return events
+def _queue_event(entry: _Queued, via_api_key: bool) -> dict[str, Any]:
+ if entry.status == "cancelled" or entry.cancel_requested:
+ message = "The job was cancelled before it started"
+ elif entry.status in ("queued", "starting"):
+ message = f"Waiting for the GPU: position {_QUEUE.index(entry) + 1} in the queue"
+ else:
+ message = entry.error or "Training did not start"
+ if not host_paths_visible(via_api_key):
+ message = redact_paths_in_text(message)
+ return {
+ "id": f"ftevent_{entry.alias}_{entry.status}",
+ "object": "fine_tuning.job.event",
+ "created_at": entry.finished_at or entry.created_at,
+ "level": "error" if entry.status == "failed" else "info",
+ "message": message,
+ "data": None,
+ "type": "message",
+ }
+
+
@router.get("/fine_tuning/jobs/{job_id}/events")
async def list_fine_tuning_events(
job_id: str,
@@ -575,7 +912,10 @@ async def list_fine_tuning_events(
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
):
- run_id = _run_id(job_id)
+ entry = _own_alias(job_id)
+ if entry is not None:
+ return _paged([_queue_event(entry, via_api_key)], after, limit)
+ run_id = await _resolve(job_id)
run = await asyncio.to_thread(get_run, run_id)
if run is None:
# Known but not started yet: no events, rather than a 404 a polling client would give up on.
@@ -620,7 +960,7 @@ def _checkpoints(run: dict[str, Any]) -> list[dict[str, Any]]:
"created_at": int(path.stat().st_mtime) or started,
# The "<run folder>/<checkpoint>" name Studio loads and exports a checkpoint by.
"fine_tuned_model_checkpoint": f"{Path(output_dir).name}/{path.name}",
- "fine_tuning_job_id": _job_id(run_id),
+ "fine_tuning_job_id": _RUN_ALIAS.get(run_id) or _job_id(run_id),
"metrics": {"step": step, "train_loss": loss, "valid_loss": eval_loss},
"step_number": step,
}
@@ -637,7 +977,9 @@ async def list_fine_tuning_checkpoints(
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
):
- run_id = _run_id(job_id)
+ if _own_alias(job_id) is not None:
+ return {"object": "list", "data": [], "has_more": False}
+ run_id = await _resolve(job_id)
run = await asyncio.to_thread(get_run, run_id)
if run is None:
await _job(run_id, current_subject, via_api_key)
@@ -651,20 +993,23 @@ async def cancel_fine_tuning_job(
current_subject: str = Depends(get_current_subject),
via_api_key: bool = Depends(authenticated_via_api_key),
):
- from routes import training as training_routes
-
- run_id = _run_id(job_id)
+ entry = _own_alias(job_id)
+ if entry is not None and entry.status == "queued":
+ _finish(entry, "cancelled")
+ return _for_caller(_queued_job(entry), via_api_key)
+ if entry is not None and entry.status == "starting":
+ # Its start is in flight: the starter stops the run as soon as it exists.
+ entry.cancel_requested = True
+ return _for_caller(_queued_job(entry), via_api_key)
+ if entry is not None:
+ raise _bad_request(f"Job {entry.alias} has already {entry.status}.")
+ run_id = await _resolve(job_id)
job = await _job(run_id, current_subject, via_api_key)
if job["status"] in ("succeeded", "failed", "cancelled"):
raise _bad_request(f"Job {job['id']} has already {job['status']}.")
- stopped = await training_routes.stop_training(
- training_routes.TrainingStopRequest(save = True, expected_job_id = run_id),
- current_subject = current_subject,
- )
+ stopped = await _stop_run(run_id, current_subject)
if stopped.status == "idle":
raise _bad_request(f"Job {job['id']} is not running.", 409)
- # The worker stops at its next safe step; the job reads "cancelled" from then on.
- _CANCELLING.add(run_id)
return {**job, "status": "cancelled"}
diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.py
index 919a2483342..e7351d4cb59 100644
--- a/studio/backend/tests/test_openai_fine_tuning.py
+++ b/studio/backend/tests/test_openai_fine_tuning.py
@@ -46,10 +46,11 @@ def studio(monkeypatch):
async def start_training(request, current_subject, via_api_key):
calls["start"].append(request)
- live["status"] = _Status("job_20261021_101500_ab12cd34", "loading_model")
- return TrainingJobResponse(
- job_id = "job_20261021_101500_ab12cd34", status = "queued", message = "queued"
+ job_id = "job_20261021_101500_ab12cd34" + (
+ "" if len(calls["start"]) == 1 else f"{len(calls['start'])}"
)
+ live["status"] = _Status(job_id, "loading_model")
+ return TrainingJobResponse(job_id = job_id, status = "queued", message = "queued")
async def stop_training(body, current_subject):
calls["stop"].append(body)
@@ -58,6 +59,8 @@ def studio(monkeypatch):
async def get_training_status(current_subject, via_api_key):
return live["status"]
+ monkeypatch.setattr(openai_fine_tuning, "_STARTED", {})
+ monkeypatch.setattr(openai_fine_tuning, "_CANCELLING", set())
monkeypatch.setattr(training_routes, "start_training", start_training)
monkeypatch.setattr(training_routes, "stop_training", stop_training)
monkeypatch.setattr(training_routes, "get_training_status", get_training_status)
@@ -372,3 +375,456 @@ def test_an_overflowed_grad_norm_keeps_events_readable(studio, tmp_path):
[step2] = [e for e in events.data if e.type == "metrics" and e.data["step"] == 2]
assert step2.data["grad_norm"] is None
assert checkpoints.data[0].step_number == 2
+
+
+def queue(studio, monkeypatch):
+ """The queue with the GPU busy and the background drainer replaced by explicit drains."""
+ client, calls, live = studio
+ monkeypatch.setattr(openai_fine_tuning, "_QUEUE", [])
+ monkeypatch.setattr(openai_fine_tuning, "_ALIASES", {})
+ monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})
+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})
+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})
+ monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())
+ monkeypatch.setattr(openai_fine_tuning, "_ensure_drainer", lambda: None)
+ gpu = {"busy": True}
+ monkeypatch.setattr(openai_fine_tuning, "_training_busy", lambda: gpu["busy"])
+ return client, calls, live, gpu
+
+
+def _create(client, file_id, **kwargs):
+ return client.fine_tuning.jobs.create(
+ model = "unsloth/Qwen3-0.6B", training_file = file_id, **kwargs
+ )
+
+
+def test_a_busy_gpu_queues_jobs_and_starts_them_in_order(queue):
+ client, calls, _live, gpu = queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ first = await _create(client, uploaded.id, suffix = "first")
+ second = await _create(client, uploaded.id, suffix = "second")
+ listed = await client.fine_tuning.jobs.list()
+ events = await client.fine_tuning.jobs.list_events(second.id)
+ await openai_fine_tuning._drain_once() # GPU still busy: nothing starts
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once() # the head starts; the GPU is now taken by it
+ gpu["busy"] = True
+ await openai_fine_tuning._drain_once()
+ after_first = [await client.fine_tuning.jobs.retrieve(j.id) for j in (first, second)]
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return first, second, listed, events, after_first
+
+ first, second, listed, events, (started, waiting) = _run(flow())
+ assert first.status == second.status == "queued"
+ assert [j.id for j in listed.data[:2]] == [second.id, first.id]
+ assert events.data[0].message == "Waiting for the GPU: position 2 in the queue"
+ assert [r.project_name for r in calls["start"]] == ["first", "second"]
+ # The started job keeps the id the client was given.
+ assert started.id == first.id and started.status == "validating_files"
+ assert waiting.status == "queued"
+
+
+def test_a_queued_job_can_be_cancelled_and_never_starts(queue):
+ client, calls, _live, gpu = queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ cancelled = await client.fine_tuning.jobs.cancel(job.id)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return cancelled, await client.fine_tuning.jobs.retrieve(job.id)
+
+ cancelled, retrieved = _run(flow())
+ assert cancelled.status == retrieved.status == "cancelled"
+ assert calls["start"] == []
+
+
+def test_a_refused_start_fails_the_queued_job(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def refuse(request, current_subject, via_api_key):
+ return TrainingJobResponse(
+ job_id = "", status = "error", message = "The model is gated.", error = "gated"
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", refuse)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return await client.fine_tuning.jobs.retrieve(job.id), openai_fine_tuning._QUEUE
+
+ job, remaining = _run(flow())
+ assert job.status == "failed" and job.error.message == "The model is gated."
+ assert remaining == []
+
+
+def test_another_account_cannot_see_or_cancel_a_queued_job(queue, monkeypatch):
+ client, _calls, _live, _gpu = queue
+ from utils.account_context import AccountContext
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(
+ openai_fine_tuning,
+ "current_account_id",
+ lambda: AccountContext("bob", "bob").account_id,
+ )
+ statuses = []
+ for call in (client.fine_tuning.jobs.retrieve, client.fine_tuning.jobs.cancel):
+ try:
+ await call(job.id)
+ except openai.NotFoundError:
+ statuses.append(404)
+ listed = await client.fine_tuning.jobs.list()
+ return statuses, [j.id for j in listed.data]
+
+ statuses, listed = _run(flow())
+ assert statuses == [404, 404] and listed == []
+
+
+def _forget_aliases(monkeypatch):
+ # What a Studio restart leaves: nothing in memory, the database as it was.
+ monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})
+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})
+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})
+ monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())
+
+
+def test_a_started_queued_job_keeps_its_id_after_a_restart(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ _forget_aliases(monkeypatch)
+ retrieved = await client.fine_tuning.jobs.retrieve(job.id)
+ listed = await client.fine_tuning.jobs.list()
+ # The started job as a cursor pages past it rather than ending the list.
+ paged = await client.fine_tuning.jobs.list(after = job.id)
+ return job, retrieved, listed, paged
+
+ job, retrieved, listed, paged = _run(flow())
+ assert retrieved.id == job.id and retrieved.status == "validating_files"
+ assert [j.id for j in listed.data] == [job.id]
+ assert paged.data == [] and not paged.has_more
+
+
+def test_cancelling_a_job_whose_start_is_in_flight_stops_it(queue, monkeypatch):
+ client, calls, live, gpu = queue
+ release = asyncio.Event()
+ entered = asyncio.Event()
+ start = training_routes.start_training
+
+ async def slow_start(request, current_subject, via_api_key):
+ entered.set()
+ await release.wait()
+ return await start(request, current_subject = current_subject, via_api_key = via_api_key)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", slow_start)
+ gpu["busy"] = False
+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())
+ await entered.wait()
+ cancelled = await client.fine_tuning.jobs.cancel(job.id)
+ release.set()
+ await drain
+ return cancelled, await client.fine_tuning.jobs.retrieve(job.id)
+
+ cancelled, retrieved = _run(flow())
+ assert cancelled.status == retrieved.status == "cancelled"
+ assert [b.expected_job_id for b in calls["stop"]] == [live["status"].job_id]
+
+
+def test_a_queued_failure_hides_host_paths_from_an_api_key(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def refuse(request, current_subject, via_api_key):
+ message = "Dataset not found: /home/alice/.unsloth/studio/datasets/chats.jsonl"
+ return TrainingJobResponse(job_id = "", status = "error", message = message, error = message)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", refuse)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return await client.fine_tuning.jobs.list_events(job.id)
+
+ events = _run(flow())
+ assert events.data[0].level == "error" and "/home/alice" not in events.data[0].message
+
+
+def test_a_diffusion_run_winning_the_gpu_keeps_the_job_queued(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def diffusion_first(request, current_subject, via_api_key):
+ return TrainingJobResponse(
+ job_id = "",
+ status = "error",
+ message = "Diffusion training already active",
+ error = "Diffusion training already active",
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", diffusion_first)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return await client.fine_tuning.jobs.retrieve(job.id)
+
+ assert _run(flow()).status == "queued"
+
+
+def test_two_creates_at_once_start_one_and_queue_the_other(queue, monkeypatch):
+ client, calls, _live, gpu = queue
+ gpu["busy"] = False
+ start = training_routes.start_training
+
+ async def start_and_take_the_gpu(request, current_subject, via_api_key):
+ await asyncio.sleep(0.01)
+ gpu["busy"] = True
+ return await start(request, current_subject = current_subject, via_api_key = via_api_key)
+
+ monkeypatch.setattr(training_routes, "start_training", start_and_take_the_gpu)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ return await asyncio.gather(_create(client, uploaded.id), _create(client, uploaded.id))
+
+ jobs = _run(flow())
+ assert sorted(j.status for j in jobs) == ["queued", "validating_files"]
+ assert len(calls["start"]) == 1
+
+
+def test_a_job_cancelled_before_it_started_does_not_hide_newer_jobs(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)
+ old = await _create(client, uploaded.id)
+ await client.fine_tuning.jobs.cancel(old.id)
+ gpu["busy"] = False
+ newer = await _create(client, uploaded.id)
+ listed = await client.fine_tuning.jobs.list()
+ return old, newer, listed, await client.fine_tuning.jobs.retrieve(old.id)
+
+ old, newer, listed, retrieved = _run(flow())
+ assert listed.data[0].id == newer.id
+ assert retrieved.status == "cancelled"
+
+
+def test_a_queued_job_keeps_its_creation_time_and_event_cursor(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)
+ job = await _create(client, uploaded.id)
+ events = await client.fine_tuning.jobs.list_events(job.id)
+ again = await client.fine_tuning.jobs.list_events(job.id, after = events.data[0].id)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ _forget_aliases(monkeypatch)
+ return job, again, await client.fine_tuning.jobs.retrieve(job.id)
+
+ job, again, started = _run(flow())
+ assert again.data == []
+ assert started.created_at == job.created_at == 1_700_000_000
+
+
+def test_an_inference_request_in_flight_never_fails_a_queued_job(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def chat_in_flight(request, current_subject, via_api_key):
+ raise openai_fine_tuning.HTTPException(
+ status_code = 409,
+ detail = "Cannot start training over the API while an inference request is in progress.",
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", chat_in_flight)
+ gpu["busy"] = False
+ for _ in range(openai_fine_tuning.QUEUE_IDLE_CONFLICTS + 2):
+ await openai_fine_tuning._drain_once()
+ return await client.fine_tuning.jobs.retrieve(job.id)
+
+ assert _run(flow()).status == "queued"
+
+
+def test_a_job_id_store_failure_keeps_the_started_job(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ def locked():
+ raise studio_db.sqlite3.OperationalError("database is locked")
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ await client.fine_tuning.jobs.list() # load the id table before it locks
+ monkeypatch.setattr(openai_fine_tuning, "_alias_db", locked)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return job, await client.fine_tuning.jobs.retrieve(job.id)
+
+ job, started = _run(flow())
+ assert started.id == job.id and started.status == "validating_files"
+
+
+def test_queue_ids_are_stored_and_loaded_per_account(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+ from utils.account_context import AccountContext
+
+ bob = AccountContext("bob", "bob")
+ reads, writes = [], []
+ alias_db = openai_fine_tuning._alias_db
+
+ def spy():
+ reads.append(openai_fine_tuning.current_account_id())
+ return alias_db()
+
+ save = openai_fine_tuning._save_alias
+
+ def save_spy(*args):
+ writes.append(openai_fine_tuning.current_account_id())
+ return save(*args)
+
+ monkeypatch.setattr(openai_fine_tuning, "_alias_db", spy)
+ monkeypatch.setattr(openai_fine_tuning, "_save_alias", save_spy)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ # The drainer runs in whatever account context it was created in, not the job's.
+ openai_fine_tuning._QUEUE[0].account = bob
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ await client.fine_tuning.jobs.list()
+ monkeypatch.setattr(openai_fine_tuning, "current_account_id", lambda: bob.account_id)
+ await client.fine_tuning.jobs.list()
+ return job
+
+ _run(flow())
+ assert writes == [bob.account_id]
+ # Bob's own table is read even though another account loaded its ids first.
+ assert bob.account_id in reads and len(set(reads)) == 2
+
+
+def _transient_409(request, current_subject, via_api_key):
+ raise openai_fine_tuning.HTTPException(
+ status_code = 409,
+ detail = "Cannot start training over the API while an inference request is in progress.",
+ )
+
+
+def test_a_cancel_during_a_start_that_hits_a_transient_conflict_ends_the_job(queue, monkeypatch):
+ client, calls, _live, gpu = queue
+ release, entered = asyncio.Event(), asyncio.Event()
+
+ async def slow_conflict(request, current_subject, via_api_key):
+ entered.set()
+ await release.wait()
+ _transient_409(request, current_subject, via_api_key)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", slow_conflict)
+ gpu["busy"] = False
+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())
+ await entered.wait()
+ await client.fine_tuning.jobs.cancel(job.id)
+ release.set()
+ await drain
+ return await client.fine_tuning.jobs.retrieve(job.id), list(openai_fine_tuning._QUEUE)
+
+ job, remaining = _run(flow())
+ assert job.status == "cancelled" and remaining == []
+
+
+def test_a_ui_run_winning_the_gpu_keeps_the_queued_job_waiting(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+
+ async def lost_to_ui(request, current_subject, via_api_key):
+ gpu["busy"] = True # the UI run is now training
+ return TrainingJobResponse(
+ job_id = "",
+ status = "error",
+ message = "Training did not start.",
+ error = "subprocess_start_failed",
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ job = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", lost_to_ui)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ return await client.fine_tuning.jobs.retrieve(job.id)
+
+ assert _run(flow()).status == "queued"
+
+
+def test_a_create_during_a_chat_request_is_queued_not_refused(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+ gpu["busy"] = False
+
+ async def chat_in_flight(request, current_subject, via_api_key):
+ _transient_409(request, current_subject, via_api_key)
+
+ monkeypatch.setattr(training_routes, "start_training", chat_in_flight)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ return await _create(client, uploaded.id)
+
+ assert _run(flow()).status == "queued"
+
+
+def test_a_sidecar_swap_conflict_queues_and_a_crash_after_cancel_stays_cancelled(
+ queue, monkeypatch
+):
+ client, _calls, _live, gpu = queue
+ release, entered = asyncio.Event(), asyncio.Event()
+
+ async def sidecar_swap(request, current_subject, via_api_key):
+ raise openai_fine_tuning.HTTPException(
+ status_code = 409,
+ detail = "A transformers installation is replacing the latest sidecar; retry when it completes.",
+ )
+
+ async def crash_after_cancel(request, current_subject, via_api_key):
+ entered.set()
+ await release.wait()
+ raise RuntimeError("worker spawn blew up")
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ monkeypatch.setattr(training_routes, "start_training", sidecar_swap)
+ gpu["busy"] = False
+ queued = await _create(client, uploaded.id)
+ monkeypatch.setattr(training_routes, "start_training", crash_after_cancel)
+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())
+ await entered.wait()
+ await client.fine_tuning.jobs.cancel(queued.id)
+ release.set()
+ await drain
+ return queued, await client.fine_tuning.jobs.retrieve(queued.id)
+
+ queued, after = _run(flow())
+ assert queued.status == "queued" and after.status == "cancelled"