Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: queue OpenAI fine-tuning jobs while the GPU is busy (#14645)
Daniel Han4d5e78ceed3d
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.pyindex 78a104b6799..db854385283 100644--- a/studio/backend/routes/openai_fine_tuning.py+++ b/studio/backend/routes/openai_fine_tuning.py@@ -7,6 +7,9 @@ A thin translation layer: uploads land where ``/api/hub/datasets/upload`` puts tsame ``start_training`` handler as ``/api/train/start`` (every guard and refusal included), and job state isread back from the training history, so a job is just a Studio training run under another name. Nothing heregrants a capability an API key does not already have through ``/api/train``.++A job created while the GPU is busy waits in a first-in-first-out queue and starts, as its own account, once+training is idle. The queue lives in this process: queued jobs do not survive a Studio restart."""from __future__ import annotations@@ -14,6 +17,9 @@ from __future__ import annotationsimport asyncioimport jsonimport re+import secrets+import weakref+from dataclasses import dataclassfrom datetime import datetime, timezonefrom pathlib import Pathfrom typing import Any, Literal, Optional@@ -27,6 +33,7 @@ from hub.utils.host_paths import host_paths_visible, redact_host_paths, redact_pfrom loggers import get_loggerfrom models.training import TrainingStartRequestfrom storage.studio_db import get_connection, get_run, list_runs+from utils.account_context import AccountContext, arun_as, current_account, current_account_idfrom utils.training_runs import drop_non_finitelogger = get_logger(__name__)@@ -203,10 +210,13 @@ def _job_object(hyperparameters = _hyperparameters(config)datasets = config.get("local_datasets") or []eval_datasets = config.get("local_eval_datasets") or []+ alias = _RUN_ALIAS.get(run_id)return {- "id": _job_id(run_id),+ "id": alias or _job_id(run_id),"object": "fine_tuning.job",- "created_at": created_at or int(datetime.now(timezone.utc).timestamp()),+ "created_at": _ALIAS_CREATED.get(alias)+ or created_at+ or int(datetime.now(timezone.utc).timestamp()),"error": {"code": "training_failed", "message": error, "param": None} if error else None,"fine_tuned_model": fine_tuned_model,"finished_at": finished_at,@@ -403,6 +413,277 @@ def _training_config(body: FineTuningJobCreate) -> dict[str, Any]:return config+QUEUE_POLL_S = 5.0+# A 409 with training idle is a conflict that will not clear by waiting (resume, offline state): give up.+QUEUE_IDLE_CONFLICTS = 12+_TERMINAL_KEEP = 256+_BUSY_ERRORS = ("Training already active", "Diffusion training already active")+# 409s that clear by themselves (a chat request in flight, a transformers install): never give up on these.+_TRANSIENT_CONFLICTS = (+ "inference request is in progress",+ "installation is in progress",+ "is replacing the latest sidecar",+)+_ALIAS_TABLE = "openai_fine_tuning_aliases"+++@dataclass+class _Queued:+ alias: str+ account: AccountContext+ subject: str+ via_api_key: bool+ request: TrainingStartRequest+ created_at: int+ status: str = "queued" # queued, starting, started, failed, cancelled+ error: Optional[str] = None+ finished_at: Optional[int] = None+ idle_conflicts: int = 0+ cancel_requested: bool = False+++_QUEUE: list[_Queued] = []+# Jobs that have not started (waiting, or ended before starting). A started job's id lives in the database.+_ALIASES: dict[str, _Queued] = {}+_ALIAS_RUN: dict[str, tuple[str, Optional[str]]] = {}+_RUN_ALIAS: dict[str, str] = {}+# A queued job keeps the creation time it was given, not the later time its run started.+_ALIAS_CREATED: dict[str, int] = {}+# Each account has its own database, so its ids load the first time that account asks.+_aliases_loaded: set[Optional[str]] = set()+_drainer: Optional[asyncio.Task] = None+_start_locks: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, asyncio.Lock]" = (+ weakref.WeakKeyDictionary()+)+++def _now() -> int:+ return int(datetime.now(timezone.utc).timestamp())+++def _start_lock() -> asyncio.Lock:+ # One start decision at a time, so two idle creates (or a create and the drainer) never race.+ loop = asyncio.get_running_loop()+ lock = _start_locks.get(loop)+ if lock is None:+ lock = _start_locks[loop] = asyncio.Lock()+ return lock+++def _alias_db():+ conn = get_connection()+ conn.execute(+ f"CREATE TABLE IF NOT EXISTS {_ALIAS_TABLE} "+ "(alias TEXT PRIMARY KEY, run_id TEXT NOT NULL, account_id TEXT, created_at INTEGER)"+ )+ return conn+++def _load_aliases() -> None:+ account_id = current_account_id()+ if account_id in _aliases_loaded:+ return+ conn = _alias_db()+ try:+ rows = conn.execute(+ f"SELECT alias, run_id, account_id, created_at FROM {_ALIAS_TABLE}"+ ).fetchall()+ finally:+ conn.close()+ for alias, run_id, account_id, created_at in rows:+ _ALIAS_RUN[alias] = (run_id, account_id)+ _RUN_ALIAS[run_id] = alias+ if created_at:+ _ALIAS_CREATED[alias] = created_at+ _aliases_loaded.add(account_id)+++def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:+ _ALIAS_RUN[alias] = (run_id, account_id)+ _RUN_ALIAS[run_id] = alias+ _ALIAS_CREATED[alias] = created_at+ try:+ conn = _alias_db()+ try:+ conn.execute(+ f"INSERT OR REPLACE INTO {_ALIAS_TABLE} (alias, run_id, account_id, created_at) "+ "VALUES (?, ?, ?, ?)",+ (alias, run_id, account_id, created_at),+ )+ conn.commit()+ finally:+ conn.close()+ except Exception: # noqa: BLE001+ # The run is already training: keep it reachable by its id in this process rather than fail it.+ logger.exception("Could not persist fine-tuning job id %s", alias)+++def _transient(exc: HTTPException) -> bool:+ return exc.status_code == 409 and any(text in str(exc.detail) for text in _TRANSIENT_CONFLICTS)+++def _training_busy() -> bool:+ from core.training import get_training_backend+ from routes import training as training_routes+ return bool(+ get_training_backend().is_training_active() or training_routes._diffusion_training_active()+ )+++def _queued_job(entry: _Queued) -> dict[str, Any]:+ job = _job_object(+ entry.alias,+ status = "cancelled"+ if entry.cancel_requested+ else entry.status.replace("starting", "queued"),+ config = entry.request.model_dump(),+ created_at = entry.created_at,+ finished_at = entry.finished_at,+ error = entry.error,+ )+ return {**job, "id": entry.alias}+++def _finish(+ entry: _Queued,+ status: str,+ error: Optional[str] = None,+) -> None:+ entry.status, entry.error, entry.finished_at = status, error, _now()+ if entry in _QUEUE:+ _QUEUE.remove(entry)+ if status == "started":+ _ALIASES.pop(entry.alias, None)+ done = [e for e in _ALIASES.values() if e.status in ("failed", "cancelled")]+ for old in done[: max(0, len(done) - _TERMINAL_KEEP)]:+ _ALIASES.pop(old.alias, None)+++async def _start_queued(entry: _Queued) -> bool:+ """Start the queue head as its own account; False while the GPU is still taken."""+ from routes import training as training_routes++ async with _start_lock():+ if entry.status != "queued":+ return True+ if _training_busy():+ return False+ entry.status = "starting"+ try:+ started = await arun_as(+ entry.account,+ training_routes.start_training(+ entry.request, current_subject = entry.subject, via_api_key = entry.via_api_key+ ),+ )+ except HTTPException as exc:+ entry.status = "queued"+ if entry.cancel_requested:+ _finish(entry, "cancelled")+ return True+ transient = _transient(exc) or _training_busy()+ entry.idle_conflicts += 0 if transient else 1+ if transient or (+ exc.status_code == 409 and entry.idle_conflicts < QUEUE_IDLE_CONFLICTS+ ):+ return False+ _finish(entry, "failed", str(exc.detail))+ return True+ if started.status == "error" or not started.job_id:+ entry.status = "queued"+ if entry.cancel_requested:+ _finish(entry, "cancelled")+ return True+ # Busy, or another start (a UI run) took the GPU while this one was admitted: keep waiting.+ if started.error in _BUSY_ERRORS or _training_busy():+ return False+ _finish(entry, "failed", started.message or started.error or "Training did not start")+ return True+ # In the job's own account database: the drainer may carry whichever account started it.+ await arun_as(+ entry.account,+ asyncio.to_thread(+ _save_alias, entry.alias, started.job_id, entry.account.account_id, entry.created_at+ ),+ )+ _remember_start(started.job_id, entry.request.model_dump(), entry.created_at)+ _finish(entry, "started")+ if entry.cancel_requested:+ # Cancelled while its start was in flight: stop it as a cancel of the running job would.+ await arun_as(entry.account, _stop_run(started.job_id, entry.subject))+ return True+++async def _drain_once() -> None:+ if not _QUEUE:+ return+ entry = _QUEUE[0]+ try:+ await _start_queued(entry)+ except Exception as exc: # noqa: BLE001+ logger.exception("Could not start queued fine-tuning job %s", entry.alias)+ if entry.cancel_requested:+ _finish(entry, "cancelled")+ else:+ _finish(entry, "failed", f"Training did not start: {type(exc).__name__}")+++async def _drain() -> None:+ while _QUEUE:+ head = _QUEUE[0]+ await _drain_once()+ if _QUEUE and _QUEUE[0] is head:+ await asyncio.sleep(QUEUE_POLL_S)+++def _ensure_drainer() -> None:+ global _drainer+ if _drainer is None or _drainer.done():+ _drainer = asyncio.get_running_loop().create_task(_drain())+++def _enqueue(request: TrainingStartRequest, current_subject: str, via_api_key: bool) -> _Queued:+ entry = _Queued(+ alias = f"ftjob_q{secrets.token_hex(8)}",+ account = current_account(),+ subject = current_subject,+ via_api_key = via_api_key,+ request = request,+ created_at = _now(),+ )+ _QUEUE.append(entry)+ _ALIASES[entry.alias] = entry+ _ensure_drainer()+ return entry+++def _own_alias(job_id: str) -> Optional[_Queued]:+ entry = _ALIASES.get(job_id)+ return entry if entry is not None and entry.account.account_id == current_account_id() else None+++async def _resolve(job_id: str) -> str:+ """The run behind a job id: a queue id once its job started, else the id itself."""+ await asyncio.to_thread(_load_aliases)+ record = _ALIAS_RUN.get(job_id)+ if record is not None and record[1] == current_account_id():+ return record[0]+ return _run_id(job_id)+++async def _stop_run(run_id: str, current_subject: str) -> Any:+ from routes import training as training_routes++ stopped = await training_routes.stop_training(+ training_routes.TrainingStopRequest(save = True, expected_job_id = run_id),+ current_subject = current_subject,+ )+ if stopped.status != "idle":+ # The worker stops at its next safe step; the job reads "cancelled" from then on.+ _CANCELLING.add(run_id)+ return stopped++@router.post("/fine_tuning/jobs")async def create_fine_tuning_job(body: FineTuningJobCreate,@@ -420,9 +701,26 @@ async def create_fine_tuning_job(first = exc.errors()[0]field = ".".join(str(part) for part in first.get("loc", ()))raise _bad_request(f"Invalid parameter {field}: {first.get('msg')}") from None- started = await training_routes.start_training(- request, current_subject = current_subject, via_api_key = via_api_key- )+ async with _start_lock():+ # First come, first served: with jobs already waiting, a new one joins the back of the queue.+ if _QUEUE or _training_busy():+ return _for_caller(+ _queued_job(_enqueue(request, current_subject, via_api_key)), via_api_key+ )+ try:+ started = await training_routes.start_training(+ request, current_subject = current_subject, via_api_key = via_api_key+ )+ except HTTPException as exc:+ if not (_transient(exc) or _training_busy()):+ raise+ return _for_caller(+ _queued_job(_enqueue(request, current_subject, via_api_key)), via_api_key+ )+ if started.error in _BUSY_ERRORS or (started.status == "error" and _training_busy()):+ return _for_caller(+ _queued_job(_enqueue(request, current_subject, via_api_key)), via_api_key+ )if started.status == "error" or not started.job_id:message = started.message or started.error or "Training did not start"raise _bad_request(@@ -471,7 +769,16 @@ async def list_fine_tuning_jobs(current_subject: str = Depends(get_current_subject),via_api_key: bool = Depends(authenticated_via_api_key),):- after_run = _run_id(after) if after else None+ await asyncio.to_thread(_load_aliases)+ own = [e for e in _ALIASES.values() if _own_alias(e.alias)]+ # Newest first, as the history lists runs; insertion order breaks same-second ties.+ pending = [(e, _for_caller(_queued_job(e), via_api_key)) for e in reversed(own)]+ pending_ids = [job["id"] for _, job in pending]+ if after in pending_ids:+ pending, after = pending[pending_ids.index(after) + 1 :], None+ elif after:+ pending = []+ after_run = await _resolve(after) if after else Noneruns = await asyncio.to_thread(_runs_after, after_run, limit + 1)if after_run and not runs:# The live job has no history row yet and is listed first, so the history follows it.@@ -485,6 +792,13 @@ async def list_fine_tuning_jobs(live_phase = _status_field(status, "phase") if status is not None else Noneif live_id and live_phase != "idle" and all(run["id"] != live_id for run in runs):jobs.insert(0, await _job(live_id, current_subject, via_api_key))+ # A job that ended before it started sorts by its creation; once a newer job ran, it is only retrievable.+ newest = max((job["created_at"] for job in jobs), default = 0)+ jobs = [+ job+ for e, job in pending+ if e.status in ("queued", "starting") or job["created_at"] > newest+ ] + jobsreturn {"object": "list", "data": jobs[:limit], "has_more": len(jobs) > limit}@@ -494,7 +808,10 @@ async def retrieve_fine_tuning_job(current_subject: str = Depends(get_current_subject),via_api_key: bool = Depends(authenticated_via_api_key),):- return await _job(_run_id(job_id), current_subject, via_api_key)+ entry = _own_alias(job_id)+ if entry is not None:+ return _for_caller(_queued_job(entry), via_api_key)+ return await _job(await _resolve(job_id), current_subject, via_api_key)def _events(run: dict[str, Any], via_api_key: bool) -> list[dict[str, Any]]:@@ -567,6 +884,26 @@ def _events(run: dict[str, Any], via_api_key: bool) -> list[dict[str, Any]]:return events+def _queue_event(entry: _Queued, via_api_key: bool) -> dict[str, Any]:+ if entry.status == "cancelled" or entry.cancel_requested:+ message = "The job was cancelled before it started"+ elif entry.status in ("queued", "starting"):+ message = f"Waiting for the GPU: position {_QUEUE.index(entry) + 1} in the queue"+ else:+ message = entry.error or "Training did not start"+ if not host_paths_visible(via_api_key):+ message = redact_paths_in_text(message)+ return {+ "id": f"ftevent_{entry.alias}_{entry.status}",+ "object": "fine_tuning.job.event",+ "created_at": entry.finished_at or entry.created_at,+ "level": "error" if entry.status == "failed" else "info",+ "message": message,+ "data": None,+ "type": "message",+ }++@router.get("/fine_tuning/jobs/{job_id}/events")async def list_fine_tuning_events(job_id: str,@@ -575,7 +912,10 @@ async def list_fine_tuning_events(current_subject: str = Depends(get_current_subject),via_api_key: bool = Depends(authenticated_via_api_key),):- run_id = _run_id(job_id)+ entry = _own_alias(job_id)+ if entry is not None:+ return _paged([_queue_event(entry, via_api_key)], after, limit)+ run_id = await _resolve(job_id)run = await asyncio.to_thread(get_run, run_id)if run is None:# Known but not started yet: no events, rather than a 404 a polling client would give up on.@@ -620,7 +960,7 @@ def _checkpoints(run: dict[str, Any]) -> list[dict[str, Any]]:"created_at": int(path.stat().st_mtime) or started,# The "<run folder>/<checkpoint>" name Studio loads and exports a checkpoint by."fine_tuned_model_checkpoint": f"{Path(output_dir).name}/{path.name}",- "fine_tuning_job_id": _job_id(run_id),+ "fine_tuning_job_id": _RUN_ALIAS.get(run_id) or _job_id(run_id),"metrics": {"step": step, "train_loss": loss, "valid_loss": eval_loss},"step_number": step,}@@ -637,7 +977,9 @@ async def list_fine_tuning_checkpoints(current_subject: str = Depends(get_current_subject),via_api_key: bool = Depends(authenticated_via_api_key),):- run_id = _run_id(job_id)+ if _own_alias(job_id) is not None:+ return {"object": "list", "data": [], "has_more": False}+ run_id = await _resolve(job_id)run = await asyncio.to_thread(get_run, run_id)if run is None:await _job(run_id, current_subject, via_api_key)@@ -651,20 +993,23 @@ async def cancel_fine_tuning_job(current_subject: str = Depends(get_current_subject),via_api_key: bool = Depends(authenticated_via_api_key),):- from routes import training as training_routes-- run_id = _run_id(job_id)+ entry = _own_alias(job_id)+ if entry is not None and entry.status == "queued":+ _finish(entry, "cancelled")+ return _for_caller(_queued_job(entry), via_api_key)+ if entry is not None and entry.status == "starting":+ # Its start is in flight: the starter stops the run as soon as it exists.+ entry.cancel_requested = True+ return _for_caller(_queued_job(entry), via_api_key)+ if entry is not None:+ raise _bad_request(f"Job {entry.alias} has already {entry.status}.")+ run_id = await _resolve(job_id)job = await _job(run_id, current_subject, via_api_key)if job["status"] in ("succeeded", "failed", "cancelled"):raise _bad_request(f"Job {job['id']} has already {job['status']}.")- stopped = await training_routes.stop_training(- training_routes.TrainingStopRequest(save = True, expected_job_id = run_id),- current_subject = current_subject,- )+ stopped = await _stop_run(run_id, current_subject)if stopped.status == "idle":raise _bad_request(f"Job {job['id']} is not running.", 409)- # The worker stops at its next safe step; the job reads "cancelled" from then on.- _CANCELLING.add(run_id)return {**job, "status": "cancelled"}diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.pyindex 919a2483342..e7351d4cb59 100644--- a/studio/backend/tests/test_openai_fine_tuning.py+++ b/studio/backend/tests/test_openai_fine_tuning.py@@ -46,10 +46,11 @@ def studio(monkeypatch):async def start_training(request, current_subject, via_api_key):calls["start"].append(request)- live["status"] = _Status("job_20261021_101500_ab12cd34", "loading_model")- return TrainingJobResponse(- job_id = "job_20261021_101500_ab12cd34", status = "queued", message = "queued"+ job_id = "job_20261021_101500_ab12cd34" + (+ "" if len(calls["start"]) == 1 else f"{len(calls['start'])}")+ live["status"] = _Status(job_id, "loading_model")+ return TrainingJobResponse(job_id = job_id, status = "queued", message = "queued")async def stop_training(body, current_subject):calls["stop"].append(body)@@ -58,6 +59,8 @@ def studio(monkeypatch):async def get_training_status(current_subject, via_api_key):return live["status"]+ monkeypatch.setattr(openai_fine_tuning, "_STARTED", {})+ monkeypatch.setattr(openai_fine_tuning, "_CANCELLING", set())monkeypatch.setattr(training_routes, "start_training", start_training)monkeypatch.setattr(training_routes, "stop_training", stop_training)monkeypatch.setattr(training_routes, "get_training_status", get_training_status)@@ -372,3 +375,456 @@ def test_an_overflowed_grad_norm_keeps_events_readable(studio, tmp_path):[step2] = [e for e in events.data if e.type == "metrics" and e.data["step"] == 2]assert step2.data["grad_norm"] is Noneassert checkpoints.data[0].step_number == 2+++def queue(studio, monkeypatch):+ """The queue with the GPU busy and the background drainer replaced by explicit drains."""+ client, calls, live = studio+ monkeypatch.setattr(openai_fine_tuning, "_QUEUE", [])+ monkeypatch.setattr(openai_fine_tuning, "_ALIASES", {})+ monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})+ monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())+ monkeypatch.setattr(openai_fine_tuning, "_ensure_drainer", lambda: None)+ gpu = {"busy": True}+ monkeypatch.setattr(openai_fine_tuning, "_training_busy", lambda: gpu["busy"])+ return client, calls, live, gpu+++def _create(client, file_id, **kwargs):+ return client.fine_tuning.jobs.create(+ model = "unsloth/Qwen3-0.6B", training_file = file_id, **kwargs+ )+++def test_a_busy_gpu_queues_jobs_and_starts_them_in_order(queue):+ client, calls, _live, gpu = queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ first = await _create(client, uploaded.id, suffix = "first")+ second = await _create(client, uploaded.id, suffix = "second")+ listed = await client.fine_tuning.jobs.list()+ events = await client.fine_tuning.jobs.list_events(second.id)+ await openai_fine_tuning._drain_once() # GPU still busy: nothing starts+ gpu["busy"] = False+ await openai_fine_tuning._drain_once() # the head starts; the GPU is now taken by it+ gpu["busy"] = True+ await openai_fine_tuning._drain_once()+ after_first = [await client.fine_tuning.jobs.retrieve(j.id) for j in (first, second)]+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return first, second, listed, events, after_first++ first, second, listed, events, (started, waiting) = _run(flow())+ assert first.status == second.status == "queued"+ assert [j.id for j in listed.data[:2]] == [second.id, first.id]+ assert events.data[0].message == "Waiting for the GPU: position 2 in the queue"+ assert [r.project_name for r in calls["start"]] == ["first", "second"]+ # The started job keeps the id the client was given.+ assert started.id == first.id and started.status == "validating_files"+ assert waiting.status == "queued"+++def test_a_queued_job_can_be_cancelled_and_never_starts(queue):+ client, calls, _live, gpu = queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ cancelled = await client.fine_tuning.jobs.cancel(job.id)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return cancelled, await client.fine_tuning.jobs.retrieve(job.id)++ cancelled, retrieved = _run(flow())+ assert cancelled.status == retrieved.status == "cancelled"+ assert calls["start"] == []+++def test_a_refused_start_fails_the_queued_job(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def refuse(request, current_subject, via_api_key):+ return TrainingJobResponse(+ job_id = "", status = "error", message = "The model is gated.", error = "gated"+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", refuse)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return await client.fine_tuning.jobs.retrieve(job.id), openai_fine_tuning._QUEUE++ job, remaining = _run(flow())+ assert job.status == "failed" and job.error.message == "The model is gated."+ assert remaining == []+++def test_another_account_cannot_see_or_cancel_a_queued_job(queue, monkeypatch):+ client, _calls, _live, _gpu = queue+ from utils.account_context import AccountContext++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(+ openai_fine_tuning,+ "current_account_id",+ lambda: AccountContext("bob", "bob").account_id,+ )+ statuses = []+ for call in (client.fine_tuning.jobs.retrieve, client.fine_tuning.jobs.cancel):+ try:+ await call(job.id)+ except openai.NotFoundError:+ statuses.append(404)+ listed = await client.fine_tuning.jobs.list()+ return statuses, [j.id for j in listed.data]++ statuses, listed = _run(flow())+ assert statuses == [404, 404] and listed == []+++def _forget_aliases(monkeypatch):+ # What a Studio restart leaves: nothing in memory, the database as it was.+ monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})+ monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())+++def test_a_started_queued_job_keeps_its_id_after_a_restart(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ _forget_aliases(monkeypatch)+ retrieved = await client.fine_tuning.jobs.retrieve(job.id)+ listed = await client.fine_tuning.jobs.list()+ # The started job as a cursor pages past it rather than ending the list.+ paged = await client.fine_tuning.jobs.list(after = job.id)+ return job, retrieved, listed, paged++ job, retrieved, listed, paged = _run(flow())+ assert retrieved.id == job.id and retrieved.status == "validating_files"+ assert [j.id for j in listed.data] == [job.id]+ assert paged.data == [] and not paged.has_more+++def test_cancelling_a_job_whose_start_is_in_flight_stops_it(queue, monkeypatch):+ client, calls, live, gpu = queue+ release = asyncio.Event()+ entered = asyncio.Event()+ start = training_routes.start_training++ async def slow_start(request, current_subject, via_api_key):+ entered.set()+ await release.wait()+ return await start(request, current_subject = current_subject, via_api_key = via_api_key)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", slow_start)+ gpu["busy"] = False+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())+ await entered.wait()+ cancelled = await client.fine_tuning.jobs.cancel(job.id)+ release.set()+ await drain+ return cancelled, await client.fine_tuning.jobs.retrieve(job.id)++ cancelled, retrieved = _run(flow())+ assert cancelled.status == retrieved.status == "cancelled"+ assert [b.expected_job_id for b in calls["stop"]] == [live["status"].job_id]+++def test_a_queued_failure_hides_host_paths_from_an_api_key(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def refuse(request, current_subject, via_api_key):+ message = "Dataset not found: /home/alice/.unsloth/studio/datasets/chats.jsonl"+ return TrainingJobResponse(job_id = "", status = "error", message = message, error = message)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", refuse)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return await client.fine_tuning.jobs.list_events(job.id)++ events = _run(flow())+ assert events.data[0].level == "error" and "/home/alice" not in events.data[0].message+++def test_a_diffusion_run_winning_the_gpu_keeps_the_job_queued(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def diffusion_first(request, current_subject, via_api_key):+ return TrainingJobResponse(+ job_id = "",+ status = "error",+ message = "Diffusion training already active",+ error = "Diffusion training already active",+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", diffusion_first)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return await client.fine_tuning.jobs.retrieve(job.id)++ assert _run(flow()).status == "queued"+++def test_two_creates_at_once_start_one_and_queue_the_other(queue, monkeypatch):+ client, calls, _live, gpu = queue+ gpu["busy"] = False+ start = training_routes.start_training++ async def start_and_take_the_gpu(request, current_subject, via_api_key):+ await asyncio.sleep(0.01)+ gpu["busy"] = True+ return await start(request, current_subject = current_subject, via_api_key = via_api_key)++ monkeypatch.setattr(training_routes, "start_training", start_and_take_the_gpu)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ return await asyncio.gather(_create(client, uploaded.id), _create(client, uploaded.id))++ jobs = _run(flow())+ assert sorted(j.status for j in jobs) == ["queued", "validating_files"]+ assert len(calls["start"]) == 1+++def test_a_job_cancelled_before_it_started_does_not_hide_newer_jobs(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)+ old = await _create(client, uploaded.id)+ await client.fine_tuning.jobs.cancel(old.id)+ gpu["busy"] = False+ newer = await _create(client, uploaded.id)+ listed = await client.fine_tuning.jobs.list()+ return old, newer, listed, await client.fine_tuning.jobs.retrieve(old.id)++ old, newer, listed, retrieved = _run(flow())+ assert listed.data[0].id == newer.id+ assert retrieved.status == "cancelled"+++def test_a_queued_job_keeps_its_creation_time_and_event_cursor(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)+ job = await _create(client, uploaded.id)+ events = await client.fine_tuning.jobs.list_events(job.id)+ again = await client.fine_tuning.jobs.list_events(job.id, after = events.data[0].id)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ _forget_aliases(monkeypatch)+ return job, again, await client.fine_tuning.jobs.retrieve(job.id)++ job, again, started = _run(flow())+ assert again.data == []+ assert started.created_at == job.created_at == 1_700_000_000+++def test_an_inference_request_in_flight_never_fails_a_queued_job(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def chat_in_flight(request, current_subject, via_api_key):+ raise openai_fine_tuning.HTTPException(+ status_code = 409,+ detail = "Cannot start training over the API while an inference request is in progress.",+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", chat_in_flight)+ gpu["busy"] = False+ for _ in range(openai_fine_tuning.QUEUE_IDLE_CONFLICTS + 2):+ await openai_fine_tuning._drain_once()+ return await client.fine_tuning.jobs.retrieve(job.id)++ assert _run(flow()).status == "queued"+++def test_a_job_id_store_failure_keeps_the_started_job(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ def locked():+ raise studio_db.sqlite3.OperationalError("database is locked")++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ await client.fine_tuning.jobs.list() # load the id table before it locks+ monkeypatch.setattr(openai_fine_tuning, "_alias_db", locked)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return job, await client.fine_tuning.jobs.retrieve(job.id)++ job, started = _run(flow())+ assert started.id == job.id and started.status == "validating_files"+++def test_queue_ids_are_stored_and_loaded_per_account(queue, monkeypatch):+ client, _calls, _live, gpu = queue+ from utils.account_context import AccountContext++ bob = AccountContext("bob", "bob")+ reads, writes = [], []+ alias_db = openai_fine_tuning._alias_db++ def spy():+ reads.append(openai_fine_tuning.current_account_id())+ return alias_db()++ save = openai_fine_tuning._save_alias++ def save_spy(*args):+ writes.append(openai_fine_tuning.current_account_id())+ return save(*args)++ monkeypatch.setattr(openai_fine_tuning, "_alias_db", spy)+ monkeypatch.setattr(openai_fine_tuning, "_save_alias", save_spy)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ # The drainer runs in whatever account context it was created in, not the job's.+ openai_fine_tuning._QUEUE[0].account = bob+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ await client.fine_tuning.jobs.list()+ monkeypatch.setattr(openai_fine_tuning, "current_account_id", lambda: bob.account_id)+ await client.fine_tuning.jobs.list()+ return job++ _run(flow())+ assert writes == [bob.account_id]+ # Bob's own table is read even though another account loaded its ids first.+ assert bob.account_id in reads and len(set(reads)) == 2+++def _transient_409(request, current_subject, via_api_key):+ raise openai_fine_tuning.HTTPException(+ status_code = 409,+ detail = "Cannot start training over the API while an inference request is in progress.",+ )+++def test_a_cancel_during_a_start_that_hits_a_transient_conflict_ends_the_job(queue, monkeypatch):+ client, calls, _live, gpu = queue+ release, entered = asyncio.Event(), asyncio.Event()++ async def slow_conflict(request, current_subject, via_api_key):+ entered.set()+ await release.wait()+ _transient_409(request, current_subject, via_api_key)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", slow_conflict)+ gpu["busy"] = False+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())+ await entered.wait()+ await client.fine_tuning.jobs.cancel(job.id)+ release.set()+ await drain+ return await client.fine_tuning.jobs.retrieve(job.id), list(openai_fine_tuning._QUEUE)++ job, remaining = _run(flow())+ assert job.status == "cancelled" and remaining == []+++def test_a_ui_run_winning_the_gpu_keeps_the_queued_job_waiting(queue, monkeypatch):+ client, _calls, _live, gpu = queue++ async def lost_to_ui(request, current_subject, via_api_key):+ gpu["busy"] = True # the UI run is now training+ return TrainingJobResponse(+ job_id = "",+ status = "error",+ message = "Training did not start.",+ error = "subprocess_start_failed",+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ job = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", lost_to_ui)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ return await client.fine_tuning.jobs.retrieve(job.id)++ assert _run(flow()).status == "queued"+++def test_a_create_during_a_chat_request_is_queued_not_refused(queue, monkeypatch):+ client, _calls, _live, gpu = queue+ gpu["busy"] = False++ async def chat_in_flight(request, current_subject, via_api_key):+ _transient_409(request, current_subject, via_api_key)++ monkeypatch.setattr(training_routes, "start_training", chat_in_flight)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ return await _create(client, uploaded.id)++ assert _run(flow()).status == "queued"+++def test_a_sidecar_swap_conflict_queues_and_a_crash_after_cancel_stays_cancelled(+ queue, monkeypatch+):+ client, _calls, _live, gpu = queue+ release, entered = asyncio.Event(), asyncio.Event()++ async def sidecar_swap(request, current_subject, via_api_key):+ raise openai_fine_tuning.HTTPException(+ status_code = 409,+ detail = "A transformers installation is replacing the latest sidecar; retry when it completes.",+ )++ async def crash_after_cancel(request, current_subject, via_api_key):+ entered.set()+ await release.wait()+ raise RuntimeError("worker spawn blew up")++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ monkeypatch.setattr(training_routes, "start_training", sidecar_swap)+ gpu["busy"] = False+ queued = await _create(client, uploaded.id)+ monkeypatch.setattr(training_routes, "start_training", crash_after_cancel)+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())+ await entered.wait()+ await client.fine_tuning.jobs.cancel(queued.id)+ release.set()+ await drain+ return queued, await client.fine_tuning.jobs.retrieve(queued.id)++ queued, after = _run(flow())+ assert queued.status == "queued" and after.status == "cancelled"