Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep fine-tuning job pages complete when a batch starts (#16823)
Nilay2e10d8035543
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.pyindex a9e39c15ae6..34be97aedeb 100644--- a/studio/backend/routes/openai_fine_tuning.py+++ b/studio/backend/routes/openai_fine_tuning.py@@ -560,18 +560,21 @@ def _load_aliases() -> None:).fetchall()finally:conn.close()- for alias, run_id, account_id, created_at in rows:- _ALIAS_RUN[alias] = (run_id, account_id)- _RUN_ALIAS[run_id] = alias- if created_at:- _ALIAS_CREATED[alias] = created_at+ for row in rows:+ _record_alias(*row)_aliases_loaded.add(account_id)-def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:+def _record_alias(+ alias: str, run_id: str, account_id: Optional[str], created_at: Optional[int]+) -> None:_ALIAS_RUN[alias] = (run_id, account_id)_RUN_ALIAS[run_id] = alias- _ALIAS_CREATED[alias] = created_at+ if created_at:+ _ALIAS_CREATED[alias] = created_at+++def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:try:conn = _alias_db()try:@@ -893,15 +896,12 @@ async def _start_queued(entry: _Queued) -> bool:return False_finish(entry, "failed", started.message or started.error or "Training did not start")return True- # In the job's own account database: the drainer may carry whichever account started it.- await arun_as(- entry.account,- asyncio.to_thread(- _save_alias, entry.alias, started.job_id, entry.account.account_id, entry.created_at- ),- )+ record = (entry.alias, started.job_id, entry.account.account_id, entry.created_at)+ _record_alias(*record)_remember_start(started.job_id, entry.request.model_dump(), entry.created_at)_finish(entry, "started")+ # In the job's own account database: the drainer may carry whichever account started it.+ await arun_as(entry.account, asyncio.to_thread(_save_alias, *record))if entry.cancel_requested:# Cancelled while its start was in flight: stop it as a cancel of the running job would.await arun_as(entry.account, _stop_run(started.job_id, entry.subject))@@ -943,6 +943,11 @@ async def _start_batch(group: list[_Queued]) -> Optional[bool]:batch_id = getattr(get_training_backend(), "current_job_id", None)if batch_id:_remember_batch(batch_id, run_ids)+ # Every member leaves the queue as its run before any await, so no listing sees it as neither.+ for member, run_id in zip(group, run_ids):+ _record_alias(member.alias, run_id, member.account.account_id, member.created_at)+ _remember_start(run_id, member.request.model_dump(), member.created_at)+ _finish(member, "started")for member, run_id in zip(group, run_ids):await arun_as(member.account,@@ -950,8 +955,6 @@ async def _start_batch(group: list[_Queued]) -> Optional[bool]:_save_alias, member.alias, run_id, member.account.account_id, member.created_at),)- _remember_start(run_id, member.request.model_dump(), member.created_at)- _finish(member, "started")for member, run_id in zip(group, run_ids):if member.cancel_requested:await arun_as(member.account, _cancel_run(run_id, member.subject))@@ -1180,6 +1183,14 @@ def _list_job_page(return {"object": "list", "data": jobs, "has_more": False}+def _pending_jobs(via_api_key: bool) -> dict[str, dict[str, Any]]:+ return {+ entry.alias: _for_caller(_queued_job(entry), via_api_key)+ for entry in reversed(_ALIASES.values())+ if _own_alias(entry.alias) is not None+ }++@router.get("/fine_tuning/jobs")async def list_fine_tuning_jobs(after: Optional[str] = None,@@ -1188,6 +1199,8 @@ async def list_fine_tuning_jobs(via_api_key: bool = Depends(authenticated_via_api_key),):await asyncio.to_thread(_load_aliases)+ # Before the awaits: a job that starts meanwhile may be in neither the status nor the history.+ queued = _pending_jobs(via_api_key)after_run = await _resolve(after) if after else Nonestatus = await _current_status(current_subject, via_api_key)live_id = _status_field(status, "job_id") if status is not None else None@@ -1201,11 +1214,16 @@ async def list_fine_tuning_jobs(after = job["id"]# After the awaits, so a job that started meanwhile is listed as its run.- pending = [- _for_caller(_queued_job(entry), via_api_key)- for entry in reversed(_ALIASES.values())- if _own_alias(entry.alias) is not None- ]+ current = _pending_jobs(via_api_key)+ pending = [job for alias, job in current.items() if alias not in queued]+ for alias, job in queued.items():+ run = _ALIAS_RUN.get(alias)+ if (+ alias in current+ or run is None+ or (run[0] not in live and await asyncio.to_thread(get_run, run[0]) is None)+ ):+ pending.append(current.get(alias, job))return await asyncio.to_thread(_list_job_page, pending, live, after, after_run, limit, via_api_key)diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.pyindex fc867181666..5fb043b0134 100644--- a/studio/backend/tests/test_openai_fine_tuning.py+++ b/studio/backend/tests/test_openai_fine_tuning.py@@ -4,6 +4,7 @@import asyncioimport json+import threadingimport httpximport pytest@@ -591,7 +592,8 @@ def test_a_busy_gpu_queues_jobs_and_starts_them_in_order(queue):first, second, listed, events, (started, waiting) = _run(flow())assert first.status == second.status == "queued"- assert [j.id for j in listed.data[:2]] == [second.id, first.id]+ newest = sorted((first, second), key = lambda job: (-job.created_at, job.id))+ assert [j.id for j in listed.data[:2]] == [job.id for job in newest]assert events.data[0].message == "Waiting for the GPU: position 2 in the queue"assert [r.project_name for r in calls["start"]] == ["first", "second"]@@ -1986,6 +1988,74 @@ def test_a_batch_lists_in_the_same_order_before_and_after_its_history_rows_existassert order([], live) == order(persisted, {}) == ["ftjob_qc", "ftjob_qb", "ftjob_qa"]+def test_a_cursor_on_a_batch_member_that_starts_during_the_request_still_pages(+ batch_queue, monkeypatch, tmp_path+):+ client, _calls, live, gpu, _backend = batch_queue+ previous = _finished_run(tmp_path, tmp_path / "chats.jsonl")+ live["status"] = _Status(previous, "completed")+ get_run = openai_fine_tuning.get_run+ reached, released = threading.Event(), threading.Event()++ def get_run_held_once(run_id):+ if run_id == previous and not reached.is_set():+ reached.set()+ released.wait(timeout = 30)+ return get_run(run_id)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "ab"]+ first = await client.fine_tuning.jobs.list(limit = 1)+ monkeypatch.setattr(openai_fine_tuning, "get_run", get_run_held_once)+ second = asyncio.ensure_future(+ client.fine_tuning.jobs.list(after = first.data[0].id, limit = 1)+ )+ await asyncio.to_thread(reached.wait, 30)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ released.set()+ page = await second+ seen = [j.id for j in first.data + page.data]+ while page.has_more:+ page = await client.fine_tuning.jobs.list(after = page.data[-1].id, limit = 1)+ seen += [j.id for j in page.data]+ return jobs, seen++ jobs, seen = _run(flow())+ newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))+ assert seen == [job.id for job in newest] + ["ftjob_20261021_101500_ab12cd34"]+++def test_a_batch_lists_under_its_job_ids_while_its_start_is_being_saved(batch_queue, monkeypatch):+ client, _calls, _live, gpu, _backend = batch_queue+ save_alias = openai_fine_tuning._save_alias+ saving, release = threading.Event(), threading.Event()++ def save_alias_held(*args):+ saving.set()+ release.wait(timeout = 30)+ save_alias(*args)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "ab"]+ monkeypatch.setattr(openai_fine_tuning, "_save_alias", save_alias_held)+ gpu["busy"] = False+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())+ await asyncio.to_thread(saving.wait, 30)+ listed = await client.fine_tuning.jobs.list()+ release.set()+ await drain+ return jobs, listed++ jobs, listed = _run(flow())+ newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))+ assert [(j.id, j.status) for j in listed.data] == [+ (job.id, "validating_files") for job in newest+ ]++@pytest.mark.parametrize("step", [10, 0])@pytest.mark.parametrize("persisted", [False, True])def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(