Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep fine-tuning job pages complete when a batch starts (#16823)

Nilay2e10d8035543
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.py
index a9e39c15ae6..34be97aedeb 100644
--- a/studio/backend/routes/openai_fine_tuning.py
+++ b/studio/backend/routes/openai_fine_tuning.py
@@ -560,18 +560,21 @@ def _load_aliases() -> None:
).fetchall()
finally:
conn.close()
- for alias, run_id, account_id, created_at in rows:
- _ALIAS_RUN[alias] = (run_id, account_id)
- _RUN_ALIAS[run_id] = alias
- if created_at:
- _ALIAS_CREATED[alias] = created_at
+ for row in rows:
+ _record_alias(*row)
_aliases_loaded.add(account_id)
-def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:
+def _record_alias(
+ alias: str, run_id: str, account_id: Optional[str], created_at: Optional[int]
+) -> None:
_ALIAS_RUN[alias] = (run_id, account_id)
_RUN_ALIAS[run_id] = alias
- _ALIAS_CREATED[alias] = created_at
+ if created_at:
+ _ALIAS_CREATED[alias] = created_at
+
+
+def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:
try:
conn = _alias_db()
try:
@@ -893,15 +896,12 @@ async def _start_queued(entry: _Queued) -> bool:
return False
_finish(entry, "failed", started.message or started.error or "Training did not start")
return True
- # In the job's own account database: the drainer may carry whichever account started it.
- await arun_as(
- entry.account,
- asyncio.to_thread(
- _save_alias, entry.alias, started.job_id, entry.account.account_id, entry.created_at
- ),
- )
+ record = (entry.alias, started.job_id, entry.account.account_id, entry.created_at)
+ _record_alias(*record)
_remember_start(started.job_id, entry.request.model_dump(), entry.created_at)
_finish(entry, "started")
+ # In the job's own account database: the drainer may carry whichever account started it.
+ await arun_as(entry.account, asyncio.to_thread(_save_alias, *record))
if entry.cancel_requested:
# Cancelled while its start was in flight: stop it as a cancel of the running job would.
await arun_as(entry.account, _stop_run(started.job_id, entry.subject))
@@ -943,6 +943,11 @@ async def _start_batch(group: list[_Queued]) -> Optional[bool]:
batch_id = getattr(get_training_backend(), "current_job_id", None)
if batch_id:
_remember_batch(batch_id, run_ids)
+ # Every member leaves the queue as its run before any await, so no listing sees it as neither.
+ for member, run_id in zip(group, run_ids):
+ _record_alias(member.alias, run_id, member.account.account_id, member.created_at)
+ _remember_start(run_id, member.request.model_dump(), member.created_at)
+ _finish(member, "started")
for member, run_id in zip(group, run_ids):
await arun_as(
member.account,
@@ -950,8 +955,6 @@ async def _start_batch(group: list[_Queued]) -> Optional[bool]:
_save_alias, member.alias, run_id, member.account.account_id, member.created_at
),
)
- _remember_start(run_id, member.request.model_dump(), member.created_at)
- _finish(member, "started")
for member, run_id in zip(group, run_ids):
if member.cancel_requested:
await arun_as(member.account, _cancel_run(run_id, member.subject))
@@ -1180,6 +1183,14 @@ def _list_job_page(
return {"object": "list", "data": jobs, "has_more": False}
+def _pending_jobs(via_api_key: bool) -> dict[str, dict[str, Any]]:
+ return {
+ entry.alias: _for_caller(_queued_job(entry), via_api_key)
+ for entry in reversed(_ALIASES.values())
+ if _own_alias(entry.alias) is not None
+ }
+
+
@router.get("/fine_tuning/jobs")
async def list_fine_tuning_jobs(
after: Optional[str] = None,
@@ -1188,6 +1199,8 @@ async def list_fine_tuning_jobs(
via_api_key: bool = Depends(authenticated_via_api_key),
):
await asyncio.to_thread(_load_aliases)
+ # Before the awaits: a job that starts meanwhile may be in neither the status nor the history.
+ queued = _pending_jobs(via_api_key)
after_run = await _resolve(after) if after else None
status = await _current_status(current_subject, via_api_key)
live_id = _status_field(status, "job_id") if status is not None else None
@@ -1201,11 +1214,16 @@ async def list_fine_tuning_jobs(
after = job["id"]
# After the awaits, so a job that started meanwhile is listed as its run.
- pending = [
- _for_caller(_queued_job(entry), via_api_key)
- for entry in reversed(_ALIASES.values())
- if _own_alias(entry.alias) is not None
- ]
+ current = _pending_jobs(via_api_key)
+ pending = [job for alias, job in current.items() if alias not in queued]
+ for alias, job in queued.items():
+ run = _ALIAS_RUN.get(alias)
+ if (
+ alias in current
+ or run is None
+ or (run[0] not in live and await asyncio.to_thread(get_run, run[0]) is None)
+ ):
+ pending.append(current.get(alias, job))
return await asyncio.to_thread(
_list_job_page, pending, live, after, after_run, limit, via_api_key
)
diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.py
index fc867181666..5fb043b0134 100644
--- a/studio/backend/tests/test_openai_fine_tuning.py
+++ b/studio/backend/tests/test_openai_fine_tuning.py
@@ -4,6 +4,7 @@
import asyncio
import json
+import threading
import httpx
import pytest
@@ -591,7 +592,8 @@ def test_a_busy_gpu_queues_jobs_and_starts_them_in_order(queue):
first, second, listed, events, (started, waiting) = _run(flow())
assert first.status == second.status == "queued"
- assert [j.id for j in listed.data[:2]] == [second.id, first.id]
+ newest = sorted((first, second), key = lambda job: (-job.created_at, job.id))
+ assert [j.id for j in listed.data[:2]] == [job.id for job in newest]
assert events.data[0].message == "Waiting for the GPU: position 2 in the queue"
assert [r.project_name for r in calls["start"]] == ["first", "second"]
@@ -1986,6 +1988,74 @@ def test_a_batch_lists_in_the_same_order_before_and_after_its_history_rows_exist
assert order([], live) == order(persisted, {}) == ["ftjob_qc", "ftjob_qb", "ftjob_qa"]
+def test_a_cursor_on_a_batch_member_that_starts_during_the_request_still_pages(
+ batch_queue, monkeypatch, tmp_path
+):
+ client, _calls, live, gpu, _backend = batch_queue
+ previous = _finished_run(tmp_path, tmp_path / "chats.jsonl")
+ live["status"] = _Status(previous, "completed")
+ get_run = openai_fine_tuning.get_run
+ reached, released = threading.Event(), threading.Event()
+
+ def get_run_held_once(run_id):
+ if run_id == previous and not reached.is_set():
+ reached.set()
+ released.wait(timeout = 30)
+ return get_run(run_id)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "ab"]
+ first = await client.fine_tuning.jobs.list(limit = 1)
+ monkeypatch.setattr(openai_fine_tuning, "get_run", get_run_held_once)
+ second = asyncio.ensure_future(
+ client.fine_tuning.jobs.list(after = first.data[0].id, limit = 1)
+ )
+ await asyncio.to_thread(reached.wait, 30)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ released.set()
+ page = await second
+ seen = [j.id for j in first.data + page.data]
+ while page.has_more:
+ page = await client.fine_tuning.jobs.list(after = page.data[-1].id, limit = 1)
+ seen += [j.id for j in page.data]
+ return jobs, seen
+
+ jobs, seen = _run(flow())
+ newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))
+ assert seen == [job.id for job in newest] + ["ftjob_20261021_101500_ab12cd34"]
+
+
+def test_a_batch_lists_under_its_job_ids_while_its_start_is_being_saved(batch_queue, monkeypatch):
+ client, _calls, _live, gpu, _backend = batch_queue
+ save_alias = openai_fine_tuning._save_alias
+ saving, release = threading.Event(), threading.Event()
+
+ def save_alias_held(*args):
+ saving.set()
+ release.wait(timeout = 30)
+ save_alias(*args)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "ab"]
+ monkeypatch.setattr(openai_fine_tuning, "_save_alias", save_alias_held)
+ gpu["busy"] = False
+ drain = asyncio.ensure_future(openai_fine_tuning._drain_once())
+ await asyncio.to_thread(saving.wait, 30)
+ listed = await client.fine_tuning.jobs.list()
+ release.set()
+ await drain
+ return jobs, listed
+
+ jobs, listed = _run(flow())
+ newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))
+ assert [(j.id, j.status) for j in listed.data] == [
+ (job.id, "validating_files") for job in newest
+ ]
+
+
@pytest.mark.parametrize("step", [10, 0])
@pytest.mark.parametrize("persisted", [False, True])
def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(