Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep fine-tuning jobs in order when a batch starts ahead of them (#16832)
Nilayb0072c712d96
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.pyindex 8053fe2cfbd..a9e39c15ae6 100644--- a/studio/backend/routes/openai_fine_tuning.py+++ b/studio/backend/routes/openai_fine_tuning.py@@ -6,17 +6,17 @@from __future__ import annotationsimport asyncio+import heapqimport importlib.utilimport jsonimport osimport reimport secretsimport weakref-from collections import dequefrom collections.abc import Iteratorfrom dataclasses import dataclassfrom datetime import datetime, timezone-from itertools import chain, groupby+from itertools import chainfrom pathlib import Pathfrom typing import Any, Literal, Optional@@ -1131,19 +1131,28 @@ def _run_created(run: dict[str, Any]) -> int:def _ordered_jobs(pending: list[dict[str, Any]], live: dict[str, dict[str, Any]]) -> Iterator[tuple[dict[str, Any], bool]]:- # By creation time, queue entries first on ties: a started job keeps its place across pages.- remaining = deque(sorted(pending, key = lambda job: job["created_at"], reverse = True))- # Batch members share started_at: order them by creation time, as the live batch lists them.- runs = groupby(_iter_runs(), key = lambda run: run.get("started_at"))- runs = chain.from_iterable(sorted(g, key = _run_created, reverse = True) for _, g in runs)- # A live run can acquire its history row between the status and history reads.- history = ((run, True) for run in runs if run["id"] not in live)- for row, is_run in chain(((job, False) for job in live.values()), history):- created_at = _run_created(row) if is_run else row["created_at"]- while remaining and remaining[0]["created_at"] >= created_at:- yield remaining.popleft(), False- yield row, is_run- yield from ((job, False) for job in remaining)+ # Newest created first, same-second ties by public id: a job keeps its place from queue to+ # history and across restarts. History is newest start first and no job starts before it is+ # created, so a buffered row created after the current row's start is final.+ def item(created_at: int, public_id: str, seq: int, row: dict[str, Any], is_run: bool):+ return (-created_at, public_id, seq), row, is_run++ heap = [+ item(job["created_at"] or 0, job["id"], seq, job, False)+ for seq, job in enumerate(chain(pending, live.values()))+ ]+ heapq.heapify(heap)+ for seq, run in enumerate(_iter_runs(), len(heap)):+ # A live run can acquire its history row between the status and history reads.+ if run["id"] in live:+ continue+ started = _unix(run.get("started_at")) or 0+ while heap and -heap[0][0][0] > started:+ yield heapq.heappop(heap)[1:]+ public_id = _RUN_ALIAS.get(run["id"]) or _job_id(run["id"])+ heapq.heappush(heap, item(min(_run_created(run), started), public_id, seq, run, True))+ while heap:+ yield heapq.heappop(heap)[1:]def _list_job_page(diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.pyindex 723c613c788..fc867181666 100644--- a/studio/backend/tests/test_openai_fine_tuning.py+++ b/studio/backend/tests/test_openai_fine_tuning.py@@ -1275,6 +1275,103 @@ def test_listing_pages_through_a_batch_that_is_still_loading(batch_queue):assert sorted(seen) == sorted(j.id for j in jobs) and len(seen) == 3+def test_a_job_that_waited_for_a_batch_keeps_its_place_once_it_starts(batch_queue, monkeypatch):+ client, _calls, _live, gpu, _backend = batch_queue+ clock = {"t": 1_700_000_000}+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: clock["t"])++ def history_row(run_id, started_at):+ studio_db.create_run(+ id = run_id,+ model_name = "unsloth/Qwen3-0.6B",+ dataset_name = "chats.jsonl",+ config_json = json.dumps({"model_name": "unsloth/Qwen3-0.6B"}),+ started_at = started_at,+ total_steps = 4,+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = []+ for suffix in "abc":+ clock["t"] += 10+ extra = {"use_dora": True} if suffix == "b" else {}+ jobs.append(await _create(client, uploaded.id, suffix = suffix, extra_body = extra))+ first = await client.fine_tuning.jobs.list(limit = 1)+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ for run_id in openai_fine_tuning._BATCHES["job_20261021_101500_batch000"]:+ history_row(run_id, "2026-11-04T00:00:00+00:00")+ await openai_fine_tuning._drain_once()+ history_row("job_20261021_101500_ab12cd34", "2026-11-04T00:10:00+00:00")+ seen, after = [j.id for j in first.data], first.data[-1].id+ for _ in range(5):+ page = await client.fine_tuning.jobs.list(limit = 1, after = after)+ seen += [j.id for j in page.data]+ if not page.has_more:+ break+ after = page.data[-1].id+ return jobs, seen++ jobs, seen = _run(flow())+ assert seen[0] == jobs[2].id+ assert sorted(seen) == sorted(j.id for j in jobs)+++def test_paging_stays_complete_when_a_loading_batch_gets_its_history_rows(batch_queue, monkeypatch):+ client, _calls, live, gpu, backend = batch_queue+ clock = {"t": 1_700_000_000}+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: clock["t"])+ starts = []++ async def start_training_batch(requests, current_subject, via_api_key):+ backend.current_job_id = f"job_20261021_10150{len(starts)}_batch"+ run_ids = [f"{backend.current_job_id}_member{i}" for i in range(len(requests))]+ starts.append(run_ids)+ backend.running = list(run_ids)+ live["status"] = _Status(backend.current_job_id, "loading_model")+ return [+ TrainingJobResponse(job_id = run_id, status = "queued", message = "queued")+ for run_id in run_ids+ ]++ monkeypatch.setattr(training_routes, "start_training_batch", start_training_batch)++ def history_rows(run_ids, started_at):+ for run_id in run_ids:+ studio_db.create_run(+ id = run_id,+ model_name = "unsloth/Qwen3-0.6B",+ dataset_name = "chats.jsonl",+ config_json = json.dumps({"model_name": "unsloth/Qwen3-0.6B"}),+ started_at = started_at,+ total_steps = 4,+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = []+ for suffix in "abcd":+ clock["t"] += 10+ model = "unsloth/Qwen3-1.7B" if suffix in "bd" else "unsloth/Qwen3-0.6B"+ jobs.append(+ await client.fine_tuning.jobs.create(+ model = model, training_file = uploaded.id, suffix = suffix+ )+ )+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ history_rows(starts[0], "2026-11-04T00:00:00+00:00")+ await openai_fine_tuning._drain_once()+ first = await client.fine_tuning.jobs.list(limit = 2)+ history_rows(starts[1], "2026-11-04T00:10:00+00:00")+ rest = await client.fine_tuning.jobs.list(limit = 10, after = first.data[-1].id)+ return jobs, [j.id for j in first.data] + [j.id for j in rest.data]++ jobs, seen = _run(flow())+ assert seen == [jobs[i].id for i in (3, 2, 1, 0)]++@pytest.mark.parametrize("model_kind",[@@ -1887,3 +1984,104 @@ def test_a_batch_lists_in_the_same_order_before_and_after_its_history_rows_existpersisted = [{"id": "run_a", "started_at": started}, {"id": "run_c", "started_at": started}]assert order([], live) == order(persisted, {}) == ["ftjob_qc", "ftjob_qb", "ftjob_qa"]++[email protected]("step", [10, 0])[email protected]("persisted", [False, True])+def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(+ batch_queue, monkeypatch, persisted, step+):+ client, calls, live, gpu, _backend = batch_queue+ clock = {"t": 1_700_000_000}+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: clock["t"])++ def history_row(run_id, started_at):+ studio_db.create_run(+ id = run_id,+ model_name = "unsloth/Qwen3-0.6B",+ dataset_name = "chats.jsonl",+ config_json = json.dumps({"model_name": "unsloth/Qwen3-0.6B"}),+ started_at = started_at,+ total_steps = 4,+ )++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = []+ for suffix, extra in (("a", {}), ("b", {"use_dora": True}), ("c", {})):+ jobs.append(await _create(client, uploaded.id, suffix = suffix, extra_body = extra))+ clock["t"] += step+ gpu["busy"] = False+ await openai_fine_tuning._drain_once()+ for i in range(2):+ history_row(f"job_20261021_101500_member0{i}", "2026-11-04T00:00:00+00:00")+ first = await client.fine_tuning.jobs.list(limit = 1)+ listed = [j.id async for j in client.fine_tuning.jobs.list(limit = 1)]+ await openai_fine_tuning._drain_once()+ if persisted:+ history_row("job_20261021_101500_ab12cd34", "2026-11-04T01:00:00+00:00")+ live["status"] = _Status()+ after = first.data[0].id+ rest = [j.id async for j in client.fine_tuning.jobs.list(after = after, limit = 1)]+ return jobs, listed, [j.id for j in first.data] + rest++ (a, b, c), listed, walked = _run(flow())+ assert [r.project_name for r in calls["batch"][0][0]] == ["a", "c"]+ assert [r.project_name for r in calls["start"]] == ["b"]+ assert walked == listed and sorted(walked) == sorted([a.id, b.id, c.id])+ if step:+ assert walked == [c.id, b.id, a.id]+++def test_history_lists_by_creation_time_reading_only_what_a_page_needs(monkeypatch):+ runs = [+ {"id": f"run{i}", "started_at": f"2026-11-04T00:{i // 60:02d}:{i % 60:02d}+00:00"}+ for i in range(300)+ ]+ created = {run["id"]: openai_fine_tuning._unix(run["started_at"]) for run in runs}+ created["run_waited"] = created["run0"] - 1+ history = [{"id": "run_waited", "started_at": "2026-11-04T01:00:00+00:00"}] + runs[::-1]+ pulled = []++ def iter_runs():+ for run in history:+ pulled.append(run["id"])+ yield run++ monkeypatch.setattr(openai_fine_tuning, "_iter_runs", iter_runs)+ monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {i: f"ftjob_q{i}" for i in created})+ monkeypatch.setattr(+ openai_fine_tuning, "_ALIAS_CREATED", {f"ftjob_q{i}": t for i, t in created.items()}+ )++ jobs = openai_fine_tuning._ordered_jobs([], {})+ assert [next(jobs)[0]["id"] for _ in range(3)] == ["run299", "run298", "run297"]+ assert len(pulled) <= 6+ assert [row["id"] for row, _ in jobs][-2:] == ["run0", "run_waited"]+++def test_a_queue_poll_reads_one_history_row_when_the_backlog_waited(monkeypatch):+ history = [+ {"id": f"run{i}", "started_at": f"2026-11-04T{i // 60:02d}:{i % 60:02d}:00+00:00"}+ for i in range(300, 0, -1)+ ]+ t0 = openai_fine_tuning._unix("2026-11-04T00:00:00+00:00")+ pulled = []++ def iter_runs():+ for run in history:+ pulled.append(run["id"])+ yield run++ monkeypatch.setattr(openai_fine_tuning, "_iter_runs", iter_runs)+ monkeypatch.setattr(+ openai_fine_tuning, "_RUN_ALIAS", {r["id"]: f"ftjob_{r['id']}" for r in history}+ )+ monkeypatch.setattr(+ openai_fine_tuning, "_ALIAS_CREATED", {f"ftjob_{r['id']}": t0 for r in history}+ )+ pending = [{"id": f"ftjob_q{i}", "created_at": t0 + 20_000 - i} for i in range(2)]++ page = openai_fine_tuning._list_job_page(pending, {}, None, None, 1, False)+ assert [job["id"] for job in page["data"]] == ["ftjob_q0"]+ assert page["has_more"] and pulled == ["run300"]