Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep fine-tuning jobs in order when a batch starts ahead of them (#16832)

Nilayb0072c712d96
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.py
index 8053fe2cfbd..a9e39c15ae6 100644
--- a/studio/backend/routes/openai_fine_tuning.py
+++ b/studio/backend/routes/openai_fine_tuning.py
@@ -6,17 +6,17 @@
from __future__ import annotations
import asyncio
+import heapq
import importlib.util
import json
import os
import re
import secrets
import weakref
-from collections import deque
from collections.abc import Iterator
from dataclasses import dataclass
from datetime import datetime, timezone
-from itertools import chain, groupby
+from itertools import chain
from pathlib import Path
from typing import Any, Literal, Optional
@@ -1131,19 +1131,28 @@ def _run_created(run: dict[str, Any]) -> int:
def _ordered_jobs(
pending: list[dict[str, Any]], live: dict[str, dict[str, Any]]
) -> Iterator[tuple[dict[str, Any], bool]]:
- # By creation time, queue entries first on ties: a started job keeps its place across pages.
- remaining = deque(sorted(pending, key = lambda job: job["created_at"], reverse = True))
- # Batch members share started_at: order them by creation time, as the live batch lists them.
- runs = groupby(_iter_runs(), key = lambda run: run.get("started_at"))
- runs = chain.from_iterable(sorted(g, key = _run_created, reverse = True) for _, g in runs)
- # A live run can acquire its history row between the status and history reads.
- history = ((run, True) for run in runs if run["id"] not in live)
- for row, is_run in chain(((job, False) for job in live.values()), history):
- created_at = _run_created(row) if is_run else row["created_at"]
- while remaining and remaining[0]["created_at"] >= created_at:
- yield remaining.popleft(), False
- yield row, is_run
- yield from ((job, False) for job in remaining)
+ # Newest created first, same-second ties by public id: a job keeps its place from queue to
+ # history and across restarts. History is newest start first and no job starts before it is
+ # created, so a buffered row created after the current row's start is final.
+ def item(created_at: int, public_id: str, seq: int, row: dict[str, Any], is_run: bool):
+ return (-created_at, public_id, seq), row, is_run
+
+ heap = [
+ item(job["created_at"] or 0, job["id"], seq, job, False)
+ for seq, job in enumerate(chain(pending, live.values()))
+ ]
+ heapq.heapify(heap)
+ for seq, run in enumerate(_iter_runs(), len(heap)):
+ # A live run can acquire its history row between the status and history reads.
+ if run["id"] in live:
+ continue
+ started = _unix(run.get("started_at")) or 0
+ while heap and -heap[0][0][0] > started:
+ yield heapq.heappop(heap)[1:]
+ public_id = _RUN_ALIAS.get(run["id"]) or _job_id(run["id"])
+ heapq.heappush(heap, item(min(_run_created(run), started), public_id, seq, run, True))
+ while heap:
+ yield heapq.heappop(heap)[1:]
def _list_job_page(
diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.py
index 723c613c788..fc867181666 100644
--- a/studio/backend/tests/test_openai_fine_tuning.py
+++ b/studio/backend/tests/test_openai_fine_tuning.py
@@ -1275,6 +1275,103 @@ def test_listing_pages_through_a_batch_that_is_still_loading(batch_queue):
assert sorted(seen) == sorted(j.id for j in jobs) and len(seen) == 3
+def test_a_job_that_waited_for_a_batch_keeps_its_place_once_it_starts(batch_queue, monkeypatch):
+ client, _calls, _live, gpu, _backend = batch_queue
+ clock = {"t": 1_700_000_000}
+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: clock["t"])
+
+ def history_row(run_id, started_at):
+ studio_db.create_run(
+ id = run_id,
+ model_name = "unsloth/Qwen3-0.6B",
+ dataset_name = "chats.jsonl",
+ config_json = json.dumps({"model_name": "unsloth/Qwen3-0.6B"}),
+ started_at = started_at,
+ total_steps = 4,
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = []
+ for suffix in "abc":
+ clock["t"] += 10
+ extra = {"use_dora": True} if suffix == "b" else {}
+ jobs.append(await _create(client, uploaded.id, suffix = suffix, extra_body = extra))
+ first = await client.fine_tuning.jobs.list(limit = 1)
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ for run_id in openai_fine_tuning._BATCHES["job_20261021_101500_batch000"]:
+ history_row(run_id, "2026-11-04T00:00:00+00:00")
+ await openai_fine_tuning._drain_once()
+ history_row("job_20261021_101500_ab12cd34", "2026-11-04T00:10:00+00:00")
+ seen, after = [j.id for j in first.data], first.data[-1].id
+ for _ in range(5):
+ page = await client.fine_tuning.jobs.list(limit = 1, after = after)
+ seen += [j.id for j in page.data]
+ if not page.has_more:
+ break
+ after = page.data[-1].id
+ return jobs, seen
+
+ jobs, seen = _run(flow())
+ assert seen[0] == jobs[2].id
+ assert sorted(seen) == sorted(j.id for j in jobs)
+
+
+def test_paging_stays_complete_when_a_loading_batch_gets_its_history_rows(batch_queue, monkeypatch):
+ client, _calls, live, gpu, backend = batch_queue
+ clock = {"t": 1_700_000_000}
+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: clock["t"])
+ starts = []
+
+ async def start_training_batch(requests, current_subject, via_api_key):
+ backend.current_job_id = f"job_20261021_10150{len(starts)}_batch"
+ run_ids = [f"{backend.current_job_id}_member{i}" for i in range(len(requests))]
+ starts.append(run_ids)
+ backend.running = list(run_ids)
+ live["status"] = _Status(backend.current_job_id, "loading_model")
+ return [
+ TrainingJobResponse(job_id = run_id, status = "queued", message = "queued")
+ for run_id in run_ids
+ ]
+
+ monkeypatch.setattr(training_routes, "start_training_batch", start_training_batch)
+
+ def history_rows(run_ids, started_at):
+ for run_id in run_ids:
+ studio_db.create_run(
+ id = run_id,
+ model_name = "unsloth/Qwen3-0.6B",
+ dataset_name = "chats.jsonl",
+ config_json = json.dumps({"model_name": "unsloth/Qwen3-0.6B"}),
+ started_at = started_at,
+ total_steps = 4,
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = []
+ for suffix in "abcd":
+ clock["t"] += 10
+ model = "unsloth/Qwen3-1.7B" if suffix in "bd" else "unsloth/Qwen3-0.6B"
+ jobs.append(
+ await client.fine_tuning.jobs.create(
+ model = model, training_file = uploaded.id, suffix = suffix
+ )
+ )
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ history_rows(starts[0], "2026-11-04T00:00:00+00:00")
+ await openai_fine_tuning._drain_once()
+ first = await client.fine_tuning.jobs.list(limit = 2)
+ history_rows(starts[1], "2026-11-04T00:10:00+00:00")
+ rest = await client.fine_tuning.jobs.list(limit = 10, after = first.data[-1].id)
+ return jobs, [j.id for j in first.data] + [j.id for j in rest.data]
+
+ jobs, seen = _run(flow())
+ assert seen == [jobs[i].id for i in (3, 2, 1, 0)]
+
+
@pytest.mark.parametrize(
"model_kind",
[
@@ -1887,3 +1984,104 @@ def test_a_batch_lists_in_the_same_order_before_and_after_its_history_rows_exist
persisted = [{"id": "run_a", "started_at": started}, {"id": "run_c", "started_at": started}]
assert order([], live) == order(persisted, {}) == ["ftjob_qc", "ftjob_qb", "ftjob_qa"]
+
+
[email protected]("step", [10, 0])
[email protected]("persisted", [False, True])
+def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(
+ batch_queue, monkeypatch, persisted, step
+):
+ client, calls, live, gpu, _backend = batch_queue
+ clock = {"t": 1_700_000_000}
+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: clock["t"])
+
+ def history_row(run_id, started_at):
+ studio_db.create_run(
+ id = run_id,
+ model_name = "unsloth/Qwen3-0.6B",
+ dataset_name = "chats.jsonl",
+ config_json = json.dumps({"model_name": "unsloth/Qwen3-0.6B"}),
+ started_at = started_at,
+ total_steps = 4,
+ )
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = []
+ for suffix, extra in (("a", {}), ("b", {"use_dora": True}), ("c", {})):
+ jobs.append(await _create(client, uploaded.id, suffix = suffix, extra_body = extra))
+ clock["t"] += step
+ gpu["busy"] = False
+ await openai_fine_tuning._drain_once()
+ for i in range(2):
+ history_row(f"job_20261021_101500_member0{i}", "2026-11-04T00:00:00+00:00")
+ first = await client.fine_tuning.jobs.list(limit = 1)
+ listed = [j.id async for j in client.fine_tuning.jobs.list(limit = 1)]
+ await openai_fine_tuning._drain_once()
+ if persisted:
+ history_row("job_20261021_101500_ab12cd34", "2026-11-04T01:00:00+00:00")
+ live["status"] = _Status()
+ after = first.data[0].id
+ rest = [j.id async for j in client.fine_tuning.jobs.list(after = after, limit = 1)]
+ return jobs, listed, [j.id for j in first.data] + rest
+
+ (a, b, c), listed, walked = _run(flow())
+ assert [r.project_name for r in calls["batch"][0][0]] == ["a", "c"]
+ assert [r.project_name for r in calls["start"]] == ["b"]
+ assert walked == listed and sorted(walked) == sorted([a.id, b.id, c.id])
+ if step:
+ assert walked == [c.id, b.id, a.id]
+
+
+def test_history_lists_by_creation_time_reading_only_what_a_page_needs(monkeypatch):
+ runs = [
+ {"id": f"run{i}", "started_at": f"2026-11-04T00:{i // 60:02d}:{i % 60:02d}+00:00"}
+ for i in range(300)
+ ]
+ created = {run["id"]: openai_fine_tuning._unix(run["started_at"]) for run in runs}
+ created["run_waited"] = created["run0"] - 1
+ history = [{"id": "run_waited", "started_at": "2026-11-04T01:00:00+00:00"}] + runs[::-1]
+ pulled = []
+
+ def iter_runs():
+ for run in history:
+ pulled.append(run["id"])
+ yield run
+
+ monkeypatch.setattr(openai_fine_tuning, "_iter_runs", iter_runs)
+ monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {i: f"ftjob_q{i}" for i in created})
+ monkeypatch.setattr(
+ openai_fine_tuning, "_ALIAS_CREATED", {f"ftjob_q{i}": t for i, t in created.items()}
+ )
+
+ jobs = openai_fine_tuning._ordered_jobs([], {})
+ assert [next(jobs)[0]["id"] for _ in range(3)] == ["run299", "run298", "run297"]
+ assert len(pulled) <= 6
+ assert [row["id"] for row, _ in jobs][-2:] == ["run0", "run_waited"]
+
+
+def test_a_queue_poll_reads_one_history_row_when_the_backlog_waited(monkeypatch):
+ history = [
+ {"id": f"run{i}", "started_at": f"2026-11-04T{i // 60:02d}:{i % 60:02d}:00+00:00"}
+ for i in range(300, 0, -1)
+ ]
+ t0 = openai_fine_tuning._unix("2026-11-04T00:00:00+00:00")
+ pulled = []
+
+ def iter_runs():
+ for run in history:
+ pulled.append(run["id"])
+ yield run
+
+ monkeypatch.setattr(openai_fine_tuning, "_iter_runs", iter_runs)
+ monkeypatch.setattr(
+ openai_fine_tuning, "_RUN_ALIAS", {r["id"]: f"ftjob_{r['id']}" for r in history}
+ )
+ monkeypatch.setattr(
+ openai_fine_tuning, "_ALIAS_CREATED", {f"ftjob_{r['id']}": t0 for r in history}
+ )
+ pending = [{"id": f"ftjob_q{i}", "created_at": t0 + 20_000 - i} for i in range(2)]
+
+ page = openai_fine_tuning._list_job_page(pending, {}, None, None, 1, False)
+ assert [job["id"] for job in page["data"]] == ["ftjob_q0"]
+ assert page["has_more"] and pulled == ["run300"]