Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep same-second fine-tuning jobs in submission order across restarts (#16835)
oobabooga4ee2a51ebc84
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.pyindex 34be97aedeb..b775741000c 100644--- a/studio/backend/routes/openai_fine_tuning.py+++ b/studio/backend/routes/openai_fine_tuning.py@@ -12,6 +12,8 @@ import jsonimport osimport reimport secrets+import sqlite3+import timeimport weakreffrom collections.abc import Iteratorfrom dataclasses import dataclass@@ -520,6 +522,9 @@ _ALIAS_RUN: dict[str, tuple[str, Optional[str]]] = {}_RUN_ALIAS: dict[str, str] = {}_ALIAS_CREATED: dict[str, int] = {}+# Persisted ns stamp: orders jobs created in the same second, also after a restart.+_ALIAS_CREATED_NS: dict[str, int] = {}+_last_created_ns = 0_aliases_loaded: set[Optional[str]] = set()_drainer: Optional[asyncio.Task] = None@@ -546,6 +551,12 @@ def _alias_db():f"CREATE TABLE IF NOT EXISTS {_ALIAS_TABLE} ""(alias TEXT PRIMARY KEY, run_id TEXT NOT NULL, account_id TEXT, created_at INTEGER)")+ if "created_ns" not in {row[1] for row in conn.execute(f"PRAGMA table_info({_ALIAS_TABLE})")}:+ try:+ conn.execute(f"ALTER TABLE {_ALIAS_TABLE} ADD COLUMN created_ns INTEGER")+ except sqlite3.OperationalError as exc:+ if "duplicate column" not in str(exc):+ raisereturn conn@@ -556,12 +567,14 @@ def _load_aliases() -> None:conn = _alias_db()try:rows = conn.execute(- f"SELECT alias, run_id, account_id, created_at FROM {_ALIAS_TABLE}"+ f"SELECT alias, run_id, account_id, created_at, created_ns FROM {_ALIAS_TABLE}").fetchall()finally:conn.close()- for row in rows:+ for *row, created_ns in rows:_record_alias(*row)+ if created_ns:+ _ALIAS_CREATED_NS[row[0]] = created_ns_aliases_loaded.add(account_id)@@ -574,14 +587,16 @@ def _record_alias(_ALIAS_CREATED[alias] = created_at-def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:+def _save_alias(+ alias: str, run_id: str, account_id: Optional[str], created_at: Optional[int]+) -> None:try:conn = _alias_db()try:conn.execute(- f"INSERT OR REPLACE INTO {_ALIAS_TABLE} (alias, run_id, account_id, created_at) "- "VALUES (?, ?, ?, ?)",- (alias, run_id, account_id, created_at),+ f"INSERT OR REPLACE INTO {_ALIAS_TABLE} "+ "(alias, run_id, account_id, created_at, created_ns) VALUES (?, ?, ?, ?, ?)",+ (alias, run_id, account_id, created_at, _ALIAS_CREATED_NS.get(alias)),)conn.commit()finally:@@ -989,6 +1004,13 @@ def _ensure_drainer() -> None:_drainer = asyncio.get_running_loop().create_task(_drain())+def _next_created_ns() -> int:+ global _last_created_ns+ # Strictly increasing: Windows clocks can tick every ~16 ms.+ _last_created_ns = max(time.time_ns(), _last_created_ns + 1)+ return _last_created_ns++def _enqueue(request: TrainingStartRequest, current_subject: str, via_api_key: bool) -> _Queued:entry = _Queued(alias = f"ftjob_q{secrets.token_hex(8)}",@@ -1000,6 +1022,7 @@ def _enqueue(request: TrainingStartRequest, current_subject: str, via_api_key: b)_QUEUE.append(entry)_ALIASES[entry.alias] = entry+ _ALIAS_CREATED_NS[entry.alias] = _next_created_ns()_ensure_drainer()return entry@@ -1097,8 +1120,13 @@ async def create_fine_tuning_job(raise _bad_request(message if host_paths_visible(via_api_key) else redact_paths_in_text(message), 409)- created_at = int(datetime.now(timezone.utc).timestamp())+ created_at = _now()+ # Only the ns stamp is saved: created_at keeps coming from the run, as before.+ record = (_job_id(started.job_id), started.job_id, current_account_id(), None)+ _ALIAS_CREATED_NS[record[0]] = _next_created_ns()+ _record_alias(*record)_remember_start(started.job_id, request.model_dump(), created_at)+ await asyncio.to_thread(_save_alias, *record)return _for_caller(_job_object(started.job_id,@@ -1134,11 +1162,12 @@ def _run_created(run: dict[str, Any]) -> int:def _ordered_jobs(pending: list[dict[str, Any]], live: dict[str, dict[str, Any]]) -> Iterator[tuple[dict[str, Any], bool]]:- # Newest created first, same-second ties by public id: a job keeps its place from queue to- # history and across restarts. History is newest start first and no job starts before it is- # created, so a buffered row created after the current row's start is final.+ # Newest created first, same-second ties by the persisted ns stamp, then public id: a job+ # keeps its place from queue to history and across restarts. History is newest start first+ # and no job starts before it is created, so a buffered row created after the current row's+ # start is final.def item(created_at: int, public_id: str, seq: int, row: dict[str, Any], is_run: bool):- return (-created_at, public_id, seq), row, is_run+ return (-created_at, -_ALIAS_CREATED_NS.get(public_id, 0), public_id, seq), row, is_runheap = [item(job["created_at"] or 0, job["id"], seq, job, False)diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.pyindex 5fb043b0134..84ff8984dcb 100644--- a/studio/backend/tests/test_openai_fine_tuning.py+++ b/studio/backend/tests/test_openai_fine_tuning.py@@ -556,6 +556,7 @@ def queue(studio, monkeypatch):monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED_NS", {})monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())monkeypatch.setattr(openai_fine_tuning, "_ensure_drainer", lambda: None)@@ -592,7 +593,7 @@ def test_a_busy_gpu_queues_jobs_and_starts_them_in_order(queue):first, second, listed, events, (started, waiting) = _run(flow())assert first.status == second.status == "queued"- newest = sorted((first, second), key = lambda job: (-job.created_at, job.id))+ newest = (second, first)assert [j.id for j in listed.data[:2]] == [job.id for job in newest]assert events.data[0].message == "Waiting for the GPU: position 2 in the queue"assert [r.project_name for r in calls["start"]] == ["first", "second"]@@ -667,6 +668,7 @@ def _forget_aliases(monkeypatch):monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED_NS", {})monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())@@ -691,6 +693,61 @@ def test_a_started_queued_job_keeps_its_id_after_a_restart(queue, monkeypatch):assert paged.data == [] and not paged.has_more+def test_an_alias_table_from_an_older_build_loads_and_gains_the_ns_column(queue):+ table = openai_fine_tuning._ALIAS_TABLE+ conn = studio_db.get_connection()+ conn.execute(+ f"CREATE TABLE {table} "+ "(alias TEXT PRIMARY KEY, run_id TEXT NOT NULL, account_id TEXT, created_at INTEGER)"+ )+ conn.execute(f"INSERT INTO {table} VALUES ('ftjob_qold', 'run_old', NULL, 100)")+ conn.commit()+ conn.close()++ openai_fine_tuning._load_aliases()+ openai_fine_tuning._ALIAS_CREATED_NS["ftjob_qnew"] = 7+ openai_fine_tuning._save_alias("ftjob_qnew", "run_new", None, 100)+ conn = studio_db.get_connection()+ rows = conn.execute(f"SELECT alias, created_ns FROM {table} ORDER BY alias").fetchall()+ conn.close()+ assert openai_fine_tuning._ALIAS_RUN["ftjob_qold"] == ("run_old", None)+ assert [tuple(r) for r in rows] == [("ftjob_qnew", 7), ("ftjob_qold", None)]+++def test_jobs_created_within_one_clock_tick_list_newest_submitted_first(queue, monkeypatch):+ client, _calls, _live, _gpu = queue+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)+ monkeypatch.setattr(openai_fine_tuning.time, "time_ns", lambda: 1_700_000_000 * 10**9)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "abc"]+ return jobs, await client.fine_tuning.jobs.list()++ jobs, listed = _run(flow())+ assert [j.id for j in listed.data] == [j.id for j in jobs[::-1]]+++def test_a_job_started_directly_lists_ahead_of_an_older_same_second_job(queue, monkeypatch):+ client, _calls, _live, gpu = queue+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)++ async def flow():+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")+ queued = await _create(client, uploaded.id, suffix = "queued")+ await client.fine_tuning.jobs.cancel(queued.id)+ gpu["busy"] = False+ direct = await _create(client, uploaded.id, suffix = "direct")+ first = await client.fine_tuning.jobs.list(limit = 1)+ _forget_aliases(monkeypatch)+ restarted = await client.fine_tuning.jobs.list(limit = 1)+ return direct, first, restarted++ direct, first, restarted = _run(flow())+ assert not direct.id.startswith("ftjob_q")+ assert first.data[0].id == restarted.data[0].id == direct.id++def test_cancelling_a_job_whose_start_is_in_flight_stops_it(queue, monkeypatch):client, calls, live, gpu = queuerelease = asyncio.Event()@@ -2023,7 +2080,7 @@ def test_a_cursor_on_a_batch_member_that_starts_during_the_request_still_pages(return jobs, seenjobs, seen = _run(flow())- newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))+ newest = jobs[::-1]assert seen == [job.id for job in newest] + ["ftjob_20261021_101500_ab12cd34"]@@ -2050,14 +2107,14 @@ def test_a_batch_lists_under_its_job_ids_while_its_start_is_being_saved(batch_qureturn jobs, listedjobs, listed = _run(flow())- newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))+ newest = jobs[::-1]assert [(j.id, j.status) for j in listed.data] == [(job.id, "validating_files") for job in newest]@pytest.mark.parametrize("step", [10, 0])[email protected]("persisted", [False, True])[email protected]("persisted", [False, True, "restart"])def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(batch_queue, monkeypatch, persisted, step):@@ -2091,6 +2148,8 @@ def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(if persisted:history_row("job_20261021_101500_ab12cd34", "2026-11-04T01:00:00+00:00")live["status"] = _Status()+ if persisted == "restart":+ _forget_aliases(monkeypatch)after = first.data[0].idrest = [j.id async for j in client.fine_tuning.jobs.list(after = after, limit = 1)]return jobs, listed, [j.id for j in first.data] + rest@@ -2098,9 +2157,7 @@ def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages((a, b, c), listed, walked = _run(flow())assert [r.project_name for r in calls["batch"][0][0]] == ["a", "c"]assert [r.project_name for r in calls["start"]] == ["b"]- assert walked == listed and sorted(walked) == sorted([a.id, b.id, c.id])- if step:- assert walked == [c.id, b.id, a.id]+ assert walked == listed == [c.id, b.id, a.id]def test_history_lists_by_creation_time_reading_only_what_a_page_needs(monkeypatch):