Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep same-second fine-tuning jobs in submission order across restarts (#16835)

oobabooga4ee2a51ebc84
diff --git a/studio/backend/routes/openai_fine_tuning.py b/studio/backend/routes/openai_fine_tuning.py
index 34be97aedeb..b775741000c 100644
--- a/studio/backend/routes/openai_fine_tuning.py
+++ b/studio/backend/routes/openai_fine_tuning.py
@@ -12,6 +12,8 @@ import json
import os
import re
import secrets
+import sqlite3
+import time
import weakref
from collections.abc import Iterator
from dataclasses import dataclass
@@ -520,6 +522,9 @@ _ALIAS_RUN: dict[str, tuple[str, Optional[str]]] = {}
_RUN_ALIAS: dict[str, str] = {}
_ALIAS_CREATED: dict[str, int] = {}
+# Persisted ns stamp: orders jobs created in the same second, also after a restart.
+_ALIAS_CREATED_NS: dict[str, int] = {}
+_last_created_ns = 0
_aliases_loaded: set[Optional[str]] = set()
_drainer: Optional[asyncio.Task] = None
@@ -546,6 +551,12 @@ def _alias_db():
f"CREATE TABLE IF NOT EXISTS {_ALIAS_TABLE} "
"(alias TEXT PRIMARY KEY, run_id TEXT NOT NULL, account_id TEXT, created_at INTEGER)"
)
+ if "created_ns" not in {row[1] for row in conn.execute(f"PRAGMA table_info({_ALIAS_TABLE})")}:
+ try:
+ conn.execute(f"ALTER TABLE {_ALIAS_TABLE} ADD COLUMN created_ns INTEGER")
+ except sqlite3.OperationalError as exc:
+ if "duplicate column" not in str(exc):
+ raise
return conn
@@ -556,12 +567,14 @@ def _load_aliases() -> None:
conn = _alias_db()
try:
rows = conn.execute(
- f"SELECT alias, run_id, account_id, created_at FROM {_ALIAS_TABLE}"
+ f"SELECT alias, run_id, account_id, created_at, created_ns FROM {_ALIAS_TABLE}"
).fetchall()
finally:
conn.close()
- for row in rows:
+ for *row, created_ns in rows:
_record_alias(*row)
+ if created_ns:
+ _ALIAS_CREATED_NS[row[0]] = created_ns
_aliases_loaded.add(account_id)
@@ -574,14 +587,16 @@ def _record_alias(
_ALIAS_CREATED[alias] = created_at
-def _save_alias(alias: str, run_id: str, account_id: Optional[str], created_at: int) -> None:
+def _save_alias(
+ alias: str, run_id: str, account_id: Optional[str], created_at: Optional[int]
+) -> None:
try:
conn = _alias_db()
try:
conn.execute(
- f"INSERT OR REPLACE INTO {_ALIAS_TABLE} (alias, run_id, account_id, created_at) "
- "VALUES (?, ?, ?, ?)",
- (alias, run_id, account_id, created_at),
+ f"INSERT OR REPLACE INTO {_ALIAS_TABLE} "
+ "(alias, run_id, account_id, created_at, created_ns) VALUES (?, ?, ?, ?, ?)",
+ (alias, run_id, account_id, created_at, _ALIAS_CREATED_NS.get(alias)),
)
conn.commit()
finally:
@@ -989,6 +1004,13 @@ def _ensure_drainer() -> None:
_drainer = asyncio.get_running_loop().create_task(_drain())
+def _next_created_ns() -> int:
+ global _last_created_ns
+ # Strictly increasing: Windows clocks can tick every ~16 ms.
+ _last_created_ns = max(time.time_ns(), _last_created_ns + 1)
+ return _last_created_ns
+
+
def _enqueue(request: TrainingStartRequest, current_subject: str, via_api_key: bool) -> _Queued:
entry = _Queued(
alias = f"ftjob_q{secrets.token_hex(8)}",
@@ -1000,6 +1022,7 @@ def _enqueue(request: TrainingStartRequest, current_subject: str, via_api_key: b
)
_QUEUE.append(entry)
_ALIASES[entry.alias] = entry
+ _ALIAS_CREATED_NS[entry.alias] = _next_created_ns()
_ensure_drainer()
return entry
@@ -1097,8 +1120,13 @@ async def create_fine_tuning_job(
raise _bad_request(
message if host_paths_visible(via_api_key) else redact_paths_in_text(message), 409
)
- created_at = int(datetime.now(timezone.utc).timestamp())
+ created_at = _now()
+ # Only the ns stamp is saved: created_at keeps coming from the run, as before.
+ record = (_job_id(started.job_id), started.job_id, current_account_id(), None)
+ _ALIAS_CREATED_NS[record[0]] = _next_created_ns()
+ _record_alias(*record)
_remember_start(started.job_id, request.model_dump(), created_at)
+ await asyncio.to_thread(_save_alias, *record)
return _for_caller(
_job_object(
started.job_id,
@@ -1134,11 +1162,12 @@ def _run_created(run: dict[str, Any]) -> int:
def _ordered_jobs(
pending: list[dict[str, Any]], live: dict[str, dict[str, Any]]
) -> Iterator[tuple[dict[str, Any], bool]]:
- # Newest created first, same-second ties by public id: a job keeps its place from queue to
- # history and across restarts. History is newest start first and no job starts before it is
- # created, so a buffered row created after the current row's start is final.
+ # Newest created first, same-second ties by the persisted ns stamp, then public id: a job
+ # keeps its place from queue to history and across restarts. History is newest start first
+ # and no job starts before it is created, so a buffered row created after the current row's
+ # start is final.
def item(created_at: int, public_id: str, seq: int, row: dict[str, Any], is_run: bool):
- return (-created_at, public_id, seq), row, is_run
+ return (-created_at, -_ALIAS_CREATED_NS.get(public_id, 0), public_id, seq), row, is_run
heap = [
item(job["created_at"] or 0, job["id"], seq, job, False)
diff --git a/studio/backend/tests/test_openai_fine_tuning.py b/studio/backend/tests/test_openai_fine_tuning.py
index 5fb043b0134..84ff8984dcb 100644
--- a/studio/backend/tests/test_openai_fine_tuning.py
+++ b/studio/backend/tests/test_openai_fine_tuning.py
@@ -556,6 +556,7 @@ def queue(studio, monkeypatch):
monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})
monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})
monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})
+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED_NS", {})
monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())
monkeypatch.setattr(openai_fine_tuning, "_ensure_drainer", lambda: None)
@@ -592,7 +593,7 @@ def test_a_busy_gpu_queues_jobs_and_starts_them_in_order(queue):
first, second, listed, events, (started, waiting) = _run(flow())
assert first.status == second.status == "queued"
- newest = sorted((first, second), key = lambda job: (-job.created_at, job.id))
+ newest = (second, first)
assert [j.id for j in listed.data[:2]] == [job.id for job in newest]
assert events.data[0].message == "Waiting for the GPU: position 2 in the queue"
assert [r.project_name for r in calls["start"]] == ["first", "second"]
@@ -667,6 +668,7 @@ def _forget_aliases(monkeypatch):
monkeypatch.setattr(openai_fine_tuning, "_RUN_ALIAS", {})
monkeypatch.setattr(openai_fine_tuning, "_ALIAS_RUN", {})
monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED", {})
+ monkeypatch.setattr(openai_fine_tuning, "_ALIAS_CREATED_NS", {})
monkeypatch.setattr(openai_fine_tuning, "_aliases_loaded", set())
@@ -691,6 +693,61 @@ def test_a_started_queued_job_keeps_its_id_after_a_restart(queue, monkeypatch):
assert paged.data == [] and not paged.has_more
+def test_an_alias_table_from_an_older_build_loads_and_gains_the_ns_column(queue):
+ table = openai_fine_tuning._ALIAS_TABLE
+ conn = studio_db.get_connection()
+ conn.execute(
+ f"CREATE TABLE {table} "
+ "(alias TEXT PRIMARY KEY, run_id TEXT NOT NULL, account_id TEXT, created_at INTEGER)"
+ )
+ conn.execute(f"INSERT INTO {table} VALUES ('ftjob_qold', 'run_old', NULL, 100)")
+ conn.commit()
+ conn.close()
+
+ openai_fine_tuning._load_aliases()
+ openai_fine_tuning._ALIAS_CREATED_NS["ftjob_qnew"] = 7
+ openai_fine_tuning._save_alias("ftjob_qnew", "run_new", None, 100)
+ conn = studio_db.get_connection()
+ rows = conn.execute(f"SELECT alias, created_ns FROM {table} ORDER BY alias").fetchall()
+ conn.close()
+ assert openai_fine_tuning._ALIAS_RUN["ftjob_qold"] == ("run_old", None)
+ assert [tuple(r) for r in rows] == [("ftjob_qnew", 7), ("ftjob_qold", None)]
+
+
+def test_jobs_created_within_one_clock_tick_list_newest_submitted_first(queue, monkeypatch):
+ client, _calls, _live, _gpu = queue
+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)
+ monkeypatch.setattr(openai_fine_tuning.time, "time_ns", lambda: 1_700_000_000 * 10**9)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ jobs = [await _create(client, uploaded.id, suffix = s) for s in "abc"]
+ return jobs, await client.fine_tuning.jobs.list()
+
+ jobs, listed = _run(flow())
+ assert [j.id for j in listed.data] == [j.id for j in jobs[::-1]]
+
+
+def test_a_job_started_directly_lists_ahead_of_an_older_same_second_job(queue, monkeypatch):
+ client, _calls, _live, gpu = queue
+ monkeypatch.setattr(openai_fine_tuning, "_now", lambda: 1_700_000_000)
+
+ async def flow():
+ uploaded = await client.files.create(file = ("chats.jsonl", CHATS), purpose = "fine-tune")
+ queued = await _create(client, uploaded.id, suffix = "queued")
+ await client.fine_tuning.jobs.cancel(queued.id)
+ gpu["busy"] = False
+ direct = await _create(client, uploaded.id, suffix = "direct")
+ first = await client.fine_tuning.jobs.list(limit = 1)
+ _forget_aliases(monkeypatch)
+ restarted = await client.fine_tuning.jobs.list(limit = 1)
+ return direct, first, restarted
+
+ direct, first, restarted = _run(flow())
+ assert not direct.id.startswith("ftjob_q")
+ assert first.data[0].id == restarted.data[0].id == direct.id
+
+
def test_cancelling_a_job_whose_start_is_in_flight_stops_it(queue, monkeypatch):
client, calls, live, gpu = queue
release = asyncio.Event()
@@ -2023,7 +2080,7 @@ def test_a_cursor_on_a_batch_member_that_starts_during_the_request_still_pages(
return jobs, seen
jobs, seen = _run(flow())
- newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))
+ newest = jobs[::-1]
assert seen == [job.id for job in newest] + ["ftjob_20261021_101500_ab12cd34"]
@@ -2050,14 +2107,14 @@ def test_a_batch_lists_under_its_job_ids_while_its_start_is_being_saved(batch_qu
return jobs, listed
jobs, listed = _run(flow())
- newest = sorted(jobs, key = lambda job: (-job.created_at, job.id))
+ newest = jobs[::-1]
assert [(j.id, j.status) for j in listed.data] == [
(job.id, "validating_files") for job in newest
]
@pytest.mark.parametrize("step", [10, 0])
[email protected]("persisted", [False, True])
[email protected]("persisted", [False, True, "restart"])
def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(
batch_queue, monkeypatch, persisted, step
):
@@ -2091,6 +2148,8 @@ def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(
if persisted:
history_row("job_20261021_101500_ab12cd34", "2026-11-04T01:00:00+00:00")
live["status"] = _Status()
+ if persisted == "restart":
+ _forget_aliases(monkeypatch)
after = first.data[0].id
rest = [j.id async for j in client.fine_tuning.jobs.list(after = after, limit = 1)]
return jobs, listed, [j.id for j in first.data] + rest
@@ -2098,9 +2157,7 @@ def test_a_job_batched_past_keeps_its_place_when_it_starts_between_pages(
(a, b, c), listed, walked = _run(flow())
assert [r.project_name for r in calls["batch"][0][0]] == ["a", "c"]
assert [r.project_name for r in calls["start"]] == ["b"]
- assert walked == listed and sorted(walked) == sorted([a.id, b.id, c.id])
- if step:
- assert walked == [c.id, b.id, a.id]
+ assert walked == listed == [c.id, b.id, a.id]
def test_history_lists_by_creation_time_reading_only_what_a_page_needs(monkeypatch):