Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep users signed in when auth.db is busy (#14151)
Etherl8cf09febfedb
diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.pyindex c2e4436176a..bb5503dae53 100644--- a/studio/backend/auth/authentication.py+++ b/studio/backend/auth/authentication.py@@ -266,6 +266,13 @@ def is_desktop_access_token(token: str) -> bool:return account is not None and account.is_owner+def new_refresh_token_value() -> Tuple[str, str]:+ """A fresh opaque refresh token and its ISO expiry, not yet stored."""+ token = secrets.token_urlsafe(48)+ expires_at = datetime.now(timezone.utc) + timedelta(days = REFRESH_TOKEN_EXPIRE_DAYS)+ return token, expires_at.isoformat()++def create_refresh_token(subject: str,*,@@ -276,12 +283,11 @@ def create_refresh_token(JWTs); expire after REFRESH_TOKEN_EXPIRE_DAYS. ``secret`` stamps the token with the credential version thecaller verified, so a rotation cannot leave a token minted from the replaced credential valid."""- token = secrets.token_urlsafe(48)- expires_at = datetime.now(timezone.utc) + timedelta(days = REFRESH_TOKEN_EXPIRE_DAYS)+ token, expires_at = new_refresh_token_value()save_refresh_token(token,subject,- expires_at.isoformat(),+ expires_at,is_desktop = desktop,secret_gen = credential_generation(secret) if secret is not None else None,)diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.pyindex 691228a4e3e..ff4ce8c972f 100644--- a/studio/backend/auth/storage.py+++ b/studio/backend/auth/storage.py@@ -10,6 +10,7 @@ import hashlibimport hmacimport ipaddressimport json+import loggingimport osimport reimport secrets@@ -18,11 +19,13 @@ import tempfileimport threadingfrom datetime import datetime, timedelta, timezoneimport uuid-from typing import Iterator, Optional, Tuple+from typing import Iterator, NamedTuple, Optional, Tuplefrom storage import sqlite_gatefrom utils.paths import auth_db_path, ensure_dir+logger = logging.getLogger(__name__)+DB_PATH = auth_db_path()DEFAULT_ADMIN_USERNAME = "unsloth"@@ -264,10 +267,12 @@ def credential_generation_guard(username: str, expect_gen: Optional[str]) -> Ite_auth_schema_ready: set[tuple[str, int, int, int]] = set()+_BUSY_TIMEOUT_MS = 5000+-def get_connection() -> sqlite3.Connection:+def get_connection(*, check_same_thread: bool = True) -> sqlite3.Connection:ensure_dir(DB_PATH.parent)- conn = sqlite_gate.connect(DB_PATH)+ conn = sqlite_gate.connect(DB_PATH, check_same_thread = check_same_thread)# sqlite3.connect would create the DB 0644 under a 022 umask, exposing identity secrets and password hashes.for _path, _mode in ((DB_PATH.parent, 0o700), (DB_PATH, 0o600)):try:@@ -278,10 +283,12 @@ def get_connection() -> sqlite3.Connection:# Set busy_timeout before journal_mode=WAL: switching journal mode needs a lock and would otherwise# raise SQLITE_BUSY.try:- conn.execute("PRAGMA busy_timeout=5000")+ conn.execute(f"PRAGMA busy_timeout={int(_BUSY_TIMEOUT_MS)}")conn.execute("PRAGMA journal_mode=WAL")except sqlite3.Error:pass+ # Per connection, so before the schema-cache return below.+ sqlite_gate.apply_wal_synchronous(conn)file_stat = DB_PATH.stat()schema_key = (str(DB_PATH),@@ -387,6 +394,49 @@ _ACCOUNT_COLUMNS = (_owner_id_repaired: set[str] = set()+# Without a keeper each close is the last WAL participant and checkpoints under sqlite_gate's+# process-wide lock. Lifespan-only, never lazy: tests swap and unlink auth.db.+_wal_keeper_lock = threading.Lock()+_wal_keeper: Optional[tuple[str, sqlite3.Connection]] = None+++def _close_keeper(conn: sqlite3.Connection) -> None:+ try:+ conn.close()+ except Exception as exc:+ logger.warning("Could not close the auth.db WAL keeper: %s", exc)+++def open_wal_keeper() -> bool:+ """Hold auth.db's WAL open for the server's lifetime, replacing any earlier keeper."""+ global _wal_keeper+ with _wal_keeper_lock:+ if _wal_keeper is not None:+ _close_keeper(_wal_keeper[1])+ _wal_keeper = None+ conn = get_connection(check_same_thread = False)+ try:+ mode = conn.execute("PRAGMA journal_mode").fetchone()[0]+ except (sqlite3.Error, TypeError, IndexError) as exc:+ _close_keeper(conn)+ logger.warning("Could not read auth.db journal mode: %s", exc)+ return False+ if not isinstance(mode, str) or mode.lower() != "wal":+ _close_keeper(conn)+ logger.info("auth.db is in %s mode, not WAL; WAL keeper not engaged.", mode)+ return False+ _wal_keeper = (str(DB_PATH.resolve()), conn)+ return True+++def close_wal_keeper() -> None:+ global _wal_keeper+ with _wal_keeper_lock:+ if _wal_keeper is not None:+ _close_keeper(_wal_keeper[1])+ _wal_keeper = None++def _repair_owner_account_id(conn: sqlite3.Connection) -> None:db_key = str(DB_PATH)if db_key in _owner_id_repaired:@@ -1458,42 +1508,113 @@ def save_refresh_token(conn.close()+def _take_refresh_token(conn: sqlite3.Connection, token: str, now: str):+ """Inside the caller's BEGIN IMMEDIATE: purge expired rows, then delete-and-return the token's+ row. Returns ``(row, user)`` (the auth_user row) when the token is live and its credential+ generation still matches, else None; the delete stands either way."""+ conn.execute(+ "DELETE FROM refresh_tokens WHERE expires_at < ?",+ (now,),+ )+ row = conn.execute(+ f"""+ DELETE FROM refresh_tokens+ WHERE token_hash {_FENCED_HASH_SQL} AND expires_at >= ?+ RETURNING username, is_desktop, secret_gen+ """,+ (*_hash_candidates(_hash_token(token)), now),+ ).fetchone()+ if row is None:+ return None+ user = conn.execute(+ f"""+ SELECT {_SECRET_SQL} AS jwt_secret, role, account_id, must_change_password+ FROM auth_user WHERE username = ?+ """,+ (row["username"],),+ ).fetchone()+ if user is None:+ return None+ secret = user["jwt_secret"]+ if row["secret_gen"] is not None and row["secret_gen"] != credential_generation(secret):+ return None+ return row, user++def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]:"""Atomically validate-and-delete a refresh token for single-use rotation. DELETE RETURNING fusesvalidate and delete into one statement so two concurrent refresh requests cannot both consumethe same token. Returns ``(username, is_desktop, jwt_secret)``; the caller must mint thereplacement tokens against that secret so a rotation landing mid-refresh cannot issue apost-rotation session from a pre-rotation token."""- token_hash = _hash_token(token)now = datetime.now(timezone.utc).isoformat()conn = get_connection()try:# One transaction with the delete: an unstamped legacy row has no generation, so a later read could# hand a reset's new secret to an older token.conn.execute("BEGIN IMMEDIATE")+ taken = _take_refresh_token(conn, token, now)+ conn.commit()+ if taken is None:+ return None+ row, user = taken+ return row["username"], bool(row["is_desktop"]), user["jwt_secret"]+ finally:+ conn.close()+++class RotatedRefresh(NamedTuple):+ username: str+ is_desktop: bool+ jwt_secret: str+ account_id: Optional[str]+ must_change_password: bool+++def rotate_refresh_token(+ token: str, new_token: str, new_expires_at: str+) -> Optional[RotatedRefresh]:+ """Consume ``token`` and store ``new_token`` in its place, in one transaction. If anything fails+ before the commit, including a busy auth.db, the old token is still valid, so a retry can+ succeed instead of logging the user out. None means the old token was invalid or expired."""+ from utils.account_context import ROLE_OWNER++ now = datetime.now(timezone.utc).isoformat()+ conn = get_connection()+ try:+ conn.execute("BEGIN IMMEDIATE")+ taken = _take_refresh_token(conn, token, now)+ if taken is None:+ conn.commit()+ return None+ row, user = taken+ username = row["username"]+ is_desktop = bool(row["is_desktop"]) and user["role"] == ROLE_OWNER+ secret = user["jwt_secret"]conn.execute(- "DELETE FROM refresh_tokens WHERE expires_at < ?",- (now,),- )- cur = conn.execute(- f"""- DELETE FROM refresh_tokens- WHERE token_hash {_FENCED_HASH_SQL} AND expires_at >= ?- RETURNING username, is_desktop, secret_gen+ """+ INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop, secret_gen)+ VALUES (?, ?, ?, ?, ?)""",- (*_hash_candidates(token_hash), now),+ (+ _fenced_hash(_hash_token(new_token), username),+ username,+ new_expires_at,+ int(is_desktop),+ credential_generation(secret),+ ),)- row = cur.fetchone()- if row is None:- conn.commit()- return None- secret = _current_secret(conn, row["username"])conn.commit()- if secret is None:- return None- if row["secret_gen"] is not None and row["secret_gen"] != credential_generation(secret):- return None- return row["username"], bool(row["is_desktop"]), secret+ return RotatedRefresh(+ username = username,+ is_desktop = is_desktop,+ jwt_secret = secret,+ account_id = user["account_id"],+ must_change_password = False if is_desktop else bool(user["must_change_password"]),+ )+ except Exception:+ conn.rollback()+ raisefinally:conn.close()@@ -1861,7 +1982,13 @@ def validate_api_key_account(raw_key: str, *, touch: bool = True) -> Optional[Tuconn = get_connection()try:if touch:- conn.execute("BEGIN IMMEDIATE")+ try:+ conn.execute("BEGIN IMMEDIATE")+ except sqlite3.OperationalError as exc:+ if not sqlite_gate.is_busy_error(exc):+ raise+ # last_used_at is informational: a busy auth.db must not fail the request.+ touch = Falsecur = conn.execute(f"""SELECT k.id, k.username, k.name, k.is_active, k.expires_at,diff --git a/studio/backend/main.py b/studio/backend/main.pyindex d9cbfdce257..f2954dbcbf7 100644--- a/studio/backend/main.py+++ b/studio/backend/main.py@@ -840,6 +840,11 @@ async def lifespan(app: FastAPI):open_wal_keeper()except Exception as exc:_lifespan_log.warning("studio.db WAL keeper failed at startup: %s", exc)+ # Same for auth.db, before account reconciliation and admin seeding open it.+ try:+ storage.open_wal_keeper()+ except Exception as exc:+ _lifespan_log.warning("auth.db WAL keeper failed at startup: %s", exc)from utils.account_context import OWNER as _owner_account, run_as as _run_as@@ -1076,6 +1081,10 @@ async def lifespan(app: FastAPI):from storage.studio_db import close_wal_keeperclose_wal_keeper()+ try:+ storage.close_wal_keeper()+ except Exception as exc:+ _lifespan_log.warning("auth.db WAL keeper failed to close: %s", exc)app = FastAPI(@@ -1800,6 +1809,7 @@ app.add_middleware("X-Unsloth-Refusal","x-typesafe-request-id","X-Unsloth-Monitor-ID",+ "Retry-After",*_hub_endpoint_proxy.EXPOSED_HEADERS,*_browser_routes.EXPOSED_HEADERS,],diff --git a/studio/backend/routes/auth.py b/studio/backend/routes/auth.pyindex 5668569b967..67c40b7882a 100644--- a/studio/backend/routes/auth.py+++ b/studio/backend/routes/auth.py@@ -11,6 +11,7 @@ import importlib.utilimport ipaddressimport osimport shlex+import sqlite3import sysimport threadingimport time@@ -33,6 +34,8 @@ from models.auth import ()from models.users import Tokenfrom auth import storage, hashing, policy+from loggers import get_logger+from storage import sqlite_gatefrom utils.account_context import OWNER_ACCOUNT_IDfrom auth.authentication import (authenticated_via_desktop_jwt,@@ -43,11 +46,13 @@ from auth.authentication import (get_current_subject,get_current_subject_allow_password_change,is_keyless,+ new_refresh_token_value,refresh_access_token,security,)router = APIRouter()+logger = get_logger(__name__)def _account_id_of(username: str) -> "str | None":@@ -613,30 +618,37 @@ def desktop_login(payload: DesktopLoginRequest, request: Request) -> Token | Res)+# Sync def: rotation is one transaction, so a busy auth.db wait must not freeze the event loop.@router.post("/refresh", response_model = Token)-async def refresh(payload: RefreshTokenRequest) -> Token:+def refresh(payload: RefreshTokenRequest) -> Token:"""Exchange a refresh token for a new access+refresh pair (single-use)."""- consumed = storage.consume_refresh_token(payload.refresh_token)- if consumed is None:+ new_refresh_token, new_expires_at = new_refresh_token_value()+ try:+ rotated = storage.rotate_refresh_token(+ payload.refresh_token, new_refresh_token, new_expires_at+ )+ except sqlite3.OperationalError as exc:+ if not sqlite_gate.is_busy_error(exc):+ raise+ logger.warning("Token refresh deferred: auth.db is busy (%s)", exc)+ raise HTTPException(+ status_code = status.HTTP_503_SERVICE_UNAVAILABLE,+ detail = "Unsloth Studio is busy. Try again in a moment.",+ headers = {"Retry-After": "2"},+ )+ if rotated is None:raise HTTPException(status_code = status.HTTP_401_UNAUTHORIZED,detail = "Invalid or expired refresh token",)- username, is_desktop, jwt_secret = consumed- account = storage.get_account(username)- if is_desktop:- is_desktop = account is not None and account.is_owner- new_access_token = create_access_token(subject = username, desktop = is_desktop, secret = jwt_secret)- new_refresh_token = create_refresh_token(- subject = username, desktop = is_desktop, secret = jwt_secret- )-return Token(- access_token = new_access_token,+ access_token = create_access_token(+ subject = rotated.username, desktop = rotated.is_desktop, secret = rotated.jwt_secret+ ),refresh_token = new_refresh_token,token_type = "bearer",- must_change_password = False if is_desktop else storage.requires_password_change(username),- account_id = account.account_id if account is not None else None,+ must_change_password = rotated.must_change_password,+ account_id = rotated.account_id,)diff --git a/studio/backend/storage/sqlite_gate.py b/studio/backend/storage/sqlite_gate.pyindex 0c9dc0e5c71..d9877c78a54 100644--- a/studio/backend/storage/sqlite_gate.py+++ b/studio/backend/storage/sqlite_gate.py@@ -40,3 +40,25 @@ def connect(database: str | os.PathLike[str], **kwargs: Any) -> sqlite3.Connecti"""sqlite3.connect with opens and closes serialized process-wide."""with _CONNECTION_GATE:return sqlite3.connect(str(database), factory = GatedConnection, **kwargs)+++def is_busy_error(exc: sqlite3.OperationalError) -> bool:+ """Is this the writer lock being held elsewhere, rather than a real fault? sqlite reports it as+ plain OperationalError, so the message is the only signal."""+ message = str(exc).lower()+ return "locked" in message or "busy" in message+++def apply_wal_synchronous(conn: sqlite3.Connection) -> None:+ """synchronous=NORMAL, only while the file is really in WAL mode (per connection).++ FULL fsyncs every commit under the writer lock (one busy-disk commit blocked 37s); NORMAL is+ sqlite's recommended WAL pairing, durable across app crashes. journal_mode=WAL silently declines+ on filesystems without shared memory, and NORMAL on a rollback journal drops the fsync that+ keeps it consistent, so this reads the mode in force and those installs keep FULL."""+ try:+ mode = conn.execute("PRAGMA journal_mode").fetchone()[0]+ except (sqlite3.Error, TypeError, IndexError):+ return+ if isinstance(mode, str) and mode.lower() == "wal":+ conn.execute("PRAGMA synchronous=NORMAL")diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.pyindex 6eb38a932fc..47eecfb191a 100644--- a/studio/backend/storage/studio_db.py+++ b/studio/backend/storage/studio_db.py@@ -1308,12 +1308,9 @@ def bulk_upsert_prompt_lists(lists: list[dict]) -> int:conn.close()-def is_sqlite_busy_error(exc: sqlite3.OperationalError) -> bool:- """Is this the writer lock being held elsewhere, rather than a real fault? sqlite reports it as- plain OperationalError, so the message is the only signal. Lives here because studio.db is the- contended file and several modules need the same answer; storage.api_usage_db delegates to it."""- message = str(exc).lower()- return "locked" in message or "busy" in message+# Old names kept: storage.api_usage_db and core.research_runs import them from here.+is_sqlite_busy_error = sqlite_gate.is_busy_error+_apply_wal_synchronous = sqlite_gate.apply_wal_synchronous# sqlite's own default, kept for every caller that may run on the event loop@@ -1324,25 +1321,6 @@ _BUSY_TIMEOUT_SECONDS = 5.0_CONTENDED_BUSY_TIMEOUT_SECONDS = 30.0-def _apply_wal_synchronous(conn: sqlite3.Connection) -> None:- """Drop to synchronous=NORMAL, but only while the file is really in WAL mode. Under WAL, sqlite- still defaults to synchronous=FULL, which fsyncs on every commit while holding the writer lock:- on a machine whose disk is busy one commit blocked for 37s, and every other writer spent that- window timing out with "database is locked". NORMAL is sqlite's own recommended pairing for WAL:- it can lose the last transactions to a host power loss, but the database is never corrupted and- commits stay durable across an application crash. The WAL check is not decoration. `PRAGMA- journal_mode=WAL` silently declines on filesystems without proper shared-memory support (network- shares, some FUSE and container-mounted paths), leaving the file on a rollback journal where- NORMAL drops the very fsync that keeps it consistent. Those installs keep FULL. journal_mode is- persistent in the file, so this reads what is in force rather than what was requested."""- try:- mode = conn.execute("PRAGMA journal_mode").fetchone()[0]- except (sqlite3.Error, TypeError, IndexError):- return- if isinstance(mode, str) and mode.lower() == "wal":- conn.execute("PRAGMA synchronous=NORMAL")--def reset_schema_state_for_tests() -> None:with _schema_lock:_schema_ready.clear()diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 5ae43195ada..95b03f86a5e 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -486,6 +486,17 @@ def _isolate_wal_keepers():studio_db._wal_unsupported.intersection_update(unsupported)[email protected](autouse = True)+def _isolate_auth_wal_keeper():+ """Close an auth.db keeper a test opened, so it never outlives that test's DB_PATH."""+ auth_storage = sys.modules.get("auth.storage")+ before = auth_storage._wal_keeper if auth_storage is not None else None+ yield+ auth_storage = sys.modules.get("auth.storage")+ if auth_storage is not None and auth_storage._wal_keeper is not before:+ auth_storage.close_wal_keeper()++@pytest.fixture(autouse = True)def _isolate_audio_gallery(monkeypatch, tmp_path):"""Keep generated-clip persistence out of the developer's real gallery.diff --git a/studio/backend/tests/test_auth_db_lock_contention.py b/studio/backend/tests/test_auth_db_lock_contention.pynew file mode 100644index 00000000000..eaec4403e5e--- /dev/null+++ b/studio/backend/tests/test_auth_db_lock_contention.py@@ -0,0 +1,352 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A busy auth.db must not log the user out.++On a Colab session a large GGUF download saturated the disk, /api/auth/refresh hit "database is+locked" after the busy timeout, the route answered a plain 500 after the old token was already+consumed, and the UI signed the user out. These pin the three links: auth.db commits skip the WAL+fsync, the refresh rotation is one transaction (a failure leaves the old token valid), and a busy+database answers a retryable 503.+"""++import hashlib+import inspect+import logging+import secrets+import sqlite3+import threading+from pathlib import Path++import pytest+from fastapi import FastAPI+from fastapi.testclient import TestClient++import routes.auth as auth_routes+from auth import policy, storage+from auth.authentication import new_refresh_token_value+from storage import sqlite_gate++FULL, NORMAL = 2, 1+++def fresh_db(tmp_path, monkeypatch):+ monkeypatch.setattr(storage, "DB_PATH", tmp_path / "auth" / "auth.db")+ monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", tmp_path / ".bootstrap_password")+ monkeypatch.setattr(storage, "_bootstrap_password", None)+ policy.invalidate_account_cache()+ yield+ policy.invalidate_account_cache()+++def owner(fresh_db):+ storage.create_initial_user(+ username = storage.DEFAULT_ADMIN_USERNAME,+ password = "human-password-123",+ jwt_secret = secrets.token_urlsafe(64),+ must_change_password = False,+ )+ return storage.DEFAULT_ADMIN_USERNAME+++def client(owner):+ app = FastAPI()+ app.include_router(auth_routes.router, prefix = "/api/auth")+ with TestClient(app) as http:+ yield http+++def _refresh(http, token):+ return http.post("/api/auth/refresh", json = {"refresh_token": token})+++def _issue(username):+ token, expires_at = new_refresh_token_value()+ storage.save_refresh_token(token, username, expires_at)+ return token+++def _synchronous(conn) -> int:+ return int(conn.execute("PRAGMA synchronous").fetchone()[0])+++def _live_tokens() -> int:+ conn = storage.get_connection()+ try:+ return conn.execute("SELECT COUNT(*) FROM refresh_tokens").fetchone()[0]+ finally:+ conn.close()+++def test_auth_connections_use_normal_sync_under_wal(owner):+ # owner already opened once, so this also covers the schema-cache early return.+ for _ in range(2):+ conn = storage.get_connection()+ try:+ assert str(conn.execute("PRAGMA journal_mode").fetchone()[0]).lower() == "wal"+ assert _synchronous(conn) == NORMAL+ finally:+ conn.close()+++def test_rollback_journal_keeps_full_sync(tmp_path):+ conn = sqlite3.connect(str(tmp_path / "rollback.db"))+ try:+ conn.execute("PRAGMA journal_mode=DELETE")+ sqlite_gate.apply_wal_synchronous(conn)+ assert _synchronous(conn) == FULL+ finally:+ conn.close()+++ "answer",+ [None, (None,), ("",), (123,), sqlite3.OperationalError("pragma unavailable")],+)+def test_unreadable_journal_mode_is_not_treated_as_wal(answer):+ class Connection:+ def __init__(self):+ self.executed = []++ def execute(self, sql, *args):+ self.executed.append(sql)+ if isinstance(answer, Exception):+ raise answer+ return type("Cursor", (), {"fetchone": lambda _self: answer})()++ conn = Connection()+ sqlite_gate.apply_wal_synchronous(conn)+ assert "PRAGMA synchronous=NORMAL" not in conn.executed+++def test_busy_auth_db_answers_retryable_503_and_keeps_the_token(client, owner, monkeypatch):+ token = _issue(owner)+ monkeypatch.setattr(storage, "_BUSY_TIMEOUT_MS", 50)+ holder = sqlite3.connect(str(storage.DB_PATH), isolation_level = None)+ try:+ holder.execute("BEGIN IMMEDIATE")+ busy = _refresh(client, token)+ finally:+ holder.execute("ROLLBACK")+ holder.close()++ assert busy.status_code == 503+ assert busy.headers["Retry-After"] == "2"+ assert busy.json()["detail"] == "Unsloth Studio is busy. Try again in a moment."++ ok = _refresh(client, token)+ assert ok.status_code == 200+ body = ok.json()+ assert body["access_token"] and body["refresh_token"] != token+ assert body["account_id"] == "owner"+ assert body["must_change_password"] is False++ assert _refresh(client, body["refresh_token"]).status_code == 200+ assert _refresh(client, token).status_code == 401+++def _last_used(key_id):+ with sqlite3.connect(str(storage.DB_PATH)) as conn:+ return conn.execute("SELECT last_used_at FROM api_keys WHERE id = ?", (key_id,)).fetchone()[+ 0+ ]+++def test_busy_auth_db_still_validates_an_api_key_without_recording_use(owner, monkeypatch):+ raw, row = storage.create_api_key(owner, name = "cli")+ assert storage.validate_api_key_account(raw) is not None+ used = _last_used(row["id"])+ assert used is not None++ monkeypatch.setattr(storage, "_BUSY_TIMEOUT_MS", 50)+ holder = sqlite3.connect(str(storage.DB_PATH), isolation_level = None)+ try:+ holder.execute("BEGIN IMMEDIATE")+ verified = storage.validate_api_key_account(raw)+ finally:+ holder.execute("ROLLBACK")+ holder.close()++ assert verified is not None+ assert verified[0]["username"] == owner+ assert _last_used(row["id"]) == used+++def test_a_non_busy_error_on_the_api_key_touch_still_raises(owner, monkeypatch):+ raw, _row = storage.create_api_key(owner, name = "cli")+ real = storage.get_connection++ class _Broken:+ def __init__(self, conn):+ self._conn = conn++ def execute(self, sql, *args):+ if sql == "BEGIN IMMEDIATE":+ raise sqlite3.OperationalError("disk I/O error")+ return self._conn.execute(sql, *args)++ def __getattr__(self, name):+ return getattr(self._conn, name)++ monkeypatch.setattr(storage, "get_connection", lambda *a, **k: _Broken(real(*a, **k)))+ with pytest.raises(sqlite3.OperationalError, match = "disk I/O error"):+ storage.validate_api_key_account(raw)+++def test_failed_replacement_insert_leaves_the_old_token_valid(client, owner):+ token = _issue(owner)+ conn = storage.get_connection()+ try:+ conn.execute(+ "CREATE TRIGGER reject_insert BEFORE INSERT ON refresh_tokens "+ "BEGIN SELECT RAISE(ABORT, 'insert rejected'); END"+ )+ conn.commit()+ finally:+ conn.close()++ new_token, expires_at = new_refresh_token_value()+ with pytest.raises(sqlite3.IntegrityError):+ storage.rotate_refresh_token(token, new_token, expires_at)+ assert _live_tokens() == 1++ conn = storage.get_connection()+ try:+ conn.execute("DROP TRIGGER reject_insert")+ conn.commit()+ finally:+ conn.close()+ assert _refresh(client, token).status_code == 200+++def test_refresh_token_stays_single_use_and_logout_revokes_the_rotation(client, owner):+ token = _issue(owner)+ first = _refresh(client, token)+ assert first.status_code == 200+ assert _refresh(client, token).status_code == 401++ storage.revoke_user_refresh_tokens(owner)+ assert _live_tokens() == 0+ assert _refresh(client, first.json()["refresh_token"]).status_code == 401+++def test_non_busy_operational_error_is_not_mapped_to_503(owner, monkeypatch):+ def broken(*_args):+ raise sqlite3.OperationalError("disk I/O error")++ monkeypatch.setattr(storage, "rotate_refresh_token", broken)+ app = FastAPI()+ app.include_router(auth_routes.router, prefix = "/api/auth")+ with TestClient(app, raise_server_exceptions = False) as http:+ response = _refresh(http, "t")+ assert response.status_code == 500+++def _wal() -> Path:+ return Path(f"{storage.DB_PATH}-wal")+++def _short_lived_write(username, n = 1):+ for _ in range(n):+ _issue(username)+++def test_keeper_holds_the_wal_until_released(owner):+ assert not _wal().exists(), "without a keeper the last close checkpoints and removes it"+ assert storage.open_wal_keeper() is True+ _short_lived_write(owner, 20)+ assert _wal().is_file()++ storage.close_wal_keeper()+ assert storage._wal_keeper is None+ assert not _wal().exists()+++def test_closing_a_connection_under_the_keeper_does_not_checkpoint(owner):+ """The checkpoint is what fsyncs inside sqlite_gate's process-wide lock."""+ assert storage.open_wal_keeper() is True+ try:+ _short_lived_write(owner)+ main_file = hashlib.sha256(storage.DB_PATH.read_bytes()).hexdigest()+ wal_size = _wal().stat().st_size++ _short_lived_write(owner)++ assert _wal().stat().st_size >= wal_size+ assert hashlib.sha256(storage.DB_PATH.read_bytes()).hexdigest() == main_file+ finally:+ storage.close_wal_keeper()+++def test_opening_again_replaces_the_keeper_of_an_earlier_db(owner, tmp_path, monkeypatch):+ assert storage.open_wal_keeper() is True+ stale = storage._wal_keeper[1]++ monkeypatch.setattr(storage, "DB_PATH", tmp_path / "other" / "auth.db")+ assert storage.open_wal_keeper() is True+ assert storage._wal_keeper[0] == str(storage.DB_PATH.resolve())+ with pytest.raises(sqlite3.ProgrammingError, match = "closed database"):+ stale.execute("SELECT 1")+ storage.close_wal_keeper()+ assert storage._wal_keeper is None++[email protected]("then", ["close", "reopen"])+def test_a_keeper_opened_on_an_exited_thread_is_released(owner, then):+ opened = threading.Thread(target = storage.open_wal_keeper)+ opened.start()+ opened.join()+ stale = storage._wal_keeper[1]++ if then == "reopen":+ assert storage.open_wal_keeper() is True+ assert storage._wal_keeper[1] is not stale+ storage.close_wal_keeper()+ with pytest.raises(sqlite3.ProgrammingError, match = "closed database"):+ stale.execute("SELECT 1")+ assert not _wal().exists()+++def test_keeper_declines_a_rollback_journal(owner, monkeypatch, caplog):+ real = storage.get_connection++ def refused_wal(**kwargs):+ conn = real(**kwargs)+ conn.execute("PRAGMA journal_mode=DELETE")+ return conn++ monkeypatch.setattr(storage, "get_connection", refused_wal)+ with caplog.at_level(logging.INFO, logger = storage.logger.name):+ assert storage.open_wal_keeper() is False+ assert storage._wal_keeper is None+ assert "not WAL" in caplog.text+++def test_the_lifespan_holds_the_auth_keeper_across_every_writer():+ """Reads the source rather than running the lifespan, which imports the whole stack."""+ import main++ source = inspect.getsource(main.lifespan)+ served = source.index("yield")+ opened = source.index("storage.open_wal_keeper()")+ assert opened < source.index("startup_reconciliation_accounts()") < served+ assert opened < source.index("storage.ensure_default_admin()") < served+ assert (+ served+ < source.index("await run_lifespan_shutdown(")+ < source.index("storage.close_wal_keeper()")+ )+++def test_a_cross_origin_client_can_read_the_refresh_retry_after():+ # tauri://localhost and tunnel pages are cross-origin: an unexposed header reads as null there.+ import main+ exposed = [+ header+ for middleware in main.app.user_middleware+ for header in middleware.kwargs.get("expose_headers", ())+ ]+ assert "Retry-After" in exposeddiff --git a/studio/backend/tests/test_config_mutations_on_event_loop.py b/studio/backend/tests/test_config_mutations_on_event_loop.pyindex 05ff6758de4..8d50a026e24 100644--- a/studio/backend/tests/test_config_mutations_on_event_loop.py+++ b/studio/backend/tests/test_config_mutations_on_event_loop.py@@ -12,14 +12,15 @@ it read, with nothing below them serializing the pair:and `credential_secrets` has no foreign key to `providers`.* login clears its admission check before verify_password reaches _record_login_failure, sothe bucket lock guards each call but not the sequence.- * refresh consumes a token and inserts its replacement, and logout deletes every token in- between without rotating the credential generation.They are await-free, so the loop is what makes those sequences atomic. In the threadpool, twoimports duplicate a server row, an update racing a delete writes a credential for a provider that-is gone, a burst of guesses passes admission together, and a logout leaves the refresh token that-landed after it. The read-only handlers beside them do belong in the threadpool, so both-directions are pinned here.+is gone, and a burst of guesses passes admission together. The read-only handlers beside them do+belong in the threadpool, so both directions are pinned here.++Refresh used to be pinned to the loop too, because it consumed a token and inserted its replacement+in two commits that a logout could land between. It now rotates in one SQLite transaction, so it+runs in the threadpool, where a busy auth.db wait does not freeze every other request.Asserts which thread each handler ran on rather than racing two requests."""@@ -97,16 +98,6 @@ _MUTATIONS = {None,401,),- "auth-refresh": _Case(- "post",- "/api/auth/refresh",- {"refresh_token": "t"},- auth_routes,- "storage",- "consume_refresh_token",- None,- 401,- ),"auth-logout": _Case("post","/api/auth/logout",@@ -132,6 +123,20 @@ _READS = {}+_ATOMIC_MUTATIONS = {+ "auth-refresh": _Case(+ "post",+ "/api/auth/refresh",+ {"refresh_token": "t"},+ auth_routes,+ "storage",+ "rotate_refresh_token",+ None,+ 401,+ ),+}++def _record(threads: list[int], result):def _call(*_args, **_kwargs):threads.append(threading.get_ident())@@ -188,3 +193,9 @@ def test_a_state_changing_handler_runs_on_the_event_loop_thread(monkeypatch, casdef test_a_read_only_handler_stays_off_the_event_loop_thread(monkeypatch, case):threads, loop_thread = _drive(monkeypatch, case)assert threads[0] != loop_thread, f"{case.path} read its store on the event loop thread"++[email protected]("case", _ATOMIC_MUTATIONS.values(), ids = list(_ATOMIC_MUTATIONS))+def test_an_atomic_mutation_stays_off_the_event_loop_thread(monkeypatch, case):+ threads, loop_thread = _drive(monkeypatch, case)+ assert threads[0] != loop_thread, f"{case.path} blocked the event loop on SQLite"diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.tsindex 6bda08f7b2c..e72d00cbb3f 100644--- a/studio/frontend/scripts/check-bundle-budget.ts+++ b/studio/frontend/scripts/check-bundle-budget.ts@@ -248,8 +248,12 @@ export const BUDGET = {// (6,825,902 -> 6,826,090 B raw, 2,054,124 -> 2,054,190 B transfer against 94464babd6 and 1de12ab309, one machine// and build, Node 24, chunk count unchanged at 95): the picker catalog is imported by startup code (chat-adapter.ts).// Raised by that growth, to the measured build plus the same 160 B raw / 189 B transfer margin.- transferBytes: 2_054_379,- rawBytes: 6_826_250,+ // Keeping the session when auth.db is busy (#14151) adds 829 B raw / 393 B transfer (6,826,090 -> 6,826,919 B+ // raw, 2,054,190 -> 2,054,583 B transfer against a33463fb16, one machine and build, Node 24, chunk count+ // unchanged at 96): the refresh retry and outcome live in features/auth/api.ts, which every authFetch imports.+ // Raised by that growth, to the measured build plus the same 160 B raw / 189 B transfer margin.+ transferBytes: 2_054_772,+ rawBytes: 6_827_079,};// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering thediff --git a/studio/frontend/src/app/auth-guards.ts b/studio/frontend/src/app/auth-guards.tsindex d35f7020f9c..82d88ae4e6a 100644--- a/studio/frontend/src/app/auth-guards.ts+++ b/studio/frontend/src/app/auth-guards.ts@@ -10,14 +10,16 @@ import {hasAuthToken,hasRefreshToken,mustChangePassword,- refreshSession,+ refreshSessionOutcome,setMustChangePassword,} from "@/features/auth";async function hasActiveSession(): Promise<boolean> {if (hasAuthToken()) return true;if (!hasRefreshToken()) return false;- return refreshSession();+ const outcome = await refreshSessionOutcome();+ // A busy backend is not a signed-out user: a refresh token that survived is still a session.+ return outcome === "ok" || (outcome === "unavailable" && hasRefreshToken());}interface AuthStatus {diff --git a/studio/frontend/src/features/auth/api.ts b/studio/frontend/src/features/auth/api.tsindex fda7e6ab5ce..4ce435e9742 100644--- a/studio/frontend/src/features/auth/api.ts+++ b/studio/frontend/src/features/auth/api.ts@@ -25,7 +25,7 @@ type AuthFetchOptions = {};let isRedirecting = false;-let refreshInflight: Promise<boolean> | null = null;+let refreshInflight: Promise<RefreshOutcome> | null = null;let refreshInflightToken: string | null = null;let logoutGeneration = 0;@@ -307,38 +307,89 @@ async function retryWithTauriAutoAuth(return null;}-export async function refreshSession(): Promise<boolean> {- const refreshToken = getRefreshToken();- if (!refreshToken) return false;- if (refreshInflight && refreshInflightToken === refreshToken) {- return refreshInflight;+/** "rejected": the server refused the token or the user signed out. "unavailable": no answer yet. */+export type RefreshOutcome = "ok" | "rejected" | "unavailable";++// Never abort an in-flight POST: if the server already rotated the pair, dropping its answer loses the new token.+const REFRESH_MAX_ATTEMPTS = 3;+const REFRESH_BUDGET_MS = 8000;+const REFRESH_RETRY_DEFAULT_MS = 1000;+const REFRESH_RETRY_MIN_MS = 250;+const REFRESH_RETRY_MAX_MS = 5000;++function refreshRetryDelay(response: Response): number {+ const header = response.headers.get("Retry-After")?.trim();+ const seconds = header ? Number(header) : Number.NaN;+ if (!Number.isFinite(seconds) || seconds < 0) {+ return REFRESH_RETRY_DEFAULT_MS;}+ return Math.min(+ Math.max(seconds * 1000, REFRESH_RETRY_MIN_MS),+ REFRESH_RETRY_MAX_MS,+ );+}+async function runRefresh(refreshToken: string): Promise<RefreshOutcome> {const startGeneration = logoutGeneration;- const promise = (async () => {+ const startedAt = Date.now();+ const superseded = (): RefreshOutcome | null => {+ if (startGeneration !== logoutGeneration) return "rejected";+ if (getRefreshToken() !== refreshToken) return "unavailable";+ return null;+ };+ for (let attempt = 1; ; attempt++) {+ let response: Response;try {- const response = await fetchWithTauriNetworkRetry(- apiUrl("/api/auth/refresh"),- {- method: "POST",- headers: { "Content-Type": "application/json" },- body: JSON.stringify({ refresh_token: refreshToken }),- },- );- if (!response.ok) {- clearAuthTokensIfCurrent(refreshToken);- return false;+ response = await fetchWithTauriNetworkRetry(apiUrl("/api/auth/refresh"), {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({ refresh_token: refreshToken }),+ });+ } catch {+ return "unavailable";+ }+ // Only a 401 rejects the token; a 5xx, 429 or proxy error says nothing about the session.+ if (response.status === 401) {+ clearAuthTokensIfCurrent(refreshToken);+ return "rejected";+ }+ if (response.ok) {+ let payload: RefreshResponse;+ try {+ payload = (await response.json()) as RefreshResponse;+ } catch {+ return "unavailable";}- const payload = (await response.json()) as RefreshResponse;- if (startGeneration !== logoutGeneration) return false;- if (getRefreshToken() !== refreshToken) return false;+ const stale = superseded();+ if (stale) return stale;storeAuthTokens(payload.access_token, payload.refresh_token);setMustChangePassword(payload.must_change_password ?? false);- return true;- } catch {- return false;+ return "ok";}- })();+ if (response.status !== 503 && response.status !== 429) {+ return "unavailable";+ }+ const delay = refreshRetryDelay(response);+ if (+ attempt >= REFRESH_MAX_ATTEMPTS ||+ Date.now() - startedAt + delay > REFRESH_BUDGET_MS+ ) {+ return "unavailable";+ }+ await wait(delay);+ const stale = superseded();+ if (stale) return stale;+ }+}++export async function refreshSessionOutcome(): Promise<RefreshOutcome> {+ const refreshToken = getRefreshToken();+ if (!refreshToken) return "rejected";+ if (refreshInflight && refreshInflightToken === refreshToken) {+ return refreshInflight;+ }++ const promise = runRefresh(refreshToken);refreshInflight = promise;refreshInflightToken = refreshToken;try {@@ -351,6 +402,10 @@ export async function refreshSession(): Promise<boolean> {}}+export async function refreshSession(): Promise<boolean> {+ return (await refreshSessionOutcome()) === "ok";+}+export async function authFetch(input: RequestInfo | URL,init?: RequestInit,@@ -400,8 +455,9 @@ export async function authFetch(if (response.status !== 401) return response;const refreshToken = getRefreshToken();- const refreshed = await refreshSession();- if (!refreshed) {+ const outcome = await refreshSessionOutcome();+ if (outcome === "unavailable") return response;+ if (outcome === "rejected") {if (isTauri) {return ((await retryWithTauriAutoAuth(diff --git a/studio/frontend/src/features/auth/components/auth-form.tsx b/studio/frontend/src/features/auth/components/auth-form.tsxindex 5ddedd432f5..5282584bda5 100644--- a/studio/frontend/src/features/auth/components/auth-form.tsx+++ b/studio/frontend/src/features/auth/components/auth-form.tsx@@ -15,7 +15,7 @@ import { Eye, EyeOff } from "lucide-react";import { useEffect, useRef, useState } from "react";import type { ReactElement } from "react";import type { SyntheticEvent } from "react";-import { refreshSession } from "../api";+import { refreshSession, refreshSessionOutcome } from "../api";import {deadlineFromStatus,formatCountdown,@@ -271,9 +271,12 @@ export function AuthForm({ mode }: AuthFormProps): ReactElement | null {let accessToken = getAuthToken();if (hasRefreshToken()) {- const refreshed = await refreshSession();+ const outcome = await refreshSessionOutcome();+ if (outcome === "unavailable") {+ throw new Error("Unsloth is busy. Try again in a moment.");+ }accessToken = getAuthToken();- if (!refreshed) {+ if (outcome === "rejected") {clearAuthTokens();accessToken = null;}diff --git a/studio/frontend/src/features/auth/index.ts b/studio/frontend/src/features/auth/index.tsindex e96fa473c0a..4fbbbd22d5e 100644--- a/studio/frontend/src/features/auth/index.ts+++ b/studio/frontend/src/features/auth/index.ts@@ -3,7 +3,12 @@export { LoginPage } from "./login-page";export { ChangePasswordPage } from "./change-password-page";-export { authFetch, logout, refreshSession } from "./api";+export {+ authFetch,+ logout,+ refreshSession,+ refreshSessionOutcome,+} from "./api";export { AUTH_SESSION_ENDING_EVENT } from "./session-events";export {AUTH_SESSION_CLEARED_EVENT,diff --git a/studio/frontend/src/features/auth/tauri-auto-auth.ts b/studio/frontend/src/features/auth/tauri-auto-auth.tsindex da0518c81cf..e482d77ebb3 100644--- a/studio/frontend/src/features/auth/tauri-auto-auth.ts+++ b/studio/frontend/src/features/auth/tauri-auto-auth.ts@@ -10,7 +10,7 @@ import {setMustChangePassword,storeAuthTokens,} from "./session";-import { refreshSession } from "./api";+import { refreshSession, refreshSessionOutcome } from "./api";import { setLoginMode } from "./login-client";import {OWNER_BROWSER_ACCOUNT,@@ -86,7 +86,10 @@ async function doTauriAutoAuth(options: TauriAutoAuthOptions): Promise<boolean>if ("login_required" in tokens) {tauriLoginRequired = true;// A managed account signs in in the document; do not discard the session it holds.- if (hasRefreshToken() && (await refreshSession()) && hasAuthToken()) {+ const outcome = hasRefreshToken()+ ? await refreshSessionOutcome()+ : "rejected";+ if (outcome === "ok" && hasAuthToken()) {clearTauriAuthFailure();if (mustChangePassword()) {const { router } = await import("@/app/router");@@ -94,6 +97,11 @@ async function doTauriAutoAuth(options: TauriAutoAuthOptions): Promise<boolean>}return true;}+ // Keep the session; the forced startup probe still releases the startup screen.+ if (outcome === "unavailable") {+ clearTauriAuthFailure();+ return options.force === true;+ }clearAuthTokens();clearTauriAuthFailure();// The forced startup probe must release the startup screen so the login form can mount; ordinary API recovery still returns false.diff --git a/studio/frontend/src/features/settings/api/debug-logs.ts b/studio/frontend/src/features/settings/api/debug-logs.tsindex a60140ec880..aa45d9b32e0 100644--- a/studio/frontend/src/features/settings/api/debug-logs.ts+++ b/studio/frontend/src/features/settings/api/debug-logs.ts@@ -230,12 +230,7 @@ export async function exportAllLogs(): Promise<string | null> {// Rust uses the token once and as-is, so it cannot retry a 401 by// refreshing the way `authFetch` does; a desktop left idle past the// access-token lifetime would fail while holding a good refresh token.- // Refreshing only HERE, rather than before every export, is what keeps a- // transient 500 on /api/auth/refresh from signing the whole app out:- // `refreshSession` clears both stored tokens on any non-2xx, so doing it- // speculatively would end the session of a user whose token was fine. A- // 401 means the token really was rejected, which is the one case where- // losing it costs nothing that was not already lost.+ // Refresh only on a 401, so a spare refresh never rotates the pair for nothing.if (desktopExportStatus(error) !== 401) throw desktopExportError(error);try {if (!(await refreshSession())) throw error;diff --git a/studio/frontend/tests/account-desktop-auth.test.ts b/studio/frontend/tests/account-desktop-auth.test.tsindex e26a4a6f527..88b8c8c1914 100644--- a/studio/frontend/tests/account-desktop-auth.test.ts+++ b/studio/frontend/tests/account-desktop-auth.test.ts@@ -34,6 +34,7 @@ function fixture(t: { after(fn: () => void): void }, tauri = true) {let invokeCalls = 0;let refreshCalls = 0;let refreshSucceeds = false;+ let refreshOutcome: "ok" | "rejected" | "unavailable" | null = null;let statusCalls = 0;let status: {initialized: boolean;@@ -57,6 +58,13 @@ function fixture(t: { after(fn: () => void): void }, tauri = true) {refresh = "refreshed-refresh";return true;},+ refreshSessionOutcome: async () => {+ if (refreshOutcome) {+ refreshCalls++;+ return refreshOutcome;+ }+ return (await session.refreshSession()) ? "ok" : "rejected";+ },};const apiBase = { isTauri: tauri, apiUrl: (path: string) => path };const transitions: { account: unknown; route: string }[] = [];@@ -112,6 +120,9 @@ function fixture(t: { after(fn: () => void): void }, tauri = true) {setResponse: (value: unknown) => { response = value; },setStatus: (value: typeof status) => { status = value; },setRefreshSucceeds: (value: boolean) => { refreshSucceeds = value; },+ setRefreshOutcome: (value: "ok" | "rejected" | "unavailable" | null) => {+ refreshOutcome = value;+ },get access() { return access; },get refresh() { return refresh; },get invokeCalls() { return invokeCalls; },@@ -162,6 +173,30 @@ test("forced multi-account startup opens login and releases the startup screen wawait assert.rejects(f.guards.requirePasswordChangeFlow(), { to: "/login" });});+test("a managed session survives forced startup while its refresh gets no answer", async (t) => {+ const f = fixture(t);+ f.session.storeAuthTokens("alice-access", "alice-refresh");+ f.setResponse(multi);+ f.setRefreshOutcome("unavailable");+ assert.equal(await f.auth.tauriAutoAuth({ force: true }), true);+ assert.equal(f.access, "alice-access");+ assert.equal(f.refresh, "alice-refresh");+ assert.deepEqual(f.navigations, []);+ assert.equal(f.auth.getTauriAuthFailure(), null);+ await f.guards.requireAuth();+});++test("a managed session whose refresh is rejected at startup still opens login", async (t) => {+ const f = fixture(t);+ f.session.storeAuthTokens("alice-access", "alice-refresh");+ f.setResponse(multi);+ f.setRefreshOutcome("rejected");+ assert.equal(await f.auth.tauriAutoAuth({ force: true }), true);+ assert.equal(f.access, null);+ assert.equal(f.refresh, null);+ assert.deepEqual(f.navigations, [{ to: "/login", replace: true }]);+});+test("ordinary multi-account API recovery never reports an authenticated session", async (t) => {const f = fixture(t);f.setResponse(multi);diff --git a/studio/frontend/tests/auth-refresh-transient.test.ts b/studio/frontend/tests/auth-refresh-transient.test.tsnew file mode 100644index 00000000000..a4a4ea32a9c--- /dev/null+++ b/studio/frontend/tests/auth-refresh-transient.test.ts@@ -0,0 +1,286 @@+// SPDX-License-Identifier: AGPL-3.0-only+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++// A busy auth.db answers /api/auth/refresh with 503 + Retry-After and keeps the refresh+// token valid. Only a 401 may end the session; anything else keeps both tokens.++import assert from "node:assert/strict";+import test, { type TestContext } from "node:test";+import { loadWithStubs } from "./helpers/module-stubs.ts";+import type * as AuthApi from "../src/features/auth/api.ts";+import type * as Guards from "../src/app/auth-guards.ts";++type Refresh = () => Response | Promise<Response>;++const busy = (retryAfter: string | null = "1") =>+ new Response(JSON.stringify({ detail: "busy" }), {+ status: 503,+ headers: retryAfter === null ? {} : { "Retry-After": retryAfter },+ });+const rotated = () =>+ Response.json({+ access_token: "new-access",+ refresh_token: "new-refresh",+ must_change_password: false,+ });++function fixture(t: TestContext, refreshes: Refresh[]) {+ const previousWindow = globalThis.window;+ const previousFetch = globalThis.fetch;+ const previousSetTimeout = globalThis.setTimeout;+ const previousNow = Date.now;+ const location = { pathname: "/chat", href: "" };+ const tokens = {+ access: "old-access" as string | null,+ refresh: "old-refresh" as string | null,+ };+ let now = 0;+ let clears = 0;+ let unexpected = 0;+ const delays: number[] = [];+ const refreshBodies: string[] = [];+ const apiTokens: (string | null)[] = [];+ const state = { onWait: null as null | (() => void) };+ globalThis.window = { location } as unknown as Window & typeof globalThis;+ // A fake clock: a wait advances it and resolves on the next turn, not in real time.+ Date.now = () => now;+ globalThis.setTimeout = ((fn: () => void, ms?: number) => {+ delays.push(ms ?? 0);+ const hook = state.onWait;+ state.onWait = null;+ hook?.();+ setImmediate(() => {+ now += ms ?? 0;+ fn();+ });+ return 0;+ }) as unknown as typeof setTimeout;+ globalThis.fetch = async (input, init) => {+ const path = String(input);+ if (path.endsWith("/status")) {+ return Response.json({+ login_mode: "single",+ requires_password_change: false,+ initialized: true,+ });+ }+ if (path.endsWith("/logout")) return new Response(null, { status: 200 });+ if (path.endsWith("/refresh")) {+ refreshBodies.push(String(init?.body));+ const next = refreshes.shift();+ if (!next) {+ unexpected++;+ return new Response(null, { status: 418 });+ }+ return next();+ }+ const auth = new Headers(init?.headers).get("Authorization");+ apiTokens.push(auth);+ return new Response(null, {+ status: auth === "Bearer new-access" ? 200 : 401,+ });+ };+ t.after(() => {+ globalThis.window = previousWindow;+ globalThis.fetch = previousFetch;+ globalThis.setTimeout = previousSetTimeout;+ Date.now = previousNow;+ assert.equal(unexpected, 0, "a refresh ran past its script");+ });+ const session = {+ getAuthToken: () => tokens.access,+ getRefreshToken: () => tokens.refresh,+ hasAuthToken: () => !!tokens.access,+ hasRefreshToken: () => !!tokens.refresh,+ mustChangePassword: () => false,+ setMustChangePassword: () => undefined,+ storeAuthTokens: (access: string, refresh: string) => {+ tokens.access = access;+ tokens.refresh = refresh;+ },+ clearAuthTokens: () => {+ clears++;+ tokens.access = null;+ tokens.refresh = null;+ },+ };+ const apiBase = {+ apiUrl: (path: string) => path,+ getApiPort: () => null,+ isTauri: false,+ };+ const api = loadWithStubs<typeof AuthApi>(+ new URL("../src/features/auth/api.ts", import.meta.url),+ {+ "@/lib/api-base": apiBase,+ "@/lib/account-transition": { accountTransitionPending: () => false },+ "./session": session,+ },+ );+ const settle = async () => {+ for (let i = 0; i < 5; i++) {+ await new Promise<void>((resolve) => setImmediate(resolve));+ }+ };+ return {+ api,+ session,+ apiBase,+ tokens,+ location,+ delays,+ refreshBodies,+ apiTokens,+ state,+ settle,+ clears: () => clears,+ };+}++test("a 503 then 200 is retried and stores the rotated pair", async (t) => {+ const f = fixture(t, [() => busy("0"), () => busy("1"), rotated]);+ assert.equal(await f.api.refreshSession(), true);+ assert.equal(f.refreshBodies.length, 3);+ assert.deepEqual(f.delays, [250, 1000]);+ assert.deepEqual(f.tokens, { access: "new-access", refresh: "new-refresh" });+});++test("a persistent 503 is bounded and keeps both tokens", async (t) => {+ const f = fixture(t, [() => busy(null), () => busy("60"), () => busy()]);+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");+ assert.equal(f.refreshBodies.length, 3);+ assert.deepEqual(f.delays, [1000, 5000]);+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });+ assert.equal(f.clears(), 0);+ await f.settle();+ assert.equal(f.location.href, "");+});++test("retries stop once the wait would pass the budget", async (t) => {+ const f = fixture(t, [() => busy("5"), () => busy("5")]);+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");+ assert.equal(f.refreshBodies.length, 2);+ assert.deepEqual(f.delays, [5000]);+});++test("a 429 is retried like a 503", async (t) => {+ const f = fixture(t, [+ () => new Response(null, { status: 429, headers: { "Retry-After": "2" } }),+ rotated,+ ]);+ assert.equal(await f.api.refreshSessionOutcome(), "ok");+ assert.deepEqual(f.delays, [2000]);+});++test("a 500 is not retried and keeps both tokens", async (t) => {+ const f = fixture(t, [() => new Response("boom", { status: 500 })]);+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");+ assert.equal(f.refreshBodies.length, 1);+ assert.deepEqual(f.delays, []);+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });+ assert.equal(f.clears(), 0);+});++test("a network error keeps both tokens", async (t) => {+ const f = fixture(t, [+ () => {+ throw new TypeError("Failed to fetch");+ },+ ]);+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });+});++test("a malformed 200 body keeps both tokens", async (t) => {+ const f = fixture(t, [() => new Response("not json", { status: 200 })]);+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });+});++test("a 401 still clears the tokens and redirects to /login", async (t) => {+ const f = fixture(t, [() => new Response(null, { status: 401 })]);+ const response = await f.api.authFetch("/api/models");+ assert.equal(response.status, 401);+ assert.equal(f.clears() > 0, true);+ assert.deepEqual(f.tokens, { access: null, refresh: null });+ await f.settle();+ assert.equal(f.location.href, "/login");+});++test("a pair rotated by another tab mid-retry stops without replay or clear", async (t) => {+ const f = fixture(t, [() => busy("1")]);+ f.state.onWait = () => {+ f.tokens.access = "peer-access";+ f.tokens.refresh = "peer-refresh";+ };+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");+ assert.equal(f.refreshBodies.length, 1, "the old token was replayed");+ assert.deepEqual(f.tokens, { access: "peer-access", refresh: "peer-refresh" });+ assert.equal(f.clears(), 0);+});++test("a logout during the retry wait ends the refresh", async (t) => {+ const f = fixture(t, [() => busy("1")]);+ let loggedOut: Promise<void> | null = null;+ f.state.onWait = () => {+ loggedOut = f.api.logout();+ };+ assert.equal(await f.api.refreshSessionOutcome(), "rejected");+ await loggedOut;+ assert.equal(f.refreshBodies.length, 1, "the refresh outlived the logout");+ assert.deepEqual(f.tokens, { access: null, refresh: null });+});++test("authFetch hands back the original 401 while refresh is unavailable", async (t) => {+ const f = fixture(t, [() => busy("0"), () => busy("0"), () => busy("0")]);+ const response = await f.api.authFetch("/api/models");+ assert.equal(response.status, 401);+ assert.equal(f.refreshBodies.length, 3);+ assert.deepEqual(f.apiTokens, ["Bearer old-access"], "the request was replayed");+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });+ assert.equal(f.clears(), 0);+ await f.settle();+ assert.equal(f.location.href, "");+});++test("authFetch replays the request with the refreshed token", async (t) => {+ const f = fixture(t, [rotated]);+ const response = await f.api.authFetch("/api/models");+ assert.equal(response.status, 200);+ assert.deepEqual(f.apiTokens, ["Bearer old-access", "Bearer new-access"]);+});++function loadGuards(f: ReturnType<typeof fixture>) {+ return loadWithStubs<typeof Guards>(+ new URL("../src/app/auth-guards.ts", import.meta.url),+ {+ "@tanstack/react-router": {+ redirect: (target: { to: string }) =>+ Object.assign(new Error(`redirect ${target.to}`), { target }),+ },+ "@/lib/api-base": f.apiBase,+ "@/features/auth/tauri-auto-auth": { isTauriLoginRequired: () => true },+ "@/features/auth/login-client": { setLoginMode: () => undefined },+ "@/features/auth": {+ ...f.session,+ getPostAuthRoute: () => "/chat",+ refreshSession: f.api.refreshSession,+ refreshSessionOutcome: f.api.refreshSessionOutcome,+ },+ },+ );+}++test("the route guard keeps a session whose refresh is unavailable", async (t) => {+ const f = fixture(t, [() => busy("0"), () => busy("0"), () => busy("0")]);+ f.tokens.access = null;+ await loadGuards(f).requireAuth();+ assert.equal(f.tokens.refresh, "old-refresh");+});++test("the route guard still sends a rejected refresh to /login", async (t) => {+ const f = fixture(t, [() => new Response(null, { status: 401 })]);+ f.tokens.access = null;+ await assert.rejects(loadGuards(f).requireAuth(), /redirect \/login/);+ assert.equal(f.tokens.refresh, null);+});diff --git a/studio/frontend/tests/debug-log-export.test.ts b/studio/frontend/tests/debug-log-export.test.tsindex 49455aa05be..08bb620feae 100644--- a/studio/frontend/tests/debug-log-export.test.ts+++ b/studio/frontend/tests/debug-log-export.test.ts@@ -328,9 +328,9 @@ test("openLogsFolder preserves the selected path with an older backend", async (});test("an export that works does not touch the session", async () => {- // refreshSession clears both stored tokens on ANY non-2xx, so refreshing- // before every export would let a transient 500 on /api/auth/refresh sign the- // user out of the whole app just for clicking Download all logs.+ // refreshSession rotates the stored pair, so refreshing before every export+ // would spend a refresh token, and risk a sign-out, just for clicking+ // Download all logs.const world = makeWorld({isTauri: true,invoke: async () => SAVED_PATH,diff --git a/studio/frontend/tests/login-account-modes.test.ts b/studio/frontend/tests/login-account-modes.test.tsindex 2e73d408e5b..f4ff0af2079 100644--- a/studio/frontend/tests/login-account-modes.test.ts+++ b/studio/frontend/tests/login-account-modes.test.ts@@ -50,7 +50,14 @@ const tick = () => new Promise<void>((resolve) => setImmediate(resolve));function mountForm(client: typeof LoginClient,- session: { access: string | null; change: boolean; owner?: boolean },+ session: {+ access: string | null;+ change: boolean;+ owner?: boolean;+ refresh?: boolean;+ outcome?: "ok" | "rejected" | "unavailable";+ clears?: number;+ },switched = false,failTransitionOnce = false,) {@@ -129,15 +136,20 @@ function mountForm(},},"lucide-react": { Eye: "Eye", EyeOff: "EyeOff" },- "../api": { refreshSession: async () => true },+ "../api": {+ refreshSession: async () => true,+ refreshSessionOutcome: async () => session.outcome ?? "ok",+ },"../bootstrap-deadline": deadline,"../session": {clearAuthTokens: () => {session.access = null;+ session.refresh = false;+ session.clears = (session.clears ?? 0) + 1;},getAuthToken: () => session.access,hasAuthToken: () => Boolean(session.access),- hasRefreshToken: () => false,+ hasRefreshToken: () => Boolean(session.refresh),getPostAuthRoute: () => (session.change ? "/change-password" : "/chat"),mustChangePassword: () => session.change,setMustChangePassword: (change: boolean) => {@@ -591,3 +603,66 @@ test("a deactivated account restricts the legacy hint in peer tabs without changunsubscribe();assert.equal(handlers.size, 0);});++async function submitPasswordChange(+ session: Parameters<typeof mountForm>[1],+ requests: string[],+) {+ globalThis.fetch = async (path) => {+ requests.push(String(path));+ return String(path).endsWith("/status")+ ? Response.json({+ initialized: true,+ requires_password_change: true,+ login_mode: "single",+ })+ : Response.json(token);+ };+ const form = mountForm(client(), session);+ let tree = await form.initialize("change-password");+ for (const [id, value] of [+ ["current-password", "old-password-1"],+ ["new-password", "new-password-1"],+ ["confirm-password", "new-password-1"],+ ]) {+ const change = elements(tree).find((element) => element.props.id === id)+ ?.props.onChange as (event: unknown) => void;+ change({ target: { value } });+ tree = form.render("change-password");+ }+ const submit = elements(tree).find((element) => element.type === "form")+ ?.props.onSubmit as (event: unknown) => Promise<void>;+ await submit({ preventDefault() {} });+ return form.render("change-password");+}++test("a busy refresh during a password change keeps the session and asks to retry", async (t) => {+ environment(t);+ const requests: string[] = [];+ const session: Parameters<typeof mountForm>[1] = {+ access: "access",+ change: true,+ refresh: true,+ outcome: "unavailable",+ };+ const tree = await submitPasswordChange(session, requests);+ assert.equal(session.access, "access");+ assert.equal(session.refresh, true);+ assert.equal(session.clears ?? 0, 0);+ assert.ok(!requests.some((path) => path.endsWith("/api/auth/change-password")));+ assert.match(textContent(tree), /Unsloth is busy/);+});++test("a rejected refresh during a password change still drops the stored session", async (t) => {+ environment(t);+ const requests: string[] = [];+ const session: Parameters<typeof mountForm>[1] = {+ access: "access",+ change: true,+ refresh: true,+ outcome: "rejected",+ };+ await submitPasswordChange(session, requests);+ assert.equal(session.clears, 1);+ assert.ok(requests.some((path) => path.endsWith("/api/auth/login")));+});