Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep users signed in when auth.db is busy (#14151)

Etherl8cf09febfedb
diff --git a/studio/backend/auth/authentication.py b/studio/backend/auth/authentication.py
index c2e4436176a..bb5503dae53 100644
--- a/studio/backend/auth/authentication.py
+++ b/studio/backend/auth/authentication.py
@@ -266,6 +266,13 @@ def is_desktop_access_token(token: str) -> bool:
return account is not None and account.is_owner
+def new_refresh_token_value() -> Tuple[str, str]:
+ """A fresh opaque refresh token and its ISO expiry, not yet stored."""
+ token = secrets.token_urlsafe(48)
+ expires_at = datetime.now(timezone.utc) + timedelta(days = REFRESH_TOKEN_EXPIRE_DAYS)
+ return token, expires_at.isoformat()
+
+
def create_refresh_token(
subject: str,
*,
@@ -276,12 +283,11 @@ def create_refresh_token(
JWTs); expire after REFRESH_TOKEN_EXPIRE_DAYS. ``secret`` stamps the token with the credential version the
caller verified, so a rotation cannot leave a token minted from the replaced credential valid.
"""
- token = secrets.token_urlsafe(48)
- expires_at = datetime.now(timezone.utc) + timedelta(days = REFRESH_TOKEN_EXPIRE_DAYS)
+ token, expires_at = new_refresh_token_value()
save_refresh_token(
token,
subject,
- expires_at.isoformat(),
+ expires_at,
is_desktop = desktop,
secret_gen = credential_generation(secret) if secret is not None else None,
)
diff --git a/studio/backend/auth/storage.py b/studio/backend/auth/storage.py
index 691228a4e3e..ff4ce8c972f 100644
--- a/studio/backend/auth/storage.py
+++ b/studio/backend/auth/storage.py
@@ -10,6 +10,7 @@ import hashlib
import hmac
import ipaddress
import json
+import logging
import os
import re
import secrets
@@ -18,11 +19,13 @@ import tempfile
import threading
from datetime import datetime, timedelta, timezone
import uuid
-from typing import Iterator, Optional, Tuple
+from typing import Iterator, NamedTuple, Optional, Tuple
from storage import sqlite_gate
from utils.paths import auth_db_path, ensure_dir
+logger = logging.getLogger(__name__)
+
DB_PATH = auth_db_path()
DEFAULT_ADMIN_USERNAME = "unsloth"
@@ -264,10 +267,12 @@ def credential_generation_guard(username: str, expect_gen: Optional[str]) -> Ite
_auth_schema_ready: set[tuple[str, int, int, int]] = set()
+_BUSY_TIMEOUT_MS = 5000
+
-def get_connection() -> sqlite3.Connection:
+def get_connection(*, check_same_thread: bool = True) -> sqlite3.Connection:
ensure_dir(DB_PATH.parent)
- conn = sqlite_gate.connect(DB_PATH)
+ conn = sqlite_gate.connect(DB_PATH, check_same_thread = check_same_thread)
# sqlite3.connect would create the DB 0644 under a 022 umask, exposing identity secrets and password hashes.
for _path, _mode in ((DB_PATH.parent, 0o700), (DB_PATH, 0o600)):
try:
@@ -278,10 +283,12 @@ def get_connection() -> sqlite3.Connection:
# Set busy_timeout before journal_mode=WAL: switching journal mode needs a lock and would otherwise
# raise SQLITE_BUSY.
try:
- conn.execute("PRAGMA busy_timeout=5000")
+ conn.execute(f"PRAGMA busy_timeout={int(_BUSY_TIMEOUT_MS)}")
conn.execute("PRAGMA journal_mode=WAL")
except sqlite3.Error:
pass
+ # Per connection, so before the schema-cache return below.
+ sqlite_gate.apply_wal_synchronous(conn)
file_stat = DB_PATH.stat()
schema_key = (
str(DB_PATH),
@@ -387,6 +394,49 @@ _ACCOUNT_COLUMNS = (
_owner_id_repaired: set[str] = set()
+# Without a keeper each close is the last WAL participant and checkpoints under sqlite_gate's
+# process-wide lock. Lifespan-only, never lazy: tests swap and unlink auth.db.
+_wal_keeper_lock = threading.Lock()
+_wal_keeper: Optional[tuple[str, sqlite3.Connection]] = None
+
+
+def _close_keeper(conn: sqlite3.Connection) -> None:
+ try:
+ conn.close()
+ except Exception as exc:
+ logger.warning("Could not close the auth.db WAL keeper: %s", exc)
+
+
+def open_wal_keeper() -> bool:
+ """Hold auth.db's WAL open for the server's lifetime, replacing any earlier keeper."""
+ global _wal_keeper
+ with _wal_keeper_lock:
+ if _wal_keeper is not None:
+ _close_keeper(_wal_keeper[1])
+ _wal_keeper = None
+ conn = get_connection(check_same_thread = False)
+ try:
+ mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
+ except (sqlite3.Error, TypeError, IndexError) as exc:
+ _close_keeper(conn)
+ logger.warning("Could not read auth.db journal mode: %s", exc)
+ return False
+ if not isinstance(mode, str) or mode.lower() != "wal":
+ _close_keeper(conn)
+ logger.info("auth.db is in %s mode, not WAL; WAL keeper not engaged.", mode)
+ return False
+ _wal_keeper = (str(DB_PATH.resolve()), conn)
+ return True
+
+
+def close_wal_keeper() -> None:
+ global _wal_keeper
+ with _wal_keeper_lock:
+ if _wal_keeper is not None:
+ _close_keeper(_wal_keeper[1])
+ _wal_keeper = None
+
+
def _repair_owner_account_id(conn: sqlite3.Connection) -> None:
db_key = str(DB_PATH)
if db_key in _owner_id_repaired:
@@ -1458,42 +1508,113 @@ def save_refresh_token(
conn.close()
+def _take_refresh_token(conn: sqlite3.Connection, token: str, now: str):
+ """Inside the caller's BEGIN IMMEDIATE: purge expired rows, then delete-and-return the token's
+ row. Returns ``(row, user)`` (the auth_user row) when the token is live and its credential
+ generation still matches, else None; the delete stands either way."""
+ conn.execute(
+ "DELETE FROM refresh_tokens WHERE expires_at < ?",
+ (now,),
+ )
+ row = conn.execute(
+ f"""
+ DELETE FROM refresh_tokens
+ WHERE token_hash {_FENCED_HASH_SQL} AND expires_at >= ?
+ RETURNING username, is_desktop, secret_gen
+ """,
+ (*_hash_candidates(_hash_token(token)), now),
+ ).fetchone()
+ if row is None:
+ return None
+ user = conn.execute(
+ f"""
+ SELECT {_SECRET_SQL} AS jwt_secret, role, account_id, must_change_password
+ FROM auth_user WHERE username = ?
+ """,
+ (row["username"],),
+ ).fetchone()
+ if user is None:
+ return None
+ secret = user["jwt_secret"]
+ if row["secret_gen"] is not None and row["secret_gen"] != credential_generation(secret):
+ return None
+ return row, user
+
+
def consume_refresh_token(token: str) -> Optional[Tuple[str, bool, str]]:
"""Atomically validate-and-delete a refresh token for single-use rotation. DELETE RETURNING fuses
validate and delete into one statement so two concurrent refresh requests cannot both consume
the same token. Returns ``(username, is_desktop, jwt_secret)``; the caller must mint the
replacement tokens against that secret so a rotation landing mid-refresh cannot issue a
post-rotation session from a pre-rotation token."""
- token_hash = _hash_token(token)
now = datetime.now(timezone.utc).isoformat()
conn = get_connection()
try:
# One transaction with the delete: an unstamped legacy row has no generation, so a later read could
# hand a reset's new secret to an older token.
conn.execute("BEGIN IMMEDIATE")
+ taken = _take_refresh_token(conn, token, now)
+ conn.commit()
+ if taken is None:
+ return None
+ row, user = taken
+ return row["username"], bool(row["is_desktop"]), user["jwt_secret"]
+ finally:
+ conn.close()
+
+
+class RotatedRefresh(NamedTuple):
+ username: str
+ is_desktop: bool
+ jwt_secret: str
+ account_id: Optional[str]
+ must_change_password: bool
+
+
+def rotate_refresh_token(
+ token: str, new_token: str, new_expires_at: str
+) -> Optional[RotatedRefresh]:
+ """Consume ``token`` and store ``new_token`` in its place, in one transaction. If anything fails
+ before the commit, including a busy auth.db, the old token is still valid, so a retry can
+ succeed instead of logging the user out. None means the old token was invalid or expired."""
+ from utils.account_context import ROLE_OWNER
+
+ now = datetime.now(timezone.utc).isoformat()
+ conn = get_connection()
+ try:
+ conn.execute("BEGIN IMMEDIATE")
+ taken = _take_refresh_token(conn, token, now)
+ if taken is None:
+ conn.commit()
+ return None
+ row, user = taken
+ username = row["username"]
+ is_desktop = bool(row["is_desktop"]) and user["role"] == ROLE_OWNER
+ secret = user["jwt_secret"]
conn.execute(
- "DELETE FROM refresh_tokens WHERE expires_at < ?",
- (now,),
- )
- cur = conn.execute(
- f"""
- DELETE FROM refresh_tokens
- WHERE token_hash {_FENCED_HASH_SQL} AND expires_at >= ?
- RETURNING username, is_desktop, secret_gen
+ """
+ INSERT INTO refresh_tokens (token_hash, username, expires_at, is_desktop, secret_gen)
+ VALUES (?, ?, ?, ?, ?)
""",
- (*_hash_candidates(token_hash), now),
+ (
+ _fenced_hash(_hash_token(new_token), username),
+ username,
+ new_expires_at,
+ int(is_desktop),
+ credential_generation(secret),
+ ),
)
- row = cur.fetchone()
- if row is None:
- conn.commit()
- return None
- secret = _current_secret(conn, row["username"])
conn.commit()
- if secret is None:
- return None
- if row["secret_gen"] is not None and row["secret_gen"] != credential_generation(secret):
- return None
- return row["username"], bool(row["is_desktop"]), secret
+ return RotatedRefresh(
+ username = username,
+ is_desktop = is_desktop,
+ jwt_secret = secret,
+ account_id = user["account_id"],
+ must_change_password = False if is_desktop else bool(user["must_change_password"]),
+ )
+ except Exception:
+ conn.rollback()
+ raise
finally:
conn.close()
@@ -1861,7 +1982,13 @@ def validate_api_key_account(raw_key: str, *, touch: bool = True) -> Optional[Tu
conn = get_connection()
try:
if touch:
- conn.execute("BEGIN IMMEDIATE")
+ try:
+ conn.execute("BEGIN IMMEDIATE")
+ except sqlite3.OperationalError as exc:
+ if not sqlite_gate.is_busy_error(exc):
+ raise
+ # last_used_at is informational: a busy auth.db must not fail the request.
+ touch = False
cur = conn.execute(
f"""
SELECT k.id, k.username, k.name, k.is_active, k.expires_at,
diff --git a/studio/backend/main.py b/studio/backend/main.py
index d9cbfdce257..f2954dbcbf7 100644
--- a/studio/backend/main.py
+++ b/studio/backend/main.py
@@ -840,6 +840,11 @@ async def lifespan(app: FastAPI):
open_wal_keeper()
except Exception as exc:
_lifespan_log.warning("studio.db WAL keeper failed at startup: %s", exc)
+ # Same for auth.db, before account reconciliation and admin seeding open it.
+ try:
+ storage.open_wal_keeper()
+ except Exception as exc:
+ _lifespan_log.warning("auth.db WAL keeper failed at startup: %s", exc)
from utils.account_context import OWNER as _owner_account, run_as as _run_as
@@ -1076,6 +1081,10 @@ async def lifespan(app: FastAPI):
from storage.studio_db import close_wal_keeper
close_wal_keeper()
+ try:
+ storage.close_wal_keeper()
+ except Exception as exc:
+ _lifespan_log.warning("auth.db WAL keeper failed to close: %s", exc)
app = FastAPI(
@@ -1800,6 +1809,7 @@ app.add_middleware(
"X-Unsloth-Refusal",
"x-typesafe-request-id",
"X-Unsloth-Monitor-ID",
+ "Retry-After",
*_hub_endpoint_proxy.EXPOSED_HEADERS,
*_browser_routes.EXPOSED_HEADERS,
],
diff --git a/studio/backend/routes/auth.py b/studio/backend/routes/auth.py
index 5668569b967..67c40b7882a 100644
--- a/studio/backend/routes/auth.py
+++ b/studio/backend/routes/auth.py
@@ -11,6 +11,7 @@ import importlib.util
import ipaddress
import os
import shlex
+import sqlite3
import sys
import threading
import time
@@ -33,6 +34,8 @@ from models.auth import (
)
from models.users import Token
from auth import storage, hashing, policy
+from loggers import get_logger
+from storage import sqlite_gate
from utils.account_context import OWNER_ACCOUNT_ID
from auth.authentication import (
authenticated_via_desktop_jwt,
@@ -43,11 +46,13 @@ from auth.authentication import (
get_current_subject,
get_current_subject_allow_password_change,
is_keyless,
+ new_refresh_token_value,
refresh_access_token,
security,
)
router = APIRouter()
+logger = get_logger(__name__)
def _account_id_of(username: str) -> "str | None":
@@ -613,30 +618,37 @@ def desktop_login(payload: DesktopLoginRequest, request: Request) -> Token | Res
)
+# Sync def: rotation is one transaction, so a busy auth.db wait must not freeze the event loop.
@router.post("/refresh", response_model = Token)
-async def refresh(payload: RefreshTokenRequest) -> Token:
+def refresh(payload: RefreshTokenRequest) -> Token:
"""Exchange a refresh token for a new access+refresh pair (single-use)."""
- consumed = storage.consume_refresh_token(payload.refresh_token)
- if consumed is None:
+ new_refresh_token, new_expires_at = new_refresh_token_value()
+ try:
+ rotated = storage.rotate_refresh_token(
+ payload.refresh_token, new_refresh_token, new_expires_at
+ )
+ except sqlite3.OperationalError as exc:
+ if not sqlite_gate.is_busy_error(exc):
+ raise
+ logger.warning("Token refresh deferred: auth.db is busy (%s)", exc)
+ raise HTTPException(
+ status_code = status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail = "Unsloth Studio is busy. Try again in a moment.",
+ headers = {"Retry-After": "2"},
+ )
+ if rotated is None:
raise HTTPException(
status_code = status.HTTP_401_UNAUTHORIZED,
detail = "Invalid or expired refresh token",
)
- username, is_desktop, jwt_secret = consumed
- account = storage.get_account(username)
- if is_desktop:
- is_desktop = account is not None and account.is_owner
- new_access_token = create_access_token(subject = username, desktop = is_desktop, secret = jwt_secret)
- new_refresh_token = create_refresh_token(
- subject = username, desktop = is_desktop, secret = jwt_secret
- )
-
return Token(
- access_token = new_access_token,
+ access_token = create_access_token(
+ subject = rotated.username, desktop = rotated.is_desktop, secret = rotated.jwt_secret
+ ),
refresh_token = new_refresh_token,
token_type = "bearer",
- must_change_password = False if is_desktop else storage.requires_password_change(username),
- account_id = account.account_id if account is not None else None,
+ must_change_password = rotated.must_change_password,
+ account_id = rotated.account_id,
)
diff --git a/studio/backend/storage/sqlite_gate.py b/studio/backend/storage/sqlite_gate.py
index 0c9dc0e5c71..d9877c78a54 100644
--- a/studio/backend/storage/sqlite_gate.py
+++ b/studio/backend/storage/sqlite_gate.py
@@ -40,3 +40,25 @@ def connect(database: str | os.PathLike[str], **kwargs: Any) -> sqlite3.Connecti
"""sqlite3.connect with opens and closes serialized process-wide."""
with _CONNECTION_GATE:
return sqlite3.connect(str(database), factory = GatedConnection, **kwargs)
+
+
+def is_busy_error(exc: sqlite3.OperationalError) -> bool:
+ """Is this the writer lock being held elsewhere, rather than a real fault? sqlite reports it as
+ plain OperationalError, so the message is the only signal."""
+ message = str(exc).lower()
+ return "locked" in message or "busy" in message
+
+
+def apply_wal_synchronous(conn: sqlite3.Connection) -> None:
+ """synchronous=NORMAL, only while the file is really in WAL mode (per connection).
+
+ FULL fsyncs every commit under the writer lock (one busy-disk commit blocked 37s); NORMAL is
+ sqlite's recommended WAL pairing, durable across app crashes. journal_mode=WAL silently declines
+ on filesystems without shared memory, and NORMAL on a rollback journal drops the fsync that
+ keeps it consistent, so this reads the mode in force and those installs keep FULL."""
+ try:
+ mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
+ except (sqlite3.Error, TypeError, IndexError):
+ return
+ if isinstance(mode, str) and mode.lower() == "wal":
+ conn.execute("PRAGMA synchronous=NORMAL")
diff --git a/studio/backend/storage/studio_db.py b/studio/backend/storage/studio_db.py
index 6eb38a932fc..47eecfb191a 100644
--- a/studio/backend/storage/studio_db.py
+++ b/studio/backend/storage/studio_db.py
@@ -1308,12 +1308,9 @@ def bulk_upsert_prompt_lists(lists: list[dict]) -> int:
conn.close()
-def is_sqlite_busy_error(exc: sqlite3.OperationalError) -> bool:
- """Is this the writer lock being held elsewhere, rather than a real fault? sqlite reports it as
- plain OperationalError, so the message is the only signal. Lives here because studio.db is the
- contended file and several modules need the same answer; storage.api_usage_db delegates to it."""
- message = str(exc).lower()
- return "locked" in message or "busy" in message
+# Old names kept: storage.api_usage_db and core.research_runs import them from here.
+is_sqlite_busy_error = sqlite_gate.is_busy_error
+_apply_wal_synchronous = sqlite_gate.apply_wal_synchronous
# sqlite's own default, kept for every caller that may run on the event loop
@@ -1324,25 +1321,6 @@ _BUSY_TIMEOUT_SECONDS = 5.0
_CONTENDED_BUSY_TIMEOUT_SECONDS = 30.0
-def _apply_wal_synchronous(conn: sqlite3.Connection) -> None:
- """Drop to synchronous=NORMAL, but only while the file is really in WAL mode. Under WAL, sqlite
- still defaults to synchronous=FULL, which fsyncs on every commit while holding the writer lock:
- on a machine whose disk is busy one commit blocked for 37s, and every other writer spent that
- window timing out with "database is locked". NORMAL is sqlite's own recommended pairing for WAL:
- it can lose the last transactions to a host power loss, but the database is never corrupted and
- commits stay durable across an application crash. The WAL check is not decoration. `PRAGMA
- journal_mode=WAL` silently declines on filesystems without proper shared-memory support (network
- shares, some FUSE and container-mounted paths), leaving the file on a rollback journal where
- NORMAL drops the very fsync that keeps it consistent. Those installs keep FULL. journal_mode is
- persistent in the file, so this reads what is in force rather than what was requested."""
- try:
- mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
- except (sqlite3.Error, TypeError, IndexError):
- return
- if isinstance(mode, str) and mode.lower() == "wal":
- conn.execute("PRAGMA synchronous=NORMAL")
-
-
def reset_schema_state_for_tests() -> None:
with _schema_lock:
_schema_ready.clear()
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 5ae43195ada..95b03f86a5e 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -486,6 +486,17 @@ def _isolate_wal_keepers():
studio_db._wal_unsupported.intersection_update(unsupported)
[email protected](autouse = True)
+def _isolate_auth_wal_keeper():
+ """Close an auth.db keeper a test opened, so it never outlives that test's DB_PATH."""
+ auth_storage = sys.modules.get("auth.storage")
+ before = auth_storage._wal_keeper if auth_storage is not None else None
+ yield
+ auth_storage = sys.modules.get("auth.storage")
+ if auth_storage is not None and auth_storage._wal_keeper is not before:
+ auth_storage.close_wal_keeper()
+
+
@pytest.fixture(autouse = True)
def _isolate_audio_gallery(monkeypatch, tmp_path):
"""Keep generated-clip persistence out of the developer's real gallery.
diff --git a/studio/backend/tests/test_auth_db_lock_contention.py b/studio/backend/tests/test_auth_db_lock_contention.py
new file mode 100644
index 00000000000..eaec4403e5e
--- /dev/null
+++ b/studio/backend/tests/test_auth_db_lock_contention.py
@@ -0,0 +1,352 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A busy auth.db must not log the user out.
+
+On a Colab session a large GGUF download saturated the disk, /api/auth/refresh hit "database is
+locked" after the busy timeout, the route answered a plain 500 after the old token was already
+consumed, and the UI signed the user out. These pin the three links: auth.db commits skip the WAL
+fsync, the refresh rotation is one transaction (a failure leaves the old token valid), and a busy
+database answers a retryable 503.
+"""
+
+import hashlib
+import inspect
+import logging
+import secrets
+import sqlite3
+import threading
+from pathlib import Path
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+import routes.auth as auth_routes
+from auth import policy, storage
+from auth.authentication import new_refresh_token_value
+from storage import sqlite_gate
+
+FULL, NORMAL = 2, 1
+
+
+def fresh_db(tmp_path, monkeypatch):
+ monkeypatch.setattr(storage, "DB_PATH", tmp_path / "auth" / "auth.db")
+ monkeypatch.setattr(storage, "_BOOTSTRAP_PW_PATH", tmp_path / ".bootstrap_password")
+ monkeypatch.setattr(storage, "_bootstrap_password", None)
+ policy.invalidate_account_cache()
+ yield
+ policy.invalidate_account_cache()
+
+
+def owner(fresh_db):
+ storage.create_initial_user(
+ username = storage.DEFAULT_ADMIN_USERNAME,
+ password = "human-password-123",
+ jwt_secret = secrets.token_urlsafe(64),
+ must_change_password = False,
+ )
+ return storage.DEFAULT_ADMIN_USERNAME
+
+
+def client(owner):
+ app = FastAPI()
+ app.include_router(auth_routes.router, prefix = "/api/auth")
+ with TestClient(app) as http:
+ yield http
+
+
+def _refresh(http, token):
+ return http.post("/api/auth/refresh", json = {"refresh_token": token})
+
+
+def _issue(username):
+ token, expires_at = new_refresh_token_value()
+ storage.save_refresh_token(token, username, expires_at)
+ return token
+
+
+def _synchronous(conn) -> int:
+ return int(conn.execute("PRAGMA synchronous").fetchone()[0])
+
+
+def _live_tokens() -> int:
+ conn = storage.get_connection()
+ try:
+ return conn.execute("SELECT COUNT(*) FROM refresh_tokens").fetchone()[0]
+ finally:
+ conn.close()
+
+
+def test_auth_connections_use_normal_sync_under_wal(owner):
+ # owner already opened once, so this also covers the schema-cache early return.
+ for _ in range(2):
+ conn = storage.get_connection()
+ try:
+ assert str(conn.execute("PRAGMA journal_mode").fetchone()[0]).lower() == "wal"
+ assert _synchronous(conn) == NORMAL
+ finally:
+ conn.close()
+
+
+def test_rollback_journal_keeps_full_sync(tmp_path):
+ conn = sqlite3.connect(str(tmp_path / "rollback.db"))
+ try:
+ conn.execute("PRAGMA journal_mode=DELETE")
+ sqlite_gate.apply_wal_synchronous(conn)
+ assert _synchronous(conn) == FULL
+ finally:
+ conn.close()
+
+
+ "answer",
+ [None, (None,), ("",), (123,), sqlite3.OperationalError("pragma unavailable")],
+)
+def test_unreadable_journal_mode_is_not_treated_as_wal(answer):
+ class Connection:
+ def __init__(self):
+ self.executed = []
+
+ def execute(self, sql, *args):
+ self.executed.append(sql)
+ if isinstance(answer, Exception):
+ raise answer
+ return type("Cursor", (), {"fetchone": lambda _self: answer})()
+
+ conn = Connection()
+ sqlite_gate.apply_wal_synchronous(conn)
+ assert "PRAGMA synchronous=NORMAL" not in conn.executed
+
+
+def test_busy_auth_db_answers_retryable_503_and_keeps_the_token(client, owner, monkeypatch):
+ token = _issue(owner)
+ monkeypatch.setattr(storage, "_BUSY_TIMEOUT_MS", 50)
+ holder = sqlite3.connect(str(storage.DB_PATH), isolation_level = None)
+ try:
+ holder.execute("BEGIN IMMEDIATE")
+ busy = _refresh(client, token)
+ finally:
+ holder.execute("ROLLBACK")
+ holder.close()
+
+ assert busy.status_code == 503
+ assert busy.headers["Retry-After"] == "2"
+ assert busy.json()["detail"] == "Unsloth Studio is busy. Try again in a moment."
+
+ ok = _refresh(client, token)
+ assert ok.status_code == 200
+ body = ok.json()
+ assert body["access_token"] and body["refresh_token"] != token
+ assert body["account_id"] == "owner"
+ assert body["must_change_password"] is False
+
+ assert _refresh(client, body["refresh_token"]).status_code == 200
+ assert _refresh(client, token).status_code == 401
+
+
+def _last_used(key_id):
+ with sqlite3.connect(str(storage.DB_PATH)) as conn:
+ return conn.execute("SELECT last_used_at FROM api_keys WHERE id = ?", (key_id,)).fetchone()[
+ 0
+ ]
+
+
+def test_busy_auth_db_still_validates_an_api_key_without_recording_use(owner, monkeypatch):
+ raw, row = storage.create_api_key(owner, name = "cli")
+ assert storage.validate_api_key_account(raw) is not None
+ used = _last_used(row["id"])
+ assert used is not None
+
+ monkeypatch.setattr(storage, "_BUSY_TIMEOUT_MS", 50)
+ holder = sqlite3.connect(str(storage.DB_PATH), isolation_level = None)
+ try:
+ holder.execute("BEGIN IMMEDIATE")
+ verified = storage.validate_api_key_account(raw)
+ finally:
+ holder.execute("ROLLBACK")
+ holder.close()
+
+ assert verified is not None
+ assert verified[0]["username"] == owner
+ assert _last_used(row["id"]) == used
+
+
+def test_a_non_busy_error_on_the_api_key_touch_still_raises(owner, monkeypatch):
+ raw, _row = storage.create_api_key(owner, name = "cli")
+ real = storage.get_connection
+
+ class _Broken:
+ def __init__(self, conn):
+ self._conn = conn
+
+ def execute(self, sql, *args):
+ if sql == "BEGIN IMMEDIATE":
+ raise sqlite3.OperationalError("disk I/O error")
+ return self._conn.execute(sql, *args)
+
+ def __getattr__(self, name):
+ return getattr(self._conn, name)
+
+ monkeypatch.setattr(storage, "get_connection", lambda *a, **k: _Broken(real(*a, **k)))
+ with pytest.raises(sqlite3.OperationalError, match = "disk I/O error"):
+ storage.validate_api_key_account(raw)
+
+
+def test_failed_replacement_insert_leaves_the_old_token_valid(client, owner):
+ token = _issue(owner)
+ conn = storage.get_connection()
+ try:
+ conn.execute(
+ "CREATE TRIGGER reject_insert BEFORE INSERT ON refresh_tokens "
+ "BEGIN SELECT RAISE(ABORT, 'insert rejected'); END"
+ )
+ conn.commit()
+ finally:
+ conn.close()
+
+ new_token, expires_at = new_refresh_token_value()
+ with pytest.raises(sqlite3.IntegrityError):
+ storage.rotate_refresh_token(token, new_token, expires_at)
+ assert _live_tokens() == 1
+
+ conn = storage.get_connection()
+ try:
+ conn.execute("DROP TRIGGER reject_insert")
+ conn.commit()
+ finally:
+ conn.close()
+ assert _refresh(client, token).status_code == 200
+
+
+def test_refresh_token_stays_single_use_and_logout_revokes_the_rotation(client, owner):
+ token = _issue(owner)
+ first = _refresh(client, token)
+ assert first.status_code == 200
+ assert _refresh(client, token).status_code == 401
+
+ storage.revoke_user_refresh_tokens(owner)
+ assert _live_tokens() == 0
+ assert _refresh(client, first.json()["refresh_token"]).status_code == 401
+
+
+def test_non_busy_operational_error_is_not_mapped_to_503(owner, monkeypatch):
+ def broken(*_args):
+ raise sqlite3.OperationalError("disk I/O error")
+
+ monkeypatch.setattr(storage, "rotate_refresh_token", broken)
+ app = FastAPI()
+ app.include_router(auth_routes.router, prefix = "/api/auth")
+ with TestClient(app, raise_server_exceptions = False) as http:
+ response = _refresh(http, "t")
+ assert response.status_code == 500
+
+
+def _wal() -> Path:
+ return Path(f"{storage.DB_PATH}-wal")
+
+
+def _short_lived_write(username, n = 1):
+ for _ in range(n):
+ _issue(username)
+
+
+def test_keeper_holds_the_wal_until_released(owner):
+ assert not _wal().exists(), "without a keeper the last close checkpoints and removes it"
+ assert storage.open_wal_keeper() is True
+ _short_lived_write(owner, 20)
+ assert _wal().is_file()
+
+ storage.close_wal_keeper()
+ assert storage._wal_keeper is None
+ assert not _wal().exists()
+
+
+def test_closing_a_connection_under_the_keeper_does_not_checkpoint(owner):
+ """The checkpoint is what fsyncs inside sqlite_gate's process-wide lock."""
+ assert storage.open_wal_keeper() is True
+ try:
+ _short_lived_write(owner)
+ main_file = hashlib.sha256(storage.DB_PATH.read_bytes()).hexdigest()
+ wal_size = _wal().stat().st_size
+
+ _short_lived_write(owner)
+
+ assert _wal().stat().st_size >= wal_size
+ assert hashlib.sha256(storage.DB_PATH.read_bytes()).hexdigest() == main_file
+ finally:
+ storage.close_wal_keeper()
+
+
+def test_opening_again_replaces_the_keeper_of_an_earlier_db(owner, tmp_path, monkeypatch):
+ assert storage.open_wal_keeper() is True
+ stale = storage._wal_keeper[1]
+
+ monkeypatch.setattr(storage, "DB_PATH", tmp_path / "other" / "auth.db")
+ assert storage.open_wal_keeper() is True
+ assert storage._wal_keeper[0] == str(storage.DB_PATH.resolve())
+ with pytest.raises(sqlite3.ProgrammingError, match = "closed database"):
+ stale.execute("SELECT 1")
+ storage.close_wal_keeper()
+ assert storage._wal_keeper is None
+
+
[email protected]("then", ["close", "reopen"])
+def test_a_keeper_opened_on_an_exited_thread_is_released(owner, then):
+ opened = threading.Thread(target = storage.open_wal_keeper)
+ opened.start()
+ opened.join()
+ stale = storage._wal_keeper[1]
+
+ if then == "reopen":
+ assert storage.open_wal_keeper() is True
+ assert storage._wal_keeper[1] is not stale
+ storage.close_wal_keeper()
+ with pytest.raises(sqlite3.ProgrammingError, match = "closed database"):
+ stale.execute("SELECT 1")
+ assert not _wal().exists()
+
+
+def test_keeper_declines_a_rollback_journal(owner, monkeypatch, caplog):
+ real = storage.get_connection
+
+ def refused_wal(**kwargs):
+ conn = real(**kwargs)
+ conn.execute("PRAGMA journal_mode=DELETE")
+ return conn
+
+ monkeypatch.setattr(storage, "get_connection", refused_wal)
+ with caplog.at_level(logging.INFO, logger = storage.logger.name):
+ assert storage.open_wal_keeper() is False
+ assert storage._wal_keeper is None
+ assert "not WAL" in caplog.text
+
+
+def test_the_lifespan_holds_the_auth_keeper_across_every_writer():
+ """Reads the source rather than running the lifespan, which imports the whole stack."""
+ import main
+
+ source = inspect.getsource(main.lifespan)
+ served = source.index("yield")
+ opened = source.index("storage.open_wal_keeper()")
+ assert opened < source.index("startup_reconciliation_accounts()") < served
+ assert opened < source.index("storage.ensure_default_admin()") < served
+ assert (
+ served
+ < source.index("await run_lifespan_shutdown(")
+ < source.index("storage.close_wal_keeper()")
+ )
+
+
+def test_a_cross_origin_client_can_read_the_refresh_retry_after():
+ # tauri://localhost and tunnel pages are cross-origin: an unexposed header reads as null there.
+ import main
+ exposed = [
+ header
+ for middleware in main.app.user_middleware
+ for header in middleware.kwargs.get("expose_headers", ())
+ ]
+ assert "Retry-After" in exposed
diff --git a/studio/backend/tests/test_config_mutations_on_event_loop.py b/studio/backend/tests/test_config_mutations_on_event_loop.py
index 05ff6758de4..8d50a026e24 100644
--- a/studio/backend/tests/test_config_mutations_on_event_loop.py
+++ b/studio/backend/tests/test_config_mutations_on_event_loop.py
@@ -12,14 +12,15 @@ it read, with nothing below them serializing the pair:
and `credential_secrets` has no foreign key to `providers`.
* login clears its admission check before verify_password reaches _record_login_failure, so
the bucket lock guards each call but not the sequence.
- * refresh consumes a token and inserts its replacement, and logout deletes every token in
- between without rotating the credential generation.
They are await-free, so the loop is what makes those sequences atomic. In the threadpool, two
imports duplicate a server row, an update racing a delete writes a credential for a provider that
-is gone, a burst of guesses passes admission together, and a logout leaves the refresh token that
-landed after it. The read-only handlers beside them do belong in the threadpool, so both
-directions are pinned here.
+is gone, and a burst of guesses passes admission together. The read-only handlers beside them do
+belong in the threadpool, so both directions are pinned here.
+
+Refresh used to be pinned to the loop too, because it consumed a token and inserted its replacement
+in two commits that a logout could land between. It now rotates in one SQLite transaction, so it
+runs in the threadpool, where a busy auth.db wait does not freeze every other request.
Asserts which thread each handler ran on rather than racing two requests.
"""
@@ -97,16 +98,6 @@ _MUTATIONS = {
None,
401,
),
- "auth-refresh": _Case(
- "post",
- "/api/auth/refresh",
- {"refresh_token": "t"},
- auth_routes,
- "storage",
- "consume_refresh_token",
- None,
- 401,
- ),
"auth-logout": _Case(
"post",
"/api/auth/logout",
@@ -132,6 +123,20 @@ _READS = {
}
+_ATOMIC_MUTATIONS = {
+ "auth-refresh": _Case(
+ "post",
+ "/api/auth/refresh",
+ {"refresh_token": "t"},
+ auth_routes,
+ "storage",
+ "rotate_refresh_token",
+ None,
+ 401,
+ ),
+}
+
+
def _record(threads: list[int], result):
def _call(*_args, **_kwargs):
threads.append(threading.get_ident())
@@ -188,3 +193,9 @@ def test_a_state_changing_handler_runs_on_the_event_loop_thread(monkeypatch, cas
def test_a_read_only_handler_stays_off_the_event_loop_thread(monkeypatch, case):
threads, loop_thread = _drive(monkeypatch, case)
assert threads[0] != loop_thread, f"{case.path} read its store on the event loop thread"
+
+
[email protected]("case", _ATOMIC_MUTATIONS.values(), ids = list(_ATOMIC_MUTATIONS))
+def test_an_atomic_mutation_stays_off_the_event_loop_thread(monkeypatch, case):
+ threads, loop_thread = _drive(monkeypatch, case)
+ assert threads[0] != loop_thread, f"{case.path} blocked the event loop on SQLite"
diff --git a/studio/frontend/scripts/check-bundle-budget.ts b/studio/frontend/scripts/check-bundle-budget.ts
index 6bda08f7b2c..e72d00cbb3f 100644
--- a/studio/frontend/scripts/check-bundle-budget.ts
+++ b/studio/frontend/scripts/check-bundle-budget.ts
@@ -248,8 +248,12 @@ export const BUDGET = {
// (6,825,902 -> 6,826,090 B raw, 2,054,124 -> 2,054,190 B transfer against 94464babd6 and 1de12ab309, one machine
// and build, Node 24, chunk count unchanged at 95): the picker catalog is imported by startup code (chat-adapter.ts).
// Raised by that growth, to the measured build plus the same 160 B raw / 189 B transfer margin.
- transferBytes: 2_054_379,
- rawBytes: 6_826_250,
+ // Keeping the session when auth.db is busy (#14151) adds 829 B raw / 393 B transfer (6,826,090 -> 6,826,919 B
+ // raw, 2,054,190 -> 2,054,583 B transfer against a33463fb16, one machine and build, Node 24, chunk count
+ // unchanged at 96): the refresh retry and outcome live in features/auth/api.ts, which every authFetch imports.
+ // Raised by that growth, to the measured build plus the same 160 B raw / 189 B transfer margin.
+ transferBytes: 2_054_772,
+ rawBytes: 6_827_079,
};
// The chunk count is reported but not budgeted. Splitting a page out of the entry raises it while lowering the
diff --git a/studio/frontend/src/app/auth-guards.ts b/studio/frontend/src/app/auth-guards.ts
index d35f7020f9c..82d88ae4e6a 100644
--- a/studio/frontend/src/app/auth-guards.ts
+++ b/studio/frontend/src/app/auth-guards.ts
@@ -10,14 +10,16 @@ import {
hasAuthToken,
hasRefreshToken,
mustChangePassword,
- refreshSession,
+ refreshSessionOutcome,
setMustChangePassword,
} from "@/features/auth";
async function hasActiveSession(): Promise<boolean> {
if (hasAuthToken()) return true;
if (!hasRefreshToken()) return false;
- return refreshSession();
+ const outcome = await refreshSessionOutcome();
+ // A busy backend is not a signed-out user: a refresh token that survived is still a session.
+ return outcome === "ok" || (outcome === "unavailable" && hasRefreshToken());
}
interface AuthStatus {
diff --git a/studio/frontend/src/features/auth/api.ts b/studio/frontend/src/features/auth/api.ts
index fda7e6ab5ce..4ce435e9742 100644
--- a/studio/frontend/src/features/auth/api.ts
+++ b/studio/frontend/src/features/auth/api.ts
@@ -25,7 +25,7 @@ type AuthFetchOptions = {
};
let isRedirecting = false;
-let refreshInflight: Promise<boolean> | null = null;
+let refreshInflight: Promise<RefreshOutcome> | null = null;
let refreshInflightToken: string | null = null;
let logoutGeneration = 0;
@@ -307,38 +307,89 @@ async function retryWithTauriAutoAuth(
return null;
}
-export async function refreshSession(): Promise<boolean> {
- const refreshToken = getRefreshToken();
- if (!refreshToken) return false;
- if (refreshInflight && refreshInflightToken === refreshToken) {
- return refreshInflight;
+/** "rejected": the server refused the token or the user signed out. "unavailable": no answer yet. */
+export type RefreshOutcome = "ok" | "rejected" | "unavailable";
+
+// Never abort an in-flight POST: if the server already rotated the pair, dropping its answer loses the new token.
+const REFRESH_MAX_ATTEMPTS = 3;
+const REFRESH_BUDGET_MS = 8000;
+const REFRESH_RETRY_DEFAULT_MS = 1000;
+const REFRESH_RETRY_MIN_MS = 250;
+const REFRESH_RETRY_MAX_MS = 5000;
+
+function refreshRetryDelay(response: Response): number {
+ const header = response.headers.get("Retry-After")?.trim();
+ const seconds = header ? Number(header) : Number.NaN;
+ if (!Number.isFinite(seconds) || seconds < 0) {
+ return REFRESH_RETRY_DEFAULT_MS;
}
+ return Math.min(
+ Math.max(seconds * 1000, REFRESH_RETRY_MIN_MS),
+ REFRESH_RETRY_MAX_MS,
+ );
+}
+async function runRefresh(refreshToken: string): Promise<RefreshOutcome> {
const startGeneration = logoutGeneration;
- const promise = (async () => {
+ const startedAt = Date.now();
+ const superseded = (): RefreshOutcome | null => {
+ if (startGeneration !== logoutGeneration) return "rejected";
+ if (getRefreshToken() !== refreshToken) return "unavailable";
+ return null;
+ };
+ for (let attempt = 1; ; attempt++) {
+ let response: Response;
try {
- const response = await fetchWithTauriNetworkRetry(
- apiUrl("/api/auth/refresh"),
- {
- method: "POST",
- headers: { "Content-Type": "application/json" },
- body: JSON.stringify({ refresh_token: refreshToken }),
- },
- );
- if (!response.ok) {
- clearAuthTokensIfCurrent(refreshToken);
- return false;
+ response = await fetchWithTauriNetworkRetry(apiUrl("/api/auth/refresh"), {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({ refresh_token: refreshToken }),
+ });
+ } catch {
+ return "unavailable";
+ }
+ // Only a 401 rejects the token; a 5xx, 429 or proxy error says nothing about the session.
+ if (response.status === 401) {
+ clearAuthTokensIfCurrent(refreshToken);
+ return "rejected";
+ }
+ if (response.ok) {
+ let payload: RefreshResponse;
+ try {
+ payload = (await response.json()) as RefreshResponse;
+ } catch {
+ return "unavailable";
}
- const payload = (await response.json()) as RefreshResponse;
- if (startGeneration !== logoutGeneration) return false;
- if (getRefreshToken() !== refreshToken) return false;
+ const stale = superseded();
+ if (stale) return stale;
storeAuthTokens(payload.access_token, payload.refresh_token);
setMustChangePassword(payload.must_change_password ?? false);
- return true;
- } catch {
- return false;
+ return "ok";
}
- })();
+ if (response.status !== 503 && response.status !== 429) {
+ return "unavailable";
+ }
+ const delay = refreshRetryDelay(response);
+ if (
+ attempt >= REFRESH_MAX_ATTEMPTS ||
+ Date.now() - startedAt + delay > REFRESH_BUDGET_MS
+ ) {
+ return "unavailable";
+ }
+ await wait(delay);
+ const stale = superseded();
+ if (stale) return stale;
+ }
+}
+
+export async function refreshSessionOutcome(): Promise<RefreshOutcome> {
+ const refreshToken = getRefreshToken();
+ if (!refreshToken) return "rejected";
+ if (refreshInflight && refreshInflightToken === refreshToken) {
+ return refreshInflight;
+ }
+
+ const promise = runRefresh(refreshToken);
refreshInflight = promise;
refreshInflightToken = refreshToken;
try {
@@ -351,6 +402,10 @@ export async function refreshSession(): Promise<boolean> {
}
}
+export async function refreshSession(): Promise<boolean> {
+ return (await refreshSessionOutcome()) === "ok";
+}
+
export async function authFetch(
input: RequestInfo | URL,
init?: RequestInit,
@@ -400,8 +455,9 @@ export async function authFetch(
if (response.status !== 401) return response;
const refreshToken = getRefreshToken();
- const refreshed = await refreshSession();
- if (!refreshed) {
+ const outcome = await refreshSessionOutcome();
+ if (outcome === "unavailable") return response;
+ if (outcome === "rejected") {
if (isTauri) {
return (
(await retryWithTauriAutoAuth(
diff --git a/studio/frontend/src/features/auth/components/auth-form.tsx b/studio/frontend/src/features/auth/components/auth-form.tsx
index 5ddedd432f5..5282584bda5 100644
--- a/studio/frontend/src/features/auth/components/auth-form.tsx
+++ b/studio/frontend/src/features/auth/components/auth-form.tsx
@@ -15,7 +15,7 @@ import { Eye, EyeOff } from "lucide-react";
import { useEffect, useRef, useState } from "react";
import type { ReactElement } from "react";
import type { SyntheticEvent } from "react";
-import { refreshSession } from "../api";
+import { refreshSession, refreshSessionOutcome } from "../api";
import {
deadlineFromStatus,
formatCountdown,
@@ -271,9 +271,12 @@ export function AuthForm({ mode }: AuthFormProps): ReactElement | null {
let accessToken = getAuthToken();
if (hasRefreshToken()) {
- const refreshed = await refreshSession();
+ const outcome = await refreshSessionOutcome();
+ if (outcome === "unavailable") {
+ throw new Error("Unsloth is busy. Try again in a moment.");
+ }
accessToken = getAuthToken();
- if (!refreshed) {
+ if (outcome === "rejected") {
clearAuthTokens();
accessToken = null;
}
diff --git a/studio/frontend/src/features/auth/index.ts b/studio/frontend/src/features/auth/index.ts
index e96fa473c0a..4fbbbd22d5e 100644
--- a/studio/frontend/src/features/auth/index.ts
+++ b/studio/frontend/src/features/auth/index.ts
@@ -3,7 +3,12 @@
export { LoginPage } from "./login-page";
export { ChangePasswordPage } from "./change-password-page";
-export { authFetch, logout, refreshSession } from "./api";
+export {
+ authFetch,
+ logout,
+ refreshSession,
+ refreshSessionOutcome,
+} from "./api";
export { AUTH_SESSION_ENDING_EVENT } from "./session-events";
export {
AUTH_SESSION_CLEARED_EVENT,
diff --git a/studio/frontend/src/features/auth/tauri-auto-auth.ts b/studio/frontend/src/features/auth/tauri-auto-auth.ts
index da0518c81cf..e482d77ebb3 100644
--- a/studio/frontend/src/features/auth/tauri-auto-auth.ts
+++ b/studio/frontend/src/features/auth/tauri-auto-auth.ts
@@ -10,7 +10,7 @@ import {
setMustChangePassword,
storeAuthTokens,
} from "./session";
-import { refreshSession } from "./api";
+import { refreshSession, refreshSessionOutcome } from "./api";
import { setLoginMode } from "./login-client";
import {
OWNER_BROWSER_ACCOUNT,
@@ -86,7 +86,10 @@ async function doTauriAutoAuth(options: TauriAutoAuthOptions): Promise<boolean>
if ("login_required" in tokens) {
tauriLoginRequired = true;
// A managed account signs in in the document; do not discard the session it holds.
- if (hasRefreshToken() && (await refreshSession()) && hasAuthToken()) {
+ const outcome = hasRefreshToken()
+ ? await refreshSessionOutcome()
+ : "rejected";
+ if (outcome === "ok" && hasAuthToken()) {
clearTauriAuthFailure();
if (mustChangePassword()) {
const { router } = await import("@/app/router");
@@ -94,6 +97,11 @@ async function doTauriAutoAuth(options: TauriAutoAuthOptions): Promise<boolean>
}
return true;
}
+ // Keep the session; the forced startup probe still releases the startup screen.
+ if (outcome === "unavailable") {
+ clearTauriAuthFailure();
+ return options.force === true;
+ }
clearAuthTokens();
clearTauriAuthFailure();
// The forced startup probe must release the startup screen so the login form can mount; ordinary API recovery still returns false.
diff --git a/studio/frontend/src/features/settings/api/debug-logs.ts b/studio/frontend/src/features/settings/api/debug-logs.ts
index a60140ec880..aa45d9b32e0 100644
--- a/studio/frontend/src/features/settings/api/debug-logs.ts
+++ b/studio/frontend/src/features/settings/api/debug-logs.ts
@@ -230,12 +230,7 @@ export async function exportAllLogs(): Promise<string | null> {
// Rust uses the token once and as-is, so it cannot retry a 401 by
// refreshing the way `authFetch` does; a desktop left idle past the
// access-token lifetime would fail while holding a good refresh token.
- // Refreshing only HERE, rather than before every export, is what keeps a
- // transient 500 on /api/auth/refresh from signing the whole app out:
- // `refreshSession` clears both stored tokens on any non-2xx, so doing it
- // speculatively would end the session of a user whose token was fine. A
- // 401 means the token really was rejected, which is the one case where
- // losing it costs nothing that was not already lost.
+ // Refresh only on a 401, so a spare refresh never rotates the pair for nothing.
if (desktopExportStatus(error) !== 401) throw desktopExportError(error);
try {
if (!(await refreshSession())) throw error;
diff --git a/studio/frontend/tests/account-desktop-auth.test.ts b/studio/frontend/tests/account-desktop-auth.test.ts
index e26a4a6f527..88b8c8c1914 100644
--- a/studio/frontend/tests/account-desktop-auth.test.ts
+++ b/studio/frontend/tests/account-desktop-auth.test.ts
@@ -34,6 +34,7 @@ function fixture(t: { after(fn: () => void): void }, tauri = true) {
let invokeCalls = 0;
let refreshCalls = 0;
let refreshSucceeds = false;
+ let refreshOutcome: "ok" | "rejected" | "unavailable" | null = null;
let statusCalls = 0;
let status: {
initialized: boolean;
@@ -57,6 +58,13 @@ function fixture(t: { after(fn: () => void): void }, tauri = true) {
refresh = "refreshed-refresh";
return true;
},
+ refreshSessionOutcome: async () => {
+ if (refreshOutcome) {
+ refreshCalls++;
+ return refreshOutcome;
+ }
+ return (await session.refreshSession()) ? "ok" : "rejected";
+ },
};
const apiBase = { isTauri: tauri, apiUrl: (path: string) => path };
const transitions: { account: unknown; route: string }[] = [];
@@ -112,6 +120,9 @@ function fixture(t: { after(fn: () => void): void }, tauri = true) {
setResponse: (value: unknown) => { response = value; },
setStatus: (value: typeof status) => { status = value; },
setRefreshSucceeds: (value: boolean) => { refreshSucceeds = value; },
+ setRefreshOutcome: (value: "ok" | "rejected" | "unavailable" | null) => {
+ refreshOutcome = value;
+ },
get access() { return access; },
get refresh() { return refresh; },
get invokeCalls() { return invokeCalls; },
@@ -162,6 +173,30 @@ test("forced multi-account startup opens login and releases the startup screen w
await assert.rejects(f.guards.requirePasswordChangeFlow(), { to: "/login" });
});
+test("a managed session survives forced startup while its refresh gets no answer", async (t) => {
+ const f = fixture(t);
+ f.session.storeAuthTokens("alice-access", "alice-refresh");
+ f.setResponse(multi);
+ f.setRefreshOutcome("unavailable");
+ assert.equal(await f.auth.tauriAutoAuth({ force: true }), true);
+ assert.equal(f.access, "alice-access");
+ assert.equal(f.refresh, "alice-refresh");
+ assert.deepEqual(f.navigations, []);
+ assert.equal(f.auth.getTauriAuthFailure(), null);
+ await f.guards.requireAuth();
+});
+
+test("a managed session whose refresh is rejected at startup still opens login", async (t) => {
+ const f = fixture(t);
+ f.session.storeAuthTokens("alice-access", "alice-refresh");
+ f.setResponse(multi);
+ f.setRefreshOutcome("rejected");
+ assert.equal(await f.auth.tauriAutoAuth({ force: true }), true);
+ assert.equal(f.access, null);
+ assert.equal(f.refresh, null);
+ assert.deepEqual(f.navigations, [{ to: "/login", replace: true }]);
+});
+
test("ordinary multi-account API recovery never reports an authenticated session", async (t) => {
const f = fixture(t);
f.setResponse(multi);
diff --git a/studio/frontend/tests/auth-refresh-transient.test.ts b/studio/frontend/tests/auth-refresh-transient.test.ts
new file mode 100644
index 00000000000..a4a4ea32a9c
--- /dev/null
+++ b/studio/frontend/tests/auth-refresh-transient.test.ts
@@ -0,0 +1,286 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+// A busy auth.db answers /api/auth/refresh with 503 + Retry-After and keeps the refresh
+// token valid. Only a 401 may end the session; anything else keeps both tokens.
+
+import assert from "node:assert/strict";
+import test, { type TestContext } from "node:test";
+import { loadWithStubs } from "./helpers/module-stubs.ts";
+import type * as AuthApi from "../src/features/auth/api.ts";
+import type * as Guards from "../src/app/auth-guards.ts";
+
+type Refresh = () => Response | Promise<Response>;
+
+const busy = (retryAfter: string | null = "1") =>
+ new Response(JSON.stringify({ detail: "busy" }), {
+ status: 503,
+ headers: retryAfter === null ? {} : { "Retry-After": retryAfter },
+ });
+const rotated = () =>
+ Response.json({
+ access_token: "new-access",
+ refresh_token: "new-refresh",
+ must_change_password: false,
+ });
+
+function fixture(t: TestContext, refreshes: Refresh[]) {
+ const previousWindow = globalThis.window;
+ const previousFetch = globalThis.fetch;
+ const previousSetTimeout = globalThis.setTimeout;
+ const previousNow = Date.now;
+ const location = { pathname: "/chat", href: "" };
+ const tokens = {
+ access: "old-access" as string | null,
+ refresh: "old-refresh" as string | null,
+ };
+ let now = 0;
+ let clears = 0;
+ let unexpected = 0;
+ const delays: number[] = [];
+ const refreshBodies: string[] = [];
+ const apiTokens: (string | null)[] = [];
+ const state = { onWait: null as null | (() => void) };
+ globalThis.window = { location } as unknown as Window & typeof globalThis;
+ // A fake clock: a wait advances it and resolves on the next turn, not in real time.
+ Date.now = () => now;
+ globalThis.setTimeout = ((fn: () => void, ms?: number) => {
+ delays.push(ms ?? 0);
+ const hook = state.onWait;
+ state.onWait = null;
+ hook?.();
+ setImmediate(() => {
+ now += ms ?? 0;
+ fn();
+ });
+ return 0;
+ }) as unknown as typeof setTimeout;
+ globalThis.fetch = async (input, init) => {
+ const path = String(input);
+ if (path.endsWith("/status")) {
+ return Response.json({
+ login_mode: "single",
+ requires_password_change: false,
+ initialized: true,
+ });
+ }
+ if (path.endsWith("/logout")) return new Response(null, { status: 200 });
+ if (path.endsWith("/refresh")) {
+ refreshBodies.push(String(init?.body));
+ const next = refreshes.shift();
+ if (!next) {
+ unexpected++;
+ return new Response(null, { status: 418 });
+ }
+ return next();
+ }
+ const auth = new Headers(init?.headers).get("Authorization");
+ apiTokens.push(auth);
+ return new Response(null, {
+ status: auth === "Bearer new-access" ? 200 : 401,
+ });
+ };
+ t.after(() => {
+ globalThis.window = previousWindow;
+ globalThis.fetch = previousFetch;
+ globalThis.setTimeout = previousSetTimeout;
+ Date.now = previousNow;
+ assert.equal(unexpected, 0, "a refresh ran past its script");
+ });
+ const session = {
+ getAuthToken: () => tokens.access,
+ getRefreshToken: () => tokens.refresh,
+ hasAuthToken: () => !!tokens.access,
+ hasRefreshToken: () => !!tokens.refresh,
+ mustChangePassword: () => false,
+ setMustChangePassword: () => undefined,
+ storeAuthTokens: (access: string, refresh: string) => {
+ tokens.access = access;
+ tokens.refresh = refresh;
+ },
+ clearAuthTokens: () => {
+ clears++;
+ tokens.access = null;
+ tokens.refresh = null;
+ },
+ };
+ const apiBase = {
+ apiUrl: (path: string) => path,
+ getApiPort: () => null,
+ isTauri: false,
+ };
+ const api = loadWithStubs<typeof AuthApi>(
+ new URL("../src/features/auth/api.ts", import.meta.url),
+ {
+ "@/lib/api-base": apiBase,
+ "@/lib/account-transition": { accountTransitionPending: () => false },
+ "./session": session,
+ },
+ );
+ const settle = async () => {
+ for (let i = 0; i < 5; i++) {
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ }
+ };
+ return {
+ api,
+ session,
+ apiBase,
+ tokens,
+ location,
+ delays,
+ refreshBodies,
+ apiTokens,
+ state,
+ settle,
+ clears: () => clears,
+ };
+}
+
+test("a 503 then 200 is retried and stores the rotated pair", async (t) => {
+ const f = fixture(t, [() => busy("0"), () => busy("1"), rotated]);
+ assert.equal(await f.api.refreshSession(), true);
+ assert.equal(f.refreshBodies.length, 3);
+ assert.deepEqual(f.delays, [250, 1000]);
+ assert.deepEqual(f.tokens, { access: "new-access", refresh: "new-refresh" });
+});
+
+test("a persistent 503 is bounded and keeps both tokens", async (t) => {
+ const f = fixture(t, [() => busy(null), () => busy("60"), () => busy()]);
+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");
+ assert.equal(f.refreshBodies.length, 3);
+ assert.deepEqual(f.delays, [1000, 5000]);
+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });
+ assert.equal(f.clears(), 0);
+ await f.settle();
+ assert.equal(f.location.href, "");
+});
+
+test("retries stop once the wait would pass the budget", async (t) => {
+ const f = fixture(t, [() => busy("5"), () => busy("5")]);
+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");
+ assert.equal(f.refreshBodies.length, 2);
+ assert.deepEqual(f.delays, [5000]);
+});
+
+test("a 429 is retried like a 503", async (t) => {
+ const f = fixture(t, [
+ () => new Response(null, { status: 429, headers: { "Retry-After": "2" } }),
+ rotated,
+ ]);
+ assert.equal(await f.api.refreshSessionOutcome(), "ok");
+ assert.deepEqual(f.delays, [2000]);
+});
+
+test("a 500 is not retried and keeps both tokens", async (t) => {
+ const f = fixture(t, [() => new Response("boom", { status: 500 })]);
+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");
+ assert.equal(f.refreshBodies.length, 1);
+ assert.deepEqual(f.delays, []);
+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });
+ assert.equal(f.clears(), 0);
+});
+
+test("a network error keeps both tokens", async (t) => {
+ const f = fixture(t, [
+ () => {
+ throw new TypeError("Failed to fetch");
+ },
+ ]);
+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");
+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });
+});
+
+test("a malformed 200 body keeps both tokens", async (t) => {
+ const f = fixture(t, [() => new Response("not json", { status: 200 })]);
+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");
+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });
+});
+
+test("a 401 still clears the tokens and redirects to /login", async (t) => {
+ const f = fixture(t, [() => new Response(null, { status: 401 })]);
+ const response = await f.api.authFetch("/api/models");
+ assert.equal(response.status, 401);
+ assert.equal(f.clears() > 0, true);
+ assert.deepEqual(f.tokens, { access: null, refresh: null });
+ await f.settle();
+ assert.equal(f.location.href, "/login");
+});
+
+test("a pair rotated by another tab mid-retry stops without replay or clear", async (t) => {
+ const f = fixture(t, [() => busy("1")]);
+ f.state.onWait = () => {
+ f.tokens.access = "peer-access";
+ f.tokens.refresh = "peer-refresh";
+ };
+ assert.equal(await f.api.refreshSessionOutcome(), "unavailable");
+ assert.equal(f.refreshBodies.length, 1, "the old token was replayed");
+ assert.deepEqual(f.tokens, { access: "peer-access", refresh: "peer-refresh" });
+ assert.equal(f.clears(), 0);
+});
+
+test("a logout during the retry wait ends the refresh", async (t) => {
+ const f = fixture(t, [() => busy("1")]);
+ let loggedOut: Promise<void> | null = null;
+ f.state.onWait = () => {
+ loggedOut = f.api.logout();
+ };
+ assert.equal(await f.api.refreshSessionOutcome(), "rejected");
+ await loggedOut;
+ assert.equal(f.refreshBodies.length, 1, "the refresh outlived the logout");
+ assert.deepEqual(f.tokens, { access: null, refresh: null });
+});
+
+test("authFetch hands back the original 401 while refresh is unavailable", async (t) => {
+ const f = fixture(t, [() => busy("0"), () => busy("0"), () => busy("0")]);
+ const response = await f.api.authFetch("/api/models");
+ assert.equal(response.status, 401);
+ assert.equal(f.refreshBodies.length, 3);
+ assert.deepEqual(f.apiTokens, ["Bearer old-access"], "the request was replayed");
+ assert.deepEqual(f.tokens, { access: "old-access", refresh: "old-refresh" });
+ assert.equal(f.clears(), 0);
+ await f.settle();
+ assert.equal(f.location.href, "");
+});
+
+test("authFetch replays the request with the refreshed token", async (t) => {
+ const f = fixture(t, [rotated]);
+ const response = await f.api.authFetch("/api/models");
+ assert.equal(response.status, 200);
+ assert.deepEqual(f.apiTokens, ["Bearer old-access", "Bearer new-access"]);
+});
+
+function loadGuards(f: ReturnType<typeof fixture>) {
+ return loadWithStubs<typeof Guards>(
+ new URL("../src/app/auth-guards.ts", import.meta.url),
+ {
+ "@tanstack/react-router": {
+ redirect: (target: { to: string }) =>
+ Object.assign(new Error(`redirect ${target.to}`), { target }),
+ },
+ "@/lib/api-base": f.apiBase,
+ "@/features/auth/tauri-auto-auth": { isTauriLoginRequired: () => true },
+ "@/features/auth/login-client": { setLoginMode: () => undefined },
+ "@/features/auth": {
+ ...f.session,
+ getPostAuthRoute: () => "/chat",
+ refreshSession: f.api.refreshSession,
+ refreshSessionOutcome: f.api.refreshSessionOutcome,
+ },
+ },
+ );
+}
+
+test("the route guard keeps a session whose refresh is unavailable", async (t) => {
+ const f = fixture(t, [() => busy("0"), () => busy("0"), () => busy("0")]);
+ f.tokens.access = null;
+ await loadGuards(f).requireAuth();
+ assert.equal(f.tokens.refresh, "old-refresh");
+});
+
+test("the route guard still sends a rejected refresh to /login", async (t) => {
+ const f = fixture(t, [() => new Response(null, { status: 401 })]);
+ f.tokens.access = null;
+ await assert.rejects(loadGuards(f).requireAuth(), /redirect \/login/);
+ assert.equal(f.tokens.refresh, null);
+});
diff --git a/studio/frontend/tests/debug-log-export.test.ts b/studio/frontend/tests/debug-log-export.test.ts
index 49455aa05be..08bb620feae 100644
--- a/studio/frontend/tests/debug-log-export.test.ts
+++ b/studio/frontend/tests/debug-log-export.test.ts
@@ -328,9 +328,9 @@ test("openLogsFolder preserves the selected path with an older backend", async (
});
test("an export that works does not touch the session", async () => {
- // refreshSession clears both stored tokens on ANY non-2xx, so refreshing
- // before every export would let a transient 500 on /api/auth/refresh sign the
- // user out of the whole app just for clicking Download all logs.
+ // refreshSession rotates the stored pair, so refreshing before every export
+ // would spend a refresh token, and risk a sign-out, just for clicking
+ // Download all logs.
const world = makeWorld({
isTauri: true,
invoke: async () => SAVED_PATH,
diff --git a/studio/frontend/tests/login-account-modes.test.ts b/studio/frontend/tests/login-account-modes.test.ts
index 2e73d408e5b..f4ff0af2079 100644
--- a/studio/frontend/tests/login-account-modes.test.ts
+++ b/studio/frontend/tests/login-account-modes.test.ts
@@ -50,7 +50,14 @@ const tick = () => new Promise<void>((resolve) => setImmediate(resolve));
function mountForm(
client: typeof LoginClient,
- session: { access: string | null; change: boolean; owner?: boolean },
+ session: {
+ access: string | null;
+ change: boolean;
+ owner?: boolean;
+ refresh?: boolean;
+ outcome?: "ok" | "rejected" | "unavailable";
+ clears?: number;
+ },
switched = false,
failTransitionOnce = false,
) {
@@ -129,15 +136,20 @@ function mountForm(
},
},
"lucide-react": { Eye: "Eye", EyeOff: "EyeOff" },
- "../api": { refreshSession: async () => true },
+ "../api": {
+ refreshSession: async () => true,
+ refreshSessionOutcome: async () => session.outcome ?? "ok",
+ },
"../bootstrap-deadline": deadline,
"../session": {
clearAuthTokens: () => {
session.access = null;
+ session.refresh = false;
+ session.clears = (session.clears ?? 0) + 1;
},
getAuthToken: () => session.access,
hasAuthToken: () => Boolean(session.access),
- hasRefreshToken: () => false,
+ hasRefreshToken: () => Boolean(session.refresh),
getPostAuthRoute: () => (session.change ? "/change-password" : "/chat"),
mustChangePassword: () => session.change,
setMustChangePassword: (change: boolean) => {
@@ -591,3 +603,66 @@ test("a deactivated account restricts the legacy hint in peer tabs without chang
unsubscribe();
assert.equal(handlers.size, 0);
});
+
+async function submitPasswordChange(
+ session: Parameters<typeof mountForm>[1],
+ requests: string[],
+) {
+ globalThis.fetch = async (path) => {
+ requests.push(String(path));
+ return String(path).endsWith("/status")
+ ? Response.json({
+ initialized: true,
+ requires_password_change: true,
+ login_mode: "single",
+ })
+ : Response.json(token);
+ };
+ const form = mountForm(client(), session);
+ let tree = await form.initialize("change-password");
+ for (const [id, value] of [
+ ["current-password", "old-password-1"],
+ ["new-password", "new-password-1"],
+ ["confirm-password", "new-password-1"],
+ ]) {
+ const change = elements(tree).find((element) => element.props.id === id)
+ ?.props.onChange as (event: unknown) => void;
+ change({ target: { value } });
+ tree = form.render("change-password");
+ }
+ const submit = elements(tree).find((element) => element.type === "form")
+ ?.props.onSubmit as (event: unknown) => Promise<void>;
+ await submit({ preventDefault() {} });
+ return form.render("change-password");
+}
+
+test("a busy refresh during a password change keeps the session and asks to retry", async (t) => {
+ environment(t);
+ const requests: string[] = [];
+ const session: Parameters<typeof mountForm>[1] = {
+ access: "access",
+ change: true,
+ refresh: true,
+ outcome: "unavailable",
+ };
+ const tree = await submitPasswordChange(session, requests);
+ assert.equal(session.access, "access");
+ assert.equal(session.refresh, true);
+ assert.equal(session.clears ?? 0, 0);
+ assert.ok(!requests.some((path) => path.endsWith("/api/auth/change-password")));
+ assert.match(textContent(tree), /Unsloth is busy/);
+});
+
+test("a rejected refresh during a password change still drops the stored session", async (t) => {
+ environment(t);
+ const requests: string[] = [];
+ const session: Parameters<typeof mountForm>[1] = {
+ access: "access",
+ change: true,
+ refresh: true,
+ outcome: "rejected",
+ };
+ await submitPasswordChange(session, requests);
+ assert.equal(session.clears, 1);
+ assert.ok(requests.some((path) => path.endsWith("/api/auth/login")));
+});