Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop rescanning every model folder on each request that names a model not on disk (#11872)
oobabooga9ff97b6273b0
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.pyindex e186a6f305e..151e60aa2cd 100644--- a/studio/backend/core/inference/local_model_resolver.py+++ b/studio/backend/core/inference/local_model_resolver.py@@ -22,7 +22,14 @@ from typing import Optionalfrom core.inference.model_ids import public_model_idfrom loggers import get_logger-from utils.account_context import account_thread, current_account_id, is_owner_context+from utils.account_context import (+ AccountContext,+ account_thread,+ current_account,+ current_account_id,+ is_owner_context,+ run_as,+)logger = get_logger(__name__)@@ -74,11 +81,42 @@ _warming = False# is rebuilt off the request path. Callers still pair invalidate_index() with warm_index_soon() for the case where it# has retired._warm_pending = False+_warm_accounts: dict[Optional[str], AccountContext] = {}+_warm_active_account: Optional[AccountContext] = None+_warm_retry_scopes: set[Optional[str]] = set()_last_scan_s = 0.0+_managed_last_scan_s: dict[str, float] = {}# rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously_WARM_DUTY = 10.0+def _scope_for_account(account: AccountContext) -> Optional[str]:+ return None if account.is_owner else account.account_id+++def _account_scope() -> Optional[str]:+ return _scope_for_account(current_account())+++def _account_duty_window(account_id: Optional[str]) -> float:+ scan_s = _last_scan_s if account_id is None else _managed_last_scan_s.get(account_id, 0.0)+ return max(_CACHE_TTL_S, scan_s * _WARM_DUTY)+++def _duty_window() -> float:+ """Minimum snapshot age for a background rescan."""+ return _account_duty_window(_account_scope())+++def _record_scan_duration(duration: float) -> None:+ global _last_scan_s+ account_id = _account_scope()+ if account_id is None:+ _last_scan_s = duration+ else:+ _managed_last_scan_s[account_id] = duration++def _is_abs_path_id(value: str) -> bool:"""True when an id is an absolute filesystem path (the ./models and LM Studio scanners use theon-disk path as the id) rather than a repo id like org/name.@@ -958,7 +996,12 @@ def recently_downloaded(repo_id: str) -> bool:return repo_id.strip().lower() in _just_downloaded-def _snapshot_is_trusted(timestamp: float, now: float) -> bool:+def _snapshot_is_trusted(+ timestamp: float,+ now: float,+ *,+ duty_window: Optional[float] = None,+) -> bool:"""Whether a snapshot stamped *timestamp* may answer a model switch at *now*.Positive is an ordinary scan, trusted for the TTL. Negative is when an@@ -969,7 +1012,7 @@ def _snapshot_is_trusted(timestamp: float, now: float) -> bool:if timestamp > 0.0:return now - timestamp < _CACHE_TTL_Sif timestamp < 0.0:- return now + timestamp < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY)+ return now + timestamp < (duty_window if duty_window is not None else _duty_window())return False@@ -978,6 +1021,13 @@ def _snapshot_is_trusted(timestamp: float, now: float) -> bool:_generation = 0+# Confirmed misses keyed by account and name, valid for the recorded invalidation generation.+_misses: dict[tuple[Optional[str], str], int] = {}+_MAX_MISSES = 256+# Request bodies may be hundreds of MB; longer names are not retained.+_MAX_MISS_NAME = 4096++def index_generation() -> int:"""How many times the local scan has been invalidated since this process started."""return _generation@@ -996,21 +1046,31 @@ def invalidate_index(*, additions_only: bool = False) -> None:now = time.monotonic()_generation += 1- def _invalidated(snapshot):+ def _invalidated(snapshot, account_id):timestamp, retained = snapshot# Entries and trust state publish together: a lock-free reader never sees a fresh# timestamp paired with revoked trust.- stamp = -now if additions_only and _snapshot_is_trusted(timestamp, now) else 0.0+ stamp = (+ -now+ if additions_only+ and _snapshot_is_trusted(+ timestamp,+ now,+ duty_window = _account_duty_window(account_id),+ )+ else 0.0+ )return (stamp, retained)# Every account's snapshot: what changed on disk is not scoped to whoever noticed.- _scan = _invalidated(_scan)+ _scan = _invalidated(_scan, None)for account_id, snapshot in list(_managed_scans.items()):- _managed_scans[account_id] = _invalidated(snapshot)+ _managed_scans[account_id] = _invalidated(snapshot, account_id)# This may have waited out a scan on _lock, so the warmer that just published can still own the slot with a snapshot# that is stale again. See _warm_pending.with _warm_lock:- if _warming:+ if _warm_active_account is not None:+ _warm_retry_scopes.add(_scope_for_account(_warm_active_account))_warm_pending = True@@ -1024,7 +1084,11 @@ def _index() -> dict[str, _LocalGgufEntry]:# would serve what was just revokedif ts > 0.0 and now - ts < _CACHE_TTL_S:return cached- fresh = _build_index()+ # Request scans also set the background rescan interval.+ try:+ fresh = _build_index()+ finally:+ _record_scan_duration(time.monotonic() - now)# Stamp AFTER the scan, not with the pre-scan ``now``: a multi-root scan on an install with many local models# can itself exceed the TTL, which would store the cache already expired and make every request rebuild the# index.@@ -1080,42 +1144,110 @@ def warm_index_soon() -> None:"""global _warming, _warm_pendingstamp = _snapshot()[0]- if stamp > 0.0 and time.monotonic() - stamp < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY):+ if stamp > 0.0 and time.monotonic() - stamp < _duty_window():return+ account = current_account()+ scope = _scope_for_account(account)with _warm_lock:+ if (+ _warm_active_account is not None and scope == _scope_for_account(_warm_active_account)+ ) or scope in _warm_accounts:+ return+ _warm_accounts[scope] = accountif _warming:return_warming = True_warm_pending = Falsedef _run() -> None:- global _warming, _warm_pending, _last_scan_s+ global _warm_active_account, _warming, _warm_pendingreleased = Falsetry:while True:- started = time.monotonic()+ with _warm_lock:+ if not _warm_accounts:+ _warming, _warm_pending, released = False, False, True+ return+ scope, account = next(iter(_warm_accounts.items()))+ del _warm_accounts[scope]+ _warm_active_account = accounttry:- _index()+ run_as(account, _index)except Exception:pass- _last_scan_s = time.monotonic() - startedwith _warm_lock:- if _warm_pending:- _warm_pending = False- continue- _warming, released = False, True- return+ _warm_active_account = None+ if scope in _warm_retry_scopes:+ _warm_retry_scopes.discard(scope)+ _warm_accounts[scope] = account+ _warm_pending = bool(_warm_retry_scopes)finally:- # Only on a BaseException: leaving the slot held would kill background warming for the life of the process- # and put scans back on requests.+ # Only on a BaseException: leaving scopes queued without a worker would kill background warming for the+ # life of the process and put scans back on requests.if not released:with _warm_lock:+ _warm_accounts.clear()+ _warm_retry_scopes.clear()+ _warm_active_account = None_warming = _warm_pending = False- # Pinned to the caller's account, or the rebuild would publish under the owner's scope.+ # Capture a real account context for the thread; queued scans rebind to their own account above.account_thread(target = _run, name = "local-model-index-warm", daemon = True).start()+def _miss_scope() -> Optional[str]:+ return _account_scope()+++def _scope_is_warming(scope: Optional[str]) -> bool:+ with _warm_lock:+ return scope in _warm_accounts or (+ _warm_active_account is not None and _scope_for_account(_warm_active_account) == scope+ )+++def resolve_local_gguf_for_switch(+ requested: str, *, include_companion_scope: bool = False+) -> Optional[tuple]:+ """Resolve an auto-switch target without repeatedly scanning for known misses.++ Reuse misses until invalidation or two duty windows of snapshot age. After one+ window, requests trigger a background scan; the second allows time to finish.+ New names and newly indexed hits use the normal resolver freshness checks.+ """+ if not isinstance(requested, str) or not requested.strip():+ return None+ requested = requested.strip()+ key = (_miss_scope(), requested)+ generation = _generation+ started = time.monotonic()+ ts, index = _snapshot()+ if (+ _misses.get(key) == generation+ and ts > 0.0+ and (started - ts < 2 * _duty_window() or _scope_is_warming(key[0]))+ and _resolve_from_index(requested, index) is None+ ):+ warm_index_soon()+ return None+ resolved = resolve_local_gguf(requested, include_companion_scope = include_companion_scope)+ if resolved is None:+ ts, index = _snapshot()+ # Require a scan fresh at call start or completed since, with no intervening invalidation.+ # Failed rebuilds leave an older snapshot that cannot prove a miss.+ if (+ len(requested) <= _MAX_MISS_NAME+ and ts > 0.0+ and started - ts < _CACHE_TTL_S+ and generation == _generation+ and _resolve_from_index(requested, index) is None+ ):+ if len(_misses) >= _MAX_MISSES:+ _misses.clear()+ _misses[key] = generation+ return resolved++def resolve_local_gguf(requested: str,*,diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.pyindex a052fe93a6f..c58a5e43f27 100644--- a/studio/backend/routes/inference.py+++ b/studio/backend/routes/inference.py@@ -9681,7 +9681,7 @@ async def _maybe_auto_switch_model(local_gguf_companion_roots,local_gguf_companion_state,local_target_is_gguf,- resolve_local_gguf,+ resolve_local_gguf_for_switch,resolve_trusted_cached_local_gguf,warm_index_soon,)@@ -9800,10 +9800,8 @@ async def _maybe_auto_switch_model(repo_level_companion_resolution = Falsestashed_gguf_companion_roots: tuple[str, ...] = ()if auto_switch_on and not reload_only:- # Fresh hits and entries retained across an additions-only download are- # safe to use immediately. An expired/config-invalidated hit, a cold- # cache, and every miss must refresh before an unrelated resident model- # can answer or an entry from a removed scan root can trigger a switch.+ # Use trusted hits immediately. The switch resolver refreshes stale hits+ # and unconfirmed misses, keeping removed scan roots out of switches.resolved = resolve_trusted_cached_local_gguf(requested_model,include_companion_scope = True,@@ -9812,7 +9810,7 @@ async def _maybe_auto_switch_model(warm_index_soon()else:resolved = await asyncio.to_thread(- resolve_local_gguf,+ resolve_local_gguf_for_switch,requested_model,include_companion_scope = True,)diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.pyindex 0480dc8d49d..866b6df82f6 100644--- a/studio/backend/tests/conftest.py+++ b/studio/backend/tests/conftest.py@@ -395,6 +395,7 @@ def _no_background_model_scan(monkeypatch):# assertion became a 503 "still indexing". Cold-path tests reset _scan themselves;# _build_index is untouched so tests calling it directly still walk for real.monkeypatch.setattr(local_model_resolver, "_scan", (time.monotonic(), {}))+ monkeypatch.setattr(local_model_resolver, "_misses", {})@pytest.fixture(scope = "session")diff --git a/studio/backend/tests/test_account_local_model_resolver.py b/studio/backend/tests/test_account_local_model_resolver.pyindex e5bc2d6038f..a30cc94a26c 100644--- a/studio/backend/tests/test_account_local_model_resolver.py+++ b/studio/backend/tests/test_account_local_model_resolver.py@@ -8,6 +8,8 @@ one's checkpoint path and the load then refuses it as foreign."""from __future__ import annotationsimport asyncio+import threading+import timefrom types import SimpleNamespaceimport pytest@@ -19,10 +21,11 @@ from core.inference import local_model_resolver as resolverfrom hub.services.models import account_access as accessfrom storage import studio_dbfrom utils import openai_auto_switch_settings as switch_settings-from utils.account_context import OWNER, AccountContext, arun_as, run_as+from utils.account_context import OWNER, AccountContext, arun_as, current_account_id, run_asALICE = AccountContext("a" * 32, "alice")BOB = AccountContext("b" * 32, "bob")+_REAL_WARM_INDEX_SOON = resolver.warm_index_soon@pytest.fixture@@ -114,3 +117,92 @@ def test_the_owner_keeps_one_index_on_a_single_user_install(home, monkeypatch):for _ in range(3):assert resolver.resolve_local_gguf("owner-model")[0] == str(owner_model)assert len(scans) == 1+++def test_a_fast_account_scan_does_not_expire_another_accounts_slow_scan_miss(home, monkeypatch):+ clock = SimpleNamespace(now = 1_000.0)+ monkeypatch.setattr(resolver.time, "monotonic", lambda: clock.now)+ monkeypatch.setattr(resolver, "_managed_scans", {})+ monkeypatch.setattr(resolver, "_managed_last_scan_s", {}, raising = False)+ monkeypatch.setattr(resolver, "_last_scan_s", 0.0)+ monkeypatch.setattr(resolver, "_misses", {})+ scans = []++ def _scan():+ account_id = current_account_id()+ scans.append(account_id)+ clock.now += 17.0 if account_id == ALICE.account_id else 0.1+ return {}++ monkeypatch.setattr(resolver, "_build_index", _scan)+ assert run_as(ALICE, resolver.resolve_local_gguf_for_switch, "missing") is None+ assert run_as(BOB, resolver.resolve_local_gguf_for_switch, "missing") is None++ # Bob's cheap scan must not replace the duty window earned by Alice's slow one.+ clock.now += 10.0+ assert run_as(ALICE, resolver.resolve_local_gguf_for_switch, "missing") is None+ assert scans == [ALICE.account_id, BOB.account_id]+++def test_a_warm_scan_queues_behind_another_accounts_scan(home, monkeypatch):+ now = time.monotonic()+ monkeypatch.setattr(+ resolver,+ "_managed_scans",+ {+ ALICE.account_id: (now - 2 * resolver._CACHE_TTL_S - 1, {}),+ BOB.account_id: (now - resolver._CACHE_TTL_S - 1, {}),+ },+ )+ monkeypatch.setattr(+ resolver,+ "_managed_last_scan_s",+ {ALICE.account_id: 0.0, BOB.account_id: 0.0},+ )+ monkeypatch.setattr(resolver, "_warming", False)+ monkeypatch.setattr(resolver, "_warm_pending", False)+ monkeypatch.setattr(resolver, "_warm_accounts", {}, raising = False)+ monkeypatch.setattr(resolver, "_warm_active_account", None, raising = False)+ monkeypatch.setattr(resolver, "_warm_retry_scopes", set(), raising = False)+ monkeypatch.setattr(+ resolver,+ "_misses",+ {(ALICE.account_id, "missing"): resolver._generation},+ )+ scans = []+ bob_started = threading.Event()+ release_bob = threading.Event()+ alice_finished = threading.Event()++ def _scan():+ bob_started.set()+ account_id = current_account_id()+ scans.append(account_id)+ if account_id == BOB.account_id:+ assert release_bob.wait(5)+ else:+ alice_finished.set()+ return {}++ monkeypatch.setattr(resolver, "_index", _scan)+ stale = run_as(+ BOB,+ lambda: (+ resolver._snapshot()[0],+ resolver.time.monotonic(),+ resolver._duty_window(),+ ),+ )+ assert stale[1] - stale[0] > stale[2], stale+ run_as(BOB, _REAL_WARM_INDEX_SOON)+ assert bob_started.wait(5), (resolver._warming, scans)+ run_as(ALICE, _REAL_WARM_INDEX_SOON)+ assert run_as(ALICE, resolver.resolve_local_gguf_for_switch, "missing") is None+ assert scans == [BOB.account_id], "the queued refresh must keep the miss nonblocking"+ release_bob.set()++ try:+ assert alice_finished.wait(1)+ assert scans == [BOB.account_id, ALICE.account_id]+ finally:+ release_bob.set()diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.pyindex 6e9eaa95c6b..f2a7d3923f9 100644--- a/studio/backend/tests/test_openai_auto_switch.py+++ b/studio/backend/tests/test_openai_auto_switch.py@@ -9239,6 +9239,125 @@ def test_an_invalidated_index_rebuilds_on_a_host_that_just_booted(monkeypatch):assert built == [1], kwargs+def _settable_resolver_clock(monkeypatch):+ """Advance resolver time by assigning ``clock.now``."""+ clock = types.SimpleNamespace(now = 1000.0)+ monkeypatch.setattr(resolver, "time", types.SimpleNamespace(monotonic = lambda: clock.now))+ return clock+++def _counted_scans(monkeypatch, index = None):+ scans = []+ monkeypatch.setattr(resolver, "_build_index", lambda: scans.append(1) or dict(index or {}))+ monkeypatch.setattr(resolver, "_last_scan_s", 0.0)+ return scans+++ "loaded, requested",+ [+ ("unsloth/A-GGUF", "claude-haiku-4-5"),+ ("/elsewhere/unscanned.gguf", "/elsewhere/unscanned.gguf"),+ ("/elsewhere/unscanned.gguf", "unscanned"),+ ],+)+def test_a_name_that_is_not_on_disk_rescans_once_not_per_request(monkeypatch, loaded, requested):+ clock = _settable_resolver_clock(monkeypatch)+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))+ scans = _counted_scans(monkeypatch)+ warmed = []++ def _warm_lands():+ # Simulate a background scan without counting it as blocking.+ warmed.append(1)+ resolver._scan = (clock.now, {})++ monkeypatch.setattr(resolver, "warm_index_soon", _warm_lands)+ real_resolve = resolver.resolve_local_gguf+ backend, rec = _wired(monkeypatch, _FakeBackend(loaded, "Q4_K_M"), None)+ monkeypatch.setattr(resolver, "resolve_local_gguf", real_resolve)++ for _ in range(4):+ clock.now += resolver._CACHE_TTL_S + 1+ _run_hook(requested)++ assert scans == [1]+ assert warmed+ assert rec.calls == []+ assert backend.model_identifier == loaded+++def test_a_remembered_miss_still_finds_a_model_that_appears_later(monkeypatch):+ clock = _settable_resolver_clock(monkeypatch)+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))+ scans = _counted_scans(monkeypatch)+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)+ assert resolver.resolve_local_gguf_for_switch("org/b") is None+ assert scans == [1]++ # Simulate background discovery without invalidation.+ added = {"org/b": _entry("org/b", "Q4_K_M")}+ clock.now += resolver._CACHE_TTL_S + 1+ monkeypatch.setattr(resolver, "_scan", (clock.now, added))+ assert resolver.resolve_local_gguf_for_switch("org/b") is not None++ # Stale hits must rescan before switching.+ clock.now += resolver._CACHE_TTL_S + 1+ assert resolver.resolve_local_gguf_for_switch("org/b") is None+ assert scans == [1, 1]+++def test_an_invalidation_forgets_a_remembered_miss(monkeypatch):+ # Download completion must make the new model discoverable immediately.+ clock = _settable_resolver_clock(monkeypatch)+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))+ scans = _counted_scans(monkeypatch)+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)+ assert resolver.resolve_local_gguf_for_switch("org/b") is None++ monkeypatch.setattr(+ resolver, "_build_index", lambda: scans.append(1) or {"org/b": _entry("org/b")}+ )+ resolver.invalidate_index(additions_only = True)+ assert resolver.resolve_local_gguf_for_switch("org/b") is not None+ assert scans == [1, 1]+++def test_a_remembered_miss_expires_and_a_failed_scan_proves_nothing(monkeypatch):+ clock = _settable_resolver_clock(monkeypatch)+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))+ scans = _counted_scans(monkeypatch)+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)+ assert resolver.resolve_local_gguf_for_switch("org/b") is None++ # Misses expire if background refresh never completes.+ clock.now += 2 * resolver._duty_window() + 1+ assert resolver.resolve_local_gguf_for_switch("org/b") is None+ assert scans == [1, 1]++ def _fail():+ scans.append(1)+ raise OSError("scan root vanished")++ monkeypatch.setattr(resolver, "_build_index", _fail)+ monkeypatch.setattr(resolver, "_misses", {})+ clock.now += resolver._CACHE_TTL_S + 1+ assert resolver.resolve_local_gguf_for_switch("org/b") is None+ clock.now += resolver._CACHE_TTL_S + 1+ assert resolver.resolve_local_gguf_for_switch("org/b") is None+ assert scans == [1, 1, 1, 1]+++def test_an_oversized_name_is_not_remembered(monkeypatch):+ clock = _settable_resolver_clock(monkeypatch)+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))+ _counted_scans(monkeypatch)+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)+ assert resolver.resolve_local_gguf_for_switch("x" * (resolver._MAX_MISS_NAME + 1)) is None+ assert resolver.resolve_local_gguf_for_switch("x" * resolver._MAX_MISS_NAME) is None+ assert [len(name) for _scope, name in resolver._misses] == [resolver._MAX_MISS_NAME]++# The resident short circuit is the one path that answers without consulting the# index, so it must never say yes where the pre-existing resident check says no.# Anything it accepts, main would have accepted too: a miss only costs the scan.