Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop rescanning every model folder on each request that names a model not on disk (#11872)

oobabooga9ff97b6273b0
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py
index e186a6f305e..151e60aa2cd 100644
--- a/studio/backend/core/inference/local_model_resolver.py
+++ b/studio/backend/core/inference/local_model_resolver.py
@@ -22,7 +22,14 @@ from typing import Optional
from core.inference.model_ids import public_model_id
from loggers import get_logger
-from utils.account_context import account_thread, current_account_id, is_owner_context
+from utils.account_context import (
+ AccountContext,
+ account_thread,
+ current_account,
+ current_account_id,
+ is_owner_context,
+ run_as,
+)
logger = get_logger(__name__)
@@ -74,11 +81,42 @@ _warming = False
# is rebuilt off the request path. Callers still pair invalidate_index() with warm_index_soon() for the case where it
# has retired.
_warm_pending = False
+_warm_accounts: dict[Optional[str], AccountContext] = {}
+_warm_active_account: Optional[AccountContext] = None
+_warm_retry_scopes: set[Optional[str]] = set()
_last_scan_s = 0.0
+_managed_last_scan_s: dict[str, float] = {}
# rescan at most a tenth of the time: on the TTL alone a slow scan would run continuously
_WARM_DUTY = 10.0
+def _scope_for_account(account: AccountContext) -> Optional[str]:
+ return None if account.is_owner else account.account_id
+
+
+def _account_scope() -> Optional[str]:
+ return _scope_for_account(current_account())
+
+
+def _account_duty_window(account_id: Optional[str]) -> float:
+ scan_s = _last_scan_s if account_id is None else _managed_last_scan_s.get(account_id, 0.0)
+ return max(_CACHE_TTL_S, scan_s * _WARM_DUTY)
+
+
+def _duty_window() -> float:
+ """Minimum snapshot age for a background rescan."""
+ return _account_duty_window(_account_scope())
+
+
+def _record_scan_duration(duration: float) -> None:
+ global _last_scan_s
+ account_id = _account_scope()
+ if account_id is None:
+ _last_scan_s = duration
+ else:
+ _managed_last_scan_s[account_id] = duration
+
+
def _is_abs_path_id(value: str) -> bool:
"""True when an id is an absolute filesystem path (the ./models and LM Studio scanners use the
on-disk path as the id) rather than a repo id like org/name.
@@ -958,7 +996,12 @@ def recently_downloaded(repo_id: str) -> bool:
return repo_id.strip().lower() in _just_downloaded
-def _snapshot_is_trusted(timestamp: float, now: float) -> bool:
+def _snapshot_is_trusted(
+ timestamp: float,
+ now: float,
+ *,
+ duty_window: Optional[float] = None,
+) -> bool:
"""Whether a snapshot stamped *timestamp* may answer a model switch at *now*.
Positive is an ordinary scan, trusted for the TTL. Negative is when an
@@ -969,7 +1012,7 @@ def _snapshot_is_trusted(timestamp: float, now: float) -> bool:
if timestamp > 0.0:
return now - timestamp < _CACHE_TTL_S
if timestamp < 0.0:
- return now + timestamp < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY)
+ return now + timestamp < (duty_window if duty_window is not None else _duty_window())
return False
@@ -978,6 +1021,13 @@ def _snapshot_is_trusted(timestamp: float, now: float) -> bool:
_generation = 0
+# Confirmed misses keyed by account and name, valid for the recorded invalidation generation.
+_misses: dict[tuple[Optional[str], str], int] = {}
+_MAX_MISSES = 256
+# Request bodies may be hundreds of MB; longer names are not retained.
+_MAX_MISS_NAME = 4096
+
+
def index_generation() -> int:
"""How many times the local scan has been invalidated since this process started."""
return _generation
@@ -996,21 +1046,31 @@ def invalidate_index(*, additions_only: bool = False) -> None:
now = time.monotonic()
_generation += 1
- def _invalidated(snapshot):
+ def _invalidated(snapshot, account_id):
timestamp, retained = snapshot
# Entries and trust state publish together: a lock-free reader never sees a fresh
# timestamp paired with revoked trust.
- stamp = -now if additions_only and _snapshot_is_trusted(timestamp, now) else 0.0
+ stamp = (
+ -now
+ if additions_only
+ and _snapshot_is_trusted(
+ timestamp,
+ now,
+ duty_window = _account_duty_window(account_id),
+ )
+ else 0.0
+ )
return (stamp, retained)
# Every account's snapshot: what changed on disk is not scoped to whoever noticed.
- _scan = _invalidated(_scan)
+ _scan = _invalidated(_scan, None)
for account_id, snapshot in list(_managed_scans.items()):
- _managed_scans[account_id] = _invalidated(snapshot)
+ _managed_scans[account_id] = _invalidated(snapshot, account_id)
# This may have waited out a scan on _lock, so the warmer that just published can still own the slot with a snapshot
# that is stale again. See _warm_pending.
with _warm_lock:
- if _warming:
+ if _warm_active_account is not None:
+ _warm_retry_scopes.add(_scope_for_account(_warm_active_account))
_warm_pending = True
@@ -1024,7 +1084,11 @@ def _index() -> dict[str, _LocalGgufEntry]:
# would serve what was just revoked
if ts > 0.0 and now - ts < _CACHE_TTL_S:
return cached
- fresh = _build_index()
+ # Request scans also set the background rescan interval.
+ try:
+ fresh = _build_index()
+ finally:
+ _record_scan_duration(time.monotonic() - now)
# Stamp AFTER the scan, not with the pre-scan ``now``: a multi-root scan on an install with many local models
# can itself exceed the TTL, which would store the cache already expired and make every request rebuild the
# index.
@@ -1080,42 +1144,110 @@ def warm_index_soon() -> None:
"""
global _warming, _warm_pending
stamp = _snapshot()[0]
- if stamp > 0.0 and time.monotonic() - stamp < max(_CACHE_TTL_S, _last_scan_s * _WARM_DUTY):
+ if stamp > 0.0 and time.monotonic() - stamp < _duty_window():
return
+ account = current_account()
+ scope = _scope_for_account(account)
with _warm_lock:
+ if (
+ _warm_active_account is not None and scope == _scope_for_account(_warm_active_account)
+ ) or scope in _warm_accounts:
+ return
+ _warm_accounts[scope] = account
if _warming:
return
_warming = True
_warm_pending = False
def _run() -> None:
- global _warming, _warm_pending, _last_scan_s
+ global _warm_active_account, _warming, _warm_pending
released = False
try:
while True:
- started = time.monotonic()
+ with _warm_lock:
+ if not _warm_accounts:
+ _warming, _warm_pending, released = False, False, True
+ return
+ scope, account = next(iter(_warm_accounts.items()))
+ del _warm_accounts[scope]
+ _warm_active_account = account
try:
- _index()
+ run_as(account, _index)
except Exception:
pass
- _last_scan_s = time.monotonic() - started
with _warm_lock:
- if _warm_pending:
- _warm_pending = False
- continue
- _warming, released = False, True
- return
+ _warm_active_account = None
+ if scope in _warm_retry_scopes:
+ _warm_retry_scopes.discard(scope)
+ _warm_accounts[scope] = account
+ _warm_pending = bool(_warm_retry_scopes)
finally:
- # Only on a BaseException: leaving the slot held would kill background warming for the life of the process
- # and put scans back on requests.
+ # Only on a BaseException: leaving scopes queued without a worker would kill background warming for the
+ # life of the process and put scans back on requests.
if not released:
with _warm_lock:
+ _warm_accounts.clear()
+ _warm_retry_scopes.clear()
+ _warm_active_account = None
_warming = _warm_pending = False
- # Pinned to the caller's account, or the rebuild would publish under the owner's scope.
+ # Capture a real account context for the thread; queued scans rebind to their own account above.
account_thread(target = _run, name = "local-model-index-warm", daemon = True).start()
+def _miss_scope() -> Optional[str]:
+ return _account_scope()
+
+
+def _scope_is_warming(scope: Optional[str]) -> bool:
+ with _warm_lock:
+ return scope in _warm_accounts or (
+ _warm_active_account is not None and _scope_for_account(_warm_active_account) == scope
+ )
+
+
+def resolve_local_gguf_for_switch(
+ requested: str, *, include_companion_scope: bool = False
+) -> Optional[tuple]:
+ """Resolve an auto-switch target without repeatedly scanning for known misses.
+
+ Reuse misses until invalidation or two duty windows of snapshot age. After one
+ window, requests trigger a background scan; the second allows time to finish.
+ New names and newly indexed hits use the normal resolver freshness checks.
+ """
+ if not isinstance(requested, str) or not requested.strip():
+ return None
+ requested = requested.strip()
+ key = (_miss_scope(), requested)
+ generation = _generation
+ started = time.monotonic()
+ ts, index = _snapshot()
+ if (
+ _misses.get(key) == generation
+ and ts > 0.0
+ and (started - ts < 2 * _duty_window() or _scope_is_warming(key[0]))
+ and _resolve_from_index(requested, index) is None
+ ):
+ warm_index_soon()
+ return None
+ resolved = resolve_local_gguf(requested, include_companion_scope = include_companion_scope)
+ if resolved is None:
+ ts, index = _snapshot()
+ # Require a scan fresh at call start or completed since, with no intervening invalidation.
+ # Failed rebuilds leave an older snapshot that cannot prove a miss.
+ if (
+ len(requested) <= _MAX_MISS_NAME
+ and ts > 0.0
+ and started - ts < _CACHE_TTL_S
+ and generation == _generation
+ and _resolve_from_index(requested, index) is None
+ ):
+ if len(_misses) >= _MAX_MISSES:
+ _misses.clear()
+ _misses[key] = generation
+ return resolved
+
+
def resolve_local_gguf(
requested: str,
*,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index a052fe93a6f..c58a5e43f27 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -9681,7 +9681,7 @@ async def _maybe_auto_switch_model(
local_gguf_companion_roots,
local_gguf_companion_state,
local_target_is_gguf,
- resolve_local_gguf,
+ resolve_local_gguf_for_switch,
resolve_trusted_cached_local_gguf,
warm_index_soon,
)
@@ -9800,10 +9800,8 @@ async def _maybe_auto_switch_model(
repo_level_companion_resolution = False
stashed_gguf_companion_roots: tuple[str, ...] = ()
if auto_switch_on and not reload_only:
- # Fresh hits and entries retained across an additions-only download are
- # safe to use immediately. An expired/config-invalidated hit, a cold
- # cache, and every miss must refresh before an unrelated resident model
- # can answer or an entry from a removed scan root can trigger a switch.
+ # Use trusted hits immediately. The switch resolver refreshes stale hits
+ # and unconfirmed misses, keeping removed scan roots out of switches.
resolved = resolve_trusted_cached_local_gguf(
requested_model,
include_companion_scope = True,
@@ -9812,7 +9810,7 @@ async def _maybe_auto_switch_model(
warm_index_soon()
else:
resolved = await asyncio.to_thread(
- resolve_local_gguf,
+ resolve_local_gguf_for_switch,
requested_model,
include_companion_scope = True,
)
diff --git a/studio/backend/tests/conftest.py b/studio/backend/tests/conftest.py
index 0480dc8d49d..866b6df82f6 100644
--- a/studio/backend/tests/conftest.py
+++ b/studio/backend/tests/conftest.py
@@ -395,6 +395,7 @@ def _no_background_model_scan(monkeypatch):
# assertion became a 503 "still indexing". Cold-path tests reset _scan themselves;
# _build_index is untouched so tests calling it directly still walk for real.
monkeypatch.setattr(local_model_resolver, "_scan", (time.monotonic(), {}))
+ monkeypatch.setattr(local_model_resolver, "_misses", {})
@pytest.fixture(scope = "session")
diff --git a/studio/backend/tests/test_account_local_model_resolver.py b/studio/backend/tests/test_account_local_model_resolver.py
index e5bc2d6038f..a30cc94a26c 100644
--- a/studio/backend/tests/test_account_local_model_resolver.py
+++ b/studio/backend/tests/test_account_local_model_resolver.py
@@ -8,6 +8,8 @@ one's checkpoint path and the load then refuses it as foreign."""
from __future__ import annotations
import asyncio
+import threading
+import time
from types import SimpleNamespace
import pytest
@@ -19,10 +21,11 @@ from core.inference import local_model_resolver as resolver
from hub.services.models import account_access as access
from storage import studio_db
from utils import openai_auto_switch_settings as switch_settings
-from utils.account_context import OWNER, AccountContext, arun_as, run_as
+from utils.account_context import OWNER, AccountContext, arun_as, current_account_id, run_as
ALICE = AccountContext("a" * 32, "alice")
BOB = AccountContext("b" * 32, "bob")
+_REAL_WARM_INDEX_SOON = resolver.warm_index_soon
@pytest.fixture
@@ -114,3 +117,92 @@ def test_the_owner_keeps_one_index_on_a_single_user_install(home, monkeypatch):
for _ in range(3):
assert resolver.resolve_local_gguf("owner-model")[0] == str(owner_model)
assert len(scans) == 1
+
+
+def test_a_fast_account_scan_does_not_expire_another_accounts_slow_scan_miss(home, monkeypatch):
+ clock = SimpleNamespace(now = 1_000.0)
+ monkeypatch.setattr(resolver.time, "monotonic", lambda: clock.now)
+ monkeypatch.setattr(resolver, "_managed_scans", {})
+ monkeypatch.setattr(resolver, "_managed_last_scan_s", {}, raising = False)
+ monkeypatch.setattr(resolver, "_last_scan_s", 0.0)
+ monkeypatch.setattr(resolver, "_misses", {})
+ scans = []
+
+ def _scan():
+ account_id = current_account_id()
+ scans.append(account_id)
+ clock.now += 17.0 if account_id == ALICE.account_id else 0.1
+ return {}
+
+ monkeypatch.setattr(resolver, "_build_index", _scan)
+ assert run_as(ALICE, resolver.resolve_local_gguf_for_switch, "missing") is None
+ assert run_as(BOB, resolver.resolve_local_gguf_for_switch, "missing") is None
+
+ # Bob's cheap scan must not replace the duty window earned by Alice's slow one.
+ clock.now += 10.0
+ assert run_as(ALICE, resolver.resolve_local_gguf_for_switch, "missing") is None
+ assert scans == [ALICE.account_id, BOB.account_id]
+
+
+def test_a_warm_scan_queues_behind_another_accounts_scan(home, monkeypatch):
+ now = time.monotonic()
+ monkeypatch.setattr(
+ resolver,
+ "_managed_scans",
+ {
+ ALICE.account_id: (now - 2 * resolver._CACHE_TTL_S - 1, {}),
+ BOB.account_id: (now - resolver._CACHE_TTL_S - 1, {}),
+ },
+ )
+ monkeypatch.setattr(
+ resolver,
+ "_managed_last_scan_s",
+ {ALICE.account_id: 0.0, BOB.account_id: 0.0},
+ )
+ monkeypatch.setattr(resolver, "_warming", False)
+ monkeypatch.setattr(resolver, "_warm_pending", False)
+ monkeypatch.setattr(resolver, "_warm_accounts", {}, raising = False)
+ monkeypatch.setattr(resolver, "_warm_active_account", None, raising = False)
+ monkeypatch.setattr(resolver, "_warm_retry_scopes", set(), raising = False)
+ monkeypatch.setattr(
+ resolver,
+ "_misses",
+ {(ALICE.account_id, "missing"): resolver._generation},
+ )
+ scans = []
+ bob_started = threading.Event()
+ release_bob = threading.Event()
+ alice_finished = threading.Event()
+
+ def _scan():
+ bob_started.set()
+ account_id = current_account_id()
+ scans.append(account_id)
+ if account_id == BOB.account_id:
+ assert release_bob.wait(5)
+ else:
+ alice_finished.set()
+ return {}
+
+ monkeypatch.setattr(resolver, "_index", _scan)
+ stale = run_as(
+ BOB,
+ lambda: (
+ resolver._snapshot()[0],
+ resolver.time.monotonic(),
+ resolver._duty_window(),
+ ),
+ )
+ assert stale[1] - stale[0] > stale[2], stale
+ run_as(BOB, _REAL_WARM_INDEX_SOON)
+ assert bob_started.wait(5), (resolver._warming, scans)
+ run_as(ALICE, _REAL_WARM_INDEX_SOON)
+ assert run_as(ALICE, resolver.resolve_local_gguf_for_switch, "missing") is None
+ assert scans == [BOB.account_id], "the queued refresh must keep the miss nonblocking"
+ release_bob.set()
+
+ try:
+ assert alice_finished.wait(1)
+ assert scans == [BOB.account_id, ALICE.account_id]
+ finally:
+ release_bob.set()
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 6e9eaa95c6b..f2a7d3923f9 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -9239,6 +9239,125 @@ def test_an_invalidated_index_rebuilds_on_a_host_that_just_booted(monkeypatch):
assert built == [1], kwargs
+def _settable_resolver_clock(monkeypatch):
+ """Advance resolver time by assigning ``clock.now``."""
+ clock = types.SimpleNamespace(now = 1000.0)
+ monkeypatch.setattr(resolver, "time", types.SimpleNamespace(monotonic = lambda: clock.now))
+ return clock
+
+
+def _counted_scans(monkeypatch, index = None):
+ scans = []
+ monkeypatch.setattr(resolver, "_build_index", lambda: scans.append(1) or dict(index or {}))
+ monkeypatch.setattr(resolver, "_last_scan_s", 0.0)
+ return scans
+
+
+ "loaded, requested",
+ [
+ ("unsloth/A-GGUF", "claude-haiku-4-5"),
+ ("/elsewhere/unscanned.gguf", "/elsewhere/unscanned.gguf"),
+ ("/elsewhere/unscanned.gguf", "unscanned"),
+ ],
+)
+def test_a_name_that_is_not_on_disk_rescans_once_not_per_request(monkeypatch, loaded, requested):
+ clock = _settable_resolver_clock(monkeypatch)
+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))
+ scans = _counted_scans(monkeypatch)
+ warmed = []
+
+ def _warm_lands():
+ # Simulate a background scan without counting it as blocking.
+ warmed.append(1)
+ resolver._scan = (clock.now, {})
+
+ monkeypatch.setattr(resolver, "warm_index_soon", _warm_lands)
+ real_resolve = resolver.resolve_local_gguf
+ backend, rec = _wired(monkeypatch, _FakeBackend(loaded, "Q4_K_M"), None)
+ monkeypatch.setattr(resolver, "resolve_local_gguf", real_resolve)
+
+ for _ in range(4):
+ clock.now += resolver._CACHE_TTL_S + 1
+ _run_hook(requested)
+
+ assert scans == [1]
+ assert warmed
+ assert rec.calls == []
+ assert backend.model_identifier == loaded
+
+
+def test_a_remembered_miss_still_finds_a_model_that_appears_later(monkeypatch):
+ clock = _settable_resolver_clock(monkeypatch)
+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))
+ scans = _counted_scans(monkeypatch)
+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)
+ assert resolver.resolve_local_gguf_for_switch("org/b") is None
+ assert scans == [1]
+
+ # Simulate background discovery without invalidation.
+ added = {"org/b": _entry("org/b", "Q4_K_M")}
+ clock.now += resolver._CACHE_TTL_S + 1
+ monkeypatch.setattr(resolver, "_scan", (clock.now, added))
+ assert resolver.resolve_local_gguf_for_switch("org/b") is not None
+
+ # Stale hits must rescan before switching.
+ clock.now += resolver._CACHE_TTL_S + 1
+ assert resolver.resolve_local_gguf_for_switch("org/b") is None
+ assert scans == [1, 1]
+
+
+def test_an_invalidation_forgets_a_remembered_miss(monkeypatch):
+ # Download completion must make the new model discoverable immediately.
+ clock = _settable_resolver_clock(monkeypatch)
+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))
+ scans = _counted_scans(monkeypatch)
+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)
+ assert resolver.resolve_local_gguf_for_switch("org/b") is None
+
+ monkeypatch.setattr(
+ resolver, "_build_index", lambda: scans.append(1) or {"org/b": _entry("org/b")}
+ )
+ resolver.invalidate_index(additions_only = True)
+ assert resolver.resolve_local_gguf_for_switch("org/b") is not None
+ assert scans == [1, 1]
+
+
+def test_a_remembered_miss_expires_and_a_failed_scan_proves_nothing(monkeypatch):
+ clock = _settable_resolver_clock(monkeypatch)
+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))
+ scans = _counted_scans(monkeypatch)
+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)
+ assert resolver.resolve_local_gguf_for_switch("org/b") is None
+
+ # Misses expire if background refresh never completes.
+ clock.now += 2 * resolver._duty_window() + 1
+ assert resolver.resolve_local_gguf_for_switch("org/b") is None
+ assert scans == [1, 1]
+
+ def _fail():
+ scans.append(1)
+ raise OSError("scan root vanished")
+
+ monkeypatch.setattr(resolver, "_build_index", _fail)
+ monkeypatch.setattr(resolver, "_misses", {})
+ clock.now += resolver._CACHE_TTL_S + 1
+ assert resolver.resolve_local_gguf_for_switch("org/b") is None
+ clock.now += resolver._CACHE_TTL_S + 1
+ assert resolver.resolve_local_gguf_for_switch("org/b") is None
+ assert scans == [1, 1, 1, 1]
+
+
+def test_an_oversized_name_is_not_remembered(monkeypatch):
+ clock = _settable_resolver_clock(monkeypatch)
+ monkeypatch.setattr(resolver, "_scan", (clock.now - 60.0, {}))
+ _counted_scans(monkeypatch)
+ monkeypatch.setattr(resolver, "warm_index_soon", lambda: None)
+ assert resolver.resolve_local_gguf_for_switch("x" * (resolver._MAX_MISS_NAME + 1)) is None
+ assert resolver.resolve_local_gguf_for_switch("x" * resolver._MAX_MISS_NAME) is None
+ assert [len(name) for _scope, name in resolver._misses] == [resolver._MAX_MISS_NAME]
+
+
# The resident short circuit is the one path that answers without consulting the
# index, so it must never say yes where the pre-existing resident check says no.
# Anything it accepts, main would have accepted too: a miss only costs the scan.