Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Edge cases for the /v1/models servability cache: concurrency, catalog swaps and deletes."""
from __future__ import annotations
import sys
import threading
import time
from pathlib import Path
from types import SimpleNamespace
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
import routes.inference as inf # noqa: E402
from core.inference import local_model_resolver as resolver
@pytest.fixture(autouse = True)
def _clean_cache():
def _reset():
inf._SERVABLE_SCAN_CACHE["entry"] = None
_reset()
yield
_reset()
def _catalog(n = 3, tag = "m"):
return [
SimpleNamespace(id = f"repo/{tag}{i}", path = f"/models/{tag}{i}", task = None) for i in range(n)
]
@pytest.fixture
def stub(monkeypatch):
counts = {"servable": 0, "resident": 0}
def _servable(info):
counts["servable"] += 1
return (True, ("Q4_K_M",))
monkeypatch.setattr("core.inference.local_model_resolver.local_servable_model", _servable)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
def _resident(key, **kw):
counts["resident"] += 1
return False
monkeypatch.setattr(inf, "_resolves_to_resident", _resident)
return counts
def test_concurrent_callers_do_not_corrupt_the_cache(stub):
catalog = _catalog(5)
results: list[int] = []
errors: list[BaseException] = []
barrier = threading.Barrier(16)
def _go():
try:
barrier.wait()
for _ in range(20):
rows = inf._servable_catalog_rows(catalog, 111.0)
results.append(len(rows))
except BaseException as exc: # noqa: BLE001
errors.append(exc)
threads = [threading.Thread(target = _go) for _ in range(16)]
for t in threads:
t.start()
for t in threads:
t.join(timeout = 60)
assert not errors, f"concurrent access raised: {errors}"
assert set(results) == {5}, "every caller must see the whole catalog"
def test_a_replaced_catalog_is_never_served_from_the_old_entry(stub):
first, second = _catalog(2, "a"), _catalog(3, "b")
rows = inf._servable_catalog_rows(first, 111.0)
assert [r[0].id for r in rows] == ["repo/a0", "repo/a1"]
rows = inf._servable_catalog_rows(second, 111.0)
assert [r[0].id for r in rows] == ["repo/b0", "repo/b1", "repo/b2"]
def test_the_zero_stamp_of_a_fresh_process_does_not_pin_the_cache(stub):
# _CATALOG_CACHE['at'] only advances in _cached_local_catalog; an external catalog keeps 0.0.
first, second = _catalog(1, "a"), _catalog(1, "b")
assert inf._servable_catalog_rows(first, 0.0)[0][0].id == "repo/a0"
assert inf._servable_catalog_rows(second, 0.0)[0][0].id == "repo/b0"
def test_residency_is_never_cached(stub):
catalog = _catalog(2)
for _ in range(10):
inf._servable_catalog_rows(catalog, 111.0)
assert stub["servable"] == 2, "the scan is cached"
assert stub["resident"] == 20, "residency is not"
def test_residency_flips_are_visible_immediately(monkeypatch):
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: (True, ()),
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
flag = {"resident": False}
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: flag["resident"])
catalog = _catalog(1)
assert inf._servable_catalog_rows(catalog, 111.0)[0][3] is False
flag["resident"] = True
assert inf._servable_catalog_rows(catalog, 111.0)[0][3] is True
def test_an_empty_catalog_is_cached_without_confusing_a_miss(stub):
assert inf._servable_catalog_rows([], 111.0) == []
assert inf._servable_catalog_rows([], 111.0) == []
assert stub["servable"] == 0
def test_no_stamp_never_populates_the_cache(stub):
catalog = _catalog(2)
inf._servable_catalog_rows(catalog)
assert inf._SERVABLE_SCAN_CACHE["entry"] is None
inf._servable_catalog_rows(catalog)
assert stub["servable"] == 4
def test_media_and_stt_tasks_stay_excluded(monkeypatch):
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: (True, ()),
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
catalog = [
SimpleNamespace(id = "repo/text", path = "/m/text", task = None),
SimpleNamespace(id = "repo/stt", path = "/m/stt", task = inf._STT_MODEL_TASK),
SimpleNamespace(id = "repo/tts", path = "/m/tts", task = inf._TTS_MODEL_TASK),
]
rows = inf._servable_catalog_rows(catalog, 111.0)
assert [r[0].id for r in rows] == ["repo/text"]
def test_a_raising_resolver_is_not_cached_as_an_empty_catalog(monkeypatch):
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: (_ for _ in ()).throw(RuntimeError("scan failed")),
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
with pytest.raises(RuntimeError):
inf._servable_catalog_rows(_catalog(1), 111.0)
assert inf._SERVABLE_SCAN_CACHE["entry"] is None, "a failed scan must not be cached"
def test_the_cache_entry_is_published_atomically(stub):
# The fast path reads without the lock, so separate fields could be seen half-replaced.
first, second = _catalog(2, "a"), _catalog(3, "b")
inf._servable_catalog_rows(first, 111.0)
entry = inf._SERVABLE_SCAN_CACHE["entry"]
assert entry is not None and len(entry) == 5, "the tuple must be one object"
at, cached, generation, _scanned_at, rows = entry
assert at == 111.0 and cached is first and len(rows) == 2
assert isinstance(generation, tuple) and len(generation) == 3
inf._servable_catalog_rows(second, 222.0)
at, cached, _generation, _at2, rows = inf._SERVABLE_SCAN_CACHE["entry"]
assert (at, cached is second, len(rows)) == (
222.0,
True,
3,
), "stamp, catalog, generation and rows move together"
def test_residency_resolves_the_current_snapshot_each_call(monkeypatch):
# A download can move the snapshot pointer within the catalog TTL; do not cache the dir.
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: (False, ()),
)
snapshot = {"dir": "/models/m0/snapshots/old"}
monkeypatch.setattr(
"core.inference.local_model_resolver.local_load_dir", lambda p: snapshot["dir"]
)
seen: list[str] = []
def _resident(key, **kw):
seen.append(key)
return key == "/models/m0/snapshots/new"
monkeypatch.setattr(inf, "_resolves_to_resident", _resident)
catalog = _catalog(1)
assert inf._servable_catalog_rows(catalog, 111.0)[0][3] is False
snapshot["dir"] = "/models/m0/snapshots/new"
assert (
inf._servable_catalog_rows(catalog, 111.0)[0][3] is True
), "the snapshot must be re-resolved per call, not cached with the scan"
assert seen == ["/models/m0/snapshots/old", "/models/m0/snapshots/new"]
def test_large_catalog_stays_correct(stub):
catalog = _catalog(500)
rows = inf._servable_catalog_rows(catalog, 111.0)
assert len(rows) == 500
assert stub["servable"] == 500
rows = inf._servable_catalog_rows(catalog, 111.0)
assert len(rows) == 500
assert stub["servable"] == 500, "second call must be free"
def test_a_deleted_model_leaves_the_listing_within_the_catalog_ttl(monkeypatch):
"""A delete bumps the resolver generation, not the catalog cache, so the listing must key on it."""
catalog = _catalog(2, "d")
gone: set[str] = set()
def _servable(info):
return None if info.path in gone else (True, ("Q4_K_M",))
monkeypatch.setattr("core.inference.local_model_resolver.local_servable_model", _servable)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
assert [r[0].id for r in inf._servable_catalog_rows(catalog, 111.0)] == ["repo/d0", "repo/d1"]
gone.add("/models/d1")
resolver.invalidate_index()
assert [r[0].id for r in inf._servable_catalog_rows(catalog, 111.0)] == ["repo/d0"]
def test_an_additions_only_invalidation_also_refreshes_the_scan(monkeypatch):
"""A finished download invalidates additions-only, and a new quant must appear
without waiting out the catalog TTL."""
catalog = _catalog(1, "q")
quants = {"/models/q0": ("Q4_K_M",)}
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: (True, quants[info.path]),
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
assert inf._servable_catalog_rows(catalog, 222.0)[0][2] == ("Q4_K_M",)
quants["/models/q0"] = ("Q8_0", "Q4_K_M")
resolver.invalidate_index(additions_only = True)
assert inf._servable_catalog_rows(catalog, 222.0)[0][2] == ("Q8_0", "Q4_K_M")
def test_an_invalidation_during_a_scan_is_not_stamped_in(monkeypatch):
"""The generation is read before the scan, so an invalidation that lands while the
scan runs makes the stored entry stale rather than being cached as already seen."""
catalog = _catalog(1, "r")
state = {"racing": True}
def _servable(info):
if state["racing"]:
state["racing"] = False
resolver.invalidate_index()
return (True, ("Q4_K_M",))
monkeypatch.setattr("core.inference.local_model_resolver.local_servable_model", _servable)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
inf._servable_catalog_rows(catalog, 333.0)
entry = inf._SERVABLE_SCAN_CACHE["entry"]
assert entry is not None
assert entry[2] != resolver.index_generation(), "the racing scan must not read as fresh"
def test_the_generation_only_moves_on_invalidation(monkeypatch):
"""The generation changes only on invalidation, else every request rescans the filesystem."""
before = resolver.index_generation()
catalog = _catalog(3, "s")
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: (True, ("Q4_K_M",)),
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
calls = {"n": 0}
def _resident(key, **kw):
calls["n"] += 1
return False
monkeypatch.setattr(inf, "_resolves_to_resident", _resident)
for _ in range(5):
inf._servable_catalog_rows(catalog, 444.0)
assert resolver.index_generation() == before
assert calls["n"] == 15
def test_a_hub_cache_deletion_leaves_the_listing(monkeypatch):
"""deletion.py invalidates the HF cache scans and nothing else, so the resolver
generation alone would keep advertising a deleted cached repo for the catalog TTL."""
from hub.utils import inventory_scan
catalog = _catalog(2, "h")
gone: set[str] = set()
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: None if info.path in gone else (True, ("Q4_K_M",)),
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
assert len(inf._servable_catalog_rows(catalog, 555.0)) == 2
gone.add("/models/h1")
inventory_scan.invalidate_hf_cache_scans()
assert [r[0].id for r in inf._servable_catalog_rows(catalog, 555.0)] == ["repo/h0"]
def test_a_hardware_redetect_reveals_newly_servable_checkpoints(monkeypatch):
"""hardware.DEVICE drives non-GGUF servability and an Apple MLX self-repair flips CPU to MLX
after startup, so an early /v1/models must not pin 'unservable' for the catalog TTL."""
from utils.hardware import hardware as hw
catalog = _catalog(1, "k")
servable = {"ok": False}
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: (False, ()) if servable["ok"] else None,
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
assert inf._servable_catalog_rows(catalog, 666.0) == []
servable["ok"] = True
monkeypatch.setattr(hw, "DETECTION_GENERATION", hw.DETECTION_GENERATION + 1)
assert len(inf._servable_catalog_rows(catalog, 666.0)) == 1
def test_the_generation_key_names_all_three_signals():
"""A missing counter would pin the key and undo invalidation, so all three are asserted."""
from core.inference.local_model_resolver import index_generation
from hub.utils.inventory_scan import hf_cache_scans_epoch
from utils.hardware import hardware as hw
assert inf._servability_generation() == (
index_generation(),
int(hf_cache_scans_epoch()),
int(hw.DETECTION_GENERATION),
)
def test_every_delete_branch_invalidates_the_scan():
"""A model deleted via delete_finetuned_model must invalidate the scan on every successful branch."""
import inspect
from routes import models as models_route
lines = inspect.getsource(models_route.delete_finetuned_model).split("\n")
# The variant branch returns a multi-line dict: match the status, not a one-line prefix.
successes = [
n
for n, line in enumerate(lines)
if '"status": "deleted"' in line or '"deleted"' in line and "status" in line
]
invalidations = [n for n, line in enumerate(lines) if "_invalidate_local_scans()" in line]
assert len(successes) == 2, f"expected two success returns, found {len(successes)}"
assert (
len(invalidations) == 2
), f"expected one invalidation per success branch, found {len(invalidations)}"
for at in successes:
before = [n for n in invalidations if n < at]
assert before, "a successful deletion branch reports success without invalidating"
assert not [
n for n in successes if before[-1] < n < at
], "the invalidation must belong to this branch, not to one above it"
def test_a_generation_bump_while_waiting_for_the_lock_is_not_accepted(monkeypatch):
"""A generation bump during the lock wait must reject the scan, which started before the delete."""
catalog = _catalog(1, "w")
scanned = {"n": 0}
def _servable(info):
scanned["n"] += 1
return (True, ("Q4_K_M",))
monkeypatch.setattr("core.inference.local_model_resolver.local_servable_model", _servable)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
generation_at_queue = inf._servability_generation()
stale_rows = [("scanned-before-the-delete",)]
class _LockThatLosesTheRace:
"""Stands in for the wait: the other scanner publishes and a delete lands here."""
def __enter__(self):
inf._SERVABLE_SCAN_CACHE["entry"] = (
777.0,
catalog,
generation_at_queue,
time.monotonic(),
stale_rows,
)
resolver.invalidate_index()
return self
def __exit__(self, *exc):
return False
monkeypatch.setattr(inf, "_SERVABLE_SCAN_CACHE_LOCK", _LockThatLosesTheRace())
rows = inf._servable_catalog_rows(catalog, 777.0)
assert rows != stale_rows, "the waiter accepted rows scanned before the delete"
assert scanned["n"] > 0, "it must have rescanned rather than reused the entry"
def test_the_invalidation_helper_stays_off_the_event_loop():
"""invalidate_index takes the resolver lock that _index() holds across a full filesystem scan,
so an async route calling it inline would stall requests and inference streams."""
import asyncio as _asyncio
import inspect
from routes import models as models_route
assert _asyncio.iscoroutinefunction(models_route._invalidate_local_scans)
source = inspect.getsource(models_route._invalidate_local_scans)
assert (
"asyncio.to_thread(invalidate_index)" in source
), "the invalidation must be offloaded, matching the other async sites in this file"
route = inspect.getsource(models_route.delete_finetuned_model)
calls = route.count("_invalidate_local_scans()")
awaited = route.count("await _invalidate_local_scans()")
assert calls == awaited == 2, f"{calls} call(s), {awaited} awaited"
def test_a_hit_is_rejected_when_the_generation_moves_mid_read(monkeypatch):
"""Seqlock: generation, entry, generation. A delete completing between the first two
reads was accepted anyway, and the lock-free path is where most requests land."""
catalog = _catalog(1, "s")
scanned = {"n": 0}
def _servable(info):
scanned["n"] += 1
return (True, ("Q4_K_M",))
monkeypatch.setattr("core.inference.local_model_resolver.local_servable_model", _servable)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
real = inf._servability_generation
entry_generation = real()
inf._SERVABLE_SCAN_CACHE["entry"] = (
888.0,
catalog,
entry_generation,
time.monotonic(),
[("stale",)],
)
calls = {"n": 0}
def _moves_after_the_first_read():
calls["n"] += 1
if calls["n"] == 1:
return entry_generation
resolver.invalidate_index()
return real()
monkeypatch.setattr(inf, "_servability_generation", _moves_after_the_first_read)
rows = inf._servable_catalog_rows(catalog, 888.0)
assert rows != [("stale",)], "a delete completing mid-read must not be served"
assert scanned["n"] > 0, "the request must fall through to a real scan"
def test_an_out_of_band_file_change_is_picked_up_within_the_scan_ttl(monkeypatch):
"""No counter moves when a file is removed from a scan folder by hand, and the
per-request scan used to notice immediately. The entry's own TTL bounds that."""
catalog = _catalog(2, "o")
gone: set[str] = set()
monkeypatch.setattr(
"core.inference.local_model_resolver.local_servable_model",
lambda info: None if info.path in gone else (True, ("Q4_K_M",)),
)
monkeypatch.setattr("core.inference.local_model_resolver.local_load_dir", lambda p: p)
monkeypatch.setattr(inf, "_resolves_to_resident", lambda key, **kw: False)
clock = {"t": 1000.0}
monkeypatch.setattr(inf.time, "monotonic", lambda: clock["t"])
assert len(inf._servable_catalog_rows(catalog, 999.0)) == 2
gone.add("/models/o1")
assert len(inf._servable_catalog_rows(catalog, 999.0)) == 2, "still inside the TTL"
clock["t"] += inf._SERVABLE_SCAN_TTL_S + 0.1
assert [r[0].id for r in inf._servable_catalog_rows(catalog, 999.0)] == ["repo/o0"]