Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix GGUF catalog snapshot selection (#10210)

Lee Jackson132f1155a62f
diff --git a/studio/backend/core/inference/llama_keepwarm.py b/studio/backend/core/inference/llama_keepwarm.py
index 595c5bc9c85..e3bd02f715c 100644
--- a/studio/backend/core/inference/llama_keepwarm.py
+++ b/studio/backend/core/inference/llama_keepwarm.py
@@ -806,7 +806,9 @@ def _loaded_identity(backend):
# Third slot is the advertised id (repo id) an auto-switch load sets on the backend; it's the override key, so an
# idle stash keyed by the concrete load path doesn't drop the user's saved launch flags on the alias reload.
advertised = getattr(backend, "_openai_advertised_id", None) or backend.model_identifier
- return (backend.model_identifier, getattr(backend, "hf_variant", None), advertised)
+ identity = (backend.model_identifier, getattr(backend, "hf_variant", None), advertised)
+ companion_roots = tuple(getattr(backend, "_openai_gguf_companion_roots", ()) or ())
+ return (*identity, companion_roots) if companion_roots else identity
def _note_idle_unload_event(freed) -> None:
diff --git a/studio/backend/core/inference/local_model_resolver.py b/studio/backend/core/inference/local_model_resolver.py
index d871981e971..adcb55bfa60 100644
--- a/studio/backend/core/inference/local_model_resolver.py
+++ b/studio/backend/core/inference/local_model_resolver.py
@@ -32,6 +32,7 @@ class _LocalGgufEntry:
load_path: str
variants: tuple[str, ...] # local quant labels; () for a standalone .gguf
is_gguf: bool = True # False routes the load to the inference orchestrator
+ repo_level_companions: bool = False
aliases: tuple[tuple[str, str], ...] = () # see _legacy_variant_aliases
@@ -115,6 +116,90 @@ def _resolve_load_dir(p, loader_id: Optional[str] = None):
return load_dir
+def _resolve_gguf_load_snapshot(p):
+ """Newest complete snapshot in this exact cache repo, or the selector fallback.
+
+ A later Hub revision can contain only a newly fetched companion such as an
+ MTP drafter or mmproj while the complete model weights remain in an older
+ snapshot. The generic resolver intentionally picks the newest snapshot, but
+ doing that for GGUF discovery makes the repo disappear from ``/v1/models``.
+ """
+ snapshots = p / "snapshots"
+ try:
+ if not snapshots.is_dir():
+ return p
+ except OSError:
+ return None
+
+ # Reuse the Hub inventory's selection rule, scoped to the row's exact repo
+ # directory so case-colliding repos cannot cross-load or trigger another root scan.
+ from hub.utils.gguf import select_gguf_cache_snapshot_for_repo_dir
+
+ selected = select_gguf_cache_snapshot_for_repo_dir(p)
+ if selected is None:
+ return None
+ return selected
+
+
+def local_gguf_companion_roots(load_path: str, *, repo_level: bool = False) -> tuple[str, ...]:
+ """Trusted sibling snapshots for a repo-level HF cache resolution.
+
+ The selected snapshot remains first so a colocated companion wins. Other
+ revisions are returned newest first for the case where a later download
+ contains only a compatible mmproj. Exact revision paths and paths outside
+ an exact ``models--*`` cache-repo layout never widen their companion search.
+ """
+ from pathlib import Path
+ from hub.utils.hf_cache_state import snapshot_selection_key
+
+ if not repo_level:
+ return ()
+ selected = Path(load_path)
+ snapshots = selected.parent
+ repo = snapshots.parent
+ if snapshots.name != "snapshots" or not repo.name.startswith("models--"):
+ return ()
+ try:
+ if not selected.is_dir():
+ return ()
+ except OSError:
+ return ()
+ siblings = []
+ try:
+ for path in snapshots.iterdir():
+ if path == selected:
+ continue
+ try:
+ if path.is_dir():
+ siblings.append(path)
+ except OSError as exc:
+ logger.debug("Skipping unreadable companion snapshot %s: %s", path, exc)
+ except OSError as exc:
+ logger.debug("Stopping at unreadable companion snapshots dir %s: %s", snapshots, exc)
+ siblings.sort(key = snapshot_selection_key, reverse = True)
+ return (str(selected), *(str(path) for path in siblings))
+
+
+def local_gguf_companion_state(roots: tuple[str, ...]) -> tuple:
+ """File metadata for trusted snapshots, including newly completed companions."""
+ from pathlib import Path
+
+ state = []
+ for root in roots:
+ try:
+ for path in sorted(Path(root).rglob("*")):
+ if path.suffix.lower() != ".gguf":
+ continue
+ try:
+ stat = path.stat()
+ state.append((str(path), stat.st_size, stat.st_mtime_ns))
+ except OSError:
+ state.append((str(path), -1, -1))
+ except OSError:
+ state.append((root, -1, -1))
+ return tuple(state)
+
+
def _legacy_variant_aliases(variants) -> tuple[tuple[str, str], ...]:
"""``(legacy label, current label)`` for each id this module used to publish and no longer does.
@@ -152,7 +237,12 @@ def _legacy_variant_aliases(variants) -> tuple[tuple[str, str], ...]:
return ()
-def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
+def _local_gguf_entry(
+ loader_id: str,
+ info,
+ *,
+ exact_snapshot: bool = False,
+) -> Optional[_LocalGgufEntry]:
"""Build an entry only when GGUF quants are on disk (not Transformers/
safetensors), listing only on-disk quants. ``load_path`` is a concrete local
path so /load resolves the variant locally and never fetches a remote one."""
@@ -177,10 +267,53 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
if p.suffix.lower() != ".gguf" or detect_gguf_model(str(p)) is None:
return None
return _LocalGgufEntry(loader_id, str(p), ())
- load_dir = _resolve_load_dir(p)
- # Asked of the lister, not filtered after: two checkpoints can share a quant, and
- # grouping keeps one, so a dead file would take its readable namesake down with it.
- variants, _ = list_local_gguf_variants(str(load_dir), require_existing_files = True)
+ cache_repo_dir = None
+ if getattr(info, "source", None) == "hf_cache":
+ try:
+ if p.name.startswith("models--") and (p / "snapshots").is_dir():
+ cache_repo_dir = p
+ except OSError:
+ return None
+ if (
+ cache_repo_dir is None
+ and p.parent.name == "snapshots"
+ and p.parent.parent.name.startswith("models--")
+ ):
+ cache_repo_dir = p.parent.parent
+ if cache_repo_dir is not None:
+ selected = _resolve_gguf_load_snapshot(cache_repo_dir)
+ if selected is None:
+ load_dir = p
+ variants, _ = list_local_gguf_variants(str(load_dir), require_existing_files = True)
+ else:
+ selected_variants, _has_vision, complete, load_dir = selected
+ variants, _ = list_local_gguf_variants(str(load_dir), require_existing_files = True)
+ if complete:
+ complete_keys = {str(quant).casefold() for quant in complete}
+ complete_files = {
+ str(variant.filename).casefold()
+ for variant in selected_variants
+ if getattr(variant, "filename", None)
+ and getattr(variant, "quant", None)
+ and str(variant.quant).casefold() in complete_keys
+ }
+ variants = [
+ variant
+ for variant in variants
+ if getattr(variant, "filename", None)
+ and str(variant.filename).casefold() in complete_files
+ ]
+ else:
+ load_dir = _resolve_load_dir(p) if p.name.startswith("models--") else p
+ variants, _ = list_local_gguf_variants(str(load_dir), require_existing_files = True)
+ if exact_snapshot:
+ from hub.utils.gguf import list_local_gguf_variants as inventory_variants
+ from hub.utils.inventory_scan import complete_snapshot_variants
+
+ complete = complete_snapshot_variants(str(load_dir))
+ offered, _ = inventory_variants(str(load_dir))
+ complete_files = {str(v.filename).casefold() for v in offered if v.quant in complete}
+ variants = [v for v in variants if str(v.filename).casefold() in complete_files]
quants = tuple(v.quant for v in variants if getattr(v, "quant", None))
if not quants:
return None
@@ -200,7 +333,11 @@ def _local_gguf_entry(loader_id: str, info) -> Optional[_LocalGgufEntry]:
if best and quants[0] != best:
quants = (best, *(q for q in quants if q != best))
return _LocalGgufEntry(
- loader_id, str(load_dir), quants, aliases = _legacy_variant_aliases(variants)
+ loader_id,
+ str(load_dir),
+ quants,
+ repo_level_companions = cache_repo_dir is not None,
+ aliases = _legacy_variant_aliases(variants),
)
except Exception:
return None
@@ -586,16 +723,26 @@ def _build_index() -> dict[str, _LocalGgufEntry]:
entry = _local_servable_entry(loader_id, info)
if entry is None:
continue
- # index every alias (including the path) so a client can resolve by any of them, even though only the non-path
- # loader_id is advertised
- for key in (
- raw_id,
- getattr(info, "model_id", None),
- getattr(info, "display_name", None),
- public_model_id(raw_id),
+ # Repo and display aliases intentionally select across revisions. An inactive-cache row's absolute id and
+ # basename name one exact revision instead: index them only when that revision has its own complete weights.
+ path_alias_entry = entry
+ if entry.repo_level_companions and _is_abs_path_id(raw_id):
+ from types import SimpleNamespace
+ path_alias_entry = _local_gguf_entry(
+ loader_id, SimpleNamespace(path = raw_id), exact_snapshot = True
+ )
+ snapshot_name = (
+ Path(raw_id).name if entry.repo_level_companions and _is_abs_path_id(raw_id) else None
+ )
+ for key, alias_entry in (
+ (raw_id, path_alias_entry),
+ (snapshot_name, path_alias_entry),
+ (getattr(info, "model_id", None), entry),
+ (getattr(info, "display_name", None), entry),
+ (public_model_id(raw_id), path_alias_entry),
):
- if key:
- index.setdefault(key.strip().lower(), entry)
+ if key and alias_entry is not None:
+ index.setdefault(key.strip().lower(), alias_entry)
# Other revisions of the same repo resolve to their own weights, so a pin on one keeps working after Hugging
# Face writes a newer snapshot.
if entry.is_gguf:
@@ -745,7 +892,9 @@ def index_is_built() -> bool:
return _scan[0] > 0.0
-def resolve_trusted_cached_local_gguf(requested: str) -> Optional[tuple[str, Optional[str], str]]:
+def resolve_trusted_cached_local_gguf(
+ requested: str, *, include_companion_scope: bool = False
+) -> Optional[tuple]:
"""Resolve a positive cache hit only when its snapshot is safe to trust.
A snapshot is trustworthy while fresh, or after an explicit additions-only
@@ -753,9 +902,16 @@ def resolve_trusted_cached_local_gguf(requested: str) -> Optional[tuple[str, Opt
must be rebuilt before it can trigger a model switch. The identity checks close
the race where invalidation publishes a different snapshot during resolution or
while the trust state is being evaluated.
+
+ With ``include_companion_scope=True``, append whether sibling snapshots belong
+ to this repo-level resolution and may be searched for compatible companions.
"""
snapshot = _scan
- resolved = _resolve_from_index(requested, snapshot[1])
+ resolved = _resolve_from_index(
+ requested,
+ snapshot[1],
+ include_companion_scope = include_companion_scope,
+ )
if resolved is None or _scan is not snapshot:
return None
trusted = _snapshot_is_trusted(snapshot[0], time.monotonic())
@@ -808,8 +964,11 @@ def warm_index_soon() -> None:
def resolve_local_gguf(
- requested: str, *, allow_scan: bool = True
-) -> Optional[tuple[str, Optional[str], str]]:
+ requested: str,
+ *,
+ allow_scan: bool = True,
+ include_companion_scope: bool = False,
+) -> Optional[tuple]:
"""Return ``(load_path, gguf_variant, loader_id)`` for a local match, else None.
``load_path`` is the concrete on-disk path to hand /load (so it never fetches
@@ -823,13 +982,19 @@ def resolve_local_gguf(
``allow_scan=False`` answers from the last built index and never rebuilds. It is
a raw snapshot read for callers that separately decide whether the snapshot is
trustworthy; use :func:`resolve_trusted_cached_local_gguf` for model switching.
+ With ``include_companion_scope=True``, append whether sibling snapshots belong
+ to this repo-level resolution and may be searched for compatible companions.
"""
if not isinstance(requested, str) or not requested.strip():
return None
requested = requested.strip()
try:
index = _index() if allow_scan else _scan[1]
- return _resolve_from_index(requested, index)
+ return _resolve_from_index(
+ requested,
+ index,
+ include_companion_scope = include_companion_scope,
+ )
except Exception:
# Best-effort: any resolver failure falls through to the loaded model, so a malformed name can never turn a
# servable request into a 500.
@@ -837,14 +1002,22 @@ def resolve_local_gguf(
def _resolve_from_index(
- requested: str, index: dict[str, _LocalGgufEntry]
-) -> Optional[tuple[str, Optional[str], str]]:
+ requested: str,
+ index: dict[str, _LocalGgufEntry],
+ *,
+ include_companion_scope: bool = False,
+) -> Optional[tuple]:
"""Resolve *requested* against one immutable published index mapping."""
try:
+
+ def _result(entry: _LocalGgufEntry, variant: Optional[str]) -> tuple:
+ result = (entry.load_path, variant, entry.loader_id)
+ return (*result, entry.repo_level_companions) if include_companion_scope else result
+
entry = index.get(requested.lower())
if entry is not None:
variant = entry.variants[0] if entry.variants else None
- return entry.load_path, variant, entry.loader_id
+ return _result(entry, variant)
base, sep, variant = requested.rpartition(":")
if not sep:
@@ -855,19 +1028,19 @@ def _resolve_from_index(
wanted = variant.strip().lower()
for v in entry.variants:
if v.lower() == wanted:
- return entry.load_path, v, entry.loader_id
+ return _result(entry, v)
# ahead of both branches below: one refuses a retired spelling, the other answers it
# with a different quant
for legacy, current in entry.aliases:
if legacy == wanted:
- return entry.load_path, current, entry.loader_id
+ return _result(entry, current)
from core.inference.openai_auto_download import looks_like_quant
if looks_like_quant(variant):
return None
# ":latest" or ":8b" names no file, so it means the repo; a real quant that is not on disk still misses, or a
# swap would serve the wrong weights.
- return entry.load_path, (entry.variants[0] if entry.variants else None), entry.loader_id
+ return _result(entry, entry.variants[0] if entry.variants else None)
except Exception:
return None
diff --git a/studio/backend/hub/utils/gguf.py b/studio/backend/hub/utils/gguf.py
index 9e3c885c985..a4838726b12 100644
--- a/studio/backend/hub/utils/gguf.py
+++ b/studio/backend/hub/utils/gguf.py
@@ -9,7 +9,7 @@ import os
import re
from dataclasses import dataclass
from pathlib import Path
-from typing import Optional, Sequence
+from typing import Iterable, Optional, Sequence
from loggers import get_logger
from utils.paths.path_utils import (
@@ -1050,22 +1050,12 @@ def list_empty_gguf_variant_dirs(repo_id: str, root: Optional[Path] = None) -> s
return {label for key, label in empty.items() if key not in nonempty}
-def select_gguf_cache_snapshot(
- repo_id: str, root: Optional[Path] = None
+def _select_gguf_snapshot(
+ snapshots: Iterable[Path],
) -> Optional[tuple[list[GgufVariantInfo], bool, set, Path]]:
- """``list_gguf_variants_from_hf_cache`` plus the snapshot it answered from.
-
- A repo dir holds every revision, so a caller that then reads metadata from wherever this
- listing came from needs the snapshot, not the dir: the dir includes revisions it skipped.
- """
# Local import: inventory_scan imports this module.
from hub.utils.inventory_scan import complete_snapshot_variants
- snapshots = (
- iter_hf_cache_snapshots(repo_id, root = root)
- if root is not None
- else iter_hf_cache_snapshots(repo_id)
- )
# Pick the snapshot the inventory row does: newest holding a whole quant, else first non-empty.
fallback: Optional[tuple[list[GgufVariantInfo], bool, set, Path]] = None
for snapshot in snapshots:
@@ -1080,6 +1070,43 @@ def select_gguf_cache_snapshot(
return fallback
+def select_gguf_cache_snapshot(
+ repo_id: str, root: Optional[Path] = None
+) -> Optional[tuple[list[GgufVariantInfo], bool, set, Path]]:
+ """``list_gguf_variants_from_hf_cache`` plus the snapshot it answered from.
+
+ A repo dir holds every revision, so a caller that then reads metadata from wherever this
+ listing came from needs the snapshot, not the dir: the dir includes revisions it skipped.
+ """
+ snapshots = (
+ iter_hf_cache_snapshots(repo_id, root = root)
+ if root is not None
+ else iter_hf_cache_snapshots(repo_id)
+ )
+ return _select_gguf_snapshot(snapshots)
+
+
+def select_gguf_cache_snapshot_for_repo_dir(
+ repo_dir: Path,
+) -> Optional[tuple[list[GgufVariantInfo], bool, set, Path]]:
+ """Select only among snapshots belonging to the exact scanned cache directory."""
+ from hub.utils.hf_cache_state import snapshot_selection_key
+
+ snapshots_dir = Path(repo_dir) / "snapshots"
+ snapshots = []
+ try:
+ for snapshot in snapshots_dir.iterdir():
+ try:
+ if snapshot.is_dir():
+ snapshots.append(snapshot)
+ except OSError as exc:
+ logger.debug("Skipping unreadable cache snapshot %s: %s", snapshot, exc)
+ except OSError as exc:
+ logger.debug("Stopping at unreadable cache snapshots dir %s: %s", snapshots_dir, exc)
+ snapshots.sort(key = snapshot_selection_key, reverse = True)
+ return _select_gguf_snapshot(snapshots)
+
+
def merge_sibling_snapshot_variants(
repo_id: str,
selected: tuple[list[GgufVariantInfo], bool, set, Path],
diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py
index dc402ea4ff1..c777bfa09ef 100644
--- a/studio/backend/models/inference.py
+++ b/studio/backend/models/inference.py
@@ -14,6 +14,7 @@ from pydantic import (
BaseModel,
Discriminator,
Field,
+ PrivateAttr,
Tag,
field_validator,
model_validator,
@@ -36,6 +37,7 @@ class LoadRequest(BaseModel):
"""Request to load a model for inference"""
model_path: str = Field(..., description = "Model identifier or local path")
+ _gguf_companion_roots: tuple[str, ...] = PrivateAttr(default = ())
load_request_id: Optional[str] = Field(
None,
min_length = 1,
diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py
index 7eb3c6529b8..f632b86bf1c 100644
--- a/studio/backend/routes/inference.py
+++ b/studio/backend/routes/inference.py
@@ -7027,6 +7027,7 @@ def _target_is_vision(
load_path: str,
gguf_variant: Optional[str] = None,
need_image: bool = True,
+ gguf_companion_roots: tuple[str, ...] = (),
) -> bool:
# A local GGUF's vision capability is its companion mmproj, a filesystem check
# (no model load). Matches the loaded backend's is_vision, so rejecting a swap
@@ -7046,6 +7047,7 @@ def _target_is_vision(
hf_token = os.environ.get("HF_TOKEN"),
gguf_variant = gguf_variant,
require_image = need_image,
+ gguf_companion_roots = gguf_companion_roots or None,
)
)
except Exception as exc:
@@ -7083,6 +7085,7 @@ def _target_accepts_request_input(
gguf_variant: Optional[str] = None,
need_image: bool = True,
needs_video: bool = False,
+ gguf_companion_roots: tuple[str, ...] = (),
) -> bool:
"""Whether a switch target can accept this request's image or audio input.
@@ -7092,7 +7095,14 @@ def _target_accepts_request_input(
tokenizer's special tokens.
"""
if is_gguf:
- return _target_is_vision(load_path, gguf_variant, need_image)
+ if not gguf_companion_roots:
+ return _target_is_vision(load_path, gguf_variant, need_image)
+ return _target_is_vision(
+ load_path,
+ gguf_variant,
+ need_image,
+ gguf_companion_roots,
+ )
# input_video is llama.cpp's own part type, so a clip is refused right after the load.
if needs_video:
return False
@@ -7855,6 +7865,22 @@ def _loaded_identity_satisfies(requested: str) -> bool:
# alias it recorded and land here.
if advertised is None and identifier and _looks_like_local_path(identifier):
return False
+ companion_roots = tuple(getattr(llama_backend, "_openai_gguf_companion_roots", ()) or ())
+ if companion_roots:
+ from core.inference.local_model_resolver import (
+ local_gguf_companion_roots,
+ local_gguf_companion_state,
+ )
+
+ # A revision pin drops repo scope; a repo alias must notice newly cached companions.
+ if _looks_like_local_path(base) or companion_roots != local_gguf_companion_roots(
+ identifier, repo_level = True
+ ):
+ return False
+ if getattr(
+ llama_backend, "_openai_gguf_companion_state", ()
+ ) != local_gguf_companion_state(companion_roots):
+ return False
return _matches_any(base, (identifier, advertised)) and _loaded_satisfies(requested)
backend = get_inference_backend()
active = getattr(backend, "active_model_name", None)
@@ -8217,6 +8243,8 @@ async def _maybe_auto_switch_model(
model_override_load_kwargs,
)
from core.inference.local_model_resolver import (
+ local_gguf_companion_roots,
+ local_gguf_companion_state,
local_target_is_gguf,
resolve_local_gguf,
resolve_trusted_cached_local_gguf,
@@ -8333,16 +8361,25 @@ async def _maybe_auto_switch_model(
# resolve so only the reload-stash path runs and no name is ever matched.
reload_only = requested_model == _RELOAD_ONLY_MODEL
resolved = None
+ repo_level_companion_resolution = False
+ stashed_gguf_companion_roots: tuple[str, ...] = ()
if auto_switch_on and not reload_only:
# Fresh hits and entries retained across an additions-only download are
# safe to use immediately. An expired/config-invalidated hit, a cold
# cache, and every miss must refresh before an unrelated resident model
# can answer or an entry from a removed scan root can trigger a switch.
- resolved = resolve_trusted_cached_local_gguf(requested_model)
+ resolved = resolve_trusted_cached_local_gguf(
+ requested_model,
+ include_companion_scope = True,
+ )
if resolved is not None:
warm_index_soon()
else:
- resolved = await asyncio.to_thread(resolve_local_gguf, requested_model)
+ resolved = await asyncio.to_thread(
+ resolve_local_gguf,
+ requested_model,
+ include_companion_scope = True,
+ )
if resolved is None:
# Not on disk. Opt-in: fetch in the background and ask the caller to retry.
if auto_switch_on and not reload_only:
@@ -8374,8 +8411,10 @@ async def _maybe_auto_switch_model(
if claim_resident:
_claim_slot_for_non_preview(fastapi_request)
return
- if len(last) == 3:
- target_id, variant, override_id = last
+ if len(last) >= 3:
+ target_id, variant, override_id = last[:3]
+ if len(last) >= 4:
+ stashed_gguf_companion_roots = tuple(last[3] or ())
else: # pre-3-tuple stash: fall back to the path as the override key
target_id, variant = last
override_id = target_id
@@ -8396,9 +8435,32 @@ async def _maybe_auto_switch_model(
# load_path is a concrete local path (never the bare repo id), so /load
# takes the local branch and cannot trigger a download. override_id is the
# advertised repo id, the launch-override key and the public model id.
- target_id, variant, override_id = resolved
+ if len(resolved) >= 4:
+ target_id, variant, override_id, repo_level_companion_resolution = resolved[:4]
+ else:
+ target_id, variant, override_id = resolved
+ requested_base, _requested_variant = split_model_ref(requested_model)
+ repo_level_companion_resolution = _matches_any(
+ requested_base,
+ (override_id, public_model_id(override_id)),
+ )
# Not inferred from the quant: a GGUF loaded from a local directory carries none.
target_is_gguf = await asyncio.to_thread(local_target_is_gguf, target_id, override_id)
+ gguf_companion_roots: tuple[str, ...] = ()
+ if target_is_gguf:
+ if resolved is None:
+ gguf_companion_roots = stashed_gguf_companion_roots
+ else:
+ gguf_companion_roots = await asyncio.to_thread(
+ local_gguf_companion_roots,
+ target_id,
+ repo_level = repo_level_companion_resolution,
+ )
+ gguf_companion_state = (
+ await asyncio.to_thread(local_gguf_companion_state, gguf_companion_roots)
+ if gguf_companion_roots
+ else ()
+ )
# no orchestrator means nothing non-GGUF is resident, so the cold build only precedes a 400.
if (
gguf_only
@@ -8449,6 +8511,13 @@ async def _maybe_auto_switch_model(
loaded_keys.add(advertised.lower())
if loaded_keys.isdisjoint({target_id.lower(), override_id.lower()}):
return False
+ loaded_companion_roots = tuple(
+ getattr(backend, "_openai_gguf_companion_roots", ()) or ()
+ )
+ if loaded_companion_roots != gguf_companion_roots:
+ return False
+ if getattr(backend, "_openai_gguf_companion_state", ()) != gguf_companion_state:
+ return False
if bare:
return True
if variant:
@@ -8499,6 +8568,7 @@ async def _maybe_auto_switch_model(
variant,
target_requires_image,
require_video,
+ gguf_companion_roots,
)
):
raise HTTPException(
@@ -8720,8 +8790,10 @@ async def _maybe_auto_switch_model(
if speech_type is not None and caller_hf_token is None:
load_internal_kw["anonymous_hf_access"] = True
try:
+ load_request = LoadRequest(**load_kwargs)
+ load_request._gguf_companion_roots = gguf_companion_roots
await _load_model_impl(
- LoadRequest(**load_kwargs),
+ load_request,
fastapi_request,
current_subject,
current_request_counted = True,
@@ -8743,8 +8815,10 @@ async def _maybe_auto_switch_model(
exc.detail,
)
load_kwargs.pop("gpu_ids", None)
+ load_request = LoadRequest(**load_kwargs)
+ load_request._gguf_companion_roots = gguf_companion_roots
await _load_model_impl(
- LoadRequest(**load_kwargs),
+ load_request,
fastapi_request,
current_subject,
current_request_counted = True,
@@ -13843,6 +13917,16 @@ async def _load_model_impl(
# Resolve once so dedupe, admission and launch use the same slot count.
_n_parallel = _resolve_parallel_slots(request, fastapi_request)
+ from core.inference.local_model_resolver import local_gguf_companion_state
+
+ # Capture before metadata resolution; a download completing during launch must be noticed next time.
+ gguf_companion_state = (
+ await asyncio.to_thread(
+ local_gguf_companion_state, tuple(request._gguf_companion_roots)
+ )
+ if request._gguf_companion_roots
+ else ()
+ )
def _reuse_loaded_gguf(
intent: GgufLoadIntent, *, display_name: Optional[str] = None
@@ -13853,6 +13937,10 @@ async def _load_model_impl(
# (the intent match lowercases the identifier). Checked before the adopt so
# a mismatch never adopts the caller's placement.
_same_loaded_identifier(llama_backend.model_identifier, model_identifier)
+ and tuple(getattr(llama_backend, "_openai_gguf_companion_roots", ()) or ())
+ == tuple(request._gguf_companion_roots)
+ and getattr(llama_backend, "_openai_gguf_companion_state", ())
+ == gguf_companion_state
and llama_backend.adopt_load_intent_if_matched(intent)
and getattr(llama_backend, "_audio_probed", True)
):
@@ -13983,6 +14071,7 @@ async def _load_model_impl(
# pass that touches a drafter candidate, so the boundary has
# to travel with it rather than being applied afterwards.
drafter_accept = _native_drafter_accept if native_grant_backed else None,
+ gguf_companion_roots = request._gguf_companion_roots or None,
)
# Guard and call go to the worker together: from_identifier can import transformers
@@ -14474,6 +14563,8 @@ async def _load_model_impl(
# Clear any idle-unload reload stash now, not only on the next poll.
from core.inference.llama_keepwarm import note_model_loaded
+ llama_backend._openai_gguf_companion_roots = tuple(request._gguf_companion_roots)
+ llama_backend._openai_gguf_companion_state = gguf_companion_state
await asyncio.to_thread(note_model_loaded, llama_backend)
# A plain load advertises its own identifier; auto-switch overwrites
# this with the repo id right after _load_model_impl returns.
diff --git a/studio/backend/tests/test_detect_mmproj_file.py b/studio/backend/tests/test_detect_mmproj_file.py
index 9a985b6386d..2414218b294 100644
--- a/studio/backend/tests/test_detect_mmproj_file.py
+++ b/studio/backend/tests/test_detect_mmproj_file.py
@@ -365,3 +365,16 @@ def test_a_zero_byte_projector_does_not_shadow_a_whole_one(tmp_path: Path):
whole = _touch(tmp_path / "mmproj-Q8_0.gguf")
assert detect_mmproj_file(str(weight)) == str(whole.resolve())
+
+
+def test_trusted_companion_snapshot_finds_nested_projector(tmp_path: Path):
+ weights = tmp_path / "weights"
+ sibling = tmp_path / "companion"
+ weights.mkdir()
+ (sibling / "vision").mkdir(parents = True)
+ weight = _touch(weights / "Model-Q4_K_M.gguf")
+ projector = _touch(sibling / "vision" / "mmproj-Model-F16.gguf")
+ assert detect_mmproj_file(str(weight), search_root = str(sibling)) is None
+ assert detect_mmproj_file(
+ str(weight), search_root = str(sibling), allow_disjoint_search_root = True
+ ) == str(projector.resolve())
diff --git a/studio/backend/tests/test_gguf_load_cache_reuse.py b/studio/backend/tests/test_gguf_load_cache_reuse.py
index d2e2f8950bd..3812d67ed7a 100644
--- a/studio/backend/tests/test_gguf_load_cache_reuse.py
+++ b/studio/backend/tests/test_gguf_load_cache_reuse.py
@@ -710,7 +710,8 @@ class TestLoadHubDownloadExclusion:
assert intent.gguf_path is None
assert intent.hf_variant == "Q8_0"
- def test_resident_gguf_reuse_precedes_model_metadata_resolution(self):
+ @pytest.mark.parametrize("reuse_case", ["unchanged", "new_roots", "completed_file"])
+ def test_resident_gguf_reuse_precedes_model_metadata_resolution(self, reuse_case, tmp_path):
from models.inference import LoadRequest
route = _load_route_module(
@@ -728,6 +729,16 @@ class TestLoadHubDownloadExclusion:
holds_no_vram = False,
)
request = LoadRequest(model_path = REPO, gguf_variant = VARIANT)
+ if reuse_case == "new_roots":
+ request._gguf_companion_roots = ("weights-revision", "companion-revision")
+ elif reuse_case == "completed_file":
+ request._gguf_companion_roots = (str(tmp_path),)
+ backend._openai_gguf_companion_roots = request._gguf_companion_roots
+ backend._openai_gguf_companion_state = ()
+ (tmp_path / "mmproj-F16.gguf").write_bytes(b"GGUF companion")
+
+ class MetadataReached(BaseException):
+ pass
with (
_reuse_route(route, backend, response),
@@ -735,13 +746,15 @@ class TestLoadHubDownloadExclusion:
route,
"ModelConfig",
SimpleNamespace(
- from_identifier = lambda **_kwargs: (_ for _ in ()).throw(
- AssertionError("resident reuse must not resolve model metadata")
- )
+ from_identifier = lambda **_kwargs: (_ for _ in ()).throw(MetadataReached())
),
),
patch.object(route, "_active_gguf_intent", return_value = object()),
):
+ if reuse_case != "unchanged":
+ with pytest.raises(MetadataReached):
+ _run_route_load(route, request)
+ return
result = _run_route_load(route, request)
assert result is response
diff --git a/studio/backend/tests/test_openai_auto_switch.py b/studio/backend/tests/test_openai_auto_switch.py
index 8878e42706a..909f2d249ab 100644
--- a/studio/backend/tests/test_openai_auto_switch.py
+++ b/studio/backend/tests/test_openai_auto_switch.py
@@ -128,6 +128,10 @@ class _LoadRecorder:
# Mirror _load_model_impl: a load advertises its own id until the
# auto-switch caller overwrites it with the repo id.
self.backend._openai_advertised_id = None
+ self.backend._openai_gguf_companion_roots = tuple(request._gguf_companion_roots)
+ self.backend._openai_gguf_companion_state = resolver.local_gguf_companion_state(
+ tuple(request._gguf_companion_roots)
+ )
from core.inference import llama_keepwarm as kw
kw.note_model_loaded(self.backend)
@@ -376,7 +380,7 @@ def test_an_expired_positive_hit_refreshes_before_switching(monkeypatch):
# The third call is the refusal wording asking what the RESIDENT model is; like the second it
# passes allow_scan = False, which is why the rescan count below is still one.
assert calls == [
- ("unsloth/B-GGUF", {}),
+ ("unsloth/B-GGUF", {"include_companion_scope": True}),
("unsloth/B-GGUF", {"allow_scan": False}),
("unsloth/A-GGUF:Q4_K_M", {"allow_scan": False}),
]
@@ -415,7 +419,7 @@ def test_a_stale_miss_refreshes_before_the_resident_model_can_answer(monkeypatch
_run_hook("unsloth/B-GGUF")
- assert calls == [("unsloth/B-GGUF", {})]
+ assert calls == [("unsloth/B-GGUF", {"include_companion_scope": True})]
assert scans == [1]
assert len(rec.calls) == 1
@@ -604,7 +608,7 @@ def test_local_gguf_entry_filters_non_gguf_and_recurses(tmp_path):
repo = tmp_path / "models--org--repo"
(repo / "snapshots" / "abc" / "BF16").mkdir(parents = True)
(repo / "snapshots" / "abc" / "BF16" / "model-BF16.gguf").write_text("x")
- e2 = resolver._local_gguf_entry("org/repo", SimpleNamespace(path = str(repo)))
+ e2 = resolver._local_gguf_entry("org/repo", SimpleNamespace(path = str(repo), source = "hf_cache"))
assert e2 is not None and e2.variants
@@ -2027,7 +2031,9 @@ def test_hf_cache_entry_loads_from_local_snapshot_path(tmp_path):
snap.mkdir(parents = True)
(snap / "model-Q4_K_M.gguf").write_bytes(b"GGUF stub")
- entry = resolver._local_gguf_entry("org/Repo", SimpleNamespace(id = "org/Repo", path = str(repo)))
+ entry = resolver._local_gguf_entry(
+ "org/Repo", SimpleNamespace(id = "org/Repo", path = str(repo), source = "hf_cache")
+ )
assert entry is not None
assert entry.loader_id == "org/Repo" # advertised id unchanged
assert "snapshots" in entry.load_path # loads from the concrete snapshot dir
@@ -2035,6 +2041,788 @@ def test_hf_cache_entry_loads_from_local_snapshot_path(tmp_path):
assert entry.variants # quant detected on disk
+def test_hf_cache_entry_skips_newer_companion_only_snapshot(tmp_path):
+ """A companion fetched at a newer revision must not hide older complete weights."""
+ import os
+ from pathlib import Path
+ from types import SimpleNamespace
+
+ repo = tmp_path / "models--unsloth--Qwen3.8-Flash-Next-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ quant_dir = old / "UD-Q4_K_XL"
+ quant_dir.mkdir(parents = True)
+ for shard in range(1, 5):
+ (quant_dir / f"Qwen3.8-Flash-Next-UD-Q4_K_XL-{shard:05d}-of-00004.gguf").write_bytes(
+ b"GGUF stub"
+ )
+
+ newer = repo / "snapshots" / "companion-revision" / "MTP"
+ newer.mkdir(parents = True)
+ (newer / "mtp-Qwen3.8-Flash-Next-Q8_0.gguf").write_bytes(b"GGUF companion")
+ os.utime(old, (1_000, 1_000))
+ os.utime(newer.parent, (2_000, 2_000))
+
+ # The generic cache resolver correctly picks the latest revision; GGUF
+ # discovery needs the stronger rule because this revision has no weights.
+ assert Path(resolver._resolve_load_dir(repo)) == newer.parent.resolve()
+ info = SimpleNamespace(id = "unsloth/Qwen3.8-Flash-Next-GGUF", path = str(repo), source = "hf_cache")
+ entry = resolver._local_gguf_entry(
+ "unsloth/Qwen3.8-Flash-Next-GGUF",
+ info,
+ )
+
+ assert entry is not None
+ assert Path(entry.load_path) == old
+ assert entry.variants == ("UD-Q4_K_XL",)
+ assert resolver.local_servable_model(info) == (True, ("UD-Q4_K_XL",))
+
+
[email protected]("repo_level", [True, False])
+ "filename, legacy, canonical",
+ [
+ ("DeepSeek-R1-BF16-Q4_K_M.gguf", "BF16", "Q4_K_M"),
+ ("Meta-Llama-3-8B.gguf", "8B", "Meta-Llama-3-8B"),
+ ],
+)
+def test_snapshot_selection_preserves_legacy_alias_companion_scope(
+ tmp_path, repo_level, filename, legacy, canonical
+):
+ import os
+ from types import SimpleNamespace
+
+ repo = tmp_path / "models--org--alias-scope"
+ old = repo / "snapshots" / "weights"
+ newer = repo / "snapshots" / "companions"
+ old.mkdir(parents = True)
+ newer.mkdir()
+ (old / filename).write_bytes(b"GGUF weights")
+ (newer / "mmproj-F16.gguf").write_bytes(b"GGUF companion")
+ os.utime(old, (1_000, 1_000))
+ os.utime(newer, (2_000, 2_000))
+ info = SimpleNamespace(
+ path = str(repo if repo_level else old),
+ source = "hf_cache" if repo_level else "local",
+ )
+ entry = resolver._local_gguf_entry("org/alias-scope", info, exact_snapshot = not repo_level)
+ assert entry is not None
+ assert entry.load_path == str(old)
+ assert entry.variants == (canonical,)
+ index = {"org/alias-scope": entry}
+ for label in (legacy, canonical):
+ assert resolver._resolve_from_index(
+ f"org/alias-scope:{label}", index, include_companion_scope = True
+ ) == (str(old), canonical, "org/alias-scope", repo_level)
+ assert resolver.local_gguf_companion_roots(
+ entry.load_path, repo_level = entry.repo_level_companions
+ ) == ((str(old), str(newer)) if repo_level else ())
+ assert (
+ resolver._local_gguf_entry(
+ "org/alias-scope", SimpleNamespace(path = str(newer)), exact_snapshot = True
+ )
+ is None
+ )
+
+
+def test_hf_cache_entry_keeps_newer_companions_for_auto_switch(tmp_path, monkeypatch):
+ """Selected weights stay pinned while request and load probes see companions."""
+ from pathlib import Path
+ from types import SimpleNamespace
+ from utils.models import model_config as model_config_module
+ from utils.models.drafters import dflash as dflash_module
+
+ ModelConfig = model_config_module.ModelConfig
+ detect_mmproj_file = model_config_module.detect_mmproj_file
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ mmproj = newer / "mmproj-vision-model-F16.gguf"
+ mmproj.write_bytes(b"GGUF companion")
+ mtp = newer / "mtp-vision-model-Q4_0.gguf"
+ mtp.write_bytes(b"GGUF drafter")
+ dspark = newer / "dspark-vision-model-Q8_0.gguf"
+ dspark.write_bytes(b"GGUF drafter")
+ dflash = newer / "dflash-kquant.gguf"
+ dflash.write_bytes(b"GGUF drafter")
+ os.utime(old, (1_000, 1_000))
+ os.utime(newer, (2_000, 2_000))
+
+ entry = resolver._local_gguf_entry(
+ "org/Vision-GGUF",
+ SimpleNamespace(id = "org/Vision-GGUF", path = str(repo), source = "hf_cache"),
+ )
+
+ assert entry is not None
+ assert Path(entry.load_path) == old
+ assert resolver.local_gguf_companion_roots(entry.load_path) == ()
+ assert resolver._resolve_from_index(
+ "Vision-GGUF",
+ {"vision-gguf": entry},
+ include_companion_scope = True,
+ ) == (entry.load_path, entry.variants[0], "org/Vision-GGUF", True)
+ roots = resolver.local_gguf_companion_roots(entry.load_path, repo_level = True)
+ assert tuple(map(Path, roots)) == (old, newer)
+ assert detect_mmproj_file(str(old / "vision-model-Q4_K_M.gguf"), search_root = str(newer)) is None
+ monkeypatch.setattr(
+ dflash_module,
+ "read_gguf_architecture",
+ lambda path: "dflash" if Path(path).name == dflash.name else None,
+ )
+ assert inference_route._target_accepts_request_input(
+ entry.load_path,
+ True,
+ True,
+ False,
+ entry.variants[0],
+ True,
+ False,
+ roots,
+ )
+ config = ModelConfig.from_identifier(
+ entry.load_path,
+ gguf_variant = entry.variants[0],
+ gguf_companion_roots = roots,
+ )
+ assert config is not None
+ assert config.gguf_mmproj_file == str(mmproj.resolve())
+ assert config.gguf_mtp_file == str(mtp.resolve())
+ assert config.gguf_dspark_file == str(dspark.resolve())
+ assert config.gguf_dflash_file == str(dflash.resolve())
+
+
+def test_hf_cache_entry_skips_unreadable_sibling_for_mmproj(tmp_path, monkeypatch):
+ from pathlib import Path
+ from types import SimpleNamespace
+ from utils.models import model_config as model_config_module
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ (newer / "mmproj-vision-model-F16.gguf").write_bytes(b"GGUF companion")
+ unreadable = repo / "snapshots" / "unreadable-revision"
+ unreadable.mkdir(parents = True)
+ os.utime(old, (1_000, 1_000))
+ os.utime(newer, (2_000, 2_000))
+ os.utime(unreadable, (3_000, 3_000))
+
+ entry = resolver._local_gguf_entry(
+ "org/Vision-GGUF",
+ SimpleNamespace(id = "org/Vision-GGUF", path = str(repo), source = "hf_cache"),
+ )
+ assert entry is not None
+ roots = resolver.local_gguf_companion_roots(entry.load_path, repo_level = True)
+ assert tuple(map(Path, roots)) == (old, unreadable, newer)
+
+ original_iter = model_config_module._iter_gguf_files
+
+ def _raise_for_unreadable(directory, recursive = False):
+ if Path(directory).resolve() == unreadable.resolve():
+ raise PermissionError("simulated unreadable snapshot")
+ return original_iter(directory, recursive)
+
+ monkeypatch.setattr(model_config_module, "_iter_gguf_files", _raise_for_unreadable)
+ assert model_config_module.is_vision_model(
+ entry.load_path,
+ gguf_variant = entry.variants[0],
+ gguf_companion_roots = roots,
+ )
+
+
+def test_image_preflight_uses_the_projector_selected_for_load(tmp_path, monkeypatch):
+ from utils.models import model_config as config_module
+
+ selected = tmp_path / "selected"
+ sibling = tmp_path / "sibling"
+ selected.mkdir()
+ sibling.mkdir()
+ (selected / "model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ audio = selected / "mmproj-model-F16.gguf"
+ image = sibling / "mmproj-model-F16.gguf"
+ audio.write_bytes(b"GGUF audio")
+ image.write_bytes(b"GGUF image")
+ roots = (str(selected), str(sibling))
+ monkeypatch.setattr(
+ config_module, "mmproj_accepts_image", lambda path: str(path) == str(image.resolve())
+ )
+ config = config_module.ModelConfig.from_identifier(
+ str(selected), gguf_variant = "Q4_K_M", gguf_companion_roots = roots
+ )
+ assert config.gguf_mmproj_file == str(audio.resolve())
+ assert not config_module.is_vision_model(
+ str(selected),
+ gguf_variant = "Q4_K_M",
+ gguf_companion_roots = roots,
+ require_image = True,
+ )
+
+
+def test_companion_roots_skip_only_the_unreadable_sibling(tmp_path, monkeypatch):
+ from pathlib import Path
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ selected = repo / "snapshots" / "weights-revision"
+ readable = repo / "snapshots" / "companion-revision"
+ unreadable = repo / "snapshots" / "unreadable-revision"
+ for path in (selected, readable, unreadable):
+ path.mkdir(parents = True)
+
+ original_is_dir = Path.is_dir
+
+ def _is_dir(path):
+ if path == unreadable:
+ raise PermissionError("simulated unreadable snapshot")
+ return original_is_dir(path)
+
+ monkeypatch.setattr(Path, "is_dir", _is_dir)
+
+ assert tuple(
+ map(Path, resolver.local_gguf_companion_roots(str(selected), repo_level = True))
+ ) == (selected, readable)
+
+
+def test_snapshot_selector_skips_only_the_unreadable_child(tmp_path, monkeypatch):
+ from pathlib import Path
+ from hub.utils.gguf import select_gguf_cache_snapshot_for_repo_dir
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ readable = repo / "snapshots" / "weights-revision"
+ unreadable = repo / "snapshots" / "unreadable-revision"
+ readable.mkdir(parents = True)
+ unreadable.mkdir()
+ (readable / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+
+ original_is_dir = Path.is_dir
+
+ def _is_dir(path):
+ if path == unreadable:
+ raise PermissionError("simulated unreadable snapshot")
+ return original_is_dir(path)
+
+ monkeypatch.setattr(Path, "is_dir", _is_dir)
+
+ selected = select_gguf_cache_snapshot_for_repo_dir(repo)
+ assert selected is not None
+ assert selected[3] == readable
+
+
+def test_disjoint_companion_roots_preserve_selected_snapshot_ancestor_walk(tmp_path):
+ """A selected snapshot still walks intermediate parents before sibling revisions."""
+ from utils.models.model_config import detect_mmproj_file
+
+ snapshot = tmp_path / "models--org--Vision-GGUF" / "snapshots" / "weights-revision"
+ checkpoint = snapshot / "checkpoint"
+ quant = checkpoint / "Q4_K_M"
+ quant.mkdir(parents = True)
+ weights = quant / "vision-model-Q4_K_M.gguf"
+ weights.write_bytes(b"GGUF weights")
+ mmproj = checkpoint / "mmproj-vision-model-F16.gguf"
+ mmproj.write_bytes(b"GGUF companion")
+
+ assert detect_mmproj_file(
+ str(weights),
+ search_root = str(snapshot),
+ allow_disjoint_search_root = True,
+ ) == str(mmproj.resolve())
+
+
+def test_auto_switch_carries_hf_cache_companion_roots_into_load(tmp_path, monkeypatch):
+ from pathlib import Path
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ (newer / "mmproj-vision-model-F16.gguf").write_bytes(b"GGUF companion")
+ os.utime(old, (1_000, 1_000))
+ os.utime(newer, (2_000, 2_000))
+
+ backend = _FakeBackend("org/Other-GGUF", "Q4_K_M")
+ recorder = _LoadRecorder(backend)
+ _wire(
+ monkeypatch,
+ enabled = True,
+ resolves_to = (str(old), "Q4_K_M", "org/Vision-GGUF"),
+ backend = backend,
+ recorder = recorder,
+ )
+
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/Vision-GGUF",
+ object(),
+ "tester",
+ require_vision = True,
+ )
+ )
+
+ assert len(recorder.calls) == 1
+ assert tuple(map(Path, recorder.calls[0]._gguf_companion_roots)) == (old, newer)
+ assert "_gguf_companion_roots" not in recorder.calls[0].model_dump()
+ assert (
+ LoadRequest(
+ model_path = str(old),
+ _gguf_companion_roots = (str(tmp_path / "untrusted"),),
+ )._gguf_companion_roots
+ == ()
+ )
+
+
+def test_auto_switch_display_alias_keeps_repo_level_companion_scope(tmp_path, monkeypatch):
+ from pathlib import Path
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ (newer / "mmproj-vision-model-F16.gguf").write_bytes(b"GGUF companion")
+
+ backend = _FakeBackend("org/Other-GGUF", "Q4_K_M")
+ recorder = _LoadRecorder(backend)
+ _wire(
+ monkeypatch,
+ enabled = True,
+ resolves_to = (str(old), "Q4_K_M", "org/Vision-GGUF", True),
+ backend = backend,
+ recorder = recorder,
+ )
+
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "Vision-GGUF",
+ object(),
+ "tester",
+ )
+ )
+
+ assert len(recorder.calls) == 1
+ assert tuple(map(Path, recorder.calls[0]._gguf_companion_roots)) == (old, newer)
+
+
[email protected]("advertised", [False, True])
+def test_repo_level_request_reloads_resident_snapshot_without_companion_roots(
+ tmp_path, monkeypatch, advertised
+):
+ from pathlib import Path
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ (newer / "mmproj-vision-model-F16.gguf").write_bytes(b"GGUF companion")
+
+ backend = _FakeBackend(str(old), "Q4_K_M")
+ if advertised:
+ backend._openai_advertised_id = "org/Vision-GGUF"
+ backend._openai_gguf_companion_roots = (str(old),)
+ recorder = _LoadRecorder(backend)
+ _wire(
+ monkeypatch,
+ enabled = True,
+ resolves_to = (str(old), "Q4_K_M", "org/Vision-GGUF", True),
+ backend = backend,
+ recorder = recorder,
+ )
+
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ "org/Vision-GGUF",
+ object(),
+ "tester",
+ )
+ )
+
+ assert len(recorder.calls) == 1
+ assert tuple(map(Path, recorder.calls[0]._gguf_companion_roots)) == (old, newer)
+ assert tuple(map(Path, backend._openai_gguf_companion_roots)) == (old, newer)
+
+
[email protected]("precreated", [False, True])
+def test_resident_repo_reloads_when_companion_finishes_in_existing_snapshot(
+ tmp_path, monkeypatch, precreated
+):
+ repo = tmp_path / "models--org--Vision-GGUF"
+ selected = repo / "snapshots" / "weights"
+ companion = repo / "snapshots" / "companion"
+ selected.mkdir(parents = True)
+ companion.mkdir()
+ if precreated:
+ (companion / "mmproj-vision-model-F16.gguf").write_bytes(b"")
+ (selected / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ backend = _FakeBackend("org/Other-GGUF", "Q4_K_M")
+ recorder = _LoadRecorder(backend)
+ _wire(
+ monkeypatch,
+ enabled = True,
+ resolves_to = (str(selected), "Q4_K_M", "org/Vision-GGUF", True),
+ backend = backend,
+ recorder = recorder,
+ )
+
+ async def run():
+ await inference_route._maybe_auto_switch_model("org/Vision-GGUF", object(), "tester")
+ assert len(recorder.calls) == 1
+ await inference_route._maybe_auto_switch_model("org/Vision-GGUF", object(), "tester")
+ assert len(recorder.calls) == 1
+ (companion / "mmproj-vision-model-F16.gguf").write_bytes(b"GGUF companion")
+ await inference_route._maybe_auto_switch_model("org/Vision-GGUF", object(), "tester")
+ assert len(recorder.calls) == 2
+
+ asyncio.run(run())
+
+
+def test_auto_switch_exact_revision_does_not_widen_companion_roots(tmp_path, monkeypatch):
+ from pathlib import Path
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ (newer / "mmproj-other-model-F16.gguf").write_bytes(b"GGUF companion")
+
+ backend = _FakeBackend("org/Other-GGUF", "Q4_K_M")
+ recorder = _LoadRecorder(backend)
+ _wire(
+ monkeypatch,
+ enabled = True,
+ resolves_to = (str(old), "Q4_K_M", "org/Vision-GGUF"),
+ backend = backend,
+ recorder = recorder,
+ )
+
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ old.name,
+ object(),
+ "tester",
+ )
+ )
+
+ assert len(recorder.calls) == 1
+ assert recorder.calls[0]._gguf_companion_roots == ()
+ assert resolver.local_gguf_companion_roots(str(old), repo_level = False) == ()
+ assert tuple(map(Path, resolver.local_gguf_companion_roots(str(old), repo_level = True))) == (
+ old,
+ newer,
+ )
+
+
+def test_idle_stash_reload_carries_hf_cache_companion_roots(tmp_path, monkeypatch):
+ from pathlib import Path
+ from core.inference import llama_keepwarm as kw
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ (newer / "mmproj-vision-model-F16.gguf").write_bytes(b"GGUF companion")
+ os.utime(old, (1_000, 1_000))
+ os.utime(newer, (2_000, 2_000))
+
+ backend = _FakeBackend(None)
+ recorder = _LoadRecorder(backend)
+ _wire(monkeypatch, enabled = True, resolves_to = None, backend = backend, recorder = recorder)
+ monkeypatch.setattr(kw, "_inflight", 0)
+ monkeypatch.setattr(
+ kw,
+ "_last_unloaded_model",
+ (str(old), "Q4_K_M", "org/Vision-GGUF", (str(old), str(newer))),
+ )
+
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ inference_route._RELOAD_ONLY_MODEL,
+ object(),
+ "tester",
+ require_vision = True,
+ )
+ )
+
+ assert len(recorder.calls) == 1
+ assert tuple(map(Path, recorder.calls[0]._gguf_companion_roots)) == (old, newer)
+
+
+def test_loaded_identity_stashes_only_roots_used_by_the_load():
+ from core.inference import llama_keepwarm as kw
+
+ backend = _FakeBackend("cache/snapshots/weights-revision", "Q4_K_M")
+ backend._openai_advertised_id = "org/Vision-GGUF"
+ backend._openai_gguf_companion_roots = ("weights-revision", "companion-revision")
+
+ assert kw._loaded_identity(backend) == (
+ "cache/snapshots/weights-revision",
+ "Q4_K_M",
+ "org/Vision-GGUF",
+ ("weights-revision", "companion-revision"),
+ )
+ backend._openai_gguf_companion_roots = ()
+ assert kw._loaded_identity(backend) == (
+ "cache/snapshots/weights-revision",
+ "Q4_K_M",
+ "org/Vision-GGUF",
+ )
+
+
+def test_idle_stash_reload_of_manual_snapshot_does_not_add_sibling_roots(tmp_path, monkeypatch):
+ from core.inference import llama_keepwarm as kw
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ newer = repo / "snapshots" / "companion-revision"
+ newer.mkdir(parents = True)
+ (newer / "mmproj-other-model-F16.gguf").write_bytes(b"GGUF companion")
+
+ backend = _FakeBackend(None)
+ recorder = _LoadRecorder(backend)
+ _wire(monkeypatch, enabled = True, resolves_to = None, backend = backend, recorder = recorder)
+ monkeypatch.setattr(kw, "_inflight", 0)
+ monkeypatch.setattr(
+ kw,
+ "_last_unloaded_model",
+ (str(old), "Q4_K_M", "org/Vision-GGUF"),
+ )
+
+ asyncio.run(
+ inference_route._maybe_auto_switch_model(
+ inference_route._RELOAD_ONLY_MODEL,
+ object(),
+ "tester",
+ )
+ )
+
+ assert len(recorder.calls) == 1
+ assert recorder.calls[0]._gguf_companion_roots == ()
+
+
+def test_companion_root_scan_does_not_block_the_event_loop(tmp_path, monkeypatch):
+ import time
+
+ old = tmp_path / "models--org--Vision-GGUF" / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "vision-model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+
+ backend = _FakeBackend("org/Other-GGUF", "Q4_K_M")
+ recorder = _LoadRecorder(backend)
+ _wire(
+ monkeypatch,
+ enabled = True,
+ resolves_to = (str(old), "Q4_K_M", "org/Vision-GGUF"),
+ backend = backend,
+ recorder = recorder,
+ )
+ entered = threading.Event()
+ release = threading.Event()
+
+ def _slow_companion_scan(_load_path, *, repo_level = False):
+ assert repo_level is True
+ entered.set()
+ release.wait(1.0)
+ return ()
+
+ monkeypatch.setattr(resolver, "local_gguf_companion_roots", _slow_companion_scan)
+
+ async def _drive():
+ started = time.monotonic()
+ task = asyncio.create_task(
+ inference_route._maybe_auto_switch_model(
+ "org/Vision-GGUF",
+ object(),
+ "tester",
+ )
+ )
+ assert await asyncio.to_thread(entered.wait, 2.0)
+ loop_was_responsive = time.monotonic() - started < 0.5
+ release.set()
+ await task
+ assert loop_was_responsive
+
+ asyncio.run(_drive())
+ assert len(recorder.calls) == 1
+
+
+def test_inactive_hf_cache_entry_skips_newer_companion_only_snapshot(tmp_path):
+ """Inactive cache rows point at a snapshot but still select complete weights."""
+ import os
+ from pathlib import Path
+ import routes.models as models_route
+
+ repo = tmp_path / "models--org--Repo"
+ old = repo / "snapshots" / "weights-revision"
+ old.mkdir(parents = True)
+ (old / "model-Q4_K_M.gguf").write_bytes(b"GGUF stub")
+
+ newer = repo / "snapshots" / "companion-revision" / "MTP"
+ newer.mkdir(parents = True)
+ (newer / "mtp-model-Q8_0.gguf").write_bytes(b"GGUF companion")
+ os.utime(old, (1_000, 1_000))
+ os.utime(newer.parent, (2_000, 2_000))
+
+ [info] = models_route._scan_hf_cache(
+ tmp_path,
+ active_cache = False,
+ classify_format = False,
+ )
+ assert Path(info.path) == newer.parent.resolve()
+
+ entry = resolver._local_gguf_entry("org/Repo", info)
+
+ assert entry is not None
+ assert Path(entry.load_path) == old
+ assert entry.variants == ("Q4_K_M",)
+
+
+def test_hf_cache_entry_stays_within_the_scanned_case_variant(tmp_path):
+ """A cache row must load from its exact repo directory, not a case-folded peer."""
+ import os
+ from pathlib import Path
+ from types import SimpleNamespace
+
+ scanned_repo = tmp_path / "models--Org--Repo"
+ scanned_snapshot = scanned_repo / "snapshots" / "scanned-revision"
+ scanned_snapshot.mkdir(parents = True)
+ (scanned_snapshot / "model-Q4_K_M.gguf").write_bytes(b"GGUF stub")
+
+ other_repo = tmp_path / "models--org--repo"
+ other_snapshot = other_repo / "snapshots" / "other-revision"
+ other_snapshot.mkdir(parents = True)
+ (other_snapshot / "model-Q8_0.gguf").write_bytes(b"GGUF stub")
+ os.utime(scanned_snapshot, (1_000, 1_000))
+ os.utime(other_snapshot, (2_000, 2_000))
+
+ entry = resolver._local_gguf_entry(
+ "Org/Repo",
+ SimpleNamespace(id = "Org/Repo", path = str(scanned_repo), source = "hf_cache"),
+ )
+
+ assert entry is not None
+ assert Path(entry.load_path) == scanned_snapshot
+ assert entry.variants == ("Q4_K_M",)
+
+
+def test_hf_cache_entry_excludes_torn_quant_from_selected_snapshot(tmp_path):
+ """Only complete quants from the selected snapshot may be advertised as loadable."""
+ from types import SimpleNamespace
+
+ repo = tmp_path / "models--org--Repo"
+ snapshot = repo / "snapshots" / "revision"
+ snapshot.mkdir(parents = True)
+ (snapshot / "model-Q4_K_M.gguf").write_bytes(b"GGUF stub")
+ (snapshot / "model-Q8_0-00001-of-00002.gguf").write_bytes(b"GGUF stub")
+
+ entry = resolver._local_gguf_entry(
+ "org/Repo",
+ SimpleNamespace(id = "org/Repo", path = str(repo), source = "hf_cache"),
+ )
+
+ assert entry is not None
+ assert entry.variants == ("Q4_K_M",)
+
+
+def test_hf_cache_entry_keeps_partial_only_fallback(tmp_path):
+ """A repo with no complete quant keeps the pre-existing fallback listing."""
+ from types import SimpleNamespace
+
+ repo = tmp_path / "models--org--Repo"
+ snapshot = repo / "snapshots" / "revision"
+ snapshot.mkdir(parents = True)
+ (snapshot / "model-Q4_K_M-00001-of-00002.gguf").write_bytes(b"GGUF stub")
+
+ entry = resolver._local_gguf_entry(
+ "org/Repo",
+ SimpleNamespace(id = "org/Repo", path = str(repo), source = "hf_cache"),
+ )
+
+ assert entry is not None
+ assert entry.variants == ("Q4_K_M",)
+
+
+def test_selected_snapshot_preserves_local_variant_labels(tmp_path):
+ """Selected snapshots use picker identities and still accept legacy API pins."""
+ from types import SimpleNamespace
+
+ repo = tmp_path / "models--org--Repo"
+ snapshot = repo / "snapshots" / "revision"
+ snapshot.mkdir(parents = True)
+ (snapshot / "model-small.gguf").write_bytes(b"small")
+ (snapshot / "model-large.gguf").write_bytes(b"larger")
+
+ entry = resolver._local_gguf_entry(
+ "org/Repo",
+ SimpleNamespace(id = "org/Repo", path = str(repo), source = "hf_cache"),
+ )
+
+ assert entry is not None
+ assert set(entry.variants) == {"model-small", "model-large"}
+ for legacy in ("small", "large"):
+ assert resolver._resolve_from_index(
+ f"org/Repo:{legacy}", {"org/repo": entry}, include_companion_scope = True
+ ) == (str(snapshot), f"model-{legacy}", "org/Repo", True)
+
+
+def test_model_dir_with_snapshots_subdir_keeps_root_gguf(tmp_path):
+ """A regular model dir is not an HF cache just because snapshots/ exists."""
+ from types import SimpleNamespace
+
+ nested = tmp_path / "snapshots" / "revision"
+ nested.mkdir(parents = True)
+ (nested / "mmproj-model-F16.gguf").write_bytes(b"GGUF companion")
+ (tmp_path / "model-Q4_K_M.gguf").write_bytes(b"GGUF stub")
+
+ entry = resolver._local_gguf_entry(
+ "custom/model",
+ SimpleNamespace(id = "custom/model", path = str(tmp_path), source = "models_dir"),
+ )
+
+ assert entry is not None
+ assert entry.load_path == str(tmp_path)
+ assert entry.variants == ("Q4_K_M",)
+
+
+def test_hf_cache_entries_do_not_rescan_the_cache_root(monkeypatch, tmp_path):
+ """Each scanned row already owns an exact repo directory under the cache root."""
+ from pathlib import Path
+ from types import SimpleNamespace
+
+ infos = []
+ for index in range(3):
+ repo = tmp_path / f"models--org--Repo-{index}"
+ snapshot = repo / "snapshots" / "revision"
+ snapshot.mkdir(parents = True)
+ (snapshot / "model-Q4_K_M.gguf").write_bytes(b"GGUF stub")
+ infos.append(SimpleNamespace(id = f"org/Repo-{index}", path = str(repo), source = "hf_cache"))
+
+ original_iterdir = Path.iterdir
+ root_scans = 0
+
+ def counting_iterdir(path):
+ nonlocal root_scans
+ if path == tmp_path:
+ root_scans += 1
+ return original_iterdir(path)
+
+ monkeypatch.setattr(Path, "iterdir", counting_iterdir)
+ assert all(resolver._local_gguf_entry(info.id, info) is not None for info in infos)
+ assert root_scans == 0
+
+
# ── review round 5: concurrent-swap, repo-id identity, /v1/models id, gate, 503 ──
@@ -2047,6 +2835,8 @@ def _revision_pair(root, complete: bool):
(old / "model-Q8_0.gguf").write_bytes(b"GGUF stub")
name = "model-Q4_K_M.gguf" if complete else "model-Q4_K_M-00001-of-00003.gguf"
(new / name).write_bytes(b"GGUF stub")
+ os.utime(old, (1_000, 1_000))
+ os.utime(new, (2_000, 2_000))
return old, new
@@ -2719,13 +3509,69 @@ def test_index_advertises_alias_not_filesystem_path(tmp_path, monkeypatch):
resolver._scan = (0.0, {})
# The advertised id is the alias, never the absolute path.
- advertised = sorted({entry.loader_id for entry in resolver._index().values()})
+ index = resolver._index()
+ advertised = sorted({entry.loader_id for entry in index.values()})
assert advertised == ["org/Repo-GGUF"]
+ assert gguf.name.lower() not in index
# But the model is still resolvable by its on-disk path (an indexed alias).
resolver._scan = (0.0, {})
assert resolver.resolve_local_gguf(str(gguf)) is not None
+def test_inactive_cache_revision_aliases_keep_exact_companion_scope(tmp_path, monkeypatch):
+ from types import SimpleNamespace
+
+ import routes.models as models_route
+ from storage import studio_db
+ from utils import hf_cache_settings
+ import utils.paths as paths
+
+ repo = tmp_path / "models--org--Vision-GGUF"
+ selected = repo / "snapshots" / "weights-revision"
+ selected.mkdir(parents = True)
+ (selected / "model-Q4_K_M.gguf").write_bytes(b"GGUF weights")
+ (selected / "model-Q8_0-00001-of-00002.gguf").write_bytes(b"GGUF partial")
+ companion = repo / "snapshots" / "companion-revision" / "MTP"
+ companion.mkdir(parents = True)
+ (companion / "mtp-model-Q8_0.gguf").write_bytes(b"GGUF companion")
+ os.utime(selected, (1_000, 1_000))
+ os.utime(companion.parent, (2_000, 2_000))
+
+ info = SimpleNamespace(
+ id = str(selected),
+ path = str(selected),
+ model_id = "org/Vision-GGUF",
+ display_name = "Vision-GGUF",
+ source = "hf_cache",
+ )
+ monkeypatch.setattr(models_route, "_scan_models_dir", lambda *a, **k: [info])
+ monkeypatch.setattr(models_route, "_scan_hf_cache", lambda *a, **k: [])
+ monkeypatch.setattr(models_route, "_scan_lmstudio_dir", lambda *a, **k: [])
+ monkeypatch.setattr(models_route, "_resolve_hf_cache_dir", lambda: tmp_path / "active")
+ monkeypatch.setattr(models_route, "_is_hidden_model", lambda *a, **k: False)
+ monkeypatch.setattr(hf_cache_settings, "known_hf_hub_caches", lambda: [])
+ monkeypatch.setattr(paths, "legacy_hf_cache_dir", lambda: None)
+ monkeypatch.setattr(paths, "hf_default_cache_dir", lambda: None)
+ monkeypatch.setattr(paths, "lmstudio_model_dirs", lambda: [])
+ monkeypatch.setattr(studio_db, "list_scan_folders", lambda: [])
+
+ index = resolver._build_index()
+
+ repo_entry = index["org/vision-gguf"]
+ display_entry = index["vision-gguf"]
+ path_entry = index[str(selected).lower()]
+ revision_entry = index[selected.name.lower()]
+ assert repo_entry.repo_level_companions is True
+ assert display_entry.repo_level_companions is True
+ assert path_entry.repo_level_companions is False
+ assert revision_entry.repo_level_companions is False
+ assert path_entry.variants == ("Q4_K_M",)
+ assert revision_entry.variants == ("Q4_K_M",)
+ assert {
+ entry.load_path for entry in (repo_entry, display_entry, path_entry, revision_entry)
+ } == {str(selected)}
+
+
def test_build_index_survives_a_failing_scanner(tmp_path, monkeypatch):
# gemini: one bad scanner (e.g. a permission error on ./models) must drop only
# that source, not abort the whole index and lose what the others found.
@@ -5396,7 +6242,7 @@ def test_auto_switch_serializes_across_event_loops(monkeypatch):
backend._openai_advertised_id = None
monkeypatch.setattr(settings, "get_openai_auto_switch_enabled", lambda: True)
- monkeypatch.setattr(resolver, "resolve_local_gguf", lambda m: (m, "Q8_0", m))
+ monkeypatch.setattr(resolver, "resolve_local_gguf", lambda m, **_kw: (m, "Q8_0", m))
monkeypatch.setattr(inference_route, "get_llama_cpp_backend", lambda: backend)
monkeypatch.setattr(inference_route, "_load_model_impl", _slow_load)
monkeypatch.setattr(inference_route, "_auto_switch_waiters", {})
diff --git a/studio/backend/utils/models/model_config.py b/studio/backend/utils/models/model_config.py
index 2bbef0d87f4..cd472f9b370 100644
--- a/studio/backend/utils/models/model_config.py
+++ b/studio/backend/utils/models/model_config.py
@@ -1131,6 +1131,7 @@ def is_vision_model(
revision: Optional[str] = None,
gguf_variant: Optional[str] = None,
require_image: bool = True,
+ gguf_companion_roots: Optional[Tuple[str, ...]] = None,
) -> bool:
"""Detect VLMs via the config architecture (works for fine-tunes); transformers-5.x
models are checked in a .venv_t5/ subprocess. Cached per (model_name, token,
@@ -1154,7 +1155,21 @@ def is_vision_model(
gguf_file = detect_gguf_model(local_path)
if gguf_file:
companion_root = _local_gguf_companion_search_root(local_path, gguf_file)
- mmproj_file = detect_mmproj_file(gguf_file, search_root = companion_root)
+ companion_roots = gguf_companion_roots or (companion_root,)
+ mmproj_file = next(
+ (
+ found
+ for root in companion_roots
+ if (
+ found := detect_mmproj_file(
+ gguf_file,
+ search_root = root,
+ allow_disjoint_search_root = gguf_companion_roots is not None,
+ )
+ )
+ ),
+ None,
+ )
# An audio-only projector serves this model's audio input, never an image.
is_vision = mmproj_file is not None and (
not require_image or mmproj_accepts_image(mmproj_file)
@@ -1930,13 +1945,18 @@ def _local_gguf_load_path(path: Path) -> Path:
return (first or path).absolute()
-def detect_mmproj_file(path: str, search_root: Optional[str] = None) -> Optional[str]:
+def detect_mmproj_file(
+ path: str,
+ search_root: Optional[str] = None,
+ allow_disjoint_search_root: bool = False,
+) -> Optional[str]:
"""Find the mmproj GGUF for a model.
``path``: directory or a .gguf file. ``search_root``: optional ancestor
to also walk (snapshot layouts where the weight is in ``snapshot/BF16/``
- but the projector sits at ``snapshot/``). Returns the projector path or
- ``None``."""
+ but the projector sits at ``snapshot/``). A trusted cache resolver may set
+ ``allow_disjoint_search_root`` for another revision of the same repository.
+ Returns the projector path or ``None``."""
p = Path(path)
start_dir = p.parent if p.is_file() else p
if not start_dir.is_dir():
@@ -1945,6 +1965,7 @@ def detect_mmproj_file(path: str, search_root: Optional[str] = None) -> Optional
# Walk incrementally so a sibling subdir's mmproj cannot leak in.
seen: set[Path] = set()
scan_order: list[Path] = []
+ recursive_root: Optional[Path] = None
def _add(d: Path) -> None:
try:
@@ -1970,24 +1991,33 @@ def detect_mmproj_file(path: str, search_root: Optional[str] = None) -> Optional
try:
root_resolved = Path(search_root).resolve()
start_resolved = start_dir.resolve()
- if root_resolved == start_resolved or (
+ root_contains_start = root_resolved == start_resolved or (
start_resolved.is_relative_to(root_resolved)
if hasattr(start_resolved, "is_relative_to")
else str(start_resolved).startswith(str(root_resolved) + "/")
- ):
+ )
+ if root_contains_start:
cur = start_resolved
while cur != root_resolved and cur.parent != cur:
cur = cur.parent
_add(cur)
if cur == root_resolved:
break
+ elif allow_disjoint_search_root:
+ _add(root_resolved)
+ if allow_disjoint_search_root:
+ recursive_root = root_resolved
except OSError:
pass
candidates: list[Path] = []
seen_resolved: set[Path] = set()
for d in scan_order:
- for f in _iter_gguf_files(d):
+ try:
+ files = list(_iter_gguf_files(d, recursive = d == recursive_root))
+ except OSError:
+ continue
+ for f in files:
try:
resolved = f.resolve()
# Interrupted download: llama-server can't open it and it must not shadow a real projector.
@@ -3946,6 +3976,7 @@ class ModelConfig:
is_lora: bool = False,
gguf_variant: Optional[str] = None,
drafter_accept: Optional[Callable[[str, str, str, str], bool]] = None,
+ gguf_companion_roots: Optional[Tuple[str, ...]] = None,
) -> Optional["ModelConfig"]:
"""Create ModelConfig from a clean model identifier (HF repo or local
path), for FastAPI routes that send sanitized paths.
@@ -3966,6 +3997,9 @@ class ModelConfig:
route rejects only after the read already happened. Left None by
every caller that has no boundary to impose, which sees the same
candidates in the same order as before.
+ gguf_companion_roots: Trusted snapshot directories belonging to the
+ resolver-selected local cache entry. Used only to locate a
+ compatible mmproj without changing the selected main weights.
Returns:
ModelConfig or None if it cannot be created.
@@ -4046,35 +4080,54 @@ class ModelConfig:
candidate, gguf_file, kind, companion_root
)
- mmproj_file = detect_mmproj_file(gguf_file, search_root = companion_root)
+ mmproj_file = next(
+ (
+ found
+ for root in (gguf_companion_roots or (companion_root,))
+ if (
+ found := detect_mmproj_file(
+ gguf_file,
+ search_root = root,
+ allow_disjoint_search_root = gguf_companion_roots is not None,
+ )
+ )
+ ),
+ None,
+ )
if mmproj_file:
gguf_is_vision = True
logger.info(f"Detected mmproj for vision: {mmproj_file}")
elif base_is_vision:
logger.warning(f"Base model is vision but no mmproj file found in {gguf_dir}")
- # Separate MTP drafter sibling (Gemma 4), mirroring mmproj.
- mtp_file = detect_mtp_file(
- gguf_file,
- search_root = companion_root,
- accept = _drafter_accept_for("mtp"),
- )
+ companion_roots = gguf_companion_roots or (companion_root,)
+
+ def _find_drafter(detector, kind: str) -> Optional[str]:
+ return next(
+ (
+ found
+ for root in companion_roots
+ if (
+ found := detector(
+ gguf_file,
+ search_root = root,
+ accept = _drafter_accept_for(kind),
+ )
+ )
+ ),
+ None,
+ )
+
+ # Separate speculative-decoding companions, mirroring mmproj.
+ mtp_file = _find_drafter(detect_mtp_file, "mtp")
if mtp_file:
logger.info(f"Detected MTP drafter: {mtp_file}")
# DSpark and DFlash take the boundary for the same reason, even
# though only the DFlash scan opens a candidate: all three are the
# same discovery, and a kind that skipped the check would hand the
# load route a sidecar it has to reject a second time.
- dspark_file = detect_dspark_file(
- gguf_file,
- search_root = companion_root,
- accept = _drafter_accept_for("dspark"),
- )
- dflash_file = detect_dflash_file(
- gguf_file,
- search_root = companion_root,
- accept = _drafter_accept_for("dflash"),
- )
+ dspark_file = _find_drafter(detect_dspark_file, "dspark")
+ dflash_file = _find_drafter(detect_dflash_file, "dflash")
return cls(
identifier = identifier,