Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
import asyncio
import base64
import copy
import json
import os
import sys
import threading
from types import SimpleNamespace
import numpy as np
import pytest
from fastapi import HTTPException
_backend = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if _backend not in sys.path:
sys.path.insert(0, _backend)
inference_route = pytest.importorskip("routes.inference", reason = "inference stack not installed")
rag_embeddings = pytest.importorskip("core.rag.embeddings", reason = "rag stack not installed")
rag_config = pytest.importorskip("core.rag.config", reason = "rag stack not installed")
MODEL = "unsloth/bge-small-en-v1.5"
IDENTITY = f"sentence-transformers:{MODEL}"
class _Request:
def __init__(self, body):
self._body = body
self.method = "POST"
self.url = SimpleNamespace(path = "/v1/embeddings")
self.state = SimpleNamespace(skip_api_monitor = True)
self.scope = {"type": "http", "path": "/v1/embeddings"}
async def json(self):
return self._body
async def is_disconnected(self):
return False
def _vectors(texts, **_):
return np.array([[1.0, 0.0]] * len(texts), dtype = np.float32)
@pytest.fixture
def studio_embedder(monkeypatch):
async def passthrough(request, current_subject, **_kwargs):
return await request.json()
def no_proxy():
raise AssertionError("the llama-server proxy must not run")
monkeypatch.setattr(inference_route, "_should_validate_before_switch", lambda: False)
monkeypatch.setattr(inference_route, "_auto_switch_from_request_body", passthrough)
monkeypatch.setattr(inference_route, "_cancelable_nonstreaming_client", no_proxy)
monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: MODEL)
monkeypatch.setattr(rag_embeddings, "encode", _vectors)
def _encode_with_identity(
texts,
*,
model_name = None,
normalize = True,
):
return rag_embeddings.encode(texts, model_name = model_name, normalize = normalize), IDENTITY
monkeypatch.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)
monkeypatch.setattr(rag_embeddings, "token_counter", lambda model_name = None: len)
monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: None)
monkeypatch.setattr(rag_embeddings, "dim", lambda model_name = None: 2)
return monkeypatch
def _call(body):
response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
assert response.status_code == 200
import json
return json.loads(response.body)
def _http_error(body):
from fastapi import HTTPException
with pytest.raises(HTTPException) as exc:
asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
return exc.value
def test_nothing_loaded_serves_from_the_studio_embedder(studio_embedder):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
payload = _call({"input": ["alpha", "beta"], "model": "text-embedding-3-small"})
assert payload["object"] == "list"
assert payload["model"] == IDENTITY
assert [row["index"] for row in payload["data"]] == [0, 1]
assert payload["data"][0]["embedding"] == [1.0, 0.0]
assert payload["usage"] == {"prompt_tokens": 9, "total_tokens": 9}
def test_chat_model_loaded_serves_from_the_studio_embedder(studio_embedder):
started = []
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
)
studio_embedder.setattr(
inference_route, "_direct_llama_request_started", lambda: started.append(1)
)
payload = _call({"input": "alpha"})
assert payload["model"] == IDENTITY
assert started == []
def test_resident_embedding_gguf_still_uses_the_proxy(studio_embedder):
import httpx
class _Client:
async def post(self, *_args, **_kwargs):
return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})
async def aclose(self):
return None
def boom(*_args, **_kwargs):
raise AssertionError("the studio embedder must not run")
studio_embedder.setattr(rag_embeddings, "encode", boom)
studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_embedding_gguf = True,
base_url = "http://llama.test",
context_length = 512,
model_identifier = "org/E-GGUF",
),
)
payload = _call({"input": "alpha", "model": "org/E-GGUF"})
assert payload == {"data": [{"embedding": [0.5]}]}
@pytest.mark.parametrize("encoding_format", ["float", "base64"])
@pytest.mark.parametrize("dimensions, status", [(2, 200), (256, 400)])
def test_resident_embedding_gguf_checks_dimensions_against_its_width(
studio_embedder, encoding_format, dimensions, status
):
import httpx
forwarded = []
class _Client:
async def post(
self,
*_args,
json = None,
**_kwargs,
):
forwarded.append(json)
vector = [0.5, 0.5]
if encoding_format == "base64":
vector = base64.b64encode(np.asarray(vector, dtype = np.float32).tobytes()).decode()
return httpx.Response(200, json = {"data": [{"embedding": vector}]})
async def aclose(self):
return None
studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_embedding_gguf = True,
base_url = "http://llama.test",
context_length = 512,
model_identifier = "org/E-GGUF",
),
)
body = {
"input": "alpha",
"model": "org/E-GGUF",
"encoding_format": encoding_format,
"dimensions": dimensions,
}
if status == 200:
_call(body)
else:
error = _http_error(body)
assert error.status_code == 400
assert "'dimensions' is not supported" in error.detail
assert "dimensions" not in forwarded[0]
def test_base64_encoding_format(studio_embedder):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
payload = _call({"input": "alpha", "encoding_format": "base64"})
raw = base64.b64decode(payload["data"][0]["embedding"])
assert np.frombuffer(raw, dtype = np.float32).tolist() == [1.0, 0.0]
@pytest.mark.parametrize(
"body",
[
{"input": ""},
{"input": []},
{"input": [[1, 2, 3]]},
{"input": ["alpha", ""]},
{"input": "alpha", "encoding_format": "int8"},
{"input": "alpha", "encoding_format": ""},
{"input": "alpha", "encoding_format": False},
{"input": "alpha", "encoding_format": 0},
{"input": "alpha", "dimensions": 999},
],
)
def test_invalid_requests_are_400(studio_embedder, body):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
assert _http_error(body).status_code == 400
def test_over_length_input_is_truncated_by_default(studio_embedder):
encoded = []
def _encode(texts, **_):
encoded.append(list(texts))
return _vectors(texts)
studio_embedder.setattr(rag_embeddings, "encode", _encode)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
payload = _call({"input": ["ab", "abcd"]})
assert encoded == [["ab", "abc"]]
assert len(payload["data"]) == 2
assert payload["usage"]["prompt_tokens"] == 5
def test_over_length_input_is_truncated_with_batched_counts(studio_embedder):
encoded, singles, batches = [], [], []
def _count(text):
singles.append(text)
return len(text)
def _many(texts):
batches.append(list(texts))
return [len(t) for t in texts]
_count.many = _many
studio_embedder.setattr(
rag_embeddings, "encode", lambda texts, **_: encoded.append(list(texts)) or _vectors(texts)
)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "token_counter", lambda model_name = None: _count)
studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
payload = _call({"input": ["ab", "abcd", "abcdefgh"]})
assert encoded == [["ab", "abc", "abc"]]
assert payload["usage"]["prompt_tokens"] == 8
assert singles == [] and batches[0] == ["ab", "abcd", "abcdefgh"]
def test_over_length_input_is_rejected_when_truncate_is_false(studio_embedder):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
error = _http_error({"input": ["ab", "abcd"], "truncate": False})
assert error.status_code == 400
assert "3-token limit" in error.detail
def test_non_boolean_truncate_is_400(studio_embedder):
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
error = _http_error({"input": "ab", "truncate": "yes"})
assert error.status_code == 400
assert "'truncate'" in error.detail
_CLS, _SEP = 101, 102
def _llama_overflow_server(
ubatch,
n_ctx,
can_split = False,
):
import httpx
embedded, tokenized = [], []
def _tokens(item):
if isinstance(item, list):
return item
return [_CLS, *(len(word) for word in item.split()), _SEP]
class _Client:
async def post(
self,
url,
*_args,
json = None,
**_kwargs,
):
if url.endswith("/tokenize"):
tokenized.append(json["content"])
tokens = _tokens(json["content"])
return httpx.Response(
200, json = {"tokens": tokens if json["add_special"] else tokens[1:-1]}
)
# The route reuses the dict.
embedded.append(copy.deepcopy(json))
items = [json["input"]] if isinstance(json["input"], str) else json["input"]
for item in items:
n = len(_tokens(item))
if can_split and n >= n_ctx:
return httpx.Response(
400,
json = {
"error": {
"message": f"request ({n} tokens) exceeds the available context "
f"size ({n_ctx} tokens), try increasing it"
}
},
)
if can_split:
continue
if n > ubatch:
return httpx.Response(
500,
json = {
"error": {
"message": f"input ({n} tokens) is too large to process. increase "
f"the physical batch size (current batch size: {ubatch})"
}
},
)
if n > n_ctx:
return httpx.Response(
400,
json = {
"error": {
"message": f"input ({n} tokens) is larger than the max context "
f"size ({n_ctx} tokens). skipping"
}
},
)
return httpx.Response(200, json = {"data": [{"embedding": [0.5]} for _ in items]})
async def aclose(self):
return None
return _Client, embedded, tokenized
_OVERFLOW_LIMITS = [(4, 512), (512, 4)]
def _resident_embedding_gguf(studio_embedder, client):
studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", client)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_embedding_gguf = True,
base_url = "http://llama.test",
context_length = 512,
model_identifier = "org/E-GGUF",
),
)
@pytest.mark.parametrize("ubatch, n_ctx", _OVERFLOW_LIMITS)
def test_resident_embedding_gguf_truncates_an_oversized_input(studio_embedder, ubatch, n_ctx):
client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)
_resident_embedding_gguf(studio_embedder, client)
payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})
assert len(payload["data"]) == 2
assert len(embedded) == 2
assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]
assert "truncate" not in embedded[0]
def test_resident_embedding_gguf_truncates_for_a_splittable_slot(studio_embedder):
client, embedded, _ = _llama_overflow_server(512, 5, can_split = True)
_resident_embedding_gguf(studio_embedder, client)
payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})
assert len(payload["data"]) == 2
assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]
@pytest.mark.parametrize("ubatch, n_ctx", _OVERFLOW_LIMITS)
def test_resident_embedding_gguf_keeps_the_error_when_truncate_is_false(
studio_embedder, ubatch, n_ctx
):
client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)
_resident_embedding_gguf(studio_embedder, client)
response = asyncio.run(
inference_route.openai_embeddings(
_Request({"input": "aa bbb c dddd e", "model": "org/E-GGUF", "truncate": False}),
"tester",
)
)
assert response.status_code == (500 if ubatch < n_ctx else 400)
assert len(embedded) == 1
assert "truncate" not in embedded[0]
def test_embedder_failure_is_502(studio_embedder):
def boom(*_args, **_kwargs):
raise RuntimeError("llama-server embedder POST /v1/embeddings -> 500")
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "encode", boom)
assert _http_error({"input": "alpha"}).status_code == 502
def test_identity_redaction_leaves_the_backend_tag_alone(tmp_path, monkeypatch):
"""Redacting a local model name must not rewrite the backend tag, e.g. sentence-transformers."""
from core.rag import config as rag_config
(tmp_path / "transformers").mkdir()
monkeypatch.chdir(tmp_path)
monkeypatch.setattr(
inference_route,
"_public_embedding_name",
lambda name: "transformers-deadbeef" if name == "transformers" else name,
)
monkeypatch.setattr(
rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "transformers-GGUF"
)
public = inference_route._public_embedding_identity(
"sentence-transformers:transformers", "transformers", "transformers-deadbeef"
)
assert public == "sentence-transformers:transformers-deadbeef"
assert rag_config.embedding_identity_model(public) == "transformers-deadbeef"
def test_identity_redaction_covers_the_gguf_repo_segment(monkeypatch):
from core.rag import config as rag_config
monkeypatch.setattr(
rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "/models/bge-GGUF"
)
monkeypatch.setattr(
inference_route,
"_public_embedding_name",
lambda name: {"/models/bge": "bge-1111", "/models/bge-GGUF": "bge-GGUF-2222"}.get(
name, name
),
)
public = inference_route._public_embedding_identity(
"llama-server:%2Fmodels%2Fbge:%2Fmodels%2Fbge-GGUF".replace("%2F", "/"),
"/models/bge",
"bge-1111",
)
assert public == "llama-server:bge-1111:bge-GGUF-2222"
def test_identity_redaction_covers_a_local_repo_under_a_hub_model(monkeypatch):
"""Redact a local path in RAG_EMBED_GGUF_REPO even when the model segment is a plain hub id."""
from core.rag import config as rag_config
monkeypatch.setattr(
rag_config,
"effective_gguf_repo_for_embedding_model",
lambda model: "/srv/private/embed.gguf",
)
monkeypatch.setattr(
inference_route,
"_public_embedding_name",
lambda name: "embed.gguf-3333" if name == "/srv/private/embed.gguf" else name,
)
public = inference_route._public_embedding_identity(
f"llama-server:{MODEL}:/srv/private/embed.gguf", MODEL, MODEL
)
assert public == f"llama-server:{MODEL}:embed.gguf-3333"
assert "/srv/private" not in public
@pytest.mark.parametrize("named", [123, 1.5, True, ["a"], {"a": 1}])
def test_a_non_string_model_is_not_a_server_error(studio_embedder, named):
"""Both pre-switch readers decline a non-string selector, so the empty-slot branch sees it."""
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
payload = _call({"input": "alpha", "model": named})
assert payload["model"] == IDENTITY
@pytest.mark.parametrize(
"exc,fragment",
[
(
rag_embeddings.EmbeddingModelDownloadRequiredError(
"Embedding model 'unsloth/bge-small-en-v1.5' is not downloaded yet. "
"Finish its Settings download before indexing documents."
),
"not downloaded yet",
),
(
rag_embeddings.UnsafeEmbeddingModelError(
"Embedding model 'evil/model' requires remote code execution."
),
"remote code",
),
],
)
def test_actionable_embedder_errors_keep_their_message(studio_embedder, exc, fragment):
"""The caller can fix these two, but only if it is told what they are.
Picking a model in Settings and querying before its download finishes is the common
one, and it reaches this route through `unsloth start openclaw`: the generated
openclaw.json points memory search here, so this detail string is the entire
diagnosis the user sees. Routed through _friendly_error -- written for llama-server
transport faults -- it became a 502 "An internal error occurred", which says nothing
and suggests the wrong owner.
"""
def boom(*_args, **_kwargs):
raise exc
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "encode", boom)
error = _http_error({"input": "alpha"})
assert error.status_code == 409
assert fragment in error.detail
assert "An internal error occurred" not in error.detail
def test_pending_gguf_download_is_classified_like_the_st_one(tmp_path, monkeypatch):
"""A pending GGUF download must be typed like the sentence-transformers one, so both reach the 409."""
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
monkeypatch.setattr(
embed_llama_server.config,
"effective_gguf_repo_for_embedding_model",
lambda model: "unsloth/bge-small-en-v1.5-GGUF",
)
monkeypatch.setattr(backend, "_resolve_local_gguf", lambda model: None)
monkeypatch.setattr(backend, "_planned_family_path", lambda model, repo: None)
monkeypatch.setattr(backend, "_resolve_cached_gguf", lambda *a, **k: None)
import utils.embedding_model_settings as ems
monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)
with pytest.raises(rag_embeddings.EmbeddingModelDownloadRequiredError) as excinfo:
backend._resolve_model_path("unsloth/bge-small-en-v1.5")
assert "not downloaded yet" in str(excinfo.value)
def test_actionable_errors_do_not_leak_a_local_path(studio_embedder, monkeypatch):
"""Pass-through error text must not expose a local path, since the route hashes labels elsewhere."""
studio_embedder.setattr(
rag_config, "effective_embedding_model", lambda: "/srv/models/bge-small"
)
def boom(*_args, **_kwargs):
raise rag_embeddings.UnsafeEmbeddingModelError(
"Embedding model '/srv/models/bge-small' is flagged as unsafe by Hugging Face's "
"security scan; refusing to load. Set a different RAG embedding model."
)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "encode", boom)
error = _http_error({"input": "alpha"})
assert error.status_code == 409
assert "/srv/models" not in error.detail
assert "flagged as unsafe" in error.detail
assert "bge-small-" in error.detail
def test_guard_model_security_names_the_configured_model_not_the_snapshot(monkeypatch):
"""The scan needs the snapshot path; the message must not carry it."""
monkeypatch.setattr(rag_embeddings, "_ambient_hf_token", lambda: None)
import utils.security as security
monkeypatch.setattr(
security,
"evaluate_file_security",
lambda *a, **k: SimpleNamespace(blocked = True),
)
monkeypatch.setattr(security, "security_load_subdirs", lambda *a, **k: ())
monkeypatch.setattr(rag_embeddings, "_st_module_subdirs", lambda *a, **k: ())
with pytest.raises(rag_embeddings.UnsafeEmbeddingModelError) as excinfo:
rag_embeddings._guard_model_security(
"/home/me/.cache/huggingface/hub/models--org--bge/snapshots/abc123",
display = "org/bge",
)
assert "org/bge" in str(excinfo.value)
assert ".cache/huggingface" not in str(excinfo.value)
def test_studio_embedder_requests_are_admission_limited(studio_embedder):
lock = threading.Lock()
gate = threading.Event()
active = {"now": 0, "peak": 0}
def encode(texts, **_kwargs):
with lock:
active["now"] += 1
active["peak"] = max(active["peak"], active["now"])
gate.wait(timeout = 5)
with lock:
active["now"] -= 1
return _vectors(texts)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "encode", encode)
async def run():
tasks = [
asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
for _ in range(inference_route._STUDIO_EMBED_CONCURRENCY + 2)
]
for _ in range(500):
await asyncio.sleep(0.01)
if active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY:
break
assert active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY
gate.set()
responses = await asyncio.gather(*tasks)
assert all(response.status_code == 200 for response in responses)
asyncio.run(run())
assert active["peak"] == inference_route._STUDIO_EMBED_CONCURRENCY
def test_studio_fallback_untracks_the_request_from_the_llama_slot(studio_embedder):
# A 2xx reaching _finish would claim a llama slot the studio embedder never used, so untrack first.
from core.inference import llama_keepwarm as kw
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
)
request = _Request({"input": "alpha"})
before = kw._inflight
kw._inflight = before + 1
try:
response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert response.status_code == 200
assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is True
assert kw._inflight == before
finally:
kw._inflight = before
def test_resident_embedding_gguf_still_claims_the_slot(studio_embedder):
import httpx
class _Client:
async def post(self, *_args, **_kwargs):
return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})
async def aclose(self):
return None
from core.inference import llama_keepwarm as kw
studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_embedding_gguf = True,
base_url = "http://llama.test",
context_length = 512,
model_identifier = "org/E-GGUF",
),
)
request = _Request({"input": "alpha", "model": "org/E-GGUF"})
asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is None
def test_context_gauge_is_not_pinned_by_a_batch(studio_embedder):
# api_monitor divides summed prompt_tokens by this, so per-text limit overstates context use.
seen = []
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 8)
studio_embedder.setattr(
inference_route,
"_monitor_openai_chunk",
lambda monitor_id, payload, context_length = None, **_: seen.append(context_length),
)
request = _Request({"input": ["alpha", "beta", "gamma"]})
request.state.skip_api_monitor = False
asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert seen == [None]
seen.clear()
request = _Request({"input": "alpha"})
request.state.skip_api_monitor = False
asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert seen == [8]
def test_cancelled_requests_do_not_leak_admission_permits(studio_embedder):
# to_thread cannot cancel the worker, so permits must outlive the awaiting task's cancellation.
lock = threading.Lock()
gate = threading.Event()
active = {"now": 0, "peak": 0}
def encode(texts, **_kwargs):
with lock:
active["now"] += 1
active["peak"] = max(active["peak"], active["now"])
gate.wait(timeout = 5)
with lock:
active["now"] -= 1
return _vectors(texts)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "encode", encode)
cap = inference_route._STUDIO_EMBED_CONCURRENCY
async def run():
first = [
asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
for _ in range(cap)
]
for _ in range(500):
await asyncio.sleep(0.01)
if active["now"] == cap:
break
assert active["now"] == cap
for task in first:
task.cancel()
await asyncio.gather(*first, return_exceptions = True)
second = [
asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))
for _ in range(cap)
]
await asyncio.sleep(0.3)
assert active["peak"] == cap, f"admission cap exceeded: peak={active['peak']}"
gate.set()
await asyncio.gather(*second, return_exceptions = True)
asyncio.run(run())
assert active["peak"] == cap
def test_reported_model_follows_the_backend_that_made_the_vectors(studio_embedder):
# ST failure swaps to llama-server (different space); the response must name the real space.
llama_identity = f"llama-server:{MODEL}:unsloth/bge-small-en-v1.5-GGUF"
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(
rag_embeddings,
"encode_with_identity",
lambda texts, **_kwargs: (_vectors(texts), llama_identity),
)
payload = _call({"input": "alpha"})
assert payload["model"] == llama_identity
def test_embedding_helpers_are_pinned_to_the_captured_model(studio_embedder):
# A Settings change while queued must not mix one model's limit/dim with another's vectors.
seen = {"max_tokens": [], "token_counter": [], "dim": [], "encode": []}
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(
rag_embeddings,
"max_tokens",
lambda model_name = None: (seen["max_tokens"].append(model_name), None)[1],
)
studio_embedder.setattr(
rag_embeddings,
"token_counter",
lambda model_name = None: (seen["token_counter"].append(model_name), len)[1],
)
studio_embedder.setattr(
rag_embeddings,
"dim",
lambda model_name = None: (seen["dim"].append(model_name), 2)[1],
)
studio_embedder.setattr(
rag_embeddings,
"encode_with_identity",
lambda texts, **kw: (
seen["encode"].append(kw.get("model_name")),
(_vectors(texts), IDENTITY),
)[1],
)
_call({"input": "alpha", "dimensions": 2})
assert seen == {
"max_tokens": [MODEL],
"token_counter": [MODEL],
"dim": [MODEL],
"encode": [MODEL],
}
def test_studio_fallback_releases_the_preview_busy_guard(studio_embedder):
# Admission precedes routing and preview reads the admitted tally, so slow encodes keep 503ing swaps.
from core.inference import llama_keepwarm as kw
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
)
request = _Request({"input": "alpha"})
# Relative: the tally is module state shared with the suite.
before = kw.other_admitted_inference_count()
kw.note_admitted_inference(request.scope)
assert kw.other_admitted_inference_count() == before + 1
assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is True
try:
asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert kw.other_admitted_inference_count() == before
assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is None
finally:
kw._admitted_inference = before
def test_cancelled_request_closes_its_monitor_row(studio_embedder):
# Running rows dodge retention trimming, so a request cancelled while queued closes its own row.
closed = []
gate = threading.Event()
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(inference_route.api_monitor, "start", lambda **_kwargs: "entry-1")
studio_embedder.setattr(
inference_route.api_monitor,
"finish",
lambda entry_id, *args, **_kw: closed.append((entry_id, args[0] if args else None)),
)
studio_embedder.setattr(
rag_embeddings,
"encode_with_identity",
lambda texts, **_kw: (gate.wait(timeout = 5), (_vectors(texts), IDENTITY))[1],
)
async def run():
request = _Request({"input": "alpha"})
request.state.skip_api_monitor = False
task = asyncio.create_task(inference_route.openai_embeddings(request, "tester"))
await asyncio.sleep(0.1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
gate.set()
asyncio.run(run())
assert closed == [("entry-1", "cancelled")]
def _identity_names(monkeypatch):
monkeypatch.setattr(
rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
)
monkeypatch.setattr(rag_embeddings, "embedding_identity", lambda model_name = None: IDENTITY)
@pytest.mark.parametrize("requested", [MODEL, f"{MODEL}-GGUF", IDENTITY, MODEL.upper()])
def test_naming_the_configured_embedder_skips_the_chat_slot_check(studio_embedder, requested):
from fastapi import HTTPException
async def reject(request, current_subject, **_kwargs):
raise HTTPException(status_code = 404, detail = "model_not_found")
_identity_names(studio_embedder)
studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", reject)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
)
payload = _call({"input": "alpha", "model": requested})
assert payload["model"] == IDENTITY
assert payload["data"][0]["embedding"] == [1.0, 0.0]
def test_other_model_names_still_run_auto_switch(studio_embedder):
seen = []
async def record(request, current_subject, **_kwargs):
seen.append(current_subject)
return await request.json()
_identity_names(studio_embedder)
studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
_call({"input": "alpha", "model": "text-embedding-3-small"})
assert seen == ["tester"]
def test_st_max_tokens_leaves_room_for_the_special_tokens(monkeypatch):
model = SimpleNamespace(
max_seq_length = 512,
tokenizer = SimpleNamespace(num_special_tokens_to_add = lambda: 2),
)
monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)
assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 510
def _gguf(tmp_path, entries):
import io
import struct
buf = io.BytesIO()
buf.write(b"GGUF" + struct.pack("<IQQ", 3, 0, len(entries)))
for key, vtype, value in entries:
raw = key.encode()
buf.write(struct.pack("<Q", len(raw)) + raw + struct.pack("<I", vtype))
if vtype == 8:
raw = value.encode()
buf.write(struct.pack("<Q", len(raw)) + raw)
elif vtype == 9:
elem_type, items = value
buf.write(struct.pack("<IQ", elem_type, len(items)))
for item in items:
raw = item.encode()
buf.write(struct.pack("<Q", len(raw)) + raw)
elif vtype == 4:
buf.write(struct.pack("<I", value))
elif vtype == 10:
buf.write(struct.pack("<Q", value))
path = tmp_path / "embedder.gguf"
path.write_bytes(buf.getvalue())
return str(path)
def test_gguf_context_length_is_read_from_the_header(tmp_path):
from core.rag import embed_llama_server
path = _gguf(
tmp_path,
[
("general.architecture", 8, "bert"),
("tokenizer.ggml.tokens", 9, (8, ["[PAD]", "[CLS]", "hello"])),
("llama.context_length", 4, 4096),
("bert.context_length", 10, 512),
],
)
assert embed_llama_server._gguf_context_length(path) == 512
assert embed_llama_server._gguf_context_length(str(tmp_path / "missing.gguf")) is None
def test_llama_max_tokens_comes_from_the_gguf_minus_its_special_tokens(tmp_path, monkeypatch):
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
backend._model_path = _gguf(
tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
)
monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
# Larger than the header, so the GGUF is still what binds; confirmed, so it caches.
monkeypatch.setattr(backend, "_server_context", lambda: 4096)
monkeypatch.setattr(backend, "_server_batch", lambda: 4096)
posts = []
def post(
path,
payload,
model_name = None,
):
posts.append((path, payload))
return {"tokens": [101, 102]}
monkeypatch.setattr(backend, "_post", post)
assert backend.max_tokens() == 510
assert backend.max_tokens() == 510
assert posts == [("/tokenize", {"content": "", "add_special": True})]
backend._adopt_model_path(backend._model_path, "unsloth/other-GGUF")
assert backend._max_tokens is None
def test_llama_max_tokens_is_dropped_when_the_binary_is_swapped(tmp_path, monkeypatch):
"""The token limit depends on the binary's n_ctx and n_ubatch, so a binary swap must drop it."""
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
backend._model_path = _gguf(
tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
)
backend._model_repo = "unsloth/bge-small-en-v1.5-GGUF"
backend._max_tokens = 510
backend._binary = "/opt/old/llama-server"
backend._binary_path_revision = 1
monkeypatch.setattr(backend, "_process_alive", lambda: True)
monkeypatch.setattr(backend, "_kill_process", lambda: None)
monkeypatch.setattr(backend, "_spawn", lambda model_name = None: None)
monkeypatch.setattr(
embed_llama_server.config,
"effective_gguf_repo_for_embedding_model",
lambda model: "unsloth/bge-small-en-v1.5-GGUF",
)
monkeypatch.setattr(
embed_llama_server.config,
"effective_embedding_model",
lambda: "unsloth/bge-small-en-v1.5",
)
import utils.llama_cpp_path_settings as path_settings
monkeypatch.setattr(path_settings, "custom_llama_cpp_path_revision", lambda: 2)
backend._ensure_ready()
assert backend._max_tokens is None
assert backend._model_path is not None
def test_st_max_tokens_reserves_the_default_prompt(monkeypatch):
tokenizer = SimpleNamespace(
num_special_tokens_to_add = lambda: 2,
encode = lambda text, add_special_tokens = False: text.split(),
)
model = SimpleNamespace(
max_seq_length = 512,
tokenizer = tokenizer,
prompts = {"query": "Represent this sentence for retrieval:"},
default_prompt_name = "query",
)
monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)
assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 512 - 2 - 5
def test_llama_max_tokens_is_capped_by_the_running_context(tmp_path, monkeypatch):
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
backend._model_path = _gguf(
tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
)
monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
monkeypatch.setattr(backend, "_server_context", lambda: 256)
monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
assert backend.max_tokens() == 254
def test_llama_max_tokens_is_capped_by_the_ubatch_we_launched_with(tmp_path, monkeypatch):
"""/props has no batch field, so the launch ubatch caps the limit; past it llama-server 500s."""
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
backend._model_path = _gguf(
tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
)
monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
monkeypatch.setattr(backend, "_server_context", lambda: 8192)
monkeypatch.setattr(
backend, "_server_props", lambda: {"default_generation_settings": {"n_ctx": 8192}}
)
monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
assert backend.max_tokens() == embed_llama_server._UBATCH_SIZE - 2
assert "-ub" in backend._build_cmd("llama-server", "m.gguf", 1, use_gpu = False)
def test_only_a_smaller_reported_ubatch_lowers_the_launch_value(monkeypatch):
"""n_batch is the logical batch; llama.cpp takes min(n_batch, n_ubatch or n_batch), so
against the -ub this backend launches with it is never the physical limit."""
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
ub = embed_llama_server._UBATCH_SIZE
monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub // 2})
assert backend._server_batch() == ub // 2
monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub * 4})
assert backend._server_batch() == ub
monkeypatch.setattr(backend, "_server_props", lambda: {"n_batch": ub * 4})
assert backend._server_batch() == ub
def test_a_stale_tagged_identity_is_refused_not_answered(studio_embedder):
"""The current space is llama-server; the client resubmits the sentence-transformers tag."""
from fastapi import HTTPException
studio_embedder.setattr(
rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
)
studio_embedder.setattr(
rag_embeddings, "embedding_identity", lambda model_name = None: f"llama-server:{MODEL}"
)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
# Warm index needed: cold, any slashed name counts as decisive.
from core.inference import local_model_resolver
studio_embedder.setattr(local_model_resolver, "index_is_built", lambda: True)
studio_embedder.setattr(local_model_resolver, "resolve_local_gguf", lambda *a, **k: None)
stale = IDENTITY
assert inference_route._names_studio_embedder(stale) is None
assert inference_route._reference_is_decisive(stale) is True
with pytest.raises(HTTPException) as exc:
asyncio.run(
inference_route.openai_embeddings(
_Request({"input": "alpha", "model": stale}), "tester"
)
)
assert exc.value.status_code == 503
@pytest.mark.parametrize(
("body", "switched"),
[
({"input": [""], "model": "org/chat-GGUF"}, False),
({"input": ["alpha", 7], "model": "org/chat-GGUF"}, False),
({"input": "alpha", "encoding_format": "int8", "model": "org/chat-GGUF"}, False),
({"input": [[1, 2, 3]], "model": "org/chat-GGUF"}, True),
],
)
def test_invalid_fallback_input_is_rejected_before_the_switch(studio_embedder, body, switched):
seen = []
async def record(request, current_subject, **_kwargs):
seen.append(current_subject)
return await request.json()
studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)
studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
)
assert _http_error(body).status_code == 400
assert bool(seen) is switched
def test_local_path_models_are_not_exposed(studio_embedder, tmp_path):
from core.rag.config import _escape_identity_segment
model_dir = str(tmp_path / "bge")
studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
studio_embedder.setattr(
rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
)
identity = f"sentence-transformers:{_escape_identity_segment(model_dir)}"
studio_embedder.setattr(
rag_embeddings,
"encode_with_identity",
lambda texts, **_kwargs: (_vectors(texts), identity),
)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
reported = _call({"input": "alpha"})["model"]
assert reported.startswith("sentence-transformers:bge-") and str(tmp_path) not in reported
studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
error = _http_error({"input": "alphabet", "truncate": False})
assert error.status_code == 400
assert "bge" in error.detail and str(tmp_path) not in error.detail
@pytest.mark.parametrize("answers", [True, False])
def test_resident_embedding_gguf_answering_the_name_keeps_the_proxy(studio_embedder, answers):
import httpx
class _Client:
async def post(self, *_args, **_kwargs):
return httpx.Response(200, json = {"data": [{"embedding": [0.5]}], "model": "proxy"})
async def aclose(self):
return None
_identity_names(studio_embedder)
studio_embedder.setattr(inference_route, "_loaded_satisfies", lambda requested: answers)
studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(
is_loaded = True,
is_embedding_gguf = True,
base_url = "http://llama.test",
context_length = 512,
model_identifier = f"{MODEL}-GGUF",
),
)
payload = _call({"input": "alpha", "model": MODEL})
assert payload["model"] == ("proxy" if answers else IDENTITY)
def test_public_embedding_name_hides_any_local_path_shape():
public = inference_route._public_embedding_name
windows = public("C:\\models\\private\\bge")
assert windows.startswith("bge-") and len(windows) == len("bge-") + 8
assert "models" not in windows and "private" not in windows
assert public("/tmp/models/bge/").startswith("bge-")
assert public("/checkpoints/run-a/model") != public("/checkpoints/run-b/model")
assert public("unsloth/bge-small-en-v1.5") == "unsloth/bge-small-en-v1.5"
def test_flat_token_array_passes_the_pre_switch_check(studio_embedder):
seen = []
async def record(request, current_subject, **_kwargs):
seen.append(current_subject)
return await request.json()
studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)
studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
studio_embedder.setattr(
inference_route,
"get_llama_cpp_backend",
lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
)
assert _http_error({"input": [1, 2, 3]}).status_code == 400
assert seen == ["tester"]
def test_local_path_aliases_match_case_exactly(studio_embedder, tmp_path):
model_dir = str(tmp_path / "Foo")
studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
studio_embedder.setattr(
rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
)
assert inference_route._names_studio_embedder(model_dir)
assert not inference_route._names_studio_embedder(str(tmp_path / "foo"))
# An identity escapes the colons in its segments (a Windows path has one after the drive).
segment = rag_config._escape_identity_segment(model_dir)
assert inference_route._names_studio_embedder(f"sentence-transformers:{segment}")
def test_untagged_aliases_never_probe_the_backend(studio_embedder):
def boom(*_args, **_kwargs):
raise AssertionError("embedding_identity must not run for untagged names")
_identity_names(studio_embedder)
studio_embedder.setattr(rag_embeddings, "embedding_identity", boom)
assert inference_route._names_studio_embedder(MODEL.upper())
assert not inference_route._names_studio_embedder("text-embedding-3-small")
def test_tagged_identity_must_match_the_current_space(studio_embedder):
_identity_names(studio_embedder)
assert inference_route._names_studio_embedder(IDENTITY) == MODEL
stale = f"llama-server:{MODEL}:{MODEL}-GGUF"
assert inference_route._names_studio_embedder(stale) is None
def test_disconnected_client_leaves_the_queue_without_embedding(studio_embedder):
lock = threading.Lock()
gate = threading.Event()
calls = {"n": 0}
def encode(texts, **_kwargs):
with lock:
calls["n"] += 1
gate.wait(timeout = 5)
return _vectors(texts)
class _Gone(_Request):
async def is_disconnected(self):
return True
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "encode", encode)
cap = inference_route._STUDIO_EMBED_CONCURRENCY
async def run():
blockers = [
asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
for _ in range(cap)
]
for _ in range(500):
await asyncio.sleep(0.01)
if calls["n"] == cap:
break
with pytest.raises(HTTPException) as exc:
await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")
assert exc.value.status_code == 499
assert calls["n"] == cap
gate.set()
await asyncio.gather(*blockers)
response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")
assert response.status_code == 200
assert calls["n"] == cap + 1
asyncio.run(run())
def test_batch_cap_applies_only_to_the_studio_fallback():
from fastapi import HTTPException
body = {"input": ["x"] * (inference_route._STUDIO_EMBED_MAX_INPUTS + 1)}
assert len(inference_route._embeddings_items(body, tokens_ok = True)) == len(body["input"])
with pytest.raises(HTTPException) as exc:
inference_route._embeddings_texts(body)
assert exc.value.status_code == 400
def test_a_decisively_named_model_is_refused_when_nothing_is_loaded(studio_embedder):
# A reference clearly meant for this server 404s rather than being served by other weights.
_identity_names(studio_embedder)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(inference_route, "_reference_is_decisive", lambda name: True)
error = _http_error({"input": "alpha", "model": "unsloth/embeddinggemma-300m-GGUF:Q8_0"})
assert error.status_code in (404, 503)
def test_a_foreign_model_name_still_reaches_the_studio_embedder(studio_embedder):
# The other half: a vendor id is no evidence, so it keeps falling through instead of 404ing.
_identity_names(studio_embedder)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
assert _call({"input": "alpha", "model": "text-embedding-3-small"})["model"] == IDENTITY
def test_reference_is_decisive_only_on_positive_evidence(monkeypatch):
from core.inference import local_model_resolver as _resolver
monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)
# An explicit quant label is evidence no foreign id carries.
assert inference_route._reference_is_decisive("unsloth/gemma-3-270m-it-GGUF:Q8_0") is True
assert inference_route._reference_is_decisive("text-embedding-3-small") is False
assert inference_route._reference_is_decisive("openai/text-embedding-3-large") is False
def test_advertised_local_identity_is_accepted_as_an_alias(studio_embedder, tmp_path):
model_dir = str(tmp_path / "bge")
studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
studio_embedder.setattr(
rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
)
from core.rag.config import _escape_identity_segment
studio_embedder.setattr(
rag_embeddings,
"embedding_identity",
lambda model_name = None: f"sentence-transformers:{_escape_identity_segment(model_dir)}",
)
public = inference_route._public_embedding_name(model_dir)
assert inference_route._names_studio_embedder(f"sentence-transformers:{public}") == model_dir
assert inference_route._names_studio_embedder(public) == model_dir
assert inference_route._names_studio_embedder("bge-00000000") is None
def test_disconnected_client_is_dropped_even_with_a_free_permit(studio_embedder):
calls = []
class _Gone(_Request):
async def is_disconnected(self):
return True
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(
rag_embeddings, "encode", lambda texts, **_: calls.append(texts) or _vectors(texts)
)
async def run():
with pytest.raises(HTTPException) as exc:
await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")
assert exc.value.status_code == 499
assert calls == []
response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")
assert response.status_code == 200
asyncio.run(run())
def test_alias_match_pins_the_model_for_the_request(studio_embedder):
settings = iter([MODEL, "unsloth/other-model", "unsloth/other-model"])
seen = []
def _encode_with_identity(
texts,
*,
model_name = None,
normalize = True,
):
seen.append(model_name)
return _vectors(texts), IDENTITY
_identity_names(studio_embedder)
studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: next(settings))
studio_embedder.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
_call({"input": "alpha", "model": MODEL})
assert seen == [MODEL]
def test_llama_max_tokens_never_exceeds_one_physical_batch(tmp_path, monkeypatch):
# Embedding is non-causal, so llama.cpp refuses rather than splits over-batch inputs.
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
backend._model_path = _gguf(
tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
)
monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
monkeypatch.setattr(backend, "_server_context", lambda: 8192)
monkeypatch.setattr(backend, "_server_batch", lambda: 512)
monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
assert backend.max_tokens() == 510
def test_the_embed_server_does_not_enlarge_its_batch(tmp_path):
# n_vocab*n_ubatch*4 is allocated at startup, so max_tokens bounds the advert instead.
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
model = _gguf(tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)])
cmd = backend._build_cmd("llama-server", model, 9999, use_gpu = True)
assert "-b" not in cmd
# Pinned at llama.cpp's own default, so nothing is allocated that was not already, and
# max_tokens has a batch to bound the advert with (/props publishes none).
assert cmd[cmd.index("-ub") + 1] == str(embed_llama_server._UBATCH_SIZE)
def test_an_unconfirmed_context_limit_is_not_cached(tmp_path, monkeypatch):
# A header context can exceed the running one; freezing it outlives the /props correction.
from core.rag import embed_llama_server
backend = embed_llama_server.LlamaServerBackend()
backend._model_path = _gguf(
tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
)
monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
monkeypatch.setattr(backend, "_server_batch", lambda: None)
monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
monkeypatch.setattr(backend, "_server_context", lambda: None)
assert backend.max_tokens() == 8190
assert backend._max_tokens is None
monkeypatch.setattr(backend, "_server_context", lambda: 512)
monkeypatch.setattr(backend, "_server_batch", lambda: 512)
assert backend.max_tokens() == 510
assert backend._max_tokens == 510
def test_props_probe_never_raises_before_the_server_is_up():
from core.rag import embed_llama_server
# An un-started server has no port; the probe must swallow that and use the launch batch.
assert (
embed_llama_server.LlamaServerBackend()._server_batch() == embed_llama_server._UBATCH_SIZE
)
def test_cancel_during_the_final_disconnect_probe_releases_the_permit(studio_embedder):
lock = threading.Lock()
gate = threading.Event()
active = {"now": 0, "peak": 0}
def encode(texts, **_kwargs):
with lock:
active["now"] += 1
active["peak"] = max(active["peak"], active["now"])
gate.wait(timeout = 5)
with lock:
active["now"] -= 1
return _vectors(texts)
class _Stuck(_Request):
async def is_disconnected(self):
await asyncio.sleep(3600)
return False
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
studio_embedder.setattr(rag_embeddings, "encode", encode)
cap = inference_route._STUDIO_EMBED_CONCURRENCY
async def run():
stuck = asyncio.create_task(inference_route.openai_embeddings(_Stuck({"input": "x"}), "t"))
await asyncio.sleep(0.05)
stuck.cancel()
with pytest.raises(asyncio.CancelledError):
await stuck
tasks = [
asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))
for _ in range(cap)
]
for _ in range(500):
await asyncio.sleep(0.01)
if active["now"] == cap:
break
assert active["now"] == cap
gate.set()
await asyncio.gather(*tasks)
asyncio.run(run())
def test_a_boolean_is_not_a_token_id():
# bool subclasses int, so `[true]` read as a token array could swap the resident GGUF.
from fastapi import HTTPException
with pytest.raises(HTTPException) as exc:
inference_route._embeddings_items({"input": [True]}, tokens_ok = True)
assert exc.value.status_code == 400
with pytest.raises(HTTPException):
inference_route._embeddings_items({"input": [[1, True, 3]]}, tokens_ok = True)
assert inference_route._embeddings_items({"input": [1, 2, 3]}, tokens_ok = True) == [[1, 2, 3]]
def test_a_cold_index_does_not_make_a_local_name_foreign(monkeypatch):
# Before the first scan resolve_local_gguf returns None for downloaded models; that is not foreign.
from core.inference import local_model_resolver as _resolver
monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)
monkeypatch.setattr(_resolver, "index_is_built", lambda: False)
monkeypatch.setattr(_resolver, "warm_index_soon", lambda: None)
assert inference_route._reference_is_decisive("org/my-local-model") is True
assert inference_route._reference_is_decisive("/models/local.gguf") is True
assert inference_route._reference_is_decisive("text-embedding-3-small") is False
# Once the index is built, absence really is evidence.
monkeypatch.setattr(_resolver, "index_is_built", lambda: True)
assert inference_route._reference_is_decisive("org/not-here") is False
@pytest.mark.parametrize("stash_embeds", [False, True])
def test_default_request_restores_the_chat_slot_only_for_an_embedding_stash(
studio_embedder, stash_embeds
):
seen = []
async def record(request, current_subject, **_kwargs):
seen.append(current_subject)
return await request.json()
studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
studio_embedder.setattr(inference_route, "_stashed_gguf_embeds", lambda: stash_embeds)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
payload = _call({"input": "alpha"})
assert payload["model"] == IDENTITY
assert seen == (["tester"] if stash_embeds else [])
def test_stashed_gguf_embeds_reads_the_stashed_model_header(monkeypatch, tmp_path):
import core.inference.llama_keepwarm as keepwarm
import core.inference.local_model_resolver as resolver
monkeypatch.setattr(keepwarm, "get_last_unloaded_model", lambda: None)
assert inference_route._stashed_gguf_embeds() is False
monkeypatch.setattr(
keepwarm, "get_last_unloaded_model", lambda: ("org/E-GGUF", "Q8_0", "org/E-GGUF")
)
monkeypatch.setattr(
resolver, "resolve_local_gguf", lambda ref, **_: ("/tmp/e.gguf", "Q8_0", "org/E-GGUF")
)
class _Probe:
is_embedding_gguf = True
def _read_gguf_metadata(self, path):
self.path = path
monkeypatch.setattr(inference_route, "_probe_backend", _Probe)
assert inference_route._stashed_gguf_embeds() is True
def test_resident_embedding_gguf_recuts_for_a_context_below_the_batch(studio_embedder):
client, embedded, tokenized = _llama_overflow_server(6, 4)
_resident_embedding_gguf(studio_embedder, client)
payload = _call({"input": ["aa bbb c dddd e ff gg hh"], "model": "org/E-GGUF"})
assert len(payload["data"]) == 1
assert [body["input"] for body in embedded[1:]] == [
[[_CLS, 2, 3, 1, 4, _SEP]],
[[_CLS, 2, 3, _SEP]],
]
assert len(tokenized) == 2
def test_resident_embedding_gguf_skips_tokenizing_strings_that_cannot_overflow(studio_embedder):
client, embedded, tokenized = _llama_overflow_server(20, 512)
_resident_embedding_gguf(studio_embedder, client)
long = " ".join(["a"] * 30)
_call({"input": [long, "x", "y z", "\ufdfa"], "model": "org/E-GGUF"})
assert embedded[1]["input"][1:] == ["x", "y z", "\ufdfa"]
assert set(tokenized) == {long, "\ufdfa"}
def test_resident_embedding_gguf_cancel_during_the_retry_is_499(studio_embedder):
from fastapi import HTTPException
server, _, _ = _llama_overflow_server(4, 512)
request = _Request({"input": ["aa bbb c dddd e"], "model": "org/E-GGUF"})
disconnected = False
async def _is_disconnected():
return disconnected
request.is_disconnected = _is_disconnected
class _Client(server):
closed = False
async def post(self, url, *args, **kwargs):
nonlocal disconnected
if self.closed:
raise RuntimeError("Cannot send a request, as the client has been closed.")
response = await super().post(url, *args, **kwargs)
if response.status_code == 500:
disconnected = True
while not self.closed:
await asyncio.sleep(0.05)
return response
async def aclose(self):
self.closed = True
_resident_embedding_gguf(studio_embedder, _Client)
with pytest.raises(HTTPException) as exc:
asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert exc.value.status_code == 499
def test_batch_counts_tokens_in_one_call_with_the_same_usage(studio_embedder):
calls = {"single": 0, "many": 0}
def _counter(model_name = None):
def _count(text):
calls["single"] += 1
return len(text)
def _many(texts):
calls["many"] += 1
return [len(text) for text in texts]
_count.many = _many
return _count
studio_embedder.setattr(rag_embeddings, "token_counter", _counter)
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
texts = [f"text {i} " * (i % 7 + 1) for i in range(64)]
payload = _call({"input": texts})
assert calls == {"single": 0, "many": 1}
assert payload["usage"]["prompt_tokens"] == sum(len(text) for text in texts)
assert [row["embedding"] for row in payload["data"]] == [[1.0, 0.0]] * 64
def test_float_rows_serialize_like_the_element_loop():
rows = np.random.default_rng(0).standard_normal((4, 33)).astype(np.float32)
rows[0, :3] = [np.nan, np.inf, -0.0]
for row in rows:
fast = inference_route._embedding_payload(row, "float")
assert json.dumps(fast) == json.dumps([float(x) for x in row])
assert all(type(x) is float for x in fast)
assert inference_route._embedding_payload([0.5, 1], "float") == [0.5, 1.0]
class _Monitor:
def __init__(self):
self.calls = []
def start(self, **_kwargs):
return "mid"
def fail(self, monitor_id, detail):
self.calls.append(("fail", detail))
def finish(self, monitor_id, *args):
self.calls.append(("finish", *args))
def _counting_gguf_proxy(
studio_embedder,
body,
status = 200,
):
import httpx
parses = []
class _Response(httpx.Response):
def json(self, **kwargs):
parses.append(threading.get_ident())
return super().json(**kwargs)
class _Client:
async def post(self, *_args, **_kwargs):
return _Response(status, content = body)
async def aclose(self):
return None
_resident_embedding_gguf(studio_embedder, _Client)
return parses
def _upstream_body(dim = 4):
rows = [{"object": "embedding", "index": 0, "embedding": [0.25] * dim}]
usage = {"prompt_tokens": 3, "total_tokens": 3}
return json.dumps({"object": "list", "data": rows, "usage": usage}).encode()
def test_gguf_proxy_skips_the_parse_when_nothing_reads_it(studio_embedder):
body = _upstream_body()
parses = _counting_gguf_proxy(studio_embedder, body)
response = asyncio.run(
inference_route.openai_embeddings(_Request({"input": "alpha"}), "tester")
)
assert response.body == body
assert parses == []
@pytest.mark.parametrize("upstream", [_upstream_body(), b"{not json"], ids = ["json", "invalid"])
def test_gguf_proxy_parses_once_off_the_loop_for_monitor_and_dimensions(studio_embedder, upstream):
parses = _counting_gguf_proxy(studio_embedder, upstream)
monitor, seen = _Monitor(), []
studio_embedder.setattr(inference_route, "api_monitor", monitor)
studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)
studio_embedder.setattr(
inference_route,
"_monitor_openai_chunk",
lambda monitor_id, data, context_length = None, **_: seen.append(data),
)
request = _Request({"input": "alpha", "dimensions": 4})
request.state.skip_api_monitor = False
if upstream.startswith(b"{not"):
assert _http_error_for(request).status_code == 400
assert seen == [] and monitor.calls[0][0] == "fail"
else:
response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert response.body == upstream
assert seen == [json.loads(upstream)]
assert monitor.calls == [("finish",)]
assert len(parses) == 1 and parses[0] != threading.get_ident()
def _http_error_for(request):
with pytest.raises(HTTPException) as exc:
asyncio.run(inference_route.openai_embeddings(request, "tester"))
return exc.value
@pytest.mark.parametrize("encoding_format", ["float", "base64"])
def test_studio_payload_is_serialized_off_the_loop_with_the_same_bytes(
studio_embedder, encoding_format
):
rows = np.random.default_rng(0).standard_normal((3, 2)).astype(np.float32)
rows[0, 0] = np.nan
threads = []
real_payload = inference_route._embedding_payload
def _payload(vector, fmt):
threads.append(threading.get_ident())
return real_payload(vector, fmt)
studio_embedder.setattr(inference_route, "_embedding_payload", _payload)
studio_embedder.setattr(rag_embeddings, "encode", lambda texts, **_: rows[: len(texts)])
studio_embedder.setattr(
inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
)
body = {"input": ["a", "bb", "ccc"], "encoding_format": encoding_format}
response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
expected = {
"object": "list",
"data": [
{"object": "embedding", "index": i, "embedding": real_payload(row, encoding_format)}
for i, row in enumerate(rows)
],
"model": IDENTITY,
"usage": {"prompt_tokens": 6, "total_tokens": 6},
}
assert response.body == json.dumps(expected).encode()
assert threads and threading.get_ident() not in threads
_USAGE = {"prompt_tokens": 3, "total_tokens": 3}
_ROWS = [{"embedding": [0.25] * 4, "index": 0, "object": "embedding"}]
@pytest.mark.parametrize(
"upstream, full_parses",
[
(json.dumps({"data": _ROWS, "model": "m", "object": "list", "usage": _USAGE}), 0),
(json.dumps({"data": _ROWS, "model": '"usage": {}', "usage": _USAGE}, indent = 2), 0),
(json.dumps({"usage": _USAGE, "data": _ROWS}), 1),
(json.dumps({"data": _ROWS, "x": {"usage": _USAGE}}), 1),
(json.dumps({"data": _ROWS}), 1),
],
)
def test_gguf_proxy_monitor_reads_usage_from_the_tail(studio_embedder, upstream, full_parses):
upstream = upstream.encode()
parses = _counting_gguf_proxy(studio_embedder, upstream)
monitor, seen = _Monitor(), []
studio_embedder.setattr(inference_route, "api_monitor", monitor)
studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)
studio_embedder.setattr(
inference_route,
"_monitor_openai_chunk",
lambda monitor_id, data, context_length = None, **_: seen.append(data.get("usage")),
)
request = _Request({"input": "alpha"})
request.state.skip_api_monitor = False
response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
assert response.body == upstream
assert seen == [json.loads(upstream).get("usage")]
assert len(parses) == full_parses and monitor.calls == [("finish",)]