Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

68.9 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import asyncio
import base64
import copy
import json
import os
import sys
import threading
from types import SimpleNamespace

import numpy as np
import pytest
from fastapi import HTTPException

_backend = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
if _backend not in sys.path:
    sys.path.insert(0, _backend)

inference_route = pytest.importorskip("routes.inference", reason = "inference stack not installed")
rag_embeddings = pytest.importorskip("core.rag.embeddings", reason = "rag stack not installed")
rag_config = pytest.importorskip("core.rag.config", reason = "rag stack not installed")

MODEL = "unsloth/bge-small-en-v1.5"
IDENTITY = f"sentence-transformers:{MODEL}"


class _Request:
    def __init__(self, body):
        self._body = body
        self.method = "POST"
        self.url = SimpleNamespace(path = "/v1/embeddings")
        self.state = SimpleNamespace(skip_api_monitor = True)
        self.scope = {"type": "http", "path": "/v1/embeddings"}

    async def json(self):
        return self._body

    async def is_disconnected(self):
        return False


def _vectors(texts, **_):
    return np.array([[1.0, 0.0]] * len(texts), dtype = np.float32)


@pytest.fixture
def studio_embedder(monkeypatch):
    async def passthrough(request, current_subject, **_kwargs):
        return await request.json()

    def no_proxy():
        raise AssertionError("the llama-server proxy must not run")

    monkeypatch.setattr(inference_route, "_should_validate_before_switch", lambda: False)
    monkeypatch.setattr(inference_route, "_auto_switch_from_request_body", passthrough)
    monkeypatch.setattr(inference_route, "_cancelable_nonstreaming_client", no_proxy)
    monkeypatch.setattr(rag_config, "effective_embedding_model", lambda: MODEL)
    monkeypatch.setattr(rag_embeddings, "encode", _vectors)

    def _encode_with_identity(
        texts,
        *,
        model_name = None,
        normalize = True,
    ):
        return rag_embeddings.encode(texts, model_name = model_name, normalize = normalize), IDENTITY

    monkeypatch.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)
    monkeypatch.setattr(rag_embeddings, "token_counter", lambda model_name = None: len)
    monkeypatch.setattr(rag_embeddings, "max_tokens", lambda model_name = None: None)
    monkeypatch.setattr(rag_embeddings, "dim", lambda model_name = None: 2)
    return monkeypatch


def _call(body):
    response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
    assert response.status_code == 200
    import json

    return json.loads(response.body)


def _http_error(body):
    from fastapi import HTTPException
    with pytest.raises(HTTPException) as exc:
        asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
    return exc.value


def test_nothing_loaded_serves_from_the_studio_embedder(studio_embedder):
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    payload = _call({"input": ["alpha", "beta"], "model": "text-embedding-3-small"})
    assert payload["object"] == "list"
    assert payload["model"] == IDENTITY
    assert [row["index"] for row in payload["data"]] == [0, 1]
    assert payload["data"][0]["embedding"] == [1.0, 0.0]
    assert payload["usage"] == {"prompt_tokens": 9, "total_tokens": 9}


def test_chat_model_loaded_serves_from_the_studio_embedder(studio_embedder):
    started = []
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
    )
    studio_embedder.setattr(
        inference_route, "_direct_llama_request_started", lambda: started.append(1)
    )
    payload = _call({"input": "alpha"})
    assert payload["model"] == IDENTITY
    assert started == []


def test_resident_embedding_gguf_still_uses_the_proxy(studio_embedder):
    import httpx

    class _Client:
        async def post(self, *_args, **_kwargs):
            return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})

        async def aclose(self):
            return None

    def boom(*_args, **_kwargs):
        raise AssertionError("the studio embedder must not run")

    studio_embedder.setattr(rag_embeddings, "encode", boom)
    studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(
            is_loaded = True,
            is_embedding_gguf = True,
            base_url = "http://llama.test",
            context_length = 512,
            model_identifier = "org/E-GGUF",
        ),
    )
    payload = _call({"input": "alpha", "model": "org/E-GGUF"})
    assert payload == {"data": [{"embedding": [0.5]}]}


@pytest.mark.parametrize("encoding_format", ["float", "base64"])
@pytest.mark.parametrize("dimensions, status", [(2, 200), (256, 400)])
def test_resident_embedding_gguf_checks_dimensions_against_its_width(
    studio_embedder, encoding_format, dimensions, status
):
    import httpx

    forwarded = []

    class _Client:
        async def post(
            self,
            *_args,
            json = None,
            **_kwargs,
        ):
            forwarded.append(json)
            vector = [0.5, 0.5]
            if encoding_format == "base64":
                vector = base64.b64encode(np.asarray(vector, dtype = np.float32).tobytes()).decode()
            return httpx.Response(200, json = {"data": [{"embedding": vector}]})

        async def aclose(self):
            return None

    studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(
            is_loaded = True,
            is_embedding_gguf = True,
            base_url = "http://llama.test",
            context_length = 512,
            model_identifier = "org/E-GGUF",
        ),
    )
    body = {
        "input": "alpha",
        "model": "org/E-GGUF",
        "encoding_format": encoding_format,
        "dimensions": dimensions,
    }
    if status == 200:
        _call(body)
    else:
        error = _http_error(body)
        assert error.status_code == 400
        assert "'dimensions' is not supported" in error.detail
    assert "dimensions" not in forwarded[0]


def test_base64_encoding_format(studio_embedder):
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    payload = _call({"input": "alpha", "encoding_format": "base64"})
    raw = base64.b64decode(payload["data"][0]["embedding"])
    assert np.frombuffer(raw, dtype = np.float32).tolist() == [1.0, 0.0]


@pytest.mark.parametrize(
    "body",
    [
        {"input": ""},
        {"input": []},
        {"input": [[1, 2, 3]]},
        {"input": ["alpha", ""]},
        {"input": "alpha", "encoding_format": "int8"},
        {"input": "alpha", "encoding_format": ""},
        {"input": "alpha", "encoding_format": False},
        {"input": "alpha", "encoding_format": 0},
        {"input": "alpha", "dimensions": 999},
    ],
)
def test_invalid_requests_are_400(studio_embedder, body):
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    assert _http_error(body).status_code == 400


def test_over_length_input_is_truncated_by_default(studio_embedder):
    encoded = []

    def _encode(texts, **_):
        encoded.append(list(texts))
        return _vectors(texts)

    studio_embedder.setattr(rag_embeddings, "encode", _encode)
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
    payload = _call({"input": ["ab", "abcd"]})
    assert encoded == [["ab", "abc"]]
    assert len(payload["data"]) == 2
    assert payload["usage"]["prompt_tokens"] == 5


def test_over_length_input_is_truncated_with_batched_counts(studio_embedder):
    encoded, singles, batches = [], [], []

    def _count(text):
        singles.append(text)
        return len(text)

    def _many(texts):
        batches.append(list(texts))
        return [len(t) for t in texts]

    _count.many = _many
    studio_embedder.setattr(
        rag_embeddings, "encode", lambda texts, **_: encoded.append(list(texts)) or _vectors(texts)
    )
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "token_counter", lambda model_name = None: _count)
    studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
    payload = _call({"input": ["ab", "abcd", "abcdefgh"]})
    assert encoded == [["ab", "abc", "abc"]]
    assert payload["usage"]["prompt_tokens"] == 8
    assert singles == [] and batches[0] == ["ab", "abcd", "abcdefgh"]


def test_over_length_input_is_rejected_when_truncate_is_false(studio_embedder):
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
    error = _http_error({"input": ["ab", "abcd"], "truncate": False})
    assert error.status_code == 400
    assert "3-token limit" in error.detail


def test_non_boolean_truncate_is_400(studio_embedder):
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    error = _http_error({"input": "ab", "truncate": "yes"})
    assert error.status_code == 400
    assert "'truncate'" in error.detail


_CLS, _SEP = 101, 102


def _llama_overflow_server(
    ubatch,
    n_ctx,
    can_split = False,
):
    import httpx

    embedded, tokenized = [], []

    def _tokens(item):
        if isinstance(item, list):
            return item
        return [_CLS, *(len(word) for word in item.split()), _SEP]

    class _Client:
        async def post(
            self,
            url,
            *_args,
            json = None,
            **_kwargs,
        ):
            if url.endswith("/tokenize"):
                tokenized.append(json["content"])
                tokens = _tokens(json["content"])
                return httpx.Response(
                    200, json = {"tokens": tokens if json["add_special"] else tokens[1:-1]}
                )
            # The route reuses the dict.
            embedded.append(copy.deepcopy(json))
            items = [json["input"]] if isinstance(json["input"], str) else json["input"]
            for item in items:
                n = len(_tokens(item))
                if can_split and n >= n_ctx:
                    return httpx.Response(
                        400,
                        json = {
                            "error": {
                                "message": f"request ({n} tokens) exceeds the available context "
                                f"size ({n_ctx} tokens), try increasing it"
                            }
                        },
                    )
                if can_split:
                    continue
                if n > ubatch:
                    return httpx.Response(
                        500,
                        json = {
                            "error": {
                                "message": f"input ({n} tokens) is too large to process. increase "
                                f"the physical batch size (current batch size: {ubatch})"
                            }
                        },
                    )
                if n > n_ctx:
                    return httpx.Response(
                        400,
                        json = {
                            "error": {
                                "message": f"input ({n} tokens) is larger than the max context "
                                f"size ({n_ctx} tokens). skipping"
                            }
                        },
                    )
            return httpx.Response(200, json = {"data": [{"embedding": [0.5]} for _ in items]})

        async def aclose(self):
            return None

    return _Client, embedded, tokenized


_OVERFLOW_LIMITS = [(4, 512), (512, 4)]


def _resident_embedding_gguf(studio_embedder, client):
    studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", client)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(
            is_loaded = True,
            is_embedding_gguf = True,
            base_url = "http://llama.test",
            context_length = 512,
            model_identifier = "org/E-GGUF",
        ),
    )


@pytest.mark.parametrize("ubatch, n_ctx", _OVERFLOW_LIMITS)
def test_resident_embedding_gguf_truncates_an_oversized_input(studio_embedder, ubatch, n_ctx):
    client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)
    _resident_embedding_gguf(studio_embedder, client)
    payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})
    assert len(payload["data"]) == 2
    assert len(embedded) == 2
    assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]
    assert "truncate" not in embedded[0]


def test_resident_embedding_gguf_truncates_for_a_splittable_slot(studio_embedder):
    client, embedded, _ = _llama_overflow_server(512, 5, can_split = True)
    _resident_embedding_gguf(studio_embedder, client)
    payload = _call({"input": ["aa bbb c dddd e", "x y"], "model": "org/E-GGUF"})
    assert len(payload["data"]) == 2
    assert embedded[1]["input"] == [[_CLS, 2, 3, _SEP], "x y"]


@pytest.mark.parametrize("ubatch, n_ctx", _OVERFLOW_LIMITS)
def test_resident_embedding_gguf_keeps_the_error_when_truncate_is_false(
    studio_embedder, ubatch, n_ctx
):
    client, embedded, _ = _llama_overflow_server(ubatch, n_ctx)
    _resident_embedding_gguf(studio_embedder, client)
    response = asyncio.run(
        inference_route.openai_embeddings(
            _Request({"input": "aa bbb c dddd e", "model": "org/E-GGUF", "truncate": False}),
            "tester",
        )
    )
    assert response.status_code == (500 if ubatch < n_ctx else 400)
    assert len(embedded) == 1
    assert "truncate" not in embedded[0]


def test_embedder_failure_is_502(studio_embedder):
    def boom(*_args, **_kwargs):
        raise RuntimeError("llama-server embedder POST /v1/embeddings -> 500")

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "encode", boom)
    assert _http_error({"input": "alpha"}).status_code == 502


def test_identity_redaction_leaves_the_backend_tag_alone(tmp_path, monkeypatch):
    """Redacting a local model name must not rewrite the backend tag, e.g. sentence-transformers."""
    from core.rag import config as rag_config

    (tmp_path / "transformers").mkdir()
    monkeypatch.chdir(tmp_path)
    monkeypatch.setattr(
        inference_route,
        "_public_embedding_name",
        lambda name: "transformers-deadbeef" if name == "transformers" else name,
    )
    monkeypatch.setattr(
        rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "transformers-GGUF"
    )
    public = inference_route._public_embedding_identity(
        "sentence-transformers:transformers", "transformers", "transformers-deadbeef"
    )
    assert public == "sentence-transformers:transformers-deadbeef"
    assert rag_config.embedding_identity_model(public) == "transformers-deadbeef"


def test_identity_redaction_covers_the_gguf_repo_segment(monkeypatch):
    from core.rag import config as rag_config

    monkeypatch.setattr(
        rag_config, "effective_gguf_repo_for_embedding_model", lambda model: "/models/bge-GGUF"
    )
    monkeypatch.setattr(
        inference_route,
        "_public_embedding_name",
        lambda name: {"/models/bge": "bge-1111", "/models/bge-GGUF": "bge-GGUF-2222"}.get(
            name, name
        ),
    )
    public = inference_route._public_embedding_identity(
        "llama-server:%2Fmodels%2Fbge:%2Fmodels%2Fbge-GGUF".replace("%2F", "/"),
        "/models/bge",
        "bge-1111",
    )
    assert public == "llama-server:bge-1111:bge-GGUF-2222"


def test_identity_redaction_covers_a_local_repo_under_a_hub_model(monkeypatch):
    """Redact a local path in RAG_EMBED_GGUF_REPO even when the model segment is a plain hub id."""
    from core.rag import config as rag_config

    monkeypatch.setattr(
        rag_config,
        "effective_gguf_repo_for_embedding_model",
        lambda model: "/srv/private/embed.gguf",
    )
    monkeypatch.setattr(
        inference_route,
        "_public_embedding_name",
        lambda name: "embed.gguf-3333" if name == "/srv/private/embed.gguf" else name,
    )
    public = inference_route._public_embedding_identity(
        f"llama-server:{MODEL}:/srv/private/embed.gguf", MODEL, MODEL
    )
    assert public == f"llama-server:{MODEL}:embed.gguf-3333"
    assert "/srv/private" not in public


@pytest.mark.parametrize("named", [123, 1.5, True, ["a"], {"a": 1}])
def test_a_non_string_model_is_not_a_server_error(studio_embedder, named):
    """Both pre-switch readers decline a non-string selector, so the empty-slot branch sees it."""
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    payload = _call({"input": "alpha", "model": named})
    assert payload["model"] == IDENTITY


@pytest.mark.parametrize(
    "exc,fragment",
    [
        (
            rag_embeddings.EmbeddingModelDownloadRequiredError(
                "Embedding model 'unsloth/bge-small-en-v1.5' is not downloaded yet. "
                "Finish its Settings download before indexing documents."
            ),
            "not downloaded yet",
        ),
        (
            rag_embeddings.UnsafeEmbeddingModelError(
                "Embedding model 'evil/model' requires remote code execution."
            ),
            "remote code",
        ),
    ],
)
def test_actionable_embedder_errors_keep_their_message(studio_embedder, exc, fragment):
    """The caller can fix these two, but only if it is told what they are.

    Picking a model in Settings and querying before its download finishes is the common
    one, and it reaches this route through `unsloth start openclaw`: the generated
    openclaw.json points memory search here, so this detail string is the entire
    diagnosis the user sees. Routed through _friendly_error -- written for llama-server
    transport faults -- it became a 502 "An internal error occurred", which says nothing
    and suggests the wrong owner.
    """

    def boom(*_args, **_kwargs):
        raise exc

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "encode", boom)
    error = _http_error({"input": "alpha"})
    assert error.status_code == 409
    assert fragment in error.detail
    assert "An internal error occurred" not in error.detail


def test_pending_gguf_download_is_classified_like_the_st_one(tmp_path, monkeypatch):
    """A pending GGUF download must be typed like the sentence-transformers one, so both reach the 409."""
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    monkeypatch.setattr(
        embed_llama_server.config,
        "effective_gguf_repo_for_embedding_model",
        lambda model: "unsloth/bge-small-en-v1.5-GGUF",
    )
    monkeypatch.setattr(backend, "_resolve_local_gguf", lambda model: None)
    monkeypatch.setattr(backend, "_planned_family_path", lambda model, repo: None)
    monkeypatch.setattr(backend, "_resolve_cached_gguf", lambda *a, **k: None)
    import utils.embedding_model_settings as ems

    monkeypatch.setattr(ems, "get_stored_download_pending", lambda model: True)

    with pytest.raises(rag_embeddings.EmbeddingModelDownloadRequiredError) as excinfo:
        backend._resolve_model_path("unsloth/bge-small-en-v1.5")
    assert "not downloaded yet" in str(excinfo.value)


def test_actionable_errors_do_not_leak_a_local_path(studio_embedder, monkeypatch):
    """Pass-through error text must not expose a local path, since the route hashes labels elsewhere."""
    studio_embedder.setattr(
        rag_config, "effective_embedding_model", lambda: "/srv/models/bge-small"
    )

    def boom(*_args, **_kwargs):
        raise rag_embeddings.UnsafeEmbeddingModelError(
            "Embedding model '/srv/models/bge-small' is flagged as unsafe by Hugging Face's "
            "security scan; refusing to load. Set a different RAG embedding model."
        )

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "encode", boom)
    error = _http_error({"input": "alpha"})
    assert error.status_code == 409
    assert "/srv/models" not in error.detail
    assert "flagged as unsafe" in error.detail
    assert "bge-small-" in error.detail


def test_guard_model_security_names_the_configured_model_not_the_snapshot(monkeypatch):
    """The scan needs the snapshot path; the message must not carry it."""
    monkeypatch.setattr(rag_embeddings, "_ambient_hf_token", lambda: None)
    import utils.security as security

    monkeypatch.setattr(
        security,
        "evaluate_file_security",
        lambda *a, **k: SimpleNamespace(blocked = True),
    )
    monkeypatch.setattr(security, "security_load_subdirs", lambda *a, **k: ())
    monkeypatch.setattr(rag_embeddings, "_st_module_subdirs", lambda *a, **k: ())
    with pytest.raises(rag_embeddings.UnsafeEmbeddingModelError) as excinfo:
        rag_embeddings._guard_model_security(
            "/home/me/.cache/huggingface/hub/models--org--bge/snapshots/abc123",
            display = "org/bge",
        )
    assert "org/bge" in str(excinfo.value)
    assert ".cache/huggingface" not in str(excinfo.value)


def test_studio_embedder_requests_are_admission_limited(studio_embedder):
    lock = threading.Lock()
    gate = threading.Event()
    active = {"now": 0, "peak": 0}

    def encode(texts, **_kwargs):
        with lock:
            active["now"] += 1
            active["peak"] = max(active["peak"], active["now"])
        gate.wait(timeout = 5)
        with lock:
            active["now"] -= 1
        return _vectors(texts)

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "encode", encode)

    async def run():
        tasks = [
            asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
            for _ in range(inference_route._STUDIO_EMBED_CONCURRENCY + 2)
        ]
        for _ in range(500):
            await asyncio.sleep(0.01)
            if active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY:
                break
        assert active["now"] == inference_route._STUDIO_EMBED_CONCURRENCY
        gate.set()
        responses = await asyncio.gather(*tasks)
        assert all(response.status_code == 200 for response in responses)

    asyncio.run(run())
    assert active["peak"] == inference_route._STUDIO_EMBED_CONCURRENCY


def test_studio_fallback_untracks_the_request_from_the_llama_slot(studio_embedder):
    # A 2xx reaching _finish would claim a llama slot the studio embedder never used, so untrack first.
    from core.inference import llama_keepwarm as kw

    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
    )
    request = _Request({"input": "alpha"})
    before = kw._inflight
    kw._inflight = before + 1
    try:
        response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
        assert response.status_code == 200
        assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is True
        assert kw._inflight == before
    finally:
        kw._inflight = before


def test_resident_embedding_gguf_still_claims_the_slot(studio_embedder):
    import httpx

    class _Client:
        async def post(self, *_args, **_kwargs):
            return httpx.Response(200, json = {"data": [{"embedding": [0.5]}]})

        async def aclose(self):
            return None

    from core.inference import llama_keepwarm as kw

    studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(
            is_loaded = True,
            is_embedding_gguf = True,
            base_url = "http://llama.test",
            context_length = 512,
            model_identifier = "org/E-GGUF",
        ),
    )
    request = _Request({"input": "alpha", "model": "org/E-GGUF"})
    asyncio.run(inference_route.openai_embeddings(request, "tester"))
    assert request.scope.get(kw._UNTRACKED_SCOPE_KEY) is None


def test_context_gauge_is_not_pinned_by_a_batch(studio_embedder):
    # api_monitor divides summed prompt_tokens by this, so per-text limit overstates context use.
    seen = []
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 8)
    studio_embedder.setattr(
        inference_route,
        "_monitor_openai_chunk",
        lambda monitor_id, payload, context_length = None, **_: seen.append(context_length),
    )
    request = _Request({"input": ["alpha", "beta", "gamma"]})
    request.state.skip_api_monitor = False
    asyncio.run(inference_route.openai_embeddings(request, "tester"))
    assert seen == [None]

    seen.clear()
    request = _Request({"input": "alpha"})
    request.state.skip_api_monitor = False
    asyncio.run(inference_route.openai_embeddings(request, "tester"))
    assert seen == [8]


def test_cancelled_requests_do_not_leak_admission_permits(studio_embedder):
    # to_thread cannot cancel the worker, so permits must outlive the awaiting task's cancellation.
    lock = threading.Lock()
    gate = threading.Event()
    active = {"now": 0, "peak": 0}

    def encode(texts, **_kwargs):
        with lock:
            active["now"] += 1
            active["peak"] = max(active["peak"], active["now"])
        gate.wait(timeout = 5)
        with lock:
            active["now"] -= 1
        return _vectors(texts)

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "encode", encode)
    cap = inference_route._STUDIO_EMBED_CONCURRENCY

    async def run():
        first = [
            asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
            for _ in range(cap)
        ]
        for _ in range(500):
            await asyncio.sleep(0.01)
            if active["now"] == cap:
                break
        assert active["now"] == cap

        for task in first:
            task.cancel()
        await asyncio.gather(*first, return_exceptions = True)

        second = [
            asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))
            for _ in range(cap)
        ]
        await asyncio.sleep(0.3)
        assert active["peak"] == cap, f"admission cap exceeded: peak={active['peak']}"
        gate.set()
        await asyncio.gather(*second, return_exceptions = True)

    asyncio.run(run())
    assert active["peak"] == cap


def test_reported_model_follows_the_backend_that_made_the_vectors(studio_embedder):
    # ST failure swaps to llama-server (different space); the response must name the real space.
    llama_identity = f"llama-server:{MODEL}:unsloth/bge-small-en-v1.5-GGUF"
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(
        rag_embeddings,
        "encode_with_identity",
        lambda texts, **_kwargs: (_vectors(texts), llama_identity),
    )
    payload = _call({"input": "alpha"})
    assert payload["model"] == llama_identity


def test_embedding_helpers_are_pinned_to_the_captured_model(studio_embedder):
    # A Settings change while queued must not mix one model's limit/dim with another's vectors.
    seen = {"max_tokens": [], "token_counter": [], "dim": [], "encode": []}
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(
        rag_embeddings,
        "max_tokens",
        lambda model_name = None: (seen["max_tokens"].append(model_name), None)[1],
    )
    studio_embedder.setattr(
        rag_embeddings,
        "token_counter",
        lambda model_name = None: (seen["token_counter"].append(model_name), len)[1],
    )
    studio_embedder.setattr(
        rag_embeddings,
        "dim",
        lambda model_name = None: (seen["dim"].append(model_name), 2)[1],
    )
    studio_embedder.setattr(
        rag_embeddings,
        "encode_with_identity",
        lambda texts, **kw: (
            seen["encode"].append(kw.get("model_name")),
            (_vectors(texts), IDENTITY),
        )[1],
    )
    _call({"input": "alpha", "dimensions": 2})
    assert seen == {
        "max_tokens": [MODEL],
        "token_counter": [MODEL],
        "dim": [MODEL],
        "encode": [MODEL],
    }


def test_studio_fallback_releases_the_preview_busy_guard(studio_embedder):
    # Admission precedes routing and preview reads the admitted tally, so slow encodes keep 503ing swaps.
    from core.inference import llama_keepwarm as kw

    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
    )
    request = _Request({"input": "alpha"})
    # Relative: the tally is module state shared with the suite.
    before = kw.other_admitted_inference_count()
    kw.note_admitted_inference(request.scope)
    assert kw.other_admitted_inference_count() == before + 1
    assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is True
    try:
        asyncio.run(inference_route.openai_embeddings(request, "tester"))
        assert kw.other_admitted_inference_count() == before
        assert request.scope.get(kw._ADMITTED_SCOPE_KEY) is None
    finally:
        kw._admitted_inference = before


def test_cancelled_request_closes_its_monitor_row(studio_embedder):
    # Running rows dodge retention trimming, so a request cancelled while queued closes its own row.
    closed = []
    gate = threading.Event()
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(inference_route.api_monitor, "start", lambda **_kwargs: "entry-1")
    studio_embedder.setattr(
        inference_route.api_monitor,
        "finish",
        lambda entry_id, *args, **_kw: closed.append((entry_id, args[0] if args else None)),
    )
    studio_embedder.setattr(
        rag_embeddings,
        "encode_with_identity",
        lambda texts, **_kw: (gate.wait(timeout = 5), (_vectors(texts), IDENTITY))[1],
    )

    async def run():
        request = _Request({"input": "alpha"})
        request.state.skip_api_monitor = False
        task = asyncio.create_task(inference_route.openai_embeddings(request, "tester"))
        await asyncio.sleep(0.1)
        task.cancel()
        with pytest.raises(asyncio.CancelledError):
            await task
        gate.set()

    asyncio.run(run())
    assert closed == [("entry-1", "cancelled")]


def _identity_names(monkeypatch):
    monkeypatch.setattr(
        rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
    )
    monkeypatch.setattr(rag_embeddings, "embedding_identity", lambda model_name = None: IDENTITY)


@pytest.mark.parametrize("requested", [MODEL, f"{MODEL}-GGUF", IDENTITY, MODEL.upper()])
def test_naming_the_configured_embedder_skips_the_chat_slot_check(studio_embedder, requested):
    from fastapi import HTTPException

    async def reject(request, current_subject, **_kwargs):
        raise HTTPException(status_code = 404, detail = "model_not_found")

    _identity_names(studio_embedder)
    studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", reject)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
    )
    payload = _call({"input": "alpha", "model": requested})
    assert payload["model"] == IDENTITY
    assert payload["data"][0]["embedding"] == [1.0, 0.0]


def test_other_model_names_still_run_auto_switch(studio_embedder):
    seen = []

    async def record(request, current_subject, **_kwargs):
        seen.append(current_subject)
        return await request.json()

    _identity_names(studio_embedder)
    studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    _call({"input": "alpha", "model": "text-embedding-3-small"})
    assert seen == ["tester"]


def test_st_max_tokens_leaves_room_for_the_special_tokens(monkeypatch):
    model = SimpleNamespace(
        max_seq_length = 512,
        tokenizer = SimpleNamespace(num_special_tokens_to_add = lambda: 2),
    )
    monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)
    assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 510


def _gguf(tmp_path, entries):
    import io
    import struct

    buf = io.BytesIO()
    buf.write(b"GGUF" + struct.pack("<IQQ", 3, 0, len(entries)))
    for key, vtype, value in entries:
        raw = key.encode()
        buf.write(struct.pack("<Q", len(raw)) + raw + struct.pack("<I", vtype))
        if vtype == 8:
            raw = value.encode()
            buf.write(struct.pack("<Q", len(raw)) + raw)
        elif vtype == 9:
            elem_type, items = value
            buf.write(struct.pack("<IQ", elem_type, len(items)))
            for item in items:
                raw = item.encode()
                buf.write(struct.pack("<Q", len(raw)) + raw)
        elif vtype == 4:
            buf.write(struct.pack("<I", value))
        elif vtype == 10:
            buf.write(struct.pack("<Q", value))
    path = tmp_path / "embedder.gguf"
    path.write_bytes(buf.getvalue())
    return str(path)


def test_gguf_context_length_is_read_from_the_header(tmp_path):
    from core.rag import embed_llama_server

    path = _gguf(
        tmp_path,
        [
            ("general.architecture", 8, "bert"),
            ("tokenizer.ggml.tokens", 9, (8, ["[PAD]", "[CLS]", "hello"])),
            ("llama.context_length", 4, 4096),
            ("bert.context_length", 10, 512),
        ],
    )
    assert embed_llama_server._gguf_context_length(path) == 512
    assert embed_llama_server._gguf_context_length(str(tmp_path / "missing.gguf")) is None


def test_llama_max_tokens_comes_from_the_gguf_minus_its_special_tokens(tmp_path, monkeypatch):
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    backend._model_path = _gguf(
        tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
    )
    monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
    # Larger than the header, so the GGUF is still what binds; confirmed, so it caches.
    monkeypatch.setattr(backend, "_server_context", lambda: 4096)
    monkeypatch.setattr(backend, "_server_batch", lambda: 4096)
    posts = []

    def post(
        path,
        payload,
        model_name = None,
    ):
        posts.append((path, payload))
        return {"tokens": [101, 102]}

    monkeypatch.setattr(backend, "_post", post)
    assert backend.max_tokens() == 510
    assert backend.max_tokens() == 510
    assert posts == [("/tokenize", {"content": "", "add_special": True})]
    backend._adopt_model_path(backend._model_path, "unsloth/other-GGUF")
    assert backend._max_tokens is None


def test_llama_max_tokens_is_dropped_when_the_binary_is_swapped(tmp_path, monkeypatch):
    """The token limit depends on the binary's n_ctx and n_ubatch, so a binary swap must drop it."""
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    backend._model_path = _gguf(
        tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
    )
    backend._model_repo = "unsloth/bge-small-en-v1.5-GGUF"
    backend._max_tokens = 510
    backend._binary = "/opt/old/llama-server"
    backend._binary_path_revision = 1
    monkeypatch.setattr(backend, "_process_alive", lambda: True)
    monkeypatch.setattr(backend, "_kill_process", lambda: None)
    monkeypatch.setattr(backend, "_spawn", lambda model_name = None: None)
    monkeypatch.setattr(
        embed_llama_server.config,
        "effective_gguf_repo_for_embedding_model",
        lambda model: "unsloth/bge-small-en-v1.5-GGUF",
    )
    monkeypatch.setattr(
        embed_llama_server.config,
        "effective_embedding_model",
        lambda: "unsloth/bge-small-en-v1.5",
    )
    import utils.llama_cpp_path_settings as path_settings

    monkeypatch.setattr(path_settings, "custom_llama_cpp_path_revision", lambda: 2)

    backend._ensure_ready()

    assert backend._max_tokens is None
    assert backend._model_path is not None


def test_st_max_tokens_reserves_the_default_prompt(monkeypatch):
    tokenizer = SimpleNamespace(
        num_special_tokens_to_add = lambda: 2,
        encode = lambda text, add_special_tokens = False: text.split(),
    )
    model = SimpleNamespace(
        max_seq_length = 512,
        tokenizer = tokenizer,
        prompts = {"query": "Represent this sentence for retrieval:"},
        default_prompt_name = "query",
    )
    monkeypatch.setattr(rag_embeddings, "_get", lambda model_name = None: model)
    assert rag_embeddings._SentenceTransformersBackend().max_tokens() == 512 - 2 - 5


def test_llama_max_tokens_is_capped_by_the_running_context(tmp_path, monkeypatch):
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    backend._model_path = _gguf(
        tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 512)]
    )
    monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
    monkeypatch.setattr(backend, "_server_context", lambda: 256)
    monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
    assert backend.max_tokens() == 254


def test_llama_max_tokens_is_capped_by_the_ubatch_we_launched_with(tmp_path, monkeypatch):
    """/props has no batch field, so the launch ubatch caps the limit; past it llama-server 500s."""
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    backend._model_path = _gguf(
        tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
    )
    monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
    monkeypatch.setattr(backend, "_server_context", lambda: 8192)
    monkeypatch.setattr(
        backend, "_server_props", lambda: {"default_generation_settings": {"n_ctx": 8192}}
    )
    monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
    assert backend.max_tokens() == embed_llama_server._UBATCH_SIZE - 2
    assert "-ub" in backend._build_cmd("llama-server", "m.gguf", 1, use_gpu = False)


def test_only_a_smaller_reported_ubatch_lowers_the_launch_value(monkeypatch):
    """n_batch is the logical batch; llama.cpp takes min(n_batch, n_ubatch or n_batch), so
    against the -ub this backend launches with it is never the physical limit."""
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    ub = embed_llama_server._UBATCH_SIZE
    monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub // 2})
    assert backend._server_batch() == ub // 2
    monkeypatch.setattr(backend, "_server_props", lambda: {"n_ubatch": ub * 4})
    assert backend._server_batch() == ub
    monkeypatch.setattr(backend, "_server_props", lambda: {"n_batch": ub * 4})
    assert backend._server_batch() == ub


def test_a_stale_tagged_identity_is_refused_not_answered(studio_embedder):
    """The current space is llama-server; the client resubmits the sentence-transformers tag."""
    from fastapi import HTTPException

    studio_embedder.setattr(
        rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
    )
    studio_embedder.setattr(
        rag_embeddings, "embedding_identity", lambda model_name = None: f"llama-server:{MODEL}"
    )
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    # Warm index needed: cold, any slashed name counts as decisive.
    from core.inference import local_model_resolver

    studio_embedder.setattr(local_model_resolver, "index_is_built", lambda: True)
    studio_embedder.setattr(local_model_resolver, "resolve_local_gguf", lambda *a, **k: None)
    stale = IDENTITY
    assert inference_route._names_studio_embedder(stale) is None
    assert inference_route._reference_is_decisive(stale) is True
    with pytest.raises(HTTPException) as exc:
        asyncio.run(
            inference_route.openai_embeddings(
                _Request({"input": "alpha", "model": stale}), "tester"
            )
        )
    assert exc.value.status_code == 503


@pytest.mark.parametrize(
    ("body", "switched"),
    [
        ({"input": [""], "model": "org/chat-GGUF"}, False),
        ({"input": ["alpha", 7], "model": "org/chat-GGUF"}, False),
        ({"input": "alpha", "encoding_format": "int8", "model": "org/chat-GGUF"}, False),
        ({"input": [[1, 2, 3]], "model": "org/chat-GGUF"}, True),
    ],
)
def test_invalid_fallback_input_is_rejected_before_the_switch(studio_embedder, body, switched):
    seen = []

    async def record(request, current_subject, **_kwargs):
        seen.append(current_subject)
        return await request.json()

    studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)
    studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
    )
    assert _http_error(body).status_code == 400
    assert bool(seen) is switched


def test_local_path_models_are_not_exposed(studio_embedder, tmp_path):
    from core.rag.config import _escape_identity_segment

    model_dir = str(tmp_path / "bge")
    studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
    studio_embedder.setattr(
        rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
    )
    identity = f"sentence-transformers:{_escape_identity_segment(model_dir)}"
    studio_embedder.setattr(
        rag_embeddings,
        "encode_with_identity",
        lambda texts, **_kwargs: (_vectors(texts), identity),
    )
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    reported = _call({"input": "alpha"})["model"]
    assert reported.startswith("sentence-transformers:bge-") and str(tmp_path) not in reported
    studio_embedder.setattr(rag_embeddings, "max_tokens", lambda model_name = None: 3)
    error = _http_error({"input": "alphabet", "truncate": False})
    assert error.status_code == 400
    assert "bge" in error.detail and str(tmp_path) not in error.detail


@pytest.mark.parametrize("answers", [True, False])
def test_resident_embedding_gguf_answering_the_name_keeps_the_proxy(studio_embedder, answers):
    import httpx

    class _Client:
        async def post(self, *_args, **_kwargs):
            return httpx.Response(200, json = {"data": [{"embedding": [0.5]}], "model": "proxy"})

        async def aclose(self):
            return None

    _identity_names(studio_embedder)
    studio_embedder.setattr(inference_route, "_loaded_satisfies", lambda requested: answers)
    studio_embedder.setattr(inference_route, "_cancelable_nonstreaming_client", _Client)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(
            is_loaded = True,
            is_embedding_gguf = True,
            base_url = "http://llama.test",
            context_length = 512,
            model_identifier = f"{MODEL}-GGUF",
        ),
    )
    payload = _call({"input": "alpha", "model": MODEL})
    assert payload["model"] == ("proxy" if answers else IDENTITY)


def test_public_embedding_name_hides_any_local_path_shape():
    public = inference_route._public_embedding_name
    windows = public("C:\\models\\private\\bge")
    assert windows.startswith("bge-") and len(windows) == len("bge-") + 8
    assert "models" not in windows and "private" not in windows
    assert public("/tmp/models/bge/").startswith("bge-")
    assert public("/checkpoints/run-a/model") != public("/checkpoints/run-b/model")
    assert public("unsloth/bge-small-en-v1.5") == "unsloth/bge-small-en-v1.5"


def test_flat_token_array_passes_the_pre_switch_check(studio_embedder):
    seen = []

    async def record(request, current_subject, **_kwargs):
        seen.append(current_subject)
        return await request.json()

    studio_embedder.setattr(inference_route, "_should_validate_before_switch", lambda: True)
    studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
    studio_embedder.setattr(
        inference_route,
        "get_llama_cpp_backend",
        lambda: SimpleNamespace(is_loaded = True, is_embedding_gguf = False),
    )
    assert _http_error({"input": [1, 2, 3]}).status_code == 400
    assert seen == ["tester"]


def test_local_path_aliases_match_case_exactly(studio_embedder, tmp_path):
    model_dir = str(tmp_path / "Foo")
    studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
    studio_embedder.setattr(
        rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
    )
    assert inference_route._names_studio_embedder(model_dir)
    assert not inference_route._names_studio_embedder(str(tmp_path / "foo"))
    # An identity escapes the colons in its segments (a Windows path has one after the drive).
    segment = rag_config._escape_identity_segment(model_dir)
    assert inference_route._names_studio_embedder(f"sentence-transformers:{segment}")


def test_untagged_aliases_never_probe_the_backend(studio_embedder):
    def boom(*_args, **_kwargs):
        raise AssertionError("embedding_identity must not run for untagged names")

    _identity_names(studio_embedder)
    studio_embedder.setattr(rag_embeddings, "embedding_identity", boom)
    assert inference_route._names_studio_embedder(MODEL.upper())
    assert not inference_route._names_studio_embedder("text-embedding-3-small")


def test_tagged_identity_must_match_the_current_space(studio_embedder):
    _identity_names(studio_embedder)
    assert inference_route._names_studio_embedder(IDENTITY) == MODEL
    stale = f"llama-server:{MODEL}:{MODEL}-GGUF"
    assert inference_route._names_studio_embedder(stale) is None


def test_disconnected_client_leaves_the_queue_without_embedding(studio_embedder):
    lock = threading.Lock()
    gate = threading.Event()
    calls = {"n": 0}

    def encode(texts, **_kwargs):
        with lock:
            calls["n"] += 1
        gate.wait(timeout = 5)
        return _vectors(texts)

    class _Gone(_Request):
        async def is_disconnected(self):
            return True

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "encode", encode)
    cap = inference_route._STUDIO_EMBED_CONCURRENCY

    async def run():
        blockers = [
            asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "x"}), "t"))
            for _ in range(cap)
        ]
        for _ in range(500):
            await asyncio.sleep(0.01)
            if calls["n"] == cap:
                break
        with pytest.raises(HTTPException) as exc:
            await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")
        assert exc.value.status_code == 499
        assert calls["n"] == cap
        gate.set()
        await asyncio.gather(*blockers)
        response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")
        assert response.status_code == 200
        assert calls["n"] == cap + 1

    asyncio.run(run())


def test_batch_cap_applies_only_to_the_studio_fallback():
    from fastapi import HTTPException

    body = {"input": ["x"] * (inference_route._STUDIO_EMBED_MAX_INPUTS + 1)}
    assert len(inference_route._embeddings_items(body, tokens_ok = True)) == len(body["input"])
    with pytest.raises(HTTPException) as exc:
        inference_route._embeddings_texts(body)
    assert exc.value.status_code == 400


def test_a_decisively_named_model_is_refused_when_nothing_is_loaded(studio_embedder):
    # A reference clearly meant for this server 404s rather than being served by other weights.
    _identity_names(studio_embedder)
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(inference_route, "_reference_is_decisive", lambda name: True)
    error = _http_error({"input": "alpha", "model": "unsloth/embeddinggemma-300m-GGUF:Q8_0"})
    assert error.status_code in (404, 503)


def test_a_foreign_model_name_still_reaches_the_studio_embedder(studio_embedder):
    # The other half: a vendor id is no evidence, so it keeps falling through instead of 404ing.
    _identity_names(studio_embedder)
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    assert _call({"input": "alpha", "model": "text-embedding-3-small"})["model"] == IDENTITY


def test_reference_is_decisive_only_on_positive_evidence(monkeypatch):
    from core.inference import local_model_resolver as _resolver

    monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)
    # An explicit quant label is evidence no foreign id carries.
    assert inference_route._reference_is_decisive("unsloth/gemma-3-270m-it-GGUF:Q8_0") is True
    assert inference_route._reference_is_decisive("text-embedding-3-small") is False
    assert inference_route._reference_is_decisive("openai/text-embedding-3-large") is False


def test_advertised_local_identity_is_accepted_as_an_alias(studio_embedder, tmp_path):
    model_dir = str(tmp_path / "bge")
    studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: model_dir)
    studio_embedder.setattr(
        rag_config, "effective_gguf_repo_for_embedding_model", lambda model: f"{model}-GGUF"
    )
    from core.rag.config import _escape_identity_segment

    studio_embedder.setattr(
        rag_embeddings,
        "embedding_identity",
        lambda model_name = None: f"sentence-transformers:{_escape_identity_segment(model_dir)}",
    )
    public = inference_route._public_embedding_name(model_dir)
    assert inference_route._names_studio_embedder(f"sentence-transformers:{public}") == model_dir
    assert inference_route._names_studio_embedder(public) == model_dir
    assert inference_route._names_studio_embedder("bge-00000000") is None


def test_disconnected_client_is_dropped_even_with_a_free_permit(studio_embedder):
    calls = []

    class _Gone(_Request):
        async def is_disconnected(self):
            return True

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(
        rag_embeddings, "encode", lambda texts, **_: calls.append(texts) or _vectors(texts)
    )

    async def run():
        with pytest.raises(HTTPException) as exc:
            await inference_route.openai_embeddings(_Gone({"input": "gone"}), "t")
        assert exc.value.status_code == 499
        assert calls == []
        response = await inference_route.openai_embeddings(_Request({"input": "later"}), "t")
        assert response.status_code == 200

    asyncio.run(run())


def test_alias_match_pins_the_model_for_the_request(studio_embedder):
    settings = iter([MODEL, "unsloth/other-model", "unsloth/other-model"])
    seen = []

    def _encode_with_identity(
        texts,
        *,
        model_name = None,
        normalize = True,
    ):
        seen.append(model_name)
        return _vectors(texts), IDENTITY

    _identity_names(studio_embedder)
    studio_embedder.setattr(rag_config, "effective_embedding_model", lambda: next(settings))
    studio_embedder.setattr(rag_embeddings, "encode_with_identity", _encode_with_identity)
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    _call({"input": "alpha", "model": MODEL})
    assert seen == [MODEL]


def test_llama_max_tokens_never_exceeds_one_physical_batch(tmp_path, monkeypatch):
    # Embedding is non-causal, so llama.cpp refuses rather than splits over-batch inputs.
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    backend._model_path = _gguf(
        tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
    )
    monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
    monkeypatch.setattr(backend, "_server_context", lambda: 8192)
    monkeypatch.setattr(backend, "_server_batch", lambda: 512)
    monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
    assert backend.max_tokens() == 510


def test_the_embed_server_does_not_enlarge_its_batch(tmp_path):
    # n_vocab*n_ubatch*4 is allocated at startup, so max_tokens bounds the advert instead.
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    model = _gguf(tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)])
    cmd = backend._build_cmd("llama-server", model, 9999, use_gpu = True)
    assert "-b" not in cmd
    # Pinned at llama.cpp's own default, so nothing is allocated that was not already, and
    # max_tokens has a batch to bound the advert with (/props publishes none).
    assert cmd[cmd.index("-ub") + 1] == str(embed_llama_server._UBATCH_SIZE)


def test_an_unconfirmed_context_limit_is_not_cached(tmp_path, monkeypatch):
    # A header context can exceed the running one; freezing it outlives the /props correction.
    from core.rag import embed_llama_server

    backend = embed_llama_server.LlamaServerBackend()
    backend._model_path = _gguf(
        tmp_path, [("general.architecture", 8, "bert"), ("bert.context_length", 4, 8192)]
    )
    monkeypatch.setattr(backend, "_ensure_ready", lambda model_name = None: None)
    monkeypatch.setattr(backend, "_server_batch", lambda: None)
    monkeypatch.setattr(backend, "_post", lambda *a, **k: {"tokens": [101, 102]})
    monkeypatch.setattr(backend, "_server_context", lambda: None)
    assert backend.max_tokens() == 8190
    assert backend._max_tokens is None
    monkeypatch.setattr(backend, "_server_context", lambda: 512)
    monkeypatch.setattr(backend, "_server_batch", lambda: 512)
    assert backend.max_tokens() == 510
    assert backend._max_tokens == 510


def test_props_probe_never_raises_before_the_server_is_up():
    from core.rag import embed_llama_server

    # An un-started server has no port; the probe must swallow that and use the launch batch.
    assert (
        embed_llama_server.LlamaServerBackend()._server_batch() == embed_llama_server._UBATCH_SIZE
    )


def test_cancel_during_the_final_disconnect_probe_releases_the_permit(studio_embedder):
    lock = threading.Lock()
    gate = threading.Event()
    active = {"now": 0, "peak": 0}

    def encode(texts, **_kwargs):
        with lock:
            active["now"] += 1
            active["peak"] = max(active["peak"], active["now"])
        gate.wait(timeout = 5)
        with lock:
            active["now"] -= 1
        return _vectors(texts)

    class _Stuck(_Request):
        async def is_disconnected(self):
            await asyncio.sleep(3600)
            return False

    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    studio_embedder.setattr(rag_embeddings, "encode", encode)
    cap = inference_route._STUDIO_EMBED_CONCURRENCY

    async def run():
        stuck = asyncio.create_task(inference_route.openai_embeddings(_Stuck({"input": "x"}), "t"))
        await asyncio.sleep(0.05)
        stuck.cancel()
        with pytest.raises(asyncio.CancelledError):
            await stuck
        tasks = [
            asyncio.create_task(inference_route.openai_embeddings(_Request({"input": "y"}), "t"))
            for _ in range(cap)
        ]
        for _ in range(500):
            await asyncio.sleep(0.01)
            if active["now"] == cap:
                break
        assert active["now"] == cap
        gate.set()
        await asyncio.gather(*tasks)

    asyncio.run(run())


def test_a_boolean_is_not_a_token_id():
    # bool subclasses int, so `[true]` read as a token array could swap the resident GGUF.
    from fastapi import HTTPException

    with pytest.raises(HTTPException) as exc:
        inference_route._embeddings_items({"input": [True]}, tokens_ok = True)
    assert exc.value.status_code == 400
    with pytest.raises(HTTPException):
        inference_route._embeddings_items({"input": [[1, True, 3]]}, tokens_ok = True)
    assert inference_route._embeddings_items({"input": [1, 2, 3]}, tokens_ok = True) == [[1, 2, 3]]


def test_a_cold_index_does_not_make_a_local_name_foreign(monkeypatch):
    # Before the first scan resolve_local_gguf returns None for downloaded models; that is not foreign.
    from core.inference import local_model_resolver as _resolver

    monkeypatch.setattr(_resolver, "resolve_local_gguf", lambda ref, allow_scan = False: None)
    monkeypatch.setattr(_resolver, "index_is_built", lambda: False)
    monkeypatch.setattr(_resolver, "warm_index_soon", lambda: None)
    assert inference_route._reference_is_decisive("org/my-local-model") is True
    assert inference_route._reference_is_decisive("/models/local.gguf") is True
    assert inference_route._reference_is_decisive("text-embedding-3-small") is False

    # Once the index is built, absence really is evidence.
    monkeypatch.setattr(_resolver, "index_is_built", lambda: True)
    assert inference_route._reference_is_decisive("org/not-here") is False


@pytest.mark.parametrize("stash_embeds", [False, True])
def test_default_request_restores_the_chat_slot_only_for_an_embedding_stash(
    studio_embedder, stash_embeds
):
    seen = []

    async def record(request, current_subject, **_kwargs):
        seen.append(current_subject)
        return await request.json()

    studio_embedder.setattr(inference_route, "_auto_switch_from_request_body", record)
    studio_embedder.setattr(inference_route, "_stashed_gguf_embeds", lambda: stash_embeds)
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    payload = _call({"input": "alpha"})
    assert payload["model"] == IDENTITY
    assert seen == (["tester"] if stash_embeds else [])


def test_stashed_gguf_embeds_reads_the_stashed_model_header(monkeypatch, tmp_path):
    import core.inference.llama_keepwarm as keepwarm
    import core.inference.local_model_resolver as resolver

    monkeypatch.setattr(keepwarm, "get_last_unloaded_model", lambda: None)
    assert inference_route._stashed_gguf_embeds() is False
    monkeypatch.setattr(
        keepwarm, "get_last_unloaded_model", lambda: ("org/E-GGUF", "Q8_0", "org/E-GGUF")
    )
    monkeypatch.setattr(
        resolver, "resolve_local_gguf", lambda ref, **_: ("/tmp/e.gguf", "Q8_0", "org/E-GGUF")
    )

    class _Probe:
        is_embedding_gguf = True

        def _read_gguf_metadata(self, path):
            self.path = path

    monkeypatch.setattr(inference_route, "_probe_backend", _Probe)
    assert inference_route._stashed_gguf_embeds() is True


def test_resident_embedding_gguf_recuts_for_a_context_below_the_batch(studio_embedder):
    client, embedded, tokenized = _llama_overflow_server(6, 4)
    _resident_embedding_gguf(studio_embedder, client)
    payload = _call({"input": ["aa bbb c dddd e ff gg hh"], "model": "org/E-GGUF"})
    assert len(payload["data"]) == 1
    assert [body["input"] for body in embedded[1:]] == [
        [[_CLS, 2, 3, 1, 4, _SEP]],
        [[_CLS, 2, 3, _SEP]],
    ]
    assert len(tokenized) == 2


def test_resident_embedding_gguf_skips_tokenizing_strings_that_cannot_overflow(studio_embedder):
    client, embedded, tokenized = _llama_overflow_server(20, 512)
    _resident_embedding_gguf(studio_embedder, client)
    long = " ".join(["a"] * 30)
    _call({"input": [long, "x", "y z", "\ufdfa"], "model": "org/E-GGUF"})
    assert embedded[1]["input"][1:] == ["x", "y z", "\ufdfa"]
    assert set(tokenized) == {long, "\ufdfa"}


def test_resident_embedding_gguf_cancel_during_the_retry_is_499(studio_embedder):
    from fastapi import HTTPException

    server, _, _ = _llama_overflow_server(4, 512)
    request = _Request({"input": ["aa bbb c dddd e"], "model": "org/E-GGUF"})
    disconnected = False

    async def _is_disconnected():
        return disconnected

    request.is_disconnected = _is_disconnected

    class _Client(server):
        closed = False

        async def post(self, url, *args, **kwargs):
            nonlocal disconnected
            if self.closed:
                raise RuntimeError("Cannot send a request, as the client has been closed.")
            response = await super().post(url, *args, **kwargs)
            if response.status_code == 500:
                disconnected = True
                while not self.closed:
                    await asyncio.sleep(0.05)
            return response

        async def aclose(self):
            self.closed = True

    _resident_embedding_gguf(studio_embedder, _Client)
    with pytest.raises(HTTPException) as exc:
        asyncio.run(inference_route.openai_embeddings(request, "tester"))
    assert exc.value.status_code == 499


def test_batch_counts_tokens_in_one_call_with_the_same_usage(studio_embedder):
    calls = {"single": 0, "many": 0}

    def _counter(model_name = None):
        def _count(text):
            calls["single"] += 1
            return len(text)

        def _many(texts):
            calls["many"] += 1
            return [len(text) for text in texts]

        _count.many = _many
        return _count

    studio_embedder.setattr(rag_embeddings, "token_counter", _counter)
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    texts = [f"text {i} " * (i % 7 + 1) for i in range(64)]
    payload = _call({"input": texts})
    assert calls == {"single": 0, "many": 1}
    assert payload["usage"]["prompt_tokens"] == sum(len(text) for text in texts)
    assert [row["embedding"] for row in payload["data"]] == [[1.0, 0.0]] * 64


def test_float_rows_serialize_like_the_element_loop():
    rows = np.random.default_rng(0).standard_normal((4, 33)).astype(np.float32)
    rows[0, :3] = [np.nan, np.inf, -0.0]
    for row in rows:
        fast = inference_route._embedding_payload(row, "float")
        assert json.dumps(fast) == json.dumps([float(x) for x in row])
        assert all(type(x) is float for x in fast)
    assert inference_route._embedding_payload([0.5, 1], "float") == [0.5, 1.0]


class _Monitor:
    def __init__(self):
        self.calls = []

    def start(self, **_kwargs):
        return "mid"

    def fail(self, monitor_id, detail):
        self.calls.append(("fail", detail))

    def finish(self, monitor_id, *args):
        self.calls.append(("finish", *args))


def _counting_gguf_proxy(
    studio_embedder,
    body,
    status = 200,
):
    import httpx

    parses = []

    class _Response(httpx.Response):
        def json(self, **kwargs):
            parses.append(threading.get_ident())
            return super().json(**kwargs)

    class _Client:
        async def post(self, *_args, **_kwargs):
            return _Response(status, content = body)

        async def aclose(self):
            return None

    _resident_embedding_gguf(studio_embedder, _Client)
    return parses


def _upstream_body(dim = 4):
    rows = [{"object": "embedding", "index": 0, "embedding": [0.25] * dim}]
    usage = {"prompt_tokens": 3, "total_tokens": 3}
    return json.dumps({"object": "list", "data": rows, "usage": usage}).encode()


def test_gguf_proxy_skips_the_parse_when_nothing_reads_it(studio_embedder):
    body = _upstream_body()
    parses = _counting_gguf_proxy(studio_embedder, body)
    response = asyncio.run(
        inference_route.openai_embeddings(_Request({"input": "alpha"}), "tester")
    )
    assert response.body == body
    assert parses == []


@pytest.mark.parametrize("upstream", [_upstream_body(), b"{not json"], ids = ["json", "invalid"])
def test_gguf_proxy_parses_once_off_the_loop_for_monitor_and_dimensions(studio_embedder, upstream):
    parses = _counting_gguf_proxy(studio_embedder, upstream)
    monitor, seen = _Monitor(), []
    studio_embedder.setattr(inference_route, "api_monitor", monitor)
    studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)
    studio_embedder.setattr(
        inference_route,
        "_monitor_openai_chunk",
        lambda monitor_id, data, context_length = None, **_: seen.append(data),
    )
    request = _Request({"input": "alpha", "dimensions": 4})
    request.state.skip_api_monitor = False
    if upstream.startswith(b"{not"):
        assert _http_error_for(request).status_code == 400
        assert seen == [] and monitor.calls[0][0] == "fail"
    else:
        response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
        assert response.body == upstream
        assert seen == [json.loads(upstream)]
        assert monitor.calls == [("finish",)]
    assert len(parses) == 1 and parses[0] != threading.get_ident()


def _http_error_for(request):
    with pytest.raises(HTTPException) as exc:
        asyncio.run(inference_route.openai_embeddings(request, "tester"))
    return exc.value


@pytest.mark.parametrize("encoding_format", ["float", "base64"])
def test_studio_payload_is_serialized_off_the_loop_with_the_same_bytes(
    studio_embedder, encoding_format
):
    rows = np.random.default_rng(0).standard_normal((3, 2)).astype(np.float32)
    rows[0, 0] = np.nan
    threads = []
    real_payload = inference_route._embedding_payload

    def _payload(vector, fmt):
        threads.append(threading.get_ident())
        return real_payload(vector, fmt)

    studio_embedder.setattr(inference_route, "_embedding_payload", _payload)
    studio_embedder.setattr(rag_embeddings, "encode", lambda texts, **_: rows[: len(texts)])
    studio_embedder.setattr(
        inference_route, "get_llama_cpp_backend", lambda: SimpleNamespace(is_loaded = False)
    )
    body = {"input": ["a", "bb", "ccc"], "encoding_format": encoding_format}
    response = asyncio.run(inference_route.openai_embeddings(_Request(body), "tester"))
    expected = {
        "object": "list",
        "data": [
            {"object": "embedding", "index": i, "embedding": real_payload(row, encoding_format)}
            for i, row in enumerate(rows)
        ],
        "model": IDENTITY,
        "usage": {"prompt_tokens": 6, "total_tokens": 6},
    }
    assert response.body == json.dumps(expected).encode()
    assert threads and threading.get_ident() not in threads


_USAGE = {"prompt_tokens": 3, "total_tokens": 3}
_ROWS = [{"embedding": [0.25] * 4, "index": 0, "object": "embedding"}]


@pytest.mark.parametrize(
    "upstream, full_parses",
    [
        (json.dumps({"data": _ROWS, "model": "m", "object": "list", "usage": _USAGE}), 0),
        (json.dumps({"data": _ROWS, "model": '"usage": {}', "usage": _USAGE}, indent = 2), 0),
        (json.dumps({"usage": _USAGE, "data": _ROWS}), 1),
        (json.dumps({"data": _ROWS, "x": {"usage": _USAGE}}), 1),
        (json.dumps({"data": _ROWS}), 1),
    ],
)
def test_gguf_proxy_monitor_reads_usage_from_the_tail(studio_embedder, upstream, full_parses):
    upstream = upstream.encode()
    parses = _counting_gguf_proxy(studio_embedder, upstream)
    monitor, seen = _Monitor(), []
    studio_embedder.setattr(inference_route, "api_monitor", monitor)
    studio_embedder.setattr(inference_route, "_request_used_api_key", lambda request: False)
    studio_embedder.setattr(
        inference_route,
        "_monitor_openai_chunk",
        lambda monitor_id, data, context_length = None, **_: seen.append(data.get("usage")),
    )
    request = _Request({"input": "alpha"})
    request.state.skip_api_monitor = False
    response = asyncio.run(inference_route.openai_embeddings(request, "tester"))
    assert response.body == upstream
    assert seen == [json.loads(upstream).get("usage")]
    assert len(parses) == full_parses and monitor.calls == [("finish",)]