Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Settings > Documents & RAG > Embedding device: RAG_EMBED_DEVICE first, then the owner's saved choice."""
from pathlib import Path
import sys
from types import SimpleNamespace
import pytest
_BACKEND_DIR = str(Path(__file__).resolve().parent.parent)
if _BACKEND_DIR not in sys.path:
sys.path.insert(0, _BACKEND_DIR)
from fastapi import HTTPException
from core.rag import config, embeddings
from core.rag.embed_llama_server import LlamaServerBackend
from routes import settings as settings_routes
from storage import studio_db
from utils import embedding_model_settings as ems
from utils.account_context import AccountContext, bind_account, reset_account
from utils.hardware.hardware import DeviceType
ALICE = AccountContext("a" * 32, "alice")
@pytest.fixture(autouse = True)
def isolated_home(monkeypatch, tmp_path):
monkeypatch.setenv("UNSLOTH_STUDIO_HOME", str(tmp_path))
monkeypatch.setattr(studio_db, "_schema_ready", set())
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
ems.forget_cached_embedding_device()
yield
ems.forget_cached_embedding_device()
@pytest.fixture
def released(monkeypatch):
calls = []
monkeypatch.setattr(embeddings, "release_backend", lambda: calls.append(1) or True)
return calls
def _put(device):
return settings_routes.update_embedding_device(
settings_routes.EmbeddingDevicePayload(device = device), current_subject = "owner"
)
def test_unset_keeps_each_backends_hardware_default():
assert config.embed_device_preference() == "auto"
assert config.embed_device_locked() is False
def test_saved_gpu_moves_every_model_to_the_gpu(monkeypatch, released):
"""Both backends read the one preference, so the choice is not tied to a model."""
monkeypatch.setattr(embeddings, "get_device", lambda: DeviceType.CUDA)
assert embeddings._device() == "cpu"
response = _put("gpu")
assert response.embedding_device == "gpu"
assert response.embedding_device_locked is False
assert released == [1]
assert embeddings._device() == "cuda"
assert LlamaServerBackend()._use_gpu() is True
# A Settings choice is a preference: a failed GPU start still falls back to CPU.
assert config.embed_device_requires_gpu() is False
def test_saved_cpu_keeps_llama_server_off_the_gpu(monkeypatch, released):
monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
assert LlamaServerBackend()._use_gpu() is True
_put("cpu")
assert LlamaServerBackend()._use_gpu() is False
def test_saving_the_same_device_again_keeps_the_embedder_resident(released):
_put("gpu")
_put("gpu")
assert released == [1]
_put("auto")
assert released == [1, 1]
assert config.embed_device_preference() == "auto"
def test_the_choice_is_installation_wide(released):
_put("gpu")
token = bind_account(ALICE)
try:
assert config.embed_device_preference() == "gpu"
finally:
reset_account(token)
@pytest.mark.parametrize("env, expected", [("gpu", "gpu"), (" cuda ", "gpu"), ("cpu", "cpu")])
def test_a_device_in_the_environment_wins_and_locks_settings(monkeypatch, released, env, expected):
ems.set_embedding_device("cpu" if expected == "gpu" else "gpu")
monkeypatch.setattr(config, "EMBED_DEVICE", env)
assert config.embed_device_preference() == expected
assert settings_routes._embedding_device_state() == (expected, True)
with pytest.raises(HTTPException) as exc:
_put("auto")
assert exc.value.status_code == 409
assert released == []
def test_environment_auto_leaves_the_choice_to_settings(monkeypatch, released):
monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
_put("gpu")
assert config.embed_device_preference() == "gpu"
def test_an_unrecognized_stored_value_reads_as_auto():
studio_db.upsert_app_settings({ems.EMBEDDING_DEVICE_SETTING_KEY: "banana"})
assert ems.get_stored_embedding_device() == "auto"
with pytest.raises(ValueError):
ems.set_embedding_device("banana")
def test_a_read_overlapping_a_save_does_not_cache_the_old_device(monkeypatch, released):
"""A settings poll that read the old value before a PUT landed must not pin it in the cache."""
real_get = studio_db.get_app_setting
def slow_get(key, fallback = None):
value = real_get(key, fallback)
if key == ems.EMBEDDING_DEVICE_SETTING_KEY and value is None:
# The PUT commits between this reader's DB read and its cache fill.
monkeypatch.setattr(studio_db, "get_app_setting", real_get)
_put("gpu")
return value
monkeypatch.setattr(studio_db, "get_app_setting", slow_get)
# Frozen clock: the stale entry must still be inside its TTL when the next reader asks.
monkeypatch.setattr(ems, "time", SimpleNamespace(monotonic = lambda: 1000.0))
assert ems.get_stored_embedding_device() == "auto"
assert config.embed_device_preference() == "gpu"
@pytest.mark.parametrize("env, retried", [("auto", True), ("gpu", False)])
def test_a_failed_gpu_load_retries_on_cpu_unless_the_environment_insists(
monkeypatch, tmp_path, released, env, retried
):
"""Out of VRAM beside a chat model: Settings' GPU falls back to CPU in the same embedding space."""
attempts = []
class FakeSentenceTransformer:
def __init__(self, name, **kwargs):
attempts.append(kwargs["device"])
if kwargs["device"] != "cpu":
raise RuntimeError("CUDA out of memory")
monkeypatch.setitem(
sys.modules,
"sentence_transformers",
SimpleNamespace(SentenceTransformer = FakeSentenceTransformer),
)
monkeypatch.setattr(embeddings, "_install_torchao_stub_once", lambda: None)
monkeypatch.setattr(embeddings, "_guard_model_security", lambda *_a, **_k: None)
monkeypatch.setattr(embeddings, "_load_device", lambda: "cuda")
monkeypatch.setattr(
"utils.hf_cache_settings.active_hf_hub_cache", lambda: str(tmp_path / "hub")
)
monkeypatch.setattr(embeddings, "_model", None)
monkeypatch.setattr(embeddings, "_name", None)
_put("gpu")
monkeypatch.setattr(config, "EMBED_DEVICE", env)
if retried:
embeddings._get("Org/Embedder")
assert attempts == ["cuda", "cpu"]
else:
with pytest.raises(RuntimeError, match = "out of memory"):
embeddings._get("Org/Embedder")
assert attempts == ["cuda"]