Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: move the RAG embedder to CPU when its GPU llama-server keeps crashing (#13739)
Daniel Han1461277ffb97
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.pyindex 1f42cefc5b0..84575a6e40f 100644--- a/studio/backend/core/rag/embed_llama_server.py+++ b/studio/backend/core/rag/embed_llama_server.py@@ -7,7 +7,7 @@ Opt-in (``RAG_EMBED_BACKEND=llama-server``). Runs a dedicated ``llama-server --esubprocess on its own port and calls its OpenAI-style ``/v1/embeddings`` + ``/tokenize``, fullyisolated from the chat backend.-Device is ``auto`` (GPU when present, else CPU, falling back to CPU if a GPU start fails);+Device is ``auto`` (GPU when present, else CPU, falling back to CPU if a GPU server fails);``RAG_EMBED_DEVICE`` forces it. We call only llama_cpp's *static* helpers (no torch), copying theinstance-coupled bits locally, since constructing a ``LlamaCppBackend`` runs an ``__init__`` reaperthat kills any Unsloth llama-server, so each request re-spawns ours if it died.@@ -206,8 +206,9 @@ class LlamaServerBackend:self._model_repo: str | None = Noneself._binary: str | None = Noneself._binary_path_revision: int | None = None- # Sticky after an auto GPU start fails: later spawns stay on CPU.+ # Sticky after an auto GPU start fails or a GPU server crashes twice on one request.self._force_cpu = False+ self._on_gpu = False# trust_env=False skips HTTP(S)_PROXY; full URLs per request survive a respawn.self._client = httpx.Client(timeout = config.EMBED_REQUEST_TIMEOUT_S, trust_env = False)self._api_key: str | None = None@@ -991,6 +992,7 @@ class LlamaServerBackend:**child_popen_kwargs(),)self._process = proc+ self._on_gpu = use_gpu# child_popen_kwargs() is empty on macOS, so the crash record is the only thing that can reap it# after a force quit.adopt_pid(proc.pid)@@ -1179,9 +1181,14 @@ class LlamaServerBackend:return resp.json()except (*_TRANSPORT_ERRORS, httpx.TimeoutException) as e:last_exc = e+ crashed = not isinstance(e, httpx.TimeoutException) and self._server_exited()if attempt == 0:self._restart(model_name)continue+ # A second crash is the GPU build failing on this input, not the reaper (#13663).+ if crashed and self._fall_back_to_cpu():+ self._restart(model_name)+ return self._post_active(path, payload, model_name)except httpx.HTTPStatusError as e:body = e.response.text[:500] if e.response is not None else ""raise RuntimeError(@@ -1189,6 +1196,32 @@ class LlamaServerBackend:) from eraise RuntimeError(f"llama-server embedder POST {path} failed after retry") from last_exc+ def _server_exited(self) -> bool:+ """Whether the server died under the request; logs its exit code and output tail."""+ proc = self._process+ if proc is None:+ return False+ try:+ code = proc.wait(timeout = 2)+ except subprocess.TimeoutExpired:+ return False+ if self._stdout_thread is not None:+ self._stdout_thread.join(timeout = 2)+ tail = "\n".join(self._stdout_lines[-30:])+ logger.warning(+ "llama-server embedder exited during a request (code %s). Last output:\n%s",+ code,+ tail[:2000],+ )+ return True++ def _fall_back_to_cpu(self) -> bool:+ if not self._on_gpu or self._force_cpu or config.embed_device_requires_gpu():+ return False+ logger.warning("llama-server embedder crashed twice on the GPU; continuing on CPU")+ self._force_cpu = True+ return True+def encode(self,texts,diff --git a/studio/backend/tests/test_rag_embed_llama_server.py b/studio/backend/tests/test_rag_embed_llama_server.pyindex 5df7a4636df..0815b2e2209 100644--- a/studio/backend/tests/test_rag_embed_llama_server.py+++ b/studio/backend/tests/test_rag_embed_llama_server.py@@ -727,6 +727,121 @@ def test_post_restarts_once_on_read_timeout(monkeypatch):assert restarts["n"] == 1 # timeout self-heals like a transport error+class _CrashingGpuServer:+ """Spawn and client stand-ins: a GPU server dies under every request (#13663), a CPU one answers."""++ def __init__(+ self,+ monkeypatch,+ backend,+ *,+ timeout = False,+ cpu_crashes = False,+ ):+ import httpx++ self.spawns = []+ monkeypatch.setattr(backend, "_ensure_ready", lambda *_a, **_k: None)+ monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))++ def fake_spawn_once(use_gpu, model_name = None):+ self.spawns.append("gpu" if use_gpu else "cpu")+ backend._process = _FakeProc(alive = True, returncode = 0)+ backend._on_gpu = use_gpu+ backend._port = 9000++ monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)+ fake_spawn_once(backend._use_gpu())++ class _Client:+ def post(+ self,+ url,+ json,+ headers = None,+ ):+ if backend._on_gpu or cpu_crashes:+ if timeout:+ raise httpx.ReadTimeout("timed out")+ backend._process._alive = False+ backend._process.returncode = 3221226505+ raise httpx.ReadError("connection reset by peer")++ class _R:+ def raise_for_status(self_inner):+ return None++ def json(self_inner):+ return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}++ return _R()++ backend._client = _Client()+++def test_a_gpu_server_crashing_twice_on_one_request_finishes_on_cpu(monkeypatch):+ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")+ b = LlamaServerBackend()+ fake = _CrashingGpuServer(monkeypatch, b)+ out = b._post("/v1/embeddings", {"input": ["x"]})+ assert out["data"][0]["embedding"] == [1.0, 0.0]+ assert fake.spawns == ["gpu", "gpu", "cpu"]+ b._post("/v1/embeddings", {"input": ["y"]})+ assert fake.spawns == ["gpu", "gpu", "cpu"]+++def test_a_settings_gpu_choice_still_falls_back_after_repeated_crashes(monkeypatch):+ import utils.embedding_model_settings as ems++ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")+ monkeypatch.setattr(ems, "get_stored_embedding_device", lambda: "gpu")+ b = LlamaServerBackend()+ fake = _CrashingGpuServer(monkeypatch, b)+ b._post("/v1/embeddings", {"input": ["x"]})+ assert fake.spawns == ["gpu", "gpu", "cpu"]+++def test_literal_gpu_env_raises_instead_of_moving_a_crashing_server_to_cpu(monkeypatch):+ monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")+ b = LlamaServerBackend()+ fake = _CrashingGpuServer(monkeypatch, b)+ with pytest.raises(RuntimeError, match = "failed after retry"):+ b._post("/v1/embeddings", {"input": ["x"]})+ assert fake.spawns == ["gpu", "gpu"]+ assert b._force_cpu is False+++def test_repeated_timeouts_on_a_live_gpu_server_do_not_move_it_to_cpu(monkeypatch):+ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")+ b = LlamaServerBackend()+ fake = _CrashingGpuServer(monkeypatch, b, timeout = True)+ with pytest.raises(RuntimeError, match = "failed after retry"):+ b._post("/v1/embeddings", {"input": ["x"]})+ assert fake.spawns == ["gpu", "gpu"]+ assert b._force_cpu is False+++def test_a_cpu_server_crashing_twice_raises_without_a_third_spawn(monkeypatch):+ monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")+ b = LlamaServerBackend()+ fake = _CrashingGpuServer(monkeypatch, b, cpu_crashes = True)+ with pytest.raises(RuntimeError, match = "failed after retry"):+ b._post("/v1/embeddings", {"input": ["x"]})+ assert fake.spawns == ["cpu", "cpu"]+++def test_a_crash_logs_the_server_exit_code_and_last_output(monkeypatch, caplog):+ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")+ b = LlamaServerBackend()+ _CrashingGpuServer(monkeypatch, b)+ b._stdout_lines = ["rocBLAS error from hip error code: 'hipErrorInvalidKernelFile':218"]+ with caplog.at_level("WARNING", logger = mod.logger.name):+ b._post("/v1/embeddings", {"input": ["x"]})+ assert "exited during a request (code 3221226505)" in caplog.text+ assert "hipErrorInvalidKernelFile" in caplog.text+ assert "continuing on CPU" in caplog.text++class TestTheEmbeddingLoaderChangesAreMacOsOnly:"""The RAG server's Linux and Windows launches must be as they were.