Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: move the RAG embedder to CPU when its GPU llama-server keeps crashing (#13739)

Daniel Han1461277ffb97
diff --git a/studio/backend/core/rag/embed_llama_server.py b/studio/backend/core/rag/embed_llama_server.py
index 1f42cefc5b0..84575a6e40f 100644
--- a/studio/backend/core/rag/embed_llama_server.py
+++ b/studio/backend/core/rag/embed_llama_server.py
@@ -7,7 +7,7 @@ Opt-in (``RAG_EMBED_BACKEND=llama-server``). Runs a dedicated ``llama-server --e
subprocess on its own port and calls its OpenAI-style ``/v1/embeddings`` + ``/tokenize``, fully
isolated from the chat backend.
-Device is ``auto`` (GPU when present, else CPU, falling back to CPU if a GPU start fails);
+Device is ``auto`` (GPU when present, else CPU, falling back to CPU if a GPU server fails);
``RAG_EMBED_DEVICE`` forces it. We call only llama_cpp's *static* helpers (no torch), copying the
instance-coupled bits locally, since constructing a ``LlamaCppBackend`` runs an ``__init__`` reaper
that kills any Unsloth llama-server, so each request re-spawns ours if it died.
@@ -206,8 +206,9 @@ class LlamaServerBackend:
self._model_repo: str | None = None
self._binary: str | None = None
self._binary_path_revision: int | None = None
- # Sticky after an auto GPU start fails: later spawns stay on CPU.
+ # Sticky after an auto GPU start fails or a GPU server crashes twice on one request.
self._force_cpu = False
+ self._on_gpu = False
# trust_env=False skips HTTP(S)_PROXY; full URLs per request survive a respawn.
self._client = httpx.Client(timeout = config.EMBED_REQUEST_TIMEOUT_S, trust_env = False)
self._api_key: str | None = None
@@ -991,6 +992,7 @@ class LlamaServerBackend:
**child_popen_kwargs(),
)
self._process = proc
+ self._on_gpu = use_gpu
# child_popen_kwargs() is empty on macOS, so the crash record is the only thing that can reap it
# after a force quit.
adopt_pid(proc.pid)
@@ -1179,9 +1181,14 @@ class LlamaServerBackend:
return resp.json()
except (*_TRANSPORT_ERRORS, httpx.TimeoutException) as e:
last_exc = e
+ crashed = not isinstance(e, httpx.TimeoutException) and self._server_exited()
if attempt == 0:
self._restart(model_name)
continue
+ # A second crash is the GPU build failing on this input, not the reaper (#13663).
+ if crashed and self._fall_back_to_cpu():
+ self._restart(model_name)
+ return self._post_active(path, payload, model_name)
except httpx.HTTPStatusError as e:
body = e.response.text[:500] if e.response is not None else ""
raise RuntimeError(
@@ -1189,6 +1196,32 @@ class LlamaServerBackend:
) from e
raise RuntimeError(f"llama-server embedder POST {path} failed after retry") from last_exc
+ def _server_exited(self) -> bool:
+ """Whether the server died under the request; logs its exit code and output tail."""
+ proc = self._process
+ if proc is None:
+ return False
+ try:
+ code = proc.wait(timeout = 2)
+ except subprocess.TimeoutExpired:
+ return False
+ if self._stdout_thread is not None:
+ self._stdout_thread.join(timeout = 2)
+ tail = "\n".join(self._stdout_lines[-30:])
+ logger.warning(
+ "llama-server embedder exited during a request (code %s). Last output:\n%s",
+ code,
+ tail[:2000],
+ )
+ return True
+
+ def _fall_back_to_cpu(self) -> bool:
+ if not self._on_gpu or self._force_cpu or config.embed_device_requires_gpu():
+ return False
+ logger.warning("llama-server embedder crashed twice on the GPU; continuing on CPU")
+ self._force_cpu = True
+ return True
+
def encode(
self,
texts,
diff --git a/studio/backend/tests/test_rag_embed_llama_server.py b/studio/backend/tests/test_rag_embed_llama_server.py
index 5df7a4636df..0815b2e2209 100644
--- a/studio/backend/tests/test_rag_embed_llama_server.py
+++ b/studio/backend/tests/test_rag_embed_llama_server.py
@@ -727,6 +727,121 @@ def test_post_restarts_once_on_read_timeout(monkeypatch):
assert restarts["n"] == 1 # timeout self-heals like a transport error
+class _CrashingGpuServer:
+ """Spawn and client stand-ins: a GPU server dies under every request (#13663), a CPU one answers."""
+
+ def __init__(
+ self,
+ monkeypatch,
+ backend,
+ *,
+ timeout = False,
+ cpu_crashes = False,
+ ):
+ import httpx
+
+ self.spawns = []
+ monkeypatch.setattr(backend, "_ensure_ready", lambda *_a, **_k: None)
+ monkeypatch.setattr(LlamaServerBackend, "_gpu_available", staticmethod(lambda: True))
+
+ def fake_spawn_once(use_gpu, model_name = None):
+ self.spawns.append("gpu" if use_gpu else "cpu")
+ backend._process = _FakeProc(alive = True, returncode = 0)
+ backend._on_gpu = use_gpu
+ backend._port = 9000
+
+ monkeypatch.setattr(backend, "_spawn_once", fake_spawn_once)
+ fake_spawn_once(backend._use_gpu())
+
+ class _Client:
+ def post(
+ self,
+ url,
+ json,
+ headers = None,
+ ):
+ if backend._on_gpu or cpu_crashes:
+ if timeout:
+ raise httpx.ReadTimeout("timed out")
+ backend._process._alive = False
+ backend._process.returncode = 3221226505
+ raise httpx.ReadError("connection reset by peer")
+
+ class _R:
+ def raise_for_status(self_inner):
+ return None
+
+ def json(self_inner):
+ return {"data": [{"index": 0, "embedding": [1.0, 0.0]}]}
+
+ return _R()
+
+ backend._client = _Client()
+
+
+def test_a_gpu_server_crashing_twice_on_one_request_finishes_on_cpu(monkeypatch):
+ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
+ b = LlamaServerBackend()
+ fake = _CrashingGpuServer(monkeypatch, b)
+ out = b._post("/v1/embeddings", {"input": ["x"]})
+ assert out["data"][0]["embedding"] == [1.0, 0.0]
+ assert fake.spawns == ["gpu", "gpu", "cpu"]
+ b._post("/v1/embeddings", {"input": ["y"]})
+ assert fake.spawns == ["gpu", "gpu", "cpu"]
+
+
+def test_a_settings_gpu_choice_still_falls_back_after_repeated_crashes(monkeypatch):
+ import utils.embedding_model_settings as ems
+
+ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
+ monkeypatch.setattr(ems, "get_stored_embedding_device", lambda: "gpu")
+ b = LlamaServerBackend()
+ fake = _CrashingGpuServer(monkeypatch, b)
+ b._post("/v1/embeddings", {"input": ["x"]})
+ assert fake.spawns == ["gpu", "gpu", "cpu"]
+
+
+def test_literal_gpu_env_raises_instead_of_moving_a_crashing_server_to_cpu(monkeypatch):
+ monkeypatch.setattr(config, "EMBED_DEVICE", "gpu")
+ b = LlamaServerBackend()
+ fake = _CrashingGpuServer(monkeypatch, b)
+ with pytest.raises(RuntimeError, match = "failed after retry"):
+ b._post("/v1/embeddings", {"input": ["x"]})
+ assert fake.spawns == ["gpu", "gpu"]
+ assert b._force_cpu is False
+
+
+def test_repeated_timeouts_on_a_live_gpu_server_do_not_move_it_to_cpu(monkeypatch):
+ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
+ b = LlamaServerBackend()
+ fake = _CrashingGpuServer(monkeypatch, b, timeout = True)
+ with pytest.raises(RuntimeError, match = "failed after retry"):
+ b._post("/v1/embeddings", {"input": ["x"]})
+ assert fake.spawns == ["gpu", "gpu"]
+ assert b._force_cpu is False
+
+
+def test_a_cpu_server_crashing_twice_raises_without_a_third_spawn(monkeypatch):
+ monkeypatch.setattr(config, "EMBED_DEVICE", "cpu")
+ b = LlamaServerBackend()
+ fake = _CrashingGpuServer(monkeypatch, b, cpu_crashes = True)
+ with pytest.raises(RuntimeError, match = "failed after retry"):
+ b._post("/v1/embeddings", {"input": ["x"]})
+ assert fake.spawns == ["cpu", "cpu"]
+
+
+def test_a_crash_logs_the_server_exit_code_and_last_output(monkeypatch, caplog):
+ monkeypatch.setattr(config, "EMBED_DEVICE", "auto")
+ b = LlamaServerBackend()
+ _CrashingGpuServer(monkeypatch, b)
+ b._stdout_lines = ["rocBLAS error from hip error code: 'hipErrorInvalidKernelFile':218"]
+ with caplog.at_level("WARNING", logger = mod.logger.name):
+ b._post("/v1/embeddings", {"input": ["x"]})
+ assert "exited during a request (code 3221226505)" in caplog.text
+ assert "hipErrorInvalidKernelFile" in caplog.text
+ assert "continuing on CPU" in caplog.text
+
+
class TestTheEmbeddingLoaderChangesAreMacOsOnly:
"""The RAG server's Linux and Windows launches must be as they were.