Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(embedder): serialize shared local GGUF context calls (#5548)

Evo6dd0727680b6
diff --git a/openviking/models/embedder/local_embedders.py b/openviking/models/embedder/local_embedders.py
index 5d6819645..350f5b8e0 100644
--- a/openviking/models/embedder/local_embedders.py
+++ b/openviking/models/embedder/local_embedders.py
@@ -8,6 +8,7 @@ import importlib
import os
from dataclasses import dataclass
from pathlib import Path
+from threading import Lock
from typing import Any, Dict, List, Optional
import requests
@@ -106,6 +107,7 @@ class LocalDenseEmbedder(DenseEmbedderBase):
self._resolved_model_path = self._resolve_model_path()
self._llama = self._load_model()
+ self._llama_lock = Lock()
def _import_llama(self):
try:
@@ -191,8 +193,10 @@ class LocalDenseEmbedder(DenseEmbedderBase):
raise RuntimeError("Unexpected llama-cpp-python embedding response format")
def _embed_formatted_text(self, formatted: str) -> EmbedResult:
- payload = self._llama.create_embedding(formatted)
- return EmbedResult(dense_vector=self._extract_embedding(payload))
+ # llama.cpp mutates the shared context, including during native calls that release the GIL.
+ with self._llama_lock:
+ payload = self._llama.create_embedding(formatted)
+ return EmbedResult(dense_vector=self._extract_embedding(payload))
def embed(self, text: str, is_query: bool = False) -> EmbedResult:
formatted = self._format_text(text, is_query=is_query)
@@ -219,6 +223,7 @@ class LocalDenseEmbedder(DenseEmbedderBase):
return self._dimension
def close(self):
- close_fn = getattr(self._llama, "close", None)
- if callable(close_fn):
- close_fn()
+ with self._llama_lock:
+ close_fn = getattr(self._llama, "close", None)
+ if callable(close_fn):
+ close_fn()
diff --git a/tests/unit/test_local_embedder.py b/tests/unit/test_local_embedder.py
index c36c30804..d945ff2ae 100644
--- a/tests/unit/test_local_embedder.py
+++ b/tests/unit/test_local_embedder.py
@@ -1,7 +1,9 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
+from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
+from threading import Event, Lock
from types import SimpleNamespace
import pytest
@@ -108,11 +110,11 @@ def test_local_embedder_downloads_default_model_and_prefixes_query(monkeypatch,
downloaded["count"] += 1
return _FakeResponse(b"gguf")
+ monkeypatch.setattr("openviking.models.embedder.local_embedders.requests.get", _fake_get)
monkeypatch.setattr(
"openviking.models.embedder.local_embedders.importlib.import_module",
lambda _name: SimpleNamespace(Llama=_FakeLlama),
)
- monkeypatch.setattr("openviking.models.embedder.local_embedders.requests.get", _fake_get)
embedder = LocalDenseEmbedder(cache_dir=str(tmp_path))
@@ -122,3 +124,70 @@ def test_local_embedder_downloads_default_model_and_prefixes_query(monkeypatch,
result = embedder.embed("测试问题", is_query=True)
assert len(result.dense_vector) == 512
assert _FakeLlama.inputs[-1] == f"{DEFAULT_BGE_ZH_QUERY_INSTRUCTION}测试问题"
+
+
[email protected]("concurrent_operation", ["embed", "close"])
[email protected]("fail_first", [False, True])
+def test_local_embedder_serializes_native_calls(
+ monkeypatch, tmp_path, concurrent_operation, fail_first
+):
+ entered = Event()
+ release = Event()
+ overlapping = Event()
+ state_lock = Lock()
+ active = 0
+
+ class BlockingLlama(_FakeLlama):
+ def create_embedding(self, payload):
+ nonlocal active
+ with state_lock:
+ active += 1
+ if active > 1:
+ overlapping.set()
+ entered.set()
+ assert release.wait(5)
+ with state_lock:
+ active -= 1
+ if fail_first and payload == "first":
+ raise ValueError("native embedding failed")
+ return super().create_embedding(payload)
+
+ def close(self):
+ with state_lock:
+ if active:
+ overlapping.set()
+
+ model_path = tmp_path / "model.gguf"
+ model_path.write_bytes(b"gguf")
+ monkeypatch.setattr(
+ "openviking.models.embedder.local_embedders.importlib.import_module",
+ lambda _name: SimpleNamespace(Llama=BlockingLlama),
+ )
+ embedder = LocalDenseEmbedder(model_path=str(model_path), config={"max_retries": 0})
+ started = Event()
+
+ def run_second_call():
+ started.set()
+ if concurrent_operation == "embed":
+ return embedder.embed("second", is_query=True)
+ return embedder.close()
+
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ first = executor.submit(embedder.embed, "first")
+ try:
+ assert entered.wait(5)
+ second = executor.submit(run_second_call)
+ assert started.wait(5)
+ assert not overlapping.wait(0.1)
+ finally:
+ release.set()
+ if fail_first:
+ with pytest.raises(RuntimeError, match="native embedding failed"):
+ first.result(timeout=5)
+ else:
+ assert first.result(timeout=5).dense_vector == [0.1] * 512
+ result = second.result(timeout=5)
+ if concurrent_operation == "embed":
+ assert result.dense_vector == [0.1] * 512
+ assert BlockingLlama.inputs[-1] == f"{DEFAULT_BGE_ZH_QUERY_INSTRUCTION}second"
+ assert not overlapping.is_set()