Skip to content

Agent Zero AI framework

Reuse preloaded local embedding models

Alessandro5a96eda07873
diff --git a/AGENTS.md b/AGENTS.md
index c0fd03f43..111053d87 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -29,6 +29,7 @@
- Use Linux paths and commands in examples.
- When a live Dockerized Agent Zero target is explicitly named, verify that exact runtime instead of assuming a fixed localhost port.
- Message-loop completion flows through a response tool with `break_loop`; plain or malformed Chat Completions text enters repair, and native Responses output text is normalized through the same response-tool path.
+- Reuse the startup-preloaded local embedding model for matching runtime configurations; wrappers retain their own rate-limit configuration while sharing the underlying inference model.
- Prompt Markdown may retain fenced JSON examples for readability; final system-prompt rendering removes only their JSON fence markers before model calls and preserves non-JSON fences.
- Copy live core-plugin changes back into tracked source under `plugins/`.
- Develop new custom plugins under ignored `usr/plugins/`; tracked bundled plugins live under `plugins/`.
diff --git a/models.py b/models.py
index e1fa5c423..1e354e73a 100644
--- a/models.py
+++ b/models.py
@@ -1,7 +1,9 @@
from dataclasses import dataclass, field
from enum import Enum
+import json
import logging
import os
+import threading
from typing import (
Any,
Awaitable,
@@ -828,6 +830,23 @@ class LiteLLMEmbeddingWrapper(Embeddings):
return item.get("embedding") if isinstance(item, dict) else item.embedding # type: ignore
+_LOCAL_EMBEDDING_MODELS: dict[tuple[str, str], SentenceTransformer] = {}
+_LOCAL_EMBEDDING_MODELS_LOCK = threading.Lock()
+
+
+def _get_local_embedding_model(
+ model: str, kwargs: dict[str, Any]
+) -> SentenceTransformer:
+ key = (model, json.dumps(kwargs, sort_keys=True, default=repr))
+ with _LOCAL_EMBEDDING_MODELS_LOCK:
+ cached = _LOCAL_EMBEDDING_MODELS.get(key)
+ if cached is None:
+ cached = SentenceTransformer(model, **kwargs)
+ _LOCAL_EMBEDDING_MODELS.clear()
+ _LOCAL_EMBEDDING_MODELS[key] = cached
+ return cached
+
+
class LocalSentenceTransformerWrapper(Embeddings):
"""Local wrapper for sentence-transformers models to avoid HuggingFace API calls"""
@@ -856,7 +875,7 @@ class LocalSentenceTransformerWrapper(Embeddings):
}
st_kwargs = {k: v for k, v in (kwargs or {}).items() if k in st_allowed_keys}
- self.model = SentenceTransformer(model, **st_kwargs)
+ self.model = _get_local_embedding_model(model, st_kwargs)
self.model_name = model
self.a0_model_conf = model_config
diff --git a/preload.py b/preload.py
index b0287a160..914c41969 100644
--- a/preload.py
+++ b/preload.py
@@ -25,7 +25,10 @@ async def preload():
emb_cfg = get_embedding_model_config_object()
if emb_cfg.provider.lower() == "huggingface":
emb_mod = models.get_embedding_model(
- "huggingface", emb_cfg.name
+ emb_cfg.provider,
+ emb_cfg.name,
+ model_config=emb_cfg,
+ **emb_cfg.build_kwargs(),
)
emb_txt = await emb_mod.aembed_query("test")
return emb_txt
diff --git a/tests/test_embedding_model_preload.py b/tests/test_embedding_model_preload.py
new file mode 100644
index 000000000..50f4634de
--- /dev/null
+++ b/tests/test_embedding_model_preload.py
@@ -0,0 +1,189 @@
+from concurrent.futures import ThreadPoolExecutor
+import threading
+from types import SimpleNamespace
+
+import pytest
+
+import models
+
+
+def _clear_local_embedding_models():
+ with models._LOCAL_EMBEDDING_MODELS_LOCK:
+ models._LOCAL_EMBEDDING_MODELS.clear()
+
+
+def test_local_embedding_preload_is_reused_with_runtime_model_config(monkeypatch):
+ created = []
+
+ class FakeSentenceTransformer:
+ def __init__(self, model, **kwargs):
+ created.append((model, kwargs))
+
+ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)
+ _clear_local_embedding_models()
+
+ try:
+ preload = models.LocalSentenceTransformerWrapper(
+ "huggingface",
+ "sentence-transformers/example",
+ device="cpu",
+ model_kwargs={"revision": "stable", "trust_remote_code": False},
+ )
+ runtime_config = SimpleNamespace(name="runtime")
+ runtime = models.LocalSentenceTransformerWrapper(
+ "huggingface",
+ "sentence-transformers/example",
+ model_config=runtime_config,
+ model_kwargs={"trust_remote_code": False, "revision": "stable"},
+ device="cpu",
+ )
+
+ assert runtime.model is preload.model
+ assert runtime.a0_model_conf is runtime_config
+ assert created == [
+ (
+ "example",
+ {
+ "device": "cpu",
+ "model_kwargs": {
+ "revision": "stable",
+ "trust_remote_code": False,
+ },
+ },
+ )
+ ]
+ finally:
+ _clear_local_embedding_models()
+
+
+def test_local_embedding_cache_tracks_effective_constructor_options(monkeypatch):
+ created = []
+
+ class FakeSentenceTransformer:
+ def __init__(self, model, **kwargs):
+ created.append((model, kwargs))
+
+ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)
+ _clear_local_embedding_models()
+
+ try:
+ first = models.LocalSentenceTransformerWrapper(
+ "huggingface", "sentence-transformers/example", device="cpu"
+ )
+ second = models.LocalSentenceTransformerWrapper(
+ "huggingface", "sentence-transformers/example", device="cuda"
+ )
+
+ assert second.model is not first.model
+ assert created == [
+ ("example", {"device": "cpu"}),
+ ("example", {"device": "cuda"}),
+ ]
+ assert len(models._LOCAL_EMBEDDING_MODELS) == 1
+ finally:
+ _clear_local_embedding_models()
+
+
+def test_concurrent_preload_and_runtime_share_one_model(monkeypatch):
+ created = []
+ construction_started = threading.Event()
+ release_construction = threading.Event()
+
+ class FakeSentenceTransformer:
+ def __init__(self, model, **kwargs):
+ created.append((model, kwargs))
+ construction_started.set()
+ assert release_construction.wait(timeout=2)
+
+ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)
+ _clear_local_embedding_models()
+
+ try:
+ with ThreadPoolExecutor(max_workers=2) as executor:
+ first = executor.submit(
+ models.LocalSentenceTransformerWrapper,
+ "huggingface",
+ "sentence-transformers/example",
+ )
+ assert construction_started.wait(timeout=2)
+ second = executor.submit(
+ models.LocalSentenceTransformerWrapper,
+ "huggingface",
+ "sentence-transformers/example",
+ )
+ release_construction.set()
+
+ assert second.result().model is first.result().model
+
+ assert created == [("example", {})]
+ finally:
+ release_construction.set()
+ _clear_local_embedding_models()
+
+
+def test_failed_model_change_keeps_the_working_cached_model(monkeypatch):
+ created = []
+
+ class FakeSentenceTransformer:
+ def __init__(self, model, **kwargs):
+ created.append((model, kwargs))
+ if model == "broken":
+ raise RuntimeError("model unavailable")
+
+ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)
+ _clear_local_embedding_models()
+
+ try:
+ working = models.LocalSentenceTransformerWrapper(
+ "huggingface", "sentence-transformers/working"
+ )
+ with pytest.raises(RuntimeError, match="model unavailable"):
+ models.LocalSentenceTransformerWrapper(
+ "huggingface", "sentence-transformers/broken"
+ )
+ reused = models.LocalSentenceTransformerWrapper(
+ "huggingface", "sentence-transformers/working"
+ )
+
+ assert reused.model is working.model
+ assert created == [("working", {}), ("broken", {})]
+ finally:
+ _clear_local_embedding_models()
+
+
+async def test_preload_uses_the_runtime_embedding_configuration(monkeypatch):
+ import preload
+ from plugins._model_config.helpers import model_config
+
+ config = SimpleNamespace(
+ provider="huggingface",
+ name="sentence-transformers/example",
+ build_kwargs=lambda: {"device": "cpu"},
+ )
+ calls = []
+ embedded = []
+
+ class FakeEmbeddings:
+ async def aembed_query(self, text):
+ embedded.append(text)
+
+ def get_embedding_model(provider, name, **kwargs):
+ calls.append((provider, name, kwargs))
+ return FakeEmbeddings()
+
+ monkeypatch.setattr(
+ model_config, "get_embedding_model_config_object", lambda: config
+ )
+ monkeypatch.setattr(preload.models, "get_embedding_model", get_embedding_model)
+
+ await preload.preload()
+
+ assert calls == [
+ (
+ "huggingface",
+ "sentence-transformers/example",
+ {"model_config": config, "device": "cpu"},
+ )
+ ]
+ assert embedded == ["test"]