Agent Zero AI framework
Reuse preloaded local embedding models
Alessandro5a96eda07873
diff --git a/AGENTS.md b/AGENTS.mdindex c0fd03f43..111053d87 100644--- a/AGENTS.md+++ b/AGENTS.md@@ -29,6 +29,7 @@- Use Linux paths and commands in examples.- When a live Dockerized Agent Zero target is explicitly named, verify that exact runtime instead of assuming a fixed localhost port.- Message-loop completion flows through a response tool with `break_loop`; plain or malformed Chat Completions text enters repair, and native Responses output text is normalized through the same response-tool path.+- Reuse the startup-preloaded local embedding model for matching runtime configurations; wrappers retain their own rate-limit configuration while sharing the underlying inference model.- Prompt Markdown may retain fenced JSON examples for readability; final system-prompt rendering removes only their JSON fence markers before model calls and preserves non-JSON fences.- Copy live core-plugin changes back into tracked source under `plugins/`.- Develop new custom plugins under ignored `usr/plugins/`; tracked bundled plugins live under `plugins/`.diff --git a/models.py b/models.pyindex e1fa5c423..1e354e73a 100644--- a/models.py+++ b/models.py@@ -1,7 +1,9 @@from dataclasses import dataclass, fieldfrom enum import Enum+import jsonimport loggingimport os+import threadingfrom typing import (Any,Awaitable,@@ -828,6 +830,23 @@ class LiteLLMEmbeddingWrapper(Embeddings):return item.get("embedding") if isinstance(item, dict) else item.embedding # type: ignore+_LOCAL_EMBEDDING_MODELS: dict[tuple[str, str], SentenceTransformer] = {}+_LOCAL_EMBEDDING_MODELS_LOCK = threading.Lock()+++def _get_local_embedding_model(+ model: str, kwargs: dict[str, Any]+) -> SentenceTransformer:+ key = (model, json.dumps(kwargs, sort_keys=True, default=repr))+ with _LOCAL_EMBEDDING_MODELS_LOCK:+ cached = _LOCAL_EMBEDDING_MODELS.get(key)+ if cached is None:+ cached = SentenceTransformer(model, **kwargs)+ _LOCAL_EMBEDDING_MODELS.clear()+ _LOCAL_EMBEDDING_MODELS[key] = cached+ return cached++class LocalSentenceTransformerWrapper(Embeddings):"""Local wrapper for sentence-transformers models to avoid HuggingFace API calls"""@@ -856,7 +875,7 @@ class LocalSentenceTransformerWrapper(Embeddings):}st_kwargs = {k: v for k, v in (kwargs or {}).items() if k in st_allowed_keys}- self.model = SentenceTransformer(model, **st_kwargs)+ self.model = _get_local_embedding_model(model, st_kwargs)self.model_name = modelself.a0_model_conf = model_configdiff --git a/preload.py b/preload.pyindex b0287a160..914c41969 100644--- a/preload.py+++ b/preload.py@@ -25,7 +25,10 @@ async def preload():emb_cfg = get_embedding_model_config_object()if emb_cfg.provider.lower() == "huggingface":emb_mod = models.get_embedding_model(- "huggingface", emb_cfg.name+ emb_cfg.provider,+ emb_cfg.name,+ model_config=emb_cfg,+ **emb_cfg.build_kwargs(),)emb_txt = await emb_mod.aembed_query("test")return emb_txtdiff --git a/tests/test_embedding_model_preload.py b/tests/test_embedding_model_preload.pynew file mode 100644index 000000000..50f4634de--- /dev/null+++ b/tests/test_embedding_model_preload.py@@ -0,0 +1,189 @@+from concurrent.futures import ThreadPoolExecutor+import threading+from types import SimpleNamespace++import pytest++import models+++def _clear_local_embedding_models():+ with models._LOCAL_EMBEDDING_MODELS_LOCK:+ models._LOCAL_EMBEDDING_MODELS.clear()+++def test_local_embedding_preload_is_reused_with_runtime_model_config(monkeypatch):+ created = []++ class FakeSentenceTransformer:+ def __init__(self, model, **kwargs):+ created.append((model, kwargs))++ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)+ _clear_local_embedding_models()++ try:+ preload = models.LocalSentenceTransformerWrapper(+ "huggingface",+ "sentence-transformers/example",+ device="cpu",+ model_kwargs={"revision": "stable", "trust_remote_code": False},+ )+ runtime_config = SimpleNamespace(name="runtime")+ runtime = models.LocalSentenceTransformerWrapper(+ "huggingface",+ "sentence-transformers/example",+ model_config=runtime_config,+ model_kwargs={"trust_remote_code": False, "revision": "stable"},+ device="cpu",+ )++ assert runtime.model is preload.model+ assert runtime.a0_model_conf is runtime_config+ assert created == [+ (+ "example",+ {+ "device": "cpu",+ "model_kwargs": {+ "revision": "stable",+ "trust_remote_code": False,+ },+ },+ )+ ]+ finally:+ _clear_local_embedding_models()+++def test_local_embedding_cache_tracks_effective_constructor_options(monkeypatch):+ created = []++ class FakeSentenceTransformer:+ def __init__(self, model, **kwargs):+ created.append((model, kwargs))++ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)+ _clear_local_embedding_models()++ try:+ first = models.LocalSentenceTransformerWrapper(+ "huggingface", "sentence-transformers/example", device="cpu"+ )+ second = models.LocalSentenceTransformerWrapper(+ "huggingface", "sentence-transformers/example", device="cuda"+ )++ assert second.model is not first.model+ assert created == [+ ("example", {"device": "cpu"}),+ ("example", {"device": "cuda"}),+ ]+ assert len(models._LOCAL_EMBEDDING_MODELS) == 1+ finally:+ _clear_local_embedding_models()+++def test_concurrent_preload_and_runtime_share_one_model(monkeypatch):+ created = []+ construction_started = threading.Event()+ release_construction = threading.Event()++ class FakeSentenceTransformer:+ def __init__(self, model, **kwargs):+ created.append((model, kwargs))+ construction_started.set()+ assert release_construction.wait(timeout=2)++ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)+ _clear_local_embedding_models()++ try:+ with ThreadPoolExecutor(max_workers=2) as executor:+ first = executor.submit(+ models.LocalSentenceTransformerWrapper,+ "huggingface",+ "sentence-transformers/example",+ )+ assert construction_started.wait(timeout=2)+ second = executor.submit(+ models.LocalSentenceTransformerWrapper,+ "huggingface",+ "sentence-transformers/example",+ )+ release_construction.set()++ assert second.result().model is first.result().model++ assert created == [("example", {})]+ finally:+ release_construction.set()+ _clear_local_embedding_models()+++def test_failed_model_change_keeps_the_working_cached_model(monkeypatch):+ created = []++ class FakeSentenceTransformer:+ def __init__(self, model, **kwargs):+ created.append((model, kwargs))+ if model == "broken":+ raise RuntimeError("model unavailable")++ monkeypatch.setattr(models, "SentenceTransformer", FakeSentenceTransformer)+ _clear_local_embedding_models()++ try:+ working = models.LocalSentenceTransformerWrapper(+ "huggingface", "sentence-transformers/working"+ )+ with pytest.raises(RuntimeError, match="model unavailable"):+ models.LocalSentenceTransformerWrapper(+ "huggingface", "sentence-transformers/broken"+ )+ reused = models.LocalSentenceTransformerWrapper(+ "huggingface", "sentence-transformers/working"+ )++ assert reused.model is working.model+ assert created == [("working", {}), ("broken", {})]+ finally:+ _clear_local_embedding_models()+++async def test_preload_uses_the_runtime_embedding_configuration(monkeypatch):+ import preload+ from plugins._model_config.helpers import model_config++ config = SimpleNamespace(+ provider="huggingface",+ name="sentence-transformers/example",+ build_kwargs=lambda: {"device": "cpu"},+ )+ calls = []+ embedded = []++ class FakeEmbeddings:+ async def aembed_query(self, text):+ embedded.append(text)++ def get_embedding_model(provider, name, **kwargs):+ calls.append((provider, name, kwargs))+ return FakeEmbeddings()++ monkeypatch.setattr(+ model_config, "get_embedding_model_config_object", lambda: config+ )+ monkeypatch.setattr(preload.models, "get_embedding_model", get_embedding_model)++ await preload.preload()++ assert calls == [+ (+ "huggingface",+ "sentence-transformers/example",+ {"model_config": config, "device": "cpu"},+ )+ ]+ assert embedded == ["test"]