Skip to content

Agent Zero AI framework

auto embeddings

frdela2c03c06d95d
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 88929a6df..08f0c097c 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -1,6 +1,7 @@
{
"version": "0.2.0",
"configurations": [
+
{
"name": "Debug run_ui.py",
"type": "debugpy",
@@ -10,15 +11,6 @@
"justMyCode": false,
"args": ["--development=true", "-Xfrozen_modules=off"]
},
- {
- "name": "Debug run_cli.py",
- "type": "debugpy",
- "request": "launch",
- "program": "./run_cli.py",
- "console": "integratedTerminal",
- "justMyCode": false,
- "args": ["--development=true", "-Xfrozen_modules=off"]
- },
{
"name": "Debug current file",
"type": "debugpy",
diff --git a/agent.py b/agent.py
index 4a40523f5..fad78a423 100644
--- a/agent.py
+++ b/agent.py
@@ -2,31 +2,21 @@ import asyncio
from collections import OrderedDict
from dataclasses import dataclass, field
from datetime import datetime
-import time, importlib, inspect, os, json
-import token
from typing import Any, Awaitable, Coroutine, Optional, Dict, TypedDict
import uuid
import models
-from langchain_core.prompt_values import ChatPromptValue
from python.helpers import extract_tools, rate_limiter, files, errors, history, tokens
from python.helpers.print_style import PrintStyle
from langchain_core.prompts import (
ChatPromptTemplate,
- MessagesPlaceholder,
- HumanMessagePromptTemplate,
- StringPromptTemplate,
)
-from langchain_core.prompts.image import ImagePromptTemplate
from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage
-from langchain_core.language_models.chat_models import BaseChatModel
-from langchain_core.language_models.llms import BaseLLM
-from langchain_core.embeddings import Embeddings
+
import python.helpers.log as Log
from python.helpers.dirty_json import DirtyJson
from python.helpers.defer import DeferredTask
from typing import Callable
-from python.helpers.history import OutputMessage
from python.helpers.localization import Localization
diff --git a/docker/run/fs/exe/node_eval.js b/docker/run/fs/exe/node_eval.js
index aab45b11e..6fdbfbf8c 100644
--- a/docker/run/fs/exe/node_eval.js
+++ b/docker/run/fs/exe/node_eval.js
@@ -7,8 +7,9 @@ const Module = require('module');
// Enhance `require` to search CWD first, then globally
function customRequire(moduleName) {
try {
- // Try resolving from CWD's node_modules
- const cwdPath = path.resolve(process.cwd(), 'node_modules', moduleName);
+ // Try resolving from CWD's node_modules using Node's require.resolve
+ const cwdPath = require.resolve(moduleName, { paths: [path.join(process.cwd(), 'node_modules')] });
+ // console.log("resolved path:", cwdPath);
return require(cwdPath);
} catch (cwdErr) {
try {
diff --git a/preload.py b/preload.py
index 2cf59171d..9c8e3ca66 100644
--- a/preload.py
+++ b/preload.py
@@ -1,6 +1,7 @@
import asyncio
from python.helpers import runtime, whisper, settings
from python.helpers.print_style import PrintStyle
+import models
PrintStyle().print("Running preload...")
runtime.initialize()
@@ -10,12 +11,31 @@ async def preload():
try:
set = settings.get_default_settings()
+ # preload whisper model
+ async def preload_whisper():
+ try:
+ return await whisper.preload(set["stt_model_size"])
+ except Exception as e:
+ PrintStyle().error(f"Error in preload_whisper: {e}")
+
+ # preload embedding model
+ async def preload_embedding():
+ if set["embed_model_provider"] == models.ModelProvider.HUGGINGFACE.name:
+ try:
+ emb_mod = models.get_huggingface_embedding(set["embed_model_name"])
+ emb_txt = await emb_mod.aembed_query("test")
+ return emb_txt
+ except Exception as e:
+ PrintStyle().error(f"Error in preload_embedding: {e}")
+
+
# async tasks to preload
- tasks = [whisper.preload(set["stt_model_size"])]
+ tasks = [preload_whisper(), preload_embedding()]
- return asyncio.gather(*tasks, return_exceptions=True)
+ await asyncio.gather(*tasks, return_exceptions=True)
+ PrintStyle().print("Preload completed")
except Exception as e:
- PrintStyle().print(f"Error in preload: {e}")
+ PrintStyle().error(f"Error in preload: {e}")
# preload transcription model
diff --git a/python/helpers/files.py b/python/helpers/files.py
index 15d7e6fc5..d5674c934 100644
--- a/python/helpers/files.py
+++ b/python/helpers/files.py
@@ -285,3 +285,8 @@ def move_file(relative_path: str, new_path: str):
new_abs_path = get_abs_path(new_path)
os.makedirs(os.path.dirname(new_abs_path), exist_ok=True)
os.rename(abs_path, new_abs_path)
+
+def safe_file_name(filename:str)-> str:
+ # Replace any character that's not alphanumeric, dash, underscore, or dot with underscore
+ import re
+ return re.sub(r'[^a-zA-Z0-9-._]', '_', filename)
\ No newline at end of file
diff --git a/python/helpers/memory.py b/python/helpers/memory.py
index 0422a69d1..29d85cb44 100644
--- a/python/helpers/memory.py
+++ b/python/helpers/memory.py
@@ -23,7 +23,7 @@ import uuid
from python.helpers import knowledge_import
from python.helpers.log import Log, LogItem
from enum import Enum
-from agent import Agent
+from agent import Agent, ModelConfig
import models
@@ -36,6 +36,9 @@ class MyFaiss(FAISS):
async def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:
return self.get_by_ids(ids)
+ def get_all_docs(self):
+ return self.docstore._dict # type: ignore
+
class Memory:
@@ -55,14 +58,9 @@ class Memory:
type="util",
heading=f"Initializing VectorDB in '/{memory_subdir}'",
)
- db = Memory.initialize(
+ db, created = Memory.initialize(
log_item,
- models.get_model(
- models.ModelType.EMBEDDING,
- agent.config.embeddings_model.provider,
- agent.config.embeddings_model.name,
- **agent.config.embeddings_model.kwargs,
- ),
+ agent.config.embeddings_model,
memory_subdir,
False,
)
@@ -90,10 +88,10 @@ class Memory:
@staticmethod
def initialize(
log_item: LogItem | None,
- embeddings_model: Embeddings,
+ model_config: ModelConfig,
memory_subdir: str,
in_memory=False,
- ) -> MyFaiss:
+ ) -> tuple[MyFaiss, bool]:
PrintStyle.standard("Initializing VectorDB...")
@@ -114,20 +112,26 @@ class Memory:
os.makedirs(em_dir, exist_ok=True)
store = LocalFileStore(em_dir)
+ embeddings_model = models.get_model(
+ models.ModelType.EMBEDDING,
+ model_config.provider,
+ model_config.name,
+ **model_config.kwargs,
+ )
+ embeddings_model_id = files.safe_file_name(
+ model_config.provider.name + "_" + model_config.name
+ )
+
# here we setup the embeddings model with the chosen cache storage
embedder = CacheBackedEmbeddings.from_bytes_store(
- embeddings_model,
- store,
- namespace=getattr(
- embeddings_model,
- "model",
- getattr(embeddings_model, "model_name", "default"),
- ),
+ embeddings_model, store, namespace=embeddings_model_id
)
- # self.db = Chroma(
- # embedding_function=self.embedder,
- # persist_directory=db_dir)
+ # initial DB and docs variables
+ db: MyFaiss | None = None
+ docs: dict[str, Document] | None = None
+
+ created = False
# if db folder exists and is not empty:
if os.path.exists(db_dir) and files.exists(db_dir, "index.faiss"):
@@ -138,8 +142,27 @@ class Memory:
distance_strategy=DistanceStrategy.COSINE,
# normalize_L2=True,
relevance_score_fn=Memory._cosine_normalizer,
- )
- else:
+ ) # type: ignore
+
+ # if there is a mismatch in embeddings used, re-index the whole DB
+ emb_ok = False
+ emb_set_file = files.get_abs_path(db_dir, "embedding.json")
+ if files.exists(emb_set_file):
+ embedding_set = json.loads(files.read_file(emb_set_file))
+ if (
+ embedding_set["model_provider"] == model_config.provider.name
+ and embedding_set["model_name"] == model_config.name
+ ):
+ # model matches
+ emb_ok = True
+
+ # re-index - create new DB and insert existing docs
+ if db and not emb_ok:
+ docs = db.get_all_docs()
+ db = None
+
+ # DB not loaded, create one
+ if not db:
index = faiss.IndexFlatIP(len(embedder.embed_query("example")))
db = MyFaiss(
@@ -151,7 +174,31 @@ class Memory:
# normalize_L2=True,
relevance_score_fn=Memory._cosine_normalizer,
)
- return db # type: ignore
+
+ # insert docs if reindexing
+ if docs:
+ PrintStyle.standard("Indexing memories...")
+ if log_item:
+ log_item.stream(progress="\nIndexing memories")
+ db.add_documents(documents=list(docs.values()), ids=list(docs.keys()))
+
+ # save DB
+ Memory._save_db_file(db, memory_subdir)
+ # save meta file
+ meta_file_path = files.get_abs_path(db_dir, "embedding.json")
+ files.write_file(
+ meta_file_path,
+ json.dumps(
+ {
+ "model_provider": model_config.provider.name,
+ "model_name": model_config.name,
+ }
+ ),
+ )
+
+ created = True
+
+ return db, created
def __init__(
self,
@@ -243,9 +290,10 @@ class Memory:
):
comparator = Memory._get_comparator(filter) if filter else None
- #rate limiter
+ # rate limiter
await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model, input=query)
+ model_config=self.agent.config.embeddings_model, input=query
+ )
return await self.db.asearch(
query,
@@ -309,25 +357,30 @@ class Memory:
ids = [str(uuid.uuid4()) for _ in range(len(docs))]
timestamp = self.get_timestamp()
-
if ids:
for doc, id in zip(docs, ids):
doc.metadata["id"] = id # add ids to documents metadata
doc.metadata["timestamp"] = timestamp # add timestamp
if not doc.metadata.get("area", ""):
doc.metadata["area"] = Memory.Area.MAIN.value
-
- #rate limiter
+
+ # rate limiter
docs_txt = "".join(self.format_docs_plain(docs))
await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model, input=docs_txt)
+ model_config=self.agent.config.embeddings_model, input=docs_txt
+ )
self.db.add_documents(documents=docs, ids=ids)
self._save_db() # persist
return ids
def _save_db(self):
- self.db.save_local(folder_path=self._abs_db_dir(self.memory_subdir))
+ Memory._save_db_file(self.db, self.memory_subdir)
+
+ @staticmethod
+ def _save_db_file(db: MyFaiss, memory_subdir: str):
+ abs_dir = Memory._abs_db_dir(memory_subdir)
+ db.save_local(folder_path=abs_dir)
@staticmethod
def _get_comparator(condition: str):
@@ -382,3 +435,8 @@ def get_custom_knowledge_subdir_abs(agent: Agent) -> str:
if dir != "default":
return files.get_abs_path("knowledge", dir)
raise Exception("No custom knowledge subdir set")
+
+
+def reload():
+ # clear the memory index, this will force all DBs to reload
+ Memory.index = {}
diff --git a/python/helpers/settings.py b/python/helpers/settings.py
index 12b7c8a5d..f07afc880 100644
--- a/python/helpers/settings.py
+++ b/python/helpers/settings.py
@@ -150,7 +150,6 @@ def convert_out(settings: Settings) -> SettingsOutput:
}
)
-
chat_model_fields.append(
{
"id": "chat_model_vision",
@@ -730,9 +729,10 @@ def get_settings() -> Settings:
def set_settings(settings: Settings):
global _settings
+ previous = _settings
_settings = normalize_settings(settings)
_write_settings_file(_settings)
- _apply_settings()
+ _apply_settings(previous)
def normalize_settings(settings: Settings) -> Settings:
@@ -795,7 +795,7 @@ def get_default_settings() -> Settings:
return Settings(
chat_model_provider=ModelProvider.OPENAI.name,
chat_model_name="gpt-4o",
- chat_model_kwargs={ "temperature": "0" },
+ chat_model_kwargs={"temperature": "0"},
chat_model_ctx_length=120000,
chat_model_ctx_history=0.7,
chat_model_vision=False,
@@ -806,19 +806,19 @@ def get_default_settings() -> Settings:
util_model_name="gpt-4o-mini",
util_model_ctx_length=120000,
util_model_ctx_input=0.7,
- util_model_kwargs={ "temperature": "0" },
+ util_model_kwargs={"temperature": "0"},
util_model_rl_requests=60,
util_model_rl_input=0,
util_model_rl_output=0,
- embed_model_provider=ModelProvider.OPENAI.name,
- embed_model_name="text-embedding-3-small",
+ embed_model_provider=ModelProvider.HUGGINGFACE.name,
+ embed_model_name="sentence-transformers/all-MiniLM-L6-v2",
embed_model_kwargs={},
embed_model_rl_requests=0,
embed_model_rl_input=0,
browser_model_provider=ModelProvider.OPENAI.name,
browser_model_name="gpt-4o",
browser_model_vision=False,
- browser_model_kwargs={ "temperature": "0" },
+ browser_model_kwargs={"temperature": "0"},
api_keys={},
auth_login="",
auth_password="",
@@ -839,7 +839,7 @@ def get_default_settings() -> Settings:
)
-def _apply_settings():
+def _apply_settings(previous: Settings | None):
global _settings
if _settings:
from agent import AgentContext
@@ -858,6 +858,15 @@ def _apply_settings():
whisper.preload, _settings["stt_model_size"]
) # TODO overkill, replace with background task
+ # force memory reload on embedding model change
+ if previous and (
+ _settings["embed_model_name"] != previous["embed_model_name"]
+ or _settings["embed_model_provider"] != previous["embed_model_provider"]
+ or _settings["embed_model_kwargs"] != previous["embed_model_kwargs"]
+ ):
+ from python.helpers.memory import reload as memory_reload
+ memory_reload()
+
def _env_to_dict(data: str):
env_dict = {}