Agent Zero AI framework
rag tool finalizing
frdel9c8703cd7c7f
diff --git a/python/api/message.py b/python/api/message.pyindex de397355a..69e2db12a 100644--- a/python/api/message.py+++ b/python/api/message.py@@ -31,7 +31,7 @@ class Message(ApiHandler):attachment_paths = []upload_folder_int = "/a0/tmp/uploads"- upload_folder_ext = files.get_abs_path("tmp/uploads")+ upload_folder_ext = files.get_abs_path("tmp/uploads") # for development environmentif attachments:os.makedirs(upload_folder_ext, exist_ok=True)diff --git a/python/helpers/document_query.py b/python/helpers/document_query.pyindex c059e0c49..0959e286e 100644--- a/python/helpers/document_query.py+++ b/python/helpers/document_query.py@@ -4,6 +4,8 @@ import asyncioimport aiohttpimport json+from python.helpers.vector_db import VectorDB+os.environ["USER_AGENT"] = "@mixedbread-ai/unstructured" # noqa E402from langchain_unstructured import UnstructuredLoader # noqa E402@@ -32,15 +34,14 @@ from langchain_community.vectorstores.utils import (from langchain_core.embeddings import Embeddingsfrom python.helpers.print_style import PrintStyle-from python.helpers import files #, rfc_files-rfc_files = files # TODO: fix-+from python.helpers import filesfrom agent import Agent-import modelsfrom langchain.text_splitter import RecursiveCharacterTextSplitter+DEFAULT_SEARCH_THRESHOLD = 0.6+class DocumentQueryStore:"""FAISS Store for document query results.@@ -55,107 +56,25 @@ class DocumentQueryStore:_stores: dict[str, "DocumentQueryStore"] = {}@staticmethod- async def get(agent: Agent):- """Get or create a DocumentQueryStore instance for the specified agent."""+ def get(agent: Agent):+ """Create a DocumentQueryStore instance for the specified agent."""if not agent or not agent.config:raise ValueError("Agent and agent config must be provided")- memory_subdir = agent.config.memory_subdir or "default"- store_key = f"{memory_subdir}/document_query"-- if store_key not in DocumentQueryStore._stores:- # Initialize embeddings model from agent config- embeddings_model = agent.get_embedding_model()-- # Initialize store- store = DocumentQueryStore(agent, embeddings_model, memory_subdir)- DocumentQueryStore._stores[store_key] = store- return store- else:- return DocumentQueryStore._stores[store_key]-- @staticmethod- async def reload(agent: Agent):- """Reload the DocumentQueryStore for the specified agent."""- memory_subdir = agent.config.memory_subdir or "default"- store_key = f"{memory_subdir}/document_query"-- if store_key in DocumentQueryStore._stores:- del DocumentQueryStore._stores[store_key]-- return await DocumentQueryStore.get(agent)+ # Initialize store+ store = DocumentQueryStore(agent)+ return storedef __init__(self,agent: Agent,- embeddings_model: Embeddings,- memory_subdir: str,):"""Initialize a DocumentQueryStore instance."""self.agent = agent- self.memory_subdir = memory_subdir-- # Get directory paths- db_dir = self._get_db_dir()- em_dir = os.path.join(db_dir, "embeddings")-- # Create directories- os.makedirs(db_dir, exist_ok=True)- os.makedirs(em_dir, exist_ok=True)-- # Setup embeddings cache- store = LocalFileStore(em_dir)- self.embeddings = CacheBackedEmbeddings.from_bytes_store(- embeddings_model,- store,- namespace=f"document_query_{getattr(embeddings_model, 'model', getattr(embeddings_model, 'model_name', 'default'))}"- )-- # Initialize vector store- index_path = os.path.join(db_dir, "index.faiss")- docstore_path = os.path.join(db_dir, "docstore.json")-- if os.path.exists(index_path) and os.path.exists(docstore_path):- PrintStyle.standard(f"Loading existing vector store from {db_dir}")- try:- self.vectorstore = FAISS.load_local(- folder_path=db_dir,- embeddings=self.embeddings,- allow_dangerous_deserialization=True,- distance_strategy=DistanceStrategy.COSINE,- )- except Exception as e:- PrintStyle.error(f"Error loading vector store: {str(e)}")- self._initialize_new_vectorstore()- else:- PrintStyle.standard(f"Creating new vector store in '{db_dir}'")- self._initialize_new_vectorstore()-- def _initialize_new_vectorstore(self):- """Initialize a new vector store."""- dimension = len(self.embeddings.embed_query("test"))- index = faiss.IndexFlatIP(dimension)- self.vectorstore = FAISS(- embedding_function=self.embeddings,- index=index,- docstore=InMemoryDocstore(),- index_to_docstore_id={},- distance_strategy=DistanceStrategy.COSINE,- )-- def _get_db_dir(self) -> str:- """Get the absolute path to the database directory."""- return files.get_abs_path("memory", self.memory_subdir, "document_query")-- def _save_vectorstore(self):- """Save the vector store to disk."""- db_dir = self._get_db_dir()- PrintStyle.standard(f"Saving vector store to {db_dir}")- self.vectorstore.save_local(folder_path=db_dir)- PrintStyle.standard(f"Vector store saved with {len(self.vectorstore.index_to_docstore_id)} documents")+ self.vector_db: VectorDB | None = None@staticmethod- def _normalize_uri(uri: str) -> str:+ def normalize_uri(uri: str) -> str:"""Normalize a document URI to ensure consistent lookup.@@ -166,7 +85,7 @@ class DocumentQueryStore:Normalized URI"""# Convert to lowercase- normalized = uri.lower()+ normalized = uri.strip() # uri.lower()# Parse the URL to get schemeparsed = urlparse(normalized)@@ -174,17 +93,23 @@ class DocumentQueryStore:# Normalize based on schemeif scheme == "file":- if not normalized.startswith("file:"):- normalized = "file:" + normalized- if normalized.startswith("file://"):- normalized = normalized.replace("file://", "file:")+ path = files.fix_dev_path(+ normalized.removeprefix("file://").removeprefix("file:")+ )+ normalized = f"file://{path}"+elif scheme in ["http", "https"]:# Always use https for web URLs- normalized = normalized.replace("http://", "https://")+ normalized = normalized.replace("http://", "https://") # TODO why?return normalized- async def add_document(self, text: str, document_uri: str, metadata: dict | None = None) -> bool:+ def init_vector_db(self):+ return VectorDB(self.agent)++ async def add_document(+ self, text: str, document_uri: str, metadata: dict | None = None+ ) -> bool:"""Add a document to the store with the given URI.@@ -197,7 +122,7 @@ class DocumentQueryStore:True if successful, False otherwise"""# Normalize the URI- document_uri = self._normalize_uri(document_uri)+ document_uri = self.normalize_uri(document_uri)# Delete existing document if it exists to avoid duplicatesawait self.delete_document(document_uri)@@ -209,8 +134,7 @@ class DocumentQueryStore:# Split text into chunkstext_splitter = RecursiveCharacterTextSplitter(- chunk_size=self.DEFAULT_CHUNK_SIZE,- chunk_overlap=self.DEFAULT_CHUNK_OVERLAP+ chunk_size=self.DEFAULT_CHUNK_SIZE, chunk_overlap=self.DEFAULT_CHUNK_OVERLAP)chunks = text_splitter.split_text(text)@@ -230,14 +154,17 @@ class DocumentQueryStore:try:docs_text = "".join(chunk.page_content for chunk in docs)await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model,- input=docs_text+ model_config=self.agent.config.embeddings_model, input=docs_text)- # Add documents to vector store- self.vectorstore.add_documents(documents=docs)- self._save_vectorstore()- PrintStyle.standard(f"Added document '{document_uri}' with {len(docs)} chunks")+ # Initialize vector db if not already initialized+ if not self.vector_db:+ self.vector_db = self.init_vector_db()++ ids = await self.vector_db.insert_documents(docs)+ PrintStyle.standard(+ f"Added document '{document_uri}' with {len(docs)} chunks"+ )return Trueexcept Exception as e:PrintStyle.error(f"Error adding document '{document_uri}': {str(e)}")@@ -253,8 +180,13 @@ class DocumentQueryStore:Returns:The complete document if found, None otherwise"""++ # DB not initialized, no documents inside+ if not self.vector_db:+ return None+# Normalize the URI- document_uri = self._normalize_uri(document_uri)+ document_uri = self.normalize_uri(document_uri)# Get all chunks for this documentdocs = await self._get_document_chunks(document_uri)@@ -283,14 +215,19 @@ class DocumentQueryStore:Returns:List of document chunks"""++ # DB not initialized, no documents inside+ if not self.vector_db:+ return []+# Normalize the URI- document_uri = self._normalize_uri(document_uri)+ document_uri = self.normalize_uri(document_uri)- # Access docstore directly- chunks = []- for doc_id, doc in self.vectorstore.docstore._dict.items(): # type: ignore- if isinstance(doc.metadata, dict) and doc.metadata.get("document_uri") == document_uri:- chunks.append(doc)+ # get docs from vector db++ chunks = await self.vector_db.search_by_metadata(+ filter=f"document_uri == '{document_uri}'",+ )PrintStyle.standard(f"Found {len(chunks)} chunks for document: {document_uri}")return chunks@@ -305,8 +242,13 @@ class DocumentQueryStore:Returns:True if the document exists, False otherwise"""++ # DB not initialized, no documents inside+ if not self.vector_db:+ return False+# Normalize the URI- document_uri = self._normalize_uri(document_uri)+ document_uri = self.normalize_uri(document_uri)chunks = await self._get_document_chunks(document_uri)return len(chunks) > 0@@ -321,84 +263,36 @@ class DocumentQueryStore:Returns:True if deleted, False if not found"""++ # DB not initialized, no documents inside+ if not self.vector_db:+ return False+# Normalize the URI- document_uri = self._normalize_uri(document_uri)+ document_uri = self.normalize_uri(document_uri)- chunks = await self._get_document_chunks(document_uri)+ chunks = await self.vector_db.search_by_metadata(+ filter=f"document_uri == '{document_uri}'",+ )if not chunks:return False# Collect IDs to delete- ids_to_delete = []- for chunk in chunks:- for doc_id, doc_ref in self.vectorstore.docstore._dict.items(): # type: ignore- if doc_ref == chunk:- ids_to_delete.append(doc_id)+ ids_to_delete = [chunk.metadata["id"] for chunk in chunks]# Delete from vector storeif ids_to_delete:- self.vectorstore.delete(ids_to_delete)- self._save_vectorstore()- PrintStyle.standard(f"Deleted document '{document_uri}' with {len(ids_to_delete)} chunks")+ dels = await self.vector_db.delete_documents_by_ids(ids_to_delete)+ PrintStyle.standard(+ f"Deleted document '{document_uri}' with {len(dels)} chunks"+ )return Truereturn False- async def expire_documents(self, older_than_days: float) -> int:- """- Delete documents older than the specified number of days.-- Args:- older_than_days: Number of days (can be fractional) before current time-- Returns:- Number of documents deleted- """- if older_than_days <= 0:- return 0-- # Calculate cutoff timestamp- cutoff_date = datetime.now().timestamp() - (older_than_days * 24 * 60 * 60)-- # Find expired documents- expired_uris = set()-- # Check all documents in the store- for doc_id, doc in self.vectorstore.docstore._dict.items(): # type: ignore- if not isinstance(doc.metadata, dict):- continue-- # Only process each document once (first chunk)- if doc.metadata.get("chunk_index", 0) != 0:- continue-- doc_uri = doc.metadata.get("document_uri")- if not doc_uri:- continue-- try:- # Check timestamp- timestamp_str = doc.metadata.get("timestamp")- if not timestamp_str:- continue-- doc_timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S").timestamp()- if doc_timestamp < cutoff_date:- expired_uris.add(doc_uri)- except (ValueError, TypeError):- # Skip documents with invalid timestamps- continue-- # Delete expired documents- deleted_count = 0- for uri in expired_uris:- if await self.delete_document(uri):- deleted_count += 1-- PrintStyle.standard(f"Expired {deleted_count} documents older than {older_than_days} days")- return deleted_count-- async def search_documents(self, query: str, limit: int = 10, threshold: float = 0.5) -> List[Document]:+ async def search_documents(+ self, query: str, limit: int = 10, threshold: float = 0.5, filter: str = ""+ ) -> List[Document]:"""Search for documents similar to the query across the entire store.@@ -410,34 +304,30 @@ class DocumentQueryStore:Returns:List of matching documents"""++ # DB not initialized, no documents inside+ if not self.vector_db:+ return []+# Handle empty queryif not query:- PrintStyle.standard("Empty search query, returning empty results")return []- # Apply rate limiter- await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model,- input=query- )-# Perform searchtry:- results = self.vectorstore.similarity_search_with_score(- query=query,- k=limit,- score_threshold=threshold+ results = await self.vector_db.search_by_similarity_threshold(+ query=query, limit=limit, threshold=threshold, filter=filter)- # Extract documents from results (which are (doc, score) pairs)- docs = [doc for doc, score in results]- PrintStyle.standard(f"Search '{query}' returned {len(docs)} results")- return docs+ PrintStyle.standard(f"Search '{query}' returned {len(results)} results")+ return resultsexcept Exception as e:PrintStyle.error(f"Error searching documents: {str(e)}")return []- async def search_document(self, document_uri: str, query: str, limit: int = 10, threshold: float = 0.5) -> List[Document]:+ async def search_document(+ self, document_uri: str, query: str, limit: int = 10, threshold: float = 0.5+ ) -> List[Document]:"""Search for content within a specific document.@@ -450,59 +340,10 @@ class DocumentQueryStore:Returns:List of matching document chunks"""- # Normalize the URI- document_uri = self._normalize_uri(document_uri)-- # Handle empty query- if not query:- PrintStyle.standard("Empty search query, returning empty results")- return []-- # Check if document exists- if not await self.document_exists(document_uri):- PrintStyle.error(f"Document not found: {document_uri}")- return []-- # Apply rate limiter- await self.agent.rate_limiter(- model_config=self.agent.config.embeddings_model,- input=query+ return await self.search_documents(+ query, limit, threshold, f"document_uri == '{document_uri}'")- # Perform search with document filter- try:- # Create metadata filter function- def filter_fn(doc_metadata):- return doc_metadata.get("document_uri") == document_uri-- results = self.vectorstore.similarity_search_with_score(- query=query,- k=limit,- score_threshold=threshold,- filter=filter_fn- )-- # Extract documents from results- docs = [doc for doc, score in results]- PrintStyle.standard(f"Search '{query}' in document '{document_uri}' returned {len(docs)} results")-- # Try with lower threshold if no results- if not docs and threshold > 0.3:- PrintStyle.standard("No results found, trying with lower threshold (0.3)")- results = self.vectorstore.similarity_search_with_score(- query=query,- k=limit,- score_threshold=0.3,- filter=filter_fn- )- docs = [doc for doc, score in results]- PrintStyle.standard(f"Retry search returned {len(docs)} results")-- return docs- except Exception as e:- PrintStyle.error(f"Error searching within document: {str(e)}")- return []-async def list_documents(self) -> List[str]:"""Get a list of all document URIs in the store.@@ -510,9 +351,13 @@ class DocumentQueryStore:Returns:List of document URIs"""+ # DB not initialized, no documents inside+ if not self.vector_db:+ return []+# Extract unique URIsuris = set()- for doc in self.vectorstore.docstore._dict.values(): # type: ignore+ for doc in self.vector_db.db.get_all_docs().values():if isinstance(doc.metadata, dict):uri = doc.metadata.get("document_uri")if uri:@@ -525,27 +370,37 @@ class DocumentQueryHelper:def __init__(self, agent: Agent):self.agent = agent- self.store: DocumentQueryStore = asyncio.run(DocumentQueryStore.get(agent))+ self.store = DocumentQueryStore.get(agent)- async def document_qa(self, document_uri: str, questions: Sequence[str]) -> Tuple[bool, str]:- _ = await self.document_get_content(document_uri)+ async def document_qa(+ self, document_uri: str, questions: Sequence[str]+ ) -> Tuple[bool, str]:+ # index document+ _ = await self.document_get_content(document_uri, True)content = ""for question in questions:human_content = f'Search Query: "{question}"'- system_content = self.agent.parse_prompt("fw.document_query.optmimize_query.md")-- optimized_query = await self.agent.call_utility_model(- system=system_content,- message=human_content+ system_content = self.agent.parse_prompt(+ "fw.document_query.optmimize_query.md")+ optimized_query = (+ await self.agent.call_utility_model(+ system=system_content, message=human_content+ )+ ).strip()++ normalized_uri = self.store.normalize_uri(document_uri)chunks = await self.store.search_document(- document_uri=document_uri,- query=str(optimized_query),- limit=10000,- threshold=0.66+ document_uri=normalized_uri,+ query=optimized_query,+ limit=100,+ threshold=DEFAULT_SEARCH_THRESHOLD,+ )+ content += (+ "\n\n----\n\n".join([chunk.page_content for chunk in chunks])+ + "\n\n----\n\n")- content += "\n\n----\n\n".join([chunk.page_content for chunk in chunks]) + "\n\n----\n\n"if not content:content = f"!!! No content found for document: {document_uri} matching queries: {json.dumps(questions)}"@@ -553,19 +408,23 @@ class DocumentQueryHelper:questions_str = "\n".join([f" * {question}" for question in questions])- qa_system_message = self.agent.parse_prompt("fw.document_query.system_prompt.md")+ qa_system_message = self.agent.parse_prompt(+ "fw.document_query.system_prompt.md"+ )qa_user_message = f"# Document:\n{content}\n\n# Queries:\n{questions_str}"ai_response = await self.agent.call_chat_model(- prompt=ChatPromptTemplate.from_messages([- SystemMessage(content=qa_system_message),- HumanMessage(content=qa_user_message),- ])+ prompt=ChatPromptTemplate.from_messages(+ [+ SystemMessage(content=qa_system_message),+ HumanMessage(content=qa_user_message),+ ]+ ))return True, str(ai_response)- async def document_get_content(self, document_uri: str) -> str:+ async def document_get_content(self, document_uri: str, add_to_db: bool = False) -> str:url = urlparse(document_uri)scheme = url.scheme or "file"mimetype, encoding = mimetypes.guess_type(document_uri)@@ -579,7 +438,11 @@ class DocumentQueryHelper:while not response and retries < 3:try:async with aiohttp.ClientSession() as session:- response = await session.head(document_uri, timeout=aiohttp.ClientTimeout(total=2.0), allow_redirects=True)+ response = await session.head(+ document_uri,+ timeout=aiohttp.ClientTimeout(total=2.0),+ allow_redirects=True,+ )if response.status > 399:raise Exception(response.status)break@@ -589,32 +452,41 @@ class DocumentQueryHelper:retries += 1if not response:- raise ValueError(f"DocumentQueryHelper::document_get_content: Document fetch error: {document_uri} ({last_error})")+ raise ValueError(+ f"DocumentQueryHelper::document_get_content: Document fetch error: {document_uri} ({last_error})"+ )mimetype = response.headers["content-type"]if "content-length" in response.headers:- content_length = float(response.headers["content-length"]) / 1024 / 1024 # MB+ content_length = (+ float(response.headers["content-length"]) / 1024 / 1024+ ) # MBif content_length > 50.0:- raise ValueError(f"Document content length exceeds max. 50MB: {content_length} MB ({document_uri})")- if mimetype and '; charset=' in mimetype:- mimetype = mimetype.split('; charset=')[0]+ raise ValueError(+ f"Document content length exceeds max. 50MB: {content_length} MB ({document_uri})"+ )+ if mimetype and "; charset=" in mimetype:+ mimetype = mimetype.split("; charset=")[0]if scheme == "file":try:- document_uri = os.path.abspath(url.path)+ document_uri = files.fix_dev_path(url.path)except Exception as e:raise ValueError(f"Invalid document path '{url.path}'") from eif encoding:- raise ValueError(f"Compressed documents are unsupported '{encoding}' ({document_uri})")+ raise ValueError(+ f"Compressed documents are unsupported '{encoding}' ({document_uri})"+ )if mimetype == "application/octet-stream":- raise ValueError(f"Unsupported document mimetype '{mimetype}' ({document_uri})")+ raise ValueError(+ f"Unsupported document mimetype '{mimetype}' ({document_uri})"+ )# Use the store's normalization method- document_uri_norm = self.store._normalize_uri(document_uri)+ document_uri_norm = self.store.normalize_uri(document_uri)- await self.store.expire_documents(7)exists = await self.store.document_exists(document_uri_norm)document_content = ""if not exists:@@ -627,14 +499,19 @@ class DocumentQueryHelper:elif mimetype == "application/pdf":document_content = self.handle_pdf_document(document_uri, scheme)else:- document_content = self.handle_unstructured_document(document_uri, scheme)- await self.store.add_document(document_content, document_uri_norm)+ document_content = self.handle_unstructured_document(+ document_uri, scheme+ )+ if add_to_db:+ await self.store.add_document(document_content, document_uri_norm)else:doc = await self.store.get_document(document_uri_norm)if doc:document_content = doc.page_contentelse:- raise ValueError(f"DocumentQueryHelper::document_get_content: Document not found: {document_uri_norm}")+ raise ValueError(+ f"DocumentQueryHelper::document_get_content: Document not found: {document_uri_norm}"+ )return document_contentdef handle_image_document(self, document: str, scheme: str) -> str:@@ -646,14 +523,19 @@ class DocumentQueryHelper:parts: list[Document] = loader.load()elif scheme == "file":# Use RFC file operations instead of TextLoader- file_content_bytes = rfc_files.read_file_bin(document)- file_content = file_content_bytes.decode('utf-8')+ file_content_bytes = files.read_file_bin(document)+ file_content = file_content_bytes.decode("utf-8")# Create Document manually since we're not using TextLoaderparts = [Document(page_content=file_content, metadata={"source": document})]else:raise ValueError(f"Unsupported scheme: {scheme}")- return "\n".join([element.page_content for element in MarkdownifyTransformer().transform_documents(parts)])+ return "\n".join(+ [+ element.page_content+ for element in MarkdownifyTransformer().transform_documents(parts)+ ]+ )def handle_text_document(self, document: str, scheme: str) -> str:if scheme in ["http", "https"]:@@ -661,10 +543,12 @@ class DocumentQueryHelper:elements: list[Document] = loader.load()elif scheme == "file":# Use RFC file operations instead of TextLoader- file_content_bytes = rfc_files.read_file_bin(document)- file_content = file_content_bytes.decode('utf-8')+ file_content_bytes = files.read_file_bin(document)+ file_content = file_content_bytes.decode("utf-8")# Create Document manually since we're not using TextLoader- elements = [Document(page_content=file_content, metadata={"source": document})]+ elements = [+ Document(page_content=file_content, metadata={"source": document})+ ]else:raise ValueError(f"Unsupported scheme: {scheme}")@@ -674,27 +558,33 @@ class DocumentQueryHelper:temp_file_path = ""if scheme == "file":# Use RFC file operations to read the PDF file as binary- file_content_bytes = rfc_files.read_file_bin(document)+ file_content_bytes = files.read_file_bin(document)# Create a temporary file for PyMuPDFLoader since it needs a file pathimport tempfile- with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:++ with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:temp_file.write(file_content_bytes)temp_file_path = temp_file.nameelif scheme in ["http", "https"]:# download the file from the web url to a temporary file using python libraries for downloadingimport requestsimport tempfile- with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:++ with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:response = requests.get(document, timeout=10.0)if response.status_code != 200:- raise ValueError(f"DocumentQueryHelper::handle_pdf_document: Failed to download PDF from {document}: {response.status_code}")+ raise ValueError(+ f"DocumentQueryHelper::handle_pdf_document: Failed to download PDF from {document}: {response.status_code}"+ )temp_file.write(response.content)temp_file_path = temp_file.nameelse:raise ValueError(f"Unsupported scheme: {scheme}")if not os.path.exists(temp_file_path):- raise ValueError(f"DocumentQueryHelper::handle_pdf_document: Temporary file not found: {temp_file_path}")+ raise ValueError(+ f"DocumentQueryHelper::handle_pdf_document: Temporary file not found: {temp_file_path}"+ )try:try:@@ -710,17 +600,21 @@ class DocumentQueryHelper:elements: list[Document] = loader.load()contents = "\n".join([element.page_content for element in elements])except Exception as e:- PrintStyle.error(f"DocumentQueryHelper::handle_pdf_document: Error loading with PyMuPDF: {e}")+ PrintStyle.error(+ f"DocumentQueryHelper::handle_pdf_document: Error loading with PyMuPDF: {e}"+ )contents = ""if not contents:import pdf2imageimport pytesseract- PrintStyle.debug(f"DocumentQueryHelper::handle_pdf_document: FALLBACK Converting PDF to images: {temp_file_path}")+ PrintStyle.debug(+ f"DocumentQueryHelper::handle_pdf_document: FALLBACK Converting PDF to images: {temp_file_path}"+ )# Convert PDF to images- pages = pdf2image.convert_from_path(temp_file_path) # type: ignore+ pages = pdf2image.convert_from_path(temp_file_path) # type: ignorefor page in pages:contents += pytesseract.image_to_string(page) + "\n\n"@@ -742,10 +636,11 @@ class DocumentQueryHelper:elements = loader.load()elif scheme == "file":# Use RFC file operations to read the file as binary- file_content_bytes = rfc_files.read_file_bin(document)+ file_content_bytes = files.read_file_bin(document)# Create a temporary file for UnstructuredLoader since it needs a file pathimport tempfileimport os+# Get file extension to preserve it for proper processing_, ext = os.path.splitext(document)with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as temp_file:diff --git a/python/helpers/files.py b/python/helpers/files.pyindex 826d5a4c0..f562a543b 100644--- a/python/helpers/files.py+++ b/python/helpers/files.py@@ -253,8 +253,16 @@ def make_dirs(relative_path: str):def get_abs_path(*relative_paths):+ "Convert relative paths to absolute paths based on the base directory."return os.path.join(get_base_dir(), *relative_paths)+def fix_dev_path(path:str):+ "On dev environment, convert /a0/... paths to local absolute paths"+ from python.helpers.runtime import is_development+ if is_development():+ if path.startswith("/a0/"):+ path = path.replace("/a0/", "")+ return get_abs_path(path)def exists(*relative_paths):path = get_abs_path(*relative_paths)diff --git a/python/helpers/vector_db.py b/python/helpers/vector_db.pyindex 21bd55ce6..128cccfbf 100644--- a/python/helpers/vector_db.py+++ b/python/helpers/vector_db.py@@ -27,27 +27,40 @@ class MyFaiss(FAISS):async def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:return self.get_by_ids(ids)+ def get_all_docs(self) -> dict[str, Document]:+ return self.docstore._dict # type: ignore+class VectorDB:- def __init__(self, agent: Agent):- self.agent = agent- self.store = InMemoryByteStore()- self.model = agent.get_embedding_model()-- self.embedder = CacheBackedEmbeddings.from_bytes_store(- self.model,- self.store,- namespace=getattr(- self.model,- "model",- getattr(self.model, "model_name", "default"),- ),++ _cached_embeddings: dict[str, CacheBackedEmbeddings] = {}++ @staticmethod+ def _get_embeddings(agent: Agent):+ model = agent.get_embedding_model()+ namespace = getattr(+ model,+ "model",+ getattr(model, "model_name", "default"),)+ if namespace not in VectorDB._cached_embeddings:+ store = InMemoryByteStore()+ VectorDB._cached_embeddings[namespace] = (+ CacheBackedEmbeddings.from_bytes_store(+ model,+ store,+ namespace=namespace,+ )+ )+ return VectorDB._cached_embeddings[namespace]- self.index = faiss.IndexFlatIP(len(self.embedder.embed_query("example")))+ def __init__(self, agent: Agent):+ self.agent = agent+ self.embeddings = self._get_embeddings(agent)+ self.index = faiss.IndexFlatIP(len(self.embeddings.embed_query("example")))self.db = MyFaiss(- embedding_function=self.embedder,+ embedding_function=self.embeddings,index=self.index,docstore=InMemoryDocstore(),index_to_docstore_id={},@@ -56,7 +69,7 @@ class VectorDB:relevance_score_fn=cosine_normalizer,)- async def search_similarity_threshold(+ async def search_by_similarity_threshold(self, query: str, limit: int, threshold: float, filter: str = ""):comparator = get_comparator(filter) if filter else None@@ -74,6 +87,18 @@ class VectorDB:filter=comparator,)+ async def search_by_metadata(self, filter: str, limit: int = 0) -> list[Document]:+ comparator = get_comparator(filter)+ all_docs = self.db.get_all_docs()+ result = []+ for doc in all_docs.values():+ if comparator(doc.metadata):+ result.append(doc)+ # stop if limit reached and limit > 0+ if limit > 0 and len(result) >= limit:+ break+ return result+async def insert_documents(self, docs: list[Document]):ids = [str(uuid.uuid4()) for _ in range(len(docs))]@@ -90,6 +115,16 @@ class VectorDB:self.db.add_documents(documents=docs, ids=ids)return ids+ async def delete_documents_by_ids(self, ids: list[str]):+ # aget_by_ids is not yet implemented in faiss, need to do a workaround+ rem_docs = await self.db.aget_by_ids(+ ids+ ) # existing docs to remove (prevents error)+ if rem_docs:+ rem_ids = [doc.metadata["id"] for doc in rem_docs] # ids to remove+ await self.db.adelete(ids=rem_ids)+ return rem_docs+def format_docs_plain(docs: list[Document]) -> list[str]:result = []@@ -113,7 +148,8 @@ def cosine_normalizer(val: float) -> float:def get_comparator(condition: str):def comparator(data: dict[str, Any]):try:- return eval(condition, {}, data)+ result = eval(condition, {}, data)+ return resultexcept Exception as e:# PrintStyle.error(f"Error evaluating condition: {e}")return False