Skip to content

Agent Zero AI framework

rag tool finalizing

frdel9c8703cd7c7f
diff --git a/python/api/message.py b/python/api/message.py
index de397355a..69e2db12a 100644
--- a/python/api/message.py
+++ b/python/api/message.py
@@ -31,7 +31,7 @@ class Message(ApiHandler):
attachment_paths = []
upload_folder_int = "/a0/tmp/uploads"
- upload_folder_ext = files.get_abs_path("tmp/uploads")
+ upload_folder_ext = files.get_abs_path("tmp/uploads") # for development environment
if attachments:
os.makedirs(upload_folder_ext, exist_ok=True)
diff --git a/python/helpers/document_query.py b/python/helpers/document_query.py
index c059e0c49..0959e286e 100644
--- a/python/helpers/document_query.py
+++ b/python/helpers/document_query.py
@@ -4,6 +4,8 @@ import asyncio
import aiohttp
import json
+from python.helpers.vector_db import VectorDB
+
os.environ["USER_AGENT"] = "@mixedbread-ai/unstructured" # noqa E402
from langchain_unstructured import UnstructuredLoader # noqa E402
@@ -32,15 +34,14 @@ from langchain_community.vectorstores.utils import (
from langchain_core.embeddings import Embeddings
from python.helpers.print_style import PrintStyle
-from python.helpers import files #, rfc_files
-rfc_files = files # TODO: fix
-
+from python.helpers import files
from agent import Agent
-import models
from langchain.text_splitter import RecursiveCharacterTextSplitter
+DEFAULT_SEARCH_THRESHOLD = 0.6
+
class DocumentQueryStore:
"""
FAISS Store for document query results.
@@ -55,107 +56,25 @@ class DocumentQueryStore:
_stores: dict[str, "DocumentQueryStore"] = {}
@staticmethod
- async def get(agent: Agent):
- """Get or create a DocumentQueryStore instance for the specified agent."""
+ def get(agent: Agent):
+ """Create a DocumentQueryStore instance for the specified agent."""
if not agent or not agent.config:
raise ValueError("Agent and agent config must be provided")
- memory_subdir = agent.config.memory_subdir or "default"
- store_key = f"{memory_subdir}/document_query"
-
- if store_key not in DocumentQueryStore._stores:
- # Initialize embeddings model from agent config
- embeddings_model = agent.get_embedding_model()
-
- # Initialize store
- store = DocumentQueryStore(agent, embeddings_model, memory_subdir)
- DocumentQueryStore._stores[store_key] = store
- return store
- else:
- return DocumentQueryStore._stores[store_key]
-
- @staticmethod
- async def reload(agent: Agent):
- """Reload the DocumentQueryStore for the specified agent."""
- memory_subdir = agent.config.memory_subdir or "default"
- store_key = f"{memory_subdir}/document_query"
-
- if store_key in DocumentQueryStore._stores:
- del DocumentQueryStore._stores[store_key]
-
- return await DocumentQueryStore.get(agent)
+ # Initialize store
+ store = DocumentQueryStore(agent)
+ return store
def __init__(
self,
agent: Agent,
- embeddings_model: Embeddings,
- memory_subdir: str,
):
"""Initialize a DocumentQueryStore instance."""
self.agent = agent
- self.memory_subdir = memory_subdir
-
- # Get directory paths
- db_dir = self._get_db_dir()
- em_dir = os.path.join(db_dir, "embeddings")
-
- # Create directories
- os.makedirs(db_dir, exist_ok=True)
- os.makedirs(em_dir, exist_ok=True)
-
- # Setup embeddings cache
- store = LocalFileStore(em_dir)
- self.embeddings = CacheBackedEmbeddings.from_bytes_store(
- embeddings_model,
- store,
- namespace=f"document_query_{getattr(embeddings_model, 'model', getattr(embeddings_model, 'model_name', 'default'))}"
- )
-
- # Initialize vector store
- index_path = os.path.join(db_dir, "index.faiss")
- docstore_path = os.path.join(db_dir, "docstore.json")
-
- if os.path.exists(index_path) and os.path.exists(docstore_path):
- PrintStyle.standard(f"Loading existing vector store from {db_dir}")
- try:
- self.vectorstore = FAISS.load_local(
- folder_path=db_dir,
- embeddings=self.embeddings,
- allow_dangerous_deserialization=True,
- distance_strategy=DistanceStrategy.COSINE,
- )
- except Exception as e:
- PrintStyle.error(f"Error loading vector store: {str(e)}")
- self._initialize_new_vectorstore()
- else:
- PrintStyle.standard(f"Creating new vector store in '{db_dir}'")
- self._initialize_new_vectorstore()
-
- def _initialize_new_vectorstore(self):
- """Initialize a new vector store."""
- dimension = len(self.embeddings.embed_query("test"))
- index = faiss.IndexFlatIP(dimension)
- self.vectorstore = FAISS(
- embedding_function=self.embeddings,
- index=index,
- docstore=InMemoryDocstore(),
- index_to_docstore_id={},
- distance_strategy=DistanceStrategy.COSINE,
- )
-
- def _get_db_dir(self) -> str:
- """Get the absolute path to the database directory."""
- return files.get_abs_path("memory", self.memory_subdir, "document_query")
-
- def _save_vectorstore(self):
- """Save the vector store to disk."""
- db_dir = self._get_db_dir()
- PrintStyle.standard(f"Saving vector store to {db_dir}")
- self.vectorstore.save_local(folder_path=db_dir)
- PrintStyle.standard(f"Vector store saved with {len(self.vectorstore.index_to_docstore_id)} documents")
+ self.vector_db: VectorDB | None = None
@staticmethod
- def _normalize_uri(uri: str) -> str:
+ def normalize_uri(uri: str) -> str:
"""
Normalize a document URI to ensure consistent lookup.
@@ -166,7 +85,7 @@ class DocumentQueryStore:
Normalized URI
"""
# Convert to lowercase
- normalized = uri.lower()
+ normalized = uri.strip() # uri.lower()
# Parse the URL to get scheme
parsed = urlparse(normalized)
@@ -174,17 +93,23 @@ class DocumentQueryStore:
# Normalize based on scheme
if scheme == "file":
- if not normalized.startswith("file:"):
- normalized = "file:" + normalized
- if normalized.startswith("file://"):
- normalized = normalized.replace("file://", "file:")
+ path = files.fix_dev_path(
+ normalized.removeprefix("file://").removeprefix("file:")
+ )
+ normalized = f"file://{path}"
+
elif scheme in ["http", "https"]:
# Always use https for web URLs
- normalized = normalized.replace("http://", "https://")
+ normalized = normalized.replace("http://", "https://") # TODO why?
return normalized
- async def add_document(self, text: str, document_uri: str, metadata: dict | None = None) -> bool:
+ def init_vector_db(self):
+ return VectorDB(self.agent)
+
+ async def add_document(
+ self, text: str, document_uri: str, metadata: dict | None = None
+ ) -> bool:
"""
Add a document to the store with the given URI.
@@ -197,7 +122,7 @@ class DocumentQueryStore:
True if successful, False otherwise
"""
# Normalize the URI
- document_uri = self._normalize_uri(document_uri)
+ document_uri = self.normalize_uri(document_uri)
# Delete existing document if it exists to avoid duplicates
await self.delete_document(document_uri)
@@ -209,8 +134,7 @@ class DocumentQueryStore:
# Split text into chunks
text_splitter = RecursiveCharacterTextSplitter(
- chunk_size=self.DEFAULT_CHUNK_SIZE,
- chunk_overlap=self.DEFAULT_CHUNK_OVERLAP
+ chunk_size=self.DEFAULT_CHUNK_SIZE, chunk_overlap=self.DEFAULT_CHUNK_OVERLAP
)
chunks = text_splitter.split_text(text)
@@ -230,14 +154,17 @@ class DocumentQueryStore:
try:
docs_text = "".join(chunk.page_content for chunk in docs)
await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model,
- input=docs_text
+ model_config=self.agent.config.embeddings_model, input=docs_text
)
- # Add documents to vector store
- self.vectorstore.add_documents(documents=docs)
- self._save_vectorstore()
- PrintStyle.standard(f"Added document '{document_uri}' with {len(docs)} chunks")
+ # Initialize vector db if not already initialized
+ if not self.vector_db:
+ self.vector_db = self.init_vector_db()
+
+ ids = await self.vector_db.insert_documents(docs)
+ PrintStyle.standard(
+ f"Added document '{document_uri}' with {len(docs)} chunks"
+ )
return True
except Exception as e:
PrintStyle.error(f"Error adding document '{document_uri}': {str(e)}")
@@ -253,8 +180,13 @@ class DocumentQueryStore:
Returns:
The complete document if found, None otherwise
"""
+
+ # DB not initialized, no documents inside
+ if not self.vector_db:
+ return None
+
# Normalize the URI
- document_uri = self._normalize_uri(document_uri)
+ document_uri = self.normalize_uri(document_uri)
# Get all chunks for this document
docs = await self._get_document_chunks(document_uri)
@@ -283,14 +215,19 @@ class DocumentQueryStore:
Returns:
List of document chunks
"""
+
+ # DB not initialized, no documents inside
+ if not self.vector_db:
+ return []
+
# Normalize the URI
- document_uri = self._normalize_uri(document_uri)
+ document_uri = self.normalize_uri(document_uri)
- # Access docstore directly
- chunks = []
- for doc_id, doc in self.vectorstore.docstore._dict.items(): # type: ignore
- if isinstance(doc.metadata, dict) and doc.metadata.get("document_uri") == document_uri:
- chunks.append(doc)
+ # get docs from vector db
+
+ chunks = await self.vector_db.search_by_metadata(
+ filter=f"document_uri == '{document_uri}'",
+ )
PrintStyle.standard(f"Found {len(chunks)} chunks for document: {document_uri}")
return chunks
@@ -305,8 +242,13 @@ class DocumentQueryStore:
Returns:
True if the document exists, False otherwise
"""
+
+ # DB not initialized, no documents inside
+ if not self.vector_db:
+ return False
+
# Normalize the URI
- document_uri = self._normalize_uri(document_uri)
+ document_uri = self.normalize_uri(document_uri)
chunks = await self._get_document_chunks(document_uri)
return len(chunks) > 0
@@ -321,84 +263,36 @@ class DocumentQueryStore:
Returns:
True if deleted, False if not found
"""
+
+ # DB not initialized, no documents inside
+ if not self.vector_db:
+ return False
+
# Normalize the URI
- document_uri = self._normalize_uri(document_uri)
+ document_uri = self.normalize_uri(document_uri)
- chunks = await self._get_document_chunks(document_uri)
+ chunks = await self.vector_db.search_by_metadata(
+ filter=f"document_uri == '{document_uri}'",
+ )
if not chunks:
return False
# Collect IDs to delete
- ids_to_delete = []
- for chunk in chunks:
- for doc_id, doc_ref in self.vectorstore.docstore._dict.items(): # type: ignore
- if doc_ref == chunk:
- ids_to_delete.append(doc_id)
+ ids_to_delete = [chunk.metadata["id"] for chunk in chunks]
# Delete from vector store
if ids_to_delete:
- self.vectorstore.delete(ids_to_delete)
- self._save_vectorstore()
- PrintStyle.standard(f"Deleted document '{document_uri}' with {len(ids_to_delete)} chunks")
+ dels = await self.vector_db.delete_documents_by_ids(ids_to_delete)
+ PrintStyle.standard(
+ f"Deleted document '{document_uri}' with {len(dels)} chunks"
+ )
return True
return False
- async def expire_documents(self, older_than_days: float) -> int:
- """
- Delete documents older than the specified number of days.
-
- Args:
- older_than_days: Number of days (can be fractional) before current time
-
- Returns:
- Number of documents deleted
- """
- if older_than_days <= 0:
- return 0
-
- # Calculate cutoff timestamp
- cutoff_date = datetime.now().timestamp() - (older_than_days * 24 * 60 * 60)
-
- # Find expired documents
- expired_uris = set()
-
- # Check all documents in the store
- for doc_id, doc in self.vectorstore.docstore._dict.items(): # type: ignore
- if not isinstance(doc.metadata, dict):
- continue
-
- # Only process each document once (first chunk)
- if doc.metadata.get("chunk_index", 0) != 0:
- continue
-
- doc_uri = doc.metadata.get("document_uri")
- if not doc_uri:
- continue
-
- try:
- # Check timestamp
- timestamp_str = doc.metadata.get("timestamp")
- if not timestamp_str:
- continue
-
- doc_timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S").timestamp()
- if doc_timestamp < cutoff_date:
- expired_uris.add(doc_uri)
- except (ValueError, TypeError):
- # Skip documents with invalid timestamps
- continue
-
- # Delete expired documents
- deleted_count = 0
- for uri in expired_uris:
- if await self.delete_document(uri):
- deleted_count += 1
-
- PrintStyle.standard(f"Expired {deleted_count} documents older than {older_than_days} days")
- return deleted_count
-
- async def search_documents(self, query: str, limit: int = 10, threshold: float = 0.5) -> List[Document]:
+ async def search_documents(
+ self, query: str, limit: int = 10, threshold: float = 0.5, filter: str = ""
+ ) -> List[Document]:
"""
Search for documents similar to the query across the entire store.
@@ -410,34 +304,30 @@ class DocumentQueryStore:
Returns:
List of matching documents
"""
+
+ # DB not initialized, no documents inside
+ if not self.vector_db:
+ return []
+
# Handle empty query
if not query:
- PrintStyle.standard("Empty search query, returning empty results")
return []
- # Apply rate limiter
- await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model,
- input=query
- )
-
# Perform search
try:
- results = self.vectorstore.similarity_search_with_score(
- query=query,
- k=limit,
- score_threshold=threshold
+ results = await self.vector_db.search_by_similarity_threshold(
+ query=query, limit=limit, threshold=threshold, filter=filter
)
- # Extract documents from results (which are (doc, score) pairs)
- docs = [doc for doc, score in results]
- PrintStyle.standard(f"Search '{query}' returned {len(docs)} results")
- return docs
+ PrintStyle.standard(f"Search '{query}' returned {len(results)} results")
+ return results
except Exception as e:
PrintStyle.error(f"Error searching documents: {str(e)}")
return []
- async def search_document(self, document_uri: str, query: str, limit: int = 10, threshold: float = 0.5) -> List[Document]:
+ async def search_document(
+ self, document_uri: str, query: str, limit: int = 10, threshold: float = 0.5
+ ) -> List[Document]:
"""
Search for content within a specific document.
@@ -450,59 +340,10 @@ class DocumentQueryStore:
Returns:
List of matching document chunks
"""
- # Normalize the URI
- document_uri = self._normalize_uri(document_uri)
-
- # Handle empty query
- if not query:
- PrintStyle.standard("Empty search query, returning empty results")
- return []
-
- # Check if document exists
- if not await self.document_exists(document_uri):
- PrintStyle.error(f"Document not found: {document_uri}")
- return []
-
- # Apply rate limiter
- await self.agent.rate_limiter(
- model_config=self.agent.config.embeddings_model,
- input=query
+ return await self.search_documents(
+ query, limit, threshold, f"document_uri == '{document_uri}'"
)
- # Perform search with document filter
- try:
- # Create metadata filter function
- def filter_fn(doc_metadata):
- return doc_metadata.get("document_uri") == document_uri
-
- results = self.vectorstore.similarity_search_with_score(
- query=query,
- k=limit,
- score_threshold=threshold,
- filter=filter_fn
- )
-
- # Extract documents from results
- docs = [doc for doc, score in results]
- PrintStyle.standard(f"Search '{query}' in document '{document_uri}' returned {len(docs)} results")
-
- # Try with lower threshold if no results
- if not docs and threshold > 0.3:
- PrintStyle.standard("No results found, trying with lower threshold (0.3)")
- results = self.vectorstore.similarity_search_with_score(
- query=query,
- k=limit,
- score_threshold=0.3,
- filter=filter_fn
- )
- docs = [doc for doc, score in results]
- PrintStyle.standard(f"Retry search returned {len(docs)} results")
-
- return docs
- except Exception as e:
- PrintStyle.error(f"Error searching within document: {str(e)}")
- return []
-
async def list_documents(self) -> List[str]:
"""
Get a list of all document URIs in the store.
@@ -510,9 +351,13 @@ class DocumentQueryStore:
Returns:
List of document URIs
"""
+ # DB not initialized, no documents inside
+ if not self.vector_db:
+ return []
+
# Extract unique URIs
uris = set()
- for doc in self.vectorstore.docstore._dict.values(): # type: ignore
+ for doc in self.vector_db.db.get_all_docs().values():
if isinstance(doc.metadata, dict):
uri = doc.metadata.get("document_uri")
if uri:
@@ -525,27 +370,37 @@ class DocumentQueryHelper:
def __init__(self, agent: Agent):
self.agent = agent
- self.store: DocumentQueryStore = asyncio.run(DocumentQueryStore.get(agent))
+ self.store = DocumentQueryStore.get(agent)
- async def document_qa(self, document_uri: str, questions: Sequence[str]) -> Tuple[bool, str]:
- _ = await self.document_get_content(document_uri)
+ async def document_qa(
+ self, document_uri: str, questions: Sequence[str]
+ ) -> Tuple[bool, str]:
+ # index document
+ _ = await self.document_get_content(document_uri, True)
content = ""
for question in questions:
human_content = f'Search Query: "{question}"'
- system_content = self.agent.parse_prompt("fw.document_query.optmimize_query.md")
-
- optimized_query = await self.agent.call_utility_model(
- system=system_content,
- message=human_content
+ system_content = self.agent.parse_prompt(
+ "fw.document_query.optmimize_query.md"
)
+ optimized_query = (
+ await self.agent.call_utility_model(
+ system=system_content, message=human_content
+ )
+ ).strip()
+
+ normalized_uri = self.store.normalize_uri(document_uri)
chunks = await self.store.search_document(
- document_uri=document_uri,
- query=str(optimized_query),
- limit=10000,
- threshold=0.66
+ document_uri=normalized_uri,
+ query=optimized_query,
+ limit=100,
+ threshold=DEFAULT_SEARCH_THRESHOLD,
+ )
+ content += (
+ "\n\n----\n\n".join([chunk.page_content for chunk in chunks])
+ + "\n\n----\n\n"
)
- content += "\n\n----\n\n".join([chunk.page_content for chunk in chunks]) + "\n\n----\n\n"
if not content:
content = f"!!! No content found for document: {document_uri} matching queries: {json.dumps(questions)}"
@@ -553,19 +408,23 @@ class DocumentQueryHelper:
questions_str = "\n".join([f" * {question}" for question in questions])
- qa_system_message = self.agent.parse_prompt("fw.document_query.system_prompt.md")
+ qa_system_message = self.agent.parse_prompt(
+ "fw.document_query.system_prompt.md"
+ )
qa_user_message = f"# Document:\n{content}\n\n# Queries:\n{questions_str}"
ai_response = await self.agent.call_chat_model(
- prompt=ChatPromptTemplate.from_messages([
- SystemMessage(content=qa_system_message),
- HumanMessage(content=qa_user_message),
- ])
+ prompt=ChatPromptTemplate.from_messages(
+ [
+ SystemMessage(content=qa_system_message),
+ HumanMessage(content=qa_user_message),
+ ]
+ )
)
return True, str(ai_response)
- async def document_get_content(self, document_uri: str) -> str:
+ async def document_get_content(self, document_uri: str, add_to_db: bool = False) -> str:
url = urlparse(document_uri)
scheme = url.scheme or "file"
mimetype, encoding = mimetypes.guess_type(document_uri)
@@ -579,7 +438,11 @@ class DocumentQueryHelper:
while not response and retries < 3:
try:
async with aiohttp.ClientSession() as session:
- response = await session.head(document_uri, timeout=aiohttp.ClientTimeout(total=2.0), allow_redirects=True)
+ response = await session.head(
+ document_uri,
+ timeout=aiohttp.ClientTimeout(total=2.0),
+ allow_redirects=True,
+ )
if response.status > 399:
raise Exception(response.status)
break
@@ -589,32 +452,41 @@ class DocumentQueryHelper:
retries += 1
if not response:
- raise ValueError(f"DocumentQueryHelper::document_get_content: Document fetch error: {document_uri} ({last_error})")
+ raise ValueError(
+ f"DocumentQueryHelper::document_get_content: Document fetch error: {document_uri} ({last_error})"
+ )
mimetype = response.headers["content-type"]
if "content-length" in response.headers:
- content_length = float(response.headers["content-length"]) / 1024 / 1024 # MB
+ content_length = (
+ float(response.headers["content-length"]) / 1024 / 1024
+ ) # MB
if content_length > 50.0:
- raise ValueError(f"Document content length exceeds max. 50MB: {content_length} MB ({document_uri})")
- if mimetype and '; charset=' in mimetype:
- mimetype = mimetype.split('; charset=')[0]
+ raise ValueError(
+ f"Document content length exceeds max. 50MB: {content_length} MB ({document_uri})"
+ )
+ if mimetype and "; charset=" in mimetype:
+ mimetype = mimetype.split("; charset=")[0]
if scheme == "file":
try:
- document_uri = os.path.abspath(url.path)
+ document_uri = files.fix_dev_path(url.path)
except Exception as e:
raise ValueError(f"Invalid document path '{url.path}'") from e
if encoding:
- raise ValueError(f"Compressed documents are unsupported '{encoding}' ({document_uri})")
+ raise ValueError(
+ f"Compressed documents are unsupported '{encoding}' ({document_uri})"
+ )
if mimetype == "application/octet-stream":
- raise ValueError(f"Unsupported document mimetype '{mimetype}' ({document_uri})")
+ raise ValueError(
+ f"Unsupported document mimetype '{mimetype}' ({document_uri})"
+ )
# Use the store's normalization method
- document_uri_norm = self.store._normalize_uri(document_uri)
+ document_uri_norm = self.store.normalize_uri(document_uri)
- await self.store.expire_documents(7)
exists = await self.store.document_exists(document_uri_norm)
document_content = ""
if not exists:
@@ -627,14 +499,19 @@ class DocumentQueryHelper:
elif mimetype == "application/pdf":
document_content = self.handle_pdf_document(document_uri, scheme)
else:
- document_content = self.handle_unstructured_document(document_uri, scheme)
- await self.store.add_document(document_content, document_uri_norm)
+ document_content = self.handle_unstructured_document(
+ document_uri, scheme
+ )
+ if add_to_db:
+ await self.store.add_document(document_content, document_uri_norm)
else:
doc = await self.store.get_document(document_uri_norm)
if doc:
document_content = doc.page_content
else:
- raise ValueError(f"DocumentQueryHelper::document_get_content: Document not found: {document_uri_norm}")
+ raise ValueError(
+ f"DocumentQueryHelper::document_get_content: Document not found: {document_uri_norm}"
+ )
return document_content
def handle_image_document(self, document: str, scheme: str) -> str:
@@ -646,14 +523,19 @@ class DocumentQueryHelper:
parts: list[Document] = loader.load()
elif scheme == "file":
# Use RFC file operations instead of TextLoader
- file_content_bytes = rfc_files.read_file_bin(document)
- file_content = file_content_bytes.decode('utf-8')
+ file_content_bytes = files.read_file_bin(document)
+ file_content = file_content_bytes.decode("utf-8")
# Create Document manually since we're not using TextLoader
parts = [Document(page_content=file_content, metadata={"source": document})]
else:
raise ValueError(f"Unsupported scheme: {scheme}")
- return "\n".join([element.page_content for element in MarkdownifyTransformer().transform_documents(parts)])
+ return "\n".join(
+ [
+ element.page_content
+ for element in MarkdownifyTransformer().transform_documents(parts)
+ ]
+ )
def handle_text_document(self, document: str, scheme: str) -> str:
if scheme in ["http", "https"]:
@@ -661,10 +543,12 @@ class DocumentQueryHelper:
elements: list[Document] = loader.load()
elif scheme == "file":
# Use RFC file operations instead of TextLoader
- file_content_bytes = rfc_files.read_file_bin(document)
- file_content = file_content_bytes.decode('utf-8')
+ file_content_bytes = files.read_file_bin(document)
+ file_content = file_content_bytes.decode("utf-8")
# Create Document manually since we're not using TextLoader
- elements = [Document(page_content=file_content, metadata={"source": document})]
+ elements = [
+ Document(page_content=file_content, metadata={"source": document})
+ ]
else:
raise ValueError(f"Unsupported scheme: {scheme}")
@@ -674,27 +558,33 @@ class DocumentQueryHelper:
temp_file_path = ""
if scheme == "file":
# Use RFC file operations to read the PDF file as binary
- file_content_bytes = rfc_files.read_file_bin(document)
+ file_content_bytes = files.read_file_bin(document)
# Create a temporary file for PyMuPDFLoader since it needs a file path
import tempfile
- with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
temp_file.write(file_content_bytes)
temp_file_path = temp_file.name
elif scheme in ["http", "https"]:
# download the file from the web url to a temporary file using python libraries for downloading
import requests
import tempfile
- with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_file:
response = requests.get(document, timeout=10.0)
if response.status_code != 200:
- raise ValueError(f"DocumentQueryHelper::handle_pdf_document: Failed to download PDF from {document}: {response.status_code}")
+ raise ValueError(
+ f"DocumentQueryHelper::handle_pdf_document: Failed to download PDF from {document}: {response.status_code}"
+ )
temp_file.write(response.content)
temp_file_path = temp_file.name
else:
raise ValueError(f"Unsupported scheme: {scheme}")
if not os.path.exists(temp_file_path):
- raise ValueError(f"DocumentQueryHelper::handle_pdf_document: Temporary file not found: {temp_file_path}")
+ raise ValueError(
+ f"DocumentQueryHelper::handle_pdf_document: Temporary file not found: {temp_file_path}"
+ )
try:
try:
@@ -710,17 +600,21 @@ class DocumentQueryHelper:
elements: list[Document] = loader.load()
contents = "\n".join([element.page_content for element in elements])
except Exception as e:
- PrintStyle.error(f"DocumentQueryHelper::handle_pdf_document: Error loading with PyMuPDF: {e}")
+ PrintStyle.error(
+ f"DocumentQueryHelper::handle_pdf_document: Error loading with PyMuPDF: {e}"
+ )
contents = ""
if not contents:
import pdf2image
import pytesseract
- PrintStyle.debug(f"DocumentQueryHelper::handle_pdf_document: FALLBACK Converting PDF to images: {temp_file_path}")
+ PrintStyle.debug(
+ f"DocumentQueryHelper::handle_pdf_document: FALLBACK Converting PDF to images: {temp_file_path}"
+ )
# Convert PDF to images
- pages = pdf2image.convert_from_path(temp_file_path) # type: ignore
+ pages = pdf2image.convert_from_path(temp_file_path) # type: ignore
for page in pages:
contents += pytesseract.image_to_string(page) + "\n\n"
@@ -742,10 +636,11 @@ class DocumentQueryHelper:
elements = loader.load()
elif scheme == "file":
# Use RFC file operations to read the file as binary
- file_content_bytes = rfc_files.read_file_bin(document)
+ file_content_bytes = files.read_file_bin(document)
# Create a temporary file for UnstructuredLoader since it needs a file path
import tempfile
import os
+
# Get file extension to preserve it for proper processing
_, ext = os.path.splitext(document)
with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as temp_file:
diff --git a/python/helpers/files.py b/python/helpers/files.py
index 826d5a4c0..f562a543b 100644
--- a/python/helpers/files.py
+++ b/python/helpers/files.py
@@ -253,8 +253,16 @@ def make_dirs(relative_path: str):
def get_abs_path(*relative_paths):
+ "Convert relative paths to absolute paths based on the base directory."
return os.path.join(get_base_dir(), *relative_paths)
+def fix_dev_path(path:str):
+ "On dev environment, convert /a0/... paths to local absolute paths"
+ from python.helpers.runtime import is_development
+ if is_development():
+ if path.startswith("/a0/"):
+ path = path.replace("/a0/", "")
+ return get_abs_path(path)
def exists(*relative_paths):
path = get_abs_path(*relative_paths)
diff --git a/python/helpers/vector_db.py b/python/helpers/vector_db.py
index 21bd55ce6..128cccfbf 100644
--- a/python/helpers/vector_db.py
+++ b/python/helpers/vector_db.py
@@ -27,27 +27,40 @@ class MyFaiss(FAISS):
async def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:
return self.get_by_ids(ids)
+ def get_all_docs(self) -> dict[str, Document]:
+ return self.docstore._dict # type: ignore
+
class VectorDB:
- def __init__(self, agent: Agent):
- self.agent = agent
- self.store = InMemoryByteStore()
- self.model = agent.get_embedding_model()
-
- self.embedder = CacheBackedEmbeddings.from_bytes_store(
- self.model,
- self.store,
- namespace=getattr(
- self.model,
- "model",
- getattr(self.model, "model_name", "default"),
- ),
+
+ _cached_embeddings: dict[str, CacheBackedEmbeddings] = {}
+
+ @staticmethod
+ def _get_embeddings(agent: Agent):
+ model = agent.get_embedding_model()
+ namespace = getattr(
+ model,
+ "model",
+ getattr(model, "model_name", "default"),
)
+ if namespace not in VectorDB._cached_embeddings:
+ store = InMemoryByteStore()
+ VectorDB._cached_embeddings[namespace] = (
+ CacheBackedEmbeddings.from_bytes_store(
+ model,
+ store,
+ namespace=namespace,
+ )
+ )
+ return VectorDB._cached_embeddings[namespace]
- self.index = faiss.IndexFlatIP(len(self.embedder.embed_query("example")))
+ def __init__(self, agent: Agent):
+ self.agent = agent
+ self.embeddings = self._get_embeddings(agent)
+ self.index = faiss.IndexFlatIP(len(self.embeddings.embed_query("example")))
self.db = MyFaiss(
- embedding_function=self.embedder,
+ embedding_function=self.embeddings,
index=self.index,
docstore=InMemoryDocstore(),
index_to_docstore_id={},
@@ -56,7 +69,7 @@ class VectorDB:
relevance_score_fn=cosine_normalizer,
)
- async def search_similarity_threshold(
+ async def search_by_similarity_threshold(
self, query: str, limit: int, threshold: float, filter: str = ""
):
comparator = get_comparator(filter) if filter else None
@@ -74,6 +87,18 @@ class VectorDB:
filter=comparator,
)
+ async def search_by_metadata(self, filter: str, limit: int = 0) -> list[Document]:
+ comparator = get_comparator(filter)
+ all_docs = self.db.get_all_docs()
+ result = []
+ for doc in all_docs.values():
+ if comparator(doc.metadata):
+ result.append(doc)
+ # stop if limit reached and limit > 0
+ if limit > 0 and len(result) >= limit:
+ break
+ return result
+
async def insert_documents(self, docs: list[Document]):
ids = [str(uuid.uuid4()) for _ in range(len(docs))]
@@ -90,6 +115,16 @@ class VectorDB:
self.db.add_documents(documents=docs, ids=ids)
return ids
+ async def delete_documents_by_ids(self, ids: list[str]):
+ # aget_by_ids is not yet implemented in faiss, need to do a workaround
+ rem_docs = await self.db.aget_by_ids(
+ ids
+ ) # existing docs to remove (prevents error)
+ if rem_docs:
+ rem_ids = [doc.metadata["id"] for doc in rem_docs] # ids to remove
+ await self.db.adelete(ids=rem_ids)
+ return rem_docs
+
def format_docs_plain(docs: list[Document]) -> list[str]:
result = []
@@ -113,7 +148,8 @@ def cosine_normalizer(val: float) -> float:
def get_comparator(condition: str):
def comparator(data: dict[str, Any]):
try:
- return eval(condition, {}, data)
+ result = eval(condition, {}, data)
+ return result
except Exception as e:
# PrintStyle.error(f"Error evaluating condition: {e}")
return False