Agent Zero AI framework
RAG tool merge
frdela9d39876099c
diff --cc docker/run/fs/ins/install_additional.shindex 53e78c43e,53e78c43e..b2d79b6dc--- a/docker/run/fs/ins/install_additional.sh+++ b/docker/run/fs/ins/install_additional.sh@@@ -4,4 -4,4 +4,9 @@@# bash /ins/install_playwright.sh "$@"# searxng - moved to base image--# bash /ins/install_searxng.sh "$@"++# bash /ins/install_searxng.sh "$@"++++#TODO : move to base image++++apt-get install -y --no-install-recommends \++ tesseract-ocr-all poppler-utilsdiff --cc python/helpers/document_query.pyindex 000000000,2b6ec7689..c059e0c49mode 000000,100644..100644--- a/python/helpers/document_query.py+++ b/python/helpers/document_query.py@@@ -1,0 -1,768 +1,770 @@@+ import mimetypes+ import os+ import asyncio+ import aiohttp+ import json++ os.environ["USER_AGENT"] = "@mixedbread-ai/unstructured" # noqa E402+ from langchain_unstructured import UnstructuredLoader # noqa E402++ from urllib.parse import urlparse+ from typing import Sequence, List, Optional, Tuple+ from datetime import datetime++ from langchain_community.document_loaders import AsyncHtmlLoader+ from langchain_community.document_loaders.text import TextLoader+ from langchain_community.document_loaders.pdf import PyMuPDFLoader+ from langchain_community.document_transformers import MarkdownifyTransformer+ from langchain_community.document_loaders.parsers.images import TesseractBlobParser++ from langchain_core.documents import Document+ from langchain.prompts import ChatPromptTemplate+ from langchain.schema import SystemMessage, HumanMessage-from langchain.storage import LocalFileStore++from langchain.storage import LocalFileStore, InMemoryStore+ from langchain.embeddings import CacheBackedEmbeddings++ from langchain_community.vectorstores import FAISS+ import faiss+ from langchain_community.docstore.in_memory import InMemoryDocstore+ from langchain_community.vectorstores.utils import (+ DistanceStrategy,+ )+ from langchain_core.embeddings import Embeddings++ from python.helpers.print_style import PrintStyle-from python.helpers import files, rfc_files++from python.helpers import files #, rfc_files++rfc_files = files # TODO: fix+++ from agent import Agent+ import models++ from langchain.text_splitter import RecursiveCharacterTextSplitter+++ class DocumentQueryStore:+ """+ FAISS Store for document query results.+ Manages documents identified by URI for storage, retrieval, and searching.+ """++ # Default chunking parameters+ DEFAULT_CHUNK_SIZE = 1000+ DEFAULT_CHUNK_OVERLAP = 100++ # Cache for initialized stores+ _stores: dict[str, "DocumentQueryStore"] = {}++ @staticmethod+ async def get(agent: Agent):+ """Get or create a DocumentQueryStore instance for the specified agent."""+ if not agent or not agent.config:+ raise ValueError("Agent and agent config must be provided")++ memory_subdir = agent.config.memory_subdir or "default"+ store_key = f"{memory_subdir}/document_query"++ if store_key not in DocumentQueryStore._stores:+ # Initialize embeddings model from agent config+ embeddings_model = agent.get_embedding_model()++ # Initialize store+ store = DocumentQueryStore(agent, embeddings_model, memory_subdir)+ DocumentQueryStore._stores[store_key] = store+ return store+ else:+ return DocumentQueryStore._stores[store_key]++ @staticmethod+ async def reload(agent: Agent):+ """Reload the DocumentQueryStore for the specified agent."""+ memory_subdir = agent.config.memory_subdir or "default"+ store_key = f"{memory_subdir}/document_query"++ if store_key in DocumentQueryStore._stores:+ del DocumentQueryStore._stores[store_key]++ return await DocumentQueryStore.get(agent)++ def __init__(+ self,+ agent: Agent,+ embeddings_model: Embeddings,+ memory_subdir: str,+ ):+ """Initialize a DocumentQueryStore instance."""+ self.agent = agent+ self.memory_subdir = memory_subdir++ # Get directory paths+ db_dir = self._get_db_dir()+ em_dir = os.path.join(db_dir, "embeddings")++ # Create directories+ os.makedirs(db_dir, exist_ok=True)+ os.makedirs(em_dir, exist_ok=True)++ # Setup embeddings cache+ store = LocalFileStore(em_dir)+ self.embeddings = CacheBackedEmbeddings.from_bytes_store(+ embeddings_model,+ store,+ namespace=f"document_query_{getattr(embeddings_model, 'model', getattr(embeddings_model, 'model_name', 'default'))}"+ )++ # Initialize vector store+ index_path = os.path.join(db_dir, "index.faiss")+ docstore_path = os.path.join(db_dir, "docstore.json")++ if os.path.exists(index_path) and os.path.exists(docstore_path):+ PrintStyle.standard(f"Loading existing vector store from {db_dir}")+ try:+ self.vectorstore = FAISS.load_local(+ folder_path=db_dir,+ embeddings=self.embeddings,+ allow_dangerous_deserialization=True,+ distance_strategy=DistanceStrategy.COSINE,+ )+ except Exception as e:+ PrintStyle.error(f"Error loading vector store: {str(e)}")+ self._initialize_new_vectorstore()+ else:- PrintStyle.standard("Creating new vector store in '{db_dir}'")++ PrintStyle.standard(f"Creating new vector store in '{db_dir}'")+ self._initialize_new_vectorstore()++ def _initialize_new_vectorstore(self):+ """Initialize a new vector store."""+ dimension = len(self.embeddings.embed_query("test"))+ index = faiss.IndexFlatIP(dimension)+ self.vectorstore = FAISS(+ embedding_function=self.embeddings,+ index=index,+ docstore=InMemoryDocstore(),+ index_to_docstore_id={},+ distance_strategy=DistanceStrategy.COSINE,+ )++ def _get_db_dir(self) -> str:+ """Get the absolute path to the database directory."""+ return files.get_abs_path("memory", self.memory_subdir, "document_query")++ def _save_vectorstore(self):+ """Save the vector store to disk."""+ db_dir = self._get_db_dir()+ PrintStyle.standard(f"Saving vector store to {db_dir}")+ self.vectorstore.save_local(folder_path=db_dir)+ PrintStyle.standard(f"Vector store saved with {len(self.vectorstore.index_to_docstore_id)} documents")++ @staticmethod+ def _normalize_uri(uri: str) -> str:+ """+ Normalize a document URI to ensure consistent lookup.++ Args:+ uri: The URI to normalize++ Returns:+ Normalized URI+ """+ # Convert to lowercase+ normalized = uri.lower()++ # Parse the URL to get scheme+ parsed = urlparse(normalized)+ scheme = parsed.scheme or "file"++ # Normalize based on scheme+ if scheme == "file":+ if not normalized.startswith("file:"):+ normalized = "file:" + normalized+ if normalized.startswith("file://"):+ normalized = normalized.replace("file://", "file:")+ elif scheme in ["http", "https"]:+ # Always use https for web URLs+ normalized = normalized.replace("http://", "https://")++ return normalized+- async def add_document(self, text: str, document_uri: str, metadata: dict = None) -> bool:++ async def add_document(self, text: str, document_uri: str, metadata: dict | None = None) -> bool:+ """+ Add a document to the store with the given URI.++ Args:+ text: The document text content+ document_uri: The URI that uniquely identifies this document+ metadata: Optional metadata for the document++ Returns:+ True if successful, False otherwise+ """+ # Normalize the URI+ document_uri = self._normalize_uri(document_uri)++ # Delete existing document if it exists to avoid duplicates+ await self.delete_document(document_uri)++ # Initialize metadata+ doc_metadata = metadata or {}+ doc_metadata["document_uri"] = document_uri+ doc_metadata["timestamp"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")++ # Split text into chunks+ text_splitter = RecursiveCharacterTextSplitter(+ chunk_size=self.DEFAULT_CHUNK_SIZE,+ chunk_overlap=self.DEFAULT_CHUNK_OVERLAP+ )+ chunks = text_splitter.split_text(text)++ # Create documents+ docs = []+ for i, chunk in enumerate(chunks):+ chunk_metadata = doc_metadata.copy()+ chunk_metadata["chunk_index"] = i+ chunk_metadata["total_chunks"] = len(chunks)+ docs.append(Document(page_content=chunk, metadata=chunk_metadata))++ if not docs:+ PrintStyle.error(f"No chunks created for document: {document_uri}")+ return False++ # Apply rate limiter+ try:+ docs_text = "".join(chunk.page_content for chunk in docs)+ await self.agent.rate_limiter(+ model_config=self.agent.config.embeddings_model,+ input=docs_text+ )++ # Add documents to vector store+ self.vectorstore.add_documents(documents=docs)+ self._save_vectorstore()+ PrintStyle.standard(f"Added document '{document_uri}' with {len(docs)} chunks")+ return True+ except Exception as e:+ PrintStyle.error(f"Error adding document '{document_uri}': {str(e)}")+ return False++ async def get_document(self, document_uri: str) -> Optional[Document]:+ """+ Retrieve a document by its URI.++ Args:+ document_uri: The URI of the document to retrieve++ Returns:+ The complete document if found, None otherwise+ """+ # Normalize the URI+ document_uri = self._normalize_uri(document_uri)++ # Get all chunks for this document+ docs = await self._get_document_chunks(document_uri)+ if not docs:+ PrintStyle.error(f"Document not found: {document_uri}")+ return None++ # Combine chunks into a single document+ chunks = sorted(docs, key=lambda x: x.metadata.get("chunk_index", 0))+ full_content = "\n".join(chunk.page_content for chunk in chunks)++ # Use metadata from first chunk+ metadata = chunks[0].metadata.copy()+ metadata.pop("chunk_index", None)+ metadata.pop("total_chunks", None)++ return Document(page_content=full_content, metadata=metadata)++ async def _get_document_chunks(self, document_uri: str) -> List[Document]:+ """+ Get all chunks for a document.++ Args:+ document_uri: The URI of the document++ Returns:+ List of document chunks+ """+ # Normalize the URI+ document_uri = self._normalize_uri(document_uri)++ # Access docstore directly+ chunks = []+ for doc_id, doc in self.vectorstore.docstore._dict.items(): # type: ignore+ if isinstance(doc.metadata, dict) and doc.metadata.get("document_uri") == document_uri:+ chunks.append(doc)++ PrintStyle.standard(f"Found {len(chunks)} chunks for document: {document_uri}")+ return chunks++ async def document_exists(self, document_uri: str) -> bool:+ """+ Check if a document exists in the store.++ Args:+ document_uri: The URI of the document to check++ Returns:+ True if the document exists, False otherwise+ """+ # Normalize the URI+ document_uri = self._normalize_uri(document_uri)++ chunks = await self._get_document_chunks(document_uri)+ return len(chunks) > 0++ async def delete_document(self, document_uri: str) -> bool:+ """+ Delete a document from the store.++ Args:+ document_uri: The URI of the document to delete++ Returns:+ True if deleted, False if not found+ """+ # Normalize the URI+ document_uri = self._normalize_uri(document_uri)++ chunks = await self._get_document_chunks(document_uri)+ if not chunks:+ return False++ # Collect IDs to delete+ ids_to_delete = []+ for chunk in chunks:+ for doc_id, doc_ref in self.vectorstore.docstore._dict.items(): # type: ignore+ if doc_ref == chunk:+ ids_to_delete.append(doc_id)++ # Delete from vector store+ if ids_to_delete:+ self.vectorstore.delete(ids_to_delete)+ self._save_vectorstore()+ PrintStyle.standard(f"Deleted document '{document_uri}' with {len(ids_to_delete)} chunks")+ return True++ return False++ async def expire_documents(self, older_than_days: float) -> int:+ """+ Delete documents older than the specified number of days.++ Args:+ older_than_days: Number of days (can be fractional) before current time++ Returns:+ Number of documents deleted+ """+ if older_than_days <= 0:+ return 0++ # Calculate cutoff timestamp+ cutoff_date = datetime.now().timestamp() - (older_than_days * 24 * 60 * 60)++ # Find expired documents+ expired_uris = set()++ # Check all documents in the store+ for doc_id, doc in self.vectorstore.docstore._dict.items(): # type: ignore+ if not isinstance(doc.metadata, dict):+ continue++ # Only process each document once (first chunk)+ if doc.metadata.get("chunk_index", 0) != 0:+ continue++ doc_uri = doc.metadata.get("document_uri")+ if not doc_uri:+ continue++ try:+ # Check timestamp+ timestamp_str = doc.metadata.get("timestamp")+ if not timestamp_str:+ continue++ doc_timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S").timestamp()+ if doc_timestamp < cutoff_date:+ expired_uris.add(doc_uri)+ except (ValueError, TypeError):+ # Skip documents with invalid timestamps+ continue++ # Delete expired documents+ deleted_count = 0+ for uri in expired_uris:+ if await self.delete_document(uri):+ deleted_count += 1++ PrintStyle.standard(f"Expired {deleted_count} documents older than {older_than_days} days")+ return deleted_count++ async def search_documents(self, query: str, limit: int = 10, threshold: float = 0.5) -> List[Document]:+ """+ Search for documents similar to the query across the entire store.++ Args:+ query: The search query string+ limit: Maximum number of results to return+ threshold: Minimum similarity score threshold (0-1)++ Returns:+ List of matching documents+ """+ # Handle empty query+ if not query:+ PrintStyle.standard("Empty search query, returning empty results")+ return []++ # Apply rate limiter+ await self.agent.rate_limiter(+ model_config=self.agent.config.embeddings_model,+ input=query+ )++ # Perform search+ try:+ results = self.vectorstore.similarity_search_with_score(+ query=query,+ k=limit,+ score_threshold=threshold+ )++ # Extract documents from results (which are (doc, score) pairs)+ docs = [doc for doc, score in results]+ PrintStyle.standard(f"Search '{query}' returned {len(docs)} results")+ return docs+ except Exception as e:+ PrintStyle.error(f"Error searching documents: {str(e)}")+ return []++ async def search_document(self, document_uri: str, query: str, limit: int = 10, threshold: float = 0.5) -> List[Document]:+ """+ Search for content within a specific document.++ Args:+ document_uri: The URI of the document to search within+ query: The search query string+ limit: Maximum number of results to return+ threshold: Minimum similarity score threshold (0-1)++ Returns:+ List of matching document chunks+ """+ # Normalize the URI+ document_uri = self._normalize_uri(document_uri)++ # Handle empty query+ if not query:+ PrintStyle.standard("Empty search query, returning empty results")+ return []++ # Check if document exists+ if not await self.document_exists(document_uri):+ PrintStyle.error(f"Document not found: {document_uri}")+ return []++ # Apply rate limiter+ await self.agent.rate_limiter(+ model_config=self.agent.config.embeddings_model,+ input=query+ )++ # Perform search with document filter+ try:+ # Create metadata filter function+ def filter_fn(doc_metadata):+ return doc_metadata.get("document_uri") == document_uri++ results = self.vectorstore.similarity_search_with_score(+ query=query,+ k=limit,+ score_threshold=threshold,+ filter=filter_fn+ )++ # Extract documents from results+ docs = [doc for doc, score in results]+ PrintStyle.standard(f"Search '{query}' in document '{document_uri}' returned {len(docs)} results")++ # Try with lower threshold if no results+ if not docs and threshold > 0.3:+ PrintStyle.standard("No results found, trying with lower threshold (0.3)")+ results = self.vectorstore.similarity_search_with_score(+ query=query,+ k=limit,+ score_threshold=0.3,+ filter=filter_fn+ )+ docs = [doc for doc, score in results]+ PrintStyle.standard(f"Retry search returned {len(docs)} results")++ return docs+ except Exception as e:+ PrintStyle.error(f"Error searching within document: {str(e)}")+ return []++ async def list_documents(self) -> List[str]:+ """+ Get a list of all document URIs in the store.++ Returns:+ List of document URIs+ """+ # Extract unique URIs+ uris = set()+ for doc in self.vectorstore.docstore._dict.values(): # type: ignore+ if isinstance(doc.metadata, dict):+ uri = doc.metadata.get("document_uri")+ if uri:+ uris.add(uri)++ return sorted(list(uris))+++ class DocumentQueryHelper:++ def __init__(self, agent: Agent):+ self.agent = agent+ self.store: DocumentQueryStore = asyncio.run(DocumentQueryStore.get(agent))++ async def document_qa(self, document_uri: str, questions: Sequence[str]) -> Tuple[bool, str]:+ _ = await self.document_get_content(document_uri)+ content = ""+ for question in questions:+ human_content = f'Search Query: "{question}"'+ system_content = self.agent.parse_prompt("fw.document_query.optmimize_query.md")++ optimized_query = await self.agent.call_utility_model(+ system=system_content,+ message=human_content+ )++ chunks = await self.store.search_document(+ document_uri=document_uri,+ query=str(optimized_query),+ limit=10000,+ threshold=0.66+ )+ content += "\n\n----\n\n".join([chunk.page_content for chunk in chunks]) + "\n\n----\n\n"++ if not content:+ content = f"!!! No content found for document: {document_uri} matching queries: {json.dumps(questions)}"+ return False, content++ questions_str = "\n".join([f" * {question}" for question in questions])++ qa_system_message = self.agent.parse_prompt("fw.document_query.system_prompt.md")+ qa_user_message = f"# Document:\n{content}\n\n# Queries:\n{questions_str}"++ ai_response = await self.agent.call_chat_model(+ prompt=ChatPromptTemplate.from_messages([+ SystemMessage(content=qa_system_message),+ HumanMessage(content=qa_user_message),+ ])+ )++ return True, str(ai_response)++ async def document_get_content(self, document_uri: str) -> str:+ url = urlparse(document_uri)+ scheme = url.scheme or "file"+ mimetype, encoding = mimetypes.guess_type(document_uri)+ mimetype = mimetype or "application/octet-stream"++ if mimetype == "application/octet-stream":+ if url.scheme in ["http", "https"]:+ response: aiohttp.ClientResponse | None = None+ retries = 0+ last_error = ""+ while not response and retries < 3:+ try:+ async with aiohttp.ClientSession() as session:+ response = await session.head(document_uri, timeout=aiohttp.ClientTimeout(total=2.0), allow_redirects=True)+ if response.status > 399:+ raise Exception(response.status)+ break+ except Exception as e:+ await asyncio.sleep(1)+ last_error = str(e)+ retries += 1++ if not response:+ raise ValueError(f"DocumentQueryHelper::document_get_content: Document fetch error: {document_uri} ({last_error})")++ mimetype = response.headers["content-type"]+ if "content-length" in response.headers:+ content_length = float(response.headers["content-length"]) / 1024 / 1024 # MB+ if content_length > 50.0:+ raise ValueError(f"Document content length exceeds max. 50MB: {content_length} MB ({document_uri})")+ if mimetype and '; charset=' in mimetype:+ mimetype = mimetype.split('; charset=')[0]++ if scheme == "file":+ try:+ document_uri = os.path.abspath(url.path)+ except Exception as e:+ raise ValueError(f"Invalid document path '{url.path}'") from e++ if encoding:+ raise ValueError(f"Compressed documents are unsupported '{encoding}' ({document_uri})")++ if mimetype == "application/octet-stream":+ raise ValueError(f"Unsupported document mimetype '{mimetype}' ({document_uri})")++ # Use the store's normalization method+ document_uri_norm = self.store._normalize_uri(document_uri)++ await self.store.expire_documents(7)+ exists = await self.store.document_exists(document_uri_norm)+ document_content = ""+ if not exists:+ if mimetype.startswith("image/"):+ document_content = self.handle_image_document(document_uri, scheme)+ elif mimetype == "text/html":+ document_content = self.handle_html_document(document_uri, scheme)+ elif mimetype.startswith("text/") or mimetype == "application/json":+ document_content = self.handle_text_document(document_uri, scheme)+ elif mimetype == "application/pdf":+ document_content = self.handle_pdf_document(document_uri, scheme)+ else:+ document_content = self.handle_unstructured_document(document_uri, scheme)+ await self.store.add_document(document_content, document_uri_norm)+ else:+ doc = await self.store.get_document(document_uri_norm)+ if doc:+ document_content = doc.page_content+ else:+ raise ValueError(f"DocumentQueryHelper::document_get_content: Document not found: {document_uri_norm}")+ return document_content++ def handle_image_document(self, document: str, scheme: str) -> str:+ return self.handle_unstructured_document(document, scheme)++ def handle_html_document(self, document: str, scheme: str) -> str:+ if scheme in ["http", "https"]:+ loader = AsyncHtmlLoader(web_path=document)+ parts: list[Document] = loader.load()+ elif scheme == "file":+ # Use RFC file operations instead of TextLoader- file_content_bytes = rfc_files.read_file_binary(document)++ file_content_bytes = rfc_files.read_file_bin(document)+ file_content = file_content_bytes.decode('utf-8')+ # Create Document manually since we're not using TextLoader+ parts = [Document(page_content=file_content, metadata={"source": document})]+ else:+ raise ValueError(f"Unsupported scheme: {scheme}")++ return "\n".join([element.page_content for element in MarkdownifyTransformer().transform_documents(parts)])++ def handle_text_document(self, document: str, scheme: str) -> str:+ if scheme in ["http", "https"]:+ loader = AsyncHtmlLoader(web_path=document)+ elements: list[Document] = loader.load()+ elif scheme == "file":+ # Use RFC file operations instead of TextLoader- file_content_bytes = rfc_files.read_file_binary(document)++ file_content_bytes = rfc_files.read_file_bin(document)+ file_content = file_content_bytes.decode('utf-8')+ # Create Document manually since we're not using TextLoader+ elements = [Document(page_content=file_content, metadata={"source": document})]+ else:+ raise ValueError(f"Unsupported scheme: {scheme}")++ return "\n".join([element.page_content for element in elements])++ def handle_pdf_document(self, document: str, scheme: str) -> str:+ temp_file_path = ""+ if scheme == "file":+ # Use RFC file operations to read the PDF file as binary- file_content_bytes = rfc_files.read_file_binary(document)++ file_content_bytes = rfc_files.read_file_bin(document)+ # Create a temporary file for PyMuPDFLoader since it needs a file path+ import tempfile+ with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:+ temp_file.write(file_content_bytes)+ temp_file_path = temp_file.name+ elif scheme in ["http", "https"]:+ # download the file from the web url to a temporary file using python libraries for downloading+ import requests+ import tempfile+ with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:+ response = requests.get(document, timeout=10.0)+ if response.status_code != 200:+ raise ValueError(f"DocumentQueryHelper::handle_pdf_document: Failed to download PDF from {document}: {response.status_code}")+ temp_file.write(response.content)+ temp_file_path = temp_file.name+ else:+ raise ValueError(f"Unsupported scheme: {scheme}")++ if not os.path.exists(temp_file_path):+ raise ValueError(f"DocumentQueryHelper::handle_pdf_document: Temporary file not found: {temp_file_path}")++ try:+ try:+ loader = PyMuPDFLoader(+ temp_file_path,+ mode="single",+ extract_tables="markdown",+ extract_images=True,+ images_inner_format="text",+ images_parser=TesseractBlobParser(),+ pages_delimiter="\n",+ )+ elements: list[Document] = loader.load()+ contents = "\n".join([element.page_content for element in elements])+ except Exception as e:+ PrintStyle.error(f"DocumentQueryHelper::handle_pdf_document: Error loading with PyMuPDF: {e}")+ contents = ""++ if not contents:+ import pdf2image+ import pytesseract++ PrintStyle.debug(f"DocumentQueryHelper::handle_pdf_document: FALLBACK Converting PDF to images: {temp_file_path}")++ # Convert PDF to images- pages = pdf2image.convert_from_path(temp_file_path)++ pages = pdf2image.convert_from_path(temp_file_path) # type: ignore+ for page in pages:+ contents += pytesseract.image_to_string(page) + "\n\n"++ return contents+ finally:+ os.unlink(temp_file_path)++ def handle_unstructured_document(self, document: str, scheme: str) -> str:+ elements: list[Document] = []+ if scheme in ["http", "https"]:+ # loader = UnstructuredURLLoader(urls=[document], mode="single")+ loader = UnstructuredLoader(+ web_url=document,+ mode="single",+ partition_via_api=False,+ # chunking_strategy="by_page",+ strategy="hi_res",+ )+ elements = loader.load()+ elif scheme == "file":+ # Use RFC file operations to read the file as binary- file_content_bytes = rfc_files.read_file_binary(document)++ file_content_bytes = rfc_files.read_file_bin(document)+ # Create a temporary file for UnstructuredLoader since it needs a file path+ import tempfile+ import os+ # Get file extension to preserve it for proper processing+ _, ext = os.path.splitext(document)+ with tempfile.NamedTemporaryFile(delete=False, suffix=ext) as temp_file:+ temp_file.write(file_content_bytes)+ temp_file_path = temp_file.name++ try:+ loader = UnstructuredLoader(+ file_path=temp_file_path,+ mode="single",+ partition_via_api=False,+ # chunking_strategy="by_page",+ strategy="hi_res",+ )+ elements = loader.load()+ finally:+ # Clean up temporary file+ os.unlink(temp_file_path)+ else:+ raise ValueError(f"Unsupported scheme: {scheme}")++ return "\n".join([element.page_content for element in elements])diff --cc python/helpers/rfc_files.pyindex 000000000,6fd65779c..2357828femode 000000,100644..100644--- a/python/helpers/rfc_files.py+++ b/python/helpers/rfc_files.py@@@ -1,0 -1,622 +1,622 @@@+ import os+ import shutil+ import fnmatch+ import base64+ import tempfile+ import zipfile+ from python.helpers import runtime+++ def get_abs_path(*relative_paths):+ """Convert relative paths to absolute paths based on the base directory."""+ if not relative_paths:+ return os.path.abspath(os.path.dirname(__file__) + "/../..")++ base_dir = os.path.abspath(os.path.dirname(__file__) + "/../..")+ return os.path.join(base_dir, *relative_paths)+++ # =====================================================+ # RFC-ENABLED FILESYSTEM OPERATIONS+ # =====================================================+-def read_file_binary(relative_path: str, backup_dirs=None) -> bytes:++def read_file_bin(relative_path: str, backup_dirs=None) -> bytes:+ """+ Read binary file content.++ Args:+ relative_path: Path to the file relative to base directory+ backup_dirs: List of backup directories to search in++ Returns:+ File content as bytes+ """+ if backup_dirs is None:+ backup_dirs = []++ # Find the file in directories+ absolute_path = find_file_in_dirs(relative_path, backup_dirs)++ # Use RFC routing for development mode+ b64_content = runtime.call_development_function_sync(+ _read_file_binary_impl, absolute_path+ )+ return base64.b64decode(b64_content)+++ def read_file_base64(relative_path: str, backup_dirs=None) -> str:+ """+ Read file content and return as base64 string.++ Args:+ relative_path: Path to the file relative to base directory+ backup_dirs: List of backup directories to search in++ Returns:+ File content as base64 encoded string+ """+ if backup_dirs is None:+ backup_dirs = []++ # Find the file in directories+ absolute_path = find_file_in_dirs(relative_path, backup_dirs)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _read_file_as_base64_impl, absolute_path+ )+++ def write_file_binary(relative_path: str, content: bytes) -> bool:+ """+ Write binary content to a file.++ Args:+ relative_path: Path to the file relative to base directory+ content: Binary content to write++ Returns:+ True if successful+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ b64_content = base64.b64encode(content).decode('utf-8')+ return runtime.call_development_function_sync(+ _write_file_binary_impl, abs_path, b64_content+ )+++ def write_file_base64(relative_path: str, content: str) -> bool:+ """+ Write base64 content to a file.++ Args:+ relative_path: Path to the file relative to base directory+ content: Base64 encoded content to write++ Returns:+ True if successful+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _write_file_from_base64_impl, abs_path, content+ )+++ def delete_file(relative_path: str) -> bool:+ """+ Delete a file.++ Args:+ relative_path: Path to the file relative to base directory++ Returns:+ True if successful+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _delete_file_impl, abs_path+ )+++ def delete_directory(relative_path: str) -> bool:+ """+ Delete a directory recursively.++ Args:+ relative_path: Path to the directory relative to base directory++ Returns:+ True if successful+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _delete_folder_impl, abs_path+ )+++ def list_directory(relative_path: str, include_hidden: bool = False) -> list:+ """+ List directory contents.++ Args:+ relative_path: Path to the directory relative to base directory+ include_hidden: Whether to include hidden files/folders++ Returns:+ List of directory items with metadata+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _list_folder_impl, abs_path, include_hidden+ )+++ def make_directories(relative_path: str) -> bool:+ """+ Create directories recursively.++ Args:+ relative_path: Path to create relative to base directory++ Returns:+ True if successful+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _make_dirs_impl, abs_path+ )+++ def path_exists(relative_path: str) -> bool:+ """+ Check if a path exists.++ Args:+ relative_path: Path to check relative to base directory++ Returns:+ True if path exists+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _path_exists_impl, abs_path+ )+++ def file_exists(relative_path: str) -> bool:+ """+ Check if a file exists.++ Args:+ relative_path: Path to check relative to base directory++ Returns:+ True if file exists+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _file_exists_impl, abs_path+ )+++ def folder_exists(relative_path: str) -> bool:+ """+ Check if a folder exists.++ Args:+ relative_path: Path to check relative to base directory++ Returns:+ True if folder exists+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _folder_exists_impl, abs_path+ )+++ def get_subdirectories(relative_path: str, include: str | list[str] = "*", exclude: str | list[str] | None = None) -> list[str]:+ """+ Get subdirectories in a directory.++ Args:+ relative_path: Path to the directory relative to base directory+ include: Pattern(s) to include+ exclude: Pattern(s) to exclude++ Returns:+ List of subdirectory names+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _get_subdirectories_impl, abs_path, include, exclude+ )+++ def zip_directory(relative_path: str) -> str:+ """+ Create a zip archive of a directory.++ Args:+ relative_path: Path to the directory relative to base directory++ Returns:+ Path to the created zip file+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _zip_dir_impl, abs_path+ )+++ def move_file(source_path: str, destination_path: str) -> bool:+ """+ Move a file from source to destination.++ Args:+ source_path: Source path relative to base directory+ destination_path: Destination path relative to base directory++ Returns:+ True if successful+ """+ source_abs = get_abs_path(source_path)+ dest_abs = get_abs_path(destination_path)++ # Use RFC routing for development mode+ return runtime.call_development_function_sync(+ _move_file_impl, source_abs, dest_abs+ )+++ def read_directory_as_zip(relative_path: str) -> bytes:+ """+ Read entire directory contents as a zip file.++ Args:+ relative_path: Path to the directory relative to base directory++ Returns:+ Zip file content as bytes+ """+ abs_path = get_abs_path(relative_path)++ # Use RFC routing for development mode+ b64_zip = runtime.call_development_function_sync(+ _read_directory_impl, abs_path+ )+ return base64.b64decode(b64_zip)+++ def find_file_in_dirs(file_path: str, backup_dirs: list[str]) -> str:+ """+ Find a file in the main directory or backup directories.++ Args:+ file_path: Relative file path to search for+ backup_dirs: List of backup directories to search in++ Returns:+ Absolute path to the found file++ Raises:+ FileNotFoundError: If file is not found in any directory+ """+ # Try the main path first+ main_path = get_abs_path(file_path)+ if runtime.call_development_function_sync(_file_exists_impl, main_path):+ return main_path++ # Try backup directories+ for backup_dir in backup_dirs:+ backup_path = os.path.join(backup_dir, file_path)+ if runtime.call_development_function_sync(_file_exists_impl, backup_path):+ return backup_path++ # File not found anywhere+ raise FileNotFoundError(f"File not found: {file_path}")+++ # =====================================================+ # IMPLEMENTATION FUNCTIONS (Container Operations)+ # =====================================================++ def _read_file_binary_impl(file_path: str) -> str:+ """+ Implementation function to read a file in binary mode.+ Returns base64 encoded content for RFC transport.+ """+ if not os.path.exists(file_path):+ raise FileNotFoundError(f"File not found: {file_path}")++ if not os.path.isfile(file_path):+ raise Exception(f"Path is not a file: {file_path}")++ try:+ with open(file_path, 'rb') as file:+ content = file.read()+ return base64.b64encode(content).decode('utf-8')+ except Exception as e:+ raise Exception(f"Failed to read file {file_path}: {str(e)}")+++ def _write_file_binary_impl(file_path: str, b64_content: str) -> bool:+ """+ Implementation function to write binary content to a file.+ Expects base64 encoded content from RFC transport.+ """+ try:+ # Ensure b64_content is properly UTF-8 encoded before base64 decoding+ if isinstance(b64_content, str):+ b64_content_bytes = b64_content.encode('utf-8')+ else:+ b64_content_bytes = b64_content++ # Decode base64 content+ content = base64.b64decode(b64_content_bytes)++ # Create directory if it doesn't exist+ os.makedirs(os.path.dirname(file_path), exist_ok=True)++ # Write file+ with open(file_path, 'wb') as file:+ file.write(content)++ return True+ except Exception as e:+ raise Exception(f"Failed to write file {file_path}: {str(e)}")+++ def _delete_file_impl(file_path: str) -> bool:+ """+ Implementation function to delete a file.+ """+ if not os.path.exists(file_path):+ raise FileNotFoundError(f"File not found: {file_path}")++ if not os.path.isfile(file_path):+ raise Exception(f"Path is not a file: {file_path}")++ try:+ os.remove(file_path)+ return True+ except Exception as e:+ raise Exception(f"Failed to delete file {file_path}: {str(e)}")+++ def _delete_folder_impl(folder_path: str) -> bool:+ """+ Implementation function to delete a folder recursively.+ """+ if not os.path.exists(folder_path):+ raise FileNotFoundError(f"Folder not found: {folder_path}")++ if not os.path.isdir(folder_path):+ raise Exception(f"Path is not a directory: {folder_path}")++ try:+ shutil.rmtree(folder_path)+ return True+ except Exception as e:+ raise Exception(f"Failed to delete folder {folder_path}: {str(e)}")+++ def _list_folder_impl(folder_path: str, include_hidden: bool = False) -> list:+ """+ Implementation function to list folder contents.+ """+ if not os.path.exists(folder_path):+ raise FileNotFoundError(f"Folder not found: {folder_path}")++ if not os.path.isdir(folder_path):+ raise Exception(f"Path is not a directory: {folder_path}")++ try:+ items = []+ for item_name in os.listdir(folder_path):+ # Skip hidden files if not requested+ if not include_hidden and item_name.startswith('.'):+ continue++ item_path = os.path.join(folder_path, item_name)+ stat_info = os.stat(item_path)++ item_info = {+ "name": item_name,+ "path": item_path,+ "is_file": os.path.isfile(item_path),+ "is_dir": os.path.isdir(item_path),+ "size": stat_info.st_size,+ "modified": stat_info.st_mtime+ }+ items.append(item_info)++ # Sort by name for consistent output+ items.sort(key=lambda x: str(x["name"]).lower())+ return items++ except Exception as e:+ raise Exception(f"Failed to list folder {folder_path}: {str(e)}")+++ def _make_dirs_impl(folder_path: str) -> bool:+ """+ Implementation function to create directories.+ """+ try:+ os.makedirs(folder_path, exist_ok=True)+ return True+ except Exception as e:+ raise Exception(f"Failed to create directories {folder_path}: {str(e)}")+++ def _path_exists_impl(file_path: str) -> bool:+ """Implementation function to check if path exists."""+ return os.path.exists(file_path)+++ def _file_exists_impl(file_path: str) -> bool:+ """Implementation function to check if file exists."""+ return os.path.exists(file_path) and os.path.isfile(file_path)+++ def _folder_exists_impl(folder_path: str) -> bool:+ """Implementation function to check if folder exists."""+ return os.path.exists(folder_path) and os.path.isdir(folder_path)+++ def _get_subdirectories_impl(folder_path: str, include: str | list[str], exclude: str | list[str] | None) -> list[str]:+ """+ Implementation function to get subdirectories.+ """+ if not os.path.exists(folder_path):+ return []++ if isinstance(include, str):+ include = [include]+ if isinstance(exclude, str):+ exclude = [exclude]++ return [+ subdir+ for subdir in os.listdir(folder_path)+ if os.path.isdir(os.path.join(folder_path, subdir))+ and any(fnmatch.fnmatch(subdir, inc) for inc in include)+ and (exclude is None or not any(fnmatch.fnmatch(subdir, exc) for exc in exclude))+ ]+++ def _zip_dir_impl(folder_path: str) -> str:+ """+ Implementation function to create a zip archive of a directory.+ """+ zip_file_path = tempfile.NamedTemporaryFile(suffix=".zip", delete=False).name+ base_name = os.path.basename(folder_path)++ with zipfile.ZipFile(zip_file_path, "w", compression=zipfile.ZIP_DEFLATED) as zip_file:+ for root, _, files in os.walk(folder_path):+ for file in files:+ file_path = os.path.join(root, file)+ rel_path = os.path.relpath(file_path, folder_path)+ zip_file.write(file_path, os.path.join(base_name, rel_path))++ return zip_file_path+++ def _move_file_impl(source_path: str, destination_path: str) -> bool:+ """+ Implementation function to move a file.+ """+ try:+ os.makedirs(os.path.dirname(destination_path), exist_ok=True)+ os.rename(source_path, destination_path)+ return True+ except Exception as e:+ raise Exception(f"Failed to move file {source_path} to {destination_path}: {str(e)}")+++ def _read_directory_impl(dir_path: str) -> str:+ """+ Implementation function to zip a directory and return base64 encoded zip.+ """+ if not os.path.exists(dir_path):+ raise FileNotFoundError(f"Directory not found: {dir_path}")++ if not os.path.isdir(dir_path):+ raise Exception(f"Path is not a directory: {dir_path}")++ temp_zip_path = None+ try:+ # Create temporary zip file+ with tempfile.NamedTemporaryFile(suffix='.zip', delete=False) as temp_zip:+ temp_zip_path = temp_zip.name++ # Create zip archive+ with zipfile.ZipFile(temp_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:+ for root, dirs, files in os.walk(dir_path):+ for file in files:+ file_path = os.path.join(root, file)+ arcname = os.path.relpath(file_path, dir_path)+ zipf.write(file_path, arcname)++ # Read zip file and encode as base64+ with open(temp_zip_path, 'rb') as zipf:+ zip_content = zipf.read()+ b64_zip = base64.b64encode(zip_content).decode('utf-8')++ # Clean up temporary file+ os.unlink(temp_zip_path)++ return b64_zip++ except Exception as e:+ # Clean up temporary file if it exists+ if temp_zip_path is not None and os.path.exists(temp_zip_path):+ os.unlink(temp_zip_path)+ raise Exception(f"Failed to zip directory {dir_path}: {str(e)}")+++ def _read_file_as_base64_impl(file_path: str) -> str:+ """+ Implementation function to read a file and return its content as base64.+ """+ if not os.path.exists(file_path):+ raise FileNotFoundError(f"File not found: {file_path}")++ if not os.path.isfile(file_path):+ raise Exception(f"Path is not a file: {file_path}")++ try:+ with open(file_path, 'rb') as file:+ content = file.read()+ return base64.b64encode(content).decode('utf-8')+ except Exception as e:+ raise Exception(f"Failed to read file {file_path}: {str(e)}")+++ def _write_file_from_base64_impl(file_path: str, content: str) -> bool:+ """+ Implementation function to write base64 content to a file.+ """+ try:+ # Ensure content is properly UTF-8 encoded before base64 decoding+ if isinstance(content, str):+ content_bytes = content.encode('utf-8')+ else:+ content_bytes = content++ # Decode base64 content+ decoded_content = base64.b64decode(content_bytes)++ # Create directory if it doesn't exist+ os.makedirs(os.path.dirname(file_path), exist_ok=True)++ # Write file+ with open(file_path, 'wb') as file:+ file.write(decoded_content)++ return True+ except Exception as e:+ raise Exception(f"Failed to write file {file_path}: {str(e)}")diff --cc requirements.txtindex 760ca4373,94a5bc7ee..f7d98965b--- a/requirements.txt+++ b/requirements.txt@@@ -16,8 -17,9 +16,9 @@@ langchain-google-genai==2.1.langchain-groq==0.2.2langchain-huggingface==0.1.2langchain-mistralai==0.2.4-langchain-ollama==0.2.2-langchain-openai==0.3.1+langchain-ollama==0.3.0+langchain-openai==0.3.11+ langchain-unstructured[all-docs]==0.1.6openai-whisper==20240930lxml_html_clean==0.3.1markdown==3.7@@@ -30,8 -32,12 +31,12 @@@ python-dotenv==1.1.pytz==2024.2sentence-transformers==3.0.1tiktoken==0.8.0- unstructured==0.15.13- unstructured-client==0.25.9+ unstructured[all-docs]==0.16.23+ unstructured-client==0.31.0webcolors==24.6.0nest-asyncio==1.6.0-markdownify==0.14.1++markdownify==1.1.0+ pymupdf==1.25.3+ pytesseract==0.3.13+ pdf2image==1.17.0-crontab==1.0.1+crontab==1.0.1