A programming framework for agentic AI
Add additional docs in retrieval agent if required (#1028)
Shobhit Vishnoiebd5de9501ae
diff --git a/autogen/agentchat/contrib/qdrant_retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/qdrant_retrieve_user_proxy_agent.pyindex 88a2d318a..0e2e0d6c9 100644--- a/autogen/agentchat/contrib/qdrant_retrieve_user_proxy_agent.py+++ b/autogen/agentchat/contrib/qdrant_retrieve_user_proxy_agent.py@@ -47,6 +47,9 @@ class QdrantRetrieveUserProxyAgent(RetrieveUserProxyAgent):will be used. If you want to use other vector db, extend this class and override the `retrieve_docs` function.- docs_path (Optional, Union[str, List[str]]): the path to the docs directory. It can also be the path to a single file,the url to a single file or a list of directories, files and urls. Default is None, which works only if the collection is already created.+ - extra_docs (Optional, bool): when true, allows adding documents with unique IDs without overwriting existing ones; when false, it replaces existing documents using default IDs, risking collection overwrite.,+ when set to true it enables the system to assign unique IDs starting from "length+i" for new document chunks, preventing the replacement of existing documents and facilitating the addition of more content to the collection..+ By default, "extra_docs" is set to false, starting document IDs from zero. This poses a risk as new documents might overwrite existing ones, potentially causing unintended loss or alteration of data in the collection.- collection_name (Optional, str): the name of the collection.If key not provided, a default name `autogen-docs` will be used.- model (Optional, str): the model to use for the retrieve chat.@@ -116,6 +119,7 @@ class QdrantRetrieveUserProxyAgent(RetrieveUserProxyAgent):custom_text_split_function=self.custom_text_split_function,custom_text_types=self._custom_text_types,recursive=self._recursive,+ extra_docs=self._extra_docs,parallel=self._parallel,on_disk=self._on_disk,quantization_config=self._quantization_config,@@ -146,6 +150,7 @@ def create_qdrant_from_dir(custom_text_split_function: Callable = None,custom_text_types: List[str] = TEXT_FORMATS,recursive: bool = True,+ extra_docs: bool = False,parallel: int = 0,on_disk: bool = False,quantization_config: Optional[models.QuantizationConfig] = None,@@ -169,6 +174,7 @@ def create_qdrant_from_dir(Default is None, will use the default function in `autogen.retrieve_utils.split_text_to_chunks`.custom_text_types (Optional, List[str]): a list of file types to be processed. Default is TEXT_FORMATS.recursive (Optional, bool): whether to search documents recursively in the dir_path. Default is True.+ extra_docs (Optional, bool): whether to add more documents in the collection. Default is Falseparallel (Optional, int): How many parallel workers to use for embedding. Defaults to the number of CPU coreson_disk (Optional, bool): Whether to store the collection on disk. Default is False.quantization_config: Quantization configuration. If None, quantization will be disabled.@@ -194,9 +200,10 @@ def create_qdrant_from_dir()logger.info(f"Found {len(chunks)} chunks.")+ collection = None# Check if collection by same name exists, if not, create it with custom optionstry:- client.get_collection(collection_name=collection_name)+ collection = client.get_collection(collection_name=collection_name)except Exception:client.create_collection(collection_name=collection_name,@@ -204,12 +211,21 @@ def create_qdrant_from_dir(on_disk=on_disk, quantization_config=quantization_config, hnsw_config=hnsw_config),)- client.get_collection(collection_name=collection_name)+ collection = client.get_collection(collection_name=collection_name)++ length = 0+ if extra_docs:+ length = len(collection.get()["ids"])# Upsert in batch of 100 or less if the total number of chunks is less than 100for i in range(0, len(chunks), min(100, len(chunks))):end_idx = i + min(100, len(chunks) - i)- client.add(collection_name, documents=chunks[i:end_idx], ids=[j for j in range(i, end_idx)], parallel=parallel)+ client.add(+ collection_name,+ documents=chunks[i:end_idx],+ ids=[(j + length) for j in range(i, end_idx)],+ parallel=parallel,+ )# Create a payload index for the document field# Enables highly efficient payload filtering. Reference: https://qdrant.tech/documentation/concepts/indexing/#indexingdiff --git a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/retrieve_user_proxy_agent.pyindex 28879d315..e07c2e578 100644--- a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py+++ b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py@@ -100,6 +100,9 @@ class RetrieveUserProxyAgent(UserProxyAgent):will be used. If you want to use other vector db, extend this class and override the `retrieve_docs` function.- docs_path (Optional, Union[str, List[str]]): the path to the docs directory. It can also be the path to a single file,the url to a single file or a list of directories, files and urls. Default is None, which works only if the collection is already created.+ - extra_docs (Optional, bool): when true, allows adding documents with unique IDs without overwriting existing ones; when false, it replaces existing documents using default IDs, risking collection overwrite.,+ when set to true it enables the system to assign unique IDs starting from "length+i" for new document chunks, preventing the replacement of existing documents and facilitating the addition of more content to the collection..+ By default, "extra_docs" is set to false, starting document IDs from zero. This poses a risk as new documents might overwrite existing ones, potentially causing unintended loss or alteration of data in the collection.- collection_name (Optional, str): the name of the collection.If key not provided, a default name `autogen-docs` will be used.- model (Optional, str): the model to use for the retrieve chat.@@ -171,6 +174,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):self._task = self._retrieve_config.get("task", "default")self._client = self._retrieve_config.get("client", chromadb.Client())self._docs_path = self._retrieve_config.get("docs_path", None)+ self._extra_docs = self._retrieve_config.get("extra_docs", False)self._collection_name = self._retrieve_config.get("collection_name", "autogen-docs")if "docs_path" not in self._retrieve_config:logger.warning(@@ -392,6 +396,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):custom_text_split_function=self.custom_text_split_function,custom_text_types=self._custom_text_types,recursive=self._recursive,+ extra_docs=self._extra_docs,)self._collection = Trueself._get_or_create = Truediff --git a/autogen/retrieve_utils.py b/autogen/retrieve_utils.pyindex d65fabb44..31467b07c 100644--- a/autogen/retrieve_utils.py+++ b/autogen/retrieve_utils.py@@ -250,6 +250,7 @@ def create_vector_db_from_dir(custom_text_split_function: Callable = None,custom_text_types: List[str] = TEXT_FORMATS,recursive: bool = True,+ extra_docs: bool = False,) -> API:"""Create a vector db from all the files in a given directory, the directory can also be a single file or a url toa single file. We support chromadb compatible APIs to create the vector db, this function is not required if@@ -274,7 +275,7 @@ def create_vector_db_from_dir(Default is None, will use the default function in `autogen.retrieve_utils.split_text_to_chunks`.custom_text_types (Optional, List[str]): a list of file types to be processed. Default is TEXT_FORMATS.recursive (Optional, bool): whether to search documents recursively in the dir_path. Default is True.-+ extra_docs (Optional, bool): whether to add more documents in the collection. Default is FalseReturns:API: the chromadb client."""@@ -296,6 +297,10 @@ def create_vector_db_from_dir(metadata={"hnsw:space": "ip", "hnsw:construction_ef": 30, "hnsw:M": 32}, # ip, l2, cosine)+ length = 0+ if extra_docs:+ length = len(collection.get()["ids"])+if custom_text_split_function is not None:chunks = split_files_to_chunks(get_files_from_dir(dir_path, custom_text_types, recursive),@@ -314,7 +319,7 @@ def create_vector_db_from_dir(end_idx = i + min(40000, len(chunks) - i)collection.upsert(documents=chunks[i:end_idx],- ids=[f"doc_{j}" for j in range(i, end_idx)], # unique for each doc+ ids=[f"doc_{j+length}" for j in range(i, end_idx)], # unique for each doc)except ValueError as e:logger.warning(f"{e}")