A programming framework for agentic AI
Add support to customized vectordb and embedding functions (#161)
Li Jiangfa6e2a52c00a
diff --git a/.github/workflows/build.yml b/.github/workflows/build.ymlindex 63ca0a254..5e5fd186b 100644--- a/.github/workflows/build.yml+++ b/.github/workflows/build.yml@@ -40,7 +40,7 @@ jobs:python -m pip install --upgrade pip wheelpip install -e .python -c "import autogen"- pip install -e.[mathchat,retrievechat] datasets pytest+ pip install -e.[mathchat,retrievechat,test] datasets pytestpip uninstall -y openai- name: Test with pytestif: matrix.python-version != '3.10'diff --git a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/retrieve_user_proxy_agent.pyindex 0fc83bdb7..0f29aa62d 100644--- a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py+++ b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py@@ -67,6 +67,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):self,name="RetrieveChatAgent", # default set to RetrieveChatAgenthuman_input_mode: Optional[str] = "ALWAYS",+ is_termination_msg: Optional[Callable[[Dict], bool]] = None,retrieve_config: Optional[Dict] = None, # config for the retrieve agent**kwargs,):@@ -82,14 +83,17 @@ class RetrieveUserProxyAgent(UserProxyAgent):the number of auto reply reaches the max_consecutive_auto_reply.(3) When "NEVER", the agent will never prompt for human input. Under this mode, the conversation stopswhen the number of auto reply reaches the max_consecutive_auto_reply or when is_termination_msg is True.+ is_termination_msg (function): a function that takes a message in the form of a dictionary+ and returns a boolean value indicating if this received message is a termination message.+ The dict can contain the following keys: "content", "role", "name", "function_call".retrieve_config (dict or None): config for the retrieve agent.To use default config, set to None. Otherwise, set to a dictionary with the following keys:- task (Optional, str): the task of the retrieve chat. Possible values are "code", "qa" and "default". Systemprompt will be different for different tasks. The default value is `default`, which supports both code and qa.- - client (Optional, chromadb.Client): the chromadb client.- If key not provided, a default client `chromadb.Client()` will be used.+ - client (Optional, chromadb.Client): the chromadb client. If key not provided, a default client `chromadb.Client()`+ will be used. If you want to use other vector db, extend this class and override the `retrieve_docs` function.- docs_path (Optional, str): the path to the docs directory. It can also be the path to a single file,- or the url to a single file. If key not provided, a default path `./docs` will be used.+ or the url to a single file. Default is None, which works only if the collection is already created.- collection_name (Optional, str): the name of the collection.If key not provided, a default name `autogen-docs` will be used.- model (Optional, str): the model to use for the retrieve chat.@@ -106,16 +110,45 @@ class RetrieveUserProxyAgent(UserProxyAgent):If key not provided, a default model `all-MiniLM-L6-v2` will be used. All available modelscan be found at `https://www.sbert.net/docs/pretrained_models.html`. The default model is afast model. If you want to use a high performance model, `all-mpnet-base-v2` is recommended.+ - embedding_function (Optional, Callable): the embedding function for creating the vector db. Default is None,+ SentenceTransformer with the given `embedding_model` will be used. If you want to use OpenAI, Cohere, HuggingFace or+ other embedding functions, you can pass it here, follow the examples in `https://docs.trychroma.com/embeddings`.- customized_prompt (Optional, str): the customized prompt for the retrieve chat. Default is None.- customized_answer_prefix (Optional, str): the customized answer prefix for the retrieve chat. Default is "".If not "" and the customized_answer_prefix is not in the answer, `Update Context` will be triggered.- update_context (Optional, bool): if False, will not apply `Update Context` for interactive retrieval. Default is True.- get_or_create (Optional, bool): if True, will create/recreate a collection for the retrieve chat.- This is the same as that used in chromadb. Default is False.+ This is the same as that used in chromadb. Default is False. Will be set to False if docs_path is None.- custom_token_count_function(Optional, Callable): a custom function to count the number of tokens in a string.The function should take a string as input and return three integers (token_count, tokens_per_message, tokens_per_name).Default is None, tiktoken will be used and may not be accurate for non-OpenAI models.**kwargs (dict): other kwargs in [UserProxyAgent](../user_proxy_agent#__init__).++ Example of overriding retrieve_docs:+ If you have set up a customized vector db, and it's not compatible with chromadb, you can easily plug in it with below code.+ ```python+ class MyRetrieveUserProxyAgent(RetrieveUserProxyAgent):+ def query_vector_db(+ self,+ query_texts: List[str],+ n_results: int = 10,+ search_string: str = "",+ **kwargs,+ ) -> Dict[str, Union[List[str], List[List[str]]]]:+ # define your own query function here+ pass++ def retrieve_docs(self, problem: str, n_results: int = 20, search_string: str = "", **kwargs):+ results = self.query_vector_db(+ query_texts=[problem],+ n_results=n_results,+ search_string=search_string,+ **kwargs,+ )++ self._results = results+ print("doc_ids: ", results["ids"])+ ```"""super().__init__(name=name,@@ -126,7 +159,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):self._retrieve_config = {} if retrieve_config is None else retrieve_configself._task = self._retrieve_config.get("task", "default")self._client = self._retrieve_config.get("client", chromadb.Client())- self._docs_path = self._retrieve_config.get("docs_path", "./docs")+ self._docs_path = self._retrieve_config.get("docs_path", None)self._collection_name = self._retrieve_config.get("collection_name", "autogen-docs")self._model = self._retrieve_config.get("model", "gpt-4")self._max_tokens = self.get_max_tokens(self._model)@@ -134,20 +167,26 @@ class RetrieveUserProxyAgent(UserProxyAgent):self._chunk_mode = self._retrieve_config.get("chunk_mode", "multi_lines")self._must_break_at_empty_line = self._retrieve_config.get("must_break_at_empty_line", True)self._embedding_model = self._retrieve_config.get("embedding_model", "all-MiniLM-L6-v2")+ self._embedding_function = self._retrieve_config.get("embedding_function", None)self.customized_prompt = self._retrieve_config.get("customized_prompt", None)self.customized_answer_prefix = self._retrieve_config.get("customized_answer_prefix", "").upper()self.update_context = self._retrieve_config.get("update_context", True)- self._get_or_create = self._retrieve_config.get("get_or_create", False)+ self._get_or_create = (+ self._retrieve_config.get("get_or_create", False) if self._docs_path is not None else False+ )self.custom_token_count_function = self._retrieve_config.get("custom_token_count_function", None)self._context_max_tokens = self._max_tokens * 0.8- self._collection = False # the collection is not created+ self._collection = True if self._docs_path is None else False # whether the collection is createdself._ipython = get_ipython()self._doc_idx = -1 # the index of the current used docself._results = {} # the results of the current queryself._intermediate_answers = set() # the intermediate answersself._doc_contents = [] # the contents of the current used docself._doc_ids = [] # the ids of the current used doc- self._is_termination_msg = self._is_termination_msg_retrievechat # update the termination message function+ # update the termination message function+ self._is_termination_msg = (+ self._is_termination_msg_retrievechat if is_termination_msg is None else is_termination_msg+ )self.register_reply(Agent, RetrieveUserProxyAgent._generate_retrieve_user_reply, position=1)def _is_termination_msg_retrievechat(self, message):@@ -188,7 +227,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):self._doc_contents = [] # the contents of the current used docself._doc_ids = [] # the ids of the current used doc- def _get_context(self, results):+ def _get_context(self, results: Dict[str, Union[List[str], List[List[str]]]]):doc_contents = ""current_tokens = 0_doc_idx = self._doc_idx@@ -297,6 +336,22 @@ class RetrieveUserProxyAgent(UserProxyAgent):return False, Nonedef retrieve_docs(self, problem: str, n_results: int = 20, search_string: str = ""):+ """Retrieve docs based on the given problem and assign the results to the class property `_results`.+ In case you want to customize the retrieval process, such as using a different vector db whose APIs are not+ compatible with chromadb or filter results with metadata, you can override this function. Just keep the current+ parameters and add your own parameters with default values, and keep the results in below type.++ Type of the results: Dict[str, List[List[Any]]], should have keys "ids" and "documents", "ids" for the ids of+ the retrieved docs and "documents" for the contents of the retrieved docs. Any other keys are optional. Refer+ to `chromadb.api.types.QueryResult` as an example.+ ids: List[string]+ documents: List[List[string]]++ Args:+ problem (str): the problem to be solved.+ n_results (int): the number of results to be retrieved.+ search_string (str): only docs containing this string will be retrieved.+ """if not self._collection or self._get_or_create:print("Trying to create collection.")create_vector_db_from_dir(@@ -308,6 +363,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):must_break_at_empty_line=self._must_break_at_empty_line,embedding_model=self._embedding_model,get_or_create=self._get_or_create,+ embedding_function=self._embedding_function,)self._collection = Trueself._get_or_create = False@@ -319,6 +375,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):client=self._client,collection_name=self._collection_name,embedding_model=self._embedding_model,+ embedding_function=self._embedding_function,)self._results = resultsprint("doc_ids: ", results["ids"])diff --git a/autogen/retrieve_utils.py b/autogen/retrieve_utils.pyindex fbe7c2878..721b1ec29 100644--- a/autogen/retrieve_utils.py+++ b/autogen/retrieve_utils.py@@ -6,6 +6,7 @@ import globimport tiktokenimport chromadbfrom chromadb.api import API+from chromadb.api.types import QueryResultimport chromadb.utils.embedding_functions as efimport loggingimport pypdf@@ -263,12 +264,36 @@ def create_vector_db_from_dir(chunk_mode: str = "multi_lines",must_break_at_empty_line: bool = True,embedding_model: str = "all-MiniLM-L6-v2",+ embedding_function: Callable = None,):- """Create a vector db from all the files in a given directory."""+ """Create a vector db from all the files in a given directory, the directory can also be a single file or a url to+ a single file. We support chromadb compatible APIs to create the vector db, this function is not required if+ you prepared your own vector db.++ Args:+ dir_path (str): the path to the directory, file or url.+ max_tokens (Optional, int): the maximum number of tokens per chunk. Default is 4000.+ client (Optional, API): the chromadb client. Default is None.+ db_path (Optional, str): the path to the chromadb. Default is "/tmp/chromadb.db".+ collection_name (Optional, str): the name of the collection. Default is "all-my-documents".+ get_or_create (Optional, bool): Whether to get or create the collection. Default is False. If True, the collection+ will be recreated if it already exists.+ chunk_mode (Optional, str): the chunk mode. Default is "multi_lines".+ must_break_at_empty_line (Optional, bool): Whether to break at empty line. Default is True.+ embedding_model (Optional, str): the embedding model to use. Default is "all-MiniLM-L6-v2". Will be ignored if+ embedding_function is not None.+ embedding_function (Optional, Callable): the embedding function to use. Default is None, SentenceTransformer with+ the given `embedding_model` will be used. If you want to use OpenAI, Cohere, HuggingFace or other embedding+ functions, you can pass it here, follow the examples in `https://docs.trychroma.com/embeddings`.+ """if client is None:client = chromadb.PersistentClient(path=db_path)try:- embedding_function = ef.SentenceTransformerEmbeddingFunction(embedding_model)+ embedding_function = (+ ef.SentenceTransformerEmbeddingFunction(embedding_model)+ if embedding_function is None+ else embedding_function+ )collection = client.create_collection(collection_name,get_or_create=get_or_create,@@ -300,14 +325,41 @@ def query_vector_db(collection_name: str = "all-my-documents",search_string: str = "",embedding_model: str = "all-MiniLM-L6-v2",-) -> Dict[str, List[str]]:- """Query a vector db."""+ embedding_function: Callable = None,+) -> QueryResult:+ """Query a vector db. We support chromadb compatible APIs, it's not required if you prepared your own vector db+ and query function.++ Args:+ query_texts (List[str]): the query texts.+ n_results (Optional, int): the number of results to return. Default is 10.+ client (Optional, API): the chromadb compatible client. Default is None, a chromadb client will be used.+ db_path (Optional, str): the path to the vector db. Default is "/tmp/chromadb.db".+ collection_name (Optional, str): the name of the collection. Default is "all-my-documents".+ search_string (Optional, str): the search string. Default is "".+ embedding_model (Optional, str): the embedding model to use. Default is "all-MiniLM-L6-v2". Will be ignored if+ embedding_function is not None.+ embedding_function (Optional, Callable): the embedding function to use. Default is None, SentenceTransformer with+ the given `embedding_model` will be used. If you want to use OpenAI, Cohere, HuggingFace or other embedding+ functions, you can pass it here, follow the examples in `https://docs.trychroma.com/embeddings`.++ Returns:+ QueryResult: the query result. The format is:+ class QueryResult(TypedDict):+ ids: List[IDs]+ embeddings: Optional[List[List[Embedding]]]+ documents: Optional[List[List[Document]]]+ metadatas: Optional[List[List[Metadata]]]+ distances: Optional[List[List[float]]]+ """if client is None:client = chromadb.PersistentClient(path=db_path)# the collection's embedding function is always the default one, but we want to use the one we used to create the# collection. So we compute the embeddings ourselves and pass it to the query function.collection = client.get_collection(collection_name)- embedding_function = ef.SentenceTransformerEmbeddingFunction(embedding_model)+ embedding_function = (+ ef.SentenceTransformerEmbeddingFunction(embedding_model) if embedding_function is None else embedding_function+ )query_embeddings = embedding_function(query_texts)# Query/search n most similar results. You can also .get by idresults = collection.query(diff --git a/setup.py b/setup.pyindex 37c9d2d88..a42432eb0 100644--- a/setup.py+++ b/setup.py@@ -40,6 +40,7 @@ setuptools.setup(extras_require={"test": ["chromadb",+ "lancedb","coverage>=5.3","datasets","ipykernel",diff --git a/test/agentchat/test_retrievechat.py b/test/agentchat/test_retrievechat.pyindex bde5730cb..99e395de5 100644--- a/test/agentchat/test_retrievechat.py+++ b/test/agentchat/test_retrievechat.py@@ -12,6 +12,7 @@ try:)from autogen.retrieve_utils import create_vector_db_from_dir, query_vector_dbimport chromadb+ from chromadb.utils import embedding_functions as efskip_test = Falseexcept ImportError:@@ -49,6 +50,7 @@ def test_retrievechat():},)+ sentence_transformer_ef = ef.SentenceTransformerEmbeddingFunction()ragproxyagent = RetrieveUserProxyAgent(name="ragproxyagent",human_input_mode="NEVER",@@ -58,6 +60,7 @@ def test_retrievechat():"chunk_token_size": 2000,"model": config_list[0]["model"],"client": chromadb.PersistentClient(path="/tmp/chromadb"),+ "embedding_function": sentence_transformer_ef,},)diff --git a/test/test_retrieve_utils.py b/test/test_retrieve_utils.pyindex fdb93d26c..be215facb 100644--- a/test/test_retrieve_utils.py+++ b/test/test_retrieve_utils.py@@ -100,6 +100,70 @@ class TestRetrieveUtils:results = query_vector_db(["autogen"], client=client)assert isinstance(results, dict) and any("autogen" in res[0].lower() for res in results.get("documents", []))+ def test_custom_vector_db(self):+ try:+ import lancedb+ except ImportError:+ return+ from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent++ db_path = "/tmp/lancedb"++ def create_lancedb():+ db = lancedb.connect(db_path)+ data = [+ {"vector": [1.1, 1.2], "id": 1, "documents": "This is a test document spark"},+ {"vector": [0.2, 1.8], "id": 2, "documents": "This is another test document"},+ {"vector": [0.1, 0.3], "id": 3, "documents": "This is a third test document spark"},+ {"vector": [0.5, 0.7], "id": 4, "documents": "This is a fourth test document"},+ {"vector": [2.1, 1.3], "id": 5, "documents": "This is a fifth test document spark"},+ {"vector": [5.1, 8.3], "id": 6, "documents": "This is a sixth test document"},+ ]+ try:+ db.create_table("my_table", data)+ except OSError:+ pass++ class MyRetrieveUserProxyAgent(RetrieveUserProxyAgent):+ def query_vector_db(+ self,+ query_texts,+ n_results=10,+ search_string="",+ ):+ if query_texts:+ vector = [0.1, 0.3]+ db = lancedb.connect(db_path)+ table = db.open_table("my_table")+ query = table.search(vector).where(f"documents LIKE '%{search_string}%'").limit(n_results).to_df()+ return {"ids": query["id"].tolist(), "documents": query["documents"].tolist()}++ def retrieve_docs(self, problem: str, n_results: int = 20, search_string: str = ""):+ results = self.query_vector_db(+ query_texts=[problem],+ n_results=n_results,+ search_string=search_string,+ )++ self._results = results+ print("doc_ids: ", results["ids"])++ ragragproxyagent = MyRetrieveUserProxyAgent(+ name="ragproxyagent",+ human_input_mode="NEVER",+ max_consecutive_auto_reply=2,+ retrieve_config={+ "task": "qa",+ "chunk_token_size": 2000,+ "client": "__",+ "embedding_model": "all-mpnet-base-v2",+ },+ )++ create_lancedb()+ ragragproxyagent.retrieve_docs("This is a test document spark", n_results=10, search_string="spark")+ assert ragragproxyagent._results["ids"] == [3, 1, 5]+if __name__ == "__main__":pytest.main()