A programming framework for agentic AI
Add llamaindex agent integration (#2831)
Diego Colombo6e8331b75409
diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.ymlindex 98a47e1e5..ebce2cd4e 100644--- a/.github/workflows/contrib-tests.yml+++ b/.github/workflows/contrib-tests.yml@@ -444,3 +444,34 @@ jobs:with:file: ./coverage.xmlflags: unittest++ LlamaIndexAgent:+ runs-on: ${{ matrix.os }}+ strategy:+ fail-fast: false+ matrix:+ os: [ubuntu-latest, macos-latest, windows-2019]+ python-version: ["3.11"]+ steps:+ - uses: actions/checkout@v4+ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@v5+ with:+ python-version: ${{ matrix.python-version }}+ - name: Install packages and dependencies for all tests+ run: |+ python -m pip install --upgrade pip wheel+ pip install pytest-cov>=5+ - name: Install packages and dependencies for LlamaIndexConverableAgent+ run: |+ pip install -e .+ pip install llama-index+ pip install llama-index-llms-openai+ - name: Coverage+ run: |+ pytest test/agentchat/contrib/test_llamaindex_conversable_agent.py --skip-openai+ - name: Upload coverage to Codecov+ uses: codecov/codecov-action@v3+ with:+ file: ./coverage.xml+ flags: unittestsdiff --git a/autogen/agentchat/contrib/llamaindex_conversable_agent.py b/autogen/agentchat/contrib/llamaindex_conversable_agent.pynew file mode 100644index 000000000..f7a9c3e61--- /dev/null+++ b/autogen/agentchat/contrib/llamaindex_conversable_agent.py@@ -0,0 +1,109 @@+from typing import Dict, List, Optional, Tuple, Union++from autogen import OpenAIWrapper+from autogen.agentchat import Agent, ConversableAgent+from autogen.agentchat.contrib.vectordb.utils import get_logger++logger = get_logger(__name__)++try:+ from llama_index.core.agent.runner.base import AgentRunner+ from llama_index.core.chat_engine.types import AgentChatResponse+ from llama_index_client import ChatMessage+except ImportError as e:+ logger.fatal("Failed to import llama-index. Try running 'pip install llama-index'")+ raise e+++class LLamaIndexConversableAgent(ConversableAgent):++ def __init__(+ self,+ name: str,+ llama_index_agent: AgentRunner,+ description: Optional[str] = None,+ **kwargs,+ ):+ """+ Args:+ name (str): agent name.+ llama_index_agent (AgentRunner): llama index agent.+ Please override this attribute if you want to reprogram the agent.+ description (str): a short description of the agent. This description is used by other agents+ (e.g. the GroupChatManager) to decide when to call upon this agent.+ **kwargs (dict): Please refer to other kwargs in+ [ConversableAgent](../conversable_agent#__init__).+ """++ if llama_index_agent is None:+ raise ValueError("llama_index_agent must be provided")++ if description is None or description.isspace():+ raise ValueError("description must be provided")++ super().__init__(+ name,+ description=description,+ **kwargs,+ )++ self._llama_index_agent = llama_index_agent++ # Override the `generate_oai_reply`+ self.replace_reply_func(ConversableAgent.generate_oai_reply, LLamaIndexConversableAgent._generate_oai_reply)++ self.replace_reply_func(ConversableAgent.a_generate_oai_reply, LLamaIndexConversableAgent._a_generate_oai_reply)++ def _generate_oai_reply(+ self,+ messages: Optional[List[Dict]] = None,+ sender: Optional[Agent] = None,+ config: Optional[OpenAIWrapper] = None,+ ) -> Tuple[bool, Union[str, Dict, None]]:+ """Generate a reply using autogen.oai."""+ user_message, history = self._extract_message_and_history(messages=messages, sender=sender)++ chatResponse: AgentChatResponse = self._llama_index_agent.chat(message=user_message, chat_history=history)++ extracted_response = chatResponse.response++ return (True, extracted_response)++ async def _a_generate_oai_reply(+ self,+ messages: Optional[List[Dict]] = None,+ sender: Optional[Agent] = None,+ config: Optional[OpenAIWrapper] = None,+ ) -> Tuple[bool, Union[str, Dict, None]]:+ """Generate a reply using autogen.oai."""+ user_message, history = self._extract_message_and_history(messages=messages, sender=sender)++ chatResponse: AgentChatResponse = await self._llama_index_agent.achat(+ message=user_message, chat_history=history+ )++ extracted_response = chatResponse.response++ return (True, extracted_response)++ def _extract_message_and_history(+ self, messages: Optional[List[Dict]] = None, sender: Optional[Agent] = None+ ) -> Tuple[str, List[ChatMessage]]:+ """Extract the message and history from the messages."""+ if not messages:+ messages = self._oai_messages[sender]++ if not messages:+ return "", []++ message = messages[-1].get("content", "")++ history = messages[:-1]+ history_messages: List[ChatMessage] = []+ for history_message in history:+ content = history_message.get("content", "")+ role = history_message.get("role", "user")+ if role:+ if role == "user" or role == "assistant":+ history_messages.append(ChatMessage(content=content, role=role, additional_kwargs={}))+ return message, history_messagesdiff --git a/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynbnew file mode 100644index 000000000..ddff0a010--- /dev/null+++ b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb@@ -0,0 +1,225 @@+{+ "cells": [+ {+ "attachments": {},+ "cell_type": "markdown",+ "id": "9a71fa36",+ "metadata": {+ "editable": true,+ "slideshow": {+ "slide_type": ""+ },+ "tags": []+ },+ "source": [+ "# Groupchat with Llamaindex agents\n",+ "\n",+ "[Llamaindex agents](https://docs.llamaindex.ai/en/stable/optimizing/agentic_strategies/agentic_strategies/) have the ability to use planning strategies to answer user questions. They can be integrated in Autogen in easy ways\n",+ "\n",+ "## Requirements"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "c528cd6d",+ "metadata": {},+ "outputs": [],+ "source": [+ "! pip install pyautogen\n",+ "! pip install llama-index\n",+ "! pip install llama-index-tools-wikipedia\n",+ "! pip install llama-index-readers-wikipedia\n",+ "! pip install wikipedia"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "id": "5ebd2397",+ "metadata": {+ "editable": true,+ "slideshow": {+ "slide_type": ""+ },+ "tags": []+ },+ "source": [+ "## Set your API Endpoint"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 2,+ "id": "dca301a4",+ "metadata": {+ "editable": true,+ "slideshow": {+ "slide_type": ""+ },+ "tags": []+ },+ "outputs": [],+ "source": [+ "import os\n",+ "\n",+ "import autogen\n",+ "\n",+ "config_list = [{\"model\": \"gpt-3.5-turbo-0125\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]"+ ]+ },+ {+ "cell_type": "markdown",+ "id": "76c11ea8",+ "metadata": {},+ "source": [+ "## Set Llamaindex"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "id": "2d3d298e",+ "metadata": {},+ "outputs": [],+ "source": [+ "from llama_index.core import Settings\n",+ "from llama_index.core.agent import ReActAgent\n",+ "from llama_index.embeddings.openai import OpenAIEmbedding\n",+ "from llama_index.llms.openai import OpenAI\n",+ "from llama_index.tools.wikipedia import WikipediaToolSpec\n",+ "\n",+ "llm = OpenAI(\n",+ " model=\"gpt-3.5-turbo-0125\",\n",+ " temperature=0.0,\n",+ " api_key=os.environ.get(\"OPENAPI_API_KEY\", \"\"),\n",+ ")\n",+ "\n",+ "embed_model = OpenAIEmbedding(\n",+ " model=\"text-embedding-ada-002\",\n",+ " temperature=0.0,\n",+ " api_key=os.environ.get(\"OPENAPI_API_KEY\", \"\"),\n",+ ")\n",+ "\n",+ "Settings.llm = llm\n",+ "Settings.embed_model = embed_model\n",+ "\n",+ "# create a react agent to use wikipedia tool\n",+ "wiki_spec = WikipediaToolSpec()\n",+ "# Get the search wikipedia tool\n",+ "wikipedia_tool = wiki_spec.to_tool_list()[1]\n",+ "\n",+ "location_specialist = ReActAgent.from_tools(tools=[wikipedia_tool], llm=llm, max_iterations=10, verbose=True)"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "id": "2b9526e7",+ "metadata": {},+ "source": [+ "## Create agents\n",+ "\n",+ "In this example, we will create a Llamaindex agent to answer questions fecting data from wikipedia and a user proxy agent."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "1a10c9fe-1fbc-40c6-b655-5d2256864ce8",+ "metadata": {},+ "outputs": [],+ "source": [+ "from llamaindex_conversable_agent import LLamaIndexConversableAgent\n",+ "\n",+ "llm_config = {\n",+ " \"temperature\": 0,\n",+ " \"config_list\": config_list,\n",+ "}\n",+ "\n",+ "trip_assistant = LLamaIndexConversableAgent(\n",+ " \"trip_specialist\",\n",+ " llama_index_agent=location_specialist,\n",+ " system_message=\"You help customers finding more about places they would like to visit. You can use external resources to provide more details as you engage with the customer.\",\n",+ " description=\"This agents helps customers discover locations to visit, things to do, and other details about a location. It can use external resources to provide more details. This agent helps in finding attractions, history and all that there si to know about a place\",\n",+ ")\n",+ "\n",+ "user_proxy = autogen.UserProxyAgent(\n",+ " name=\"Admin\",\n",+ " human_input_mode=\"ALWAYS\",\n",+ " code_execution_config=False,\n",+ ")"+ ]+ },+ {+ "attachments": {},+ "cell_type": "markdown",+ "id": "966c96a4-cc8a-4400-b8db-a21b7142e33c",+ "metadata": {},+ "source": [+ "Next, let's set up our group chat."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": 3,+ "id": "354b4a8f-7a96-455b-9f17-cbc19d880462",+ "metadata": {},+ "outputs": [],+ "source": [+ "groupchat = autogen.GroupChat(\n",+ " agents=[trip_assistant, user_proxy],\n",+ " messages=[],\n",+ " max_round=500,\n",+ " speaker_selection_method=\"round_robin\",\n",+ " enable_clear_history=True,\n",+ ")\n",+ "manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=llm_config)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "id": "d5518947",+ "metadata": {},+ "outputs": [],+ "source": [+ "chat_result = user_proxy.initiate_chat(\n",+ " manager,\n",+ " message=\"\"\"\n",+ "What can i find in Tokyo related to Hayao Miyazaki and its moveis like Spirited Away?.\n",+ "\"\"\",\n",+ ")"+ ]+ }+ ],+ "metadata": {+ "front_matter": {+ "description": "Integrate llamaindex agents with Autogen.",+ "tags": [+ "react",+ "llama index",+ "software engineering"+ ]+ },+ "kernelspec": {+ "display_name": "Python 3 (ipykernel)",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.11.9"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 5+}diff --git a/test/agentchat/contrib/test_llamaindex_conversable_agent.py b/test/agentchat/contrib/test_llamaindex_conversable_agent.pynew file mode 100644index 000000000..ec31e1f6a--- /dev/null+++ b/test/agentchat/contrib/test_llamaindex_conversable_agent.py@@ -0,0 +1,102 @@+#!/usr/bin/env python3 -m pytest++import os+import sys+import unittest+from unittest.mock import MagicMock, patch++import pytest+from conftest import MOCK_OPEN_AI_API_KEY++from autogen import GroupChat, GroupChatManager+from autogen.agentchat.contrib.llamaindex_conversable_agent import LLamaIndexConversableAgent+from autogen.agentchat.conversable_agent import ConversableAgent++sys.path.append(os.path.join(os.path.dirname(__file__), "../.."))+sys.path.append(os.path.join(os.path.dirname(__file__), ".."))+from conftest import reason, skip_openai++skip_reasons = [reason]+try:+ from llama_index.core.agent import ReActAgent+ from llama_index.core.chat_engine.types import AgentChatResponse+ from llama_index.llms.openai import OpenAI++ skip_for_dependencies = False+ skip_reason = ""+except ImportError as e:+ skip_for_dependencies = True+ skip_reason = f"dependency not installed: {e.msg}"+ pass+++openaiKey = MOCK_OPEN_AI_API_KEY++[email protected](skip_for_dependencies, reason=skip_reason)+@patch("llama_index.core.agent.ReActAgent.chat")+def test_group_chat_with_llama_index_conversable_agent(chat_mock: MagicMock) -> None:+ """+ Tests the group chat functionality with two MultimodalConversable Agents.+ Verifies that the chat is correctly limited by the max_round parameter.+ Each agent is set to describe an image in a unique style, but the chat should not exceed the specified max_rounds.+ """+ llm = OpenAI(+ model="gpt-4",+ temperature=0.0,+ api_key=openaiKey,+ )++ chat_mock.return_value = AgentChatResponse(+ response="Visit ghibli studio in Tokyo, Japan. It is a must-visit place for fans of Hayao Miyazaki and his movies like Spirited Away."+ )++ location_specialist = ReActAgent.from_tools(llm=llm, max_iterations=5)++ # create an autogen agent using the react agent+ trip_assistant = LLamaIndexConversableAgent(+ "trip_specialist",+ llama_index_agent=location_specialist,+ system_message="You help customers finding more about places they would like to visit. You can use external resources to provide more details as you engage with the customer.",+ description="This agents helps customers discover locations to visit, things to do, and other details about a location. It can use external resources to provide more details. This agent helps in finding attractions, history and all that there si to know about a place",+ )++ llm_config = False+ max_round = 5++ user_proxy = ConversableAgent(+ "customer",+ max_consecutive_auto_reply=10,+ human_input_mode="NEVER",+ llm_config=False,+ default_auto_reply="Thank you. TERMINATE",+ )++ group_chat = GroupChat(+ agents=[user_proxy, trip_assistant],+ messages=[],+ max_round=100,+ send_introductions=False,+ speaker_selection_method="round_robin",+ )++ group_chat_manager = GroupChatManager(+ groupchat=group_chat,+ llm_config=llm_config,+ is_termination_msg=lambda x: x.get("content", "").rstrip().find("TERMINATE") >= 0,+ )++ # Initiating the group chat and observing the number of rounds+ user_proxy.initiate_chat(+ group_chat_manager,+ message="What can i find in Tokyo related to Hayao Miyazaki and its moveis like Spirited Away?.",+ )++ # Assertions to check if the number of rounds does not exceed max_round+ assert all(len(arr) <= max_round for arr in trip_assistant._oai_messages.values()), "Agent 1 exceeded max rounds"+ assert all(len(arr) <= max_round for arr in user_proxy._oai_messages.values()), "User proxy exceeded max rounds"+++if __name__ == "__main__":+ """Runs this file's tests from the command line."""+ test_group_chat_with_llama_index_conversable_agent()diff --git a/website/docs/ecosystem/img/ecosystem-llamaindex.png b/website/docs/ecosystem/img/ecosystem-llamaindex.pngnew file mode 100644index 000000000..ba2ab0a6d--- /dev/null+++ b/website/docs/ecosystem/img/ecosystem-llamaindex.png@@ -0,0 +1,3 @@+version https://git-lfs.github.com/spec/v1+oid sha256:08965da22eddf8c9253e96c1424d9a9b67f210018a3e39372746b93f1d787c04+size 23548diff --git a/website/docs/ecosystem/llamaindex.md b/website/docs/ecosystem/llamaindex.mdnew file mode 100644index 000000000..4e9cffeed--- /dev/null+++ b/website/docs/ecosystem/llamaindex.md@@ -0,0 +1,7 @@+# Llamaindex++++[Llamaindex](https://www.llamaindex.ai/) allows the users to create Llamaindex agents and integrate them in autogen conversation patterns.++- [Llamaindex + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb)