Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Updated sync callbacks. Bug where events are being overwritten and causing collision in supabase

Howard Gil3ba67a3971c1
diff --git a/agentops/__init__.py b/agentops/__init__.py
index 40ffa69d..5961de96 100755
--- a/agentops/__init__.py
+++ b/agentops/__init__.py
@@ -1,7 +1,7 @@
# agentops/__init__.py
from .client import Client
-from .event import ActionEvent, LLMEvent, ToolEvent, ErrorEvent
+from .event import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEvent
from .logger import AgentOpsLogger
from .enums import Models, LLMMessageFormat
from .decorators import record_function
diff --git a/agentops/enums.py b/agentops/enums.py
index 8f704968..d4d1cfe1 100644
--- a/agentops/enums.py
+++ b/agentops/enums.py
@@ -8,11 +8,11 @@ class EventType(Enum):
TOOL = "tools"
ERROR = "errors"
-
-class Result(Enum):
- SUCCESS = "Success"
- FAIL = "Fail"
- INDETERMINATE = "Indeterminate"
+# TODO: Not used
+# class Result(Enum):
+# SUCCESS = "Success"
+# FAIL = "Fail"
+# INDETERMINATE = "Indeterminate"
class Models(Enum):
diff --git a/agentops/event.py b/agentops/event.py
index 0f192f99..83c958c8 100644
--- a/agentops/event.py
+++ b/agentops/event.py
@@ -2,7 +2,7 @@
AgentOps events.
Data Class:
- Event: Represents discrete events to be recorded.
+ Event: Repents discrete events to be recorded.
"""
from dataclasses import dataclass, field
diff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.py
index e19f0999..b8539fcd 100644
--- a/agentops/langchain_callback_handler.py
+++ b/agentops/langchain_callback_handler.py
@@ -1,21 +1,18 @@
-from typing import Dict, Any, List, Optional, Union
+from typing import Dict, Any, List, Optional, Sequence
from uuid import UUID
from langchain_core.agents import AgentFinish, AgentAction
from langchain_core.outputs import LLMResult
from langchain_core.documents import Document
-
from langchain_core.outputs import LLMResult
+from langchain.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler
-
-from agentops import Client as AOClient
-from agentops import Event
from tenacity import RetryCallState
-from langchain.callbacks.base import BaseCallbackHandler, AsyncCallbackHandler
-
+from agentops import Client as AOClient
+from agentops import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEvent
+from agentops import LLMMessageFormat
from agentops.helpers import get_ISO_time
-from typing import Any, Dict, List, Optional, Sequence
class LangchainCallbackHandler(BaseCallbackHandler):
@@ -38,7 +35,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):
self.ao_client = AOClient(**{k: v for k, v in client_params.items()
if v is not None}, override=False)
- # keypair <run_id: str, Event>
+ # keypair <run_id: str, ActionEvent>
self.events: Dict[Any, Event] = {}
# LLM Callbacks
@@ -53,28 +50,29 @@ class LangchainCallbackHandler(BaseCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id] = Event(
- event_type="llm",
- action_type='llm',
- tags=tags,
+ self.events[run_id] = LLMEvent(
+ # tags=tags, # TODO: Are these tags coming from langchain?
+ params={**serialized, **kwargs, **({} if metadata is None else metadata)},
model=kwargs['invocation_params']['model'],
- params={**kwargs, **({} if metadata is None else metadata)},
- prompt=prompts[0],
- init_timestamp=get_ISO_time()
+ prompt_messages=prompts[0]
)
def on_llm_error(
self,
error: BaseException,
- *,
+ *, # TODO: What this
run_id: UUID,
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].result = "Fail"
-
- self.ao_client.record(self.events[run_id])
+ llmEvent: LLMEvent = self.events[run_id]
+ errorEvent = ErrorEvent(
+ trigger_event=llmEvent,
+ details=str(error), # TODO: do we need to str(error)
+ timestamp=get_ISO_time()
+ )
+ self.ao_client.record(llmEvent)
+ self.ao_client.record(errorEvent)
def on_llm_end(
self,
@@ -84,21 +82,30 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].returns = {
+ llmEvent: LLMEvent = self.events[run_id]
+
+ llmEvent.end_timestamp = get_ISO_time()
+ llmEvent.returns = {
"content": response.generations[0][0].message.content,
"generations": response.generations
}
- if response.llm_output is not None:
- self.events[run_id].prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
- self.events[run_id].completion_tokens = response.llm_output['token_usage']['completion_tokens']
-
- if len(response.generations) > 0:
- self.events[run_id].result = "Success"
- else:
- self.events[run_id].result = "Fail"
- self.ao_client.record(self.events[run_id])
+ if response.llm_output is not None:
+ llmEvent.completion_message = response.generations[0][0].message.content # TODO
+ llmEvent.completion_message_format = LLMMessageFormat.STRING # TODO
+ llmEvent.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']
+ llmEvent.completion_tokens = response.llm_output['token_usage']['completion_tokens']
+ llmEvent.format_messages() # TODO: Find somewhere logical to call this on the user's behalf. They shouldn't call it
+
+ self.ao_client.record(llmEvent)
+
+ if len(response.generations) == 0:
+ errorEvent = ErrorEvent(
+ trigger_event=self.events[run_id],
+ details="on_llm_end: No generations", # TODO: more descriptive error
+ timestamp=get_ISO_time()
+ )
+ self.ao_client.record(errorEvent)
# Chain callbacks
def on_chain_start(
@@ -112,12 +119,11 @@ class LangchainCallbackHandler(BaseCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id] = Event(
- event_type="chain",
- init_timestamp=get_ISO_time(),
- tags=tags,
- params={**inputs, **kwargs, **
+ self.events[run_id] = ActionEvent(
+ params={**serialized, **inputs, **kwargs, **
({} if metadata is None else metadata)},
+ action_type="chain"
+ # tags=tags,
)
def on_chain_end(
@@ -128,11 +134,12 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].result = "Success"
- self.events[run_id].returns = outputs
+ actionEvent: ActionEvent = self.events[run_id]
- self.ao_client.record(self.events[run_id])
+ actionEvent.returns = outputs
+ actionEvent.end_timestamp = get_ISO_time()
+
+ self.ao_client.record(actionEvent)
def on_chain_error(
self,
@@ -142,11 +149,12 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].result = "Fail"
- self.events[run_id].returns = str(error)
+ actionEvent: ActionEvent = self.events[run_id]
+ self.ao_client.record(actionEvent)
- self.ao_client.record(self.events[run_id])
+ # TODO: do we need to str(error)
+ errorEvent = ErrorEvent(trigger_event=actionEvent, details=str(error), timestamp=get_ISO_time())
+ self.ao_client.record(errorEvent)
# Tool callbacks
def on_tool_start(
@@ -160,12 +168,12 @@ class LangchainCallbackHandler(BaseCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Any:
- """Run when tool starts running."""
- self.events[run_id] = Event(
- event_type="tool",
- init_timestamp=get_ISO_time(),
- tags=tags,
+ self.events[run_id] = ToolEvent(
+ # tags=tags,
+ # TODO: we use serialized here but kwargs in on_llm_start
params={**serialized, **({} if metadata is None else metadata)},
+ logs=input_str # TODO: is this the right attribution?
+ # TODO: agent_id?
)
def on_tool_end(
@@ -176,17 +184,17 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
+ toolEvent: ToolEvent = self.events[run_id]
+
+ toolEvent.end_timestamp = get_ISO_time()
+ toolEvent.returns = output
+ self.ao_client.record(toolEvent)
+
# Tools are capable of failing `on_tool_end` quietly.
# This is a workaround to make sure we can log it as an error.
if kwargs.get('name') == '_Exception':
- self.events[run_id].result = "Fail"
- else:
- self.events[run_id].result = "Success"
-
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].returns = output
-
- self.ao_client.record(self.events[run_id])
+ errorEvent = ErrorEvent(trigger_event=toolEvent, details=output, timestamp=get_ISO_time())
+ self.ao_client.record(errorEvent)
def on_tool_error(
self,
@@ -196,11 +204,12 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].result = "Fail"
- self.events[run_id].returns = str(error)
+ toolEvent: ToolEvent = self.events[run_id]
+ self.ao_client.record(toolEvent)
- self.ao_client.record(self.events[run_id])
+ # TODO: do we need to str(error)
+ errorEvent = ErrorEvent(trigger_event=toolEvent, details=str(error), timestamp=get_ISO_time())
+ self.ao_client.record(errorEvent)
# Retriever callbacks
def on_retriever_start(
@@ -214,9 +223,10 @@ class LangchainCallbackHandler(BaseCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> None:
- self.events[run_id] = Event(
- event_type="retriever",
- init_timestamp=get_ISO_time()
+ self.events[run_id] = ActionEvent(
+ params={**serialized, **kwargs, "query": query, **({} if metadata is None else metadata)},
+ action_type="retreiver"
+ # tags=tags,
)
def on_retriever_end(
@@ -228,10 +238,12 @@ class LangchainCallbackHandler(BaseCallbackHandler):
tags: Optional[List[str]] = None,
**kwargs: Any,
) -> None:
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].result = "Success"
+ actionEvent: ActionEvent = self.events[run_id]
- self.ao_client.record(self.events[run_id])
+ actionEvent.logs = documents # TODO: Adding this. Might want to add elsewhere e.g. params
+ actionEvent.end_timestamp = get_ISO_time()
+
+ self.ao_client.record(actionEvent)
def on_retriever_error(
self,
@@ -242,10 +254,12 @@ class LangchainCallbackHandler(BaseCallbackHandler):
tags: Optional[List[str]] = None,
**kwargs: Any,
) -> None:
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].result = "Fail"
+ actionEvent: ActionEvent = self.events[run_id]
+ self.ao_client.record(actionEvent)
- self.ao_client.record(self.events[run_id])
+ # TODO: do we need to str(error)
+ errorEvent = ErrorEvent(trigger_event=actionEvent, details=str(error), timestamp=get_ISO_time())
+ self.ao_client.record(errorEvent)
# Agent callbacks
def on_agent_action(
@@ -256,11 +270,9 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- """Run on agent action."""
- self.events[run_id] = Event(
- event_type="agent",
- init_timestamp=get_ISO_time(),
- params={**kwargs},
+ self.events[run_id] = ActionEvent(
+ params={"action": action, **kwargs},
+ action_type="agent"
)
def on_agent_finish(
@@ -271,15 +283,14 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- """Run on agent finish."""
- self.events[run_id].end_timestamp = get_ISO_time()
- self.events[run_id].result = "Success"
- self.events[run_id].returns = finish.to_json()
+ actionEvent: ActionEvent = self.events[run_id]
+ actionEvent.returns = finish.to_json()
+ actionEvent.end_timestamp = get_ISO_time()
- self.ao_client.record(self.events[run_id])
+ self.ao_client.record(actionEvent)
# TODO: Create a way for the end user to set this based on their conditions
- self.ao_client.end_session("Success")
+ # self.ao_client.end_session("Success") #TODO: calling end_session here causes "No current session"
# Misc.
def on_retry(
@@ -290,16 +301,13 @@ class LangchainCallbackHandler(BaseCallbackHandler):
parent_run_id: Optional[UUID] = None,
**kwargs: Any,
) -> Any:
- """Run on a retry event."""
- event = Event(
- event_type="retry",
- init_timestamp=get_ISO_time(),
- end_timestamp=get_ISO_time(),
+ actionEvent = ActionEvent(
params={**kwargs},
- result="Indeterminate",
- returns=retry_state
+ returns=retry_state,
+ action_type="retry",
+ # result="Indeterminate" # TODO: currently have no way of recording Indeterminate
)
- self.ao_client.record(event)
+ self.ao_client.record(actionEvent)
@property
def session_id(self):
@@ -326,7 +334,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
self.ao_client = AOClient(**{k: v for k, v in client_params.items()
if v is not None}, override=False)
- # keypair <run_id: str, Event>
+ # keypair <run_id: str, ActionEvent>
self.events: Dict[Any, Event] = {}
# LLM Callbacks
@@ -341,13 +349,13 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id] = Event(
+ self.events[run_id] = LLMEvent(
event_type="llm",
action_type='llm',
tags=tags,
model=kwargs['invocation_params']['model'],
params={**kwargs, **({} if metadata is None else metadata)},
- prompt=prompts[0],
+ prompt_messages=prompts[0],
init_timestamp=get_ISO_time()
)
@@ -400,7 +408,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> Any:
- self.events[run_id] = Event(
+ self.events[run_id] = ActionEvent(
event_type="chain",
init_timestamp=get_ISO_time(),
tags=tags,
@@ -450,7 +458,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
**kwargs: Any,
) -> Any:
"""Run when tool starts running."""
- self.events[run_id] = Event(
+ self.events[run_id] = ToolEvent(
event_type="tool",
init_timestamp=get_ISO_time(),
tags=tags,
@@ -503,7 +511,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
metadata: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> None:
- self.events[run_id] = Event(
+ self.events[run_id] = ActionEvent(
event_type="retriever",
init_timestamp=get_ISO_time()
)
@@ -546,7 +554,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
**kwargs: Any,
) -> Any:
"""Run on agent action."""
- self.events[run_id] = Event(
+ self.events[run_id] = ActionEvent(
event_type="agent",
init_timestamp=get_ISO_time(),
params={**kwargs},
@@ -580,7 +588,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):
**kwargs: Any,
) -> Any:
"""Run on a retry event."""
- event = Event(
+ event = ActionEvent(
event_type="retry",
init_timestamp=get_ISO_time(),
end_timestamp=get_ISO_time(),
diff --git a/examples/langchain_examples.ipynb b/examples/langchain_examples.ipynb
index 469b0226..3298fb6e 100644
--- a/examples/langchain_examples.ipynb
+++ b/examples/langchain_examples.ipynb
@@ -14,7 +14,7 @@
},
{
"cell_type": "code",
- "execution_count": 1,
+ "execution_count": null,
"id": "initial_id",
"metadata": {
"ExecuteTime": {
@@ -41,7 +41,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": null,
"id": "585f00bb186711a7",
"metadata": {
"ExecuteTime": {
@@ -51,7 +51,7 @@
},
"outputs": [],
"source": [
- "from agentops.langchain_callback_handler import LangchainCallbackHandler"
+ "from agentops.langchain_callback_handler import LangchainCallbackHandler as AgentOpsLangchainCallbackHandler"
]
},
{
@@ -64,7 +64,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": null,
"id": "1490411415d7317c",
"metadata": {
"ExecuteTime": {
@@ -72,18 +72,7 @@
"start_time": "2023-12-15T20:21:11.479154Z"
}
},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "True"
- ]
- },
- "execution_count": 3,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
"load_dotenv()"
]
@@ -93,34 +82,26 @@
"id": "8371ec020e634dd0",
"metadata": {},
"source": [
- "This is where AgentOps comes into play. Before creating our LLM instance via Langchain, first we'll create an instance of the AO LangchainCallbackHandler. After the handler is initialiezd, a session will be recorded automatically.\n",
+ "This is where AgentOps comes into play. Before creating our LLM instance via Langchain, first we'll create an instance of the AO LangchainCallbackHandler. After the handler is initialized, a session will be recorded automatically.\n",
"\n",
"Pass in your API key, and optionally any tags to describe this session for easier lookup in the AO dashboard. You can also retrieve the `session_id` of the newly created session."
]
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": null,
"id": "deec8da7-5b88-487e-bb07-e1ec79147b72",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Agent Ops session ID: d08efbb9-05aa-4bbe-bc9c-abfe78cad8dc\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"AGENTOPS_API_KEY = os.environ.get('AGENTOPS_API_KEY')\n",
"OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')\n",
"\n",
- "handler = LangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Langchain Example'])\n",
+ "agentops_handler = AgentOpsLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Langchain Example'])\n",
"\n",
- "print(\"Agent Ops session ID: \" + handler.session_id)\n",
+ "print(\"Agent Ops session ID: \" + str(agentops_handler.session_id))\n",
"llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY,\n",
- " callbacks=[handler],\n",
+ " callbacks=[agentops_handler],\n",
" model='gpt-3.5-turbo')"
]
},
@@ -134,7 +115,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": null,
"id": "6abf26f9",
"metadata": {},
"outputs": [],
@@ -158,13 +139,13 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": null,
"id": "71d56635-d0db-4362-b140-5072abee249f",
"metadata": {},
"outputs": [],
"source": [
"for t in tools:\n",
- " t.callbacks = [handler]"
+ " t.callbacks = [agentops_handler]"
]
},
{
@@ -177,7 +158,7 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": null,
"id": "d538b20aa954ee80",
"metadata": {
"ExecuteTime": {
@@ -191,57 +172,20 @@
" llm,\n",
" agent=AgentType.CHAT_ZERO_SHOT_REACT_DESCRIPTION,\n",
" verbose=True,\n",
- " callbacks=[handler], # You must pass in a callback handler to record your agent\n",
+ " callbacks=[agentops_handler], # You must pass in a callback handler to record your agent\n",
" handle_parsing_errors=True)"
]
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": null,
"id": "6dfb127553751384",
"metadata": {
"scrolled": true
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "\n",
- "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
- "\u001b[32;1m\u001b[1;3mCould not parse LLM output: I can use the `find_movie` tool to find the available movies playing.\u001b[0m\n",
- "Observation: Invalid or incomplete response\n",
- "Thought:\u001b[32;1m\u001b[1;3mCould not parse LLM output: I should use the `find_movie` tool to find the available movies playing.\u001b[0m\n",
- "Observation: Invalid or incomplete response\n",
- "Thought:\u001b[32;1m\u001b[1;3mI should use the `find_movie` tool to find the available movies playing.\n",
- "Action:\n",
- "```json\n",
- "{\n",
- " \"action\": \"find_movie\",\n",
- " \"action_input\": \"\"\n",
- "}\n",
- "```\u001b[0m\n",
- "Observation: \u001b[36;1m\u001b[1;3mCitizen Kane\u001b[0m\n",
- "Thought:\u001b[32;1m\u001b[1;3mThe movie playing is \"Citizen Kane\".\n",
- "Final Answer: Citizen Kane\u001b[0m\n",
- "\n",
- "\u001b[1m> Finished chain.\u001b[0m\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "'Citizen Kane'"
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "agent.run(\"What movies are playing?\", callbacks=[handler])"
+ "agent.run(\"What movies are playing?\", callbacks=[agentops_handler])"
]
},
{
@@ -271,7 +215,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": null,
"id": "973f00a2-24da-48b3-adf7-97872d0d5afd",
"metadata": {},
"outputs": [],
@@ -285,7 +229,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": null,
"id": "16d8136c-cac2-47a5-85d8-cc466089feea",
"metadata": {},
"outputs": [],
@@ -295,7 +239,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": null,
"id": "b5986ed0-02d0-4e5f-99cd-c1fdfd7e5f1c",
"metadata": {},
"outputs": [],
@@ -309,31 +253,23 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": null,
"id": "bdce6ac1-bdbc-46ba-a9f1-7fd384187016",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Agent Ops session ID: d228326a-6392-4863-8ba5-a27ead34f586\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "handler = AsyncLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Async Example'])\n",
+ "agentops_handler = AsyncLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Async Example'])\n",
"\n",
"llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY,\n",
- " callbacks=[handler],\n",
+ " callbacks=[agentops_handler],\n",
" model='gpt-3.5-turbo')\n",
"\n",
- "print(\"Agent Ops session ID: \" + handler.session_id)"
+ "print(\"Agent Ops session ID: \" + agentops_handler.session_id)"
]
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": null,
"id": "d728c326-9a41-498b-bb64-9720577aac3e",
"metadata": {},
"outputs": [],
@@ -346,58 +282,25 @@
"tools = [find_movie]\n",
"\n",
"for t in tools:\n",
- " t.callbacks = [handler]"
+ " t.callbacks = [agentops_handler]"
]
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": null,
"id": "5d01e2b2-42f1-45ed-8a9d-fdfe82154926",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Agent Ops session ID: d228326a-6392-4863-8ba5-a27ead34f586\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
- "print(\"Agent Ops session ID: \" + handler.session_id)"
+ "print(\"Agent Ops session ID: \" + agentops_handler.session_id)"
]
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": null,
"id": "30eddc8b-0c90-4d96-9f1f-45da38c0d984",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\n",
- "\n",
- "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
- "\n",
- "Observation: Invalid or incomplete response\n",
- "Thought:\u001b[32;1m\u001b[1;3mI need to use the `find_movie` tool to find the available movies playing.\n",
- "Action:\n",
- "```json\n",
- "{\n",
- " \"action\": \"find_movie\",\n",
- " \"action_input\": \"\"\n",
- "}\n",
- "```\u001b[0m\n",
- "Observation: \u001b[36;1m\u001b[1;3mCitizen Kane\u001b[0m\n",
- "Thought:\u001b[32;1m\u001b[1;3mThe movie \"Citizen Kane\" is playing. \n",
- "Final Answer: Citizen Kane\u001b[0m\n",
- "\n",
- "\u001b[1m> Finished chain.\u001b[0m\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"agent = initialize_agent(\n",
" tools,\n",
@@ -405,7 +308,7 @@
" agent=AgentType.CHAT_ZERO_SHOT_REACT_DESCRIPTION,\n",
" verbose=True,\n",
" handle_parsing_errors=True,\n",
- " callbacks=[handler]\n",
+ " callbacks=[agentops_handler]\n",
")\n",
"\n",
"response = await agent.arun('What movies are playing?')"
@@ -452,7 +355,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.3"
+ "version": "3.12.2"
}
},
"nbformat": 4,
diff --git a/tests/langchain_handlers/test_langchain_handler.py b/tests/langchain_handlers/test_langchain_handler.py
new file mode 100644
index 00000000..550600a6
--- /dev/null
+++ b/tests/langchain_handlers/test_langchain_handler.py
@@ -0,0 +1,42 @@
+import os
+from langchain.chat_models import ChatOpenAI
+from langchain.agents import initialize_agent, AgentType
+from dotenv import load_dotenv
+from langchain.agents import tool
+from agentops.langchain_callback_handler import LangchainCallbackHandler as AgentOpsLangchainCallbackHandler
+
+
+load_dotenv()
+
+
+AGENTOPS_API_KEY = os.environ.get('AGENTOPS_API_KEY')
+OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
+
+agentops_handler = AgentOpsLangchainCallbackHandler(api_key=AGENTOPS_API_KEY, tags=['Langchain Example'])
+
+print("Agent Ops session ID: " + str(agentops_handler.session_id))
+llm = ChatOpenAI(openai_api_key=OPENAI_API_KEY,
+ callbacks=[agentops_handler],
+ model='gpt-3.5-turbo')
+
+
+@tool
+def find_movie(term) -> str:
+ """Find available movies"""
+ return 'Citizen Kane'
+
+
+tools = [find_movie]
+
+for t in tools:
+ t.callbacks = [agentops_handler]
+
+agent = initialize_agent(tools,
+ llm,
+ agent=AgentType.CHAT_ZERO_SHOT_REACT_DESCRIPTION,
+ verbose=True,
+ callbacks=[agentops_handler], # You must pass in a callback handler to record your agent
+ handle_parsing_errors=True)
+
+
+agent.run("What movies are playing?", callbacks=[agentops_handler])