Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Eng-113-events-docstring (#128)
Howard Giladf66d7ea871
diff --git a/agentops/__init__.py b/agentops/__init__.pyindex 09efb23f..e09375a0 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -4,9 +4,8 @@ from typing import Optional, Listfrom .client import Clientfrom .config import Configuration-from .event import ActionEvent, LLMEvent, ToolEvent, ErrorEvent, Eventfrom .event import Event, ActionEvent, LLMEvent, ToolEvent, ErrorEvent-from .enums import Models, LLMMessageFormat, EndState+from .enums import Modelsfrom .decorators import record_functionfrom os import environdiff --git a/agentops/client.py b/agentops/client.pyindex 7580b8ca..9b802f5f 100644--- a/agentops/client.py+++ b/agentops/client.py@@ -47,6 +47,7 @@ class Client(metaclass=MetaClient):auto_start_session (bool): Whether to start a session automatically when the client is created.Attributes:_session (Session, optional): A Session is a grouping of events (e.g. a run of your agent).+ _worker (Worker, optional): A Worker manages the event queue and sends session updates to the AgentOps api server"""def __init__(self,diff --git a/agentops/enums.py b/agentops/enums.pyindex 1d6d6677..2c93be15 100644--- a/agentops/enums.py+++ b/agentops/enums.py@@ -23,11 +23,6 @@ class Models(Enum):TEXT_EMBEDDING_ADA_002 = "text-embedding-ada-002"-class LLMMessageFormat(Enum):- STRING = "string"- CHATML = "chatml"--class EndState(Enum):SUCCESS = "Success"FAIL = "Fail"diff --git a/agentops/event.py b/agentops/event.pyindex 02102b49..b4b10d0e 100644--- a/agentops/event.py+++ b/agentops/event.py@@ -8,12 +8,33 @@ Data Class:from dataclasses import dataclass, fieldfrom typing import List, Optionalfrom .helpers import get_ISO_time-from .enums import EventType, Models, LLMMessageFormat+from .enums import EventType, Modelsfrom uuid import UUID, uuid4@dataclassclass Event:++ """+ Abstract base class for events that will be recorded. Should not be instantiated directly.++ event_type(str): The type of event. Defined in enums.EventType. Some values are 'llm', 'action', 'api', 'tool', 'error'.+ params(dict, optional): The parameters of the function containing the triggered event, e.g. {'x': 1} in example below+ returns(str, optional): The return value of the function containing the triggered event, e.g. 2 in example below+ init_timestamp(str): A timestamp indicating when the event began. Defaults to the time when this Event was instantiated.+ end_timestamp(str): A timestamp indicating when the event ended. Defaults to the time when this Event was instantiated.+ id(UUID): A unique identifier for the event. Defaults to a new UUID.++ foo(x=1) {+ ...+ // params equals {'x': 1}+ record(ActionEvent(params=**kwargs, ...))+ ...+ // returns equals 2+ return x+1+ }+ """+event_type: str # EventType.ENUM.valueparams: Optional[dict] = Nonereturns: Optional[str] = None@@ -25,6 +46,15 @@ class Event:@dataclassclass ActionEvent(Event):+ """+ For generic events++ agent_id(UUID, optional): The unique identifier of the agent that triggered the event.+ action_type(str, optional): High level name describing the action+ logs(str, optional): For detailed information/logging related to the action+ screenshot(str, optional): url to snapshot if agent interacts with UI+ """+event_type: str = EventType.ACTION.value# TODO: Should not be optional, but non-default argument 'agent_id' follows default argument erroragent_id: Optional[UUID] = None@@ -40,51 +70,63 @@ class ActionEvent(Event):@dataclassclass LLMEvent(Event):++ """+ For recording calls to LLMs. AgentOps auto-instruments calls to the most popular LLMs e.g. GPT, Claude, Gemini, etc.++ agent_id(UUID, optional): The unique identifier of the agent that triggered the event.+ thread_id(UUID, optional): The unique identifier of the contextual thread that a message pertains to.+ prompt_message(str, list, optional): The message or messages that were used to prompt the LLM. Preferably in ChatML format which is more fully supported by AgentOps.+ prompt_tokens(int, optional): The number of tokens in the prompt message.+ completion_message(str, object, optional): The message or returned by the LLM. Preferably in ChatML format which is more fully supported by AgentOps.+ completion_tokens(int, optional): The number of tokens in the completion message.+ model(Models, str, optional): LLM model e.g. "gpt-4". Models defined in enums.Models are more fully supported by AgentOps e.g. extra features in dashboard.++ """+event_type: str = EventType.LLM.valueagent_id: Optional[UUID] = Nonethread_id: Optional[UUID] = None- prompt_messages: str | List = None # TODO: remove from serialization- prompt_messages_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization- # TODO: remove and just create it in __post_init__ so it can never be set by user?- _formatted_prompt_messages: object = field(init=False, default=None)- completion_message: str | object = None # TODO: remove from serialization- completion_message_format: LLMMessageFormat = LLMMessageFormat.STRING # TODO: remove from serialization- # TODO: remove and just create it in __post_init__ so it can never be set by user?- _formatted_completion_message: object = field(init=False, default=None)- model: Optional[Models | str] = None+ prompt_message: str | List = Noneprompt_tokens: Optional[int] = None+ completion_message: str | object = Nonecompletion_tokens: Optional[int] = None-- def format_messages(self):- if self.prompt_messages:- # TODO should we just figure out if it's chatml so user doesn't have to pass anything?- if self.prompt_messages_format == LLMMessageFormat.STRING:- self._formatted_prompt_messages = {"type": "string", "string": self.prompt_messages}- elif self.prompt_messages_format == LLMMessageFormat.CHATML:- self._formatted_prompt_messages = {"type": "chatml", "messages": self.prompt_messages}-- if self.completion_message:- if self.completion_message_format == LLMMessageFormat.STRING:- self._formatted_completion_message = {"type": "string", "string": self.completion_message}- elif self.completion_message_format == LLMMessageFormat.CHATML:- self._formatted_completion_message = {"type": "chatml", "message": self.completion_message}-- def __post_init__(self):- # format if prompt/completion messages were passed when LLMEvent was created- self.format_messages()+ model: Optional[Models | str] = None@dataclassclass ToolEvent(Event):+ """+ For recording calls to tools e.g. searchWeb, fetchFromDB++ agent_id(UUID, optional): The unique identifier of the agent that triggered the event.+ name(str, optional): A name describing the tool or the actual function name if applicable e.g. searchWeb, fetchFromDB.+ logs(str, dict, optional): For detailed information/logging related to the tool.++ """event_type: str = EventType.TOOL.valueagent_id: Optional[UUID] = Nonename: Optional[str] = Nonelogs: Optional[str | dict] = None-# Does not inherit from Event because error will (optionally) be linked to an ActionEvent, LLMEvent, etc that will have the details++@dataclassclass ErrorEvent():++ """+ For recording any errors e.g. ones related to agent execution++ trigger_event(Event, optional): The event object that triggered the error if applicable.+ error_type(str, optional): The type of error e.g. "ValueError".+ code(str, optional): A code that can be used to identify the error e.g. 501.+ details(str, optional): Detailed information about the error.+ logs(str, optional): For detailed information/logging related to the error.+ timestamp(str): A timestamp indicating when the error occurred. Defaults to the time when this ErrorEvent was instantiated.++ """+trigger_event: Optional[Event] = None # TODO: remove from serialization?error_type: Optional[str] = Nonecode: Optional[str] = Nonediff --git a/agentops/langchain_callback_handler.py b/agentops/langchain_callback_handler.pyindex 4a05f50d..daca246d 100644--- a/agentops/langchain_callback_handler.py+++ b/agentops/langchain_callback_handler.py@@ -12,7 +12,6 @@ from tenacity import RetryCallStatefrom agentops import Client as AOClientfrom agentops import ActionEvent, LLMEvent, ToolEvent, ErrorEvent-from agentops import LLMMessageFormatfrom agentops.helpers import get_ISO_timefrom .helpers import debug_print_function_params@@ -65,7 +64,7 @@ class LangchainCallbackHandler(BaseCallbackHandler):**({} if metadata is None else metadata),**kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logsmodel=kwargs['invocation_params']['model'],- prompt_messages=prompts[0]+ prompt_message=prompts[0]# tags=tags # TODO)@@ -101,16 +100,14 @@ class LangchainCallbackHandler(BaseCallbackHandler):llm_event.end_timestamp = get_ISO_time()if response.llm_output is not None:llm_event.completion_message = response.generations[0][0].message.content # TODO- llm_event.completion_message_format = LLMMessageFormat.STRING # TODOllm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']- llm_event.format_messages() # TODO: Find somewhere logical to call this on the user's behalf. They shouldn't call itself.ao_client.record(llm_event)if len(response.generations) == 0:# TODO: more descriptive errorerror_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],- details="on_llm_end: No generations", timestamp=get_ISO_time())+ details="on_llm_end: No generations", timestamp=get_ISO_time())self.ao_client.record(error_event)@debug_print_function_params@@ -366,7 +363,7 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):**({} if metadata is None else metadata),**kwargs}, # TODO: params is inconsistent, in ToolEvent we put it in logsmodel=kwargs['invocation_params']['model'],- prompt_messages=prompts[0]+ prompt_message=prompts[0])@debug_print_function_params@@ -428,16 +425,14 @@ class AsyncLangchainCallbackHandler(AsyncCallbackHandler):llm_event.end_timestamp = get_ISO_time()if response.llm_output is not None:llm_event.completion_message = response.generations[0][0].message.content # TODO- llm_event.completion_message_format = LLMMessageFormat.STRING # TODOllm_event.prompt_tokens = response.llm_output['token_usage']['prompt_tokens']llm_event.completion_tokens = response.llm_output['token_usage']['completion_tokens']- llm_event.format_messages() # TODO: Find somewhere logical to call this on the user's behalf. They shouldn't call itself.ao_client.record(llm_event)if len(response.generations) == 0:# TODO: more descriptive errorerror_event = ErrorEvent(trigger_event=self.events.llm[str(run_id)],- details="on_llm_end: No generations", timestamp=get_ISO_time())+ details="on_llm_end: No generations", timestamp=get_ISO_time())self.ao_client.record(error_event)@debug_print_function_paramsdiff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex ed9ce6cf..6b03c840 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -6,7 +6,6 @@ import loggingfrom .event import LLMEvent, ErrorEventfrom .helpers import get_ISO_time, check_call_stack_for_agent_idimport inspect-from .enums import LLMMessageFormatclass LlmTracker:@@ -41,6 +40,7 @@ class LlmTracker:)try:+ # NOTE: prompt/completion usage not returned in response when streamingmodel = chunk['model']choices = chunk['choices']token = choices[0]['delta'].get('content', '')@@ -50,14 +50,11 @@ class LlmTracker:if finish_reason:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_messages = kwargs["messages"]- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML+ self.llm_event.prompt_message = kwargs["messages"]self.llm_event.completion_message = {"role": "assistant", "content": self.completion}- self.llm_event.completion_message_format = LLMMessageFormat.CHATMLself.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion}self.llm_event.model = modelself.llm_event.end_timestamp = get_ISO_time()- self.llm_event.format_messages()self.client.record(self.llm_event)except Exception as e:@@ -90,14 +87,13 @@ class LlmTracker:# v0.0.0 responses are dictstry:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_messages = kwargs["messages"]- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML- self.llm_event.completion_message = response['choices'][0]['message']- self.llm_event.completion_message_format = LLMMessageFormat.CHATML+ self.llm_event.prompt_message = kwargs["messages"]+ self.llm_event.prompt_tokens = response['usage']['prompt_tokens']+ self.llm_event.completion_message = response['choices'][0]['message']['content']+ self.llm_event.completion_tokens = response['usage']['completion_tokens']self.llm_event.returns = {"content": response['choices'][0]['message']['content']}self.llm_event.model = response["model"]self.llm_event.end_timestamp = get_ISO_time()- self.llm_event.format_messages()self.client.record(self.llm_event)except Exception as e:@@ -125,6 +121,7 @@ class LlmTracker:)try:+ # NOTE: prompt/completion usage not returned in response when streamingmodel = chunk.modelchoices = chunk.choicestoken = choices[0].delta.content@@ -137,15 +134,12 @@ class LlmTracker:if finish_reason:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_messages = kwargs["messages"]- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML+ self.llm_event.prompt_message = kwargs["messages"]self.llm_event.completion_message = {"role": "assistant", "content": self.completion}- self.llm_event.completion_message_format = LLMMessageFormat.CHATMLself.llm_event.returns = {"finish_reason": finish_reason, "content": self.completion,"function_call": function_call, "tool_calls": tool_calls, "role": role}self.llm_event.model = modelself.llm_event.end_timestamp = get_ISO_time()- self.llm_event.format_messages()self.client.record(self.llm_event)except Exception as e:@@ -185,13 +179,12 @@ class LlmTracker:# v1.0.0+ responses are objectstry:self.llm_event.agent_id = check_call_stack_for_agent_id()- self.llm_event.prompt_messages = kwargs["messages"]- self.llm_event.prompt_messages_format = LLMMessageFormat.CHATML- self.llm_event.completion_message = response.choices[0].message.model_dump()- self.llm_event.completion_message_format = LLMMessageFormat.CHATML+ self.llm_event.prompt_message = kwargs["messages"]+ self.llm_event.prompt_tokens = response.usage.prompt_tokens+ self.llm_event.completion_message = response.choices[0].message.model_dump().get('content')+ self.llm_event.completion_tokens = response.usage.completion_tokensself.llm_event.returns = response.model_dump()self.llm_event.model = response.model- self.llm_event.format_messages()self.client.record(self.llm_event)except Exception as e:diff --git a/tests/openai_handlers/_test_llm_tracker_ge_1_async.py b/tests/openai_handlers/_test_llm_tracker_ge_1_async.pyindex de78fa4c..8d580094 100644--- a/tests/openai_handlers/_test_llm_tracker_ge_1_async.py+++ b/tests/openai_handlers/_test_llm_tracker_ge_1_async.py@@ -1,38 +1,64 @@-from openai import AsyncOpenAI # Assuming AsyncOpenAI is the async version of the client+from openai import AsyncOpenAIimport asyncioimport agentopsfrom agentops import record_function+from dotenv import load_dotenv-ao_client = agentops.Client()+load_dotenv()+agentops.init()-@record_function('sample function being recorded')-async def call_openai_async():++@record_function('openai v1 async no streaming')+async def call_openai_v1_async_no_streaming():+ client = AsyncOpenAI()++ chat_completion = await client.chat.completions.create(+ model="gpt-3.5-turbo",+ messages=[+ {+ "role": "system",+ "content": "You are an expert haiku writer"+ },+ {+ "role": "user",+ "content": "write me a haiku about devops"+ }+ ]+ )++ print(chat_completion)+ # raise ValueError("This is an intentional error for testing.")+++@record_function('openai v1 async with streaming')+async def call_openai_v1_async_streaming():client = AsyncOpenAI() # Using the async client- response = await client.chat.completions.create( # Await the async call+ chat_completion = await client.chat.completions.create(model="gpt-3.5-turbo",messages=[{"role": "system",- "content": "You will be provided with a piece of code, and your task is to explain it in a concise way."+ "content": "You are an expert haiku writer"},{"role": "user",- "content": "class Log:\n def __init__(self, path):\n dirname = os.path.dirname(path)\n os.makedirs(dirname, exist_ok=True)\n f = open(path, \"a+\")\n \n # Check that the file is newline-terminated\n size = os.path.getsize(path)\n if size > 0:\n f.seek(size - 1)\n end = f.read(1)\n if end != \"\\n\":\n f.write(\"\\n\")\n self.f = f\n self.path = path\n \n def log(self, event):\n event[\"_event_id\"] = str(uuid.uuid4())\n json.dump(event, self.f)\n self.f.write(\"\\n\")\n \n def state(self):\n state = {\"complete\": set(), \"last\": None}\n for line in open(self.path):\n event = json.loads(line)\n if event[\"type\"] == \"submit\" and event[\"success\"]:\n state[\"complete\"].add(event[\"id\"])\n state[\"last\"] = event\n return state"+ "content": "write me a haiku about devops"}],- temperature=0.7,- max_tokens=64,- top_p=1+ stream=True)- print(response)+ async for chunk in chat_completion:+ chunk_message = chunk.choices[0].delta.content+ print(chunk_message)# raise ValueError("This is an intentional error for testing.")async def main():- await call_openai_async()- ao_client.end_session('Success') # This would also need to be made async if it makes network calls+ await call_openai_v1_async_no_streaming()+ await call_openai_v1_async_streaming()+ agentops.end_session('Success') # This would also need to be made async if it makes network callsasyncio.run(main())diff --git a/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py b/tests/openai_handlers/_test_llm_tracker_ge_1_sync.pyindex 415e0af0..e602e5c6 100644--- a/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py+++ b/tests/openai_handlers/_test_llm_tracker_ge_1_sync.py@@ -4,9 +4,11 @@ from agentops import record_functionfrom packaging.version import parsefrom importlib import import_moduleimport sys+from dotenv import load_dotenv+load_dotenv()-ao_client = agentops.Client()+agentops.init()api = 'openai'if api in sys.modules:@@ -17,11 +19,31 @@ if api in sys.modules:print('openai version: ', module_version)-@record_function('sample function being recorded')-def call_openai():+@record_function('openai v1 sync no streaming')+def call_openai_v1_sync_no_streaming():client = OpenAI()+ chat_completion = client.chat.completions.create(+ model="gpt-3.5-turbo",+ messages=[+ {+ "role": "system",+ "content": "You are an expert haiku writer"+ },+ {+ "role": "user",+ "content": "write me a haiku about devops"+ }+ ]+ )++ print(chat_completion)+ # raise ValueError("This is an intentional error for testing.")+- response = client.chat.completions.create(+@record_function('openai v1 sync with streaming')+def call_openai_v1_sync_streaming():+ client = OpenAI()+ chat_completion = client.chat.completions.create(model="gpt-3.5-turbo",messages=[{@@ -33,15 +55,17 @@ def call_openai():"content": "write me a haiku about devops"}],- temperature=0.7,- max_tokens=64,- top_p=1+ stream=True)- print(response)+ for chunk in chat_completion:+ chunk_message = chunk.choices[0].delta.content+ print(chunk_message)+# raise ValueError("This is an intentional error for testing.")-call_openai()+call_openai_v1_sync_no_streaming()+call_openai_v1_sync_streaming()-ao_client.end_session('Success')+agentops.end_session('Success')diff --git a/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py b/tests/openai_handlers/_test_llm_tracker_lt_1_sync.pyindex 9f33104c..5555ad76 100644--- a/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py+++ b/tests/openai_handlers/_test_llm_tracker_lt_1_sync.py@@ -1,12 +1,14 @@-import openaifrom openai import ChatCompletionimport agentopsfrom agentops import record_functionfrom packaging.version import parsefrom importlib import import_moduleimport sys+from dotenv import load_dotenv-ao_client = agentops.Client()+load_dotenv()++agentops.init()api = 'openai'if api in sys.modules:@@ -17,19 +19,51 @@ if api in sys.modules:print('openai version: ', module_version)-@record_function('sample function being recorded')-def call_openai():+@record_function('openai v0 sync no streaming')+def call_openai_v0_sync_no_streaming():+ chat_completion = ChatCompletion.create(+ model='gpt-3.5-turbo',+ messages=[+ {+ "role": "system",+ "content": "You are an expert haiku writer"+ },+ {+ "role": "user",+ "content": "write me a haiku about devops"+ }+ ]+ )++ print(chat_completion)+ # raise ValueError("This is an intentional error for testing.")- message = [{"role": "user", "content": "Hello"},- {"role": "assistant", "content": "Hi there!"}]- response = ChatCompletion.create(- model='gpt-3.5-turbo', messages=message, temperature=0.5)+@record_function('openai v0 sync with streaming')+def call_openai_v0_sync_with_streaming():+ chat_completion = ChatCompletion.create(+ model='gpt-3.5-turbo',+ messages=[+ {+ "role": "system",+ "content": "You are an expert haiku writer"+ },+ {+ "role": "user",+ "content": "write me a haiku about devops"+ }+ ],+ stream=True+ )- print(response)+ for chunk in chat_completion:+ print(chunk.choices[0])+ chunk_message = chunk.choices[0].delta["content"]+ print(chunk_message)# raise ValueError("This is an intentional error for testing.")-call_openai()+call_openai_v0_sync_no_streaming()+call_openai_v0_sync_with_streaming()-ao_client.end_session('Success')+agentops.end_session('Success')