Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Functioning sync and streaming handlers for openai
reibs7d22a879a563
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.pyindex ba7162e9..ffb777d7 100644--- a/agentops/llm_tracker.py+++ b/agentops/llm_tracker.py@@ -25,8 +25,9 @@ class LlmTracker:self.client = clientself.event_stream = None- def _handle_response_openai(self, response, kwargs, init_timestamp):+ def _handle_response_v0_openai(self, response, kwargs, init_timestamp):"""Handle responses for OpenAI versions <v1.0.0"""+def handle_stream_chunk(chunk):try:model = chunk['model']@@ -50,25 +51,27 @@ class LlmTracker:self.event_stream.returns['content'] += tokenif finish_reason:+ if not self.event_stream.returns:+ self.event_stream.returns = {}self.event_stream.returns['finish_reason'] = finish_reason# Update end_timestampself.event_stream.end_timestamp = get_ISO_time()self.client.record(self.event_stream)self.event_stream = None- except:+ except Exception as e:print(f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generatorif inspect.isasyncgen(response):- async def generator():+ async def async_generator():async for chunk in response:handle_stream_chunk(chunk)yield chunk- return generator()+ return async_generator()- if inspect.isgenerator(response):+ elif inspect.isgenerator(response):def generator():for chunk in response:handle_stream_chunk(chunk)@@ -111,23 +114,31 @@ class LlmTracker:completion_tokens=response.usage.completion_tokens))# Standard response- except:+ except Exception as e:print(- f"Unable to parse response for LLM call {kwargs} - skipping upload to AgentOps")+ f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")return response- def _handle_async_response_openai(self, response, kwargs, init_timestamp):- def handle_stream_chunk(chunk):+ def _handle_response_v1_openai(self, response, kwargs, init_timestamp):+ """Handle responses for OpenAI versions >v1.0.0"""+ from openai import Stream, AsyncStream+ from openai.types.chat import ChatCompletionChunk+ from openai.resources import AsyncCompletions++ def handle_stream_chunk(chunk: ChatCompletionChunk):try:- model = chunk['model']- choices = chunk['choices']- token = choices[0]['delta'].get('content', '')- finish_reason = choices[0]['finish_reason']+ model = chunk.model+ choices = chunk.choices+ token = choices[0].delta.content+ finish_reason = choices[0].finish_reason+ function_call = choices[0].delta.function_call+ tool_calls = choices[0].delta.tool_calls+ role = choices[0].delta.roleif self.event_stream == None:self.event_stream = Event(- event_type='openai stream',+ event_type='openai chat completion stream',params=kwargs,result='Success',returns={"finish_reason": None,@@ -138,58 +149,108 @@ class LlmTracker:init_timestamp=init_timestamp)else:+ if token == None:+ token = ''self.event_stream.returns['content'] += tokenif finish_reason:+ if not self.event_stream.returns:+ self.event_stream.returns = {}self.event_stream.returns['finish_reason'] = finish_reason+ self.event_stream.returns['function_call'] = function_call+ self.event_stream.returns['tool_calls'] = tool_calls+ self.event_stream.returns['role'] = role# Update end_timestampself.event_stream.end_timestamp = get_ISO_time()self.client.record(self.event_stream)self.event_stream = None- except:+ except Exception as e:print(f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")# if the response is a generator, decorate the generator- if inspect.isasyncgen(response):- async def generator():- async for chunk in response:+ if isinstance(response, Stream):+ def generator():+ for chunk in response:handle_stream_chunk(chunk)-yield chunkreturn generator()- if inspect.isgenerator(response):- def generator():- for chunk in response:+ # For asynchronous AsyncStream+ elif isinstance(response, AsyncStream):+ async def async_generator():+ async for chunk in response:handle_stream_chunk(chunk)+ yield chunk+ return async_generator()+ # For async AsyncCompletion+ elif isinstance(response, AsyncCompletions):+ async def async_generator():+ async for chunk in response:+ handle_stream_chunk(chunk)yield chunk- return generator()+ return async_generator()- else:- try:- self.client.record(Event(- event_type=response['object'],- params=kwargs,- result='Success',- returns={"content":- response['choices'][0]['message']['content']},- action_type='llm',- model=response['model'],- prompt=kwargs['messages'],- init_timestamp=init_timestamp- ))- except:- print(- f"Unable to parse response for LLM call {kwargs} - skipping upload to AgentOps")+ # v1.0.0+ responses are objects+ try:+ self.client.record(Event(+ event_type=response.object,+ params=kwargs,+ result='Success',+ returns={+ # TODO: Will need to make the completion the key for content, splat out the model dump+ "content": response.choices[0].message.model_dump()},+ action_type='llm',+ model=response.model,+ prompt=kwargs['messages'],+ init_timestamp=init_timestamp,+ prompt_tokens=response.usage.prompt_tokens,+ completion_tokens=response.usage.completion_tokens+ ))+ # Standard response+ except Exception as e:+ print(+ f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")- return response+ return response++ def override_openai_v1_completion(self):+ from openai.resources.chat import completions++ # Store the original method+ original_create = completions.Completions.create++ # Define the patched function+ def patched_function(*args, **kwargs):+ init_timestamp = get_ISO_time()+ # Call the original function with its original arguments+ result = original_create(*args, **kwargs)+ return self._handle_response_v1_openai(result, kwargs, init_timestamp)++ # Override the original method with the patched one+ completions.Completions.create = patched_function++ def override_openai_v1_async_completion(self):+ from openai.resources.chat import completions++ # Store the original method+ original_create = completions.AsyncCompletions.create+ # Define the patched function++ async def patched_function(*args, **kwargs):+ # Call the original function with its original arguments+ init_timestamp = get_ISO_time()+ result = await original_create(*args, **kwargs)+ return self._handle_response_v1_openai(result, kwargs, init_timestamp)++ # Override the original method with the patched one+ completions.AsyncCompletions.create = patched_functiondef _override_method(self, api, method_path, module):def handle_response(result, kwargs, init_timestamp):if api == "openai":- return self._handle_response_openai(result, kwargs, init_timestamp)+ return self._handle_response_v0_openai(result, kwargs, init_timestamp)return resultdef wrap_method(original_method):@@ -236,52 +297,11 @@ class LlmTracker:self.override_openai_v1_completion()self.override_openai_v1_async_completion()+ # Patch all methods in every APIif hasattr(module, '__version__'):module_version = parse(module.__version__)for version in sorted(self.SUPPORTED_APIS[api], key=parse, reverse=True):if module_version >= parse(version):- # Patch all methods in every APIfor method_path in self.SUPPORTED_APIS[api][version]:self._override_method(api, method_path, module)break- else:- for method_path in self.SUPPORTED_APIS[api]['0.0.0']:- self._override_method(api, method_path, module)-- def override_openai_v1_completion(self):- from openai.resources.chat import completions-- # Store the original method- original_create = completions.Completions.create-- # Define the patched function- def patched_function(*args, **kwargs):- init_timestamp = get_ISO_time()- # Call the original function with its original arguments- result = original_create(*args, **kwargs)- self._handle_response_openai(result, kwargs, init_timestamp)-- return result-- # Override the original method with the patched one- completions.Completions.create = patched_function-- def override_openai_v1_async_completion(self):- from openai.resources.chat import completions-- # Store the original method- original_create = completions.AsyncCompletions.create- # Define the patched function-- def patched_function(*args, **kwargs):- print('patched async function')- # Call the original function with its original arguments- result = original_create(*args, **kwargs)- self._handle_response_openai(result, kwargs, get_ISO_time())-- # You can add additional logic here if needed- return result-- # Override the original method with the patched one- completions.AsyncCompletions.create = patched_function- print('patched async')diff --git a/tests/mock_agent_oai_v1.py b/tests/mock_agent_oai_v1.pydeleted file mode 100644index 3faa604a..00000000--- a/tests/mock_agent_oai_v1.py+++ /dev/null@@ -1,51 +0,0 @@-from openai.resources.chat import completions-import openai-from openai import OpenAI, AsyncOpenAI-import agentops-import asyncio-from dotenv import load_dotenv-load_dotenv()-client = OpenAI()---async_client = AsyncOpenAI()---# completions.Completions.create = lambda *args, **kwargs: print('test2')--print('Running OpenAI v1.0.0+')---# Assuming that instantiating agentops.Client will trigger the LlmTracker to override methods-ao_client = agentops.Client(tags=['mock agent'])---# Now the client.chat.completions.create should be the overridden method-# chat_completion = client.chat.completions.create(-# messages=[-# {-# "role": "user",-# "content": "Say this is a test",-# }-# ],-# model="gpt-3.5-turbo",-# )--# Test the async version of client.chat.completions.create---async def test_async_chat_completion():- return await async_client.chat.completions.create(- messages=[- {- "role": "user",- "content": "Say this is a test",- }- ],- model="gpt-3.5-turbo",- )---asyncio.run(test_async_chat_completion())--ao_client.end_session('Success')diff --git a/tests/async_call_agent.py b/tests/openai_handlers/mock_agent_oai_v0.pysimilarity index 80%rename from tests/async_call_agent.pyrename to tests/openai_handlers/mock_agent_oai_v0.pyindex 7a45e842..c1ed32fa 100644--- a/tests/async_call_agent.py+++ b/tests/openai_handlers/mock_agent_oai_v0.py@@ -1,23 +1,15 @@-<<<<<<< Updated upstreamimport openaiimport agentopsimport asyncioimport os-=======-import agentops-import asyncio->>>>>>> Stashed changes-ao_client = agentops.Client(api_key='',- tags=['mock tests'])+print('Running OpenAI <v1.0.0')+++ao_client = agentops.Client(tags=['mock tests'])-<<<<<<< Updated upstreamopenai.api_key = os.environ['OPENAI_API_KEY']-=======-import openai-openai.api_key = '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'->>>>>>> Stashed changesasync def stream_achat():diff --git a/tests/openai_handlers/mock_agent_oai_v1.py b/tests/openai_handlers/mock_agent_oai_v1.pynew file mode 100644index 00000000..39c3212f--- /dev/null+++ b/tests/openai_handlers/mock_agent_oai_v1.py@@ -0,0 +1,97 @@+import time+from openai.resources.chat import completions+import openai+from openai import OpenAI, AsyncOpenAI+import agentops+import asyncio+from dotenv import load_dotenv+load_dotenv()+client = OpenAI()+++async_client = AsyncOpenAI()+++print('Running OpenAI v1.0.0+')+++# Assuming that instantiating agentops.Client will trigger the LlmTracker to override methods+ao_client = agentops.Client(tags=['mock agent'])+++# Now the client.chat.completions.create should be the overridden method+print('Chat completion')+chat_completion = client.chat.completions.create(+ messages=[+ {+ "role": "user",+ "content": "Say this is a test",+ }+ ],+ model="gpt-3.5-turbo",+)++# test streaming+print('Chat completion streaming')+chat_completion = client.chat.completions.create(+ messages=[+ {+ "role": "user",+ "content": "Say this is a test",+ }+ ],+ model="gpt-3.5-turbo",+ stream=True+)+start_time = time.time()++collected_chunks = []+collected_messages = []++print(chat_completion)++for chunk in chat_completion:+ chunk_time = time.time() - start_time # calculate the time delay of the chunk+ collected_chunks.append(chunk) # save the event response+ chunk_message = chunk.choices[0].delta.content # extract the message+ collected_messages.append(chunk_message) # save the message+ # print the delay and text+ print(+ f"Message received {chunk_time:.2f} seconds after request: {chunk_message}")+++# # Test the async version of client.chat.completions.create++async def test_async_chat_completion():+ return await async_client.chat.completions.create(+ messages=[+ {+ "role": "user",+ "content": "Say this is an async test",+ }+ ],+ model="gpt-3.5-turbo",+ )+++async def test_async_chat_completion_stream():+ chat_completion_stream = await async_client.chat.completions.create(+ messages=[+ {+ "role": "user",+ "content": "Say this is an async test",+ }+ ],+ model="gpt-3.5-turbo",+ stream=True+ )+ async for chunk in chat_completion_stream:+ print(chunk)+++print('Running async tests')+asyncio.run(test_async_chat_completion())+print('Running async streaming test')+asyncio.run(test_async_chat_completion_stream())++ao_client.end_session('Success')