Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Functioning sync and streaming handlers for openai

reibs7d22a879a563
diff --git a/agentops/llm_tracker.py b/agentops/llm_tracker.py
index ba7162e9..ffb777d7 100644
--- a/agentops/llm_tracker.py
+++ b/agentops/llm_tracker.py
@@ -25,8 +25,9 @@ class LlmTracker:
self.client = client
self.event_stream = None
- def _handle_response_openai(self, response, kwargs, init_timestamp):
+ def _handle_response_v0_openai(self, response, kwargs, init_timestamp):
"""Handle responses for OpenAI versions <v1.0.0"""
+
def handle_stream_chunk(chunk):
try:
model = chunk['model']
@@ -50,25 +51,27 @@ class LlmTracker:
self.event_stream.returns['content'] += token
if finish_reason:
+ if not self.event_stream.returns:
+ self.event_stream.returns = {}
self.event_stream.returns['finish_reason'] = finish_reason
# Update end_timestamp
self.event_stream.end_timestamp = get_ISO_time()
self.client.record(self.event_stream)
self.event_stream = None
- except:
+ except Exception as e:
print(
f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
if inspect.isasyncgen(response):
- async def generator():
+ async def async_generator():
async for chunk in response:
handle_stream_chunk(chunk)
yield chunk
- return generator()
+ return async_generator()
- if inspect.isgenerator(response):
+ elif inspect.isgenerator(response):
def generator():
for chunk in response:
handle_stream_chunk(chunk)
@@ -111,23 +114,31 @@ class LlmTracker:
completion_tokens=response.usage.completion_tokens
))
# Standard response
- except:
+ except Exception as e:
print(
- f"Unable to parse response for LLM call {kwargs} - skipping upload to AgentOps")
+ f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
return response
- def _handle_async_response_openai(self, response, kwargs, init_timestamp):
- def handle_stream_chunk(chunk):
+ def _handle_response_v1_openai(self, response, kwargs, init_timestamp):
+ """Handle responses for OpenAI versions >v1.0.0"""
+ from openai import Stream, AsyncStream
+ from openai.types.chat import ChatCompletionChunk
+ from openai.resources import AsyncCompletions
+
+ def handle_stream_chunk(chunk: ChatCompletionChunk):
try:
- model = chunk['model']
- choices = chunk['choices']
- token = choices[0]['delta'].get('content', '')
- finish_reason = choices[0]['finish_reason']
+ model = chunk.model
+ choices = chunk.choices
+ token = choices[0].delta.content
+ finish_reason = choices[0].finish_reason
+ function_call = choices[0].delta.function_call
+ tool_calls = choices[0].delta.tool_calls
+ role = choices[0].delta.role
if self.event_stream == None:
self.event_stream = Event(
- event_type='openai stream',
+ event_type='openai chat completion stream',
params=kwargs,
result='Success',
returns={"finish_reason": None,
@@ -138,58 +149,108 @@ class LlmTracker:
init_timestamp=init_timestamp
)
else:
+ if token == None:
+ token = ''
self.event_stream.returns['content'] += token
if finish_reason:
+ if not self.event_stream.returns:
+ self.event_stream.returns = {}
self.event_stream.returns['finish_reason'] = finish_reason
+ self.event_stream.returns['function_call'] = function_call
+ self.event_stream.returns['tool_calls'] = tool_calls
+ self.event_stream.returns['role'] = role
# Update end_timestamp
self.event_stream.end_timestamp = get_ISO_time()
self.client.record(self.event_stream)
self.event_stream = None
- except:
+ except Exception as e:
print(
f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
# if the response is a generator, decorate the generator
- if inspect.isasyncgen(response):
- async def generator():
- async for chunk in response:
+ if isinstance(response, Stream):
+ def generator():
+ for chunk in response:
handle_stream_chunk(chunk)
-
yield chunk
return generator()
- if inspect.isgenerator(response):
- def generator():
- for chunk in response:
+ # For asynchronous AsyncStream
+ elif isinstance(response, AsyncStream):
+ async def async_generator():
+ async for chunk in response:
handle_stream_chunk(chunk)
+ yield chunk
+ return async_generator()
+ # For async AsyncCompletion
+ elif isinstance(response, AsyncCompletions):
+ async def async_generator():
+ async for chunk in response:
+ handle_stream_chunk(chunk)
yield chunk
- return generator()
+ return async_generator()
- else:
- try:
- self.client.record(Event(
- event_type=response['object'],
- params=kwargs,
- result='Success',
- returns={"content":
- response['choices'][0]['message']['content']},
- action_type='llm',
- model=response['model'],
- prompt=kwargs['messages'],
- init_timestamp=init_timestamp
- ))
- except:
- print(
- f"Unable to parse response for LLM call {kwargs} - skipping upload to AgentOps")
+ # v1.0.0+ responses are objects
+ try:
+ self.client.record(Event(
+ event_type=response.object,
+ params=kwargs,
+ result='Success',
+ returns={
+ # TODO: Will need to make the completion the key for content, splat out the model dump
+ "content": response.choices[0].message.model_dump()},
+ action_type='llm',
+ model=response.model,
+ prompt=kwargs['messages'],
+ init_timestamp=init_timestamp,
+ prompt_tokens=response.usage.prompt_tokens,
+ completion_tokens=response.usage.completion_tokens
+ ))
+ # Standard response
+ except Exception as e:
+ print(
+ f"Unable to parse a chunk for LLM call {kwargs} - skipping upload to AgentOps")
- return response
+ return response
+
+ def override_openai_v1_completion(self):
+ from openai.resources.chat import completions
+
+ # Store the original method
+ original_create = completions.Completions.create
+
+ # Define the patched function
+ def patched_function(*args, **kwargs):
+ init_timestamp = get_ISO_time()
+ # Call the original function with its original arguments
+ result = original_create(*args, **kwargs)
+ return self._handle_response_v1_openai(result, kwargs, init_timestamp)
+
+ # Override the original method with the patched one
+ completions.Completions.create = patched_function
+
+ def override_openai_v1_async_completion(self):
+ from openai.resources.chat import completions
+
+ # Store the original method
+ original_create = completions.AsyncCompletions.create
+ # Define the patched function
+
+ async def patched_function(*args, **kwargs):
+ # Call the original function with its original arguments
+ init_timestamp = get_ISO_time()
+ result = await original_create(*args, **kwargs)
+ return self._handle_response_v1_openai(result, kwargs, init_timestamp)
+
+ # Override the original method with the patched one
+ completions.AsyncCompletions.create = patched_function
def _override_method(self, api, method_path, module):
def handle_response(result, kwargs, init_timestamp):
if api == "openai":
- return self._handle_response_openai(result, kwargs, init_timestamp)
+ return self._handle_response_v0_openai(result, kwargs, init_timestamp)
return result
def wrap_method(original_method):
@@ -236,52 +297,11 @@ class LlmTracker:
self.override_openai_v1_completion()
self.override_openai_v1_async_completion()
+ # Patch all methods in every API
if hasattr(module, '__version__'):
module_version = parse(module.__version__)
for version in sorted(self.SUPPORTED_APIS[api], key=parse, reverse=True):
if module_version >= parse(version):
- # Patch all methods in every API
for method_path in self.SUPPORTED_APIS[api][version]:
self._override_method(api, method_path, module)
break
- else:
- for method_path in self.SUPPORTED_APIS[api]['0.0.0']:
- self._override_method(api, method_path, module)
-
- def override_openai_v1_completion(self):
- from openai.resources.chat import completions
-
- # Store the original method
- original_create = completions.Completions.create
-
- # Define the patched function
- def patched_function(*args, **kwargs):
- init_timestamp = get_ISO_time()
- # Call the original function with its original arguments
- result = original_create(*args, **kwargs)
- self._handle_response_openai(result, kwargs, init_timestamp)
-
- return result
-
- # Override the original method with the patched one
- completions.Completions.create = patched_function
-
- def override_openai_v1_async_completion(self):
- from openai.resources.chat import completions
-
- # Store the original method
- original_create = completions.AsyncCompletions.create
- # Define the patched function
-
- def patched_function(*args, **kwargs):
- print('patched async function')
- # Call the original function with its original arguments
- result = original_create(*args, **kwargs)
- self._handle_response_openai(result, kwargs, get_ISO_time())
-
- # You can add additional logic here if needed
- return result
-
- # Override the original method with the patched one
- completions.AsyncCompletions.create = patched_function
- print('patched async')
diff --git a/tests/mock_agent_oai_v1.py b/tests/mock_agent_oai_v1.py
deleted file mode 100644
index 3faa604a..00000000
--- a/tests/mock_agent_oai_v1.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from openai.resources.chat import completions
-import openai
-from openai import OpenAI, AsyncOpenAI
-import agentops
-import asyncio
-from dotenv import load_dotenv
-load_dotenv()
-client = OpenAI()
-
-
-async_client = AsyncOpenAI()
-
-
-# completions.Completions.create = lambda *args, **kwargs: print('test2')
-
-print('Running OpenAI v1.0.0+')
-
-
-# Assuming that instantiating agentops.Client will trigger the LlmTracker to override methods
-ao_client = agentops.Client(tags=['mock agent'])
-
-
-# Now the client.chat.completions.create should be the overridden method
-# chat_completion = client.chat.completions.create(
-# messages=[
-# {
-# "role": "user",
-# "content": "Say this is a test",
-# }
-# ],
-# model="gpt-3.5-turbo",
-# )
-
-# Test the async version of client.chat.completions.create
-
-
-async def test_async_chat_completion():
- return await async_client.chat.completions.create(
- messages=[
- {
- "role": "user",
- "content": "Say this is a test",
- }
- ],
- model="gpt-3.5-turbo",
- )
-
-
-asyncio.run(test_async_chat_completion())
-
-ao_client.end_session('Success')
diff --git a/tests/async_call_agent.py b/tests/openai_handlers/mock_agent_oai_v0.py
similarity index 80%
rename from tests/async_call_agent.py
rename to tests/openai_handlers/mock_agent_oai_v0.py
index 7a45e842..c1ed32fa 100644
--- a/tests/async_call_agent.py
+++ b/tests/openai_handlers/mock_agent_oai_v0.py
@@ -1,23 +1,15 @@
-<<<<<<< Updated upstream
import openai
import agentops
import asyncio
import os
-=======
-import agentops
-import asyncio
->>>>>>> Stashed changes
-ao_client = agentops.Client(api_key='',
- tags=['mock tests'])
+print('Running OpenAI <v1.0.0')
+
+
+ao_client = agentops.Client(tags=['mock tests'])
-<<<<<<< Updated upstream
openai.api_key = os.environ['OPENAI_API_KEY']
-=======
-import openai
-openai.api_key = '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'
->>>>>>> Stashed changes
async def stream_achat():
diff --git a/tests/openai_handlers/mock_agent_oai_v1.py b/tests/openai_handlers/mock_agent_oai_v1.py
new file mode 100644
index 00000000..39c3212f
--- /dev/null
+++ b/tests/openai_handlers/mock_agent_oai_v1.py
@@ -0,0 +1,97 @@
+import time
+from openai.resources.chat import completions
+import openai
+from openai import OpenAI, AsyncOpenAI
+import agentops
+import asyncio
+from dotenv import load_dotenv
+load_dotenv()
+client = OpenAI()
+
+
+async_client = AsyncOpenAI()
+
+
+print('Running OpenAI v1.0.0+')
+
+
+# Assuming that instantiating agentops.Client will trigger the LlmTracker to override methods
+ao_client = agentops.Client(tags=['mock agent'])
+
+
+# Now the client.chat.completions.create should be the overridden method
+print('Chat completion')
+chat_completion = client.chat.completions.create(
+ messages=[
+ {
+ "role": "user",
+ "content": "Say this is a test",
+ }
+ ],
+ model="gpt-3.5-turbo",
+)
+
+# test streaming
+print('Chat completion streaming')
+chat_completion = client.chat.completions.create(
+ messages=[
+ {
+ "role": "user",
+ "content": "Say this is a test",
+ }
+ ],
+ model="gpt-3.5-turbo",
+ stream=True
+)
+start_time = time.time()
+
+collected_chunks = []
+collected_messages = []
+
+print(chat_completion)
+
+for chunk in chat_completion:
+ chunk_time = time.time() - start_time # calculate the time delay of the chunk
+ collected_chunks.append(chunk) # save the event response
+ chunk_message = chunk.choices[0].delta.content # extract the message
+ collected_messages.append(chunk_message) # save the message
+ # print the delay and text
+ print(
+ f"Message received {chunk_time:.2f} seconds after request: {chunk_message}")
+
+
+# # Test the async version of client.chat.completions.create
+
+async def test_async_chat_completion():
+ return await async_client.chat.completions.create(
+ messages=[
+ {
+ "role": "user",
+ "content": "Say this is an async test",
+ }
+ ],
+ model="gpt-3.5-turbo",
+ )
+
+
+async def test_async_chat_completion_stream():
+ chat_completion_stream = await async_client.chat.completions.create(
+ messages=[
+ {
+ "role": "user",
+ "content": "Say this is an async test",
+ }
+ ],
+ model="gpt-3.5-turbo",
+ stream=True
+ )
+ async for chunk in chat_completion_stream:
+ print(chunk)
+
+
+print('Running async tests')
+asyncio.run(test_async_chat_completion())
+print('Running async streaming test')
+asyncio.run(test_async_chat_completion_stream())
+
+ao_client.end_session('Success')