Skip to content

Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI

Revert "Add integration tests for Anthropic/Cohere/Groq/Litellm/Mistr… (#614)

teocnsad43fa78e272
diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml
index 8da0b585..357624a5 100644
--- a/.github/workflows/python-testing.yml
+++ b/.github/workflows/python-testing.yml
@@ -24,13 +24,6 @@ jobs:
runs-on: ubuntu-latest
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
- AGENTOPS_API_KEY: ${{ secrets.AGENTOPS_API_KEY }}
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
- AI21_API_KEY: ${{ secrets.AI21_API_KEY }}
- MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
- GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
- FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
strategy:
matrix:
@@ -49,9 +42,3 @@ jobs:
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
AGENTOPS_API_KEY: ${{ secrets.AGENTOPS_API_KEY }}
- ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
- COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
- AI21_API_KEY: ${{ secrets.AI21_API_KEY }}
- MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
- GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
- FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
diff --git a/tests/core_manual_tests/providers/ai21_canary.py b/tests/core_manual_tests/providers/ai21_canary.py
index 048b8467..e38be654 100644
--- a/tests/core_manual_tests/providers/ai21_canary.py
+++ b/tests/core_manual_tests/providers/ai21_canary.py
@@ -1,110 +1,64 @@
import asyncio
-import os
+
import agentops
from dotenv import load_dotenv
+import os
import ai21
from ai21.models.chat import ChatMessage
-from ai21.clients.studio.resources.chat import ChatCompletions, AsyncChatCompletions
load_dotenv()
+agentops.init(default_tags=["ai21-provider-test"])
+
+api_key = os.getenv("AI2I_API_KEY")
+ai21_client = ai21.AI21Client(api_key=api_key)
+async_ai21_client = ai21.AsyncAI21Client(api_key=api_key)
+
+messages = [
+ ChatMessage(content="You are an expert mathematician.", role="system"),
+ ChatMessage(
+ content="Write a summary of 5 lines on the Shockley diode equation.",
+ role="user",
+ ),
+]
+
+response = ai21_client.chat.completions.create(
+ model="jamba-1.5-mini",
+ messages=messages,
+)
+
+
+stream_response = ai21_client.chat.completions.create(
+ model="jamba-1.5-mini",
+ messages=messages,
+ stream=True,
+)
+
+response = ""
+for chunk in stream_response:
+ response += chunk.choices[0].delta.content
+print(response)
+
+
+async def async_test():
+ async_response = await async_ai21_client.chat.completions.create(
+ model="jamba-1.5-mini",
+ messages=messages,
+ )
+ print(async_response)
+
+
+asyncio.run(async_test())
+
+agentops.stop_instrumenting()
+
+untracked_response = ai21_client.chat.completions.create(
+ model="jamba-1.5-mini",
+ messages=messages,
+)
+
+
+agentops.end_session(end_state="Success")
-# Check for required API key
-if not os.getenv("AI21_API_KEY"):
- raise ValueError("AI21_API_KEY environment variable is required")
-
-def test_ai21_integration():
- """Integration test demonstrating all four AI21 call patterns:
- 1. Sync (non-streaming)
- 2. Sync (streaming)
- 3. Async (non-streaming)
- 4. Async (streaming)
-
- Verifies that AgentOps correctly tracks all LLM calls via analytics.
- """
- # Initialize AgentOps without auto-starting session
- agentops.init(auto_start_session=False)
- session = agentops.start_session()
-
- api_key = os.getenv("AI21_API_KEY")
- # Initialize provider
- from agentops.llms.providers.ai21 import AI21Provider
- provider = AI21Provider(None) # AI21 doesn't need a client instance
- provider.override()
-
- # Pass session to provider
- provider.client = session
- ai21_client = ai21.AI21Client(api_key=api_key)
- async_ai21_client = ai21.AsyncAI21Client(api_key=api_key)
- chat_client = ChatCompletions(client=ai21_client)
- async_chat_client = AsyncChatCompletions(client=async_ai21_client)
-
- # Create message objects
- base_messages = [
- ChatMessage(role="system", content="You are a helpful AI assistant"),
- ChatMessage(role="user", content="Hello from the test suite")
- ]
- sync_messages = base_messages.copy()
- sync_stream_messages = base_messages.copy()
- async_messages = base_messages.copy()
- async_stream_messages = base_messages.copy()
-
- def sync_no_stream():
- chat_client.create(
- model="jamba-instruct",
- system="You are a helpful AI assistant",
- messages=sync_messages,
- maxTokens=10
- )
-
- def sync_stream():
- stream_response = chat_client.create(
- model="jamba-instruct",
- system="You are a helpful AI assistant",
- messages=sync_stream_messages,
- maxTokens=10,
- stream=True
- )
- for chunk in stream_response:
- _ = chunk.choices[0].delta.content if hasattr(chunk.choices[0].delta, "content") else ""
-
- async def async_no_stream():
- await async_chat_client.create(
- model="jamba-instruct",
- system="You are a helpful AI assistant",
- messages=async_messages,
- maxTokens=10
- )
-
- async def async_stream():
- async_stream_response = await async_chat_client.create(
- model="jamba-instruct",
- system="You are a helpful AI assistant",
- messages=async_stream_messages,
- maxTokens=10,
- stream=True
- )
- async for chunk in async_stream_response:
- _ = chunk.choices[0].delta.content if hasattr(chunk.choices[0].delta, 'content') else ''
-
- async def run_async_tests():
- await async_no_stream()
- await async_stream()
-
- # Call each function with proper error handling
- try:
- sync_no_stream()
- sync_stream()
- asyncio.run(run_async_tests())
- except Exception as e:
- print(f"Error during AI21 test: {str(e)}")
- raise
- finally:
- session.end_session("Success")
- analytics = session.get_analytics()
- print(f"Analytics: {analytics}")
- assert analytics["LLM calls"] >= 4, f"Expected at least 4 LLM calls, but got {analytics['LLM calls']}"
-
- # Analytics verification is handled in the finally block
-
-if __name__ == "__main__":
- test_ai21_integration()
+###
+# Used to verify that one session is created with one LLM event
+###
diff --git a/tests/core_manual_tests/providers/anthropic_canary.py b/tests/core_manual_tests/providers/anthropic_canary.py
index c667acf9..52acf0ab 100644
--- a/tests/core_manual_tests/providers/anthropic_canary.py
+++ b/tests/core_manual_tests/providers/anthropic_canary.py
@@ -1,110 +1,78 @@
import asyncio
-import os
+
import agentops
from dotenv import load_dotenv
import anthropic
load_dotenv()
+agentops.init(default_tags=["anthropic-provider-test"])
+anthropic_client = anthropic.Anthropic()
+async_anthropic_client = anthropic.AsyncAnthropic()
+
+response = anthropic_client.messages.create(
+ max_tokens=1024,
+ model="claude-3-5-sonnet-20240620",
+ messages=[
+ {
+ "role": "user",
+ "content": "say hi",
+ }
+ ],
+)
+
-def test_anthropic_integration():
- """Integration test demonstrating all four Anthropic call patterns:
- 1. Sync (non-streaming)
- 2. Sync (streaming)
- 3. Async (non-streaming)
- 4. Async (streaming)
+stream_response = anthropic_client.messages.create(
+ max_tokens=1024,
+ model="claude-3-5-sonnet-20240620",
+ messages=[
+ {
+ "role": "user",
+ "content": "say hi 2",
+ }
+ ],
+ stream=True,
+)
- Verifies that AgentOps correctly tracks all LLM calls via analytics.
- """
- # Initialize AgentOps without auto-starting session
- agentops.init(auto_start_session=False)
- session = agentops.start_session()
+response = ""
+for event in stream_response:
+ if event.type == "content_block_delta":
+ response += event.delta.text
+ elif event.type == "message_stop":
+ print(response)
- # Initialize clients and provider
- anthropic_client = anthropic.Anthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
- async_anthropic_client = anthropic.AsyncAnthropic(api_key=os.getenv("ANTHROPIC_API_KEY"))
- from agentops.llms.providers.anthropic import AnthropicProvider
- provider = AnthropicProvider(anthropic_client)
- provider.override()
-
- # Pass session to provider
- provider.client = session
- def sync_no_stream():
- anthropic_client.messages.create(
- max_tokens=1024,
- model="claude-3-5-sonnet-20240620",
- messages=[
- {
- "role": "user",
- "content": "Hello from sync no stream",
- }
- ],
- session=session
- )
+async def async_test():
+ async_response = await async_anthropic_client.messages.create(
+ max_tokens=1024,
+ model="claude-3-5-sonnet-20240620",
+ messages=[
+ {
+ "role": "user",
+ "content": "say hi 3",
+ }
+ ],
+ )
+ print(async_response)
- def sync_stream():
- stream_response = anthropic_client.messages.create(
- max_tokens=1024,
- model="claude-3-5-sonnet-20240620",
- messages=[
- {
- "role": "user",
- "content": "Hello from sync streaming",
- }
- ],
- stream=True,
- session=session
- )
- for _ in stream_response:
- pass
- async def async_no_stream():
- await async_anthropic_client.messages.create(
- max_tokens=1024,
- model="claude-3-5-sonnet-20240620",
- messages=[
- {
- "role": "user",
- "content": "Hello from async no stream",
- }
- ],
- session=session
- )
+asyncio.run(async_test())
- async def async_stream():
- async_stream_response = await async_anthropic_client.messages.create(
- max_tokens=1024,
- model="claude-3-5-sonnet-20240620",
- messages=[
- {
- "role": "user",
- "content": "Hello from async streaming",
- }
- ],
- stream=True,
- session=session
- )
- async for _ in async_stream_response:
- pass
+agentops.stop_instrumenting()
- async def run_async_tests():
- await async_no_stream()
- await async_stream()
+untracked_response = anthropic_client.messages.create(
+ max_tokens=1024,
+ model="claude-3-5-sonnet-20240620",
+ messages=[
+ {
+ "role": "user",
+ "content": "say hi 4",
+ }
+ ],
+)
- # Call each function with proper error handling
- try:
- sync_no_stream()
- sync_stream()
- asyncio.run(run_async_tests())
- except Exception as e:
- print(f"Error during Anthropic test: {str(e)}")
- raise
- session.end_session("Success")
- analytics = session.get_analytics()
- print(analytics)
- # Verify that all LLM calls were tracked
- assert analytics["LLM calls"] >= 4, f"Expected at least 4 LLM calls, but got {analytics['LLM calls']}"
+agentops.end_session(end_state="Success")
-if __name__ == "__main__":
- test_anthropic_integration()
+###
+# Used to verify that one session is created with one LLM event
+###
diff --git a/tests/core_manual_tests/providers/cohere_canary.py b/tests/core_manual_tests/providers/cohere_canary.py
index 1797e274..fcec2006 100644
--- a/tests/core_manual_tests/providers/cohere_canary.py
+++ b/tests/core_manual_tests/providers/cohere_canary.py
@@ -1,148 +1,45 @@
import asyncio
-import inspect
-import os
+
import agentops
from dotenv import load_dotenv
import cohere
-from agentops.helpers import get_ISO_time
-from agentops.llms.providers.cohere import CohereProvider
load_dotenv()
+agentops.init(default_tags=["cohere-provider-test"])
+co = cohere.Client()
+aco = cohere.AsyncClient()
-def test_cohere_integration():
- """Integration test demonstrating all four Cohere call patterns:
- 1. Sync (non-streaming)
- 2. Sync (streaming)
- 3. Async (non-streaming)
- 4. Async (streaming)
-
- Verifies that AgentOps correctly tracks all LLM calls via analytics.
- """
- # Get API key with error handling
- api_key = os.getenv("COHERE_API_KEY")
- if not api_key:
- raise ValueError("COHERE_API_KEY environment variable is required")
- print("\nInitializing test with Cohere API key...")
+chat = co.chat(message="say hi 1")
- # Initialize AgentOps without auto-starting session
- agentops.init(auto_start_session=False)
- session = agentops.start_session()
- print(f"Started new session with ID: {session.session_id}")
+stream = co.chat_stream(message="say hi 2")
+response = ""
+for event in stream:
+ if event.event_type == "text-generation":
+ response += event.text
+ print(event.text, end="")
+ elif event.event_type == "stream-end":
+ print("\n")
+ print(event)
+ print("\n")
- # Initialize client and provider with error handling
- try:
- co = cohere.Client(api_key=api_key)
- aco = cohere.AsyncClient(api_key=api_key)
- from agentops.llms.providers.cohere import CohereProvider
- provider = CohereProvider(co)
- provider.client = session # Pass session to provider before override
- provider.override() # This will handle both sync and async clients
-
- # Set up async client with the same session
- aco.session = session
- # Ensure the async client's provider also has the session
- aco.provider = provider
- print("Successfully initialized Cohere clients and provider")
- print(f"Provider session ID: {provider.client.session_id}")
- except Exception as e:
- print(f"Error initializing Cohere clients: {str(e)}")
- raise
- def sync_no_stream():
- try:
- print("\nExecuting sync_no_stream...")
- response = co.chat(message="Hello from sync no stream", model="command", session=session)
- print(f"sync_no_stream completed successfully with response: {response.text}")
- except Exception as e:
- print(f"Error in sync_no_stream: {str(e)}")
- raise
+async def main():
+ async_response = await aco.chat(message="say hi 3")
+ print(async_response)
- def sync_stream():
- try:
- print("\nExecuting sync_stream...")
- stream = co.chat_stream(message="Hello from sync streaming", model="command", session=session)
- completion = ""
- for chunk in stream:
- if hasattr(chunk, 'text'):
- completion += chunk.text
- print(f"Received sync chunk: {chunk}")
- print(f"sync_stream completed successfully with completion: {completion}")
- except Exception as e:
- print(f"Error in sync_stream: {str(e)}")
- raise
- async def async_no_stream():
- try:
- print("\nExecuting async_no_stream...")
- async with asyncio.timeout(30):
- response = await aco.chat(message="Hello from async no stream", model="command", session=session)
- print(f"async_no_stream completed successfully with response: {response.text}")
- except asyncio.TimeoutError:
- print("Warning: async_no_stream timed out")
- raise
- except Exception as e:
- print(f"Error in async_no_stream: {str(e)}")
- raise
+asyncio.run(main())
- async def async_stream(provider, session):
- try:
- print("\nStarting async_stream call...")
- async with asyncio.timeout(30): # Add timeout to prevent hanging
- # Ensure provider has the current session
- provider.client = session
- # Create a new stream with the provider to ensure proper event tracking
- stream = await aco.chat_stream(
- message="Hello from async streaming",
- model="command",
- session=session
- )
- print("Stream created, starting iteration...")
- async for chunk in stream:
- print(f"Received async chunk: {chunk}")
- print("Stream completed successfully")
- except asyncio.TimeoutError:
- print("Warning: Async stream timed out")
- raise
- except Exception as e:
- print(f"Error in async_stream: {str(e)}")
- raise
+agentops.stop_instrumenting()
- async def run_async_tests():
- print("\nRunning async tests...")
- print("Starting async_no_stream...")
- await async_no_stream()
- print("Completed async_no_stream")
-
- print("\nStarting first async_stream...")
- await async_stream(provider, session)
- print("Completed first async_stream")
-
- print("\nStarting second async_stream...")
- await async_stream(provider, session) # Run twice to ensure we get all LLM calls
- print("Completed second async_stream")
-
- print("\nStarting third async_stream...")
- await async_stream(provider, session) # Run thrice to ensure we get all LLM calls
- print("Completed third async_stream")
-
- print("\nAll async tests completed successfully")
-
- # End session and verify analytics after all tests
- session.end_session("Success")
- analytics = session.get_analytics()
- print(f"\nAnalytics: {analytics}")
- assert analytics["LLM calls"] >= 4, f"Expected at least 4 LLM calls, but got {analytics['LLM calls']}"
+# for cohere, because the client is not a singleton, calling `stop_instrumenting()` only affects
+# new clients created, not existing clients.
- # Call each function with proper error handling
- try:
- sync_no_stream()
- sync_stream()
- asyncio.run(run_async_tests())
- except Exception as e:
- print(f"Error during Cohere test: {str(e)}")
- raise
+co_untracked = cohere.Client()
+chat_untracked = co_untracked.chat(message="say hi untracked")
- print("\nTest completed successfully")
+agentops.end_session(end_state="Success")
-if __name__ == "__main__":
- test_cohere_integration()
+###
+# Used to verify that one session is created with one LLM event
+###
diff --git a/tests/core_manual_tests/providers/groq_canary.py b/tests/core_manual_tests/providers/groq_canary.py
index 0eae7f98..1d91f33c 100644
--- a/tests/core_manual_tests/providers/groq_canary.py
+++ b/tests/core_manual_tests/providers/groq_canary.py
@@ -1,94 +1,60 @@
import asyncio
-import os
+
import agentops
from dotenv import load_dotenv
+import os
from groq import Groq, AsyncGroq
load_dotenv()
-
-def test_groq_integration():
- """Integration test demonstrating all four Groq call patterns:
- 1. Sync (non-streaming)
- 2. Sync (streaming)
- 3. Async (non-streaming)
- 4. Async (streaming)
-
- Verifies that AgentOps correctly tracks all LLM calls via analytics.
- """
- # Initialize AgentOps without auto-starting session
- agentops.init(auto_start_session=False)
- session = agentops.start_session()
-
- # Initialize client and provider
- groq_client = Groq(api_key=os.getenv("GROQ_API_KEY"))
- from agentops.llms.providers.groq import GroqProvider
- provider = GroqProvider(groq_client)
- provider.override()
-
- # Pass session to provider
- provider.client = session
- async_groq_client = AsyncGroq(api_key=os.getenv("GROQ_API_KEY"))
-
- def sync_no_stream():
- groq_client.chat.completions.create(
- model="llama3-70b-8192",
- messages=[
- {"role": "user", "content": "Hello from sync no stream"},
- ],
- session=session
- )
-
- def sync_stream():
- stream_response = groq_client.chat.completions.create(
- model="llama3-70b-8192",
- messages=[
- {"role": "user", "content": "Hello from sync streaming"},
- ],
- stream=True,
- session=session
- )
- for _ in stream_response:
- pass
-
- async def async_no_stream():
- await async_groq_client.chat.completions.create(
- model="llama3-70b-8192",
- messages=[
- {"role": "user", "content": "Hello from async no stream"},
- ],
- session=session
- )
-
- async def async_stream():
- async_stream_response = await async_groq_client.chat.completions.create(
- model="llama3-70b-8192",
- messages=[
- {"role": "user", "content": "Hello from async streaming"},
- ],
- stream=True,
- session=session
- )
- async for _ in async_stream_response:
- pass
-
- async def run_async_tests():
- await async_no_stream()
- await async_stream()
-
- # Call each function with proper error handling
- try:
- sync_no_stream()
- sync_stream()
- asyncio.run(run_async_tests())
- except Exception as e:
- print(f"Error during Groq test: {str(e)}")
- raise
-
- session.end_session("Success")
- analytics = session.get_analytics()
- print(analytics)
- # Verify that all LLM calls were tracked
- assert analytics["LLM calls"] >= 4, f"Expected at least 4 LLM calls, but got {analytics['LLM calls']}"
-
-if __name__ == "__main__":
- test_groq_integration()
+agentops.init(default_tags=["groq-provider-test"])
+groq_client = Groq(api_key=os.getenv("GROQ_API_KEY"))
+async_groq_client = AsyncGroq(api_key=os.getenv("GROQ_API_KEY"))
+
+messages = [{"role": "user", "content": "Hello"}]
+
+# option 1: use session.patch
+res = groq_client.chat.completions.create(
+ model="llama3-70b-8192",
+ messages=[
+ {"role": "user", "content": "Say hello"},
+ ],
+)
+
+stream_res = groq_client.chat.completions.create(
+ model="llama3-70b-8192",
+ messages=[
+ {"role": "user", "content": "Say hello"},
+ ],
+ stream=True,
+)
+
+for chunk in stream_res:
+ print(chunk)
+
+
+async def async_test():
+ async_res = await async_groq_client.chat.completions.create(
+ model="llama3-70b-8192",
+ messages=[
+ {"role": "user", "content": "Say hello"},
+ ],
+ )
+ print(async_res)
+
+
+asyncio.run(async_test())
+
+agentops.stop_instrumenting()
+
+untracked_res = groq_client.chat.completions.create(
+ model="llama3-70b-8192",
+ messages=[
+ {"role": "user", "content": "Say hello"},
+ ],
+)
+
+agentops.end_session(end_state="Success")
+
+###
+# Used to verify that one session is created with one LLM event
+###
diff --git a/tests/core_manual_tests/providers/litellm_canary.py b/tests/core_manual_tests/providers/litellm_canary.py
index 099d3961..0b9cf98c 100644
--- a/tests/core_manual_tests/providers/litellm_canary.py
+++ b/tests/core_manual_tests/providers/litellm_canary.py
@@ -1,88 +1,42 @@
import asyncio
-import os
+
import agentops
from dotenv import load_dotenv
import litellm
load_dotenv()
+agentops.init(default_tags=["litellm-provider-test"])
-def test_litellm_integration():
- """Integration test demonstrating all four LiteLLM call patterns:
- 1. Sync (non-streaming)
- 2. Sync (streaming)
- 3. Async (non-streaming)
- 4. Async (streaming)
+response = litellm.completion(model="gpt-3.5-turbo", messages=[{"content": "Hello, how are you?", "role": "user"}])
- Verifies that AgentOps correctly tracks all LLM calls via analytics.
- """
- # Initialize AgentOps without auto-starting session
- agentops.init(auto_start_session=False)
- session = agentops.start_session()
-
- # Initialize LiteLLM provider
- from agentops.llms.providers.litellm import LiteLLMProvider
- provider = LiteLLMProvider(None) # LiteLLM doesn't need a client
- provider.override()
-
- # Pass session to provider
- provider.client = session
+stream_response = litellm.completion(
+ model="gpt-3.5-turbo",
+ messages=[{"content": "Hello, how are you?", "role": "user"}],
+ stream=True,
+)
+print(stream_response)
+for chunk in stream_response:
+ print(chunk)
- def sync_no_stream():
- litellm.completion(
- model="gpt-3.5-turbo",
- messages=[{"content": "Hello from sync no stream", "role": "user"}],
- session=session
- )
- def sync_stream():
- stream_response = litellm.completion(
- model="gpt-3.5-turbo",
- messages=[{"content": "Hello from sync streaming", "role": "user"}],
- stream=True,
- session=session
- )
- for _ in stream_response:
- pass
+async def main():
+ async_response = await litellm.acompletion(
+ model="gpt-3.5-turbo",
+ messages=[{"content": "Hello, how are you?", "role": "user"}],
+ )
+ print(async_response)
- async def async_no_stream():
- await litellm.acompletion(
- model="gpt-3.5-turbo",
- messages=[{"content": "Hello from async no stream", "role": "user"}],
- session=session
- )
- async def async_stream():
- async_stream_response = await litellm.acompletion(
- model="gpt-3.5-turbo",
- messages=[{"content": "Hello from async streaming", "role": "user"}],
- stream=True,
- session=session
- )
- # Handle streaming response
- if isinstance(async_stream_response, str):
- _ = async_stream_response
- else:
- async for chunk in async_stream_response:
- _ = chunk.choices[0].delta.content if hasattr(chunk.choices[0].delta, "content") else ""
+asyncio.run(main())
- async def run_async_tests():
- await async_no_stream()
- await async_stream()
+agentops.stop_instrumenting()
- # Call each function with proper error handling
- try:
- sync_no_stream()
- sync_stream()
- asyncio.run(run_async_tests())
- except Exception as e:
- print(f"Error during LiteLLM test: {str(e)}")
- raise
+untracked_response = litellm.completion(
+ model="gpt-3.5-turbo", messages=[{"content": "Hello, how are you?", "role": "user"}]
+)
- session.end_session("Success")
- analytics = session.get_analytics()
- print(analytics)
- # Verify that all LLM calls were tracked
- assert analytics["LLM calls"] >= 4, f"Expected at least 4 LLM calls, but got {analytics['LLM calls']}"
+agentops.end_session(end_state="Success")
-if __name__ == "__main__":
- test_litellm_integration()
+###
+# Used to verify that one session is created with one LLM event
+###
diff --git a/tests/core_manual_tests/providers/mistral_canary.py b/tests/core_manual_tests/providers/mistral_canary.py
index 13b77472..9fdfd134 100644
--- a/tests/core_manual_tests/providers/mistral_canary.py
+++ b/tests/core_manual_tests/providers/mistral_canary.py
@@ -1,85 +1,86 @@
import asyncio
-import os
+
import agentops
+import os
from dotenv import load_dotenv
-from mistralai.client import MistralClient
-from mistralai.models.chat_completion import ChatMessage
+from mistralai import Mistral
load_dotenv()
+MISTRAL_API_KEY = os.getenv("MISTRAL_API_KEY")
+
+agentops.init(default_tags=["mistral-provider-test"])
+client = Mistral(MISTRAL_API_KEY)
+
+response = client.chat.complete(
+ model="open-mistral-nemo",
+ messages=[
+ {
+ "role": "user",
+ "content": "Say Hello",
+ },
+ ],
+)
+
+
+stream_response = client.chat.stream(
+ model="open-mistral-nemo",
+ messages=[
+ {
+ "role": "user",
+ "content": "Say Hello again",
+ }
+ ],
+)
+
+response = ""
+for event in stream_response:
+ if event.data.choices[0].finish_reason == "stop":
+ print(response)
+ else:
+ response += event.data.choices[0].delta.content
+
+
+async def async_test():
+ async_response = await client.chat.complete_async(
+ model="open-mistral-nemo",
+ messages=[
+ {
+ "role": "user",
+ "content": "Say Hello in the Hindi language",
+ }
+ ],
+ )
+ print(async_response.choices[0].message.content)
+
+
+async def async_stream_test():
+ async_stream_response = await client.chat.stream_async(
+ model="open-mistral-nemo",
+ messages=[
+ {
+ "role": "user",
+ "content": "Say Hello in the Japanese language",
+ }
+ ],
+ )
+
+ response = ""
+ async for event in async_stream_response:
+ if event.data.choices[0].finish_reason == "stop":
+ print(response)
+ else:
+ response += event.data.choices[0].delta.content
+
+
+async def main():
+ await async_test()
+ await async_stream_test()
+
+
+asyncio.run(main())
+
+agentops.end_session(end_state="Success")
-# Check for required API key
-if not os.getenv("MISTRAL_API_KEY"):
- raise ValueError("MISTRAL_API_KEY environment variable is required")
-
-def test_mistral_integration():
- """Integration test demonstrating all four Mistral call patterns:
- 1. Sync (non-streaming)
- 2. Sync (streaming)
- 3. Async (non-streaming)
- 4. Async (streaming)
-
- Verifies that AgentOps correctly tracks all LLM calls via analytics.
- """
- # Initialize AgentOps without auto-starting session
- agentops.init(auto_start_session=False)
- session = agentops.start_session()
-
- # Initialize client and provider
- client = MistralClient(api_key=os.getenv("MISTRAL_API_KEY"))
- from agentops.llms.providers.mistral import MistralProvider
- provider = MistralProvider(client)
- provider.override()
-
- # Pass session to provider
- provider.client = session
-
- def sync_no_stream():
- client.chat(
- model="mistral-tiny",
- messages=[ChatMessage(role="user", content="Hello from sync no stream")]
- )
-
- async def sync_stream():
- stream_response = await client.chat_stream(
- model="mistral-tiny",
- messages=[ChatMessage(role="user", content="Hello from sync streaming")]
- )
- async for chunk in stream_response:
- _ = chunk.delta.content if hasattr(chunk.delta, "content") else ""
-
- async def async_no_stream():
- # Mistral doesn't have async methods, use sync
- client.chat(
- model="mistral-tiny",
- messages=[ChatMessage(role="user", content="Hello from async no stream")]
- )
-
- async def async_stream():
- stream_response = await client.chat_stream(
- model="mistral-tiny",
- messages=[ChatMessage(role="user", content="Hello from async streaming")]
- )
- async for chunk in stream_response:
- _ = chunk.delta.content if hasattr(chunk.delta, 'content') else ''
-
- async def run_async_tests():
- await async_no_stream()
- await async_stream()
-
- # Call each function with proper error handling
- try:
- sync_no_stream()
- asyncio.run(sync_stream())
- asyncio.run(run_async_tests())
- except Exception as e:
- print(f"Error during Mistral test: {str(e)}")
- raise
-
- session.end_session("Success")
- analytics = session.get_analytics()
- print(analytics)
- # Verify that all LLM calls were tracked
- assert analytics["LLM calls"] >= 4, f"Expected at least 4 LLM calls, but got {analytics['LLM calls']}"
-
-if __name__ == "__main__":
- test_mistral_integration()
+###
+# Used to verify that one session is created with one LLM event
+###
diff --git a/tox.ini b/tox.ini
index 47b25c5d..b4167ae3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,33 +32,15 @@ deps =
langchain
termcolor
python-dotenv
- anthropic>=0.18.1
- cohere>=5.13.4
- groq>=0.4.2
- litellm>=1.30.7
- mistralai==0.4.2
- ai21>=2.1.3
- aioitertools>=0.11.0
-e .
commands =
- coverage run --source . -m pytest tests/core_manual_tests/providers/anthropic_canary.py tests/core_manual_tests/providers/cohere_canary.py tests/core_manual_tests/providers/groq_canary.py tests/core_manual_tests/providers/litellm_canary.py tests/core_manual_tests/providers/mistral_canary.py tests/core_manual_tests/providers/ai21_canary.py -v
+ coverage run --source . -m pytest
coverage report -m
coverage xml
mypy: mypy .
passenv =
OPENAI_API_KEY
AGENTOPS_API_KEY
- ANTHROPIC_API_KEY
- COHERE_API_KEY
- GROQ_API_KEY
- AI21_API_KEY
- MISTRAL_API_KEY
- ANTHROPIC_API_KEY
- COHERE_API_KEY
- AI21_API_KEY
- MISTRAL_API_KEY
- GROQ_API_KEY
- FIREWORKS_API_KEY
[coverage:run]
branch = True