Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI
Python agentagentopsagents-sdkaianthropicautogencost-estimationcrewaievalsevaluation-metricsgroqlangchainllmmistralollamaopenaiopenai-agents
Examples integration tests (#1135)
Alex Reibman18a7037d5056
diff --git a/.github/workflows/examples-integration-test.yml b/.github/workflows/examples-integration-test.ymlnew file mode 100644index 00000000..ccddbd19--- /dev/null+++ b/.github/workflows/examples-integration-test.yml@@ -0,0 +1,192 @@+name: Examples Integration Test++# This workflow runs all example scripts to ensure they work correctly+# and that LLM spans are properly tracked in AgentOps using the+# integrated validation functionality.++on:+ push:+ branches: [ main, develop ]+ paths:+ - 'examples/**/*.py'+ - 'agentops/**'+ - '.github/workflows/examples-integration-test.yml'+ pull_request:+ branches: [ main, develop ]+ paths:+ - 'examples/**/*.py'+ - 'agentops/**'+ - '.github/workflows/examples-integration-test.yml'+ workflow_dispatch:++env:+ PYTHON_VERSION: '3.11'++jobs:+ test-examples:+ runs-on: ubuntu-latest+ timeout-minutes: 30++ strategy:+ fail-fast: false+ matrix:+ example:+ # OpenAI examples+ - { path: 'examples/openai/openai_example_sync.py', name: 'OpenAI Sync' }+ - { path: 'examples/openai/openai_example_async.py', name: 'OpenAI Async' }+ - { path: 'examples/openai/multi_tool_orchestration.py', name: 'OpenAI Multi-Tool' }+ - { path: 'examples/openai/web_search.py', name: 'OpenAI Web Search' }++ # Anthropic examples+ - { path: 'examples/anthropic/anthropic-example-sync.py', name: 'Anthropic Sync' }+ - { path: 'examples/anthropic/anthropic-example-async.py', name: 'Anthropic Async' }+ - { path: 'examples/anthropic/agentops-anthropic-understanding-tools.py', name: 'Anthropic Tools' }++ # LangChain examples+ - { path: 'examples/langchain/langchain_examples.py', name: 'LangChain' }++ # LiteLLM examples+ - { path: 'examples/litellm/litellm_example.py', name: 'LiteLLM' }++ # Google Generative AI examples+ - { path: 'examples/google_genai/gemini_example.py', name: 'Google Gemini' }++ # xAI examples+ - { path: 'examples/xai/grok_examples.py', name: 'xAI Grok' }+ - { path: 'examples/xai/grok_vision_examples.py', name: 'xAI Grok Vision' }++ # CrewAI examples+ - { path: 'examples/crewai/job_posting.py', name: 'CrewAI Job Posting' }+ - { path: 'examples/crewai/markdown_validator.py', name: 'CrewAI Markdown' }++ # AutoGen examples+ - { path: 'examples/autogen/AgentChat.py', name: 'AutoGen Agent Chat' }+ - { path: 'examples/autogen/MathAgent.py', name: 'AutoGen Math Agent' }++ # AG2 examples+ - { path: 'examples/ag2/async_human_input.py', name: 'AG2 Async Human Input' }+ - { path: 'examples/ag2/tools_wikipedia_search.py', name: 'AG2 Wikipedia Search' }++ # Context Manager examples+ - { path: 'examples/context_manager/basic_usage.py', name: 'Context Manager Basic' }+ - { path: 'examples/context_manager/error_handling.py', name: 'Context Manager Errors' }+ - { path: 'examples/context_manager/parallel_traces.py', name: 'Context Manager Parallel' }+ - { path: 'examples/context_manager/production_patterns.py', name: 'Context Manager Production' }++ # Agno examples+ - { path: 'examples/agno/agno_async_operations.py', name: 'Agno Async Operations' }+ - { path: 'examples/agno/agno_basic_agents.py', name: 'Agno Basic Agents' }+ - { path: 'examples/agno/agno_research_team.py', name: 'Agno Research Team' }+ - { path: 'examples/agno/agno_tool_integrations.py', name: 'Agno Tool Integrations' }+ - { path: 'examples/agno/agno_workflow_setup.py', name: 'Agno Workflow Setup' }++ # Google ADK examples+ - { path: 'examples/google_adk/human_approval.py', name: 'Google ADK Human Approval' }++ # LlamaIndex examples+ - { path: 'examples/llamaindex/llamaindex_example.py', name: 'LlamaIndex' }++ # Mem0 examples+ - { path: 'examples/mem0/mem0_memoryclient_example.py', name: 'Mem0 Memory Client' }++ # Watsonx examples+ - { path: 'examples/watsonx/watsonx-streaming.py', name: 'Watsonx Streaming' }+ - { path: 'examples/watsonx/watsonx-text-chat.py', name: 'Watsonx Text Chat' }+ - { path: 'examples/watsonx/watsonx-tokeniation-model.py', name: 'Watsonx Tokenization' }++ # LangGraph examples+ - { path: 'examples/langgraph/langgraph_example.py', name: 'LangGraph' }++ # Smolagents examples+ - { path: 'examples/smolagents/multi_smolagents_system.py', name: 'Smolagents Multi System' }+ - { path: 'examples/smolagents/text_to_sql.py', name: 'Smolagents Text to SQL' }++ # OpenAI Agents examples+ - { path: 'examples/openai_agents/agent_guardrails.py', name: 'OpenAI Agents Guardrails' }+ - { path: 'examples/openai_agents/agent_patterns.py', name: 'OpenAI Agents Patterns' }+ - { path: 'examples/openai_agents/agents_tools.py', name: 'OpenAI Agents Tools' }+ - { path: 'examples/openai_agents/customer_service_agent.py', name: 'OpenAI Agents Customer Service' }++ # Add more examples as needed++ steps:+ - uses: actions/checkout@v4++ - name: Set up Python+ uses: actions/setup-python@v4+ with:+ python-version: ${{ env.PYTHON_VERSION }}++ - name: Install AgentOps+ run: |+ pip install -e .++ - name: Install example dependencies+ run: |+ # Install common dependencies+ pip install python-dotenv requests++ # Install from requirements.txt in the example's directory+ example_dir=$(dirname "${{ matrix.example.path }}")+ if [ -f "$example_dir/requirements.txt" ]; then+ echo "Installing dependencies from $example_dir/requirements.txt"+ pip install -r "$example_dir/requirements.txt"+ else+ echo "No requirements.txt found in $example_dir"+ fi++ - name: Run example - ${{ matrix.example.name }}+ env:+ AGENTOPS_API_KEY: ${{ secrets.AGENTOPS_API_KEY }}+ OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}+ ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}+ GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}+ XAI_API_KEY: ${{ secrets.XAI_API_KEY }}+ WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}+ WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}+ WATSONX_URL: ${{ secrets.WATSONX_URL }}+ MEM0_API_KEY: ${{ secrets.MEM0_API_KEY }}+ GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}+ COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}+ GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}+ FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}+ MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}+ AI21_API_KEY: ${{ secrets.AI21_API_KEY }}+ TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}+ EXA_API_KEY: ${{ secrets.EXA_API_KEY }}+ LLAMA_API_KEY: ${{ secrets.LLAMA_API_KEY }}+ PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }}+ REPLICATE_API_TOKEN: ${{ secrets.REPLICATE_API_TOKEN }}+ PYTHONPATH: ${{ github.workspace }}+ run: |+ echo "Running ${{ matrix.example.name }}..."+ python "${{ matrix.example.path }}" || exit 1++ - name: Check for errors+ if: failure()+ run: |+ echo "Example ${{ matrix.example.name }} failed!"+ echo "Path: ${{ matrix.example.path }}"++ # Show last 50 lines of any log files+ if [ -f agentops.log ]; then+ echo "=== AgentOps Log ==="+ tail -n 50 agentops.log+ fi++ summary:+ needs: test-examples+ runs-on: ubuntu-latest+ if: always()++ steps:+ - name: Summary+ run: |+ echo "## Examples Integration Test Summary" >> $GITHUB_STEP_SUMMARY+ echo "" >> $GITHUB_STEP_SUMMARY++ if [ "${{ needs.test-examples.result }}" == "success" ]; then+ echo "✅ All examples passed!" >> $GITHUB_STEP_SUMMARY+ else+ echo "❌ Some examples failed. Check the logs above." >> $GITHUB_STEP_SUMMARY+ fi\ No newline at end of filediff --git a/agentops/__init__.py b/agentops/__init__.pyindex fc4ceb27..a0308931 100755--- a/agentops/__init__.py+++ b/agentops/__init__.py@@ -34,6 +34,9 @@ from agentops.logging.config import loggerfrom agentops.helpers.deprecation import deprecated, warn_deprecated_paramimport threading+# Import validation functions+from agentops.validation import validate_trace_spans, print_validation_summary, ValidationError+# Thread-safe client management_client_lock = threading.Lock()_client = None@@ -442,37 +445,40 @@ def update_trace_metadata(metadata: Dict[str, Any], prefix: str = "trace.metadat__all__ = [- "init",- "configure",- "get_client",- "record",- "start_trace",- "end_trace",- "update_trace_metadata",+ # Legacy exports"start_session","end_session","track_agent","track_tool","end_all_sessions",+ "Session","ToolEvent","ErrorEvent","ActionEvent","LLMEvent",- "Session",+ # Modern exports+ "init",+ "start_trace",+ "end_trace",+ "update_trace_metadata",+ "Client",+ "get_client",+ # Decorators"trace","session","agent","task","workflow","operation",- "guardrail",- "tracer","tool",- # Trace state enums+ "guardrail",+ # Enums"TraceState","SUCCESS","ERROR","UNSET",- # OpenTelemetry status codes (for advanced users)- "StatusCode",+ # Validation+ "validate_trace_spans",+ "print_validation_summary",+ "ValidationError",]diff --git a/agentops/validation.py b/agentops/validation.pynew file mode 100644index 00000000..7b9cda2e--- /dev/null+++ b/agentops/validation.py@@ -0,0 +1,357 @@+"""+AgentOps Validation Module++This module provides functions to validate that spans have been sent to AgentOps+using the public API. This is useful for testing and verification purposes.+"""++import time+import requests+from typing import Optional, Dict, List, Any, Tuple++from agentops.logging import logger+from agentops.exceptions import ApiServerException+++class ValidationError(Exception):+ """Raised when span validation fails."""++ pass+++def get_jwt_token(api_key: Optional[str] = None) -> str:+ """+ Exchange API key for JWT token.++ Args:+ api_key: Optional API key. If not provided, uses AGENTOPS_API_KEY env var.++ Returns:+ JWT bearer token++ Raises:+ ApiServerException: If token exchange fails+ """+ if api_key is None:+ from agentops import get_client++ client = get_client()+ if client and client.config.api_key:+ api_key = client.config.api_key+ else:+ import os++ api_key = os.getenv("AGENTOPS_API_KEY")+ if not api_key:+ raise ValueError("No API key provided and AGENTOPS_API_KEY environment variable not set")++ try:+ response = requests.post(+ "https://api.agentops.ai/public/v1/auth/access_token", json={"api_key": api_key}, timeout=10+ )+ response.raise_for_status()+ return response.json()["bearer"]+ except requests.exceptions.RequestException as e:+ raise ApiServerException(f"Failed to get JWT token: {e}")+++def get_trace_details(trace_id: str, jwt_token: str) -> Dict[str, Any]:+ """+ Get trace details from AgentOps API.++ Args:+ trace_id: The trace ID to query+ jwt_token: JWT authentication token++ Returns:+ Trace details including spans++ Raises:+ ApiServerException: If API request fails+ """+ try:+ response = requests.get(+ f"https://api.agentops.ai/public/v1/traces/{trace_id}",+ headers={"Authorization": f"Bearer {jwt_token}"},+ timeout=10,+ )+ response.raise_for_status()+ return response.json()+ except requests.exceptions.RequestException as e:+ raise ApiServerException(f"Failed to get trace details: {e}")+++def get_trace_metrics(trace_id: str, jwt_token: str) -> Dict[str, Any]:+ """+ Get trace metrics from AgentOps API.++ Args:+ trace_id: The trace ID to query+ jwt_token: JWT authentication token++ Returns:+ Trace metrics including token counts and costs++ Raises:+ ApiServerException: If API request fails+ """+ try:+ response = requests.get(+ f"https://api.agentops.ai/public/v1/traces/{trace_id}/metrics",+ headers={"Authorization": f"Bearer {jwt_token}"},+ timeout=10,+ )+ response.raise_for_status()+ return response.json()+ except requests.exceptions.RequestException as e:+ raise ApiServerException(f"Failed to get trace metrics: {e}")+++def check_llm_spans(spans: List[Dict[str, Any]]) -> Tuple[bool, List[str]]:+ """+ Check if any LLM spans are present in the trace.++ Args:+ spans: List of span dictionaries++ Returns:+ Tuple of (has_llm_spans, llm_span_names)+ """+ llm_spans = []++ for span in spans:+ span_name = span.get("span_name", "unnamed_span")++ # Check span attributes for LLM span kind+ span_attributes = span.get("span_attributes", {})+ is_llm_span = False++ # If we have span_attributes, check them+ if span_attributes:+ # Try different possible structures for span kind+ span_kind = None++ # Structure 1: span_attributes.agentops.span.kind+ if isinstance(span_attributes, dict):+ agentops_attrs = span_attributes.get("agentops", {})+ if isinstance(agentops_attrs, dict):+ span_info = agentops_attrs.get("span", {})+ if isinstance(span_info, dict):+ span_kind = span_info.get("kind", "")++ # Structure 2: Direct in span_attributes+ if not span_kind and isinstance(span_attributes, dict):+ # Try looking for agentops.span.kind as a flattened key+ span_kind = span_attributes.get("agentops.span.kind", "")++ # Structure 3: Look for SpanAttributes.AGENTOPS_SPAN_KIND+ if not span_kind and isinstance(span_attributes, dict):+ from agentops.semconv import SpanAttributes++ span_kind = span_attributes.get(SpanAttributes.AGENTOPS_SPAN_KIND, "")++ # Check if this is an LLM span by span kind+ is_llm_span = span_kind == "llm"++ # Alternative check: Look for gen_ai.prompt or gen_ai.completion attributes+ # These are standard semantic conventions for LLM spans+ if not is_llm_span and isinstance(span_attributes, dict):+ gen_ai_attrs = span_attributes.get("gen_ai", {})+ if isinstance(gen_ai_attrs, dict):+ # If we have prompt or completion data, it's an LLM span+ if "prompt" in gen_ai_attrs or "completion" in gen_ai_attrs:+ is_llm_span = True++ # Check for LLM_REQUEST_TYPE attribute (used by provider instrumentations)+ if not is_llm_span and isinstance(span_attributes, dict):+ from agentops.semconv import SpanAttributes, LLMRequestTypeValues++ # Check for LLM request type - try both gen_ai.* and llm.* prefixes+ # The instrumentation sets gen_ai.* but the API might return llm.*+ llm_request_type = span_attributes.get(SpanAttributes.LLM_REQUEST_TYPE, "")+ if not llm_request_type:+ # Try the llm.* prefix version+ llm_request_type = span_attributes.get("llm.request.type", "")++ # Check if it's a chat or completion request (the main LLM types)+ if llm_request_type in [LLMRequestTypeValues.CHAT.value, LLMRequestTypeValues.COMPLETION.value]:+ is_llm_span = True++ if is_llm_span:+ llm_spans.append(span_name)++ return len(llm_spans) > 0, llm_spans+++def validate_trace_spans(+ trace_id: Optional[str] = None,+ trace_context: Optional[Any] = None,+ max_retries: int = 10,+ retry_delay: float = 1.0,+ check_llm: bool = True,+ min_spans: int = 1,+ api_key: Optional[str] = None,+) -> Dict[str, Any]:+ """+ Validate that spans have been sent to AgentOps.++ Args:+ trace_id: Direct trace ID to validate+ trace_context: TraceContext object from start_trace (alternative to trace_id)+ max_retries: Maximum number of retries to wait for spans to appear+ retry_delay: Delay between retries in seconds+ check_llm: Whether to specifically check for LLM spans+ min_spans: Minimum number of spans expected+ api_key: Optional API key (uses environment variable if not provided)++ Returns:+ Dictionary containing validation results and metrics++ Raises:+ ValidationError: If validation fails+ ValueError: If neither trace_id nor trace_context is provided+ """+ # Extract trace ID+ if trace_id is None and trace_context is None:+ # Try to get from current span+ try:+ from opentelemetry.trace import get_current_span++ current_span = get_current_span()+ if current_span and hasattr(current_span, "get_span_context"):+ span_context = current_span.get_span_context()+ if hasattr(span_context, "trace_id") and span_context.trace_id:+ if isinstance(span_context.trace_id, int):+ trace_id = format(span_context.trace_id, "032x")+ else:+ trace_id = str(span_context.trace_id)+ except ImportError:+ pass++ elif trace_context is not None and trace_id is None:+ # Extract from TraceContext+ if hasattr(trace_context, "span") and trace_context.span:+ span_context = trace_context.span.get_span_context()+ if hasattr(span_context, "trace_id"):+ if isinstance(span_context.trace_id, int):+ trace_id = format(span_context.trace_id, "032x")+ else:+ trace_id = str(span_context.trace_id)++ if trace_id is None:+ raise ValueError("No trace ID found. Provide either trace_id or trace_context parameter.")++ # Get JWT token+ jwt_token = get_jwt_token(api_key)++ logger.info(f"Validating spans for trace ID: {trace_id}")++ for attempt in range(max_retries):+ try:+ # Get trace details+ trace_details = get_trace_details(trace_id, jwt_token)+ spans = trace_details.get("spans", [])++ if len(spans) >= min_spans:+ logger.info(f"Found {len(spans)} span(s) in trace")++ # Prepare result+ result = {+ "trace_id": trace_id,+ "span_count": len(spans),+ "spans": spans,+ "has_llm_spans": False,+ "llm_span_names": [],+ "metrics": None,+ }++ # Get metrics first - if we have token usage, we definitely have LLM spans+ try:+ metrics = get_trace_metrics(trace_id, jwt_token)+ result["metrics"] = metrics++ if metrics:+ logger.info(+ f"Trace metrics - Total tokens: {metrics.get('total_tokens', 0)}, "+ f"Cost: ${metrics.get('total_cost', '0.0000')}"+ )++ # If we have token usage > 0, we definitely have LLM activity+ if metrics.get("total_tokens", 0) > 0:+ result["has_llm_spans"] = True+ logger.info("LLM activity confirmed via token usage metrics")+ except Exception as e:+ logger.warning(f"Could not retrieve metrics: {e}")++ # Check for LLM spans if requested and not already confirmed via metrics+ if check_llm and not result["has_llm_spans"]:+ has_llm_spans, llm_span_names = check_llm_spans(spans)+ result["has_llm_spans"] = has_llm_spans+ result["llm_span_names"] = llm_span_names++ if has_llm_spans:+ logger.info(f"Found LLM spans: {', '.join(llm_span_names)}")+ else:+ logger.warning("No LLM spans found via attribute inspection")++ # Final validation+ if check_llm and not result["has_llm_spans"]:+ raise ValidationError(+ f"No LLM activity detected in trace {trace_id}. "+ f"Found spans: {[s.get('span_name', 'unnamed') for s in spans]}, "+ f"Token usage: {result.get('metrics', {}).get('total_tokens', 0)}"+ )++ return result++ else:+ logger.debug(+ f"Only {len(spans)} spans found, expected at least {min_spans}. "+ f"Retrying... ({attempt + 1}/{max_retries})"+ )++ except ApiServerException as e:+ logger.debug(f"API error during validation: {e}. Retrying... ({attempt + 1}/{max_retries})")++ if attempt < max_retries - 1:+ time.sleep(retry_delay)++ raise ValidationError(+ f"Validation failed for trace {trace_id} after {max_retries} attempts. "+ f"Expected at least {min_spans} spans"+ + (", including LLM activity" if check_llm else "")+ + ". Please check that tracking is properly configured."+ )+++def print_validation_summary(result: Dict[str, Any]) -> None:+ """+ Print a user-friendly summary of validation results.++ Args:+ result: Validation result dictionary from validate_trace_spans+ """+ print("\n" + "=" * 50)+ print("🔍 AgentOps Span Validation Results")+ print("=" * 50)++ print(f"✅ Found {result['span_count']} span(s) in trace")++ if result.get("has_llm_spans"):+ if result.get("llm_span_names"):+ print(f"✅ Found LLM spans: {', '.join(result['llm_span_names'])}")+ else:+ # LLM activity confirmed via metrics+ print("✅ LLM activity confirmed via token usage metrics")+ elif "has_llm_spans" in result:+ print("⚠️ No LLM activity detected")++ if result.get("metrics"):+ metrics = result["metrics"]+ print("\n📊 Trace Metrics:")+ print(f" - Total tokens: {metrics.get('total_tokens', 0)}")+ print(f" - Prompt tokens: {metrics.get('prompt_tokens', 0)}")+ print(f" - Completion tokens: {metrics.get('completion_tokens', 0)}")+ print(f" - Total cost: ${metrics.get('total_cost', '0.0000')}")++ print("\n✅ Validation successful!")diff --git a/examples/README_TESTING.md b/examples/README_TESTING.mdnew file mode 100644index 00000000..4d9e8105--- /dev/null+++ b/examples/README_TESTING.md@@ -0,0 +1,103 @@+# AgentOps Examples Integration Testing++This directory contains example scripts demonstrating how to use AgentOps with various LLM providers and frameworks. Each example includes automatic validation to ensure that LLM spans are properly tracked by AgentOps.++## What's Being Tested++Each example script now includes automated span validation that:++1. **Runs the example** - Executes the normal example code+2. **Validates span tracking** - Uses the AgentOps integrated validation to verify that:+ - Spans were successfully sent to AgentOps+ - LLM calls were properly instrumented and tracked+ - Token counts and costs were recorded++## How It Works++### 1. Integrated Validation++AgentOps now includes built-in validation functionality (`agentops.validate_trace_spans`) that:+- Exchanges API keys for JWT tokens using the public API+- Queries the AgentOps API for trace and span data+- Validates that expected spans are present+- Retrieves metrics like token usage and costs++### 2. Example Structure++Each example follows this pattern:++```python+import agentops++# Initialize AgentOps+agentops.init()++# Start a trace+tracer = agentops.start_trace("example-name")++# ... perform operations with LLMs ...++# End the trace+agentops.end_trace(tracer, end_state="Success")++# Validate spans were tracked+try:+ result = agentops.validate_trace_spans(trace_context=tracer)+ agentops.print_validation_summary(result)+except agentops.ValidationError as e:+ print(f"❌ Error validating spans: {e}")+ raise+```++### 3. CI/CD Integration++The GitHub Actions workflow (`examples-integration-test.yml`) runs all examples automatically on:+- Push to main/develop branches+- Pull requests+- Manual workflow dispatch++Each example is run in isolation with proper error handling and reporting.++## Running Tests Locally++To run a specific example with validation:++```bash+cd examples/openai+python openai_example_sync.py+```++To run all examples:++```bash+# From the examples directory+for script in $(find . -name "*.py" -type f | grep -v "__pycache__"); do+ echo "Running $script..."+ python "$script"+done+```++## Adding New Examples++When adding a new example:++1. Include the standard validation at the end:+ ```python+ # Validate spans were tracked+ try:+ result = agentops.validate_trace_spans(trace_context=tracer)+ agentops.print_validation_summary(result)+ except agentops.ValidationError as e:+ print(f"❌ Error validating spans: {e}")+ raise+ ```++2. Add the example to the GitHub Actions matrix in `.github/workflows/examples-integration-test.yml`++3. Ensure the example has proper error handling++## Requirements++- Valid `AGENTOPS_API_KEY` environment variable+- API keys for the specific LLM provider being tested+- Python 3.12+ with required dependencies\ No newline at end of filediff --git a/examples/ag2/async_human_input.py b/examples/ag2/async_human_input.pyindex 6484773a..a534b475 100644--- a/examples/ag2/async_human_input.py+++ b/examples/ag2/async_human_input.py@@ -25,7 +25,7 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="AG2 Async Human Input")tracer = agentops.start_trace(trace_name="AG2 Agent chat with Async Human Inputs", tags=["ag2-chat-async-human-inputs", "agentops-example"])@@ -104,3 +104,13 @@ async def main():# await main()agentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/ag2/requirements.txt b/examples/ag2/requirements.txtnew file mode 100644index 00000000..4359d0e3--- /dev/null+++ b/examples/ag2/requirements.txt@@ -0,0 +1,3 @@+ag2+nest-asyncio+wikipedia-api\ No newline at end of filediff --git a/examples/ag2/tools_wikipedia_search.py b/examples/ag2/tools_wikipedia_search.pyindex 95aff730..c5a3c5e9 100644--- a/examples/ag2/tools_wikipedia_search.py+++ b/examples/ag2/tools_wikipedia_search.py@@ -22,7 +22,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_# ### Agent Configuration## Configure an assistant agent and user proxy to be used for LLM recommendation and execution respectively.-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="AG2 Wikipedia Search Tools")tracer = agentops.start_trace(trace_name="AG2 Wikipedia Search Tools", tags=["ag2-wikipedia-search-tools", "agentops-example"])@@ -70,3 +70,13 @@ response = user_proxy.initiate_chat()agentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/agno/agno_async_operations.py b/examples/agno/agno_async_operations.pyindex 5e994e2c..4ab778b0 100644--- a/examples/agno/agno_async_operations.py+++ b/examples/agno/agno_async_operations.py@@ -26,7 +26,7 @@ load_dotenv()os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here")-agentops.init(auto_start_session=False, tags=["agno-example", "async-operation"])+agentops.init(auto_start_session=False, trace_name="Agno Async Operations", tags=["agno-example", "async-operation"])async def demonstrate_async_operations():@@ -75,6 +75,16 @@ async def demonstrate_async_operations():print(f"An error occurred: {e}")agentops.end_trace(tracer, end_state="Error")+ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+if __name__ == "__main__":asyncio.run(demonstrate_async_operations())diff --git a/examples/agno/agno_basic_agents.py b/examples/agno/agno_basic_agents.pyindex ef61a642..25a0cd41 100644--- a/examples/agno/agno_basic_agents.py+++ b/examples/agno/agno_basic_agents.py@@ -35,7 +35,9 @@ load_dotenv()os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_api_key_here")-agentops.init(auto_start_session=False, tags=["agno-example", "basics", "agents-and-teams"])+agentops.init(+ auto_start_session=False, trace_name="Agno Basic Agents", tags=["agno-example", "basics", "agents-and-teams"]+)def demonstrate_basic_agents():@@ -88,6 +90,16 @@ def demonstrate_basic_agents():print(f"An error occurred: {e}")agentops.end_trace(tracer, end_state="Error")+ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+if __name__ == "__main__":demonstrate_basic_agents()diff --git a/examples/agno/agno_research_team.py b/examples/agno/agno_research_team.pyindex c9ddc5e9..8fbb89a7 100644--- a/examples/agno/agno_research_team.py+++ b/examples/agno/agno_research_team.py@@ -67,7 +67,7 @@ from dotenv import load_dotenvload_dotenv()# Initialize AgentOps for monitoring and analytics-agentops.init(auto_start_session=False, tags=["agno-example", "research-team"])+agentops.init(auto_start_session=False, trace_name="Agno Research Team", tags=["agno-example", "research-team"])def demonstrate_research_team():@@ -204,5 +204,15 @@ def demonstrate_research_team():except Exception:agentops.end_trace(tracer, end_state="Error")+ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+demonstrate_research_team()diff --git a/examples/agno/agno_tool_integrations.py b/examples/agno/agno_tool_integrations.pyindex 3a28f0e9..859e92f0 100644--- a/examples/agno/agno_tool_integrations.py+++ b/examples/agno/agno_tool_integrations.py@@ -33,8 +33,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_agentops_ap# Initialize AgentOpsagentops.init(- auto_start_session=False,- tags=["agno-tools", "tool-integration", "demo"]+ auto_start_session=False, trace_name="Agno Tool Integrations", tags=["agno-tools", "tool-integration", "demo"])@@ -56,7 +55,7 @@ def demonstrate_tool_integration():role="Research information using Google Search",model=OpenAIChat(id="gpt-4o-mini"),tools=[GoogleSearchTools()],- instructions="You are a research assistant. Use Google Search to find accurate, up-to-date information."+ instructions="You are a research assistant. Use Google Search to find accurate, up-to-date information.",)response = search_agent.run("What are the latest developments in AI agents?")@@ -70,15 +69,11 @@ def demonstrate_tool_integration():name="Research Agent",role="Comprehensive research using multiple tools",model=OpenAIChat(id="gpt-4o-mini"),- tools=[- GoogleSearchTools(),- ArxivTools(),- DuckDuckGoTools()- ],+ tools=[GoogleSearchTools(), ArxivTools(), DuckDuckGoTools()],instructions="""You are a comprehensive research assistant.Use Google Search for general information, Arxiv for academic papers,and DuckDuckGo as an alternative search engine.- Provide well-researched, balanced information from multiple sources."""+ Provide well-researched, balanced information from multiple sources.""",)response = research_agent.run(@@ -96,12 +91,10 @@ def demonstrate_tool_integration():role="Find and summarize academic papers",model=OpenAIChat(id="gpt-4o-mini"),tools=[ArxivTools()],- instructions="You are an academic research assistant. Use Arxiv to find relevant papers and provide concise summaries."+ instructions="You are an academic research assistant. Use Arxiv to find relevant papers and provide concise summaries.",)- response = academic_agent.run(- "Find recent papers about tool augmented language models"- )+ response = academic_agent.run("Find recent papers about tool augmented language models")print(f"Academic Agent Response:\n{response.content}")# Example 4: Comparing Search Tools@@ -112,13 +105,10 @@ def demonstrate_tool_integration():name="Comparison Agent",role="Compare results from different search engines",model=OpenAIChat(id="gpt-4o-mini"),- tools=[- GoogleSearchTools(),- DuckDuckGoTools()- ],+ tools=[GoogleSearchTools(), DuckDuckGoTools()],instructions="""Compare search results from Google and DuckDuckGo.Note any differences in results, ranking, or information quality.- Be objective in your comparison."""+ Be objective in your comparison.""",)response = comparison_agent.run(@@ -144,6 +134,16 @@ def demonstrate_tool_integration():agentops.end_trace(tracer, end_state="Error")raise+ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+if __name__ == "__main__":demonstrate_tool_integration()diff --git a/examples/agno/agno_workflow_setup.py b/examples/agno/agno_workflow_setup.pyindex d3aa0c9e..42f44d64 100644--- a/examples/agno/agno_workflow_setup.py+++ b/examples/agno/agno_workflow_setup.py@@ -27,7 +27,7 @@ from typing import Iteratorload_dotenv()-agentops.init(auto_start_session=False, tags=["agno-example", "workflow-setup"])+agentops.init(auto_start_session=False, trace_name="Agno Workflow Setup", tags=["agno-example", "workflow-setup"])class CacheWorkflow(Workflow):@@ -113,5 +113,15 @@ def demonstrate_workflows():except Exception:agentops.end_trace(tracer, end_state="Error")+ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+asyncio.run(demonstrate_workflows())diff --git a/examples/agno/requirements.txt b/examples/agno/requirements.txtnew file mode 100644index 00000000..96ab8a51--- /dev/null+++ b/examples/agno/requirements.txt@@ -0,0 +1,2 @@+agno+aiohttp\ No newline at end of filediff --git a/examples/anthropic/agentops-anthropic-understanding-tools.py b/examples/anthropic/agentops-anthropic-understanding-tools.pyindex bbce678f..ed689825 100644--- a/examples/anthropic/agentops-anthropic-understanding-tools.py+++ b/examples/anthropic/agentops-anthropic-understanding-tools.py@@ -17,7 +17,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["ANTHROPIC_API_KEY"] = os.getenv("ANTHROPIC_API_KEY", "your_anthropic_api_key_here")## Now let's set the client as Anthropic and make an AgentOps trace-agentops.init(tags=["anthropic-example-tool-tutorials", "agentops-example"])+agentops.init(trace_name="Anthropic Understanding Tools", tags=["anthropic-example-tool-tutorials", "agentops-example"])client = Anthropic()# Now to create a simple dummy tool! We are going to make a tool that will tell us about the demon infestation levels for 3 areas. From there, we will have VEGA, our AI determine the best place for the Doom Slayer to attack.locations = [@@ -371,3 +371,14 @@ response = client.messages.create(message = response.content[0].textprint(message)+++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/anthropic/anthropic-example-async.py b/examples/anthropic/anthropic-example-async.pyindex f1bff79a..26f6b2b6 100644--- a/examples/anthropic/anthropic-example-async.py+++ b/examples/anthropic/anthropic-example-async.py@@ -22,7 +22,7 @@ os.environ["ANTHROPIC_API_KEY"] = os.getenv("ANTHROPIC_API_KEY", "your_anthropic## Now let's set the client as Anthropic and open an agentops trace!client = Anthropic()-agentops.init(tags=["anthropic-async", "agentops-example"])+agentops.init(trace_name="Anthropic Async Example", tags=["anthropic-async", "agentops-example"])# Now we create three personality presets;## Legion is a relentless and heavy-hitting Titan that embodies brute strength and defensive firepower, Northstar is a precise and agile sniper that excels in long-range combat and flight, while Ronin is a swift and aggressive melee specialist who thrives on close-quarters hit-and-run tactics.@@ -108,3 +108,14 @@ async def main():# Run the main functionasyncio.run(main())# We can observe the trace in the AgentOps dashboard by going to the trace URL provided above.+++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/anthropic/anthropic-example-sync.py b/examples/anthropic/anthropic-example-sync.pyindex f9742996..08db9058 100644--- a/examples/anthropic/anthropic-example-sync.py+++ b/examples/anthropic/anthropic-example-sync.py@@ -21,7 +21,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["ANTHROPIC_API_KEY"] = os.getenv("ANTHROPIC_API_KEY", "your_anthropic_api_key_here")# Now let's set the client as Anthropic and an AgentOps session!client = Anthropic()-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="Anthropic Sync Example")tracer = agentops.start_trace(trace_name="Anthropic Sync Example", tags=["anthropic-example", "agentops-example"])# Remember that story we made earlier? As of writing, claude-3-5-sonnet-20240620 (the version we will be using) has a 150k word, 680k character length. We also get an 8192 context length. This is great because we can actually set an example for the script!#@@ -110,3 +110,13 @@ for event in stream:## Now we will end the session with a success message. We can also end the session with a failure or intdeterminate status. By default, the session will be marked as indeterminate.agentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/anthropic/requirements.txt b/examples/anthropic/requirements.txtnew file mode 100644index 00000000..d5d39b6d--- /dev/null+++ b/examples/anthropic/requirements.txt@@ -0,0 +1 @@+anthropic\ No newline at end of filediff --git a/examples/autogen/AgentChat.py b/examples/autogen/AgentChat.pyindex b0ef8d41..5958e90d 100644--- a/examples/autogen/AgentChat.py+++ b/examples/autogen/AgentChat.py@@ -9,9 +9,6 @@# Then import themimport osfrom dotenv import load_dotenv-from IPython.core.error import (- StdinNotImplementedError,-)import asyncioimport agentops@@ -35,7 +32,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")# When initializing AgentOps, you can pass in optional tags to help filter sessions-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="Autogen Agent Chat Example")tracer = agentops.start_trace(trace_name="Microsoft Agent Chat Example", tags=["autogen-chat", "microsoft-autogen", "agentops-example"])@@ -50,7 +47,7 @@ tracer = agentops.start_trace(# * Errors# # Simple Chat Example# Define model and API key-model_name = "gpt-4-turbo" # Or "gpt-4o" / "gpt-4o-mini" as per migration guide examples+model_name = "gpt-4o-mini" # Or "gpt-4o" / "gpt-4o-mini" as per migration guide examplesapi_key = os.getenv("OPENAI_API_KEY")# Create the model client@@ -82,16 +79,22 @@ async def main():await Console().run(stream)agentops.end_trace(tracer, end_state="Success")- except StdinNotImplementedError:- print("StdinNotImplementedError: This typically happens in non-interactive environments.")- print("Skipping interactive part of chat for automation.")- agentops.end_trace(tracer, end_state="Indeterminate")except Exception as e:print(f"An error occurred: {e}")agentops.end_trace(tracer, end_state="Error")finally:await model_client.close()+ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+if __name__ == "__main__":try:diff --git a/examples/autogen/MathAgent.py b/examples/autogen/MathAgent.pyindex 64348b65..c594f05c 100644--- a/examples/autogen/MathAgent.py+++ b/examples/autogen/MathAgent.py@@ -11,9 +11,6 @@ from typing import Annotated, Literalimport asyncioimport osfrom dotenv import load_dotenv-from IPython.core.error import (- StdinNotImplementedError,-)import agentops@@ -32,7 +29,7 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="Autogen Math Agent Example")tracer = agentops.start_trace(trace_name="Microsoft Autogen Tool Example", tags=["autogen-tool", "microsoft-autogen", "agentops-example"])@@ -97,15 +94,22 @@ async def main():agentops.end_trace(tracer, end_state="Success")- except StdinNotImplementedError:- print("StdinNotImplementedError: This typically happens in non-interactive environments.")- agentops.end_trace(tracer, end_state="Indeterminate")except Exception as e:print(f"An error occurred: {e}")agentops.end_trace(tracer, end_state="Error")finally:await model_client.close()+ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+if __name__ == "__main__":try:diff --git a/examples/autogen/requirements.txt b/examples/autogen/requirements.txtnew file mode 100644index 00000000..032705de--- /dev/null+++ b/examples/autogen/requirements.txt@@ -0,0 +1 @@+pyautogen\ No newline at end of filediff --git a/examples/context_manager/basic_usage.py b/examples/context_manager/basic_usage.pyindex 12299133..94d57c5a 100644--- a/examples/context_manager/basic_usage.py+++ b/examples/context_manager/basic_usage.py@@ -40,10 +40,14 @@ def basic_context_manager_example():print("Basic Context Manager Example")# Initialize AgentOps- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Basic Example",+ tags=["context-manager", "agentops-example"],+ )# Use native TraceContext context manager- with agentops.start_trace("basic_example", tags=["basic", "demo"]):+ with agentops.start_trace("Context Manager Basic Example", tags=["basic", "demo"]):print("Trace started")# Create and use agent@@ -58,10 +62,14 @@ def multiple_parallel_traces():"""Example showing multiple parallel traces."""print("\nMultiple Parallel Traces")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Multiple Parallel Traces",+ tags=["context-manager", "agentops-example"],+ )# First trace- with agentops.start_trace("task_1", tags=["parallel", "task-1"]):+ with agentops.start_trace("Context Manager Task 1", tags=["parallel", "task-1"]):print("Task 1 started")agent1 = SimpleAgent("Agent1")result1 = agent1.process_data("task 1 data")@@ -81,7 +89,11 @@ def error_handling_example():"""Example showing error handling with context manager."""print("\nError Handling Example")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Error Handling",+ tags=["context-manager", "agentops-example"],+ )try:with agentops.start_trace("error_example", tags=["error-handling"]):@@ -103,7 +115,11 @@ def nested_traces_example():"""Example showing nested traces (which are parallel, not parent-child)."""print("\nNested Traces Example")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Nested Traces",+ tags=["context-manager", "agentops-example"],+ )# Outer tracewith agentops.start_trace("main_workflow", tags=["workflow", "main"]):@@ -138,3 +154,13 @@ if __name__ == "__main__":nested_traces_example()print("\nAll examples completed!")++ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/context_manager/error_handling.py b/examples/context_manager/error_handling.pyindex 9f53c3cb..f42d0a8e 100644--- a/examples/context_manager/error_handling.py+++ b/examples/context_manager/error_handling.py@@ -66,13 +66,17 @@ def basic_exception_handling():"""Basic example of exception handling with context managers."""print("Basic Exception Handling")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Basic Exception Handling",+ tags=["context-manager", "error-handling", "agentops-example"],+ )error_types = ["value_error", "type_error", "runtime_error", "success"]for error_type in error_types:try:- with agentops.start_trace(f"basic_{error_type}", tags=["basic", "error-handling"]):+ with agentops.start_trace(f"Context Manager Basic {error_type}", tags=["basic", "error-handling"]):print(f"Started trace for {error_type}")agent = ErrorProneAgent(f"BasicAgent_{error_type}")@@ -93,7 +97,11 @@ def nested_exception_handling():"""Example of exception handling in nested traces."""print("\nNested Exception Handling")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Nested Exception Handling",+ tags=["context-manager", "error-handling", "agentops-example"],+ )try:with agentops.start_trace("outer_operation", tags=["nested", "outer"]):@@ -124,7 +132,11 @@ def retry_pattern():"""Example of retry pattern with context managers."""print("\nRetry Pattern")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Retry Pattern",+ tags=["context-manager", "error-handling", "agentops-example"],+ )max_retries = 3for attempt in range(max_retries):@@ -160,7 +172,11 @@ def graceful_degradation():"""Example of graceful degradation pattern."""print("\nGraceful Degradation")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Graceful Degradation",+ tags=["context-manager", "error-handling", "agentops-example"],+ )try:with agentops.start_trace("primary_service", tags=["degradation", "primary"]):@@ -188,7 +204,11 @@ def partial_success_handling():"""Example of partial success handling."""print("\nPartial Success Handling")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Partial Success Handling",+ tags=["context-manager", "error-handling", "agentops-example"],+ )steps = ["step1", "step2", "fail", "step4"]@@ -210,7 +230,11 @@ def custom_exception_handling():"""Example of handling custom exceptions."""print("\nCustom Exception Handling")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Custom Exception Handling",+ tags=["context-manager", "error-handling", "agentops-example"],+ )try:with agentops.start_trace("custom_exception", tags=["custom", "exception"]):@@ -232,7 +256,11 @@ def finally_blocks_example():"""Example of exception handling with finally blocks."""print("\nFinally Blocks Example")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Finally Blocks",+ tags=["context-manager", "error-handling", "agentops-example"],+ )cleanup_actions = []@@ -263,7 +291,11 @@ def exception_chaining_example():"""Example of exception chaining and context preservation."""print("\nException Chaining Example")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Exception Chaining",+ tags=["context-manager", "error-handling", "agentops-example"],+ )try:with agentops.start_trace("exception_chaining", tags=["chaining", "context"]):@@ -315,3 +347,13 @@ if __name__ == "__main__":exception_chaining_example()print("\nAll error handling examples completed!")++ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/context_manager/parallel_traces.py b/examples/context_manager/parallel_traces.pyindex 1e05a26a..f8436a44 100644--- a/examples/context_manager/parallel_traces.py+++ b/examples/context_manager/parallel_traces.py@@ -51,7 +51,11 @@ def sequential_parallel_traces():"""Example of sequential parallel traces - each trace is independent."""print("Sequential Parallel Traces")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Sequential Parallel Traces",+ tags=["context-manager", "agentops-example"],+ )tasks = ["task_1", "task_2", "task_3"]results = []@@ -75,7 +79,11 @@ def nested_parallel_traces():"""Example showing that nested context managers create parallel traces."""print("\nNested Parallel Traces")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Nested Parallel Traces",+ tags=["context-manager", "agentops-example"],+ )# Outer trace for the overall workflowwith agentops.start_trace("workflow_main", tags=["workflow", "main"]):@@ -110,7 +118,11 @@ def concurrent_traces_with_threads():"""Example of truly concurrent traces using threading."""print("\nConcurrent Traces with Threading")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Concurrent Threads",+ tags=["context-manager", "agentops-example"],+ )def worker_function(worker_id: int, task_data: str):"""Function to run in a separate thread with its own trace."""@@ -149,7 +161,11 @@ def concurrent_traces_with_executor():"""Example using ThreadPoolExecutor for concurrent traces."""print("\nConcurrent Traces with ThreadPoolExecutor")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Concurrent Executor",+ tags=["context-manager", "agentops-example"],+ )def process_with_trace(task_id: int, data: str) -> str:"""Process data within its own trace context."""@@ -195,7 +211,9 @@ def trace_with_different_tag_types():"""Example showing different ways to tag parallel traces."""print("\nTraces with Different Tag Types")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY, trace_name="Context Manager Tag Types", tags=["context-manager", "agentops-example"]+ )# Trace with list tagswith agentops.start_trace("list_tags_trace", tags=["list", "example", "demo"]):diff --git a/examples/context_manager/production_patterns.py b/examples/context_manager/production_patterns.pyindex 8dedc50e..8cb131f6 100644--- a/examples/context_manager/production_patterns.py+++ b/examples/context_manager/production_patterns.py@@ -125,7 +125,11 @@ def api_endpoint_pattern():"""Example of using context managers in API endpoint pattern."""print("API Endpoint Pattern")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager API Endpoint Pattern",+ tags=["context-manager", "production", "agentops-example"],+ )# Simulate API requestsrequests = [@@ -154,7 +158,11 @@ def batch_processing_pattern():"""Example of batch processing with context managers."""print("\nBatch Processing Pattern")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Batch Processing Pattern",+ tags=["context-manager", "production", "agentops-example"],+ )# Simulate batch databatch_data = [{"id": f"item_{i:03d}", "type": "data_record", "payload": {"value": i * 10}} for i in range(1, 6)]@@ -188,7 +196,11 @@ def microservice_pattern():"""Example of microservice communication pattern."""print("\nMicroservice Communication Pattern")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Microservice Pattern",+ tags=["context-manager", "production", "agentops-example"],+ )def authenticate_user(user_id: str) -> bool:"""Simulate authentication service."""@@ -239,7 +251,11 @@ def monitoring_pattern():"""Example of monitoring with context managers."""print("\nMonitoring Pattern")- agentops.init(api_key=AGENTOPS_API_KEY)+ agentops.init(+ api_key=AGENTOPS_API_KEY,+ trace_name="Context Manager Monitoring Pattern",+ tags=["context-manager", "production", "agentops-example"],+ )class AlertManager:"""Simple alert manager for demonstration."""@@ -338,3 +354,13 @@ if __name__ == "__main__":monitoring_pattern()print("\nAll production pattern examples completed!")++ # Let's check programmatically that spans were recorded in AgentOps+ print("\n" + "=" * 50)+ print("Now let's verify that our LLM calls were tracked properly...")+ try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+ except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/context_manager/requirements.txt b/examples/context_manager/requirements.txtnew file mode 100644index 00000000..434f10ac--- /dev/null+++ b/examples/context_manager/requirements.txt@@ -0,0 +1,2 @@+# Context manager examples only need agentops+# python-dotenv and requests are installed as common deps\ No newline at end of filediff --git a/examples/crewai/job_posting.py b/examples/crewai/job_posting.pyindex 3f8d6021..db0067c2 100644--- a/examples/crewai/job_posting.py+++ b/examples/crewai/job_posting.py@@ -20,7 +20,9 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_os.environ["SERPER_API_KEY"] = os.getenv("SERPER_API_KEY", "your_serper_api_key_here")# Initialize AgentOps client-agentops.init(auto_start_session=False)+agentops.init(+ auto_start_session=False, trace_name="CrewAI Job Posting", tags=["crewai", "job-posting", "agentops-example"]+)web_search_tool = WebsiteSearchTool()serper_dev_tool = SerperDevTool()@@ -133,10 +135,10 @@ class Tasks:tracer = agentops.start_trace(trace_name="CrewAI Job Posting", tags=["crew-job-posting-example", "agentops-example"])tasks = Tasks()agents = Agents()-company_description = input("What is the company description?\n")-company_domain = input("What is the company domain?\n")-hiring_needs = input("What are the hiring needs?\n")-specific_benefits = input("What are specific_benefits you offer?\n")+company_description = "We are a software company that builds AI-powered tools for businesses."+company_domain = "https://www.agentops.ai"+hiring_needs = "We are looking for a software engineer with 3 years of experience in Python and Django."+specific_benefits = "We offer a competitive salary, health insurance, and a 401k plan."# Create Agentsresearcher_agent = agents.research_agent()@@ -172,3 +174,13 @@ print("Final Job Posting:")print(result)agentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/crewai/markdown_validator.py b/examples/crewai/markdown_validator.pyindex ea9d3676..1d56cb42 100644--- a/examples/crewai/markdown_validator.py+++ b/examples/crewai/markdown_validator.py@@ -24,7 +24,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")# The first step in any AgentOps integration is to call `agentops.init()`-agentops.init(tags=["markdown_validator", "agentops-example"])+agentops.init(trace_name="CrewAI Markdown Validator", tags=["markdown_validator", "agentops-example"])# Lets start by creating our markdown validator tool@@ -104,3 +104,14 @@ syntax_review_task = Task(# Now lets run our task!syntax_review_task.execute_sync()+++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/crewai/requirements.txt b/examples/crewai/requirements.txtnew file mode 100644index 00000000..f2e17dad--- /dev/null+++ b/examples/crewai/requirements.txt@@ -0,0 +1,3 @@+crewai+crewai-tools+pymarkdown\ No newline at end of filediff --git a/examples/google_adk/human_approval.py b/examples/google_adk/human_approval.pyindex 1cbe55e7..7bf08844 100644--- a/examples/google_adk/human_approval.py+++ b/examples/google_adk/human_approval.py@@ -31,7 +31,12 @@ nest_asyncio.apply()AGENTOPS_API_KEY = os.getenv("AGENTOPS_API_KEY") or "your_agentops_api_key_here"# Initialize AgentOps - Just 2 lines!-agentops.init(AGENTOPS_API_KEY, trace_name="adk-human-approval-notebook", auto_start_session=False)+agentops.init(+ AGENTOPS_API_KEY,+ trace_name="adk-human-approval-notebook",+ auto_start_session=False,+ tags=["google-adk", "human-approval", "agentops-example"],+)# Define some constants for our application.APP_NAME = "human_approval_app_notebook"@@ -209,3 +214,13 @@ try:except Exception as e:print(f"Error: {e}")agentops.end_trace(end_state="Error")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/google_adk/requirements.txt b/examples/google_adk/requirements.txtnew file mode 100644index 00000000..476e7f45--- /dev/null+++ b/examples/google_adk/requirements.txt@@ -0,0 +1,5 @@+google-adk+google-genai+nest-asyncio+agentops+python-dotenv\ No newline at end of filediff --git a/examples/google_genai/gemini_example.py b/examples/google_genai/gemini_example.pyindex 6a845080..62e4f019 100644--- a/examples/google_genai/gemini_example.py+++ b/examples/google_genai/gemini_example.py@@ -15,7 +15,7 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["GEMINI_API_KEY"] = os.getenv("GEMINI_API_KEY", "your_gemini_api_key_here")# Initialize AgentOps and Gemini client-agentops.init(tags=["gemini-example", "agentops-example"])+agentops.init(trace_name="Google Gemini Example", tags=["gemini-example", "agentops-example"])client = genai.Client()# Test synchronous generation@@ -46,3 +46,13 @@ token_response = client.models.count_tokens(model="gemini-1.5-flash", contents="This is a test sentence to count tokens.")print(f"Token count: {token_response.total_tokens}")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/google_genai/requirements.txt b/examples/google_genai/requirements.txtnew file mode 100644index 00000000..c969438c--- /dev/null+++ b/examples/google_genai/requirements.txt@@ -0,0 +1,2 @@+google-generativeai+google-genai\ No newline at end of filediff --git a/examples/langchain/langchain_examples.py b/examples/langchain/langchain_examples.pyindex 33455bbd..1c130788 100644--- a/examples/langchain/langchain_examples.py+++ b/examples/langchain/langchain_examples.py@@ -77,3 +77,17 @@ agent_executor.invoke({"input": "What comedies are playing?"}, config={"callback# ## Check your session# Finally, check your run on [AgentOps](https://app.agentops.ai). You will see a session recorded with the LLM calls and tool usage.++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ import agentops++ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except ImportError:+ print("\n❌ Error: agentops library not installed. Please install it to validate spans.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/langchain/requirements.txt b/examples/langchain/requirements.txtnew file mode 100644index 00000000..5497821c--- /dev/null+++ b/examples/langchain/requirements.txt@@ -0,0 +1,2 @@+langchain+langchain-openai\ No newline at end of filediff --git a/examples/langgraph/langgraph_example.py b/examples/langgraph/langgraph_example.pyindex 2a42b84f..3a393001 100644--- a/examples/langgraph/langgraph_example.py+++ b/examples/langgraph/langgraph_example.py@@ -10,7 +10,11 @@ from dotenv import load_dotenvload_dotenv()-agentops.init(os.getenv("AGENTOPS_API_KEY"))+agentops.init(+ os.getenv("AGENTOPS_API_KEY"),+ trace_name="LangGraph Tool Usage Example",+ tags=["langgraph", "tool-usage", "agentops-example"],+)@tool@@ -116,3 +120,15 @@ def run_example():if __name__ == "__main__":run_example()print("✅ Check your AgentOps dashboard for the trace!")+++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that we have enough spans tracked properly...")+try:+ # LangGraph doesn't emit LLM spans in the same format, so we just check span count+ result = agentops.validate_trace_spans(trace_context=None, check_llm=False, min_spans=5)+ print(f"\n✅ Success! {result['span_count']} spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/langgraph/requirements.txt b/examples/langgraph/requirements.txtnew file mode 100644index 00000000..0450c988--- /dev/null+++ b/examples/langgraph/requirements.txt@@ -0,0 +1,2 @@+langgraph+langchain-openai\ No newline at end of filediff --git a/examples/litellm/litellm_example.py b/examples/litellm/litellm_example.pyindex 8c9126c8..2e4d5102 100644--- a/examples/litellm/litellm_example.py+++ b/examples/litellm/litellm_example.py@@ -28,7 +28,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here") # or the provider of your choosing-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="LiteLLM Example")tracer = agentops.start_trace(trace_name="LiteLLM Example", tags=["litellm-example", "agentops-example"])# Note: AgentOps requires that you call LiteLLM completions differently than the LiteLLM's docs mention@@ -46,7 +46,17 @@ tracer = agentops.start_trace(trace_name="LiteLLM Example", tags=["litellm-examp# litellm.completion()# ```messages = [{"role": "user", "content": "Write a 12 word poem about secret agents."}]-response = litellm.completion(model="gpt-4", messages=messages) # or the model of your choosing+response = litellm.completion(model="gpt-4o-mini", messages=messages) # or the model of your choosingprint(response.choices[0].message.content)agentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/litellm/requirements.txt b/examples/litellm/requirements.txtnew file mode 100644index 00000000..e215fd14--- /dev/null+++ b/examples/litellm/requirements.txt@@ -0,0 +1 @@+litellm\ No newline at end of filediff --git a/examples/llamaindex/llamaindex_example.py b/examples/llamaindex/llamaindex_example.pyindex ed7826a5..d416f163 100644--- a/examples/llamaindex/llamaindex_example.py+++ b/examples/llamaindex/llamaindex_example.py@@ -8,7 +8,7 @@ from llama_index.instrumentation.agentops import AgentOpsHandlerfrom llama_index.embeddings.huggingface import HuggingFaceEmbeddingfrom llama_index.llms.huggingface import HuggingFaceLLM-handler = AgentOpsHandler()+handler = AgentOpsHandler(tags=["llamaindex", "rag", "agentops-example"])handler.init()load_dotenv()@@ -64,3 +64,15 @@ print("\n" + "=" * 50)print("🎉 Example completed successfully!")print("📊 Check your AgentOps dashboard to see the recorded session with LLM calls and operations.")print("🔗 The session link should be printed above by AgentOps.")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ import agentops++ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/llamaindex/requirements.txt b/examples/llamaindex/requirements.txtnew file mode 100644index 00000000..bafc641c--- /dev/null+++ b/examples/llamaindex/requirements.txt@@ -0,0 +1,3 @@+llama-index+llama-index-llms-openai+llama-index-instrumentation-agentops\ No newline at end of filediff --git a/examples/mem0/mem0_memory_example.py b/examples/mem0/mem0_memory_example.pyindex 4a51e79d..6f59f07e 100644--- a/examples/mem0/mem0_memory_example.py+++ b/examples/mem0/mem0_memory_example.py@@ -26,8 +26,8 @@ load_dotenv()os.environ["AGENTOPS_LOG_LEVEL"] = "DEBUG"# Set environment variables before importing-os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY")-os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY")+os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "")+os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "")# Now import mem0 - it will be instrumented by agentopsfrom mem0 import Memory, AsyncMemory # noqa E402@@ -57,7 +57,7 @@ def demonstrate_sync_memory(local_config, sample_messages, sample_preferences, umust complete before the next one begins."""- agentops.start_trace("mem0_memory_example", tags=["mem0_memory_example"])+ tracer = agentops.start_trace("Mem0 Memory Example", tags=["mem0_memory_example"])try:# Initialize sync Memory with local configurationmemory = Memory.from_config(local_config)@@ -96,9 +96,10 @@ def demonstrate_sync_memory(local_config, sample_messages, sample_preferences, udelete_all_result = memory.delete_all(user_id=user_id)print(f"Delete all result: {delete_all_result}")- agentops.end_trace(end_state="success")- except Exception:- agentops.end_trace(end_state="error")+ agentops.end_trace(tracer, end_state="Success")+ except Exception as e:+ agentops.end_trace(tracer, end_state="Error")+ raise easync def demonstrate_async_memory(local_config, sample_messages, sample_preferences, user_id):@@ -122,7 +123,7 @@ async def demonstrate_async_memory(local_config, sample_messages, sample_prefereby running multiple memory operations in parallel."""- agentops.start_trace("mem0_memory_async_example")+ tracer = agentops.start_trace("Mem0 Memory Async Example", tags=["mem0", "async", "memory-management"])try:# Initialize async Memory with configurationasync_memory = await AsyncMemory.from_config(local_config)@@ -177,12 +178,16 @@ async def demonstrate_async_memory(local_config, sample_messages, sample_preferedelete_all_result = await async_memory.delete_all(user_id=user_id)print(f"Delete all result: {delete_all_result}")- agentops.end_trace(end_state="success")+ agentops.end_trace(tracer, end_state="Success")- except Exception:- agentops.end_trace(end_state="error")+ except Exception as e:+ agentops.end_trace(tracer, end_state="Error")+ raise e+# Initialize AgentOps+agentops.init(trace_name="Mem0 Memory Example", tags=["mem0", "memory-management", "agentops-example"])+# Configuration for local memory (Memory)# This configuration specifies the LLM provider and model settingslocal_config = {@@ -222,3 +227,15 @@ sample_preferences = [# Execute both sync and async demonstrationsdemonstrate_sync_memory(local_config, sample_messages, sample_preferences, user_id)asyncio.run(demonstrate_async_memory(local_config, sample_messages, sample_preferences, user_id))++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ # Note: Using trace_id since we ran multiple traces+ # In a real application, you would store each tracer and validate individually+ result = agentops.validate_trace_spans(check_llm=False) # Don't check for LLM spans as this uses memory operations+ agentops.print_validation_summary(result)+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/mem0/mem0_memoryclient_example.py b/examples/mem0/mem0_memoryclient_example.pyindex cfb060cb..74d307ac 100644--- a/examples/mem0/mem0_memoryclient_example.py+++ b/examples/mem0/mem0_memoryclient_example.py@@ -54,7 +54,7 @@ def demonstrate_sync_memory_client(sample_messages, sample_preferences, user_id)Cloud benefit: All memory operations are handled by Mem0's infrastructure,providing scalability and persistence without local storage management."""- agentops.start_trace("mem0_memoryclient_example", tags=["mem0_memoryclient_example"])+ agentops.start_trace("Mem0 MemoryClient Example", tags=["mem0_memoryclient_example"])try:# Initialize sync MemoryClient with API key for cloud accessclient = MemoryClient(api_key=mem0_api_key)@@ -106,7 +106,7 @@ async def demonstrate_async_memory_client(sample_messages, sample_preferences, uconcurrently, significantly reducing total execution time compared to sequential calls.This is especially beneficial when dealing with network I/O to cloud services."""- agentops.start_trace("mem0_memoryclient_example", tags=["mem0_memoryclient_example"])+ agentops.start_trace("Mem0 MemoryClient Async Example", tags=["mem0_memoryclient_example"])try:# Initialize async MemoryClient for concurrent cloud operationsasync_client = AsyncMemoryClient(api_key=mem0_api_key)@@ -183,3 +183,13 @@ sample_preferences = [# Note: The async version typically completes faster due to concurrent operationsdemonstrate_sync_memory_client(sample_messages, sample_preferences, user_id)asyncio.run(demonstrate_async_memory_client(sample_messages, sample_preferences, user_id))++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/mem0/requirements.txt b/examples/mem0/requirements.txtnew file mode 100644index 00000000..2713cbc8--- /dev/null+++ b/examples/mem0/requirements.txt@@ -0,0 +1 @@+mem0ai\ No newline at end of filediff --git a/examples/openai/multi_tool_orchestration.py b/examples/openai/multi_tool_orchestration.pyindex 552751b8..f3586a75 100644--- a/examples/openai/multi_tool_orchestration.py+++ b/examples/openai/multi_tool_orchestration.py@@ -28,9 +28,13 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")os.environ["PINECONE_API_KEY"] = os.getenv("PINECONE_API_KEY", "your_pinecone_api_key_here")-agentops.init(auto_start_session=True)+agentops.init(+ auto_start_session=True,+ trace_name="OpenAI Multi-Tool Orchestration",+ tags=["openai", "multi-tool", "agentops-example"],+)tracer = agentops.start_trace(- trace_name="Multi-Tool Orchestration with RAG",+ trace_name="OpenAI Multi-Tool Orchestration with RAG",tags=["multi-tool-orchestration-rag-demo", "openai-responses", "agentops-example"],)client = OpenAI()@@ -342,4 +346,15 @@ print(response_2)print(response_2.output_text)agentops.end_trace(tracer, end_state="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise++# Here, we have seen how to utilize OpenAI's Responses API to implement a Retrieval-Augmented Generation (RAG) approach with multi-tool calling capabilities. It showcases an example where the model selects the appropriate tool based on the input query: general questions may be handled by built-in tools such as web-search, while specific medical inquiries related to internal knowledge are addressed by retrieving context from a vector database (such as Pinecone) via function calls. Additonally, we have showcased how multiple tool calls can be sequentially combined to generate a final response based on our instructions provided to responses API. Happy coding!diff --git a/examples/openai/openai_example_async.py b/examples/openai/openai_example_async.pyindex 1e32f52b..0602f72f 100644--- a/examples/openai/openai_example_async.py+++ b/examples/openai/openai_example_async.py@@ -21,7 +21,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")# Next we initialize the AgentOps client.-agentops.init(auto_start_session=True)+agentops.init(auto_start_session=True, trace_name="OpenAI Async Example", tags=["openai", "async", "agentops-example"])tracer = agentops.start_trace(trace_name="OpenAI Async Example", tags=["openai-async-example", "openai", "agentops-example"])@@ -37,7 +37,10 @@ You are given a prompt and you need to generate a story based on the prompt."""user_prompt = [- {"type": "text", "text": "Write a mystery thriller story based on your understanding of the provided image."},+ {+ "type": "text",+ "text": "Write a very short mystery thriller story based on your understanding of the provided image.",+ },{"type": "image_url","image_url": {"url": "https://www.cosy.sbg.ac.at/~pmeerw/Watermarking/lena_color.gif"},@@ -78,5 +81,16 @@ async def main_stream():asyncio.run(main_stream())agentops.end_trace(tracer, end_state="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise++# Note that the response is a generator that yields chunks of the story. We can track this with AgentOps by navigating to the trace url and viewing the run.# All done!diff --git a/examples/openai/openai_example_sync.py b/examples/openai/openai_example_sync.pyindex 6b535251..9c7f49cd 100644--- a/examples/openai/openai_example_sync.py+++ b/examples/openai/openai_example_sync.py@@ -20,7 +20,7 @@ os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")# Next we initialize the AgentOps client.-agentops.init(auto_start_session=True)+agentops.init(auto_start_session=True, trace_name="OpenAI Sync Example", tags=["openai", "sync", "agentops-example"])tracer = agentops.start_trace(trace_name="OpenAI Sync Example", tags=["openai-sync-example", "openai", "agentops-example"])@@ -35,7 +35,7 @@ You have experience in writing for children and adults alike.You are given a prompt and you need to generate a story based on the prompt."""-user_prompt = "Write a story about a cyber-warrior trapped in the imperial time period."+user_prompt = "Write a very short story about a cyber-warrior trapped in the imperial time period."messages = [{"role": "system", "content": system_prompt},@@ -64,5 +64,15 @@ for chunk in stream:agentops.end_trace(tracer, end_state="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ result = agentops.validate_trace_spans(trace_context=tracer)+ agentops.print_validation_summary(result)+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+# Note that the response is a generator that yields chunks of the story. We can track this with AgentOps by navigating to the trace url and viewing the run.# All done!diff --git a/examples/openai/requirements.txt b/examples/openai/requirements.txtnew file mode 100644index 00000000..f066e4c3--- /dev/null+++ b/examples/openai/requirements.txt@@ -0,0 +1,3 @@+openai+pandas # for multi_tool_orchestration.py+tavily-python # for web_search.py\ No newline at end of filediff --git a/examples/openai/web_search.py b/examples/openai/web_search.pyindex d632e0cc..627d79ce 100644--- a/examples/openai/web_search.py+++ b/examples/openai/web_search.py@@ -26,7 +26,9 @@ load_dotenv()os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")-agentops.init(auto_start_session=True)+agentops.init(+ auto_start_session=True, trace_name="OpenAI Web Search Example", tags=["openai", "web-search", "agentops-example"]+)tracer = agentops.start_trace(trace_name="OpenAI Responses Example", tags=["openai-responses-example", "openai", "agentops-example"])@@ -101,6 +103,17 @@ response_multimodal = client.responses.create(print(json.dumps(response_multimodal.__dict__, default=lambda o: o.__dict__, indent=4))agentops.end_trace(tracer, end_state="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise++# In the above example, we were able to use the `web_search` tool to search the web for news related to the image in one API call instead of multiple round trips that would be required if we were using the Chat Completions API.# With the responses API# 🔥 a single API call can handle:diff --git a/examples/openai_agents/agent_guardrails.py b/examples/openai_agents/agent_guardrails.pyindex 2429dab4..ebfa32ac 100644--- a/examples/openai_agents/agent_guardrails.py+++ b/examples/openai_agents/agent_guardrails.py@@ -33,7 +33,7 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")-agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], tags=["agentops-example"])+agentops.init(api_key=os.environ["AGENTOPS_API_KEY"], trace_name="OpenAI Agents Guardrails", tags=["agentops-example"])# OpenAI Agents SDK guardrail example with agentops guardrails decorator for observability@@ -81,3 +81,14 @@ async def main():if __name__ == "__main__":asyncio.run(main())+++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/openai_agents/agent_patterns.py b/examples/openai_agents/agent_patterns.pyindex 5482fc7b..3e6af902 100644--- a/examples/openai_agents/agent_patterns.py+++ b/examples/openai_agents/agent_patterns.py@@ -78,7 +78,11 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")# Initialize AgentOps-agentops.init(auto_start_session=False)+agentops.init(+ auto_start_session=False,+ trace_name="OpenAI Agents Patterns",+ tags=["openai-agents", "patterns", "agentops-example"],+)# Note: tracer will be defined in each section's cell for clarity, using the specific tags for that pattern.# ## 1. Agents as Tools Pattern#@@ -88,7 +92,7 @@ agentops.init(auto_start_session=False)## This pattern demonstrates using agents as callable tools within other agents. The orchestrator agent receives a user message and then picks which specialized agents to call as tools.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Agents as Tools Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(trace_name="OpenAI Agents as Tools Pattern", tags=["agentops-example", "openai-agents"])# Define specialized translation agentsspanish_agent = Agent(@@ -174,7 +178,9 @@ agentops.end_trace(tracer, end_state="Success")## This pattern demonstrates breaking down a complex task into a series of smaller, sequential steps. Each step is performed by an agent, and the output of one agent is used as input to the next.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Deterministic Flow Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(+ trace_name="OpenAI Agents Deterministic Flow Pattern", tags=["agentops-example", "openai-agents"]+)# Define the story generation workflowstory_outline_agent = Agent(@@ -246,7 +252,9 @@ agentops.end_trace(tracer, end_state="Success")## For this demo, we'll allow the user to choose which tool use behavior to test:# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Forcing Tool Use Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(+ trace_name="OpenAI Agents Forcing Tool Use Pattern", tags=["agentops-example", "openai-agents"]+)# Define the weather tool and agent@@ -325,7 +333,9 @@ agentops.end_trace(tracer, end_state="Success")## This pattern demonstrates how to use input guardrails to validate user inputs before they reach the main agent. Guardrails can prevent inappropriate or off-topic requests from being processed.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Input Guardrails Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(+ trace_name="OpenAI Agents Input Guardrails Pattern", tags=["agentops-example", "openai-agents"]+)# Define the guardrail@@ -388,7 +398,9 @@ agentops.end_trace(tracer, end_state="Success")## This pattern shows how to use one LLM to evaluate and improve the output of another. The first agent generates content, and the second agent judges the quality and provides feedback for improvement.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="LLM as a Judge Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(+ trace_name="OpenAI Agents LLM as a Judge Pattern", tags=["agentops-example", "openai-agents"]+)# Define the story generation and evaluation agentsstory_outline_generator = Agent(@@ -468,7 +480,9 @@ agentops.end_trace(tracer, end_state="Success")## This pattern demonstrates how to use output guardrails to validate agent outputs after they are generated. This can help prevent sensitive information from being shared or ensure outputs meet quality standards.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Output Guardrails Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(+ trace_name="OpenAI Agents Output Guardrails Pattern", tags=["agentops-example", "openai-agents"]+)# The agent's output type@@ -535,7 +549,9 @@ agentops.end_trace(tracer, end_state="Success")## This pattern shows how to run multiple agents in parallel to improve latency or generate multiple options to choose from. In this example, we run translation agents multiple times and pick the best result.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Output Guardrails Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(+ trace_name="OpenAI Agents Parallelization Pattern", tags=["agentops-example", "openai-agents"]+)# Define agents for parallelizationspanish_translation_agent = Agent(@@ -592,7 +608,7 @@ agentops.end_trace(tracer, end_state="Success")## This pattern demonstrates handoffs and routing between specialized agents. The triage agent receives the first message and hands off to the appropriate agent based on the language of the request.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Routing Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(trace_name="OpenAI Agents Routing Pattern", tags=["agentops-example", "openai-agents"])# Define language-specific agentsfrench_routing_agent = Agent(@@ -645,7 +661,9 @@ agentops.end_trace(tracer, end_state="Success")## This pattern shows how to use guardrails during streaming to provide real-time validation. Unlike output guardrails that run after completion, streaming guardrails can interrupt the generation process early.# Start the AgentOps trace session-tracer = agentops.start_trace(trace_name="Streaming Guardrails Pattern", tags=["agentops-example", "openai-agents"])+tracer = agentops.start_trace(+ trace_name="OpenAI Agents Streaming Guardrails Pattern", tags=["agentops-example", "openai-agents"]+)# Define streaming guardrail agentstreaming_agent = Agent(@@ -723,6 +741,17 @@ if __name__ == "__main__":# End the AgentOps trace sessionagentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+# ## Conclusion## This notebook has demonstrated 9 key agent patterns that are commonly used in production AI applications. Each pattern showcases how agents can be orchestrated to perform complex tasks, validate inputs and outputs, and improve overall application performance.diff --git a/examples/openai_agents/agents_tools.py b/examples/openai_agents/agents_tools.pyindex 71c8644e..ed01e752 100644--- a/examples/openai_agents/agents_tools.py+++ b/examples/openai_agents/agents_tools.py@@ -48,7 +48,11 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")-agentops.init(auto_start_session=False, tags=["agentops-example"])+agentops.init(+ auto_start_session=False,+ trace_name="OpenAI Agents Tools Examples",+ tags=["openai-agents", "tools", "agentops-example"],+)# ## 1. Code Interpreter Tool#@@ -61,7 +65,7 @@ agentops.init(auto_start_session=False, tags=["agentops-example"])# - Handle data processing tasks# Start the AgentOps trace sessiontracer = agentops.start_trace(- trace_name="Code Interpreter Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"]+ trace_name="OpenAI Agents Code Interpreter Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"])@@ -111,7 +115,7 @@ agentops.end_trace(tracer, end_state="Success")# **Note:** This example requires a pre-configured vector store ID.# Start the AgentOps trace sessiontracer = agentops.start_trace(- trace_name="File Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"]+ trace_name="OpenAI Agents File Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"])@@ -157,7 +161,7 @@ agentops.end_trace(tracer, end_state="Success")# - Automatic image saving and display# Start the AgentOps trace sessiontracer = agentops.start_trace(- trace_name="Image Generation Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"]+ trace_name="OpenAI Agents Image Generation Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"])@@ -222,7 +226,7 @@ agentops.end_trace(tracer, end_state="Success")# - Configurable search parameters# Start the AgentOps trace sessiontracer = agentops.start_trace(- trace_name="Web Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"]+ trace_name="OpenAI Agents Web Search Tool Example", tags=["tools-demo", "openai-agents", "agentops-example"])@@ -248,6 +252,17 @@ async def run_web_search_demo():# End the AgentOps trace sessionagentops.end_trace(tracer, end_state="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise++# ## Conclusion## Each tool extends agent capabilities and enables sophisticated automation. **AgentOps makes tool observability effortless** - simply import the library and all your tool interactions are automatically tracked, visualized, and analyzed. This enables you to:diff --git a/examples/openai_agents/customer_service_agent.py b/examples/openai_agents/customer_service_agent.pyindex 92fa34bb..dffad335 100644--- a/examples/openai_agents/customer_service_agent.py+++ b/examples/openai_agents/customer_service_agent.py@@ -53,8 +53,12 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")-agentops.init(tags=["customer-service-agent", "openai-agents", "agentops-example"], auto_start_session=False)-tracer = agentops.start_trace(trace_name="Customer Service Agent")+agentops.init(+ trace_name="OpenAI Agents Customer Service",+ tags=["customer-service-agent", "openai-agents", "agentops-example"],+ auto_start_session=False,+)+tracer = agentops.start_trace(trace_name="OpenAI Agents Customer Service Agent")# Context model for the airline agent@@ -102,13 +106,13 @@ async def update_seat(context: RunContextWrapper[AirlineAgentContext], confirmatreturn f"Updated seat to {new_seat} for confirmation number {confirmation_number}"-### HOOKS+# HOOKSasync def on_seat_booking_handoff(context: RunContextWrapper[AirlineAgentContext]) -> None:flight_number = f"FLT-{random.randint(100, 999)}"context.context.flight_number = flight_number-### AGENTS+# AGENTSfaq_agent = Agent[AirlineAgentContext](name="FAQ Agent",handoff_description="A helpful agent that can answer questions about the airline.",@@ -187,6 +191,17 @@ async def main():# await main()agentops.end_trace(tracer, status="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise++# ## Conclusion## **AgentOps makes observability effortless** - simply import the library and all your interactions are automatically tracked, visualized, and analyzed. This enables you to:diff --git a/examples/openai_agents/requirements.txt b/examples/openai_agents/requirements.txtnew file mode 100644index 00000000..11a0a416--- /dev/null+++ b/examples/openai_agents/requirements.txt@@ -0,0 +1,2 @@+openai+openai-agents\ No newline at end of filediff --git a/examples/smolagents/multi_smolagents_system.py b/examples/smolagents/multi_smolagents_system.pyindex c14740b3..acbb6965 100644--- a/examples/smolagents/multi_smolagents_system.py+++ b/examples/smolagents/multi_smolagents_system.py@@ -47,9 +47,10 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="Smolagents Multi-Agent System")tracer = agentops.start_trace(- trace_name="Orchestrate a Multi-Agent System", tags=["smolagents", "example", "multi-agent", "agentops-example"]+ trace_name="Smolagents Multi-Agent System Orchestration",+ tags=["smolagents", "example", "multi-agent", "agentops-example"],)model = LiteLLMModel("openai/gpt-4o-mini")# ## Create a Web Search Tool@@ -82,7 +83,7 @@ def visit_webpage(url: str) -> str:except RequestException as e:return f"Error fetching the webpage: {str(e)}"- except Exception as e:+ except agentops.ValidationError as e:return f"An unexpected error occurred: {str(e)}"@@ -111,4 +112,15 @@ answer = manager_agent.run(print(answer)# Awesome! We've successfully run a multi-agent system. Let's end the agentops session with a "Success" state. You can also end the session with a "Failure" or "Indeterminate" state, which is set as default.agentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+# You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session.diff --git a/examples/smolagents/requirements.txt b/examples/smolagents/requirements.txtnew file mode 100644index 00000000..9a3d792c--- /dev/null+++ b/examples/smolagents/requirements.txt@@ -0,0 +1,4 @@+smolagents+pandas+duckduckgo-search+sqlalchemy\ No newline at end of filediff --git a/examples/smolagents/text_to_sql.py b/examples/smolagents/text_to_sql.pyindex 10eeefa6..32282b6f 100644--- a/examples/smolagents/text_to_sql.py+++ b/examples/smolagents/text_to_sql.py@@ -98,9 +98,9 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")os.environ["OPENAI_API_KEY"] = os.getenv("OPENAI_API_KEY", "your_openai_api_key_here")-agentops.init(auto_start_session=False)+agentops.init(auto_start_session=False, trace_name="Smolagents Text-to-SQL")tracer = agentops.start_trace(- trace_name="Text-to-SQL", tags=["smolagents", "example", "text-to-sql", "agentops-example"]+ trace_name="Smolagents Text-to-SQL", tags=["smolagents", "example", "text-to-sql", "agentops-example"])model = LiteLLMModel("openai/gpt-4o-mini")agent = CodeAgent(@@ -156,4 +156,15 @@ agent = CodeAgent(agent.run("Which waiter got more total money from tips?")# All done! Now we can end the agentops session with a "Success" state. You can also end the session with a "Failure" or "Indeterminate" state, where the "Indeterminate" state is used by default.agentops.end_trace(tracer, end_state="Success")++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise+# You can view the session in the [AgentOps dashboard](https://app.agentops.ai/sessions) by clicking the link provided after ending the session.diff --git a/examples/watsonx/requirements.txt b/examples/watsonx/requirements.txtnew file mode 100644index 00000000..b3e3ac41--- /dev/null+++ b/examples/watsonx/requirements.txt@@ -0,0 +1 @@+ibm-watsonx-ai\ No newline at end of filediff --git a/examples/watsonx/watsonx-streaming.py b/examples/watsonx/watsonx-streaming.pyindex a78214f4..c62d2732 100644--- a/examples/watsonx/watsonx-streaming.py+++ b/examples/watsonx/watsonx-streaming.py@@ -15,7 +15,7 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")# Initialize AgentOps-agentops.init(tags=["watsonx-streaming", "agentops-example"])+agentops.init(trace_name="WatsonX Streaming Example", tags=["watsonx-streaming", "agentops-example"])# ## Initialize IBM Watsonx AI Credentials## To use IBM Watsonx AI, you need to set up your credentials and project ID.@@ -117,3 +117,13 @@ print(full_chat_stream_response)# Close connectionsgen_model.close_persistent_connection()chat_model.close_persistent_connection()++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/watsonx/watsonx-text-chat.py b/examples/watsonx/watsonx-text-chat.pyindex c2139158..af49599a 100644--- a/examples/watsonx/watsonx-text-chat.py+++ b/examples/watsonx/watsonx-text-chat.py@@ -15,7 +15,7 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")# Initialize AgentOps-agentops.init(tags=["watsonx-text-chat", "agentops-example"])+agentops.init(trace_name="WatsonX Text Chat Example", tags=["watsonx-text-chat", "agentops-example"])# ## Initialize IBM Watsonx AI Credentials## To use IBM Watsonx AI, you need to set up your credentials and project ID.@@ -76,3 +76,13 @@ print(f"Chat Response:\\n{chat_response['choices'][0]['message']['content']}")# Close connectionsgen_model.close_persistent_connection()chat_model.close_persistent_connection()++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/watsonx/watsonx-tokeniation-model.py b/examples/watsonx/watsonx-tokeniation-model.pyindex 90d8210d..4d039e52 100644--- a/examples/watsonx/watsonx-tokeniation-model.py+++ b/examples/watsonx/watsonx-tokeniation-model.py@@ -15,7 +15,7 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")# Initialize AgentOps-agentops.init(tags=["watsonx-tokenization", "agentops-example"])+agentops.init(trace_name="WatsonX Tokenization Model Example", tags=["watsonx-tokenization", "agentops-example"])# ## Initialize IBM Watsonx AI Credentials## To use IBM Watsonx AI, you need to set up your credentials and project ID.@@ -95,3 +95,13 @@ print(f"\\nLlama tokens: {llama_tokens}")# Close connectionsmodel.close_persistent_connection()llama_model.close_persistent_connection()++# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=None)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raisediff --git a/examples/xai/grok_examples.py b/examples/xai/grok_examples.pyindex 43718a2c..860f93b0 100644--- a/examples/xai/grok_examples.py+++ b/examples/xai/grok_examples.py@@ -17,8 +17,8 @@ load_dotenv()os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_here")# Next we initialize the AgentOps client.-agentops.init(auto_start_session=False)-tracer = agentops.start_trace(trace_name="XAI Example", tags=["xai-example", "grok", "agentops-example"])+agentops.init(auto_start_session=False, trace_name="XAI Grok Example", tags=["xai", "grok", "agentops-example"])+tracer = agentops.start_trace(trace_name="XAI Grok Example", tags=["xai-example", "grok", "agentops-example"])# And we are all set! Note the seesion url above. We will use it to track the chatbot.#@@ -77,4 +77,15 @@ print(f"Response in {native_language}:\n{generated_response}")# Awesome! We can now transliterate from English to any language! And all of this can be tracked with AgentOps by going to the session url above.agentops.end_trace(tracer, end_state="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise++# We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state.diff --git a/examples/xai/grok_vision_examples.py b/examples/xai/grok_vision_examples.pyindex 3f620d63..151c3bdf 100644--- a/examples/xai/grok_vision_examples.py+++ b/examples/xai/grok_vision_examples.py@@ -18,8 +18,12 @@ os.environ["AGENTOPS_API_KEY"] = os.getenv("AGENTOPS_API_KEY", "your_api_key_heros.environ["XAI_API_KEY"] = os.getenv("XAI_API_KEY", "your_xai_api_key_here")# Next we initialize the AgentOps client.-agentops.init(auto_start_session=False)-tracer = agentops.start_trace(trace_name="XAI Vision Example", tags=["xai-example", "grok-vision", "agentops-example"])+agentops.init(+ auto_start_session=False, trace_name="XAI Grok Vision Example", tags=["xai", "grok-vision", "agentops-example"]+)+tracer = agentops.start_trace(+ trace_name="XAI Grok Vision Example", tags=["xai-example", "grok-vision", "agentops-example"]+)# And we are all set! Note the seesion url above. We will use it to track the program's performance.#@@ -60,4 +64,15 @@ print(response.choices[0].message.content)# Awesome! It returns a fascinating response explaining the image and also deciphering the text content. All of this can be tracked with AgentOps by going to the session url above.agentops.end_trace(tracer, end_state="Success")+# Let's check programmatically that spans were recorded in AgentOps+print("\n" + "=" * 50)+print("Now let's verify that our LLM calls were tracked properly...")+try:+ agentops.validate_trace_spans(trace_context=tracer)+ print("\n✅ Success! All LLM spans were properly recorded in AgentOps.")+except agentops.ValidationError as e:+ print(f"\n❌ Error validating spans: {e}")+ raise++# We end the session with a success state and a success reason. This is useful if you want to track the success or failure of the chatbot. In that case you can set the end state to failure and provide a reason. By default the session will have an indeterminate end state.diff --git a/examples/xai/requirements.txt b/examples/xai/requirements.txtnew file mode 100644index 00000000..85a3cb06--- /dev/null+++ b/examples/xai/requirements.txt@@ -0,0 +1 @@+openai # xAI uses OpenAI client\ No newline at end of filediff --git a/tests/unit/test_validation.py b/tests/unit/test_validation.pynew file mode 100644index 00000000..ac717358--- /dev/null+++ b/tests/unit/test_validation.py@@ -0,0 +1,405 @@+"""+Unit tests for the AgentOps validation module.+"""++import pytest+from unittest.mock import patch, Mock+import requests++from agentops.validation import (+ get_jwt_token,+ get_trace_details,+ check_llm_spans,+ validate_trace_spans,+ ValidationError,+ print_validation_summary,+)+from agentops.exceptions import ApiServerException+++class TestGetJwtToken:+ """Test JWT token exchange functionality."""++ @patch("agentops.validation.requests.post")+ def test_get_jwt_token_success(self, mock_post):+ """Test successful JWT token retrieval."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"bearer": "test-token"}+ mock_post.return_value = mock_response++ token = get_jwt_token("test-api-key")+ assert token == "test-token"++ mock_post.assert_called_once_with(+ "https://api.agentops.ai/public/v1/auth/access_token", json={"api_key": "test-api-key"}, timeout=10+ )++ @patch("agentops.validation.requests.post")+ def test_get_jwt_token_failure(self, mock_post):+ """Test JWT token retrieval failure."""+ mock_response = Mock()+ mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("401 Unauthorized")+ mock_post.return_value = mock_response++ with pytest.raises(ApiServerException, match="Failed to get JWT token"):+ get_jwt_token("invalid-api-key")++ @patch("os.getenv")+ @patch("agentops.get_client")+ @patch("agentops.validation.requests.post")+ def test_get_jwt_token_from_env(self, mock_post, mock_get_client, mock_getenv):+ """Test JWT token retrieval using environment variable."""+ mock_get_client.return_value = None+ mock_getenv.return_value = "env-api-key"++ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"bearer": "env-token"}+ mock_post.return_value = mock_response++ token = get_jwt_token()+ assert token == "env-token"++ mock_getenv.assert_called_once_with("AGENTOPS_API_KEY")+++class TestGetTraceDetails:+ """Test trace details retrieval."""++ @patch("agentops.validation.requests.get")+ def test_get_trace_details_success(self, mock_get):+ """Test successful trace details retrieval."""+ mock_response = Mock()+ mock_response.status_code = 200+ mock_response.json.return_value = {"trace_id": "test-trace", "spans": [{"span_name": "test-span"}]}+ mock_get.return_value = mock_response++ details = get_trace_details("test-trace", "test-token")+ assert details["trace_id"] == "test-trace"+ assert len(details["spans"]) == 1++ mock_get.assert_called_once_with(+ "https://api.agentops.ai/public/v1/traces/test-trace",+ headers={"Authorization": "Bearer test-token"},+ timeout=10,+ )++ @patch("agentops.validation.requests.get")+ def test_get_trace_details_failure(self, mock_get):+ """Test trace details retrieval failure."""+ mock_response = Mock()+ mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError("404 Not Found")+ mock_get.return_value = mock_response++ with pytest.raises(ApiServerException, match="Failed to get trace details"):+ get_trace_details("invalid-trace", "test-token")+++class TestCheckLlmSpans:+ """Test LLM span checking."""++ def test_check_llm_spans_found(self):+ """Test when LLM spans are found."""+ spans = [+ {"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}},+ {"span_name": "Some other span"},+ {"span_name": "anthropic.messages.create", "span_attributes": {"agentops": {"span": {"kind": "llm"}}}},+ ]++ has_llm, llm_names = check_llm_spans(spans)+ assert has_llm is True+ assert len(llm_names) == 2+ assert "OpenAI Chat Completion" in llm_names+ assert "anthropic.messages.create" in llm_names++ def test_check_llm_spans_not_found(self):+ """Test when no LLM spans are found."""+ spans = [{"span_name": "database.query"}, {"span_name": "http.request"}]++ has_llm, llm_names = check_llm_spans(spans)+ assert has_llm is False+ assert len(llm_names) == 0++ def test_check_llm_spans_empty(self):+ """Test with empty spans list."""+ has_llm, llm_names = check_llm_spans([])+ assert has_llm is False+ assert len(llm_names) == 0++ def test_check_llm_spans_with_request_type(self):+ """Test when LLM spans are identified by LLM_REQUEST_TYPE attribute."""+ from agentops.semconv import SpanAttributes, LLMRequestTypeValues++ spans = [+ {+ "span_name": "openai.chat.completion",+ "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value},+ },+ {+ "span_name": "anthropic.messages.create",+ "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value},+ },+ {+ "span_name": "llm.completion",+ "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.COMPLETION.value},+ },+ {+ "span_name": "embedding.create",+ "span_attributes": {SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.EMBEDDING.value},+ },+ {"span_name": "database.query"},+ ]++ has_llm, llm_names = check_llm_spans(spans)+ assert has_llm is True+ assert len(llm_names) == 3 # Only chat and completion types count as LLM+ assert "openai.chat.completion" in llm_names+ assert "anthropic.messages.create" in llm_names+ assert "llm.completion" in llm_names+ assert "embedding.create" not in llm_names # Embeddings are not LLM spans++ def test_check_llm_spans_real_world(self):+ """Test with real-world span structures from OpenAI and Anthropic."""+ from agentops.semconv import SpanAttributes, LLMRequestTypeValues++ # This simulates what we actually get from the OpenAI and Anthropic instrumentations+ spans = [+ {+ "span_name": "openai.chat.completion",+ "span_attributes": {+ SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value,+ SpanAttributes.LLM_SYSTEM: "OpenAI",+ SpanAttributes.LLM_REQUEST_MODEL: "gpt-4",+ },+ },+ {+ "span_name": "anthropic.messages.create",+ "span_attributes": {+ SpanAttributes.LLM_REQUEST_TYPE: LLMRequestTypeValues.CHAT.value,+ SpanAttributes.LLM_SYSTEM: "Anthropic",+ SpanAttributes.LLM_REQUEST_MODEL: "claude-3-opus-20240229",+ },+ },+ ]++ has_llm, llm_names = check_llm_spans(spans)+ assert has_llm is True+ assert len(llm_names) == 2+ assert "openai.chat.completion" in llm_names+ assert "anthropic.messages.create" in llm_names+++class TestValidateTraceSpans:+ """Test the main validation function."""++ @patch("agentops.validation.get_jwt_token")+ @patch("agentops.validation.get_trace_details")+ @patch("agentops.validation.get_trace_metrics")+ def test_validate_trace_spans_success(self, mock_metrics, mock_details, mock_token):+ """Test successful validation."""+ mock_token.return_value = "test-token"+ mock_details.return_value = {+ "spans": [+ {"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}},+ {"span_name": "Other span"},+ ]+ }+ mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"}++ result = validate_trace_spans(trace_id="test-trace")++ assert result["trace_id"] == "test-trace"+ assert result["span_count"] == 2+ assert result["has_llm_spans"] is True+ # LLM activity can be confirmed via metrics or span inspection+ assert result["metrics"]["total_tokens"] == 100++ @patch("agentops.validation.get_jwt_token")+ @patch("agentops.validation.get_trace_details")+ @patch("agentops.validation.get_trace_metrics")+ def test_validate_trace_spans_success_via_metrics(self, mock_metrics, mock_details, mock_token):+ """Test successful validation when LLM activity is confirmed via metrics."""+ mock_token.return_value = "test-token"+ mock_details.return_value = {+ "spans": [+ {+ "span_name": "openai.chat.completion",+ "span_attributes": {}, # No specific LLM attributes+ },+ {"span_name": "Other span"},+ ]+ }+ # But we have token usage, proving LLM activity+ mock_metrics.return_value = {"total_tokens": 1066, "total_cost": "0.0006077"}++ result = validate_trace_spans(trace_id="test-trace")++ assert result["trace_id"] == "test-trace"+ assert result["span_count"] == 2+ assert result["has_llm_spans"] is True # Confirmed via metrics+ assert result["metrics"]["total_tokens"] == 1066++ @patch("agentops.validation.get_jwt_token")+ @patch("agentops.validation.get_trace_details")+ @patch("agentops.validation.get_trace_metrics")+ def test_validate_trace_spans_no_llm(self, mock_metrics, mock_details, mock_token):+ """Test validation failure when no LLM spans found and no token usage."""+ mock_token.return_value = "test-token"+ mock_details.return_value = {"spans": [{"span_name": "database.query"}]}+ # No token usage either+ mock_metrics.return_value = {"total_tokens": 0, "total_cost": "0.0000"}++ with pytest.raises(ValidationError, match="No LLM activity detected"):+ validate_trace_spans(trace_id="test-trace", check_llm=True)++ @patch("agentops.validation.get_jwt_token")+ @patch("agentops.validation.get_trace_details")+ @patch("agentops.validation.get_trace_metrics")+ def test_validate_trace_spans_retry(self, mock_metrics, mock_details, mock_token):+ """Test validation with retries."""+ mock_token.return_value = "test-token"++ # First two calls return empty, third returns spans+ mock_details.side_effect = [+ {"spans": []},+ {"spans": []},+ {"spans": [{"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}]},+ ]++ # Mock metrics for the successful attempt+ mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"}++ result = validate_trace_spans(trace_id="test-trace", max_retries=3, retry_delay=0.01)++ assert result["span_count"] == 1+ assert mock_details.call_count == 3++ @patch("opentelemetry.trace.get_current_span")+ def test_validate_trace_spans_no_trace_id(self, mock_get_current_span):+ """Test validation without trace ID."""+ # Mock get_current_span to return None+ mock_get_current_span.return_value = None++ with pytest.raises(ValueError, match="No trace ID found"):+ validate_trace_spans()++ @patch("opentelemetry.trace.get_current_span")+ @patch("agentops.validation.get_jwt_token")+ @patch("agentops.validation.get_trace_details")+ @patch("agentops.validation.get_trace_metrics")+ def test_validate_trace_spans_from_current_span(self, mock_metrics, mock_details, mock_token, mock_get_span):+ """Test extracting trace ID from current span."""+ # Mock the current span+ mock_span_context = Mock()+ mock_span_context.trace_id = 12345678901234567890++ mock_span = Mock()+ mock_span.get_span_context.return_value = mock_span_context++ mock_get_span.return_value = mock_span++ mock_token.return_value = "test-token"+ mock_details.return_value = {+ "spans": [{"span_name": "OpenAI Chat Completion", "span_attributes": {"agentops.span.kind": "llm"}}]+ }+ mock_metrics.return_value = {"total_tokens": 100, "total_cost": "0.0025"}++ result = validate_trace_spans()+ assert result["trace_id"] == "0000000000000000ab54a98ceb1f0ad2" # hex format of trace ID+++class TestPrintValidationSummary:+ """Test validation summary printing."""++ def test_print_validation_summary(self, capsys):+ """Test printing validation summary."""+ result = {+ "span_count": 3,+ "has_llm_spans": True,+ "llm_span_names": ["OpenAI Chat", "Claude Message"],+ "metrics": {"total_tokens": 150, "prompt_tokens": 100, "completion_tokens": 50, "total_cost": "0.0030"},+ }++ print_validation_summary(result)++ captured = capsys.readouterr()+ assert "Found 3 span(s)" in captured.out+ assert "OpenAI Chat" in captured.out+ assert "Total tokens: 150" in captured.out+ assert "Total cost: $0.0030" in captured.out+ assert "✅ Validation successful!" in captured.out++ def test_print_validation_summary_metrics_only(self, capsys):+ """Test printing validation summary when LLM activity confirmed via metrics only."""+ result = {+ "span_count": 2,+ "has_llm_spans": True,+ "llm_span_names": [], # No specific LLM span names found+ "metrics": {+ "total_tokens": 1066,+ "prompt_tokens": 800,+ "completion_tokens": 266,+ "total_cost": "0.0006077",+ },+ }++ print_validation_summary(result)++ captured = capsys.readouterr()+ assert "Found 2 span(s)" in captured.out+ assert "LLM activity confirmed via token usage metrics" in captured.out+ assert "Total tokens: 1066" in captured.out+ assert "Total cost: $0.0006077" in captured.out+ assert "✅ Validation successful!" in captured.out++ def test_print_validation_summary_llm_prefix(self, capsys):+ """Test with spans using llm.* prefix (as returned by API)."""+ result = {+ "span_count": 1,+ "has_llm_spans": True,+ "llm_span_names": ["openai.chat.completion"],+ "metrics": {"total_tokens": 150, "prompt_tokens": 100, "completion_tokens": 50, "total_cost": "0.0030"},+ }++ print_validation_summary(result)++ captured = capsys.readouterr()+ assert "Found 1 span(s)" in captured.out+ assert "openai.chat.completion" in captured.out+ assert "✅ Validation successful!" in captured.out+++class TestCheckLlmSpansWithLlmPrefix:+ """Test LLM span checking with llm.* prefix attributes."""++ def test_check_llm_spans_with_llm_prefix(self):+ """Test when spans use llm.request.type instead of gen_ai.request.type."""+ spans = [+ {+ "span_name": "openai.chat.completion",+ "span_attributes": {+ "llm.request.type": "chat",+ "llm.system": "OpenAI",+ "llm.request.model": "gpt-4",+ "llm.usage.total_tokens": 150,+ },+ },+ {+ "span_name": "anthropic.messages.create",+ "span_attributes": {+ "llm.request.type": "chat",+ "llm.system": "Anthropic",+ "llm.request.model": "claude-3-opus",+ "llm.usage.total_tokens": 300,+ },+ },+ {"span_name": "embedding.create", "span_attributes": {"llm.request.type": "embedding"}},+ {"span_name": "database.query"},+ ]++ has_llm, llm_names = check_llm_spans(spans)+ assert has_llm is True+ assert len(llm_names) == 2 # Only chat types+ assert "openai.chat.completion" in llm_names+ assert "anthropic.messages.create" in llm_names