Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
feat: Support Multiple OpenTelemetry Processors (#962)
Patcher8d0aa7802e80
diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.ymlindex e165aafe..f7bc1279 100644--- a/.github/workflows/pylint.yml+++ b/.github/workflows/pylint.yml@@ -7,46 +7,8 @@ on:- 'sdk/python/**'jobs:- check:- runs-on: ubuntu-latest-- steps:- - name: Checkout base code- uses: actions/checkout@v5- with:- ref: ${{ github.event.pull_request.head.sha }}-- - name: Fetch main branch- run: git fetch origin main-- - name: Check if version was updated in pyproject.toml- run: |- sudo apt-get install -y python3-tomli-- echo "Checking current version in pyproject.toml"- python3 -c "import tomli; print(tomli.load(open('sdk/python/pyproject.toml', 'rb')))" || { echo "Failed to read pyproject.toml"; exit 1; }-- current_version=$(python3 -c "import tomli; data=tomli.load(open('sdk/python/pyproject.toml', 'rb')); print(data.get('tool', {}).get('poetry', {}).get('version', 'Version not found'))")- main_version=$(git show origin/main:sdk/python/pyproject.toml | python3 -c "import tomli; import sys; data=tomli.load(sys.stdin.buffer); print(data.get('tool', {}).get('poetry', {}).get('version', 'Version not found'))")-- echo "Current branch version: $current_version"- echo "Main branch version: $main_version"-- if [ "$current_version" = "Version not found" ] || [ "$main_version" = "Version not found" ]; then- echo "Error: Could not find version in pyproject.toml." >&2- exit 1- fi-- if [ "$current_version" = "$main_version" ]; then- echo "Version in pyproject.toml was not updated. Please update the version." >&2- exit 1- else- echo "Version in pyproject.toml was updated."- fi-lint:runs-on: ubuntu-latest- needs: checkstrategy:matrix:python-version: ["3.10", "3.11", "3.12"]diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.ymlindex 12b716b6..6ee0b8cd 100644--- a/.github/workflows/pypi-release.yml+++ b/.github/workflows/pypi-release.yml@@ -15,6 +15,23 @@ jobs:steps:- uses: actions/checkout@v5+ - name: Validate tag version matches pyproject.toml+ if: github.ref_type == 'tag'+ working-directory: sdk/python+ run: |+ pip install tomli+ TAG_VERSION="${GITHUB_REF#refs/tags/py-}"+ PYPROJECT_VERSION=$(python3 -c "import tomli; print(tomli.load(open('pyproject.toml', 'rb'))['tool']['poetry']['version'])")++ echo "Tag version: $TAG_VERSION"+ echo "pyproject.toml version: $PYPROJECT_VERSION"++ if [ "$TAG_VERSION" != "$PYPROJECT_VERSION" ]; then+ echo "::error::Tag py-$TAG_VERSION does not match pyproject.toml version $PYPROJECT_VERSION"+ exit 1+ fi+ echo "✓ Version validation passed"+- name: Set up Pythonuses: actions/setup-python@v5with:diff --git a/operator/providers/base/setup-instrumentation.py b/operator/providers/base/setup-instrumentation.pyindex e186655c..a5a48a24 100644--- a/operator/providers/base/setup-instrumentation.py+++ b/operator/providers/base/setup-instrumentation.py@@ -20,58 +20,61 @@ to the shared /openlit-sdk/ directory.Used by: OpenLIT Kubernetes operator init containersEnvironment: Python 3.x in Alpine-based container"""+import osimport shutilimport sys+def setup_instrumentation():"""Copy instrumentation packages to target directory"""- provider = os.environ.get('INSTRUMENTATION_PROVIDER', 'openlit')- source_dir = f'/instrumentations/{provider}'- target_dir = '/openlit-sdk'-+ provider = os.environ.get("INSTRUMENTATION_PROVIDER", "openlit")+ source_dir = f"/instrumentations/{provider}"+ target_dir = "/openlit-sdk"+print(f"🚀 Setting up {provider} instrumentation...")-+# Ensure target directory existsos.makedirs(target_dir, exist_ok=True)-+# Copy packagesif os.path.exists(source_dir):# Copy all contents from source to targetfor item in os.listdir(source_dir):source_path = os.path.join(source_dir, item)target_path = os.path.join(target_dir, item)-+if os.path.isdir(source_path):if os.path.exists(target_path):shutil.rmtree(target_path)shutil.copytree(source_path, target_path)else:shutil.copy2(source_path, target_path)-+# Copy sitecustomize.py to the root for auto-import- sitecustomize_src = os.path.join(source_dir, 'sitecustomize.py')- sitecustomize_dst = os.path.join(target_dir, 'sitecustomize.py')+ sitecustomize_src = os.path.join(source_dir, "sitecustomize.py")+ sitecustomize_dst = os.path.join(target_dir, "sitecustomize.py")if os.path.exists(sitecustomize_src):shutil.copy2(sitecustomize_src, sitecustomize_dst)print("✅ sitecustomize.py copied to root for auto-import")-+print(f"✅ {provider} instrumentation ready!")print(f"🎯 Provider: {provider}")print(f"📦 Packages copied to: {target_dir}")-+# Show some package infotry:- packages = [f for f in os.listdir(target_dir) if not f.startswith('.')][:10]+ packages = [f for f in os.listdir(target_dir) if not f.startswith(".")][:10]print(f"📚 Key packages: {', '.join(packages[:5])}")if len(packages) > 5:print(f" ... and {len(packages) - 5} more")except Exception as e:print(f"📦 Package listing failed: {e}")-+else:print(f"❌ Source directory not found: {source_dir}")sys.exit(1)-if __name__ == '__main__':- setup_instrumentation()\ No newline at end of file++if __name__ == "__main__":+ setup_instrumentation()diff --git a/operator/providers/base/sitecustomize.py b/operator/providers/base/sitecustomize.pyindex a67efb91..c022ba2f 100644--- a/operator/providers/base/sitecustomize.py+++ b/operator/providers/base/sitecustomize.py@@ -1,7 +1,9 @@"""Base auto-instrumentation sitecustomize.py for custom configurations"""+import osimport sys-sys.path.insert(0, '/openlit-sdk')++sys.path.insert(0, "/openlit-sdk")try:# Basic OpenTelemetry setup for custom configurations@@ -9,14 +11,14 @@ try:from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporterfrom opentelemetry.sdk.trace import TracerProviderfrom opentelemetry.sdk.trace.export import BatchSpanProcessor-+# Configure OpenTelemetry with environment variablestracer_provider = TracerProvider()-+# Support both HTTP and gRPC endpoints- endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT')+ endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")if endpoint:- if endpoint.endswith('/v1/traces'):+ if endpoint.endswith("/v1/traces"):# HTTP endpointotlp_exporter = OTLPSpanExporter(endpoint=endpoint)else:@@ -24,19 +26,20 @@ try:print(f"🔧 gRPC endpoint detected: {endpoint}")print("📦 Make sure to include gRPC exporter in CUSTOM_PACKAGES")otlp_exporter = OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")-+tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))trace.set_tracer_provider(tracer_provider)-+print("✅ Base OpenTelemetry initialized for custom instrumentation!")print(f"🎯 Service: {os.environ.get('OTEL_SERVICE_NAME', 'custom-app')}")print(f"🔗 Endpoint: {endpoint}")else:print("⚠️ No OTEL_EXPORTER_OTLP_ENDPOINT provided")print("🔧 Set up tracing in your custom packages")-+except Exception as e:print(f"⚠️ Base instrumentation setup failed: {e}")print("🔧 Your custom packages should handle instrumentation setup")import traceback- traceback.print_exc()\ No newline at end of file++ traceback.print_exc()diff --git a/operator/providers/openinference/setup-instrumentation.py b/operator/providers/openinference/setup-instrumentation.pyindex e186655c..a5a48a24 100644--- a/operator/providers/openinference/setup-instrumentation.py+++ b/operator/providers/openinference/setup-instrumentation.py@@ -20,58 +20,61 @@ to the shared /openlit-sdk/ directory.Used by: OpenLIT Kubernetes operator init containersEnvironment: Python 3.x in Alpine-based container"""+import osimport shutilimport sys+def setup_instrumentation():"""Copy instrumentation packages to target directory"""- provider = os.environ.get('INSTRUMENTATION_PROVIDER', 'openlit')- source_dir = f'/instrumentations/{provider}'- target_dir = '/openlit-sdk'-+ provider = os.environ.get("INSTRUMENTATION_PROVIDER", "openlit")+ source_dir = f"/instrumentations/{provider}"+ target_dir = "/openlit-sdk"+print(f"🚀 Setting up {provider} instrumentation...")-+# Ensure target directory existsos.makedirs(target_dir, exist_ok=True)-+# Copy packagesif os.path.exists(source_dir):# Copy all contents from source to targetfor item in os.listdir(source_dir):source_path = os.path.join(source_dir, item)target_path = os.path.join(target_dir, item)-+if os.path.isdir(source_path):if os.path.exists(target_path):shutil.rmtree(target_path)shutil.copytree(source_path, target_path)else:shutil.copy2(source_path, target_path)-+# Copy sitecustomize.py to the root for auto-import- sitecustomize_src = os.path.join(source_dir, 'sitecustomize.py')- sitecustomize_dst = os.path.join(target_dir, 'sitecustomize.py')+ sitecustomize_src = os.path.join(source_dir, "sitecustomize.py")+ sitecustomize_dst = os.path.join(target_dir, "sitecustomize.py")if os.path.exists(sitecustomize_src):shutil.copy2(sitecustomize_src, sitecustomize_dst)print("✅ sitecustomize.py copied to root for auto-import")-+print(f"✅ {provider} instrumentation ready!")print(f"🎯 Provider: {provider}")print(f"📦 Packages copied to: {target_dir}")-+# Show some package infotry:- packages = [f for f in os.listdir(target_dir) if not f.startswith('.')][:10]+ packages = [f for f in os.listdir(target_dir) if not f.startswith(".")][:10]print(f"📚 Key packages: {', '.join(packages[:5])}")if len(packages) > 5:print(f" ... and {len(packages) - 5} more")except Exception as e:print(f"📦 Package listing failed: {e}")-+else:print(f"❌ Source directory not found: {source_dir}")sys.exit(1)-if __name__ == '__main__':- setup_instrumentation()\ No newline at end of file++if __name__ == "__main__":+ setup_instrumentation()diff --git a/operator/providers/openinference/sitecustomize.py b/operator/providers/openinference/sitecustomize.pyindex 809444b4..7735112b 100644--- a/operator/providers/openinference/sitecustomize.py+++ b/operator/providers/openinference/sitecustomize.py@@ -3,81 +3,122 @@ OpenInference auto-instrumentation sitecustomize.pyUsing Pure OpenTelemetry approach that works with OpenLITBased on successful comprehensive local test results"""+import osimport sys-sys.path.insert(0, '/openlit-sdk')++sys.path.insert(0, "/openlit-sdk")+def setup_openinference():"""Setup comprehensive OpenInference instrumentation using pure OpenTelemetry"""-+print("🚀 COMPREHENSIVE OpenInference → OpenLIT Auto-Instrumentation")print("📋 Using Pure OpenTelemetry + ALL Available OpenInference Instrumentors")print("=" * 80)-+try:# 1. Setup Pure OpenTelemetry (that works with OpenLIT)from opentelemetry import trace- from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import (+ OTLPSpanExporter,+ )from opentelemetry.sdk.trace import TracerProviderfrom opentelemetry.sdk.trace.export import SimpleSpanProcessorfrom opentelemetry.sdk.resources import Resourcefrom opentelemetry.semconv.resource import ResourceAttributes-+print("🔧 Setting up Pure OpenTelemetry...")-+# Get configuration from environment variables- service_name = os.environ.get('OTEL_SERVICE_NAME', 'openinference-app')- otlp_endpoint = os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://localhost:4318/v1/traces')- environment = os.environ.get('OTEL_DEPLOYMENT_ENVIRONMENT', 'production')-+ service_name = os.environ.get("OTEL_SERVICE_NAME", "openinference-app")+ otlp_endpoint = os.environ.get(+ "OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318/v1/traces"+ )+ environment = os.environ.get("OTEL_DEPLOYMENT_ENVIRONMENT", "production")+# Create resource with service name- resource = Resource.create({- ResourceAttributes.SERVICE_NAME: service_name,- ResourceAttributes.SERVICE_VERSION: "1.0.0",- ResourceAttributes.DEPLOYMENT_ENVIRONMENT: environment- })-+ resource = Resource.create(+ {+ ResourceAttributes.SERVICE_NAME: service_name,+ ResourceAttributes.SERVICE_VERSION: "1.0.0",+ ResourceAttributes.DEPLOYMENT_ENVIRONMENT: environment,+ }+ )+# Create tracer providertracer_provider = TracerProvider(resource=resource)-+# Create OTLP exporter (that works with OpenLIT)otlp_exporter = OTLPSpanExporter(endpoint=otlp_endpoint)-+# Create span processor (use Simple for immediate sending)span_processor = SimpleSpanProcessor(otlp_exporter)tracer_provider.add_span_processor(span_processor)-+# Set global tracer providertrace.set_tracer_provider(tracer_provider)-+print("✅ Pure OpenTelemetry configured for OpenLIT!")print(f"🎯 Service: {service_name}")print(f"🔗 Endpoint: {otlp_endpoint}")print(f"🌍 Environment: {environment}")-+# 2. Enable ALL Available OpenInference Instrumentorsprint("\n📦 Enabling ALL OpenInference Instrumentors...")-+instrumentors = []-+# Enable all available instrumentorsinstrumentor_configs = [("openinference.instrumentation.openai", "OpenAIInstrumentor", "OpenAI"),- ("openinference.instrumentation.anthropic", "AnthropicInstrumentor", "Anthropic"),- ("openinference.instrumentation.langchain", "LangChainInstrumentor", "LangChain"),- ("openinference.instrumentation.llama_index", "LlamaIndexInstrumentor", "LlamaIndex"),+ (+ "openinference.instrumentation.anthropic",+ "AnthropicInstrumentor",+ "Anthropic",+ ),+ (+ "openinference.instrumentation.langchain",+ "LangChainInstrumentor",+ "LangChain",+ ),+ (+ "openinference.instrumentation.llama_index",+ "LlamaIndexInstrumentor",+ "LlamaIndex",+ ),("openinference.instrumentation.bedrock", "BedrockInstrumentor", "Bedrock"),- ("openinference.instrumentation.mistralai", "MistralAIInstrumentor", "MistralAI"),+ (+ "openinference.instrumentation.mistralai",+ "MistralAIInstrumentor",+ "MistralAI",+ ),("openinference.instrumentation.groq", "GroqInstrumentor", "Groq"),- ("openinference.instrumentation.vertexai", "VertexAIInstrumentor", "VertexAI"),+ (+ "openinference.instrumentation.vertexai",+ "VertexAIInstrumentor",+ "VertexAI",+ ),("openinference.instrumentation.dspy", "DSPyInstrumentor", "DSPy"),- ("openinference.instrumentation.instructor", "InstructorInstrumentor", "Instructor"),+ (+ "openinference.instrumentation.instructor",+ "InstructorInstrumentor",+ "Instructor",+ ),("openinference.instrumentation.litellm", "LiteLLMInstrumentor", "LiteLLM"),- ("openinference.instrumentation.haystack", "HaystackInstrumentor", "Haystack"),- ("openinference.instrumentation.guardrails", "GuardrailsInstrumentor", "Guardrails"),+ (+ "openinference.instrumentation.haystack",+ "HaystackInstrumentor",+ "Haystack",+ ),+ (+ "openinference.instrumentation.guardrails",+ "GuardrailsInstrumentor",+ "Guardrails",+ ),("openinference.instrumentation.portkey", "PortkeyInstrumentor", "Portkey"),]-+for module_name, class_name, display_name in instrumentor_configs:try:module = __import__(module_name, fromlist=[class_name])@@ -87,10 +128,14 @@ def setup_openinference():print(f"✅ {display_name} Instrumentor enabled")except Exception as e:# Don't print warnings for missing optional dependencies- if "No module named" not in str(e) and "DependencyConflict" not in str(e):+ if "No module named" not in str(e) and "DependencyConflict" not in str(+ e+ ):print(f"⚠️ {display_name} Instrumentor failed: {e}")-- print(f"\n📊 Successfully enabled {len(instrumentors)} instrumentors: {', '.join(instrumentors)}")++ print(+ f"\n📊 Successfully enabled {len(instrumentors)} instrumentors: {', '.join(instrumentors)}"+ )print("\n" + "=" * 80)print("🎉 COMPREHENSIVE OPENINFERENCE AUTO-INSTRUMENTATION COMPLETED!")print("📊 Configuration Summary:")@@ -100,19 +145,22 @@ def setup_openinference():print(f" • Instrumentors: {len(instrumentors)} enabled")print(f" • Auto-Instrumentation: ✅ All available frameworks")print("\n🔗 This approach works with OpenLIT while Phoenix OTEL doesn't!")-+return True-+except Exception as e:print(f"❌ OpenInference comprehensive setup failed: {e}")import traceback+traceback.print_exc()return False+# Initialize OpenInference instrumentationtry:setup_openinference()except Exception as e:print(f"❌ Failed to initialize OpenInference: {e}")import traceback- traceback.print_exc()\ No newline at end of file++ traceback.print_exc()diff --git a/operator/providers/openlit/setup-instrumentation.py b/operator/providers/openlit/setup-instrumentation.pyindex f9b62417..a844bc57 100644--- a/operator/providers/openlit/setup-instrumentation.py+++ b/operator/providers/openlit/setup-instrumentation.py@@ -20,58 +20,61 @@ to the shared /openlit-sdk/ directory.Used by: OpenLIT Kubernetes operator init containersEnvironment: Python 3.x in Alpine-based container"""+import osimport shutilimport sys+def setup_instrumentation():"""Copy instrumentation packages to target directory"""- provider = os.environ.get('INSTRUMENTATION_PROVIDER', 'openlit')- source_dir = f'/instrumentations/{provider}'- target_dir = os.environ.get('TARGET_PATH', '/instrumentation-packages')-+ provider = os.environ.get("INSTRUMENTATION_PROVIDER", "openlit")+ source_dir = f"/instrumentations/{provider}"+ target_dir = os.environ.get("TARGET_PATH", "/instrumentation-packages")+print(f"🚀 Setting up {provider} instrumentation...")-+# Ensure target directory existsos.makedirs(target_dir, exist_ok=True)-+# Copy packagesif os.path.exists(source_dir):# Copy all contents from source to targetfor item in os.listdir(source_dir):source_path = os.path.join(source_dir, item)target_path = os.path.join(target_dir, item)-+if os.path.isdir(source_path):if os.path.exists(target_path):shutil.rmtree(target_path)shutil.copytree(source_path, target_path)else:shutil.copy2(source_path, target_path)-+# Copy sitecustomize.py to the root for auto-import- sitecustomize_src = os.path.join(source_dir, 'sitecustomize.py')- sitecustomize_dst = os.path.join(target_dir, 'sitecustomize.py')+ sitecustomize_src = os.path.join(source_dir, "sitecustomize.py")+ sitecustomize_dst = os.path.join(target_dir, "sitecustomize.py")if os.path.exists(sitecustomize_src):shutil.copy2(sitecustomize_src, sitecustomize_dst)print("✅ sitecustomize.py copied to root for auto-import")-+print(f"✅ {provider} instrumentation ready!")print(f"🎯 Provider: {provider}")print(f"📦 Packages copied to: {target_dir}")-+# Show some package infotry:- packages = [f for f in os.listdir(target_dir) if not f.startswith('.')][:10]+ packages = [f for f in os.listdir(target_dir) if not f.startswith(".")][:10]print(f"📚 Key packages: {', '.join(packages[:5])}")if len(packages) > 5:print(f" ... and {len(packages) - 5} more")except Exception as e:print(f"📦 Package listing failed: {e}")-+else:print(f"❌ Source directory not found: {source_dir}")sys.exit(1)-if __name__ == '__main__':- setup_instrumentation()\ No newline at end of file++if __name__ == "__main__":+ setup_instrumentation()diff --git a/operator/providers/openlit/sitecustomize.py b/operator/providers/openlit/sitecustomize.pyindex 9fbf2c58..faabae25 100644--- a/operator/providers/openlit/sitecustomize.py+++ b/operator/providers/openlit/sitecustomize.py@@ -1,39 +1,49 @@"""OpenLIT auto-instrumentation sitecustomize.py using CLI bootstrap approach"""+import osimport sys# Use configurable instrumentation path from environment or fallback to default-instrumentation_path = os.environ.get('PYTHONPATH', '/instrumentation-packages').split(':')[0]+instrumentation_path = os.environ.get("PYTHONPATH", "/instrumentation-packages").split(+ ":"+)[0]sys.path.insert(0, instrumentation_path)try:# Use OpenLIT CLI bootstrap initialization (proper auto-instrumentation approach)# Simply importing the module will automatically call initialize()import openlit.cli.bootstrap.sitecustomize-+print("✅ OpenLIT auto-instrumentation initialized via CLI bootstrap!")print(f"🎯 Service: {os.environ.get('OTEL_SERVICE_NAME', 'openlit-app')}")print(f"🔗 Endpoint: {os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}")- print(f"🌍 Environment: {os.environ.get('OTEL_DEPLOYMENT_ENVIRONMENT', 'production')}")+ print(+ f"🌍 Environment: {os.environ.get('OTEL_DEPLOYMENT_ENVIRONMENT', 'production')}"+ )print("🚀 Using OpenLIT CLI bootstrap for zero-code instrumentation")-+except Exception as e:print(f"⚠️ OpenLIT CLI bootstrap failed: {e}")# Fallback to manual init if CLI approach failstry:import openlit-+openlit.init(- otlp_endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT'),- otlp_headers=os.environ.get('OTEL_EXPORTER_OTLP_HEADERS'),- application_name=os.environ.get('OTEL_SERVICE_NAME', 'openlit-app'),- environment=os.environ.get('OTEL_DEPLOYMENT_ENVIRONMENT', 'production'),- capture_message_content=os.environ.get('OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT', 'true').lower() == 'true',- detailed_tracing=os.environ.get('OPENLIT_DETAILED_TRACING', 'true').lower() == 'true'+ otlp_endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT"),+ otlp_headers=os.environ.get("OTEL_EXPORTER_OTLP_HEADERS"),+ application_name=os.environ.get("OTEL_SERVICE_NAME", "openlit-app"),+ environment=os.environ.get("OTEL_DEPLOYMENT_ENVIRONMENT", "production"),+ capture_message_content=os.environ.get(+ "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "true"+ ).lower()+ == "true",+ detailed_tracing=os.environ.get("OPENLIT_DETAILED_TRACING", "true").lower()+ == "true",)-+print("✅ OpenLIT fallback initialization successful!")except Exception as fallback_e:print(f"❌ OpenLIT fallback failed: {fallback_e}")import traceback- traceback.print_exc()\ No newline at end of file++ traceback.print_exc()diff --git a/operator/providers/openllmetry/setup-instrumentation.py b/operator/providers/openllmetry/setup-instrumentation.pyindex e186655c..a5a48a24 100644--- a/operator/providers/openllmetry/setup-instrumentation.py+++ b/operator/providers/openllmetry/setup-instrumentation.py@@ -20,58 +20,61 @@ to the shared /openlit-sdk/ directory.Used by: OpenLIT Kubernetes operator init containersEnvironment: Python 3.x in Alpine-based container"""+import osimport shutilimport sys+def setup_instrumentation():"""Copy instrumentation packages to target directory"""- provider = os.environ.get('INSTRUMENTATION_PROVIDER', 'openlit')- source_dir = f'/instrumentations/{provider}'- target_dir = '/openlit-sdk'-+ provider = os.environ.get("INSTRUMENTATION_PROVIDER", "openlit")+ source_dir = f"/instrumentations/{provider}"+ target_dir = "/openlit-sdk"+print(f"🚀 Setting up {provider} instrumentation...")-+# Ensure target directory existsos.makedirs(target_dir, exist_ok=True)-+# Copy packagesif os.path.exists(source_dir):# Copy all contents from source to targetfor item in os.listdir(source_dir):source_path = os.path.join(source_dir, item)target_path = os.path.join(target_dir, item)-+if os.path.isdir(source_path):if os.path.exists(target_path):shutil.rmtree(target_path)shutil.copytree(source_path, target_path)else:shutil.copy2(source_path, target_path)-+# Copy sitecustomize.py to the root for auto-import- sitecustomize_src = os.path.join(source_dir, 'sitecustomize.py')- sitecustomize_dst = os.path.join(target_dir, 'sitecustomize.py')+ sitecustomize_src = os.path.join(source_dir, "sitecustomize.py")+ sitecustomize_dst = os.path.join(target_dir, "sitecustomize.py")if os.path.exists(sitecustomize_src):shutil.copy2(sitecustomize_src, sitecustomize_dst)print("✅ sitecustomize.py copied to root for auto-import")-+print(f"✅ {provider} instrumentation ready!")print(f"🎯 Provider: {provider}")print(f"📦 Packages copied to: {target_dir}")-+# Show some package infotry:- packages = [f for f in os.listdir(target_dir) if not f.startswith('.')][:10]+ packages = [f for f in os.listdir(target_dir) if not f.startswith(".")][:10]print(f"📚 Key packages: {', '.join(packages[:5])}")if len(packages) > 5:print(f" ... and {len(packages) - 5} more")except Exception as e:print(f"📦 Package listing failed: {e}")-+else:print(f"❌ Source directory not found: {source_dir}")sys.exit(1)-if __name__ == '__main__':- setup_instrumentation()\ No newline at end of file++if __name__ == "__main__":+ setup_instrumentation()diff --git a/operator/providers/openllmetry/sitecustomize.py b/operator/providers/openllmetry/sitecustomize.pyindex 50fc9d0f..ef0b0e5e 100644--- a/operator/providers/openllmetry/sitecustomize.py+++ b/operator/providers/openllmetry/sitecustomize.py@@ -1,43 +1,51 @@"""OpenLLMetry auto-instrumentation sitecustomize.py"""+import osimport sys-sys.path.insert(0, '/openlit-sdk')++sys.path.insert(0, "/openlit-sdk")try:# Use traceloop-sdk for OpenLLMetry instrumentationfrom traceloop.sdk import Traceloop-+# Initialize Traceloop with environment variablesTraceloop.init(- app_name=os.environ.get('TRACELOOP_APP_NAME', 'openllmetry-app'),- api_endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT') or- os.environ.get('TRACELOOP_API_ENDPOINT', 'http://localhost:4318'),- headers=os.environ.get('OTEL_EXPORTER_OTLP_HEADERS'),- disable_batch=os.environ.get('TRACELOOP_DISABLE_BATCH', 'false').lower() == 'true'+ app_name=os.environ.get("TRACELOOP_APP_NAME", "openllmetry-app"),+ api_endpoint=os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT")+ or os.environ.get("TRACELOOP_API_ENDPOINT", "http://localhost:4318"),+ headers=os.environ.get("OTEL_EXPORTER_OTLP_HEADERS"),+ disable_batch=os.environ.get("TRACELOOP_DISABLE_BATCH", "false").lower()+ == "true",)-+print("✅ OpenLLMetry auto-instrumentation initialized with traceloop-sdk!")print(f"🎯 App: {os.environ.get('TRACELOOP_APP_NAME', 'openllmetry-app')}")print(f"🔗 Endpoint: {os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'default')}")-+except Exception as e:print(f"⚠️ OpenLLMetry initialization failed: {e}")# Fallback to basic OpenTelemetry setuptry:from opentelemetry import trace- from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter+ from opentelemetry.exporter.otlp.proto.http.trace_exporter import (+ OTLPSpanExporter,+ )from opentelemetry.sdk.trace import TracerProviderfrom opentelemetry.sdk.trace.export import BatchSpanProcessor-+tracer_provider = TracerProvider()otlp_exporter = OTLPSpanExporter(- endpoint=os.environ.get('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://localhost:4318/v1/traces')+ endpoint=os.environ.get(+ "OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318/v1/traces"+ ))tracer_provider.add_span_processor(BatchSpanProcessor(otlp_exporter))trace.set_tracer_provider(tracer_provider)-+print("✅ OpenLLMetry fallback initialization successful!")except Exception as fallback_e:print(f"❌ OpenLLMetry fallback failed: {fallback_e}")import traceback- traceback.print_exc()\ No newline at end of file++ traceback.print_exc()diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex 30da4f52..49fb1a37 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.36.4"+version = "1.36.5"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/__helpers.py b/sdk/python/src/openlit/__helpers.pyindex 7543c37d..901a584e 100644--- a/sdk/python/src/openlit/__helpers.py+++ b/sdk/python/src/openlit/__helpers.py@@ -23,6 +23,23 @@ from openlit.semcov import SemanticConventionlogger = logging.getLogger(__name__)+def parse_exporters(env_var_name):+ """+ Parse comma-separated exporter names from environment variable.+ Returns None if not set (signals to use default behavior).++ Args:+ env_var_name: Name of the environment variable to parse++ Returns:+ List of exporter names (lowercase, stripped) or None if env var not set+ """+ exporters_str = os.getenv(env_var_name)+ if not exporters_str:+ return None+ return [e.strip().lower() for e in exporters_str.split(",") if e.strip()]++def response_as_dict(response):"""Return parsed response as a dictdiff --git a/sdk/python/src/openlit/instrumentation/agno/async_agno.py b/sdk/python/src/openlit/instrumentation/agno/async_agno.pyindex 7ed740aa..8155c900 100644--- a/sdk/python/src/openlit/instrumentation/agno/async_agno.py+++ b/sdk/python/src/openlit/instrumentation/agno/async_agno.py@@ -245,7 +245,9 @@ def async_agent_run_stream_wrap(except Exception: # noqa: WPS429RunOutput = None # type: ignore # pylint: disable=invalid-name# `yield_run_response` is backwards compatibility for agno < 2.3- yield_run_output = kwargs.get("yield_run_output", None) or kwargs.get("yield_run_response", None)+ yield_run_output = kwargs.get("yield_run_output", None) or kwargs.get(+ "yield_run_response", None+ )new_kwargs = dict(kwargs)new_kwargs["yield_run_output"] = Trueasync for response in wrapped(*args, **new_kwargs):diff --git a/sdk/python/src/openlit/otel/events.py b/sdk/python/src/openlit/otel/events.pyindex 820389b8..46a03db2 100644--- a/sdk/python/src/openlit/otel/events.py+++ b/sdk/python/src/openlit/otel/events.py@@ -3,6 +3,7 @@ Setups up OpenTelemetry events emitter"""import os+import loggingfrom opentelemetry import _events, _logsfrom opentelemetry.sdk.resources import (SERVICE_NAME,@@ -23,6 +24,10 @@ if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") == "grpc":else:from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter+from openlit.__helpers import parse_exporters++logger = logging.getLogger(__name__)+# Global flag to check if the events provider initialization is complete.EVENTS_SET = False@@ -82,20 +87,54 @@ def setup_events(os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers_str- # Configure the span exporter and processor based on whether the endpoint is effectively set.- if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):- event_exporter = OTLPLogExporter()- # pylint: disable=line-too-long- logger_provider.add_log_record_processor(- SimpleLogRecordProcessor(event_exporter)- ) if disable_batch else logger_provider.add_log_record_processor(- BatchLogRecordProcessor(event_exporter)- )+ # Check for OTEL_LOGS_EXPORTER env var for multiple exporters support+ exporters_config = parse_exporters("OTEL_LOGS_EXPORTER")+ processors_added = False++ if exporters_config is not None:+ # New behavior: use specified exporters from OTEL_LOGS_EXPORTER+ for exporter_name in exporters_config:+ if exporter_name == "otlp":+ event_exporter = OTLPLogExporter()+ log_processor = (+ BatchLogRecordProcessor(event_exporter)+ if not disable_batch+ else SimpleLogRecordProcessor(event_exporter)+ )+ logger_provider.add_log_record_processor(log_processor)+ processors_added = True+ elif exporter_name == "console":+ event_exporter = ConsoleLogExporter()+ log_processor = SimpleLogRecordProcessor(event_exporter)+ logger_provider.add_log_record_processor(log_processor)+ processors_added = True+ elif exporter_name == "none":+ # "none" means no exporter, skip+ continue+ else:+ logger.warning("Unknown log exporter: %s", exporter_name)++ # Warn if no valid exporters were configured+ if not processors_added:+ logger.warning(+ "OTEL_LOGS_EXPORTER is set but no valid exporters configured. "+ "Log export is disabled. Valid exporters: otlp, console"+ )else:- event_exporter = ConsoleLogExporter()- logger_provider.add_log_record_processor(- SimpleLogRecordProcessor(event_exporter)- )+ # Default behavior: use OTEL_EXPORTER_OTLP_ENDPOINT check+ if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):+ event_exporter = OTLPLogExporter()+ # pylint: disable=line-too-long+ log_processor = (+ BatchLogRecordProcessor(event_exporter)+ if not disable_batch+ else SimpleLogRecordProcessor(event_exporter)+ )+ else:+ event_exporter = ConsoleLogExporter()+ log_processor = SimpleLogRecordProcessor(event_exporter)++ logger_provider.add_log_record_processor(log_processor)_logs.set_logger_provider(logger_provider)event_provider = EventLoggerProvider()diff --git a/sdk/python/src/openlit/otel/metrics.py b/sdk/python/src/openlit/otel/metrics.pyindex d70cb9c4..fd2c6445 100644--- a/sdk/python/src/openlit/otel/metrics.py+++ b/sdk/python/src/openlit/otel/metrics.py@@ -4,6 +4,7 @@ Setups up OpenTelemetry Meter"""import os+import loggingfrom opentelemetry import metricsfrom opentelemetry.sdk.metrics import MeterProviderfrom opentelemetry.sdk.metrics.export import (@@ -17,6 +18,7 @@ from opentelemetry.sdk.resources import ()from opentelemetry.sdk.resources import Resourcefrom openlit.semcov import SemanticConvention+from openlit.__helpers import parse_exportersif os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") == "grpc":from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (@@ -27,9 +29,12 @@ else:OTLPMetricExporter,)+logger = logging.getLogger(__name__)+# Global flag to check if the meter provider initialization is complete.METER_SET = False+_DB_CLIENT_OPERATION_DURATION_BUCKETS = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10]_GEN_AI_CLIENT_OPERATION_DURATION_BUCKETS = [@@ -170,16 +175,45 @@ def setup_meter(application_name, environment, meter, otlp_endpoint, otlp_header# Now, we have either converted the dict to a string or used the provided string.os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers_str- # Configure the span exporter and processor based on whether the endpoint is effectively set.- if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):- metric_exporter = OTLPMetricExporter()- else:- metric_exporter = ConsoleMetricExporter()+ # Check for OTEL_METRICS_EXPORTER env var for multiple exporters support+ exporters_config = parse_exporters("OTEL_METRICS_EXPORTER")+ metric_readers = []++ if exporters_config is not None:+ # New behavior: use specified exporters from OTEL_METRICS_EXPORTER+ for exporter_name in exporters_config:+ if exporter_name == "otlp":+ metric_exporter = OTLPMetricExporter()+ metric_readers.append(+ PeriodicExportingMetricReader(metric_exporter)+ )+ elif exporter_name == "console":+ metric_exporter = ConsoleMetricExporter()+ metric_readers.append(+ PeriodicExportingMetricReader(metric_exporter)+ )+ elif exporter_name == "none":+ # "none" means no exporter, skip+ continue+ else:+ logger.warning("Unknown metric exporter: %s", exporter_name)- metric_reader = PeriodicExportingMetricReader(metric_exporter)+ # Warn if no valid exporters were configured+ if not metric_readers:+ logger.warning(+ "OTEL_METRICS_EXPORTER is set but no valid exporters configured. "+ "Metric export is disabled. Valid exporters: otlp, console"+ )+ else:+ # Default behavior: use OTEL_EXPORTER_OTLP_ENDPOINT check+ if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):+ metric_exporter = OTLPMetricExporter()+ else:+ metric_exporter = ConsoleMetricExporter()+ metric_readers.append(PeriodicExportingMetricReader(metric_exporter))meter_provider = MeterProvider(- resource=resource, metric_readers=[metric_reader]+ resource=resource, metric_readers=metric_readers)metrics.set_meter_provider(meter_provider)diff --git a/sdk/python/src/openlit/otel/tracing.py b/sdk/python/src/openlit/otel/tracing.pyindex 1dfb0356..20d046d5 100644--- a/sdk/python/src/openlit/otel/tracing.py+++ b/sdk/python/src/openlit/otel/tracing.py@@ -3,6 +3,7 @@ Setups up OpenTelemetry tracer"""import os+import loggingfrom opentelemetry import tracefrom opentelemetry.sdk.resources import (SERVICE_NAME,@@ -19,6 +20,10 @@ if os.environ.get("OTEL_EXPORTER_OTLP_PROTOCOL") == "grpc":else:from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter+from openlit.__helpers import parse_exporters++logger = logging.getLogger(__name__)+# Global flag to check if the tracer provider initialization is complete.TRACER_SET = False@@ -70,20 +75,54 @@ def setup_tracing(os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers_str- # Configure the span exporter and processor based on whether the endpoint is effectively set.- if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):- span_exporter = OTLPSpanExporter()- # pylint: disable=line-too-long- span_processor = (- BatchSpanProcessor(span_exporter)- if not disable_batch- else SimpleSpanProcessor(span_exporter)- )+ # Check for OTEL_TRACES_EXPORTER env var for multiple exporters support+ exporters_config = parse_exporters("OTEL_TRACES_EXPORTER")+ processors_added = False++ if exporters_config is not None:+ # New behavior: use specified exporters from OTEL_TRACES_EXPORTER+ for exporter_name in exporters_config:+ if exporter_name == "otlp":+ span_exporter = OTLPSpanExporter()+ span_processor = (+ BatchSpanProcessor(span_exporter)+ if not disable_batch+ else SimpleSpanProcessor(span_exporter)+ )+ trace.get_tracer_provider().add_span_processor(span_processor)+ processors_added = True+ elif exporter_name == "console":+ span_exporter = ConsoleSpanExporter()+ span_processor = SimpleSpanProcessor(span_exporter)+ trace.get_tracer_provider().add_span_processor(span_processor)+ processors_added = True+ elif exporter_name == "none":+ # "none" means no exporter, skip+ continue+ else:+ logger.warning("Unknown trace exporter: %s", exporter_name)++ # Warn if no valid exporters were configured+ if not processors_added:+ logger.warning(+ "OTEL_TRACES_EXPORTER is set but no valid exporters configured. "+ "Trace export is disabled. Valid exporters: otlp, console"+ )else:- span_exporter = ConsoleSpanExporter()- span_processor = SimpleSpanProcessor(span_exporter)+ # Default behavior: use OTEL_EXPORTER_OTLP_ENDPOINT check+ if os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT"):+ span_exporter = OTLPSpanExporter()+ # pylint: disable=line-too-long+ span_processor = (+ BatchSpanProcessor(span_exporter)+ if not disable_batch+ else SimpleSpanProcessor(span_exporter)+ )+ else:+ span_exporter = ConsoleSpanExporter()+ span_processor = SimpleSpanProcessor(span_exporter)- trace.get_tracer_provider().add_span_processor(span_processor)+ trace.get_tracer_provider().add_span_processor(span_processor)TRACER_SET = Truediff --git a/sdk/python/tests/test_agno.py b/sdk/python/tests/test_agno.pyindex a0369bd5..c3e7319f 100644--- a/sdk/python/tests/test_agno.py+++ b/sdk/python/tests/test_agno.py@@ -18,8 +18,7 @@ import openlit# Initialize OpenLIT monitoringopenlit.init(- environment="openlit-python-testing",- application_name="openlit-python-agno-test"+ environment="openlit-python-testing", application_name="openlit-python-agno-test")@@ -31,10 +30,7 @@ def test_agent_run():AssertionError: If the agent response is not as expected."""try:- agent = Agent(- model=Cohere(id="command-r-08-2024"),- markdown=True- )+ agent = Agent(model=Cohere(id="command-r-08-2024"), markdown=True)response = agent.run("Say 'test passed' in 2 words")assert response is not None@@ -56,10 +52,7 @@ def test_agent_async_run():async def run_async():try:- agent = Agent(- model=Cohere(id="command-r-08-2024"),- markdown=True- )+ agent = Agent(model=Cohere(id="command-r-08-2024"), markdown=True)response = await agent.arun("Say 'async test passed' in 3 words")assert response is not None