Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
[Feat]: Refactor Crawl4AI OpenTelemetry Monitoring Instrumentaion (#825)
Patcher35defdf7eb33
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.tomlindex b0425161..8dfb2886 100644--- a/sdk/python/pyproject.toml+++ b/sdk/python/pyproject.toml@@ -1,6 +1,6 @@[tool.poetry]name = "openlit"-version = "1.34.37"+version = "1.34.38"description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"authors = ["OpenLIT"]license = "Apache-2.0"diff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.py b/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.pyindex 4a613870..b3612d46 100644--- a/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.py+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.py@@ -1,20 +1,25 @@# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code-"""Initializer of Auto Instrumentation of Crawl4AI Functions"""+"""+Initializer of Auto Instrumentation of Crawl4AI Functions.+Supports Crawl4AI 0.7.x with comprehensive operation coverage.+"""from typing import Collectionimport importlib.metadatafrom opentelemetry.instrumentation.instrumentor import BaseInstrumentorfrom wrapt import wrap_function_wrapper-from openlit.instrumentation.crawl4ai.crawl4ai import wrap_crawl-from openlit.instrumentation.crawl4ai.async_crawl4ai import async_wrap_crawl+from openlit.instrumentation.crawl4ai.crawl4ai import general_wrap+from openlit.instrumentation.crawl4ai.async_crawl4ai import async_general_wrap-_instruments = ("crawl4ai >= 0.4.0",)+_instruments = ("crawl4ai >= 0.7.0",)class Crawl4AIInstrumentor(BaseInstrumentor):"""- An instrumentor for crawl4ai's client library.+ An instrumentor for Crawl4AI's client library.+ Supports comprehensive 0.7.x operations including async crawling,+ batch processing, deep crawling, and extraction strategies."""def instrumentation_dependencies(self) -> Collection[str]:@@ -28,40 +33,329 @@ class Crawl4AIInstrumentor(BaseInstrumentor):pricing_info = kwargs.get("pricing_info", {})capture_message_content = kwargs.get("capture_message_content", False)disable_metrics = kwargs.get("disable_metrics")- version = importlib.metadata.version("crawl4ai")-- wrap_function_wrapper(- "crawl4ai.web_crawler",- "WebCrawler.run",- wrap_crawl(- "crawl4ai.web_crawl",- version,- environment,- application_name,- tracer,- pricing_info,- capture_message_content,- metrics,- disable_metrics,- ),- )-- wrap_function_wrapper(- "crawl4ai.async_webcrawler",- "AsyncWebCrawler.arun",- async_wrap_crawl(- "crawl4ai.web_crawl",- version,- environment,- application_name,- tracer,- pricing_info,- capture_message_content,- metrics,- disable_metrics,- ),- )++ try:+ version = importlib.metadata.version("crawl4ai")+ except importlib.metadata.PackageNotFoundError:+ version = "unknown"++ # Wrap AsyncWebCrawler.arun() - Primary single URL crawling+ try:+ wrap_function_wrapper(+ "crawl4ai.async_webcrawler",+ "AsyncWebCrawler.arun",+ async_general_wrap(+ "crawl4ai.arun",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ # Wrap AsyncWebCrawler.arun_many() - Batch/streaming crawling (0.7.x)+ try:+ wrap_function_wrapper(+ "crawl4ai.async_webcrawler",+ "AsyncWebCrawler.arun_many",+ async_general_wrap(+ "crawl4ai.arun_many",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ # Wrap legacy WebCrawler.run() for backward compatibility+ try:+ wrap_function_wrapper(+ "crawl4ai.web_crawler",+ "WebCrawler.run",+ general_wrap(+ "crawl4ai.run",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ # Wrap deep crawling operations (0.7.x)+ try:+ wrap_function_wrapper(+ "crawl4ai.deep_crawling.strategies",+ "BFSDeepCrawlStrategy.arun",+ async_general_wrap(+ "crawl4ai.deep_crawl.bfs",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.deep_crawling.strategies",+ "DFSDeepCrawlStrategy.arun",+ async_general_wrap(+ "crawl4ai.deep_crawl.dfs",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.deep_crawling.strategies",+ "BestFirstCrawlingStrategy.arun",+ async_general_wrap(+ "crawl4ai.deep_crawl.best_first",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ # Wrap LLM and Extraction Strategy Operations (0.7.x)+ try:+ wrap_function_wrapper(+ "crawl4ai.extraction_strategy",+ "LLMExtractionStrategy.extract",+ general_wrap(+ "crawl4ai.extraction.llm.extract",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.extraction_strategy",+ "LLMExtractionStrategy.run",+ general_wrap(+ "crawl4ai.extraction.llm.run",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.extraction_strategy",+ "JsonCssExtractionStrategy.extract",+ general_wrap(+ "crawl4ai.extraction.css.extract",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.extraction_strategy",+ "JsonXPathExtractionStrategy.extract",+ general_wrap(+ "crawl4ai.extraction.xpath.extract",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.extraction_strategy",+ "CosineStrategy.extract",+ general_wrap(+ "crawl4ai.extraction.cosine.extract",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.extraction_strategy",+ "RegexExtractionStrategy.extract",+ general_wrap(+ "crawl4ai.extraction.regex.extract",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ # Wrap Core Processing Operations (0.7.x)+ try:+ wrap_function_wrapper(+ "crawl4ai.async_webcrawler",+ "AsyncWebCrawler.aprocess_html",+ async_general_wrap(+ "crawl4ai.aprocess_html",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.async_webcrawler",+ "AsyncWebCrawler.aseed_urls",+ async_general_wrap(+ "crawl4ai.aseed_urls",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ # Wrap Monitor Operations for Business Intelligence+ try:+ wrap_function_wrapper(+ "crawl4ai.components.crawler_monitor",+ "CrawlerMonitor.update_task",+ general_wrap(+ "crawl4ai.monitor.update_task",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versions++ try:+ wrap_function_wrapper(+ "crawl4ai.components.crawler_monitor",+ "CrawlerMonitor.get_summary",+ general_wrap(+ "crawl4ai.monitor.get_summary",+ version,+ environment,+ application_name,+ tracer,+ pricing_info,+ capture_message_content,+ metrics,+ disable_metrics,+ ),+ )+ except Exception:+ pass # Module may not exist in all versionsdef _uninstrument(self, **kwargs):- # Proper uninstrumentation logic to revert patched methods- pass+ """+ Proper uninstrumentation logic to revert patched methods.++ Note: wrapt automatically handles uninstrumentation for+ wrap_function_wrapper calls when the instrumentor is disabled.+ """+ # Uninstrumentation is handled automatically by wrapt+ returndiff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py b/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.pyindex afb5403b..07c34df0 100644--- a/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py@@ -1,25 +1,33 @@# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument"""-Module for monitoring Crawl4AI calls.+Module for monitoring Crawl4AI asynchronous calls.+Supports comprehensive 0.7.x operations with business intelligence and enhanced observability.+Includes support for async generators and streaming responses."""import logging-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import (- SERVICE_NAME,- TELEMETRY_SDK_NAME,- DEPLOYMENT_ENVIRONMENT,-)-from openlit.__helpers import (- handle_exception,+import time+from opentelemetry.trace import SpanKind+from opentelemetry import context as context_api++from openlit.__helpers import handle_exception+from openlit.instrumentation.crawl4ai.utils import (+ Crawl4AIInstrumentationContext,+ get_operation_name,+ create_crawl_span_name,+ set_crawl_attributes,+ process_crawl_response,+ capture_message_content_if_enabled,+ process_llm_extraction_response,+ create_extraction_span_name,+ capture_extraction_content,)-from openlit.semcov import SemanticConvention# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def async_wrap_crawl(+def async_general_wrap(gen_ai_endpoint,version,environment,@@ -31,121 +39,320 @@ def async_wrap_crawl(disable_metrics,):"""- Generates a telemetry wrapper for chat completions to collect metrics.+ Generates a telemetry wrapper for async Crawl4AI operations to collect comprehensive metrics.Args:gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.+ version: Version of the Crawl4AI package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Crawl4AI Agent.+ application_name: Name of the application using Crawl4AI.tracer: OpenTelemetry tracer for creating spans.pricing_info: Information used for calculating the cost of Crawl4AI usage.capture_message_content: Flag indicating whether to trace the actual content.+ metrics: OpenLIT metrics dictionary for performance tracking.+ disable_metrics: Flag to disable metrics collection.Returns:- A function that wraps the chat completions method to add telemetry.+ A function that wraps async Crawl4AI methods to add comprehensive telemetry."""- async def wrapper(wrapped, instance, args, kwargs):+ def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat.completions' API call to add telemetry.+ Wraps async Crawl4AI operations to add comprehensive telemetry.- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ This collects metrics such as execution time, URL information, configuration details,+ and result metrics, while handling errors gracefully for enhanced observability.+ Supports both regular async methods and async generators for streaming.Args:- wrapped: The original 'chat.completions' method to be wrapped.+ wrapped: The original async Crawl4AI method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat.completions' method.- kwargs: Keyword arguments for the 'chat.completions' method.+ args: Positional arguments for the Crawl4AI method.+ kwargs: Keyword arguments for the Crawl4AI method.Returns:- The response from the original 'chat.completions' method.+ The response from the original async Crawl4AI method."""- # pylint: disable=line-too-long- with tracer.start_as_current_span(- gen_ai_endpoint, kind=SpanKind.CLIENT- ) as span:- response = await wrapped(*args, **kwargs)+ async def async_wrapper():+ # Check if instrumentation is suppressed+ if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY):+ return await wrapped(*args, **kwargs)- try:- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(- SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_CRAWL4AI,- )- span.set_attribute(- SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_AGENT,- )- span.set_attribute(SemanticConvention.GEN_AI_ENDPOINT, gen_ai_endpoint)- span.set_attribute(SERVICE_NAME, application_name)- span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_TYPE,- SemanticConvention.GEN_AI_AGENT_TYPE_BROWSER,- )- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_ENABLE_CACHE,- not kwargs.get("disable_cache", False),- )+ # Get operation name and create context+ operation_name = get_operation_name(gen_ai_endpoint)+ ctx = Crawl4AIInstrumentationContext(+ instance, args, kwargs, version, environment, application_name+ )- url = (- kwargs.get("url")- if "url" in kwargs- else str(args[0])- if args- else None- )- if url is not None:- span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, url)+ # Create span name following the framework guide pattern+ if operation_name == "extract":+ # For extraction operations, determine strategy type from the endpoint+ strategy_type = None+ if "llm" in gen_ai_endpoint:+ strategy_type = "llm"+ elif "css" in gen_ai_endpoint:+ strategy_type = "css"+ elif "xpath" in gen_ai_endpoint:+ strategy_type = "xpath"+ elif "cosine" in gen_ai_endpoint:+ strategy_type = "cosine"+ elif "regex" in gen_ai_endpoint:+ strategy_type = "regex"- extraction_strategy = kwargs.get(- "extraction_strategy", "NoExtractionStrategy"- )- extraction_name = (- extraction_strategy.name- if hasattr(extraction_strategy, "name")- else extraction_strategy+ target_url = ctx.url if ctx.url != "unknown" else "content"+ span_name = create_extraction_span_name(+ operation_name, strategy_type, target_url)+ else:+ # For crawl operations, use the new improved naming+ span_name = create_crawl_span_name(operation_name, ctx, gen_ai_endpoint)- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_STRATEGY, extraction_name- )+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()- if extraction_name == "LLMExtractionStrategy" and hasattr(- extraction_strategy, "provider"- ):- _, llm_model = extraction_strategy.provider.split("/")- span.set_attribute(- SemanticConvention.GEN_AI_REQUEST_MODEL, llm_model- )+ try:+ # Set comprehensive span attributes+ set_crawl_attributes(span, ctx, operation_name)- elif extraction_name == "CosineStrategy":- span.set_attribute(- SemanticConvention.GEN_AI_REQUEST_MODEL, "all-MiniLM-L6-v2"- )+ # Execute the original async function+ response = await wrapped(*args, **kwargs)++ # Handle async generators (streaming responses)+ if hasattr(response, "__aiter__"):+ return await _handle_async_generator(+ span,+ response,+ ctx,+ start_time,+ operation_name,+ capture_message_content,+ metrics,+ disable_metrics,+ )- elif extraction_name == "JsonCssExtractionStrategy" and hasattr(- extraction_strategy, "schema"- ):- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_SCHEMA,- str(extraction_strategy.schema),+ # Handle regular responses+ return await _handle_regular_response(+ span,+ response,+ ctx,+ start_time,+ operation_name,+ capture_message_content,+ metrics,+ disable_metrics,+ instance, # Add missing parameter+ args, # Add missing parameter)- span.set_status(Status(StatusCode.OK))+ except Exception as e:+ # Calculate duration even for errors+ end_time = time.time()+ duration_ms = (end_time - start_time) * 1000+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)- # Return original response- return response+ # Handle and log the exception+ handle_exception(span, e)+ logger.error("Error in async Crawl4AI operation: %s", e)- except Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ # Record error metrics if enabled+ if not disable_metrics and metrics:+ try:+ error_key = f"crawl4ai.{operation_name}.error"+ if error_key not in metrics:+ metrics[error_key] = 0+ metrics[error_key] += 1+ except Exception as metrics_error:+ logger.debug(+ "Error recording error metrics: %s", metrics_error+ )- # Return original response- return response+ # Re-raise the exception to maintain original behavior+ raise++ return async_wrapper()return wrapper+++async def _handle_regular_response(+ span,+ response,+ ctx,+ start_time,+ operation_name,+ capture_message_content,+ metrics,+ disable_metrics,+ instance,+ args,+):+ """Handle regular (non-streaming) async responses."""++ # Calculate duration for business intelligence+ end_time = time.time()+ duration_ms = (end_time - start_time) * 1000+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)++ # Process response based on operation type+ if operation_name == "extract":+ # Process extraction strategy response with LLM-specific metrics+ if hasattr(instance, "__class__"):+ # For extraction strategy operations, the instance is the strategy+ process_llm_extraction_response(span, instance, response, ctx)++ # Capture extraction content if enabled+ extraction_input = None+ if len(args) >= 3:+ extraction_input = args[2] # html content+ elif len(args) >= 2:+ extraction_input = args[1] # html_content parameter++ capture_extraction_content(+ span, extraction_input, response, capture_message_content+ )+ else:+ # Process crawl operations+ process_crawl_response(span, response, ctx)++ # Capture message content for crawl operations+ capture_message_content_if_enabled(span, ctx, response, capture_message_content)++ # Record metrics if enabled+ if not disable_metrics and metrics:+ try:+ # Record operation metrics+ metrics_key = f"crawl4ai.{operation_name}.duration"+ if metrics_key not in metrics:+ metrics[metrics_key] = []+ metrics[metrics_key].append(duration_ms)++ # Record success/failure metrics+ success_key = f"crawl4ai.{operation_name}.success"+ if success_key not in metrics:+ metrics[success_key] = 0++ # Determine success based on response+ is_success = True+ if hasattr(response, "success"):+ is_success = response.success+ elif isinstance(response, list):+ is_success = all(getattr(r, "success", True) for r in response)++ if is_success:+ metrics[success_key] += 1++ except Exception as metrics_error:+ logger.debug("Error recording metrics: %s", metrics_error)++ return response+++async def _handle_async_generator(+ span,+ response_generator,+ ctx,+ start_time,+ operation_name,+ capture_message_content,+ metrics,+ disable_metrics,+):+ """Handle async generator responses (streaming)."""++ class TrackedAsyncGenerator:+ """Async generator wrapper to track streaming results."""++ def __init__(+ self,+ generator,+ span,+ ctx,+ start_time,+ operation_name,+ capture_message_content,+ metrics,+ disable_metrics,+ ):+ self.generator = generator+ self.span = span+ self.ctx = ctx+ self.start_time = start_time+ self.operation_name = operation_name+ self.capture_message_content = capture_message_content+ self.metrics = metrics+ self.disable_metrics = disable_metrics+ self.results = []+ self.completed = False++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ result = await self.generator.__anext__()+ self.results.append(result)++ # Update span with current result+ if hasattr(result, "url"):+ self.span.set_attribute("gen_ai.crawl.current_url", result.url)++ return result++ except StopAsyncIteration:+ # Generator is complete, finalize metrics+ if not self.completed:+ await self._finalize_metrics()+ self.completed = True+ raise+ except Exception as e:+ # Handle errors in streaming+ handle_exception(self.span, e)+ logger.error("Error in streaming crawl operation: %s", e)+ raise++ async def _finalize_metrics(self):+ """Finalize metrics when streaming is complete."""+ end_time = time.time()+ duration_ms = (end_time - self.start_time) * 1000+ self.span.set_attribute("gen_ai.client.operation.duration", duration_ms)++ # Process aggregated results+ if self.results:+ process_crawl_response(self.span, self.results, self.ctx)++ # Capture message content for streaming+ capture_message_content_if_enabled(+ self.span, self.ctx, self.results, self.capture_message_content+ )++ # Record streaming metrics+ if not self.disable_metrics and self.metrics:+ try:+ metrics_key = f"crawl4ai.{self.operation_name}.stream.duration"+ if metrics_key not in self.metrics:+ self.metrics[metrics_key] = []+ self.metrics[metrics_key].append(duration_ms)++ # Record stream completion+ stream_key = f"crawl4ai.{self.operation_name}.stream.completed"+ if stream_key not in self.metrics:+ self.metrics[stream_key] = 0+ self.metrics[stream_key] += 1++ except Exception as metrics_error:+ logger.debug(+ "Error recording streaming metrics: %s", metrics_error+ )++ # Return the tracked generator+ return TrackedAsyncGenerator(+ response_generator,+ span,+ ctx,+ start_time,+ operation_name,+ capture_message_content,+ metrics,+ disable_metrics,+ )diff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py b/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.pyindex 30302966..f5a27f11 100644--- a/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py@@ -1,25 +1,32 @@# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument"""-Module for monitoring Crawl4AI calls.+Module for monitoring Crawl4AI synchronous calls.+Supports comprehensive 0.7.x operations with business intelligence and enhanced observability."""import logging-from opentelemetry.trace import SpanKind, Status, StatusCode-from opentelemetry.sdk.resources import (- SERVICE_NAME,- TELEMETRY_SDK_NAME,- DEPLOYMENT_ENVIRONMENT,+import time+from opentelemetry.trace import SpanKind+from opentelemetry import context as context_api++from openlit.__helpers import handle_exception+from openlit.instrumentation.crawl4ai.utils import (+ Crawl4AIInstrumentationContext,+ get_operation_name,+ create_crawl_span_name,+ set_crawl_attributes,+ process_crawl_response,+ capture_message_content_if_enabled,+ process_llm_extraction_response,+ create_extraction_span_name,+ capture_extraction_content,)-from openlit.__helpers import (- handle_exception,-)-from openlit.semcov import SemanticConvention# Initialize logger for logging potential issues and operationslogger = logging.getLogger(__name__)-def wrap_crawl(+def general_wrap(gen_ai_endpoint,version,environment,@@ -31,121 +38,166 @@ def wrap_crawl(disable_metrics,):"""- Generates a telemetry wrapper for chat completions to collect metrics.+ Generates a telemetry wrapper for Crawl4AI operations to collect comprehensive metrics.Args:gen_ai_endpoint: Endpoint identifier for logging and tracing.- version: Version of the monitoring package.+ version: Version of the Crawl4AI package.environment: Deployment environment (e.g., production, staging).- application_name: Name of the application using the Crawl4AI Agent.+ application_name: Name of the application using Crawl4AI.tracer: OpenTelemetry tracer for creating spans.pricing_info: Information used for calculating the cost of Crawl4AI usage.capture_message_content: Flag indicating whether to trace the actual content.+ metrics: OpenLIT metrics dictionary for performance tracking.+ disable_metrics: Flag to disable metrics collection.Returns:- A function that wraps the chat completions method to add telemetry.+ A function that wraps Crawl4AI methods to add comprehensive telemetry."""def wrapper(wrapped, instance, args, kwargs):"""- Wraps the 'chat.completions' API call to add telemetry.+ Wraps Crawl4AI operations to add comprehensive telemetry.- This collects metrics such as execution time, cost, and token usage, and handles errors- gracefully, adding details to the trace for observability.+ This collects metrics such as execution time, URL information, configuration details,+ and result metrics, while handling errors gracefully for enhanced observability.Args:- wrapped: The original 'chat.completions' method to be wrapped.+ wrapped: The original Crawl4AI method to be wrapped.instance: The instance of the class where the original method is defined.- args: Positional arguments for the 'chat.completions' method.- kwargs: Keyword arguments for the 'chat.completions' method.+ args: Positional arguments for the Crawl4AI method.+ kwargs: Keyword arguments for the Crawl4AI method.Returns:- The response from the original 'chat.completions' method.+ The response from the original Crawl4AI method."""- # pylint: disable=line-too-long- with tracer.start_as_current_span(- gen_ai_endpoint, kind=SpanKind.CLIENT- ) as span:- response = wrapped(*args, **kwargs)+ # Check if instrumentation is suppressed+ if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY):+ return wrapped(*args, **kwargs)++ # Get operation name and create context+ operation_name = get_operation_name(gen_ai_endpoint)+ ctx = Crawl4AIInstrumentationContext(+ instance, args, kwargs, version, environment, application_name+ )++ # Create span name following the framework guide pattern+ if operation_name == "extract":+ # For extraction operations, determine strategy type from the endpoint+ strategy_type = None+ if "llm" in gen_ai_endpoint:+ strategy_type = "llm"+ elif "css" in gen_ai_endpoint:+ strategy_type = "css"+ elif "xpath" in gen_ai_endpoint:+ strategy_type = "xpath"+ elif "cosine" in gen_ai_endpoint:+ strategy_type = "cosine"+ elif "regex" in gen_ai_endpoint:+ strategy_type = "regex"++ target_url = ctx.url if ctx.url != "unknown" else "content"+ span_name = create_extraction_span_name(+ operation_name, strategy_type, target_url+ )+ else:+ # For crawl operations, use the new improved naming+ span_name = create_crawl_span_name(operation_name, ctx, gen_ai_endpoint)++ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:+ start_time = time.time()try:- # Set base span attribues- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")- span.set_attribute(- SemanticConvention.GEN_AI_SYSTEM,- SemanticConvention.GEN_AI_SYSTEM_CRAWL4AI,- )- span.set_attribute(- SemanticConvention.GEN_AI_OPERATION,- SemanticConvention.GEN_AI_OPERATION_TYPE_AGENT,- )- span.set_attribute(SemanticConvention.GEN_AI_ENDPOINT, gen_ai_endpoint)- span.set_attribute(SERVICE_NAME, application_name)- span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_TYPE,- SemanticConvention.GEN_AI_AGENT_TYPE_BROWSER,- )- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_ENABLE_CACHE,- not kwargs.get("disable_cache", False),- )-- url = (- kwargs.get("url")- if "url" in kwargs- else str(args[0])- if args- else None- )- if url is not None:- span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, url)-- extraction_strategy = kwargs.get(- "extraction_strategy", "NoExtractionStrategy"- )- extraction_name = (- extraction_strategy.name- if hasattr(extraction_strategy, "name")- else extraction_strategy- )-- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_STRATEGY, extraction_name- )-- if extraction_name == "LLMExtractionStrategy" and hasattr(- extraction_strategy, "provider"- ):- _, llm_model = extraction_strategy.provider.split("/")- span.set_attribute(- SemanticConvention.GEN_AI_REQUEST_MODEL, llm_model- )-- elif extraction_name == "CosineStrategy":- span.set_attribute(- SemanticConvention.GEN_AI_REQUEST_MODEL, "all-MiniLM-L6-v2"+ # Set comprehensive span attributes+ set_crawl_attributes(span, ctx, operation_name)++ # Execute the original function+ response = wrapped(*args, **kwargs)++ # Calculate duration for business intelligence+ end_time = time.time()+ duration_ms = (end_time - start_time) * 1000+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)++ # Process response based on operation type+ if operation_name == "extract":+ # Process extraction strategy response with LLM-specific metrics+ if hasattr(instance, "__class__"):+ # For extraction strategy operations, the instance is the strategy+ process_llm_extraction_response(span, instance, response, ctx)++ # Capture extraction content if enabled+ extraction_input = None+ if len(args) >= 3:+ extraction_input = args[2] # html content+ elif len(args) >= 2:+ extraction_input = args[1] # html_content parameter++ capture_extraction_content(+ span, extraction_input, response, capture_message_content)+ else:+ # Process crawl operations+ process_crawl_response(span, response, ctx)- elif extraction_name == "JsonCssExtractionStrategy" and hasattr(- extraction_strategy, "schema"- ):- span.set_attribute(- SemanticConvention.GEN_AI_AGENT_SCHEMA,- str(extraction_strategy.schema),+ # Capture message content for crawl operations+ capture_message_content_if_enabled(+ span, ctx, response, capture_message_content)- span.set_status(Status(StatusCode.OK))+ # Record metrics if enabled+ if not disable_metrics and metrics:+ try:+ # Record operation metrics+ metrics_key = f"crawl4ai.{operation_name}.duration"+ if metrics_key not in metrics:+ metrics[metrics_key] = []+ metrics[metrics_key].append(duration_ms)++ # Record success/failure metrics+ success_key = f"crawl4ai.{operation_name}.success"+ if success_key not in metrics:+ metrics[success_key] = 0++ # Determine success based on response+ is_success = True+ if hasattr(response, "success"):+ is_success = response.success+ elif isinstance(response, list):+ is_success = all(+ getattr(r, "success", True) for r in response+ )++ if is_success:+ metrics[success_key] += 1++ except Exception as metrics_error:+ logger.debug("Error recording metrics: %s", metrics_error)- # Return original responsereturn responseexcept Exception as e:- handle_exception(span, e)- logger.error("Error in trace creation: %s", e)+ # Calculate duration even for errors+ end_time = time.time()+ duration_ms = (end_time - start_time) * 1000+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)- # Return original response- return response+ # Handle and log the exception+ handle_exception(span, e)+ logger.error("Error in Crawl4AI operation: %s", e)++ # Record error metrics if enabled+ if not disable_metrics and metrics:+ try:+ error_key = f"crawl4ai.{operation_name}.error"+ if error_key not in metrics:+ metrics[error_key] = 0+ metrics[error_key] += 1+ except Exception as metrics_error:+ logger.debug("Error recording error metrics: %s", metrics_error)++ # Re-raise the exception to maintain original behavior+ raisereturn wrapperdiff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/utils.py b/sdk/python/src/openlit/instrumentation/crawl4ai/utils.pynew file mode 100644index 00000000..e7f34613--- /dev/null+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/utils.py@@ -0,0 +1,828 @@+"""+Utilities for Crawl4AI instrumentation.+Includes operation mapping, telemetry processing, and context caching for 0.7.x support.+"""++import logging+from typing import Any, Optional, List++from opentelemetry.trace import Status, StatusCode+from opentelemetry.sdk.resources import (+ SERVICE_NAME,+ TELEMETRY_SDK_NAME,+ DEPLOYMENT_ENVIRONMENT,+)++from openlit.__helpers import handle_exception+from openlit.semcov import SemanticConvention++# Initialize logger for logging potential issues and operations+logger = logging.getLogger(__name__)++# Operation mapping for Crawl4AI endpoints following the framework guide pattern+CRAWL_OPERATION_MAP = {+ "crawl4ai.arun": "crawl",+ "crawl4ai.arun_many": "crawl", # Use same operation name for consistency+ "crawl4ai.run": "crawl",+ "crawl4ai.deep_crawl.bfs": "crawl_deep",+ "crawl4ai.deep_crawl.dfs": "crawl_deep",+ "crawl4ai.deep_crawl.best_first": "crawl_deep",+ "crawl4ai.extract": "extract",+ "crawl4ai.scrape": "scrape",+ # LLM and Extraction Strategy Operations (0.7.x) - All map to "extract"+ "crawl4ai.extraction.llm.extract": "extract",+ "crawl4ai.extraction.llm.run": "extract",+ "crawl4ai.extraction.css.extract": "extract",+ "crawl4ai.extraction.xpath.extract": "extract",+ "crawl4ai.extraction.cosine.extract": "extract",+ "crawl4ai.extraction.regex.extract": "extract",+ "crawl4ai.extraction.lxml.extract": "extract",+ # Processing and Analysis Operations+ "crawl4ai.aprocess_html": "process_html",+ "crawl4ai.aseed_urls": "seed_urls",+ # Monitor and Analysis+ "crawl4ai.monitor.update_task": "monitor_task",+ "crawl4ai.monitor.get_summary": "monitor_summary",+}+++class Crawl4AIInstrumentationContext:+ """Context object to cache expensive extractions and reduce performance overhead."""++ __slots__ = (+ "instance",+ "args",+ "kwargs",+ "version",+ "environment",+ "application_name",+ "_url",+ "_urls",+ "_config",+ "_browser_config",+ "_session_id",+ "_cache_mode",+ "_extraction_strategy",+ "_operation_type",+ )++ def __init__(self, instance, args, kwargs, version, environment, application_name):+ self.instance = instance+ self.args = args+ self.kwargs = kwargs+ self.version = version+ self.environment = environment+ self.application_name = application_name++ # Cache expensive operations with lazy loading+ self._url = None+ self._urls = None+ self._config = None+ self._browser_config = None+ self._session_id = None+ self._cache_mode = None+ self._extraction_strategy = None+ self._operation_type = None++ @property+ def url(self) -> Optional[str]:+ """Get primary URL with caching."""+ if self._url is None:+ if len(self.args) > 0:+ self._url = str(self.args[0])+ elif "url" in self.kwargs:+ self._url = str(self.kwargs["url"])+ else:+ self._url = "unknown"+ return self._url++ @property+ def urls(self) -> Optional[List[str]]:+ """Get URLs list for batch operations with caching."""+ if self._urls is None:+ if len(self.args) > 0 and isinstance(self.args[0], list):+ self._urls = [str(url) for url in self.args[0]]+ elif "urls" in self.kwargs and isinstance(self.kwargs["urls"], list):+ self._urls = [str(url) for url in self.kwargs["urls"]]+ else:+ self._urls = []+ return self._urls++ @property+ def config(self) -> Optional[Any]:+ """Get CrawlerRunConfig with caching."""+ if self._config is None:+ self._config = self.kwargs.get("config")+ return self._config++ @property+ def browser_config(self) -> Optional[Any]:+ """Get BrowserConfig with caching."""+ if self._browser_config is None:+ if hasattr(self.instance, "config"):+ self._browser_config = self.instance.config+ return self._browser_config++ @property+ def session_id(self) -> Optional[str]:+ """Get session ID with caching."""+ if self._session_id is None:+ config = self.config+ if config and hasattr(config, "session_id"):+ self._session_id = config.session_id+ return self._session_id++ @property+ def cache_mode(self) -> Optional[str]:+ """Get cache mode with caching."""+ if self._cache_mode is None:+ config = self.config+ if config and hasattr(config, "cache_mode"):+ self._cache_mode = str(config.cache_mode)+ return self._cache_mode++ @property+ def extraction_strategy(self) -> Optional[str]:+ """Get extraction strategy with caching."""+ if self._extraction_strategy is None:+ config = self.config+ if config and hasattr(config, "extraction_strategy"):+ strategy = config.extraction_strategy+ if strategy:+ self._extraction_strategy = strategy.__class__.__name__+ return self._extraction_strategy+++def get_operation_name(endpoint: str) -> str:+ """Get operation name from endpoint using the operation map."""+ return CRAWL_OPERATION_MAP.get(endpoint, "crawl")+++def create_span_name(operation_name: str, target: str) -> str:+ """Create span name following the '{operation_type} {operation_name}' format."""+ return f"{operation_name} {target}"+++def create_crawl_span_name(+ operation_name: str, ctx: Crawl4AIInstrumentationContext, gen_ai_endpoint: str+) -> str:+ """Create descriptive span name for crawl operations, handling batch operations specially."""++ # Check if this is a batch operation+ is_batch = "arun_many" in gen_ai_endpoint or (ctx.urls and len(ctx.urls) > 1)++ if is_batch and ctx.urls:+ # For batch operations, show the URLs but limit length+ if len(ctx.urls) <= 3:+ # Show all URLs for small batches+ urls_str = ", ".join(ctx.urls)+ if len(urls_str) <= 80: # Keep reasonable length+ return f"{operation_name} [{urls_str}]"++ # For larger batches or long URLs, show count and sample+ sample_urls = ctx.urls[:2]+ remaining_count = len(ctx.urls) - 2+ if remaining_count > 0:+ urls_sample = ", ".join(sample_urls) + f" +{remaining_count} more"+ else:+ urls_sample = ", ".join(sample_urls)++ return f"{operation_name} [{urls_sample}]"++ else:+ # Single URL operation+ target_url = ctx.url if ctx.url != "unknown" else "unknown"+ return f"{operation_name} {target_url}"+++def set_crawl_attributes(+ span, ctx: Crawl4AIInstrumentationContext, operation_name: str+):+ """Set comprehensive Crawl4AI span attributes."""++ # Core framework attributes+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")+ span.set_attribute(+ SemanticConvention.GEN_AI_SYSTEM, SemanticConvention.GEN_AI_SYSTEM_CRAWL4AI+ )++ # Map operation to semantic convention+ if operation_name == "crawl":+ span.set_attribute(+ SemanticConvention.GEN_AI_OPERATION,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL,+ )+ elif operation_name == "crawl_batch":+ span.set_attribute(+ SemanticConvention.GEN_AI_OPERATION,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL, # Fixed: use CRAWL not CRAWL_BATCH+ )+ elif operation_name == "crawl_deep":+ span.set_attribute(+ SemanticConvention.GEN_AI_OPERATION,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL_DEEP,+ )+ elif operation_name == "extract":+ span.set_attribute(+ SemanticConvention.GEN_AI_OPERATION,+ SemanticConvention.GEN_AI_OPERATION_TYPE_EXTRACT,+ )+ elif operation_name == "scrape":+ span.set_attribute(+ SemanticConvention.GEN_AI_OPERATION,+ SemanticConvention.GEN_AI_OPERATION_TYPE_SCRAPE,+ )+ elif operation_name in ["process_html", "seed_urls"]:+ span.set_attribute(+ SemanticConvention.GEN_AI_OPERATION,+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL, # Processing operations use crawl type+ )+ else:+ # Default fallback for unknown operations+ span.set_attribute(+ SemanticConvention.GEN_AI_OPERATION,+ SemanticConvention.GEN_AI_OPERATION_TYPE_EXTRACT, # Changed: use extract instead of agent+ )++ # Environment attributes+ span.set_attribute(SERVICE_NAME, ctx.application_name)+ span.set_attribute(DEPLOYMENT_ENVIRONMENT, ctx.environment)+ span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION, ctx.version)++ # Agent type for browser-based crawling+ span.set_attribute(+ SemanticConvention.GEN_AI_AGENT_TYPE,+ SemanticConvention.GEN_AI_AGENT_TYPE_BROWSER,+ )++ # URL information+ if operation_name == "crawl_batch" and ctx.urls:+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_URL_COUNT, len(ctx.urls))+ # Set primary URL as browse URL+ if ctx.urls:+ span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, ctx.urls[0])+ elif ctx.url and ctx.url != "unknown":+ span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, ctx.url)++ # Configuration attributes+ if ctx.session_id:+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_SESSION_ID, ctx.session_id)++ if ctx.cache_mode:+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_CACHE_MODE, ctx.cache_mode)++ if ctx.extraction_strategy:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_EXTRACTION_STRATEGY, ctx.extraction_strategy+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_AGENT_STRATEGY, ctx.extraction_strategy+ )++ # Browser configuration+ browser_config = ctx.browser_config+ if browser_config:+ if hasattr(browser_config, "browser_type"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_BROWSER_TYPE,+ browser_config.browser_type,+ )++ if hasattr(browser_config, "headless"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_HEADLESS, browser_config.headless+ )++ if hasattr(browser_config, "viewport_width"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_VIEWPORT_WIDTH,+ browser_config.viewport_width,+ )++ if hasattr(browser_config, "viewport_height"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_VIEWPORT_HEIGHT,+ browser_config.viewport_height,+ )++ if hasattr(browser_config, "user_agent"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_USER_AGENT,+ str(browser_config.user_agent),+ )++ # Run configuration+ config = ctx.config+ if config:+ if hasattr(config, "word_count_threshold"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_WORD_COUNT_THRESHOLD,+ config.word_count_threshold,+ )++ if hasattr(config, "css_selector") and config.css_selector:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_CSS_SELECTOR, config.css_selector+ )++ if hasattr(config, "excluded_tags") and config.excluded_tags:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_EXCLUDED_TAGS, str(config.excluded_tags)+ )++ if hasattr(config, "screenshot"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_SCREENSHOT, config.screenshot+ )++ if hasattr(config, "pdf"):+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_PDF, config.pdf)++ if hasattr(config, "wait_for") and config.wait_for:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_WAIT_FOR, str(config.wait_for)+ )++ if hasattr(config, "page_timeout"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_PAGE_TIMEOUT, config.page_timeout+ )++ if hasattr(config, "js_code") and config.js_code:+ js_code_str = (+ str(config.js_code)+ if not isinstance(config.js_code, list)+ else "; ".join(config.js_code)+ )+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_JS_CODE, js_code_str)++ # Deep crawling attributes+ if hasattr(config, "deep_crawl_strategy") and config.deep_crawl_strategy:+ strategy_name = config.deep_crawl_strategy.__class__.__name__+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_DEEP_STRATEGY, strategy_name+ )++ if hasattr(config.deep_crawl_strategy, "max_depth"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_DEPTH,+ config.deep_crawl_strategy.max_depth,+ )+++def process_crawl_response(+ span, response, ctx: Crawl4AIInstrumentationContext, **kwargs+):+ """Process Crawl4AI response and extract business intelligence."""++ try:+ if response is None:+ return++ # Handle single result vs batch results+ if isinstance(response, list):+ # Batch crawling results+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_URL_COUNT, len(response))++ # Aggregate statistics from batch+ total_html_length = 0+ total_markdown_length = 0+ total_links = 0+ total_images = 0+ success_count = 0++ for result in response:+ if hasattr(result, "success") and result.success:+ success_count += 1++ if hasattr(result, "html") and result.html:+ total_html_length += len(result.html)++ if hasattr(result, "markdown") and result.markdown:+ total_markdown_length += len(result.markdown)++ if hasattr(result, "links") and result.links:+ total_links += len(result.links)++ if (+ hasattr(result, "media")+ and result.media+ and hasattr(result.media, "images")+ ):+ total_images += len(result.media.images)++ # Set aggregate metrics+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_SUCCESS,+ success_count == len(response),+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_HTML_LENGTH, total_html_length+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_MARKDOWN_LENGTH,+ total_markdown_length,+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_LINKS_COUNT, total_links+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_IMAGES_COUNT, total_images+ )++ else:+ # Single result+ if hasattr(response, "success"):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_SUCCESS, response.success+ )++ if hasattr(response, "status_code") and response.status_code is not None:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_STATUS_CODE,+ response.status_code,+ )++ if hasattr(response, "html") and response.html:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_HTML_LENGTH,+ len(response.html),+ )++ if hasattr(response, "markdown") and response.markdown:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_MARKDOWN_LENGTH,+ len(response.markdown),+ )++ if hasattr(response, "links") and response.links:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_LINKS_COUNT,+ len(response.links),+ )++ if (+ hasattr(response, "media")+ and response.media+ and hasattr(response.media, "images")+ ):+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_IMAGES_COUNT,+ len(response.media.images),+ )++ if hasattr(response, "redirected_url") and response.redirected_url:+ span.set_attribute(+ SemanticConvention.GEN_AI_CRAWL_RESULT_REDIRECTED_URL,+ response.redirected_url,+ )++ # Set success status+ span.set_status(Status(StatusCode.OK))++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error processing crawl response: %s", e)+++def format_content(content, max_length: int = 1000) -> str:+ """Format content for tracing with length limits."""+ if content is None:+ return ""++ content_str = str(content)+ if len(content_str) > max_length:+ return content_str[:max_length] + "..."+ return content_str+++def capture_message_content_if_enabled(+ span, ctx: Crawl4AIInstrumentationContext, response, capture_message_content: bool+):+ """Capture input/output content if message content capture is enabled."""++ if not capture_message_content:+ return++ try:+ # Capture input URL(s) as prompt+ if ctx.url and ctx.url != "unknown":+ span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, ctx.url)+ elif ctx.urls:+ urls_str = ", ".join(ctx.urls[:5]) # Limit to first 5 URLs+ if len(ctx.urls) > 5:+ urls_str += f" and {len(ctx.urls) - 5} more"+ span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, urls_str)++ # Capture output content based on response type+ if response is None:+ return++ if isinstance(response, list):+ # For batch responses, capture summary+ completion_summary = f"Crawled {len(response)} URLs"+ span.set_attribute(+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, completion_summary+ )+ else:+ # For single responses, capture markdown content (limited)+ if hasattr(response, "markdown") and response.markdown:+ formatted_content = format_content(response.markdown, max_length=2000)+ span.set_attribute(+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, formatted_content+ )+ elif hasattr(response, "html") and response.html:+ formatted_content = format_content(response.html, max_length=1000)+ span.set_attribute(+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, formatted_content+ )++ except Exception as e:+ logger.error("Error capturing message content: %s", e)+++def process_llm_extraction_response(+ span, extraction_strategy, response, ctx: Crawl4AIInstrumentationContext+):+ """Process LLM extraction strategy response and capture comprehensive metrics."""++ try:+ # Capture LLM extraction strategy details+ if hasattr(extraction_strategy, "__class__"):+ strategy_name = extraction_strategy.__class__.__name__++ # Map strategy class to semantic convention+ if "LLM" in strategy_name:+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_LLM,+ )+ elif "Css" in strategy_name:+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_CSS,+ )+ elif "XPath" in strategy_name:+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_XPATH,+ )+ elif "Cosine" in strategy_name:+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_COSINE,+ )+ elif "Regex" in strategy_name:+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_REGEX,+ )++ # Process LLM-specific attributes for LLMExtractionStrategy+ if hasattr(extraction_strategy, "llm_config"):+ llm_config = extraction_strategy.llm_config+ if llm_config:+ # Provider information+ if hasattr(llm_config, "provider"):+ provider_parts = str(llm_config.provider).split("/")+ if len(provider_parts) >= 2:+ span.set_attribute(+ SemanticConvention.GEN_AI_LLM_PROVIDER, provider_parts[0]+ )+ span.set_attribute(+ SemanticConvention.GEN_AI_LLM_MODEL, provider_parts[1]+ )+ else:+ span.set_attribute(+ SemanticConvention.GEN_AI_LLM_PROVIDER,+ str(llm_config.provider),+ )++ # Base URL if available+ if hasattr(llm_config, "base_url") and llm_config.base_url:+ span.set_attribute(+ SemanticConvention.GEN_AI_LLM_BASE_URL, llm_config.base_url+ )++ # Extraction configuration+ if hasattr(extraction_strategy, "extraction_type"):+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_TYPE,+ extraction_strategy.extraction_type,+ )++ if (+ hasattr(extraction_strategy, "instruction")+ and extraction_strategy.instruction+ ):+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_INSTRUCTION,+ format_content(extraction_strategy.instruction, max_length=500),+ )++ if hasattr(extraction_strategy, "input_format"):+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_INPUT_FORMAT,+ extraction_strategy.input_format,+ )++ # Chunking configuration+ if hasattr(extraction_strategy, "apply_chunking"):+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_APPLY_CHUNKING,+ extraction_strategy.apply_chunking,+ )++ if hasattr(extraction_strategy, "chunk_token_threshold"):+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_CHUNK_TOKEN_THRESHOLD,+ extraction_strategy.chunk_token_threshold,+ )++ if hasattr(extraction_strategy, "overlap_rate"):+ span.set_attribute(+ SemanticConvention.GEN_AI_EXTRACTION_OVERLAP_RATE,+ extraction_strategy.overlap_rate,+ )++ # LLM model parameters+ if (+ hasattr(extraction_strategy, "extra_args")+ and extraction_strategy.extra_args+ ):+ extra_args = extraction_strategy.extra_args+ if "temperature" in extra_args:+ span.set_attribute(+ SemanticConvention.GEN_AI_LLM_TEMPERATURE, extra_args["temperature"]+ )+ if "max_tokens" in extra_args:+ span.set_attribute(+ SemanticConvention.GEN_AI_LLM_MAX_TOKENS, extra_args["max_tokens"]+ )+ if "top_p" in extra_args:+ span.set_attribute(+ SemanticConvention.GEN_AI_LLM_TOP_P, extra_args["top_p"]+ )++ # Token usage tracking (critical for business intelligence)+ if (+ hasattr(extraction_strategy, "total_usage")+ and extraction_strategy.total_usage+ ):+ total_usage = extraction_strategy.total_usage+ if hasattr(total_usage, "prompt_tokens"):+ span.set_attribute(+ SemanticConvention.GEN_AI_TOKEN_USAGE_INPUT,+ total_usage.prompt_tokens,+ )+ if hasattr(total_usage, "completion_tokens"):+ span.set_attribute(+ SemanticConvention.GEN_AI_TOKEN_USAGE_OUTPUT,+ total_usage.completion_tokens,+ )+ if hasattr(total_usage, "total_tokens"):+ span.set_attribute(+ SemanticConvention.GEN_AI_TOKEN_USAGE_TOTAL,+ total_usage.total_tokens,+ )++ # Individual chunk usage+ if hasattr(extraction_strategy, "usages") and extraction_strategy.usages:+ chunk_count = len(extraction_strategy.usages)+ span.set_attribute(SemanticConvention.GEN_AI_TOKEN_CHUNK_COUNT, chunk_count)++ # Aggregate chunk usage information+ total_chunk_tokens = 0+ for usage in extraction_strategy.usages:+ if hasattr(usage, "total_tokens"):+ total_chunk_tokens += usage.total_tokens+ if total_chunk_tokens > 0:+ span.set_attribute(+ SemanticConvention.GEN_AI_TOKEN_CHUNK_USAGE, total_chunk_tokens+ )++ # Extraction success/failure+ if response is not None:+ span.set_attribute(SemanticConvention.GEN_AI_EXTRACTION_SUCCESS, True)++ # Count extracted items for business intelligence+ if isinstance(response, list):+ span.set_attribute("gen_ai.extraction.items_count", len(response))+ elif isinstance(response, dict):+ span.set_attribute("gen_ai.extraction.items_count", 1)+ else:+ span.set_attribute(SemanticConvention.GEN_AI_EXTRACTION_SUCCESS, False)++ except Exception as e:+ handle_exception(span, e)+ logger.error("Error processing LLM extraction response: %s", e)+++def process_crawler_monitor_metrics(span, monitor_data):+ """Process CrawlerMonitor metrics and integrate with OpenLIT telemetry."""++ try:+ if not monitor_data:+ return++ # Task-level metrics+ if "task_id" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_TASK_ID, monitor_data["task_id"]+ )++ if "status" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_TASK_STATUS, monitor_data["status"]+ )++ # Memory metrics (business intelligence)+ if "memory_usage" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_MEMORY_USAGE,+ monitor_data["memory_usage"],+ )++ if "peak_memory" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_PEAK_MEMORY,+ monitor_data["peak_memory"],+ )++ # Performance metrics+ if "retry_count" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_RETRY_COUNT,+ monitor_data["retry_count"],+ )++ if "wait_time" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_WAIT_TIME, monitor_data["wait_time"]+ )++ # Queue and completion metrics+ if "queue_size" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_QUEUE_SIZE, monitor_data["queue_size"]+ )++ if "completion_rate" in monitor_data:+ span.set_attribute(+ SemanticConvention.GEN_AI_MONITOR_COMPLETION_RATE,+ monitor_data["completion_rate"],+ )++ except Exception as e:+ logger.error("Error processing crawler monitor metrics: %s", e)+++def create_extraction_span_name(+ operation_type: str, strategy_type: str, target: str+) -> str:+ """Create span name for extraction operations following the 'extract {target}' format."""+ # Always use "extract" as the operation name regardless of strategy type+ return f"extract {target}"+++def capture_extraction_content(+ span, extraction_input, extraction_output, capture_message_content: bool+):+ """Capture extraction input/output content if enabled."""++ if not capture_message_content:+ return++ try:+ # Capture extraction input as prompt (URL or content snippet)+ if extraction_input:+ if isinstance(extraction_input, str):+ input_content = format_content(extraction_input, max_length=1000)+ span.set_attribute(+ SemanticConvention.GEN_AI_CONTENT_PROMPT, input_content+ )+ elif hasattr(extraction_input, "__iter__"):+ # Handle list/tuple of inputs+ combined_input = " ".join(str(item)[:200] for item in extraction_input)+ input_content = format_content(combined_input, max_length=1000)+ span.set_attribute(+ SemanticConvention.GEN_AI_CONTENT_PROMPT, input_content+ )++ # Capture extraction output as completion+ if extraction_output is not None:+ if isinstance(extraction_output, (list, dict)):+ # Format structured output+ output_summary = f"Extracted {len(extraction_output) if hasattr(extraction_output, '__len__') else 1} items"+ span.set_attribute(+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, output_summary+ )+ else:+ # Handle string output+ output_content = format_content(str(extraction_output), max_length=1000)+ span.set_attribute(+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, output_content+ )++ except Exception as e:+ logger.error("Error capturing extraction content: %s", e)diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.pyindex b6c7c03e..6e31699f 100644--- a/sdk/python/src/openlit/semcov/__init__.py+++ b/sdk/python/src/openlit/semcov/__init__.py@@ -421,6 +421,107 @@ class SemanticConvention:GEN_AI_AGENT_TYPE_BROWSER = "browser"+ # Crawl4AI Specific Attributes (0.7.x Support)+ GEN_AI_CRAWL_URL_COUNT = "gen_ai.crawl.url_count"+ GEN_AI_CRAWL_DEPTH = "gen_ai.crawl.depth"+ GEN_AI_CRAWL_SESSION_ID = "gen_ai.crawl.session_id"+ GEN_AI_CRAWL_CACHE_MODE = "gen_ai.crawl.cache_mode"+ GEN_AI_CRAWL_EXTRACTION_STRATEGY = "gen_ai.crawl.extraction_strategy"+ GEN_AI_CRAWL_SCRAPING_STRATEGY = "gen_ai.crawl.scraping_strategy"+ GEN_AI_CRAWL_BROWSER_TYPE = "gen_ai.crawl.browser_type"+ GEN_AI_CRAWL_HEADLESS = "gen_ai.crawl.headless"+ GEN_AI_CRAWL_VIEWPORT_WIDTH = "gen_ai.crawl.viewport_width"+ GEN_AI_CRAWL_VIEWPORT_HEIGHT = "gen_ai.crawl.viewport_height"+ GEN_AI_CRAWL_USER_AGENT = "gen_ai.crawl.user_agent"+ GEN_AI_CRAWL_WORD_COUNT_THRESHOLD = "gen_ai.crawl.word_count_threshold"+ GEN_AI_CRAWL_CSS_SELECTOR = "gen_ai.crawl.css_selector"+ GEN_AI_CRAWL_EXCLUDED_TAGS = "gen_ai.crawl.excluded_tags"+ GEN_AI_CRAWL_SCREENSHOT = "gen_ai.crawl.screenshot"+ GEN_AI_CRAWL_PDF = "gen_ai.crawl.pdf"+ GEN_AI_CRAWL_WAIT_FOR = "gen_ai.crawl.wait_for"+ GEN_AI_CRAWL_PAGE_TIMEOUT = "gen_ai.crawl.page_timeout"+ GEN_AI_CRAWL_JS_CODE = "gen_ai.crawl.js_code"+ GEN_AI_CRAWL_RESULT_SUCCESS = "gen_ai.crawl.result.success"+ GEN_AI_CRAWL_RESULT_STATUS_CODE = "gen_ai.crawl.result.status_code"+ GEN_AI_CRAWL_RESULT_HTML_LENGTH = "gen_ai.crawl.result.html_length"+ GEN_AI_CRAWL_RESULT_MARKDOWN_LENGTH = "gen_ai.crawl.result.markdown_length"+ GEN_AI_CRAWL_RESULT_LINKS_COUNT = "gen_ai.crawl.result.links_count"+ GEN_AI_CRAWL_RESULT_IMAGES_COUNT = "gen_ai.crawl.result.images_count"+ GEN_AI_CRAWL_RESULT_REDIRECTED_URL = "gen_ai.crawl.result.redirected_url"+ GEN_AI_CRAWL_DEEP_STRATEGY = "gen_ai.crawl.deep_strategy"+ GEN_AI_CRAWL_PROXY_CONFIG = "gen_ai.crawl.proxy_config"++ # Crawl4AI LLM Extraction Attributes (0.7.x)+ GEN_AI_EXTRACTION_STRATEGY_TYPE = "gen_ai.extraction.strategy.type"+ GEN_AI_EXTRACTION_TYPE = "gen_ai.extraction.type"+ GEN_AI_EXTRACTION_INSTRUCTION = "gen_ai.extraction.instruction"+ GEN_AI_EXTRACTION_SCHEMA = "gen_ai.extraction.schema"+ GEN_AI_EXTRACTION_INPUT_FORMAT = "gen_ai.extraction.input_format"+ GEN_AI_EXTRACTION_CHUNK_COUNT = "gen_ai.extraction.chunk_count"+ GEN_AI_EXTRACTION_CHUNK_TOKEN_THRESHOLD = "gen_ai.extraction.chunk_token_threshold"+ GEN_AI_EXTRACTION_OVERLAP_RATE = "gen_ai.extraction.overlap_rate"+ GEN_AI_EXTRACTION_APPLY_CHUNKING = "gen_ai.extraction.apply_chunking"+ GEN_AI_EXTRACTION_SUCCESS = "gen_ai.extraction.success"+ GEN_AI_EXTRACTION_ERROR = "gen_ai.extraction.error"++ # LLM Provider and Model Information+ GEN_AI_LLM_PROVIDER = "gen_ai.llm.provider"+ GEN_AI_LLM_MODEL = "gen_ai.llm.model"+ GEN_AI_LLM_BASE_URL = "gen_ai.llm.base_url"+ GEN_AI_LLM_TEMPERATURE = "gen_ai.llm.temperature"+ GEN_AI_LLM_MAX_TOKENS = "gen_ai.llm.max_tokens"+ GEN_AI_LLM_TOP_P = "gen_ai.llm.top_p"++ # Token Usage and Cost Tracking (Enhanced Business Intelligence)+ GEN_AI_TOKEN_USAGE_INPUT = "gen_ai.token.usage.input"+ GEN_AI_TOKEN_USAGE_OUTPUT = "gen_ai.token.usage.output"+ GEN_AI_TOKEN_USAGE_TOTAL = "gen_ai.token.usage.total"+ GEN_AI_TOKEN_COST_INPUT = "gen_ai.token.cost.input"+ GEN_AI_TOKEN_COST_OUTPUT = "gen_ai.token.cost.output"+ GEN_AI_TOKEN_COST_TOTAL = "gen_ai.token.cost.total"+ GEN_AI_TOKEN_CHUNK_USAGE = "gen_ai.token.chunk_usage"+ GEN_AI_TOKEN_CHUNK_COUNT = "gen_ai.token.chunk_count"++ # CrawlerMonitor Integration Attributes+ GEN_AI_MONITOR_TASK_ID = "gen_ai.monitor.task_id"+ GEN_AI_MONITOR_TASK_STATUS = "gen_ai.monitor.task_status"+ GEN_AI_MONITOR_MEMORY_USAGE = "gen_ai.monitor.memory_usage"+ GEN_AI_MONITOR_PEAK_MEMORY = "gen_ai.monitor.peak_memory"+ GEN_AI_MONITOR_RETRY_COUNT = "gen_ai.monitor.retry_count"+ GEN_AI_MONITOR_WAIT_TIME = "gen_ai.monitor.wait_time"+ GEN_AI_MONITOR_QUEUE_SIZE = "gen_ai.monitor.queue_size"+ GEN_AI_MONITOR_COMPLETION_RATE = "gen_ai.monitor.completion_rate"++ # Extraction Strategy Types+ GEN_AI_EXTRACTION_STRATEGY_LLM = "llm"+ GEN_AI_EXTRACTION_STRATEGY_CSS = "css"+ GEN_AI_EXTRACTION_STRATEGY_XPATH = "xpath"+ GEN_AI_EXTRACTION_STRATEGY_COSINE = "cosine"+ GEN_AI_EXTRACTION_STRATEGY_REGEX = "regex"+ GEN_AI_EXTRACTION_STRATEGY_LXML = "lxml"++ # LLM Extraction Types+ GEN_AI_EXTRACTION_TYPE_SCHEMA = "schema"+ GEN_AI_EXTRACTION_TYPE_BLOCK = "block"++ # Monitor Task Status Values+ GEN_AI_MONITOR_STATUS_PENDING = "pending"+ GEN_AI_MONITOR_STATUS_RUNNING = "running"+ GEN_AI_MONITOR_STATUS_COMPLETED = "completed"+ GEN_AI_MONITOR_STATUS_FAILED = "failed"+ GEN_AI_MONITOR_STATUS_RETRYING = "retrying"++ # Crawl4AI Operation Types (reusing existing pattern)+ GEN_AI_OPERATION_TYPE_CRAWL = "crawl"+ GEN_AI_OPERATION_TYPE_CRAWL_DEEP = "crawl_deep"+ GEN_AI_OPERATION_TYPE_EXTRACT = "extract"+ GEN_AI_OPERATION_TYPE_EXTRACT_LLM = "extract_llm"+ GEN_AI_OPERATION_TYPE_EXTRACT_CSS = "extract_css"+ GEN_AI_OPERATION_TYPE_EXTRACT_XPATH = "extract_xpath"+ GEN_AI_OPERATION_TYPE_EXTRACT_COSINE = "extract_cosine"+ GEN_AI_OPERATION_TYPE_EXTRACT_REGEX = "extract_regex"+ GEN_AI_OPERATION_TYPE_SCRAPE = "scrape"+# GPUGPU_INDEX = "gpu.index"GPU_UUID = "gpu.uuid"