Skip to content

Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.

[Feat]: Refactor Crawl4AI OpenTelemetry Monitoring Instrumentaion (#825)

Patcher35defdf7eb33
diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml
index b0425161..8dfb2886 100644
--- a/sdk/python/pyproject.toml
+++ b/sdk/python/pyproject.toml
@@ -1,6 +1,6 @@
[tool.poetry]
name = "openlit"
-version = "1.34.37"
+version = "1.34.38"
description = "OpenTelemetry-native Auto instrumentation library for monitoring LLM Applications and GPUs, facilitating the integration of observability into your GenAI-driven projects"
authors = ["OpenLIT"]
license = "Apache-2.0"
diff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.py b/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.py
index 4a613870..b3612d46 100644
--- a/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.py
+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/__init__.py
@@ -1,20 +1,25 @@
# pylint: disable=useless-return, bad-staticmethod-argument, disable=duplicate-code
-"""Initializer of Auto Instrumentation of Crawl4AI Functions"""
+"""
+Initializer of Auto Instrumentation of Crawl4AI Functions.
+Supports Crawl4AI 0.7.x with comprehensive operation coverage.
+"""
from typing import Collection
import importlib.metadata
from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
from wrapt import wrap_function_wrapper
-from openlit.instrumentation.crawl4ai.crawl4ai import wrap_crawl
-from openlit.instrumentation.crawl4ai.async_crawl4ai import async_wrap_crawl
+from openlit.instrumentation.crawl4ai.crawl4ai import general_wrap
+from openlit.instrumentation.crawl4ai.async_crawl4ai import async_general_wrap
-_instruments = ("crawl4ai >= 0.4.0",)
+_instruments = ("crawl4ai >= 0.7.0",)
class Crawl4AIInstrumentor(BaseInstrumentor):
"""
- An instrumentor for crawl4ai's client library.
+ An instrumentor for Crawl4AI's client library.
+ Supports comprehensive 0.7.x operations including async crawling,
+ batch processing, deep crawling, and extraction strategies.
"""
def instrumentation_dependencies(self) -> Collection[str]:
@@ -28,40 +33,329 @@ class Crawl4AIInstrumentor(BaseInstrumentor):
pricing_info = kwargs.get("pricing_info", {})
capture_message_content = kwargs.get("capture_message_content", False)
disable_metrics = kwargs.get("disable_metrics")
- version = importlib.metadata.version("crawl4ai")
-
- wrap_function_wrapper(
- "crawl4ai.web_crawler",
- "WebCrawler.run",
- wrap_crawl(
- "crawl4ai.web_crawl",
- version,
- environment,
- application_name,
- tracer,
- pricing_info,
- capture_message_content,
- metrics,
- disable_metrics,
- ),
- )
-
- wrap_function_wrapper(
- "crawl4ai.async_webcrawler",
- "AsyncWebCrawler.arun",
- async_wrap_crawl(
- "crawl4ai.web_crawl",
- version,
- environment,
- application_name,
- tracer,
- pricing_info,
- capture_message_content,
- metrics,
- disable_metrics,
- ),
- )
+
+ try:
+ version = importlib.metadata.version("crawl4ai")
+ except importlib.metadata.PackageNotFoundError:
+ version = "unknown"
+
+ # Wrap AsyncWebCrawler.arun() - Primary single URL crawling
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.async_webcrawler",
+ "AsyncWebCrawler.arun",
+ async_general_wrap(
+ "crawl4ai.arun",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ # Wrap AsyncWebCrawler.arun_many() - Batch/streaming crawling (0.7.x)
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.async_webcrawler",
+ "AsyncWebCrawler.arun_many",
+ async_general_wrap(
+ "crawl4ai.arun_many",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ # Wrap legacy WebCrawler.run() for backward compatibility
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.web_crawler",
+ "WebCrawler.run",
+ general_wrap(
+ "crawl4ai.run",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ # Wrap deep crawling operations (0.7.x)
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.deep_crawling.strategies",
+ "BFSDeepCrawlStrategy.arun",
+ async_general_wrap(
+ "crawl4ai.deep_crawl.bfs",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.deep_crawling.strategies",
+ "DFSDeepCrawlStrategy.arun",
+ async_general_wrap(
+ "crawl4ai.deep_crawl.dfs",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.deep_crawling.strategies",
+ "BestFirstCrawlingStrategy.arun",
+ async_general_wrap(
+ "crawl4ai.deep_crawl.best_first",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ # Wrap LLM and Extraction Strategy Operations (0.7.x)
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.extraction_strategy",
+ "LLMExtractionStrategy.extract",
+ general_wrap(
+ "crawl4ai.extraction.llm.extract",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.extraction_strategy",
+ "LLMExtractionStrategy.run",
+ general_wrap(
+ "crawl4ai.extraction.llm.run",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.extraction_strategy",
+ "JsonCssExtractionStrategy.extract",
+ general_wrap(
+ "crawl4ai.extraction.css.extract",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.extraction_strategy",
+ "JsonXPathExtractionStrategy.extract",
+ general_wrap(
+ "crawl4ai.extraction.xpath.extract",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.extraction_strategy",
+ "CosineStrategy.extract",
+ general_wrap(
+ "crawl4ai.extraction.cosine.extract",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.extraction_strategy",
+ "RegexExtractionStrategy.extract",
+ general_wrap(
+ "crawl4ai.extraction.regex.extract",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ # Wrap Core Processing Operations (0.7.x)
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.async_webcrawler",
+ "AsyncWebCrawler.aprocess_html",
+ async_general_wrap(
+ "crawl4ai.aprocess_html",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.async_webcrawler",
+ "AsyncWebCrawler.aseed_urls",
+ async_general_wrap(
+ "crawl4ai.aseed_urls",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ # Wrap Monitor Operations for Business Intelligence
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.components.crawler_monitor",
+ "CrawlerMonitor.update_task",
+ general_wrap(
+ "crawl4ai.monitor.update_task",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
+
+ try:
+ wrap_function_wrapper(
+ "crawl4ai.components.crawler_monitor",
+ "CrawlerMonitor.get_summary",
+ general_wrap(
+ "crawl4ai.monitor.get_summary",
+ version,
+ environment,
+ application_name,
+ tracer,
+ pricing_info,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ),
+ )
+ except Exception:
+ pass # Module may not exist in all versions
def _uninstrument(self, **kwargs):
- # Proper uninstrumentation logic to revert patched methods
- pass
+ """
+ Proper uninstrumentation logic to revert patched methods.
+
+ Note: wrapt automatically handles uninstrumentation for
+ wrap_function_wrapper calls when the instrumentor is disabled.
+ """
+ # Uninstrumentation is handled automatically by wrapt
+ return
diff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py b/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py
index afb5403b..07c34df0 100644
--- a/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py
+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py
@@ -1,25 +1,33 @@
# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
"""
-Module for monitoring Crawl4AI calls.
+Module for monitoring Crawl4AI asynchronous calls.
+Supports comprehensive 0.7.x operations with business intelligence and enhanced observability.
+Includes support for async generators and streaming responses.
"""
import logging
-from opentelemetry.trace import SpanKind, Status, StatusCode
-from opentelemetry.sdk.resources import (
- SERVICE_NAME,
- TELEMETRY_SDK_NAME,
- DEPLOYMENT_ENVIRONMENT,
-)
-from openlit.__helpers import (
- handle_exception,
+import time
+from opentelemetry.trace import SpanKind
+from opentelemetry import context as context_api
+
+from openlit.__helpers import handle_exception
+from openlit.instrumentation.crawl4ai.utils import (
+ Crawl4AIInstrumentationContext,
+ get_operation_name,
+ create_crawl_span_name,
+ set_crawl_attributes,
+ process_crawl_response,
+ capture_message_content_if_enabled,
+ process_llm_extraction_response,
+ create_extraction_span_name,
+ capture_extraction_content,
)
-from openlit.semcov import SemanticConvention
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def async_wrap_crawl(
+def async_general_wrap(
gen_ai_endpoint,
version,
environment,
@@ -31,121 +39,320 @@ def async_wrap_crawl(
disable_metrics,
):
"""
- Generates a telemetry wrapper for chat completions to collect metrics.
+ Generates a telemetry wrapper for async Crawl4AI operations to collect comprehensive metrics.
Args:
gen_ai_endpoint: Endpoint identifier for logging and tracing.
- version: Version of the monitoring package.
+ version: Version of the Crawl4AI package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the Crawl4AI Agent.
+ application_name: Name of the application using Crawl4AI.
tracer: OpenTelemetry tracer for creating spans.
pricing_info: Information used for calculating the cost of Crawl4AI usage.
capture_message_content: Flag indicating whether to trace the actual content.
+ metrics: OpenLIT metrics dictionary for performance tracking.
+ disable_metrics: Flag to disable metrics collection.
Returns:
- A function that wraps the chat completions method to add telemetry.
+ A function that wraps async Crawl4AI methods to add comprehensive telemetry.
"""
- async def wrapper(wrapped, instance, args, kwargs):
+ def wrapper(wrapped, instance, args, kwargs):
"""
- Wraps the 'chat.completions' API call to add telemetry.
+ Wraps async Crawl4AI operations to add comprehensive telemetry.
- This collects metrics such as execution time, cost, and token usage, and handles errors
- gracefully, adding details to the trace for observability.
+ This collects metrics such as execution time, URL information, configuration details,
+ and result metrics, while handling errors gracefully for enhanced observability.
+ Supports both regular async methods and async generators for streaming.
Args:
- wrapped: The original 'chat.completions' method to be wrapped.
+ wrapped: The original async Crawl4AI method to be wrapped.
instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'chat.completions' method.
- kwargs: Keyword arguments for the 'chat.completions' method.
+ args: Positional arguments for the Crawl4AI method.
+ kwargs: Keyword arguments for the Crawl4AI method.
Returns:
- The response from the original 'chat.completions' method.
+ The response from the original async Crawl4AI method.
"""
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(
- gen_ai_endpoint, kind=SpanKind.CLIENT
- ) as span:
- response = await wrapped(*args, **kwargs)
+ async def async_wrapper():
+ # Check if instrumentation is suppressed
+ if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY):
+ return await wrapped(*args, **kwargs)
- try:
- # Set base span attribues
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(
- SemanticConvention.GEN_AI_SYSTEM,
- SemanticConvention.GEN_AI_SYSTEM_CRAWL4AI,
- )
- span.set_attribute(
- SemanticConvention.GEN_AI_OPERATION,
- SemanticConvention.GEN_AI_OPERATION_TYPE_AGENT,
- )
- span.set_attribute(SemanticConvention.GEN_AI_ENDPOINT, gen_ai_endpoint)
- span.set_attribute(SERVICE_NAME, application_name)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_TYPE,
- SemanticConvention.GEN_AI_AGENT_TYPE_BROWSER,
- )
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_ENABLE_CACHE,
- not kwargs.get("disable_cache", False),
- )
+ # Get operation name and create context
+ operation_name = get_operation_name(gen_ai_endpoint)
+ ctx = Crawl4AIInstrumentationContext(
+ instance, args, kwargs, version, environment, application_name
+ )
- url = (
- kwargs.get("url")
- if "url" in kwargs
- else str(args[0])
- if args
- else None
- )
- if url is not None:
- span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, url)
+ # Create span name following the framework guide pattern
+ if operation_name == "extract":
+ # For extraction operations, determine strategy type from the endpoint
+ strategy_type = None
+ if "llm" in gen_ai_endpoint:
+ strategy_type = "llm"
+ elif "css" in gen_ai_endpoint:
+ strategy_type = "css"
+ elif "xpath" in gen_ai_endpoint:
+ strategy_type = "xpath"
+ elif "cosine" in gen_ai_endpoint:
+ strategy_type = "cosine"
+ elif "regex" in gen_ai_endpoint:
+ strategy_type = "regex"
- extraction_strategy = kwargs.get(
- "extraction_strategy", "NoExtractionStrategy"
- )
- extraction_name = (
- extraction_strategy.name
- if hasattr(extraction_strategy, "name")
- else extraction_strategy
+ target_url = ctx.url if ctx.url != "unknown" else "content"
+ span_name = create_extraction_span_name(
+ operation_name, strategy_type, target_url
)
+ else:
+ # For crawl operations, use the new improved naming
+ span_name = create_crawl_span_name(operation_name, ctx, gen_ai_endpoint)
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_STRATEGY, extraction_name
- )
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
- if extraction_name == "LLMExtractionStrategy" and hasattr(
- extraction_strategy, "provider"
- ):
- _, llm_model = extraction_strategy.provider.split("/")
- span.set_attribute(
- SemanticConvention.GEN_AI_REQUEST_MODEL, llm_model
- )
+ try:
+ # Set comprehensive span attributes
+ set_crawl_attributes(span, ctx, operation_name)
- elif extraction_name == "CosineStrategy":
- span.set_attribute(
- SemanticConvention.GEN_AI_REQUEST_MODEL, "all-MiniLM-L6-v2"
- )
+ # Execute the original async function
+ response = await wrapped(*args, **kwargs)
+
+ # Handle async generators (streaming responses)
+ if hasattr(response, "__aiter__"):
+ return await _handle_async_generator(
+ span,
+ response,
+ ctx,
+ start_time,
+ operation_name,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ )
- elif extraction_name == "JsonCssExtractionStrategy" and hasattr(
- extraction_strategy, "schema"
- ):
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_SCHEMA,
- str(extraction_strategy.schema),
+ # Handle regular responses
+ return await _handle_regular_response(
+ span,
+ response,
+ ctx,
+ start_time,
+ operation_name,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ instance, # Add missing parameter
+ args, # Add missing parameter
)
- span.set_status(Status(StatusCode.OK))
+ except Exception as e:
+ # Calculate duration even for errors
+ end_time = time.time()
+ duration_ms = (end_time - start_time) * 1000
+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)
- # Return original response
- return response
+ # Handle and log the exception
+ handle_exception(span, e)
+ logger.error("Error in async Crawl4AI operation: %s", e)
- except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ # Record error metrics if enabled
+ if not disable_metrics and metrics:
+ try:
+ error_key = f"crawl4ai.{operation_name}.error"
+ if error_key not in metrics:
+ metrics[error_key] = 0
+ metrics[error_key] += 1
+ except Exception as metrics_error:
+ logger.debug(
+ "Error recording error metrics: %s", metrics_error
+ )
- # Return original response
- return response
+ # Re-raise the exception to maintain original behavior
+ raise
+
+ return async_wrapper()
return wrapper
+
+
+async def _handle_regular_response(
+ span,
+ response,
+ ctx,
+ start_time,
+ operation_name,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ instance,
+ args,
+):
+ """Handle regular (non-streaming) async responses."""
+
+ # Calculate duration for business intelligence
+ end_time = time.time()
+ duration_ms = (end_time - start_time) * 1000
+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)
+
+ # Process response based on operation type
+ if operation_name == "extract":
+ # Process extraction strategy response with LLM-specific metrics
+ if hasattr(instance, "__class__"):
+ # For extraction strategy operations, the instance is the strategy
+ process_llm_extraction_response(span, instance, response, ctx)
+
+ # Capture extraction content if enabled
+ extraction_input = None
+ if len(args) >= 3:
+ extraction_input = args[2] # html content
+ elif len(args) >= 2:
+ extraction_input = args[1] # html_content parameter
+
+ capture_extraction_content(
+ span, extraction_input, response, capture_message_content
+ )
+ else:
+ # Process crawl operations
+ process_crawl_response(span, response, ctx)
+
+ # Capture message content for crawl operations
+ capture_message_content_if_enabled(span, ctx, response, capture_message_content)
+
+ # Record metrics if enabled
+ if not disable_metrics and metrics:
+ try:
+ # Record operation metrics
+ metrics_key = f"crawl4ai.{operation_name}.duration"
+ if metrics_key not in metrics:
+ metrics[metrics_key] = []
+ metrics[metrics_key].append(duration_ms)
+
+ # Record success/failure metrics
+ success_key = f"crawl4ai.{operation_name}.success"
+ if success_key not in metrics:
+ metrics[success_key] = 0
+
+ # Determine success based on response
+ is_success = True
+ if hasattr(response, "success"):
+ is_success = response.success
+ elif isinstance(response, list):
+ is_success = all(getattr(r, "success", True) for r in response)
+
+ if is_success:
+ metrics[success_key] += 1
+
+ except Exception as metrics_error:
+ logger.debug("Error recording metrics: %s", metrics_error)
+
+ return response
+
+
+async def _handle_async_generator(
+ span,
+ response_generator,
+ ctx,
+ start_time,
+ operation_name,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+):
+ """Handle async generator responses (streaming)."""
+
+ class TrackedAsyncGenerator:
+ """Async generator wrapper to track streaming results."""
+
+ def __init__(
+ self,
+ generator,
+ span,
+ ctx,
+ start_time,
+ operation_name,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ ):
+ self.generator = generator
+ self.span = span
+ self.ctx = ctx
+ self.start_time = start_time
+ self.operation_name = operation_name
+ self.capture_message_content = capture_message_content
+ self.metrics = metrics
+ self.disable_metrics = disable_metrics
+ self.results = []
+ self.completed = False
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ try:
+ result = await self.generator.__anext__()
+ self.results.append(result)
+
+ # Update span with current result
+ if hasattr(result, "url"):
+ self.span.set_attribute("gen_ai.crawl.current_url", result.url)
+
+ return result
+
+ except StopAsyncIteration:
+ # Generator is complete, finalize metrics
+ if not self.completed:
+ await self._finalize_metrics()
+ self.completed = True
+ raise
+ except Exception as e:
+ # Handle errors in streaming
+ handle_exception(self.span, e)
+ logger.error("Error in streaming crawl operation: %s", e)
+ raise
+
+ async def _finalize_metrics(self):
+ """Finalize metrics when streaming is complete."""
+ end_time = time.time()
+ duration_ms = (end_time - self.start_time) * 1000
+ self.span.set_attribute("gen_ai.client.operation.duration", duration_ms)
+
+ # Process aggregated results
+ if self.results:
+ process_crawl_response(self.span, self.results, self.ctx)
+
+ # Capture message content for streaming
+ capture_message_content_if_enabled(
+ self.span, self.ctx, self.results, self.capture_message_content
+ )
+
+ # Record streaming metrics
+ if not self.disable_metrics and self.metrics:
+ try:
+ metrics_key = f"crawl4ai.{self.operation_name}.stream.duration"
+ if metrics_key not in self.metrics:
+ self.metrics[metrics_key] = []
+ self.metrics[metrics_key].append(duration_ms)
+
+ # Record stream completion
+ stream_key = f"crawl4ai.{self.operation_name}.stream.completed"
+ if stream_key not in self.metrics:
+ self.metrics[stream_key] = 0
+ self.metrics[stream_key] += 1
+
+ except Exception as metrics_error:
+ logger.debug(
+ "Error recording streaming metrics: %s", metrics_error
+ )
+
+ # Return the tracked generator
+ return TrackedAsyncGenerator(
+ response_generator,
+ span,
+ ctx,
+ start_time,
+ operation_name,
+ capture_message_content,
+ metrics,
+ disable_metrics,
+ )
diff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py b/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py
index 30302966..f5a27f11 100644
--- a/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py
+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py
@@ -1,25 +1,32 @@
# pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
"""
-Module for monitoring Crawl4AI calls.
+Module for monitoring Crawl4AI synchronous calls.
+Supports comprehensive 0.7.x operations with business intelligence and enhanced observability.
"""
import logging
-from opentelemetry.trace import SpanKind, Status, StatusCode
-from opentelemetry.sdk.resources import (
- SERVICE_NAME,
- TELEMETRY_SDK_NAME,
- DEPLOYMENT_ENVIRONMENT,
+import time
+from opentelemetry.trace import SpanKind
+from opentelemetry import context as context_api
+
+from openlit.__helpers import handle_exception
+from openlit.instrumentation.crawl4ai.utils import (
+ Crawl4AIInstrumentationContext,
+ get_operation_name,
+ create_crawl_span_name,
+ set_crawl_attributes,
+ process_crawl_response,
+ capture_message_content_if_enabled,
+ process_llm_extraction_response,
+ create_extraction_span_name,
+ capture_extraction_content,
)
-from openlit.__helpers import (
- handle_exception,
-)
-from openlit.semcov import SemanticConvention
# Initialize logger for logging potential issues and operations
logger = logging.getLogger(__name__)
-def wrap_crawl(
+def general_wrap(
gen_ai_endpoint,
version,
environment,
@@ -31,121 +38,166 @@ def wrap_crawl(
disable_metrics,
):
"""
- Generates a telemetry wrapper for chat completions to collect metrics.
+ Generates a telemetry wrapper for Crawl4AI operations to collect comprehensive metrics.
Args:
gen_ai_endpoint: Endpoint identifier for logging and tracing.
- version: Version of the monitoring package.
+ version: Version of the Crawl4AI package.
environment: Deployment environment (e.g., production, staging).
- application_name: Name of the application using the Crawl4AI Agent.
+ application_name: Name of the application using Crawl4AI.
tracer: OpenTelemetry tracer for creating spans.
pricing_info: Information used for calculating the cost of Crawl4AI usage.
capture_message_content: Flag indicating whether to trace the actual content.
+ metrics: OpenLIT metrics dictionary for performance tracking.
+ disable_metrics: Flag to disable metrics collection.
Returns:
- A function that wraps the chat completions method to add telemetry.
+ A function that wraps Crawl4AI methods to add comprehensive telemetry.
"""
def wrapper(wrapped, instance, args, kwargs):
"""
- Wraps the 'chat.completions' API call to add telemetry.
+ Wraps Crawl4AI operations to add comprehensive telemetry.
- This collects metrics such as execution time, cost, and token usage, and handles errors
- gracefully, adding details to the trace for observability.
+ This collects metrics such as execution time, URL information, configuration details,
+ and result metrics, while handling errors gracefully for enhanced observability.
Args:
- wrapped: The original 'chat.completions' method to be wrapped.
+ wrapped: The original Crawl4AI method to be wrapped.
instance: The instance of the class where the original method is defined.
- args: Positional arguments for the 'chat.completions' method.
- kwargs: Keyword arguments for the 'chat.completions' method.
+ args: Positional arguments for the Crawl4AI method.
+ kwargs: Keyword arguments for the Crawl4AI method.
Returns:
- The response from the original 'chat.completions' method.
+ The response from the original Crawl4AI method.
"""
- # pylint: disable=line-too-long
- with tracer.start_as_current_span(
- gen_ai_endpoint, kind=SpanKind.CLIENT
- ) as span:
- response = wrapped(*args, **kwargs)
+ # Check if instrumentation is suppressed
+ if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY):
+ return wrapped(*args, **kwargs)
+
+ # Get operation name and create context
+ operation_name = get_operation_name(gen_ai_endpoint)
+ ctx = Crawl4AIInstrumentationContext(
+ instance, args, kwargs, version, environment, application_name
+ )
+
+ # Create span name following the framework guide pattern
+ if operation_name == "extract":
+ # For extraction operations, determine strategy type from the endpoint
+ strategy_type = None
+ if "llm" in gen_ai_endpoint:
+ strategy_type = "llm"
+ elif "css" in gen_ai_endpoint:
+ strategy_type = "css"
+ elif "xpath" in gen_ai_endpoint:
+ strategy_type = "xpath"
+ elif "cosine" in gen_ai_endpoint:
+ strategy_type = "cosine"
+ elif "regex" in gen_ai_endpoint:
+ strategy_type = "regex"
+
+ target_url = ctx.url if ctx.url != "unknown" else "content"
+ span_name = create_extraction_span_name(
+ operation_name, strategy_type, target_url
+ )
+ else:
+ # For crawl operations, use the new improved naming
+ span_name = create_crawl_span_name(operation_name, ctx, gen_ai_endpoint)
+
+ with tracer.start_as_current_span(span_name, kind=SpanKind.CLIENT) as span:
+ start_time = time.time()
try:
- # Set base span attribues
- span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
- span.set_attribute(
- SemanticConvention.GEN_AI_SYSTEM,
- SemanticConvention.GEN_AI_SYSTEM_CRAWL4AI,
- )
- span.set_attribute(
- SemanticConvention.GEN_AI_OPERATION,
- SemanticConvention.GEN_AI_OPERATION_TYPE_AGENT,
- )
- span.set_attribute(SemanticConvention.GEN_AI_ENDPOINT, gen_ai_endpoint)
- span.set_attribute(SERVICE_NAME, application_name)
- span.set_attribute(DEPLOYMENT_ENVIRONMENT, environment)
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_TYPE,
- SemanticConvention.GEN_AI_AGENT_TYPE_BROWSER,
- )
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_ENABLE_CACHE,
- not kwargs.get("disable_cache", False),
- )
-
- url = (
- kwargs.get("url")
- if "url" in kwargs
- else str(args[0])
- if args
- else None
- )
- if url is not None:
- span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, url)
-
- extraction_strategy = kwargs.get(
- "extraction_strategy", "NoExtractionStrategy"
- )
- extraction_name = (
- extraction_strategy.name
- if hasattr(extraction_strategy, "name")
- else extraction_strategy
- )
-
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_STRATEGY, extraction_name
- )
-
- if extraction_name == "LLMExtractionStrategy" and hasattr(
- extraction_strategy, "provider"
- ):
- _, llm_model = extraction_strategy.provider.split("/")
- span.set_attribute(
- SemanticConvention.GEN_AI_REQUEST_MODEL, llm_model
- )
-
- elif extraction_name == "CosineStrategy":
- span.set_attribute(
- SemanticConvention.GEN_AI_REQUEST_MODEL, "all-MiniLM-L6-v2"
+ # Set comprehensive span attributes
+ set_crawl_attributes(span, ctx, operation_name)
+
+ # Execute the original function
+ response = wrapped(*args, **kwargs)
+
+ # Calculate duration for business intelligence
+ end_time = time.time()
+ duration_ms = (end_time - start_time) * 1000
+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)
+
+ # Process response based on operation type
+ if operation_name == "extract":
+ # Process extraction strategy response with LLM-specific metrics
+ if hasattr(instance, "__class__"):
+ # For extraction strategy operations, the instance is the strategy
+ process_llm_extraction_response(span, instance, response, ctx)
+
+ # Capture extraction content if enabled
+ extraction_input = None
+ if len(args) >= 3:
+ extraction_input = args[2] # html content
+ elif len(args) >= 2:
+ extraction_input = args[1] # html_content parameter
+
+ capture_extraction_content(
+ span, extraction_input, response, capture_message_content
)
+ else:
+ # Process crawl operations
+ process_crawl_response(span, response, ctx)
- elif extraction_name == "JsonCssExtractionStrategy" and hasattr(
- extraction_strategy, "schema"
- ):
- span.set_attribute(
- SemanticConvention.GEN_AI_AGENT_SCHEMA,
- str(extraction_strategy.schema),
+ # Capture message content for crawl operations
+ capture_message_content_if_enabled(
+ span, ctx, response, capture_message_content
)
- span.set_status(Status(StatusCode.OK))
+ # Record metrics if enabled
+ if not disable_metrics and metrics:
+ try:
+ # Record operation metrics
+ metrics_key = f"crawl4ai.{operation_name}.duration"
+ if metrics_key not in metrics:
+ metrics[metrics_key] = []
+ metrics[metrics_key].append(duration_ms)
+
+ # Record success/failure metrics
+ success_key = f"crawl4ai.{operation_name}.success"
+ if success_key not in metrics:
+ metrics[success_key] = 0
+
+ # Determine success based on response
+ is_success = True
+ if hasattr(response, "success"):
+ is_success = response.success
+ elif isinstance(response, list):
+ is_success = all(
+ getattr(r, "success", True) for r in response
+ )
+
+ if is_success:
+ metrics[success_key] += 1
+
+ except Exception as metrics_error:
+ logger.debug("Error recording metrics: %s", metrics_error)
- # Return original response
return response
except Exception as e:
- handle_exception(span, e)
- logger.error("Error in trace creation: %s", e)
+ # Calculate duration even for errors
+ end_time = time.time()
+ duration_ms = (end_time - start_time) * 1000
+ span.set_attribute("gen_ai.client.operation.duration", duration_ms)
- # Return original response
- return response
+ # Handle and log the exception
+ handle_exception(span, e)
+ logger.error("Error in Crawl4AI operation: %s", e)
+
+ # Record error metrics if enabled
+ if not disable_metrics and metrics:
+ try:
+ error_key = f"crawl4ai.{operation_name}.error"
+ if error_key not in metrics:
+ metrics[error_key] = 0
+ metrics[error_key] += 1
+ except Exception as metrics_error:
+ logger.debug("Error recording error metrics: %s", metrics_error)
+
+ # Re-raise the exception to maintain original behavior
+ raise
return wrapper
diff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/utils.py b/sdk/python/src/openlit/instrumentation/crawl4ai/utils.py
new file mode 100644
index 00000000..e7f34613
--- /dev/null
+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/utils.py
@@ -0,0 +1,828 @@
+"""
+Utilities for Crawl4AI instrumentation.
+Includes operation mapping, telemetry processing, and context caching for 0.7.x support.
+"""
+
+import logging
+from typing import Any, Optional, List
+
+from opentelemetry.trace import Status, StatusCode
+from opentelemetry.sdk.resources import (
+ SERVICE_NAME,
+ TELEMETRY_SDK_NAME,
+ DEPLOYMENT_ENVIRONMENT,
+)
+
+from openlit.__helpers import handle_exception
+from openlit.semcov import SemanticConvention
+
+# Initialize logger for logging potential issues and operations
+logger = logging.getLogger(__name__)
+
+# Operation mapping for Crawl4AI endpoints following the framework guide pattern
+CRAWL_OPERATION_MAP = {
+ "crawl4ai.arun": "crawl",
+ "crawl4ai.arun_many": "crawl", # Use same operation name for consistency
+ "crawl4ai.run": "crawl",
+ "crawl4ai.deep_crawl.bfs": "crawl_deep",
+ "crawl4ai.deep_crawl.dfs": "crawl_deep",
+ "crawl4ai.deep_crawl.best_first": "crawl_deep",
+ "crawl4ai.extract": "extract",
+ "crawl4ai.scrape": "scrape",
+ # LLM and Extraction Strategy Operations (0.7.x) - All map to "extract"
+ "crawl4ai.extraction.llm.extract": "extract",
+ "crawl4ai.extraction.llm.run": "extract",
+ "crawl4ai.extraction.css.extract": "extract",
+ "crawl4ai.extraction.xpath.extract": "extract",
+ "crawl4ai.extraction.cosine.extract": "extract",
+ "crawl4ai.extraction.regex.extract": "extract",
+ "crawl4ai.extraction.lxml.extract": "extract",
+ # Processing and Analysis Operations
+ "crawl4ai.aprocess_html": "process_html",
+ "crawl4ai.aseed_urls": "seed_urls",
+ # Monitor and Analysis
+ "crawl4ai.monitor.update_task": "monitor_task",
+ "crawl4ai.monitor.get_summary": "monitor_summary",
+}
+
+
+class Crawl4AIInstrumentationContext:
+ """Context object to cache expensive extractions and reduce performance overhead."""
+
+ __slots__ = (
+ "instance",
+ "args",
+ "kwargs",
+ "version",
+ "environment",
+ "application_name",
+ "_url",
+ "_urls",
+ "_config",
+ "_browser_config",
+ "_session_id",
+ "_cache_mode",
+ "_extraction_strategy",
+ "_operation_type",
+ )
+
+ def __init__(self, instance, args, kwargs, version, environment, application_name):
+ self.instance = instance
+ self.args = args
+ self.kwargs = kwargs
+ self.version = version
+ self.environment = environment
+ self.application_name = application_name
+
+ # Cache expensive operations with lazy loading
+ self._url = None
+ self._urls = None
+ self._config = None
+ self._browser_config = None
+ self._session_id = None
+ self._cache_mode = None
+ self._extraction_strategy = None
+ self._operation_type = None
+
+ @property
+ def url(self) -> Optional[str]:
+ """Get primary URL with caching."""
+ if self._url is None:
+ if len(self.args) > 0:
+ self._url = str(self.args[0])
+ elif "url" in self.kwargs:
+ self._url = str(self.kwargs["url"])
+ else:
+ self._url = "unknown"
+ return self._url
+
+ @property
+ def urls(self) -> Optional[List[str]]:
+ """Get URLs list for batch operations with caching."""
+ if self._urls is None:
+ if len(self.args) > 0 and isinstance(self.args[0], list):
+ self._urls = [str(url) for url in self.args[0]]
+ elif "urls" in self.kwargs and isinstance(self.kwargs["urls"], list):
+ self._urls = [str(url) for url in self.kwargs["urls"]]
+ else:
+ self._urls = []
+ return self._urls
+
+ @property
+ def config(self) -> Optional[Any]:
+ """Get CrawlerRunConfig with caching."""
+ if self._config is None:
+ self._config = self.kwargs.get("config")
+ return self._config
+
+ @property
+ def browser_config(self) -> Optional[Any]:
+ """Get BrowserConfig with caching."""
+ if self._browser_config is None:
+ if hasattr(self.instance, "config"):
+ self._browser_config = self.instance.config
+ return self._browser_config
+
+ @property
+ def session_id(self) -> Optional[str]:
+ """Get session ID with caching."""
+ if self._session_id is None:
+ config = self.config
+ if config and hasattr(config, "session_id"):
+ self._session_id = config.session_id
+ return self._session_id
+
+ @property
+ def cache_mode(self) -> Optional[str]:
+ """Get cache mode with caching."""
+ if self._cache_mode is None:
+ config = self.config
+ if config and hasattr(config, "cache_mode"):
+ self._cache_mode = str(config.cache_mode)
+ return self._cache_mode
+
+ @property
+ def extraction_strategy(self) -> Optional[str]:
+ """Get extraction strategy with caching."""
+ if self._extraction_strategy is None:
+ config = self.config
+ if config and hasattr(config, "extraction_strategy"):
+ strategy = config.extraction_strategy
+ if strategy:
+ self._extraction_strategy = strategy.__class__.__name__
+ return self._extraction_strategy
+
+
+def get_operation_name(endpoint: str) -> str:
+ """Get operation name from endpoint using the operation map."""
+ return CRAWL_OPERATION_MAP.get(endpoint, "crawl")
+
+
+def create_span_name(operation_name: str, target: str) -> str:
+ """Create span name following the '{operation_type} {operation_name}' format."""
+ return f"{operation_name} {target}"
+
+
+def create_crawl_span_name(
+ operation_name: str, ctx: Crawl4AIInstrumentationContext, gen_ai_endpoint: str
+) -> str:
+ """Create descriptive span name for crawl operations, handling batch operations specially."""
+
+ # Check if this is a batch operation
+ is_batch = "arun_many" in gen_ai_endpoint or (ctx.urls and len(ctx.urls) > 1)
+
+ if is_batch and ctx.urls:
+ # For batch operations, show the URLs but limit length
+ if len(ctx.urls) <= 3:
+ # Show all URLs for small batches
+ urls_str = ", ".join(ctx.urls)
+ if len(urls_str) <= 80: # Keep reasonable length
+ return f"{operation_name} [{urls_str}]"
+
+ # For larger batches or long URLs, show count and sample
+ sample_urls = ctx.urls[:2]
+ remaining_count = len(ctx.urls) - 2
+ if remaining_count > 0:
+ urls_sample = ", ".join(sample_urls) + f" +{remaining_count} more"
+ else:
+ urls_sample = ", ".join(sample_urls)
+
+ return f"{operation_name} [{urls_sample}]"
+
+ else:
+ # Single URL operation
+ target_url = ctx.url if ctx.url != "unknown" else "unknown"
+ return f"{operation_name} {target_url}"
+
+
+def set_crawl_attributes(
+ span, ctx: Crawl4AIInstrumentationContext, operation_name: str
+):
+ """Set comprehensive Crawl4AI span attributes."""
+
+ # Core framework attributes
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
+ span.set_attribute(
+ SemanticConvention.GEN_AI_SYSTEM, SemanticConvention.GEN_AI_SYSTEM_CRAWL4AI
+ )
+
+ # Map operation to semantic convention
+ if operation_name == "crawl":
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OPERATION,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL,
+ )
+ elif operation_name == "crawl_batch":
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OPERATION,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL, # Fixed: use CRAWL not CRAWL_BATCH
+ )
+ elif operation_name == "crawl_deep":
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OPERATION,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL_DEEP,
+ )
+ elif operation_name == "extract":
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OPERATION,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_EXTRACT,
+ )
+ elif operation_name == "scrape":
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OPERATION,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_SCRAPE,
+ )
+ elif operation_name in ["process_html", "seed_urls"]:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OPERATION,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_CRAWL, # Processing operations use crawl type
+ )
+ else:
+ # Default fallback for unknown operations
+ span.set_attribute(
+ SemanticConvention.GEN_AI_OPERATION,
+ SemanticConvention.GEN_AI_OPERATION_TYPE_EXTRACT, # Changed: use extract instead of agent
+ )
+
+ # Environment attributes
+ span.set_attribute(SERVICE_NAME, ctx.application_name)
+ span.set_attribute(DEPLOYMENT_ENVIRONMENT, ctx.environment)
+ span.set_attribute(SemanticConvention.GEN_AI_SDK_VERSION, ctx.version)
+
+ # Agent type for browser-based crawling
+ span.set_attribute(
+ SemanticConvention.GEN_AI_AGENT_TYPE,
+ SemanticConvention.GEN_AI_AGENT_TYPE_BROWSER,
+ )
+
+ # URL information
+ if operation_name == "crawl_batch" and ctx.urls:
+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_URL_COUNT, len(ctx.urls))
+ # Set primary URL as browse URL
+ if ctx.urls:
+ span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, ctx.urls[0])
+ elif ctx.url and ctx.url != "unknown":
+ span.set_attribute(SemanticConvention.GEN_AI_AGENT_BROWSE_URL, ctx.url)
+
+ # Configuration attributes
+ if ctx.session_id:
+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_SESSION_ID, ctx.session_id)
+
+ if ctx.cache_mode:
+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_CACHE_MODE, ctx.cache_mode)
+
+ if ctx.extraction_strategy:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_EXTRACTION_STRATEGY, ctx.extraction_strategy
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_AGENT_STRATEGY, ctx.extraction_strategy
+ )
+
+ # Browser configuration
+ browser_config = ctx.browser_config
+ if browser_config:
+ if hasattr(browser_config, "browser_type"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_BROWSER_TYPE,
+ browser_config.browser_type,
+ )
+
+ if hasattr(browser_config, "headless"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_HEADLESS, browser_config.headless
+ )
+
+ if hasattr(browser_config, "viewport_width"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_VIEWPORT_WIDTH,
+ browser_config.viewport_width,
+ )
+
+ if hasattr(browser_config, "viewport_height"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_VIEWPORT_HEIGHT,
+ browser_config.viewport_height,
+ )
+
+ if hasattr(browser_config, "user_agent"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_USER_AGENT,
+ str(browser_config.user_agent),
+ )
+
+ # Run configuration
+ config = ctx.config
+ if config:
+ if hasattr(config, "word_count_threshold"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_WORD_COUNT_THRESHOLD,
+ config.word_count_threshold,
+ )
+
+ if hasattr(config, "css_selector") and config.css_selector:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_CSS_SELECTOR, config.css_selector
+ )
+
+ if hasattr(config, "excluded_tags") and config.excluded_tags:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_EXCLUDED_TAGS, str(config.excluded_tags)
+ )
+
+ if hasattr(config, "screenshot"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_SCREENSHOT, config.screenshot
+ )
+
+ if hasattr(config, "pdf"):
+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_PDF, config.pdf)
+
+ if hasattr(config, "wait_for") and config.wait_for:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_WAIT_FOR, str(config.wait_for)
+ )
+
+ if hasattr(config, "page_timeout"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_PAGE_TIMEOUT, config.page_timeout
+ )
+
+ if hasattr(config, "js_code") and config.js_code:
+ js_code_str = (
+ str(config.js_code)
+ if not isinstance(config.js_code, list)
+ else "; ".join(config.js_code)
+ )
+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_JS_CODE, js_code_str)
+
+ # Deep crawling attributes
+ if hasattr(config, "deep_crawl_strategy") and config.deep_crawl_strategy:
+ strategy_name = config.deep_crawl_strategy.__class__.__name__
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_DEEP_STRATEGY, strategy_name
+ )
+
+ if hasattr(config.deep_crawl_strategy, "max_depth"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_DEPTH,
+ config.deep_crawl_strategy.max_depth,
+ )
+
+
+def process_crawl_response(
+ span, response, ctx: Crawl4AIInstrumentationContext, **kwargs
+):
+ """Process Crawl4AI response and extract business intelligence."""
+
+ try:
+ if response is None:
+ return
+
+ # Handle single result vs batch results
+ if isinstance(response, list):
+ # Batch crawling results
+ span.set_attribute(SemanticConvention.GEN_AI_CRAWL_URL_COUNT, len(response))
+
+ # Aggregate statistics from batch
+ total_html_length = 0
+ total_markdown_length = 0
+ total_links = 0
+ total_images = 0
+ success_count = 0
+
+ for result in response:
+ if hasattr(result, "success") and result.success:
+ success_count += 1
+
+ if hasattr(result, "html") and result.html:
+ total_html_length += len(result.html)
+
+ if hasattr(result, "markdown") and result.markdown:
+ total_markdown_length += len(result.markdown)
+
+ if hasattr(result, "links") and result.links:
+ total_links += len(result.links)
+
+ if (
+ hasattr(result, "media")
+ and result.media
+ and hasattr(result.media, "images")
+ ):
+ total_images += len(result.media.images)
+
+ # Set aggregate metrics
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_SUCCESS,
+ success_count == len(response),
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_HTML_LENGTH, total_html_length
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_MARKDOWN_LENGTH,
+ total_markdown_length,
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_LINKS_COUNT, total_links
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_IMAGES_COUNT, total_images
+ )
+
+ else:
+ # Single result
+ if hasattr(response, "success"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_SUCCESS, response.success
+ )
+
+ if hasattr(response, "status_code") and response.status_code is not None:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_STATUS_CODE,
+ response.status_code,
+ )
+
+ if hasattr(response, "html") and response.html:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_HTML_LENGTH,
+ len(response.html),
+ )
+
+ if hasattr(response, "markdown") and response.markdown:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_MARKDOWN_LENGTH,
+ len(response.markdown),
+ )
+
+ if hasattr(response, "links") and response.links:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_LINKS_COUNT,
+ len(response.links),
+ )
+
+ if (
+ hasattr(response, "media")
+ and response.media
+ and hasattr(response.media, "images")
+ ):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_IMAGES_COUNT,
+ len(response.media.images),
+ )
+
+ if hasattr(response, "redirected_url") and response.redirected_url:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CRAWL_RESULT_REDIRECTED_URL,
+ response.redirected_url,
+ )
+
+ # Set success status
+ span.set_status(Status(StatusCode.OK))
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error processing crawl response: %s", e)
+
+
+def format_content(content, max_length: int = 1000) -> str:
+ """Format content for tracing with length limits."""
+ if content is None:
+ return ""
+
+ content_str = str(content)
+ if len(content_str) > max_length:
+ return content_str[:max_length] + "..."
+ return content_str
+
+
+def capture_message_content_if_enabled(
+ span, ctx: Crawl4AIInstrumentationContext, response, capture_message_content: bool
+):
+ """Capture input/output content if message content capture is enabled."""
+
+ if not capture_message_content:
+ return
+
+ try:
+ # Capture input URL(s) as prompt
+ if ctx.url and ctx.url != "unknown":
+ span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, ctx.url)
+ elif ctx.urls:
+ urls_str = ", ".join(ctx.urls[:5]) # Limit to first 5 URLs
+ if len(ctx.urls) > 5:
+ urls_str += f" and {len(ctx.urls) - 5} more"
+ span.set_attribute(SemanticConvention.GEN_AI_CONTENT_PROMPT, urls_str)
+
+ # Capture output content based on response type
+ if response is None:
+ return
+
+ if isinstance(response, list):
+ # For batch responses, capture summary
+ completion_summary = f"Crawled {len(response)} URLs"
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, completion_summary
+ )
+ else:
+ # For single responses, capture markdown content (limited)
+ if hasattr(response, "markdown") and response.markdown:
+ formatted_content = format_content(response.markdown, max_length=2000)
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, formatted_content
+ )
+ elif hasattr(response, "html") and response.html:
+ formatted_content = format_content(response.html, max_length=1000)
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, formatted_content
+ )
+
+ except Exception as e:
+ logger.error("Error capturing message content: %s", e)
+
+
+def process_llm_extraction_response(
+ span, extraction_strategy, response, ctx: Crawl4AIInstrumentationContext
+):
+ """Process LLM extraction strategy response and capture comprehensive metrics."""
+
+ try:
+ # Capture LLM extraction strategy details
+ if hasattr(extraction_strategy, "__class__"):
+ strategy_name = extraction_strategy.__class__.__name__
+
+ # Map strategy class to semantic convention
+ if "LLM" in strategy_name:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_LLM,
+ )
+ elif "Css" in strategy_name:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_CSS,
+ )
+ elif "XPath" in strategy_name:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_XPATH,
+ )
+ elif "Cosine" in strategy_name:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_COSINE,
+ )
+ elif "Regex" in strategy_name:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_TYPE,
+ SemanticConvention.GEN_AI_EXTRACTION_STRATEGY_REGEX,
+ )
+
+ # Process LLM-specific attributes for LLMExtractionStrategy
+ if hasattr(extraction_strategy, "llm_config"):
+ llm_config = extraction_strategy.llm_config
+ if llm_config:
+ # Provider information
+ if hasattr(llm_config, "provider"):
+ provider_parts = str(llm_config.provider).split("/")
+ if len(provider_parts) >= 2:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_LLM_PROVIDER, provider_parts[0]
+ )
+ span.set_attribute(
+ SemanticConvention.GEN_AI_LLM_MODEL, provider_parts[1]
+ )
+ else:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_LLM_PROVIDER,
+ str(llm_config.provider),
+ )
+
+ # Base URL if available
+ if hasattr(llm_config, "base_url") and llm_config.base_url:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_LLM_BASE_URL, llm_config.base_url
+ )
+
+ # Extraction configuration
+ if hasattr(extraction_strategy, "extraction_type"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_TYPE,
+ extraction_strategy.extraction_type,
+ )
+
+ if (
+ hasattr(extraction_strategy, "instruction")
+ and extraction_strategy.instruction
+ ):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_INSTRUCTION,
+ format_content(extraction_strategy.instruction, max_length=500),
+ )
+
+ if hasattr(extraction_strategy, "input_format"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_INPUT_FORMAT,
+ extraction_strategy.input_format,
+ )
+
+ # Chunking configuration
+ if hasattr(extraction_strategy, "apply_chunking"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_APPLY_CHUNKING,
+ extraction_strategy.apply_chunking,
+ )
+
+ if hasattr(extraction_strategy, "chunk_token_threshold"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_CHUNK_TOKEN_THRESHOLD,
+ extraction_strategy.chunk_token_threshold,
+ )
+
+ if hasattr(extraction_strategy, "overlap_rate"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_EXTRACTION_OVERLAP_RATE,
+ extraction_strategy.overlap_rate,
+ )
+
+ # LLM model parameters
+ if (
+ hasattr(extraction_strategy, "extra_args")
+ and extraction_strategy.extra_args
+ ):
+ extra_args = extraction_strategy.extra_args
+ if "temperature" in extra_args:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_LLM_TEMPERATURE, extra_args["temperature"]
+ )
+ if "max_tokens" in extra_args:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_LLM_MAX_TOKENS, extra_args["max_tokens"]
+ )
+ if "top_p" in extra_args:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_LLM_TOP_P, extra_args["top_p"]
+ )
+
+ # Token usage tracking (critical for business intelligence)
+ if (
+ hasattr(extraction_strategy, "total_usage")
+ and extraction_strategy.total_usage
+ ):
+ total_usage = extraction_strategy.total_usage
+ if hasattr(total_usage, "prompt_tokens"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_TOKEN_USAGE_INPUT,
+ total_usage.prompt_tokens,
+ )
+ if hasattr(total_usage, "completion_tokens"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_TOKEN_USAGE_OUTPUT,
+ total_usage.completion_tokens,
+ )
+ if hasattr(total_usage, "total_tokens"):
+ span.set_attribute(
+ SemanticConvention.GEN_AI_TOKEN_USAGE_TOTAL,
+ total_usage.total_tokens,
+ )
+
+ # Individual chunk usage
+ if hasattr(extraction_strategy, "usages") and extraction_strategy.usages:
+ chunk_count = len(extraction_strategy.usages)
+ span.set_attribute(SemanticConvention.GEN_AI_TOKEN_CHUNK_COUNT, chunk_count)
+
+ # Aggregate chunk usage information
+ total_chunk_tokens = 0
+ for usage in extraction_strategy.usages:
+ if hasattr(usage, "total_tokens"):
+ total_chunk_tokens += usage.total_tokens
+ if total_chunk_tokens > 0:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_TOKEN_CHUNK_USAGE, total_chunk_tokens
+ )
+
+ # Extraction success/failure
+ if response is not None:
+ span.set_attribute(SemanticConvention.GEN_AI_EXTRACTION_SUCCESS, True)
+
+ # Count extracted items for business intelligence
+ if isinstance(response, list):
+ span.set_attribute("gen_ai.extraction.items_count", len(response))
+ elif isinstance(response, dict):
+ span.set_attribute("gen_ai.extraction.items_count", 1)
+ else:
+ span.set_attribute(SemanticConvention.GEN_AI_EXTRACTION_SUCCESS, False)
+
+ except Exception as e:
+ handle_exception(span, e)
+ logger.error("Error processing LLM extraction response: %s", e)
+
+
+def process_crawler_monitor_metrics(span, monitor_data):
+ """Process CrawlerMonitor metrics and integrate with OpenLIT telemetry."""
+
+ try:
+ if not monitor_data:
+ return
+
+ # Task-level metrics
+ if "task_id" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_TASK_ID, monitor_data["task_id"]
+ )
+
+ if "status" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_TASK_STATUS, monitor_data["status"]
+ )
+
+ # Memory metrics (business intelligence)
+ if "memory_usage" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_MEMORY_USAGE,
+ monitor_data["memory_usage"],
+ )
+
+ if "peak_memory" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_PEAK_MEMORY,
+ monitor_data["peak_memory"],
+ )
+
+ # Performance metrics
+ if "retry_count" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_RETRY_COUNT,
+ monitor_data["retry_count"],
+ )
+
+ if "wait_time" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_WAIT_TIME, monitor_data["wait_time"]
+ )
+
+ # Queue and completion metrics
+ if "queue_size" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_QUEUE_SIZE, monitor_data["queue_size"]
+ )
+
+ if "completion_rate" in monitor_data:
+ span.set_attribute(
+ SemanticConvention.GEN_AI_MONITOR_COMPLETION_RATE,
+ monitor_data["completion_rate"],
+ )
+
+ except Exception as e:
+ logger.error("Error processing crawler monitor metrics: %s", e)
+
+
+def create_extraction_span_name(
+ operation_type: str, strategy_type: str, target: str
+) -> str:
+ """Create span name for extraction operations following the 'extract {target}' format."""
+ # Always use "extract" as the operation name regardless of strategy type
+ return f"extract {target}"
+
+
+def capture_extraction_content(
+ span, extraction_input, extraction_output, capture_message_content: bool
+):
+ """Capture extraction input/output content if enabled."""
+
+ if not capture_message_content:
+ return
+
+ try:
+ # Capture extraction input as prompt (URL or content snippet)
+ if extraction_input:
+ if isinstance(extraction_input, str):
+ input_content = format_content(extraction_input, max_length=1000)
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CONTENT_PROMPT, input_content
+ )
+ elif hasattr(extraction_input, "__iter__"):
+ # Handle list/tuple of inputs
+ combined_input = " ".join(str(item)[:200] for item in extraction_input)
+ input_content = format_content(combined_input, max_length=1000)
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CONTENT_PROMPT, input_content
+ )
+
+ # Capture extraction output as completion
+ if extraction_output is not None:
+ if isinstance(extraction_output, (list, dict)):
+ # Format structured output
+ output_summary = f"Extracted {len(extraction_output) if hasattr(extraction_output, '__len__') else 1} items"
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, output_summary
+ )
+ else:
+ # Handle string output
+ output_content = format_content(str(extraction_output), max_length=1000)
+ span.set_attribute(
+ SemanticConvention.GEN_AI_CONTENT_COMPLETION, output_content
+ )
+
+ except Exception as e:
+ logger.error("Error capturing extraction content: %s", e)
diff --git a/sdk/python/src/openlit/semcov/__init__.py b/sdk/python/src/openlit/semcov/__init__.py
index b6c7c03e..6e31699f 100644
--- a/sdk/python/src/openlit/semcov/__init__.py
+++ b/sdk/python/src/openlit/semcov/__init__.py
@@ -421,6 +421,107 @@ class SemanticConvention:
GEN_AI_AGENT_TYPE_BROWSER = "browser"
+ # Crawl4AI Specific Attributes (0.7.x Support)
+ GEN_AI_CRAWL_URL_COUNT = "gen_ai.crawl.url_count"
+ GEN_AI_CRAWL_DEPTH = "gen_ai.crawl.depth"
+ GEN_AI_CRAWL_SESSION_ID = "gen_ai.crawl.session_id"
+ GEN_AI_CRAWL_CACHE_MODE = "gen_ai.crawl.cache_mode"
+ GEN_AI_CRAWL_EXTRACTION_STRATEGY = "gen_ai.crawl.extraction_strategy"
+ GEN_AI_CRAWL_SCRAPING_STRATEGY = "gen_ai.crawl.scraping_strategy"
+ GEN_AI_CRAWL_BROWSER_TYPE = "gen_ai.crawl.browser_type"
+ GEN_AI_CRAWL_HEADLESS = "gen_ai.crawl.headless"
+ GEN_AI_CRAWL_VIEWPORT_WIDTH = "gen_ai.crawl.viewport_width"
+ GEN_AI_CRAWL_VIEWPORT_HEIGHT = "gen_ai.crawl.viewport_height"
+ GEN_AI_CRAWL_USER_AGENT = "gen_ai.crawl.user_agent"
+ GEN_AI_CRAWL_WORD_COUNT_THRESHOLD = "gen_ai.crawl.word_count_threshold"
+ GEN_AI_CRAWL_CSS_SELECTOR = "gen_ai.crawl.css_selector"
+ GEN_AI_CRAWL_EXCLUDED_TAGS = "gen_ai.crawl.excluded_tags"
+ GEN_AI_CRAWL_SCREENSHOT = "gen_ai.crawl.screenshot"
+ GEN_AI_CRAWL_PDF = "gen_ai.crawl.pdf"
+ GEN_AI_CRAWL_WAIT_FOR = "gen_ai.crawl.wait_for"
+ GEN_AI_CRAWL_PAGE_TIMEOUT = "gen_ai.crawl.page_timeout"
+ GEN_AI_CRAWL_JS_CODE = "gen_ai.crawl.js_code"
+ GEN_AI_CRAWL_RESULT_SUCCESS = "gen_ai.crawl.result.success"
+ GEN_AI_CRAWL_RESULT_STATUS_CODE = "gen_ai.crawl.result.status_code"
+ GEN_AI_CRAWL_RESULT_HTML_LENGTH = "gen_ai.crawl.result.html_length"
+ GEN_AI_CRAWL_RESULT_MARKDOWN_LENGTH = "gen_ai.crawl.result.markdown_length"
+ GEN_AI_CRAWL_RESULT_LINKS_COUNT = "gen_ai.crawl.result.links_count"
+ GEN_AI_CRAWL_RESULT_IMAGES_COUNT = "gen_ai.crawl.result.images_count"
+ GEN_AI_CRAWL_RESULT_REDIRECTED_URL = "gen_ai.crawl.result.redirected_url"
+ GEN_AI_CRAWL_DEEP_STRATEGY = "gen_ai.crawl.deep_strategy"
+ GEN_AI_CRAWL_PROXY_CONFIG = "gen_ai.crawl.proxy_config"
+
+ # Crawl4AI LLM Extraction Attributes (0.7.x)
+ GEN_AI_EXTRACTION_STRATEGY_TYPE = "gen_ai.extraction.strategy.type"
+ GEN_AI_EXTRACTION_TYPE = "gen_ai.extraction.type"
+ GEN_AI_EXTRACTION_INSTRUCTION = "gen_ai.extraction.instruction"
+ GEN_AI_EXTRACTION_SCHEMA = "gen_ai.extraction.schema"
+ GEN_AI_EXTRACTION_INPUT_FORMAT = "gen_ai.extraction.input_format"
+ GEN_AI_EXTRACTION_CHUNK_COUNT = "gen_ai.extraction.chunk_count"
+ GEN_AI_EXTRACTION_CHUNK_TOKEN_THRESHOLD = "gen_ai.extraction.chunk_token_threshold"
+ GEN_AI_EXTRACTION_OVERLAP_RATE = "gen_ai.extraction.overlap_rate"
+ GEN_AI_EXTRACTION_APPLY_CHUNKING = "gen_ai.extraction.apply_chunking"
+ GEN_AI_EXTRACTION_SUCCESS = "gen_ai.extraction.success"
+ GEN_AI_EXTRACTION_ERROR = "gen_ai.extraction.error"
+
+ # LLM Provider and Model Information
+ GEN_AI_LLM_PROVIDER = "gen_ai.llm.provider"
+ GEN_AI_LLM_MODEL = "gen_ai.llm.model"
+ GEN_AI_LLM_BASE_URL = "gen_ai.llm.base_url"
+ GEN_AI_LLM_TEMPERATURE = "gen_ai.llm.temperature"
+ GEN_AI_LLM_MAX_TOKENS = "gen_ai.llm.max_tokens"
+ GEN_AI_LLM_TOP_P = "gen_ai.llm.top_p"
+
+ # Token Usage and Cost Tracking (Enhanced Business Intelligence)
+ GEN_AI_TOKEN_USAGE_INPUT = "gen_ai.token.usage.input"
+ GEN_AI_TOKEN_USAGE_OUTPUT = "gen_ai.token.usage.output"
+ GEN_AI_TOKEN_USAGE_TOTAL = "gen_ai.token.usage.total"
+ GEN_AI_TOKEN_COST_INPUT = "gen_ai.token.cost.input"
+ GEN_AI_TOKEN_COST_OUTPUT = "gen_ai.token.cost.output"
+ GEN_AI_TOKEN_COST_TOTAL = "gen_ai.token.cost.total"
+ GEN_AI_TOKEN_CHUNK_USAGE = "gen_ai.token.chunk_usage"
+ GEN_AI_TOKEN_CHUNK_COUNT = "gen_ai.token.chunk_count"
+
+ # CrawlerMonitor Integration Attributes
+ GEN_AI_MONITOR_TASK_ID = "gen_ai.monitor.task_id"
+ GEN_AI_MONITOR_TASK_STATUS = "gen_ai.monitor.task_status"
+ GEN_AI_MONITOR_MEMORY_USAGE = "gen_ai.monitor.memory_usage"
+ GEN_AI_MONITOR_PEAK_MEMORY = "gen_ai.monitor.peak_memory"
+ GEN_AI_MONITOR_RETRY_COUNT = "gen_ai.monitor.retry_count"
+ GEN_AI_MONITOR_WAIT_TIME = "gen_ai.monitor.wait_time"
+ GEN_AI_MONITOR_QUEUE_SIZE = "gen_ai.monitor.queue_size"
+ GEN_AI_MONITOR_COMPLETION_RATE = "gen_ai.monitor.completion_rate"
+
+ # Extraction Strategy Types
+ GEN_AI_EXTRACTION_STRATEGY_LLM = "llm"
+ GEN_AI_EXTRACTION_STRATEGY_CSS = "css"
+ GEN_AI_EXTRACTION_STRATEGY_XPATH = "xpath"
+ GEN_AI_EXTRACTION_STRATEGY_COSINE = "cosine"
+ GEN_AI_EXTRACTION_STRATEGY_REGEX = "regex"
+ GEN_AI_EXTRACTION_STRATEGY_LXML = "lxml"
+
+ # LLM Extraction Types
+ GEN_AI_EXTRACTION_TYPE_SCHEMA = "schema"
+ GEN_AI_EXTRACTION_TYPE_BLOCK = "block"
+
+ # Monitor Task Status Values
+ GEN_AI_MONITOR_STATUS_PENDING = "pending"
+ GEN_AI_MONITOR_STATUS_RUNNING = "running"
+ GEN_AI_MONITOR_STATUS_COMPLETED = "completed"
+ GEN_AI_MONITOR_STATUS_FAILED = "failed"
+ GEN_AI_MONITOR_STATUS_RETRYING = "retrying"
+
+ # Crawl4AI Operation Types (reusing existing pattern)
+ GEN_AI_OPERATION_TYPE_CRAWL = "crawl"
+ GEN_AI_OPERATION_TYPE_CRAWL_DEEP = "crawl_deep"
+ GEN_AI_OPERATION_TYPE_EXTRACT = "extract"
+ GEN_AI_OPERATION_TYPE_EXTRACT_LLM = "extract_llm"
+ GEN_AI_OPERATION_TYPE_EXTRACT_CSS = "extract_css"
+ GEN_AI_OPERATION_TYPE_EXTRACT_XPATH = "extract_xpath"
+ GEN_AI_OPERATION_TYPE_EXTRACT_COSINE = "extract_cosine"
+ GEN_AI_OPERATION_TYPE_EXTRACT_REGEX = "extract_regex"
+ GEN_AI_OPERATION_TYPE_SCRAPE = "scrape"
+
# GPU
GPU_INDEX = "gpu.index"
GPU_UUID = "gpu.uuid"