Same agent Smaller bill. The tools and infrastructure to get more work from your AI. Fewer tokens. Less compute. Lower cost.
TypeScript agent-harnessagent-harness-engineeringai-engineeringai-observabilityamd-gpuclaude-codeclickhousecodexcoding-agentcoding-agentscursorgenaigpu-monitoringllm-observabilityllmopsnvidia-smiobservabilityopen-sourceopenaiopentelemetry
fix(browser-use/crawl4ai): report operation duration in seconds (#1609)
RichardoMu912da1407107
diff --git a/sdk/python/src/openlit/instrumentation/browser_use/async_browser_use.py b/sdk/python/src/openlit/instrumentation/browser_use/async_browser_use.pyindex ef0302cd..b1a36545 100644--- a/sdk/python/src/openlit/instrumentation/browser_use/async_browser_use.py+++ b/sdk/python/src/openlit/instrumentation/browser_use/async_browser_use.py@@ -90,9 +90,10 @@ def async_general_wrap(# Calculate durationend_time = time.time()- duration_ms = (end_time - start_time) * 1000+ duration = end_time - start_time+ duration_ms = duration * 1000main_span.set_attribute(- SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration_ms+ SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration)# Process response with enhanced details@@ -118,9 +119,10 @@ def async_general_wrap(except Exception as e:# Calculate duration even for errorsend_time = time.time()- duration_ms = (end_time - start_time) * 1000+ duration = end_time - start_time+ duration_ms = duration * 1000main_span.set_attribute(- SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration_ms+ SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration)# Handle and log the exceptiondiff --git a/sdk/python/src/openlit/instrumentation/browser_use/browser_use.py b/sdk/python/src/openlit/instrumentation/browser_use/browser_use.pyindex b959f4b1..e04316f2 100644--- a/sdk/python/src/openlit/instrumentation/browser_use/browser_use.py+++ b/sdk/python/src/openlit/instrumentation/browser_use/browser_use.py@@ -73,9 +73,10 @@ def general_wrap(# Calculate durationend_time = time.time()- duration_ms = (end_time - start_time) * 1000+ duration = end_time - start_time+ duration_ms = duration * 1000main_span.set_attribute(- SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration_ms+ SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration)# Process response with enhanced details@@ -101,9 +102,10 @@ def general_wrap(except Exception as e:# Calculate duration even for errorsend_time = time.time()- duration_ms = (end_time - start_time) * 1000+ duration = end_time - start_time+ duration_ms = duration * 1000main_span.set_attribute(- SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration_ms+ SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION, duration)# Handle and log the exceptiondiff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py b/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.pyindex 07c34df0..6dc42372 100644--- a/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/async_crawl4ai.py@@ -148,8 +148,8 @@ def async_general_wrap(except Exception as e:# Calculate duration even for errorsend_time = time.time()- duration_ms = (end_time - start_time) * 1000- span.set_attribute("gen_ai.client.operation.duration", duration_ms)+ duration = end_time - start_time+ span.set_attribute("gen_ai.client.operation.duration", duration)# Handle and log the exceptionhandle_exception(span, e)@@ -191,8 +191,9 @@ async def _handle_regular_response(# Calculate duration for business intelligenceend_time = time.time()- duration_ms = (end_time - start_time) * 1000- span.set_attribute("gen_ai.client.operation.duration", duration_ms)+ duration = end_time - start_time+ duration_ms = duration * 1000+ span.set_attribute("gen_ai.client.operation.duration", duration)# Process response based on operation typeif operation_name == "extract":@@ -314,8 +315,9 @@ async def _handle_async_generator(async def _finalize_metrics(self):"""Finalize metrics when streaming is complete."""end_time = time.time()- duration_ms = (end_time - self.start_time) * 1000- self.span.set_attribute("gen_ai.client.operation.duration", duration_ms)+ duration = end_time - self.start_time+ duration_ms = duration * 1000+ self.span.set_attribute("gen_ai.client.operation.duration", duration)# Process aggregated resultsif self.results:diff --git a/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py b/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.pyindex f5a27f11..8ef57a39 100644--- a/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py+++ b/sdk/python/src/openlit/instrumentation/crawl4ai/crawl4ai.py@@ -117,8 +117,9 @@ def general_wrap(# Calculate duration for business intelligenceend_time = time.time()- duration_ms = (end_time - start_time) * 1000- span.set_attribute("gen_ai.client.operation.duration", duration_ms)+ duration = end_time - start_time+ duration_ms = duration * 1000+ span.set_attribute("gen_ai.client.operation.duration", duration)# Process response based on operation typeif operation_name == "extract":@@ -180,8 +181,8 @@ def general_wrap(except Exception as e:# Calculate duration even for errorsend_time = time.time()- duration_ms = (end_time - start_time) * 1000- span.set_attribute("gen_ai.client.operation.duration", duration_ms)+ duration = end_time - start_time+ span.set_attribute("gen_ai.client.operation.duration", duration)# Handle and log the exceptionhandle_exception(span, e)diff --git a/sdk/python/tests/test_async_operation_duration_unit.py b/sdk/python/tests/test_async_operation_duration_unit.pynew file mode 100644index 00000000..b3d1fa77--- /dev/null+++ b/sdk/python/tests/test_async_operation_duration_unit.py@@ -0,0 +1,164 @@+# pylint: disable=protected-access, missing-function-docstring, duplicate-code, too-few-public-methods+"""Async runtime coverage for the `gen_ai.client.operation.duration` unit.++`test_operation_duration_unit.py` asserts the unit on the synchronous wrappers and+adds a static guard over the sources. Five of the nine original sites are in the+async modules, whose runtime behaviour nothing exercises: the async paths record+the attribute from three different places, and a static guard cannot tell which+`* 1000` is the semconv attribute and which is openlit's internal millisecond+bookkeeping.++These tests await the real async wrapper with a frozen clock and assert the+recorded attribute is elapsed **seconds** on both shapes:++- a normal awaited call,+- a call whose wrapped function raises.+"""++import asyncio+from unittest.mock import patch++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++import openlit+from openlit._config import OpenlitConfig+from openlit.instrumentation.crawl4ai import async_crawl4ai as async_mod+from openlit.semcov import SemanticConvention++# Initialise OpenLIT the way the rest of this suite does, so class-level+# configuration such as custom_span_attributes is populated.+openlit.init(+ environment="openlit-python-testing",+ application_name="openlit-python-async-duration-test",+)++DURATION = SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION+ELAPSED_SECONDS = 2.5++[email protected](autouse=True)+def _reset_config():+ OpenlitConfig.reset_to_defaults()+ yield+ OpenlitConfig.reset_to_defaults()+++def _tracer_with_exporter():+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _recorded_durations(exporter):+ """Every value the finished spans carry for the duration attribute."""+ return [+ span.attributes[DURATION]+ for span in exporter.get_finished_spans()+ if span.attributes and DURATION in span.attributes+ ]+++class _Clock:+ """A clock that advances by exactly ELAPSED_SECONDS on its second read."""++ def __init__(self):+ self.calls = 0++ def __call__(self):+ self.calls += 1+ return 1000.0 if self.calls == 1 else 1000.0 + ELAPSED_SECONDS+++def _wrapped(tracer, wrapped):+ return async_mod.async_general_wrap(+ "crawl",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics={},+ disable_metrics=True,+ )(wrapped, None, (), {})+++def test_normal_await_records_seconds():+ tracer, exporter = _tracer_with_exporter()++ async def _call(*_args, **_kwargs):+ return {"success": True}++ with patch.object(async_mod.time, "time", _Clock()):+ asyncio.run(_wrapped(tracer, _call))++ recorded = _recorded_durations(exporter)+ assert recorded, "the async wrapper recorded no duration at all"+ for value in recorded:+ # 2500.0 before the fix: milliseconds, 1000x past the seconds ladder+ # openlit declares for this histogram.+ assert value == pytest.approx(ELAPSED_SECONDS), (+ f"duration is declared in seconds, got {value!r}"+ )+++def test_error_path_records_seconds():+ tracer, exporter = _tracer_with_exporter()++ async def _raises(*_args, **_kwargs):+ raise RuntimeError("boom")++ with patch.object(async_mod.time, "time", _Clock()):+ with pytest.raises(RuntimeError):+ asyncio.run(_wrapped(tracer, _raises))++ recorded = _recorded_durations(exporter)+ assert recorded, "the error path recorded no duration at all"+ for value in recorded:+ assert value == pytest.approx(ELAPSED_SECONDS), (+ f"the error path must use the same unit, got {value!r}"+ )++def test_internal_millisecond_bookkeeping_is_not_the_attribute():+ """openlit keeps an internal millisecond aggregate; that is not the attribute.++ A purely static guard cannot separate the two, because both derive from the+ same elapsed value in the same function. This pins the distinction: the+ semconv attribute is seconds while the internal list stays in milliseconds.+ """+ tracer, exporter = _tracer_with_exporter()+ metrics: dict = {}++ async def _call(*_args, **_kwargs):+ return {"success": True}++ with patch.object(async_mod.time, "time", _Clock()):+ asyncio.run(+ async_mod.async_general_wrap(+ "crawl",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=metrics,+ disable_metrics=False,+ )(_call, None, (), {})+ )++ for value in _recorded_durations(exporter):+ assert value == pytest.approx(ELAPSED_SECONDS)++ for key, samples in metrics.items():+ if key.endswith("duration"):+ for sample in samples:+ assert sample == pytest.approx(ELAPSED_SECONDS * 1000), (+ f"internal {key} is milliseconds by design, got {sample!r}"+ )diff --git a/sdk/python/tests/test_browser_operation_duration_unit.py b/sdk/python/tests/test_browser_operation_duration_unit.pynew file mode 100644index 00000000..29d0c7cf--- /dev/null+++ b/sdk/python/tests/test_browser_operation_duration_unit.py@@ -0,0 +1,177 @@+# pylint: disable=protected-access, missing-function-docstring, duplicate-code, too-few-public-methods+"""Runtime coverage for the synchronous and asynchronous Browser-Use wrappers.++`test_async_operation_duration_unit.py` only drives+`crawl4ai.async_crawl4ai.async_general_wrap`. A revert experiment shows the two+`browser_use` modules (four of the nine corrected sites) are not pinned:+re-applying the `* 1000` factor in `browser_use.py` or+`async_browser_use.py` leaves that suite green. These tests await/call the real+Browser-Use wrappers with a frozen clock and assert the recorded+`gen_ai.client.operation.duration` is elapsed **seconds** on the normal and the+error path of each wrapper.+"""++import asyncio+from unittest.mock import patch++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++import openlit+from openlit._config import OpenlitConfig+from openlit.instrumentation.browser_use import (+ async_browser_use as async_mod,+)+from openlit.instrumentation.browser_use import browser_use as sync_mod+from openlit.semcov import SemanticConvention++openlit.init(+ environment="openlit-python-testing",+ application_name="openlit-python-browser-duration-test",+)++DURATION = SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION+ELAPSED_SECONDS = 2.5++[email protected](autouse=True)+def _reset_config():+ OpenlitConfig.reset_to_defaults()+ yield+ OpenlitConfig.reset_to_defaults()+++def _tracer_and_exporter():+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _recorded_durations(exporter):+ return [+ span.attributes[DURATION]+ for span in exporter.get_finished_spans()+ if span.attributes and DURATION in span.attributes+ ]+++class _Clock:+ """Advances by exactly ELAPSED_SECONDS between the first and second read."""++ def __init__(self):+ self.calls = 0++ def __call__(self):+ self.calls += 1+ return 1000.0 if self.calls == 1 else 1000.0 + ELAPSED_SECONDS+++def _assert_seconds(recorded):+ assert recorded, "the wrapper recorded no operation duration"+ for value in recorded:+ assert value == pytest.approx(ELAPSED_SECONDS), (+ f"duration is declared in seconds, got {value!r}"+ )+++def test_sync_records_seconds():+ tracer, exporter = _tracer_and_exporter()++ def _call(*_args, **_kwargs):+ return {"success": True}++ wrapped = sync_mod.general_wrap(+ "navigate",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics={},+ disable_metrics=True,+ )++ with patch.object(sync_mod.time, "time", _Clock()):+ wrapped(_call, None, (), {})++ _assert_seconds(_recorded_durations(exporter))+++def test_sync_error_path_records_seconds():+ tracer, exporter = _tracer_and_exporter()++ def _raises(*_args, **_kwargs):+ raise RuntimeError("boom")++ wrapped = sync_mod.general_wrap(+ "navigate",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics={},+ disable_metrics=True,+ )++ with patch.object(sync_mod.time, "time", _Clock()):+ with pytest.raises(RuntimeError):+ wrapped(_raises, None, (), {})++ _assert_seconds(_recorded_durations(exporter))+++def test_async_browser_records_seconds():+ tracer, exporter = _tracer_and_exporter()++ async def _call(*_args, **_kwargs):+ return {"success": True}++ wrapped = async_mod.async_general_wrap(+ "navigate",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics={},+ disable_metrics=True,+ )++ with patch.object(async_mod.time, "time", _Clock()):+ asyncio.run(wrapped(_call, None, (), {}))++ _assert_seconds(_recorded_durations(exporter))+++def test_async_browser_error_path_records_seconds():+ tracer, exporter = _tracer_and_exporter()++ async def _raises(*_args, **_kwargs):+ raise RuntimeError("boom")++ wrapped = async_mod.async_general_wrap(+ "navigate",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics={},+ disable_metrics=True,+ )++ with patch.object(async_mod.time, "time", _Clock()):+ with pytest.raises(RuntimeError):+ asyncio.run(wrapped(_raises, None, (), {}))++ _assert_seconds(_recorded_durations(exporter))diff --git a/sdk/python/tests/test_crawl4ai_operation_duration_unit.py b/sdk/python/tests/test_crawl4ai_operation_duration_unit.pynew file mode 100644index 00000000..24fd2144--- /dev/null+++ b/sdk/python/tests/test_crawl4ai_operation_duration_unit.py@@ -0,0 +1,207 @@+# pylint: disable=protected-access, missing-function-docstring, duplicate-code, too-few-public-methods+"""Runtime coverage for the synchronous Crawl4AI wrapper and the streaming path.++``test_async_operation_duration_unit.py`` drives ``async_crawl4ai.async_general_wrap``+for an awaited coroutine (async sites at ``async_crawl4ai.py`` lines 151/194), and+``test_browser_operation_duration_unit.py`` drives the four browser_use sites. Two+of the nine corrected sites are still not pinned by either file:++* ``crawl4ai/crawl4ai.py`` lines 120 (success) and 184 (error): the synchronous+ wrapper is never called, so re-applying ``* 1000`` there leaves the suite green.+* ``crawl4ai/async_crawl4ai.py`` line 318 (``_finalize_metrics``): reached only after+ the tracked async generator is fully consumed. The span is already ended at that+ point, so the seconds attribute there is dropped (tracked separately as #1618);+ the value that remains observable is openlit's internal streaming collection,+ which is intentionally kept in **milliseconds**.++The first two tests call the real synchronous wrapper with a frozen clock and assert+the recorded attribute is elapsed seconds on the normal and the error path. The last+test consumes a real async generator and pins the streaming collection at+``crawl4ai.crawl.stream.duration == 2500.0`` milliseconds, without asserting the+(absent) span attribute.+"""++import asyncio+from unittest.mock import patch++import pytest+from opentelemetry.sdk.trace import TracerProvider+from opentelemetry.sdk.trace.export import SimpleSpanProcessor+from opentelemetry.sdk.trace.export.in_memory_span_exporter import (+ InMemorySpanExporter,+)++import openlit+from openlit._config import OpenlitConfig+from openlit.instrumentation.crawl4ai import async_crawl4ai as async_mod+from openlit.instrumentation.crawl4ai import crawl4ai as sync_mod+from openlit.semcov import SemanticConvention++openlit.init(+ environment="openlit-python-testing",+ application_name="openlit-python-crawl4ai-duration-test",+)++DURATION = SemanticConvention.GEN_AI_CLIENT_OPERATION_DURATION+ELAPSED_SECONDS = 2.5+STREAM_DURATION_KEY = "crawl4ai.crawl.stream.duration"++[email protected](autouse=True)+def _reset_config():+ OpenlitConfig.reset_to_defaults()+ yield+ OpenlitConfig.reset_to_defaults()+++def _tracer_and_exporter():+ exporter = InMemorySpanExporter()+ provider = TracerProvider()+ provider.add_span_processor(SimpleSpanProcessor(exporter))+ return provider.get_tracer(__name__), exporter+++def _recorded_durations(exporter):+ return [+ span.attributes[DURATION]+ for span in exporter.get_finished_spans()+ if span.attributes and DURATION in span.attributes+ ]+++class _Clock:+ """Advances by exactly ELAPSED_SECONDS between the first and second read."""++ def __init__(self):+ self.calls = 0++ def __call__(self):+ self.calls += 1+ return 1000.0 if self.calls == 1 else 1000.0 + ELAPSED_SECONDS+++def _sync_wrap(tracer, metrics=None, disable_metrics=True):+ return sync_mod.general_wrap(+ "crawl",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics={} if metrics is None else metrics,+ disable_metrics=disable_metrics,+ )+++def test_sync_crawl_records_seconds():+ tracer, exporter = _tracer_and_exporter()++ def _call(*_args, **_kwargs):+ return {"success": True}++ wrapped = _sync_wrap(tracer)+ with patch.object(sync_mod.time, "time", _Clock()):+ wrapped(_call, None, (), {})++ recorded = _recorded_durations(exporter)+ assert recorded, "the synchronous wrapper recorded no operation duration"+ for value in recorded:+ assert value == pytest.approx(ELAPSED_SECONDS), (+ f"duration is declared in seconds, got {value!r}"+ )+++def test_sync_crawl_error_path_records_seconds():+ tracer, exporter = _tracer_and_exporter()++ def _raises(*_args, **_kwargs):+ raise RuntimeError("boom")++ wrapped = _sync_wrap(tracer)+ with patch.object(sync_mod.time, "time", _Clock()):+ with pytest.raises(RuntimeError):+ wrapped(_raises, None, (), {})++ recorded = _recorded_durations(exporter)+ assert recorded, "the synchronous error path recorded no operation duration"+ for value in recorded:+ assert value == pytest.approx(ELAPSED_SECONDS), (+ f"the error path must use the same unit, got {value!r}"+ )+++async def _consume(aiterable):+ results = []+ async for item in aiterable:+ results.append(item)+ return results+++def test_streaming_finalize_records_millisecond_collection_not_span_attribute():+ """Site 9: only the internal collection is observable after the span ends.++ ``_finalize_metrics`` runs from inside ``TrackedAsyncGenerator.__anext__`` when+ the consumer exhausts the generator, by which time the enclosing span context has+ already exited. We therefore assert:++ * the seconds attribute is absent from the exported (ended) span, and+ * the streaming aggregate ``crawl4ai.crawl.stream.duration`` is recorded in+ milliseconds (2.5 s elapsed -> 2500.0).+ """+ tracer, _exporter = _tracer_and_exporter()+ # The wrapper gates internal collection on a truthy ``metrics`` mapping+ # (``if not disable_metrics and metrics:``). In production openlit passes its+ # shared, already-populated metrics registry; seed an unrelated key so the gate+ # is open and the streaming aggregate is actually recorded.+ metrics: dict = {"crawl4ai.crawl.success": 0}++ async def _chunks():+ yield {"chunk": 1}+ yield {"chunk": 2}++ async def _call(*_args, **_kwargs):+ # An async coroutine that resolves to an async-iterator is the shape the+ # wrapper's streaming branch awaits (it does `response = await wrapped(...)`+ # then checks ``hasattr(response, "__aiter__")``), matching crawl4ai methods+ # that hand back a stream object rather than native async-generator functions.+ return _chunks()++ wrapped = async_mod.async_general_wrap(+ "crawl",+ "0.0.0-test",+ "test-env",+ "test-app",+ tracer,+ pricing_info={},+ capture_message_content=False,+ metrics=metrics,+ disable_metrics=False,+ )++ with patch.object(async_mod.time, "time", _Clock()):+ tracked = wrapped(_call, None, (), {})+ # wrapper returns a coroutine; awaiting it yields the tracked generator+ generator = asyncio.run(tracked)+ chunks = asyncio.run(_consume(generator))++ assert chunks == [{"chunk": 1}, {"chunk": 2}]++ assert STREAM_DURATION_KEY in metrics, (+ "exhausting the tracked generator did not finalize streaming metrics"+ )+ samples = metrics[STREAM_DURATION_KEY]+ assert samples, "the streaming duration collection is empty"+ for sample in samples:+ assert sample == pytest.approx(ELAPSED_SECONDS * 1000), (+ f"internal {STREAM_DURATION_KEY} is milliseconds by design, got {sample!r}"+ )++ # The span is closed before _finalize_metrics runs, so its set_attribute there is+ # dropped. This documents the #1618 lifecycle defect rather than asserting the+ # (unreachable) attribute.+ finished = [s for s in _exporter.get_finished_spans() if s.name.startswith("crawl")]+ assert finished, "no crawl span was exported"+ assert not any(+ s.attributes and DURATION in s.attributes for s in finished+ ), "streaming duration unexpectedly reached the ended span (#1618 unchanged)"