Skip to content

Automate browser based workflows with AI

feat(observability): coarse scrape attribution on skyvern.agent.scrape span (SKY-10319) (#6482)

LawyZheng9499cb52bbcb
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index b465a87c9..7210e46f3 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -330,6 +330,48 @@ async def _wait_for_scrape_ready(skyvern_frame: SkyvernFrame) -> None:
await skyvern_frame.safe_wait_for_animation_end(caller="scraper.scrape_ready")
+def _record_scrape_span_attrs(
+ *,
+ elements: list,
+ html: str,
+ text_content: str,
+ url: str,
+ take_screenshots: bool,
+ draw_boxes: bool,
+ scroll: bool,
+ max_screenshot_number: int,
+ screenshots: list[bytes],
+ id_to_frame_dict: dict,
+ empty_page_retry: bool,
+) -> None:
+ """Attach coarse scrape attribution to the active ``skyvern.agent.scrape`` span.
+
+ Falls back to ``scrape_trigger="unspecified"`` when the caller did not set
+ SkyvernContext.scrape_trigger, so dashboards can quantify the unattributed
+ bucket instead of dropping the data point.
+ """
+ span = otel_trace.get_current_span()
+ span.set_attribute("element_count", len(elements))
+ span.set_attribute("html_bytes", len(html) if html else 0)
+ span.set_attribute("text_bytes", len(text_content) if text_content else 0)
+ span.set_attribute("page_url", strip_query_params(url))
+ span.set_attribute("take_screenshots", take_screenshots)
+ span.set_attribute("draw_boxes", draw_boxes)
+ span.set_attribute("scroll", scroll)
+ span.set_attribute("max_screenshot_number", max_screenshot_number)
+ span.set_attribute("screenshot_count", len(screenshots))
+ span.set_attribute("screenshot_bytes", sum(len(s) for s in screenshots))
+ span.set_attribute("frame_count", len(id_to_frame_dict))
+ span.set_attribute("empty_page_retry", empty_page_retry)
+ ctx = skyvern_context.current()
+ span.set_attribute(
+ "scrape_trigger",
+ ctx.scrape_trigger if ctx and ctx.scrape_trigger else "unspecified",
+ )
+ if ctx and ctx.scrape_screenshots_consumed is not None:
+ span.set_attribute("screenshots_consumed", ctx.scrape_screenshots_consumed)
+
+
@traced(name="skyvern.agent.scrape")
async def scrape_web_unsafe(
browser_state: BrowserState,
@@ -385,9 +427,11 @@ async def scrape_web_unsafe(
await asyncio.sleep(wait_seconds)
elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
+ empty_page_retry = False
if not elements and not support_empty_page:
LOG.warning("No elements found on the page, wait and retry")
await empty_page_retry_wait()
+ empty_page_retry = True
elements, element_tree = await get_interactable_element_tree(page, scrape_exclude, must_included_tags)
element_tree = await cleanup_element_tree(page, url, copy.deepcopy(element_tree))
@@ -471,11 +515,19 @@ async def scrape_web_unsafe(
exc_info=True,
)
- _scrape_span = otel_trace.get_current_span()
- _scrape_span.set_attribute("element_count", len(elements))
- _scrape_span.set_attribute("html_bytes", len(html) if html else 0)
- _scrape_span.set_attribute("text_bytes", len(text_content) if text_content else 0)
- _scrape_span.set_attribute("page_url", strip_query_params(url))
+ _record_scrape_span_attrs(
+ elements=elements,
+ html=html,
+ text_content=text_content,
+ url=url,
+ take_screenshots=take_screenshots,
+ draw_boxes=draw_boxes,
+ scroll=scroll,
+ max_screenshot_number=max_screenshot_number,
+ screenshots=screenshots,
+ id_to_frame_dict=id_to_frame_dict,
+ empty_page_retry=empty_page_retry,
+ )
return ScrapedPage(
elements=elements,
diff --git a/tests/unit/test_scrape_span_attrs.py b/tests/unit/test_scrape_span_attrs.py
new file mode 100644
index 000000000..a77f20937
--- /dev/null
+++ b/tests/unit/test_scrape_span_attrs.py
@@ -0,0 +1,132 @@
+"""Coarse observability attrs on the ``skyvern.agent.scrape`` span.
+
+Pure-unit coverage of ``_record_scrape_span_attrs``: a real tracer + the shared
+``span_exporter`` fixture, no browser, no Playwright. Asserts the input/mode,
+output-shape, caller-bucket, and outcome attrs are emitted under both an
+attributed (``scrape_trigger`` set) and an unattributed (fallback) caller.
+"""
+
+from __future__ import annotations
+
+from zoneinfo import ZoneInfo
+
+import pytest
+from opentelemetry import trace as otel_trace
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+from skyvern.forge.sdk.core import skyvern_context
+from skyvern.forge.sdk.core.skyvern_context import SkyvernContext
+from skyvern.webeye.scraper.scraper import _record_scrape_span_attrs
+
+SCRAPE_SPAN_NAME = "skyvern.agent.scrape"
+
+
+def _span_by_name(spans, name):
+ return next((s for s in spans if s.name == name), None)
+
+
+def _record_inside_span(**kwargs) -> None:
+ tracer = otel_trace.get_tracer("skyvern")
+ with tracer.start_as_current_span(SCRAPE_SPAN_NAME):
+ _record_scrape_span_attrs(**kwargs)
+
+
+_BASE_KWARGS = dict(
+ elements=[{"id": "x"}, {"id": "y"}],
+ html="<html></html>",
+ text_content="hello world",
+ url="https://example.com/path?secret=abc",
+ take_screenshots=True,
+ draw_boxes=False,
+ scroll=True,
+ max_screenshot_number=3,
+ screenshots=[b"aaa", b"bbbb"],
+ id_to_frame_dict={"f1": object(), "f2": object(), "f3": object()},
+ empty_page_retry=False,
+)
+
+
+def test_scrape_span_records_mode_output_and_unspecified_trigger_when_no_context(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ skyvern_context.reset()
+ _record_inside_span(**_BASE_KWARGS)
+
+ span = _span_by_name(span_exporter.get_finished_spans(), SCRAPE_SPAN_NAME)
+ assert span is not None
+ attrs = span.attributes or {}
+
+ # Output shape (cheap signals only)
+ assert attrs.get("element_count") == 2
+ assert attrs.get("html_bytes") == len("<html></html>")
+ assert attrs.get("text_bytes") == len("hello world")
+ assert attrs.get("screenshot_count") == 2
+ assert attrs.get("screenshot_bytes") == len(b"aaa") + len(b"bbbb")
+ assert attrs.get("frame_count") == 3
+
+ # Input/mode params
+ assert attrs.get("take_screenshots") is True
+ assert attrs.get("draw_boxes") is False
+ assert attrs.get("scroll") is True
+ assert attrs.get("max_screenshot_number") == 3
+
+ # Outcome
+ assert attrs.get("empty_page_retry") is False
+
+ # Caller bucket: unspecified when no SkyvernContext is active
+ assert attrs.get("scrape_trigger") == "unspecified"
+ # screenshots_consumed is intentionally absent when the caller did not set it
+ assert "screenshots_consumed" not in attrs
+
+ # URL is stripped of query params to bound cardinality
+ page_url = attrs.get("page_url")
+ assert page_url is not None
+ assert "secret" not in page_url
+
+
+def test_scrape_span_uses_context_trigger_when_caller_sets_it(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ ctx = SkyvernContext(tz_info=ZoneInfo("UTC"))
+ ctx.scrape_trigger = "verification"
+ ctx.scrape_screenshots_consumed = True
+ skyvern_context.set(ctx)
+ try:
+ _record_inside_span(**_BASE_KWARGS)
+ finally:
+ skyvern_context.reset()
+
+ span = _span_by_name(span_exporter.get_finished_spans(), SCRAPE_SPAN_NAME)
+ assert span is not None
+ attrs = span.attributes or {}
+ assert attrs.get("scrape_trigger") == "verification"
+ assert attrs.get("screenshots_consumed") is True
+
+
+def test_scrape_span_marks_empty_page_retry_when_set(
+ span_exporter: InMemorySpanExporter,
+) -> None:
+ kwargs = dict(_BASE_KWARGS)
+ kwargs["empty_page_retry"] = True
+ skyvern_context.reset()
+ _record_inside_span(**kwargs)
+
+ span = _span_by_name(span_exporter.get_finished_spans(), SCRAPE_SPAN_NAME)
+ assert span is not None
+ attrs = span.attributes or {}
+ assert attrs.get("empty_page_retry") is True
+
+
[email protected]("html_value, expected_bytes", [("", 0), ("abc", 3)])
+def test_scrape_span_handles_empty_html(
+ span_exporter: InMemorySpanExporter, html_value: str, expected_bytes: int
+) -> None:
+ kwargs = dict(_BASE_KWARGS)
+ kwargs["html"] = html_value
+ skyvern_context.reset()
+ _record_inside_span(**kwargs)
+
+ span = _span_by_name(span_exporter.get_finished_spans(), SCRAPE_SPAN_NAME)
+ assert span is not None
+ attrs = span.attributes or {}
+ assert attrs.get("html_bytes") == expected_bytes