Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Harden skyvern-api parsing/validation boundaries (SKY-12523, SKY-12625, SKY-12515) (#7539)
Aaron Perez61d1ef255779
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.pyindex e76d76b77..88446516f 100644--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py@@ -42,6 +42,7 @@ from skyvern.forge.sdk.api.llm.utils import (is_truncated_response,llm_messages_builder,llm_messages_builder_with_history,+ loads_with_repair,parse_api_response,)from skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequest@@ -1841,7 +1842,7 @@ class LLMAPIHandlerFactory:if context and len(context.hashed_href_map) > 0:llm_content = json.dumps(parsed_response)rendered_content = Template(llm_content).render(context.hashed_href_map)- parsed_response = json.loads(rendered_content)+ parsed_response = loads_with_repair(rendered_content)rendered_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:if _should_bundle:@@ -2397,7 +2398,7 @@ class LLMAPIHandlerFactory:if context and len(context.hashed_href_map) > 0:llm_content = json.dumps(parsed_response)rendered_content = Template(llm_content).render(context.hashed_href_map)- parsed_response = json.loads(rendered_content)+ parsed_response = loads_with_repair(rendered_content)rendered_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:if _should_bundle:@@ -2998,7 +2999,7 @@ class LLMCaller:if context and len(context.hashed_href_map) > 0:llm_content = json.dumps(parsed_response)rendered_content = Template(llm_content).render(context.hashed_href_map)- parsed_response = json.loads(rendered_content)+ parsed_response = loads_with_repair(rendered_content)rendered_response_json = json.dumps(parsed_response, indent=2)if should_persist_llm_artifacts:if _should_bundle:diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.pyindex ec4170eba..f053d6f9b 100644--- a/skyvern/forge/sdk/api/llm/utils.py+++ b/skyvern/forge/sdk/api/llm/utils.py@@ -379,6 +379,20 @@ def _fix_and_parse_json_string(json_string: str) -> dict[str, Any]:return _fix_cutoff_json(json_string, error_position)+def loads_with_repair(content: str) -> Any:+ """Parse JSON, repairing invalid control characters instead of raising.++ Rendering the hashed-href map back into a serialized LLM response injects raw+ page values (hrefs, attribute text) that can carry literal control characters,+ which strict ``json.loads`` rejects. json_repair tolerates them.+ """+ try:+ return json.loads(content)+ except json.JSONDecodeError:+ LOG.warning("Strict JSON parse failed; falling back to json_repair.", exc_info=True)+ return json_repair.loads(content)++def _try_to_extract_json_from_markdown_format(text: str) -> str:pattern = r"```json\s*(.*?)\s*```"match = re.search(pattern, text, re.DOTALL)diff --git a/skyvern/forge/sdk/api/llm/utils_test.py b/skyvern/forge/sdk/api/llm/utils_test.pyindex 377fd4821..40e4a245a 100644--- a/skyvern/forge/sdk/api/llm/utils_test.py+++ b/skyvern/forge/sdk/api/llm/utils_test.py@@ -8,7 +8,29 @@ import litellmimport pytestfrom skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat-from skyvern.forge.sdk.api.llm.utils import _fix_cutoff_json, _fix_unescaped_quotes_in_json, parse_api_response+from skyvern.forge.sdk.api.llm.utils import (+ _fix_cutoff_json,+ _fix_unescaped_quotes_in_json,+ loads_with_repair,+ parse_api_response,+)+++class TestLoadsWithRepair:+ """Tests for loads_with_repair, used when rendering hashed-href values back into JSON."""++ def test_valid_json_passes_through(self) -> None:+ assert loads_with_repair('{"url": "https://example.com"}') == {"url": "https://example.com"}++ def test_invalid_control_character_does_not_raise(self) -> None:+ # A raw control character inside a string value is what strict json.loads rejects+ # ("Invalid control character at ...") when hashed hrefs are rendered into the payload.+ rendered = '{"path": "/a\x1fb"}'+ with pytest.raises(json.JSONDecodeError):+ json.loads(rendered)+ result = loads_with_repair(rendered)+ assert isinstance(result, dict)+ assert "path" in resultclass TestParseApiResponse:diff --git a/skyvern/services/webhook_delivery.py b/skyvern/services/webhook_delivery.pyindex 5ed3d7b53..2fd91838b 100644--- a/skyvern/services/webhook_delivery.py+++ b/skyvern/services/webhook_delivery.py@@ -7,10 +7,12 @@ import randomfrom dataclasses import dataclassfrom datetime import datetime, timezonefrom email.utils import parsedate_to_datetime+from urllib.parse import urlparseimport httpximport structlog+from skyvern.exceptions import InvalidUrlfrom skyvern.forge import appLOG = structlog.get_logger()@@ -88,6 +90,13 @@ async def deliver_webhook_with_retries(max_attempts: int = WEBHOOK_DELIVERY_MAX_ATTEMPTS,base_delay_seconds: float = WEBHOOK_DELIVERY_RETRY_BASE_DELAY_SECONDS,) -> httpx.Response:+ parsed_url = urlparse(url)+ if parsed_url.scheme not in ("http", "https") or not parsed_url.netloc:+ # Reject scheme-less/host-less targets before the outbound call so the+ # failure is classified as invalid input rather than a raw+ # httpx.UnsupportedProtocol from the transport.+ raise InvalidUrl(url)+last_response: httpx.Response | None = Nonelast_exc: Exception | None = Nonediff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.pyindex 31a336eb2..283e21086 100644--- a/skyvern/webeye/scraper/scraper.py+++ b/skyvern/webeye/scraper/scraper.py@@ -39,10 +39,14 @@ from skyvern.webeye.utils.page import SkyvernFrameLOG = structlog.get_logger()-async def build_scraping_failed_reason(browser_state: BrowserState, requested_url: str) -> str:+async def build_scraping_failed_reason(+ browser_state: BrowserState, requested_url: str, *, timed_out: bool = False+) -> str:"""Build the user-facing ScrapingFailed reason with the requested URL plus the landed URL when they differ.Query strings are stripped because OAuth/SSO URLs commonly carry secrets in query params.+ ``timed_out`` keeps the word "timeout" in the reason so failure_classifier routes page-analysis+ timeouts to PAGE_LOAD_TIMEOUT instead of lumping them into DATA_EXTRACTION_FAILURE."""safe_requested = strip_query_params(requested_url)safe_landed: str | None = None@@ -55,10 +59,16 @@ async def build_scraping_failed_reason(browser_state: BrowserState, requested_urexcept Exception:LOG.debug("Could not resolve landed URL for ScrapingFailed reason", exc_info=True)- base = (- "Skyvern failed to load the website. "- "The page may have navigated unexpectedly or become unresponsive during analysis."- )+ if timed_out:+ base = (+ "Skyvern hit a page-analysis timeout while loading the website. "+ "The page took too long to load or become responsive during analysis."+ )+ else:+ base = (+ "Skyvern failed to load the website. "+ "The page may have navigated unexpectedly or become unresponsive during analysis."+ )if safe_landed and safe_landed != safe_requested and safe_landed not in {"about:blank", ""}:return f"{base} Requested URL: {safe_requested}. Current URL: {safe_landed}."return f"{base} URL: {safe_requested}."@@ -256,7 +266,9 @@ async def scrape_website(if isinstance(e, FailedToTakeScreenshot):raise eelse:- raise ScrapingFailed(reason=await build_scraping_failed_reason(browser_state, url)) from e+ raise ScrapingFailed(+ reason=await build_scraping_failed_reason(browser_state, url, timed_out=isinstance(e, TimeoutError))+ ) from eLOG.info("Scraping failed, will retry", max_retries=max_retries, num_retry=num_retry, url=url, wait_seconds=0.5)await asyncio.sleep(0.5)return await scrape_website(@@ -594,7 +606,14 @@ async def filter_frames(filtered_frames.append(frame)continue- decision = await scrape_exclude(frame.page, frame)+ try:+ frame_page = frame.page+ except AssertionError:+ # Playwright's Frame.page asserts self._page; a frame can detach between+ # the is_detached() check above and here, leaving _page unset.+ continue++ decision = await scrape_exclude(frame_page, frame)if decision.placeholder is not None and decision.placeholder not in placeholder_nodes:placeholder_nodes.append(decision.placeholder)if decision.exclude:diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.pyindex 96421bfe4..732e245d3 100644--- a/tests/unit/test_failure_classifier.py+++ b/tests/unit/test_failure_classifier.py@@ -1,8 +1,12 @@from __future__ import annotations+from unittest.mock import AsyncMock, MagicMock+import pytest+from skyvern.exceptions import ScrapingFailedfrom skyvern.forge.failure_classifier import classify_from_failure_reason+from skyvern.webeye.scraper.scraper import build_scraping_failed_reasonclass NoProxyAvailable(Exception):@@ -294,3 +298,26 @@ def test_inactivity_timeout_is_not_infrastructure() -> None:assert "INFRASTRUCTURE_ERROR" not in categoriesassert "PAGE_LOAD_TIMEOUT" in categories+++async def test_page_analysis_timeout_reason_ranks_page_load_timeout() -> None:+ browser_state = MagicMock()+ browser_state.get_working_page = AsyncMock(return_value=None)++ timeout_reason = await build_scraping_failed_reason(browser_state, "https://example.com/path", timed_out=True)+ assert "timeout" in timeout_reason.lower()+ # A page-analysis timeout is wrapped in ScrapingFailed; its reason must still rank+ # PAGE_LOAD_TIMEOUT above DATA_EXTRACTION_FAILURE so it stays distinguishable.+ categories = _categories_for(timeout_reason, ScrapingFailed(reason=timeout_reason))+ assert categories[0] == "PAGE_LOAD_TIMEOUT"+++async def test_non_timeout_scraping_reason_stays_data_extraction() -> None:+ browser_state = MagicMock()+ browser_state.get_working_page = AsyncMock(return_value=None)++ generic_reason = await build_scraping_failed_reason(browser_state, "https://example.com/path", timed_out=False)+ assert "timeout" not in generic_reason.lower()+ assert _categories_for(generic_reason, ScrapingFailed(reason=generic_reason)) == ["DATA_EXTRACTION_FAILURE"]diff --git a/tests/unit/test_scrape_frame_decision.py b/tests/unit/test_scrape_frame_decision.pyindex 86194f106..642a72d91 100644--- a/tests/unit/test_scrape_frame_decision.py+++ b/tests/unit/test_scrape_frame_decision.py@@ -120,6 +120,28 @@ class TestFilterFrames:assert frames == [keep]assert placeholders == []+ @pytest.mark.asyncio+ async def test_frame_detached_after_check_skips_instead_of_asserting(self) -> None:+ # Playwright's Frame.page asserts self._page; a frame detached between the+ # is_detached() check and the page access raises AssertionError mid-scrape.+ class _DetachRacingFrame:+ def is_detached(self) -> bool:+ return False++ @property+ def page(self) -> object:+ raise AssertionError++ keep = _make_frame()+ racing = _DetachRacingFrame()++ async def _exclude(page: object, frame: object) -> ScrapeFrameDecision:+ return ScrapeFrameDecision(exclude=False)++ frames, placeholders = await filter_frames([keep, racing], _exclude)+ assert frames == [keep]+ assert placeholders == []+@pytest.fixture(autouse=True)def _skyvern_ctx() -> Iterator[None]:diff --git a/tests/unit/test_webhook_delivery.py b/tests/unit/test_webhook_delivery.pynew file mode 100644index 000000000..fdf5eb8eb--- /dev/null+++ b/tests/unit/test_webhook_delivery.py@@ -0,0 +1,44 @@+from unittest.mock import AsyncMock, MagicMock++import httpx+import pytest++from skyvern.exceptions import InvalidUrl+from skyvern.forge import app+from skyvern.services import webhook_delivery++[email protected]("bad_url", ["example.com/webhook", "ftp://example.com", "", "//example.com/x"])+async def test_scheme_less_url_rejected_before_delivery(monkeypatch: pytest.MonkeyPatch, bad_url: str) -> None:+ deliver = AsyncMock()+ monkeypatch.setattr(app, "AGENT_FUNCTION", MagicMock(deliver_webhook=deliver), raising=False)++ with pytest.raises(InvalidUrl):+ await webhook_delivery.deliver_webhook_with_retries(+ url=bad_url,+ payload="{}",+ headers={},+ timeout_seconds=1.0,+ organization_id="o_1",+ run_id="wr_1",+ )+ deliver.assert_not_awaited()+++async def test_valid_https_url_reaches_delivery(monkeypatch: pytest.MonkeyPatch) -> None:+ resp = httpx.Response(200, request=httpx.Request("POST", "https://example.com/webhook"))+ deliver = AsyncMock(return_value=resp)+ monkeypatch.setattr(app, "AGENT_FUNCTION", MagicMock(deliver_webhook=deliver), raising=False)++ out = await webhook_delivery.deliver_webhook_with_retries(+ url="https://example.com/webhook",+ payload="{}",+ headers={},+ timeout_seconds=1.0,+ organization_id="o_1",+ run_id="wr_1",+ )+ assert out.status_code == 200+ deliver.assert_awaited_once()