Skip to content

Automate browser based workflows with AI

Harden skyvern-api parsing/validation boundaries (SKY-12523, SKY-12625, SKY-12515) (#7539)

Aaron Perez61d1ef255779
diff --git a/skyvern/forge/sdk/api/llm/api_handler_factory.py b/skyvern/forge/sdk/api/llm/api_handler_factory.py
index e76d76b77..88446516f 100644
--- a/skyvern/forge/sdk/api/llm/api_handler_factory.py
+++ b/skyvern/forge/sdk/api/llm/api_handler_factory.py
@@ -42,6 +42,7 @@ from skyvern.forge.sdk.api.llm.utils import (
is_truncated_response,
llm_messages_builder,
llm_messages_builder_with_history,
+ loads_with_repair,
parse_api_response,
)
from skyvern.forge.sdk.artifact.manager import BulkArtifactCreationRequest
@@ -1841,7 +1842,7 @@ class LLMAPIHandlerFactory:
if context and len(context.hashed_href_map) > 0:
llm_content = json.dumps(parsed_response)
rendered_content = Template(llm_content).render(context.hashed_href_map)
- parsed_response = json.loads(rendered_content)
+ parsed_response = loads_with_repair(rendered_content)
rendered_response_json = json.dumps(parsed_response, indent=2)
if should_persist_llm_artifacts:
if _should_bundle:
@@ -2397,7 +2398,7 @@ class LLMAPIHandlerFactory:
if context and len(context.hashed_href_map) > 0:
llm_content = json.dumps(parsed_response)
rendered_content = Template(llm_content).render(context.hashed_href_map)
- parsed_response = json.loads(rendered_content)
+ parsed_response = loads_with_repair(rendered_content)
rendered_response_json = json.dumps(parsed_response, indent=2)
if should_persist_llm_artifacts:
if _should_bundle:
@@ -2998,7 +2999,7 @@ class LLMCaller:
if context and len(context.hashed_href_map) > 0:
llm_content = json.dumps(parsed_response)
rendered_content = Template(llm_content).render(context.hashed_href_map)
- parsed_response = json.loads(rendered_content)
+ parsed_response = loads_with_repair(rendered_content)
rendered_response_json = json.dumps(parsed_response, indent=2)
if should_persist_llm_artifacts:
if _should_bundle:
diff --git a/skyvern/forge/sdk/api/llm/utils.py b/skyvern/forge/sdk/api/llm/utils.py
index ec4170eba..f053d6f9b 100644
--- a/skyvern/forge/sdk/api/llm/utils.py
+++ b/skyvern/forge/sdk/api/llm/utils.py
@@ -379,6 +379,20 @@ def _fix_and_parse_json_string(json_string: str) -> dict[str, Any]:
return _fix_cutoff_json(json_string, error_position)
+def loads_with_repair(content: str) -> Any:
+ """Parse JSON, repairing invalid control characters instead of raising.
+
+ Rendering the hashed-href map back into a serialized LLM response injects raw
+ page values (hrefs, attribute text) that can carry literal control characters,
+ which strict ``json.loads`` rejects. json_repair tolerates them.
+ """
+ try:
+ return json.loads(content)
+ except json.JSONDecodeError:
+ LOG.warning("Strict JSON parse failed; falling back to json_repair.", exc_info=True)
+ return json_repair.loads(content)
+
+
def _try_to_extract_json_from_markdown_format(text: str) -> str:
pattern = r"```json\s*(.*?)\s*```"
match = re.search(pattern, text, re.DOTALL)
diff --git a/skyvern/forge/sdk/api/llm/utils_test.py b/skyvern/forge/sdk/api/llm/utils_test.py
index 377fd4821..40e4a245a 100644
--- a/skyvern/forge/sdk/api/llm/utils_test.py
+++ b/skyvern/forge/sdk/api/llm/utils_test.py
@@ -8,7 +8,29 @@ import litellm
import pytest
from skyvern.forge.sdk.api.llm.exceptions import InvalidLLMResponseFormat
-from skyvern.forge.sdk.api.llm.utils import _fix_cutoff_json, _fix_unescaped_quotes_in_json, parse_api_response
+from skyvern.forge.sdk.api.llm.utils import (
+ _fix_cutoff_json,
+ _fix_unescaped_quotes_in_json,
+ loads_with_repair,
+ parse_api_response,
+)
+
+
+class TestLoadsWithRepair:
+ """Tests for loads_with_repair, used when rendering hashed-href values back into JSON."""
+
+ def test_valid_json_passes_through(self) -> None:
+ assert loads_with_repair('{"url": "https://example.com"}') == {"url": "https://example.com"}
+
+ def test_invalid_control_character_does_not_raise(self) -> None:
+ # A raw control character inside a string value is what strict json.loads rejects
+ # ("Invalid control character at ...") when hashed hrefs are rendered into the payload.
+ rendered = '{"path": "/a\x1fb"}'
+ with pytest.raises(json.JSONDecodeError):
+ json.loads(rendered)
+ result = loads_with_repair(rendered)
+ assert isinstance(result, dict)
+ assert "path" in result
class TestParseApiResponse:
diff --git a/skyvern/services/webhook_delivery.py b/skyvern/services/webhook_delivery.py
index 5ed3d7b53..2fd91838b 100644
--- a/skyvern/services/webhook_delivery.py
+++ b/skyvern/services/webhook_delivery.py
@@ -7,10 +7,12 @@ import random
from dataclasses import dataclass
from datetime import datetime, timezone
from email.utils import parsedate_to_datetime
+from urllib.parse import urlparse
import httpx
import structlog
+from skyvern.exceptions import InvalidUrl
from skyvern.forge import app
LOG = structlog.get_logger()
@@ -88,6 +90,13 @@ async def deliver_webhook_with_retries(
max_attempts: int = WEBHOOK_DELIVERY_MAX_ATTEMPTS,
base_delay_seconds: float = WEBHOOK_DELIVERY_RETRY_BASE_DELAY_SECONDS,
) -> httpx.Response:
+ parsed_url = urlparse(url)
+ if parsed_url.scheme not in ("http", "https") or not parsed_url.netloc:
+ # Reject scheme-less/host-less targets before the outbound call so the
+ # failure is classified as invalid input rather than a raw
+ # httpx.UnsupportedProtocol from the transport.
+ raise InvalidUrl(url)
+
last_response: httpx.Response | None = None
last_exc: Exception | None = None
diff --git a/skyvern/webeye/scraper/scraper.py b/skyvern/webeye/scraper/scraper.py
index 31a336eb2..283e21086 100644
--- a/skyvern/webeye/scraper/scraper.py
+++ b/skyvern/webeye/scraper/scraper.py
@@ -39,10 +39,14 @@ from skyvern.webeye.utils.page import SkyvernFrame
LOG = structlog.get_logger()
-async def build_scraping_failed_reason(browser_state: BrowserState, requested_url: str) -> str:
+async def build_scraping_failed_reason(
+ browser_state: BrowserState, requested_url: str, *, timed_out: bool = False
+) -> str:
"""Build the user-facing ScrapingFailed reason with the requested URL plus the landed URL when they differ.
Query strings are stripped because OAuth/SSO URLs commonly carry secrets in query params.
+ ``timed_out`` keeps the word "timeout" in the reason so failure_classifier routes page-analysis
+ timeouts to PAGE_LOAD_TIMEOUT instead of lumping them into DATA_EXTRACTION_FAILURE.
"""
safe_requested = strip_query_params(requested_url)
safe_landed: str | None = None
@@ -55,10 +59,16 @@ async def build_scraping_failed_reason(browser_state: BrowserState, requested_ur
except Exception:
LOG.debug("Could not resolve landed URL for ScrapingFailed reason", exc_info=True)
- base = (
- "Skyvern failed to load the website. "
- "The page may have navigated unexpectedly or become unresponsive during analysis."
- )
+ if timed_out:
+ base = (
+ "Skyvern hit a page-analysis timeout while loading the website. "
+ "The page took too long to load or become responsive during analysis."
+ )
+ else:
+ base = (
+ "Skyvern failed to load the website. "
+ "The page may have navigated unexpectedly or become unresponsive during analysis."
+ )
if safe_landed and safe_landed != safe_requested and safe_landed not in {"about:blank", ""}:
return f"{base} Requested URL: {safe_requested}. Current URL: {safe_landed}."
return f"{base} URL: {safe_requested}."
@@ -256,7 +266,9 @@ async def scrape_website(
if isinstance(e, FailedToTakeScreenshot):
raise e
else:
- raise ScrapingFailed(reason=await build_scraping_failed_reason(browser_state, url)) from e
+ raise ScrapingFailed(
+ reason=await build_scraping_failed_reason(browser_state, url, timed_out=isinstance(e, TimeoutError))
+ ) from e
LOG.info("Scraping failed, will retry", max_retries=max_retries, num_retry=num_retry, url=url, wait_seconds=0.5)
await asyncio.sleep(0.5)
return await scrape_website(
@@ -594,7 +606,14 @@ async def filter_frames(
filtered_frames.append(frame)
continue
- decision = await scrape_exclude(frame.page, frame)
+ try:
+ frame_page = frame.page
+ except AssertionError:
+ # Playwright's Frame.page asserts self._page; a frame can detach between
+ # the is_detached() check above and here, leaving _page unset.
+ continue
+
+ decision = await scrape_exclude(frame_page, frame)
if decision.placeholder is not None and decision.placeholder not in placeholder_nodes:
placeholder_nodes.append(decision.placeholder)
if decision.exclude:
diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py
index 96421bfe4..732e245d3 100644
--- a/tests/unit/test_failure_classifier.py
+++ b/tests/unit/test_failure_classifier.py
@@ -1,8 +1,12 @@
from __future__ import annotations
+from unittest.mock import AsyncMock, MagicMock
+
import pytest
+from skyvern.exceptions import ScrapingFailed
from skyvern.forge.failure_classifier import classify_from_failure_reason
+from skyvern.webeye.scraper.scraper import build_scraping_failed_reason
class NoProxyAvailable(Exception):
@@ -294,3 +298,26 @@ def test_inactivity_timeout_is_not_infrastructure() -> None:
assert "INFRASTRUCTURE_ERROR" not in categories
assert "PAGE_LOAD_TIMEOUT" in categories
+
+
+async def test_page_analysis_timeout_reason_ranks_page_load_timeout() -> None:
+ browser_state = MagicMock()
+ browser_state.get_working_page = AsyncMock(return_value=None)
+
+ timeout_reason = await build_scraping_failed_reason(browser_state, "https://example.com/path", timed_out=True)
+ assert "timeout" in timeout_reason.lower()
+ # A page-analysis timeout is wrapped in ScrapingFailed; its reason must still rank
+ # PAGE_LOAD_TIMEOUT above DATA_EXTRACTION_FAILURE so it stays distinguishable.
+ categories = _categories_for(timeout_reason, ScrapingFailed(reason=timeout_reason))
+ assert categories[0] == "PAGE_LOAD_TIMEOUT"
+
+
+async def test_non_timeout_scraping_reason_stays_data_extraction() -> None:
+ browser_state = MagicMock()
+ browser_state.get_working_page = AsyncMock(return_value=None)
+
+ generic_reason = await build_scraping_failed_reason(browser_state, "https://example.com/path", timed_out=False)
+ assert "timeout" not in generic_reason.lower()
+ assert _categories_for(generic_reason, ScrapingFailed(reason=generic_reason)) == ["DATA_EXTRACTION_FAILURE"]
diff --git a/tests/unit/test_scrape_frame_decision.py b/tests/unit/test_scrape_frame_decision.py
index 86194f106..642a72d91 100644
--- a/tests/unit/test_scrape_frame_decision.py
+++ b/tests/unit/test_scrape_frame_decision.py
@@ -120,6 +120,28 @@ class TestFilterFrames:
assert frames == [keep]
assert placeholders == []
+ @pytest.mark.asyncio
+ async def test_frame_detached_after_check_skips_instead_of_asserting(self) -> None:
+ # Playwright's Frame.page asserts self._page; a frame detached between the
+ # is_detached() check and the page access raises AssertionError mid-scrape.
+ class _DetachRacingFrame:
+ def is_detached(self) -> bool:
+ return False
+
+ @property
+ def page(self) -> object:
+ raise AssertionError
+
+ keep = _make_frame()
+ racing = _DetachRacingFrame()
+
+ async def _exclude(page: object, frame: object) -> ScrapeFrameDecision:
+ return ScrapeFrameDecision(exclude=False)
+
+ frames, placeholders = await filter_frames([keep, racing], _exclude)
+ assert frames == [keep]
+ assert placeholders == []
+
@pytest.fixture(autouse=True)
def _skyvern_ctx() -> Iterator[None]:
diff --git a/tests/unit/test_webhook_delivery.py b/tests/unit/test_webhook_delivery.py
new file mode 100644
index 000000000..fdf5eb8eb
--- /dev/null
+++ b/tests/unit/test_webhook_delivery.py
@@ -0,0 +1,44 @@
+from unittest.mock import AsyncMock, MagicMock
+
+import httpx
+import pytest
+
+from skyvern.exceptions import InvalidUrl
+from skyvern.forge import app
+from skyvern.services import webhook_delivery
+
+
[email protected]("bad_url", ["example.com/webhook", "ftp://example.com", "", "//example.com/x"])
+async def test_scheme_less_url_rejected_before_delivery(monkeypatch: pytest.MonkeyPatch, bad_url: str) -> None:
+ deliver = AsyncMock()
+ monkeypatch.setattr(app, "AGENT_FUNCTION", MagicMock(deliver_webhook=deliver), raising=False)
+
+ with pytest.raises(InvalidUrl):
+ await webhook_delivery.deliver_webhook_with_retries(
+ url=bad_url,
+ payload="{}",
+ headers={},
+ timeout_seconds=1.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+ deliver.assert_not_awaited()
+
+
+async def test_valid_https_url_reaches_delivery(monkeypatch: pytest.MonkeyPatch) -> None:
+ resp = httpx.Response(200, request=httpx.Request("POST", "https://example.com/webhook"))
+ deliver = AsyncMock(return_value=resp)
+ monkeypatch.setattr(app, "AGENT_FUNCTION", MagicMock(deliver_webhook=deliver), raising=False)
+
+ out = await webhook_delivery.deliver_webhook_with_retries(
+ url="https://example.com/webhook",
+ payload="{}",
+ headers={},
+ timeout_seconds=1.0,
+ organization_id="o_1",
+ run_id="wr_1",
+ )
+ assert out.status_code == 200
+ deliver.assert_awaited_once()