Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix: harden css-vars.test.ts existsSync filter for OSS sync target (#5852)
Andrew Neilsonae076804a781
diff --git a/skyvern/config.py b/skyvern/config.pyindex ef1304f01..4d2c2e490 100644--- a/skyvern/config.py+++ b/skyvern/config.py@@ -100,6 +100,8 @@ class Settings(BaseSettings):DISABLE_CONNECTION_POOL: bool = FalseDATABASE_POOL_SIZE: int = 5DATABASE_POOL_MAX_OVERFLOW: int = 10+ DATABASE_POOL_TIMEOUT: int = 30+ DATABASE_POOL_RECYCLE: int = 1800PROMPT_ACTION_HISTORY_WINDOW: int = 1TASK_RESPONSE_ACTION_SCREENSHOT_COUNT: int = 3diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex 2a8775579..3084d8d77 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -79,7 +79,6 @@ def _build_engine(database_string: str) -> AsyncEngine:return engine- # PostgreSQL path (unchanged)connect_args: dict[str, Any] = {}if settings.DISABLE_CONNECTION_POOL:if "postgresql+psycopg" in database_string:@@ -105,6 +104,8 @@ def _build_engine(database_string: str) -> AsyncEngine:pool_pre_ping=True,pool_size=settings.DATABASE_POOL_SIZE,max_overflow=settings.DATABASE_POOL_MAX_OVERFLOW,+ pool_timeout=settings.DATABASE_POOL_TIMEOUT,+ pool_recycle=settings.DATABASE_POOL_RECYCLE,)diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex ea2276500..8cb4a3148 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -2,6 +2,7 @@ import asyncioimport copyimport jsonimport os+import reimport timeimport urllib.parseimport uuid@@ -25,7 +26,7 @@ from skyvern.constants import (DROPDOWN_MENU_MAX_DISTANCE,SKYVERN_ID_ATTR,)-from skyvern.errors.errors import TOTPExpiredError, filter_to_user_defined_codes+from skyvern.errors.errors import TOTPExpiredError, UserDefinedError, filter_to_user_defined_codesfrom skyvern.exceptions import (EmptySelect,ErrEmptyTweakValue,@@ -105,7 +106,6 @@ from skyvern.webeye.actions.actions import (SelectOption,SelectOptionAction,UploadFileAction,- UserDefinedError,WebAction,)from skyvern.webeye.actions.responses import ActionAbort, ActionFailure, ActionResult, ActionSuccess@@ -4906,6 +4906,40 @@ async def _get_input_or_select_context(return input_or_select_context+def _match_user_defined_error_from_reasoning(task: Task, step: Step, reasoning: str) -> list[UserDefinedError]:+ # If the LLM returns no structured errors but its terminate reasoning+ # explicitly mentions a configured code or description, preserve that+ # machine-readable code for task/run/webhook error aggregation.+ normalized_reasoning = reasoning.lower()+ matched_errors: list[UserDefinedError] = []+ for error_code, error_description in (task.error_code_mapping or {}).items():+ # Only match structured codes directly. Generic single-word codes like+ # "timeout" can appear in unrelated reasoning and look falsely authoritative.+ code_matches = (+ "_" in error_code and re.search(rf"\b{re.escape(error_code.lower())}\b", normalized_reasoning) is not None+ )+ description_matches = isinstance(error_description, str) and error_description.lower() in normalized_reasoning+ if code_matches or description_matches:+ matched_errors.append(+ UserDefinedError(+ error_code=error_code,+ reasoning=reasoning,+ confidence_float=1.0,+ )+ )+ if matched_errors:+ if len(matched_errors) > 1:+ LOG.warning(+ "Multiple user-defined error mappings matched terminate reasoning; using first match",+ task_id=task.task_id,+ step_id=step.step_id,+ matched_error_codes=[error.error_code for error in matched_errors],+ selected_error_code=matched_errors[0].error_code,+ )+ return [matched_errors[0]]+ return []++async def extract_user_defined_errors(task: Task, step: Step, scraped_page: ScrapedPage, reasoning: str | None = None) -> list[UserDefinedError]:@@ -4938,4 +4972,6 @@ async def extract_user_defined_errors(dropped_codes=dropped,allowed_codes=sorted((task.error_code_mapping or {}).keys()),)+ if not kept and reasoning:+ return _match_user_defined_error_from_reasoning(task=task, step=step, reasoning=reasoning)return keptdiff --git a/tests/unit/forge/sdk/db/test_db_engine_config.py b/tests/unit/forge/sdk/db/test_db_engine_config.pynew file mode 100644index 000000000..27773c180--- /dev/null+++ b/tests/unit/forge/sdk/db/test_db_engine_config.py@@ -0,0 +1,60 @@+from unittest.mock import MagicMock, patch++import pytest+from sqlalchemy.ext.asyncio import AsyncEngine++from skyvern.forge.sdk.db import agent_db+++def _fake_engine() -> AsyncEngine:+ engine = MagicMock(spec=AsyncEngine)+ engine.dialect = MagicMock()+ engine.dialect.name = "postgresql"+ engine.pool = MagicMock()+ return engine+++def _call_build_engine(url: str = "postgresql+psycopg://user:pass@host/db") -> dict:+ captured: dict = {}++ def _capture(_url: str, **kwargs: object) -> AsyncEngine:+ captured.update(kwargs)+ return _fake_engine()++ with patch("skyvern.forge.sdk.db.agent_db.create_async_engine", side_effect=_capture):+ agent_db._build_engine(url)++ return captured+++def test_pool_timeout_forwarded_to_queue_pool(monkeypatch: pytest.MonkeyPatch) -> None:+ import skyvern.config++ monkeypatch.setattr(skyvern.config.settings, "DISABLE_CONNECTION_POOL", False)+ monkeypatch.setattr(skyvern.config.settings, "DATABASE_POOL_TIMEOUT", 7)++ captured = _call_build_engine()++ assert captured.get("pool_timeout") == 7+++def test_pool_recycle_forwarded_to_queue_pool(monkeypatch: pytest.MonkeyPatch) -> None:+ import skyvern.config++ monkeypatch.setattr(skyvern.config.settings, "DISABLE_CONNECTION_POOL", False)+ monkeypatch.setattr(skyvern.config.settings, "DATABASE_POOL_RECYCLE", 900)++ captured = _call_build_engine()++ assert captured.get("pool_recycle") == 900+++def test_null_pool_path_does_not_forward_queue_pool_params(monkeypatch: pytest.MonkeyPatch) -> None:+ import skyvern.config++ monkeypatch.setattr(skyvern.config.settings, "DISABLE_CONNECTION_POOL", True)++ captured = _call_build_engine()++ assert "pool_timeout" not in captured+ assert "pool_recycle" not in captureddiff --git a/tests/unit/test_handle_terminate_action.py b/tests/unit/test_handle_terminate_action.pyindex 520257e52..f28fa2c6a 100644--- a/tests/unit/test_handle_terminate_action.py+++ b/tests/unit/test_handle_terminate_action.py@@ -4,13 +4,15 @@ Verifies that when extract_user_defined_errors fails (e.g., CDP disconnection),the original error codes from the action reasoning are preserved instead of being lost."""+from datetime import timezone+from types import SimpleNamespacefrom unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom skyvern.errors.errors import UserDefinedErrorfrom skyvern.webeye.actions.actions import TerminateAction-from skyvern.webeye.actions.handler import handle_terminate_action+from skyvern.webeye.actions.handler import extract_user_defined_errors, handle_terminate_actionfrom skyvern.webeye.actions.responses import ActionSuccess@@ -27,6 +29,35 @@ def _make_step() -> MagicMock:return step+async def _extract_with_empty_llm(+ task: MagicMock, step: MagicMock, reasoning: str, monkeypatch: pytest.MonkeyPatch+) -> list[UserDefinedError]:+ scraped_page = MagicMock()+ refreshed_page = MagicMock()+ refreshed_page.build_element_tree.return_value = ""+ refreshed_page.url = "chrome-error://chromewebdata/"+ refreshed_page.screenshots = []+ scraped_page.refresh = AsyncMock(return_value=refreshed_page)++ monkeypatch.setattr(+ "skyvern.webeye.actions.handler.app.EXTRACTION_LLM_API_HANDLER",+ AsyncMock(return_value={"errors": []}),+ raising=False,+ )+ with (+ patch(+ "skyvern.webeye.actions.handler.get_action_history",+ new_callable=AsyncMock,+ return_value=[],+ ),+ patch(+ "skyvern.webeye.actions.handler.skyvern_context.ensure_context",+ return_value=SimpleNamespace(tz_info=timezone.utc),+ ),+ ):+ return await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page, reasoning=reasoning)++@pytest.mark.asyncioasync def test_terminate_preserves_errors_when_extract_fails() -> None:"""When extract_user_defined_errors raises, action.errors from LLM reasoning should be preserved."""@@ -81,6 +112,80 @@ async def test_terminate_uses_extracted_errors_on_success() -> None:assert action.errors == extracted_errors+async def test_extract_user_defined_errors_falls_back_to_reasoning_match(monkeypatch: pytest.MonkeyPatch) -> None:+ task = _make_task(error_code_mapping={"portal_inaccessible": "portal is inaccessible"})+ task.navigation_goal = "Download invoices"+ task.navigation_payload = {}+ step = _make_step()++ reasoning = (+ "The page is a browser network error (ERR_CONNECTION_CLOSED) indicating the portal is inaccessible; "+ "terminating avoids further actions that cannot succeed."+ )+ errors = await _extract_with_empty_llm(task=task, step=step, reasoning=reasoning, monkeypatch=monkeypatch)++ assert len(errors) == 1+ assert errors[0].error_code == "portal_inaccessible"+ assert errors[0].reasoning == reasoning+++async def test_extract_user_defined_errors_uses_word_boundary_for_code_match(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ task = _make_task(+ error_code_mapping={+ "closed": "connection closed",+ "portal_inaccessible": "portal is inaccessible",+ }+ )+ task.navigation_goal = "Download invoices"+ task.navigation_payload = {}+ step = _make_step()++ reasoning = (+ "The page is a browser network error (ERR_CONNECTION_CLOSED) indicating the portal is inaccessible; "+ "terminating avoids further actions that cannot succeed."+ )+ errors = await _extract_with_empty_llm(task=task, step=step, reasoning=reasoning, monkeypatch=monkeypatch)++ assert len(errors) == 1+ assert errors[0].error_code == "portal_inaccessible"+++async def test_extract_user_defined_errors_logs_additional_reasoning_matches(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ task = _make_task(+ error_code_mapping={+ "portal_inaccessible": "portal is inaccessible",+ "network_error": "browser network error",+ }+ )+ task.navigation_goal = "Download invoices"+ task.navigation_payload = {}+ step = _make_step()++ reasoning = (+ "The page is a browser network error (ERR_CONNECTION_CLOSED) indicating the portal is inaccessible; "+ "terminating avoids further actions that cannot succeed."+ )+ with patch("skyvern.webeye.actions.handler.LOG.warning") as mock_warning:+ errors = await _extract_with_empty_llm(task=task, step=step, reasoning=reasoning, monkeypatch=monkeypatch)++ assert len(errors) == 1+ assert errors[0].error_code == "portal_inaccessible"+ mock_warning.assert_called_once_with(+ "Multiple user-defined error mappings matched terminate reasoning; using first match",+ task_id=task.task_id,+ step_id=step.step_id,+ matched_error_codes=["portal_inaccessible", "network_error"],+ selected_error_code="portal_inaccessible",+ )++@pytest.mark.asyncioasync def test_terminate_skips_extraction_without_error_code_mapping() -> None:"""When task has no error_code_mapping, extract_user_defined_errors should not be called."""