Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: improve reCAPTCHA/hCaptcha solving with prompt fix and proactive detection (SKY-8421) (#5171)
LawyZheng4c185b98321a
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.pyindex fb1dbef65..5eb78e324 100644--- a/skyvern/forge/agent.py+++ b/skyvern/forge/agent.py@@ -982,7 +982,7 @@ class ForgeAgent:context.step_id = step.step_idstep = await self.update_step(step=step, status=StepStatus.running)- await app.AGENT_FUNCTION.prepare_step_execution(+ injected_actions = await app.AGENT_FUNCTION.prepare_step_execution(organization=organization, task=task, step=step, browser_state=browser_state)@@ -1025,7 +1025,17 @@ class ForgeAgent:detailed_agent_step_output.extract_action_prompt = extract_action_promptactions: list[Action]- if engine == RunEngine.openai_cua:+ # If prepare_step_execution injected actions (e.g. proactive captcha solving),+ # skip LLM entirely and use the injected actions directly.+ if injected_actions is not None:+ LOG.info(+ "Using injected actions from prepare_step_execution, skipping LLM",+ step_id=step.step_id,+ num_actions=len(injected_actions),+ action_types=[a.action_type for a in injected_actions],+ )+ actions = injected_actions+ elif engine == RunEngine.openai_cua:actions, new_cua_response = await self._generate_cua_actions(task=task,step=step,diff --git a/skyvern/forge/agent_functions.py b/skyvern/forge/agent_functions.pyindex f8de2f045..eabd57446 100644--- a/skyvern/forge/agent_functions.py+++ b/skyvern/forge/agent_functions.py@@ -477,11 +477,14 @@ class AgentFunction:task: Task,step: Step,browser_state: BrowserState,- ) -> None:+ ) -> list[Action] | None:"""Get prepared for the step execution. It's called at the first beginning when step running.++ Returns:+ A list of actions to inject into the step (skipping LLM), or None for normal flow."""- return+ return Noneasync def post_step_execution(self, task: Task, step: Step) -> None:returndiff --git a/skyvern/forge/prompts/skyvern/extract-action-static.j2 b/skyvern/forge/prompts/skyvern/extract-action-static.j2index 11f432a93..2e17a6330 100644--- a/skyvern/forge/prompts/skyvern/extract-action-static.j2+++ b/skyvern/forge/prompts/skyvern/extract-action-static.j2@@ -19,7 +19,7 @@ Reply in JSON format with the following keys:"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data."confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence"action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}- "id": str, // The id of the element to take action on. The id has to be one from the elements list+ "id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null."captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER"."text": str, // Text for INPUT_TEXT action only"key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". null if not KEYPRESS action.diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2index 27aa8d301..a703a7361 100644--- a/skyvern/forge/prompts/skyvern/extract-action.j2+++ b/skyvern/forge/prompts/skyvern/extract-action.j2@@ -19,7 +19,7 @@ Reply in JSON format with the following keys:"user_detail_answer": str, // The answer to the `user_detail_query`. The source of this answer can be user goal or user details. This answer CAN contain specific user data."confidence_float": float, // The confidence of the action. Pick a number between 0.0 and 1.0. 0.0 means no confidence, 1.0 means full confidence"action_type": str, // It's a string enum: "CLICK", "HOVER", "INPUT_TEXT", "UPLOAD_FILE", "SELECT_OPTION", "WAIT", "SOLVE_CAPTCHA", "COMPLETE", "TERMINATE", "KEYPRESS", "SCROLL"{{', "CLOSE_PAGE"' if has_magic_link_page else ""}}. "CLICK" is an element you'd like to click. "HOVER" is used to move the mouse over an element without clicking, particularly when revealing hover-only menus or buttons before clicking, or when the UI hints that a control (like a CTA button) only appears after hovering a card, tile, or model name. "INPUT_TEXT" is an element you'd like to input text into. "UPLOAD_FILE" is an element you'd like to upload a file into. "SELECT_OPTION" is an element you'd like to select an option from. "WAIT" action should be used if there are no actions to take and there is some indication on screen that waiting could yield more actions. "WAIT" should not be used if there are actions to take. "SOLVE_CAPTCHA" should be used if there's a captcha to solve on the screen. "COMPLETE" is used when the {{"complete criterion has been met" if complete_criterion else "user goal has been achieved"}} AND if there's any data extraction goal, you should be able to get data from the page. Never return a COMPLETE action unless the {{ "complete criterion is met" if complete_criterion else "user goal is achieved" }}. "TERMINATE" is used to terminate the whole task with a failure when it doesn't seem like the user goal can be achieved. Do not use "TERMINATE" if waiting could lead the user towards the goal. Only return "TERMINATE" if you are on a page where the user goal cannot be achieved. All other actions are ignored when "TERMINATE" is returned. "KEYPRESS" is used to press a keyboard key when no clickable button or element achieves the same result. Only use KEYPRESS when pressing a key is the sole way to proceed (e.g., pressing Enter to submit a search with no search button, or Escape to close a modal with no close button). KEYPRESS does not require an element id. Requires the "key" field. "SCROLL" is used to scroll within a specific scrollable container on the page (not the page itself). Only use SCROLL when a required action is blocked because the target element is hidden, disabled, or unreachable until the container is scrolled (e.g., an "Agree" button that only becomes enabled after scrolling to the bottom of a terms and conditions box). Do not use SCROLL for general page navigation. Provide the element id of an interactable element within or near the scrollable container. Requires the "direction" field. SCROLL does not require a "text", "option", or "file_url" field.{{' "CLOSE_PAGE" is used to close the current page when it is impossible to achieve the user goal on the current page.' if has_magic_link_page else ''}}- "id": str, // The id of the element to take action on. The id has to be one from the elements list+ "id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null."captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER"."text": str, // Text for INPUT_TEXT action only"key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". null if not KEYPRESS action.diff --git a/tests/unit/test_parallel_verification.py b/tests/unit/test_parallel_verification.pyindex b3324a502..ecdae7cb3 100644--- a/tests/unit/test_parallel_verification.py+++ b/tests/unit/test_parallel_verification.py@@ -219,7 +219,7 @@ async def test_agent_step_skips_user_goal_check_when_feature_disabled(monkeypatc"skyvern.forge.agent.LLMAPIHandlerFactory.get_override_llm_api_handler",lambda *_args, **_kwargs: llm_handler_mock,)- monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution", AsyncMock())+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution", AsyncMock(return_value=None))monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_action_execution", AsyncMock())monkeypatch.setattr("skyvern.forge.agent.asyncio.sleep", AsyncMock(return_value=None))monkeypatch.setattr("skyvern.forge.agent.random.uniform", lambda *_args, **_kwargs: 0)@@ -344,7 +344,7 @@ async def test_agent_step_persists_artifacts_when_using_speculative_plan(action_handler_mock = AsyncMock(return_value=[ActionSuccess()])monkeypatch.setattr("skyvern.forge.agent.ActionHandler.handle_action", action_handler_mock)- monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution", AsyncMock())+ monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.prepare_step_execution", AsyncMock(return_value=None))monkeypatch.setattr("skyvern.forge.agent.app.AGENT_FUNCTION.post_action_execution", AsyncMock())monkeypatch.setattr("skyvern.forge.agent.asyncio.sleep", AsyncMock(return_value=None))monkeypatch.setattr("skyvern.forge.agent.random.uniform", lambda *_args, **_kwargs: 0)