Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Support None/Invalid selector in ai_upload_file (#3884)
Stanislav Novosad833ffff1eabb
diff --git a/skyvern/core/script_generations/real_skyvern_page_ai.py b/skyvern/core/script_generations/real_skyvern_page_ai.pyindex ff475f4bc..d2bc76b04 100644--- a/skyvern/core/script_generations/real_skyvern_page_ai.py+++ b/skyvern/core/script_generations/real_skyvern_page_ai.py@@ -13,7 +13,6 @@ from skyvern.constants import SPECIAL_FIELD_VERIFICATION_CODEfrom skyvern.core.script_generations.skyvern_page_ai import SkyvernPageAifrom skyvern.forge import appfrom skyvern.forge.prompts import prompt_engine-from skyvern.forge.sdk.api.files import download_filefrom skyvern.forge.sdk.core import skyvern_contextfrom skyvern.forge.sdk.schemas.totp_codes import OTPTypefrom skyvern.services.otp_service import poll_otp_value@@ -22,11 +21,13 @@ from skyvern.webeye.actions import handler_utilsfrom skyvern.webeye.actions.actions import (ActionStatus,InputTextAction,+ UploadFileAction,)from skyvern.webeye.actions.handler import (handle_click_action,handle_input_text_action,handle_select_option_action,+ handle_upload_file_action,)from skyvern.webeye.actions.parse_actions import parse_actionsfrom skyvern.webeye.scraper.scraper import ScrapedPage@@ -41,6 +42,9 @@ INPUT_GOAL = """- The intention to fill out an input: {intention}.SELECT_OPTION_GOAL = """- The intention to select an option: {intention}.- The overall goal that the user wants to achieve: {prompt}."""+UPLOAD_GOAL = """- The intention to upload a file: {intention}.+- The overall goal that the user wants to achieve: {prompt}."""+async def _get_element_id_by_selector(selector: str, page: Page) -> str | None:locator = page.locator(selector)@@ -312,7 +316,7 @@ class RealSkyvernPageAi(SkyvernPageAi):async def ai_upload_file(self,- selector: str,+ selector: str | None,files: str,intention: str,data: str | dict[str, Any] | None = None,@@ -320,30 +324,90 @@ class RealSkyvernPageAi(SkyvernPageAi):) -> str:"""Upload a file using AI to process the file URL."""+ context = skyvern_context.ensure_context()+ files = files or ""+ action: UploadFileAction | None = None+ organization_id = context.organization_id+ task_id = context.task_id+ step_id = context.step_id+ workflow_run_id = context.workflow_run_id+ task = await app.DATABASE.get_task(task_id, organization_id) if task_id and organization_id else None+ step = await app.DATABASE.get_step(step_id, organization_id) if step_id and organization_id else None+if intention:try:- context = skyvern_context.current()- prompt = context.prompt if context else None- data = _get_context_data(data)- script_generation_file_url_prompt = prompt_engine.load_prompt(- template="script-generation-file-url-generation",- intention=intention,- data=data,- goal=prompt,- )- json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(- prompt=script_generation_file_url_prompt,- prompt_name="script-generation-file-url-generation",- organization_id=context.organization_id if context else None,- )- files = json_response.get("answer", files)+ prompt = context.prompt+ data = data or {}++ refreshed_page = await self.scraped_page.generate_scraped_page_without_screenshots()+ self.scraped_page = refreshed_page++ # Try to get element_id from selector if selector is provided+ element_id = await _get_element_id_by_selector(selector, self.page) if selector else None++ if element_id:+ # The selector/element is valid, using a simpler/smaller prompt+ script_generation_file_url_prompt = prompt_engine.load_prompt(+ template="script-generation-file-url-generation",+ intention=intention,+ data=data,+ goal=prompt,+ )+ json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ prompt=script_generation_file_url_prompt,+ prompt_name="script-generation-file-url-generation",+ step=step,+ organization_id=organization_id,+ )+ files = json_response.get("answer", files)++ action = UploadFileAction(+ element_id=element_id,+ file_url=files,+ status=ActionStatus.pending,+ organization_id=organization_id,+ workflow_run_id=workflow_run_id,+ task_id=task_id,+ step_id=context.step_id if context else None,+ reasoning=intention,+ intention=intention,+ response=files,+ )+ else:+ # Use a heavier single-upload-action when selector is not found+ element_tree = refreshed_page.build_element_tree()+ payload_str = _get_context_data(data)+ merged_goal = UPLOAD_GOAL.format(intention=intention, prompt=prompt)++ single_upload_prompt = prompt_engine.load_prompt(+ template="single-upload-action",+ navigation_goal=merged_goal,+ navigation_payload_str=payload_str,+ current_url=self.page.url,+ elements=element_tree,+ local_datetime=datetime.now(context.tz_info or datetime.now().astimezone().tzinfo).isoformat(),+ )+ json_response = await app.SINGLE_INPUT_AGENT_LLM_API_HANDLER(+ prompt=single_upload_prompt,+ prompt_name="single-upload-action",+ step=step,+ organization_id=organization_id,+ )++ actions_json = json_response.get("actions", [])+ if actions_json and task and step:+ actions = parse_actions(task, step.step_id, step.order, refreshed_page, actions_json)+ if actions and isinstance(actions[0], UploadFileAction):+ action = cast(UploadFileAction, actions[0])+ files = action.file_urlexcept Exception:- LOG.exception(f"Failed to adapt value for input text action on selector={selector}, file={files}")- if not files:- raise ValueError("file url must be provided")- file_path = await download_file(files)- locator = self.page.locator(selector)- await locator.set_input_files(file_path, timeout=timeout)+ LOG.exception(f"Failed to adapt value for upload file action on selector={selector}, file={files}")++ if action and organization_id and task and step:+ result = await handle_upload_file_action(action, self.page, self.scraped_page, task, step)+ if result and result[-1].success is False:+ raise Exception(result[-1].exception_message)+return filesasync def ai_select_option(diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 6d82650f8..a354aa5ac 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -616,7 +616,7 @@ class SkyvernPage:@action_wrap(ActionType.UPLOAD_FILE)async def upload_file(self,- selector: str,+ selector: str | None,files: str,ai: str | None = "fallback",intention: str | None = None,@@ -628,12 +628,14 @@ class SkyvernPage:ai = context.ai_mode_overrideif ai == "fallback":error_to_raise = None- try:- file_path = await download_file(files)- locator = self.page.locator(selector)- await locator.set_input_files(file_path)- except Exception as e:- error_to_raise = e+ if selector:+ try:+ file_path = await download_file(files)+ locator = self.page.locator(selector)+ await locator.set_input_files(file_path)+ except Exception as e:+ error_to_raise = e+if intention:return await self._ai.ai_upload_file(selector=selector,@@ -654,6 +656,10 @@ class SkyvernPage:data=data,timeout=timeout,)++ if not selector:+ raise ValueError("Selector is required but was not provided")+file_path = await download_file(files)locator = self.page.locator(selector)await locator.set_input_files(file_path, timeout=timeout)diff --git a/skyvern/core/script_generations/skyvern_page_ai.py b/skyvern/core/script_generations/skyvern_page_ai.pyindex 4a31a1256..0770b7acd 100644--- a/skyvern/core/script_generations/skyvern_page_ai.py+++ b/skyvern/core/script_generations/skyvern_page_ai.py@@ -33,7 +33,7 @@ class SkyvernPageAi(Protocol):async def ai_upload_file(self,- selector: str,+ selector: str | None,files: str,intention: str,data: str | dict[str, Any] | None = None,diff --git a/skyvern/library/skyvern_browser_page_ai.py b/skyvern/library/skyvern_browser_page_ai.pyindex fce620052..0b7aca8da 100644--- a/skyvern/library/skyvern_browser_page_ai.py+++ b/skyvern/library/skyvern_browser_page_ai.py@@ -106,7 +106,7 @@ class SdkSkyvernPageAi(SkyvernPageAi):async def ai_upload_file(self,- selector: str,+ selector: str | None,files: str,intention: str,data: str | dict[str, Any] | None = None,