Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
fix(SKY-8978): allow KEYPRESS to type digits/letters as INPUT_TEXT fallback (#5568)
LawyZhengdae0e97e5aa9
diff --git a/skyvern/forge/prompts/skyvern/extract-action-static.j2 b/skyvern/forge/prompts/skyvern/extract-action-static.j2index 2e17a6330..7ed2cbbf6 100644--- a/skyvern/forge/prompts/skyvern/extract-action-static.j2+++ b/skyvern/forge/prompts/skyvern/extract-action-static.j2@@ -22,7 +22,7 @@ Reply in JSON format with the following keys:"id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null."captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER"."text": str, // Text for INPUT_TEXT action only- "key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". null if not KEYPRESS action.+ "key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". For "0"-"9" and "a"-"z", use KEYPRESS only when no suitable element is available for INPUT_TEXT. null if not KEYPRESS action."direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action."file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise."download": bool, // Can only be true for CLICK or SELECT_OPTION actions. If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2index a703a7361..1f84c75ee 100644--- a/skyvern/forge/prompts/skyvern/extract-action.j2+++ b/skyvern/forge/prompts/skyvern/extract-action.j2@@ -22,7 +22,7 @@ Reply in JSON format with the following keys:"id": str, // The id of the element to take action on. The id has to be one from the elements list. For target-less actions (SOLVE_CAPTCHA, WAIT, COMPLETE, TERMINATE, KEYPRESS), set this to null."captcha_type": str, // The type of captcha for SOLVE_CAPTCHA action only. null if not SOLVE_CAPTCHA action. It's a string enum: "TEXT_CAPTCHA", "RECAPTCHA", "HCAPTCHA", "MTCAPTCHA", "FUNCAPTCHA", "CLOUDFLARE", "OTHER"."text": str, // Text for INPUT_TEXT action only- "key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". null if not KEYPRESS action.+ "key": str, // The keyboard key to press for KEYPRESS action only. Allowed values: "Enter", "Tab", "Escape", "ArrowDown", "ArrowUp". For "0"-"9" and "a"-"z", use KEYPRESS only when no suitable element is available for INPUT_TEXT. null if not KEYPRESS action."direction": str, // The direction to scroll for SCROLL action only. Allowed values: "up", "down". null if not SCROLL action."file_url": str, // The url of the file to upload if applicable. This field must be present for UPLOAD_FILE but can also be present for CLICK only if the click is to upload the file. It should be null otherwise."download": bool, // Can only be true for CLICK or SELECT_OPTION actions. If true, the browser will trigger a download by clicking the element. If false, the browser will click the element without triggering a download.diff --git a/skyvern/webeye/actions/parse_actions.py b/skyvern/webeye/actions/parse_actions.pyindex 30a3f7975..36864b6e4 100644--- a/skyvern/webeye/actions/parse_actions.py+++ b/skyvern/webeye/actions/parse_actions.py@@ -209,11 +209,13 @@ def parse_action(base_action_dict["skyvern_element_hash"] = Nonebase_action_dict["skyvern_element_data"] = None# Support both "key" (single key from prompt) and "keys" (list, from code/legacy)- # Limited to navigation/submission keys to prevent misuse on regular form fields+ # Navigation/submission keys are always allowed. Single digits and letters are allowed+ # as a fallback for typing into hidden/overlay input components (e.g., 2FA digit boxes).allowed_keys = {"Enter", "Tab", "Escape", "ArrowDown", "ArrowUp"}key = action.get("key")if key:- if key not in allowed_keys:+ is_single_char = len(key) == 1 and (key.isdigit() or key.isalpha())+ if key not in allowed_keys and not is_single_char:LOG.warning("KEYPRESS action has unsupported key, skipping action", key=key)return NullAction(**base_action_dict)keys = [key]