Skip to content

Automate browser based workflows with AI

add date_format context for input and select action context, which will be used as context for caching (#3951)

Shuchang Zheng43e689027617
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.py
index 740ca760a..d462bfd06 100644
--- a/skyvern/core/script_generations/generate_script.py
+++ b/skyvern/core/script_generations/generate_script.py
@@ -431,6 +431,21 @@ def _action_to_stmt(act: dict[str, Any], task: dict[str, Any], assign_to_output:
comma=cst.Comma(),
)
)
+ action_context = act.get("input_or_select_context")
+ if action_context and action_context.get("date_format") and method in ["type", "fill", "select_option"]:
+ date_format_value = action_context.get("date_format")
+ data = {"date_format": date_format_value}
+ args.append(
+ cst.Arg(
+ keyword=cst.Name("data"),
+ value=_value(data),
+ whitespace_after_arg=cst.ParenthesizedWhitespace(
+ indent=True,
+ last_line=cst.SimpleWhitespace(INDENT),
+ ),
+ )
+ )
+
intention = act.get("intention") or act.get("reasoning") or ""
if intention and method not in ACTIONS_OPT_OUT_INTENTION_FOR_PROMPT:
args.extend(
diff --git a/skyvern/forge/prompts/skyvern/extract-action-static.j2 b/skyvern/forge/prompts/skyvern/extract-action-static.j2
index f3b6353fb..265fa3a2a 100644
--- a/skyvern/forge/prompts/skyvern/extract-action-static.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action-static.j2
@@ -27,6 +27,10 @@ Reply in JSON format with the following keys:
"label": str, // the label of the option if any. MAKE SURE YOU USE THIS LABEL TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION LABEL HERE
"index": int, // the index corresponding to the option index under the select element.
"value": str // the value of the option. MAKE SURE YOU USE THIS VALUE TO SELECT THE OPTION. DO NOT PUT ANYTHING OTHER THAN A VALID OPTION VALUE HERE
+ },
+ "click_context": { // The context for CLICK action only. null if not CLICK action
+ "thought": str, // Describe how you decided that this action is a single choice option or multi-choice option.
+ "single_option_click": bool, // True if the click is the only choice to proceed towards the goal, regardless of different user context or input. False if there are multiple valid options that depend on user input. Examples: clicking a login button to login is True (it's the only way to login); clicking a radio button for a multi-choice question (e.g., selecting "male", "female", or "other" for gender) is False (the choice depends on user input). When clicking on radio buttons, dropdown options, or any element that represents one of multiple possible selections, this should be False.
}{% if parse_select_feature_enabled %},
"context": { // The context for INPUT_TEXT or SELECT_OPTION action only. null if not INPUT_TEXT or SELECT_OPTION action. Extract the following detailed information from the "reasoning", and double-check the information by analysing the HTML elements.
"thought": str, // A string to describe how you double-check the context information to ensure the accuracy.
@@ -35,6 +39,7 @@ Reply in JSON format with the following keys:
"is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.
"is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.
"is_date_related": bool, // True if the field is related to date input or select, otherwise false.
+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.
}{% endif %}
}],{% if verification_code_check %}
"verification_code_reasoning": str, // Let's think step by step. Describe what you see and think if there is somewhere on the current page where you must enter the verification code now for login or any verification step. Explain why you believe a verification code needs to be entered somewhere or not. Do not imagine any place to enter the code if the code has not been sent yet.
diff --git a/skyvern/forge/prompts/skyvern/extract-action.j2 b/skyvern/forge/prompts/skyvern/extract-action.j2
index 0bb804914..f7db072e3 100644
--- a/skyvern/forge/prompts/skyvern/extract-action.j2
+++ b/skyvern/forge/prompts/skyvern/extract-action.j2
@@ -39,6 +39,7 @@ Reply in JSON format with the following keys:
"is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.
"is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.
"is_date_related": bool, // True if the field is related to date input or select, otherwise false.
+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.
}{% endif %}
}],{% if verification_code_check %}
"verification_code_reasoning": str, // Let's think step by step. Describe what you see and think if there is somewhere on the current page where you must enter the verification code now for login or any verification step. Explain why you believe a verification code needs to be entered somewhere or not. Do not imagine any place to enter the code if the code has not been sent yet.
diff --git a/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2 b/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2
index 54210d388..81e341b38 100644
--- a/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2
+++ b/skyvern/forge/prompts/skyvern/parse-input-or-select-context.j2
@@ -10,6 +10,7 @@ Reply in the following JSON format:
"is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.
"is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.
"is_date_related": bool, // True if the field is related to date input or select, otherwise false.
+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.
}
Existing reasoning context:
diff --git a/skyvern/forge/prompts/skyvern/single-input-action.j2 b/skyvern/forge/prompts/skyvern/single-input-action.j2
index 18f58a8ee..24b32c555 100644
--- a/skyvern/forge/prompts/skyvern/single-input-action.j2
+++ b/skyvern/forge/prompts/skyvern/single-input-action.j2
@@ -22,6 +22,7 @@ Reply in JSON format with the following keys:
"is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.
"is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.
"is_date_related": bool, // True if the field is related to date input or select, otherwise false.
+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.
}{% endif %}
}]{% if verification_code_check %}
"verification_code_reasoning": str, // Let's think step by step. Describe what you see and think if there is somewhere on the current page where you must enter the verification code now for login or any verification step. Explain why you believe a verification code needs to be entered somewhere or not. Do not imagine any place to enter the code if the code has not been sent yet.
diff --git a/skyvern/forge/prompts/skyvern/single-select-action.j2 b/skyvern/forge/prompts/skyvern/single-select-action.j2
index 6388c45cc..e3a92978c 100644
--- a/skyvern/forge/prompts/skyvern/single-select-action.j2
+++ b/skyvern/forge/prompts/skyvern/single-select-action.j2
@@ -26,6 +26,7 @@ Reply in JSON format with the following keys:
"is_search_bar": bool, // True if the element to take the action is a search bar, otherwise false.
"is_location_input": bool, // True if the element is asking user to input where he lives, otherwise false. For example, it is asking for location, or address, or other similar information. Output False if it only requires ZIP code or postal code.
"is_date_related": bool, // True if the field is related to date input or select, otherwise false.
+ "date_format": str, // The format of the date or datetime to be input. For example YYYY-MM-DD, YYYY-MM-DD HH:MM:SS, DD.MM.YYYY, MM/DD/YYYY, etc. If the field is not related to date input or select, this should be null.
}{% endif %}
}]
}
diff --git a/skyvern/webeye/actions/actions.py b/skyvern/webeye/actions/actions.py
index e23a40edf..a2110e7de 100644
--- a/skyvern/webeye/actions/actions.py
+++ b/skyvern/webeye/actions/actions.py
@@ -82,6 +82,7 @@ class InputOrSelectContext(BaseModel):
is_search_bar: bool | None = None # don't trigger custom-selection logic when it's a search bar
is_location_input: bool | None = None # address input usually requires auto completion
is_date_related: bool | None = None # date picker mini agent requires some special logic
+ date_format: str | None = None
def __repr__(self) -> str:
return f"InputOrSelectContext(field={self.field}, is_required={self.is_required}, is_search_bar={self.is_search_bar}, is_location_input={self.is_location_input}, intention={self.intention})"