Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: pass block termination criteria to script reviewer (#5435)
pedrohsdb1da33727f04e
diff --git a/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2 b/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2index 80f11f189..b51a0394d 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer-extraction.j2@@ -25,6 +25,31 @@ You are a script reviewer for a browser automation system. Your job is to make a```{{ navigation_goal }}```+{% if terminate_criterion %}++## Block Termination Criterion+When this condition is met, call `await page.terminate(errors=["<error_code>: <reason>"])`:+```+{{ terminate_criterion }}+```+You MUST check for this condition using `page.classify()` or `page.extract()` and terminate inside an `if`/`elif` branch when it is satisfied.+{% endif %}+{% if complete_criterion %}++## Block Completion Criterion+The block should call `await page.complete()` when this condition is met:+```+{{ complete_criterion }}+```+{% endif %}+{% if error_code_mapping %}++## Error Code Mapping+These are the specific error codes defined for this block. When terminating, use the matching error code:+{% for code, description in error_code_mapping.items() %}+- **{{ code }}**: {{ description }}+{% endfor %}+{% endif %}{% if parameter_keys %}## Workflow Parameter Keysdiff --git a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2 b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2index 45c325d46..c28b30a8f 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer-form-filling.j2@@ -31,6 +31,31 @@ Each entry in FIELD_MAP is a dict with these keys:```{{ navigation_goal }}```+{% if terminate_criterion %}++## Block Termination Criterion+When this condition is met, call `await page.terminate(errors=["<error_code>: <reason>"])`:+```+{{ terminate_criterion }}+```+You MUST check for this condition using `page.classify()` or `page.extract()` and terminate inside an `if`/`elif` branch when it is satisfied.+{% endif %}+{% if complete_criterion %}++## Block Completion Criterion+The block should call `await page.complete()` when this condition is met:+```+{{ complete_criterion }}+```+{% endif %}+{% if error_code_mapping %}++## Error Code Mapping+These are the specific error codes defined for this block. When terminating, use the matching error code:+{% for code, description in error_code_mapping.items() %}+- **{{ code }}**: {{ description }}+{% endfor %}+{% endif %}{% if parameter_keys %}## Workflow Parameter Keysdiff --git a/skyvern/forge/prompts/skyvern/script-reviewer.j2 b/skyvern/forge/prompts/skyvern/script-reviewer.j2index afdeb1938..69a0e05c6 100644--- a/skyvern/forge/prompts/skyvern/script-reviewer.j2+++ b/skyvern/forge/prompts/skyvern/script-reviewer.j2@@ -59,6 +59,32 @@ You are a script reviewer for a browser automation system. Your job is to update```{{ navigation_goal }}```+{% if terminate_criterion %}++## Block Termination Criterion+When this condition is met, call `await page.terminate(errors=["<error_code>: <reason>"])`:+```+{{ terminate_criterion }}+```+You MUST check for this condition using `page.classify()` or `page.extract()` and terminate inside an `if`/`elif` branch when it is satisfied. Do NOT rely on the AI fallback to handle termination — the script should detect it directly.+{% endif %}+{% if complete_criterion %}++## Block Completion Criterion+The block should call `await page.complete()` when this condition is met:+```+{{ complete_criterion }}+```+{% endif %}+{% if error_code_mapping %}++## Error Code Mapping+These are the specific error codes defined for this block. When terminating, use the matching error code:+{% for code, description in error_code_mapping.items() %}+- **{{ code }}**: {{ description }}+{% endfor %}+Use these codes in `page.terminate(errors=["<ERROR_CODE>: <details>"])` when the corresponding condition is detected.+{% endif %}{% if parameter_keys %}## Workflow Parameter Keysdiff --git a/skyvern/services/script_reviewer.py b/skyvern/services/script_reviewer.pyindex cb1287c01..1bb10b964 100644--- a/skyvern/services/script_reviewer.py+++ b/skyvern/services/script_reviewer.py@@ -155,8 +155,8 @@ class ScriptReviewer:if not episodes:return None- # Load the workflow definition to get navigation goals and parameter keys- navigation_goals, all_parameter_keys = await self._load_workflow_context(+ # Load the workflow definition to get navigation goals, parameter keys, and block criteria+ navigation_goals, all_parameter_keys, block_criteria = await self._load_workflow_context(organization_id=organization_id,workflow_permanent_id=workflow_permanent_id,)@@ -251,6 +251,7 @@ class ScriptReviewer:run_parameter_values=run_parameter_values,user_instructions=user_instructions,historical_run_params=historical_run_params,+ block_criteria=block_criteria.get(block_label),)if result:updated_blocks[block_label] = result@@ -294,7 +295,7 @@ class ScriptReviewer:)# No episodes — review all blocks with user instructions only- navigation_goals, all_parameter_keys = await self._load_workflow_context(+ navigation_goals, all_parameter_keys, block_criteria = await self._load_workflow_context(organization_id=organization_id,workflow_permanent_id=workflow_permanent_id,)@@ -324,6 +325,7 @@ class ScriptReviewer:run_parameter_values=run_parameter_values,user_instructions=user_instructions,preloaded_code=existing_code,+ block_criteria=block_criteria.get(block_label),)if result:updated_blocks[block_label] = result@@ -620,6 +622,7 @@ class ScriptReviewer:user_instructions: str | None = None,preloaded_code: str | None = None,historical_run_params: dict[str, dict[str, str]] | None = None,+ block_criteria: dict[str, str | dict[str, str] | None] | None = None,) -> BlockReviewResult | None:"""Review a single block's fallback episodes and generate updated code.@@ -709,6 +712,15 @@ class ScriptReviewer:summary["run_parameters"] = ep_paramshistory_summaries.append(summary)+ # Extract block criteria for the template+ terminate_criterion = None+ complete_criterion = None+ error_code_mapping = None+ if block_criteria:+ terminate_criterion = block_criteria.get("terminate_criterion")+ complete_criterion = block_criteria.get("complete_criterion")+ error_code_mapping = block_criteria.get("error_code_mapping")+# Build the reviewer promptreviewer_prompt = prompt_engine.load_prompt(template=template,@@ -732,6 +744,9 @@ class ScriptReviewer:historical_episodes=history_summaries,run_parameter_values=run_parameter_values,user_instructions=user_instructions,+ terminate_criterion=terminate_criterion,+ complete_criterion=complete_criterion,+ error_code_mapping=error_code_mapping,)LOG.info(@@ -1114,13 +1129,19 @@ class ScriptReviewer:self,organization_id: str,workflow_permanent_id: str,- ) -> tuple[dict[str, str], list[str]]:- """Load navigation goals and parameter keys for a workflow.-- Returns (goals_by_label, parameter_keys).+ ) -> tuple[dict[str, str], list[str], dict[str, dict[str, str | dict[str, str] | None]]]:+ """Load navigation goals, parameter keys, and block criteria for a workflow.++ Returns (goals_by_label, parameter_keys, block_criteria_by_label).+ block_criteria_by_label maps block_label -> {+ "terminate_criterion": str | None,+ "complete_criterion": str | None,+ "error_code_mapping": dict[str, str] | None,+ }"""goals: dict[str, str] = {}parameter_keys: list[str] = []+ block_criteria: dict[str, dict[str, str | dict[str, str] | None]] = {}try:workflow = await app.DATABASE.workflows.get_workflow_by_permanent_id(workflow_permanent_id=workflow_permanent_id,@@ -1134,6 +1155,16 @@ class ScriptReviewer:goal = getattr(block, "navigation_goal", None) or getattr(block, "data_extraction_goal", None)if goal:goals[block.label] = goal+ # Collect termination/completion criteria and error code mappings+ terminate_criterion = getattr(block, "terminate_criterion", None)+ complete_criterion = getattr(block, "complete_criterion", None)+ error_code_mapping = getattr(block, "error_code_mapping", None)+ if terminate_criterion or complete_criterion or error_code_mapping:+ block_criteria[block.label] = {+ "terminate_criterion": terminate_criterion,+ "complete_criterion": complete_criterion,+ "error_code_mapping": error_code_mapping,+ }# Collect parameter keys from workflow definitionfor param in workflow.workflow_definition.parameters:if param.key:@@ -1144,7 +1175,7 @@ class ScriptReviewer:workflow_permanent_id=workflow_permanent_id,exc_info=True,)- return goals, parameter_keys+ return goals, parameter_keys, block_criteriadef _extract_function_signature(self, code: str) -> str:"""Extract the async function signature from existing code."""