Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Re-apply data_schema support for natural language loop blocks (SKY-7055) — backend (#4851)
Celal Zamanoğlu2a9b44f1347b
diff --git a/fern/openapi/skyvern_openapi.json b/fern/openapi/skyvern_openapi.jsonindex 1649869bb..aaa8af20d 100644--- a/fern/openapi/skyvern_openapi.json+++ b/fern/openapi/skyvern_openapi.json@@ -9516,6 +9516,21 @@"type": "boolean","title": "Complete If Empty","default": false+ },+ "data_schema": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data Schema"}},"type": "object",@@ -9670,6 +9685,21 @@"type": "boolean","title": "Complete If Empty","default": false+ },+ "data_schema": {+ "anyOf": [+ {+ "additionalProperties": true,+ "type": "object"+ },+ {+ "type": "string"+ },+ {+ "type": "null"+ }+ ],+ "title": "Data Schema"}},"type": "object",diff --git a/skyvern/forge/prompts/skyvern/extraction_prompt_for_nat_language_loops.j2 b/skyvern/forge/prompts/skyvern/extraction_prompt_for_nat_language_loops.j2index 4c4f41cae..5a4c66cfd 100644--- a/skyvern/forge/prompts/skyvern/extraction_prompt_for_nat_language_loops.j2+++ b/skyvern/forge/prompts/skyvern/extraction_prompt_for_nat_language_loops.j2@@ -3,10 +3,10 @@ Analyze the current webpage and extract information based on this request: {{ naYou need to identify what values should be iterated over (loop_values). Each value should be the primary data that will be used in each loop iteration.For example:-- If the request is "go to each product page", extract product URLs as strings-- If the request is "extract the authors of the top 4 posts", extract author names as strings-- If the request is "summarize the text of each article", extract article URLs as strings-- If the request is "download each file", extract file URLs as strings-- If the request is "check if these articles are AI-related", extract article titles as strings+- If the request is "go to each product page", extract product URLs+- If the request is "extract the authors of the top 4 posts", extract author names+- If the request is "summarize the text of each article", extract article URLs+- If the request is "download each file", extract file URLs+- If the request is "collect product details" and a structured schema is provided, extract structured objects matching the schema (e.g., objects with name, price, url fields)-Return the results in the specified schema format with loop_values containing an array of strings, where each string is the primary value to be used in the loop iteration.\ No newline at end of file+Return the results in the specified schema format with loop_values containing an array of values, where each value matches the schema provided.diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex 3958d1f7b..c8949dec4 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -1213,6 +1213,9 @@ class ForLoopBlock(Block):loop_over: PARAMETER_TYPE | None = Noneloop_variable_reference: str | None = Nonecomplete_if_empty: bool = False+ # Note: intentionally excludes `list` (unlike BaseTaskBlock.data_schema) because a list schema+ # does not describe the shape of individual loop items -- only dict schemas are meaningful here.+ data_schema: dict[str, Any] | str | None = Nonedef get_all_parameters(self,@@ -1280,7 +1283,9 @@ class ForLoopBlock(Block):if parameter_value is None and not is_likely_parameter_path:try:# Create and execute extraction block using the current block's workflow_id- extraction_block = self._create_initial_extraction_block(self.loop_variable_reference)+ extraction_block = self._create_initial_extraction_block(+ self.loop_variable_reference, workflow_run_context=workflow_run_context+ )LOG.info("Processing natural language loop input",@@ -1470,23 +1475,70 @@ class ForLoopBlock(Block):except Exception:return None- def _create_initial_extraction_block(self, natural_language_prompt: str) -> ExtractionBlock:+ def _create_initial_extraction_block(+ self,+ natural_language_prompt: str,+ workflow_run_context: WorkflowRunContext | None = None,+ ) -> ExtractionBlock:"""Create an extraction block to process natural language input."""- # Create a schema that only extracts loop values- data_schema = {- "type": "object",- "properties": {- "loop_values": {- "type": "array",- "description": "Array of values to iterate over. Each value should be the primary data needed for the loop blocks.",- "items": {- "type": "string",- "description": "The primary value to be used in the loop iteration (e.g., URL, text, identifier, etc.)",- },- }- },- }+ # Determine the items schema for loop_values+ items_schema: dict[str, Any] | None = None+ if self.data_schema is not None:+ if isinstance(self.data_schema, dict):+ items_schema = self.data_schema+ elif isinstance(self.data_schema, str):+ # Interpolate Jinja templates before parsing, matching how BaseTaskBlock.setup_block_v2+ # handles data_schema strings (see line 652-654)+ schema_str = self.data_schema+ if workflow_run_context is not None:+ schema_str = self.format_block_parameter_template_from_workflow_run_context(+ schema_str, workflow_run_context+ )+ try:+ parsed = json.loads(schema_str)+ if isinstance(parsed, dict):+ items_schema = parsed+ else:+ LOG.warning(+ "Parsed data_schema is not a dict, falling back to default string schema",+ block_label=self.label,+ data_schema=self.data_schema,+ )+ except (json.JSONDecodeError, TypeError):+ LOG.warning(+ "Failed to parse data_schema string, falling back to default string schema",+ block_label=self.label,+ data_schema=self.data_schema,+ )++ if items_schema is not None:+ # User provided a custom schema — each loop iteration will produce a structured object+ data_schema: dict[str, Any] = {+ "type": "object",+ "properties": {+ "loop_values": {+ "type": "array",+ "description": "Array of structured values to iterate over, matching the provided schema.",+ "items": items_schema,+ }+ },+ }+ else:+ # Default: extract simple string array+ data_schema = {+ "type": "object",+ "properties": {+ "loop_values": {+ "type": "array",+ "description": "Array of values to iterate over. Each value should be the primary data needed for the loop blocks.",+ "items": {+ "type": "string",+ "description": "The primary value to be used in the loop iteration (e.g., URL, text, identifier, etc.)",+ },+ }+ },+ }# Create extraction goal that includes the natural language promptextraction_goal = prompt_engine.load_prompt(diff --git a/skyvern/forge/sdk/workflow/workflow_definition_converter.py b/skyvern/forge/sdk/workflow/workflow_definition_converter.pyindex 30602c2fb..d4efce42a 100644--- a/skyvern/forge/sdk/workflow/workflow_definition_converter.py+++ b/skyvern/forge/sdk/workflow/workflow_definition_converter.py@@ -418,6 +418,7 @@ def block_yaml_to_block(loop_variable_reference=block_yaml.loop_variable_reference,loop_blocks=loop_blocks,complete_if_empty=block_yaml.complete_if_empty,+ data_schema=block_yaml.data_schema,)elif block_yaml.block_type == BlockType.CONDITIONAL:branch_conditions = []diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex b8bf06624..3bcafcb53 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -614,6 +614,7 @@ class ForLoopBlockYAML(BlockYAML):loop_over_parameter_key: str = ""loop_variable_reference: str | None = Nonecomplete_if_empty: bool = False+ data_schema: dict[str, Any] | str | None = Noneclass BranchCriteriaYAML(BaseModel):