Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
script regeneration after ai fallback (#3330)
Shuchang Zheng83b3cfb6af32
diff --git a/skyvern/core/script_generations/generate_script.py b/skyvern/core/script_generations/generate_script.pyindex c46b1ce14..8d5fb4826 100644--- a/skyvern/core/script_generations/generate_script.py+++ b/skyvern/core/script_generations/generate_script.py@@ -1187,7 +1187,7 @@ async def generate_workflow_script(async def create_script_block(- block_code: str,+ block_code: str | bytes,script_revision_id: str,script_id: str,organization_id: str,@@ -1205,6 +1205,7 @@ async def create_script_block(block_name: Optional custom name for the block (defaults to function name)block_description: Optional description for the block"""+ block_code_bytes = block_code if isinstance(block_code, bytes) else block_code.encode("utf-8")try:# Step 3: Create script block in databasescript_block = await app.DATABASE.create_script_block(@@ -1225,7 +1226,7 @@ async def create_script_block(script_id=script_id,script_version=1, # Assuming version 1 for nowfile_path=file_path,- data=block_code.encode("utf-8"),+ data=block_code_bytes,)# Create script file record@@ -1236,8 +1237,8 @@ async def create_script_block(file_path=file_path,file_name=file_name,file_type="file",- content_hash=f"sha256:{hashlib.sha256(block_code.encode('utf-8')).hexdigest()}",- file_size=len(block_code.encode("utf-8")),+ content_hash=f"sha256:{hashlib.sha256(block_code_bytes).hexdigest()}",+ file_size=len(block_code_bytes),mime_type="text/x-python",artifact_id=artifact_id,)diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 340c0ab73..d30ef4422 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -278,7 +278,6 @@ class SkyvernPage:If the prompt generation or parsing fails for any reason we fall back toclicking the originally supplied ``xpath``."""-new_xpath = xpathif intention and data:diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.pyindex 9e17cdcb2..cbc17601a 100644--- a/skyvern/forge/sdk/db/client.py+++ b/skyvern/forge/sdk/db/client.py@@ -3967,6 +3967,7 @@ class AgentDB:select(ScriptBlockModel).filter_by(script_revision_id=script_revision_id).filter_by(organization_id=organization_id)+ .order_by(ScriptBlockModel.created_at.asc()))).all()return [convert_to_script_block(record) for record in records]diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.pyindex 2af0e6489..74c83028e 100644--- a/skyvern/services/script_service.py+++ b/skyvern/services/script_service.py@@ -7,12 +7,15 @@ import osfrom datetime import datetimefrom typing import Any, cast+import libcst as cstimport structlogfrom fastapi import BackgroundTasks, HTTPExceptionfrom jinja2.sandbox import SandboxedEnvironment+from skyvern.config import settingsfrom skyvern.constants import GET_DOWNLOADED_FILES_TIMEOUTfrom skyvern.core.script_generations.constants import SCRIPT_TASK_BLOCKS+from skyvern.core.script_generations.generate_script import _build_block_fn, create_script_blockfrom skyvern.core.script_generations.script_run_context_manager import script_run_context_managerfrom skyvern.exceptions import ScriptNotFound, WorkflowRunNotFoundfrom skyvern.forge import app@@ -22,8 +25,9 @@ from skyvern.forge.sdk.models import StepStatusfrom skyvern.forge.sdk.schemas.files import FileInfofrom skyvern.forge.sdk.schemas.tasks import TaskOutput, TaskStatusfrom skyvern.forge.sdk.workflow.models.block import TaskBlock+from skyvern.forge.sdk.workflow.models.workflow import Workflowfrom skyvern.schemas.runs import RunEngine-from skyvern.schemas.scripts import CreateScriptResponse, FileNode, ScriptFileCreate+from skyvern.schemas.scripts import CreateScriptResponse, FileEncoding, FileNode, ScriptFileCreatefrom skyvern.schemas.workflows import BlockStatus, BlockTypeLOG = structlog.get_logger(__name__)@@ -446,6 +450,7 @@ async def _run_cached_function(cache_key: str) -> Any:async def _fallback_to_ai_run(+ block_type: BlockType,cache_key: str,prompt: str | None = None,url: str | None = None,@@ -475,32 +480,38 @@ async def _fallback_to_ai_run(and context.step_id):return+ organization_id = context.organization_id+ workflow_id = context.workflow_id+ workflow_run_id = context.workflow_run_id+ workflow_permanent_id = context.workflow_permanent_id+ task_id = context.task_id+ script_step_id = context.step_idtry:- organization_id = context.organization_idLOG.info("Script trying to fallback to AI run",cache_key=cache_key,organization_id=organization_id,- workflow_id=context.workflow_id,- workflow_run_id=context.workflow_run_id,- task_id=context.task_id,- step_id=context.step_id,+ workflow_id=workflow_id,+ workflow_run_id=workflow_run_id,+ task_id=task_id,+ step_id=script_step_id,)# 1. fail the previous stepprevious_step = await app.DATABASE.update_step(- step_id=context.step_id,- task_id=context.task_id,+ step_id=script_step_id,+ task_id=task_id,organization_id=organization_id,status=StepStatus.failed,)# 2. create a new step for ai runai_step = await app.DATABASE.create_step(- task_id=context.task_id,+ task_id=task_id,organization_id=organization_id,order=previous_step.order + 1,retry_index=0,)context.step_id = ai_step.step_id+ ai_step_id = ai_step.step_id# 3. build the task block# 4. run execute_steporganization = await app.DATABASE.get_organization(organization_id=organization_id)@@ -510,21 +521,35 @@ async def _fallback_to_ai_run(if not task:raise Exception(f"Task is missing task_id={context.task_id}")workflow = await app.DATABASE.get_workflow(workflow_id=context.workflow_id, organization_id=organization_id)- if not workflow or not workflow.ai_fallback:+ if not workflow:+ return+ if not workflow.ai_fallback:+ LOG.info(+ "AI fallback is not enabled for the workflow",+ workflow_id=workflow_id,+ workflow_permanent_id=workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ )return# get the output_paramteroutput_parameter = workflow.get_output_parameter(cache_key)if not output_parameter:+ LOG.exception(+ "Output parameter not found for the workflow",+ workflow_id=workflow_id,+ workflow_permanent_id=workflow_permanent_id,+ workflow_run_id=workflow_run_id,+ )returnLOG.info("Script starting to fallback to AI run",cache_key=cache_key,organization_id=organization_id,- workflow_id=context.workflow_id,- workflow_run_id=context.workflow_run_id,- task_id=context.task_id,- step_id=context.step_id,+ workflow_id=workflow_id,+ workflow_run_id=workflow_run_id,+ task_id=task_id,+ step_id=script_step_id,)task_block = TaskBlock(@@ -553,6 +578,7 @@ async def _fallback_to_ai_run(step=ai_step,task_block=task_block,)+# Update block status to completed if workflow block was createdif workflow_run_block_id:await _update_workflow_block(@@ -562,6 +588,37 @@ async def _fallback_to_ai_run(step_id=context.step_id,label=cache_key,)++ # 5. After successful AI execution, regenerate the script block and create new version+ try:+ await _regenerate_script_block_after_ai_fallback(+ block_type=block_type,+ cache_key=cache_key,+ task_id=context.task_id,+ script_step_id=ai_step_id,+ ai_step_id=ai_step_id,+ organization_id=organization_id,+ workflow=workflow,+ workflow_run_id=context.workflow_run_id,+ prompt=prompt,+ url=url,+ engine=engine,+ complete_criterion=complete_criterion,+ terminate_criterion=terminate_criterion,+ data_extraction_goal=data_extraction_goal,+ schema=schema,+ error_code_mapping=error_code_mapping,+ max_steps=max_steps,+ complete_on_download=complete_on_download,+ download_suffix=download_suffix,+ totp_verification_url=totp_verification_url,+ totp_identifier=totp_identifier,+ complete_verification=complete_verification,+ include_action_history_in_verification=include_action_history_in_verification,+ )+ except Exception as e:+ LOG.warning("Failed to regenerate script block after AI fallback", error=str(e), exc_info=True)+ # Don't fail the entire fallback process if script regeneration failsexcept Exception as e:LOG.warning("Failed to fallback to AI run", cache_key=cache_key, exc_info=True)# Update block status to failed if workflow block was created@@ -577,6 +634,293 @@ async def _fallback_to_ai_run(raise e+async def _regenerate_script_block_after_ai_fallback(+ block_type: BlockType,+ cache_key: str,+ task_id: str,+ script_step_id: str,+ ai_step_id: str,+ organization_id: str,+ workflow: Workflow,+ workflow_run_id: str,+ prompt: str | None = None,+ url: str | None = None,+ engine: RunEngine = RunEngine.skyvern_v1,+ complete_criterion: str | None = None,+ terminate_criterion: str | None = None,+ data_extraction_goal: str | None = None,+ schema: dict[str, Any] | list | str | None = None,+ error_code_mapping: dict[str, str] | None = None,+ max_steps: int | None = None,+ complete_on_download: bool = False,+ download_suffix: str | None = None,+ totp_verification_url: str | None = None,+ totp_identifier: str | None = None,+ complete_verification: bool = True,+ include_action_history_in_verification: bool = False,+) -> None:+ """+ Regenerate the script block after a successful AI fallback and create a new script version.+ Only the specific block that fell back to AI is regenerated; all other blocks remain unchanged.++ 1. get the latest cashed script for the workflow+ 2. create a completely new script, with only the current block's script being different as it's newly generated.+ -+ """+ try:+ # Get the current script for this workflow and cache key value+ # Render the cache_key_value from workflow run parameters (same logic as generate_script_for_workflow)+ cache_key_value = ""+ if workflow.cache_key:+ try:+ parameter_tuples = await app.DATABASE.get_workflow_run_parameters(workflow_run_id=workflow_run_id)+ parameters = {wf_param.key: run_param.value for wf_param, run_param in parameter_tuples}+ cache_key_value = jinja_sandbox_env.from_string(workflow.cache_key).render(parameters)+ except Exception as e:+ LOG.warning("Failed to render cache key for script regeneration", error=str(e), exc_info=True)+ # Fallback to using cache_key as cache_key_value+ cache_key_value = cache_key++ if not cache_key_value:+ cache_key_value = cache_key # Fallback++ existing_scripts = await app.DATABASE.get_workflow_scripts_by_cache_key_value(+ organization_id=organization_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ cache_key_value=cache_key_value,+ cache_key=workflow.cache_key,+ )++ if not existing_scripts:+ LOG.error("No existing script found to regenerate", cache_key=cache_key, cache_key_value=cache_key_value)+ return++ current_script = existing_scripts[0]+ LOG.info(+ "Regenerating script block after AI fallback",+ script_id=current_script.script_id,+ script_version=current_script.version,+ cache_key=cache_key,+ cache_key_value=cache_key_value,+ )++ # Create a new script version+ new_script = await app.DATABASE.create_script(+ organization_id=organization_id,+ run_id=workflow_run_id,+ script_id=current_script.script_id, # Use same script_id for versioning+ version=current_script.version + 1,+ )++ # deprecate the current workflow script+ await app.DATABASE.delete_workflow_cache_key_value(+ organization_id=organization_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ cache_key_value=cache_key_value,+ )++ # Create workflow script mapping for the new version+ await app.DATABASE.create_workflow_script(+ organization_id=organization_id,+ script_id=new_script.script_id,+ workflow_permanent_id=workflow.workflow_permanent_id,+ cache_key=workflow.cache_key or "",+ cache_key_value=cache_key_value,+ workflow_id=workflow.workflow_id,+ workflow_run_id=workflow_run_id,+ )++ # Get all existing script blocks from the previous version+ existing_script_blocks = await app.DATABASE.get_script_blocks_by_script_revision_id(+ script_revision_id=current_script.script_revision_id,+ organization_id=organization_id,+ )++ # Copy all existing script blocks to the new version (except the one we're regenerating)+ block_file_contents = []+ starter_block_file_content_bytes = b""+ block_file_content: bytes | str = ""+ for existing_block in existing_script_blocks:+ if existing_block.script_block_label == cache_key:+ # Skip this block - we'll regenerate it+ block_file_content = await _generate_block_code_from_task(+ block_type=block_type,+ cache_key=cache_key,+ task_id=task_id,+ script_step_id=script_step_id,+ ai_step_id=ai_step_id,+ organization_id=organization_id,+ workflow=workflow,+ workflow_run_id=workflow_run_id,+ )+ else:+ # Copy the existing block to the new version+ # Get the script file content for this block and copy a new script block for it+ if existing_block.script_file_id:+ script_file = await app.DATABASE.get_script_file_by_id(+ script_revision_id=current_script.script_revision_id,+ file_id=existing_block.script_file_id,+ organization_id=organization_id,+ )++ if script_file and script_file.artifact_id:+ # Retrieve the artifact content+ artifact = await app.DATABASE.get_artifact_by_id(script_file.artifact_id, organization_id)+ if artifact:+ file_content = await app.ARTIFACT_MANAGER.retrieve_artifact(artifact)+ if file_content:+ block_file_content = file_content+ else:+ LOG.warning(+ "Failed to retrieve artifact content for existing block",+ block_label=existing_block.script_block_label,+ )+ else:+ LOG.warning(+ "Artifact not found for existing block", block_label=existing_block.script_block_label+ )+ else:+ LOG.warning(+ "Script file or artifact not found for existing block",+ block_label=existing_block.script_block_label,+ )+ else:+ LOG.warning("No script file ID for existing block", block_label=existing_block.script_block_label)++ if not block_file_content:+ LOG.warning(+ "No block file content found for existing block", block_label=existing_block.script_block_label+ )+ continue++ await create_script_block(+ block_code=block_file_content,+ script_revision_id=new_script.script_revision_id,+ script_id=new_script.script_id,+ organization_id=organization_id,+ block_name=existing_block.script_block_label,+ )+ block_file_content_bytes = (+ block_file_content if isinstance(block_file_content, bytes) else block_file_content.encode("utf-8")+ )+ if existing_block.script_block_label == settings.WORKFLOW_START_BLOCK_LABEL:+ starter_block_file_content_bytes = block_file_content_bytes+ else:+ block_file_contents.append(block_file_content_bytes)++ if starter_block_file_content_bytes:+ block_file_contents.insert(0, starter_block_file_content_bytes)+ else:+ LOG.error("Starter block file content not found")++ # 4) Persist script and files, then record mapping+ python_src = "\n\n".join([block_file_content.decode("utf-8") for block_file_content in block_file_contents])+ content_bytes = python_src.encode("utf-8")+ content_b64 = base64.b64encode(content_bytes).decode("utf-8")+ files = [+ ScriptFileCreate(+ path="main.py",+ content=content_b64,+ encoding=FileEncoding.BASE64,+ mime_type="text/x-python",+ )+ ]++ # Upload script file(s) as artifacts and create rows+ await build_file_tree(+ files=files,+ organization_id=workflow.organization_id,+ script_id=new_script.script_id,+ script_version=new_script.version,+ script_revision_id=new_script.script_revision_id,+ )++ except Exception as e:+ LOG.error("Failed to regenerate script block after AI fallback", error=str(e), exc_info=True)+ raise+++async def _get_block_definition_by_label(+ label: str, workflow: Workflow, task_id: str, organization_id: str+) -> dict[str, Any] | None:+ final_dump = None+ for block in workflow.workflow_definition.blocks:+ if block.label == label:+ final_dump = block.model_dump()+ break+ if not final_dump:+ return None++ task = await app.DATABASE.get_task(task_id=task_id, organization_id=organization_id)+ if task:+ task_dump = task.model_dump()+ final_dump.update({k: v for k, v in task_dump.items() if k not in final_dump})++ # Add run block execution metadata+ final_dump.update(+ {+ "task_id": task_id,+ "output": task.extracted_information,+ }+ )++ return final_dump+++async def _generate_block_code_from_task(+ block_type: BlockType,+ cache_key: str,+ task_id: str,+ script_step_id: str,+ ai_step_id: str,+ organization_id: str,+ workflow: Workflow,+ workflow_run_id: str,+) -> str:+ block_data = await _get_block_definition_by_label(cache_key, workflow, task_id, organization_id)+ if not block_data:+ return ""+ try:+ # Now regenerate only the specific block that fell back to AI+ task_actions = await app.DATABASE.get_task_actions_hydrated(+ task_id=task_id,+ organization_id=organization_id,+ )++ # Filter actions by step_id and exclude the final action that failed before ai fallback+ actions_to_cache = []+ for index, task_action in enumerate(task_actions):+ # if this action is the last action of the script step, right before ai fallback, we should not include it+ if (+ index < len(task_actions) - 1+ and task_action.step_id == script_step_id+ and task_actions[index + 1].step_id == ai_step_id+ ):+ continue+ action_dump = task_action.model_dump()+ action_dump["xpath"] = task_action.get_xpath()+ actions_to_cache.append(action_dump)++ if not actions_to_cache:+ LOG.warning("No actions found in successful step for script block regeneration")+ return ""++ # Generate the new block function+ block_fn_def = _build_block_fn(block_data, actions_to_cache)++ # Convert the FunctionDef to code using a temporary module+ temp_module = cst.Module(body=[block_fn_def])+ block_code = temp_module.code++ return block_code++ except Exception as block_gen_error:+ LOG.error("Failed to generate block function", error=str(block_gen_error), exc_info=True)+ # Even if block generation fails, we've created the new script version+ # which can be useful for debugging+ return ""++async def run_task(prompt: str,url: str | None = None,@@ -610,6 +954,7 @@ async def run_task(except Exception as e:LOG.exception("Failed to run task block. Falling back to AI run.")await _fallback_to_ai_run(+ block_type=BlockType.TASK,cache_key=cache_key,prompt=prompt,url=url,@@ -668,6 +1013,7 @@ async def download(except Exception as e:LOG.exception("Failed to run download block. Falling back to AI run.")await _fallback_to_ai_run(+ block_type=BlockType.FILE_DOWNLOAD,cache_key=cache_key,prompt=prompt,url=url,@@ -726,6 +1072,7 @@ async def action(except Exception as e:LOG.exception("Failed to run action block. Falling back to AI run.")await _fallback_to_ai_run(+ block_type=BlockType.ACTION,cache_key=cache_key,prompt=prompt,url=url,@@ -781,8 +1128,9 @@ async def login()except Exception as e:- LOG.exception("Failed to run login block. Falling back to AI run.")+ LOG.exception("Failed to run login block")await _fallback_to_ai_run(+ block_type=BlockType.LOGIN,cache_key=cache_key,prompt=prompt,url=url,