Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
generate description for cached action (#3603)
Shuchang Zheng1cfa23ae3878
diff --git a/skyvern/core/script_generations/skyvern_page.py b/skyvern/core/script_generations/skyvern_page.pyindex 9f3ebd95c..ec62673be 100644--- a/skyvern/core/script_generations/skyvern_page.py+++ b/skyvern/core/script_generations/skyvern_page.py@@ -268,6 +268,55 @@ class SkyvernPage:timeout=timeout,)+ async def _update_action_reasoning(+ self,+ action_id: str,+ organization_id: str,+ action_type: ActionType,+ intention: str = "",+ text: str | None = None,+ select_option: SelectOption | None = None,+ file_url: str | None = None,+ data_extraction_goal: str | None = None,+ data_extraction_schema: dict[str, Any] | list | str | None = None,+ ) -> str:+ """Generate user-facing reasoning for an action using the secondary LLM."""+ reasoning = f"Auto-generated action for {action_type.value}"+ try:+ context = skyvern_context.current()+ if not context or not context.organization_id:+ return f"Auto-generated action for {action_type.value}"++ # Build the prompt with available context+ prompt = prompt_engine.load_prompt(+ template="generate-action-reasoning",+ action_type=action_type.value,+ intention=intention,+ text=text,+ select_option=select_option.value if select_option else None,+ file_url=file_url,+ data_extraction_goal=data_extraction_goal,+ data_extraction_schema=data_extraction_schema,+ )++ # Call secondary LLM to generate reasoning+ json_response = await app.SECONDARY_LLM_API_HANDLER(+ prompt=prompt,+ prompt_name="generate-action-reasoning",+ organization_id=context.organization_id,+ )++ reasoning = json_response.get("reasoning", f"Auto-generated action for {action_type.value}")++ except Exception:+ LOG.warning("Failed to generate action reasoning, using fallback", action_type=action_type)+ await app.DATABASE.update_action_reasoning(+ organization_id=organization_id,+ action_id=action_id,+ reasoning=reasoning,+ )+ return reasoning+async def _create_action_after_execution(self,action_type: ActionType,@@ -312,14 +361,17 @@ class SkyvernPage:step_order=0, # Will be updated by the system if neededaction_order=context.action_order, # Will be updated by the system if neededintention=intention,- reasoning=f"Auto-generated action for {action_type.value}",text=text,option=select_option,file_url=file_url,response=response,created_by="script",)+ data_extraction_goal = None+ data_extraction_schema = Noneif action_type == ActionType.EXTRACT:+ data_extraction_goal = kwargs.get("prompt")+ data_extraction_schema = kwargs.get("schema")action = ExtractAction(element_id="",action_type=action_type,@@ -331,15 +383,29 @@ class SkyvernPage:step_order=0,action_order=context.action_order,intention=intention,- reasoning=f"Auto-generated action for {action_type.value}",- data_extraction_goal=kwargs.get("prompt"),- data_extraction_schema=kwargs.get("schema"),+ data_extraction_goal=data_extraction_goal,+ data_extraction_schema=data_extraction_schema,option=select_option,response=response,created_by="script",)created_action = await app.DATABASE.create_action(action)+ # Generate user-facing reasoning using secondary LLM+ asyncio.create_task(+ self._update_action_reasoning(+ action_id=str(created_action.action_id),+ organization_id=str(context.organization_id),+ action_type=action_type,+ intention=intention,+ text=text,+ select_option=select_option,+ file_url=file_url,+ data_extraction_goal=data_extraction_goal,+ data_extraction_schema=data_extraction_schema,+ )+ )+context.action_order += 1return created_actiondiff --git a/skyvern/forge/prompts/skyvern/generate-action-reasoning.j2 b/skyvern/forge/prompts/skyvern/generate-action-reasoning.j2new file mode 100644index 000000000..d71e45c41--- /dev/null+++ b/skyvern/forge/prompts/skyvern/generate-action-reasoning.j2@@ -0,0 +1,20 @@+Generating a user-facing description for an browser action to help users understand what the action is doing and why.++Action Information:+- Action Type: {{ action_type }}+{% if intention %}- Intention: {{ intention }}{% endif %}+{% if text %}- Text/Value: {{ text }}{% endif %}+{% if select_option %}- Selected Option: {{ select_option }}{% endif %}+{% if file_url %}- File URL: {{ file_url }}{% endif %}+{% if data_extraction_goal %}- Data Extraction Goal: {{ data_extraction_goal }}{% endif %}+{% if data_extraction_schema %}- Data Extraction Schema: {{ data_extraction_schema }}{% endif %}++MAKE SURE YOU OUTPUT VALID JSON. No text before or after JSON, no trailing commas, no comments (//), no unnecessary quotes, etc.++Respond with the following JSON format:+```+{+ "reasoning": str // A clear, user-friendly explanation (one sentence, 20 words max) of what this action is doing. Use present tense like "Clicking the submit button" or "Entering the email address". Focus on the action and its purpose.+}+```+diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.pyindex 1e78416c3..f49f5b0dd 100644--- a/skyvern/forge/sdk/db/client.py+++ b/skyvern/forge/sdk/db/client.py@@ -2673,6 +2673,25 @@ class AgentDB:await session.refresh(new_action)return Action.model_validate(new_action)+ async def update_action_reasoning(+ self,+ organization_id: str,+ action_id: str,+ reasoning: str,+ ) -> Action:+ async with self.Session() as session:+ action = (+ await session.scalars(+ select(ActionModel).filter_by(action_id=action_id).filter_by(organization_id=organization_id)+ )+ ).first()+ if action:+ action.reasoning = reasoning+ await session.commit()+ await session.refresh(action)+ return Action.model_validate(action)+ raise NotFoundError(f"Action {action_id}")+async def retrieve_action_plan(self, task: Task) -> list[Action]:async with self.Session() as session:subquery = (diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 4c17787fc..e47f89a87 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -525,7 +525,7 @@ class WorkflowCreateYAMLRequest(BaseModel):status: WorkflowStatus = WorkflowStatus.publishedrun_with: str | None = Noneai_fallback: bool = False- cache_key: str | None = "default"+ cache_key: str | None = Nonerun_sequentially: bool = Falsesequential_key: str | None = None