Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
workflow run block description generation (#1482)
Shuchang Zheng187c18d83c7a
diff --git a/alembic/versions/2025_01_03_2349-32e2f138f7fd_add_description_to_workflow_run_block.py b/alembic/versions/2025_01_03_2349-32e2f138f7fd_add_description_to_workflow_run_block.pynew file mode 100644index 000000000..31c98ad20--- /dev/null+++ b/alembic/versions/2025_01_03_2349-32e2f138f7fd_add_description_to_workflow_run_block.py@@ -0,0 +1,31 @@+"""add description to workflow run block++Revision ID: 32e2f138f7fd+Revises: 521241e64aed+Create Date: 2025-01-03 23:49:40.290858+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "32e2f138f7fd"+down_revision: Union[str, None] = "521241e64aed"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.add_column("workflow_run_blocks", sa.Column("description", sa.String(), nullable=True))+ # ### end Alembic commands ###+++def downgrade() -> None:+ # ### commands auto generated by Alembic - please adjust! ###+ op.drop_column("workflow_run_blocks", "description")+ # ### end Alembic commands ###diff --git a/skyvern/forge/prompts/skyvern/generate_workflow_run_block_description.j2 b/skyvern/forge/prompts/skyvern/generate_workflow_run_block_description.j2new file mode 100644index 000000000..3a29393cd--- /dev/null+++ b/skyvern/forge/prompts/skyvern/generate_workflow_run_block_description.j2@@ -0,0 +1,10 @@+We are building workflow automations. There are many blocks in a workflow. Generate a summary for the following block to help the user understand what the block is planning to do.++{{ block }}++Respond with the following JSON format:+```+{+ "summary": str // Be concise and to the point. Summarize the block in one sentence with no more than 20 words. Use languages like "Will do something", "Planning to do something", "Going to do something" and replace the "do something" with the actual action the block is planning to do.+}+```\ No newline at end of filediff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.pyindex 766247116..22986948b 100644--- a/skyvern/forge/sdk/db/client.py+++ b/skyvern/forge/sdk/db/client.py@@ -2146,11 +2146,11 @@ class AgentDB:async def update_workflow_run_block(self,workflow_run_block_id: str,+ organization_id: str | None = None,status: BlockStatus | None = None,output: dict | list | str | None = None,failure_reason: str | None = None,task_id: str | None = None,- organization_id: str | None = None,loop_values: list | None = None,current_value: str | None = None,current_index: int | None = None,@@ -2160,6 +2160,7 @@ class AgentDB:body: str | None = None,prompt: str | None = None,wait_sec: int | None = None,+ description: str | None = None,) -> WorkflowRunBlock:async with self.Session() as session:workflow_run_block = (@@ -2196,6 +2197,8 @@ class AgentDB:workflow_run_block.prompt = promptif wait_sec:workflow_run_block.wait_sec = wait_sec+ if description:+ workflow_run_block.description = descriptionawait session.commit()await session.refresh(workflow_run_block)else:diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 04bf968e4..8fd15f4ed 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -494,6 +494,7 @@ class WorkflowRunBlockModel(Base):String, ForeignKey("workflow_run_blocks.workflow_run_block_id"), nullable=True)organization_id = Column(String, ForeignKey("organizations.organization_id"), nullable=True)+ description = Column(String, nullable=True)task_id = Column(String, ForeignKey("tasks.task_id"), nullable=True)label = Column(String, nullable=True)block_type = Column(String, nullable=False)diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 9e7b30c21..6e7a6d024 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -382,6 +382,7 @@ def convert_to_workflow_run_block(workflow_run_id=workflow_run_block_model.workflow_run_id,organization_id=workflow_run_block_model.organization_id,parent_workflow_run_block_id=workflow_run_block_model.parent_workflow_run_block_id,+ description=workflow_run_block_model.description,block_type=BlockType(workflow_run_block_model.block_type),label=workflow_run_block_model.label,status=BlockStatus(workflow_run_block_model.status),diff --git a/skyvern/forge/sdk/schemas/workflow_runs.py b/skyvern/forge/sdk/schemas/workflow_runs.pyindex 60116688d..bc4c85401 100644--- a/skyvern/forge/sdk/schemas/workflow_runs.py+++ b/skyvern/forge/sdk/schemas/workflow_runs.py@@ -15,6 +15,7 @@ class WorkflowRunBlock(BaseModel):workflow_run_block_id: strworkflow_run_id: strorganization_id: str | None = None+ description: str | None = Noneparent_workflow_run_block_id: str | None = Noneblock_type: BlockTypelabel: str | None = Nonediff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex c3f96d755..7ff3fadc2 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -208,6 +208,49 @@ class Block(BaseModel, abc.ABC):block_type=self.block_type,continue_on_failure=self.continue_on_failure,)+ workflow_run_block_id = workflow_run_block.workflow_run_block_id++ description = None+ try:+ block_data = self.model_dump(+ exclude={+ "workflow_run_block_id",+ "organization_id",+ "task_id",+ "workflow_run_id",+ "parent_workflow_run_block_id",+ "label",+ "status",+ "output",+ "continue_on_failure",+ "failure_reason",+ "actions",+ "created_at",+ "modified_at",+ },+ exclude_none=True,+ )+ description_generation_prompt = prompt_engine.load_prompt(+ "generate_workflow_run_block_description",+ block=block_data,+ )+ json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=description_generation_prompt)+ description = json_response.get("summary")+ LOG.info(+ "Generated description for the workflow run block",+ description=description,+ workflow_run_block_id=workflow_run_block.workflow_run_block_id,+ )+ except Exception as e:+ LOG.exception("Failed to generate description for the workflow run block", error=e)++ if description:+ workflow_run_block = await app.DATABASE.update_workflow_run_block(+ workflow_run_block_id=workflow_run_block.workflow_run_block_id,+ description=description,+ organization_id=organization_id,+ )+# create a screenshotbrowser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)if not browser_state:@@ -220,7 +263,6 @@ class Block(BaseModel, abc.ABC):artifact_type=ArtifactType.SCREENSHOT_LLM,data=screenshot,)- workflow_run_block_id = workflow_run_block.workflow_run_block_idreturn await self.execute(workflow_run_id, workflow_run_block_id, organization_id=organization_id, **kwargs)except Exception as e:LOG.exception(