Skip to content

Automate browser based workflows with AI

workflow run block description generation (#1482)

Shuchang Zheng187c18d83c7a
diff --git a/alembic/versions/2025_01_03_2349-32e2f138f7fd_add_description_to_workflow_run_block.py b/alembic/versions/2025_01_03_2349-32e2f138f7fd_add_description_to_workflow_run_block.py
new file mode 100644
index 000000000..31c98ad20
--- /dev/null
+++ b/alembic/versions/2025_01_03_2349-32e2f138f7fd_add_description_to_workflow_run_block.py
@@ -0,0 +1,31 @@
+"""add description to workflow run block
+
+Revision ID: 32e2f138f7fd
+Revises: 521241e64aed
+Create Date: 2025-01-03 23:49:40.290858+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "32e2f138f7fd"
+down_revision: Union[str, None] = "521241e64aed"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("workflow_run_blocks", sa.Column("description", sa.String(), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("workflow_run_blocks", "description")
+ # ### end Alembic commands ###
diff --git a/skyvern/forge/prompts/skyvern/generate_workflow_run_block_description.j2 b/skyvern/forge/prompts/skyvern/generate_workflow_run_block_description.j2
new file mode 100644
index 000000000..3a29393cd
--- /dev/null
+++ b/skyvern/forge/prompts/skyvern/generate_workflow_run_block_description.j2
@@ -0,0 +1,10 @@
+We are building workflow automations. There are many blocks in a workflow. Generate a summary for the following block to help the user understand what the block is planning to do.
+
+{{ block }}
+
+Respond with the following JSON format:
+```
+{
+ "summary": str // Be concise and to the point. Summarize the block in one sentence with no more than 20 words. Use languages like "Will do something", "Planning to do something", "Going to do something" and replace the "do something" with the actual action the block is planning to do.
+}
+```
\ No newline at end of file
diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.py
index 766247116..22986948b 100644
--- a/skyvern/forge/sdk/db/client.py
+++ b/skyvern/forge/sdk/db/client.py
@@ -2146,11 +2146,11 @@ class AgentDB:
async def update_workflow_run_block(
self,
workflow_run_block_id: str,
+ organization_id: str | None = None,
status: BlockStatus | None = None,
output: dict | list | str | None = None,
failure_reason: str | None = None,
task_id: str | None = None,
- organization_id: str | None = None,
loop_values: list | None = None,
current_value: str | None = None,
current_index: int | None = None,
@@ -2160,6 +2160,7 @@ class AgentDB:
body: str | None = None,
prompt: str | None = None,
wait_sec: int | None = None,
+ description: str | None = None,
) -> WorkflowRunBlock:
async with self.Session() as session:
workflow_run_block = (
@@ -2196,6 +2197,8 @@ class AgentDB:
workflow_run_block.prompt = prompt
if wait_sec:
workflow_run_block.wait_sec = wait_sec
+ if description:
+ workflow_run_block.description = description
await session.commit()
await session.refresh(workflow_run_block)
else:
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 04bf968e4..8fd15f4ed 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -494,6 +494,7 @@ class WorkflowRunBlockModel(Base):
String, ForeignKey("workflow_run_blocks.workflow_run_block_id"), nullable=True
)
organization_id = Column(String, ForeignKey("organizations.organization_id"), nullable=True)
+ description = Column(String, nullable=True)
task_id = Column(String, ForeignKey("tasks.task_id"), nullable=True)
label = Column(String, nullable=True)
block_type = Column(String, nullable=False)
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 9e7b30c21..6e7a6d024 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -382,6 +382,7 @@ def convert_to_workflow_run_block(
workflow_run_id=workflow_run_block_model.workflow_run_id,
organization_id=workflow_run_block_model.organization_id,
parent_workflow_run_block_id=workflow_run_block_model.parent_workflow_run_block_id,
+ description=workflow_run_block_model.description,
block_type=BlockType(workflow_run_block_model.block_type),
label=workflow_run_block_model.label,
status=BlockStatus(workflow_run_block_model.status),
diff --git a/skyvern/forge/sdk/schemas/workflow_runs.py b/skyvern/forge/sdk/schemas/workflow_runs.py
index 60116688d..bc4c85401 100644
--- a/skyvern/forge/sdk/schemas/workflow_runs.py
+++ b/skyvern/forge/sdk/schemas/workflow_runs.py
@@ -15,6 +15,7 @@ class WorkflowRunBlock(BaseModel):
workflow_run_block_id: str
workflow_run_id: str
organization_id: str | None = None
+ description: str | None = None
parent_workflow_run_block_id: str | None = None
block_type: BlockType
label: str | None = None
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index c3f96d755..7ff3fadc2 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -208,6 +208,49 @@ class Block(BaseModel, abc.ABC):
block_type=self.block_type,
continue_on_failure=self.continue_on_failure,
)
+ workflow_run_block_id = workflow_run_block.workflow_run_block_id
+
+ description = None
+ try:
+ block_data = self.model_dump(
+ exclude={
+ "workflow_run_block_id",
+ "organization_id",
+ "task_id",
+ "workflow_run_id",
+ "parent_workflow_run_block_id",
+ "label",
+ "status",
+ "output",
+ "continue_on_failure",
+ "failure_reason",
+ "actions",
+ "created_at",
+ "modified_at",
+ },
+ exclude_none=True,
+ )
+ description_generation_prompt = prompt_engine.load_prompt(
+ "generate_workflow_run_block_description",
+ block=block_data,
+ )
+ json_response = await app.SECONDARY_LLM_API_HANDLER(prompt=description_generation_prompt)
+ description = json_response.get("summary")
+ LOG.info(
+ "Generated description for the workflow run block",
+ description=description,
+ workflow_run_block_id=workflow_run_block.workflow_run_block_id,
+ )
+ except Exception as e:
+ LOG.exception("Failed to generate description for the workflow run block", error=e)
+
+ if description:
+ workflow_run_block = await app.DATABASE.update_workflow_run_block(
+ workflow_run_block_id=workflow_run_block.workflow_run_block_id,
+ description=description,
+ organization_id=organization_id,
+ )
+
# create a screenshot
browser_state = app.BROWSER_MANAGER.get_for_workflow_run(workflow_run_id)
if not browser_state:
@@ -220,7 +263,6 @@ class Block(BaseModel, abc.ABC):
artifact_type=ArtifactType.SCREENSHOT_LLM,
data=screenshot,
)
- workflow_run_block_id = workflow_run_block.workflow_run_block_id
return await self.execute(workflow_run_id, workflow_run_block_id, organization_id=organization_id, **kwargs)
except Exception as e:
LOG.exception(