Skip to content

Automate browser based workflows with AI

add workflow_run.script_run and mark ai_fallback_triggered when the script falls back to ai run (#3433)

Shuchang Zheng720ebac72c18
diff --git a/alembic/versions/2025_09_15_0540-8998d998feed_db_migration_script.py b/alembic/versions/2025_09_15_0540-8998d998feed_db_migration_script.py
new file mode 100644
index 000000000..df92c601f
--- /dev/null
+++ b/alembic/versions/2025_09_15_0540-8998d998feed_db_migration_script.py
@@ -0,0 +1,31 @@
+"""db migration script
+
+Revision ID: 8998d998feed
+Revises: f78486c3f895
+Create Date: 2025-09-15 05:40:16.200764+00:00
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "8998d998feed"
+down_revision: Union[str, None] = "f78486c3f895"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.add_column("workflow_runs", sa.Column("script_run", sa.JSON(), nullable=True))
+ # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+ # ### commands auto generated by Alembic - please adjust! ###
+ op.drop_column("workflow_runs", "script_run")
+ # ### end Alembic commands ###
diff --git a/fern/running-tasks/webhooks-faq.mdx b/fern/running-tasks/webhooks-faq.mdx
index a90aecf25..f1858e7cf 100644
--- a/fern/running-tasks/webhooks-faq.mdx
+++ b/fern/running-tasks/webhooks-faq.mdx
@@ -30,6 +30,14 @@ The webhook request body is a JSON object with the following fields:
"app_url": "The URL to the run in the Skyvern app",
"created_at": "The timestamp when the run was created",
"modified_at": "The timestamp when the run was last modified",
+ "queued_at": "The timestamp when the run was queued",
+ "started_at": "The timestamp when the run started",
+ "finished_at": "The timestamp when the run finished",
+ "app_url": "The URL to the run in the Skyvern app",
+ "browser_session_id": "The ID of the browser session",
+ "max_screenshot_scrolls": "The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot",
+ "run_request": "The original request parameters used to start this task or workflow run",
+ "script_run": "The script run result containing information like whether AI fallback is triggered when the script fails",
}
```
For detailed schema, please refer to the [Run Response](/api-reference/api-reference/agent/get-run#response).
diff --git a/skyvern/forge/sdk/db/client.py b/skyvern/forge/sdk/db/client.py
index 67ac3054a..ba7cc7cef 100644
--- a/skyvern/forge/sdk/db/client.py
+++ b/skyvern/forge/sdk/db/client.py
@@ -1672,6 +1672,7 @@ class AgentDB:
status: WorkflowRunStatus | None = None,
failure_reason: str | None = None,
webhook_failure_reason: str | None = None,
+ ai_fallback_triggered: bool | None = None,
) -> WorkflowRun:
async with self.Session() as session:
workflow_run = (
@@ -1690,6 +1691,8 @@ class AgentDB:
workflow_run.failure_reason = failure_reason
if webhook_failure_reason is not None:
workflow_run.webhook_failure_reason = webhook_failure_reason
+ if ai_fallback_triggered is not None:
+ workflow_run.script_run = {"ai_fallback_triggered": ai_fallback_triggered}
await session.commit()
await session.refresh(workflow_run)
await save_workflow_run_logs(workflow_run_id)
@@ -2936,6 +2939,7 @@ class AgentDB:
http_request_parameters: dict[str, Any] | None = None,
http_request_timeout: int | None = None,
http_request_follow_redirects: bool | None = None,
+ ai_fallback_triggered: bool | None = None,
) -> WorkflowRunBlock:
async with self.Session() as session:
workflow_run_block = (
@@ -2993,6 +2997,8 @@ class AgentDB:
workflow_run_block.http_request_timeout = http_request_timeout
if http_request_follow_redirects is not None:
workflow_run_block.http_request_follow_redirects = http_request_follow_redirects
+ if ai_fallback_triggered is not None:
+ workflow_run_block.script_run = {"ai_fallback_triggered": ai_fallback_triggered}
await session.commit()
await session.refresh(workflow_run_block)
else:
diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.py
index 6409935ef..18dff7727 100644
--- a/skyvern/forge/sdk/db/models.py
+++ b/skyvern/forge/sdk/db/models.py
@@ -283,6 +283,7 @@ class WorkflowRunModel(Base):
max_screenshot_scrolling_times = Column(Integer, nullable=True)
extra_http_headers = Column(JSON, nullable=True)
browser_address = Column(String, nullable=True)
+ script_run = Column(JSON, nullable=True)
queued_at = Column(DateTime, nullable=True)
started_at = Column(DateTime, nullable=True)
diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.py
index 7bbe45017..829bf1862 100644
--- a/skyvern/forge/sdk/db/utils.py
+++ b/skyvern/forge/sdk/db/utils.py
@@ -50,7 +50,7 @@ from skyvern.forge.sdk.workflow.models.workflow import (
WorkflowRunStatus,
WorkflowStatus,
)
-from skyvern.schemas.runs import ProxyLocation
+from skyvern.schemas.runs import ProxyLocation, ScriptRunResponse
from skyvern.schemas.scripts import Script, ScriptBlock, ScriptFile
from skyvern.schemas.workflows import BlockStatus, BlockType
from skyvern.webeye.actions.actions import (
@@ -303,6 +303,7 @@ def convert_to_workflow_run(
max_screenshot_scrolls=workflow_run_model.max_screenshot_scrolling_times,
extra_http_headers=workflow_run_model.extra_http_headers,
browser_address=workflow_run_model.browser_address,
+ script_run=ScriptRunResponse.model_validate(workflow_run_model.script_run),
)
diff --git a/skyvern/forge/sdk/workflow/models/workflow.py b/skyvern/forge/sdk/workflow/models/workflow.py
index 762b3ecbd..750bd5b99 100644
--- a/skyvern/forge/sdk/workflow/models/workflow.py
+++ b/skyvern/forge/sdk/workflow/models/workflow.py
@@ -10,7 +10,7 @@ from skyvern.forge.sdk.schemas.task_v2 import TaskV2
from skyvern.forge.sdk.workflow.exceptions import WorkflowDefinitionHasDuplicateBlockLabels
from skyvern.forge.sdk.workflow.models.block import BlockTypeVar
from skyvern.forge.sdk.workflow.models.parameter import PARAMETER_TYPE, OutputParameter
-from skyvern.schemas.runs import ProxyLocation
+from skyvern.schemas.runs import ProxyLocation, ScriptRunResponse
from skyvern.schemas.workflows import WorkflowStatus
from skyvern.utils.url_validators import validate_url
@@ -135,6 +135,7 @@ class WorkflowRun(BaseModel):
workflow_title: str | None = None
max_screenshot_scrolls: int | None = None
browser_address: str | None = None
+ script_run: ScriptRunResponse | None = None
queued_at: datetime | None = None
started_at: datetime | None = None
@@ -186,3 +187,4 @@ class WorkflowRunResponseBase(BaseModel):
browser_session_id: str | None = None
max_screenshot_scrolls: int | None = None
browser_address: str | None = None
+ script_run: ScriptRunResponse | None = None
diff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.py
index 9da55b7a1..c79de2f9b 100644
--- a/skyvern/forge/sdk/workflow/service.py
+++ b/skyvern/forge/sdk/workflow/service.py
@@ -1445,6 +1445,7 @@ class WorkflowService:
max_screenshot_scrolls=workflow_run.max_screenshot_scrolls,
task_v2=task_v2,
browser_address=workflow_run.browser_address,
+ script_run=workflow_run.script_run,
)
async def clean_up_workflow(
@@ -1554,6 +1555,7 @@ class WorkflowService:
screenshot_urls=workflow_run_status_response.screenshot_urls,
failure_reason=workflow_run_status_response.failure_reason,
app_url=app_url,
+ script_run=workflow_run_status_response.script_run,
created_at=workflow_run_status_response.created_at,
modified_at=workflow_run_status_response.modified_at,
run_request=WorkflowRunRequest(
diff --git a/skyvern/schemas/runs.py b/skyvern/schemas/runs.py
index 28ec4f64d..ddba249a7 100644
--- a/skyvern/schemas/runs.py
+++ b/skyvern/schemas/runs.py
@@ -380,6 +380,10 @@ class BlockRunRequest(WorkflowRunRequest):
)
+class ScriptRunResponse(BaseModel):
+ ai_fallback_triggered: bool = False
+
+
class BaseRunResponse(BaseModel):
run_id: str = Field(
description="Unique identifier for this run. Run ID starts with `tsk_` for task runs and `wr_` for workflow runs.",
@@ -419,6 +423,10 @@ class BaseRunResponse(BaseModel):
default=None,
description="The maximum number of scrolls for the post action screenshot. When it's None or 0, it takes the current viewpoint screenshot",
)
+ script_run: ScriptRunResponse | None = Field(
+ default=None,
+ description="The script run result",
+ )
class TaskRunResponse(BaseRunResponse):
diff --git a/skyvern/services/script_service.py b/skyvern/services/script_service.py
index 67833f330..c98b5e13b 100644
--- a/skyvern/services/script_service.py
+++ b/skyvern/services/script_service.py
@@ -428,6 +428,7 @@ async def _update_workflow_block(
label: str | None = None,
failure_reason: str | None = None,
output: dict[str, Any] | list | str | None = None,
+ ai_fallback_triggered: bool = False,
) -> None:
"""Update the status of a workflow run block."""
try:
@@ -632,6 +633,13 @@ async def _fallback_to_ai_run(
task_block=task_block,
)
+ # update workflow run to indicate that there's a script run
+ if workflow_run_id:
+ await app.DATABASE.update_workflow_run(
+ workflow_run_id=workflow_run_id,
+ ai_fallback_triggered=True,
+ )
+
# Update block status to completed if workflow block was created
if workflow_run_block_id:
await _update_workflow_block(
@@ -1327,6 +1335,11 @@ async def run_script(
)
if not workflow_run:
raise WorkflowRunNotFound(workflow_run_id=workflow_run_id)
+ # update workfow run to indicate that there's a script run
+ workflow_run = await app.DATABASE.update_workflow_run(
+ workflow_run_id=workflow_run_id,
+ ai_fallback_triggered=False,
+ )
context.workflow_run_id = workflow_run_id
context.organization_id = organization_id