Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat(SKY-11213): mirror file_download block download count to the warehouse (#6717)
pedrohsdbeefd0c15fc14
diff --git a/alembic/versions/2026_06_19_2044-58b0ced36529_add_downloaded_file_count_to_workflow_run_blocks.py b/alembic/versions/2026_06_19_2044-58b0ced36529_add_downloaded_file_count_to_workflow_run_blocks.pynew file mode 100644index 000000000..4373544af--- /dev/null+++ b/alembic/versions/2026_06_19_2044-58b0ced36529_add_downloaded_file_count_to_workflow_run_blocks.py@@ -0,0 +1,28 @@+"""add downloaded_file_count to workflow_run_blocks++Revision ID: 58b0ced36529+Revises: 1fee32b3d7c6+Create Date: 2026-06-19T20:44:46.548990+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "58b0ced36529"+down_revision: Union[str, None] = "1fee32b3d7c6"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.execute("SET LOCAL lock_timeout = '5s'")+ op.add_column("workflow_run_blocks", sa.Column("downloaded_file_count", sa.Integer(), nullable=True))+++def downgrade() -> None:+ op.drop_column("workflow_run_blocks", "downloaded_file_count")diff --git a/docs/api-reference/openapi.json b/docs/api-reference/openapi.jsonindex b3c1f2403..14f6a00bd 100644--- a/docs/api-reference/openapi.json+++ b/docs/api-reference/openapi.json@@ -24791,6 +24791,17 @@"type": "null"}]+ },+ "downloaded_file_count": {+ "anyOf": [+ {+ "type": "integer"+ },+ {+ "type": "null"+ }+ ],+ "title": "Downloaded File Count"}},"type": "object",diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 9888bcaf5..30877ed50 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -1037,6 +1037,10 @@ class WorkflowRunBlockModel(Base):# column on `WorkflowRunModel` but at block granularity.script_run = Column(JSON, nullable=True)+ # Scalar mirror of output["downloaded_files"] length: the JSON output column is+ # not CDC-mirrored, so download success would not otherwise be queryable.+ downloaded_file_count = Column(Integer, nullable=True)+created_at = Column(DateTime, default=datetime.datetime.utcnow, nullable=False)modified_at = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, nullable=False)diff --git a/skyvern/forge/sdk/db/repositories/observer.py b/skyvern/forge/sdk/db/repositories/observer.pyindex 26c6e9e05..354581213 100644--- a/skyvern/forge/sdk/db/repositories/observer.py+++ b/skyvern/forge/sdk/db/repositories/observer.py@@ -24,6 +24,7 @@ from skyvern.forge.sdk.db.protocols import TaskReaderfrom skyvern.forge.sdk.db.utils import (convert_to_task_v2,convert_to_workflow_run_block,+ downloaded_file_count_from_output,serialize_proxy_location,truncate_oversized_jsonb_value,)@@ -502,6 +503,9 @@ class ObserverRepository(BaseRepository):executed_branch_result: bool | None = None,executed_branch_next_block: str | None = None,) -> WorkflowRunBlock:+ # Read before truncation: truncate_oversized_jsonb_value strips downloaded_files+ # from oversized payloads, which would zero out an otherwise-real count.+ derived_downloaded_file_count = downloaded_file_count_from_output(output)if output is not None:output = truncate_oversized_jsonb_value(output,@@ -589,6 +593,10 @@ class ObserverRepository(BaseRepository):workflow_run_block.executed_branch_result = executed_branch_resultif executed_branch_next_block is not None:workflow_run_block.executed_branch_next_block = executed_branch_next_block+ # Count derives from the output; rewrite it whenever output is written+ # (a non-download output clears a stale count). Falsy output leaves it.+ if output:+ workflow_run_block.downloaded_file_count = derived_downloaded_file_countawait session.commit()await session.refresh(workflow_run_block)else:diff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex b22852967..2dbe56a34 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -786,6 +786,14 @@ def convert_to_workflow_run_parameter()+def downloaded_file_count_from_output(output: dict | list | str | None) -> int | None:+ if isinstance(output, dict):+ downloaded_files = output.get("downloaded_files")+ if isinstance(downloaded_files, list):+ return len(downloaded_files)+ return None++def convert_to_workflow_run_block(workflow_run_block_model: WorkflowRunBlockModel,task: Task | None = None,@@ -825,6 +833,7 @@ def convert_to_workflow_run_block(script_run=ScriptRunResponse.model_validate(workflow_run_block_model.script_run)if workflow_run_block_model.script_runelse None,+ downloaded_file_count=workflow_run_block_model.downloaded_file_count,)if task:if task.finished_at and task.started_at:diff --git a/skyvern/forge/sdk/schemas/workflow_runs.py b/skyvern/forge/sdk/schemas/workflow_runs.pyindex 451216aaa..f306d9d29 100644--- a/skyvern/forge/sdk/schemas/workflow_runs.py+++ b/skyvern/forge/sdk/schemas/workflow_runs.py@@ -70,6 +70,9 @@ class WorkflowRunBlock(BaseModel):# clean cached execution, always-agent blocks, and pre-column rows.script_run: ScriptRunResponse | None = None+ # Count of files this block downloaded; scalar mirror of output["downloaded_files"].+ downloaded_file_count: int | None = None+class WorkflowRunTimelineType(StrEnum):thought = "thought"diff --git a/tests/unit/test_copilot_run_outcome_event.py b/tests/unit/test_copilot_run_outcome_event.pyindex af5c16881..06ee29dc5 100644--- a/tests/unit/test_copilot_run_outcome_event.py+++ b/tests/unit/test_copilot_run_outcome_event.py@@ -7,7 +7,6 @@ result rows; domains and person names are generic placeholders.from __future__ import annotationsimport inspect-import reimport timefrom types import SimpleNamespacefrom typing import Any@@ -204,7 +203,14 @@ def test_challenge_failure_sanitizes_halt_metadata_reason() -> None:assert ctx.turn_halt is not Noneevidence_reason = ctx.turn_halt.extra["evidence_reason"]- assert re.search(r"https://example\.com", evidence_reason) is not None+ # Assert exact equality on the full redacted string: the safe origin is preserved+ # while every credential is stripped. Equality (vs. a substring `in` check) is a+ # complete sanitization assertion. The trailing codeql marker suppresses a known+ # false positive — the value is an exact-match expectation, not URL sanitization.+ assert evidence_reason == (+ "Run output reported a blocker: Human verification challenge blocked "+ "https://example.com after [REDACTED_SECRET] was submitted." # codeql[py/incomplete-url-substring-sanitization]+ )assert "[REDACTED_SECRET]" in evidence_reasonassert "user:secret" not in evidence_reasonassert "password=" not in evidence_reason@@ -427,7 +433,13 @@ def test_display_reason_redacts_secrets_and_url_credentials() -> None:)assert reason is not None- assert re.search(r"https://example\.com", reason) is not None+ # Exact-equality assertion: safe origin kept, credentials stripped. Using `==` rather+ # than a substring `in` check is a complete sanitization assertion. The trailing codeql+ # marker suppresses a known false positive — this is exact-match, not URL sanitization.+ assert reason == (+ "Blocked at "+ "https://example.com after [REDACTED_SECRET] was submitted." # codeql[py/incomplete-url-substring-sanitization]+ )assert "[REDACTED_SECRET]" in reasonassert "user:secret" not in reasonassert "password=" not in reasondiff --git a/tests/unit/test_downloaded_file_count.py b/tests/unit/test_downloaded_file_count.pynew file mode 100644index 000000000..76d5c45c4--- /dev/null+++ b/tests/unit/test_downloaded_file_count.py@@ -0,0 +1,25 @@+from skyvern.forge.sdk.db.utils import downloaded_file_count_from_output+++def test_counts_downloaded_files() -> None:+ output = {"downloaded_files": [{"url": "a"}, {"url": "b"}, {"url": "c"}]}+ assert downloaded_file_count_from_output(output) == 3+++def test_zero_when_no_files_downloaded() -> None:+ assert downloaded_file_count_from_output({"downloaded_files": []}) == 0+++def test_none_when_key_absent() -> None:+ # Non-download blocks (e.g. an extraction block output) carry no downloaded_files key.+ assert downloaded_file_count_from_output({"extracted_information": {"k": "v"}}) is None+++def test_none_when_downloaded_files_is_null() -> None:+ assert downloaded_file_count_from_output({"downloaded_files": None}) is None+++def test_none_for_non_dict_output() -> None:+ assert downloaded_file_count_from_output(None) is None+ assert downloaded_file_count_from_output("a string value") is None+ assert downloaded_file_count_from_output(["a", "b"]) is None