Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
feat: report FILE_PARSER_ERROR code on file parser block failures (SKY-7939) (#5145)
Suchintanb76de94e5f07
diff --git a/alembic/versions/2026_03_19_0735-786acdf95243_add_error_codes_to_workflow_run_blocks.py b/alembic/versions/2026_03_19_0735-786acdf95243_add_error_codes_to_workflow_run_blocks.pynew file mode 100644index 000000000..c2c255525--- /dev/null+++ b/alembic/versions/2026_03_19_0735-786acdf95243_add_error_codes_to_workflow_run_blocks.py@@ -0,0 +1,27 @@+"""add error_codes to workflow_run_blocks++Revision ID: 786acdf95243+Revises: e8c78980ed9d+Create Date: 2026-03-19 07:35:01.666339+00:00++"""++from typing import Sequence, Union++import sqlalchemy as sa++from alembic import op++# revision identifiers, used by Alembic.+revision: str = "786acdf95243"+down_revision: Union[str, None] = "e8c78980ed9d"+branch_labels: Union[str, Sequence[str], None] = None+depends_on: Union[str, Sequence[str], None] = None+++def upgrade() -> None:+ op.add_column("workflow_run_blocks", sa.Column("error_codes", sa.JSON(), nullable=True))+++def downgrade() -> None:+ op.drop_column("workflow_run_blocks", "error_codes")diff --git a/skyvern/forge/sdk/db/agent_db.py b/skyvern/forge/sdk/db/agent_db.pyindex a384fd64a..bf3db909d 100644--- a/skyvern/forge/sdk/db/agent_db.py+++ b/skyvern/forge/sdk/db/agent_db.py@@ -3879,13 +3879,19 @@ class AgentDB(BaseAlchemyDB):def _apply_error_code_filter(query, error_code: str | None): # type: ignore[no-untyped-def]if not error_code:return query- error_code_exists = exists(+ error_code_in_tasks = exists(select(1).select_from(TaskModel).where(TaskModel.workflow_run_id == WorkflowRunModel.workflow_run_id).where(cast(TaskModel.errors, JSONB).contains(literal([{"error_code": error_code}], type_=JSONB))))- return query.where(error_code_exists)+ error_code_in_blocks = exists(+ select(1)+ .select_from(WorkflowRunBlockModel)+ .where(WorkflowRunBlockModel.workflow_run_id == WorkflowRunModel.workflow_run_id)+ .where(cast(WorkflowRunBlockModel.error_codes, JSONB).contains(literal([error_code], type_=JSONB)))+ )+ return query.where(or_(error_code_in_tasks, error_code_in_blocks))async def get_workflow_runs(self,@@ -5256,6 +5262,8 @@ class AgentDB(BaseAlchemyDB):http_request_timeout: int | None = None,http_request_follow_redirects: bool | None = None,ai_fallback_triggered: bool | None = None,+ # block-level error codes (e.g. ["FILE_PARSER_ERROR"])+ error_codes: list[str] | None = None,# human interaction blockinstructions: str | None = None,positive_descriptor: str | None = None,@@ -5283,6 +5291,8 @@ class AgentDB(BaseAlchemyDB):workflow_run_block.task_id = task_idif failure_reason:workflow_run_block.failure_reason = failure_reason+ if error_codes is not None:+ workflow_run_block.error_codes = error_codes# Use `is not None` instead of truthiness checks so that falsy# values like current_index=0, empty loop_values=[], or# current_value="" are correctly persisted. Without this,@@ -5415,6 +5425,22 @@ class AgentDB(BaseAlchemyDB):for workflow_run_block in workflow_run_blocks]+ async def get_workflow_run_block_errors(+ self,+ workflow_run_id: str,+ organization_id: str | None = None,+ ) -> list[tuple[list[str], str | None]]:+ """Return (error_codes, failure_reason) tuples for blocks with non-null error_codes."""+ async with self.Session() as session:+ query = select(WorkflowRunBlockModel.error_codes, WorkflowRunBlockModel.failure_reason).filter_by(+ workflow_run_id=workflow_run_id+ )+ if organization_id is not None:+ query = query.filter_by(organization_id=organization_id)+ query = query.where(WorkflowRunBlockModel.error_codes.isnot(None))+ rows = (await session.execute(query)).all()+ return [(row.error_codes, row.failure_reason) for row in rows]+async def create_browser_profile(self,organization_id: str,diff --git a/skyvern/forge/sdk/db/models.py b/skyvern/forge/sdk/db/models.pyindex 9ccee8785..691583879 100644--- a/skyvern/forge/sdk/db/models.py+++ b/skyvern/forge/sdk/db/models.py@@ -775,6 +775,7 @@ class WorkflowRunBlockModel(Base):output = Column(JSON, nullable=True)continue_on_failure = Column(Boolean, nullable=False, default=False)failure_reason = Column(String, nullable=True)+ error_codes = Column(JSON, nullable=True)engine = Column(String, nullable=True)# for loop blockdiff --git a/skyvern/forge/sdk/db/utils.py b/skyvern/forge/sdk/db/utils.pyindex 82eec84c2..0dbbb065e 100644--- a/skyvern/forge/sdk/db/utils.py+++ b/skyvern/forge/sdk/db/utils.py@@ -619,6 +619,7 @@ def convert_to_workflow_run_block(output=workflow_run_block_model.output,continue_on_failure=workflow_run_block_model.continue_on_failure,failure_reason=workflow_run_block_model.failure_reason,+ error_codes=workflow_run_block_model.error_codes or [],engine=workflow_run_block_model.engine,task_id=workflow_run_block_model.task_id,loop_values=workflow_run_block_model.loop_values,diff --git a/skyvern/forge/sdk/schemas/workflow_runs.py b/skyvern/forge/sdk/schemas/workflow_runs.pyindex acb4f6d77..c24ab2249 100644--- a/skyvern/forge/sdk/schemas/workflow_runs.py+++ b/skyvern/forge/sdk/schemas/workflow_runs.py@@ -25,6 +25,7 @@ class WorkflowRunBlock(BaseModel):output: dict | list | str | None = Nonecontinue_on_failure: bool = Falsefailure_reason: str | None = None+ error_codes: list[str] = []engine: RunEngine | None = Nonetask_id: str | None = Noneurl: str | None = Nonediff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.pyindex d2a34b7ae..20659d37a 100644--- a/skyvern/forge/sdk/workflow/models/block.py+++ b/skyvern/forge/sdk/workflow/models/block.py@@ -231,6 +231,7 @@ class Block(BaseModel, abc.ABC):executed_branch_expression: str | None = None,executed_branch_result: bool | None = None,executed_branch_next_block: str | None = None,+ error_codes: list[str] | None = None,) -> BlockResult:# TODO: update workflow run block status and failure reasonif isinstance(output_parameter_value, str):@@ -247,10 +248,12 @@ class Block(BaseModel, abc.ABC):executed_branch_expression=executed_branch_expression,executed_branch_result=executed_branch_result,executed_branch_next_block=executed_branch_next_block,+ error_codes=error_codes,)return BlockResult(success=success,failure_reason=failure_reason,+ error_codes=error_codes or [],output_parameter=self.output_parameter,output_parameter_value=output_parameter_value,status=status,@@ -498,6 +501,10 @@ class Block(BaseModel, abc.ABC):organization_id=organization_id,)+ def get_failure_error_codes(self) -> list[str]:+ """Return block-level error codes for unexpected failures. Override in subclasses."""+ return []+@traced()async def execute_safe(self,@@ -588,6 +595,7 @@ class Block(BaseModel, abc.ABC):status=BlockStatus.failed,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,+ error_codes=self.get_failure_error_codes() or None,)@abc.abstractmethod@@ -3370,6 +3378,9 @@ class FileParserBlock(Block):file_type: FileTypejson_schema: dict[str, Any] | None = None+ def get_failure_error_codes(self) -> list[str]:+ return ["FILE_PARSER_ERROR"]+def get_all_parameters(self,workflow_run_id: str,@@ -3765,20 +3776,32 @@ class FileParserBlock(Block):status=BlockStatus.failed,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,+ error_codes=self.get_failure_error_codes() or None,)- # Download the file- if self.file_url.startswith("s3://"):- file_path = await download_from_s3(self.get_async_aws_client(), self.file_url)- else:- file_path = await download_file(self.file_url)+ try:+ # Download the file+ if self.file_url.startswith("s3://"):+ file_path = await download_from_s3(self.get_async_aws_client(), self.file_url)+ else:+ file_path = await download_file(self.file_url)- # Auto-detect file type if not explicitly set (IMAGE/EXCEL/PDF/DOCX are explicit choices)- if self.file_type not in (FileType.IMAGE, FileType.EXCEL, FileType.PDF, FileType.DOCX):- self.file_type = self._detect_file_type_from_url(self.file_url, file_path=file_path)+ # Auto-detect file type if not explicitly set (IMAGE/EXCEL/PDF/DOCX are explicit choices)+ if self.file_type not in (FileType.IMAGE, FileType.EXCEL, FileType.PDF, FileType.DOCX):+ self.file_type = self._detect_file_type_from_url(self.file_url, file_path=file_path)- # Validate the file type- self.validate_file_type(self.file_url, file_path)+ # Validate the file type+ self.validate_file_type(self.file_url, file_path)+ except Exception as e:+ return await self.build_block_result(+ success=False,+ failure_reason=f"Failed to download or validate file: {str(e)}",+ output_parameter_value=None,+ status=BlockStatus.failed,+ workflow_run_block_id=workflow_run_block_id,+ organization_id=organization_id,+ error_codes=self.get_failure_error_codes() or None,+ )LOG.debug("FileParserBlock: After file type validation",@@ -3807,6 +3830,7 @@ class FileParserBlock(Block):status=BlockStatus.failed,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,+ error_codes=self.get_failure_error_codes() or None,)# If json_schema is provided, use AI to extract structured data@@ -3830,6 +3854,7 @@ class FileParserBlock(Block):status=BlockStatus.failed,workflow_run_block_id=workflow_run_block_id,organization_id=organization_id,+ error_codes=self.get_failure_error_codes() or None,)else:# Return raw parsed datadiff --git a/skyvern/forge/sdk/workflow/service.py b/skyvern/forge/sdk/workflow/service.pyindex c29b90c17..039ba80c9 100644--- a/skyvern/forge/sdk/workflow/service.py+++ b/skyvern/forge/sdk/workflow/service.py@@ -3896,6 +3896,24 @@ class WorkflowService:for task in workflow_run_tasks:errors.extend(task.errors)+ # Also collect block-level error codes (e.g. FILE_PARSER_ERROR) into the+ # same errors array so they appear in the top-level workflow run response,+ # matching the task-level error format. Uses a lightweight query that only+ # fetches blocks with non-null error_codes to avoid a full block load on+ # every status poll.+ block_errors = await app.DATABASE.get_workflow_run_block_errors(+ workflow_run_id=workflow_run_id, organization_id=organization_id+ )+ for error_codes, failure_reason in block_errors:+ for code in error_codes:+ errors.append(+ {+ "error_code": code,+ "reasoning": failure_reason or "",+ "confidence_float": 1.0,+ }+ )+total_steps = Nonetotal_cost = Noneif include_step_count or include_cost:diff --git a/skyvern/schemas/workflows.py b/skyvern/schemas/workflows.pyindex 1db8c0788..c3636dc5d 100644--- a/skyvern/schemas/workflows.py+++ b/skyvern/schemas/workflows.py@@ -1,6 +1,6 @@import abcimport functools-from dataclasses import dataclass+from dataclasses import dataclass, fieldfrom enum import StrEnumfrom typing import Annotated, Any, Literal@@ -399,6 +399,7 @@ class BlockResult:output_parameter_value: dict[str, Any] | list | str | None = Nonestatus: BlockStatus | None = Nonefailure_reason: str | None = None+ error_codes: list[str] = field(default_factory=list)workflow_run_block_id: str | None = Nonediff --git a/tests/unit/workflow/test_file_parser_error_code.py b/tests/unit/workflow/test_file_parser_error_code.pynew file mode 100644index 000000000..f41c33020--- /dev/null+++ b/tests/unit/workflow/test_file_parser_error_code.py@@ -0,0 +1,284 @@+"""+Tests for FILE_PARSER_ERROR error codes on FileParserBlock failures.++Covers:+- FileParserError class construction+- BlockResult with error_codes field+- FileParserBlock.execute() returns error_codes on various failure paths+- FileParserBlock.execute() returns empty error_codes on success+- build_block_result() passes error_codes through to update_workflow_run_block()+"""++from __future__ import annotations++from datetime import datetime, timezone+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock+from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType+from skyvern.schemas.workflows import BlockResult, BlockStatus, FileType+++def _make_output_parameter(key: str = "test_output") -> OutputParameter:+ return OutputParameter(+ parameter_type=ParameterType.OUTPUT,+ key=key,+ description="test",+ output_parameter_id="test-output-id",+ workflow_id="test-workflow-id",+ created_at=datetime.now(timezone.utc),+ modified_at=datetime.now(timezone.utc),+ )+++def _make_file_parser_block(+ file_url: str = "https://example.com/file.csv", file_type: FileType = FileType.CSV+) -> FileParserBlock:+ return FileParserBlock(+ label="test_file_parser",+ block_type=BlockType.FILE_URL_PARSER,+ output_parameter=_make_output_parameter(),+ file_url=file_url,+ file_type=file_type,+ )+++def _mock_workflow_run_context() -> MagicMock:+ ctx = MagicMock()+ ctx.has_parameter.return_value = False+ ctx.has_value.return_value = False+ return ctx+++class TestBlockResultErrorCodes:+ """Tests for the error_codes field on BlockResult."""++ def test_block_result_with_error_codes(self) -> None:+ result = BlockResult(+ success=False,+ output_parameter=_make_output_parameter(),+ failure_reason="File parse failed",+ error_codes=["FILE_PARSER_ERROR"],+ status=BlockStatus.failed,+ )+ assert result.error_codes == ["FILE_PARSER_ERROR"]+ assert result.success is False++ def test_block_result_without_error_codes(self) -> None:+ result = BlockResult(+ success=True,+ output_parameter=_make_output_parameter(),+ )+ assert result.error_codes == []++ def test_block_result_default_empty(self) -> None:+ result = BlockResult(+ success=False,+ output_parameter=_make_output_parameter(),+ failure_reason="some error",+ )+ assert result.error_codes == []++ def test_block_result_multiple_error_codes(self) -> None:+ result = BlockResult(+ success=False,+ output_parameter=_make_output_parameter(),+ failure_reason="multiple errors",+ error_codes=["FILE_PARSER_ERROR", "DOWNLOAD_ERROR"],+ status=BlockStatus.failed,+ )+ assert result.error_codes == ["FILE_PARSER_ERROR", "DOWNLOAD_ERROR"]+++class TestFileParserBlockGetFailureErrorCodes:+ """Tests for get_failure_error_codes() override."""++ def test_file_parser_block_returns_error_codes(self) -> None:+ block = _make_file_parser_block()+ assert block.get_failure_error_codes() == ["FILE_PARSER_ERROR"]+++class TestFileParserBlockExecuteErrorCodes:+ """Tests for FileParserBlock.execute() returning error_codes on failure paths."""++ @pytest.mark.asyncio+ async def test_jinja_template_failure_returns_error_codes(self) -> None:+ block = _make_file_parser_block(file_url="{{ invalid_jinja }}")+ mock_ctx = _mock_workflow_run_context()++ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):+ with patch.object(+ FileParserBlock, "format_potential_template_parameters", side_effect=Exception("Jinja error")+ ):+ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert result.success is False+ assert result.error_codes == ["FILE_PARSER_ERROR"]+ assert "jinja template" in result.failure_reason.lower()++ @pytest.mark.asyncio+ async def test_download_failure_returns_error_codes(self) -> None:+ block = _make_file_parser_block(file_url="https://example.com/missing.csv")+ mock_ctx = _mock_workflow_run_context()++ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):+ with patch.object(FileParserBlock, "format_potential_template_parameters"):+ with patch(+ "skyvern.forge.sdk.workflow.models.block.download_file",+ side_effect=Exception("Download failed"),+ ):+ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert result.success is False+ assert result.error_codes == ["FILE_PARSER_ERROR"]++ @pytest.mark.asyncio+ async def test_file_validation_failure_returns_error_codes(self) -> None:+ block = _make_file_parser_block(file_url="https://example.com/bad.csv", file_type=FileType.CSV)+ mock_ctx = _mock_workflow_run_context()++ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):+ with patch.object(FileParserBlock, "format_potential_template_parameters"):+ with patch(+ "skyvern.forge.sdk.workflow.models.block.download_file",+ return_value="/tmp/bad.csv",+ ):+ with patch.object(+ FileParserBlock, "validate_file_type", side_effect=Exception("Invalid file format")+ ):+ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert result.success is False+ assert result.error_codes == ["FILE_PARSER_ERROR"]++ @pytest.mark.asyncio+ async def test_unsupported_file_type_returns_error_codes(self) -> None:+ block = _make_file_parser_block(file_url="https://example.com/file.csv")+ mock_ctx = _mock_workflow_run_context()++ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):+ with patch.object(FileParserBlock, "format_potential_template_parameters"):+ with patch(+ "skyvern.forge.sdk.workflow.models.block.download_file",+ return_value="/tmp/test_file",+ ):+ with patch.object(FileParserBlock, "_detect_file_type_from_url", return_value="unsupported_type"):+ with patch.object(FileParserBlock, "validate_file_type"):+ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert result.success is False+ assert result.error_codes == ["FILE_PARSER_ERROR"]+ assert "unsupported" in result.failure_reason.lower()++ @pytest.mark.asyncio+ async def test_ai_extraction_failure_returns_error_codes(self) -> None:+ block = _make_file_parser_block(file_url="https://example.com/file.csv")+ block.json_schema = {"type": "object", "properties": {"name": {"type": "string"}}}+ mock_ctx = _mock_workflow_run_context()++ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):+ with patch.object(FileParserBlock, "format_potential_template_parameters"):+ with patch(+ "skyvern.forge.sdk.workflow.models.block.download_file",+ return_value="/tmp/test.csv",+ ):+ with patch.object(FileParserBlock, "validate_file_type"):+ with patch.object(FileParserBlock, "_parse_csv_file", return_value=[{"name": "Alice"}]):+ with patch.object(FileParserBlock, "_extract_with_ai", side_effect=Exception("LLM error")):+ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert result.success is False+ assert result.error_codes == ["FILE_PARSER_ERROR"]+ assert "extract data with ai" in result.failure_reason.lower()++ @pytest.mark.asyncio+ async def test_success_returns_empty_error_codes(self) -> None:+ block = _make_file_parser_block(file_url="https://example.com/file.csv")+ mock_ctx = _mock_workflow_run_context()++ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):+ with patch.object(FileParserBlock, "format_potential_template_parameters"):+ with patch(+ "skyvern.forge.sdk.workflow.models.block.download_file",+ return_value="/tmp/test.csv",+ ):+ with patch.object(FileParserBlock, "validate_file_type"):+ with patch.object(FileParserBlock, "_parse_csv_file", return_value=[{"name": "Alice"}]):+ with patch.object(FileParserBlock, "record_output_parameter_value", new_callable=AsyncMock):+ result = await block.execute(+ workflow_run_id="wr_test",+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ assert result.success is True+ assert result.error_codes == []+++class TestBuildBlockResultPassesErrorCodes:+ """Tests that build_block_result() passes error_codes to update_workflow_run_block()."""++ @pytest.mark.asyncio+ async def test_error_codes_passed_to_db_update(self) -> None:+ block = _make_file_parser_block()++ from skyvern.forge import app++ app.DATABASE.update_workflow_run_block.reset_mock()++ result = await block.build_block_result(+ success=False,+ failure_reason="test failure",+ status=BlockStatus.failed,+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ error_codes=["FILE_PARSER_ERROR"],+ )++ app.DATABASE.update_workflow_run_block.assert_called_once()+ call_kwargs = app.DATABASE.update_workflow_run_block.call_args[1]+ assert call_kwargs["error_codes"] == ["FILE_PARSER_ERROR"]+ assert result.error_codes == ["FILE_PARSER_ERROR"]++ @pytest.mark.asyncio+ async def test_none_error_codes_passed_to_db_update(self) -> None:+ block = _make_file_parser_block()++ from skyvern.forge import app++ app.DATABASE.update_workflow_run_block.reset_mock()++ result = await block.build_block_result(+ success=True,+ failure_reason=None,+ status=BlockStatus.completed,+ workflow_run_block_id="wrb_test",+ organization_id="org_test",+ )++ app.DATABASE.update_workflow_run_block.assert_called_once()+ call_kwargs = app.DATABASE.update_workflow_run_block.call_args[1]+ assert call_kwargs["error_codes"] is None+ assert result.error_codes == []