Skip to content

Automate browser based workflows with AI

fix(SKY-11131): surface parse errors instead of crashing FileParserBlock (#6657)

Aaron Perez7cba2b2cfca0
diff --git a/fern/docs/changelog.mdx b/fern/docs/changelog.mdx
index efa473ca4..c0297da06 100644
--- a/fern/docs/changelog.mdx
+++ b/fern/docs/changelog.mdx
@@ -56,6 +56,7 @@ mode: custom
- Fixed code block runs not generating a recording entry, causing the Recording tab to appear empty. ([#6607](https://github.com/Skyvern-AI/skyvern/pull/6607))
- Fixed code block action screenshots not appearing on the run Overview page. ([#6605](https://github.com/Skyvern-AI/skyvern/pull/6605))
- Fixed long usernames or email addresses hiding the sidebar collapse button on desktop.
+- Fixed the File Parser block crashing when a downloaded file isn't a valid PDF (e.g. a site returns a short plain-text error instead of the expected document); the block now surfaces a clear parse error instead of failing the run unexpectedly.
</Update>
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index 72bafabb2..0ffcd664a 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -129,6 +129,7 @@ from skyvern.forge.sdk.copilot.turn_intent import (
from skyvern.forge.sdk.copilot.turn_outcome import (
apply_repeated_reply_guard,
derive_response_kind,
+ with_copilot_code_mode_diagnostics,
)
from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from skyvern.forge.sdk.schemas.persistent_browser_sessions import is_final_status
@@ -949,6 +950,8 @@ def _make_agent_result(
if payload_updates:
kwargs["narrative_payload"] = {**narrative_payload, **payload_updates}
result = AgentResult(global_llm_context=final_context, turn_outcome=turn_outcome, **kwargs)
+ if ctx is not None and result.turn_outcome is not None:
+ result.turn_outcome = with_copilot_code_mode_diagnostics(result.turn_outcome, ctx)
if ctx is not None and result.completion_criteria_turn_state is None:
result.completion_criteria_turn_state = getattr(ctx, "completion_criteria_turn_state", None)
return result
diff --git a/skyvern/forge/sdk/copilot/context.py b/skyvern/forge/sdk/copilot/context.py
index 04c8dff28..bff83b40f 100644
--- a/skyvern/forge/sdk/copilot/context.py
+++ b/skyvern/forge/sdk/copilot/context.py
@@ -408,6 +408,7 @@ class CopilotContext(AgentContext):
code_only_code_schema_seen: bool = False
code_only_target_page_evidence_seen: bool = False
last_failed_workflow_yaml: str | None = None
+ code_native_pending_capability: str | None = None
# Set when a block-running tool timed out and the run's true outcome
# could not be reconciled (post-drain row was ``canceled``, non-final, or
# unreadable). Blocks further block-running tool calls until the LLM
diff --git a/skyvern/forge/sdk/copilot/runtime.py b/skyvern/forge/sdk/copilot/runtime.py
index 38405b00b..525684c29 100644
--- a/skyvern/forge/sdk/copilot/runtime.py
+++ b/skyvern/forge/sdk/copilot/runtime.py
@@ -197,6 +197,7 @@ class AgentContext:
last_failed_workflow_yaml: str | None = None
code_only_code_schema_seen: bool = False
code_only_target_page_evidence_seen: bool = False
+ code_native_pending_capability: str | None = None
repeated_failure_streak_count: int = 0
repeated_failure_nudge_emitted_at_streak: int = 0
challenge_gated_proxy_retry_count: int = 0
diff --git a/skyvern/forge/sdk/copilot/tools/banned_blocks.py b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
index 467760a8f..b036b3b95 100644
--- a/skyvern/forge/sdk/copilot/tools/banned_blocks.py
+++ b/skyvern/forge/sdk/copilot/tools/banned_blocks.py
@@ -176,6 +176,15 @@ def _render_block_policy_detail(block_type: str, policy: CopilotBlockPolicy) ->
return f"`{block_type}` is {policy.status.value} and requires {policy.required_capability}. {policy.guidance}"
+def _record_code_native_pending_capability(ctx: AgentContext | None, policy: CopilotBlockPolicy) -> None:
+ if (
+ ctx is not None
+ and policy.status == CopilotBlockPolicyStatus.CODE_NATIVE_PENDING
+ and ctx.code_native_pending_capability is None
+ ):
+ ctx.code_native_pending_capability = policy.required_capability
+
+
def _code_only_browser_unavailable_types() -> list[str]:
return sorted(
block_type
@@ -273,6 +282,7 @@ def _banned_block_reject_message(items: list[tuple[str, str]], ctx: AgentContext
if policy_entry is None:
continue
_normalized, policy = policy_entry
+ _record_code_native_pending_capability(ctx, policy)
type_labels = ", ".join(sorted(grouped[block_type]))
details.append(f"{block_type} [{type_labels}]: {_render_block_policy_detail(block_type, policy)}")
details_part = " ".join(details)
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index be89e9349..fa10b8f29 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -31,6 +31,7 @@ from .banned_blocks import (
_copilot_banned_block_types,
_copilot_block_authoring_policy,
_copilot_block_policy,
+ _record_code_native_pending_capability,
_render_block_policy_detail,
)
from .completion import _maybe_run_completion_verification_from_page_observation
@@ -90,6 +91,7 @@ async def _get_block_schema_pre_hook(
if policy_entry is None:
return None
normalized, policy = policy_entry
+ _record_code_native_pending_capability(ctx, policy)
return {
"ok": False,
"error": (
@@ -131,6 +133,7 @@ async def _validate_block_pre_hook(
if policy_entry is None:
return None
normalized, policy = policy_entry
+ _record_code_native_pending_capability(ctx, policy)
return {
"ok": False,
"error": (
diff --git a/skyvern/forge/sdk/copilot/turn_outcome.py b/skyvern/forge/sdk/copilot/turn_outcome.py
index 0ff906c16..42ad6555d 100644
--- a/skyvern/forge/sdk/copilot/turn_outcome.py
+++ b/skyvern/forge/sdk/copilot/turn_outcome.py
@@ -9,7 +9,7 @@ direction allowed.
from __future__ import annotations
from collections.abc import Iterable
-from typing import Any
+from typing import Any, Literal
import structlog
@@ -20,6 +20,7 @@ from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOut
LOG = structlog.get_logger()
IDENTICAL_REPLY_BLOCKED_TERMINAL_REASON = "identical_reply_blocked"
+CopilotComposerMode = Literal["ask", "build", "code"]
REPEATED_REPLY_ESCALATION_TEMPLATES: dict[ResponseKind, str] = {
ResponseKind.BUILD: (
@@ -162,6 +163,43 @@ def _dedup_signatures(signatures: Iterable[str]) -> list[str]:
return sorted({sig for sig in signatures if isinstance(sig, str) and sig})
+def _string_or_none(value: object) -> str | None:
+ return value if isinstance(value, str) and value else None
+
+
+def derive_copilot_code_mode_diagnostics(ctx: Any) -> dict[str, bool | str | None]:
+ turn_halt_kind = getattr(getattr(ctx, "turn_halt", None), "kind", None)
+ turn_halt_kind_value = getattr(turn_halt_kind, "value", turn_halt_kind)
+ pending_capability = _string_or_none(getattr(ctx, "code_native_pending_capability", None))
+ return {
+ "copilot_last_code_build_failed": bool(
+ getattr(ctx, "last_test_ok", None) is False or getattr(ctx, "last_failed_workflow_yaml", None)
+ ),
+ "copilot_repair_ceiling_hit": turn_halt_kind_value == "repair_ceiling_reached",
+ "copilot_pending_capability": pending_capability,
+ }
+
+
+def with_copilot_code_mode_diagnostics(outcome: TurnOutcome, ctx: Any) -> TurnOutcome:
+ return outcome.model_copy(update=derive_copilot_code_mode_diagnostics(ctx))
+
+
+def with_copilot_code_mode_metadata(
+ outcome: TurnOutcome,
+ *,
+ effective_mode: CopilotComposerMode,
+ code_available: bool,
+ turn_id: str | None,
+) -> TurnOutcome:
+ return outcome.model_copy(
+ update={
+ "copilot_effective_mode": effective_mode,
+ "copilot_code_available": code_available,
+ "copilot_turn_id": turn_id,
+ }
+ )
+
+
def build_minimal_turn_outcome(
final_text: str,
response_kind: ResponseKind,
diff --git a/skyvern/forge/sdk/routes/workflow_copilot.py b/skyvern/forge/sdk/routes/workflow_copilot.py
index 4189f6a61..719eeca6c 100644
--- a/skyvern/forge/sdk/routes/workflow_copilot.py
+++ b/skyvern/forge/sdk/routes/workflow_copilot.py
@@ -16,6 +16,7 @@ from opentelemetry import trace as otel_trace
from pydantic import ValidationError
from sse_starlette import EventSourceResponse
+from skyvern import analytics
from skyvern.config import settings
from skyvern.constants import DEFAULT_LOGIN_PROMPT
from skyvern.forge import app
@@ -35,7 +36,7 @@ from skyvern.forge.sdk.copilot.completion_criteria_store import (
criteria_to_json,
plan_persistence,
)
-from skyvern.forge.sdk.copilot.config import CopilotConfig
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig, normalize_block_authoring_policy
from skyvern.forge.sdk.copilot.context import AgentResult, ProposalDisposition, TurnNarrativePayload
from skyvern.forge.sdk.copilot.llm_config import resolve_main_copilot_handler
from skyvern.forge.sdk.copilot.output_utils import truncate_output
@@ -45,10 +46,15 @@ from skyvern.forge.sdk.copilot.recoverable_failure import (
format_recoverable_failure_reply,
merge_failure_into_context,
)
+from skyvern.forge.sdk.copilot.turn_outcome import (
+ CopilotComposerMode,
+ build_minimal_turn_outcome,
+ with_copilot_code_mode_metadata,
+)
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.routes.event_source_stream import EventSourceStream, FastAPIEventSourceStream
from skyvern.forge.sdk.routes.routers import base_router
-from skyvern.forge.sdk.schemas.copilot_turn_outcome import TurnOutcome
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
from skyvern.forge.sdk.schemas.organizations import Organization
from skyvern.forge.sdk.schemas.workflow_copilot import (
WorkflowCopilotApplyProposedWorkflowRequest,
@@ -182,6 +188,155 @@ def bind_copilot_session_id(chat_id: str | None) -> Iterator[None]:
ctx.copilot_session_id = prev
+COPILOT_CODE_MODE_OPT_OUT_EVENT = "copilot_code_mode_opt_out"
+COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON = "copilot_recoverable_failure"
+
+
+def _effective_copilot_composer_mode(
+ chat_request: WorkflowCopilotChatRequest,
+ *,
+ uses_v2: bool,
+ code_mode_fallback: bool = False,
+) -> CopilotComposerMode:
+ if chat_request.mode == "ask":
+ return "ask"
+ if chat_request.mode == "build":
+ return "code" if chat_request.code_block is True else "build"
+ if uses_v2:
+ if chat_request.code_block is not None:
+ return "code" if chat_request.code_block is True else "build"
+ return "code" if code_mode_fallback else "build"
+ return "ask"
+
+
+def _latest_assistant_turn_outcome(chat_messages: list[WorkflowCopilotChatMessage]) -> TurnOutcome | None:
+ for message in reversed(chat_messages):
+ if message.sender == WorkflowCopilotChatSender.AI and message.turn_outcome is not None:
+ return message.turn_outcome
+ return None
+
+
+def _should_emit_copilot_code_mode_opt_out(
+ *,
+ prior_turn_outcome: TurnOutcome | None,
+ to_mode: CopilotComposerMode,
+ current_code_available: bool,
+) -> bool:
+ if prior_turn_outcome is None:
+ return False
+ from_mode = prior_turn_outcome.copilot_effective_mode
+ if from_mode is None or from_mode == to_mode:
+ return False
+ if from_mode == "code" and to_mode != "code":
+ return True
+ return (
+ from_mode == "build"
+ and to_mode == "ask"
+ and (prior_turn_outcome.copilot_code_available or current_code_available)
+ )
+
+
+def _reason_category_for_copilot_code_mode_opt_out(
+ prior_turn_outcome: TurnOutcome,
+) -> str:
+ if (
+ prior_turn_outcome.copilot_last_code_build_failed
+ or prior_turn_outcome.copilot_repair_ceiling_hit
+ or prior_turn_outcome.terminal_reason == COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON
+ ):
+ return "failure"
+ if prior_turn_outcome.copilot_pending_capability:
+ return "missing_capability"
+ return "confusion"
+
+
+def _capture_copilot_code_mode_opt_out(
+ *,
+ prior_turn_outcome: TurnOutcome | None,
+ to_mode: CopilotComposerMode,
+ current_code_available: bool,
+ workflow_copilot_chat_id: str,
+ workflow_permanent_id: str,
+ organization_id: str,
+ turn_id: str | None,
+) -> None:
+ if prior_turn_outcome is None or not _should_emit_copilot_code_mode_opt_out(
+ prior_turn_outcome=prior_turn_outcome,
+ to_mode=to_mode,
+ current_code_available=current_code_available,
+ ):
+ return
+ try:
+ analytics.capture(
+ COPILOT_CODE_MODE_OPT_OUT_EVENT,
+ data={
+ "from_mode": prior_turn_outcome.copilot_effective_mode,
+ "to_mode": to_mode,
+ "reason_category": _reason_category_for_copilot_code_mode_opt_out(prior_turn_outcome),
+ "last_code_build_failed": prior_turn_outcome.copilot_last_code_build_failed,
+ "repair_ceiling_hit": prior_turn_outcome.copilot_repair_ceiling_hit,
+ "pending_capability": prior_turn_outcome.copilot_pending_capability,
+ "org_id": organization_id,
+ "workflow_permanent_id": workflow_permanent_id,
+ "workflow_copilot_chat_id": workflow_copilot_chat_id,
+ "turn_id": turn_id,
+ "prior_turn_id": prior_turn_outcome.copilot_turn_id,
+ },
+ distinct_id=workflow_copilot_chat_id,
+ )
+ except Exception:
+ LOG.warning(
+ "Failed to capture copilot code mode opt-out event",
+ workflow_copilot_chat_id=workflow_copilot_chat_id,
+ organization_id=organization_id,
+ exc_info=True,
+ )
+
+
+async def _resolve_copilot_code_available(
+ organization_id: str,
+ chat_request: WorkflowCopilotChatRequest,
+) -> bool:
+ try:
+ has_code_block_access = await app.AGENT_FUNCTION.has_code_block_access(organization_id)
+ except Exception:
+ LOG.warning("Failed to resolve copilot code block access", organization_id=organization_id, exc_info=True)
+ return False
+ if not has_code_block_access:
+ return False
+ if chat_request.code_block is not None:
+ return True
+ try:
+ copilot_config = await app.AGENT_FUNCTION.get_copilot_config_for_request(
+ organization_id,
+ code_block_mode=None,
+ )
+ except Exception:
+ LOG.warning("Failed to resolve copilot code mode availability", organization_id=organization_id, exc_info=True)
+ return False
+ return (
+ normalize_block_authoring_policy(getattr(copilot_config, "block_authoring_policy", None))
+ == BlockAuthoringPolicy.CODE_ONLY_BROWSER
+ )
+
+
+def _with_current_copilot_code_mode_metadata(
+ turn_outcome: TurnOutcome | None,
+ *,
+ effective_mode: CopilotComposerMode,
+ code_available: bool,
+ turn_id: str | None,
+) -> TurnOutcome | None:
+ if turn_outcome is None:
+ return None
+ return with_copilot_code_mode_metadata(
+ turn_outcome,
+ effective_mode=effective_mode,
+ code_available=code_available,
+ turn_id=turn_id,
+ )
+
+
@dataclass(frozen=True)
class RunInfo:
block_label: str | None
@@ -381,6 +536,12 @@ def _build_recoverable_route_agent_result(
clear_proposed_workflow=clear_proposed_workflow,
turn_id=turn_id,
narrative_payload=_make_error_narrative_payload(turn_id, turn_index, user_response),
+ turn_outcome=build_minimal_turn_outcome(
+ user_response,
+ response_kind=ResponseKind.RECOVER,
+ reason_code=failure.failure_kind,
+ terminal_reason=COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON,
+ ),
)
_record_recoverable_failure_span_attrs(failure, proposal_disposition="no_proposal")
return agent_result, failure
@@ -1606,6 +1767,23 @@ async def _new_copilot_chat_post(
global_llm_context: str | None = None
terminal_frame_emitted = False
cancel_watcher: asyncio.Task[None] | None = None
+ current_code_available = False
+ effective_mode = _effective_copilot_composer_mode(chat_request, uses_v2=True)
+ prior_turn_outcome: TurnOutcome | None = None
+
+ def capture_code_mode_opt_out_after_persist() -> None:
+ if chat is None:
+ return
+ _capture_copilot_code_mode_opt_out(
+ prior_turn_outcome=prior_turn_outcome,
+ to_mode=effective_mode,
+ current_code_available=current_code_available,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ workflow_permanent_id=chat.workflow_permanent_id,
+ organization_id=organization.organization_id,
+ turn_id=turn_id,
+ )
+
# Single-element list used as a closure flag (mutable bool by reference).
# The watcher sets [0] = True before issuing handler_task.cancel() so the
# except CancelledError block can distinguish a user-driven cancel from
@@ -1637,6 +1815,16 @@ async def _new_copilot_chat_post(
chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
)
+ prior_turn_outcome = _latest_assistant_turn_outcome(chat_messages)
+ current_code_available = await _resolve_copilot_code_available(
+ organization.organization_id,
+ chat_request,
+ )
+ effective_mode = _effective_copilot_composer_mode(
+ chat_request,
+ uses_v2=True,
+ code_mode_fallback=current_code_available,
+ )
for message in reversed(chat_messages):
if message.global_llm_context is not None:
global_llm_context = message.global_llm_context
@@ -1790,6 +1978,13 @@ async def _new_copilot_chat_post(
stored_completion_criteria=stored_completion_criteria,
)
+ agent_result.turn_outcome = _with_current_copilot_code_mode_metadata(
+ agent_result.turn_outcome,
+ effective_mode=effective_mode,
+ code_available=current_code_available,
+ turn_id=turn_id,
+ )
+
if getattr(agent_result, "cancelled", False):
# The agent absorbed the CancelledError and returned a result
# carrying ``workflow_was_persisted`` so rollback proceeds normally.
@@ -1804,6 +1999,7 @@ async def _new_copilot_chat_post(
turn_id=turn_id,
)
terminal_frame_emitted = True
+ capture_code_mode_opt_out_after_persist()
LOG.info(
"Workflow copilot v2 cancelled by user",
workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
@@ -1824,6 +2020,7 @@ async def _new_copilot_chat_post(
)
)
terminal_frame_emitted = True
+ capture_code_mode_opt_out_after_persist()
except HTTPException as exc:
if chat is not None and _should_restore_persisted_workflow(chat.auto_accept, agent_result):
try:
@@ -1864,6 +2061,12 @@ async def _new_copilot_chat_post(
turn_id=turn_id,
turn_index=turn_index,
)
+ recovered_result.turn_outcome = _with_current_copilot_code_mode_metadata(
+ recovered_result.turn_outcome,
+ effective_mode=effective_mode,
+ code_available=current_code_available,
+ turn_id=turn_id,
+ )
LOG.error(
"LLM provider error translated to recoverable workflow copilot v2 reply",
organization_id=organization.organization_id,
@@ -1884,6 +2087,7 @@ async def _new_copilot_chat_post(
)
)
terminal_frame_emitted = True
+ capture_code_mode_opt_out_after_persist()
else:
LOG.error(
"LLM provider error (copilot v2)",
@@ -1965,6 +2169,12 @@ async def _new_copilot_chat_post(
turn_id=turn_id,
turn_index=turn_index,
)
+ recovered_result.turn_outcome = _with_current_copilot_code_mode_metadata(
+ recovered_result.turn_outcome,
+ effective_mode=effective_mode,
+ code_available=current_code_available,
+ turn_id=turn_id,
+ )
LOG.error(
"Unexpected workflow copilot v2 error translated to recoverable reply",
organization_id=organization.organization_id,
@@ -1985,6 +2195,7 @@ async def _new_copilot_chat_post(
)
)
terminal_frame_emitted = True
+ capture_code_mode_opt_out_after_persist()
else:
LOG.error(
"Unexpected error in workflow copilot v2",
@@ -2164,6 +2375,7 @@ async def workflow_copilot_chat_post(
return await _new_copilot_chat_post(request, chat_request, organization)
async def stream_handler(stream: EventSourceStream) -> None:
+ turn_id = uuid.uuid4().hex
LOG.info(
"Workflow copilot chat request",
workflow_copilot_chat_id=chat_request.workflow_copilot_chat_id,
@@ -2198,6 +2410,16 @@ async def workflow_copilot_chat_post(
chat_messages = await app.DATABASE.workflow_params.get_workflow_copilot_chat_messages(
workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
)
+ prior_turn_outcome = _latest_assistant_turn_outcome(chat_messages)
+ current_code_available = await _resolve_copilot_code_available(
+ organization.organization_id,
+ chat_request,
+ )
+ effective_mode = _effective_copilot_composer_mode(
+ chat_request,
+ uses_v2=False,
+ code_mode_fallback=current_code_available,
+ )
global_llm_context = None
for message in reversed(chat_messages):
if message.global_llm_context is not None:
@@ -2243,6 +2465,16 @@ async def workflow_copilot_chat_post(
debug_run_info_text,
)
+ legacy_turn_outcome = _with_current_copilot_code_mode_metadata(
+ build_minimal_turn_outcome(
+ user_response,
+ response_kind=ResponseKind.CLARIFY if effective_mode == "ask" else ResponseKind.BUILD,
+ ),
+ effective_mode=effective_mode,
+ code_available=current_code_available,
+ turn_id=turn_id,
+ )
+
if updated_workflow and chat.auto_accept is not True:
proposed_data = updated_workflow.model_dump(mode="json")
# _copilot_yaml is what /apply-proposed-workflow re-parses into
@@ -2269,8 +2501,18 @@ async def workflow_copilot_chat_post(
sender=WorkflowCopilotChatSender.AI,
content=user_response,
global_llm_context=updated_global_llm_context,
+ turn_outcome=legacy_turn_outcome,
)
+ _capture_copilot_code_mode_opt_out(
+ prior_turn_outcome=prior_turn_outcome,
+ to_mode=effective_mode,
+ current_code_available=current_code_available,
+ workflow_copilot_chat_id=chat.workflow_copilot_chat_id,
+ workflow_permanent_id=chat.workflow_permanent_id,
+ organization_id=organization.organization_id,
+ turn_id=turn_id,
+ )
terminal_frame_emitted = True
await stream.send(
WorkflowCopilotStreamResponseUpdate(
@@ -2318,7 +2560,7 @@ async def workflow_copilot_chat_post(
)
)
finally:
- await _ensure_terminal_frame(stream, terminal_frame_emitted)
+ await _ensure_terminal_frame(stream, terminal_frame_emitted, turn_id=turn_id)
return FastAPIEventSourceStream.create(request, stream_handler)
diff --git a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
index 8a6bc4871..b52dc73ac 100644
--- a/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
+++ b/skyvern/forge/sdk/schemas/copilot_turn_outcome.py
@@ -8,7 +8,7 @@ in any ``copilot/`` business logic — derivation lives in
from __future__ import annotations
from enum import StrEnum
-from typing import Any
+from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, Field
@@ -33,3 +33,9 @@ class TurnOutcome(BaseModel):
tool_calls: list[str] = Field(default_factory=list)
terminal_reason: str | None = None
blocked_signatures: list[str] = Field(default_factory=list)
+ copilot_effective_mode: Literal["ask", "build", "code"] | None = None
+ copilot_code_available: bool = False
+ copilot_last_code_build_failed: bool = False
+ copilot_repair_ceiling_hit: bool = False
+ copilot_pending_capability: str | None = None
+ copilot_turn_id: str | None = None
diff --git a/skyvern/forge/sdk/workflow/models/block.py b/skyvern/forge/sdk/workflow/models/block.py
index a6ff3f9f7..7bacdcf1d 100644
--- a/skyvern/forge/sdk/workflow/models/block.py
+++ b/skyvern/forge/sdk/workflow/models/block.py
@@ -5904,31 +5904,40 @@ class FileParserBlock(Block):
# Parse the file based on type
parsed_data: str | list[dict[str, Any]]
- if self.file_type == FileType.CSV:
- parsed_data = await self._parse_csv_file(file_path)
- elif self.file_type == FileType.EXCEL:
- parsed_data = await self._parse_excel_file(file_path)
- elif self.file_type == FileType.PDF:
- parsed_data = await self._parse_pdf_file(
- file_path,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
- elif self.file_type == FileType.IMAGE:
- parsed_data = await self._parse_image_file(
- file_path,
- workflow_run_block_id=workflow_run_block_id,
- organization_id=organization_id,
- )
- elif self.file_type == FileType.DOCX:
- parsed_data = await self._parse_docx_file(file_path)
- else:
+ try:
+ if self.file_type == FileType.CSV:
+ parsed_data = await self._parse_csv_file(file_path)
+ elif self.file_type == FileType.EXCEL:
+ parsed_data = await self._parse_excel_file(file_path)
+ elif self.file_type == FileType.PDF:
+ parsed_data = await self._parse_pdf_file(
+ file_path,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ elif self.file_type == FileType.IMAGE:
+ parsed_data = await self._parse_image_file(
+ file_path,
+ workflow_run_block_id=workflow_run_block_id,
+ organization_id=organization_id,
+ )
+ elif self.file_type == FileType.DOCX:
+ parsed_data = await self._parse_docx_file(file_path)
+ else:
+ return await self._record_failure(
+ workflow_run_context,
+ workflow_run_id,
+ workflow_run_block_id,
+ organization_id,
+ f"Unsupported file type: {self.file_type}",
+ )
+ except Exception as e:
return await self._record_failure(
workflow_run_context,
workflow_run_id,
workflow_run_block_id,
organization_id,
- f"Unsupported file type: {self.file_type}",
+ f"Failed to parse {self.file_type} file: {str(e)}",
)
# If json_schema is provided, use AI to extract structured data
diff --git a/tests/unit/test_copilot_code_mode_opt_out.py b/tests/unit/test_copilot_code_mode_opt_out.py
new file mode 100644
index 000000000..8fbac65e5
--- /dev/null
+++ b/tests/unit/test_copilot_code_mode_opt_out.py
@@ -0,0 +1,265 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy, CopilotConfig
+from skyvern.forge.sdk.copilot.turn_outcome import (
+ derive_copilot_code_mode_diagnostics,
+ with_copilot_code_mode_metadata,
+)
+from skyvern.forge.sdk.routes import workflow_copilot as workflow_copilot_route
+from skyvern.forge.sdk.routes.workflow_copilot import (
+ COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON,
+ _build_recoverable_route_agent_result,
+ _capture_copilot_code_mode_opt_out,
+ _effective_copilot_composer_mode,
+ _reason_category_for_copilot_code_mode_opt_out,
+ _resolve_copilot_code_available,
+ _should_emit_copilot_code_mode_opt_out,
+)
+from skyvern.forge.sdk.schemas.copilot_turn_outcome import ResponseKind, TurnOutcome
+from skyvern.forge.sdk.schemas.workflow_copilot import WorkflowCopilotChatRequest
+
+
+def _request(mode: str | None, code_block: bool | None) -> WorkflowCopilotChatRequest:
+ return WorkflowCopilotChatRequest(
+ workflow_permanent_id="wpid-1",
+ workflow_id="wf-1",
+ workflow_copilot_chat_id="chat-1",
+ message="message",
+ workflow_yaml="title: Example",
+ mode=mode,
+ code_block=code_block,
+ )
+
+
+def _outcome(
+ *,
+ mode: str | None,
+ code_available: bool = True,
+ last_code_build_failed: bool = False,
+ repair_ceiling_hit: bool = False,
+ pending_capability: str | None = None,
+ turn_id: str | None = "prior-turn",
+) -> TurnOutcome:
+ return TurnOutcome(
+ response_kind=ResponseKind.BUILD,
+ copilot_effective_mode=mode,
+ copilot_code_available=code_available,
+ copilot_last_code_build_failed=last_code_build_failed,
+ copilot_repair_ceiling_hit=repair_ceiling_hit,
+ copilot_pending_capability=pending_capability,
+ copilot_turn_id=turn_id,
+ )
+
+
+ ("mode", "code_block", "uses_v2", "code_mode_fallback", "expected"),
+ [
+ ("ask", None, False, False, "ask"),
+ ("build", None, True, True, "build"),
+ ("build", False, True, True, "build"),
+ ("build", True, True, False, "code"),
+ (None, True, True, False, "code"),
+ (None, False, True, True, "build"),
+ (None, None, True, False, "build"),
+ (None, None, True, True, "code"),
+ (None, None, False, True, "ask"),
+ ],
+)
+def test_effective_copilot_composer_mode(
+ mode: str | None, code_block: bool | None, uses_v2: bool, code_mode_fallback: bool, expected: str
+) -> None:
+ assert (
+ _effective_copilot_composer_mode(
+ _request(mode, code_block),
+ uses_v2=uses_v2,
+ code_mode_fallback=code_mode_fallback,
+ )
+ == expected
+ )
+
+
+ ("prior", "to_mode", "current_code_available", "expected"),
+ [
+ (_outcome(mode="code"), "build", True, True),
+ (_outcome(mode="code"), "ask", False, True),
+ (_outcome(mode="build", code_available=True), "ask", False, True),
+ (_outcome(mode="build", code_available=False), "ask", True, True),
+ (_outcome(mode="build", code_available=False), "ask", False, False),
+ (_outcome(mode="code"), "code", True, False),
+ (_outcome(mode="build", code_available=True), "build", True, False),
+ (_outcome(mode="ask", code_available=True), "build", True, False),
+ (None, "ask", True, False),
+ (_outcome(mode=None, code_available=True), "ask", True, False),
+ ],
+)
+def test_should_emit_copilot_code_mode_opt_out_transitions(
+ prior: TurnOutcome | None,
+ to_mode: str,
+ current_code_available: bool,
+ expected: bool,
+) -> None:
+ assert (
+ _should_emit_copilot_code_mode_opt_out(
+ prior_turn_outcome=prior,
+ to_mode=to_mode,
+ current_code_available=current_code_available,
+ )
+ is expected
+ )
+
+
+ ("prior", "expected"),
+ [
+ (_outcome(mode="code", last_code_build_failed=True, pending_capability="capability"), "failure"),
+ (_outcome(mode="code", repair_ceiling_hit=True, pending_capability="capability"), "failure"),
+ (
+ TurnOutcome(
+ response_kind=ResponseKind.RECOVER,
+ copilot_effective_mode="code",
+ terminal_reason=COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON,
+ copilot_pending_capability="capability",
+ ),
+ "failure",
+ ),
+ (_outcome(mode="code", pending_capability="capability"), "missing_capability"),
+ (_outcome(mode="code"), "confusion"),
+ ],
+)
+def test_reason_category_for_copilot_code_mode_opt_out(prior: TurnOutcome, expected: str) -> None:
+ assert _reason_category_for_copilot_code_mode_opt_out(prior) == expected
+
+
+def test_capture_copilot_code_mode_opt_out_uses_chat_id_as_distinct_id(monkeypatch: pytest.MonkeyPatch) -> None:
+ capture = MagicMock()
+ monkeypatch.setattr(workflow_copilot_route.analytics, "capture", capture)
+
+ prior = _outcome(
+ mode="code",
+ last_code_build_failed=True,
+ repair_ceiling_hit=False,
+ pending_capability="credential-typed code synthesis",
+ turn_id="turn-prior",
+ )
+
+ _capture_copilot_code_mode_opt_out(
+ prior_turn_outcome=prior,
+ to_mode="ask",
+ current_code_available=True,
+ workflow_copilot_chat_id="chat-123",
+ workflow_permanent_id="wpid-123",
+ organization_id="org-123",
+ turn_id="turn-current",
+ )
+
+ capture.assert_called_once_with(
+ "copilot_code_mode_opt_out",
+ data={
+ "from_mode": "code",
+ "to_mode": "ask",
+ "reason_category": "failure",
+ "last_code_build_failed": True,
+ "repair_ceiling_hit": False,
+ "pending_capability": "credential-typed code synthesis",
+ "org_id": "org-123",
+ "workflow_permanent_id": "wpid-123",
+ "workflow_copilot_chat_id": "chat-123",
+ "turn_id": "turn-current",
+ "prior_turn_id": "turn-prior",
+ },
+ distinct_id="chat-123",
+ )
+
+
+def test_capture_copilot_code_mode_opt_out_skips_non_transition(monkeypatch: pytest.MonkeyPatch) -> None:
+ capture = MagicMock()
+ monkeypatch.setattr(workflow_copilot_route.analytics, "capture", capture)
+
+ _capture_copilot_code_mode_opt_out(
+ prior_turn_outcome=_outcome(mode="build", code_available=False),
+ to_mode="ask",
+ current_code_available=False,
+ workflow_copilot_chat_id="chat-123",
+ workflow_permanent_id="wpid-123",
+ organization_id="org-123",
+ turn_id="turn-current",
+ )
+
+ capture.assert_not_called()
+
+
+def test_build_recoverable_route_agent_result_sets_failure_turn_outcome() -> None:
+ agent_result, failure = _build_recoverable_route_agent_result(
+ RuntimeError("boom"),
+ workflow_modified=False,
+ clear_proposed_workflow=False,
+ global_llm_context=None,
+ turn_id="turn-error",
+ turn_index=2,
+ )
+
+ assert agent_result.turn_outcome is not None
+ assert agent_result.turn_outcome.response_kind is ResponseKind.RECOVER
+ assert agent_result.turn_outcome.reason_code == failure.failure_kind
+ assert agent_result.turn_outcome.terminal_reason == COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON
+ assert _reason_category_for_copilot_code_mode_opt_out(agent_result.turn_outcome) == "failure"
+
+
+async def test_resolve_copilot_code_available_uses_access_and_rollout(monkeypatch: pytest.MonkeyPatch) -> None:
+ config = CopilotConfig(block_authoring_policy=BlockAuthoringPolicy.CODE_ONLY_BROWSER)
+ agent_function = SimpleNamespace(has_code_block_access=AsyncMock(), get_copilot_config_for_request=AsyncMock())
+ monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
+
+ agent_function.has_code_block_access.return_value = False
+ assert await _resolve_copilot_code_available("org-1", _request("build", False)) is False
+ agent_function.get_copilot_config_for_request.assert_not_awaited()
+
+ agent_function.has_code_block_access.return_value = True
+ agent_function.get_copilot_config_for_request.return_value = config
+ assert await _resolve_copilot_code_available("org-1", _request("ask", None)) is True
+
+
+def test_with_copilot_code_mode_metadata_preserves_turn_outcome_fields() -> None:
+ outcome = TurnOutcome(
+ response_kind=ResponseKind.CLARIFY,
+ reason_code="request_policy_clarification",
+ terminal_reason="terminal",
+ )
+
+ updated = with_copilot_code_mode_metadata(
+ outcome,
+ effective_mode="build",
+ code_available=True,
+ turn_id="turn-123",
+ )
+
+ assert updated.response_kind == ResponseKind.CLARIFY
+ assert updated.reason_code == "request_policy_clarification"
+ assert updated.terminal_reason == "terminal"
+ assert updated.copilot_effective_mode == "build"
+ assert updated.copilot_code_available is True
+ assert updated.copilot_turn_id == "turn-123"
+
+
+def test_derive_copilot_code_mode_diagnostics_uses_context_state() -> None:
+ ctx = type("Ctx", (), {})()
+ ctx.last_test_ok = False
+ ctx.last_failed_workflow_yaml = None
+ ctx.code_native_pending_capability = "credential-typed code synthesis"
+ ctx.turn_halt = type("Halt", (), {"kind": type("Kind", (), {"value": "repair_ceiling_reached"})()})()
+
+ diagnostics = derive_copilot_code_mode_diagnostics(ctx)
+
+ assert diagnostics == {
+ "copilot_last_code_build_failed": True,
+ "copilot_repair_ceiling_hit": True,
+ "copilot_pending_capability": "credential-typed code synthesis",
+ }
diff --git a/tests/unit/test_workflow_copilot_route.py b/tests/unit/test_workflow_copilot_route.py
index ed23f2aff..ccab8b168 100644
--- a/tests/unit/test_workflow_copilot_route.py
+++ b/tests/unit/test_workflow_copilot_route.py
@@ -691,6 +691,19 @@ async def test_flag_on_route_error_after_chat_persists_recoverable_reply(
assert "The workflow was not modified" in assistant_contents[0]
assert "reference cpe_" in assistant_contents[0]
+ assistant_messages = [
+ call.kwargs
+ for call in app.DATABASE.workflow_params.create_workflow_copilot_chat_message.await_args_list
+ if call.kwargs.get("sender") == WorkflowCopilotChatSender.AI
+ ]
+ assert len(assistant_messages) == 1
+ turn_outcome = assistant_messages[0]["turn_outcome"]
+ assert turn_outcome is not None
+ assert turn_outcome.response_kind == "recover"
+ assert turn_outcome.terminal_reason == workflow_copilot_route.COPILOT_RECOVERABLE_FAILURE_TERMINAL_REASON
+ assert turn_outcome.copilot_effective_mode == "build"
+ assert turn_outcome.copilot_turn_id is not None
+
frames = [call.args[0] for call in stream.send.await_args_list if call.args]
response_frames = [frame for frame in frames if isinstance(frame, WorkflowCopilotStreamResponseUpdate)]
assert response_frames
diff --git a/tests/unit/workflow/test_file_parser_error_code.py b/tests/unit/workflow/test_file_parser_error_code.py
index 0640c807f..11b6a044f 100644
--- a/tests/unit/workflow/test_file_parser_error_code.py
+++ b/tests/unit/workflow/test_file_parser_error_code.py
@@ -16,6 +16,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
+from skyvern.forge.sdk.workflow.exceptions import InvalidFileType
from skyvern.forge.sdk.workflow.models.block import BlockType, FileParserBlock
from skyvern.forge.sdk.workflow.models.parameter import OutputParameter, ParameterType
from skyvern.schemas.workflows import BlockResult, BlockStatus, FileType
@@ -190,6 +191,68 @@ class TestFileParserBlockExecuteErrorCodes:
assert result.error_codes == ["FILE_PARSER_ERROR"]
assert "unsupported" in result.failure_reason.lower()
+ @pytest.mark.asyncio
+ async def test_parse_invalid_file_type_returns_error_codes(self) -> None:
+ """SKY-11131: a file that passes validation but fails parsing (e.g. a non-PDF
+ that pypdf opens but cannot extract) must surface a graceful failure, not raise."""
+ block = _make_file_parser_block(file_url="https://example.com/statement.pdf", file_type=FileType.PDF)
+ mock_ctx = _mock_workflow_run_context()
+
+ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):
+ with patch.object(FileParserBlock, "format_potential_template_parameters"):
+ with patch(
+ "skyvern.forge.sdk.workflow.models.block.download_file",
+ return_value="/tmp/statement.pdf",
+ ):
+ with patch.object(FileParserBlock, "validate_file_type"):
+ with patch.object(
+ FileParserBlock,
+ "_parse_pdf_file",
+ side_effect=InvalidFileType(
+ file_url="https://example.com/statement.pdf",
+ file_type=FileType.PDF,
+ error="both parsers failed",
+ ),
+ ):
+ result = await block.execute(
+ workflow_run_id="wr_test",
+ workflow_run_block_id="wrb_test",
+ organization_id="org_test",
+ )
+
+ assert result.success is False
+ assert result.error_codes == ["FILE_PARSER_ERROR"]
+ assert "parse" in result.failure_reason.lower()
+
+ @pytest.mark.asyncio
+ async def test_parse_unexpected_exception_returns_error_codes(self) -> None:
+ """SKY-11131: a raw exception from the parse stage (e.g. the vision-LLM PDF
+ fallback) must be caught and surfaced as a graceful failure."""
+ block = _make_file_parser_block(file_url="https://example.com/statement.pdf", file_type=FileType.PDF)
+ mock_ctx = _mock_workflow_run_context()
+
+ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):
+ with patch.object(FileParserBlock, "format_potential_template_parameters"):
+ with patch(
+ "skyvern.forge.sdk.workflow.models.block.download_file",
+ return_value="/tmp/statement.pdf",
+ ):
+ with patch.object(FileParserBlock, "validate_file_type"):
+ with patch.object(
+ FileParserBlock,
+ "_parse_pdf_file",
+ side_effect=Exception("No /Root object! - Is this really a PDF?"),
+ ):
+ result = await block.execute(
+ workflow_run_id="wr_test",
+ workflow_run_block_id="wrb_test",
+ organization_id="org_test",
+ )
+
+ assert result.success is False
+ assert result.error_codes == ["FILE_PARSER_ERROR"]
+ assert "parse" in result.failure_reason.lower()
+
@pytest.mark.asyncio
async def test_ai_extraction_failure_returns_error_codes(self) -> None:
block = _make_file_parser_block(file_url="https://example.com/file.csv")
@@ -393,6 +456,37 @@ class TestFileParserBlockRecordsOutputOnFailure:
self._assert_failure_output_shape(recorded, "unsupported")
assert result.output_parameter_value == recorded
+ @pytest.mark.asyncio
+ async def test_parse_failure_records_failed_output(self) -> None:
+ block = _make_file_parser_block(file_url="https://example.com/statement.pdf", file_type=FileType.PDF)
+ mock_ctx = _mock_workflow_run_context()
+
+ with patch.object(FileParserBlock, "get_workflow_run_context", return_value=mock_ctx):
+ with patch.object(FileParserBlock, "format_potential_template_parameters"):
+ with patch(
+ "skyvern.forge.sdk.workflow.models.block.download_file",
+ return_value="/tmp/statement.pdf",
+ ):
+ with patch.object(FileParserBlock, "validate_file_type"):
+ with patch.object(
+ FileParserBlock,
+ "_parse_pdf_file",
+ side_effect=Exception("No /Root object! - Is this really a PDF?"),
+ ):
+ with patch.object(
+ FileParserBlock, "record_output_parameter_value", new_callable=AsyncMock
+ ) as mock_record:
+ result = await block.execute(
+ workflow_run_id="wr_test",
+ workflow_run_block_id="wrb_test",
+ organization_id="org_test",
+ )
+
+ assert result.success is False
+ recorded = self._recorded_value(mock_record)
+ self._assert_failure_output_shape(recorded, "parse")
+ assert result.output_parameter_value == recorded
+
@pytest.mark.asyncio
async def test_ai_extraction_failure_records_failed_output(self) -> None:
block = _make_file_parser_block(file_url="https://example.com/file.csv")