Skip to content

Automate browser based workflows with AI

Remove two no-action monitors and replace the any-5xx alert with a per-route 5xx count (SKY-18419) (#8943)

Shuchang Zheng9f30f1efece5
diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioTopBar.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioTopBar.tsx
index 3755d6495..796e1eec1 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StudioTopBar.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StudioTopBar.tsx
@@ -13,6 +13,7 @@ import { AxiosError } from "axios";
import {
CalendarIcon,
InputIcon,
+ LightningBoltIcon,
Pencil1Icon,
PlayIcon,
ReloadIcon,
@@ -22,6 +23,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query";
import { Link, useNavigate, useSearchParams } from "react-router-dom";
import { usePostHog } from "posthog-js/react";
import { useWorkflowPermanentId } from "@/routes/workflows/WorkflowPermanentIdContext";
+import { usePageSlots } from "@/store/PageSlots";
import { getClient } from "@/api/AxiosClient";
import { SaveIcon } from "@/components/icons/SaveIcon";
@@ -294,7 +296,7 @@ function PanelToggle({
label,
icon,
}: {
- content: "parameters" | "schedules";
+ content: "parameters" | "schedules" | "triggers";
label: string;
icon: ReactNode;
}) {
@@ -568,6 +570,7 @@ export function RunStopButton({ stopOnly = false }: { stopOnly?: boolean }) {
}
export function StudioTopBar() {
+ const { workflowTriggersPanel } = usePageSlots();
const isGlobalWorkflow = useIsGlobalWorkflow();
const workflowDeletedAt = useStudioWorkflowDeletedAt();
const { panes, setOpenPanes } = useStudioPanes();
@@ -622,6 +625,13 @@ export function StudioTopBar() {
label="Schedule"
icon={<CalendarIcon className="size-4" />}
/>
+ {workflowTriggersPanel ? (
+ <PanelToggle
+ content="triggers"
+ label="Triggers"
+ icon={<LightningBoltIcon className="size-4" />}
+ />
+ ) : null}
<EditorOverflowMenu
triggerClassName="h-8 w-8 rounded-md border-0 bg-transparent text-muted-foreground hover:bg-accent hover:text-accent-foreground"
onVersionHistory={openVersionHistory}
diff --git a/skyvern-frontend/src/routes/workflows/studio/StudioWorkflowPanels.tsx b/skyvern-frontend/src/routes/workflows/studio/StudioWorkflowPanels.tsx
index 083db3292..94a25c828 100644
--- a/skyvern-frontend/src/routes/workflows/studio/StudioWorkflowPanels.tsx
+++ b/skyvern-frontend/src/routes/workflows/studio/StudioWorkflowPanels.tsx
@@ -2,23 +2,30 @@ import { useWorkflowPanelStore } from "@/store/WorkflowPanelStore";
import { WorkflowParametersPanel } from "../editor/panels/WorkflowParametersPanel";
import { WorkflowSchedulePanel } from "../editor/panels/schedulePanel/WorkflowSchedulePanel";
+import { usePageSlots } from "@/store/PageSlots";
+import { useWorkflowPermanentId } from "@/routes/workflows/WorkflowPermanentIdContext";
/**
- * Inputs / Schedule panels rendered at shell level so they open over any tab —
+ * Inputs / Schedule / Triggers panels render at shell level so they open over any tab —
* the editor canvas is display:none on Browser/Overview tabs and would hide them.
*/
export function StudioWorkflowPanels() {
const state = useWorkflowPanelStore((s) => s.workflowPanelState);
const close = useWorkflowPanelStore((s) => s.closeWorkflowPanel);
+ const { workflowTriggersPanel: WorkflowTriggersPanel } = usePageSlots();
+ const workflowPermanentId = useWorkflowPermanentId();
const content =
state.active &&
- (state.content === "parameters" || state.content === "schedules")
+ (state.content === "parameters" ||
+ state.content === "schedules" ||
+ state.content === "triggers")
? state.content
: null;
if (!content) {
return null;
}
+ if (content === "triggers" && !WorkflowTriggersPanel) return null;
return (
<>
@@ -29,10 +36,19 @@ export function StudioWorkflowPanels() {
<div className="absolute right-3 top-3 z-40">
<WorkflowParametersPanel />
</div>
- ) : (
+ ) : content === "schedules" ? (
<div className="absolute right-3 top-3 z-40 max-h-[calc(100%-1.5rem)] overflow-y-auto">
<WorkflowSchedulePanel onClose={close} />
</div>
+ ) : (
+ <div className="absolute right-3 top-3 z-40 max-h-[calc(100%-1.5rem)] overflow-y-auto">
+ {WorkflowTriggersPanel ? (
+ <WorkflowTriggersPanel
+ workflowPermanentId={workflowPermanentId ?? ""}
+ onClose={close}
+ />
+ ) : null}
+ </div>
)}
</>
);
diff --git a/skyvern-frontend/src/store/PageSlots.ts b/skyvern-frontend/src/store/PageSlots.ts
index 7d9f5ba6b..9087c6049 100644
--- a/skyvern-frontend/src/store/PageSlots.ts
+++ b/skyvern-frontend/src/store/PageSlots.ts
@@ -6,6 +6,10 @@ export type WorkflowAnalyticsPanelProps = {
workflow?: WorkflowApiResponse;
workflowIsLoading?: boolean;
};
+export type WorkflowTriggersPanelProps = Readonly<{
+ workflowPermanentId: string;
+ onClose: () => void;
+}>;
export type WorkflowRunMilestoneCardProps = Readonly<{
workflowRunId: string;
rerun?: Readonly<{ to: string; state?: unknown }>;
@@ -31,13 +35,20 @@ export type PermissionsResponse = Readonly<{
}>;
export type PermissionsState =
- | Readonly<{ status: "unavailable"; data?: undefined }>
+ | Readonly<{
+ status: "unavailable";
+ data?: undefined;
+ personalWorkspaceOwner?: true;
+ // The backend has no /auth/capabilities route, so it predates RBAC and enforces nothing.
+ endpointMissing?: true;
+ }>
| Readonly<{ status: "loading"; data?: undefined }>
| Readonly<{ status: "error"; data?: undefined }>
| Readonly<{ status: "ready"; data: PermissionsResponse }>;
export type PageSlots = {
workflowAnalyticsPanel?: React.ComponentType<WorkflowAnalyticsPanelProps>;
+ workflowTriggersPanel?: React.ComponentType<WorkflowTriggersPanelProps>;
workflowRunsFilterControls?: React.ComponentType;
workflowRunMilestoneCard?: React.ComponentType<WorkflowRunMilestoneCardProps>;
firstRunWaitCard?: React.ComponentType<FirstRunWaitCardProps>;
@@ -54,3 +65,28 @@ export const PageSlotsProvider = PageSlotsContext.Provider;
export function usePageSlots(): PageSlots {
return useContext(PageSlotsContext);
}
+
+export function isCapabilityAllowed(
+ permissions: PermissionsState | undefined,
+ capability: string,
+): boolean {
+ // Missing page slots are the OSS context; Cloud always provides a permission state.
+ if (!permissions) return true;
+ if (
+ permissions.status === "unavailable" &&
+ (permissions.personalWorkspaceOwner || permissions.endpointMissing)
+ ) {
+ return true;
+ }
+ return (
+ permissions.status === "ready" &&
+ permissions.data.capabilities.some(
+ (item) => item.capability === capability && item.allowed,
+ )
+ );
+}
+
+export function useHasCapability(capability: string): boolean {
+ const { permissions } = usePageSlots();
+ return isCapabilityAllowed(permissions, capability);
+}
diff --git a/skyvern-frontend/src/store/WorkflowPanelStore.ts b/skyvern-frontend/src/store/WorkflowPanelStore.ts
index a12c04bc5..2a5628a4e 100644
--- a/skyvern-frontend/src/store/WorkflowPanelStore.ts
+++ b/skyvern-frontend/src/store/WorkflowPanelStore.ts
@@ -17,7 +17,8 @@ type WorkflowPanelState = {
| "nodeLibrary"
| "history"
| "comparison"
- | "schedules";
+ | "schedules"
+ | "triggers";
data?: {
previous?: string | null;
next?: string | null;
diff --git a/skyvern/cli/mcp_tools/response.py b/skyvern/cli/mcp_tools/response.py
index 3c56f15c5..4ce09bacc 100644
--- a/skyvern/cli/mcp_tools/response.py
+++ b/skyvern/cli/mcp_tools/response.py
@@ -576,6 +576,16 @@ def _truncate_response_to_limit(
return envelope
+def within_response_limits(data: Any) -> bool:
+ """Whether a payload clears both caps the response decorators enforce.
+
+ A tool that degrades its own oversize payload has to measure the same two
+ limits: checking characters alone lets a non-ASCII response slip into the
+ truncation envelope, which keeps only top-level identifier keys.
+ """
+ return _response_size(data) <= MCP_MAX_RESPONSE_CHARS and _response_size_bytes(data) <= MCP_MAX_RESPONSE_BYTES
+
+
def truncate_response(data: Any, *, max_chars: int = MCP_MAX_RESPONSE_CHARS) -> Any:
"""Cap a response by JSON-serialized characters, preserving the existing envelope contract."""
return _truncate_response_to_limit(
@@ -1218,4 +1228,5 @@ __all__ = [
"size_capped",
"truncate_response",
"truncate_response_bytes",
+ "within_response_limits",
]
diff --git a/skyvern/cli/mcp_tools/workflow.py b/skyvern/cli/mcp_tools/workflow.py
index 180b4479a..4fa5ab7de 100644
--- a/skyvern/cli/mcp_tools/workflow.py
+++ b/skyvern/cli/mcp_tools/workflow.py
@@ -45,11 +45,13 @@ from ._workflow_http import (
update_workflow_folder_raw,
update_workflow_raw,
)
-from .response import MCP_MAX_RESPONSE_CHARS, _response_size
+from .response import MCP_MAX_RESPONSE_BYTES, MCP_MAX_RESPONSE_CHARS, _response_size, within_response_limits
LOG = structlog.get_logger()
_SUMMARY_TOP_LEVEL_KEY_LIMIT = 8
_SUMMARY_SCALAR_PREVIEW_LIMIT = 3
+_SUMMARY_DOWNLOAD_DETAIL_LIMIT = 20
+_SUMMARY_DOWNLOAD_DETAIL_BUDGET_CHARS = 6_000
_SUMMARY_ARTIFACT_PREVIEW_LIMIT = 4
_SUMMARY_STRING_PREVIEW_LIMIT = 120
_SUMMARY_RECURSION_LIMIT = 10
@@ -309,7 +311,50 @@ def _summarize_output_value(output_value: Any) -> tuple[dict[str, Any], dict[str
_MISSING = object()
-def _summarize_artifacts(run: Any, output_stats: dict[str, Any] | None) -> dict[str, Any]:
+def _download_detail(file_info: Any) -> dict[str, Any]:
+ """Reduce a downloaded file to what makes it retrievable.
+
+ The signed URL is the only field a caller cannot rebuild from the rest of a
+ reduced payload, so a summary that drops it strands the file.
+ """
+ detail = {field: _jsonable(_get_value(file_info, field)) for field in ("url", "checksum", "filename")}
+ return {key: value for key, value in detail.items() if value}
+
+
+def _bounded_download_details(downloaded_files: list[Any]) -> list[dict[str, Any]]:
+ """Keep whole download entries within a fixed share of the response budget.
+
+ Signed URLs are long and a summary is also used per row in run lists, so the
+ list is capped by characters as well as by entry count. ``downloaded_file_count``
+ reports the true total when entries are left out.
+ """
+ details: list[dict[str, Any]] = []
+ spent = 0
+ for file_info in downloaded_files:
+ if len(details) >= _SUMMARY_DOWNLOAD_DETAIL_LIMIT:
+ break
+ detail = _download_detail(file_info)
+ if not detail:
+ continue
+ spent += _response_size(detail)
+ if spent > _SUMMARY_DOWNLOAD_DETAIL_BUDGET_CHARS:
+ break
+ details.append(detail)
+ return details
+
+
+def _summarize_artifacts(
+ run: Any,
+ output_stats: dict[str, Any] | None,
+ *,
+ include_download_details: bool = False,
+) -> dict[str, Any]:
+ """Summarize a run's artifacts.
+
+ ``include_download_details`` belongs to callers that drop ``downloaded_files``
+ from their payload; a caller that keeps the field verbatim would otherwise
+ carry every url and checksum twice.
+ """
missing = _MISSING
downloaded_files_value = _get_value(run, "downloaded_files", missing)
screenshot_urls_value = _get_value(run, "screenshot_urls", missing)
@@ -335,12 +380,26 @@ def _summarize_artifacts(run: Any, output_stats: dict[str, Any] | None) -> dict[
if filenames:
summary["downloaded_file_names"] = filenames[:_SUMMARY_SCALAR_PREVIEW_LIMIT]
+ if include_download_details:
+ file_details = _bounded_download_details(downloaded_files)
+ if file_details:
+ summary["downloaded_files"] = file_details
+
if output_stats and output_stats["artifact_ids_preview"]:
summary["artifact_ids_preview"] = output_stats["artifact_ids_preview"]
return summary
+_DOWNLOAD_REFERENCE_FIELDS = ("downloaded_file_count", "downloaded_files")
+
+
+def _download_reference(artifact_summary: Any) -> dict[str, Any]:
+ if not isinstance(artifact_summary, dict):
+ return {}
+ return {field: artifact_summary[field] for field in _DOWNLOAD_REFERENCE_FIELDS if field in artifact_summary}
+
+
_RUN_LIST_DETAILS_HINT = (
"Run rows do not include recording, screenshot, artifact or output details; an omitted field means "
"unknown, not none. Call skyvern_workflow_status(run_id, verbosity='full') for a run's details."
@@ -361,7 +420,7 @@ def _serialize_run_summary(run: Any) -> dict[str, Any]:
if output_value is _MISSING or output_value is None:
output_value = None if outputs_value is _MISSING else outputs_value
output_summary, output_stats = _summarize_output_value(output_value)
- artifact_summary = _summarize_artifacts(run, output_stats)
+ artifact_summary = _summarize_artifacts(run, output_stats, include_download_details=True)
summary: dict[str, Any] = {
"run_id": run_id,
@@ -2310,7 +2369,7 @@ async def skyvern_workflow_status(
return payload
result = make_result("skyvern_workflow_status", data=_payload_for(verbosity), timing_ms=timer.timing_ms)
- if _response_size(result) <= MCP_MAX_RESPONSE_CHARS:
+ if within_response_limits(result):
return result
# The generic size cap preserves only top-level `ok`/`error`/`*_id`, and this result
@@ -2318,26 +2377,39 @@ async def skyvern_workflow_status(
# caller nothing to act on. Degrade to the compact payload instead; the full output is
# still retrievable over the HTTP API, which is not capped.
oversized_warning = (
- f"Full output exceeded the {MCP_MAX_RESPONSE_CHARS}-char MCP response limit; "
- f"returning a reduced payload. Fetch the full run over the HTTP API at /v1/runs/{run_id}."
+ f"Full output exceeded the MCP response size limit ({MCP_MAX_RESPONSE_CHARS} chars / "
+ f"{MCP_MAX_RESPONSE_BYTES} bytes); returning a reduced payload. "
+ f"Fetch the full run over the HTTP API at /v1/runs/{run_id}."
)
+ summary_data = _payload_for("summary")
summary_result = make_result(
"skyvern_workflow_status",
- data=_payload_for("summary"),
+ data=summary_data,
timing_ms=timer.timing_ms,
warnings=[oversized_warning],
)
- if _response_size(summary_result) <= MCP_MAX_RESPONSE_CHARS:
+ if within_response_limits(summary_result):
return summary_result
# The summary is itself unbounded — it copies fields like failure_reason verbatim. Fall
- # back to the identifiers a poller needs so it never lands in the stripped envelope.
- summary_data = _payload_for("summary")
+ # back to the identifiers a poller needs so it never lands in the stripped envelope,
+ # plus the download references, which are otherwise unreachable over MCP.
minimal: dict[str, Any] = {"run_id": run_id, "status": summary_data.get("status")}
if run_id.startswith("wr_"):
minimal["sdk_equivalent"] = f"await skyvern_workflow_status(run_id={run_id!r}, verbosity='summary')"
else:
minimal["sdk_equivalent"] = f"await skyvern.get_run({run_id!r})"
+
+ downloads = _download_reference(summary_data.get("artifact_summary"))
+ if downloads:
+ with_downloads = make_result(
+ "skyvern_workflow_status",
+ data={**minimal, "artifact_summary": downloads},
+ timing_ms=timer.timing_ms,
+ warnings=[oversized_warning],
+ )
+ if within_response_limits(with_downloads):
+ return with_downloads
return make_result(
"skyvern_workflow_status",
data=minimal,
diff --git a/skyvern/forge/agent.py b/skyvern/forge/agent.py
index 239e192c1..7ab0ac156 100644
--- a/skyvern/forge/agent.py
+++ b/skyvern/forge/agent.py
@@ -93,7 +93,12 @@ from skyvern.exceptions import (
from skyvern.experimentation.wait_utils import get_or_create_wait_config, get_wait_time
from skyvern.forge import app
from skyvern.forge.async_operations import AgentPhase, AsyncOperationPool
-from skyvern.forge.failure_classifier import FailureCategory, classify_from_failure_reason, is_captcha_solve_failure
+from skyvern.forge.failure_classifier import (
+ FailureCategory,
+ bounded_failure_categories,
+ classify_from_failure_reason,
+ is_captcha_solve_failure,
+)
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.aws import get_aws_client
from skyvern.forge.sdk.api.files import (
@@ -727,6 +732,30 @@ def _llm_error_category(reasoning: str) -> list[dict]:
return [{"category": FailureCategory.LLM_ERROR.value, "confidence_float": 0.9, "reasoning": reasoning}]
+def _append_mapped_category_errors(
+ errors: list[UserDefinedError],
+ categories: list[dict[str, Any]],
+ error_code_mapping: dict[str, str] | None,
+ rejected_category_names: list[str],
+) -> None:
+ error_codes = {error.error_code for error in errors}
+ category_names = [category.get("category") for category in categories]
+ category_names.extend(rejected_category_names)
+ for category_name in category_names:
+ if not isinstance(category_name, str) or category_name.upper() in FailureCategory.__members__:
+ continue
+ error_code = resolve_error_code_mapping_key(category_name, error_code_mapping)
+ if error_code and error_code not in error_codes:
+ errors.append(
+ UserDefinedError(
+ error_code=error_code,
+ reasoning=f"{error_code}: {(error_code_mapping or {})[error_code]}",
+ confidence_float=1.0,
+ )
+ )
+ error_codes.add(error_code)
+
+
def _require_actions_payload(json_response: dict[str, Any]) -> list[Any]:
actions_payload = json_response.get("actions")
if not isinstance(actions_payload, list):
@@ -10325,7 +10354,9 @@ class ForgeAgent:
)
if persisted_action.errors:
failure_reason = "; ".join(error.reasoning for error in persisted_action.errors)
- failure_category = persisted_action.failure_categories or classify_from_failure_reason(failure_reason)
+ failure_category = bounded_failure_categories(persisted_action.failure_categories) or (
+ classify_from_failure_reason(failure_reason)
+ )
LOG.info(
"Task failure classified",
task_id=task.task_id,
@@ -10590,11 +10621,9 @@ class ForgeAgent:
)
user_errors = list(failure_response.errors)
- # The summary model sometimes files a mapped code as a failure category instead of an error.
- # A taxonomy name is its own classification, even when a mapping key reuses that name.
- for category in failure_response.failure_categories:
- category_name = category.get("category")
- if not isinstance(category_name, str) or category_name.upper() in FailureCategory.__members__:
+ # Preserve mapped error recovery without treating taxonomy case variants as customer codes.
+ for category_name in failure_response._rejected_failure_categories:
+ if category_name.upper() in FailureCategory.__members__:
continue
error_code = resolve_error_code_mapping_key(category_name, task.error_code_mapping)
if error_code and error_code not in {error.error_code for error in user_errors}:
@@ -10785,6 +10814,12 @@ class ForgeAgent:
system_prompt=task.workflow_system_prompt,
)
response = MaxStepsReasonResponse.model_validate(json_response)
+ _append_mapped_category_errors(
+ response.errors,
+ response.failure_categories,
+ task.error_code_mapping,
+ response._rejected_failure_categories,
+ )
response.errors = self._filter_response_errors(task=task, step=step, errors=response.errors)
return response
except Exception:
@@ -10939,6 +10974,12 @@ class ForgeAgent:
system_prompt=task.workflow_system_prompt,
)
response = MaxStepsReasonResponse.model_validate(json_response)
+ _append_mapped_category_errors(
+ response.errors,
+ response.failure_categories,
+ task.error_code_mapping,
+ response._rejected_failure_categories,
+ )
response.errors = self._filter_response_errors(task=task, step=step, errors=response.errors)
return response
except Exception:
diff --git a/skyvern/forge/failure_classifier.py b/skyvern/forge/failure_classifier.py
index 1c8febe19..f318b668d 100644
--- a/skyvern/forge/failure_classifier.py
+++ b/skyvern/forge/failure_classifier.py
@@ -607,7 +607,48 @@ _MIN_ATTRIBUTION_CONFIDENCE = 0.7
# Shared with typed run emitters; dynamic/user-defined categories remain excluded.
_FAILURE_CATEGORY_LITERALS = frozenset(category.value for category in FailureCategory)
+
+def partition_failure_categories(value: Any) -> tuple[list[dict], list[str]]:
+ """Keep exact taxonomy values; map any non-empty invalid-only response to UNKNOWN."""
+ kept: list[dict] = []
+ rejected: list[str] = []
+ for entry in value if isinstance(value, list) else ():
+ if not isinstance(entry, dict):
+ continue
+ name = entry.get("category")
+ if not isinstance(name, str):
+ continue
+ if name not in _FAILURE_CATEGORY_LITERALS:
+ rejected.append(name)
+ continue
+ kept.append(entry)
+ if isinstance(value, list) and value and not kept:
+ kept = [
+ {
+ "category": FailureCategory.UNKNOWN.value,
+ "confidence_float": 0.5,
+ "reasoning": "Model returned no recognized failure category",
+ }
+ ]
+ return kept, rejected
+
+
+def bounded_failure_categories(value: Any) -> list[dict]:
+ """``value``'s entries narrowed to this taxonomy; see partition_failure_categories."""
+ kept, rejected = partition_failure_categories(value)
+ if rejected:
+ LOG.warning(
+ "Dropped model-written failure categories outside the known taxonomy",
+ rejected_category_count=len(rejected),
+ kept_category_count=len(kept),
+ )
+ return kept
+
+
# Persisted failure_category reason codes are separate from HTTP error codes, even where their values match.
+# Bounded reason codes emitted by out-of-module terminal producers. Defined here as the single
+# source of truth so a producer and this allowlist cannot drift: the browser-lease seam
+# (skyvern/forge/sdk/workflow/service.py::_browser_lease_failure_category) imports these.
BROWSER_SESSION_CLOSED_REASON_CODE = "browser_session_closed"
BROWSER_SESSION_STARTUP_TIMEOUT_REASON_CODE = "browser_session_startup_timeout"
BROWSER_SESSION_EXPIRED_BEFORE_RUN_REASON_CODE = "browser_session_expired_before_run"
diff --git a/skyvern/forge/sdk/db/repositories/organizations.py b/skyvern/forge/sdk/db/repositories/organizations.py
index 28a54381e..939db97bb 100644
--- a/skyvern/forge/sdk/db/repositories/organizations.py
+++ b/skyvern/forge/sdk/db/repositories/organizations.py
@@ -494,6 +494,8 @@ class OrganizationsRepository(BaseRepository):
query = query.filter_by(token=token)
if valid is not None:
query = query.filter_by(valid=valid)
+ # Twin rows can share a token; a fixed order keeps every pod resolving the same row ID.
+ query = query.order_by(OrganizationAuthTokenModel.created_at.desc(), OrganizationAuthTokenModel.id)
if token_obj := (await session.scalars(query)).first():
return await convert_to_organization_auth_token(token_obj, token_type)
else:
diff --git a/skyvern/forge/sdk/workflow/models/web_search_block.py b/skyvern/forge/sdk/workflow/models/web_search_block.py
index 50b393421..6422f0d2c 100644
--- a/skyvern/forge/sdk/workflow/models/web_search_block.py
+++ b/skyvern/forge/sdk/workflow/models/web_search_block.py
@@ -195,6 +195,7 @@ class WebSearchBlock(Block):
merged_mapping: dict[str, str],
mapping_is_page_derived: bool,
failure_reason: str | None,
+ search_served: bool,
workflow_run_id: str,
workflow_run_block_id: str,
organization_id: str | None,
@@ -223,10 +224,13 @@ class WebSearchBlock(Block):
tz_info = run_context.tz_info if run_context and run_context.tz_info else datetime.now().astimezone().tzinfo
search_data: dict[str, Any] = {
"query": response.query,
- "results": response.results,
"prompt": rendered_prompt or None,
"prompt_output": response.prompt_output,
}
+ # A search that was never served has no result list to judge, and offering an empty one
+ # invites a code describing empty results. The failure itself still gets a code.
+ if search_served:
+ search_data["results"] = response.results
if mapping_is_page_derived:
search_data["error_codes"] = merged_mapping
detection_prompt = prompt_engine.load_prompt(
@@ -343,10 +347,12 @@ class WebSearchBlock(Block):
execution_denied = False
prompt_ran = False
+ search_served = False
if failure_reason is None:
try:
async with asyncio.timeout(web_search_client.SEARCH_TIMEOUT_SECONDS):
await web_search_client.search(response, self.provider, self.num_results)
+ search_served = True
except (TimeoutError, WebSearchError) as exc:
reason = str(exc) or (
f"{response.provider.title()} search timed out after "
@@ -355,6 +361,7 @@ class WebSearchBlock(Block):
if response.fallback_reason:
reason += f" Exa ran because {response.fallback_reason.rstrip('.')}."
if response.results:
+ search_served = True
LOG.warning(
"Web search stopped before reaching the requested result count",
provider=response.provider,
@@ -483,6 +490,7 @@ class WebSearchBlock(Block):
merged_mapping,
mapping_is_page_derived,
failure_reason,
+ search_served,
workflow_run_id,
workflow_run_block_id,
organization_id,
diff --git a/skyvern/forge/sdk/workflow/web_search_client.py b/skyvern/forge/sdk/workflow/web_search_client.py
index e2956a519..3c84de644 100644
--- a/skyvern/forge/sdk/workflow/web_search_client.py
+++ b/skyvern/forge/sdk/workflow/web_search_client.py
@@ -20,6 +20,11 @@ LOG = structlog.get_logger()
SearchProvider = Literal["google", "exa"]
ProviderChoice = Literal["auto", "google", "exa"]
SEARCH_TIMEOUT_SECONDS = 180
+REQUEST_TIMEOUT_SECONDS = 30
+# The upstream slow tail is per-request, so each attempt is an independent draw. Three of them fit
+# inside SEARCH_TIMEOUT_SECONDS per provider; pagination and a provider fallback can still reach
+# that ceiling, which stays the hard bound on the whole search.
+REQUEST_ATTEMPTS = 3
def _site_restriction(query: str) -> tuple[str, str] | None:
@@ -110,24 +115,56 @@ def redact_keys(value: Any, secret_values: Collection[str] = ()) -> Any:
return value
+def timeout_message(provider: SearchProvider, attempts: int = REQUEST_ATTEMPTS) -> str:
+ attempt_count = f"{attempts} attempts" if attempts > 1 else "1 attempt"
+ return f"{provider.title()} search timed out after {attempt_count} of {REQUEST_TIMEOUT_SECONDS} seconds."
+
+
+async def _request_with_retries(
+ provider: SearchProvider, url: str, payload: dict[str, Any] | None, attempts: int
+) -> tuple[int, dict[str, Any]]:
+ """Only a timed-out attempt is retried; a served response is returned on any status, for the
+ caller to judge."""
+ attempt = 1
+ while True:
+ try:
+ status, _, body = await aiohttp_request(
+ method="GET" if provider == "google" else "POST",
+ url=url,
+ headers={"x-api-key": settings.EXA_API_KEY or ""} if provider == "exa" else None,
+ json_data=payload,
+ timeout=REQUEST_TIMEOUT_SECONDS,
+ follow_redirects=False,
+ )
+ except TimeoutError:
+ if attempt == attempts:
+ raise TimeoutError(timeout_message(provider, attempts)) from None
+ with contained_effect("log search request retry"):
+ LOG.info(
+ "Search request timed out; retrying",
+ provider=provider,
+ attempt=attempt,
+ attempts=attempts,
+ timeout_seconds=REQUEST_TIMEOUT_SECONDS,
+ )
+ attempt += 1
+ except Exception:
+ raise WebSearchError(f"{provider.title()} search request failed.") from None
+ else:
+ return status, body
+
+
async def request(
- response: SearchResponse, provider: SearchProvider, url: str, payload: dict[str, Any] | None = None
+ response: SearchResponse,
+ provider: SearchProvider,
+ url: str,
+ payload: dict[str, Any] | None = None,
+ attempts: int = REQUEST_ATTEMPTS,
) -> dict[str, Any]:
# SerpAPI authenticates in the URL; the HTTP instrumentor does not redact api_key.
token = attach(set_value(_SUPPRESS_HTTP_INSTRUMENTATION_KEY, True))
try:
- status, _, body = await aiohttp_request(
- method="GET" if provider == "google" else "POST",
- url=url,
- headers={"x-api-key": settings.EXA_API_KEY or ""} if provider == "exa" else None,
- json_data=payload,
- timeout=30,
- follow_redirects=False,
- )
- except TimeoutError:
- raise TimeoutError(f"{provider.title()} search timed out after 30 seconds.") from None
- except Exception:
- raise WebSearchError(f"{provider.title()} search request failed.") from None
+ status, body = await _request_with_retries(provider, url, payload, attempts)
finally:
detach(token)
@@ -211,7 +248,12 @@ async def google_search(response: SearchResponse, num_results: int, max_pages: i
query = urlencode({**params, "no_cache": "true"})
page_status = (response.http_status, response.http_status_provider)
try:
- fresh_body = await request(response, "google", f"https://serpapi.com/search.json?{query}")
+ # One attempt: this page is uncached and the slowest request of the search, and the
+ # page in hand reaches response.results only after this loop, so a retry here risks
+ # the global deadline discarding it.
+ fresh_body = await request(
+ response, "google", f"https://serpapi.com/search.json?{query}", attempts=1
+ )
except Exception: # noqa: BLE001
# The page already in hand is what gets reported, so its status stays the reported one.
response.http_status, response.http_status_provider = page_status
@@ -283,7 +325,8 @@ async def exa_search(response: SearchResponse, num_results: int) -> None:
append_results(response, body.get("results"), num_results)
if not response.results:
return
- # The snippet lookup is best-effort; the reported status stays the search's.
+ # The snippet lookup is best-effort; the reported status stays the search's. It gets one attempt
+ # so retrying it cannot spend the budget the caller still needs to keep the results it holds.
search_status = (response.http_status, response.http_status_provider)
try:
contents = await request(
@@ -295,6 +338,7 @@ async def exa_search(response: SearchResponse, num_results: int) -> None:
"highlights": {"maxCharacters": 1000, "query": payload["query"]},
"maxAgeHours": -1,
},
+ attempts=1,
)
if contents.get("error"):
raise WebSearchError("Exa highlights request did not complete successfully.")
@@ -347,7 +391,7 @@ async def search(response: SearchResponse, provider: ProviderChoice, num_results
except (TimeoutError, WebSearchError) as exc:
if provider != "auto" or response.results or not settings.EXA_API_KEY:
raise
- response.fallback_reason = str(exc) or "Google search timed out after 30 seconds."
+ response.fallback_reason = str(exc) or timeout_message("google")
response.provider = "exa"
response.pages.clear()
await exa_search(response, num_results)
diff --git a/skyvern/services/task_v2_service.py b/skyvern/services/task_v2_service.py
index 89de74d70..d1ac2c0e8 100644
--- a/skyvern/services/task_v2_service.py
+++ b/skyvern/services/task_v2_service.py
@@ -23,7 +23,7 @@ from skyvern.exceptions import (
UrlGenerationFailure,
)
from skyvern.forge import app
-from skyvern.forge.failure_classifier import classify_from_failure_reason
+from skyvern.forge.failure_classifier import bounded_failure_categories, classify_from_failure_reason
from skyvern.forge.prompts import prompt_engine
from skyvern.forge.sdk.api.llm.api_handler_factory import (
LLMAPIHandlerFactory,
@@ -240,7 +240,7 @@ async def _summarize_max_steps_failure_reason(
thought=thought,
system_prompt=task_v2.workflow_system_prompt,
)
- return json_response.get("reasoning", ""), json_response.get("failure_categories")
+ return json_response.get("reasoning", ""), bounded_failure_categories(json_response.get("failure_categories"))
except Exception:
LOG.warning("Failed to summarize the failure reason for task v2", exc_info=True)
return "", None
@@ -1088,7 +1088,7 @@ async def run_task_v2_helper(
user_goal_achieved = task_v2_response.get("user_goal_achieved", False)
should_terminate = task_v2_response.get("should_terminate", False)
termination_reason = task_v2_response.get("termination_reason")
- failure_categories = task_v2_response.get("failure_categories")
+ failure_categories = bounded_failure_categories(task_v2_response.get("failure_categories"))
observation = task_v2_response.get("page_info", "")
thoughts: str = task_v2_response.get("thoughts", "")
plan = task_v2_response.get("plan", "")
@@ -1415,7 +1415,7 @@ async def run_task_v2_helper(
user_goal_achieved = completion_resp.get("user_goal_achieved", False)
should_terminate = completion_resp.get("should_terminate", False)
termination_reason = completion_resp.get("termination_reason")
- completion_failure_categories = completion_resp.get("failure_categories")
+ completion_failure_categories = bounded_failure_categories(completion_resp.get("failure_categories"))
thought_content = completion_resp.get("thoughts", "")
await app.DATABASE.observer.update_thought(
thought_id=thought.observer_thought_id,
diff --git a/skyvern/utils/prompt_engine.py b/skyvern/utils/prompt_engine.py
index f7b1afbbb..6796e7177 100644
--- a/skyvern/utils/prompt_engine.py
+++ b/skyvern/utils/prompt_engine.py
@@ -2,11 +2,12 @@ from collections.abc import Callable
from typing import Any
import structlog
-from pydantic import BaseModel
+from pydantic import BaseModel, PrivateAttr, model_validator
from skyvern.constants import DEFAULT_MAX_TOKENS
from skyvern.errors.errors import UserDefinedError
from skyvern.exceptions import SkyvernContextWindowExceededError
+from skyvern.forge.failure_classifier import partition_failure_categories
from skyvern.forge.sdk.core import skyvern_context
from skyvern.forge.sdk.prompting import PromptEngine
from skyvern.utils.strings import escape_code_fences
@@ -49,10 +50,25 @@ class MaxStepsReasonResponse(BaseModel):
reasoning: str
errors: list[UserDefinedError] = []
failure_categories: list[dict] = []
+ # Model output must not be able to set or invalidate this derived state.
+ _rejected_failure_categories: list[str] = PrivateAttr(default_factory=list)
# Explicit provenance for failure_categories. Set by short-circuit paths
# to avoid the caller inferring "llm" from the presence of categories.
failure_category_source: str | None = None
+ @model_validator(mode="after")
+ def _bound_failure_categories(self) -> "MaxStepsReasonResponse":
+ kept, rejected = partition_failure_categories(self.failure_categories)
+ if rejected:
+ LOG.warning(
+ "Summary model returned failure categories outside the known taxonomy",
+ rejected_category_count=len(rejected),
+ kept_category_count=len(kept),
+ )
+ self.failure_categories = kept
+ self._rejected_failure_categories = rejected
+ return self
+
PROMPT_HARD_CEILING_TOKENS = 180_000
_CEILING_SAFETY_MARGIN_TOKENS = 2_000
diff --git a/tests/unit/test_error_detection_integration.py b/tests/unit/test_error_detection_integration.py
index 6871d0a2d..139ee180a 100644
--- a/tests/unit/test_error_detection_integration.py
+++ b/tests/unit/test_error_detection_integration.py
@@ -11,6 +11,7 @@ import pytest
from skyvern.errors.errors import ReachMaxRetriesError, UserDefinedError
from skyvern.forge.agent import ForgeAgent
+from skyvern.forge.failure_classifier import FailureCategory
from skyvern.forge.sdk.models import StepStatus
from skyvern.forge.sdk.schemas.tasks import TaskStatus
from skyvern.schemas.steps import AgentStepOutput
@@ -397,6 +398,46 @@ async def test_max_retries_without_mapped_code_still_fails_with_reach_max_retrie
assert captured["status"] == TaskStatus.failed
assert [error["error_code"] for error in captured["errors"]] == [ReachMaxRetriesError().error_code]
+ # Whatever is persisted is a category every reader can match, even when the model named none.
+ assert all(entry["category"] in FailureCategory.__members__ for entry in captured["failure_category"])
+
+
+async def test_max_retries_does_not_map_case_variant_of_builtin_category(agent, mock_browser_state):
+ summary = MaxStepsReasonResponse(
+ page_info="",
+ reasoning="The agent could not find the login form",
+ errors=[],
+ failure_categories=[{"category": "element_not_found", "confidence_float": 0.9, "reasoning": "No form"}],
+ )
+
+ captured = await _run_max_retries(agent, mock_browser_state, summary)
+
+ assert captured["status"] == TaskStatus.failed
+ assert [error["error_code"] for error in captured["errors"]] == [ReachMaxRetriesError().error_code]
+ assert captured["failure_category"][0]["category"] == FailureCategory.UNKNOWN
+
+
+async def test_max_retries_does_not_persist_a_failure_category_the_model_invented(agent, mock_browser_state):
+ """SKY-17368: every reader of failure_category matches the literal exactly, so an invented
+ name is a classification nothing can read. Known categories are persisted untouched."""
+ known = {"category": "ELEMENT_NOT_FOUND", "confidence_float": 0.8, "reasoning": "No login form"}
+ summary = MaxStepsReasonResponse.model_validate(
+ {
+ "page_info": "",
+ "reasoning": "The agent could not find the login form",
+ "errors": [],
+ "failure_categories": [
+ {"category": "LOGIN_FORM_MISSING", "confidence_float": 0.9, "reasoning": "Invented by the model"},
+ known,
+ ],
+ }
+ )
+
+ captured = await _run_max_retries(agent, mock_browser_state, summary)
+
+ assert captured["failure_category"] == [known]
@pytest.mark.asyncio
diff --git a/tests/unit/test_failure_classifier.py b/tests/unit/test_failure_classifier.py
index a610cec9a..55bc00fa4 100644
--- a/tests/unit/test_failure_classifier.py
+++ b/tests/unit/test_failure_classifier.py
@@ -22,8 +22,11 @@ from skyvern.forge.failure_classifier import (
FailureCategory,
classify_from_failure_reason,
derive_failure_attribution,
+ partition_failure_categories,
)
from skyvern.forge.sdk.api.llm.exceptions import LLMProviderError as ProviderLLMError
+from skyvern.utils.prompt_engine import MaxStepsReasonResponse
+from skyvern.webeye.actions.actions import CompleteVerifyResult
from skyvern.webeye.scraper.scraper import build_scraping_failed_reason
@@ -1345,3 +1348,67 @@ def test_browser_lease_producer_reason_codes_round_trip_through_attribution() ->
assert derive_failure_attribution(_browser_lease_failure_category(expired, workflow_run))["failure_category"] == (
FailureCategory.BROWSER_SESSION_EXPIRED
)
+
+
+_UNKNOWN_MODEL_CATEGORY = {
+ "category": "UNKNOWN",
+ "confidence_float": 0.5,
+ "reasoning": "Model returned no recognized failure category",
+}
+
+
+ "value, expected",
+ [
+ (
+ [{"category": "AUTH_FAILURE", "confidence_float": 0.9}],
+ ([{"category": "AUTH_FAILURE", "confidence_float": 0.9}], []),
+ ),
+ ([{"category": "auth_failure"}], ([_UNKNOWN_MODEL_CATEGORY], ["auth_failure"])),
+ ([{"category": " ANTI_BOT_DETECTION "}], ([_UNKNOWN_MODEL_CATEGORY], [" ANTI_BOT_DETECTION "])),
+ (
+ [{"category": "SITE_UNDER_MAINTENANCE", "confidence_float": 0.9}],
+ ([_UNKNOWN_MODEL_CATEGORY], ["SITE_UNDER_MAINTENANCE"]),
+ ),
+ # Hostile shapes do not raise or become raw persisted categories.
+ ([{"category": ["UNKNOWN"]}, "UNKNOWN", {"reasoning": "no category at all"}], ([_UNKNOWN_MODEL_CATEGORY], [])),
+ ("UNKNOWN", ([], [])),
+ (None, ([], [])),
+ ],
+)
+def test_partition_failure_categories_keeps_only_known_literals(value: object, expected: tuple) -> None:
+ assert partition_failure_categories(value) == expected
+
+
+def test_model_written_categories_are_bounded_without_changing_verification_input() -> None:
+ """Keep categories raw for the TOTP verdict; only the persisted task category is bounded."""
+ invented = {"category": "SITE_UNDER_MAINTENANCE", "confidence_float": 0.9, "reasoning": "Maintenance page"}
+ known = {"category": "ANTI_BOT_DETECTION", "confidence_float": 0.9, "reasoning": "Challenge page"}
+
+ verdict = CompleteVerifyResult.model_validate(
+ {"status": "terminate", "thoughts": "blocked", "failure_categories": [invented, known]}
+ )
+ summary = MaxStepsReasonResponse.model_validate(
+ {"page_info": "", "reasoning": "blocked", "failure_categories": [invented, known]}
+ )
+
+ assert verdict.failure_categories == [invented, known]
+ assert summary.failure_categories == [known]
+ assert summary._rejected_failure_categories == ["SITE_UNDER_MAINTENANCE"]
+
+
[email protected]("untrusted_rejected_categories", [None, "not a list", ["ELEMENT_NOT_FOUND", 3]])
+def test_summary_ignores_model_provided_rejected_categories(untrusted_rejected_categories: object) -> None:
+ summary = MaxStepsReasonResponse.model_validate(
+ {
+ "page_info": "",
+ "reasoning": "valid model reasoning",
+ "errors": [],
+ "failure_categories": [{"category": "AUTH_FAILURE", "confidence_float": 0.9}],
+ "rejected_failure_categories": untrusted_rejected_categories,
+ }
+ )
+
+ assert summary.reasoning == "valid model reasoning"
+ assert summary.failure_categories == [{"category": "AUTH_FAILURE", "confidence_float": 0.9}]
+ assert summary._rejected_failure_categories == []
diff --git a/tests/unit/test_mcp_concise_responses.py b/tests/unit/test_mcp_concise_responses.py
index 5f9af19a3..5cdd366bd 100644
--- a/tests/unit/test_mcp_concise_responses.py
+++ b/tests/unit/test_mcp_concise_responses.py
@@ -231,13 +231,18 @@ def test_workflow_status_summary_stays_bounded_for_heavy_payload() -> None:
summary_payload = workflow_tools._serialize_run_summary(heavy_run)
assert len(json.dumps(full_payload)) > 20_000
- assert len(json.dumps(summary_payload)) < 2_000
+ # The download URL is the one URL a summary keeps verbatim — it is what makes
+ # the file retrievable — so the bound covers it on top of the URL-free summary.
+ assert len(json.dumps(summary_payload)) < 2_000 + len(f"{long_url}-download")
result = make_result("skyvern_workflow_status", data=summary_payload)
- assert len(json.dumps(result)) < 2_200
+ assert len(json.dumps(result)) < 2_200 + len(f"{long_url}-download")
assert "recording_url" not in result["data"]
assert "output" not in result["data"]
assert result["data"]["artifact_summary"]["artifact_id_count"] == 36
+ assert result["data"]["artifact_summary"]["downloaded_files"] == [
+ {"url": f"{long_url}-download", "filename": "case-export.csv"}
+ ]
def test_workflow_status_summary_shrinks_simple_payload() -> None:
diff --git a/tests/unit/test_mcp_response_workflow.py b/tests/unit/test_mcp_response_workflow.py
index aea5b75c5..28441419d 100644
--- a/tests/unit/test_mcp_response_workflow.py
+++ b/tests/unit/test_mcp_response_workflow.py
@@ -287,6 +287,73 @@ async def test_workflow_status_full_over_cap_keeps_existing_summary_degradation(
assert "_truncated" not in result
+_DOWNLOADED_PDF = {
+ "url": "https://artifacts.skyvern.example/wr_bytes/statement.pdf?signature=abc123",
+ "checksum": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "filename": "statement.pdf",
+ "file_size": 102_400,
+}
+
+
+async def test_workflow_status_full_over_byte_cap_keeps_the_run_and_its_downloads(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Typographic punctuation in extracted prose costs one character and three
+ # UTF-8 bytes, so this run clears the character cap but not the byte cap.
+ run = {
+ "workflow_run_id": "wr_bytes",
+ "workflow_id": "wpid_bytes",
+ "status": "completed",
+ "downloaded_files": [_DOWNLOADED_PDF],
+ "outputs": {"extracted_information": "—" * 60_000},
+ }
+ monkeypatch.setattr(workflow_tools, "get_workflow_run_status", AsyncMock(return_value=run))
+
+ tool = await mcp.get_tool("skyvern_workflow_status")
+ result = await tool.fn(run_id="wr_bytes", verbosity="full")
+
+ assert result["ok"] is True
+ assert "_truncated" not in result
+ assert result["data"]["run_id"] == "wr_bytes"
+ assert result["data"]["status"] == "completed"
+ assert result["data"]["artifact_summary"]["downloaded_files"] == [
+ {
+ "url": _DOWNLOADED_PDF["url"],
+ "checksum": _DOWNLOADED_PDF["checksum"],
+ "filename": _DOWNLOADED_PDF["filename"],
+ }
+ ]
+ assert any("returning a reduced payload" in warning for warning in result["warnings"])
+
+
+async def test_workflow_status_minimal_fallback_still_reaches_the_downloads(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # The summary copies failure_reason verbatim, so an oversize reason drops the
+ # response to the last-resort payload.
+ run = {
+ "workflow_run_id": "wr_minimal",
+ "workflow_id": "wpid_minimal",
+ "status": "failed",
+ "failure_reason": "x" * (MCP_MAX_RESPONSE_CHARS + 1_000),
+ "downloaded_files": [_DOWNLOADED_PDF],
+ }
+ monkeypatch.setattr(workflow_tools, "get_workflow_run_status", AsyncMock(return_value=run))
+
+ tool = await mcp.get_tool("skyvern_workflow_status")
+ result = await tool.fn(run_id="wr_minimal", verbosity="full")
+
+ assert result["ok"] is True
+ assert "_truncated" not in result
+ assert result["data"]["run_id"] == "wr_minimal"
+ assert result["data"]["status"] == "failed"
+ assert result["data"]["artifact_summary"]["downloaded_file_count"] == 1
+ assert result["data"]["artifact_summary"]["downloaded_files"][0]["url"] == _DOWNLOADED_PDF["url"]
+ assert "failure_reason" not in result["data"]
+
+
@pytest.mark.asyncio
async def test_workflow_status_default_does_not_expand_backend_output(monkeypatch: pytest.MonkeyPatch) -> None:
status = AsyncMock(
diff --git a/tests/unit/workflow/test_block_anti_bot_retry_skip.py b/tests/unit/workflow/test_block_anti_bot_retry_skip.py
index 1988fb09d..6e1410645 100644
--- a/tests/unit/workflow/test_block_anti_bot_retry_skip.py
+++ b/tests/unit/workflow/test_block_anti_bot_retry_skip.py
@@ -10,7 +10,7 @@ from skyvern.exceptions import (
ScreenshotTargetClosed,
get_user_facing_exception_message,
)
-from skyvern.forge.failure_classifier import classify_from_failure_reason
+from skyvern.forge.failure_classifier import bounded_failure_categories, classify_from_failure_reason
from skyvern.forge.sdk.schemas.tasks import Task, TaskStatus
from skyvern.forge.sdk.workflow.models.block import _should_skip_retry_on_anti_bot_detection
@@ -59,6 +59,17 @@ class TestPersistedCategory:
)
assert _should_skip_retry_on_anti_bot_detection(task) is False
+ @pytest.mark.parametrize("model_category", ["SITE_BLOCKED_ME", " anti_bot_detection "])
+ def test_unrecognized_model_category_does_not_suppress_captcha_retries(self, model_category: str) -> None:
+ categories = bounded_failure_categories([{"category": model_category}])
+ task = _make_failed_task(
+ failure_category=categories,
+ failure_reason="The page showed a captcha challenge",
+ )
+
+ assert categories[0]["category"] == "UNKNOWN"
+ assert _should_skip_retry_on_anti_bot_detection(task) is False
+
def test_persisted_primary_wins_over_reason_keywords(self) -> None:
"""A persisted non-anti-bot primary must NOT be overridden by reason text.
diff --git a/tests/unit/workflow/test_web_search.py b/tests/unit/workflow/test_web_search.py
index c706398eb..7bc5f965c 100644
--- a/tests/unit/workflow/test_web_search.py
+++ b/tests/unit/workflow/test_web_search.py
@@ -4,7 +4,7 @@ import pytest
from skyvern.forge.sdk.browser_egress_policy import DestinationBlockedError
from skyvern.forge.sdk.settings_manager import SettingsManager
-from skyvern.forge.sdk.workflow import web_search
+from skyvern.forge.sdk.workflow import web_search, web_search_client
from skyvern.forge.sdk.workflow.web_search import search_web
from tests.unit.conftest import SearchApiReply, arm_search_api, serpapi_page
@@ -69,6 +69,60 @@ async def test_a_malformed_body_reports_the_status_it_was_served_with(monkeypatc
assert observed["http_status"] == 200
+ ("timeouts", "error_kind"),
+ [(1, None), (web_search_client.REQUEST_ATTEMPTS, "TimeoutError")],
+ ids=["retry-is-served", "attempts-are-bounded"],
+)
+async def test_a_timed_out_attempt_is_retried_within_a_bounded_number_of_attempts(
+ monkeypatch: pytest.MonkeyPatch, timeouts: int, error_kind: str | None
+) -> None:
+ api = arm_search_api(monkeypatch, *([TimeoutError()] * timeouts), (200, PAGE))
+
+ observed = await search_web("roofing")
+
+ # The page queued behind the timeouts proves the bound: once the attempts are spent it goes unread.
+ assert observed["error_kind"] == error_kind
+ assert len(api.urls) == (timeouts if error_kind else timeouts + 1)
+ assert [result["url"] for result in observed["results"]] == ([] if error_kind else [FIRST, SECOND, THIRD])
+
+
+async def test_a_best_effort_snippet_lookup_is_not_retried_and_keeps_the_results(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Google is rejected outright so the run falls back to Exa, whose search lands and whose
+ # snippet lookup never completes: the results it already holds must survive that.
+ api = arm_search_api(
+ monkeypatch,
+ (401, {"error": "refused"}),
+ (200, {"results": [{"url": FIRST, "title": "Title"}]}),
+ TimeoutError(),
+ exa_key="exa-test-key",
+ )
+
+ observed = await search_web("roofing")
+
+ assert observed["error_kind"] is None
+ assert [result["url"] for result in observed["results"]] == [FIRST]
+ assert [result["snippet"] for result in observed["results"]] == [""]
+ assert len(api.urls) == 3
+
+
+async def test_an_off_site_refetch_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None:
+ # The uncached refetch is the slowest request of a search and the page it would replace is not
+ # in response.results yet, so it gets one attempt rather than spending the global deadline.
+ api = arm_search_api(monkeypatch, (200, PAGE), TimeoutError())
+
+ observed = await search_web("roofing site:elsewhere.example")
+
+ assert observed["error_kind"] is None
+ assert len(api.urls) == 2
+ assert "no_cache" in api.urls[1]
+
+
@pytest.mark.asyncio
async def test_a_request_that_never_completes_reports_no_status(monkeypatch: pytest.MonkeyPatch) -> None:
arm_search_api(monkeypatch, TimeoutError(), exa_key="exa-test-key")
diff --git a/tests/unit/workflow/test_web_search_block.py b/tests/unit/workflow/test_web_search_block.py
index 16e02d9c9..db0d9bdaa 100644
--- a/tests/unit/workflow/test_web_search_block.py
+++ b/tests/unit/workflow/test_web_search_block.py
@@ -117,7 +117,11 @@ async def test_exa_search_keeps_pages_without_cached_content(
payloads: dict[str, dict[str, Any]] = {}
async def request(
- _response: SearchResponse, provider: str, url: str, payload: dict[str, Any] | None = None
+ _response: SearchResponse,
+ provider: str,
+ url: str,
+ payload: dict[str, Any] | None = None,
+ attempts: int = web_search_client.REQUEST_ATTEMPTS,
) -> dict[str, Any]:
assert payload is not None
payloads[url] = payload
@@ -249,7 +253,7 @@ async def test_search_rejects_invalid_schema_before_llm_call(
@pytest.mark.parametrize(
"second_page,num_results,expected_count",
[
- (TimeoutError("Google search timed out after 30 seconds."), 11, 9),
+ (TimeoutError(web_search_client.timeout_message("google")), 11, 9),
(WebSearchError("Google search failed (HTTP 429)."), 11, 9),
*[
(
@@ -340,7 +344,7 @@ async def test_search_first_page_timeout_preserves_fallback(
original, _, handler = search_setup
block = original.model_copy(update={"provider": provider})
request = AsyncMock(
- side_effect=[TimeoutError("Google search timed out after 30 seconds."), PROVIDER_PAGE, PROVIDER_PAGE]
+ side_effect=[TimeoutError(web_search_client.timeout_message("google")), PROVIDER_PAGE, PROVIDER_PAGE]
)
monkeypatch.setattr(web_search_client.settings, "SERPAPI_API_KEY", "test-google-key")
monkeypatch.setattr(web_search_client, "request", request)
@@ -428,6 +432,38 @@ async def test_search_no_results_detects_legacy_code_after_prompt(
assert context.values["search_output"] == output
[email protected]("served", [False, True], ids=["unserved", "served-but-empty"])
+async def test_only_a_served_search_offers_its_result_list_to_error_detection(
+ search_setup: tuple[WebSearchBlock, WorkflowRunContext, AsyncMock],
+ monkeypatch: pytest.MonkeyPatch,
+ served: bool,
+) -> None:
+ block, _, handler = search_setup
+ block.prompt = None
+ block.error_code_mapping = {"SEARCH_UNAVAILABLE": "The search could not be run."}
+ reply = {"reasoning": "Checked.", "errors": []}
+ handler.side_effect = [reply]
+ served_reply: dict[str, Any] = {"results": []}
+ request = (
+ AsyncMock(return_value=served_reply)
+ if served
+ else AsyncMock(side_effect=TimeoutError(web_search_client.timeout_message("exa")))
+ )
+ monkeypatch.setattr(web_search_client, "request", request)
+
+ result = await block.execute("workflow-run-test", "block-run-test", "org-test")
+
+ # Detection still runs on an unserved search so a failure-describing code can match, but it is
+ # not handed a result list that only our own failure emptied.
+ handler.assert_awaited_once()
+ detection_prompt = handler.await_args_list[0].kwargs["prompt"]
+ assert ('"results"' in detection_prompt) is served
+ assert result.status == (BlockStatus.completed if served else BlockStatus.timed_out)
+ if not served:
+ assert result.output_parameter_value["failure_category"][0]["category"] == "INFRASTRUCTURE_ERROR"
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("page", [PROVIDER_PAGE, {"results": []}], ids=["results", "empty-results"])
@pytest.mark.parametrize(