Skip to content

Automate browser based workflows with AI

Close the date-segment press's gaps before its run arm ramps (SKY-17094) (#8701)

pedrohsdb2935ee4e97f3
diff --git a/skyvern/errors/errors.py b/skyvern/errors/errors.py
index ba5c73d83..d10d001ff 100644
--- a/skyvern/errors/errors.py
+++ b/skyvern/errors/errors.py
@@ -1,7 +1,7 @@
from enum import StrEnum
from typing import Literal
-from pydantic import BaseModel, Field, field_validator
+from pydantic import BaseModel, Field, PrivateAttr, field_validator
from skyvern.constants import ERROR_CODE_REASONING_MAX_LENGTH
@@ -16,6 +16,12 @@ class UserDefinedError(BaseModel):
reasoning: str
confidence_float: float = Field(..., ge=0, le=1)
error_type: Literal[ErrorType.USER_DEFINED_ERROR] = ErrorType.USER_DEFINED_ERROR
+ # Private so no LLM or API payload can set it; only SkyvernDefinedError.to_user_defined_error does.
+ _skyvern_defined: bool = PrivateAttr(default=False)
+
+ @property
+ def is_skyvern_defined(self) -> bool:
+ return self._skyvern_defined
@field_validator("reasoning")
@classmethod
@@ -71,7 +77,9 @@ class SkyvernDefinedError(BaseModel):
return f"{self.reasoning}(error_code={self.error_code})"
def to_user_defined_error(self) -> UserDefinedError:
- return UserDefinedError(error_code=self.error_code, reasoning=self.reasoning, confidence_float=1.0)
+ error = UserDefinedError(error_code=self.error_code, reasoning=self.reasoning, confidence_float=1.0)
+ error._skyvern_defined = True
+ return error
class ReachMaxStepsError(SkyvernDefinedError):
diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.py
index b597c2f63..23b2ce948 100644
--- a/skyvern/forge/sdk/copilot/agent.py
+++ b/skyvern/forge/sdk/copilot/agent.py
@@ -139,6 +139,7 @@ from skyvern.forge.sdk.copilot.hooks import FinalReplyRunHooks
from skyvern.forge.sdk.copilot.interruption import INTERRUPTED_TERMINAL_SUPERSEDED_HEADLINE
from skyvern.forge.sdk.copilot.llm_errors import CopilotEmptyCompletionError
from skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error as _is_retriable_llm_error
+from skyvern.forge.sdk.copilot.model_resolver import router_fallback_llm_key
from skyvern.forge.sdk.copilot.model_telemetry import (
CopilotModelStopMetadata,
model_attempt_telemetry_scope,
@@ -4116,7 +4117,7 @@ async def _translate_to_agent_result(
def _fallback_llm_key(config: CopilotConfig, current_llm_key: str) -> str | None:
- fallback_key = config.fallback_llm_key
+ fallback_key = router_fallback_llm_key(current_llm_key) or config.fallback_llm_key
if not fallback_key or fallback_key == current_llm_key:
return None
return fallback_key
diff --git a/skyvern/forge/sdk/copilot/model_resolver.py b/skyvern/forge/sdk/copilot/model_resolver.py
index 86fdbf9dd..a06f39f60 100644
--- a/skyvern/forge/sdk/copilot/model_resolver.py
+++ b/skyvern/forge/sdk/copilot/model_resolver.py
@@ -8,7 +8,8 @@ Known limitations:
reproduced here.
* ``LLMRouterConfig`` (fallback chains) is accepted by resolving the
``main_model_group`` entry as the primary ``LLMConfig`` and passing the
- remaining provider model names through LiteLLM's ``fallbacks`` argument.
+ remaining groups, each with its own credentials, through LiteLLM's
+ ``fallbacks`` argument.
Load-balancing across multiple deployments for the same router group and
Redis-coordinated cooldowns are not applied on the copilot-v2 path. Proper
router support through the Agents SDK model interface is tracked in SKY-9256.
@@ -84,22 +85,47 @@ _DROP_FIELDS = frozenset({"thinking_level"})
_WARNED_DROP_KEYS: set[str] = set()
-def _router_model_name(config: LLMRouterConfig, model_group: str) -> str:
- """Return the concrete provider model for a router group alias."""
- entry = next((m for m in config.model_list if m.model_name == model_group), None)
- if entry is None:
- return model_group
- return str(entry.litellm_params.get("model") or entry.model_name)
+# LiteLLM runs each fallback with the primary's kwargs, so a hop that does not reset these would
+# send the primary's credentials, endpoint and service tier to its own provider.
+_PER_HOP_FIELDS = (
+ "api_key",
+ "api_version",
+ "model_info",
+ "vertex_credentials",
+ "vertex_location",
+ "thinking",
+ "service_tier",
+)
-def _router_fallback_models(config: LLMRouterConfig) -> list[str]:
+def _fallback_groups(config: LLMRouterConfig) -> list[str]:
if not config.fallback_model_group:
return []
if isinstance(config.fallback_model_group, str):
- fallback_groups = [config.fallback_model_group]
- else:
- fallback_groups = list(config.fallback_model_group)
- return [_router_model_name(config, group) for group in fallback_groups]
+ return [config.fallback_model_group]
+ return list(config.fallback_model_group)
+
+
+def _router_fallback_models(config: LLMRouterConfig) -> list[dict[str, Any]]:
+ fallbacks: list[dict[str, Any]] = []
+ for group in _fallback_groups(config):
+ entry = next((m for m in config.model_list if m.model_name == group), None)
+ params = entry.litellm_params if entry is not None else {}
+ hop = {field: params.get(field) for field in _PER_HOP_FIELDS}
+ hop["model"] = str(params.get("model") or group)
+ hop["base_url"] = params.get("api_base")
+ hop["timeout"] = params.get("timeout", settings.LLM_CONFIG_TIMEOUT)
+ fallbacks.append(hop)
+ return fallbacks
+
+
+def router_fallback_llm_key(llm_key: str) -> str | None:
+ """The registered key a router falls back to, when its fallback group is named after one."""
+ if not LLMConfigRegistry.is_registered(llm_key):
+ return None
+ config = LLMConfigRegistry.get_config(llm_key)
+ groups = _fallback_groups(config) if isinstance(config, LLMRouterConfig) else []
+ return groups[0] if groups and LLMConfigRegistry.is_registered(groups[0]) else None
def _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfig:
@@ -109,9 +135,9 @@ def _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfi
full bridge lands (SKY-9256), the copilot-v2 path needs a way to run on
orgs whose configured llm_key resolves to a router. We use the entry whose
``model_name`` matches ``main_model_group``; if none match we fall back to
- ``model_list[0]`` and warn. Fallback groups are converted to concrete
- provider model strings and passed to LiteLLM's plain ``acompletion``
- fallback path via ModelSettings.extra_args.
+ ``model_list[0]`` and warn. Each fallback group becomes a LiteLLM fallback
+ entry carrying its own model, credentials and tier, passed to the plain
+ ``acompletion`` fallback path via ModelSettings.extra_args.
The happy-path resolution is the expected code path on every copilot-v2
call in staging/prod, so it logs at INFO. WARN is reserved for the
@@ -146,7 +172,7 @@ def _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfi
llm_key=llm_key,
main_model_group=config.main_model_group,
selected_model_name=direct_model_name,
- fallback_models=fallback_models,
+ fallback_models=[hop["model"] for hop in fallback_models],
)
return LLMConfig(
diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.py
index 3cfbfb7af..d4b82f15a 100644
--- a/skyvern/forge/sdk/copilot/model_telemetry.py
+++ b/skyvern/forge/sdk/copilot/model_telemetry.py
@@ -106,6 +106,7 @@ def _usage_field(value: Any, *keys: str) -> Any:
class CopilotModelCallTelemetry:
model_call_index: int
response_model: str | None = None
+ served_provider: str | None = None
cache_mode: Literal["implicit", "explicit"] = "implicit"
cache_breakpoint_count: int = 0
cache_stable_prefix_chars: int | None = None
@@ -226,11 +227,17 @@ def _log_model_call_usage(
return
billing_model = telemetry.response_model or model
+ # An in-call fallback can be served by another provider than the one base_url points at.
+ provider_name = (
+ normalize_gen_ai_provider(telemetry.served_provider, billing_model)
+ if telemetry.served_provider
+ else _otel_provider_name(billing_model, base_url)
+ )
emit_copilot_model_usage(
CopilotModelUsageEvent(
request_model=model,
response_model=telemetry.response_model,
- provider_name=_otel_provider_name(billing_model, base_url),
+ provider_name=provider_name,
input_tokens=telemetry.input_tokens,
output_tokens=telemetry.output_tokens,
cache_read_tokens=telemetry.cache_read_tokens,
@@ -271,6 +278,13 @@ def model_call_telemetry_scope(
_current_model_call_telemetry.reset(token)
+def _capture_served_provider(telemetry: CopilotModelCallTelemetry, response: object) -> None:
+ hidden_params = getattr(response, "_hidden_params", None)
+ provider = hidden_params.get("custom_llm_provider") if isinstance(hidden_params, dict) else None
+ if isinstance(provider, str) and provider:
+ telemetry.served_provider = provider
+
+
def _capture_chat_stop_metadata(telemetry: CopilotModelCallTelemetry, response: object) -> None:
"""Capture only typed transport exit facts; no model text reaches logs."""
@@ -338,6 +352,7 @@ class _UsageCapturingStream:
chunk = await self._iterator.__anext__()
if isinstance(chunk.model, str):
self._telemetry.response_model = chunk.model
+ _capture_served_provider(self._telemetry, chunk)
# LiteLLM returns ModelResponseStream at runtime even though its public
# annotation is ChatCompletionChunk. Content chunks omit ``usage``.
usage = getattr(chunk, "usage", None)
@@ -394,8 +409,9 @@ def _is_gemini_model(model: str) -> bool:
def _gemini_on_model_chain(model: str, model_settings: ModelSettings) -> bool:
- fallbacks: list[str] = (model_settings.extra_args or {}).get("fallbacks") or []
- return any(_is_gemini_model(candidate) for candidate in [model, *fallbacks])
+ fallbacks: list[str | dict[str, Any]] = (model_settings.extra_args or {}).get("fallbacks") or []
+ models = [hop["model"] if isinstance(hop, dict) else hop for hop in fallbacks]
+ return any(_is_gemini_model(candidate) for candidate in [model, *models])
def _has_ref_key(value: object) -> bool:
@@ -630,6 +646,7 @@ class CopilotLitellmModel(LitellmModel):
if telemetry is not None:
if isinstance(result.model, str):
telemetry.response_model = result.model
+ _capture_served_provider(telemetry, result)
usage = result.get("usage")
if isinstance(usage, LiteLLMUsage):
_capture_usage(telemetry, usage)
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index e85d44d47..6e24d075c 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -5334,17 +5334,19 @@ _DATE_SEGMENT_GROUP_JS = (
# segment that actually filled.
_DECLARED_ROLE_JS = "el => el.getAttribute('role') || ''"
+# Only these input types have a caret; a number input throws on setSelectionRange but may take End as its maximum.
+_IS_TEXT_BOX_JS = (
+ "el => el.tagName === 'TEXTAREA' || "
+ "(el.tagName === 'INPUT' && ['text', 'search', 'url', 'tel', 'password'].includes(el.type))"
+)
+
_DATE_SEGMENT_READBACK_JS = (
"el => [el.value, el.textContent].filter(v => v != null && String(v).trim() !== '').join('|')"
)
-# A press at a sub-pixel segment's centre reaches the hit and bubbles through every ancestor, so it may aim
-# only when none of those is a control and nothing between the hit and the date group covers another segment.
-_DATE_SEGMENT_AIM_JS = (
- r"""(el) => {
- const CONTROL = """
- + json.dumps(_INTERACTIVE_HIT_SEL + ", label")
- + r""";
+# Where a press at a sub-pixel segment's centre would land: a layer inside the date group whose path up to the
+# group covers no other segment. Whether a control sits anywhere on the press's path is decided separately.
+_DATE_SEGMENT_AIM_JS = r"""(el) => {
const centre = (e) => {
const r = e.getBoundingClientRect();
return [r.left + r.width / 2, r.top + r.height / 2];
@@ -5352,23 +5354,59 @@ _DATE_SEGMENT_AIM_JS = (
const [x, y] = centre(el);
const root = el.getRootNode();
const hit = (root.elementsFromPoint ? root : document).elementsFromPoint(x, y)[0];
- if (!hit) return 'none';
- if (hit === el) return 'self';
+ if (!hit) return { hit: 'none' };
+ if (hit === el || el.contains(hit)) return { hit: 'self' };
const group = el.closest('[data-tv3-dateseg="group"]');
- if (!group || !group.contains(hit) || hit === group) return 'outside';
+ if (!group || !group.contains(hit) || hit === group) return { hit: 'outside' };
const others = Array.from(group.querySelectorAll('[data-tv3-dateseg]'))
.filter((s) => s !== el && s !== group)
.map(centre);
- for (let n = hit; n; n = n.parentElement || (n.getRootNode() && n.getRootNode().host) || null) {
- if (n.matches && n.matches(CONTROL)) return 'control';
- }
for (let n = hit; n && n !== group; n = n.parentElement) {
const box = n.getBoundingClientRect();
if (others.some(([sx, sy]) => sx >= box.left && sx <= box.right && sy >= box.top && sy <= box.bottom)) {
- return 'other';
+ return { hit: 'other' };
}
}
- return 'layer';
+ // CDP hit-tests only whole document pixels, so the press lands on the same whole pixel the check reads. Playwright
+ // measures a click position from the padding box and truncates the point to 0.01px; the 0.005 keeps float error
+ // from truncating onto the pixel before.
+ const docX = Math.round(x + window.scrollX);
+ const docY = Math.round(y + window.scrollY);
+ const box = hit.getBoundingClientRect();
+ const style = getComputedStyle(hit);
+ return {
+ hit: 'layer',
+ el: hit,
+ x: docX - window.scrollX - box.left - parseFloat(style.borderLeftWidth || '0') + 0.005,
+ y: docY - window.scrollY - box.top - parseFloat(style.borderTopWidth || '0') + 0.005,
+ docX,
+ docY,
+ };
+}"""
+
+# The layer carries a per-press token only while its press is checked: CDP and Playwright hold separate remote
+# objects for one node, and the page's main world is the one place both reach.
+_DATE_SEGMENT_LAYER_KEY = "Symbol.for('tv3-dateseg-layer')"
+_DATE_SEGMENT_MARK_LAYER_JS = f"(el, token) => {{ el[{_DATE_SEGMENT_LAYER_KEY}] = token; }}"
+_DATE_SEGMENT_UNMARK_LAYER_JS = (
+ f"(el, token) => {{ if (el[{_DATE_SEGMENT_LAYER_KEY}] === token) delete el[{_DATE_SEGMENT_LAYER_KEY}]; }}"
+)
+
+# Run on the deepest node under the press, which the browser resolves through closed shadow roots too: a press
+# bubbles up its composed path, so it is clear only when that path passes through the layer and holds no control.
+_DATE_SEGMENT_PRESS_PATH_JS = (
+ r"""function (token) {
+ const CONTROL = """
+ + json.dumps(_INTERACTIVE_HIT_SEL + ", label")
+ + r""";
+ let onLayer = false;
+ for (let n = this; n; n = n.assignedSlot || n.parentNode || n.host || null) {
+ if (n.nodeType === 1 && n.matches(CONTROL)) return false;
+ if (n["""
+ + _DATE_SEGMENT_LAYER_KEY
+ + r"""] === token) onLayer = true;
+ }
+ return onLayer;
}"""
)
@@ -13534,8 +13572,16 @@ def build_browser_tools(
held = _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector))
if held:
# A press leaves the caret where it landed, and Backspace deletes only what sits before it.
- if await locator.evaluate(_CARET_TO_END_JS, timeout=2000) == "end_key":
+ caret = await locator.evaluate(_CARET_TO_END_JS, timeout=2000)
+ # End is only a caret key in a text box; an ARIA spinbutton takes it as "set to maximum".
+ if caret == "end_key" and await locator.evaluate(_IS_TEXT_BOX_JS, timeout=2000):
await locator.press("End")
+ elif caret == "end_key":
+ # No caret key here, so Delete clears the digits after the press point.
+ for _ in range(len(held.group(0))):
+ await locator.press("Delete")
+ if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):
+ return
for _ in range(len(held.group(0)) if held else 0):
await locator.press("Backspace")
if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):
@@ -13543,8 +13589,75 @@ def build_browser_tools(
except Exception:
pass
+ # Set while a whole-date write takes the base path, so every step of that write behaves as control.
+ _date_segment_aim_suspended: list[bool] = [False]
+
def _date_segment_aim_on() -> bool:
- return run_arm_enabled(DATE_SEGMENT_AIM_FLAG, settings.TASK_V3_DATE_SEGMENT_AIM)
+ return not _date_segment_aim_suspended[0] and run_arm_enabled(
+ DATE_SEGMENT_AIM_FLAG, settings.TASK_V3_DATE_SEGMENT_AIM
+ )
+
+ async def _press_path_is_clear(layer: Any, x: int, y: int) -> bool:
+ # The browser's own node at a point in main-frame document coordinates, closed shadow roots included; anything
+ # unresolvable refuses, as does a browser that stops answering. CDP finds only nodes inside the current viewport.
+ top = _current_page()
+ token = secrets.token_hex(8)
+ sessions: list[Any] = []
+
+ async def probe() -> bool:
+ await layer.evaluate(_DATE_SEGMENT_MARK_LAYER_JS, token)
+ cdp = await top.context.new_cdp_session(top)
+ sessions.append(cdp)
+ await cdp.send("DOM.getDocument", {"depth": 0})
+ found = await cdp.send("DOM.getNodeForLocation", {"x": x, "y": y})
+ node = await cdp.send("DOM.resolveNode", {"backendNodeId": found["backendNodeId"]})
+ result = await cdp.send(
+ "Runtime.callFunctionOn",
+ {
+ "objectId": node["object"]["objectId"],
+ "functionDeclaration": _DATE_SEGMENT_PRESS_PATH_JS,
+ "arguments": [{"value": token}],
+ "returnByValue": True,
+ },
+ )
+ return result.get("result", {}).get("value") is True
+
+ try:
+ return await asyncio.wait_for(probe(), timeout=3)
+ except Exception:
+ return False
+ finally:
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(layer.evaluate(_DATE_SEGMENT_UNMARK_LAYER_JS, token), timeout=1)
+ for cdp in sessions:
+ with contextlib.suppress(Exception):
+ await asyncio.wait_for(cdp.detach(), timeout=1)
+
+ async def _aim_hit(locator: Any) -> tuple[str, Any, dict[str, float]]:
+ # (kind, the layer's ElementHandle for a pressable layer, its press point). The layer comes back as a handle
+ # rather than a DOM marker, which the next aim in a different shadow root could not clear.
+ try:
+ result = await locator.evaluate_handle(_DATE_SEGMENT_AIM_JS, timeout=2000)
+ except Exception:
+ return "none", None, {}
+ props: dict[str, Any] = {}
+ returned = None
+ try:
+ props = await result.get_properties()
+ kind = await props["hit"].json_value() if "hit" in props else "none"
+ layer = props["el"].as_element() if "el" in props else None
+ at = {key: float(await props[key].json_value()) for key in ("x", "y", "docX", "docY") if key in props}
+ returned = layer
+ except Exception:
+ return "none", None, {}
+ finally:
+ for name, prop in props.items():
+ if name != "el" or returned is None:
+ with contextlib.suppress(Exception):
+ await prop.dispose()
+ with contextlib.suppress(Exception):
+ await result.dispose()
+ return str(kind), returned, at
async def _aim_date_segment(page: Any, selector: str, locator: Any, *, press: bool = True) -> str:
# Some widgets move their section cursor only on a trusted pointer event and reset it when focus
@@ -13557,7 +13670,14 @@ def build_browser_tools(
return "focus"
if box["width"] * box["height"] > 1:
if not press:
- return "click"
+ # Planning only: a normal-size segment counts as clickable only when it is what its centre hits.
+ try:
+ kind, layer, _ = await _aim_hit(locator)
+ if layer is not None:
+ await layer.dispose()
+ return "click" if kind == "self" else "focus"
+ except Exception:
+ return "focus"
try:
await locator.click(timeout=2000)
except Exception:
@@ -13567,15 +13687,38 @@ def build_browser_tools(
# inside the frame cannot see, so a framed segment is only focused.
if not _date_segment_aim_on() or (_acted_realm and page.parent_frame is not None):
return "focus"
+ owner = None
+ handle = None
try:
- hit = await locator.evaluate(_DATE_SEGMENT_AIM_JS, timeout=2000)
+ handle = await locator.element_handle(timeout=2000)
+ owner = await handle.owner_frame() if handle is not None else None
except Exception:
- hit = None
- if hit != "layer":
+ owner = None
+ finally:
+ if handle is not None:
+ with contextlib.suppress(Exception):
+ await handle.dispose()
+ if owner is None or owner != _current_page().main_frame:
return "focus"
- if not press:
+ kind, layer, at = await _aim_hit(locator)
+ try:
+ if kind != "layer" or layer is None:
+ return "focus"
+ if not await _press_path_is_clear(layer, int(at["docX"]), int(at["docY"])):
+ return "focus"
+ if not press:
+ return "mirror"
+ # Pressed through Playwright as the layer ELEMENT, never forced: its actionability and hit-target checks
+ # refuse the press when the layer has been hidden or covered since the aim measured it.
+ try:
+ await layer.click(position={"x": float(at["x"]), "y": float(at["y"])}, timeout=1500)
+ except Exception:
+ return "focus"
return "mirror"
- return "mirror" if await _click_at_box_centre(page, selector) else "focus"
+ finally:
+ if layer is not None:
+ with contextlib.suppress(Exception):
+ await layer.dispose()
async def _type_one_date_segment(page: Any, selector: str, digits: str) -> tuple[bool, str]:
locator = page.locator(selector).first
@@ -13640,22 +13783,30 @@ def build_browser_tools(
async def _fill_date_segment_group(
page: Any, selector: str, components: dict[str, str], order: list[str]
) -> ToolResult:
- written: list[tuple[str, str, str]] = []
arm = _date_segment_aim_on()
- write_order = list(_DATE_SEGMENT_ORDER)
+ aimed = arm
if arm:
- # A segment that can only be focused types wherever the widget's cursor is, which a widget
- # resets to its first segment, so without a press on every segment the group is written in
- # its own order.
+ # No write order serves a group mixing pressable and focus-only segments, so such a group takes the
+ # base path, exactly as control does.
for label in order:
segment_selector = f'[data-tv3-dateseg="{label}"]'
- aimable = await _aim_date_segment(
+ planned = await _aim_date_segment(
page, segment_selector, page.locator(segment_selector).first, press=False
)
- if aimable == "focus":
- write_order = list(order)
+ if planned == "focus":
+ aimed = False
break
- for label in write_order:
+ _date_segment_aim_suspended[0] = arm and not aimed
+ try:
+ return await _write_date_segment_group(page, selector, components, order, arm=arm, aimed=aimed)
+ finally:
+ _date_segment_aim_suspended[0] = False
+
+ async def _write_date_segment_group(
+ page: Any, selector: str, components: dict[str, str], order: list[str], *, arm: bool, aimed: bool
+ ) -> ToolResult:
+ written: list[tuple[str, str, str]] = []
+ for label in _DATE_SEGMENT_ORDER:
segment_selector = f'[data-tv3-dateseg="{label}"]'
committed, aim = await _type_one_date_segment(page, segment_selector, components[label])
LOG.info(
@@ -13666,6 +13817,7 @@ def build_browser_tools(
aim=aim,
group_shape=_date_group_shape(order),
date_segment_aim_arm=arm,
+ date_segment_group_aimed=aimed,
)
if not committed:
return ToolResult.error(
diff --git a/skyvern/services/browser_recording/evidence.py b/skyvern/services/browser_recording/evidence.py
index e1c8f3ef4..81746f29e 100644
--- a/skyvern/services/browser_recording/evidence.py
+++ b/skyvern/services/browser_recording/evidence.py
@@ -252,6 +252,25 @@ def _label_kept_actions(
return kept, deleted_action_ids
+def recorded_credential_urls(
+ actions: list[Action],
+ draft_steps: list[RecordingDraftStep] | None,
+) -> list[tuple[str, str]]:
+ """Every (credential_id, url) the evidence packet would report, including actions past `MAX_EVIDENCE_CHARS`."""
+ ordered_actions = sorted(actions, key=lambda action: action.timestamp_start)
+ input_values = list(
+ dict.fromkeys(action.input_value for action in ordered_actions if isinstance(action, ActionInputText))
+ )
+ return [
+ (draft.credential_id, _redact_input_values(action.url, input_values) or "")
+ for action, draft in transfer_focus_click_credentials(apply_draft_overlay(ordered_actions, draft_steps))
+ if isinstance(action, ActionInputText)
+ and draft is not None
+ and draft.credential_id
+ and _credential_kind(action, draft) is not None
+ ]
+
+
def build_recording_evidence(
actions: list[Action],
draft_steps: list[RecordingDraftStep] | None,
diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.py
index 87b2e686c..901648527 100644
--- a/skyvern/services/browser_recording/service.py
+++ b/skyvern/services/browser_recording/service.py
@@ -26,7 +26,11 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
from skyvern.forge.sdk.browser_action_policy import canonicalize_origin
from skyvern.forge.sdk.copilot.credential_resolution import safe_admitted_url
from skyvern.services.browser_recording.code_first import actions_to_code_first_blocks
-from skyvern.services.browser_recording.evidence import RecordingEvidencePacket, build_recording_evidence
+from skyvern.services.browser_recording.evidence import (
+ RecordingEvidencePacket,
+ build_recording_evidence,
+ recorded_credential_urls,
+)
from skyvern.services.browser_recording.redact import is_secret_field, redact_console_event, texts_are_labels
from skyvern.services.browser_recording.types import (
Action,
@@ -707,7 +711,7 @@ class Processor:
"""
Process the compressed browser session recording into workflow definition blocks.
"""
- blocks, parameters, _, _, evidence = await self.process_with_evidence(
+ blocks, parameters, _, _, evidence, _ = await self.process_with_evidence(
compressed_chunks,
draft_steps=draft_steps,
recorded_actions=recorded_actions,
@@ -727,6 +731,7 @@ class Processor:
list[dict[str, t.Any]],
dict[str, t.Any],
RecordingEvidencePacket,
+ list[tuple[str, str]],
]:
if recorded_actions is None:
events = self.compressed_chunks_to_events(compressed_chunks)
@@ -770,6 +775,7 @@ class Processor:
interpretation_session_id=self.interpretation_session_id,
),
refinement_evidence,
+ recorded_credential_urls(actions, draft_steps),
)
@@ -802,7 +808,14 @@ class BrowserSessionRecordingService:
interpretation_session_id=interpretation_session_id,
)
- blocks, parameters, evidence, metadata, refinement_evidence = await processor.process_with_evidence(
+ (
+ blocks,
+ parameters,
+ evidence,
+ metadata,
+ refinement_evidence,
+ recorded_credentials,
+ ) = await processor.process_with_evidence(
compressed_chunks,
draft_steps=draft_steps,
recorded_actions=recorded_actions,
@@ -812,10 +825,9 @@ class BrowserSessionRecordingService:
return blocks, parameters, None, refinement_evidence
credential_urls: dict[str, str] = {}
- for action in refinement_evidence.actions:
- credential_id = action.credential.credential_id if action.credential is not None else None
- admitted_url = safe_admitted_url(action.url)
- if credential_id and admitted_url and canonicalize_origin(admitted_url) is not None:
+ for credential_id, url in recorded_credentials:
+ admitted_url = safe_admitted_url(url)
+ if admitted_url and canonicalize_origin(admitted_url) is not None:
credential_urls.setdefault(credential_id, admitted_url)
if credential_urls:
owned_credentials = await app.DATABASE.credentials.get_credentials_by_ids(
diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.py
index f8e284e1d..7dbd16429 100644
--- a/skyvern/webeye/actions/handler.py
+++ b/skyvern/webeye/actions/handler.py
@@ -10671,8 +10671,11 @@ async def handle_terminate_action(
step: Step,
) -> list[ActionResult]:
if task.error_code_mapping:
+ # A code Skyvern attached itself (OTP_TIMEOUT once TOTP polling ran out) records something the page cannot
+ # show, so the screenshot-based extraction must not drop it, even when the mapping does not declare it.
+ skyvern_errors = [error for error in action.errors if error.is_skyvern_defined]
try:
- action.errors = await extract_user_defined_errors(
+ extracted_errors = await extract_user_defined_errors(
task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning
)
except Exception:
@@ -10683,6 +10686,11 @@ async def handle_terminate_action(
action_errors=action.errors,
exc_info=True,
)
+ else:
+ skyvern_codes = {error.error_code for error in skyvern_errors}
+ action.errors = skyvern_errors + [
+ error for error in extracted_errors if error.error_code not in skyvern_codes
+ ]
return [ActionSuccess()]
diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.py
index f2b3f0060..77776ca6d 100644
--- a/tests/unit/test_copilot_agent_helpers.py
+++ b/tests/unit/test_copilot_agent_helpers.py
@@ -7035,6 +7035,38 @@ class TestCopilotConfig:
assert agent_module._fallback_llm_key(CopilotConfig(fallback_llm_key="PRIMARY"), "PRIMARY") is None
assert agent_module._fallback_llm_key(CopilotConfig(fallback_llm_key="SECONDARY"), "PRIMARY") == "SECONDARY"
+ def test_fallback_key_follows_a_router_whose_fallback_group_is_a_registered_key(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry
+ from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig
+
+ def router(fallback_group: str) -> LLMRouterConfig:
+ return LLMRouterConfig(
+ model_name="router",
+ model_list=[
+ LLMRouterModelConfig(model_name="AZURE_TERRA", litellm_params={"model": "azure/gpt-5.6-terra"}),
+ LLMRouterModelConfig(model_name=fallback_group, litellm_params={"model": "gpt-5.6-terra"}),
+ ],
+ required_env_vars=[],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ main_model_group="AZURE_TERRA",
+ fallback_model_group=fallback_group,
+ )
+
+ monkeypatch.setitem(
+ LLMConfigRegistry._configs, # type: ignore[attr-defined]
+ "OPENAI_TERRA",
+ LLMConfig("gpt-5.6-terra", [], supports_vision=True, add_assistant_prefix=False),
+ )
+ monkeypatch.setitem(LLMConfigRegistry._configs, "TERRA_ROUTER", router("OPENAI_TERRA")) # type: ignore[attr-defined]
+ monkeypatch.setitem(LLMConfigRegistry._configs, "ALIAS_ROUTER", router("openai-terra-fallback")) # type: ignore[attr-defined]
+ config = CopilotConfig(fallback_llm_key="SECONDARY")
+
+ assert agent_module._fallback_llm_key(config, "TERRA_ROUTER") == "OPENAI_TERRA"
+ assert agent_module._fallback_llm_key(config, "ALIAS_ROUTER") == "SECONDARY"
+
@pytest.mark.asyncio
@pytest.mark.parametrize(
("fallback_llm_key", "reply_llm_key", "attempt"),
diff --git a/tests/unit/test_copilot_model_resolver.py b/tests/unit/test_copilot_model_resolver.py
index 8a67e90be..ef39d07e5 100644
--- a/tests/unit/test_copilot_model_resolver.py
+++ b/tests/unit/test_copilot_model_resolver.py
@@ -104,11 +104,77 @@ class TestModelResolver:
assert run_config.model_settings.max_tokens == 8192
assert run_config.model_settings.extra_args is not None
assert run_config.model_settings.extra_args["timeout"] == 900.0
- assert run_config.model_settings.extra_args["fallbacks"] == [
+ assert [hop["model"] for hop in run_config.model_settings.extra_args["fallbacks"]] == [
"azure/gpt-4-1-mini",
"anthropic/claude-sonnet-4-20250514",
]
+ @pytest.mark.asyncio
+ async def test_router_fallback_hop_is_sent_with_its_own_credentials(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ import litellm
+
+ from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfig
+
+ router_config = LLMRouterConfig(
+ model_name="azure-terra-with-openai-terra",
+ model_list=[
+ LLMRouterModelConfig(
+ model_name="AZURE_TERRA",
+ litellm_params={
+ "model": "azure/gpt-5.6-terra",
+ "api_base": "https://azure.example.test",
+ "api_key": "azure-key",
+ "api_version": "2025-04-01-preview",
+ "model_info": {"model_name": "azure/gpt-5.6-terra"},
+ },
+ ),
+ LLMRouterModelConfig(model_name="OPENAI_TERRA", litellm_params={"model": "gpt-5.6-terra"}),
+ ],
+ required_env_vars=[],
+ supports_vision=True,
+ add_assistant_prefix=False,
+ main_model_group="AZURE_TERRA",
+ fallback_model_group="OPENAI_TERRA",
+ )
+ handler = _install_config(monkeypatch, router_config, "AZURE_TERRA_WITH_OPENAI_FALLBACK")
+ model_name, run_config, _, _ = resolve_model_config(handler)
+ model = run_config.model_provider.get_model(model_name)
+ assert run_config.model_settings is not None
+
+ real_acompletion = litellm.acompletion
+ hops: list[dict[str, Any]] = []
+
+ async def primary_fails(**kwargs: Any) -> Any:
+ if "fallbacks" in kwargs:
+ return await real_acompletion(**kwargs)
+ hops.append(kwargs)
+ if len(hops) == 1:
+ raise litellm.RateLimitError(message="429", llm_provider="azure", model=kwargs["model"])
+ return "served"
+
+ monkeypatch.setattr(litellm, "acompletion", primary_fails)
+ served = await litellm.acompletion(
+ model=model_name,
+ messages=[{"role": "user", "content": "hi"}],
+ api_key=model.api_key,
+ base_url=model.base_url,
+ **run_config.model_settings.extra_args,
+ )
+
+ assert served == "served"
+ assert [
+ (hop["model"], hop["api_key"], hop["base_url"], hop["api_version"], hop.get("model_info")) for hop in hops
+ ] == [
+ (
+ "azure/gpt-5.6-terra",
+ "azure-key",
+ "https://azure.example.test",
+ "2025-04-01-preview",
+ {"model_name": "azure/gpt-5.6-terra"},
+ ),
+ ("gpt-5.6-terra", None, None, None, None),
+ ]
+
def test_router_config_no_main_group_match_falls_back_to_first_entry(self, monkeypatch: pytest.MonkeyPatch) -> None:
from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfig
diff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.py
index 0834a484a..38f7febc2 100644
--- a/tests/unit/test_copilot_model_telemetry.py
+++ b/tests/unit/test_copilot_model_telemetry.py
@@ -745,6 +745,32 @@ def test_datadog_usage_attributes_fallback_spend_to_response_model(
assert events[0]["gen_ai.provider.name"] == "anthropic"
+async def test_datadog_usage_names_the_provider_that_served_an_in_call_fallback(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ events: list[dict[str, Any]] = []
+ monkeypatch.setattr(model_telemetry_module.LOG, "info", lambda _event, **fields: events.append(fields))
+ monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: None)
+ chunk = ModelResponseStream(model="gpt-5.6-terra")
+ chunk._hidden_params = {"custom_llm_provider": "openai"}
+
+ async def served_by_openai() -> AsyncIterator[ModelResponseStream]:
+ yield chunk
+
+ with model_call_telemetry_scope(
+ 6,
+ model="azure/gpt-5.6-terra",
+ base_url="https://example.openai.azure.com",
+ ) as telemetry:
+ await anext(model_telemetry_module._UsageCapturingStream(served_by_openai(), telemetry)) # type: ignore[arg-type]
+ telemetry.input_tokens = 100
+ telemetry.output_tokens = 5
+
+ assert events[0]["gen_ai.request.model"] == "azure/gpt-5.6-terra"
+ assert events[0]["gen_ai.provider.name"] == "openai"
+
+
def test_model_call_without_provider_usage_does_not_emit_datadog_event(
monkeypatch: pytest.MonkeyPatch,
) -> None:
@@ -1410,7 +1436,7 @@ def _json_keys(value: object) -> set[str]:
("code", _GEMINI, None, False),
("code", "openai/gpt-5.6", {"fallbacks": [_GEMINI]}, True),
("for_loop", _GEMINI, None, True),
- ("for_loop", "azure/gpt-5.6-sol", {"fallbacks": [_GEMINI]}, False),
+ ("for_loop", "azure/gpt-5.6-sol", {"fallbacks": [{"model": _GEMINI, "api_key": None}]}, False),
("escaped_ref_key", _GEMINI, None, False),
],
)
diff --git a/tests/unit/test_handle_terminate_action.py b/tests/unit/test_handle_terminate_action.py
index f28fa2c6a..b272113f5 100644
--- a/tests/unit/test_handle_terminate_action.py
+++ b/tests/unit/test_handle_terminate_action.py
@@ -1,18 +1,18 @@
-"""Tests for handle_terminate_action error handling.
-
-Verifies that when extract_user_defined_errors fails (e.g., CDP disconnection),
-the original error codes from the action reasoning are preserved instead of being lost.
-"""
+"""Tests for which error codes a terminate action keeps when the task has an error_code_mapping."""
+from collections.abc import Iterator
+from contextlib import contextmanager
from datetime import timezone
from types import SimpleNamespace
+from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from skyvern.errors.errors import UserDefinedError
+from skyvern.errors.errors import TimeoutGetTOTPVerificationCodeError, UserDefinedError
from skyvern.webeye.actions.actions import TerminateAction
from skyvern.webeye.actions.handler import extract_user_defined_errors, handle_terminate_action
+from skyvern.webeye.actions.models import DetailedAgentStepOutput
from skyvern.webeye.actions.responses import ActionSuccess
@@ -29,19 +29,21 @@ def _make_step() -> MagicMock:
return step
-async def _extract_with_empty_llm(
- task: MagicMock, step: MagicMock, reasoning: str, monkeypatch: pytest.MonkeyPatch
-) -> list[UserDefinedError]:
+def _make_scraped_page() -> MagicMock:
scraped_page = MagicMock()
refreshed_page = MagicMock()
refreshed_page.build_element_tree.return_value = ""
refreshed_page.url = "chrome-error://chromewebdata/"
refreshed_page.screenshots = []
scraped_page.refresh = AsyncMock(return_value=refreshed_page)
+ return scraped_page
+
+@contextmanager
+def _surface_errors_llm_returns(monkeypatch: pytest.MonkeyPatch, errors: list[dict[str, Any]]) -> Iterator[None]:
monkeypatch.setattr(
"skyvern.webeye.actions.handler.app.EXTRACTION_LLM_API_HANDLER",
- AsyncMock(return_value={"errors": []}),
+ AsyncMock(return_value={"errors": errors}),
raising=False,
)
with (
@@ -55,7 +57,102 @@ async def _extract_with_empty_llm(
return_value=SimpleNamespace(tz_info=timezone.utc),
),
):
- return await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page, reasoning=reasoning)
+ yield
+
+
+async def _extract_with_empty_llm(
+ task: MagicMock, step: MagicMock, reasoning: str, monkeypatch: pytest.MonkeyPatch
+) -> list[UserDefinedError]:
+ with _surface_errors_llm_returns(monkeypatch, []):
+ return await extract_user_defined_errors(
+ task=task, step=step, scraped_page=_make_scraped_page(), reasoning=reasoning
+ )
+
+
+ ("error_code_mapping", "llm_errors", "expected_codes"),
+ [
+ pytest.param(
+ {"LOGIN_FAILED": "the login did not complete"},
+ [],
+ ["OTP_TIMEOUT"],
+ id="mapping_without_otp_timeout_llm_returns_nothing",
+ ),
+ pytest.param(
+ {"OTP_TIMEOUT": "the 2FA code never arrived", "LOGIN_FAILED": "the login did not complete"},
+ [{"error_code": "LOGIN_FAILED", "reasoning": "Still on the login page", "confidence_float": 0.8}],
+ ["OTP_TIMEOUT", "LOGIN_FAILED"],
+ id="mapping_declares_otp_timeout_llm_omits_it",
+ ),
+ pytest.param(
+ {"OTP_TIMEOUT": "the 2FA code never arrived", "LOGIN_FAILED": "the login did not complete"},
+ [
+ {"error_code": "OTP_TIMEOUT", "reasoning": "No code arrived", "confidence_float": 0.9},
+ {"error_code": "LOGIN_FAILED", "reasoning": "Still on the login page", "confidence_float": 0.8},
+ ],
+ ["OTP_TIMEOUT", "LOGIN_FAILED"],
+ id="mapping_declares_otp_timeout_llm_repeats_it",
+ ),
+ ],
+)
+async def test_totp_timeout_code_reaches_step_errors_with_error_code_mapping(
+ monkeypatch: pytest.MonkeyPatch,
+ error_code_mapping: dict[str, str],
+ llm_errors: list[dict[str, Any]],
+ expected_codes: list[str],
+) -> None:
+ task = _make_task(error_code_mapping=error_code_mapping)
+ task.navigation_goal = "Log in"
+ task.navigation_payload = {}
+ action = TerminateAction(
+ reasoning="No TOTP verification code found. Going to terminate. Polled source: totp_verification_url=x.",
+ errors=[TimeoutGetTOTPVerificationCodeError().to_user_defined_error()],
+ )
+
+ with _surface_errors_llm_returns(monkeypatch, llm_errors):
+ results = await handle_terminate_action(
+ action=action, page=MagicMock(), scraped_page=_make_scraped_page(), task=task, step=_make_step()
+ )
+
+ step_output = DetailedAgentStepOutput(
+ scraped_page=None,
+ extract_action_prompt=None,
+ llm_response=None,
+ actions=[action],
+ action_results=results,
+ actions_and_results=[(action, results)],
+ )
+ assert [error.error_code for error in step_output.extract_errors()] == expected_codes
+
+
+ "llm_error_code",
+ [
+ pytest.param("LLM_REASONING_ERROR", id="code_not_in_mapping"),
+ pytest.param("OTP_TIMEOUT", id="code_named_like_a_skyvern_code"),
+ ],
+)
+async def test_terminate_keeps_codes_the_llm_put_on_the_action_only_if_extraction_agrees(
+ monkeypatch: pytest.MonkeyPatch, llm_error_code: str
+) -> None:
+ task = _make_task(
+ error_code_mapping={"OTP_TIMEOUT": "the 2FA code never arrived", "LOGIN_FAILED": "the login did not complete"}
+ )
+ task.navigation_goal = "Log in"
+ task.navigation_payload = {}
+ action = TerminateAction(
+ reasoning="The page did not load",
+ errors=[UserDefinedError(error_code=llm_error_code, reasoning="Page is blank", confidence_float=0.8)],
+ )
+
+ with _surface_errors_llm_returns(monkeypatch, []):
+ await handle_terminate_action(
+ action=action, page=MagicMock(), scraped_page=_make_scraped_page(), task=task, step=_make_step()
+ )
+
+ assert action.errors == []
@pytest.mark.asyncio
@@ -85,33 +182,6 @@ async def test_terminate_preserves_errors_when_extract_fails() -> None:
assert action.errors[0].error_code == "OTP_TIMEOUT"
-async def test_terminate_uses_extracted_errors_on_success() -> None:
- """When extract_user_defined_errors succeeds, its result should replace action.errors."""
- task = _make_task(error_code_mapping={"OTP_TIMEOUT": "OTP verification code not received"})
- step = _make_step()
- page = MagicMock()
- scraped_page = MagicMock()
-
- action = TerminateAction(reasoning="No TOTP verification code found")
-
- extracted_errors = [
- UserDefinedError(error_code="OTP_TIMEOUT", reasoning="Page shows OTP not received", confidence_float=0.99)
- ]
- with patch(
- "skyvern.webeye.actions.handler.extract_user_defined_errors",
- new_callable=AsyncMock,
- return_value=extracted_errors,
- ):
- results = await handle_terminate_action(
- action=action, page=page, scraped_page=scraped_page, task=task, step=step
- )
-
- assert len(results) == 1
- assert isinstance(results[0], ActionSuccess)
- assert action.errors == extracted_errors
-
-
@pytest.mark.asyncio
async def test_extract_user_defined_errors_falls_back_to_reasoning_match(monkeypatch: pytest.MonkeyPatch) -> None:
task = _make_task(error_code_mapping={"portal_inaccessible": "portal is inaccessible"})