Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Close the date-segment press's gaps before its run arm ramps (SKY-17094) (#8701)
pedrohsdb2935ee4e97f3
diff --git a/skyvern/errors/errors.py b/skyvern/errors/errors.pyindex ba5c73d83..d10d001ff 100644--- a/skyvern/errors/errors.py+++ b/skyvern/errors/errors.py@@ -1,7 +1,7 @@from enum import StrEnumfrom typing import Literal-from pydantic import BaseModel, Field, field_validator+from pydantic import BaseModel, Field, PrivateAttr, field_validatorfrom skyvern.constants import ERROR_CODE_REASONING_MAX_LENGTH@@ -16,6 +16,12 @@ class UserDefinedError(BaseModel):reasoning: strconfidence_float: float = Field(..., ge=0, le=1)error_type: Literal[ErrorType.USER_DEFINED_ERROR] = ErrorType.USER_DEFINED_ERROR+ # Private so no LLM or API payload can set it; only SkyvernDefinedError.to_user_defined_error does.+ _skyvern_defined: bool = PrivateAttr(default=False)++ @property+ def is_skyvern_defined(self) -> bool:+ return self._skyvern_defined@field_validator("reasoning")@classmethod@@ -71,7 +77,9 @@ class SkyvernDefinedError(BaseModel):return f"{self.reasoning}(error_code={self.error_code})"def to_user_defined_error(self) -> UserDefinedError:- return UserDefinedError(error_code=self.error_code, reasoning=self.reasoning, confidence_float=1.0)+ error = UserDefinedError(error_code=self.error_code, reasoning=self.reasoning, confidence_float=1.0)+ error._skyvern_defined = True+ return errorclass ReachMaxStepsError(SkyvernDefinedError):diff --git a/skyvern/forge/sdk/copilot/agent.py b/skyvern/forge/sdk/copilot/agent.pyindex b597c2f63..23b2ce948 100644--- a/skyvern/forge/sdk/copilot/agent.py+++ b/skyvern/forge/sdk/copilot/agent.py@@ -139,6 +139,7 @@ from skyvern.forge.sdk.copilot.hooks import FinalReplyRunHooksfrom skyvern.forge.sdk.copilot.interruption import INTERRUPTED_TERMINAL_SUPERSEDED_HEADLINEfrom skyvern.forge.sdk.copilot.llm_errors import CopilotEmptyCompletionErrorfrom skyvern.forge.sdk.copilot.llm_errors import is_retriable_llm_error as _is_retriable_llm_error+from skyvern.forge.sdk.copilot.model_resolver import router_fallback_llm_keyfrom skyvern.forge.sdk.copilot.model_telemetry import (CopilotModelStopMetadata,model_attempt_telemetry_scope,@@ -4116,7 +4117,7 @@ async def _translate_to_agent_result(def _fallback_llm_key(config: CopilotConfig, current_llm_key: str) -> str | None:- fallback_key = config.fallback_llm_key+ fallback_key = router_fallback_llm_key(current_llm_key) or config.fallback_llm_keyif not fallback_key or fallback_key == current_llm_key:return Nonereturn fallback_keydiff --git a/skyvern/forge/sdk/copilot/model_resolver.py b/skyvern/forge/sdk/copilot/model_resolver.pyindex 86fdbf9dd..a06f39f60 100644--- a/skyvern/forge/sdk/copilot/model_resolver.py+++ b/skyvern/forge/sdk/copilot/model_resolver.py@@ -8,7 +8,8 @@ Known limitations:reproduced here.* ``LLMRouterConfig`` (fallback chains) is accepted by resolving the``main_model_group`` entry as the primary ``LLMConfig`` and passing the- remaining provider model names through LiteLLM's ``fallbacks`` argument.+ remaining groups, each with its own credentials, through LiteLLM's+ ``fallbacks`` argument.Load-balancing across multiple deployments for the same router group andRedis-coordinated cooldowns are not applied on the copilot-v2 path. Properrouter support through the Agents SDK model interface is tracked in SKY-9256.@@ -84,22 +85,47 @@ _DROP_FIELDS = frozenset({"thinking_level"})_WARNED_DROP_KEYS: set[str] = set()-def _router_model_name(config: LLMRouterConfig, model_group: str) -> str:- """Return the concrete provider model for a router group alias."""- entry = next((m for m in config.model_list if m.model_name == model_group), None)- if entry is None:- return model_group- return str(entry.litellm_params.get("model") or entry.model_name)+# LiteLLM runs each fallback with the primary's kwargs, so a hop that does not reset these would+# send the primary's credentials, endpoint and service tier to its own provider.+_PER_HOP_FIELDS = (+ "api_key",+ "api_version",+ "model_info",+ "vertex_credentials",+ "vertex_location",+ "thinking",+ "service_tier",+)-def _router_fallback_models(config: LLMRouterConfig) -> list[str]:+def _fallback_groups(config: LLMRouterConfig) -> list[str]:if not config.fallback_model_group:return []if isinstance(config.fallback_model_group, str):- fallback_groups = [config.fallback_model_group]- else:- fallback_groups = list(config.fallback_model_group)- return [_router_model_name(config, group) for group in fallback_groups]+ return [config.fallback_model_group]+ return list(config.fallback_model_group)+++def _router_fallback_models(config: LLMRouterConfig) -> list[dict[str, Any]]:+ fallbacks: list[dict[str, Any]] = []+ for group in _fallback_groups(config):+ entry = next((m for m in config.model_list if m.model_name == group), None)+ params = entry.litellm_params if entry is not None else {}+ hop = {field: params.get(field) for field in _PER_HOP_FIELDS}+ hop["model"] = str(params.get("model") or group)+ hop["base_url"] = params.get("api_base")+ hop["timeout"] = params.get("timeout", settings.LLM_CONFIG_TIMEOUT)+ fallbacks.append(hop)+ return fallbacks+++def router_fallback_llm_key(llm_key: str) -> str | None:+ """The registered key a router falls back to, when its fallback group is named after one."""+ if not LLMConfigRegistry.is_registered(llm_key):+ return None+ config = LLMConfigRegistry.get_config(llm_key)+ groups = _fallback_groups(config) if isinstance(config, LLMRouterConfig) else []+ return groups[0] if groups and LLMConfigRegistry.is_registered(groups[0]) else Nonedef _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfig:@@ -109,9 +135,9 @@ def _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfifull bridge lands (SKY-9256), the copilot-v2 path needs a way to run onorgs whose configured llm_key resolves to a router. We use the entry whose``model_name`` matches ``main_model_group``; if none match we fall back to- ``model_list[0]`` and warn. Fallback groups are converted to concrete- provider model strings and passed to LiteLLM's plain ``acompletion``- fallback path via ModelSettings.extra_args.+ ``model_list[0]`` and warn. Each fallback group becomes a LiteLLM fallback+ entry carrying its own model, credentials and tier, passed to the plain+ ``acompletion`` fallback path via ModelSettings.extra_args.The happy-path resolution is the expected code path on every copilot-v2call in staging/prod, so it logs at INFO. WARN is reserved for the@@ -146,7 +172,7 @@ def _degrade_router_to_direct(llm_key: str, config: LLMRouterConfig) -> LLMConfillm_key=llm_key,main_model_group=config.main_model_group,selected_model_name=direct_model_name,- fallback_models=fallback_models,+ fallback_models=[hop["model"] for hop in fallback_models],)return LLMConfig(diff --git a/skyvern/forge/sdk/copilot/model_telemetry.py b/skyvern/forge/sdk/copilot/model_telemetry.pyindex 3cfbfb7af..d4b82f15a 100644--- a/skyvern/forge/sdk/copilot/model_telemetry.py+++ b/skyvern/forge/sdk/copilot/model_telemetry.py@@ -106,6 +106,7 @@ def _usage_field(value: Any, *keys: str) -> Any:class CopilotModelCallTelemetry:model_call_index: intresponse_model: str | None = None+ served_provider: str | None = Nonecache_mode: Literal["implicit", "explicit"] = "implicit"cache_breakpoint_count: int = 0cache_stable_prefix_chars: int | None = None@@ -226,11 +227,17 @@ def _log_model_call_usage(returnbilling_model = telemetry.response_model or model+ # An in-call fallback can be served by another provider than the one base_url points at.+ provider_name = (+ normalize_gen_ai_provider(telemetry.served_provider, billing_model)+ if telemetry.served_provider+ else _otel_provider_name(billing_model, base_url)+ )emit_copilot_model_usage(CopilotModelUsageEvent(request_model=model,response_model=telemetry.response_model,- provider_name=_otel_provider_name(billing_model, base_url),+ provider_name=provider_name,input_tokens=telemetry.input_tokens,output_tokens=telemetry.output_tokens,cache_read_tokens=telemetry.cache_read_tokens,@@ -271,6 +278,13 @@ def model_call_telemetry_scope(_current_model_call_telemetry.reset(token)+def _capture_served_provider(telemetry: CopilotModelCallTelemetry, response: object) -> None:+ hidden_params = getattr(response, "_hidden_params", None)+ provider = hidden_params.get("custom_llm_provider") if isinstance(hidden_params, dict) else None+ if isinstance(provider, str) and provider:+ telemetry.served_provider = provider++def _capture_chat_stop_metadata(telemetry: CopilotModelCallTelemetry, response: object) -> None:"""Capture only typed transport exit facts; no model text reaches logs."""@@ -338,6 +352,7 @@ class _UsageCapturingStream:chunk = await self._iterator.__anext__()if isinstance(chunk.model, str):self._telemetry.response_model = chunk.model+ _capture_served_provider(self._telemetry, chunk)# LiteLLM returns ModelResponseStream at runtime even though its public# annotation is ChatCompletionChunk. Content chunks omit ``usage``.usage = getattr(chunk, "usage", None)@@ -394,8 +409,9 @@ def _is_gemini_model(model: str) -> bool:def _gemini_on_model_chain(model: str, model_settings: ModelSettings) -> bool:- fallbacks: list[str] = (model_settings.extra_args or {}).get("fallbacks") or []- return any(_is_gemini_model(candidate) for candidate in [model, *fallbacks])+ fallbacks: list[str | dict[str, Any]] = (model_settings.extra_args or {}).get("fallbacks") or []+ models = [hop["model"] if isinstance(hop, dict) else hop for hop in fallbacks]+ return any(_is_gemini_model(candidate) for candidate in [model, *models])def _has_ref_key(value: object) -> bool:@@ -630,6 +646,7 @@ class CopilotLitellmModel(LitellmModel):if telemetry is not None:if isinstance(result.model, str):telemetry.response_model = result.model+ _capture_served_provider(telemetry, result)usage = result.get("usage")if isinstance(usage, LiteLLMUsage):_capture_usage(telemetry, usage)diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex e85d44d47..6e24d075c 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -5334,17 +5334,19 @@ _DATE_SEGMENT_GROUP_JS = (# segment that actually filled._DECLARED_ROLE_JS = "el => el.getAttribute('role') || ''"+# Only these input types have a caret; a number input throws on setSelectionRange but may take End as its maximum.+_IS_TEXT_BOX_JS = (+ "el => el.tagName === 'TEXTAREA' || "+ "(el.tagName === 'INPUT' && ['text', 'search', 'url', 'tel', 'password'].includes(el.type))"+)+_DATE_SEGMENT_READBACK_JS = ("el => [el.value, el.textContent].filter(v => v != null && String(v).trim() !== '').join('|')")-# A press at a sub-pixel segment's centre reaches the hit and bubbles through every ancestor, so it may aim-# only when none of those is a control and nothing between the hit and the date group covers another segment.-_DATE_SEGMENT_AIM_JS = (- r"""(el) => {- const CONTROL = """- + json.dumps(_INTERACTIVE_HIT_SEL + ", label")- + r""";+# Where a press at a sub-pixel segment's centre would land: a layer inside the date group whose path up to the+# group covers no other segment. Whether a control sits anywhere on the press's path is decided separately.+_DATE_SEGMENT_AIM_JS = r"""(el) => {const centre = (e) => {const r = e.getBoundingClientRect();return [r.left + r.width / 2, r.top + r.height / 2];@@ -5352,23 +5354,59 @@ _DATE_SEGMENT_AIM_JS = (const [x, y] = centre(el);const root = el.getRootNode();const hit = (root.elementsFromPoint ? root : document).elementsFromPoint(x, y)[0];- if (!hit) return 'none';- if (hit === el) return 'self';+ if (!hit) return { hit: 'none' };+ if (hit === el || el.contains(hit)) return { hit: 'self' };const group = el.closest('[data-tv3-dateseg="group"]');- if (!group || !group.contains(hit) || hit === group) return 'outside';+ if (!group || !group.contains(hit) || hit === group) return { hit: 'outside' };const others = Array.from(group.querySelectorAll('[data-tv3-dateseg]')).filter((s) => s !== el && s !== group).map(centre);- for (let n = hit; n; n = n.parentElement || (n.getRootNode() && n.getRootNode().host) || null) {- if (n.matches && n.matches(CONTROL)) return 'control';- }for (let n = hit; n && n !== group; n = n.parentElement) {const box = n.getBoundingClientRect();if (others.some(([sx, sy]) => sx >= box.left && sx <= box.right && sy >= box.top && sy <= box.bottom)) {- return 'other';+ return { hit: 'other' };}}- return 'layer';+ // CDP hit-tests only whole document pixels, so the press lands on the same whole pixel the check reads. Playwright+ // measures a click position from the padding box and truncates the point to 0.01px; the 0.005 keeps float error+ // from truncating onto the pixel before.+ const docX = Math.round(x + window.scrollX);+ const docY = Math.round(y + window.scrollY);+ const box = hit.getBoundingClientRect();+ const style = getComputedStyle(hit);+ return {+ hit: 'layer',+ el: hit,+ x: docX - window.scrollX - box.left - parseFloat(style.borderLeftWidth || '0') + 0.005,+ y: docY - window.scrollY - box.top - parseFloat(style.borderTopWidth || '0') + 0.005,+ docX,+ docY,+ };+}"""++# The layer carries a per-press token only while its press is checked: CDP and Playwright hold separate remote+# objects for one node, and the page's main world is the one place both reach.+_DATE_SEGMENT_LAYER_KEY = "Symbol.for('tv3-dateseg-layer')"+_DATE_SEGMENT_MARK_LAYER_JS = f"(el, token) => {{ el[{_DATE_SEGMENT_LAYER_KEY}] = token; }}"+_DATE_SEGMENT_UNMARK_LAYER_JS = (+ f"(el, token) => {{ if (el[{_DATE_SEGMENT_LAYER_KEY}] === token) delete el[{_DATE_SEGMENT_LAYER_KEY}]; }}"+)++# Run on the deepest node under the press, which the browser resolves through closed shadow roots too: a press+# bubbles up its composed path, so it is clear only when that path passes through the layer and holds no control.+_DATE_SEGMENT_PRESS_PATH_JS = (+ r"""function (token) {+ const CONTROL = """+ + json.dumps(_INTERACTIVE_HIT_SEL + ", label")+ + r""";+ let onLayer = false;+ for (let n = this; n; n = n.assignedSlot || n.parentNode || n.host || null) {+ if (n.nodeType === 1 && n.matches(CONTROL)) return false;+ if (n["""+ + _DATE_SEGMENT_LAYER_KEY+ + r"""] === token) onLayer = true;+ }+ return onLayer;}""")@@ -13534,8 +13572,16 @@ def build_browser_tools(held = _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector))if held:# A press leaves the caret where it landed, and Backspace deletes only what sits before it.- if await locator.evaluate(_CARET_TO_END_JS, timeout=2000) == "end_key":+ caret = await locator.evaluate(_CARET_TO_END_JS, timeout=2000)+ # End is only a caret key in a text box; an ARIA spinbutton takes it as "set to maximum".+ if caret == "end_key" and await locator.evaluate(_IS_TEXT_BOX_JS, timeout=2000):await locator.press("End")+ elif caret == "end_key":+ # No caret key here, so Delete clears the digits after the press point.+ for _ in range(len(held.group(0))):+ await locator.press("Delete")+ if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):+ returnfor _ in range(len(held.group(0)) if held else 0):await locator.press("Backspace")if not _DATE_SEGMENT_DIGITS_RE.search(await _read_date_segment(page, selector)):@@ -13543,8 +13589,75 @@ def build_browser_tools(except Exception:pass+ # Set while a whole-date write takes the base path, so every step of that write behaves as control.+ _date_segment_aim_suspended: list[bool] = [False]+def _date_segment_aim_on() -> bool:- return run_arm_enabled(DATE_SEGMENT_AIM_FLAG, settings.TASK_V3_DATE_SEGMENT_AIM)+ return not _date_segment_aim_suspended[0] and run_arm_enabled(+ DATE_SEGMENT_AIM_FLAG, settings.TASK_V3_DATE_SEGMENT_AIM+ )++ async def _press_path_is_clear(layer: Any, x: int, y: int) -> bool:+ # The browser's own node at a point in main-frame document coordinates, closed shadow roots included; anything+ # unresolvable refuses, as does a browser that stops answering. CDP finds only nodes inside the current viewport.+ top = _current_page()+ token = secrets.token_hex(8)+ sessions: list[Any] = []++ async def probe() -> bool:+ await layer.evaluate(_DATE_SEGMENT_MARK_LAYER_JS, token)+ cdp = await top.context.new_cdp_session(top)+ sessions.append(cdp)+ await cdp.send("DOM.getDocument", {"depth": 0})+ found = await cdp.send("DOM.getNodeForLocation", {"x": x, "y": y})+ node = await cdp.send("DOM.resolveNode", {"backendNodeId": found["backendNodeId"]})+ result = await cdp.send(+ "Runtime.callFunctionOn",+ {+ "objectId": node["object"]["objectId"],+ "functionDeclaration": _DATE_SEGMENT_PRESS_PATH_JS,+ "arguments": [{"value": token}],+ "returnByValue": True,+ },+ )+ return result.get("result", {}).get("value") is True++ try:+ return await asyncio.wait_for(probe(), timeout=3)+ except Exception:+ return False+ finally:+ with contextlib.suppress(Exception):+ await asyncio.wait_for(layer.evaluate(_DATE_SEGMENT_UNMARK_LAYER_JS, token), timeout=1)+ for cdp in sessions:+ with contextlib.suppress(Exception):+ await asyncio.wait_for(cdp.detach(), timeout=1)++ async def _aim_hit(locator: Any) -> tuple[str, Any, dict[str, float]]:+ # (kind, the layer's ElementHandle for a pressable layer, its press point). The layer comes back as a handle+ # rather than a DOM marker, which the next aim in a different shadow root could not clear.+ try:+ result = await locator.evaluate_handle(_DATE_SEGMENT_AIM_JS, timeout=2000)+ except Exception:+ return "none", None, {}+ props: dict[str, Any] = {}+ returned = None+ try:+ props = await result.get_properties()+ kind = await props["hit"].json_value() if "hit" in props else "none"+ layer = props["el"].as_element() if "el" in props else None+ at = {key: float(await props[key].json_value()) for key in ("x", "y", "docX", "docY") if key in props}+ returned = layer+ except Exception:+ return "none", None, {}+ finally:+ for name, prop in props.items():+ if name != "el" or returned is None:+ with contextlib.suppress(Exception):+ await prop.dispose()+ with contextlib.suppress(Exception):+ await result.dispose()+ return str(kind), returned, atasync def _aim_date_segment(page: Any, selector: str, locator: Any, *, press: bool = True) -> str:# Some widgets move their section cursor only on a trusted pointer event and reset it when focus@@ -13557,7 +13670,14 @@ def build_browser_tools(return "focus"if box["width"] * box["height"] > 1:if not press:- return "click"+ # Planning only: a normal-size segment counts as clickable only when it is what its centre hits.+ try:+ kind, layer, _ = await _aim_hit(locator)+ if layer is not None:+ await layer.dispose()+ return "click" if kind == "self" else "focus"+ except Exception:+ return "focus"try:await locator.click(timeout=2000)except Exception:@@ -13567,15 +13687,38 @@ def build_browser_tools(# inside the frame cannot see, so a framed segment is only focused.if not _date_segment_aim_on() or (_acted_realm and page.parent_frame is not None):return "focus"+ owner = None+ handle = Nonetry:- hit = await locator.evaluate(_DATE_SEGMENT_AIM_JS, timeout=2000)+ handle = await locator.element_handle(timeout=2000)+ owner = await handle.owner_frame() if handle is not None else Noneexcept Exception:- hit = None- if hit != "layer":+ owner = None+ finally:+ if handle is not None:+ with contextlib.suppress(Exception):+ await handle.dispose()+ if owner is None or owner != _current_page().main_frame:return "focus"- if not press:+ kind, layer, at = await _aim_hit(locator)+ try:+ if kind != "layer" or layer is None:+ return "focus"+ if not await _press_path_is_clear(layer, int(at["docX"]), int(at["docY"])):+ return "focus"+ if not press:+ return "mirror"+ # Pressed through Playwright as the layer ELEMENT, never forced: its actionability and hit-target checks+ # refuse the press when the layer has been hidden or covered since the aim measured it.+ try:+ await layer.click(position={"x": float(at["x"]), "y": float(at["y"])}, timeout=1500)+ except Exception:+ return "focus"return "mirror"- return "mirror" if await _click_at_box_centre(page, selector) else "focus"+ finally:+ if layer is not None:+ with contextlib.suppress(Exception):+ await layer.dispose()async def _type_one_date_segment(page: Any, selector: str, digits: str) -> tuple[bool, str]:locator = page.locator(selector).first@@ -13640,22 +13783,30 @@ def build_browser_tools(async def _fill_date_segment_group(page: Any, selector: str, components: dict[str, str], order: list[str]) -> ToolResult:- written: list[tuple[str, str, str]] = []arm = _date_segment_aim_on()- write_order = list(_DATE_SEGMENT_ORDER)+ aimed = armif arm:- # A segment that can only be focused types wherever the widget's cursor is, which a widget- # resets to its first segment, so without a press on every segment the group is written in- # its own order.+ # No write order serves a group mixing pressable and focus-only segments, so such a group takes the+ # base path, exactly as control does.for label in order:segment_selector = f'[data-tv3-dateseg="{label}"]'- aimable = await _aim_date_segment(+ planned = await _aim_date_segment(page, segment_selector, page.locator(segment_selector).first, press=False)- if aimable == "focus":- write_order = list(order)+ if planned == "focus":+ aimed = Falsebreak- for label in write_order:+ _date_segment_aim_suspended[0] = arm and not aimed+ try:+ return await _write_date_segment_group(page, selector, components, order, arm=arm, aimed=aimed)+ finally:+ _date_segment_aim_suspended[0] = False++ async def _write_date_segment_group(+ page: Any, selector: str, components: dict[str, str], order: list[str], *, arm: bool, aimed: bool+ ) -> ToolResult:+ written: list[tuple[str, str, str]] = []+ for label in _DATE_SEGMENT_ORDER:segment_selector = f'[data-tv3-dateseg="{label}"]'committed, aim = await _type_one_date_segment(page, segment_selector, components[label])LOG.info(@@ -13666,6 +13817,7 @@ def build_browser_tools(aim=aim,group_shape=_date_group_shape(order),date_segment_aim_arm=arm,+ date_segment_group_aimed=aimed,)if not committed:return ToolResult.error(diff --git a/skyvern/services/browser_recording/evidence.py b/skyvern/services/browser_recording/evidence.pyindex e1c8f3ef4..81746f29e 100644--- a/skyvern/services/browser_recording/evidence.py+++ b/skyvern/services/browser_recording/evidence.py@@ -252,6 +252,25 @@ def _label_kept_actions(return kept, deleted_action_ids+def recorded_credential_urls(+ actions: list[Action],+ draft_steps: list[RecordingDraftStep] | None,+) -> list[tuple[str, str]]:+ """Every (credential_id, url) the evidence packet would report, including actions past `MAX_EVIDENCE_CHARS`."""+ ordered_actions = sorted(actions, key=lambda action: action.timestamp_start)+ input_values = list(+ dict.fromkeys(action.input_value for action in ordered_actions if isinstance(action, ActionInputText))+ )+ return [+ (draft.credential_id, _redact_input_values(action.url, input_values) or "")+ for action, draft in transfer_focus_click_credentials(apply_draft_overlay(ordered_actions, draft_steps))+ if isinstance(action, ActionInputText)+ and draft is not None+ and draft.credential_id+ and _credential_kind(action, draft) is not None+ ]++def build_recording_evidence(actions: list[Action],draft_steps: list[RecordingDraftStep] | None,diff --git a/skyvern/services/browser_recording/service.py b/skyvern/services/browser_recording/service.pyindex 87b2e686c..901648527 100644--- a/skyvern/services/browser_recording/service.py+++ b/skyvern/services/browser_recording/service.py@@ -26,7 +26,11 @@ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistryfrom skyvern.forge.sdk.browser_action_policy import canonicalize_originfrom skyvern.forge.sdk.copilot.credential_resolution import safe_admitted_urlfrom skyvern.services.browser_recording.code_first import actions_to_code_first_blocks-from skyvern.services.browser_recording.evidence import RecordingEvidencePacket, build_recording_evidence+from skyvern.services.browser_recording.evidence import (+ RecordingEvidencePacket,+ build_recording_evidence,+ recorded_credential_urls,+)from skyvern.services.browser_recording.redact import is_secret_field, redact_console_event, texts_are_labelsfrom skyvern.services.browser_recording.types import (Action,@@ -707,7 +711,7 @@ class Processor:"""Process the compressed browser session recording into workflow definition blocks."""- blocks, parameters, _, _, evidence = await self.process_with_evidence(+ blocks, parameters, _, _, evidence, _ = await self.process_with_evidence(compressed_chunks,draft_steps=draft_steps,recorded_actions=recorded_actions,@@ -727,6 +731,7 @@ class Processor:list[dict[str, t.Any]],dict[str, t.Any],RecordingEvidencePacket,+ list[tuple[str, str]],]:if recorded_actions is None:events = self.compressed_chunks_to_events(compressed_chunks)@@ -770,6 +775,7 @@ class Processor:interpretation_session_id=self.interpretation_session_id,),refinement_evidence,+ recorded_credential_urls(actions, draft_steps),)@@ -802,7 +808,14 @@ class BrowserSessionRecordingService:interpretation_session_id=interpretation_session_id,)- blocks, parameters, evidence, metadata, refinement_evidence = await processor.process_with_evidence(+ (+ blocks,+ parameters,+ evidence,+ metadata,+ refinement_evidence,+ recorded_credentials,+ ) = await processor.process_with_evidence(compressed_chunks,draft_steps=draft_steps,recorded_actions=recorded_actions,@@ -812,10 +825,9 @@ class BrowserSessionRecordingService:return blocks, parameters, None, refinement_evidencecredential_urls: dict[str, str] = {}- for action in refinement_evidence.actions:- credential_id = action.credential.credential_id if action.credential is not None else None- admitted_url = safe_admitted_url(action.url)- if credential_id and admitted_url and canonicalize_origin(admitted_url) is not None:+ for credential_id, url in recorded_credentials:+ admitted_url = safe_admitted_url(url)+ if admitted_url and canonicalize_origin(admitted_url) is not None:credential_urls.setdefault(credential_id, admitted_url)if credential_urls:owned_credentials = await app.DATABASE.credentials.get_credentials_by_ids(diff --git a/skyvern/webeye/actions/handler.py b/skyvern/webeye/actions/handler.pyindex f8e284e1d..7dbd16429 100644--- a/skyvern/webeye/actions/handler.py+++ b/skyvern/webeye/actions/handler.py@@ -10671,8 +10671,11 @@ async def handle_terminate_action(step: Step,) -> list[ActionResult]:if task.error_code_mapping:+ # A code Skyvern attached itself (OTP_TIMEOUT once TOTP polling ran out) records something the page cannot+ # show, so the screenshot-based extraction must not drop it, even when the mapping does not declare it.+ skyvern_errors = [error for error in action.errors if error.is_skyvern_defined]try:- action.errors = await extract_user_defined_errors(+ extracted_errors = await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page, reasoning=action.reasoning)except Exception:@@ -10683,6 +10686,11 @@ async def handle_terminate_action(action_errors=action.errors,exc_info=True,)+ else:+ skyvern_codes = {error.error_code for error in skyvern_errors}+ action.errors = skyvern_errors + [+ error for error in extracted_errors if error.error_code not in skyvern_codes+ ]return [ActionSuccess()]diff --git a/tests/unit/test_copilot_agent_helpers.py b/tests/unit/test_copilot_agent_helpers.pyindex f2b3f0060..77776ca6d 100644--- a/tests/unit/test_copilot_agent_helpers.py+++ b/tests/unit/test_copilot_agent_helpers.py@@ -7035,6 +7035,38 @@ class TestCopilotConfig:assert agent_module._fallback_llm_key(CopilotConfig(fallback_llm_key="PRIMARY"), "PRIMARY") is Noneassert agent_module._fallback_llm_key(CopilotConfig(fallback_llm_key="SECONDARY"), "PRIMARY") == "SECONDARY"+ def test_fallback_key_follows_a_router_whose_fallback_group_is_a_registered_key(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ from skyvern.forge.sdk.api.llm.config_registry import LLMConfigRegistry+ from skyvern.schemas.llm import LLMConfig, LLMRouterConfig, LLMRouterModelConfig++ def router(fallback_group: str) -> LLMRouterConfig:+ return LLMRouterConfig(+ model_name="router",+ model_list=[+ LLMRouterModelConfig(model_name="AZURE_TERRA", litellm_params={"model": "azure/gpt-5.6-terra"}),+ LLMRouterModelConfig(model_name=fallback_group, litellm_params={"model": "gpt-5.6-terra"}),+ ],+ required_env_vars=[],+ supports_vision=True,+ add_assistant_prefix=False,+ main_model_group="AZURE_TERRA",+ fallback_model_group=fallback_group,+ )++ monkeypatch.setitem(+ LLMConfigRegistry._configs, # type: ignore[attr-defined]+ "OPENAI_TERRA",+ LLMConfig("gpt-5.6-terra", [], supports_vision=True, add_assistant_prefix=False),+ )+ monkeypatch.setitem(LLMConfigRegistry._configs, "TERRA_ROUTER", router("OPENAI_TERRA")) # type: ignore[attr-defined]+ monkeypatch.setitem(LLMConfigRegistry._configs, "ALIAS_ROUTER", router("openai-terra-fallback")) # type: ignore[attr-defined]+ config = CopilotConfig(fallback_llm_key="SECONDARY")++ assert agent_module._fallback_llm_key(config, "TERRA_ROUTER") == "OPENAI_TERRA"+ assert agent_module._fallback_llm_key(config, "ALIAS_ROUTER") == "SECONDARY"+@pytest.mark.asyncio@pytest.mark.parametrize(("fallback_llm_key", "reply_llm_key", "attempt"),diff --git a/tests/unit/test_copilot_model_resolver.py b/tests/unit/test_copilot_model_resolver.pyindex 8a67e90be..ef39d07e5 100644--- a/tests/unit/test_copilot_model_resolver.py+++ b/tests/unit/test_copilot_model_resolver.py@@ -104,11 +104,77 @@ class TestModelResolver:assert run_config.model_settings.max_tokens == 8192assert run_config.model_settings.extra_args is not Noneassert run_config.model_settings.extra_args["timeout"] == 900.0- assert run_config.model_settings.extra_args["fallbacks"] == [+ assert [hop["model"] for hop in run_config.model_settings.extra_args["fallbacks"]] == ["azure/gpt-4-1-mini","anthropic/claude-sonnet-4-20250514",]+ @pytest.mark.asyncio+ async def test_router_fallback_hop_is_sent_with_its_own_credentials(self, monkeypatch: pytest.MonkeyPatch) -> None:+ import litellm++ from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfig++ router_config = LLMRouterConfig(+ model_name="azure-terra-with-openai-terra",+ model_list=[+ LLMRouterModelConfig(+ model_name="AZURE_TERRA",+ litellm_params={+ "model": "azure/gpt-5.6-terra",+ "api_base": "https://azure.example.test",+ "api_key": "azure-key",+ "api_version": "2025-04-01-preview",+ "model_info": {"model_name": "azure/gpt-5.6-terra"},+ },+ ),+ LLMRouterModelConfig(model_name="OPENAI_TERRA", litellm_params={"model": "gpt-5.6-terra"}),+ ],+ required_env_vars=[],+ supports_vision=True,+ add_assistant_prefix=False,+ main_model_group="AZURE_TERRA",+ fallback_model_group="OPENAI_TERRA",+ )+ handler = _install_config(monkeypatch, router_config, "AZURE_TERRA_WITH_OPENAI_FALLBACK")+ model_name, run_config, _, _ = resolve_model_config(handler)+ model = run_config.model_provider.get_model(model_name)+ assert run_config.model_settings is not None++ real_acompletion = litellm.acompletion+ hops: list[dict[str, Any]] = []++ async def primary_fails(**kwargs: Any) -> Any:+ if "fallbacks" in kwargs:+ return await real_acompletion(**kwargs)+ hops.append(kwargs)+ if len(hops) == 1:+ raise litellm.RateLimitError(message="429", llm_provider="azure", model=kwargs["model"])+ return "served"++ monkeypatch.setattr(litellm, "acompletion", primary_fails)+ served = await litellm.acompletion(+ model=model_name,+ messages=[{"role": "user", "content": "hi"}],+ api_key=model.api_key,+ base_url=model.base_url,+ **run_config.model_settings.extra_args,+ )++ assert served == "served"+ assert [+ (hop["model"], hop["api_key"], hop["base_url"], hop["api_version"], hop.get("model_info")) for hop in hops+ ] == [+ (+ "azure/gpt-5.6-terra",+ "azure-key",+ "https://azure.example.test",+ "2025-04-01-preview",+ {"model_name": "azure/gpt-5.6-terra"},+ ),+ ("gpt-5.6-terra", None, None, None, None),+ ]+def test_router_config_no_main_group_match_falls_back_to_first_entry(self, monkeypatch: pytest.MonkeyPatch) -> None:from skyvern.schemas.llm import LLMRouterConfig, LLMRouterModelConfigdiff --git a/tests/unit/test_copilot_model_telemetry.py b/tests/unit/test_copilot_model_telemetry.pyindex 0834a484a..38f7febc2 100644--- a/tests/unit/test_copilot_model_telemetry.py+++ b/tests/unit/test_copilot_model_telemetry.py@@ -745,6 +745,32 @@ def test_datadog_usage_attributes_fallback_spend_to_response_model(assert events[0]["gen_ai.provider.name"] == "anthropic"+async def test_datadog_usage_names_the_provider_that_served_an_in_call_fallback(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ events: list[dict[str, Any]] = []+ monkeypatch.setattr(model_telemetry_module.LOG, "info", lambda _event, **fields: events.append(fields))+ monkeypatch.setattr(model_telemetry_module, "_model_call_cost", lambda telemetry, model: None)+ chunk = ModelResponseStream(model="gpt-5.6-terra")+ chunk._hidden_params = {"custom_llm_provider": "openai"}++ async def served_by_openai() -> AsyncIterator[ModelResponseStream]:+ yield chunk++ with model_call_telemetry_scope(+ 6,+ model="azure/gpt-5.6-terra",+ base_url="https://example.openai.azure.com",+ ) as telemetry:+ await anext(model_telemetry_module._UsageCapturingStream(served_by_openai(), telemetry)) # type: ignore[arg-type]+ telemetry.input_tokens = 100+ telemetry.output_tokens = 5++ assert events[0]["gen_ai.request.model"] == "azure/gpt-5.6-terra"+ assert events[0]["gen_ai.provider.name"] == "openai"++def test_model_call_without_provider_usage_does_not_emit_datadog_event(monkeypatch: pytest.MonkeyPatch,) -> None:@@ -1410,7 +1436,7 @@ def _json_keys(value: object) -> set[str]:("code", _GEMINI, None, False),("code", "openai/gpt-5.6", {"fallbacks": [_GEMINI]}, True),("for_loop", _GEMINI, None, True),- ("for_loop", "azure/gpt-5.6-sol", {"fallbacks": [_GEMINI]}, False),+ ("for_loop", "azure/gpt-5.6-sol", {"fallbacks": [{"model": _GEMINI, "api_key": None}]}, False),("escaped_ref_key", _GEMINI, None, False),],)diff --git a/tests/unit/test_handle_terminate_action.py b/tests/unit/test_handle_terminate_action.pyindex f28fa2c6a..b272113f5 100644--- a/tests/unit/test_handle_terminate_action.py+++ b/tests/unit/test_handle_terminate_action.py@@ -1,18 +1,18 @@-"""Tests for handle_terminate_action error handling.--Verifies that when extract_user_defined_errors fails (e.g., CDP disconnection),-the original error codes from the action reasoning are preserved instead of being lost.-"""+"""Tests for which error codes a terminate action keeps when the task has an error_code_mapping."""+from collections.abc import Iterator+from contextlib import contextmanagerfrom datetime import timezonefrom types import SimpleNamespace+from typing import Anyfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest-from skyvern.errors.errors import UserDefinedError+from skyvern.errors.errors import TimeoutGetTOTPVerificationCodeError, UserDefinedErrorfrom skyvern.webeye.actions.actions import TerminateActionfrom skyvern.webeye.actions.handler import extract_user_defined_errors, handle_terminate_action+from skyvern.webeye.actions.models import DetailedAgentStepOutputfrom skyvern.webeye.actions.responses import ActionSuccess@@ -29,19 +29,21 @@ def _make_step() -> MagicMock:return step-async def _extract_with_empty_llm(- task: MagicMock, step: MagicMock, reasoning: str, monkeypatch: pytest.MonkeyPatch-) -> list[UserDefinedError]:+def _make_scraped_page() -> MagicMock:scraped_page = MagicMock()refreshed_page = MagicMock()refreshed_page.build_element_tree.return_value = ""refreshed_page.url = "chrome-error://chromewebdata/"refreshed_page.screenshots = []scraped_page.refresh = AsyncMock(return_value=refreshed_page)+ return scraped_page++@contextmanager+def _surface_errors_llm_returns(monkeypatch: pytest.MonkeyPatch, errors: list[dict[str, Any]]) -> Iterator[None]:monkeypatch.setattr("skyvern.webeye.actions.handler.app.EXTRACTION_LLM_API_HANDLER",- AsyncMock(return_value={"errors": []}),+ AsyncMock(return_value={"errors": errors}),raising=False,)with (@@ -55,7 +57,102 @@ async def _extract_with_empty_llm(return_value=SimpleNamespace(tz_info=timezone.utc),),):- return await extract_user_defined_errors(task=task, step=step, scraped_page=scraped_page, reasoning=reasoning)+ yield+++async def _extract_with_empty_llm(+ task: MagicMock, step: MagicMock, reasoning: str, monkeypatch: pytest.MonkeyPatch+) -> list[UserDefinedError]:+ with _surface_errors_llm_returns(monkeypatch, []):+ return await extract_user_defined_errors(+ task=task, step=step, scraped_page=_make_scraped_page(), reasoning=reasoning+ )+++ ("error_code_mapping", "llm_errors", "expected_codes"),+ [+ pytest.param(+ {"LOGIN_FAILED": "the login did not complete"},+ [],+ ["OTP_TIMEOUT"],+ id="mapping_without_otp_timeout_llm_returns_nothing",+ ),+ pytest.param(+ {"OTP_TIMEOUT": "the 2FA code never arrived", "LOGIN_FAILED": "the login did not complete"},+ [{"error_code": "LOGIN_FAILED", "reasoning": "Still on the login page", "confidence_float": 0.8}],+ ["OTP_TIMEOUT", "LOGIN_FAILED"],+ id="mapping_declares_otp_timeout_llm_omits_it",+ ),+ pytest.param(+ {"OTP_TIMEOUT": "the 2FA code never arrived", "LOGIN_FAILED": "the login did not complete"},+ [+ {"error_code": "OTP_TIMEOUT", "reasoning": "No code arrived", "confidence_float": 0.9},+ {"error_code": "LOGIN_FAILED", "reasoning": "Still on the login page", "confidence_float": 0.8},+ ],+ ["OTP_TIMEOUT", "LOGIN_FAILED"],+ id="mapping_declares_otp_timeout_llm_repeats_it",+ ),+ ],+)+async def test_totp_timeout_code_reaches_step_errors_with_error_code_mapping(+ monkeypatch: pytest.MonkeyPatch,+ error_code_mapping: dict[str, str],+ llm_errors: list[dict[str, Any]],+ expected_codes: list[str],+) -> None:+ task = _make_task(error_code_mapping=error_code_mapping)+ task.navigation_goal = "Log in"+ task.navigation_payload = {}+ action = TerminateAction(+ reasoning="No TOTP verification code found. Going to terminate. Polled source: totp_verification_url=x.",+ errors=[TimeoutGetTOTPVerificationCodeError().to_user_defined_error()],+ )++ with _surface_errors_llm_returns(monkeypatch, llm_errors):+ results = await handle_terminate_action(+ action=action, page=MagicMock(), scraped_page=_make_scraped_page(), task=task, step=_make_step()+ )++ step_output = DetailedAgentStepOutput(+ scraped_page=None,+ extract_action_prompt=None,+ llm_response=None,+ actions=[action],+ action_results=results,+ actions_and_results=[(action, results)],+ )+ assert [error.error_code for error in step_output.extract_errors()] == expected_codes+++ "llm_error_code",+ [+ pytest.param("LLM_REASONING_ERROR", id="code_not_in_mapping"),+ pytest.param("OTP_TIMEOUT", id="code_named_like_a_skyvern_code"),+ ],+)+async def test_terminate_keeps_codes_the_llm_put_on_the_action_only_if_extraction_agrees(+ monkeypatch: pytest.MonkeyPatch, llm_error_code: str+) -> None:+ task = _make_task(+ error_code_mapping={"OTP_TIMEOUT": "the 2FA code never arrived", "LOGIN_FAILED": "the login did not complete"}+ )+ task.navigation_goal = "Log in"+ task.navigation_payload = {}+ action = TerminateAction(+ reasoning="The page did not load",+ errors=[UserDefinedError(error_code=llm_error_code, reasoning="Page is blank", confidence_float=0.8)],+ )++ with _surface_errors_llm_returns(monkeypatch, []):+ await handle_terminate_action(+ action=action, page=MagicMock(), scraped_page=_make_scraped_page(), task=task, step=_make_step()+ )++ assert action.errors == []@pytest.mark.asyncio@@ -85,33 +182,6 @@ async def test_terminate_preserves_errors_when_extract_fails() -> None:assert action.errors[0].error_code == "OTP_TIMEOUT"-async def test_terminate_uses_extracted_errors_on_success() -> None:- """When extract_user_defined_errors succeeds, its result should replace action.errors."""- task = _make_task(error_code_mapping={"OTP_TIMEOUT": "OTP verification code not received"})- step = _make_step()- page = MagicMock()- scraped_page = MagicMock()-- action = TerminateAction(reasoning="No TOTP verification code found")-- extracted_errors = [- UserDefinedError(error_code="OTP_TIMEOUT", reasoning="Page shows OTP not received", confidence_float=0.99)- ]- with patch(- "skyvern.webeye.actions.handler.extract_user_defined_errors",- new_callable=AsyncMock,- return_value=extracted_errors,- ):- results = await handle_terminate_action(- action=action, page=page, scraped_page=scraped_page, task=task, step=step- )-- assert len(results) == 1- assert isinstance(results[0], ActionSuccess)- assert action.errors == extracted_errors--@pytest.mark.asyncioasync def test_extract_user_defined_errors_falls_back_to_reasoning_match(monkeypatch: pytest.MonkeyPatch) -> None:task = _make_task(error_code_mapping={"portal_inaccessible": "portal is inaccessible"})