Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Copilot: send the current screenshot sharp on every call, keep history small (SKY-18247) (#8905)
Andrew Neilson4e533b512225
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsxindex 300c25716..ede29cc82 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx@@ -344,7 +344,16 @@ describe("CredentialCard sign in myself", () => {...overrides,});- it("offers signing in yourself only when the chat passes an offer", () => {+ const SIGN_IN_LABEL = "Sign in myself (this test only)";+ const SCOPE_WARNING =+ "Signing in yourself works for this test. Later runs usually won't stay signed in — connect a credential so every run can sign in.";+ const follows = (later: Element, earlier: Element) =>+ Boolean(+ earlier.compareDocumentPosition(later) & Node.DOCUMENT_POSITION_FOLLOWING,+ );++ it("offers signing in yourself only when the chat passes an offer", async () => {+ credsData.current = [{ credential_id: "cred_hn", name: "HN login" }];const offer = signIn();render(<CredentialCard@@ -355,7 +364,17 @@ describe("CredentialCard sign in myself", () => {signIn={offer}/>,);- fireEvent.click(screen.getByRole("button", { name: "Sign in myself" }));+ const picker = await screen.findByRole("combobox");+ const control = screen.getByRole("button", { name: SIGN_IN_LABEL });+ expect(control.className).not.toContain("bg-cta");+ expect(control.className).not.toContain("border-input");+ const connect = screen.getByRole("button", { name: "Connect credential" });+ const skip = screen.getByRole("button", { name: "Skip for now" });+ expect(follows(control, connect)).toBe(true);+ expect(follows(control, picker)).toBe(true);+ expect(follows(skip, control)).toBe(true);+ expect(screen.getByText(SCOPE_WARNING)).toBeTruthy();+ fireEvent.click(control);expect(offer.onStart).toHaveBeenCalledTimes(1);cleanup();@@ -367,7 +386,8 @@ describe("CredentialCard sign in myself", () => {onSkip={vi.fn()}/>,);- expect(screen.queryByRole("button", { name: "Sign in myself" })).toBeNull();+ expect(screen.queryByRole("button", { name: SIGN_IN_LABEL })).toBeNull();+ expect(screen.queryByText(SCOPE_WARNING)).toBeNull();});it("while signing in, Done finishes, Cancel skips, and a Done that found nothing says so", () => {@@ -385,6 +405,11 @@ describe("CredentialCard sign in myself", () => {expect(screen.getAllByText(/No sign-in found for portal\.example\.com/),).not.toHaveLength(0);+ expect(+ screen.getByText(+ "Use the browser to sign in, including any verification code, then click Done. This sign-in is for this test; later runs usually won't stay signed in. No password is stored.",+ ),+ ).toBeTruthy();fireEvent.click(screen.getByRole("button", { name: "Done" }));fireEvent.click(screen.getByRole("button", { name: "Cancel" }));expect(offer.onDone).toHaveBeenCalledTimes(1);@@ -423,8 +448,13 @@ describe("CredentialCard sign in myself", () => {onSkip={vi.fn()}/>,);+ expect(screen.getByText("Signed in for this test only")).toBeTruthy();+ expect(screen.getByText(/saved as 'Sign-in for portal'/)).toBeTruthy();+ fireEvent.click(screen.getByRole("button", { expanded: false }));expect(- screen.getByText("Signed in, saved as 'Sign-in for portal'"),+ screen.getByText(+ "For this test · later runs usually won't stay signed in · no password stored",+ ),).toBeTruthy();});});diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsxindex e0be00b30..33b178b32 100644--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx@@ -222,7 +222,7 @@ const UPDATE_SKIP_OUTCOME = {detail: "Credential not updated — the workflow keeps its saved sign-in.",};const SIGNED_IN_DETAIL =- "No password stored · your sign-in is saved as browser cookies in this profile";+ "For this test · later runs usually won't stay signed in · no password stored";const TIMEOUT_OUTCOME = {title: "Sign-in request timed out",meta: "test may stop at login",@@ -995,10 +995,11 @@ function CredentialAskCard({return (<ResolvedCredentialCardtone="done"- title={+ title="Signed in for this test only"+ meta={resolvedOutcome.name- ? `Signed in, saved as '${resolvedOutcome.name}'`- : "Signed in, saved as a browser profile"+ ? `saved as '${resolvedOutcome.name}'`+ : "saved as a browser profile"}detail={SIGNED_IN_DETAIL}/>@@ -1181,9 +1182,9 @@ function CredentialAskCard({key="how"className="text-[11px] leading-relaxed text-muted-foreground">- Use the browser to sign in, including any verification code, then- click Done. Skyvern saves the sign-in as a browser profile; no- password is stored.+ {+ "Use the browser to sign in, including any verification code, then click Done. This sign-in is for this test; later runs usually won't stay signed in. No password is stored."+ }</span>,...(notFound? [@@ -1263,6 +1264,18 @@ function CredentialAskCard({</span>,]: []),+ ...(signIn+ ? [+ <span+ key="sign-in-scope"+ className="text-[11px] leading-relaxed text-muted-foreground"+ >+ {+ "Signing in yourself works for this test. Later runs usually won't stay signed in — connect a credential so every run can sign in."+ }+ </span>,+ ]+ : []),...(mode === "terminal"? [<span@@ -1305,17 +1318,6 @@ function CredentialAskCard({</Button>) : null}{connectButton}- {signIn ? (- <Button- type="button"- size="sm"- variant="outline"- disabled={disabled || signIn.busy}- onClick={signIn.onStart}- >- Sign in myself- </Button>- ) : null}{orgCredentials.status === "ready" &&(pickable.length > 0 || pickerEngaged) ? (<CredentialPicker@@ -1344,6 +1346,18 @@ function CredentialAskCard({{retryButton}</>) : null}+ {signIn ? (+ <Button+ type="button"+ size="sm"+ variant="link"+ className="text-blue-600 dark:text-blue-400"+ disabled={disabled || signIn.busy}+ onClick={signIn.onStart}+ >+ Sign in myself (this test only)+ </Button>+ ) : null}<div className="ml-auto"><SkipButton onSkip={onSkip} disabled={disabled} /></div>diff --git a/skyvern/cli/core/browser_ops.py b/skyvern/cli/core/browser_ops.pyindex c01bad079..bb79ca002 100644--- a/skyvern/cli/core/browser_ops.py+++ b/skyvern/cli/core/browser_ops.py@@ -134,6 +134,7 @@ LOCALHOST_RECOVERY_HINT = (# reporting a failure for a page that is already loaded and usable._LOAD_STATE_LADDER = ("commit", "domcontentloaded", "load", "networkidle")_NAVIGATE_TITLE_TIMEOUT_SECONDS = 5.0+_VIEWPORT_PROBE_TIMEOUT_SECONDS = 2.0_UNBOUNDED_LEG_TIMEOUT_MS = 1000@@ -148,6 +149,7 @@ class NavigateResult:class ScreenshotResult:data: bytesfull_page: bool = False+ viewport_css_px: tuple[int, int] | None = None@dataclass@@ -281,6 +283,8 @@ async def do_screenshot(data = await element.screenshot()else:data = await page.screenshot(full_page=full_page)+ # Reported only when the image shows exactly the viewport, so image size over it is a valid scale.+ viewport_css_px = None if selector or full_page else await _viewport_css_px(page)except asyncio.CancelledError:cancelled = Trueraise@@ -293,7 +297,20 @@ async def do_screenshot(restore.add_done_callback(_CURSOR_RESTORES.discard)if not cancelled:await restore- return ScreenshotResult(data=data, full_page=full_page)+ return ScreenshotResult(data=data, full_page=full_page, viewport_css_px=viewport_css_px)+++async def _viewport_css_px(page: Any) -> tuple[int, int] | None:+ # Screenshots are taken at device scale, so the image's pixel size is not the CSS viewport on a HiDPI display.+ try:+ size = await asyncio.wait_for(+ page.evaluate("() => [window.innerWidth, window.innerHeight]"), timeout=_VIEWPORT_PROBE_TIMEOUT_SECONDS+ )+ except Exception:+ return None+ if isinstance(size, list) and len(size) == 2 and all(type(value) is int and value > 0 for value in size):+ return size[0], size[1]+ return Noneasync def do_act(diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.pyindex 970e0bc1a..73f3463b5 100644--- a/skyvern/cli/mcp_tools/browser.py+++ b/skyvern/cli/mcp_tools/browser.py@@ -1719,6 +1719,11 @@ async def skyvern_screenshot(session_id=ctx.session_id,)+ viewport = (+ {"viewport": {"width": result.viewport_css_px[0], "height": result.viewport_css_px[1]}}+ if result.viewport_css_px+ else {}+ )if inline:data_b64 = base64.b64encode(result.data).decode("utf-8")return action_result(@@ -1731,6 +1736,7 @@ async def skyvern_screenshot("mime": "image/png","bytes": len(result.data),"sdk_equivalent": "await page.screenshot()",+ **viewport,},artifacts=[artifact],timing_ms=timer.timing_ms,@@ -1740,7 +1746,7 @@ async def skyvern_screenshot(return action_result("skyvern_screenshot",browser_context=ctx,- data={"path": artifact.path, "sdk_equivalent": "await page.screenshot(path='screenshot.png')"},+ data={"path": artifact.path, "sdk_equivalent": "await page.screenshot(path='screenshot.png')", **viewport},artifacts=[artifact],timing_ms=timer.timing_ms,)diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.pyindex 58288b2d7..f37a2917a 100644--- a/skyvern/forge/sdk/copilot/credential_pause.py+++ b/skyvern/forge/sdk/copilot/credential_pause.py@@ -862,7 +862,10 @@ def signed_in_facts(signed_in: SignedInProfile) -> str:return (f"The user signed in to {signed_in.site} themselves in the live browser via the credential card. "f"Skyvern saved that sign-in as browser profile {defang_card_text(signed_in.profile_name)} "- f"({signed_in.browser_profile_id}) with {signed_in.cookie_count} cookies. No credential was provided."+ f"({signed_in.browser_profile_id}) with {signed_in.cookie_count} cookies. No credential was provided. "+ "This profile seeds this turn's tests only. Later turns and saved runs start from it only if it is set as "+ "the workflow's browser profile and the user accepts. Sites often end cookie-only sign-ins within minutes "+ "or hours, so saved runs may start signed out.")diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.pyindex 5da919e20..f04b058a8 100644--- a/skyvern/forge/sdk/copilot/enforcement.py+++ b/skyvern/forge/sdk/copilot/enforcement.py@@ -3,8 +3,12 @@from __future__ import annotationsimport asyncio+import base64import copy+import functools+import ioimport json+import mathimport reimport timefrom collections.abc import Mapping, Sequence@@ -20,6 +24,7 @@ from agents.run import Runnerfrom agents.run_config import RunConfigfrom agents.run_context import RunContextWrapperfrom openai.types.responses import ResponseFunctionToolCall+from PIL import Imagefrom skyvern.config import settingsfrom skyvern.forge.sdk.copilot import streaming_adapter@@ -178,10 +183,7 @@ _POST_RUN_PAGE_PATH_INTERACTION_BUDGET = 4_PRE_RUN_UNGATED_EVIDENCE_SOURCES = frozenset({"independent_run_evidence", "registered_output_parameter", "registered_artifact_content"})-# OpenAI detail=high cost per resized image. If we support other providers,-# pull from model config — this value will silently over/undercount otherwise.-# See screenshot_utils.resize_screenshot_b64 for the dimension contract this-# token count assumes.+# Charged for an image whose pixel size cannot be read.TOKENS_PER_RESIZED_IMAGE = 765# Keep the last N function_call_output items at full (head-truncated) size.@@ -481,7 +483,7 @@ def _same_page(left: str | None, right: str | None) -> bool:return left_path == right_path-def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:+def pending_screenshot_message(ctx: Any, *, sharp: bool = False) -> dict[str, Any] | None:"""Build the synthetic user message for the staged frame without draining it.Tool results stay text-only because OpenAI rejects images in tool@@ -495,8 +497,12 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:if not isinstance(pending, list) or not pending:return Nonescreenshots: list[ScreenshotEntry] = list(pending)+ images = [+ (entry.sharp_b64, entry.sharp_px) if sharp and entry.sharp_b64 else (entry.b64, entry.image_px)+ for entry in screenshots+ ]provenance_lines: list[str] = []- for entry in screenshots:+ for entry, (_, image_px) in zip(screenshots, images):provenance = entry.provenancefields = {"capture_id": entry.capture_id,@@ -512,6 +518,8 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:"session_binding": provenance.session_binding.value,"workflow_run_id": provenance.workflow_run_id or "unavailable","action_relation": provenance.action_relation.value,+ "image_px": _px(image_px),+ "viewport_css_px": _px(provenance.viewport_css_px),}rendered = "; ".join(f"{key}={value}" for key, value in fields.items())relation = (@@ -533,17 +541,21 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:"text": SCREENSHOT_SENTINEL + paired_marker + "\n".join(provenance_lines),},]- for entry in screenshots:+ for entry, (b64, _) in zip(screenshots, images):content.append({"type": "input_image",- "image_url": f"data:{entry.mime};base64,{entry.b64}",+ "image_url": f"data:{entry.mime};base64,{b64}","detail": "high",})return {"role": "user", "content": content}+def _px(size: tuple[int, int] | None) -> str:+ return f"{size[0]}x{size[1]}" if size else "unavailable"++def _consume_pending_screenshots(ctx: Any) -> dict[str, Any] | None:"""Build the screenshot message and clear the queue — the end-of-turn drain."""message = pending_screenshot_message(ctx)@@ -1115,32 +1127,49 @@ def _prune_input_list(items: list[Any]) -> list[Any]:return result[email protected]_cache(maxsize=16)+def _image_tokens(image_url: str) -> int:+ """OpenAI's patch formula x 1.2, which matched terra's billing at 1024x576 (692) and 1600x900 (1,741).+ `skyvern.utils.image_token_estimator` uses 1.0 for gpt-5.x and needs a model name, which this budget check lacks.+ """+ _, _, encoded = image_url.partition(";base64,")+ try:+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as image:+ width, height = image.size+ except Exception:+ return TOKENS_PER_RESIZED_IMAGE+ return math.floor(1.2 * math.ceil(width / 32) * math.ceil(height / 32)) + 1++def _sanitize_for_token_estimation(value: Any) -> tuple[Any, int]:"""Build a sanitized copy of *value*, replacing base64 image data witha short placeholder so blobs don't inflate the token count.- Returns ``(sanitized_value, image_count)``.+ Returns ``(sanitized_value, image_tokens)``."""if isinstance(value, dict):is_image = value.get("type") == "input_image"sanitized: dict[str, Any] = {}- image_count = 1 if is_image else 0+ image_tokens = 0+ if is_image:+ image_url = value.get("image_url")+ image_tokens = _image_tokens(image_url) if isinstance(image_url, str) else TOKENS_PER_RESIZED_IMAGEfor key, child in value.items():if is_image and key == "image_url":sanitized[key] = "[image]"continue- sanitized_child, child_images = _sanitize_for_token_estimation(child)+ sanitized_child, child_tokens = _sanitize_for_token_estimation(child)sanitized[key] = sanitized_child- image_count += child_images- return sanitized, image_count+ image_tokens += child_tokens+ return sanitized, image_tokensif isinstance(value, list):sanitized_list: list[Any] = []- image_count = 0+ image_tokens = 0for item in value:- sanitized_item, item_images = _sanitize_for_token_estimation(item)+ sanitized_item, item_tokens = _sanitize_for_token_estimation(item)sanitized_list.append(sanitized_item)- image_count += item_images- return sanitized_list, image_count+ image_tokens += item_tokens+ return sanitized_list, image_tokensreturn value, 0@@ -1148,9 +1177,9 @@ def estimate_tokens(items: list[Any]) -> int:"""Token estimate for an input list using tiktoken."""if not items:return 0- sanitized, image_count = _sanitize_for_token_estimation(items)+ sanitized, image_tokens = _sanitize_for_token_estimation(items)text = json.dumps(sanitized, separators=(",", ":"), ensure_ascii=False, default=str)- return count_tokens(text) + image_count * TOKENS_PER_RESIZED_IMAGE+ return count_tokens(text) + image_tokens_AGGRESSIVE_PRUNE_TAIL = 7diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.pyindex ae5dcf987..3a2eea6d1 100644--- a/skyvern/forge/sdk/copilot/mcp_adapter.py+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py@@ -87,6 +87,7 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (ScreenshotProvenance,capturing_tool_call,enqueue_screenshot_from_result,+ screenshot_viewport_css_px,)from skyvern.forge.sdk.copilot.secret_scrub import (encoded_secret_variants,@@ -2236,6 +2237,7 @@ class SkyvernOverlayMCPServer(MCPServer):if isinstance(workflow_run_id, str) and workflow_run_idelse None,action_relation=ScreenshotActionRelation.TOOL_RESULT,+ viewport_css_px=screenshot_viewport_css_px(copilot_result),),)diff --git a/skyvern/forge/sdk/copilot/screenshot_utils.py b/skyvern/forge/sdk/copilot/screenshot_utils.pyindex 77f658679..aa8d4e3a8 100644--- a/skyvern/forge/sdk/copilot/screenshot_utils.py+++ b/skyvern/forge/sdk/copilot/screenshot_utils.py@@ -22,6 +22,8 @@ from PIL import ImageCOPILOT_SCREENSHOT_MAX_WIDTH = 1024COPILOT_SCREENSHOT_MAX_HEIGHT = 768+COPILOT_SHARP_MAX_WIDTH = 1600+COPILOT_SHARP_MAX_HEIGHT = 1200COPILOT_JPEG_QUALITY = 60_BROWSER_ABLATION_MAX_SCREENSHOT_FRAMES = 24@@ -53,6 +55,7 @@ class ScreenshotProvenance:dispatch_browser_session_id: str | None = Noneproducer_browser_session_id: str | None = Nonesession_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE+ viewport_css_px: tuple[int, int] | None = None@classmethoddef unknown(cls, *, source_tool: str) -> ScreenshotProvenance:@@ -74,6 +77,10 @@ class ScreenshotEntry:provenance: ScreenshotProvenancecaptured_at: float = field(default_factory=time.monotonic)capture_event_id: str = field(default_factory=lambda: uuid.uuid4().hex)+ image_px: tuple[int, int] | None = None+ # Sent on every model call that carries this frame while it fits the budget; session history keeps b64.+ sharp_b64: str | None = None+ sharp_px: tuple[int, int] | None = None@dataclass(frozen=True)@@ -86,6 +93,7 @@ class CapturedFrame:dispatch_browser_session_id: str | None = Noneproducer_browser_session_id: str | None = Nonesession_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE+ viewport_css_px: tuple[int, int] | None = None@dataclass(frozen=True)@@ -153,6 +161,17 @@ def screenshot_result_facts(return producer_url, producer_session_id, binding+def screenshot_viewport_css_px(result: dict[str, Any]) -> tuple[int, int] | None:+ data = result.get("data")+ viewport = data.get("viewport") if isinstance(data, dict) else None+ if not isinstance(viewport, dict):+ return None+ width, height = viewport.get("width"), viewport.get("height")+ if type(width) is int and type(height) is int and width > 0 and height > 0:+ return width, height+ return None++def _fingerprint_projection(value: Any) -> Any:if hasattr(value, "model_dump"):value = value.model_dump(mode="json")@@ -182,34 +201,40 @@ def model_input_fingerprint(items: list[Any]) -> str:return hashlib.sha256(encoded).hexdigest()+def _fitted_jpeg_b64(img: Image.Image, box: tuple[int, int]) -> tuple[str, tuple[int, int]]:+ fitted = img.copy()+ fitted.thumbnail(box, Image.Resampling.LANCZOS)+ if fitted.mode != "RGB":+ fitted = fitted.convert("RGB")+ buf = io.BytesIO()+ fitted.save(buf, format="JPEG", quality=COPILOT_JPEG_QUALITY)+ return base64.b64encode(buf.getvalue()).decode("ascii"), fitted.size++def resize_screenshot_b64(b64_png: str,*,provenance: ScreenshotProvenance,captured_at: float | None = None,) -> ScreenshotEntry:- """Resize a base64 PNG to 1024x768 max and compress to JPEG.+ """Encode a frame as two JPEGs: a sharp copy fitted to 1600x1200 and a small copy fitted to 1024x768.- Uses thumbnail() to preserve aspect ratio — only shrinks, never enlarges.- Converts RGBA/P modes to RGB for JPEG compatibility.+ thumbnail() preserves aspect ratio and never enlarges."""raw = base64.b64decode(b64_png)capture_id = f"sha256:{hashlib.sha256(raw).hexdigest()}"- img = Image.open(io.BytesIO(raw))- img.thumbnail(- (COPILOT_SCREENSHOT_MAX_WIDTH, COPILOT_SCREENSHOT_MAX_HEIGHT),- Image.Resampling.LANCZOS,- )- if img.mode in ("RGBA", "P"):- img = img.convert("RGB")- buf = io.BytesIO()- img.save(buf, format="JPEG", quality=COPILOT_JPEG_QUALITY)+ with Image.open(io.BytesIO(raw)) as img:+ small_b64, small_px = _fitted_jpeg_b64(img, (COPILOT_SCREENSHOT_MAX_WIDTH, COPILOT_SCREENSHOT_MAX_HEIGHT))+ sharp_b64, sharp_px = _fitted_jpeg_b64(img, (COPILOT_SHARP_MAX_WIDTH, COPILOT_SHARP_MAX_HEIGHT))return ScreenshotEntry(- b64=base64.b64encode(buf.getvalue()).decode("ascii"),+ b64=small_b64,mime="image/jpeg",capture_id=capture_id,provenance=provenance,captured_at=captured_at if captured_at is not None else time.monotonic(),+ image_px=small_px,+ sharp_b64=sharp_b64,+ sharp_px=sharp_px,)@@ -310,6 +335,7 @@ class ViewportFrame:producer_session_id: str | Nonesession_binding: ProvenanceBindingstarted_at: float+ viewport_css_px: tuple[int, int] | None = Nonedef viewport_visible_effect(pre: ViewportFrame | None, post: ViewportFrame | None) -> VisibleEffect:diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.pyindex 6b81a2f46..7501a660a 100644--- a/skyvern/forge/sdk/copilot/session_factory.py+++ b/skyvern/forge/sdk/copilot/session_factory.py@@ -221,7 +221,8 @@ def _run_context(data: CallModelData[Any]) -> Any:def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget: int) -> ModelInputData:- items = list(data.model_data.input)+ # Compacted up front so the sharp-copy budget check below sees the input the budget filter will.+ items = _compact_tool_items(list(data.model_data.input))ctx = _run_context(data)# Read-only peek: the end-of-turn drain in enforcement stays the only clear, so a provider# retry or model fallback re-running this filter still carries the frame.@@ -253,6 +254,12 @@ def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget:input_fingerprint=fingerprint,)if screenshot_msg is not None:+ sharp_msg = pending_screenshot_message(ctx, sharp=True)+ # The sharp copy may cost older screenshots, which budget pruning drops first anyway, but nothing else.+ if sharp_msg is not None and estimate_tokens(_drop_older_screenshots([*items, sharp_msg])) <= token_budget:+ screenshot_msg = sharp_msg+ else:+ LOG.info("copilot_sharp_frame_over_budget")LOG.info("Injecting screenshot user message",count=len(screenshot_msg["content"]) - 1,@@ -379,13 +386,7 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budgereturn ModelInputData(input=items, instructions=instructions)# Layer 2: Drop all screenshots except the most recent- screenshot_indices = [i for i, item in enumerate(items) if is_screenshot_message(item)]- if len(screenshot_indices) > 1:- drop_indices = set(screenshot_indices[:-1])- items = [- {"role": "user", "content": SCREENSHOT_PLACEHOLDER} if i in drop_indices else item- for i, item in enumerate(items)- ]+ items = _drop_older_screenshots(items)est = estimate_tokens(items)if est <= token_budget:@@ -437,6 +438,15 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budgereturn ModelInputData(input=items, instructions=instructions)+def _drop_older_screenshots(items: list[Any]) -> list[Any]:+ screenshot_indices = [i for i, item in enumerate(items) if is_screenshot_message(item)]+ drop_indices = set(screenshot_indices[:-1])+ return [+ {"role": "user", "content": SCREENSHOT_PLACEHOLDER} if i in drop_indices else item+ for i, item in enumerate(items)+ ]++def _truncate_tool_output(item: Any, max_chars: int) -> Any:"""Truncate a function_call_output item's output if it exceeds max_chars."""if get_agent_message_field(item, "type") != "function_call_output":diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.pyindex 19e54e780..a0211c523 100644--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py@@ -67,6 +67,7 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (ScreenshotProvenance,enqueue_screenshot,screenshot_result_facts,+ screenshot_viewport_css_px,)from skyvern.forge.sdk.copilot.secret_scrub import (scrub_secrets_from_structure,@@ -422,6 +423,7 @@ async def _composition_capture_visual_summary(dispatch_browser_session_id=capture_session_id,producer_browser_session_id=producer_session_id,session_binding=session_binding,+ viewport_css_px=screenshot_viewport_css_px(screenshot_result),)if screenshot_b64else None@@ -1488,6 +1490,7 @@ async def _inspect_page_for_composition_under_custody(dispatch_browser_session_id=visual_fallback_frame.dispatch_browser_session_id,producer_browser_session_id=visual_fallback_frame.producer_browser_session_id,session_binding=visual_fallback_frame.session_binding,+ viewport_css_px=visual_fallback_frame.viewport_css_px,),captured_at=visual_fallback_frame.captured_at,)diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.pyindex b571ab7f4..cf86fa0d9 100644--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py@@ -1165,6 +1165,7 @@ async def _screenshot_post_hook("screenshot_base64": data.get("data", ""),"url": url,"title": title,+ **({"viewport": data["viewport"]} if "viewport" in data else {}),}return resultdiff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.pyindex 2ec2d2f61..23e7bbace 100644--- a/skyvern/forge/sdk/copilot/tools/run_execution.py+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py@@ -2772,6 +2772,7 @@ async def _capture_and_store_post_run_page(dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,producer_browser_session_id=captured_frame.producer_browser_session_id,session_binding=captured_frame.session_binding,+ viewport_css_px=captured_frame.viewport_css_px,),captured_at=captured_frame.captured_at,)@@ -3013,6 +3014,7 @@ async def _capture_dispatched_terminal_page_evidence(dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,producer_browser_session_id=captured_frame.producer_browser_session_id,session_binding=captured_frame.session_binding,+ viewport_css_px=captured_frame.viewport_css_px,),captured_at=captured_frame.captured_at,)diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.pyindex 9f1ae8568..7d0567b0e 100644--- a/skyvern/forge/sdk/copilot/tools/scouting.py+++ b/skyvern/forge/sdk/copilot/tools/scouting.py@@ -86,6 +86,7 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (consume_screenshot_artifact,enqueue_screenshot,screenshot_result_facts,+ screenshot_viewport_css_px,)from skyvern.forge.sdk.copilot.secret_scrub import (REDACTED_SECRET_PLACEHOLDER,@@ -559,6 +560,7 @@ async def _capture_post_interaction_screenshot(dispatch_browser_session_id=frame.dispatch_session_id,producer_browser_session_id=frame.producer_session_id,session_binding=frame.session_binding,+ viewport_css_px=frame.viewport_css_px,),captured_at=frame.started_at,)@@ -596,6 +598,7 @@ async def _take_viewport_frame(ctx: AgentContext, *, timeout_seconds: float) ->producer_session_id=producer_session_id,session_binding=session_binding,started_at=started_at,+ viewport_css_px=screenshot_viewport_css_px(result),)diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.pyindex 0ef834a56..c7c3184da 100644--- a/tests/unit/copilot_test_helpers.py+++ b/tests/unit/copilot_test_helpers.py@@ -3,6 +3,8 @@from __future__ import annotationsimport asyncio+import base64+import ioimport jsonfrom collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequencefrom contextlib import asynccontextmanager, suppress@@ -28,6 +30,7 @@ from openai.types.responses import ()from openai.types.responses.response import Responsefrom openai.types.responses.response_function_tool_call import ResponseFunctionToolCall+from PIL import Imagefrom playwright.async_api import Page, Route, async_playwrightfrom playwright.sync_api import sync_playwright@@ -63,6 +66,12 @@ from skyvern.forge.sdk.copilot.runtime import (AgentContext,OriginRunRedactionRegistry,)+from skyvern.forge.sdk.copilot.screenshot_utils import (+ ScreenshotActionRelation,+ ScreenshotEntry,+ ScreenshotProvenance,+ resize_screenshot_b64,+)from skyvern.forge.sdk.copilot.session_factory import (copilot_call_model_input_filter,copilot_session_input_callback,@@ -1356,6 +1365,33 @@ def carried_interaction(**fields: InteractionFieldValue) -> dict[str, Any]:return dict(fields)+def captured_frame_entry(width: int = 1920, height: int = 947) -> ScreenshotEntry:+ """A frame staged from a real PNG; each call is a new capture event."""+ buf = io.BytesIO()+ Image.new("RGB", (width, height), (40, 90, 160)).save(buf, format="PNG")+ return resize_screenshot_b64(+ base64.b64encode(buf.getvalue()).decode("ascii"),+ provenance=ScreenshotProvenance(+ source_tool="get_browser_screenshot",+ captured_url="https://example.com/page",+ observation_step=None,+ browser_session_id="pbs_123",+ workflow_run_id=None,+ action_relation=ScreenshotActionRelation.TOOL_RESULT,+ viewport_css_px=(width, height),+ ),+ )+++def sent_image_px(message: Any) -> list[tuple[int, int]]:+ content = message.get("content") if isinstance(message, dict) else None+ return [+ Image.open(io.BytesIO(base64.b64decode(part["image_url"].partition(";base64,")[2]))).size+ for part in (content if isinstance(content, list) else [])+ if part.get("type") == "input_image"+ ]++def make_model_input_data(items: list[Any], *, instructions: str | None = None, context: Any = None) -> Any:"""Build a fake CallModelData payload with a model_data.input list.diff --git a/tests/unit/test_cli_shared_core.py b/tests/unit/test_cli_shared_core.pyindex f5ec5d2a2..9dfa5e80d 100644--- a/tests/unit/test_cli_shared_core.py+++ b/tests/unit/test_cli_shared_core.py@@ -2,11 +2,15 @@from __future__ import annotations+import asyncio+import iofrom datetime import datetime, timezonefrom unittest.mock import AsyncMock, MagicMockimport pytest+from PIL import Imagefrom playwright.async_api import TimeoutError as PlaywrightTimeoutError+from playwright.async_api import async_playwrightfrom skyvern.cli.core import browser_opsfrom skyvern.cli.core.browser_ops import do_act, do_extract, do_navigate, do_screenshot, parse_extract_schema@@ -20,6 +24,7 @@ from skyvern.cli.core.guards import ()from skyvern.cli.core.session_ops import do_session_close, do_session_create, do_session_listfrom skyvern.client.types.extensions import Extensions+from tests.unit.copilot_test_helpers import skip_no_browserCAPTCHA_SOLVER_EXTENSION: Extensions = "captcha-solver"LOCALHOST_RECOVERY_HINT = (@@ -276,6 +281,42 @@ async def test_do_screenshot_full_page() -> None:assert result.full_page is True+async def test_do_screenshot_returns_the_capture_when_the_viewport_probe_never_answers(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ async def wedged_renderer(*_: object) -> None:+ await asyncio.Event().wait()++ monkeypatch.setattr(browser_ops, "_VIEWPORT_PROBE_TIMEOUT_SECONDS", 0.05)+ page = MagicMock()+ page.screenshot = AsyncMock(return_value=b"png-data")+ page.evaluate = wedged_renderer++ result = await do_screenshot(page)++ assert result.data == b"png-data"+ assert result.viewport_css_px is None+++@skip_no_browser+async def test_do_screenshot_reports_the_css_viewport_only_for_a_viewport_capture() -> None:+ async with async_playwright() as playwright:+ browser = await playwright.chromium.launch()+ try:+ page = await browser.new_page(viewport={"width": 800, "height": 600}, device_scale_factor=2)+ await page.set_content("<body style='height: 3000px'>tall</body>")+ viewport = await do_screenshot(page)+ full_page = await do_screenshot(page, full_page=True)+ finally:+ await browser.close()++ assert Image.open(io.BytesIO(viewport.data)).size == (1600, 1200)+ assert viewport.viewport_css_px == (800, 600)+ assert full_page.viewport_css_px is None++@pytest.mark.asyncioasync def test_do_screenshot_with_selector() -> None:page = MagicMock()diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.pyindex 4c5a6c381..baa2cc520 100644--- a/tests/unit/test_copilot_credential_pause.py+++ b/tests/unit/test_copilot_credential_pause.py@@ -2999,6 +2999,9 @@ def test_the_raw_secret_card_sees_only_the_origin(user_url: str, origin: str) ->assert credential_pause_module.raw_secret_card_origin(user_url) == origin+_SIGNED_IN_SCOPE_PHRASES = ("this turn's tests only", "start signed out")++@pytest.mark.asyncioasync def test_a_run_derived_card_answered_by_signing_in_resumes_with_facts_and_does_not_fire_again(monkeypatch: pytest.MonkeyPatch,@@ -3029,7 +3032,9 @@ async def test_a_run_derived_card_answered_by_signing_in_resumes_with_facts_and_resume_msgs = await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config)assert resume_msgs is not None- assert credential_pause_module.signed_in_facts(signed_in) in json.dumps(resume_msgs)+ resume_text = json.dumps(resume_msgs)+ assert credential_pause_module.signed_in_facts(signed_in) in resume_text+ assert all(phrase in resume_text for phrase in _SIGNED_IN_SCOPE_PHRASES)assert ctx.credential_pause_outcome == "signed_in"assert _turn_credential_prompt_reason(ctx) is Noneassert await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config) is None@@ -3052,7 +3057,7 @@ async def test_a_card_answered_by_signing_in_tells_the_tool_what_happened(monkeyresult = await _call_ask_tool(ctx)assert (result["status"], result["browser_profile_id"]) == ("signed_in", "bp_signed_in")- assert result["detail"]+ assert all(phrase in result["detail"] for phrase in _SIGNED_IN_SCOPE_PHRASES)_REGISTRATION = {diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.pyindex c1d516ff3..6fa9aaff4 100644--- a/tests/unit/test_copilot_hooks.py+++ b/tests/unit/test_copilot_hooks.py@@ -3,6 +3,7 @@from __future__ import annotationsimport asyncio+import ioimport jsonimport timefrom collections.abc import AsyncIterator, Callable@@ -15,8 +16,10 @@ from unittest.mock import AsyncMock, MagicMock, callimport pytestimport yaml+from PIL import Imagefrom structlog.testing import capture_logs+from skyvern.cli.core.browser_ops import ScreenshotResultfrom skyvern.config import settingsfrom skyvern.forge import appfrom skyvern.forge.sdk.copilot import tools as tools_module@@ -819,7 +822,9 @@ class TestMCPFailedStepLoopDetection:observed_session_ids.append(current.context.session_id if current.context else None)observed_localhost_access.append(current.context.can_access_localhost if current.context else None)assert current.api_key_hash == session_manager._api_key_hash("sk-copilot-org")- return SimpleNamespace(data=b"fake-png")+ png = io.BytesIO()+ Image.new("RGB", (1920, 947)).save(png, format="PNG")+ return ScreenshotResult(data=png.getvalue(), viewport_css_px=(1920, 947))monkeypatch.setattr(browser_tools, "do_screenshot", fake_do_screenshot)@@ -858,6 +863,7 @@ class TestMCPFailedStepLoopDetection:assert result.isError is Falseassert parsed["ok"] is Trueassert parsed["data"]["screenshot_base64"]+ assert [entry.provenance.viewport_css_px for entry in ctx.pending_screenshots] == [(1920, 947)]assert observed_session_ids == ["pbs_copilot"]assert observed_localhost_access == [True]fallback_skyvern.connect_to_cloud_browser_session.assert_not_awaited()diff --git a/tests/unit/test_copilot_screenshot_handling.py b/tests/unit/test_copilot_screenshot_handling.pyindex 50864baf3..77ee37343 100644--- a/tests/unit/test_copilot_screenshot_handling.py+++ b/tests/unit/test_copilot_screenshot_handling.py@@ -37,9 +37,11 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (from skyvern.forge.sdk.copilot.tools.scouting import _capture_post_interaction_screenshotfrom skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidencefrom tests.unit.copilot_test_helpers import (+ captured_frame_entry,install_get_run_results_harness,make_model_input_data,run_result_block_row,+ sent_image_px,)@@ -363,6 +365,21 @@ class TestCapturePostInteractionScreenshot:assert width <= COPILOT_SCREENSHOT_MAX_WIDTHassert height <= COPILOT_SCREENSHOT_MAX_HEIGHT+ @pytest.mark.asyncio+ async def test_delivered_frame_states_its_image_size_and_the_measured_viewport(self, tmp_path: Path) -> None:+ frame = tmp_path / "frame.png"+ Image.new("RGB", (1920, 947), (10, 120, 200)).save(frame, format="PNG")+ ctx = self._ctx({"ok": True, "data": {"path": str(frame), "viewport": {"width": 1920, "height": 947}}})++ assert await _capture_post_interaction_screenshot(ctx, source_tool="click", captured_url=None) is True+ result = copilot_call_model_input_filter(+ make_model_input_data([{"role": "user", "content": "go"}], context=ctx)+ )++ message = result.input[-1]+ assert sent_image_px(message) == [(1600, 789)]+ assert "image_px=1600x789; viewport_css_px=1920x947" in message["content"][0]["text"]+@pytest.mark.parametrize("data",[@@ -475,6 +492,18 @@ class TestNudgeDrainAndFilterExclusivity:assert result.input == items+ def test_a_frame_sent_sharp_is_saved_small_with_a_nudge(self) -> None:+ ctx = SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)+ items = [{"role": "user", "content": "read the reference code"}]+ seen = copilot_call_model_input_filter(make_model_input_data(items, context=ctx))+ saved = _consume_pending_screenshots(ctx)+ history = [*items, {"role": "assistant", "content": "done"}, saved]++ after = copilot_call_model_input_filter(make_model_input_data(history, context=ctx))++ assert sent_image_px(seen.input[-1]) == [(1600, 789)]+ assert sent_image_px(after.input[-1]) == [(1024, 505)]+def test_the_nudge_path_also_withholds_the_frame_from_a_non_vision_model(self) -> None:# The vision check lives in the shared builder so every delivery path inherits it;# the drain still empties the queue.diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.pyindex db8a2ec18..a071da696 100644--- a/tests/unit/test_copilot_session_factory.py+++ b/tests/unit/test_copilot_session_factory.py@@ -9,6 +9,7 @@ from types import SimpleNamespacefrom typing import Anyimport pytest+from agents.run_config import ModelInputDatafrom structlog.testing import capture_logsfrom skyvern.forge.sdk.copilot.ask_user import (@@ -19,7 +20,7 @@ from skyvern.forge.sdk.copilot.ask_user import (QuestionResponse,)from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy-from skyvern.forge.sdk.copilot.enforcement import estimate_tokens+from skyvern.forge.sdk.copilot.enforcement import estimate_tokens, pending_screenshot_messagefrom skyvern.forge.sdk.copilot.screenshot_utils import (ScreenshotActionRelation,ScreenshotEntry,@@ -30,8 +31,9 @@ from skyvern.forge.sdk.copilot.session_factory import (copilot_session_input_callback,make_copilot_call_model_input_filter,)-from tests.unit.copilot_test_helpers import make_copilot_ctx+from tests.unit.copilot_test_helpers import captured_frame_entry, make_copilot_ctxfrom tests.unit.copilot_test_helpers import make_model_input_data as _mk_input_data+from tests.unit.copilot_test_helpers import sent_image_pxdef _unread_run_results_batch() -> list[dict[str, Any]]:@@ -705,6 +707,45 @@ class TestStagedScreenshotBinding:assert len(_image_parts(delivered.input[-1])) == 1assert ctx.pending_frame_lease.capture_event_id == recaptured.capture_event_id+ def test_a_sharp_frame_over_budget_costs_older_screenshots_first_and_then_falls_back_to_the_small_copy(+ self,+ ) -> None:+ older_frame = pending_screenshot_message(+ SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)+ )+ items = [+ {"role": "user", "content": "what does the page say"},+ older_frame,+ {"type": "function_call", "call_id": "read-1", "name": "evaluate", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "read-1", "output": '{"text":"order 4412 shipped"}'},+ ]++ def call(token_budget: int) -> ModelInputData:+ ctx = SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)+ return make_copilot_call_model_input_filter(token_budget=token_budget)(_mk_input_data(items, context=ctx))++ for token_budget, new_frame_px in ((2_200, (1600, 789)), (1_200, (1024, 505))):+ result = call(token_budget)+ assert [sent_image_px(item) for item in result.input] == [[], [], [], [], [new_frame_px]]+ assert result.input[2:4] == items[2:4]++ def test_the_pending_frame_is_sent_sharp_on_every_call_that_carries_it(self) -> None:+ ctx = SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)+ initial = [{"role": "user", "content": "what does the page say"}]+ advanced = [+ *initial,+ {"type": "function_call", "call_id": "read-1", "name": "evaluate", "arguments": "{}"},+ {"type": "function_call_output", "call_id": "read-1", "output": '{"ok":true}'},+ ]++ first = copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))+ later = copilot_call_model_input_filter(_mk_input_data(advanced, context=ctx))++ for result in (first, later):+ message = result.input[-1]+ assert sent_image_px(message) == [(1600, 789)]+ assert "image_px=1600x789; viewport_css_px=1920x947" in message["content"][0]["text"]+def test_paired_message_has_machine_readable_marker(self) -> None:ctx = _ctx_with_staged_frame()diff --git a/tests/unit/test_copilot_token_accounting.py b/tests/unit/test_copilot_token_accounting.pyindex 8a3a51913..86265cbe3 100644--- a/tests/unit/test_copilot_token_accounting.py+++ b/tests/unit/test_copilot_token_accounting.py@@ -2,11 +2,14 @@from __future__ import annotations+import base64+import iofrom unittest.mock import MagicMockimport pytestfrom agents.run_context import RunContextWrapperfrom agents.usage import InputTokensDetails, OutputTokensDetails, Usage+from PIL import Imagefrom skyvern.forge.sdk.copilot.context import CopilotContextfrom skyvern.forge.sdk.copilot.enforcement import (@@ -87,6 +90,20 @@ class TestEstimateTokens:# so text tokens should be well under 100.assert text_portion < 100+ @pytest.mark.parametrize(("width", "height", "billed"), [(1024, 576, 692), (1600, 900, 1741)])+ def test_image_estimate_follows_pixel_size(self, width: int, height: int, billed: int) -> None:+ buf = io.BytesIO()+ Image.new("RGB", (width, height)).save(buf, format="JPEG")+ item = {+ "type": "input_image",+ "image_url": "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode(),+ "detail": "high",+ }++ _, image_tokens = _sanitize_for_token_estimation(item)++ assert image_tokens == billed+def test_image_metadata_preserved(self):"""type and detail fields should be included, only image_url replaced."""item = {@@ -94,8 +111,8 @@ class TestEstimateTokens:"image_url": "data:image/png;base64," + "X" * 1000,"detail": "high",}- sanitized, count = _sanitize_for_token_estimation(item)- assert count == 1+ sanitized, image_tokens = _sanitize_for_token_estimation(item)+ assert image_tokens == TOKENS_PER_RESIZED_IMAGEassert sanitized["type"] == "input_image"assert sanitized["detail"] == "high"assert sanitized["image_url"] == "[image]"diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.pyindex a47c28bb1..b6b8f8fc6 100644--- a/tests/unit/test_mcp_browser_tools.py+++ b/tests/unit/test_mcp_browser_tools.py@@ -4553,7 +4553,7 @@ async def test_navigating_paired_tools_invalidate_refs_around_navigation(return NavigateResult(url="https://example.test", title="Example", load_state="load")monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=navigate))- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=browser_ops.ScreenshotResult(data=b"png")))monkeypatch.setattr(mcp_browser, "do_extract", AsyncMock(return_value=SimpleNamespace(extracted={"value": 1})))monkeypatch.setattr(mcp_browser,@@ -4618,7 +4618,7 @@ async def test_navigating_paired_tools_surface_the_degraded_load_state("do_navigate",AsyncMock(return_value=NavigateResult(url="https://example.test", title="Example", load_state="commit")),)- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=browser_ops.ScreenshotResult(data=b"png")))monkeypatch.setattr(mcp_browser,"save_artifact",@@ -4940,7 +4940,7 @@ async def test_paired_read_failure_keeps_screenshot_and_reports_failure(@pytest.mark.asyncioasync def test_inline_screenshot_is_also_persisted(monkeypatch: pytest.MonkeyPatch) -> None:_sdk_equivalent_page(monkeypatch)- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=browser_ops.ScreenshotResult(data=b"png")))artifact = Artifact(kind="screenshot", path="/tmp/shot.png", mime="image/png", bytes=3)monkeypatch.setattr(mcp_browser, "save_artifact", Mock(return_value=artifact))@@ -5376,7 +5376,7 @@ async def test_screenshot_page_change_retry(monkeypatch: pytest.MonkeyPatch, mesctx = BrowserContext(mode="local")state = session_manager.SessionState(context=ctx, _active_page=page)patch_get_page(monkeypatch, mcp_browser, page, ctx)- read = AsyncMock(side_effect=[RuntimeError(message), SimpleNamespace(data=b"png")])+ read = AsyncMock(side_effect=[RuntimeError(message), browser_ops.ScreenshotResult(data=b"png")])monkeypatch.setattr(mcp_browser, "do_screenshot", read)monkeypatch.setattr(mcp_browser,@@ -5481,7 +5481,7 @@ async def test_read_tools_retry_backoff(monkeypatch: pytest.MonkeyPatch, tool: sif attempts <= 6:raise RuntimeError("Execution context was destroyed")if tool == "screenshot":- return SimpleNamespace(data=b"png")+ return browser_ops.ScreenshotResult(data=b"png")if tool == "get_html":return "<p>fresh</p>"return {"ok": True, "data": {"fresh": True}}diff --git a/tests/unit/test_mcp_response_size_cap.py b/tests/unit/test_mcp_response_size_cap.pyindex a67225a04..2b2247b01 100644--- a/tests/unit/test_mcp_response_size_cap.py+++ b/tests/unit/test_mcp_response_size_cap.py@@ -11,7 +11,7 @@ from unittest.mock import AsyncMock, Mockimport pytest-from skyvern.cli.core.browser_ops import NavigateResult+from skyvern.cli.core.browser_ops import NavigateResult, ScreenshotResultfrom skyvern.cli.core.result import Artifact, BrowserContextfrom skyvern.cli.mcp_tools import browser as mcp_browserfrom skyvern.cli.mcp_tools import mcp@@ -394,7 +394,7 @@ async def test_every_registered_extract_tool_caps_an_oversize_extraction("do_navigate",AsyncMock(return_value=NavigateResult(url="https://example.test", title="Example")),)- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=ScreenshotResult(data=b"png")))monkeypatch.setattr(mcp_browser,"save_artifact",@@ -428,7 +428,7 @@ def _stub_browser(monkeypatch: pytest.MonkeyPatch) -> None:"do_navigate",AsyncMock(return_value=NavigateResult(url="https://example.test", title="Example")),)- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=ScreenshotResult(data=b"png")))monkeypatch.setattr(mcp_browser,"save_artifact",@@ -443,7 +443,7 @@ async def test_navigate_and_screenshot_caps_an_oversize_inline_screenshot(monkeytool-result limit with no truncation warning."""_stub_browser(monkeypatch)oversize_png = b"\x89PNG" + b"z" * MCP_MAX_RESPONSE_CHARS- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=oversize_png)))+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=ScreenshotResult(data=oversize_png)))tool = await mcp.get_tool("skyvern_navigate_and_screenshot")result = await tool.fn(url="https://example.test", inline=True)