Skip to content

Automate browser based workflows with AI

Copilot: send the current screenshot sharp on every call, keep history small (SKY-18247) (#8905)

Andrew Neilson4e533b512225
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx
index 300c25716..ede29cc82 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.test.tsx
@@ -344,7 +344,16 @@ describe("CredentialCard sign in myself", () => {
...overrides,
});
- it("offers signing in yourself only when the chat passes an offer", () => {
+ const SIGN_IN_LABEL = "Sign in myself (this test only)";
+ const SCOPE_WARNING =
+ "Signing in yourself works for this test. Later runs usually won't stay signed in — connect a credential so every run can sign in.";
+ const follows = (later: Element, earlier: Element) =>
+ Boolean(
+ earlier.compareDocumentPosition(later) & Node.DOCUMENT_POSITION_FOLLOWING,
+ );
+
+ it("offers signing in yourself only when the chat passes an offer", async () => {
+ credsData.current = [{ credential_id: "cred_hn", name: "HN login" }];
const offer = signIn();
render(
<CredentialCard
@@ -355,7 +364,17 @@ describe("CredentialCard sign in myself", () => {
signIn={offer}
/>,
);
- fireEvent.click(screen.getByRole("button", { name: "Sign in myself" }));
+ const picker = await screen.findByRole("combobox");
+ const control = screen.getByRole("button", { name: SIGN_IN_LABEL });
+ expect(control.className).not.toContain("bg-cta");
+ expect(control.className).not.toContain("border-input");
+ const connect = screen.getByRole("button", { name: "Connect credential" });
+ const skip = screen.getByRole("button", { name: "Skip for now" });
+ expect(follows(control, connect)).toBe(true);
+ expect(follows(control, picker)).toBe(true);
+ expect(follows(skip, control)).toBe(true);
+ expect(screen.getByText(SCOPE_WARNING)).toBeTruthy();
+ fireEvent.click(control);
expect(offer.onStart).toHaveBeenCalledTimes(1);
cleanup();
@@ -367,7 +386,8 @@ describe("CredentialCard sign in myself", () => {
onSkip={vi.fn()}
/>,
);
- expect(screen.queryByRole("button", { name: "Sign in myself" })).toBeNull();
+ expect(screen.queryByRole("button", { name: SIGN_IN_LABEL })).toBeNull();
+ expect(screen.queryByText(SCOPE_WARNING)).toBeNull();
});
it("while signing in, Done finishes, Cancel skips, and a Done that found nothing says so", () => {
@@ -385,6 +405,11 @@ describe("CredentialCard sign in myself", () => {
expect(
screen.getAllByText(/No sign-in found for portal\.example\.com/),
).not.toHaveLength(0);
+ expect(
+ screen.getByText(
+ "Use the browser to sign in, including any verification code, then click Done. This sign-in is for this test; later runs usually won't stay signed in. No password is stored.",
+ ),
+ ).toBeTruthy();
fireEvent.click(screen.getByRole("button", { name: "Done" }));
fireEvent.click(screen.getByRole("button", { name: "Cancel" }));
expect(offer.onDone).toHaveBeenCalledTimes(1);
@@ -423,8 +448,13 @@ describe("CredentialCard sign in myself", () => {
onSkip={vi.fn()}
/>,
);
+ expect(screen.getByText("Signed in for this test only")).toBeTruthy();
+ expect(screen.getByText(/saved as 'Sign-in for portal'/)).toBeTruthy();
+ fireEvent.click(screen.getByRole("button", { expanded: false }));
expect(
- screen.getByText("Signed in, saved as 'Sign-in for portal'"),
+ screen.getByText(
+ "For this test · later runs usually won't stay signed in · no password stored",
+ ),
).toBeTruthy();
});
});
diff --git a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
index e0be00b30..33b178b32 100644
--- a/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
+++ b/skyvern-frontend/src/routes/workflows/copilot/cards/CredentialCard.tsx
@@ -222,7 +222,7 @@ const UPDATE_SKIP_OUTCOME = {
detail: "Credential not updated — the workflow keeps its saved sign-in.",
};
const SIGNED_IN_DETAIL =
- "No password stored · your sign-in is saved as browser cookies in this profile";
+ "For this test · later runs usually won't stay signed in · no password stored";
const TIMEOUT_OUTCOME = {
title: "Sign-in request timed out",
meta: "test may stop at login",
@@ -995,10 +995,11 @@ function CredentialAskCard({
return (
<ResolvedCredentialCard
tone="done"
- title={
+ title="Signed in for this test only"
+ meta={
resolvedOutcome.name
- ? `Signed in, saved as '${resolvedOutcome.name}'`
- : "Signed in, saved as a browser profile"
+ ? `saved as '${resolvedOutcome.name}'`
+ : "saved as a browser profile"
}
detail={SIGNED_IN_DETAIL}
/>
@@ -1181,9 +1182,9 @@ function CredentialAskCard({
key="how"
className="text-[11px] leading-relaxed text-muted-foreground"
>
- Use the browser to sign in, including any verification code, then
- click Done. Skyvern saves the sign-in as a browser profile; no
- password is stored.
+ {
+ "Use the browser to sign in, including any verification code, then click Done. This sign-in is for this test; later runs usually won't stay signed in. No password is stored."
+ }
</span>,
...(notFound
? [
@@ -1263,6 +1264,18 @@ function CredentialAskCard({
</span>,
]
: []),
+ ...(signIn
+ ? [
+ <span
+ key="sign-in-scope"
+ className="text-[11px] leading-relaxed text-muted-foreground"
+ >
+ {
+ "Signing in yourself works for this test. Later runs usually won't stay signed in — connect a credential so every run can sign in."
+ }
+ </span>,
+ ]
+ : []),
...(mode === "terminal"
? [
<span
@@ -1305,17 +1318,6 @@ function CredentialAskCard({
</Button>
) : null}
{connectButton}
- {signIn ? (
- <Button
- type="button"
- size="sm"
- variant="outline"
- disabled={disabled || signIn.busy}
- onClick={signIn.onStart}
- >
- Sign in myself
- </Button>
- ) : null}
{orgCredentials.status === "ready" &&
(pickable.length > 0 || pickerEngaged) ? (
<CredentialPicker
@@ -1344,6 +1346,18 @@ function CredentialAskCard({
{retryButton}
</>
) : null}
+ {signIn ? (
+ <Button
+ type="button"
+ size="sm"
+ variant="link"
+ className="text-blue-600 dark:text-blue-400"
+ disabled={disabled || signIn.busy}
+ onClick={signIn.onStart}
+ >
+ Sign in myself (this test only)
+ </Button>
+ ) : null}
<div className="ml-auto">
<SkipButton onSkip={onSkip} disabled={disabled} />
</div>
diff --git a/skyvern/cli/core/browser_ops.py b/skyvern/cli/core/browser_ops.py
index c01bad079..bb79ca002 100644
--- a/skyvern/cli/core/browser_ops.py
+++ b/skyvern/cli/core/browser_ops.py
@@ -134,6 +134,7 @@ LOCALHOST_RECOVERY_HINT = (
# reporting a failure for a page that is already loaded and usable.
_LOAD_STATE_LADDER = ("commit", "domcontentloaded", "load", "networkidle")
_NAVIGATE_TITLE_TIMEOUT_SECONDS = 5.0
+_VIEWPORT_PROBE_TIMEOUT_SECONDS = 2.0
_UNBOUNDED_LEG_TIMEOUT_MS = 1000
@@ -148,6 +149,7 @@ class NavigateResult:
class ScreenshotResult:
data: bytes
full_page: bool = False
+ viewport_css_px: tuple[int, int] | None = None
@dataclass
@@ -281,6 +283,8 @@ async def do_screenshot(
data = await element.screenshot()
else:
data = await page.screenshot(full_page=full_page)
+ # Reported only when the image shows exactly the viewport, so image size over it is a valid scale.
+ viewport_css_px = None if selector or full_page else await _viewport_css_px(page)
except asyncio.CancelledError:
cancelled = True
raise
@@ -293,7 +297,20 @@ async def do_screenshot(
restore.add_done_callback(_CURSOR_RESTORES.discard)
if not cancelled:
await restore
- return ScreenshotResult(data=data, full_page=full_page)
+ return ScreenshotResult(data=data, full_page=full_page, viewport_css_px=viewport_css_px)
+
+
+async def _viewport_css_px(page: Any) -> tuple[int, int] | None:
+ # Screenshots are taken at device scale, so the image's pixel size is not the CSS viewport on a HiDPI display.
+ try:
+ size = await asyncio.wait_for(
+ page.evaluate("() => [window.innerWidth, window.innerHeight]"), timeout=_VIEWPORT_PROBE_TIMEOUT_SECONDS
+ )
+ except Exception:
+ return None
+ if isinstance(size, list) and len(size) == 2 and all(type(value) is int and value > 0 for value in size):
+ return size[0], size[1]
+ return None
async def do_act(
diff --git a/skyvern/cli/mcp_tools/browser.py b/skyvern/cli/mcp_tools/browser.py
index 970e0bc1a..73f3463b5 100644
--- a/skyvern/cli/mcp_tools/browser.py
+++ b/skyvern/cli/mcp_tools/browser.py
@@ -1719,6 +1719,11 @@ async def skyvern_screenshot(
session_id=ctx.session_id,
)
+ viewport = (
+ {"viewport": {"width": result.viewport_css_px[0], "height": result.viewport_css_px[1]}}
+ if result.viewport_css_px
+ else {}
+ )
if inline:
data_b64 = base64.b64encode(result.data).decode("utf-8")
return action_result(
@@ -1731,6 +1736,7 @@ async def skyvern_screenshot(
"mime": "image/png",
"bytes": len(result.data),
"sdk_equivalent": "await page.screenshot()",
+ **viewport,
},
artifacts=[artifact],
timing_ms=timer.timing_ms,
@@ -1740,7 +1746,7 @@ async def skyvern_screenshot(
return action_result(
"skyvern_screenshot",
browser_context=ctx,
- data={"path": artifact.path, "sdk_equivalent": "await page.screenshot(path='screenshot.png')"},
+ data={"path": artifact.path, "sdk_equivalent": "await page.screenshot(path='screenshot.png')", **viewport},
artifacts=[artifact],
timing_ms=timer.timing_ms,
)
diff --git a/skyvern/forge/sdk/copilot/credential_pause.py b/skyvern/forge/sdk/copilot/credential_pause.py
index 58288b2d7..f37a2917a 100644
--- a/skyvern/forge/sdk/copilot/credential_pause.py
+++ b/skyvern/forge/sdk/copilot/credential_pause.py
@@ -862,7 +862,10 @@ def signed_in_facts(signed_in: SignedInProfile) -> str:
return (
f"The user signed in to {signed_in.site} themselves in the live browser via the credential card. "
f"Skyvern saved that sign-in as browser profile {defang_card_text(signed_in.profile_name)} "
- f"({signed_in.browser_profile_id}) with {signed_in.cookie_count} cookies. No credential was provided."
+ f"({signed_in.browser_profile_id}) with {signed_in.cookie_count} cookies. No credential was provided. "
+ "This profile seeds this turn's tests only. Later turns and saved runs start from it only if it is set as "
+ "the workflow's browser profile and the user accepts. Sites often end cookie-only sign-ins within minutes "
+ "or hours, so saved runs may start signed out."
)
diff --git a/skyvern/forge/sdk/copilot/enforcement.py b/skyvern/forge/sdk/copilot/enforcement.py
index 5da919e20..f04b058a8 100644
--- a/skyvern/forge/sdk/copilot/enforcement.py
+++ b/skyvern/forge/sdk/copilot/enforcement.py
@@ -3,8 +3,12 @@
from __future__ import annotations
import asyncio
+import base64
import copy
+import functools
+import io
import json
+import math
import re
import time
from collections.abc import Mapping, Sequence
@@ -20,6 +24,7 @@ from agents.run import Runner
from agents.run_config import RunConfig
from agents.run_context import RunContextWrapper
from openai.types.responses import ResponseFunctionToolCall
+from PIL import Image
from skyvern.config import settings
from skyvern.forge.sdk.copilot import streaming_adapter
@@ -178,10 +183,7 @@ _POST_RUN_PAGE_PATH_INTERACTION_BUDGET = 4
_PRE_RUN_UNGATED_EVIDENCE_SOURCES = frozenset(
{"independent_run_evidence", "registered_output_parameter", "registered_artifact_content"}
)
-# OpenAI detail=high cost per resized image. If we support other providers,
-# pull from model config — this value will silently over/undercount otherwise.
-# See screenshot_utils.resize_screenshot_b64 for the dimension contract this
-# token count assumes.
+# Charged for an image whose pixel size cannot be read.
TOKENS_PER_RESIZED_IMAGE = 765
# Keep the last N function_call_output items at full (head-truncated) size.
@@ -481,7 +483,7 @@ def _same_page(left: str | None, right: str | None) -> bool:
return left_path == right_path
-def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:
+def pending_screenshot_message(ctx: Any, *, sharp: bool = False) -> dict[str, Any] | None:
"""Build the synthetic user message for the staged frame without draining it.
Tool results stay text-only because OpenAI rejects images in tool
@@ -495,8 +497,12 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:
if not isinstance(pending, list) or not pending:
return None
screenshots: list[ScreenshotEntry] = list(pending)
+ images = [
+ (entry.sharp_b64, entry.sharp_px) if sharp and entry.sharp_b64 else (entry.b64, entry.image_px)
+ for entry in screenshots
+ ]
provenance_lines: list[str] = []
- for entry in screenshots:
+ for entry, (_, image_px) in zip(screenshots, images):
provenance = entry.provenance
fields = {
"capture_id": entry.capture_id,
@@ -512,6 +518,8 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:
"session_binding": provenance.session_binding.value,
"workflow_run_id": provenance.workflow_run_id or "unavailable",
"action_relation": provenance.action_relation.value,
+ "image_px": _px(image_px),
+ "viewport_css_px": _px(provenance.viewport_css_px),
}
rendered = "; ".join(f"{key}={value}" for key, value in fields.items())
relation = (
@@ -533,17 +541,21 @@ def pending_screenshot_message(ctx: Any) -> dict[str, Any] | None:
"text": SCREENSHOT_SENTINEL + paired_marker + "\n".join(provenance_lines),
},
]
- for entry in screenshots:
+ for entry, (b64, _) in zip(screenshots, images):
content.append(
{
"type": "input_image",
- "image_url": f"data:{entry.mime};base64,{entry.b64}",
+ "image_url": f"data:{entry.mime};base64,{b64}",
"detail": "high",
}
)
return {"role": "user", "content": content}
+def _px(size: tuple[int, int] | None) -> str:
+ return f"{size[0]}x{size[1]}" if size else "unavailable"
+
+
def _consume_pending_screenshots(ctx: Any) -> dict[str, Any] | None:
"""Build the screenshot message and clear the queue — the end-of-turn drain."""
message = pending_screenshot_message(ctx)
@@ -1115,32 +1127,49 @@ def _prune_input_list(items: list[Any]) -> list[Any]:
return result
[email protected]_cache(maxsize=16)
+def _image_tokens(image_url: str) -> int:
+ """OpenAI's patch formula x 1.2, which matched terra's billing at 1024x576 (692) and 1600x900 (1,741).
+ `skyvern.utils.image_token_estimator` uses 1.0 for gpt-5.x and needs a model name, which this budget check lacks.
+ """
+ _, _, encoded = image_url.partition(";base64,")
+ try:
+ with Image.open(io.BytesIO(base64.b64decode(encoded))) as image:
+ width, height = image.size
+ except Exception:
+ return TOKENS_PER_RESIZED_IMAGE
+ return math.floor(1.2 * math.ceil(width / 32) * math.ceil(height / 32)) + 1
+
+
def _sanitize_for_token_estimation(value: Any) -> tuple[Any, int]:
"""Build a sanitized copy of *value*, replacing base64 image data with
a short placeholder so blobs don't inflate the token count.
- Returns ``(sanitized_value, image_count)``.
+ Returns ``(sanitized_value, image_tokens)``.
"""
if isinstance(value, dict):
is_image = value.get("type") == "input_image"
sanitized: dict[str, Any] = {}
- image_count = 1 if is_image else 0
+ image_tokens = 0
+ if is_image:
+ image_url = value.get("image_url")
+ image_tokens = _image_tokens(image_url) if isinstance(image_url, str) else TOKENS_PER_RESIZED_IMAGE
for key, child in value.items():
if is_image and key == "image_url":
sanitized[key] = "[image]"
continue
- sanitized_child, child_images = _sanitize_for_token_estimation(child)
+ sanitized_child, child_tokens = _sanitize_for_token_estimation(child)
sanitized[key] = sanitized_child
- image_count += child_images
- return sanitized, image_count
+ image_tokens += child_tokens
+ return sanitized, image_tokens
if isinstance(value, list):
sanitized_list: list[Any] = []
- image_count = 0
+ image_tokens = 0
for item in value:
- sanitized_item, item_images = _sanitize_for_token_estimation(item)
+ sanitized_item, item_tokens = _sanitize_for_token_estimation(item)
sanitized_list.append(sanitized_item)
- image_count += item_images
- return sanitized_list, image_count
+ image_tokens += item_tokens
+ return sanitized_list, image_tokens
return value, 0
@@ -1148,9 +1177,9 @@ def estimate_tokens(items: list[Any]) -> int:
"""Token estimate for an input list using tiktoken."""
if not items:
return 0
- sanitized, image_count = _sanitize_for_token_estimation(items)
+ sanitized, image_tokens = _sanitize_for_token_estimation(items)
text = json.dumps(sanitized, separators=(",", ":"), ensure_ascii=False, default=str)
- return count_tokens(text) + image_count * TOKENS_PER_RESIZED_IMAGE
+ return count_tokens(text) + image_tokens
_AGGRESSIVE_PRUNE_TAIL = 7
diff --git a/skyvern/forge/sdk/copilot/mcp_adapter.py b/skyvern/forge/sdk/copilot/mcp_adapter.py
index ae5dcf987..3a2eea6d1 100644
--- a/skyvern/forge/sdk/copilot/mcp_adapter.py
+++ b/skyvern/forge/sdk/copilot/mcp_adapter.py
@@ -87,6 +87,7 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (
ScreenshotProvenance,
capturing_tool_call,
enqueue_screenshot_from_result,
+ screenshot_viewport_css_px,
)
from skyvern.forge.sdk.copilot.secret_scrub import (
encoded_secret_variants,
@@ -2236,6 +2237,7 @@ class SkyvernOverlayMCPServer(MCPServer):
if isinstance(workflow_run_id, str) and workflow_run_id
else None,
action_relation=ScreenshotActionRelation.TOOL_RESULT,
+ viewport_css_px=screenshot_viewport_css_px(copilot_result),
),
)
diff --git a/skyvern/forge/sdk/copilot/screenshot_utils.py b/skyvern/forge/sdk/copilot/screenshot_utils.py
index 77f658679..aa8d4e3a8 100644
--- a/skyvern/forge/sdk/copilot/screenshot_utils.py
+++ b/skyvern/forge/sdk/copilot/screenshot_utils.py
@@ -22,6 +22,8 @@ from PIL import Image
COPILOT_SCREENSHOT_MAX_WIDTH = 1024
COPILOT_SCREENSHOT_MAX_HEIGHT = 768
+COPILOT_SHARP_MAX_WIDTH = 1600
+COPILOT_SHARP_MAX_HEIGHT = 1200
COPILOT_JPEG_QUALITY = 60
_BROWSER_ABLATION_MAX_SCREENSHOT_FRAMES = 24
@@ -53,6 +55,7 @@ class ScreenshotProvenance:
dispatch_browser_session_id: str | None = None
producer_browser_session_id: str | None = None
session_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE
+ viewport_css_px: tuple[int, int] | None = None
@classmethod
def unknown(cls, *, source_tool: str) -> ScreenshotProvenance:
@@ -74,6 +77,10 @@ class ScreenshotEntry:
provenance: ScreenshotProvenance
captured_at: float = field(default_factory=time.monotonic)
capture_event_id: str = field(default_factory=lambda: uuid.uuid4().hex)
+ image_px: tuple[int, int] | None = None
+ # Sent on every model call that carries this frame while it fits the budget; session history keeps b64.
+ sharp_b64: str | None = None
+ sharp_px: tuple[int, int] | None = None
@dataclass(frozen=True)
@@ -86,6 +93,7 @@ class CapturedFrame:
dispatch_browser_session_id: str | None = None
producer_browser_session_id: str | None = None
session_binding: ProvenanceBinding = ProvenanceBinding.UNAVAILABLE
+ viewport_css_px: tuple[int, int] | None = None
@dataclass(frozen=True)
@@ -153,6 +161,17 @@ def screenshot_result_facts(
return producer_url, producer_session_id, binding
+def screenshot_viewport_css_px(result: dict[str, Any]) -> tuple[int, int] | None:
+ data = result.get("data")
+ viewport = data.get("viewport") if isinstance(data, dict) else None
+ if not isinstance(viewport, dict):
+ return None
+ width, height = viewport.get("width"), viewport.get("height")
+ if type(width) is int and type(height) is int and width > 0 and height > 0:
+ return width, height
+ return None
+
+
def _fingerprint_projection(value: Any) -> Any:
if hasattr(value, "model_dump"):
value = value.model_dump(mode="json")
@@ -182,34 +201,40 @@ def model_input_fingerprint(items: list[Any]) -> str:
return hashlib.sha256(encoded).hexdigest()
+def _fitted_jpeg_b64(img: Image.Image, box: tuple[int, int]) -> tuple[str, tuple[int, int]]:
+ fitted = img.copy()
+ fitted.thumbnail(box, Image.Resampling.LANCZOS)
+ if fitted.mode != "RGB":
+ fitted = fitted.convert("RGB")
+ buf = io.BytesIO()
+ fitted.save(buf, format="JPEG", quality=COPILOT_JPEG_QUALITY)
+ return base64.b64encode(buf.getvalue()).decode("ascii"), fitted.size
+
+
def resize_screenshot_b64(
b64_png: str,
*,
provenance: ScreenshotProvenance,
captured_at: float | None = None,
) -> ScreenshotEntry:
- """Resize a base64 PNG to 1024x768 max and compress to JPEG.
+ """Encode a frame as two JPEGs: a sharp copy fitted to 1600x1200 and a small copy fitted to 1024x768.
- Uses thumbnail() to preserve aspect ratio — only shrinks, never enlarges.
- Converts RGBA/P modes to RGB for JPEG compatibility.
+ thumbnail() preserves aspect ratio and never enlarges.
"""
raw = base64.b64decode(b64_png)
capture_id = f"sha256:{hashlib.sha256(raw).hexdigest()}"
- img = Image.open(io.BytesIO(raw))
- img.thumbnail(
- (COPILOT_SCREENSHOT_MAX_WIDTH, COPILOT_SCREENSHOT_MAX_HEIGHT),
- Image.Resampling.LANCZOS,
- )
- if img.mode in ("RGBA", "P"):
- img = img.convert("RGB")
- buf = io.BytesIO()
- img.save(buf, format="JPEG", quality=COPILOT_JPEG_QUALITY)
+ with Image.open(io.BytesIO(raw)) as img:
+ small_b64, small_px = _fitted_jpeg_b64(img, (COPILOT_SCREENSHOT_MAX_WIDTH, COPILOT_SCREENSHOT_MAX_HEIGHT))
+ sharp_b64, sharp_px = _fitted_jpeg_b64(img, (COPILOT_SHARP_MAX_WIDTH, COPILOT_SHARP_MAX_HEIGHT))
return ScreenshotEntry(
- b64=base64.b64encode(buf.getvalue()).decode("ascii"),
+ b64=small_b64,
mime="image/jpeg",
capture_id=capture_id,
provenance=provenance,
captured_at=captured_at if captured_at is not None else time.monotonic(),
+ image_px=small_px,
+ sharp_b64=sharp_b64,
+ sharp_px=sharp_px,
)
@@ -310,6 +335,7 @@ class ViewportFrame:
producer_session_id: str | None
session_binding: ProvenanceBinding
started_at: float
+ viewport_css_px: tuple[int, int] | None = None
def viewport_visible_effect(pre: ViewportFrame | None, post: ViewportFrame | None) -> VisibleEffect:
diff --git a/skyvern/forge/sdk/copilot/session_factory.py b/skyvern/forge/sdk/copilot/session_factory.py
index 6b81a2f46..7501a660a 100644
--- a/skyvern/forge/sdk/copilot/session_factory.py
+++ b/skyvern/forge/sdk/copilot/session_factory.py
@@ -221,7 +221,8 @@ def _run_context(data: CallModelData[Any]) -> Any:
def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget: int) -> ModelInputData:
- items = list(data.model_data.input)
+ # Compacted up front so the sharp-copy budget check below sees the input the budget filter will.
+ items = _compact_tool_items(list(data.model_data.input))
ctx = _run_context(data)
# Read-only peek: the end-of-turn drain in enforcement stays the only clear, so a provider
# retry or model fallback re-running this filter still carries the frame.
@@ -253,6 +254,12 @@ def _copilot_call_model_input_filter(data: CallModelData[Any], *, token_budget:
input_fingerprint=fingerprint,
)
if screenshot_msg is not None:
+ sharp_msg = pending_screenshot_message(ctx, sharp=True)
+ # The sharp copy may cost older screenshots, which budget pruning drops first anyway, but nothing else.
+ if sharp_msg is not None and estimate_tokens(_drop_older_screenshots([*items, sharp_msg])) <= token_budget:
+ screenshot_msg = sharp_msg
+ else:
+ LOG.info("copilot_sharp_frame_over_budget")
LOG.info(
"Injecting screenshot user message",
count=len(screenshot_msg["content"]) - 1,
@@ -379,13 +386,7 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budge
return ModelInputData(input=items, instructions=instructions)
# Layer 2: Drop all screenshots except the most recent
- screenshot_indices = [i for i, item in enumerate(items) if is_screenshot_message(item)]
- if len(screenshot_indices) > 1:
- drop_indices = set(screenshot_indices[:-1])
- items = [
- {"role": "user", "content": SCREENSHOT_PLACEHOLDER} if i in drop_indices else item
- for i, item in enumerate(items)
- ]
+ items = _drop_older_screenshots(items)
est = estimate_tokens(items)
if est <= token_budget:
@@ -437,6 +438,15 @@ def _filter_to_budget(items: list[Any], instructions: str | None, *, token_budge
return ModelInputData(input=items, instructions=instructions)
+def _drop_older_screenshots(items: list[Any]) -> list[Any]:
+ screenshot_indices = [i for i, item in enumerate(items) if is_screenshot_message(item)]
+ drop_indices = set(screenshot_indices[:-1])
+ return [
+ {"role": "user", "content": SCREENSHOT_PLACEHOLDER} if i in drop_indices else item
+ for i, item in enumerate(items)
+ ]
+
+
def _truncate_tool_output(item: Any, max_chars: int) -> Any:
"""Truncate a function_call_output item's output if it exceeds max_chars."""
if get_agent_message_field(item, "type") != "function_call_output":
diff --git a/skyvern/forge/sdk/copilot/tools/composition_capture.py b/skyvern/forge/sdk/copilot/tools/composition_capture.py
index 19e54e780..a0211c523 100644
--- a/skyvern/forge/sdk/copilot/tools/composition_capture.py
+++ b/skyvern/forge/sdk/copilot/tools/composition_capture.py
@@ -67,6 +67,7 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (
ScreenshotProvenance,
enqueue_screenshot,
screenshot_result_facts,
+ screenshot_viewport_css_px,
)
from skyvern.forge.sdk.copilot.secret_scrub import (
scrub_secrets_from_structure,
@@ -422,6 +423,7 @@ async def _composition_capture_visual_summary(
dispatch_browser_session_id=capture_session_id,
producer_browser_session_id=producer_session_id,
session_binding=session_binding,
+ viewport_css_px=screenshot_viewport_css_px(screenshot_result),
)
if screenshot_b64
else None
@@ -1488,6 +1490,7 @@ async def _inspect_page_for_composition_under_custody(
dispatch_browser_session_id=visual_fallback_frame.dispatch_browser_session_id,
producer_browser_session_id=visual_fallback_frame.producer_browser_session_id,
session_binding=visual_fallback_frame.session_binding,
+ viewport_css_px=visual_fallback_frame.viewport_css_px,
),
captured_at=visual_fallback_frame.captured_at,
)
diff --git a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
index b571ab7f4..cf86fa0d9 100644
--- a/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
+++ b/skyvern/forge/sdk/copilot/tools/mcp_hooks.py
@@ -1165,6 +1165,7 @@ async def _screenshot_post_hook(
"screenshot_base64": data.get("data", ""),
"url": url,
"title": title,
+ **({"viewport": data["viewport"]} if "viewport" in data else {}),
}
return result
diff --git a/skyvern/forge/sdk/copilot/tools/run_execution.py b/skyvern/forge/sdk/copilot/tools/run_execution.py
index 2ec2d2f61..23e7bbace 100644
--- a/skyvern/forge/sdk/copilot/tools/run_execution.py
+++ b/skyvern/forge/sdk/copilot/tools/run_execution.py
@@ -2772,6 +2772,7 @@ async def _capture_and_store_post_run_page(
dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,
producer_browser_session_id=captured_frame.producer_browser_session_id,
session_binding=captured_frame.session_binding,
+ viewport_css_px=captured_frame.viewport_css_px,
),
captured_at=captured_frame.captured_at,
)
@@ -3013,6 +3014,7 @@ async def _capture_dispatched_terminal_page_evidence(
dispatch_browser_session_id=captured_frame.dispatch_browser_session_id,
producer_browser_session_id=captured_frame.producer_browser_session_id,
session_binding=captured_frame.session_binding,
+ viewport_css_px=captured_frame.viewport_css_px,
),
captured_at=captured_frame.captured_at,
)
diff --git a/skyvern/forge/sdk/copilot/tools/scouting.py b/skyvern/forge/sdk/copilot/tools/scouting.py
index 9f1ae8568..7d0567b0e 100644
--- a/skyvern/forge/sdk/copilot/tools/scouting.py
+++ b/skyvern/forge/sdk/copilot/tools/scouting.py
@@ -86,6 +86,7 @@ from skyvern.forge.sdk.copilot.screenshot_utils import (
consume_screenshot_artifact,
enqueue_screenshot,
screenshot_result_facts,
+ screenshot_viewport_css_px,
)
from skyvern.forge.sdk.copilot.secret_scrub import (
REDACTED_SECRET_PLACEHOLDER,
@@ -559,6 +560,7 @@ async def _capture_post_interaction_screenshot(
dispatch_browser_session_id=frame.dispatch_session_id,
producer_browser_session_id=frame.producer_session_id,
session_binding=frame.session_binding,
+ viewport_css_px=frame.viewport_css_px,
),
captured_at=frame.started_at,
)
@@ -596,6 +598,7 @@ async def _take_viewport_frame(ctx: AgentContext, *, timeout_seconds: float) ->
producer_session_id=producer_session_id,
session_binding=session_binding,
started_at=started_at,
+ viewport_css_px=screenshot_viewport_css_px(result),
)
diff --git a/tests/unit/copilot_test_helpers.py b/tests/unit/copilot_test_helpers.py
index 0ef834a56..c7c3184da 100644
--- a/tests/unit/copilot_test_helpers.py
+++ b/tests/unit/copilot_test_helpers.py
@@ -3,6 +3,8 @@
from __future__ import annotations
import asyncio
+import base64
+import io
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, Sequence
from contextlib import asynccontextmanager, suppress
@@ -28,6 +30,7 @@ from openai.types.responses import (
)
from openai.types.responses.response import Response
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
+from PIL import Image
from playwright.async_api import Page, Route, async_playwright
from playwright.sync_api import sync_playwright
@@ -63,6 +66,12 @@ from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
OriginRunRedactionRegistry,
)
+from skyvern.forge.sdk.copilot.screenshot_utils import (
+ ScreenshotActionRelation,
+ ScreenshotEntry,
+ ScreenshotProvenance,
+ resize_screenshot_b64,
+)
from skyvern.forge.sdk.copilot.session_factory import (
copilot_call_model_input_filter,
copilot_session_input_callback,
@@ -1356,6 +1365,33 @@ def carried_interaction(**fields: InteractionFieldValue) -> dict[str, Any]:
return dict(fields)
+def captured_frame_entry(width: int = 1920, height: int = 947) -> ScreenshotEntry:
+ """A frame staged from a real PNG; each call is a new capture event."""
+ buf = io.BytesIO()
+ Image.new("RGB", (width, height), (40, 90, 160)).save(buf, format="PNG")
+ return resize_screenshot_b64(
+ base64.b64encode(buf.getvalue()).decode("ascii"),
+ provenance=ScreenshotProvenance(
+ source_tool="get_browser_screenshot",
+ captured_url="https://example.com/page",
+ observation_step=None,
+ browser_session_id="pbs_123",
+ workflow_run_id=None,
+ action_relation=ScreenshotActionRelation.TOOL_RESULT,
+ viewport_css_px=(width, height),
+ ),
+ )
+
+
+def sent_image_px(message: Any) -> list[tuple[int, int]]:
+ content = message.get("content") if isinstance(message, dict) else None
+ return [
+ Image.open(io.BytesIO(base64.b64decode(part["image_url"].partition(";base64,")[2]))).size
+ for part in (content if isinstance(content, list) else [])
+ if part.get("type") == "input_image"
+ ]
+
+
def make_model_input_data(items: list[Any], *, instructions: str | None = None, context: Any = None) -> Any:
"""Build a fake CallModelData payload with a model_data.input list.
diff --git a/tests/unit/test_cli_shared_core.py b/tests/unit/test_cli_shared_core.py
index f5ec5d2a2..9dfa5e80d 100644
--- a/tests/unit/test_cli_shared_core.py
+++ b/tests/unit/test_cli_shared_core.py
@@ -2,11 +2,15 @@
from __future__ import annotations
+import asyncio
+import io
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock
import pytest
+from PIL import Image
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from playwright.async_api import async_playwright
from skyvern.cli.core import browser_ops
from skyvern.cli.core.browser_ops import do_act, do_extract, do_navigate, do_screenshot, parse_extract_schema
@@ -20,6 +24,7 @@ from skyvern.cli.core.guards import (
)
from skyvern.cli.core.session_ops import do_session_close, do_session_create, do_session_list
from skyvern.client.types.extensions import Extensions
+from tests.unit.copilot_test_helpers import skip_no_browser
CAPTCHA_SOLVER_EXTENSION: Extensions = "captcha-solver"
LOCALHOST_RECOVERY_HINT = (
@@ -276,6 +281,42 @@ async def test_do_screenshot_full_page() -> None:
assert result.full_page is True
+async def test_do_screenshot_returns_the_capture_when_the_viewport_probe_never_answers(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def wedged_renderer(*_: object) -> None:
+ await asyncio.Event().wait()
+
+ monkeypatch.setattr(browser_ops, "_VIEWPORT_PROBE_TIMEOUT_SECONDS", 0.05)
+ page = MagicMock()
+ page.screenshot = AsyncMock(return_value=b"png-data")
+ page.evaluate = wedged_renderer
+
+ result = await do_screenshot(page)
+
+ assert result.data == b"png-data"
+ assert result.viewport_css_px is None
+
+
+@skip_no_browser
+async def test_do_screenshot_reports_the_css_viewport_only_for_a_viewport_capture() -> None:
+ async with async_playwright() as playwright:
+ browser = await playwright.chromium.launch()
+ try:
+ page = await browser.new_page(viewport={"width": 800, "height": 600}, device_scale_factor=2)
+ await page.set_content("<body style='height: 3000px'>tall</body>")
+ viewport = await do_screenshot(page)
+ full_page = await do_screenshot(page, full_page=True)
+ finally:
+ await browser.close()
+
+ assert Image.open(io.BytesIO(viewport.data)).size == (1600, 1200)
+ assert viewport.viewport_css_px == (800, 600)
+ assert full_page.viewport_css_px is None
+
+
@pytest.mark.asyncio
async def test_do_screenshot_with_selector() -> None:
page = MagicMock()
diff --git a/tests/unit/test_copilot_credential_pause.py b/tests/unit/test_copilot_credential_pause.py
index 4c5a6c381..baa2cc520 100644
--- a/tests/unit/test_copilot_credential_pause.py
+++ b/tests/unit/test_copilot_credential_pause.py
@@ -2999,6 +2999,9 @@ def test_the_raw_secret_card_sees_only_the_origin(user_url: str, origin: str) ->
assert credential_pause_module.raw_secret_card_origin(user_url) == origin
+_SIGNED_IN_SCOPE_PHRASES = ("this turn's tests only", "start signed out")
+
+
@pytest.mark.asyncio
async def test_a_run_derived_card_answered_by_signing_in_resumes_with_facts_and_does_not_fire_again(
monkeypatch: pytest.MonkeyPatch,
@@ -3029,7 +3032,9 @@ async def test_a_run_derived_card_answered_by_signing_in_resumes_with_facts_and_
resume_msgs = await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config)
assert resume_msgs is not None
- assert credential_pause_module.signed_in_facts(signed_in) in json.dumps(resume_msgs)
+ resume_text = json.dumps(resume_msgs)
+ assert credential_pause_module.signed_in_facts(signed_in) in resume_text
+ assert all(phrase in resume_text for phrase in _SIGNED_IN_SCOPE_PHRASES)
assert ctx.credential_pause_outcome == "signed_in"
assert _turn_credential_prompt_reason(ctx) is None
assert await maybe_credential_pause(ctx, _fake_result(), _make_stream(), config) is None
@@ -3052,7 +3057,7 @@ async def test_a_card_answered_by_signing_in_tells_the_tool_what_happened(monkey
result = await _call_ask_tool(ctx)
assert (result["status"], result["browser_profile_id"]) == ("signed_in", "bp_signed_in")
- assert result["detail"]
+ assert all(phrase in result["detail"] for phrase in _SIGNED_IN_SCOPE_PHRASES)
_REGISTRATION = {
diff --git a/tests/unit/test_copilot_hooks.py b/tests/unit/test_copilot_hooks.py
index c1d516ff3..6fa9aaff4 100644
--- a/tests/unit/test_copilot_hooks.py
+++ b/tests/unit/test_copilot_hooks.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import io
import json
import time
from collections.abc import AsyncIterator, Callable
@@ -15,8 +16,10 @@ from unittest.mock import AsyncMock, MagicMock, call
import pytest
import yaml
+from PIL import Image
from structlog.testing import capture_logs
+from skyvern.cli.core.browser_ops import ScreenshotResult
from skyvern.config import settings
from skyvern.forge import app
from skyvern.forge.sdk.copilot import tools as tools_module
@@ -819,7 +822,9 @@ class TestMCPFailedStepLoopDetection:
observed_session_ids.append(current.context.session_id if current.context else None)
observed_localhost_access.append(current.context.can_access_localhost if current.context else None)
assert current.api_key_hash == session_manager._api_key_hash("sk-copilot-org")
- return SimpleNamespace(data=b"fake-png")
+ png = io.BytesIO()
+ Image.new("RGB", (1920, 947)).save(png, format="PNG")
+ return ScreenshotResult(data=png.getvalue(), viewport_css_px=(1920, 947))
monkeypatch.setattr(browser_tools, "do_screenshot", fake_do_screenshot)
@@ -858,6 +863,7 @@ class TestMCPFailedStepLoopDetection:
assert result.isError is False
assert parsed["ok"] is True
assert parsed["data"]["screenshot_base64"]
+ assert [entry.provenance.viewport_css_px for entry in ctx.pending_screenshots] == [(1920, 947)]
assert observed_session_ids == ["pbs_copilot"]
assert observed_localhost_access == [True]
fallback_skyvern.connect_to_cloud_browser_session.assert_not_awaited()
diff --git a/tests/unit/test_copilot_screenshot_handling.py b/tests/unit/test_copilot_screenshot_handling.py
index 50864baf3..77ee37343 100644
--- a/tests/unit/test_copilot_screenshot_handling.py
+++ b/tests/unit/test_copilot_screenshot_handling.py
@@ -37,9 +37,11 @@ from skyvern.forge.sdk.copilot.tools.run_execution import (
from skyvern.forge.sdk.copilot.tools.scouting import _capture_post_interaction_screenshot
from skyvern.forge.sdk.copilot.verification_evidence import WorkflowVerificationEvidence
from tests.unit.copilot_test_helpers import (
+ captured_frame_entry,
install_get_run_results_harness,
make_model_input_data,
run_result_block_row,
+ sent_image_px,
)
@@ -363,6 +365,21 @@ class TestCapturePostInteractionScreenshot:
assert width <= COPILOT_SCREENSHOT_MAX_WIDTH
assert height <= COPILOT_SCREENSHOT_MAX_HEIGHT
+ @pytest.mark.asyncio
+ async def test_delivered_frame_states_its_image_size_and_the_measured_viewport(self, tmp_path: Path) -> None:
+ frame = tmp_path / "frame.png"
+ Image.new("RGB", (1920, 947), (10, 120, 200)).save(frame, format="PNG")
+ ctx = self._ctx({"ok": True, "data": {"path": str(frame), "viewport": {"width": 1920, "height": 947}}})
+
+ assert await _capture_post_interaction_screenshot(ctx, source_tool="click", captured_url=None) is True
+ result = copilot_call_model_input_filter(
+ make_model_input_data([{"role": "user", "content": "go"}], context=ctx)
+ )
+
+ message = result.input[-1]
+ assert sent_image_px(message) == [(1600, 789)]
+ assert "image_px=1600x789; viewport_css_px=1920x947" in message["content"][0]["text"]
+
@pytest.mark.parametrize(
"data",
[
@@ -475,6 +492,18 @@ class TestNudgeDrainAndFilterExclusivity:
assert result.input == items
+ def test_a_frame_sent_sharp_is_saved_small_with_a_nudge(self) -> None:
+ ctx = SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)
+ items = [{"role": "user", "content": "read the reference code"}]
+ seen = copilot_call_model_input_filter(make_model_input_data(items, context=ctx))
+ saved = _consume_pending_screenshots(ctx)
+ history = [*items, {"role": "assistant", "content": "done"}, saved]
+
+ after = copilot_call_model_input_filter(make_model_input_data(history, context=ctx))
+
+ assert sent_image_px(seen.input[-1]) == [(1600, 789)]
+ assert sent_image_px(after.input[-1]) == [(1024, 505)]
+
def test_the_nudge_path_also_withholds_the_frame_from_a_non_vision_model(self) -> None:
# The vision check lives in the shared builder so every delivery path inherits it;
# the drain still empties the queue.
diff --git a/tests/unit/test_copilot_session_factory.py b/tests/unit/test_copilot_session_factory.py
index db8a2ec18..a071da696 100644
--- a/tests/unit/test_copilot_session_factory.py
+++ b/tests/unit/test_copilot_session_factory.py
@@ -9,6 +9,7 @@ from types import SimpleNamespace
from typing import Any
import pytest
+from agents.run_config import ModelInputData
from structlog.testing import capture_logs
from skyvern.forge.sdk.copilot.ask_user import (
@@ -19,7 +20,7 @@ from skyvern.forge.sdk.copilot.ask_user import (
QuestionResponse,
)
from skyvern.forge.sdk.copilot.config import BlockAuthoringPolicy
-from skyvern.forge.sdk.copilot.enforcement import estimate_tokens
+from skyvern.forge.sdk.copilot.enforcement import estimate_tokens, pending_screenshot_message
from skyvern.forge.sdk.copilot.screenshot_utils import (
ScreenshotActionRelation,
ScreenshotEntry,
@@ -30,8 +31,9 @@ from skyvern.forge.sdk.copilot.session_factory import (
copilot_session_input_callback,
make_copilot_call_model_input_filter,
)
-from tests.unit.copilot_test_helpers import make_copilot_ctx
+from tests.unit.copilot_test_helpers import captured_frame_entry, make_copilot_ctx
from tests.unit.copilot_test_helpers import make_model_input_data as _mk_input_data
+from tests.unit.copilot_test_helpers import sent_image_px
def _unread_run_results_batch() -> list[dict[str, Any]]:
@@ -705,6 +707,45 @@ class TestStagedScreenshotBinding:
assert len(_image_parts(delivered.input[-1])) == 1
assert ctx.pending_frame_lease.capture_event_id == recaptured.capture_event_id
+ def test_a_sharp_frame_over_budget_costs_older_screenshots_first_and_then_falls_back_to_the_small_copy(
+ self,
+ ) -> None:
+ older_frame = pending_screenshot_message(
+ SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)
+ )
+ items = [
+ {"role": "user", "content": "what does the page say"},
+ older_frame,
+ {"type": "function_call", "call_id": "read-1", "name": "evaluate", "arguments": "{}"},
+ {"type": "function_call_output", "call_id": "read-1", "output": '{"text":"order 4412 shipped"}'},
+ ]
+
+ def call(token_budget: int) -> ModelInputData:
+ ctx = SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)
+ return make_copilot_call_model_input_filter(token_budget=token_budget)(_mk_input_data(items, context=ctx))
+
+ for token_budget, new_frame_px in ((2_200, (1600, 789)), (1_200, (1024, 505))):
+ result = call(token_budget)
+ assert [sent_image_px(item) for item in result.input] == [[], [], [], [], [new_frame_px]]
+ assert result.input[2:4] == items[2:4]
+
+ def test_the_pending_frame_is_sent_sharp_on_every_call_that_carries_it(self) -> None:
+ ctx = SimpleNamespace(pending_screenshots=[captured_frame_entry()], supports_vision=True)
+ initial = [{"role": "user", "content": "what does the page say"}]
+ advanced = [
+ *initial,
+ {"type": "function_call", "call_id": "read-1", "name": "evaluate", "arguments": "{}"},
+ {"type": "function_call_output", "call_id": "read-1", "output": '{"ok":true}'},
+ ]
+
+ first = copilot_call_model_input_filter(_mk_input_data(initial, context=ctx))
+ later = copilot_call_model_input_filter(_mk_input_data(advanced, context=ctx))
+
+ for result in (first, later):
+ message = result.input[-1]
+ assert sent_image_px(message) == [(1600, 789)]
+ assert "image_px=1600x789; viewport_css_px=1920x947" in message["content"][0]["text"]
+
def test_paired_message_has_machine_readable_marker(self) -> None:
ctx = _ctx_with_staged_frame()
diff --git a/tests/unit/test_copilot_token_accounting.py b/tests/unit/test_copilot_token_accounting.py
index 8a3a51913..86265cbe3 100644
--- a/tests/unit/test_copilot_token_accounting.py
+++ b/tests/unit/test_copilot_token_accounting.py
@@ -2,11 +2,14 @@
from __future__ import annotations
+import base64
+import io
from unittest.mock import MagicMock
import pytest
from agents.run_context import RunContextWrapper
from agents.usage import InputTokensDetails, OutputTokensDetails, Usage
+from PIL import Image
from skyvern.forge.sdk.copilot.context import CopilotContext
from skyvern.forge.sdk.copilot.enforcement import (
@@ -87,6 +90,20 @@ class TestEstimateTokens:
# so text tokens should be well under 100.
assert text_portion < 100
+ @pytest.mark.parametrize(("width", "height", "billed"), [(1024, 576, 692), (1600, 900, 1741)])
+ def test_image_estimate_follows_pixel_size(self, width: int, height: int, billed: int) -> None:
+ buf = io.BytesIO()
+ Image.new("RGB", (width, height)).save(buf, format="JPEG")
+ item = {
+ "type": "input_image",
+ "image_url": "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode(),
+ "detail": "high",
+ }
+
+ _, image_tokens = _sanitize_for_token_estimation(item)
+
+ assert image_tokens == billed
+
def test_image_metadata_preserved(self):
"""type and detail fields should be included, only image_url replaced."""
item = {
@@ -94,8 +111,8 @@ class TestEstimateTokens:
"image_url": "data:image/png;base64," + "X" * 1000,
"detail": "high",
}
- sanitized, count = _sanitize_for_token_estimation(item)
- assert count == 1
+ sanitized, image_tokens = _sanitize_for_token_estimation(item)
+ assert image_tokens == TOKENS_PER_RESIZED_IMAGE
assert sanitized["type"] == "input_image"
assert sanitized["detail"] == "high"
assert sanitized["image_url"] == "[image]"
diff --git a/tests/unit/test_mcp_browser_tools.py b/tests/unit/test_mcp_browser_tools.py
index a47c28bb1..b6b8f8fc6 100644
--- a/tests/unit/test_mcp_browser_tools.py
+++ b/tests/unit/test_mcp_browser_tools.py
@@ -4553,7 +4553,7 @@ async def test_navigating_paired_tools_invalidate_refs_around_navigation(
return NavigateResult(url="https://example.test", title="Example", load_state="load")
monkeypatch.setattr(mcp_browser, "do_navigate", AsyncMock(side_effect=navigate))
- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))
+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=browser_ops.ScreenshotResult(data=b"png")))
monkeypatch.setattr(mcp_browser, "do_extract", AsyncMock(return_value=SimpleNamespace(extracted={"value": 1})))
monkeypatch.setattr(
mcp_browser,
@@ -4618,7 +4618,7 @@ async def test_navigating_paired_tools_surface_the_degraded_load_state(
"do_navigate",
AsyncMock(return_value=NavigateResult(url="https://example.test", title="Example", load_state="commit")),
)
- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))
+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=browser_ops.ScreenshotResult(data=b"png")))
monkeypatch.setattr(
mcp_browser,
"save_artifact",
@@ -4940,7 +4940,7 @@ async def test_paired_read_failure_keeps_screenshot_and_reports_failure(
@pytest.mark.asyncio
async def test_inline_screenshot_is_also_persisted(monkeypatch: pytest.MonkeyPatch) -> None:
_sdk_equivalent_page(monkeypatch)
- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))
+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=browser_ops.ScreenshotResult(data=b"png")))
artifact = Artifact(kind="screenshot", path="/tmp/shot.png", mime="image/png", bytes=3)
monkeypatch.setattr(mcp_browser, "save_artifact", Mock(return_value=artifact))
@@ -5376,7 +5376,7 @@ async def test_screenshot_page_change_retry(monkeypatch: pytest.MonkeyPatch, mes
ctx = BrowserContext(mode="local")
state = session_manager.SessionState(context=ctx, _active_page=page)
patch_get_page(monkeypatch, mcp_browser, page, ctx)
- read = AsyncMock(side_effect=[RuntimeError(message), SimpleNamespace(data=b"png")])
+ read = AsyncMock(side_effect=[RuntimeError(message), browser_ops.ScreenshotResult(data=b"png")])
monkeypatch.setattr(mcp_browser, "do_screenshot", read)
monkeypatch.setattr(
mcp_browser,
@@ -5481,7 +5481,7 @@ async def test_read_tools_retry_backoff(monkeypatch: pytest.MonkeyPatch, tool: s
if attempts <= 6:
raise RuntimeError("Execution context was destroyed")
if tool == "screenshot":
- return SimpleNamespace(data=b"png")
+ return browser_ops.ScreenshotResult(data=b"png")
if tool == "get_html":
return "<p>fresh</p>"
return {"ok": True, "data": {"fresh": True}}
diff --git a/tests/unit/test_mcp_response_size_cap.py b/tests/unit/test_mcp_response_size_cap.py
index a67225a04..2b2247b01 100644
--- a/tests/unit/test_mcp_response_size_cap.py
+++ b/tests/unit/test_mcp_response_size_cap.py
@@ -11,7 +11,7 @@ from unittest.mock import AsyncMock, Mock
import pytest
-from skyvern.cli.core.browser_ops import NavigateResult
+from skyvern.cli.core.browser_ops import NavigateResult, ScreenshotResult
from skyvern.cli.core.result import Artifact, BrowserContext
from skyvern.cli.mcp_tools import browser as mcp_browser
from skyvern.cli.mcp_tools import mcp
@@ -394,7 +394,7 @@ async def test_every_registered_extract_tool_caps_an_oversize_extraction(
"do_navigate",
AsyncMock(return_value=NavigateResult(url="https://example.test", title="Example")),
)
- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))
+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=ScreenshotResult(data=b"png")))
monkeypatch.setattr(
mcp_browser,
"save_artifact",
@@ -428,7 +428,7 @@ def _stub_browser(monkeypatch: pytest.MonkeyPatch) -> None:
"do_navigate",
AsyncMock(return_value=NavigateResult(url="https://example.test", title="Example")),
)
- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=b"png")))
+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=ScreenshotResult(data=b"png")))
monkeypatch.setattr(
mcp_browser,
"save_artifact",
@@ -443,7 +443,7 @@ async def test_navigate_and_screenshot_caps_an_oversize_inline_screenshot(monkey
tool-result limit with no truncation warning."""
_stub_browser(monkeypatch)
oversize_png = b"\x89PNG" + b"z" * MCP_MAX_RESPONSE_CHARS
- monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=SimpleNamespace(data=oversize_png)))
+ monkeypatch.setattr(mcp_browser, "do_screenshot", AsyncMock(return_value=ScreenshotResult(data=oversize_png)))
tool = await mcp.get_tool("skyvern_navigate_and_screenshot")
result = await tool.fn(url="https://example.test", inline=True)