Skip to content

Automate browser based workflows with AI

Let Copilot upload a chat-attached file into a page's file picker (SKY-16825) (#8691)

Andrew Neilson9f32da7bd624
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.py
index 804fc26a1..19e433486 100644
--- a/skyvern/forge/sdk/copilot/blocker_signal.py
+++ b/skyvern/forge/sdk/copilot/blocker_signal.py
@@ -82,6 +82,7 @@ _INTERNAL_TOOL_NAME_TOKENS: tuple[str, ...] = (
"search_web",
"solve_page_challenge",
"start_fresh_browser",
+ "upload_attached_file",
"get_browser_screenshot",
"list_credentials",
"list_integrations",
diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.py
index b366d8e18..8934083bc 100644
--- a/skyvern/forge/sdk/copilot/narration.py
+++ b/skyvern/forge/sdk/copilot/narration.py
@@ -137,6 +137,7 @@ _TOOL_ACTIVITY_DISPLAY_LABELS = {
"fill_credential_field": "Entering saved credentials",
"solve_page_challenge": "Solving the page's verification challenge",
"start_fresh_browser": "Starting a fresh browser",
+ "upload_attached_file": "Attaching your file to the page",
"run_browser_code": "Running browser code",
"edit_block": "Editing block",
"add_block": "Adding block",
diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.py
index 7721be2ef..7b255aa21 100644
--- a/skyvern/forge/sdk/copilot/output_utils.py
+++ b/skyvern/forge/sdk/copilot/output_utils.py
@@ -2057,6 +2057,8 @@ def summarize_tool_result(tool_name: str, result: dict[str, Any], *, for_display
return f"{summary} (timed out)" if result.get("timed_out") else summary
if tool_name == "start_fresh_browser":
return "Started a fresh browser; the old browser's cookies, sign-ins and open tabs are gone"
+ if tool_name == "upload_attached_file":
+ return f"Placed {result.get('filename')} ({result.get('size_bytes')} bytes) in the page's file input"
if tool_name == "run_browser_code":
operations = result.get("operations")
count = len(operations) if isinstance(operations, list) else 0
diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.py
index 8286fe4e2..397f83d99 100644
--- a/skyvern/forge/sdk/copilot/tools/__init__.py
+++ b/skyvern/forge/sdk/copilot/tools/__init__.py
@@ -97,6 +97,7 @@ from ._shared import _raw_yaml_proxy_location as _raw_yaml_proxy_location
from ._shared import _same_page_ignoring_fragment as _same_page_ignoring_fragment
from ._shared import _unverified_current_workflow_labels as _unverified_current_workflow_labels
from ._shared import admitted_requested_output_reads
+from .attached_file_upload import UPLOAD_TOOL_NAME, upload_attached_file
from .banned_blocks import _COPILOT_BANNED_BLOCK_TYPES as _COPILOT_BANNED_BLOCK_TYPES
from .banned_blocks import AUTHORING_FAMILY_GUIDANCE as AUTHORING_FAMILY_GUIDANCE
from .banned_blocks import SCHEMA_FIRST_GUIDANCE as SCHEMA_FIRST_GUIDANCE
@@ -1749,6 +1750,20 @@ async def start_fresh_browser_tool(ctx: RunContextWrapper) -> str:
return json.dumps(scrub_secrets_from_structure(ctx.context, result))
+@function_tool(failure_error_function=copilot_tool_failure, name_override=UPLOAD_TOOL_NAME)
+async def upload_attached_file_tool(ctx: RunContextWrapper, file_id: str, selector: str) -> str:
+ """Set a file the user attached to this chat (`file_id` exactly as listed in the attached files) on the page's
+ <input type="file"> named by `selector`, without clicking any submit control; `ok` is true only when the input's
+ own file list (`input_files`) then holds this file's name and size, and an unlisted or removed attachment is
+ refused."""
+ authority_error = _authority_tool_error(ctx.context, UPLOAD_TOOL_NAME)
+ if authority_error:
+ return _diagnosis_repair_tool_error(ctx.context, UPLOAD_TOOL_NAME, authority_error)
+ result = await upload_attached_file(ctx.context, file_id, selector)
+ record_tool_step_result_for_ctx(ctx.context, UPLOAD_TOOL_NAME, {"file_id": file_id, "selector": selector}, result)
+ return json.dumps(scrub_secrets_from_structure(ctx.context, result))
+
+
@function_tool(
failure_error_function=copilot_tool_failure, name_override="inspect_page_for_composition", strict_mode=False
)
@@ -2073,6 +2088,7 @@ NATIVE_TOOLS = [
run_browser_code_tool,
solve_page_challenge_tool,
start_fresh_browser_tool,
+ upload_attached_file_tool,
]
@@ -2089,6 +2105,7 @@ BROWSER_BOUND_TOOL_NAMES = BLOCK_RUNNING_TOOLS | frozenset(
BROWSER_CODE_TOOL_NAME,
SOLVE_TOOL_NAME,
FRESH_BROWSER_TOOL_NAME,
+ UPLOAD_TOOL_NAME,
}
)
diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.py
index 93566e8b7..c465c2e73 100644
--- a/skyvern/forge/sdk/copilot/tools/_shared.py
+++ b/skyvern/forge/sdk/copilot/tools/_shared.py
@@ -2,13 +2,14 @@ from __future__ import annotations
import asyncio
import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal
from urllib.parse import urlparse
import structlog
import yaml
+from playwright.async_api import Page
from typing_extensions import TypedDict
from skyvern.cli.core.js_dispatch import outer_cap_seconds
@@ -39,11 +40,24 @@ from skyvern.forge.sdk.copilot.enforcement import (
_requested_output_labels_by_path,
proxy_hop_failure_reason,
)
+from skyvern.forge.sdk.copilot.mcp_adapter import _browser_session_error_disposition, _browser_session_loss_result
from skyvern.forge.sdk.copilot.nav_attribution import proxy_owns_nav_codes
from skyvern.forge.sdk.copilot.runtime import (
+ SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR,
+ SENSITIVE_ORIGIN_PAGE_ERROR,
AgentContext,
+ BrowserProbeOutcome,
+ CopilotBrowserGenerationRetired,
+ CopilotBrowserSessionUnavailable,
+ _browser_context_attachability,
+ browser_evidence_commit_lock,
+ browser_page_custody_lock,
effective_browser_session_id,
+ live_working_page,
+ mcp_browser_context,
resolve_browser_state_for_context,
+ sensitive_origin_page_has_active_run,
+ sensitive_origin_page_is_tainted,
)
from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_prompt
from skyvern.forge.sdk.copilot.task_output_envelope import (
@@ -57,6 +71,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatus
from skyvern.schemas.proxy_location import ProxyLocationInput
from skyvern.schemas.workflows import BlockType
from skyvern.utils.yaml_loader import safe_load_no_dates
+from skyvern.webeye.browser_errors import BrowserAutomationError
LOG = structlog.get_logger()
@@ -1039,3 +1054,50 @@ async def _composition_get_structured_evidence(
timeout_seconds=timeout_seconds,
)
return evidence
+
+
+def browser_is_lost(page: Page) -> bool:
+ return (
+ page.is_closed() or _browser_context_attachability(page.context) is BrowserProbeOutcome.positively_unreachable
+ )
+
+
+async def on_working_page(
+ ctx: AgentContext,
+ *,
+ tool_name: str,
+ no_page_error: str,
+ act: Callable[[Page], Awaitable[dict[str, Any]]],
+) -> dict[str, Any]:
+ async with browser_page_custody_lock(ctx), browser_evidence_commit_lock(ctx):
+ if sensitive_origin_page_has_active_run(ctx):
+ return {"ok": False, "error": SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR}
+ if sensitive_origin_page_is_tainted(ctx):
+ return {"ok": False, "error": SENSITIVE_ORIGIN_PAGE_ERROR}
+ entered_browser = False
+ try:
+ async with mcp_browser_context(ctx):
+ entered_browser = True
+ page = await live_working_page(ctx)
+ if page is None:
+ return {"ok": False, "error": no_page_error}
+ return await act(page)
+ except (CopilotBrowserGenerationRetired, CopilotBrowserSessionUnavailable) as exc:
+ disposition = await _browser_session_error_disposition(ctx, exc, tool_name=tool_name, call_path="model")
+ return _browser_session_loss_result(
+ {}, disposition=disposition, deadline_expired=ctx.browser_session_continuity_deadline_expired
+ )
+ except Exception as exc:
+ if entered_browser:
+ raise
+ # Only a classified error's message has been through CDP-endpoint redaction; an
+ # unclassified one is named by type, and its text stays in the log.
+ detail = (str(exc).rstrip(".") if isinstance(exc, BrowserAutomationError) else "") or type(exc).__name__
+ LOG.warning(
+ "copilot native browser tool could not enter its browser",
+ tool_name=tool_name,
+ browser_session_id=ctx.browser_session_id,
+ error_type=type(exc).__name__,
+ error=str(exc),
+ )
+ return {"ok": False, "error": f"{tool_name} could not reach its browser: {detail}. Nothing was done."}
diff --git a/skyvern/forge/sdk/copilot/tools/attached_file_upload.py b/skyvern/forge/sdk/copilot/tools/attached_file_upload.py
new file mode 100644
index 000000000..0a19b3be6
--- /dev/null
+++ b/skyvern/forge/sdk/copilot/tools/attached_file_upload.py
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import hashlib
+import mimetypes
+from typing import Any
+
+from playwright.async_api import FilePayload, Page
+
+from skyvern.forge import app
+from skyvern.forge.sdk.api.files import resolve_uploaded_file_id
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.runtime import AgentContext, CopilotBrowserSessionUnavailable
+from skyvern.forge.sdk.copilot.secret_redaction import redact_secretlike_filename
+from skyvern.forge.sdk.schemas.workflow_copilot import CopilotAttachedFile
+
+from ._shared import browser_is_lost, on_working_page
+
+UPLOAD_TOOL_NAME = "upload_attached_file"
+
+_SET_FILES_TIMEOUT_MS = 15000
+_NO_BROWSER = "No browser is open in this chat yet. Navigate to the page with the file input first."
+_NO_PAGE = "This chat's browser has no page open. Navigate to the page with the file input first."
+# set_input_files retargets a <label>, or an element inside one, to the label's input; read the files from there.
+_READ_FILES = """el => {
+ const input = el instanceof HTMLInputElement ? el : (el.closest("label")?.control ?? el);
+ return {
+ origin: input.ownerDocument.location.origin,
+ file_input: input instanceof HTMLInputElement && input.type === "file",
+ files: Array.from(input.files || [], file => ({name: file.name, size: file.size})),
+ };
+}"""
+_RETRY_NOTE = "Check the page before retrying: a retry can upload it twice."
+_DELIVERED_NOTE = (
+ "The file was set on the input and its change event fired, so the page may already have taken or sent it "
+ f"(some pages clear the input after reading it). {_RETRY_NOTE}"
+)
+_NOT_A_FILE_INPUT = "The selector does not name a file input, so nothing was set."
+
+
+def _unavailable(file_id: str) -> dict[str, Any]:
+ return {
+ "ok": False,
+ "error": (
+ f"Attached file {file_id!r} is not available to this chat. Nothing was uploaded; ask the user to "
+ "attach the file again."
+ ),
+ }
+
+
+def _attached_in_this_turn(ctx: AgentContext, file_id: str) -> CopilotAttachedFile | None:
+ if not isinstance(ctx, CopilotContext) or ctx.turn_context_packet is None:
+ return None
+ attached_files = ctx.turn_context_packet.attached_file_context
+ if attached_files is None:
+ return None
+ return next((attached for attached in attached_files.files if attached.file_id == file_id), None)
+
+
+async def _read_attached_bytes(ctx: AgentContext, file_id: str) -> bytes | None:
+ try:
+ storage_uri = await resolve_uploaded_file_id(file_id, ctx.organization_id)
+ return await app.STORAGE.download_managed_file(storage_uri, ctx.organization_id)
+ except (FileNotFoundError, PermissionError):
+ return None
+
+
+async def upload_attached_file(ctx: AgentContext, file_id: str, selector: str) -> dict[str, Any]:
+ attached = _attached_in_this_turn(ctx, file_id)
+ if attached is None:
+ return _unavailable(file_id)
+ session_id = ctx.browser_session_id
+ if not session_id:
+ return {"ok": False, "error": _NO_BROWSER}
+ data = await _read_attached_bytes(ctx, file_id)
+ if data is None:
+ return _unavailable(file_id)
+ payload = FilePayload(
+ name=attached.filename,
+ mimeType=mimetypes.guess_type(attached.filename)[0] or "application/octet-stream",
+ buffer=data,
+ )
+ uploaded: dict[str, Any] = {
+ "file_id": file_id,
+ "filename": redact_secretlike_filename(attached.filename),
+ "size_bytes": len(data),
+ "sha256": hashlib.sha256(data).hexdigest(),
+ }
+
+ async def _upload(page: Page) -> dict[str, Any]:
+ target = page.locator(selector)
+ files_set = False
+ try:
+ await target.set_input_files(payload, timeout=_SET_FILES_TIMEOUT_MS)
+ files_set = True
+ readback = await target.evaluate(_READ_FILES, timeout=_SET_FILES_TIMEOUT_MS)
+ except Exception as exc:
+ if browser_is_lost(page):
+ raise CopilotBrowserSessionUnavailable(session_id) from exc
+ first_line = next(iter(str(exc).strip().splitlines()), "") or type(exc).__name__
+ if files_set:
+ first_line = f"{first_line}. The file may already be on the page. {_RETRY_NOTE}"
+ return {**uploaded, "ok": False, "error": first_line}
+ # set_input_files returns without error or events on a non-file input.
+ if not readback["file_input"]:
+ return {**uploaded, "ok": False, "change_delivered": False, "error": _NOT_A_FILE_INPUT}
+ input_files = readback["files"]
+ ok = {"name": attached.filename, "size": len(data)} in input_files
+ result: dict[str, Any] = {
+ **uploaded,
+ "ok": ok,
+ "change_delivered": True,
+ "input_origin": readback["origin"],
+ "input_files": [{**item, "name": redact_secretlike_filename(item["name"])} for item in input_files],
+ }
+ if not ok:
+ result["error"] = f"The file input no longer holds this file. {_DELIVERED_NOTE}"
+ return result
+
+ return await on_working_page(ctx, tool_name=UPLOAD_TOOL_NAME, no_page_error=_NO_PAGE, act=_upload)
diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.py
index bf97899eb..b819dad2c 100644
--- a/skyvern/forge/sdk/copilot/tools/guardrails.py
+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py
@@ -168,6 +168,7 @@ def _authority_tool_error(
"run_browser_code",
"solve_page_challenge",
"start_fresh_browser",
+ "upload_attached_file",
"inspect_page_for_composition",
LOCATOR_INSPECTION_TOOL_NAME,
} and raw_secret_browser_denied(ctx):
diff --git a/skyvern/forge/sdk/copilot/tools/page_challenge.py b/skyvern/forge/sdk/copilot/tools/page_challenge.py
index f47502641..ec3b46f35 100644
--- a/skyvern/forge/sdk/copilot/tools/page_challenge.py
+++ b/skyvern/forge/sdk/copilot/tools/page_challenge.py
@@ -8,23 +8,13 @@ from playwright.async_api import Page
from skyvern.forge import app
from skyvern.forge.sdk.copilot.build_test_connect_failure import BuildTestConnectFailure
-from skyvern.forge.sdk.copilot.mcp_adapter import _browser_session_error_disposition, _browser_session_loss_result
from skyvern.forge.sdk.copilot.runtime import (
SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR,
- SENSITIVE_ORIGIN_PAGE_ERROR,
AgentContext,
- BrowserProbeOutcome,
- CopilotBrowserGenerationRetired,
CopilotBrowserSessionUnavailable,
- _browser_context_attachability,
_browser_session_acquisition_failure_result,
- browser_evidence_commit_lock,
- browser_page_custody_lock,
- live_working_page,
- mcp_browser_context,
replace_browser_session,
sensitive_origin_page_has_active_run,
- sensitive_origin_page_is_tainted,
)
from skyvern.forge.sdk.workflow.models.block import CodeBlockCaptchaError, _code_block_solve_captcha_builtin
from skyvern.webeye.utils.captcha_solver import (
@@ -33,6 +23,7 @@ from skyvern.webeye.utils.captcha_solver import (
solve_challenge_ladder,
)
+from ._shared import browser_is_lost, on_working_page
from .scouting import rendered_challenge_vendor
LOG = structlog.get_logger()
@@ -87,32 +78,14 @@ async def solve_page_challenge(
session_id = ctx.browser_session_id
if not session_id:
return {"ok": False, "error": _NO_BROWSER}
- async with browser_page_custody_lock(ctx), browser_evidence_commit_lock(ctx):
- if sensitive_origin_page_has_active_run(ctx):
- return {"ok": False, "error": SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR}
- if sensitive_origin_page_is_tainted(ctx):
- return {"ok": False, "error": SENSITIVE_ORIGIN_PAGE_ERROR}
- try:
- async with mcp_browser_context(ctx):
- page = await live_working_page(ctx)
- if page is None:
- return {"ok": False, "error": _NO_PAGE}
- if image is not None and input is not None:
- result = await _read_image(ctx, page, session_id, image, input)
- else:
- result = await _run_ladder(ctx, page, session_id)
- if result["outcome"] == "unsolved" and (
- page.is_closed()
- or _browser_context_attachability(page.context) is BrowserProbeOutcome.positively_unreachable
- ):
- raise CopilotBrowserSessionUnavailable(session_id)
- except (CopilotBrowserGenerationRetired, CopilotBrowserSessionUnavailable) as exc:
- disposition = await _browser_session_error_disposition(
- ctx, exc, tool_name=SOLVE_TOOL_NAME, call_path="model"
- )
- return _browser_session_loss_result(
- {}, disposition=disposition, deadline_expired=ctx.browser_session_continuity_deadline_expired
- )
+
+ async def _solve(page: Page) -> dict[str, Any]:
+ if image is not None and input is not None:
+ result = await _read_image(ctx, page, session_id, image, input)
+ else:
+ result = await _run_ladder(ctx, page, session_id)
+ if result["outcome"] == "unsolved" and browser_is_lost(page):
+ raise CopilotBrowserSessionUnavailable(session_id)
# A fresh browser changes whether a site shows a challenge, not whether its image can be read.
if image is not None:
return result
@@ -130,6 +103,8 @@ async def solve_page_challenge(
result["untried_in_this_request"] = [f"{FRESH_BROWSER_TOOL_NAME}: {_FRESH_BROWSER_FACT}"]
return result
+ return await on_working_page(ctx, tool_name=SOLVE_TOOL_NAME, no_page_error=_NO_PAGE, act=_solve)
+
async def _run_ladder(ctx: AgentContext, page: Page, session_id: str) -> dict[str, Any]:
if not await app.AGENT_FUNCTION.captcha_solving_available(ctx.organization_id, page.url):
diff --git a/tests/unit/forge/sdk/copilot/test_organization_usage_tool.py b/tests/unit/forge/sdk/copilot/test_organization_usage_tool.py
index 7af7ded33..2e8847d22 100644
--- a/tests/unit/forge/sdk/copilot/test_organization_usage_tool.py
+++ b/tests/unit/forge/sdk/copilot/test_organization_usage_tool.py
@@ -74,6 +74,7 @@ CATALOG_WITHOUT_THIS_TOOL = [
"run_browser_code",
"solve_page_challenge",
"start_fresh_browser",
+ "upload_attached_file",
]
diff --git a/tests/unit/test_copilot_attached_file_upload.py b/tests/unit/test_copilot_attached_file_upload.py
new file mode 100644
index 000000000..8235dfd29
--- /dev/null
+++ b/tests/unit/test_copilot_attached_file_upload.py
@@ -0,0 +1,406 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import struct
+import zlib
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+from playwright.async_api import Page, Route, async_playwright
+
+from skyvern.forge import app
+from skyvern.forge.sdk.copilot.context import CopilotContext
+from skyvern.forge.sdk.copilot.runtime import AgentContext
+from skyvern.forge.sdk.copilot.tools import _shared, attached_file_upload
+from skyvern.forge.sdk.copilot.turn_context import AttachedFileContext, TranscriptContext, TurnContextPacket
+from skyvern.forge.sdk.schemas.workflow_copilot import CopilotAttachedFile
+from skyvern.services import uploaded_file_service
+from skyvern.webeye.browser_errors import BrowserCdpAcquisitionError
+from tests.unit.conftest import make_copilot_context
+from tests.unit.test_code_block_captcha import _skip_no_browser
+from tests.unit.test_copilot_runtime import _make_ctx
+from tests.unit.test_uploaded_file_retention import (
+ ATTACKER_ORG_ID,
+ VICTIM_ORG_ID,
+ FakeStorage,
+ FakeUploadedFilesRepository,
+ _uri,
+)
+
+CHAT_ORG_ID = ATTACKER_ORG_ID
+PAGE_URL = "https://upload.fixture.test/"
+
+UPLOAD_PAGE = """<!doctype html>
+<input type="file" id="resume" accept="application/pdf">
+<input type="file" id="photo" accept="image/*"><div id="photo-receipt"></div>
+<input type="file" id="video" accept="video/*"><div id="video-receipt"></div>
+<script>
+ for (const id of ["photo", "video"]) {
+ document.getElementById(id).addEventListener("change", async (event) => {
+ const file = event.target.files[0];
+ const digest = await crypto.subtle.digest("SHA-256", await file.arrayBuffer());
+ const hex = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
+ document.getElementById(`${id}-receipt`).textContent = `${file.name}:${file.size}:${hex}`;
+ });
+ }
+</script>
+"""
+
+
+def _png() -> bytes:
+ def chunk(kind: bytes, body: bytes) -> bytes:
+ return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body))
+
+ header = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)
+ return (
+ b"\x89PNG\r\n\x1a\n"
+ + chunk(b"IHDR", header)
+ + chunk(b"IDAT", zlib.compress(b"\x00\x10\x60\xd0"))
+ + chunk(b"IEND", b"")
+ )
+
+
+def _mp4() -> bytes:
+ return struct.pack(">I", 24) + b"ftypisom" + struct.pack(">I", 512) + b"isomiso2" + struct.pack(">I", 8) + b"mdat"
+
+
+def _chat(monkeypatch: pytest.MonkeyPatch, page: Page | None, files: list[CopilotAttachedFile]) -> CopilotContext:
+ @asynccontextmanager
+ async def _admitted(_ctx: AgentContext) -> AsyncIterator[None]:
+ yield
+
+ async def _page(_ctx: AgentContext) -> Page | None:
+ return page
+
+ monkeypatch.setattr(_shared, "mcp_browser_context", _admitted)
+ monkeypatch.setattr(_shared, "live_working_page", _page)
+ ctx = make_copilot_context()
+ ctx.organization_id = CHAT_ORG_ID
+ ctx.browser_session_id = "pbs_chat"
+ ctx.turn_context_packet = TurnContextPacket(
+ transcript_context=TranscriptContext(
+ earliest_user_turn="",
+ latest_prior_user_turn="",
+ latest_assistant_turn="",
+ retained_history="",
+ omitted_any=False,
+ ),
+ attached_file_context=AttachedFileContext(files=files),
+ )
+ return ctx
+
+
+def _attach(
+ repo: FakeUploadedFilesRepository,
+ storage: FakeStorage,
+ filename: str,
+ data: bytes,
+ *,
+ organization_id: str = CHAT_ORG_ID,
+ expires_at: datetime | None = None,
+) -> CopilotAttachedFile:
+ file_id = repo.seed(organization_id, expires_at=expires_at, filename=filename)
+ storage.objects[_uri(organization_id, filename)] = data
+ return CopilotAttachedFile(file_id=file_id, filename=filename, size_bytes=len(data))
+
+
+def repo() -> FakeUploadedFilesRepository:
+ return FakeUploadedFilesRepository()
+
+
+def storage(monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository) -> FakeStorage:
+ fake = FakeStorage()
+ fake_app = SimpleNamespace(DATABASE=SimpleNamespace(uploaded_files=repo), STORAGE=fake)
+ monkeypatch.setattr(uploaded_file_service, "app", fake_app)
+ monkeypatch.setattr(app, "STORAGE", fake)
+ return fake
+
+
+@asynccontextmanager
+async def _fixture_page(html: str = UPLOAD_PAGE) -> AsyncIterator[Page]:
+ async def _serve(route: Route) -> None:
+ await route.fulfill(content_type="text/html", body=html)
+
+ async with async_playwright() as playwright:
+ browser = await playwright.chromium.launch()
+ try:
+ page = await browser.new_page()
+ # An https origin is a secure context, which crypto.subtle needs for the receipt digest.
+ await page.route(PAGE_URL, _serve)
+ await page.goto(PAGE_URL)
+ yield page
+ finally:
+ await browser.close()
+
+
+async def _receipt(page: Page, receipt_id: str) -> str:
+ await page.wait_for_function(f"document.getElementById({receipt_id!r}).textContent !== ''", timeout=5000)
+ return await page.locator(f"#{receipt_id}").inner_text()
+
+
+@_skip_no_browser
+ ("name", "data", "selector", "receipt_id"),
+ [
+ ("synthetic-photo.png", _png(), "#photo", "photo-receipt"),
+ ("synthetic-clip.mp4", _mp4(), "#video", "video-receipt"),
+ ],
+ ids=["png", "mp4"],
+)
+async def test_the_attached_bytes_reach_the_page_and_the_result_is_the_pages_own_receipt(
+ monkeypatch: pytest.MonkeyPatch,
+ repo: FakeUploadedFilesRepository,
+ storage: FakeStorage,
+ name: str,
+ data: bytes,
+ selector: str,
+ receipt_id: str,
+) -> None:
+ attached = _attach(repo, storage, name, data)
+ async with _fixture_page() as page:
+ ctx = _chat(monkeypatch, page, [attached])
+
+ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, selector)
+
+ receipt = await _receipt(page, receipt_id)
+
+ assert receipt == f"{name}:{len(data)}:{hashlib.sha256(data).hexdigest()}"
+ assert result == {
+ "ok": True,
+ "file_id": attached.file_id,
+ "filename": name,
+ "size_bytes": len(data),
+ "sha256": hashlib.sha256(data).hexdigest(),
+ "change_delivered": True,
+ "input_origin": PAGE_URL.rstrip("/"),
+ "input_files": [{"name": name, "size": len(data)}],
+ }
+
+
+def _refusal_cases(
+ repo: FakeUploadedFilesRepository, storage: FakeStorage
+) -> dict[str, tuple[str, list[CopilotAttachedFile], bool]]:
+ png = _png()
+ foreign = _attach(repo, storage, "foreign-photo.png", png, organization_id=VICTIM_ORG_ID)
+ expired = _attach(
+ repo, storage, "expired-photo.png", png, expires_at=datetime.now(timezone.utc) - timedelta(minutes=1)
+ )
+ deleted = _attach(repo, storage, "deleted-photo.png", png)
+ repo.rows[deleted.file_id].deleted_at = datetime.now(timezone.utc)
+ missing = CopilotAttachedFile(file_id="file_999999", filename="missing-photo.png")
+ unlisted = _attach(repo, storage, "unlisted-photo.png", png)
+ return {
+ "cross_org": (foreign.file_id, [foreign], True),
+ "expired": (expired.file_id, [expired], True),
+ "deleted": (deleted.file_id, [deleted], True),
+ "missing": (missing.file_id, [missing], True),
+ "not_in_this_turn": (unlisted.file_id, [], True),
+ "no_turn_packet": (unlisted.file_id, [unlisted], False),
+ }
+
+
+def _assert_refused_naming_only(result: dict[str, Any], file_id: str) -> None:
+ assert set(result) == {"ok", "error"}
+ assert result["ok"] is False
+ assert file_id in result["error"]
+ assert "Nothing was uploaded" in result["error"]
+
+
+@_skip_no_browser
[email protected]("case", ["cross_org", "expired", "deleted", "missing", "not_in_this_turn", "no_turn_packet"])
+async def test_an_unavailable_reference_uploads_nothing_and_names_only_the_id(
+ monkeypatch: pytest.MonkeyPatch,
+ repo: FakeUploadedFilesRepository,
+ storage: FakeStorage,
+ case: str,
+) -> None:
+ file_id, listed, with_packet = _refusal_cases(repo, storage)[case]
+ async with _fixture_page() as page:
+ ctx = _chat(monkeypatch, page, listed)
+ if not with_packet:
+ ctx.turn_context_packet = None
+
+ result = await attached_file_upload.upload_attached_file(ctx, file_id, "#photo")
+
+ held = await page.locator("#photo").evaluate("el => el.files.length")
+
+ _assert_refused_naming_only(result, file_id)
+ assert held == 0
+ assert storage.downloads == []
+ text = json.dumps(result)
+ assert "photo.png" not in text and "s3://" not in text and "/" not in result["error"]
+
+
+async def test_a_context_that_is_not_a_copilot_chat_is_refused(
+ repo: FakeUploadedFilesRepository, storage: FakeStorage
+) -> None:
+ attached = _attach(repo, storage, "synthetic-photo.png", b"png")
+ ctx = _make_ctx()
+ ctx.organization_id = CHAT_ORG_ID
+ ctx.browser_session_id = "pbs_chat"
+
+ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#photo")
+
+ _assert_refused_naming_only(result, attached.file_id)
+ assert storage.downloads == []
+
+
+@_skip_no_browser
+async def test_a_reattached_copy_uploads_after_the_original_was_removed(
+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage
+) -> None:
+ data = _png()
+ removed = _attach(repo, storage, "synthetic-photo.png", data)
+ repo.rows[removed.file_id].deleted_at = datetime.now(timezone.utc)
+ async with _fixture_page() as page:
+ ctx = _chat(monkeypatch, page, [removed])
+ refused = await attached_file_upload.upload_attached_file(ctx, removed.file_id, "#photo")
+ reattached = _attach(repo, storage, "synthetic-photo.png", data)
+ assert ctx.turn_context_packet is not None and ctx.turn_context_packet.attached_file_context is not None
+ ctx.turn_context_packet.attached_file_context.files.append(reattached)
+
+ result = await attached_file_upload.upload_attached_file(ctx, reattached.file_id, "#photo")
+
+ receipt = await _receipt(page, "photo-receipt")
+
+ assert refused["ok"] is False
+ assert result["ok"] is True
+ assert receipt == f"synthetic-photo.png:{len(data)}:{hashlib.sha256(data).hexdigest()}"
+
+
+@_skip_no_browser
+async def test_a_secret_shaped_filename_reaches_the_page_but_not_the_result(
+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage
+) -> None:
+ name = "export password=hunter2.csv"
+ attached = _attach(repo, storage, name, b"a,b\n1,2\n")
+ async with _fixture_page() as page:
+ ctx = _chat(monkeypatch, page, [attached])
+
+ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#photo")
+
+ receipt = await _receipt(page, "photo-receipt")
+
+ assert receipt.startswith(f"{name}:")
+ assert result["ok"] is True
+ assert "hunter2" not in json.dumps(result)
+
+
+_LABELLED_PICKERS = (
+ '<label for="photo">Photo</label><input type="file" id="photo">'
+ '<label id="wrapped"><span id="hint">Add image</span><input type="file"></label>'
+)
+
+
+@_skip_no_browser
+ "selector", ["label[for=photo]", "text=Photo", "#hint"], ids=["label_for", "text", "span_in_label"]
+)
+async def test_a_selector_naming_the_pickers_label_reports_the_input_playwright_filled(
+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage, selector: str
+) -> None:
+ data = _png()
+ attached = _attach(repo, storage, "synthetic-photo.png", data)
+ async with _fixture_page(_LABELLED_PICKERS) as page:
+ ctx = _chat(monkeypatch, page, [attached])
+
+ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, selector)
+
+ held = await page.evaluate("Array.from(document.querySelectorAll('input[type=file]'), el => el.files.length)")
+
+ assert result["ok"] is True
+ assert result["input_files"] == [{"name": "synthetic-photo.png", "size": len(data)}]
+ assert sum(held) == 1
+
+
+_CLEARS_ON_CHANGE = '<input type="file" id="picker" onchange="this.value = \'\'">'
+
+
+@_skip_no_browser
+ ("html", "change_delivered"),
+ [(_CLEARS_ON_CHANGE, True), ('<input type="text" id="picker">', False)],
+ ids=["page_clears_the_file", "not_a_file_input"],
+)
+async def test_a_file_the_input_does_not_keep_is_reported_as_not_uploaded(
+ monkeypatch: pytest.MonkeyPatch,
+ repo: FakeUploadedFilesRepository,
+ storage: FakeStorage,
+ html: str,
+ change_delivered: bool,
+) -> None:
+ attached = _attach(repo, storage, "synthetic-photo.png", _png())
+ async with _fixture_page(html) as page:
+ ctx = _chat(monkeypatch, page, [attached])
+
+ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#picker")
+
+ assert result["ok"] is False
+ assert result["change_delivered"] is change_delivered
+ assert ("upload it twice" in result["error"]) is change_delivered
+
+
+@_skip_no_browser
+async def test_a_set_input_files_error_comes_back_as_the_failure(
+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage
+) -> None:
+ attached = _attach(repo, storage, "synthetic-photo.png", _png())
+ async with _fixture_page() as page:
+ ctx = _chat(monkeypatch, page, [attached])
+
+ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "input[type=file]")
+
+ held = await page.evaluate("Array.from(document.querySelectorAll('input[type=file]'), el => el.files.length)")
+
+ assert result["ok"] is False
+ assert "strict mode violation" in result["error"]
+ assert "twice" not in result["error"]
+ assert held == [0, 0, 0]
+
+
+ ("failure", "expected_detail"),
+ [
+ (RuntimeError("connect ws://10.1.2.3:9222/devtools/browser/abc refused"), "RuntimeError"),
+ (BrowserCdpAcquisitionError("the browser session is not reachable"), "the browser session is not reachable"),
+ ],
+ ids=["unclassified", "classified"],
+)
+async def test_a_browser_that_cannot_be_reached_reports_no_endpoint_text(
+ monkeypatch: pytest.MonkeyPatch,
+ repo: FakeUploadedFilesRepository,
+ storage: FakeStorage,
+ failure: Exception,
+ expected_detail: str,
+) -> None:
+ attached = _attach(repo, storage, "synthetic-photo.png", b"png")
+ ctx = _chat(monkeypatch, None, [attached])
+
+ @asynccontextmanager
+ async def _unreachable(_ctx: AgentContext) -> AsyncIterator[None]:
+ raise failure
+ yield
+
+ monkeypatch.setattr(_shared, "mcp_browser_context", _unreachable)
+
+ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#photo")
+
+ assert result["ok"] is False
+ assert expected_detail in result["error"]
+ assert "10.1.2.3" not in json.dumps(result)
diff --git a/tests/unit/test_copilot_page_challenge.py b/tests/unit/test_copilot_page_challenge.py
index ea931a2af..800178123 100644
--- a/tests/unit/test_copilot_page_challenge.py
+++ b/tests/unit/test_copilot_page_challenge.py
@@ -17,7 +17,7 @@ from skyvern.forge.sdk.copilot.runtime import (
AgentContext,
CopilotBrowserSessionUnavailable,
)
-from skyvern.forge.sdk.copilot.tools import page_challenge
+from skyvern.forge.sdk.copilot.tools import _shared, page_challenge
from skyvern.webeye.utils.captcha_solver import MAX_IMAGE_CAPTCHA_READS, CaptchaChallengeUnsolvedError
from tests.unit.conftest import OcrRecordingAgentFunction
from tests.unit.test_code_block_captcha import _skip_no_browser
@@ -57,8 +57,8 @@ def _chat(
monkeypatch.setattr(page_challenge, "app", MagicMock(AGENT_FUNCTION=stub))
else:
monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)
- monkeypatch.setattr(page_challenge, "mcp_browser_context", _admitted)
- monkeypatch.setattr(page_challenge, "live_working_page", AsyncMock(return_value=page))
+ monkeypatch.setattr(_shared, "mcp_browser_context", _admitted)
+ monkeypatch.setattr(_shared, "live_working_page", AsyncMock(return_value=page))
monkeypatch.setattr(page_challenge, "solve_challenge_ladder", ladder)
ctx = _make_ctx()
ctx.browser_session_id = "pbs_chat"
@@ -123,7 +123,7 @@ async def test_a_browser_lost_during_the_solve_is_reported_as_session_loss_not_u
page.closed = True
ctx = _chat(monkeypatch, page, AsyncMock(side_effect=CaptchaChallengeUnsolvedError()))
disposition = AsyncMock(return_value="reestablished")
- monkeypatch.setattr(page_challenge, "_browser_session_error_disposition", disposition)
+ monkeypatch.setattr(_shared, "_browser_session_error_disposition", disposition)
result = await page_challenge.solve_page_challenge(ctx)
diff --git a/tests/unit/test_uploaded_file_retention.py b/tests/unit/test_uploaded_file_retention.py
index f0822502d..32ddd7815 100644
--- a/tests/unit/test_uploaded_file_retention.py
+++ b/tests/unit/test_uploaded_file_retention.py
@@ -154,10 +154,21 @@ class FakeStorage:
# URIs whose object is already gone. Deleting one succeeds without deleting
# anything, as S3, GCS, Azure and local storage all do for an absent key.
self.absent: set[str] = set()
+ self.objects: dict[str, bytes] = {}
+ self.downloads: list[str] = []
- async def delete_legacy_file(self, *, organization_id: str, uri: str) -> None:
+ @staticmethod
+ def _assert_org_prefix(uri: str, organization_id: str) -> None:
if not uri.startswith(f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{settings.ENV}/{organization_id}/"):
raise PermissionError(f"No permission to access storage URI: {uri}")
+
+ async def download_managed_file(self, uri: str, organization_id: str) -> bytes | None:
+ self.downloads.append(uri)
+ self._assert_org_prefix(uri, organization_id)
+ return self.objects.get(uri)
+
+ async def delete_legacy_file(self, *, organization_id: str, uri: str) -> None:
+ self._assert_org_prefix(uri, organization_id)
if self.fail_with:
raise self.fail_with
if uri in self.absent: