Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Let Copilot upload a chat-attached file into a page's file picker (SKY-16825) (#8691)
Andrew Neilson9f32da7bd624
diff --git a/skyvern/forge/sdk/copilot/blocker_signal.py b/skyvern/forge/sdk/copilot/blocker_signal.pyindex 804fc26a1..19e433486 100644--- a/skyvern/forge/sdk/copilot/blocker_signal.py+++ b/skyvern/forge/sdk/copilot/blocker_signal.py@@ -82,6 +82,7 @@ _INTERNAL_TOOL_NAME_TOKENS: tuple[str, ...] = ("search_web","solve_page_challenge","start_fresh_browser",+ "upload_attached_file","get_browser_screenshot","list_credentials","list_integrations",diff --git a/skyvern/forge/sdk/copilot/narration.py b/skyvern/forge/sdk/copilot/narration.pyindex b366d8e18..8934083bc 100644--- a/skyvern/forge/sdk/copilot/narration.py+++ b/skyvern/forge/sdk/copilot/narration.py@@ -137,6 +137,7 @@ _TOOL_ACTIVITY_DISPLAY_LABELS = {"fill_credential_field": "Entering saved credentials","solve_page_challenge": "Solving the page's verification challenge","start_fresh_browser": "Starting a fresh browser",+ "upload_attached_file": "Attaching your file to the page","run_browser_code": "Running browser code","edit_block": "Editing block","add_block": "Adding block",diff --git a/skyvern/forge/sdk/copilot/output_utils.py b/skyvern/forge/sdk/copilot/output_utils.pyindex 7721be2ef..7b255aa21 100644--- a/skyvern/forge/sdk/copilot/output_utils.py+++ b/skyvern/forge/sdk/copilot/output_utils.py@@ -2057,6 +2057,8 @@ def summarize_tool_result(tool_name: str, result: dict[str, Any], *, for_displayreturn f"{summary} (timed out)" if result.get("timed_out") else summaryif tool_name == "start_fresh_browser":return "Started a fresh browser; the old browser's cookies, sign-ins and open tabs are gone"+ if tool_name == "upload_attached_file":+ return f"Placed {result.get('filename')} ({result.get('size_bytes')} bytes) in the page's file input"if tool_name == "run_browser_code":operations = result.get("operations")count = len(operations) if isinstance(operations, list) else 0diff --git a/skyvern/forge/sdk/copilot/tools/__init__.py b/skyvern/forge/sdk/copilot/tools/__init__.pyindex 8286fe4e2..397f83d99 100644--- a/skyvern/forge/sdk/copilot/tools/__init__.py+++ b/skyvern/forge/sdk/copilot/tools/__init__.py@@ -97,6 +97,7 @@ from ._shared import _raw_yaml_proxy_location as _raw_yaml_proxy_locationfrom ._shared import _same_page_ignoring_fragment as _same_page_ignoring_fragmentfrom ._shared import _unverified_current_workflow_labels as _unverified_current_workflow_labelsfrom ._shared import admitted_requested_output_reads+from .attached_file_upload import UPLOAD_TOOL_NAME, upload_attached_filefrom .banned_blocks import _COPILOT_BANNED_BLOCK_TYPES as _COPILOT_BANNED_BLOCK_TYPESfrom .banned_blocks import AUTHORING_FAMILY_GUIDANCE as AUTHORING_FAMILY_GUIDANCEfrom .banned_blocks import SCHEMA_FIRST_GUIDANCE as SCHEMA_FIRST_GUIDANCE@@ -1749,6 +1750,20 @@ async def start_fresh_browser_tool(ctx: RunContextWrapper) -> str:return json.dumps(scrub_secrets_from_structure(ctx.context, result))+@function_tool(failure_error_function=copilot_tool_failure, name_override=UPLOAD_TOOL_NAME)+async def upload_attached_file_tool(ctx: RunContextWrapper, file_id: str, selector: str) -> str:+ """Set a file the user attached to this chat (`file_id` exactly as listed in the attached files) on the page's+ <input type="file"> named by `selector`, without clicking any submit control; `ok` is true only when the input's+ own file list (`input_files`) then holds this file's name and size, and an unlisted or removed attachment is+ refused."""+ authority_error = _authority_tool_error(ctx.context, UPLOAD_TOOL_NAME)+ if authority_error:+ return _diagnosis_repair_tool_error(ctx.context, UPLOAD_TOOL_NAME, authority_error)+ result = await upload_attached_file(ctx.context, file_id, selector)+ record_tool_step_result_for_ctx(ctx.context, UPLOAD_TOOL_NAME, {"file_id": file_id, "selector": selector}, result)+ return json.dumps(scrub_secrets_from_structure(ctx.context, result))++@function_tool(failure_error_function=copilot_tool_failure, name_override="inspect_page_for_composition", strict_mode=False)@@ -2073,6 +2088,7 @@ NATIVE_TOOLS = [run_browser_code_tool,solve_page_challenge_tool,start_fresh_browser_tool,+ upload_attached_file_tool,]@@ -2089,6 +2105,7 @@ BROWSER_BOUND_TOOL_NAMES = BLOCK_RUNNING_TOOLS | frozenset(BROWSER_CODE_TOOL_NAME,SOLVE_TOOL_NAME,FRESH_BROWSER_TOOL_NAME,+ UPLOAD_TOOL_NAME,})diff --git a/skyvern/forge/sdk/copilot/tools/_shared.py b/skyvern/forge/sdk/copilot/tools/_shared.pyindex 93566e8b7..c465c2e73 100644--- a/skyvern/forge/sdk/copilot/tools/_shared.py+++ b/skyvern/forge/sdk/copilot/tools/_shared.py@@ -2,13 +2,14 @@ from __future__ import annotationsimport asyncioimport json-from collections.abc import Mapping, Sequence+from collections.abc import Awaitable, Callable, Mapping, Sequencefrom dataclasses import dataclassfrom typing import TYPE_CHECKING, Any, Literalfrom urllib.parse import urlparseimport structlogimport yaml+from playwright.async_api import Pagefrom typing_extensions import TypedDictfrom skyvern.cli.core.js_dispatch import outer_cap_seconds@@ -39,11 +40,24 @@ from skyvern.forge.sdk.copilot.enforcement import (_requested_output_labels_by_path,proxy_hop_failure_reason,)+from skyvern.forge.sdk.copilot.mcp_adapter import _browser_session_error_disposition, _browser_session_loss_resultfrom skyvern.forge.sdk.copilot.nav_attribution import proxy_owns_nav_codesfrom skyvern.forge.sdk.copilot.runtime import (+ SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR,+ SENSITIVE_ORIGIN_PAGE_ERROR,AgentContext,+ BrowserProbeOutcome,+ CopilotBrowserGenerationRetired,+ CopilotBrowserSessionUnavailable,+ _browser_context_attachability,+ browser_evidence_commit_lock,+ browser_page_custody_lock,effective_browser_session_id,+ live_working_page,+ mcp_browser_context,resolve_browser_state_for_context,+ sensitive_origin_page_has_active_run,+ sensitive_origin_page_is_tainted,)from skyvern.forge.sdk.copilot.secret_redaction import redact_raw_secrets_for_promptfrom skyvern.forge.sdk.copilot.task_output_envelope import (@@ -57,6 +71,7 @@ from skyvern.forge.sdk.workflow.models.workflow import WorkflowRunStatusfrom skyvern.schemas.proxy_location import ProxyLocationInputfrom skyvern.schemas.workflows import BlockTypefrom skyvern.utils.yaml_loader import safe_load_no_dates+from skyvern.webeye.browser_errors import BrowserAutomationErrorLOG = structlog.get_logger()@@ -1039,3 +1054,50 @@ async def _composition_get_structured_evidence(timeout_seconds=timeout_seconds,)return evidence+++def browser_is_lost(page: Page) -> bool:+ return (+ page.is_closed() or _browser_context_attachability(page.context) is BrowserProbeOutcome.positively_unreachable+ )+++async def on_working_page(+ ctx: AgentContext,+ *,+ tool_name: str,+ no_page_error: str,+ act: Callable[[Page], Awaitable[dict[str, Any]]],+) -> dict[str, Any]:+ async with browser_page_custody_lock(ctx), browser_evidence_commit_lock(ctx):+ if sensitive_origin_page_has_active_run(ctx):+ return {"ok": False, "error": SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR}+ if sensitive_origin_page_is_tainted(ctx):+ return {"ok": False, "error": SENSITIVE_ORIGIN_PAGE_ERROR}+ entered_browser = False+ try:+ async with mcp_browser_context(ctx):+ entered_browser = True+ page = await live_working_page(ctx)+ if page is None:+ return {"ok": False, "error": no_page_error}+ return await act(page)+ except (CopilotBrowserGenerationRetired, CopilotBrowserSessionUnavailable) as exc:+ disposition = await _browser_session_error_disposition(ctx, exc, tool_name=tool_name, call_path="model")+ return _browser_session_loss_result(+ {}, disposition=disposition, deadline_expired=ctx.browser_session_continuity_deadline_expired+ )+ except Exception as exc:+ if entered_browser:+ raise+ # Only a classified error's message has been through CDP-endpoint redaction; an+ # unclassified one is named by type, and its text stays in the log.+ detail = (str(exc).rstrip(".") if isinstance(exc, BrowserAutomationError) else "") or type(exc).__name__+ LOG.warning(+ "copilot native browser tool could not enter its browser",+ tool_name=tool_name,+ browser_session_id=ctx.browser_session_id,+ error_type=type(exc).__name__,+ error=str(exc),+ )+ return {"ok": False, "error": f"{tool_name} could not reach its browser: {detail}. Nothing was done."}diff --git a/skyvern/forge/sdk/copilot/tools/attached_file_upload.py b/skyvern/forge/sdk/copilot/tools/attached_file_upload.pynew file mode 100644index 000000000..0a19b3be6--- /dev/null+++ b/skyvern/forge/sdk/copilot/tools/attached_file_upload.py@@ -0,0 +1,119 @@+from __future__ import annotations++import hashlib+import mimetypes+from typing import Any++from playwright.async_api import FilePayload, Page++from skyvern.forge import app+from skyvern.forge.sdk.api.files import resolve_uploaded_file_id+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.runtime import AgentContext, CopilotBrowserSessionUnavailable+from skyvern.forge.sdk.copilot.secret_redaction import redact_secretlike_filename+from skyvern.forge.sdk.schemas.workflow_copilot import CopilotAttachedFile++from ._shared import browser_is_lost, on_working_page++UPLOAD_TOOL_NAME = "upload_attached_file"++_SET_FILES_TIMEOUT_MS = 15000+_NO_BROWSER = "No browser is open in this chat yet. Navigate to the page with the file input first."+_NO_PAGE = "This chat's browser has no page open. Navigate to the page with the file input first."+# set_input_files retargets a <label>, or an element inside one, to the label's input; read the files from there.+_READ_FILES = """el => {+ const input = el instanceof HTMLInputElement ? el : (el.closest("label")?.control ?? el);+ return {+ origin: input.ownerDocument.location.origin,+ file_input: input instanceof HTMLInputElement && input.type === "file",+ files: Array.from(input.files || [], file => ({name: file.name, size: file.size})),+ };+}"""+_RETRY_NOTE = "Check the page before retrying: a retry can upload it twice."+_DELIVERED_NOTE = (+ "The file was set on the input and its change event fired, so the page may already have taken or sent it "+ f"(some pages clear the input after reading it). {_RETRY_NOTE}"+)+_NOT_A_FILE_INPUT = "The selector does not name a file input, so nothing was set."+++def _unavailable(file_id: str) -> dict[str, Any]:+ return {+ "ok": False,+ "error": (+ f"Attached file {file_id!r} is not available to this chat. Nothing was uploaded; ask the user to "+ "attach the file again."+ ),+ }+++def _attached_in_this_turn(ctx: AgentContext, file_id: str) -> CopilotAttachedFile | None:+ if not isinstance(ctx, CopilotContext) or ctx.turn_context_packet is None:+ return None+ attached_files = ctx.turn_context_packet.attached_file_context+ if attached_files is None:+ return None+ return next((attached for attached in attached_files.files if attached.file_id == file_id), None)+++async def _read_attached_bytes(ctx: AgentContext, file_id: str) -> bytes | None:+ try:+ storage_uri = await resolve_uploaded_file_id(file_id, ctx.organization_id)+ return await app.STORAGE.download_managed_file(storage_uri, ctx.organization_id)+ except (FileNotFoundError, PermissionError):+ return None+++async def upload_attached_file(ctx: AgentContext, file_id: str, selector: str) -> dict[str, Any]:+ attached = _attached_in_this_turn(ctx, file_id)+ if attached is None:+ return _unavailable(file_id)+ session_id = ctx.browser_session_id+ if not session_id:+ return {"ok": False, "error": _NO_BROWSER}+ data = await _read_attached_bytes(ctx, file_id)+ if data is None:+ return _unavailable(file_id)+ payload = FilePayload(+ name=attached.filename,+ mimeType=mimetypes.guess_type(attached.filename)[0] or "application/octet-stream",+ buffer=data,+ )+ uploaded: dict[str, Any] = {+ "file_id": file_id,+ "filename": redact_secretlike_filename(attached.filename),+ "size_bytes": len(data),+ "sha256": hashlib.sha256(data).hexdigest(),+ }++ async def _upload(page: Page) -> dict[str, Any]:+ target = page.locator(selector)+ files_set = False+ try:+ await target.set_input_files(payload, timeout=_SET_FILES_TIMEOUT_MS)+ files_set = True+ readback = await target.evaluate(_READ_FILES, timeout=_SET_FILES_TIMEOUT_MS)+ except Exception as exc:+ if browser_is_lost(page):+ raise CopilotBrowserSessionUnavailable(session_id) from exc+ first_line = next(iter(str(exc).strip().splitlines()), "") or type(exc).__name__+ if files_set:+ first_line = f"{first_line}. The file may already be on the page. {_RETRY_NOTE}"+ return {**uploaded, "ok": False, "error": first_line}+ # set_input_files returns without error or events on a non-file input.+ if not readback["file_input"]:+ return {**uploaded, "ok": False, "change_delivered": False, "error": _NOT_A_FILE_INPUT}+ input_files = readback["files"]+ ok = {"name": attached.filename, "size": len(data)} in input_files+ result: dict[str, Any] = {+ **uploaded,+ "ok": ok,+ "change_delivered": True,+ "input_origin": readback["origin"],+ "input_files": [{**item, "name": redact_secretlike_filename(item["name"])} for item in input_files],+ }+ if not ok:+ result["error"] = f"The file input no longer holds this file. {_DELIVERED_NOTE}"+ return result++ return await on_working_page(ctx, tool_name=UPLOAD_TOOL_NAME, no_page_error=_NO_PAGE, act=_upload)diff --git a/skyvern/forge/sdk/copilot/tools/guardrails.py b/skyvern/forge/sdk/copilot/tools/guardrails.pyindex bf97899eb..b819dad2c 100644--- a/skyvern/forge/sdk/copilot/tools/guardrails.py+++ b/skyvern/forge/sdk/copilot/tools/guardrails.py@@ -168,6 +168,7 @@ def _authority_tool_error("run_browser_code","solve_page_challenge","start_fresh_browser",+ "upload_attached_file","inspect_page_for_composition",LOCATOR_INSPECTION_TOOL_NAME,} and raw_secret_browser_denied(ctx):diff --git a/skyvern/forge/sdk/copilot/tools/page_challenge.py b/skyvern/forge/sdk/copilot/tools/page_challenge.pyindex f47502641..ec3b46f35 100644--- a/skyvern/forge/sdk/copilot/tools/page_challenge.py+++ b/skyvern/forge/sdk/copilot/tools/page_challenge.py@@ -8,23 +8,13 @@ from playwright.async_api import Pagefrom skyvern.forge import appfrom skyvern.forge.sdk.copilot.build_test_connect_failure import BuildTestConnectFailure-from skyvern.forge.sdk.copilot.mcp_adapter import _browser_session_error_disposition, _browser_session_loss_resultfrom skyvern.forge.sdk.copilot.runtime import (SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR,- SENSITIVE_ORIGIN_PAGE_ERROR,AgentContext,- BrowserProbeOutcome,- CopilotBrowserGenerationRetired,CopilotBrowserSessionUnavailable,- _browser_context_attachability,_browser_session_acquisition_failure_result,- browser_evidence_commit_lock,- browser_page_custody_lock,- live_working_page,- mcp_browser_context,replace_browser_session,sensitive_origin_page_has_active_run,- sensitive_origin_page_is_tainted,)from skyvern.forge.sdk.workflow.models.block import CodeBlockCaptchaError, _code_block_solve_captcha_builtinfrom skyvern.webeye.utils.captcha_solver import (@@ -33,6 +23,7 @@ from skyvern.webeye.utils.captcha_solver import (solve_challenge_ladder,)+from ._shared import browser_is_lost, on_working_pagefrom .scouting import rendered_challenge_vendorLOG = structlog.get_logger()@@ -87,32 +78,14 @@ async def solve_page_challenge(session_id = ctx.browser_session_idif not session_id:return {"ok": False, "error": _NO_BROWSER}- async with browser_page_custody_lock(ctx), browser_evidence_commit_lock(ctx):- if sensitive_origin_page_has_active_run(ctx):- return {"ok": False, "error": SENSITIVE_ORIGIN_ACTIVE_RUN_PAGE_ERROR}- if sensitive_origin_page_is_tainted(ctx):- return {"ok": False, "error": SENSITIVE_ORIGIN_PAGE_ERROR}- try:- async with mcp_browser_context(ctx):- page = await live_working_page(ctx)- if page is None:- return {"ok": False, "error": _NO_PAGE}- if image is not None and input is not None:- result = await _read_image(ctx, page, session_id, image, input)- else:- result = await _run_ladder(ctx, page, session_id)- if result["outcome"] == "unsolved" and (- page.is_closed()- or _browser_context_attachability(page.context) is BrowserProbeOutcome.positively_unreachable- ):- raise CopilotBrowserSessionUnavailable(session_id)- except (CopilotBrowserGenerationRetired, CopilotBrowserSessionUnavailable) as exc:- disposition = await _browser_session_error_disposition(- ctx, exc, tool_name=SOLVE_TOOL_NAME, call_path="model"- )- return _browser_session_loss_result(- {}, disposition=disposition, deadline_expired=ctx.browser_session_continuity_deadline_expired- )++ async def _solve(page: Page) -> dict[str, Any]:+ if image is not None and input is not None:+ result = await _read_image(ctx, page, session_id, image, input)+ else:+ result = await _run_ladder(ctx, page, session_id)+ if result["outcome"] == "unsolved" and browser_is_lost(page):+ raise CopilotBrowserSessionUnavailable(session_id)# A fresh browser changes whether a site shows a challenge, not whether its image can be read.if image is not None:return result@@ -130,6 +103,8 @@ async def solve_page_challenge(result["untried_in_this_request"] = [f"{FRESH_BROWSER_TOOL_NAME}: {_FRESH_BROWSER_FACT}"]return result+ return await on_working_page(ctx, tool_name=SOLVE_TOOL_NAME, no_page_error=_NO_PAGE, act=_solve)+async def _run_ladder(ctx: AgentContext, page: Page, session_id: str) -> dict[str, Any]:if not await app.AGENT_FUNCTION.captcha_solving_available(ctx.organization_id, page.url):diff --git a/tests/unit/forge/sdk/copilot/test_organization_usage_tool.py b/tests/unit/forge/sdk/copilot/test_organization_usage_tool.pyindex 7af7ded33..2e8847d22 100644--- a/tests/unit/forge/sdk/copilot/test_organization_usage_tool.py+++ b/tests/unit/forge/sdk/copilot/test_organization_usage_tool.py@@ -74,6 +74,7 @@ CATALOG_WITHOUT_THIS_TOOL = ["run_browser_code","solve_page_challenge","start_fresh_browser",+ "upload_attached_file",]diff --git a/tests/unit/test_copilot_attached_file_upload.py b/tests/unit/test_copilot_attached_file_upload.pynew file mode 100644index 000000000..8235dfd29--- /dev/null+++ b/tests/unit/test_copilot_attached_file_upload.py@@ -0,0 +1,406 @@+from __future__ import annotations++import hashlib+import json+import struct+import zlib+from collections.abc import AsyncIterator+from contextlib import asynccontextmanager+from datetime import datetime, timedelta, timezone+from types import SimpleNamespace+from typing import Any++import pytest+from playwright.async_api import Page, Route, async_playwright++from skyvern.forge import app+from skyvern.forge.sdk.copilot.context import CopilotContext+from skyvern.forge.sdk.copilot.runtime import AgentContext+from skyvern.forge.sdk.copilot.tools import _shared, attached_file_upload+from skyvern.forge.sdk.copilot.turn_context import AttachedFileContext, TranscriptContext, TurnContextPacket+from skyvern.forge.sdk.schemas.workflow_copilot import CopilotAttachedFile+from skyvern.services import uploaded_file_service+from skyvern.webeye.browser_errors import BrowserCdpAcquisitionError+from tests.unit.conftest import make_copilot_context+from tests.unit.test_code_block_captcha import _skip_no_browser+from tests.unit.test_copilot_runtime import _make_ctx+from tests.unit.test_uploaded_file_retention import (+ ATTACKER_ORG_ID,+ VICTIM_ORG_ID,+ FakeStorage,+ FakeUploadedFilesRepository,+ _uri,+)++CHAT_ORG_ID = ATTACKER_ORG_ID+PAGE_URL = "https://upload.fixture.test/"++UPLOAD_PAGE = """<!doctype html>+<input type="file" id="resume" accept="application/pdf">+<input type="file" id="photo" accept="image/*"><div id="photo-receipt"></div>+<input type="file" id="video" accept="video/*"><div id="video-receipt"></div>+<script>+ for (const id of ["photo", "video"]) {+ document.getElementById(id).addEventListener("change", async (event) => {+ const file = event.target.files[0];+ const digest = await crypto.subtle.digest("SHA-256", await file.arrayBuffer());+ const hex = Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");+ document.getElementById(`${id}-receipt`).textContent = `${file.name}:${file.size}:${hex}`;+ });+ }+</script>+"""+++def _png() -> bytes:+ def chunk(kind: bytes, body: bytes) -> bytes:+ return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body))++ header = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0)+ return (+ b"\x89PNG\r\n\x1a\n"+ + chunk(b"IHDR", header)+ + chunk(b"IDAT", zlib.compress(b"\x00\x10\x60\xd0"))+ + chunk(b"IEND", b"")+ )+++def _mp4() -> bytes:+ return struct.pack(">I", 24) + b"ftypisom" + struct.pack(">I", 512) + b"isomiso2" + struct.pack(">I", 8) + b"mdat"+++def _chat(monkeypatch: pytest.MonkeyPatch, page: Page | None, files: list[CopilotAttachedFile]) -> CopilotContext:+ @asynccontextmanager+ async def _admitted(_ctx: AgentContext) -> AsyncIterator[None]:+ yield++ async def _page(_ctx: AgentContext) -> Page | None:+ return page++ monkeypatch.setattr(_shared, "mcp_browser_context", _admitted)+ monkeypatch.setattr(_shared, "live_working_page", _page)+ ctx = make_copilot_context()+ ctx.organization_id = CHAT_ORG_ID+ ctx.browser_session_id = "pbs_chat"+ ctx.turn_context_packet = TurnContextPacket(+ transcript_context=TranscriptContext(+ earliest_user_turn="",+ latest_prior_user_turn="",+ latest_assistant_turn="",+ retained_history="",+ omitted_any=False,+ ),+ attached_file_context=AttachedFileContext(files=files),+ )+ return ctx+++def _attach(+ repo: FakeUploadedFilesRepository,+ storage: FakeStorage,+ filename: str,+ data: bytes,+ *,+ organization_id: str = CHAT_ORG_ID,+ expires_at: datetime | None = None,+) -> CopilotAttachedFile:+ file_id = repo.seed(organization_id, expires_at=expires_at, filename=filename)+ storage.objects[_uri(organization_id, filename)] = data+ return CopilotAttachedFile(file_id=file_id, filename=filename, size_bytes=len(data))+++def repo() -> FakeUploadedFilesRepository:+ return FakeUploadedFilesRepository()+++def storage(monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository) -> FakeStorage:+ fake = FakeStorage()+ fake_app = SimpleNamespace(DATABASE=SimpleNamespace(uploaded_files=repo), STORAGE=fake)+ monkeypatch.setattr(uploaded_file_service, "app", fake_app)+ monkeypatch.setattr(app, "STORAGE", fake)+ return fake+++@asynccontextmanager+async def _fixture_page(html: str = UPLOAD_PAGE) -> AsyncIterator[Page]:+ async def _serve(route: Route) -> None:+ await route.fulfill(content_type="text/html", body=html)++ async with async_playwright() as playwright:+ browser = await playwright.chromium.launch()+ try:+ page = await browser.new_page()+ # An https origin is a secure context, which crypto.subtle needs for the receipt digest.+ await page.route(PAGE_URL, _serve)+ await page.goto(PAGE_URL)+ yield page+ finally:+ await browser.close()+++async def _receipt(page: Page, receipt_id: str) -> str:+ await page.wait_for_function(f"document.getElementById({receipt_id!r}).textContent !== ''", timeout=5000)+ return await page.locator(f"#{receipt_id}").inner_text()+++@_skip_no_browser+ ("name", "data", "selector", "receipt_id"),+ [+ ("synthetic-photo.png", _png(), "#photo", "photo-receipt"),+ ("synthetic-clip.mp4", _mp4(), "#video", "video-receipt"),+ ],+ ids=["png", "mp4"],+)+async def test_the_attached_bytes_reach_the_page_and_the_result_is_the_pages_own_receipt(+ monkeypatch: pytest.MonkeyPatch,+ repo: FakeUploadedFilesRepository,+ storage: FakeStorage,+ name: str,+ data: bytes,+ selector: str,+ receipt_id: str,+) -> None:+ attached = _attach(repo, storage, name, data)+ async with _fixture_page() as page:+ ctx = _chat(monkeypatch, page, [attached])++ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, selector)++ receipt = await _receipt(page, receipt_id)++ assert receipt == f"{name}:{len(data)}:{hashlib.sha256(data).hexdigest()}"+ assert result == {+ "ok": True,+ "file_id": attached.file_id,+ "filename": name,+ "size_bytes": len(data),+ "sha256": hashlib.sha256(data).hexdigest(),+ "change_delivered": True,+ "input_origin": PAGE_URL.rstrip("/"),+ "input_files": [{"name": name, "size": len(data)}],+ }+++def _refusal_cases(+ repo: FakeUploadedFilesRepository, storage: FakeStorage+) -> dict[str, tuple[str, list[CopilotAttachedFile], bool]]:+ png = _png()+ foreign = _attach(repo, storage, "foreign-photo.png", png, organization_id=VICTIM_ORG_ID)+ expired = _attach(+ repo, storage, "expired-photo.png", png, expires_at=datetime.now(timezone.utc) - timedelta(minutes=1)+ )+ deleted = _attach(repo, storage, "deleted-photo.png", png)+ repo.rows[deleted.file_id].deleted_at = datetime.now(timezone.utc)+ missing = CopilotAttachedFile(file_id="file_999999", filename="missing-photo.png")+ unlisted = _attach(repo, storage, "unlisted-photo.png", png)+ return {+ "cross_org": (foreign.file_id, [foreign], True),+ "expired": (expired.file_id, [expired], True),+ "deleted": (deleted.file_id, [deleted], True),+ "missing": (missing.file_id, [missing], True),+ "not_in_this_turn": (unlisted.file_id, [], True),+ "no_turn_packet": (unlisted.file_id, [unlisted], False),+ }+++def _assert_refused_naming_only(result: dict[str, Any], file_id: str) -> None:+ assert set(result) == {"ok", "error"}+ assert result["ok"] is False+ assert file_id in result["error"]+ assert "Nothing was uploaded" in result["error"]+++@_skip_no_browser[email protected]("case", ["cross_org", "expired", "deleted", "missing", "not_in_this_turn", "no_turn_packet"])+async def test_an_unavailable_reference_uploads_nothing_and_names_only_the_id(+ monkeypatch: pytest.MonkeyPatch,+ repo: FakeUploadedFilesRepository,+ storage: FakeStorage,+ case: str,+) -> None:+ file_id, listed, with_packet = _refusal_cases(repo, storage)[case]+ async with _fixture_page() as page:+ ctx = _chat(monkeypatch, page, listed)+ if not with_packet:+ ctx.turn_context_packet = None++ result = await attached_file_upload.upload_attached_file(ctx, file_id, "#photo")++ held = await page.locator("#photo").evaluate("el => el.files.length")++ _assert_refused_naming_only(result, file_id)+ assert held == 0+ assert storage.downloads == []+ text = json.dumps(result)+ assert "photo.png" not in text and "s3://" not in text and "/" not in result["error"]+++async def test_a_context_that_is_not_a_copilot_chat_is_refused(+ repo: FakeUploadedFilesRepository, storage: FakeStorage+) -> None:+ attached = _attach(repo, storage, "synthetic-photo.png", b"png")+ ctx = _make_ctx()+ ctx.organization_id = CHAT_ORG_ID+ ctx.browser_session_id = "pbs_chat"++ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#photo")++ _assert_refused_naming_only(result, attached.file_id)+ assert storage.downloads == []+++@_skip_no_browser+async def test_a_reattached_copy_uploads_after_the_original_was_removed(+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage+) -> None:+ data = _png()+ removed = _attach(repo, storage, "synthetic-photo.png", data)+ repo.rows[removed.file_id].deleted_at = datetime.now(timezone.utc)+ async with _fixture_page() as page:+ ctx = _chat(monkeypatch, page, [removed])+ refused = await attached_file_upload.upload_attached_file(ctx, removed.file_id, "#photo")+ reattached = _attach(repo, storage, "synthetic-photo.png", data)+ assert ctx.turn_context_packet is not None and ctx.turn_context_packet.attached_file_context is not None+ ctx.turn_context_packet.attached_file_context.files.append(reattached)++ result = await attached_file_upload.upload_attached_file(ctx, reattached.file_id, "#photo")++ receipt = await _receipt(page, "photo-receipt")++ assert refused["ok"] is False+ assert result["ok"] is True+ assert receipt == f"synthetic-photo.png:{len(data)}:{hashlib.sha256(data).hexdigest()}"+++@_skip_no_browser+async def test_a_secret_shaped_filename_reaches_the_page_but_not_the_result(+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage+) -> None:+ name = "export password=hunter2.csv"+ attached = _attach(repo, storage, name, b"a,b\n1,2\n")+ async with _fixture_page() as page:+ ctx = _chat(monkeypatch, page, [attached])++ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#photo")++ receipt = await _receipt(page, "photo-receipt")++ assert receipt.startswith(f"{name}:")+ assert result["ok"] is True+ assert "hunter2" not in json.dumps(result)+++_LABELLED_PICKERS = (+ '<label for="photo">Photo</label><input type="file" id="photo">'+ '<label id="wrapped"><span id="hint">Add image</span><input type="file"></label>'+)+++@_skip_no_browser+ "selector", ["label[for=photo]", "text=Photo", "#hint"], ids=["label_for", "text", "span_in_label"]+)+async def test_a_selector_naming_the_pickers_label_reports_the_input_playwright_filled(+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage, selector: str+) -> None:+ data = _png()+ attached = _attach(repo, storage, "synthetic-photo.png", data)+ async with _fixture_page(_LABELLED_PICKERS) as page:+ ctx = _chat(monkeypatch, page, [attached])++ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, selector)++ held = await page.evaluate("Array.from(document.querySelectorAll('input[type=file]'), el => el.files.length)")++ assert result["ok"] is True+ assert result["input_files"] == [{"name": "synthetic-photo.png", "size": len(data)}]+ assert sum(held) == 1+++_CLEARS_ON_CHANGE = '<input type="file" id="picker" onchange="this.value = \'\'">'+++@_skip_no_browser+ ("html", "change_delivered"),+ [(_CLEARS_ON_CHANGE, True), ('<input type="text" id="picker">', False)],+ ids=["page_clears_the_file", "not_a_file_input"],+)+async def test_a_file_the_input_does_not_keep_is_reported_as_not_uploaded(+ monkeypatch: pytest.MonkeyPatch,+ repo: FakeUploadedFilesRepository,+ storage: FakeStorage,+ html: str,+ change_delivered: bool,+) -> None:+ attached = _attach(repo, storage, "synthetic-photo.png", _png())+ async with _fixture_page(html) as page:+ ctx = _chat(monkeypatch, page, [attached])++ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#picker")++ assert result["ok"] is False+ assert result["change_delivered"] is change_delivered+ assert ("upload it twice" in result["error"]) is change_delivered+++@_skip_no_browser+async def test_a_set_input_files_error_comes_back_as_the_failure(+ monkeypatch: pytest.MonkeyPatch, repo: FakeUploadedFilesRepository, storage: FakeStorage+) -> None:+ attached = _attach(repo, storage, "synthetic-photo.png", _png())+ async with _fixture_page() as page:+ ctx = _chat(monkeypatch, page, [attached])++ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "input[type=file]")++ held = await page.evaluate("Array.from(document.querySelectorAll('input[type=file]'), el => el.files.length)")++ assert result["ok"] is False+ assert "strict mode violation" in result["error"]+ assert "twice" not in result["error"]+ assert held == [0, 0, 0]+++ ("failure", "expected_detail"),+ [+ (RuntimeError("connect ws://10.1.2.3:9222/devtools/browser/abc refused"), "RuntimeError"),+ (BrowserCdpAcquisitionError("the browser session is not reachable"), "the browser session is not reachable"),+ ],+ ids=["unclassified", "classified"],+)+async def test_a_browser_that_cannot_be_reached_reports_no_endpoint_text(+ monkeypatch: pytest.MonkeyPatch,+ repo: FakeUploadedFilesRepository,+ storage: FakeStorage,+ failure: Exception,+ expected_detail: str,+) -> None:+ attached = _attach(repo, storage, "synthetic-photo.png", b"png")+ ctx = _chat(monkeypatch, None, [attached])++ @asynccontextmanager+ async def _unreachable(_ctx: AgentContext) -> AsyncIterator[None]:+ raise failure+ yield++ monkeypatch.setattr(_shared, "mcp_browser_context", _unreachable)++ result = await attached_file_upload.upload_attached_file(ctx, attached.file_id, "#photo")++ assert result["ok"] is False+ assert expected_detail in result["error"]+ assert "10.1.2.3" not in json.dumps(result)diff --git a/tests/unit/test_copilot_page_challenge.py b/tests/unit/test_copilot_page_challenge.pyindex ea931a2af..800178123 100644--- a/tests/unit/test_copilot_page_challenge.py+++ b/tests/unit/test_copilot_page_challenge.py@@ -17,7 +17,7 @@ from skyvern.forge.sdk.copilot.runtime import (AgentContext,CopilotBrowserSessionUnavailable,)-from skyvern.forge.sdk.copilot.tools import page_challenge+from skyvern.forge.sdk.copilot.tools import _shared, page_challengefrom skyvern.webeye.utils.captcha_solver import MAX_IMAGE_CAPTCHA_READS, CaptchaChallengeUnsolvedErrorfrom tests.unit.conftest import OcrRecordingAgentFunctionfrom tests.unit.test_code_block_captcha import _skip_no_browser@@ -57,8 +57,8 @@ def _chat(monkeypatch.setattr(page_challenge, "app", MagicMock(AGENT_FUNCTION=stub))else:monkeypatch.setattr(app, "AGENT_FUNCTION", agent_function)- monkeypatch.setattr(page_challenge, "mcp_browser_context", _admitted)- monkeypatch.setattr(page_challenge, "live_working_page", AsyncMock(return_value=page))+ monkeypatch.setattr(_shared, "mcp_browser_context", _admitted)+ monkeypatch.setattr(_shared, "live_working_page", AsyncMock(return_value=page))monkeypatch.setattr(page_challenge, "solve_challenge_ladder", ladder)ctx = _make_ctx()ctx.browser_session_id = "pbs_chat"@@ -123,7 +123,7 @@ async def test_a_browser_lost_during_the_solve_is_reported_as_session_loss_not_upage.closed = Truectx = _chat(monkeypatch, page, AsyncMock(side_effect=CaptchaChallengeUnsolvedError()))disposition = AsyncMock(return_value="reestablished")- monkeypatch.setattr(page_challenge, "_browser_session_error_disposition", disposition)+ monkeypatch.setattr(_shared, "_browser_session_error_disposition", disposition)result = await page_challenge.solve_page_challenge(ctx)diff --git a/tests/unit/test_uploaded_file_retention.py b/tests/unit/test_uploaded_file_retention.pyindex f0822502d..32ddd7815 100644--- a/tests/unit/test_uploaded_file_retention.py+++ b/tests/unit/test_uploaded_file_retention.py@@ -154,10 +154,21 @@ class FakeStorage:# URIs whose object is already gone. Deleting one succeeds without deleting# anything, as S3, GCS, Azure and local storage all do for an absent key.self.absent: set[str] = set()+ self.objects: dict[str, bytes] = {}+ self.downloads: list[str] = []- async def delete_legacy_file(self, *, organization_id: str, uri: str) -> None:+ @staticmethod+ def _assert_org_prefix(uri: str, organization_id: str) -> None:if not uri.startswith(f"s3://{settings.AWS_S3_BUCKET_UPLOADS}/{settings.ENV}/{organization_id}/"):raise PermissionError(f"No permission to access storage URI: {uri}")++ async def download_managed_file(self, uri: str, organization_id: str) -> bytes | None:+ self.downloads.append(uri)+ self._assert_org_prefix(uri, organization_id)+ return self.objects.get(uri)++ async def delete_legacy_file(self, *, organization_id: str, uri: str) -> None:+ self._assert_org_prefix(uri, organization_id)if self.fail_with:raise self.fail_withif uri in self.absent: