Automate browser based workflows with AI
Python aiapiautomationbrowserbrowser-automationcomputergptllmplaywrightpowerautomatepuppeteerpythonrpaseleniumvisionworkflow
Keep Task V3 observe readable when a page defines its own toJSON (#8770)
pedrohsdb84e59bc0c7a5
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.pyindex b387c194e..502654bc6 100644--- a/skyvern/forge/taskv3/tools.py+++ b/skyvern/forge/taskv3/tools.py@@ -10194,7 +10194,21 @@ async (__tv3TextArgs) => {rec.ref = typeof r === 'number' ? r : null;}__OBSERVE_TEXT_DELTA__- const payload = JSON.stringify({ textDelta: textDelta, refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });+ // A page-defined toJSON (legacy libraries put one on Array.prototype) encodes every array as a string. It is+ // lifted for this one synchronous call so page code never runs; the replacer runs after toJSON and hands+ // back the holder's own value, so even one the page made non-configurable never reaches the payload.+ const _ownStringify = (value) => {+ const protos = [Array.prototype, Object.prototype];+ const saved = protos.map((p) => Object.getOwnPropertyDescriptor(p, 'toJSON'));+ // Kept in place when it could not be put back (non-configurable, or a non-extensible prototype).+ const kept = protos.map((p, i) => !!saved[i] && !(saved[i].configurable && Object.isExtensible(p)));+ protos.forEach((p, i) => { if (saved[i] && !kept[i]) delete p.toJSON; });+ // A kept toJSON still runs before the replacer; if it throws, the reading is refused rather than the evaluate.+ try { return JSON.stringify(value, kept.some(Boolean) ? function (k) { return this[k]; } : undefined); } catch (e) { return 'null'; } finally {+ protos.forEach((p, i) => { if (saved[i] && !kept[i]) Object.defineProperty(p, 'toJSON', saved[i]); });+ }+ };+ const payload = _ownStringify({ textDelta: textDelta, refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });return __OBSERVE_RETURN__;}"""@@ -11374,6 +11388,41 @@ class _RealmChangedDuringRead(Exception):"""A realm's document was replaced between the two identity samples bracketing its read."""+class _RealmUnreadable(Exception):+ """A realm's reading is not the shape observe builds, so nothing in it can be paired or merged."""+++def _realm_reading(raw: Any) -> dict[str, Any]:+ # Reachable only when the page replaced JSON.stringify itself: every other shape the script can emit is fixed.+ try:+ data = json.loads(raw) if isinstance(raw, str) else raw+ except ValueError:+ raise _RealmUnreadable("reading is not JSON") from None+ if not isinstance(data, dict):+ raise _RealmUnreadable(f"reading is a {type(data).__name__}")+ iframes = data.get("iframes")+ lists = {key: data.get(key) for key in ("elements", "text", "textFull")}+ if isinstance(iframes, dict):+ lists["iframes.entries"] = iframes.get("entries")+ elif iframes is not None:+ raise _RealmUnreadable(f"iframes is a {type(iframes).__name__}")+ for key, value in lists.items():+ if (value is not None or key in data) and not isinstance(value, list):+ raise _RealmUnreadable(f"{key} is a {type(value).__name__}")+ for key in ("elements", "iframes.entries"):+ if not all(isinstance(e, dict) for e in lists.get(key) or []):+ raise _RealmUnreadable(f"{key} holds a non-record entry")+ if not all(isinstance(t, str) for t in lists["text"] or []) or not all(+ t is None or isinstance(t, str) for t in lists["textFull"] or []+ ):+ raise _RealmUnreadable("text holds a non-string entry")+ for key in _OBSERVE_SUMMED_KEYS:+ count = data.get(key) or 0+ if not isinstance(count, (int, float)) or not math.isfinite(count):+ raise _RealmUnreadable(f"{key} is a {type(data.get(key)).__name__}")+ return data++class _Observation(NamedTuple):"""One reading of the page, spanning its main frame and the child frames observe could read."""@@ -11939,8 +11988,17 @@ def build_browser_tools(after = await _realm_document_id(target)if after != before:raise _RealmChangedDuringRead(f"realm changed during the read: {before!r} -> {after!r}")- data = json.loads(raw) if isinstance(raw, str) else raw- inline = data.pop("textDelta", None) if isinstance(data, dict) else None+ try:+ data = _realm_reading(raw)+ except _RealmUnreadable:+ for handle in handles:+ if handle is not None:+ try:+ await handle.dispose()+ except Exception:+ pass+ raise+ inline = data.pop("textDelta", None)if text_args is not None:# Cleared on every main-frame attempt: a retry on a replacement document that yields no text# must not leave the previous document's text to be recorded as this one's.@@ -11998,7 +12056,10 @@ def build_browser_tools(if isinstance(result, BaseException):# One frame that will not answer costs its own contents and nothing else, and is# counted so the digest can say a region went unread rather than imply it was empty.- LOG.debug("taskv3 observe could not read a child frame", exc_info=result)+ if isinstance(result, _RealmUnreadable):+ LOG.info("taskv3 observe skipped a child frame", reason=str(result))+ else:+ LOG.debug("taskv3 observe could not read a child frame", exc_info=result)unreadable += 1continueframe_data, frame_handles, frame_document = result@@ -12043,7 +12104,13 @@ def build_browser_tools(return error# Bound the whole acquisition -- the digest AND the handles behind it -- so a wedged page# can't hang the turn indefinitely on a later round trip the first bound never covered.- observation = await asyncio.wait_for(_read_observation(page), timeout=30)+ try:+ observation = await asyncio.wait_for(_read_observation(page), timeout=30)+ except _RealmUnreadable as exc:+ LOG.warning("taskv3 observe could not read the page", reason=str(exc))+ return ToolResult.error(+ "observe could not read this page's controls; use look or get_html to read it instead"+ )data, handles, owners = observation.data, observation.handles, observation.ownerselements = data.get("elements", [])if len(handles) != len(elements):diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.pyindex 66e0c2799..592e99825 100644--- a/tests/unit/test_taskv3_tools.py+++ b/tests/unit/test_taskv3_tools.py@@ -5044,6 +5044,95 @@ async def test_observe_a_frames_radio_group_keeps_its_marker_beside_the_pages_owassert [("(no question found)" in line) for line in radio_lines] == [True, False, True, False], r.content+def _two_realm_page(page_script: str, frame_script: str) -> str:+ frame = f"<script>{frame_script}</script><button>Frame action</button>".replace('"', "'")+ return (+ f"<!doctype html><html><body><script>{page_script}</script><button>Page action</button>"+ f'<iframe srcdoc="{frame}" width="400" height="200"></iframe></body></html>'+ )+++# Legacy libraries define Array.prototype.toJSON, and JSON.stringify calls it on every array it meets.+_LEGACY_TOJSON = "Array.prototype.toJSON = function () { return 'legacy'; };"+# Non-configurable, so nothing can lift it; and one encoding each array as JSON text, as a library's own encoder does.+_PINNED_TOJSON = "Object.defineProperty(Array.prototype, 'toJSON', { value: function () { return 'legacy'; } });"+_PINNED_JSON_TOJSON = (+ "let busy = false; Object.defineProperty(Array.prototype, 'toJSON', { value: function () {"+ " if (busy) return Array.from(this); busy = true;"+ " try { return JSON.stringify(this); } finally { busy = false; } } });"+)+++@_skip_no_browser+ "page_script,frame_script",+ [+ (_LEGACY_TOJSON, ""),+ ("", _LEGACY_TOJSON),+ (_LEGACY_TOJSON, _LEGACY_TOJSON),+ (_PINNED_TOJSON, _PINNED_TOJSON),+ ("", _PINNED_JSON_TOJSON),+ # Lifted, so it never runs; pinned, it would throw out of the stringify and lose the realm.+ ("Array.prototype.toJSON = function () { throw new Error('legacy'); };", ""),+ # JSON.stringify never consults these for the payload's primitives, and the payload holds no Date.+ ("String.prototype.toJSON = Date.prototype.toJSON = function () { return 'legacy'; };", ""),+ ("Object.prototype.toJSON = function () { return 'legacy'; };", _LEGACY_TOJSON),+ # Non-extensible, so the property could not be put back: it stays in place and the replacer covers it.+ (_LEGACY_TOJSON + " Object.preventExtensions(Array.prototype);", ""),+ ],+)+async def test_observe_reads_realms_whose_page_defines_array_tojson(page_script: str, frame_script: str) -> None:+ descriptors = (+ "() => [Array.prototype, Object.prototype, String.prototype].map((p) => {"+ " const d = Object.getOwnPropertyDescriptor(p, 'toJSON'); if (!d) return null;"+ " if (!window.__fixtureToJSON) window.__fixtureToJSON = new Map(); const m = window.__fixtureToJSON;"+ " if (!m.has(p)) m.set(p, d.value);"+ " return [m.get(p) === d.value, d.enumerable, d.writable, d.configurable]; })"+ )+ async with _live_page(_two_realm_page(page_script, frame_script)) as page:+ before = await page.evaluate(descriptors)+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})+ after = await page.evaluate(descriptors)+ assert r.status == "ok", r.content+ assert "Page action" in r.content and "Frame action" in r.content, r.content+ # The page gets back its own function under its own descriptor (only its key order may change).+ assert after == before+++@_skip_no_browser+ "reading",+ [+ "'legacy'",+ "real('legacy')",+ "real({elements: 'legacy'})",+ "real({elements: null})",+ "real({elements: 5})",+ "real({elements: [1]})",+ "real({elements: [], iframes: {entries: 'x'}})",+ "real({elements: [], dropped: 'x'})",+ "real({elements: [], text: [1]})",+ "real({elements: [], text: [], textFull: [1]})",+ "real({elements: [], dropped: 1}).replace('1', 'NaN')",+ # What a pinned toJSON that throws does to the call.+ "(() => { throw new Error('legacy'); })()",+ ],+)+async def test_observe_skips_an_unreadable_frame_and_refuses_an_unreadable_page_without_raising(reading: str) -> None:+ # A page that replaces JSON.stringify itself decides what observe's reading of that realm says.+ script = f"const real = JSON.stringify; JSON.stringify = function () {{ return {reading}; }};"+ async with _live_page(_two_realm_page("", script)) as page:+ frame_skipped = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})+ assert frame_skipped.status == "ok", frame_skipped.content+ assert "Page action" in frame_skipped.content and "Frame action" not in frame_skipped.content+ async with _live_page(_two_realm_page(script, "")) as page:+ page_refused = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})+ assert page_refused.status == "error"+ assert "could not read this page" in page_refused.content++@_skip_no_browser@pytest.mark.asyncioasync def test_observe_a_radio_groups_description_never_puts_its_marker_mid_group() -> None: