Skip to content

Automate browser based workflows with AI

Keep Task V3 observe readable when a page defines its own toJSON (#8770)

pedrohsdb84e59bc0c7a5
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index b387c194e..502654bc6 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -10194,7 +10194,21 @@ async (__tv3TextArgs) => {
rec.ref = typeof r === 'number' ? r : null;
}
__OBSERVE_TEXT_DELTA__
- const payload = JSON.stringify({ textDelta: textDelta, refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });
+ // A page-defined toJSON (legacy libraries put one on Array.prototype) encodes every array as a string. It is
+ // lifted for this one synchronous call so page code never runs; the replacer runs after toJSON and hands
+ // back the holder's own value, so even one the page made non-configurable never reaches the payload.
+ const _ownStringify = (value) => {
+ const protos = [Array.prototype, Object.prototype];
+ const saved = protos.map((p) => Object.getOwnPropertyDescriptor(p, 'toJSON'));
+ // Kept in place when it could not be put back (non-configurable, or a non-extensible prototype).
+ const kept = protos.map((p, i) => !!saved[i] && !(saved[i].configurable && Object.isExtensible(p)));
+ protos.forEach((p, i) => { if (saved[i] && !kept[i]) delete p.toJSON; });
+ // A kept toJSON still runs before the replacer; if it throws, the reading is refused rather than the evaluate.
+ try { return JSON.stringify(value, kept.some(Boolean) ? function (k) { return this[k]; } : undefined); } catch (e) { return 'null'; } finally {
+ protos.forEach((p, i) => { if (saved[i] && !kept[i]) Object.defineProperty(p, 'toJSON', saved[i]); });
+ }
+ };
+ const payload = _ownStringify({ textDelta: textDelta, refsFresh: refsFresh, url: location.href, title: document.title, text: texts, textFull: texts.map((t) => { const f = fullText.get(t); return f && f !== t ? f : null; }), textTruncated: textFull, textDropped: textDropped, iframes: iframeInfo, frameCensus: frameCensus, dropped: dropped, truncated: truncated, truncatedInComponents: truncatedInComponents, pointerCapped: pointerCapped, pointerTruncated: pointerTruncated, pointerDropped: pointerDropped, pointerListed: pointerListed, pointerScanStopped: _pointerState.scanStopped ? 1 : 0, pointerScanFailed: _pointerState.scanFailed ? 1 : 0, unnamedAnonymous: unnamedAnonymous, unnamedBudget: unnamedBudget, unnamedDuplicated: unnamedDuplicated, unnamedUnverifiable: unnamedUnverifiable, unnamedUnsafe: unnamedUnsafe, unreadableRoot: sawUnreadableRoot, undiscoveredRoots: undiscoveredRoots, rootCount: allRoots.length - 1, hiddenListed: hiddenListed, hiddenDropped: hiddenDropped, hiddenDroppedOffCanvas: hiddenDroppedOffCanvas, hiddenDroppedVisibility: hiddenDroppedVisibility, hiddenDroppedZeroRect: hiddenDroppedZeroRect, hiddenDroppedOffViewport: hiddenDroppedOffViewport, offViewportUnreachableUnnamed: offViewportUnreachableUnnamed, offViewportUnnamedHostExempt: offViewportUnnamedHostExempt, phantomDropped: phantomDropped, markersMinted: markersWritten, markersReused: markersReused, pageMutated: mutated, elements: out });
return __OBSERVE_RETURN__;
}
"""
@@ -11374,6 +11388,41 @@ class _RealmChangedDuringRead(Exception):
"""A realm's document was replaced between the two identity samples bracketing its read."""
+class _RealmUnreadable(Exception):
+ """A realm's reading is not the shape observe builds, so nothing in it can be paired or merged."""
+
+
+def _realm_reading(raw: Any) -> dict[str, Any]:
+ # Reachable only when the page replaced JSON.stringify itself: every other shape the script can emit is fixed.
+ try:
+ data = json.loads(raw) if isinstance(raw, str) else raw
+ except ValueError:
+ raise _RealmUnreadable("reading is not JSON") from None
+ if not isinstance(data, dict):
+ raise _RealmUnreadable(f"reading is a {type(data).__name__}")
+ iframes = data.get("iframes")
+ lists = {key: data.get(key) for key in ("elements", "text", "textFull")}
+ if isinstance(iframes, dict):
+ lists["iframes.entries"] = iframes.get("entries")
+ elif iframes is not None:
+ raise _RealmUnreadable(f"iframes is a {type(iframes).__name__}")
+ for key, value in lists.items():
+ if (value is not None or key in data) and not isinstance(value, list):
+ raise _RealmUnreadable(f"{key} is a {type(value).__name__}")
+ for key in ("elements", "iframes.entries"):
+ if not all(isinstance(e, dict) for e in lists.get(key) or []):
+ raise _RealmUnreadable(f"{key} holds a non-record entry")
+ if not all(isinstance(t, str) for t in lists["text"] or []) or not all(
+ t is None or isinstance(t, str) for t in lists["textFull"] or []
+ ):
+ raise _RealmUnreadable("text holds a non-string entry")
+ for key in _OBSERVE_SUMMED_KEYS:
+ count = data.get(key) or 0
+ if not isinstance(count, (int, float)) or not math.isfinite(count):
+ raise _RealmUnreadable(f"{key} is a {type(data.get(key)).__name__}")
+ return data
+
+
class _Observation(NamedTuple):
"""One reading of the page, spanning its main frame and the child frames observe could read."""
@@ -11939,8 +11988,17 @@ def build_browser_tools(
after = await _realm_document_id(target)
if after != before:
raise _RealmChangedDuringRead(f"realm changed during the read: {before!r} -> {after!r}")
- data = json.loads(raw) if isinstance(raw, str) else raw
- inline = data.pop("textDelta", None) if isinstance(data, dict) else None
+ try:
+ data = _realm_reading(raw)
+ except _RealmUnreadable:
+ for handle in handles:
+ if handle is not None:
+ try:
+ await handle.dispose()
+ except Exception:
+ pass
+ raise
+ inline = data.pop("textDelta", None)
if text_args is not None:
# Cleared on every main-frame attempt: a retry on a replacement document that yields no text
# must not leave the previous document's text to be recorded as this one's.
@@ -11998,7 +12056,10 @@ def build_browser_tools(
if isinstance(result, BaseException):
# One frame that will not answer costs its own contents and nothing else, and is
# counted so the digest can say a region went unread rather than imply it was empty.
- LOG.debug("taskv3 observe could not read a child frame", exc_info=result)
+ if isinstance(result, _RealmUnreadable):
+ LOG.info("taskv3 observe skipped a child frame", reason=str(result))
+ else:
+ LOG.debug("taskv3 observe could not read a child frame", exc_info=result)
unreadable += 1
continue
frame_data, frame_handles, frame_document = result
@@ -12043,7 +12104,13 @@ def build_browser_tools(
return error
# Bound the whole acquisition -- the digest AND the handles behind it -- so a wedged page
# can't hang the turn indefinitely on a later round trip the first bound never covered.
- observation = await asyncio.wait_for(_read_observation(page), timeout=30)
+ try:
+ observation = await asyncio.wait_for(_read_observation(page), timeout=30)
+ except _RealmUnreadable as exc:
+ LOG.warning("taskv3 observe could not read the page", reason=str(exc))
+ return ToolResult.error(
+ "observe could not read this page's controls; use look or get_html to read it instead"
+ )
data, handles, owners = observation.data, observation.handles, observation.owners
elements = data.get("elements", [])
if len(handles) != len(elements):
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 66e0c2799..592e99825 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -5044,6 +5044,95 @@ async def test_observe_a_frames_radio_group_keeps_its_marker_beside_the_pages_ow
assert [("(no question found)" in line) for line in radio_lines] == [True, False, True, False], r.content
+def _two_realm_page(page_script: str, frame_script: str) -> str:
+ frame = f"<script>{frame_script}</script><button>Frame action</button>".replace('"', "'")
+ return (
+ f"<!doctype html><html><body><script>{page_script}</script><button>Page action</button>"
+ f'<iframe srcdoc="{frame}" width="400" height="200"></iframe></body></html>'
+ )
+
+
+# Legacy libraries define Array.prototype.toJSON, and JSON.stringify calls it on every array it meets.
+_LEGACY_TOJSON = "Array.prototype.toJSON = function () { return 'legacy'; };"
+# Non-configurable, so nothing can lift it; and one encoding each array as JSON text, as a library's own encoder does.
+_PINNED_TOJSON = "Object.defineProperty(Array.prototype, 'toJSON', { value: function () { return 'legacy'; } });"
+_PINNED_JSON_TOJSON = (
+ "let busy = false; Object.defineProperty(Array.prototype, 'toJSON', { value: function () {"
+ " if (busy) return Array.from(this); busy = true;"
+ " try { return JSON.stringify(this); } finally { busy = false; } } });"
+)
+
+
+@_skip_no_browser
+ "page_script,frame_script",
+ [
+ (_LEGACY_TOJSON, ""),
+ ("", _LEGACY_TOJSON),
+ (_LEGACY_TOJSON, _LEGACY_TOJSON),
+ (_PINNED_TOJSON, _PINNED_TOJSON),
+ ("", _PINNED_JSON_TOJSON),
+ # Lifted, so it never runs; pinned, it would throw out of the stringify and lose the realm.
+ ("Array.prototype.toJSON = function () { throw new Error('legacy'); };", ""),
+ # JSON.stringify never consults these for the payload's primitives, and the payload holds no Date.
+ ("String.prototype.toJSON = Date.prototype.toJSON = function () { return 'legacy'; };", ""),
+ ("Object.prototype.toJSON = function () { return 'legacy'; };", _LEGACY_TOJSON),
+ # Non-extensible, so the property could not be put back: it stays in place and the replacer covers it.
+ (_LEGACY_TOJSON + " Object.preventExtensions(Array.prototype);", ""),
+ ],
+)
+async def test_observe_reads_realms_whose_page_defines_array_tojson(page_script: str, frame_script: str) -> None:
+ descriptors = (
+ "() => [Array.prototype, Object.prototype, String.prototype].map((p) => {"
+ " const d = Object.getOwnPropertyDescriptor(p, 'toJSON'); if (!d) return null;"
+ " if (!window.__fixtureToJSON) window.__fixtureToJSON = new Map(); const m = window.__fixtureToJSON;"
+ " if (!m.has(p)) m.set(p, d.value);"
+ " return [m.get(p) === d.value, d.enumerable, d.writable, d.configurable]; })"
+ )
+ async with _live_page(_two_realm_page(page_script, frame_script)) as page:
+ before = await page.evaluate(descriptors)
+ r = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
+ after = await page.evaluate(descriptors)
+ assert r.status == "ok", r.content
+ assert "Page action" in r.content and "Frame action" in r.content, r.content
+ # The page gets back its own function under its own descriptor (only its key order may change).
+ assert after == before
+
+
+@_skip_no_browser
+ "reading",
+ [
+ "'legacy'",
+ "real('legacy')",
+ "real({elements: 'legacy'})",
+ "real({elements: null})",
+ "real({elements: 5})",
+ "real({elements: [1]})",
+ "real({elements: [], iframes: {entries: 'x'}})",
+ "real({elements: [], dropped: 'x'})",
+ "real({elements: [], text: [1]})",
+ "real({elements: [], text: [], textFull: [1]})",
+ "real({elements: [], dropped: 1}).replace('1', 'NaN')",
+ # What a pinned toJSON that throws does to the call.
+ "(() => { throw new Error('legacy'); })()",
+ ],
+)
+async def test_observe_skips_an_unreadable_frame_and_refuses_an_unreadable_page_without_raising(reading: str) -> None:
+ # A page that replaces JSON.stringify itself decides what observe's reading of that realm says.
+ script = f"const real = JSON.stringify; JSON.stringify = function () {{ return {reading}; }};"
+ async with _live_page(_two_realm_page("", script)) as page:
+ frame_skipped = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
+ assert frame_skipped.status == "ok", frame_skipped.content
+ assert "Page action" in frame_skipped.content and "Frame action" not in frame_skipped.content
+ async with _live_page(_two_realm_page(script, "")) as page:
+ page_refused = await _tool(build_browser_tools(_fixed_page_provider(page)), "observe").handler({})
+ assert page_refused.status == "error"
+ assert "could not read this page" in page_refused.content
+
+
@_skip_no_browser
@pytest.mark.asyncio
async def test_observe_a_radio_groups_description_never_puts_its_marker_mid_group() -> None: