Skip to content

Automate browser based workflows with AI

Bound concurrent SDK actions per organization (SKY-14670) (#8260)

Aaron Perez07222e7a7caa
diff --git a/skyvern/exceptions.py b/skyvern/exceptions.py
index 3239fbf28..19fe8a434 100644
--- a/skyvern/exceptions.py
+++ b/skyvern/exceptions.py
@@ -227,6 +227,19 @@ class RateLimitExceeded(SkyvernHTTPException):
super().__init__(message, status_code=HTTPStatus.TOO_MANY_REQUESTS)
+class ConcurrencyLimitExceeded(SkyvernHTTPException):
+ def __init__(self, organization_id: str, operation: str, limit: int):
+ self.organization_id = organization_id
+ self.operation = operation
+ self.limit = limit
+ message = (
+ f"Concurrency limit exceeded for organization {organization_id}. "
+ f"At most {limit} {operation} requests may be in flight at once. "
+ "Retry once an in-flight request finishes."
+ )
+ super().__init__(message, status_code=HTTPStatus.TOO_MANY_REQUESTS)
+
+
class InvalidOpenAIResponseFormat(SkyvernException):
def __init__(self, message: str | None = None):
super().__init__(f"Invalid response format: {message}")
diff --git a/skyvern/forge/sdk/core/rate_limiter.py b/skyvern/forge/sdk/core/rate_limiter.py
index 95501d5e3..5a9b46ceb 100644
--- a/skyvern/forge/sdk/core/rate_limiter.py
+++ b/skyvern/forge/sdk/core/rate_limiter.py
@@ -1,3 +1,4 @@
+from contextlib import AbstractAsyncContextManager, nullcontext
from typing import Protocol
@@ -21,6 +22,26 @@ class RateLimiter(Protocol):
"""
...
+ def limit_sdk_action_concurrency(self, organization_id: str) -> AbstractAsyncContextManager[None]:
+ """
+ Hold one in-flight SDK action slot for the organization for the duration of the block.
+
+ Unlike the request-rate limit above, this bounds how many SDK actions one organization may
+ have running at once. SDK actions execute synchronously in the API process, so a single
+ organization's concurrency is what an API worker's event loop actually has to absorb.
+
+ Entering raises when the organization is already at its cap, so a caller over the cap is
+ rejected rather than queued. The slot is released when the block exits, which is what lets a
+ backing-off organization resume with no operator action.
+
+ Args:
+ organization_id: The organization ID to bound
+
+ Raises:
+ Exception: If the organization is at its concurrency cap (implementation-specific)
+ """
+ ...
+
class NoopRateLimiter(RateLimiter):
"""
@@ -31,3 +52,7 @@ class NoopRateLimiter(RateLimiter):
async def rate_limit_submit_run(self, organization_id: str) -> None:
"""No-op implementation that never rate limits."""
+
+ def limit_sdk_action_concurrency(self, organization_id: str) -> AbstractAsyncContextManager[None]:
+ """No-op implementation that never bounds concurrency."""
+ return nullcontext()
diff --git a/skyvern/forge/sdk/routes/sdk.py b/skyvern/forge/sdk/routes/sdk.py
index 21b32ef2b..d83ccecc9 100644
--- a/skyvern/forge/sdk/routes/sdk.py
+++ b/skyvern/forge/sdk/routes/sdk.py
@@ -69,6 +69,16 @@ async def run_sdk_action(
organization: Organization = Depends(org_auth_service.get_current_org),
) -> RunSdkActionResponse:
"""Execute a single SDK action with the specified parameters."""
+ # The whole action runs on this process's event loop, so the cap is taken before any database
+ # write or page work: an organization over its cap costs a rejected request, not a started one.
+ async with app.RATE_LIMITER.limit_sdk_action_concurrency(organization.organization_id):
+ return await _run_sdk_action(action_request, organization)
+
+
+async def _run_sdk_action(
+ action_request: RunSdkActionRequest,
+ organization: Organization,
+) -> RunSdkActionResponse:
LOG.info(
"Running SDK action",
organization_id=organization.organization_id,
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 7bb430ce0..172f57990 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -91,17 +91,98 @@ _SHADOW_ROOTS_JS = r"""(from_root) => {
}"""
+# Document-plus-shadow equivalents of the DOM query APIs. Every reaction/commit probe below judges
+# what a Playwright action just did, and Playwright's selector engine pierces open shadow roots — so
+# a document-only probe reports "not an option" / "menu closed" / "did not commit" about elements
+# that are visible and were acted on successfully, which is a fabricated answer rather than a gap.
+_PIERCED_QUERY_JS = (
+ r"""
+ const _shadowRoots = """
+ + _SHADOW_ROOTS_JS
+ + r""";
+ // Walked once per invocation: a probe calls these helpers several times, and the roots cannot
+ // change between those calls.
+ const _rootList = _shadowRoots(document);
+ // A throw here propagates, exactly as document.querySelector did: an unparseable selector is not
+ // the same fact as "no such element", and callers that gate on the result disarm themselves if the
+ // two are conflated.
+ const pQS = (sel) => {
+ for (const root of _rootList) {
+ const el = root.querySelector(sel);
+ if (el) return el;
+ }
+ return null;
+ };
+ const pQSA = (sel) => {
+ const acc = [];
+ for (const root of _rootList) {
+ for (const el of root.querySelectorAll(sel)) acc.push(el);
+ }
+ return acc;
+ };
+ // Node.contains walks the light tree only, so a host does not contain its own shadow content.
+ // Every caller below compares elements drawn from the pierced scope, where a cross-tree pair is
+ // ordinary. Same hop parentOf() uses: a ShadowRoot has no parentNode, so step to its host.
+ const pContains = (a, b) => {
+ if (!a || !b) return false;
+ for (let n = b; n; n = n.parentNode || n.host || null) if (n === a) return true;
+ return false;
+ };
+ // The pre-snapshot carries element identity across a click or a keystroke, so the carrier has to
+ // survive whatever the page did in between -- and the two halves of the page need different ones.
+ // In the light DOM an attribute is the only carrier that survives cloneNode/innerHTML, so a
+ // container the page re-creates by cloning still reads as "existed before" rather than "appeared
+ // in reaction". Inside a shadow root we write nothing at all: stamping there makes a component
+ // watching its own root re-render, destroying the marks just made and leaving the finders reading
+ // a static list as a reaction. The WeakSet is the best carrier that costs no mutation, at one
+ // disclosed price -- a component that re-creates its own content by cloning reads as all-new.
+ // Absent (a navigation cleared window) means "no snapshot", never "everything is new".
+ const preMark = (el, inShadow) => {
+ if (inShadow) window.__tv3_pre.add(el);
+ else el.setAttribute('data-tv3-pre', '1');
+ };
+ // instanceof, not truthiness: a page that pre-defines __tv3_pre as an accessor keeps its own
+ // object through preReset, and a `has: () => false` impostor would make every element read as a
+ // reaction -- defeating the one distinction this guard exists to draw.
+ const preReady = () => window.__tv3_pre instanceof WeakSet;
+ const preHas = (el) => {
+ try { if (el.hasAttribute('data-tv3-pre')) return true; } catch (e) { /* clobbered getter */ }
+ return preReady() && window.__tv3_pre.has(el);
+ };
+ const preReset = () => {
+ pQSA('[data-tv3-pre]').forEach((e) => e.removeAttribute('data-tv3-pre'));
+ window.__tv3_pre = new WeakSet();
+ };
+ // 'body *' has no meaning inside a shadow root, whose own descendants are the equivalent scope.
+ const pScopeEach = (fn) => {
+ for (const root of _rootList) {
+ const inShadow = root !== document;
+ for (const el of root.querySelectorAll(inShadow ? '*' : 'body *')) fn(el, inShadow);
+ }
+ };
+ const pScopeAll = () => {
+ const acc = [];
+ pScopeEach((el) => acc.push(el));
+ return acc;
+ };
+"""
+)
+
# Snapshot of everything visible BEFORE typing. The finder ignores anything marked here, so only DOM
# that appeared (or became visible) IN REACTION to typing can be treated as a suggestion — static page
# text that merely happens to share a word with the value (a nearby card, nav item, prior answer) is
# never eligible. This is what makes "detect by the page's reaction" rigorous rather than a claim.
-_PRESNAPSHOT_JS = r"""() => {
- document.querySelectorAll('[data-tv3-pre]').forEach((e) => e.removeAttribute('data-tv3-pre'));
- for (const el of document.querySelectorAll('body *')) {
+_PRESNAPSHOT_JS = (
+ r"""() => {"""
+ + _PIERCED_QUERY_JS
+ + r"""
+ preReset();
+ pScopeEach((el, inShadow) => {
const r = el.getBoundingClientRect();
- if (r.width > 0 && r.height > 0) el.setAttribute('data-tv3-pre', '1');
- }
+ if (r.width > 0 && r.height > 0) preMark(el, inShadow);
+ });
}"""
+)
# Behavioral, site-agnostic suggestion finder. After the caller types a value (with a pre-snapshot taken
# first), this looks for the suggestion list the typeahead rendered IN REACTION: a small, visible,
@@ -114,19 +195,25 @@ _PRESNAPSHOT_JS = r"""() => {
# rows, so it ties/outranks any single row; clicking it would land on the wrong row), so it's dropped.
# Tags the winner with data-tv3-sugg and returns {text, score}, or null if nothing reacted.
_FIND_SUGGESTION_JS = (
- r"""(args) => {
+ r"""(args) => {"""
+ + _PIERCED_QUERY_JS
+ + r"""
const STOP = """
+ _STOPWORDS_JS
+ r""";
const toks = (s) => new Set(String(s).toLowerCase().replace(/[\/,]/g, ' ').split(/\s+/).filter((w) => w.length >= 3 && !STOP.has(w)));
const want = toks(args.value || '');
- document.querySelectorAll('[data-tv3-sugg]').forEach((e) => e.removeAttribute('data-tv3-sugg'));
- if (!want.size) return null;
- const field = document.querySelector(args.field);
- const fr = field ? field.getBoundingClientRect() : null;
+ pQSA('[data-tv3-sugg]').forEach((e) => e.removeAttribute('data-tv3-sugg'));
+ if (!want.size || !preReady()) return null;
+ const field = pQS(args.field);
+ // No field means no geometry gate, and without it the scan below is page-wide and will happily
+ // tag -- and then click -- a row far from the control the caller typed into. Refuse instead:
+ // "cannot judge" and "nothing reacted" are both safe, and a confident wrong click is not.
+ if (!field) return null;
+ const fr = field.getBoundingClientRect();
const cands = [];
- for (const el of document.querySelectorAll('body *')) {
- if (el.hasAttribute('data-tv3-pre')) continue; // existed/was visible before typing → not a reaction
+ for (const el of pScopeAll()) {
+ if (preHas(el)) continue; // existed/was visible before typing → not a reaction
const tag = el.tagName;
if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || tag === 'SCRIPT' || tag === 'STYLE' || tag === 'LABEL' || tag === 'FORM') continue;
const role = el.getAttribute('role');
@@ -150,7 +237,7 @@ _FIND_SUGGESTION_JS = (
if (!cands.length) return null;
// Drop any candidate that CONTAINS another candidate (a dropdown container over its own rows), then
// take the highest score, breaking ties toward the smallest (innermost) row.
- const leaves = cands.filter((c) => !cands.some((o) => o.el !== c.el && c.el.contains(o.el)));
+ const leaves = cands.filter((c) => !cands.some((o) => o.el !== c.el && pContains(c.el, o.el)));
const pool = leaves.length ? leaves : cands;
pool.sort((a, b) => b.score - a.score || a.h - b.h);
const best = pool[0];
@@ -174,14 +261,17 @@ _FIND_SUGGESTION_JS = (
# with the chosen suggestion, or the typed value) and (b) shows the click took effect — it changed from
# the raw typed text OR the suggestion list closed. Failing that, a hidden input in the nearest
# div/li/fieldset (never the whole <form>) whose value overlaps. Otherwise "" — nothing committed.
-_VERIFY_COMMIT_JS = r"""(args) => {
+_VERIFY_COMMIT_JS = (
+ r"""(args) => {"""
+ + _PIERCED_QUERY_JS
+ + r"""
const toks = (s) => new Set(String(s).toLowerCase().replace(/[\/,]/g, ' ').split(/\s+/).filter((w) => w.length >= 3));
const overlaps = (a, b) => { const B = toks(b); for (const w of toks(a)) if (B.has(w)) return true; return false; };
- const el = document.querySelector(args.field);
+ const el = pQS(args.field);
const typed = String(args.typed || '').trim();
const chosen = String(args.chosen || '').trim() || typed;
const cur = el ? (el.value || '').trim() : '';
- const tagged = document.querySelector('[data-tv3-sugg]');
+ const tagged = pQS('[data-tv3-sugg]');
const listClosed = !tagged || tagged.getBoundingClientRect().height === 0;
// A short normalized value ("New York" -> "NY", "United States" -> "US") has no >=3-char token to
// overlap, so accept it on causality alone (it changed / the list closed). Longer values must still
@@ -196,16 +286,18 @@ _VERIFY_COMMIT_JS = r"""(args) => {
}
return '';
}"""
+)
-# Whether the MAIN-DOCUMENT query the reaction probes use can see this element. They are all
-# document-only, so a control inside a component is invisible to them even though Playwright types
-# into it perfectly well -- the action works and only our verification of it is blind. False here
-# means "do not claim this was checked", never "do not act".
-# Why our reaction probes cannot see this selector, if they cannot: `component` (only a piercing
-# query resolves it), `unprobeable` (the main-document query cannot even parse it -- Playwright
-# syntax like `css=`, `>> nth=`, `:visible`, `text=`), or `` (probeable, so any failure to find it is
-# a fact about the page rather than about us). Both readings happen in ONE evaluation: as two round
-# trips, an ordinary re-render landing between them lets each describe a different moment.
+# Why a reaction probe's answer about this selector may not carry a claim. `unprobeable` -- in-page
+# CSS cannot even parse it (Playwright syntax like `css=`, `>> nth=`, `:visible`, `text=`).
+# `component` -- it lives inside a component. The probes DO pierce open shadow roots, so this no
+# longer means they are blind to the element itself; it means the widget's own list may still render
+# where a pierced query does not reach -- a portal mounted elsewhere in the page, or a closed root,
+# which is undetectable from script. So a missing suggestion list there is still not proof the field
+# is unfilled, and the softened reading is kept deliberately rather than for lack of reach.
+# `` -- neither applies, so any failure to find the element is a fact about the page, not about us.
+# Both readings happen in ONE evaluation: as two round trips, an ordinary re-render landing between
+# them lets each describe a different moment.
_PROBE_REACH_JS = (
r"""(sel) => {
const _roots = """
@@ -397,7 +489,10 @@ _CHECKBOX_CHECKED_JS = (
# that row's state fingerprint — aria checked/selected/pressed, class, child count, text — so an option
# click on a multi-select menu (which commits WITHOUT closing) can be verified by its state change.
# Also takes the visible-DOM pre-snapshot (data-tv3-pre) so a menu the click opens reads as a reaction.
-_CLICK_PRECHECK_JS = r"""(clicked) => {
+_CLICK_PRECHECK_JS = (
+ r"""(clicked) => {"""
+ + _PIERCED_QUERY_JS
+ + r"""
const vis = (el) => {
const r = el.getBoundingClientRect();
if (r.width === 0 || r.height === 0) return false;
@@ -421,9 +516,9 @@ _CLICK_PRECHECK_JS = r"""(clicked) => {
].join('|');
};
const openRows = [];
- for (const el of document.querySelectorAll('[data-tv3-menu]')) if (vis(el)) openRows.push(el);
+ for (const el of pQSA('[data-tv3-menu]')) if (vis(el)) openRows.push(el);
let target = null;
- try { target = document.querySelector(clicked); } catch (e) { target = null; }
+ try { target = pQS(clicked); } catch (e) { target = null; }
let isOption = false;
let containsMenu = false;
let optText = '';
@@ -433,24 +528,28 @@ _CLICK_PRECHECK_JS = r"""(clicked) => {
// The target being the row or inside it is an option pick. The target merely CONTAINING rows
// (the card around the menu) is not — and since a center-point click on the card can land on
// an arbitrary row, that case is flagged so the handler makes no claims about it at all.
- if (el === target || el.contains(target)) {
+ if (el === target || pContains(el, target)) {
isOption = true;
optText = (el.innerText || '').trim().slice(0, 80);
optState = state(el);
break;
}
- if (target.contains(el)) containsMenu = true;
+ if (pContains(target, el)) containsMenu = true;
}
}
- document.querySelectorAll('[data-tv3-pre]').forEach((e) => e.removeAttribute('data-tv3-pre'));
- for (const el of document.querySelectorAll('body *')) if (vis(el)) el.setAttribute('data-tv3-pre', '1');
+ preReset();
+ pScopeEach((el, inShadow) => { if (vis(el)) preMark(el, inShadow); });
return { menuOpen: openRows.length > 0, isOption, containsMenu, optText, optState };
}"""
+)
# Post-click menu state: how many previously-tagged menu rows are still visible (a closed menu — nodes
# destroyed or hidden — reads 0), plus the clicked row's current state fingerprint for the multi-select
# commit check. Field names are distinct from _CLICK_PRECHECK_JS's on purpose (tests dispatch on them).
-_MENU_AFTER_JS = r"""(clicked) => {
+_MENU_AFTER_JS = (
+ r"""(clicked) => {"""
+ + _PIERCED_QUERY_JS
+ + r"""
const vis = (el) => {
const r = el.getBoundingClientRect();
if (r.width === 0 || r.height === 0) return false;
@@ -473,17 +572,18 @@ _MENU_AFTER_JS = r"""(clicked) => {
};
let stillOpen = 0;
const rows = [];
- for (const el of document.querySelectorAll('[data-tv3-menu]')) if (vis(el)) { stillOpen++; rows.push(el); }
+ for (const el of pQSA('[data-tv3-menu]')) if (vis(el)) { stillOpen++; rows.push(el); }
let target = null;
- try { target = document.querySelector(clicked); } catch (e) { target = null; }
+ try { target = pQS(clicked); } catch (e) { target = null; }
let optState = '';
if (target) {
for (const el of rows) {
- if (el === target || el.contains(target)) { optState = state(el); break; }
+ if (el === target || pContains(el, target)) { optState = state(el); break; }
}
}
return { stillOpen, optState };
}"""
+)
# Behavioral, site-agnostic menu finder: after a click (with the pre-snapshot taken first),
# look for the option list the page rendered IN REACTION — a NEW container (not data-tv3-pre: a
@@ -494,15 +594,23 @@ _MENU_AFTER_JS = r"""(clicked) => {
# navigational rows are listed too. Tags rows data-tv3-menu="1..N" (top-to-bottom) — in-DOM tags that
# stay valid until the menu re-renders, so the model can pick an option without a re-observe re-minting
# ids (the staging trace's staleness trap). Existing tags are cleared only when a new menu is tagged.
-_FIND_MENU_JS = r"""(clicked) => {
+_FIND_MENU_JS = (
+ r"""(clicked) => {"""
+ + _PIERCED_QUERY_JS
+ + r"""
const vis = (r) => r.width > 0 && r.height > 0;
let trigger = null;
- try { trigger = document.querySelector(clicked); } catch (e) { return null; }
+ try { trigger = pQS(clicked); } catch (e) { return null; }
if (!trigger) return null;
+ // The reaction gate below is the whole basis for calling these rows a menu the click just opened.
+ // A navigation destroys window, so an absent snapshot here means the page under us is not the page
+ // we clicked on, and every row would read as new. Refuse: "cannot judge" beats naming three
+ // ordinary links on a fresh document as a menu and telling the model to pick one.
+ if (!preReady()) return null;
const tr = trigger.getBoundingClientRect();
const rows = [];
- for (const el of document.querySelectorAll('body *')) {
- if (el.hasAttribute('data-tv3-pre')) continue;
+ for (const el of pScopeAll()) {
+ if (preHas(el)) continue;
const tag = el.tagName;
if (tag === 'INPUT' || tag === 'TEXTAREA' || tag === 'SELECT' || tag === 'SCRIPT' || tag === 'STYLE' || tag === 'LABEL' || tag === 'FORM') continue;
if (el.children.length > 8) continue;
@@ -521,14 +629,32 @@ _FIND_MENU_JS = r"""(clicked) => {
rows.push({ el, r, txt });
}
if (rows.length < 2) return null;
- const leaves = rows.filter((c) => !rows.some((o) => o.el !== c.el && c.el.contains(o.el)));
+ const leaves = rows.filter((c) => !rows.some((o) => o.el !== c.el && pContains(c.el, o.el)));
if (leaves.length < 2) return null;
// Group by parent AND grandparent so both flat menus (card > button*N) and nested ones
// (ul > li > button) find their shared container.
const groups = new Map();
+ // parentElement is null at a shadow boundary (a ShadowRoot is not an Element), so a menu whose
+ // rows are written straight into the root -- root.innerHTML = '<div role="option">...' -- would
+ // group under nothing and never be found. The host stands in for the boundary.
+ // A host reached this way stands in for the boundary, and a host necessarily pre-exists the menu
+ // its component just rendered -- so the container-is-new check below must not be applied to it.
+ // The rows' own newness still carries the reaction evidence.
+ const boundaryStandIns = new Set();
+ const parentOf = (el) => {
+ if (!el) return null;
+ const p = el.parentNode;
+ if (!p) return null;
+ if (p.nodeType === 11) {
+ if (!p.host) return null;
+ boundaryStandIns.add(p.host);
+ return p.host;
+ }
+ return p.nodeType === 1 ? p : null;
+ };
for (const c of leaves) {
- const p1 = c.el.parentElement;
- for (const p of [p1, p1 ? p1.parentElement : null]) {
+ const p1 = parentOf(c.el);
+ for (const p of [p1, parentOf(p1)]) {
if (!p || p === document.body || p === document.documentElement) continue;
if (!groups.has(p)) groups.set(p, new Set());
groups.get(p).add(c);
@@ -538,7 +664,7 @@ _FIND_MENU_JS = r"""(clicked) => {
for (const [p, set] of groups) {
const g = Array.from(set);
if (g.length < 2) continue;
- if (p.hasAttribute('data-tv3-pre')) continue;
+ if (!boundaryStandIns.has(p) && preHas(p)) continue;
// A dialog is a page mode, not a menu — mislabeling it invites a wrong "pick an option" move.
try { if (p.closest('dialog,[role~="dialog"],[aria-modal="true"]')) continue; } catch (e) {}
const pr = p.getBoundingClientRect();
@@ -550,7 +676,7 @@ _FIND_MENU_JS = r"""(clicked) => {
if (!best || g.length > best.g.length || (g.length === best.g.length && pr.height < best.h)) best = { p, g, h: pr.height };
}
if (!best) return null;
- document.querySelectorAll('[data-tv3-menu]').forEach((e) => e.removeAttribute('data-tv3-menu'));
+ pQSA('[data-tv3-menu]').forEach((e) => e.removeAttribute('data-tv3-menu'));
best.g.sort((a, b) => a.r.top - b.r.top || a.r.left - b.r.left);
const options = [];
let n = 0;
@@ -561,6 +687,7 @@ _FIND_MENU_JS = r"""(clicked) => {
}
return { count: n, options };
}"""
+)
# Raw DOM perception: collect visible interactive elements with a stable selector each.
# Elements without a natural selector get a data-tv3 marker so later actions can target them.
@@ -1808,9 +1935,9 @@ def build_browser_tools(
async def _unverifiable_because(page: Any, selector: str) -> str | None:
# Returns the clause explaining why a check could not be run, or None when it could. "The
- # document cannot see it" is not one fact but three: a widget that unmounts its own input, a
- # control inside a component, and a selector our probe cannot parse all answer alike, and
- # only the middle one is a component. Empty on failure -- claiming a reason we did not
+ # probe cannot carry this claim" is not one fact but three: a widget that unmounts its own
+ # input, a control inside a component whose list may render beyond a pierced query's reach,
+ # and a selector our probe cannot parse. Empty on failure -- claiming a reason we did not
# establish would be its own false statement.
try:
reach = str(await page.evaluate(_PROBE_REACH_JS, selector) or "")
@@ -1855,10 +1982,11 @@ def build_browser_tools(
f"typed into {selector}; it is a typeahead — selected {opt_txt!r} (committed value: {committed!r})"
)
if opt_txt and not committed:
- # The verifier reads the field with document.querySelector, so on a control inside a
- # component it reads nothing and "did not commit" is its own unearned claim -- the
- # field is often filled. Halting a batch on that is as wrong as proceeding on a real
- # failure, so say which one we actually established.
+ # The verifier pierces open shadow roots, so inside a component it does read the
+ # field -- but a widget that portals its list elsewhere in the page, or renders it in
+ # a closed root, is still beyond it, and "did not commit" would then be its own
+ # unearned claim on a field that is often filled. Halting a batch on that is as wrong
+ # as proceeding on a real failure, so say which one we actually established.
why = await _unverifiable_because(page, selector)
if why:
return ToolResult.ok(
@@ -1872,10 +2000,11 @@ def build_browser_tools(
f"clicked suggestion {opt_txt!r} for {selector} but it did not commit — the field is NOT "
"filled; re-observe and retry, do not proceed"
)
- # No suggestion list surfaced. On a control inside a component that is not evidence of
- # absence: the finder is document-only and could not have seen one. Saying "typed into X"
- # here reads as a verified fill, and on a typeahead that silently rejects raw text it
- # turns an honest failure into a confident wrong answer on a form we then submit.
+ # No suggestion list surfaced. The finder pierces open shadow roots, so it can see a list
+ # inside one -- but not one the widget portals elsewhere in the page or renders in a
+ # closed root, so inside a component this is still not evidence of absence. Saying
+ # "typed into X" there reads as a verified fill, and on a typeahead that silently rejects
+ # raw text it turns an honest failure into a confident wrong answer on a form we submit.
why = await _unverifiable_because(page, selector)
if why:
return ToolResult.ok(
@@ -2026,10 +2155,12 @@ def build_browser_tools(
value = _resolve_text(args["value"])
committed, opt_txt = await _type_and_commit(page, selector, value, rounds=8)
if opt_txt is None:
- # The suggestion finder is document-only, so inside a component it sees no list -- which is
- # not evidence there was none, and "the field is NOT filled" is measurably false there (the
- # value is typed in). Erroring would also strand the country/phone comboboxes on the very
- # forms this targets. observe DOES pierce, so re-observing is a check the model can run.
+ # The suggestion finder pierces open shadow roots, so inside a component it now sees a
+ # list rendered in that root -- but a portalled or closed-root list stays invisible, so a
+ # missing list there is still not evidence there was none, and "the field is NOT filled"
+ # would be measurably false (the value is typed in). Erroring would also strand the
+ # country/phone comboboxes on the very forms this targets. observe pierces too, so
+ # re-observing is a check the model can run.
why = await _unverifiable_because(page, selector)
if why:
return ToolResult.ok(
diff --git a/tests/unit/test_run_sdk_action_failures.py b/tests/unit/test_run_sdk_action_failures.py
index 3c9a79a54..8d9e2575a 100644
--- a/tests/unit/test_run_sdk_action_failures.py
+++ b/tests/unit/test_run_sdk_action_failures.py
@@ -8,6 +8,7 @@ import pytest
from fastapi import HTTPException
from skyvern.exceptions import (
+ ConcurrencyLimitExceeded,
MissingBrowserStatePage,
ScrapingFailed,
ScreenshotTargetClosed,
@@ -556,3 +557,33 @@ async def test_ai_upload_file_route_rejects_legacy_file_url_escape(
assert rejection is not None
assert rejection.status_code == 400
page_ai.ai_upload_file.assert_not_awaited()
+
+
+class _RejectingSlot:
+ def __init__(self, organization_id: str) -> None:
+ self._organization_id = organization_id
+
+ async def __aenter__(self) -> None:
+ raise ConcurrencyLimitExceeded(organization_id=self._organization_id, operation="SDK action", limit=4)
+
+ async def __aexit__(self, *exc_info: object) -> None:
+ return None
+
+
+async def test_action_over_the_concurrency_cap_is_rejected_before_any_run_is_created(
+ mock_request: Any, mock_organization: Any, mock_app: Any
+) -> None:
+ """A rejected action must cost a request, not a workflow run, a task or a step."""
+ mock_request.workflow_run_id = None
+ mock_app.RATE_LIMITER.limit_sdk_action_concurrency = MagicMock(side_effect=_RejectingSlot)
+ mock_app.WORKFLOW_SERVICE.create_empty_workflow = AsyncMock()
+
+ with patch("skyvern.forge.sdk.routes.sdk.app", mock_app):
+ with pytest.raises(ConcurrencyLimitExceeded) as rejected:
+ await run_sdk_action(mock_request, organization=mock_organization)
+
+ assert rejected.value.status_code == 429
+ mock_app.WORKFLOW_SERVICE.create_empty_workflow.assert_not_awaited()
+ mock_app.DATABASE.tasks.create_task.assert_not_awaited()
+ mock_app.DATABASE.tasks.create_step.assert_not_awaited()
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 734fa6ff0..3d0d535ab 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -4673,3 +4673,507 @@ async def test_hidden_native_note_counts_only_the_controls_it_actually_listed()
assert "note: 1 native control(s) hidden behind styled proxies" in r.content, r.content
# the two that share an id are dropped, and said to be dropped, rather than counted as listed
assert "reused by another instance of the same component" in r.content
+
+
+# --- Reaction probes inside open shadow roots. Perception already pierces; these cover the
+# probes that judge what an action DID -- click precheck, menu open/close, suggestion commit --
+# plus the pre-snapshot carrier they all read, which cannot stamp an attribute inside a root
+# without provoking the re-render it would then misread. ---
+
+_LISTBOX_FIXTURE_HTML = """
+<label id="country-label">Country*</label>
+<div id="proxy" role="listbox" tabindex="0" aria-label="Country">Select a country</div>
+<div id="options" style="display:none">
+ <div id="opt-us" role="option" data-value="us">United States</div>
+ <div id="opt-ca" role="option" data-value="ca">Canada</div>
+</div>
+<select id="native" style="display:none">
+ <option value="">--</option><option value="us">United States</option><option value="ca">Canada</option>
+</select>
+<script>
+document.getElementById('proxy').addEventListener('click', function () {
+ document.getElementById('options').style.display = 'block';
+});
+document.querySelectorAll('#options [role=option]').forEach(function (o) {
+ o.addEventListener('click', function () {
+ document.getElementById('proxy').textContent = o.textContent;
+ document.getElementById('native').value = o.getAttribute('data-value');
+ document.getElementById('native').dispatchEvent(new Event('change', {bubbles: true}));
+ document.getElementById('options').style.display = 'none';
+ });
+});
+</script>"""
+
+
+@_skip_no_browser
+async def test_aria_listbox_proxy_can_be_driven_to_a_committed_selection() -> None:
+ # The whole point of enumerating the proxy: the intended interaction is click-to-open then
+ # click-the-option, and the widget writes through to the display:none native select itself.
+ # select_option cannot drive that hidden select, and this is why it does not need to.
+ async with _live_page(_LISTBOX_FIXTURE_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ opened = await _tool(tools, "click").handler({"selector": "#proxy"})
+ assert opened.status == "ok"
+ assert "opened a menu of 2 options" in opened.content
+
+ chosen = await _tool(tools, "click").handler({"selector": "#opt-ca"})
+ assert chosen.status == "ok"
+ assert await page.eval_on_selector("#native", "e => e.value") == "ca"
+
+
+@_skip_no_browser
+async def test_click_precheck_recognizes_a_shadow_hosted_menu_option() -> None:
+ # isOption gates the whole commit-verification path; reading False for a real option silently
+ # disables it for exactly the controls shadow piercing unlocks.
+ from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<ds-menu id="m"></ds-menu><script>
+ document.getElementById('m').attachShadow({mode:'open'}).innerHTML =
+ '<ul><li id="o1" data-tv3-menu="1" role="option">Alpha</li>'
+ + '<li id="o2" data-tv3-menu="2" role="option">Beta</li></ul>';
+ </script>"""
+ ) as page:
+ pre = await page.evaluate(_CLICK_PRECHECK_JS, '[data-tv3-menu="1"]')
+ assert pre["isOption"] is True
+ assert pre["menuOpen"] is True
+ assert pre["optText"] == "Alpha"
+
+
+def test_fake_page_dispatch_markers_are_unique_to_one_blob() -> None:
+ # The click/typeahead fakes route page.evaluate by matching substrings of the real JS constants,
+ # so a marker that starts appearing in a second blob silently re-routes every probe and the
+ # failures land far from the edit. (This caught `return !!` leaking into the shared helper.)
+ from skyvern.forge.taskv3 import tools as _t # noqa: PLC0415
+
+ blobs = {n: v for n in dir(_t) if n.endswith("_JS") and isinstance(v := getattr(_t, n), str)}
+ for marker, expected in (
+ ("return !!found", "_SELECTOR_EXISTS_JS"),
+ ("menuOpen", "_CLICK_PRECHECK_JS"),
+ ("stillOpen", "_MENU_AFTER_JS"),
+ ("clickable", "_FIND_MENU_JS"),
+ ):
+ owners = sorted(n for n, b in blobs.items() if marker in b)
+ assert owners == [expected], f"{marker!r} must identify only {expected}, found {owners}"
+
+
+@_skip_no_browser
+async def test_menu_after_does_not_report_an_open_shadow_menu_as_closed() -> None:
+ # The most damaging fabrication: stillOpen==0 makes _click_reaction return the affirmative
+ # "the menu closed" — a success claim invented from the query's blindness, on a menu still open.
+ from skyvern.forge.taskv3.tools import _MENU_AFTER_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<ds-menu id="m"></ds-menu><script>
+ document.getElementById('m').attachShadow({mode:'open'}).innerHTML =
+ '<ul><li id="o1" data-tv3-menu="1" role="option">Alpha</li>'
+ + '<li id="o2" data-tv3-menu="2" role="option">Beta</li></ul>';
+ </script>"""
+ ) as page:
+ assert await page.is_visible('[data-tv3-menu="1"]') is True
+ after = await page.evaluate(_MENU_AFTER_JS, '[data-tv3-menu="1"]')
+ assert after["stillOpen"] == 2
+
+
+@_skip_no_browser
+async def test_menu_finder_refuses_when_no_snapshot_arms_it() -> None:
+ # The reaction gate is the whole basis for calling these rows "a menu the click just opened".
+ # A navigation destroys window and with it the snapshot, so the probe wakes up on a document it
+ # never saw before typing or clicking, where every row reads as new. That is the state this
+ # asserts directly: no snapshot, rows present, and the only honest answer is silence.
+ from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS, _FIND_MENU_JS # noqa: PLC0415
+
+ html = """<button id="pick" style="position:absolute;top:10px;left:10px;width:120px;height:24px">Pick</button>
+ <div id="menu" style="position:absolute;top:40px;left:10px;width:200px">
+ <div role="option" style="height:20px">Alpha Corp</div>
+ <div role="option" style="height:20px">Beta LLC</div>
+ <div role="option" style="height:20px">Gamma Inc</div>
+ </div>"""
+ async with _live_page(html) as page:
+ # No precheck has run: window carries no snapshot, exactly as after a navigation.
+ assert await page.evaluate("() => window.__tv3_pre === undefined") is True
+ assert await page.evaluate(_FIND_MENU_JS, "#pick") is None
+ # ...and the same page WITH a snapshot still finds nothing, because nothing reacted — so the
+ # refusal above is about the missing snapshot, not about an unfindable menu.
+ await page.evaluate(_CLICK_PRECHECK_JS, "#pick")
+ assert await page.evaluate(_FIND_MENU_JS, "#pick") is None
+ # Prove the fixture is findable at all once the rows genuinely appear in reaction.
+ await page.evaluate("() => { document.getElementById('menu').remove(); }")
+ await page.evaluate(_CLICK_PRECHECK_JS, "#pick")
+ await page.evaluate(
+ "(h) => { const d = document.createElement('div'); d.innerHTML = h;"
+ " document.body.appendChild(d.firstElementChild); }",
+ """<div id="menu2" style="position:absolute;top:40px;left:10px;width:200px">
+ <div role="option" style="height:20px">Alpha Corp</div>
+ <div role="option" style="height:20px">Beta LLC</div>
+ <div role="option" style="height:20px">Gamma Inc</div>
+ </div>""",
+ )
+ found = await page.evaluate(_FIND_MENU_JS, "#pick")
+ assert isinstance(found, dict) and found.get("count") == 3, found
+
+
+@_skip_no_browser
+async def test_menu_finder_sees_rows_written_straight_into_a_shadow_root() -> None:
+ # Rows grouped by parentElement vanish at the shadow boundary: a ShadowRoot is not an Element, so
+ # `root.innerHTML = '<div role="option">...'` produced no group and the menu was invisible —
+ # while the same rows wrapped in a <ul> were found. Half a capability is its own trap: the shape
+ # decides whether the model is told the menu exists.
+ from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS, _FIND_MENU_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<button id="pick" style="position:absolute;top:10px;left:10px;width:120px;height:24px">Pick</button>
+ <x-menu id="m" style="position:absolute;top:40px;left:10px;display:block;width:200px;height:60px"></x-menu>
+ <script>
+ window.root = document.getElementById('m').attachShadow({mode:'open'});
+ document.getElementById('pick').addEventListener('click', function () {
+ // Direct children of the root — no wrapper element to group by.
+ window.root.innerHTML =
+ '<div role="option" style="height:20px">Alpha</div>'
+ + '<div role="option" style="height:20px">Beta</div>'
+ + '<div role="option" style="height:20px">Gamma</div>';
+ });
+ </script>"""
+ ) as page:
+ await page.evaluate(_CLICK_PRECHECK_JS, "#pick")
+ await page.click("#pick")
+ await page.wait_for_timeout(100)
+ found = await page.evaluate(_FIND_MENU_JS, "#pick")
+ assert isinstance(found, dict) and found.get("count") == 3, found
+ assert [o["text"] for o in found["options"]] == ["Alpha", "Beta", "Gamma"]
+
+
+@_skip_no_browser
+async def test_click_precheck_flags_a_shadow_host_that_wraps_the_open_menu() -> None:
+ # containsMenu is a safety valve: clicking the container AROUND a menu may land the center point
+ # on an arbitrary row, so the handler must make no open/closed/selected claim. Node.contains()
+ # only walks the light tree, so a host whose OWN shadow root holds the rows read as "not the
+ # container" — the valve disengaged and the handler was free to report "this click CLOSED the
+ # open menu — no option was selected" about a click it could not actually account for. Piercing
+ # perception is what makes those rows reachable, so this became live with it.
+ from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS, _FIND_MENU_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<button id="pick" style="position:absolute;top:10px;left:10px;width:120px;height:24px">Pick</button>
+ <x-menu id="m" style="position:absolute;top:40px;left:10px;display:block;width:200px;height:60px"></x-menu>
+ <script>
+ window.root = document.getElementById('m').attachShadow({mode:'open'});
+ document.getElementById('pick').addEventListener('click', function () {
+ window.root.innerHTML =
+ '<div role="option" style="height:20px">Alpha</div>'
+ + '<div role="option" style="height:20px">Beta</div>'
+ + '<div role="option" style="height:20px">Gamma</div>';
+ });
+ </script>"""
+ ) as page:
+ await page.evaluate(_CLICK_PRECHECK_JS, "#pick")
+ await page.click("#pick")
+ await page.wait_for_timeout(100)
+ found = await page.evaluate(_FIND_MENU_JS, "#pick")
+ assert isinstance(found, dict) and found.get("count") == 3, found
+
+ pre = await page.evaluate(_CLICK_PRECHECK_JS, "#m")
+ assert pre["menuOpen"] is True, pre
+ # The host encloses every tagged row through its own shadow root, so this click cannot be
+ # attributed to any one option.
+ assert pre["containsMenu"] is True, pre
+ assert pre["isOption"] is False, pre
+
+
+@_skip_no_browser
+async def test_pierced_query_does_not_count_a_named_getter_decoy_as_a_second_root() -> None:
+ # A form exposes its named controls as its own properties, so <fieldset name="shadowRoot"> makes
+ # form.shadowRoot that fieldset. Walking it adds a non-root to the list every probe then queries,
+ # and because the fieldset is ALREADY in the document, everything inside it gets counted twice —
+ # so a uniqueness check reads a single element as ambiguous and the probe declines a real answer.
+ # Existence probes cannot see this (an extra root only adds matches); accumulation can.
+ from skyvern.forge.taskv3.tools import _PIERCED_QUERY_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<form id="hostile">
+ <fieldset name="shadowRoot"><input id="decoy" name="decoyChild"></fieldset>
+ </form>"""
+ ) as page:
+ counts = await page.evaluate(
+ "() => {" + _PIERCED_QUERY_JS + " return {roots: _rootList.length, hits: pQSA('#decoy').length}; }"
+ )
+ assert counts == {"roots": 1, "hits": 1}, counts
+
+
+@_skip_no_browser
+async def test_pre_snapshot_does_not_provoke_the_re_render_it_then_misreads() -> None:
+ # Marking the pre-snapshot with attributes mutates the page, and a component watching its own
+ # shadow root re-renders in response — destroying every mark just made, so the finders see an
+ # unmarked subtree and report a static list as something that appeared in reaction.
+ from skyvern.forge.taskv3.tools import _FIND_MENU_JS, _FIND_SUGGESTION_JS, _PRESNAPSHOT_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<input id="city" style="position:absolute;top:20px;left:20px;width:200px;height:24px">
+ <x-list id="lst" style="position:absolute;top:50px;left:20px;display:block;width:220px;height:70px"></x-list>
+ <script>
+ var root = document.getElementById('lst').attachShadow({mode: 'open'});
+ window.renders = 0;
+ function render() {
+ window.renders++;
+ // role=option so these rows are menu-ELIGIBLE: a plain <li> is filtered out before the
+ // reaction gate is consulted, which would make the _FIND_MENU_JS assertion below pass
+ // whatever the gate does.
+ root.innerHTML = '<ul style="margin:0"><li role="option" style="height:20px">Paris Texas</li>'
+ + '<li role="option" style="height:20px">Paris France</li>'
+ + '<li role="option" style="height:20px">Parish Road</li></ul>';
+ }
+ render();
+ // Disconnected across its own write, or re-rendering would observe itself and spin.
+ var obs = new MutationObserver(function () {
+ obs.disconnect();
+ render();
+ obs.observe(root, {subtree: true, attributes: true, childList: true});
+ });
+ obs.observe(root, {subtree: true, attributes: true, childList: true});
+ </script>"""
+ ) as page:
+ await page.evaluate(_PRESNAPSHOT_JS)
+ await page.wait_for_timeout(150)
+ # Nothing on this page reacts to anything, so both finders must come back empty.
+ assert await page.evaluate(_FIND_SUGGESTION_JS, {"field": "#city", "value": "Paris France"}) is None
+ # A STRING, not a dict: _FIND_MENU_JS takes the clicked selector. A dict stringifies to
+ # "[object Object]", pQS throws on it, and the probe returns null at the trigger step —
+ # passing for a reason that has nothing to do with the reaction gate under test.
+ assert await page.evaluate(_FIND_MENU_JS, "#city") is None
+ assert await page.evaluate("() => window.renders") == 1, "the snapshot must not mutate the page"
+
+ # The fixture really is reactive: stamping an attribute the old way does re-render it, so the
+ # assertion above is about our snapshot and not about an inert page.
+ await page.evaluate(
+ "() => { document.getElementById('lst').shadowRoot.querySelector('li').setAttribute('x', '1'); }"
+ )
+ await page.wait_for_timeout(150)
+ assert await page.evaluate("() => window.renders") > 1
+
+
+@_skip_no_browser
+async def test_pre_snapshot_survives_a_container_the_page_rebuilds_by_cloning() -> None:
+ # The light-DOM half of the snapshot is an ATTRIBUTE, not node identity, because an attribute
+ # survives cloneNode/innerHTML and node identity does not. A page that re-creates a list by
+ # cloning it would otherwise read as "all of this appeared in reaction to your click", and the
+ # model would be told to pick one of its own search results as a dropdown option.
+ from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS, _FIND_MENU_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<button id="sort" style="position:absolute;top:10px;left:10px;width:120px;height:24px">Sort</button>
+ <div id="results" style="position:absolute;top:40px;left:10px;width:220px">
+ <a href="/a" role="option" style="display:block;height:20px">Result Alpha</a>
+ <a href="/b" role="option" style="display:block;height:20px">Result Beta</a>
+ <a href="/c" role="option" style="display:block;height:20px">Result Gamma</a>
+ </div>"""
+ ) as page:
+ await page.evaluate(_CLICK_PRECHECK_JS, "#sort")
+ # The page rebuilds the container: every node is new, but every node carries the mark.
+ await page.evaluate(
+ "() => { const r = document.getElementById('results'); r.parentNode.replaceChild(r.cloneNode(true), r); }"
+ )
+ assert await page.evaluate(_FIND_MENU_JS, "#sort") is None
+
+
+@_skip_no_browser
+async def test_reaction_gate_rejects_a_page_supplied_snapshot_impostor() -> None:
+ # preReady() exists to tell "no snapshot" apart from "everything is new". A page that pre-defines
+ # __tv3_pre as a non-writable accessor keeps its own object through preReset, and an impostor
+ # whose has() always returns false makes every element read as a reaction — turning the guard
+ # into the exact confident-wrong answer it was added to prevent. instanceof, not truthiness.
+ from skyvern.forge.taskv3.tools import _CLICK_PRECHECK_JS, _FIND_MENU_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<button id="pick" style="position:absolute;top:10px;left:10px;width:120px;height:24px">Pick</button>
+ <x-menu id="m" style="position:absolute;top:40px;left:10px;display:block;width:200px;height:60px"></x-menu>
+ <script>
+ // The rows live in a shadow root ON PURPOSE: in the light DOM the attribute half of the
+ // snapshot carries identity and would mask the impostor. Inside a root the WeakSet is the
+ // only carrier, so this is exactly where an impostor decides the answer.
+ document.getElementById('m').attachShadow({mode:'open'}).innerHTML =
+ '<div role="option" style="height:20px">Alpha Corp</div>'
+ + '<div role="option" style="height:20px">Beta LLC</div>'
+ + '<div role="option" style="height:20px">Gamma Inc</div>';
+ Object.defineProperty(window, '__tv3_pre', {
+ configurable: false,
+ get: function () { return { add: function () {}, has: function () { return false; } }; },
+ set: function () {},
+ });
+ </script>"""
+ ) as page:
+ await page.evaluate(_CLICK_PRECHECK_JS, "#pick")
+ # The impostor answers "not in the snapshot" for every row. Without the instanceof check the
+ # probe believes it, and reports three static links as a menu the click just opened.
+ assert await page.evaluate(_FIND_MENU_JS, "#pick") is None
+
+
+@_skip_no_browser
+async def test_suggestion_finder_refuses_when_the_field_cannot_be_resolved() -> None:
+ # Without a field there is no geometry gate, and the scan is then page-wide: it will tag, and
+ # _commit_typeahead will click, a row nowhere near the control that was typed into. "Cannot
+ # judge" and "nothing reacted" are both safe outcomes; a confident wrong click is not.
+ from skyvern.forge.taskv3.tools import _FIND_SUGGESTION_JS, _PRESNAPSHOT_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<input id="city" style="position:absolute;top:20px;left:20px;width:200px;height:24px">
+ <div id="far" style="position:absolute;top:1500px;left:600px;display:none">Paris France</div>"""
+ ) as page:
+ await page.evaluate(_PRESNAPSHOT_JS)
+ await page.evaluate("() => { document.getElementById('far').style.display = 'block'; }")
+ assert await page.evaluate(_FIND_SUGGESTION_JS, {"field": "#missing", "value": "Paris France"}) is None
+ # and a selector the CSS engine cannot parse stays loud rather than degrading to a page scan
+ with pytest.raises(Exception):
+ await page.evaluate(_FIND_SUGGESTION_JS, {"field": "text=City", "value": "Paris France"})
+
+
+@_skip_no_browser
+async def test_verify_commit_reads_a_shadow_hosted_field() -> None:
+ # Reading '' for a field that did commit turns a successful type into a reported failure.
+ from skyvern.forge.taskv3.tools import _VERIFY_COMMIT_JS # noqa: PLC0415
+
+ async with _live_page(
+ """<ds-field id="f"></ds-field><script>
+ document.getElementById('f').attachShadow({mode:'open'}).innerHTML =
+ '<input id="inner" value="United States" />';
+ </script>"""
+ ) as page:
+ assert await page.evaluate(_VERIFY_COMMIT_JS, {"field": "#inner", "typed": "United"}) == "United States"
+
+
+@_skip_no_browser
+async def test_a_shadow_selector_is_never_handed_out_when_it_would_denote_a_sibling() -> None:
+ # Unique is not enough, it must be THIS element, and that has to hold on the pierced path too: a
+ # NULL id folds to U+FFFD when escaped, so a selector built from it denotes a sibling carrying
+ # the real U+FFFD id. observe declines to name the first control and says why, rather than
+ # handing out a selector that resolves to the second -- and the sibling it does name still
+ # resolves to itself.
+ async with _live_page(
+ """<x-host id="h"></x-host><script>
+ var r = document.getElementById('h').attachShadow({mode: 'open'});
+ var real = document.createElement('button');
+ real.setAttribute('id', '\\u0000');
+ real.textContent = 'Real Control';
+ real.setAttribute('style', 'display:block;width:120px;height:20px');
+ var decoy = document.createElement('button');
+ decoy.setAttribute('id', '\\uFFFD');
+ decoy.textContent = 'Decoy';
+ decoy.setAttribute('style', 'display:block;width:120px;height:20px');
+ r.appendChild(real); r.appendChild(decoy);
+ </script>"""
+ ) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "observe").handler({})
+ assert "Real Control" not in r.content, r.content
+ assert "identifier we cannot render safely" in r.content, r.content
+ line = next(ln for ln in r.content.splitlines() if "Decoy" in ln)
+ selector = line[1 : line.index("] ")]
+ assert await page.locator(selector).first.inner_text() == "Decoy"
+
+
+# The seam between the two shipped halves: a styled listbox proxy AND the hidden native it writes
+# through to, both inside one open shadow root, with the option rows rendered in that same root. The
+# hidden-native listing reaches the <select>; the pierced reaction probes reach the rows. Nothing
+# below asserts that they compile — each step asserts what the page did.
+_PROXY_MENU_IN_SHADOW_HTML = """
+<!doctype html><html><body>
+ <ds-country id="country-host"></ds-country>
+ <script>
+ var c = document.getElementById('country-host').attachShadow({mode: 'open'});
+ c.innerHTML =
+ '<label for="sr-country">Shipping Country</label>' +
+ '<select id="sr-country" style="display:none">' +
+ '<option value="">Pick</option><option value="us">United States</option>' +
+ '<option value="ca">Canada</option></select>' +
+ '<div role="listbox" id="sr-proxy" style="width:200px;height:30px">Pick</div>' +
+ '<div id="sr-opts" style="display:none;position:absolute;top:40px;left:0;width:200px">' +
+ '<div role="option" data-v="us" style="height:20px">United States</div>' +
+ '<div role="option" data-v="ca" style="height:20px">Canada</div></div>';
+ c.getElementById('sr-proxy').addEventListener('click', function () {
+ c.getElementById('sr-opts').style.display = 'block';
+ });
+ c.querySelectorAll('#sr-opts [role=option]').forEach(function (o) {
+ o.addEventListener('click', function () {
+ var s = c.getElementById('sr-country');
+ s.value = o.getAttribute('data-v');
+ s.dispatchEvent(new Event('change', {bubbles: true}));
+ c.getElementById('sr-proxy').textContent = o.textContent;
+ c.getElementById('sr-opts').style.display = 'none';
+ });
+ });
+ c.getElementById('sr-country').addEventListener('change', function (e) {
+ window.__srChanged = e.target.value;
+ });
+ </script>
+</body></html>
+"""
+
+
+@_skip_no_browser
+async def test_pierced_probes_and_hidden_native_listing_agree_on_the_shadow_hosted_widget() -> None:
+ async with _content_page(_PROXY_MENU_IN_SHADOW_HTML) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "observe").handler({})
+ assert r.status == "ok"
+ # The hidden-native half still reaches the <select> through the root.
+ assert any("[hidden-native" in ln and "sr-country" in ln for ln in r.content.splitlines()), r.content
+
+ # The pierced reaction probes see the rows the component rendered into its own root.
+ opened = await _tool(tools, "click").handler({"selector": "#sr-proxy"})
+ assert opened.status == "ok", opened.content
+ assert "opened a menu of 2 options" in opened.content, opened.content
+
+ assert "Canada" in opened.content, opened.content
+
+ chosen = await _tool(tools, "click").handler({"selector": '[data-tv3-menu="2"]'})
+ assert chosen.status == "ok", chosen.content
+ # The element actually driven is the hidden native the other half listed — the two halves
+ # agree about what this widget is, rather than each acting on a different element.
+ assert await page.evaluate("window.__srChanged") == "ca"
+ assert (
+ await page.evaluate(
+ "() => document.getElementById('country-host').shadowRoot.getElementById('sr-country').value"
+ )
+ == "ca"
+ )
+
+
+@_skip_no_browser
+async def test_a_menu_opened_by_a_hash_href_trigger_is_still_reported() -> None:
+ # `<a href="#">` is the ordinary markup for a dropdown trigger, and clicking one moves the URL to
+ # `...#` without leaving the document. Treating any URL change as a navigation deleted these
+ # menus outright: the rows are visible, the finder returns all of them, and the tool reported
+ # nothing — so the model never learns there is a list to pick from. A real cross-document
+ # navigation needs no such guard, because it destroys the pre-snapshot that arms the finder.
+ async with _live_page(
+ """<a id="go" href="#" style="display:block;width:120px;height:24px">Menu</a>
+ <div id="menu" style="display:none;position:absolute;top:40px;left:0;width:200px">
+ <div role="option" style="height:20px">Alpha Corp</div>
+ <div role="option" style="height:20px">Beta LLC</div>
+ <div role="option" style="height:20px">Gamma Inc</div>
+ </div>
+ <script>
+ document.getElementById('go').addEventListener('click', function () {
+ document.getElementById('menu').style.display = 'block';
+ });
+ </script>"""
+ ) as page:
+ tools = build_browser_tools(_fixed_page_provider(page))
+ r = await _tool(tools, "click").handler({"selector": "#go"})
+ assert r.status == "ok", r.content
+ assert "opened a menu of 3 options" in r.content, r.content
+ assert "Alpha Corp" in r.content, r.content