Skip to content

Automate browser based workflows with AI

Surface non-ARIA validation text and rejected fields in task_v3 observe (#8255)

pedrohsdbe9c2e3bd1d86
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index d7b7743f8..f4ff054e6 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -79,6 +79,7 @@ How to work:
Rules:
- Fill fields from the task's data and satisfy required fields rather than failing over a missing value: prefer the provided values, and for an ordinary required field with no exact value, enter the most reasonable value you can. Do not invent sensitive or identifying values (government IDs, financial details, or legal/eligibility attestations); if one of those is required and not provided, stop and report it rather than guessing. Leave optional fields blank when you have no basis to fill them.
- A page message rejecting your submission and inviting you to try again is not an instruction to loop: retry at most once, and if the outcome is unchanged, finish honestly naming the rejection as the reason.
+- When a submit is refused, find the page's own message in `observe`: a `text:` line that reads as a rejection or validation message, or a field marked `*invalid`. Fix the named field if the task's data allows; otherwise finish and quote that message as the reason. A captcha widget that is merely present on the page is not evidence that it blocked the submission.
- Do not submit forms or take irreversible actions unless the goal explicitly instructs it."""
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index e67d1551f..f9ddbcc8e 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -413,13 +413,22 @@ _OBSERVE_JS = (
if (el.type === 'checkbox' || el.type === 'radio') rec.checked = !!el.checked;
else if (el.getAttribute('role') === 'checkbox' || el.getAttribute('role') === 'radio') rec.checked = el.getAttribute('aria-checked') === 'true';
if (el.getAttribute('aria-required') === 'true' || el.required) rec.required = true;
+ const isChoice = el.type === 'checkbox' || el.type === 'radio' || el.getAttribute('role') === 'checkbox' || el.getAttribute('role') === 'radio';
+ // Read .validity, never checkValidity(): that dispatches an 'invalid' event and perception must
+ // not mutate the page. Checkbox/radio .value is the static attribute ("on"), so they are excluded.
+ const ai = el.getAttribute('aria-invalid');
+ if (ai && ai !== 'false') rec.invalid = true;
+ // willValidate excludes readonly/disabled fields the agent cannot fix; password is excluded so
+ // validationMessage (which can echo the typed value) never leaks it.
+ else if (!isChoice && el.type !== 'password' && el.value && el.willValidate && !(el.form && el.form.noValidate) && el.validity && !el.validity.valid) {
+ rec.invalid = (el.validationMessage || '').slice(0, 140) || true;
+ }
// Flag typeahead/autocomplete inputs so the model treats them as combobox fills instead of typing
// raw text that never registers as a valid selection (type() also auto-commits them). See _IS_AUTOCOMPLETE_JS.
if (_isAutocomplete(el)) rec.autocomplete = true;
// Attach the surrounding question/group text for controls whose meaning lives in nearby
// non-interactive text (radio/checkbox groups, weakly-labeled fields) so the agent can answer
// without fetching raw HTML. Deduped against the previous element to keep grouped options compact.
- const isChoice = el.type === 'checkbox' || el.type === 'radio' || el.getAttribute('role') === 'checkbox' || el.getAttribute('role') === 'radio';
if (isChoice || label.length < 3) {
const g = el.closest('fieldset,[role=group],li,dd,.form-group,[class*="question"],[class*="field"]');
if (g) {
@@ -434,13 +443,19 @@ _OBSERVE_JS = (
} catch (e) { dropped++; continue; }
}
// Page-text digest: outcome states (submission confirmations, rejection banners, validation
- // summaries) live in non-interactive nodes the element list can never carry. Sources are
- // structural only — ARIA status channels first, then headings — never a body-text dump, so the
- // digest stays bounded and can't regrow the context that transcript compaction bounds.
+ // summaries) live in non-interactive nodes the element list can never carry. Three sources in
+ // priority order — ARIA status channels (uncapped within the 900 total), class/id-named message
+ // blocks (600, or 300 past what ARIA spent, whichever is larger, still inside the 900), then
+ // headings (whatever the 900 leaves) — never a body-text
+ // dump, so the digest stays bounded and can't regrow the context that transcript compaction
+ // bounds. All three carry page-controlled text at the same trust level as element labels.
const texts = [];
let textTotal = 0;
let textFull = false;
- const pushText = (t) => {
+ let textDropped = 0;
+ // `limit` is a cumulative reservation: a channel stops at its limit so the channels after it keep
+ // a floor of the 900 total instead of being starved by whichever channel ran first.
+ const pushText = (t, limit = 900) => {
t = (t || '').replace(/\s+/g, ' ').trim().slice(0, 300);
if (!t) return;
// Containment dedupe, richer message wins: an alert's text re-surfaces inside its heading's
@@ -449,7 +464,7 @@ _OBSERVE_JS = (
if (texts.some((s) => s.includes(t))) return;
const kept = texts.filter((s) => !t.includes(s));
const keptTotal = kept.reduce((total, s) => total + s.length, 0);
- if (keptTotal + t.length > 900) { textFull = true; return; }
+ if (keptTotal + t.length > limit) { textFull = textFull || limit >= 900; textDropped++; return; }
texts.length = 0; texts.push(...kept, t); textTotal = keptTotal + t.length;
};
const visible = (el) => { const r = el.getBoundingClientRect(); return r.width > 0 && r.height > 0; };
@@ -461,6 +476,42 @@ _OBSERVE_JS = (
if (textFull) break;
if (visible(el)) pushText(el.innerText);
}
+ // Rejection messages most sites render as a plain styled block with no ARIA; class/id naming is
+ // the only signal. A block with several form fields is a container (skipped unless short); one
+ // with at most one field up to the digest total is a message; longer is prose.
+ let messageCandidates = 0;
+ // A prior channel (ARIA) can already occupy the shared total before this loop starts; spend is
+ // measured relative to that starting point so this channel still gets its own floor.
+ const blockStart = textTotal;
+ const blockLimit = Math.min(900, Math.max(600, blockStart + 300));
+ // Real validation errors live inside the form; cookie banners and alert dropdowns sit above it
+ // in DOM order and must not spend the budget before the form's own error is seen.
+ // Only "error" is matched against id: id-named chrome (#alert-count, #cookie-warning) is the
+ // false-positive family this channel is most exposed to, and "error" is the one id that isn't.
+ const msgSel = '[class*="error" i],[class*="invalid" i],[class*="alert" i],[class*="warning" i],[id*="error" i]';
+ const msgCands = Array.from(document.querySelectorAll(msgSel));
+ const inForm = msgCands.filter((el) => el.closest('form'));
+ const orderedCands = inForm.concat(msgCands.filter((el) => !el.closest('form')));
+ // Stricter than visible(): this channel's selector is broad and site chrome is routinely present
+ // but hidden, whereas an ARIA live region styled invisible is not a pattern worth the extra reads.
+ const visibleText = (el) => {
+ if (!visible(el) || el.closest('[aria-hidden="true"]')) return false;
+ const r = el.getBoundingClientRect();
+ if (r.right <= 0 || r.bottom <= 0) return false;
+ const cs = getComputedStyle(el);
+ return cs.visibility !== 'hidden' && cs.opacity !== '0';
+ };
+ for (const el of orderedCands) {
+ if (textFull || textTotal - blockStart >= 600 || ++messageCandidates > 200) break;
+ // This selector set is broad, so one poisoned element degrades to "skip it", not to an
+ // emptied digest (the outer catch is for the narrow ARIA channel).
+ try {
+ if (!visibleText(el)) continue;
+ const t = (el.innerText || '').replace(/\s+/g, ' ').trim();
+ if (!t) continue;
+ if (t.length <= 300 || (t.length <= 900 && el.querySelectorAll('input,select,textarea').length < 2)) pushText(t, blockLimit);
+ } catch (e) { continue; }
+ }
for (const h of document.querySelectorAll('h1,h2,h3')) {
if (textFull) break;
if (!visible(h)) continue;
@@ -469,7 +520,7 @@ _OBSERVE_JS = (
const pt = h.parentElement ? (h.parentElement.innerText || '').replace(/\s+/g, ' ').trim() : '';
pushText(pt && pt.length <= 300 ? pt : h.innerText);
}
- } catch (e) { texts.length = 0; }
+ } catch (e) { texts.length = 0; textDropped = 0; }
// Cross-origin iframe PRESENCE: an anti-bot/captcha widget lives in one, and main-frame element
// perception can never list its contents — record host + signature so the model can see the gate
// exists. Attributes only, never the frame's document (page.frames-based traversal was considered
@@ -493,7 +544,7 @@ _OBSERVE_JS = (
iframeInfo.entries.push({ host: u.host.slice(0, 80), title: ttl, captcha: sig.test(src + ' ' + ttl) });
}
} catch (e) { iframeInfo.total = 0; iframeInfo.entries.length = 0; }
- return JSON.stringify({ url: location.href, title: document.title, text: texts, iframes: iframeInfo, dropped: dropped, elements: out });
+ return JSON.stringify({ url: location.href, title: document.title, text: texts, textDropped: textDropped, iframes: iframeInfo, dropped: dropped, elements: out });
}
"""
)
@@ -577,6 +628,10 @@ def build_browser_tools(
lines = [f"url={data.get('url')} title={data.get('title')!r} ({len(elements)} interactive elements)"]
for t in data.get("text") or []:
lines.append(f"text: {t!r}")
+ text_dropped = data.get("textDropped") or 0
+ if text_dropped:
+ # A capped digest must say it was capped: silently showing the first N reads as "that is all".
+ lines.append(f"note: {text_dropped} more page message(s) did not fit the text digest")
iframe_info = data.get("iframes") or {}
iframe_entries = iframe_info.get("entries") or []
if iframe_entries:
@@ -609,6 +664,8 @@ def build_browser_tools(
extra += f" pressed={e['pressed']}"
if e.get("required"):
extra += " *required"
+ if e.get("invalid"):
+ extra += " *invalid" if e["invalid"] is True else f" *invalid={e['invalid']!r}"
if e.get("autocomplete"):
extra += " [autocomplete→use select_combobox]"
if e.get("group"):
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 7a1ff9163..f2e13c734 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -925,6 +925,171 @@ async def test_observe_digest_total_cap_holds_under_adversarial_status_content()
texts = data.get("text") or []
assert texts # something survived
assert sum(len(t) for t in texts) <= 900
+ assert data.get("textDropped", 0) > 0 # a capped digest discloses that it was capped
+
+
+_VALIDATION_PAGE_HTML = """
+<!doctype html><html><head><title>Apply</title></head><body>
+ <h1>Talent Partner</h1>
+ <div class="error-boundary">{prose}</div>
+ <form>
+ <p class="error-message">Required profile link is empty.</p>
+ <p class="error-message" style="display:none">File exceeds the maximum upload size.</p>
+ <span class="field-error" id="phone-error">Enter a valid phone number</span>
+ <ul id="summary-errors">{summary}</ul>
+ <div class="error-summary-region">Please fix the following before continuing: your region must be set here. <select><option>US</option></select> {filler}</div>
+ <div class="alert alert-danger alert-dismissible">We could not process your application. Please correct the highlighted fields. <button type="button" class="close">x</button></div>
+ {headings}
+ <input id="site" type="url" value="N/A">
+ <input id="name" type="text" value="Jane" aria-invalid="true">
+ <input id="empty" type="text" required>
+ <input id="agree" type="checkbox" required>
+ <button id="submit" type="button">Submit</button>
+ </form>
+</body></html>
+"""
+
+
+@_skip_no_browser
+async def test_observe_digest_surfaces_validation_text_without_aria() -> None:
+ # Many sites render the submit refusal as a plain styled block with no ARIA role; the digest must
+ # still carry it, or the only anomaly the model can see after a refused submit is unrelated
+ # (e.g. a captcha widget that is always present) and the verdict names the wrong cause.
+ prose = "word " * 400 # a prose-length block whose class happens to contain "error" is not a message
+ summary = "".join(f"<li>Error {i}: field {i} must be corrected before you can submit</li>" for i in range(3))
+ headings = "".join(f"<h2>Section {i}: {'heading text ' * 6}</h2>" for i in range(12)) # ~1000 chars of headings
+ async with _content_page(
+ _VALIDATION_PAGE_HTML.format(prose=prose, summary=summary, headings=headings, filler="filler " * 30)
+ ) as page:
+ data = await _observe_data(page)
+ texts = data.get("text") or []
+ joined = " | ".join(texts)
+ assert "Required profile link is empty." in joined
+ assert "Enter a valid phone number" in joined
+ assert "Error 0: field 0 must be corrected" in joined # long field-free summary truncates, not drops
+ assert "We could not process your application" in joined # a dismiss button does not make it a container
+ assert "your region must be set here" in joined # one embedded fix control does not make it a container
+ assert any(t.startswith("Section 0:") for t in texts) # headings keep a floor of the budget
+ assert "File exceeds" not in joined # hidden
+ assert "word word word" not in joined # large container excluded
+ assert sum(len(t) for t in texts) <= 900
+
+
+@_skip_no_browser
+async def test_observe_digest_message_channel_not_starved_by_aria_total() -> None:
+ # Three ARIA status blocks alone can cross the message-block loop's shared-total break before it
+ # ever runs; the loop needs its own spend, not a check against the total the ARIA pass already ate.
+ def status_block(i: int) -> str:
+ return f'<div role="status">Status update number {i} ' + ("padding " * 29) + f"end{i}</div>"
+
+ filler = "context " * 40 # pushes the form's innerText past 300 chars so the headings channel
+ # can only take the heading alone, leaving the message-block channel as the sole source.
+ html = (
+ "<!doctype html><html><body>"
+ + "".join(status_block(i) for i in range(3))
+ + '<form><p class="error-message">Your application was rejected: the profile link is required.</p>'
+ + f"<h2>Section 0</h2><p>{filler}</p></form>"
+ + "</body></html>"
+ )
+ async with _content_page(html) as page:
+ data = await _observe_data(page)
+ texts = data.get("text") or []
+ joined = " | ".join(texts)
+ assert "Your application was rejected: the profile link is required." in joined
+ assert sum(len(t) for t in texts) <= 900
+
+
+def _pad_to(text: str, target: int) -> str:
+ s = text
+ while len(s) < target:
+ s += " word"
+ return s[:target]
+
+
+@_skip_no_browser
+async def test_observe_digest_prioritizes_form_error_over_site_chrome() -> None:
+ # Site chrome (cookie banners, alert dropdowns) sits above the form in DOM order and can spend
+ # the whole message-block budget before the loop ever reaches the real validation error.
+ alerts = _pad_to("Recent alerts dropdown notice content", 190)
+ cookie = _pad_to("This site uses cookies for analytics and marketing purposes accept", 250)
+ info = _pad_to("General info alert banner", 100)
+ html = f"""<!doctype html><html><body>
+ <div class="alerts-dropdown">{alerts}</div>
+ <div class="cookie-warning">{cookie}</div>
+ <div class="alert alert-info">{info}</div>
+ <form>
+ <p class="error-message">We could not process your application. Please correct the highlighted fields.</p>
+ <p class="error-message" style="opacity:0">Opacity hidden placeholder</p>
+ <p class="error-message" style="position:absolute;left:-9999px">Offscreen placeholder</p>
+ <p class="error-message" aria-hidden="true">Aria hidden placeholder</p>
+ </form>
+ </body></html>"""
+ async with _content_page(html) as page:
+ data = await _observe_data(page)
+ texts = data.get("text") or []
+ joined = " | ".join(texts)
+ assert "We could not process your application" in joined
+ assert "Opacity hidden placeholder" not in joined
+ assert "Offscreen placeholder" not in joined
+ assert "Aria hidden placeholder" not in joined
+
+
+@_skip_no_browser
+async def test_observe_flags_invalid_fields_with_validation_message() -> None:
+ async with _content_page(_VALIDATION_PAGE_HTML.format(prose="short", summary="", headings="", filler="")) as page:
+ data = await _observe_data(page)
+ by_sel = {e["selector"]: e for e in data["elements"]}
+ # Native constraint: "N/A" is not a URL. The browser's wording is not spec'd, so only its
+ # presence is asserted.
+ assert isinstance(by_sel["#site"]["invalid"], str) and by_sel["#site"]["invalid"]
+ assert by_sel["#name"]["invalid"] is True # aria-invalid without a native message
+ assert "invalid" not in by_sel["#empty"] # empty required is *required, not invalid
+ assert "invalid" not in by_sel["#agree"] # unchecked required checkbox: .value is "on", not a state
+
+
+@_skip_no_browser
+async def test_observe_flags_non_boolean_aria_invalid_values() -> None:
+ # aria-invalid also takes token values ("grammar", "spelling") per the ARIA spec; only the
+ # literal "false" means valid.
+ html = """<!doctype html><html><body>
+ <input id="grammar" aria-invalid="grammar">
+ <input id="spelling" aria-invalid="spelling">
+ <input id="ok" aria-invalid="false">
+ </body></html>"""
+ async with _content_page(html) as page:
+ data = await _observe_data(page)
+ by_sel = {e["selector"]: e for e in data["elements"]}
+ assert by_sel["#grammar"]["invalid"] is True
+ assert by_sel["#spelling"]["invalid"] is True
+ assert "invalid" not in by_sel["#ok"]
+
+
+@_skip_no_browser
+async def test_observe_native_invalid_excludes_unwritable_and_password_fields() -> None:
+ # readonly/disabled/novalidate fields can never be corrected or submitted as typed, so a native
+ # "invalid" verdict on them is noise; a password's validationMessage would leak its value.
+ html = """<!doctype html><html><body>
+ <input id="ro" type="email" value="not-an-email" readonly>
+ <input id="dis" type="email" value="not-an-email" disabled>
+ <form novalidate><input id="nv" type="email" value="not-an-email"></form>
+ <input id="pw" type="password" minlength="12">
+ <form><input id="plain" type="email" value="not-an-email"></form>
+ </body></html>"""
+ async with _content_page(html) as page:
+ await page.fill("#pw", "abcdefg")
+ data = await _observe_data(page)
+ by_sel = {e["selector"]: e for e in data["elements"]}
+ assert "invalid" not in by_sel["#ro"]
+ assert "invalid" not in by_sel["#dis"]
+ assert "invalid" not in by_sel["#nv"]
+ assert "invalid" not in by_sel["#pw"]
+ assert isinstance(by_sel["#plain"]["invalid"], str) and by_sel["#plain"]["invalid"]
@_skip_no_browser
@@ -1146,8 +1311,17 @@ async def test_observe_renders_text_digest_and_pressed_state() -> None:
"url": self.url,
"title": "Apply",
"text": ["Success Your submission was received."],
+ "textDropped": 3,
"elements": [
- {"i": 0, "tag": "button", "type": None, "selector": "#no", "label": "No", "pressed": True}
+ {"i": 0, "tag": "button", "type": None, "selector": "#no", "label": "No", "pressed": True},
+ {
+ "i": 1,
+ "tag": "input",
+ "type": "url",
+ "selector": "#site",
+ "label": "Site",
+ "invalid": "Enter a URL.",
+ },
],
}
)
@@ -1156,6 +1330,8 @@ async def test_observe_renders_text_digest_and_pressed_state() -> None:
r = await _tool(tools, "observe").handler({})
assert r.status == "ok"
assert "text: 'Success Your submission was received.'" in r.content
+ assert "note: 3 more page message(s) did not fit the text digest" in r.content
+ assert "*invalid='Enter a URL.'" in r.content
assert "pressed=True" in r.content