Skip to content

Automate browser based workflows with AI

Give Task V3 observe the question context it drops (curb get_html spam) (#8201)

pedrohsdb7d7ff346174f
diff --git a/skyvern/forge/taskv3/engine.py b/skyvern/forge/taskv3/engine.py
index 36603efd9..a69c8c84b 100644
--- a/skyvern/forge/taskv3/engine.py
+++ b/skyvern/forge/taskv3/engine.py
@@ -46,7 +46,9 @@ How to work:
- Perceive with `observe`: it returns the page's visible interactive elements, each with a CSS selector, label, type, current value, and (for selects) options. Call it once per page state and act from that snapshot; re-observe only after the page changes.
- Act by CSS selector: `type`, `select_option`, `click`, `press_key`, `scroll`, `wait`, `navigate`, `file_upload`.
- Be efficient — this is the whole point of the engine. After observing a form once, fill every field you can before doing anything that reloads the page. Minimize tool calls and turns.
-- Use `get_html` only when `observe` lacks detail for a specific element; do not read whole pages into the conversation.
+- Batch aggressively: in ONE turn you can `type` into many fields AND `click` many radio/checkbox options AND `select_option` on several dropdowns. Answer a whole form section in a single turn — never spend a separate turn on each click.
+- `observe` already gives you everything you need to fill a field (selector, label, type, current value, options, and the surrounding question text) — act on it directly. `get_html` is a rare last resort for ONE specific element `observe` failed to describe: NEVER call it on a whole page/form/section, NEVER call it twice for the same element, and NEVER inspect more than once before acting.
+- Inspecting the page does NOT progress the task — only `type`/`select_option`/`click` do. If your recent turns were mostly `observe`/`get_html` with little typing or clicking, you are stuck inspecting: stop, and fill every field you can from the latest `observe` snapshot using its selectors before doing anything else.
- Before calling finish with status=completed, re-check with `observe` that every required field holds its intended value and the only remaining step is the final submit; fix anything missing first. Call `finish(status, reason, extracted_output)` when the goal is achieved (completed) or impossible/blocked (failed/terminated).
Rules:
diff --git a/skyvern/forge/taskv3/tools.py b/skyvern/forge/taskv3/tools.py
index 090207610..60532f4fd 100644
--- a/skyvern/forge/taskv3/tools.py
+++ b/skyvern/forge/taskv3/tools.py
@@ -30,6 +30,7 @@ _OBSERVE_JS = r"""
const els = document.querySelectorAll(q);
const out = [];
let i = 0;
+ let lastGroup = '';
for (const el of els) {
const r = el.getBoundingClientRect();
if (r.width === 0 || r.height === 0) continue;
@@ -48,6 +49,17 @@ _OBSERVE_JS = r"""
if (el.type === 'checkbox' || el.type === 'radio') rec.checked = !!el.checked;
else if (el.getAttribute('role') === 'checkbox' || el.getAttribute('role') === 'radio') rec.checked = el.getAttribute('aria-checked') === 'true';
if (el.getAttribute('aria-required') === 'true' || el.required) rec.required = true;
+ // Attach the surrounding question/group text for controls whose meaning lives in nearby
+ // non-interactive text (radio/checkbox groups, weakly-labeled fields) so the agent can answer
+ // without fetching raw HTML. Deduped against the previous element to keep grouped options compact.
+ const isChoice = el.type === 'checkbox' || el.type === 'radio' || el.getAttribute('role') === 'checkbox' || el.getAttribute('role') === 'radio';
+ if (isChoice || label.length < 3) {
+ const g = el.closest('fieldset,[role=group],li,dd,.form-group,[class*="question"],[class*="field"]');
+ if (g) {
+ const gt = (g.innerText || '').trim().replace(/\s+/g, ' ').slice(0, 200);
+ if (gt && gt.length > label.length && gt !== lastGroup) { rec.group = gt; lastGroup = gt; }
+ }
+ }
out.push(rec);
if (++i > 250) break;
}
@@ -97,6 +109,8 @@ def build_browser_tools(
extra += f" checked={e['checked']}"
if e.get("required"):
extra += " *required"
+ if e.get("group"):
+ extra += f" group={e['group']!r}"
lines.append(
f"[{e['selector']}] {e['tag']}{('/' + e['type']) if e.get('type') else ''} {e.get('label', '')!r}{extra}"
)
diff --git a/tests/unit/test_taskv3_tools.py b/tests/unit/test_taskv3_tools.py
index 15885b3a7..e5eaded9d 100644
--- a/tests/unit/test_taskv3_tools.py
+++ b/tests/unit/test_taskv3_tools.py
@@ -63,6 +63,7 @@ class _FakePage:
"selector": "#agree",
"label": "I agree",
"checked": False,
+ "group": "Consent: I agree to the terms and privacy policy",
},
],
}
@@ -164,6 +165,16 @@ async def test_observe_renders_checkbox_checked_state() -> None:
assert "#agree" in r.content and "checked=" in r.content
+async def test_observe_renders_group_context() -> None:
+ # Controls whose meaning lives in surrounding text (radio/checkbox groups, weak labels) carry a
+ # `group` field with the question text, so the agent can answer without fetching raw HTML.
+ tools = build_browser_tools(_FakePage())
+ r = await _tool(tools, "observe").handler({})
+ assert "group=" in r.content
+ assert "Consent: I agree to the terms" in r.content
+
+
@pytest.mark.asyncio
async def test_click_type_select_dispatch_raw_ops() -> None:
page = _FakePage()