Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): select radios and checkboxes by label (#14282)

Nilay86a1c93a9eca
diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.yml
index bde53ca7a6c..8f9b778b5dc 100644
--- a/.github/workflows/studio-frontend-ci.yml
+++ b/.github/workflows/studio-frontend-ci.yml
@@ -33,6 +33,7 @@ on:
- 'tests/studio/playwright_keyboard_shortcuts.py'
- 'tests/studio/playwright_find_in_page.py'
- 'tests/studio/playwright_project_folders.py'
+ - 'tests/studio/playwright_browser_agent_select.py'
- 'tests/studio/playwright_tool_activity.py'
- 'tests/studio/playwright_stream_pacing.py'
- 'tests/studio/playwright_code_block_flicker.py'
@@ -76,6 +77,7 @@ on:
- 'tests/studio/playwright_keyboard_shortcuts.py'
- 'tests/studio/playwright_find_in_page.py'
- 'tests/studio/playwright_project_folders.py'
+ - 'tests/studio/playwright_browser_agent_select.py'
- 'tests/studio/playwright_tool_activity.py'
- 'tests/studio/playwright_stream_pacing.py'
- 'tests/studio/playwright_code_block_flicker.py'
@@ -556,6 +558,10 @@ jobs:
working-directory: ${{ github.workspace }}
timeout-minutes: 10
run: python3 tests/studio/playwright_project_folders.py --out "$RUNNER_TEMP/project-folders"
+ - name: Browser agent checks radios and checkboxes by their labels
+ working-directory: ${{ github.workspace }}
+ timeout-minutes: 5
+ run: python3 tests/studio/playwright_browser_agent_select.py
- name: Icon tooltips follow the pointer between neighbours
working-directory: ${{ github.workspace }}
diff --git a/studio/backend/core/inference/browser_agent.py b/studio/backend/core/inference/browser_agent.py
index f561c86a7b3..fdef9166e62 100644
--- a/studio/backend/core/inference/browser_agent.py
+++ b/studio/backend/core/inference/browser_agent.py
@@ -207,7 +207,8 @@ BROWSER_SELECT_TOOL = {
"name": "browser_select",
"description": (
"Choose an option in a dropdown, or set a checkbox or radio button, by ref. value is "
- 'the option text, or "true"/"false" for a checkbox.'
+ 'the option text, or "true"/"false" for a checkbox or radio button, whose own ref '
+ "is the one to pass."
),
"parameters": {
"type": "object",
@@ -479,7 +480,10 @@ def _validated_request(name: str, arguments: dict, website_policy: Optional[dict
request["text"] = text
request["submit"] = arguments.get("submit") is True
if name == "browser_select":
- request["value"] = str(arguments.get("value") or "")[:500]
+ value = arguments.get("value")
+ if isinstance(value, bool):
+ value = "true" if value else "false"
+ request["value"] = ("" if value is None else str(value))[:500]
return request
if name == "browser_scroll":
direction = str(arguments.get("direction") or "down").lower()
diff --git a/studio/backend/tests/test_browser_agent.py b/studio/backend/tests/test_browser_agent.py
index fe9661fd7c9..a6eefa3d6e5 100644
--- a/studio/backend/tests/test_browser_agent.py
+++ b/studio/backend/tests/test_browser_agent.py
@@ -102,6 +102,10 @@ def test_type_select_scroll_and_keys():
"submit": True,
}
assert _request("browser_select", {"ref": "r4", "value": "Large"})["value"] == "Large"
+ assert _request("browser_select", {"ref": "r4", "value": False})["value"] == "false"
+ assert _request("browser_select", {"ref": "r4", "value": 0})["value"] == "0"
+ assert _request("browser_select", {"ref": "r4"})["value"] == ""
+ assert _request("browser_select", {"ref": "r4", "value": True})["value"] == "true"
assert _request("browser_scroll", {}) == {"op": "scroll", "direction": "down"}
assert _request("browser_press_key", {"key": "enter"}) == {"op": "press", "key": "Enter"}
with pytest.raises(ValueError):
@@ -217,6 +221,11 @@ def test_errors_and_long_pages():
assert browser_agent.format_result(
"browser_click", {"ok": False, "error": "stale ref r9"}, None
) == ("Error: stale ref r9")
+ error = 'r4 is a radio; value must be "true", "false" or its own label. Pick the ref of the option to choose.'
+ assert (
+ browser_agent.format_result("browser_select", {"ok": False, "error": error}, None)
+ == f"Error: {error}"
+ )
long = browser_agent.format_result(
"browser_snapshot", {"ok": True, "url": "", "text": "x" * 50_000}, None
)
diff --git a/studio/frontend/src/features/browser/agent/executor.ts b/studio/frontend/src/features/browser/agent/executor.ts
index eb162b8a94b..1d20ee1377d 100644
--- a/studio/frontend/src/features/browser/agent/executor.ts
+++ b/studio/frontend/src/features/browser/agent/executor.ts
@@ -314,6 +314,7 @@ const AGENT_ERRORS = [
/^r\d{1,9} is disabled or read-only\.$/,
/^r\d{1,9} takes a value like (2026-03-14|09:30|2026-03-14T09:30|2026-03|2026-W11)\.$/,
/^r\d{1,9} is a radio button; select another option in its group instead\.$/,
+ /^r\d{1,9} is a (checkbox|radio|switch); value must be "true", "false" or its own label\. Pick the ref of the option to choose\.$/,
/^that key does nothing in this field; click or type instead\.$/,
/^r\d{1,9} has no such option; snapshot it to see its options\.$/,
/^r\d{1,9} is not a dropdown or checkbox; click it, then click the option\.$/,
diff --git a/studio/frontend/src/features/browser/agent/page-agent.ts b/studio/frontend/src/features/browser/agent/page-agent.ts
index 2b889616a4a..ae4dbeff689 100644
--- a/studio/frontend/src/features/browser/agent/page-agent.ts
+++ b/studio/frontend/src/features/browser/agent/page-agent.ts
@@ -211,9 +211,9 @@ export function installPageAgent(): PageAgent {
return words.join(" ");
};
- const nameOf = (el: Element): string => {
+ const nameOf = (el: Element, max = 80): string => {
const aria = el.getAttribute("aria-label");
- if (aria?.trim()) return clean(aria);
+ if (aria?.trim()) return clean(aria, max);
const labelledBy = el.getAttribute("aria-labelledby");
if (labelledBy) {
// Read as the user sees it: a hidden or 1px element must not name the control.
@@ -226,25 +226,25 @@ export function installPageAgent(): PageAgent {
return visibleText(label);
})
.join(" ");
- if (text.trim()) return clean(text);
+ if (text.trim()) return clean(text, max);
}
const field = el as HTMLInputElement;
if (field.labels?.length) {
const text = Array.from(field.labels)
.map((label) => visibleText(label))
.join(" ");
- if (text.trim()) return clean(text);
+ if (text.trim()) return clean(text, max);
}
const tag = el.tagName.toLowerCase();
- if (tag === "input" && ["button", "submit", "reset"].includes(field.type) && field.value) return clean(field.value);
- if (tag === "input" && field.type === "image" && el.getAttribute("alt")) return clean(el.getAttribute("alt"));
+ if (tag === "input" && ["button", "submit", "reset"].includes(field.type) && field.value) return clean(field.value, max);
+ if (tag === "input" && field.type === "image" && el.getAttribute("alt")) return clean(el.getAttribute("alt"), max);
if (tag === "input" || tag === "textarea") {
const hint = el.getAttribute("placeholder") || el.getAttribute("title") || el.getAttribute("name");
- return clean(hint);
+ return clean(hint, max);
}
- if (tag === "select") return clean(el.getAttribute("title") || el.getAttribute("name"));
+ if (tag === "select") return clean(el.getAttribute("title") || el.getAttribute("name"), max);
const text = visibleText(el);
- if (text.trim()) return clean(text);
+ if (text.trim()) return clean(text, max);
// An icon link: its picture's alt, or a title on it or inside it (an upvote arrow).
const img = el.querySelector("img[alt]");
const titled = el.querySelector("[title], [aria-label]");
@@ -253,6 +253,7 @@ export function installPageAgent(): PageAgent {
el.getAttribute("title") ||
titled?.getAttribute("aria-label") ||
titled?.getAttribute("title"),
+ max,
);
};
@@ -1242,7 +1243,21 @@ export function installPageAgent(): PageAgent {
}
const role = roleOf(el);
if (role === "checkbox" || role === "radio" || role === "switch") {
- const on = ["true", "on", "yes", "checked", "1", "check"].includes(value.trim().toLowerCase());
+ const wanted = clean(value, Infinity).toLowerCase();
+ const onWords = ["true", "on", "yes", "checked", "1", "check"];
+ const offWords = ["false", "off", "no", "unchecked", "0", "uncheck"];
+ let on = onWords.includes(wanted);
+ // State commands keep their meaning even when a label or HTML value says "No".
+ if (!on && !offWords.includes(wanted)) {
+ const label = nameOf(el, Infinity);
+ on = Boolean(wanted) && (
+ wanted === label.toLowerCase() || wanted === clean(label).toLowerCase() ||
+ (el instanceof HTMLInputElement && wanted === clean(el.value, Infinity).toLowerCase())
+ );
+ if (!on) {
+ return { ok: false, error: `${ref} is a ${role}; value must be "true", "false" or its own label. Pick the ref of the option to choose.` };
+ }
+ }
const current = el instanceof HTMLInputElement ? el.checked : el.getAttribute("aria-checked") === "true";
if (current && !on && role === "radio") {
return { ok: false, error: `${ref} is a radio button; select another option in its group instead.` };
diff --git a/tests/studio/playwright_browser_agent_select.py b/tests/studio/playwright_browser_agent_select.py
new file mode 100644
index 00000000000..3d36e8f8f22
--- /dev/null
+++ b/tests/studio/playwright_browser_agent_select.py
@@ -0,0 +1,241 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+from playwright.sync_api import sync_playwright
+
+LEAF = (
+ Path(__file__).resolve().parents[2] / "studio/frontend/src/features/browser/agent/page-agent.ts"
+)
+EXECUTOR = LEAF.with_name("executor.ts")
+LONG_LABEL = (
+ "I agree to receive information about upcoming events and promotional offers from our partners"
+)
+UNICODE_LABEL = "İletişim bilgilerimin yaklaşan etkinlikler ve iş ortaklarımızın teklifleri hakkında bilgi almak için kullanılmasını kabul ediyorum."
+
+
+def panel_errors() -> list[re.Pattern[str]]:
+ block = re.search(
+ r"const AGENT_ERRORS = \[(.*?)\n\];", EXECUTOR.read_text(encoding = "utf-8"), re.S
+ ).group(1)
+ return [re.compile(body) for body in re.findall(r"^\s*/(.*)/,$", block, re.M)]
+
+
+def shipped() -> str:
+ js = f"import({json.dumps(LEAF.as_uri())}).then((m) => console.log(JSON.stringify(m.pageAgentSource())))"
+ cmd = ["node", "--experimental-strip-types", "--no-warnings", "-e", js]
+ return json.loads(subprocess.run(cmd, check = True, capture_output = True, text = True).stdout)
+
+
[email protected](scope = "module")
+def browser():
+ with sync_playwright() as p:
+ with getattr(p, os.environ.get("PW_BROWSER", "chromium")).launch(headless = True) as browser:
+ yield browser
+
+
[email protected](scope = "module")
+def source():
+ return shipped()
+
+
[email protected](scope = "module")
+def page(browser):
+ with browser.new_page(viewport = {"width": 1280, "height": 2400}) as page:
+ yield page
+
+
+def form(page, source):
+ def install(html):
+ page.set_content(html)
+ page.evaluate(source)
+
+ def run(op, **args):
+ return page.evaluate(
+ "(c) => window[Symbol.for('unsloth.agent')].run(c)", {"op": op, **args}
+ )
+
+ text = run("snapshot")["text"]
+ refs = {name: ref for ref, name in re.findall(r'\[(r\d+)\] \w+ "([^\"]+)"', text)}
+ return page, run, refs
+
+ return install
+
+
+ "html,name,value",
+ [
+ ('<label><input type="checkbox" value="bacon">Bacon</label>', "Bacon", " BACON "),
+ (
+ '<label for="box">Extra napkins</label><input id="box" type="checkbox">',
+ "Extra napkins",
+ " EXTRA \n NAPKINS ",
+ ),
+ (
+ '<label><input type="checkbox" aria-label="Accessible">Other</label>',
+ "Accessible",
+ "accessible",
+ ),
+ (
+ '<span id="a">Named</span><span id="b">option</span><input type="checkbox" aria-labelledby="a b">',
+ "Named option",
+ "named option",
+ ),
+ ('<label><input type="checkbox" value=" code-7 ">Option</label>', "Option", "CODE-7"),
+ (
+ '<div role="checkbox" aria-checked="false" onclick="this.setAttribute(\'aria-checked\', \'true\')">Custom</div>',
+ "Custom",
+ "custom",
+ ),
+ (f'<label><input type="checkbox">{LONG_LABEL}</label>', LONG_LABEL[:79] + "…", LONG_LABEL),
+ (
+ f'<label><input type="checkbox">{LONG_LABEL}</label>',
+ LONG_LABEL[:79] + "…",
+ LONG_LABEL[:79] + "…",
+ ),
+ (
+ f'<label><input type="checkbox">{UNICODE_LABEL}</label>',
+ UNICODE_LABEL[:79] + "…",
+ UNICODE_LABEL,
+ ),
+ (
+ f'<label><input type="checkbox">{UNICODE_LABEL}</label>',
+ UNICODE_LABEL[:79] + "…",
+ UNICODE_LABEL[:79] + "…",
+ ),
+ ],
+)
+def test_own_label_or_value(form, html, name, value):
+ page, run, refs = form(html)
+ answer = run("select", ref = refs[name], value = value)
+ assert answer["ok"], answer
+ if page.locator("input").count():
+ assert page.locator("input").is_checked()
+ else:
+ assert page.locator("[role=checkbox]").get_attribute("aria-checked") == "true"
+
+
+ "word",
+ [
+ "true",
+ "on",
+ "yes",
+ "checked",
+ "1",
+ "check",
+ "false",
+ "off",
+ "no",
+ "unchecked",
+ "0",
+ "uncheck",
+ ],
+)
[email protected]("role", ["checkbox", "radio", "switch"])
+def test_state_words_precede_labels_and_values(form, word, role):
+ html = (
+ f'<label><input type="{role}" value="{word}">{word}</label>'
+ if role != "switch"
+ else f"<div role=\"switch\" aria-checked=\"false\" onclick=\"this.setAttribute('aria-checked', String(this.getAttribute('aria-checked') !== 'true'))\">{word}</div>"
+ )
+ page, run, refs = form(html)
+ ref = refs[word]
+ expected = word in ("true", "on", "yes", "checked", "1", "check")
+ for initial in (False, True):
+ page.locator("input, [role=switch]").evaluate(
+ "(el, state) => { if (el.tagName === 'INPUT') el.checked = state; else el.setAttribute('aria-checked', String(state)); }",
+ initial,
+ )
+ answer = run("select", ref = ref, value = f" {word.upper()} ")
+ state = page.locator("input, [role=switch]").evaluate(
+ "el => el.tagName === 'INPUT' ? el.checked : el.getAttribute('aria-checked') === 'true'"
+ )
+ if role == "radio" and initial and not expected:
+ assert not answer["ok"] and state is True, answer
+ else:
+ assert answer["ok"] and state is expected, answer
+
+
[email protected]("kind", ["native", "aria"])
+def test_radio_targets_and_sibling_state_words(form, kind):
+ labels = ["Small", "Large", "Yes", "No", "On", "Off", "Checked", "Unchecked", "0", "1"]
+ if kind == "native":
+ html = '<form id="size"></form>' + "".join(
+ f'<label><input type="radio" name="size" form="size" value="{label.lower()}">{label}</label>'
+ for label in labels
+ )
+ else:
+ html = (
+ '<div role="radiogroup group">'
+ + "".join(
+ f'<div role="radio option" aria-checked="false" onclick="for (const n of this.parentElement.children) n.setAttribute(\'aria-checked\', String(n === this))">{label}</div>'
+ for label in labels
+ )
+ + "</div>"
+ )
+ page, run, refs = form(html)
+
+ def selected():
+ return page.locator('input:checked, [role="radio option"][aria-checked=true]').evaluate_all(
+ "nodes => nodes.map(n => n.tagName === 'INPUT' ? n.value : n.textContent.toLowerCase())"
+ )
+
+ for target, other in (("Large", "Small"), ("Small", "Large")):
+ assert run("select", ref = refs[target], value = target)["ok"]
+ assert selected() == [target.lower()]
+ answer = run("select", ref = refs[target], value = other)
+ assert not answer["ok"] and selected() == [target.lower()], answer
+ assert any(pattern.fullmatch(answer["error"]) for pattern in panel_errors()), answer
+ for word in ("yes", "on", "checked", "1", "check", "true"):
+ assert run("select", ref = refs[target], value = word)["ok"]
+ assert selected() == [target.lower()]
+ assert run("select", ref = refs["No"], value = "true")["ok"]
+ assert selected() == ["no"]
+
+
[email protected]("value", ["extra", "", " ", "Other"])
[email protected]("role", ["checkbox", "radio", "switch"])
+def test_unknown_value_fails_without_changing_state(form, value, role):
+ html = (
+ f'<label><input type="{role}" value="" checked>Option</label>'
+ if role != "switch"
+ else '<div role="switch" aria-checked="true" onclick="this.setAttribute(\'aria-checked\', \'false\')">Option</div>'
+ )
+ page, run, refs = form(html)
+ answer = run("select", ref = refs["Option"], value = value)
+ state = page.locator("input, [role=switch]").evaluate(
+ "el => el.tagName === 'INPUT' ? el.checked : el.getAttribute('aria-checked') === 'true'"
+ )
+ assert answer["ok"] is False and state is True, answer
+ assert any(pattern.fullmatch(answer["error"]) for pattern in panel_errors()), answer
+ assert '"true", "false"' in answer["error"]
+
+
+def test_aria_name_takes_precedence_over_html_label(form):
+ page, run, refs = form('<label><input type="checkbox" aria-label="Accessible">Other</label>')
+ answer = run("select", ref = refs["Accessible"], value = "Other")
+ assert answer["ok"] is False and not page.locator("input").is_checked(), answer
+
+
+def test_dropdown_unchanged(form):
+ page, run, refs = form(
+ '<select aria-label="Size"><option value="s">Small</option><option value="l">Large</option></select>'
+ )
+ for value, expected in (("Large", "l"), ("s", "s")):
+ assert run("select", ref = refs["Size"], value = value)["ok"]
+ assert page.locator("select").input_value() == expected
+
+
+if __name__ == "__main__":
+ sys.exit(pytest.main([__file__, "-q", "--confcutdir", str(Path(__file__).parent)]))