Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): select radios and checkboxes by label (#14282)
Nilay86a1c93a9eca
diff --git a/.github/workflows/studio-frontend-ci.yml b/.github/workflows/studio-frontend-ci.ymlindex bde53ca7a6c..8f9b778b5dc 100644--- a/.github/workflows/studio-frontend-ci.yml+++ b/.github/workflows/studio-frontend-ci.yml@@ -33,6 +33,7 @@ on:- 'tests/studio/playwright_keyboard_shortcuts.py'- 'tests/studio/playwright_find_in_page.py'- 'tests/studio/playwright_project_folders.py'+ - 'tests/studio/playwright_browser_agent_select.py'- 'tests/studio/playwright_tool_activity.py'- 'tests/studio/playwright_stream_pacing.py'- 'tests/studio/playwright_code_block_flicker.py'@@ -76,6 +77,7 @@ on:- 'tests/studio/playwright_keyboard_shortcuts.py'- 'tests/studio/playwright_find_in_page.py'- 'tests/studio/playwright_project_folders.py'+ - 'tests/studio/playwright_browser_agent_select.py'- 'tests/studio/playwright_tool_activity.py'- 'tests/studio/playwright_stream_pacing.py'- 'tests/studio/playwright_code_block_flicker.py'@@ -556,6 +558,10 @@ jobs:working-directory: ${{ github.workspace }}timeout-minutes: 10run: python3 tests/studio/playwright_project_folders.py --out "$RUNNER_TEMP/project-folders"+ - name: Browser agent checks radios and checkboxes by their labels+ working-directory: ${{ github.workspace }}+ timeout-minutes: 5+ run: python3 tests/studio/playwright_browser_agent_select.py- name: Icon tooltips follow the pointer between neighboursworking-directory: ${{ github.workspace }}diff --git a/studio/backend/core/inference/browser_agent.py b/studio/backend/core/inference/browser_agent.pyindex f561c86a7b3..fdef9166e62 100644--- a/studio/backend/core/inference/browser_agent.py+++ b/studio/backend/core/inference/browser_agent.py@@ -207,7 +207,8 @@ BROWSER_SELECT_TOOL = {"name": "browser_select","description": ("Choose an option in a dropdown, or set a checkbox or radio button, by ref. value is "- 'the option text, or "true"/"false" for a checkbox.'+ 'the option text, or "true"/"false" for a checkbox or radio button, whose own ref '+ "is the one to pass."),"parameters": {"type": "object",@@ -479,7 +480,10 @@ def _validated_request(name: str, arguments: dict, website_policy: Optional[dictrequest["text"] = textrequest["submit"] = arguments.get("submit") is Trueif name == "browser_select":- request["value"] = str(arguments.get("value") or "")[:500]+ value = arguments.get("value")+ if isinstance(value, bool):+ value = "true" if value else "false"+ request["value"] = ("" if value is None else str(value))[:500]return requestif name == "browser_scroll":direction = str(arguments.get("direction") or "down").lower()diff --git a/studio/backend/tests/test_browser_agent.py b/studio/backend/tests/test_browser_agent.pyindex fe9661fd7c9..a6eefa3d6e5 100644--- a/studio/backend/tests/test_browser_agent.py+++ b/studio/backend/tests/test_browser_agent.py@@ -102,6 +102,10 @@ def test_type_select_scroll_and_keys():"submit": True,}assert _request("browser_select", {"ref": "r4", "value": "Large"})["value"] == "Large"+ assert _request("browser_select", {"ref": "r4", "value": False})["value"] == "false"+ assert _request("browser_select", {"ref": "r4", "value": 0})["value"] == "0"+ assert _request("browser_select", {"ref": "r4"})["value"] == ""+ assert _request("browser_select", {"ref": "r4", "value": True})["value"] == "true"assert _request("browser_scroll", {}) == {"op": "scroll", "direction": "down"}assert _request("browser_press_key", {"key": "enter"}) == {"op": "press", "key": "Enter"}with pytest.raises(ValueError):@@ -217,6 +221,11 @@ def test_errors_and_long_pages():assert browser_agent.format_result("browser_click", {"ok": False, "error": "stale ref r9"}, None) == ("Error: stale ref r9")+ error = 'r4 is a radio; value must be "true", "false" or its own label. Pick the ref of the option to choose.'+ assert (+ browser_agent.format_result("browser_select", {"ok": False, "error": error}, None)+ == f"Error: {error}"+ )long = browser_agent.format_result("browser_snapshot", {"ok": True, "url": "", "text": "x" * 50_000}, None)diff --git a/studio/frontend/src/features/browser/agent/executor.ts b/studio/frontend/src/features/browser/agent/executor.tsindex eb162b8a94b..1d20ee1377d 100644--- a/studio/frontend/src/features/browser/agent/executor.ts+++ b/studio/frontend/src/features/browser/agent/executor.ts@@ -314,6 +314,7 @@ const AGENT_ERRORS = [/^r\d{1,9} is disabled or read-only\.$/,/^r\d{1,9} takes a value like (2026-03-14|09:30|2026-03-14T09:30|2026-03|2026-W11)\.$/,/^r\d{1,9} is a radio button; select another option in its group instead\.$/,+ /^r\d{1,9} is a (checkbox|radio|switch); value must be "true", "false" or its own label\. Pick the ref of the option to choose\.$/,/^that key does nothing in this field; click or type instead\.$/,/^r\d{1,9} has no such option; snapshot it to see its options\.$/,/^r\d{1,9} is not a dropdown or checkbox; click it, then click the option\.$/,diff --git a/studio/frontend/src/features/browser/agent/page-agent.ts b/studio/frontend/src/features/browser/agent/page-agent.tsindex 2b889616a4a..ae4dbeff689 100644--- a/studio/frontend/src/features/browser/agent/page-agent.ts+++ b/studio/frontend/src/features/browser/agent/page-agent.ts@@ -211,9 +211,9 @@ export function installPageAgent(): PageAgent {return words.join(" ");};- const nameOf = (el: Element): string => {+ const nameOf = (el: Element, max = 80): string => {const aria = el.getAttribute("aria-label");- if (aria?.trim()) return clean(aria);+ if (aria?.trim()) return clean(aria, max);const labelledBy = el.getAttribute("aria-labelledby");if (labelledBy) {// Read as the user sees it: a hidden or 1px element must not name the control.@@ -226,25 +226,25 @@ export function installPageAgent(): PageAgent {return visibleText(label);}).join(" ");- if (text.trim()) return clean(text);+ if (text.trim()) return clean(text, max);}const field = el as HTMLInputElement;if (field.labels?.length) {const text = Array.from(field.labels).map((label) => visibleText(label)).join(" ");- if (text.trim()) return clean(text);+ if (text.trim()) return clean(text, max);}const tag = el.tagName.toLowerCase();- if (tag === "input" && ["button", "submit", "reset"].includes(field.type) && field.value) return clean(field.value);- if (tag === "input" && field.type === "image" && el.getAttribute("alt")) return clean(el.getAttribute("alt"));+ if (tag === "input" && ["button", "submit", "reset"].includes(field.type) && field.value) return clean(field.value, max);+ if (tag === "input" && field.type === "image" && el.getAttribute("alt")) return clean(el.getAttribute("alt"), max);if (tag === "input" || tag === "textarea") {const hint = el.getAttribute("placeholder") || el.getAttribute("title") || el.getAttribute("name");- return clean(hint);+ return clean(hint, max);}- if (tag === "select") return clean(el.getAttribute("title") || el.getAttribute("name"));+ if (tag === "select") return clean(el.getAttribute("title") || el.getAttribute("name"), max);const text = visibleText(el);- if (text.trim()) return clean(text);+ if (text.trim()) return clean(text, max);// An icon link: its picture's alt, or a title on it or inside it (an upvote arrow).const img = el.querySelector("img[alt]");const titled = el.querySelector("[title], [aria-label]");@@ -253,6 +253,7 @@ export function installPageAgent(): PageAgent {el.getAttribute("title") ||titled?.getAttribute("aria-label") ||titled?.getAttribute("title"),+ max,);};@@ -1242,7 +1243,21 @@ export function installPageAgent(): PageAgent {}const role = roleOf(el);if (role === "checkbox" || role === "radio" || role === "switch") {- const on = ["true", "on", "yes", "checked", "1", "check"].includes(value.trim().toLowerCase());+ const wanted = clean(value, Infinity).toLowerCase();+ const onWords = ["true", "on", "yes", "checked", "1", "check"];+ const offWords = ["false", "off", "no", "unchecked", "0", "uncheck"];+ let on = onWords.includes(wanted);+ // State commands keep their meaning even when a label or HTML value says "No".+ if (!on && !offWords.includes(wanted)) {+ const label = nameOf(el, Infinity);+ on = Boolean(wanted) && (+ wanted === label.toLowerCase() || wanted === clean(label).toLowerCase() ||+ (el instanceof HTMLInputElement && wanted === clean(el.value, Infinity).toLowerCase())+ );+ if (!on) {+ return { ok: false, error: `${ref} is a ${role}; value must be "true", "false" or its own label. Pick the ref of the option to choose.` };+ }+ }const current = el instanceof HTMLInputElement ? el.checked : el.getAttribute("aria-checked") === "true";if (current && !on && role === "radio") {return { ok: false, error: `${ref} is a radio button; select another option in its group instead.` };diff --git a/tests/studio/playwright_browser_agent_select.py b/tests/studio/playwright_browser_agent_select.pynew file mode 100644index 00000000000..3d36e8f8f22--- /dev/null+++ b/tests/studio/playwright_browser_agent_select.py@@ -0,0 +1,241 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++from __future__ import annotations++import json+import os+import re+import subprocess+import sys+from pathlib import Path++import pytest+from playwright.sync_api import sync_playwright++LEAF = (+ Path(__file__).resolve().parents[2] / "studio/frontend/src/features/browser/agent/page-agent.ts"+)+EXECUTOR = LEAF.with_name("executor.ts")+LONG_LABEL = (+ "I agree to receive information about upcoming events and promotional offers from our partners"+)+UNICODE_LABEL = "İletişim bilgilerimin yaklaşan etkinlikler ve iş ortaklarımızın teklifleri hakkında bilgi almak için kullanılmasını kabul ediyorum."+++def panel_errors() -> list[re.Pattern[str]]:+ block = re.search(+ r"const AGENT_ERRORS = \[(.*?)\n\];", EXECUTOR.read_text(encoding = "utf-8"), re.S+ ).group(1)+ return [re.compile(body) for body in re.findall(r"^\s*/(.*)/,$", block, re.M)]+++def shipped() -> str:+ js = f"import({json.dumps(LEAF.as_uri())}).then((m) => console.log(JSON.stringify(m.pageAgentSource())))"+ cmd = ["node", "--experimental-strip-types", "--no-warnings", "-e", js]+ return json.loads(subprocess.run(cmd, check = True, capture_output = True, text = True).stdout)++[email protected](scope = "module")+def browser():+ with sync_playwright() as p:+ with getattr(p, os.environ.get("PW_BROWSER", "chromium")).launch(headless = True) as browser:+ yield browser++[email protected](scope = "module")+def source():+ return shipped()++[email protected](scope = "module")+def page(browser):+ with browser.new_page(viewport = {"width": 1280, "height": 2400}) as page:+ yield page+++def form(page, source):+ def install(html):+ page.set_content(html)+ page.evaluate(source)++ def run(op, **args):+ return page.evaluate(+ "(c) => window[Symbol.for('unsloth.agent')].run(c)", {"op": op, **args}+ )++ text = run("snapshot")["text"]+ refs = {name: ref for ref, name in re.findall(r'\[(r\d+)\] \w+ "([^\"]+)"', text)}+ return page, run, refs++ return install+++ "html,name,value",+ [+ ('<label><input type="checkbox" value="bacon">Bacon</label>', "Bacon", " BACON "),+ (+ '<label for="box">Extra napkins</label><input id="box" type="checkbox">',+ "Extra napkins",+ " EXTRA \n NAPKINS ",+ ),+ (+ '<label><input type="checkbox" aria-label="Accessible">Other</label>',+ "Accessible",+ "accessible",+ ),+ (+ '<span id="a">Named</span><span id="b">option</span><input type="checkbox" aria-labelledby="a b">',+ "Named option",+ "named option",+ ),+ ('<label><input type="checkbox" value=" code-7 ">Option</label>', "Option", "CODE-7"),+ (+ '<div role="checkbox" aria-checked="false" onclick="this.setAttribute(\'aria-checked\', \'true\')">Custom</div>',+ "Custom",+ "custom",+ ),+ (f'<label><input type="checkbox">{LONG_LABEL}</label>', LONG_LABEL[:79] + "…", LONG_LABEL),+ (+ f'<label><input type="checkbox">{LONG_LABEL}</label>',+ LONG_LABEL[:79] + "…",+ LONG_LABEL[:79] + "…",+ ),+ (+ f'<label><input type="checkbox">{UNICODE_LABEL}</label>',+ UNICODE_LABEL[:79] + "…",+ UNICODE_LABEL,+ ),+ (+ f'<label><input type="checkbox">{UNICODE_LABEL}</label>',+ UNICODE_LABEL[:79] + "…",+ UNICODE_LABEL[:79] + "…",+ ),+ ],+)+def test_own_label_or_value(form, html, name, value):+ page, run, refs = form(html)+ answer = run("select", ref = refs[name], value = value)+ assert answer["ok"], answer+ if page.locator("input").count():+ assert page.locator("input").is_checked()+ else:+ assert page.locator("[role=checkbox]").get_attribute("aria-checked") == "true"+++ "word",+ [+ "true",+ "on",+ "yes",+ "checked",+ "1",+ "check",+ "false",+ "off",+ "no",+ "unchecked",+ "0",+ "uncheck",+ ],+)[email protected]("role", ["checkbox", "radio", "switch"])+def test_state_words_precede_labels_and_values(form, word, role):+ html = (+ f'<label><input type="{role}" value="{word}">{word}</label>'+ if role != "switch"+ else f"<div role=\"switch\" aria-checked=\"false\" onclick=\"this.setAttribute('aria-checked', String(this.getAttribute('aria-checked') !== 'true'))\">{word}</div>"+ )+ page, run, refs = form(html)+ ref = refs[word]+ expected = word in ("true", "on", "yes", "checked", "1", "check")+ for initial in (False, True):+ page.locator("input, [role=switch]").evaluate(+ "(el, state) => { if (el.tagName === 'INPUT') el.checked = state; else el.setAttribute('aria-checked', String(state)); }",+ initial,+ )+ answer = run("select", ref = ref, value = f" {word.upper()} ")+ state = page.locator("input, [role=switch]").evaluate(+ "el => el.tagName === 'INPUT' ? el.checked : el.getAttribute('aria-checked') === 'true'"+ )+ if role == "radio" and initial and not expected:+ assert not answer["ok"] and state is True, answer+ else:+ assert answer["ok"] and state is expected, answer++[email protected]("kind", ["native", "aria"])+def test_radio_targets_and_sibling_state_words(form, kind):+ labels = ["Small", "Large", "Yes", "No", "On", "Off", "Checked", "Unchecked", "0", "1"]+ if kind == "native":+ html = '<form id="size"></form>' + "".join(+ f'<label><input type="radio" name="size" form="size" value="{label.lower()}">{label}</label>'+ for label in labels+ )+ else:+ html = (+ '<div role="radiogroup group">'+ + "".join(+ f'<div role="radio option" aria-checked="false" onclick="for (const n of this.parentElement.children) n.setAttribute(\'aria-checked\', String(n === this))">{label}</div>'+ for label in labels+ )+ + "</div>"+ )+ page, run, refs = form(html)++ def selected():+ return page.locator('input:checked, [role="radio option"][aria-checked=true]').evaluate_all(+ "nodes => nodes.map(n => n.tagName === 'INPUT' ? n.value : n.textContent.toLowerCase())"+ )++ for target, other in (("Large", "Small"), ("Small", "Large")):+ assert run("select", ref = refs[target], value = target)["ok"]+ assert selected() == [target.lower()]+ answer = run("select", ref = refs[target], value = other)+ assert not answer["ok"] and selected() == [target.lower()], answer+ assert any(pattern.fullmatch(answer["error"]) for pattern in panel_errors()), answer+ for word in ("yes", "on", "checked", "1", "check", "true"):+ assert run("select", ref = refs[target], value = word)["ok"]+ assert selected() == [target.lower()]+ assert run("select", ref = refs["No"], value = "true")["ok"]+ assert selected() == ["no"]++[email protected]("value", ["extra", "", " ", "Other"])[email protected]("role", ["checkbox", "radio", "switch"])+def test_unknown_value_fails_without_changing_state(form, value, role):+ html = (+ f'<label><input type="{role}" value="" checked>Option</label>'+ if role != "switch"+ else '<div role="switch" aria-checked="true" onclick="this.setAttribute(\'aria-checked\', \'false\')">Option</div>'+ )+ page, run, refs = form(html)+ answer = run("select", ref = refs["Option"], value = value)+ state = page.locator("input, [role=switch]").evaluate(+ "el => el.tagName === 'INPUT' ? el.checked : el.getAttribute('aria-checked') === 'true'"+ )+ assert answer["ok"] is False and state is True, answer+ assert any(pattern.fullmatch(answer["error"]) for pattern in panel_errors()), answer+ assert '"true", "false"' in answer["error"]+++def test_aria_name_takes_precedence_over_html_label(form):+ page, run, refs = form('<label><input type="checkbox" aria-label="Accessible">Other</label>')+ answer = run("select", ref = refs["Accessible"], value = "Other")+ assert answer["ok"] is False and not page.locator("input").is_checked(), answer+++def test_dropdown_unchanged(form):+ page, run, refs = form(+ '<select aria-label="Size"><option value="s">Small</option><option value="l">Large</option></select>'+ )+ for value, expected in (("Large", "l"), ("s", "s")):+ assert run("select", ref = refs["Size"], value = value)["ok"]+ assert page.locator("select").input_value() == expected+++if __name__ == "__main__":+ sys.exit(pytest.main([__file__, "-q", "--confcutdir", str(Path(__file__).parent)]))