Agent Zero AI framework
Make the interactive Browser honor configured keyboard layouts
Jehubb2a2ecf16a6
diff --git a/plugins/_browser/AGENTS.md b/plugins/_browser/AGENTS.mdindex ce97ff7da..240cb0c81 100644--- a/plugins/_browser/AGENTS.md+++ b/plugins/_browser/AGENTS.md@@ -48,6 +48,7 @@- Annotation voice input reuses Whisper STT's configured draft/send delivery mode and shared microphone state.- Internal-browser proxy settings map directly to Playwright's persistent-context proxy option, never to Bring Your Own Browser, and changes must restart active internal runtimes.- Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver.+- Browser keyboard layout settings (`keyboard_layout`/`keyboard_variant`, e.g. `de`/`mac`) apply the configured XKB layout to the private browser display with setxkbmap and pin it on the Xpra shadow server so non-US keyboards type their printed characters; layout changes flow through `browser_runtime_config` and restart internal runtimes.- Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads.- `hooks.prepare_playwright_cache()` owns reconciliation of the pinned Patchright package and Chromium binary so repository self-updates and fresh images use the same setup path.- Browser startup must install the shared virtual-desktop route hook itself; do not make Browser depend on the Desktop plugin being enabled.diff --git a/plugins/_browser/default_config.yaml b/plugins/_browser/default_config.yamlindex e15d3d880..9f7dd057a 100644--- a/plugins/_browser/default_config.yaml+++ b/plugins/_browser/default_config.yaml@@ -49,3 +49,9 @@ host_browser_selection: ""# Optional _model_config preset used by Browser-owned model helpers.# Empty uses the effective Main Model.model_preset: ""++# XKB keyboard layout applied to the interactive Browser display so typed keys+# match the physical keyboard. German Mac example: keyboard_layout: "de",+# keyboard_variant: "mac" (Option+L then types "@"). Empty keeps US behavior.+keyboard_layout: ""+keyboard_variant: ""diff --git a/plugins/_browser/helpers/config.py b/plugins/_browser/helpers/config.pyindex f000a6f3c..eecd97003 100644--- a/plugins/_browser/helpers/config.py+++ b/plugins/_browser/helpers/config.py@@ -22,6 +22,8 @@ PROXY_SERVER_KEY = "proxy_server"PROXY_BYPASS_KEY = "proxy_bypass"PROXY_USERNAME_KEY = "proxy_username"PROXY_PASSWORD_KEY = "proxy_password"+KEYBOARD_LAYOUT_KEY = "keyboard_layout"+KEYBOARD_VARIANT_KEY = "keyboard_variant"RUNTIME_BACKENDS = {"container", "host_required"}BROWSER_TAB_SCOPES = {"per_context", "shared"}HOST_BROWSER_PRIVACY_POLICIES = {"enforce_local", "warn", "allow"}@@ -100,6 +102,12 @@ def _normalize_int(value: Any, *, default: int, minimum: int, maximum: int) -> ireturn max(minimum, min(maximum, number))+def _normalize_xkb_token(value: Any) -> str:+ return "".join(+ ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch in {"_", "-"}+ )[:32]++def _normalize_choice(value: Any, *, allowed: set[str], default: str) -> str:normalized = str(value or "").strip().lower().replace("-", "_")if normalized in allowed:@@ -125,6 +133,10 @@ def _model_config_summary(config: dict[str, Any] | None) -> str:def normalize_browser_config(settings: dict[str, Any] | None) -> dict[str, Any]:raw = settings if isinstance(settings, dict) else {}extension_paths = _normalize_extension_paths(raw.get("extension_paths", []))+ keyboard_layout = _normalize_xkb_token(raw.get(KEYBOARD_LAYOUT_KEY, ""))+ keyboard_variant = (+ _normalize_xkb_token(raw.get(KEYBOARD_VARIANT_KEY, "")) if keyboard_layout else ""+ )return {"extension_paths": extension_paths,DEFAULT_HOMEPAGE_KEY: _normalize_default_homepage(@@ -165,6 +177,8 @@ def normalize_browser_config(settings: dict[str, Any] | None) -> dict[str, Any]:PROXY_BYPASS_KEY: str(raw.get(PROXY_BYPASS_KEY, "") or "").strip()[:4096],PROXY_USERNAME_KEY: str(raw.get(PROXY_USERNAME_KEY, "") or "")[:1024],PROXY_PASSWORD_KEY: str(raw.get(PROXY_PASSWORD_KEY, "") or "")[:4096],+ KEYBOARD_LAYOUT_KEY: keyboard_layout,+ KEYBOARD_VARIANT_KEY: keyboard_variant,MODEL_PRESET_KEY: _normalize_model_preset(raw.get(MODEL_PRESET_KEY, "")),}@@ -177,6 +191,8 @@ def browser_runtime_config(settings: dict[str, Any] | None) -> dict[str, Any]:PROXY_BYPASS_KEY: config[PROXY_BYPASS_KEY],PROXY_USERNAME_KEY: config[PROXY_USERNAME_KEY],PROXY_PASSWORD_KEY: config[PROXY_PASSWORD_KEY],+ KEYBOARD_LAYOUT_KEY: config[KEYBOARD_LAYOUT_KEY],+ KEYBOARD_VARIANT_KEY: config[KEYBOARD_VARIANT_KEY],}diff --git a/plugins/_browser/helpers/interactive_view.py b/plugins/_browser/helpers/interactive_view.pyindex a603633d0..6689e55bb 100644--- a/plugins/_browser/helpers/interactive_view.py+++ b/plugins/_browser/helpers/interactive_view.py@@ -20,6 +20,21 @@ DEFAULT_HEIGHT = 768START_TIMEOUT_SECONDS = 15.0+def keyboard_options() -> dict[str, str]:+ """Resolve the configured XKB keyboard layout for browser displays."""+ from plugins._browser.helpers.config import (+ KEYBOARD_LAYOUT_KEY,+ KEYBOARD_VARIANT_KEY,+ get_browser_config,+ )++ config = get_browser_config()+ return {+ "layout": str(config.get(KEYBOARD_LAYOUT_KEY, "") or "").strip(),+ "variant": str(config.get(KEYBOARD_VARIANT_KEY, "") or "").strip(),+ }++def collect_status() -> dict[str, Any]:binaries = {name: shutil.which(name) or ""@@ -114,6 +129,7 @@ class BrowserInteractiveView:self._xvfb = processself.display = int(display_number)+ self._apply_keyboard_layout()self.resize(self.width, self.height)return self.display_name@@ -190,6 +206,43 @@ class BrowserInteractiveView:self._stop_locked()shutil.rmtree(self.state_dir, ignore_errors=True)+ def _apply_keyboard_layout(self) -> None:+ """Apply the configured XKB layout to this private display."""+ if self.display is None:+ return+ options = keyboard_options()+ if not options["layout"]:+ return+ setxkbmap = shutil.which("setxkbmap")+ if not setxkbmap:+ return+ command = [setxkbmap, "-display", self.display_name, "-layout", options["layout"]]+ if options["variant"]:+ command.extend(["-variant", options["variant"]])+ try:+ subprocess.run(+ command,+ check=False,+ stdin=subprocess.DEVNULL,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ timeout=5,+ )+ except (OSError, subprocess.TimeoutExpired):+ pass++ def _keyboard_xpra_args(self) -> list[str]:+ options = keyboard_options()+ if not options["layout"]:+ return []+ args = [+ "--keyboard-sync=no",+ "--keyboard-layout", options["layout"],+ ]+ if options["variant"]:+ args.extend(["--keyboard-variant", options["variant"]])+ return args+def _start_xpra(self, xpra: str) -> None:self.port = self._free_port()runtime_dir = self.state_dir / "runtime"@@ -227,6 +280,7 @@ class BrowserInteractiveView:"--encoding=auto","--quality=90","--speed=90",+ *self._keyboard_xpra_args(),f"--bind-tcp=127.0.0.1:{self.port}",f"--socket-dir={socket_dir}",f"--log-dir={self.state_dir}",diff --git a/plugins/_browser/webui/browser-config-store.js b/plugins/_browser/webui/browser-config-store.jsindex 8220f9e33..0c941674c 100644--- a/plugins/_browser/webui/browser-config-store.js+++ b/plugins/_browser/webui/browser-config-store.js@@ -41,6 +41,8 @@ function ensureConfig(config) {config.proxy_bypass = String(config.proxy_bypass || "").trim();config.proxy_username = String(config.proxy_username || "");config.proxy_password = String(config.proxy_password || "");+ config.keyboard_layout = normalizeXkbToken(config.keyboard_layout);+ config.keyboard_variant = normalizeXkbToken(config.keyboard_variant);config.host_browser_privacy_policy = normalizeChoice(config.host_browser_privacy_policy,HOST_PRIVACY_POLICIES,@@ -68,6 +70,14 @@ function normalizeInt(value, fallback, minimum, maximum) {return Math.max(minimum, Math.min(maximum, number));}+function normalizeXkbToken(value) {+ return String(value || "")+ .trim()+ .toLowerCase()+ .replace(/[^a-z0-9_-]/g, "")+ .slice(0, 32);+}+function normalizeRuntimeBackend(value) {const normalized = String(value || "").trim().toLowerCase().replace(/-/g, "_");if (normalized === "host_when_available") return "host_required";diff --git a/plugins/_browser/webui/browser-store.js b/plugins/_browser/webui/browser-store.jsindex 1b59f6838..e85ac38f1 100644--- a/plugins/_browser/webui/browser-store.js+++ b/plugins/_browser/webui/browser-store.js@@ -2908,10 +2908,11 @@ const model = {if (this.annotating) return;const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);if (!contextId || !this.activeBrowserId) return;- if (event.ctrlKey || event.metaKey || event.altKey) return;+ const printable = event.key && event.key.length === 1;+ const altGrText = printable && event.altKey && !event.metaKey;+ if ((event.ctrlKey || event.metaKey || event.altKey) && !altGrText) return;if (isLocalEditableTarget(event?.target)) return;event.preventDefault();- const printable = event.key && event.key.length === 1;await websocket.emit("browser_viewer_input", {context_id: contextId,browser_id: this.activeBrowserId,diff --git a/plugins/_browser/webui/config.html b/plugins/_browser/webui/config.htmlindex 765e56288..326b513b6 100644--- a/plugins/_browser/webui/config.html+++ b/plugins/_browser/webui/config.html@@ -243,6 +243,37 @@</span></label>+ <label class="browser-config-field">+ <span class="browser-config-field-label">Keyboard layout</span>+ <input+ type="text"+ x-model="$store.browserConfig.config.keyboard_layout"+ placeholder="de, fr, us, ..."+ autocomplete="off"+ />+ <span class="browser-config-field-help">+ XKB layout of the interactive Browser window, e.g. de for a German keyboard. Match your physical keyboard so characters like @, umlauts, and brackets land on the printed keys. Empty keeps US.+ <a href="https://man.archlinux.org/man/xkeyboard-config.7#LAYOUTS" target="_blank" rel="noreferrer">Full layout list</a>. Changing it restarts the internal browser on save.+ </span>+ </label>++ <label+ class="browser-config-field"+ x-show="$store.browserConfig.config.keyboard_layout"+ >+ <span class="browser-config-field-label">Keyboard variant</span>+ <input+ type="text"+ x-model="$store.browserConfig.config.keyboard_variant"+ placeholder="mac, nodeadkeys, colemak, ..."+ autocomplete="off"+ />+ <span class="browser-config-field-help">+ XKB variant for the layout above. German Mac keyboards use mac, so Option+L types @.+ <a href="https://man.archlinux.org/man/xkeyboard-config.7#LAYOUTS" target="_blank" rel="noreferrer">Variants per layout</a>. Changing it restarts the internal browser on save.+ </span>+ </label>+<label class="browser-config-switch-row"><span class="browser-config-switch-copy"><span class="browser-config-field-label">Autofocus active page</span>diff --git a/tests/test_browser_agent_regressions.py b/tests/test_browser_agent_regressions.pyindex c63ccae3a..7da3b37ef 100644--- a/tests/test_browser_agent_regressions.py+++ b/tests/test_browser_agent_regressions.py@@ -209,10 +209,58 @@ def test_browser_config_normalizes_extension_paths(tmp_path):"proxy_bypass": "","proxy_username": "","proxy_password": "",+ "keyboard_layout": "",+ "keyboard_variant": "","model_preset": "",}+def test_browser_config_normalizes_keyboard_layout():+ config = normalize_browser_config(+ {+ "keyboard_layout": " DE ",+ "keyboard_variant": " Mac ",+ }+ )++ assert config["keyboard_layout"] == "de"+ assert config["keyboard_variant"] == "mac"++ cleared = normalize_browser_config(+ {+ "keyboard_layout": "",+ "keyboard_variant": "mac",+ }+ )++ assert cleared["keyboard_layout"] == ""+ assert cleared["keyboard_variant"] == ""++ unsafe = normalize_browser_config(+ {+ "keyboard_layout": "de; rm -rf /",+ "keyboard_variant": "mac & echo pwned",+ }+ )++ for token in (unsafe["keyboard_layout"], unsafe["keyboard_variant"]):+ assert token == token.lower()+ for forbidden in (" ", ";", "&", "|", "$", "`", "'", '"'):+ assert forbidden not in token+++def test_browser_keyboard_layout_restarts_runtime():+ from plugins._browser.helpers.config import browser_runtime_config++ base = browser_runtime_config({"keyboard_layout": "de", "keyboard_variant": "mac"})++ assert base["keyboard_layout"] == "de"+ assert base["keyboard_variant"] == "mac"++ changed = browser_runtime_config({"keyboard_layout": "us", "keyboard_variant": "mac"})+ assert changed != base++def test_browser_config_normalizes_model_preset():assert normalize_browser_config({"model_preset": " Research "})["model_preset"] == "Research"assert "model" not in normalize_browser_config({"model": "main"})@@ -2466,6 +2514,67 @@ def test_browser_startup_migration_prepares_current_playwright_binary():assert "PrintStyle.warning" in extension+++def test_browser_interactive_view_keyboard_options(monkeypatch):+ import plugins._browser.helpers.interactive_view as iv_module++ view = BrowserInteractiveView("ctx-kb")++ # no layout -> no xpra keyboard args+ monkeypatch.setattr(+ iv_module, "keyboard_options", lambda: {"layout": "", "variant": ""}+ )+ assert view._keyboard_xpra_args() == []++ # layout + variant -> pinned server layout, sync disabled+ monkeypatch.setattr(+ iv_module,+ "keyboard_options",+ lambda: {"layout": "de", "variant": "mac"},+ )+ assert view._keyboard_xpra_args() == [+ "--keyboard-sync=no",+ "--keyboard-layout", "de",+ "--keyboard-variant", "mac",+ ]+++def test_browser_interactive_view_applies_keyboard_layout(monkeypatch):+ commands = []++ def fake_run(command, **kwargs):+ commands.append(list(command))+ return SimpleNamespace(returncode=0)++ monkeypatch.setattr(browser_interactive_view_module.subprocess, "run", fake_run)+ monkeypatch.setattr(+ browser_interactive_view_module.shutil,+ "which",+ lambda name: "/usr/bin/setxkbmap" if name == "setxkbmap" else None,+ )+ monkeypatch.setattr(+ browser_interactive_view_module,+ "keyboard_options",+ lambda: {"layout": "de", "variant": "mac"},+ )++ view = BrowserInteractiveView("ctx-kb-apply")+ view.display = 42+ view._apply_keyboard_layout()++ assert commands == [["/usr/bin/setxkbmap", "-display", ":42", "-layout", "de", "-variant", "mac"]]++ # no configured layout -> no setxkbmap call+ monkeypatch.setattr(+ browser_interactive_view_module,+ "keyboard_options",+ lambda: {"layout": "", "variant": ""},+ )+ view._apply_keyboard_layout()+ assert len(commands) == 1++def test_browser_interactive_views_use_isolated_loopback_sessions(monkeypatch, tmp_path):class FakeProcess:def __init__(self):