"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/
Python
feat(openclaw): add visual_anchor backend, recorder, semantic_ui completion, gemini assist
haorui-harry37ae1604a7b9
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.py b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.pyindex 46ff8bbe6..33e1c9b44 100644--- a/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.py+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.py@@ -1,15 +1,28 @@-"""SemanticUIBackend — drive applications via accessibility and keyboard shortcuts.+"""SemanticUIBackend — drive applications via accessibility APIs and keyboard shortcuts.-This backend provides stubs for AT-SPI (Linux), Windows UIA, and macOS-accessibility APIs. Full implementations require platform-specific libraries-(pyatspi2, pywinauto, pyobjc-framework-Accessibility).+Backends by platform:+ Linux: AT-SPI via python3-pyatspi (apt install python3-pyatspi)+ Fallback: xdotool for keyboard/shortcuts+ macOS: ApplicationServices / Quartz via pyobjc+ Fallback: osascript (AppleScript)+ Windows: UI Automation via pywinauto (pip install pywinauto)-Example macro step:+Action space:++ shortcut — send keyboard shortcut to focused window+ type_text — type text into focused control+ menu_click — activate a menu item by path+ button_click — click a button by label/role+ wait_for_window — wait for a window with given title to appear+ focus_window — bring a window to foreground+ get_controls — list interactive controls (for discovery)++Example YAML steps:- backend: semantic_uiaction: menu_clickparams:- menu_path: [File, Export As, PNG]+ menu_path: [File, Export As, PNG Image]- backend: semantic_uiaction: shortcut@@ -21,16 +34,196 @@ Example macro step:params:title_contains: Exporttimeout_ms: 5000++ - backend: semantic_ui+ action: button_click+ params:+ label: OK++ - backend: semantic_ui+ action: focus_window+ params:+ title_contains: Inkscape++ - backend: semantic_ui+ action: get_controls+ params:+ window_title: Inkscape"""from __future__ import annotationsimport platform+import shutil+import subprocessimport time+from typing import Optionalfrom cli_anything.openclaw.backends.base import Backend, BackendContext, StepResultfrom cli_anything.openclaw.core.macro_model import MacroStep, substitute+_SYSTEM = platform.system()+++# ── AT-SPI helpers (Linux) ────────────────────────────────────────────────────++def _atspi_available() -> bool:+ try:+ import pyatspi # noqa: F401+ return True+ except ImportError:+ return False+++def _atspi_find_app(name_fragment: str):+ """Return the first AT-SPI application matching name_fragment."""+ import pyatspi+ desktop = pyatspi.Registry.getDesktop(0)+ for app in desktop:+ if app and name_fragment.lower() in (app.name or "").lower():+ return app+ return None+++def _atspi_find_control(root, role_name: str, label_fragment: str, max_depth: int = 20):+ """BFS search for a control by role and label."""+ import pyatspi+ role_map = {+ "button": pyatspi.ROLE_PUSH_BUTTON,+ "menu": pyatspi.ROLE_MENU,+ "menu_item": pyatspi.ROLE_MENU_ITEM,+ "menu_bar": pyatspi.ROLE_MENU_BAR,+ "text": pyatspi.ROLE_TEXT,+ "combo_box": pyatspi.ROLE_COMBO_BOX,+ "check_box": pyatspi.ROLE_CHECK_BOX,+ "radio": pyatspi.ROLE_RADIO_BUTTON,+ "list_item": pyatspi.ROLE_LIST_ITEM,+ "dialog": pyatspi.ROLE_DIALOG,+ "window": pyatspi.ROLE_FRAME,+ }+ target_role = role_map.get(role_name.lower())++ from collections import deque+ queue = deque([(root, 0)])+ while queue:+ node, depth = queue.popleft()+ if depth > max_depth:+ continue+ try:+ node_role = node.getRole()+ node_name = node.name or ""+ if (target_role is None or node_role == target_role):+ if label_fragment.lower() in node_name.lower():+ return node+ for i in range(node.childCount):+ child = node.getChildAtIndex(i)+ if child:+ queue.append((child, depth + 1))+ except Exception:+ continue+ return None+++def _atspi_menu_path(app, menu_path: list[str]):+ """Navigate a menu path and activate the final item."""+ import pyatspi++ # Find the menu bar+ menu_bar = _atspi_find_control(app, "menu_bar", "", max_depth=3)+ if menu_bar is None:+ raise RuntimeError("AT-SPI: menu bar not found in application.")++ current = menu_bar+ for label in menu_path:+ item = _atspi_find_control(current, "menu", label)+ if item is None:+ item = _atspi_find_control(current, "menu_item", label)+ if item is None:+ raise RuntimeError(f"AT-SPI: menu item '{label}' not found.")+ # Activate / click+ try:+ action = item.queryAction()+ for i in range(action.nActions):+ if action.getName(i).lower() in ("click", "activate", "open"):+ action.doAction(i)+ break+ except Exception:+ pass+ current = item+ time.sleep(0.15)++ return True+++# ── xdotool helpers (Linux fallback) ─────────────────────────────────────────++def _xdotool_key(keys: str) -> None:+ if not shutil.which("xdotool"):+ raise RuntimeError("xdotool not found. Install with: apt install xdotool")+ # ctrl+shift+e → ctrl+shift+e (xdotool accepts this format directly)+ subprocess.run(["xdotool", "key", "--clearmodifiers", keys], check=True)+++def _xdotool_type(text: str) -> None:+ if not shutil.which("xdotool"):+ raise RuntimeError("xdotool not found. Install with: apt install xdotool")+ subprocess.run(["xdotool", "type", "--clearmodifiers", "--delay", "30", text], check=True)+++def _xdotool_focus(title: str) -> None:+ if not shutil.which("xdotool"):+ raise RuntimeError("xdotool not found. Install with: apt install xdotool")+ subprocess.run(+ ["xdotool", "search", "--name", title, "windowfocus", "--sync"],+ check=True,+ )+++# ── osascript helpers (macOS) ─────────────────────────────────────────────────++def _osascript(script: str) -> str:+ r = subprocess.run(+ ["osascript", "-e", script], capture_output=True, text=True+ )+ if r.returncode != 0:+ raise RuntimeError(f"osascript failed: {r.stderr.strip()}")+ return r.stdout.strip()+++def _macos_menu_click(app_name: str, menu_path: list[str]) -> None:+ if len(menu_path) < 2:+ raise ValueError("menu_path needs at least 2 elements (menu name + item).")+ menu_name = menu_path[0]+ items = menu_path[1:]+ # Build nested AppleScript path+ item_script = " of menu ".join(+ [f'menu item "{i}"' for i in reversed(items)]+ )+ script = f"""+ tell application "{app_name}"+ activate+ end tell+ tell application "System Events"+ tell process "{app_name}"+ click {item_script} of menu "{menu_name}" of menu bar 1+ end tell+ end tell+ """+ _osascript(script)+++# ── pywinauto helpers (Windows) ───────────────────────────────────────────────++def _win_find_app(title_fragment: str):+ from pywinauto import Application, findwindows+ handles = findwindows.find_windows(title_re=f".*{title_fragment}.*")+ if not handles:+ raise RuntimeError(f"Window not found: '{title_fragment}'")+ app = Application().connect(handle=handles[0])+ return app.window(handle=handles[0])+++# ── Backend ───────────────────────────────────────────────────────────────────class SemanticUIBackend(Backend):"""Drive applications through semantic (accessibility) controls."""@@ -41,7 +234,7 @@ class SemanticUIBackend(Backend):def execute(self, step: MacroStep, params: dict, context: BackendContext) -> StepResult:t0 = time.time()action = step.action- step_params = substitute(step.params, params)+ p = substitute(step.params, params)if context.dry_run:return StepResult(@@ -52,37 +245,33 @@ class SemanticUIBackend(Backend):)dispatch = {- "shortcut": self._shortcut,- "menu_click": self._menu_click,+ "shortcut": self._shortcut,+ "type_text": self._type_text,+ "menu_click": self._menu_click,+ "button_click": self._button_click,"wait_for_window": self._wait_for_window,- "button_click": self._button_click,- "type_text": self._type_text,+ "focus_window": self._focus_window,+ "get_controls": self._get_controls,}handler = dispatch.get(action)if handler is None:return StepResult(success=False,- error=f"SemanticUIBackend: unknown action '{action}'.",+ error=f"SemanticUIBackend: unknown action '{action}'. "+ f"Available: {sorted(dispatch)}",backend_used=self.name,duration_ms=(time.time() - t0) * 1000,)try:- output = handler(step_params)+ output = handler(p, context)return StepResult(success=True,output=output or {},backend_used=self.name,duration_ms=(time.time() - t0) * 1000,)- except NotImplementedError as exc:- return StepResult(- success=False,- error=str(exc),- backend_used=self.name,- duration_ms=(time.time() - t0) * 1000,- )except Exception as exc:return StepResult(success=False,@@ -92,111 +281,298 @@ class SemanticUIBackend(Backend):)def is_available(self) -> bool:- # Available when at least one accessibility library can be found- if platform.system() == "Linux":- try:- import pyatspi # noqa: F401- return True- except ImportError:- pass- elif platform.system() == "Windows":+ if _SYSTEM == "Linux":+ return _atspi_available() or bool(shutil.which("xdotool"))+ elif _SYSTEM == "Darwin":+ return bool(shutil.which("osascript"))+ elif _SYSTEM == "Windows":try:import pywinauto # noqa: F401return Trueexcept ImportError:- pass- elif platform.system() == "Darwin":- try:- import ApplicationServices # noqa: F401- return True- except ImportError:- pass+ return Falsereturn False- # ── Actions ──────────────────────────────────────────────────────────+ # ── shortcut ─────────────────────────────────────────────────────────────- def _shortcut(self, p: dict) -> dict:- """Send a keyboard shortcut to the focused window."""+ def _shortcut(self, p: dict, context: BackendContext) -> dict:keys: str = p.get("keys", "")if not keys:- raise ValueError("shortcut action requires 'keys' param.")-- sys = platform.system()- if sys == "Linux":- return self._shortcut_xdotool(keys)- elif sys == "Windows":- return self._shortcut_win32(keys)- elif sys == "Darwin":- return self._shortcut_macos(keys)- raise NotImplementedError(- f"SemanticUIBackend.shortcut: not yet implemented for platform {sys}. "- "Consider using native_api backend instead."- )+ raise ValueError("shortcut requires 'keys' param.")- def _shortcut_xdotool(self, keys: str) -> dict:- """Use xdotool to send keys on Linux."""- import shutil- if not shutil.which("xdotool"):- raise NotImplementedError(- "xdotool not found. Install with: apt install xdotool"- )- import subprocess- # Convert from ctrl+shift+e → ctrl+shift+e (xdotool format)- xdg_keys = keys.replace("+", " ")- subprocess.run(["xdotool", "key", xdg_keys], check=True)- return {"keys": keys, "method": "xdotool"}+ if _SYSTEM == "Linux":+ _xdotool_key(keys)+ return {"keys": keys, "method": "xdotool"}- def _shortcut_win32(self, keys: str) -> dict:- raise NotImplementedError(- "SemanticUIBackend.shortcut: Windows requires pywinauto. "- "pip install pywinauto"- )+ elif _SYSTEM == "Darwin":+ # Use pynput (cross-platform) or AppleScript key code+ from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackend+ from cli_anything.openclaw.core.macro_model import MacroStep as MS+ va = VisualAnchorBackend()+ step = MS(id="x", backend="visual_anchor", action="hotkey", params={"keys": keys})+ result = va._hotkey({"keys": keys}, context)+ return result- def _shortcut_macos(self, keys: str) -> dict:- raise NotImplementedError(- "SemanticUIBackend.shortcut: macOS requires pyobjc. "- "pip install pyobjc-framework-Quartz"- )+ elif _SYSTEM == "Windows":+ import pywinauto.keyboard as kb+ # Convert ctrl+s → {VK_CONTROL}s+ kb.send_keys(keys.replace("+", ""))+ return {"keys": keys, "method": "pywinauto"}++ raise NotImplementedError(f"shortcut not implemented for {_SYSTEM}")++ # ── type_text ─────────────────────────────────────────────────────────────++ def _type_text(self, p: dict, context: BackendContext) -> dict:+ text: str = p.get("text", "")+ if not text:+ raise ValueError("type_text requires 'text' param.")++ if _SYSTEM == "Linux":+ _xdotool_type(text)+ return {"typed": len(text), "method": "xdotool"}++ # macOS / Windows: fall through to visual_anchor type_text+ from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackend+ va = VisualAnchorBackend()+ return va._type_text(p, context)- def _menu_click(self, p: dict) -> dict:- """Click a menu item by path."""+ # ── menu_click ────────────────────────────────────────────────────────────++ def _menu_click(self, p: dict, context: BackendContext) -> dict:menu_path: list = p.get("menu_path", [])+ app_name: str = p.get("app_name", "")++ if not menu_path:+ raise ValueError("menu_click requires 'menu_path' param (list of strings).")++ if _SYSTEM == "Linux":+ if _atspi_available():+ if not app_name:+ raise ValueError(+ "menu_click on Linux AT-SPI requires 'app_name' param."+ )+ app = _atspi_find_app(app_name)+ if app is None:+ raise RuntimeError(f"AT-SPI: application '{app_name}' not found.")+ _atspi_menu_path(app, menu_path)+ return {"menu_path": menu_path, "method": "at-spi"}+ else:+ raise RuntimeError(+ "menu_click on Linux requires AT-SPI.\n"+ " apt install python3-pyatspi\n"+ " Or use visual_anchor backend instead."+ )++ elif _SYSTEM == "Darwin":+ if not app_name:+ raise ValueError("menu_click on macOS requires 'app_name' param.")+ _macos_menu_click(app_name, menu_path)+ return {"menu_path": menu_path, "method": "osascript"}++ elif _SYSTEM == "Windows":+ if not app_name:+ raise ValueError("menu_click on Windows requires 'app_name' param.")+ win = _win_find_app(app_name)+ # pywinauto menu navigation+ menu = win.menu()+ for item in menu_path:+ menu = menu.item_by_path(item)+ menu.click_input()+ return {"menu_path": menu_path, "method": "pywinauto"}++ raise NotImplementedError(f"menu_click not implemented for {_SYSTEM}")++ # ── button_click ──────────────────────────────────────────────────────────++ def _button_click(self, p: dict, context: BackendContext) -> dict:+ label: str = p.get("label", "")+ app_name: str = p.get("app_name", "")++ if not label:+ raise ValueError("button_click requires 'label' param.")++ if _SYSTEM == "Linux" and _atspi_available():+ if not app_name:+ raise ValueError("button_click on Linux AT-SPI requires 'app_name'.")+ app = _atspi_find_app(app_name)+ if app is None:+ raise RuntimeError(f"AT-SPI: application '{app_name}' not found.")+ btn = _atspi_find_control(app, "button", label)+ if btn is None:+ raise RuntimeError(f"AT-SPI: button '{label}' not found in '{app_name}'.")+ action = btn.queryAction()+ for i in range(action.nActions):+ if action.getName(i).lower() == "click":+ action.doAction(i)+ return {"clicked": label, "method": "at-spi"}+ raise RuntimeError(f"AT-SPI: no click action on button '{label}'.")++ elif _SYSTEM == "Darwin":+ script = f"""+ tell application "System Events"+ click button "{label}" of front window of (first process whose frontmost is true)+ end tell+ """+ _osascript(script)+ return {"clicked": label, "method": "osascript"}++ elif _SYSTEM == "Windows":+ win = _win_find_app(app_name or "")+ win.child_window(title=label, control_type="Button").click_input()+ return {"clicked": label, "method": "pywinauto"}+raise NotImplementedError(- f"SemanticUIBackend.menu_click: not yet implemented. "- f"Menu path: {menu_path}. "- "Use native_api or file_transform backends instead when possible."+ f"button_click not fully implemented for {_SYSTEM} without AT-SPI.\n"+ "Use visual_anchor backend as fallback.")- def _wait_for_window(self, p: dict) -> dict:- """Wait for a window matching title criteria."""- title_contains: str = p.get("title_contains", "")+ # ── wait_for_window ───────────────────────────────────────────────────────++ def _wait_for_window(self, p: dict, context: BackendContext) -> dict:+ title: str = p.get("title_contains", "")timeout_ms: int = int(p.get("timeout_ms", 5000))- raise NotImplementedError(- f"SemanticUIBackend.wait_for_window: not yet implemented. "- f"Looking for: '{title_contains}'. "- "Consider using postconditions (file_exists) to detect completion."- )+ poll_ms: int = int(p.get("poll_ms", 300))- def _button_click(self, p: dict) -> dict:- """Click a UI button by label."""- raise NotImplementedError(- f"SemanticUIBackend.button_click: not yet implemented. "- "Use native_api or file_transform backends instead when possible."+ if not title:+ raise ValueError("wait_for_window requires 'title_contains' param.")++ deadline = time.time() + timeout_ms / 1000.0++ if _SYSTEM == "Linux":+ while time.time() < deadline:+ if shutil.which("wmctrl"):+ r = subprocess.run(+ ["wmctrl", "-l"], capture_output=True, text=True+ )+ if title.lower() in r.stdout.lower():+ return {"found": title, "method": "wmctrl"}+ elif shutil.which("xdotool"):+ r = subprocess.run(+ ["xdotool", "search", "--name", title],+ capture_output=True, text=True,+ )+ if r.returncode == 0 and r.stdout.strip():+ return {"found": title, "method": "xdotool"}+ time.sleep(poll_ms / 1000.0)++ elif _SYSTEM == "Darwin":+ while time.time() < deadline:+ script = f"""+ tell application "System Events"+ set ws to name of every window of every process+ set found to false+ repeat with wlist in ws+ repeat with wname in wlist+ if "{title}" is in (wname as text) then+ set found to true+ end if+ end repeat+ end repeat+ return found+ end tell+ """+ result = _osascript(script)+ if result.lower() == "true":+ return {"found": title, "method": "osascript"}+ time.sleep(poll_ms / 1000.0)++ elif _SYSTEM == "Windows":+ import pywinauto.findwindows as fw+ while time.time() < deadline:+ try:+ handles = fw.find_windows(title_re=f".*{title}.*")+ if handles:+ return {"found": title, "method": "pywinauto"}+ except Exception:+ pass+ time.sleep(poll_ms / 1000.0)++ raise TimeoutError(+ f"wait_for_window: window containing '{title}' did not appear "+ f"within {timeout_ms}ms.")- def _type_text(self, p: dict) -> dict:- """Type text into the focused input field."""- text: str = p.get("text", "")- if not text:- raise ValueError("type_text action requires 'text' param.")- sys = platform.system()- if sys == "Linux":- import shutil- if not shutil.which("xdotool"):- raise NotImplementedError("xdotool not found. apt install xdotool")- import subprocess- subprocess.run(["xdotool", "type", "--clearmodifiers", text], check=True)- return {"text": text, "method": "xdotool"}+ # ── focus_window ──────────────────────────────────────────────────────────++ def _focus_window(self, p: dict, context: BackendContext) -> dict:+ title: str = p.get("title_contains", "")+ if not title:+ raise ValueError("focus_window requires 'title_contains' param.")++ if _SYSTEM == "Linux":+ if shutil.which("wmctrl"):+ subprocess.run(["wmctrl", "-a", title], check=True)+ return {"focused": title, "method": "wmctrl"}+ _xdotool_focus(title)+ return {"focused": title, "method": "xdotool"}++ elif _SYSTEM == "Darwin":+ _osascript(f'tell application "{title}" to activate')+ return {"focused": title, "method": "osascript"}++ elif _SYSTEM == "Windows":+ win = _win_find_app(title)+ win.set_focus()+ return {"focused": title, "method": "pywinauto"}++ raise NotImplementedError(f"focus_window not implemented for {_SYSTEM}")++ # ── get_controls ──────────────────────────────────────────────────────────++ def _get_controls(self, p: dict, context: BackendContext) -> dict:+ """List interactive controls in a window (for macro authoring / discovery)."""+ window_title: str = p.get("window_title", "")+ max_depth: int = int(p.get("max_depth", 5))++ if _SYSTEM == "Linux" and _atspi_available():+ import pyatspi+ app = _atspi_find_app(window_title) if window_title else None+ root = app or pyatspi.Registry.getDesktop(0)+ controls = []+ interactive_roles = {+ pyatspi.ROLE_PUSH_BUTTON,+ pyatspi.ROLE_MENU,+ pyatspi.ROLE_MENU_ITEM,+ pyatspi.ROLE_TEXT,+ pyatspi.ROLE_COMBO_BOX,+ pyatspi.ROLE_CHECK_BOX,+ pyatspi.ROLE_RADIO_BUTTON,+ pyatspi.ROLE_TOGGLE_BUTTON,+ }+ from collections import deque+ queue = deque([(root, 0)])+ while queue:+ node, depth = queue.popleft()+ if depth > max_depth:+ continue+ try:+ if node.getRole() in interactive_roles:+ controls.append({+ "role": node.getRoleName(),+ "name": node.name,+ })+ for i in range(node.childCount):+ child = node.getChildAtIndex(i)+ if child:+ queue.append((child, depth + 1))+ except Exception:+ continue+ return {"controls": controls, "count": len(controls)}++ elif _SYSTEM == "Windows":+ win = _win_find_app(window_title)+ controls = []+ for ctrl in win.descendants():+ try:+ controls.append({+ "role": ctrl.element_info.control_type,+ "name": ctrl.element_info.name,+ })+ except Exception:+ pass+ return {"controls": controls, "count": len(controls)}+raise NotImplementedError(- f"SemanticUIBackend.type_text: not yet implemented for {sys}."+ f"get_controls not implemented for {_SYSTEM} without AT-SPI / pywinauto.")diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/backends/visual_anchor.py b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/visual_anchor.pynew file mode 100644index 000000000..cfecc158d--- /dev/null+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/visual_anchor.py@@ -0,0 +1,579 @@+"""VisualAnchorBackend — find UI elements by image template and interact.++Approach:+ 1. Capture full screen with mss (pure Python, cross-platform)+ 2. Find the template image inside the screenshot using numpy correlation+ 3. Use pynput to click / type / scroll at the discovered coordinates++This backend never uses hardcoded absolute coordinates in macro definitions.+Instead, macros store small PNG templates of the UI elements they want to+interact with, and coordinates are computed at runtime.++Supported actions:++ click_image — find template on screen and click its center+ click_relative — click at (x_pct, y_pct) relative to a named window bounds+ wait_image — wait until template appears on screen+ type_text — type a string (keyboard injection, no coordinates needed)+ hotkey — send a keyboard shortcut+ scroll — scroll at the position of a template image+ capture_region — screenshot a region and save it (for template creation)++Example YAML steps:++ - backend: visual_anchor+ action: click_image+ params:+ template: templates/export_button.png+ confidence: 0.85 # 0..1, lower = more tolerant+ timeout_ms: 5000 # wait this long for the image to appear++ - backend: visual_anchor+ action: click_relative+ params:+ window_title: "Draw.io" # partial window title match+ x_pct: 0.5 # 50% across the window+ y_pct: 0.1 # 10% down the window++ - backend: visual_anchor+ action: type_text+ params:+ text: "output.png"+ interval_ms: 30 # delay between key presses++ - backend: visual_anchor+ action: hotkey+ params:+ keys: ctrl+shift+e # + separated++ - backend: visual_anchor+ action: wait_image+ params:+ template: templates/dialog_ok.png+ timeout_ms: 10000++ - backend: visual_anchor+ action: capture_region+ params:+ output: templates/my_button.png+ x: 100+ y: 200+ width: 80+ height: 30+"""++from __future__ import annotations++import os+import time+from pathlib import Path+from typing import Optional++from cli_anything.openclaw.backends.base import Backend, BackendContext, StepResult+from cli_anything.openclaw.core.macro_model import MacroStep, substitute+++# ── lazy imports (only needed when backend is actually used) ──────────────────++def _require_numpy():+ try:+ import numpy as np+ return np+ except ImportError:+ raise ImportError(+ "numpy is required for the visual_anchor backend.\n"+ " pip install numpy"+ )+++def _require_pil():+ try:+ from PIL import Image+ return Image+ except ImportError:+ raise ImportError(+ "Pillow is required for the visual_anchor backend.\n"+ " pip install Pillow"+ )+++def _require_mss():+ try:+ import mss+ return mss+ except ImportError:+ raise ImportError(+ "mss is required for screen capture.\n"+ " pip install mss"+ )+++def _require_pynput():+ try:+ from pynput import mouse as _m, keyboard as _k+ return _m, _k+ except ImportError:+ raise ImportError(+ "pynput is required for mouse/keyboard control.\n"+ " pip install pynput"+ )+++# ── template matching ─────────────────────────────────────────────────────────++def _load_image_as_array(path: str):+ """Load an image file as a numpy uint8 RGB array."""+ np = _require_numpy()+ Image = _require_pil()+ img = Image.open(path).convert("RGB")+ return np.array(img, dtype=np.uint8)+++def _screenshot_as_array():+ """Capture the full screen and return as numpy RGB array."""+ np = _require_numpy()+ mss = _require_mss()+ Image = _require_pil()+ with mss.mss() as sct:+ # Monitor 1 = first physical monitor (index 0 = all monitors combined)+ monitor = sct.monitors[1]+ raw = sct.grab(monitor)+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")+ return np.array(img, dtype=np.uint8), monitor+++def _find_template(+ screen: "np.ndarray",+ template: "np.ndarray",+ confidence: float = 0.85,+ step: int = 1,+) -> Optional[tuple[int, int, float]]:+ """Find template in screen. Returns (center_x, center_y, score) or None.++ score is 0..1 where 1 = perfect match.+ Confidence threshold: only return match if score >= confidence.+ """+ np = _require_numpy()++ sh, sw = screen.shape[:2]+ th, tw = template.shape[:2]++ if th > sh or tw > sw:+ return None++ screen_f = screen.astype(np.float32)+ tmpl_f = template.astype(np.float32)+ tmpl_norm = tmpl_f - tmpl_f.mean()+ tmpl_std = tmpl_f.std()+ if tmpl_std < 1e-6:+ return None # blank template++ best_score = -1.0+ best_pos: Optional[tuple[int, int]] = None++ for y in range(0, sh - th + 1, step):+ for x in range(0, sw - tw + 1, step):+ region = screen_f[y:y + th, x:x + tw]+ region_norm = region - region.mean()+ region_std = region.std()+ if region_std < 1e-6:+ continue+ score = float(+ (region_norm * tmpl_norm).sum()+ / (th * tw * region_std * tmpl_std)+ )+ if score > best_score:+ best_score = score+ best_pos = (x + tw // 2, y + th // 2)++ if best_pos is None or best_score < confidence:+ return None++ return (best_pos[0], best_pos[1], best_score)+++def _wait_for_template(+ template_array: "np.ndarray",+ confidence: float,+ timeout_ms: int,+ poll_ms: int = 300,+) -> Optional[tuple[int, int, float]]:+ """Poll until template found on screen or timeout. Returns match or None."""+ deadline = time.time() + timeout_ms / 1000.0+ while time.time() < deadline:+ screen, _ = _screenshot_as_array()+ result = _find_template(screen, template_array, confidence)+ if result is not None:+ return result+ time.sleep(poll_ms / 1000.0)+ return None+++# ── window bounds helper ──────────────────────────────────────────────────────++def _get_window_bounds(title_fragment: str) -> Optional[dict]:+ """Return {x, y, width, height} of the first window whose title contains+ title_fragment. Works on Linux (xwininfo + wmctrl) and macOS (AppleScript).+ Returns None if not found or not available.+ """+ import subprocess+ import shutil+ import platform++ system = platform.system()++ if system == "Linux":+ # Try wmctrl first (most reliable)+ if shutil.which("wmctrl"):+ r = subprocess.run(+ ["wmctrl", "-lG"], capture_output=True, text=True+ )+ for line in r.stdout.splitlines():+ parts = line.split(None, 9)+ if len(parts) >= 9 and title_fragment.lower() in parts[-1].lower():+ try:+ # wmctrl -lG: wid desktop x y w h host title+ x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5])+ return {"x": x, "y": y, "width": w, "height": h}+ except ValueError:+ pass+ # Fallback: xwininfo+ if shutil.which("xwininfo"):+ r = subprocess.run(+ ["xwininfo", "-name", title_fragment],+ capture_output=True, text=True+ )+ bounds = {}+ for line in r.stdout.splitlines():+ line = line.strip()+ if "Absolute upper-left X:" in line:+ bounds["x"] = int(line.split()[-1])+ elif "Absolute upper-left Y:" in line:+ bounds["y"] = int(line.split()[-1])+ elif "Width:" in line:+ bounds["width"] = int(line.split()[-1])+ elif "Height:" in line:+ bounds["height"] = int(line.split()[-1])+ if len(bounds) == 4:+ return bounds++ elif system == "Darwin":+ # macOS: use AppleScript to get window position+ script = f"""+ tell application "System Events"+ set ws to every window of every process whose name contains "{title_fragment}"+ if ws is not {{}} then+ set w to item 1 of item 1 of ws+ set p to position of w+ set s to size of w+ return (item 1 of p as text) & "," & (item 2 of p as text) & "," & (item 1 of s as text) & "," & (item 2 of s as text)+ end if+ end tell+ """+ r = subprocess.run(["osascript", "-e", script], capture_output=True, text=True)+ if r.returncode == 0 and r.stdout.strip():+ parts = r.stdout.strip().split(",")+ if len(parts) == 4:+ try:+ return {+ "x": int(parts[0]), "y": int(parts[1]),+ "width": int(parts[2]), "height": int(parts[3])+ }+ except ValueError:+ pass++ return None+++# ── Backend ───────────────────────────────────────────────────────────────────++class VisualAnchorBackend(Backend):+ """Find UI elements by image template and interact with them."""++ name = "visual_anchor"+ priority = 75 # between file_transform(70) and gui_macro(80)++ def execute(self, step: MacroStep, params: dict, context: BackendContext) -> StepResult:+ t0 = time.time()+ action = step.action+ p = substitute(step.params, params)++ if context.dry_run:+ return StepResult(+ success=True,+ output={"dry_run": True, "action": action},+ backend_used=self.name,+ duration_ms=(time.time() - t0) * 1000,+ )++ dispatch = {+ "click_image": self._click_image,+ "click_relative": self._click_relative,+ "wait_image": self._wait_image,+ "type_text": self._type_text,+ "hotkey": self._hotkey,+ "scroll": self._scroll,+ "capture_region": self._capture_region,+ }++ handler = dispatch.get(action)+ if handler is None:+ return StepResult(+ success=False,+ error=f"VisualAnchorBackend: unknown action '{action}'. "+ f"Available: {sorted(dispatch)}",+ backend_used=self.name,+ duration_ms=(time.time() - t0) * 1000,+ )++ try:+ output = handler(p, context)+ return StepResult(+ success=True,+ output=output or {},+ backend_used=self.name,+ duration_ms=(time.time() - t0) * 1000,+ )+ except Exception as exc:+ return StepResult(+ success=False,+ error=f"VisualAnchorBackend.{action}: {exc}",+ backend_used=self.name,+ duration_ms=(time.time() - t0) * 1000,+ )++ def is_available(self) -> bool:+ for pkg in ("mss", "numpy", "PIL", "pynput"):+ try:+ __import__(pkg if pkg != "PIL" else "PIL.Image")+ except ImportError:+ return False+ return True++ # ── Actions ──────────────────────────────────────────────────────────────++ def _click_image(self, p: dict, context: BackendContext) -> dict:+ """Find template on screen and click its center."""+ template_path = p.get("template", "")+ if not template_path or not Path(template_path).is_file():+ raise FileNotFoundError(+ f"Template image not found: '{template_path}'. "+ "Use 'macro record' or 'capture_region' to create one."+ )++ confidence = float(p.get("confidence", 0.85))+ timeout_ms = int(p.get("timeout_ms", 5000))+ button = p.get("button", "left") # left | right | middle+ double = bool(p.get("double", False))++ template_arr = _load_image_as_array(template_path)+ match = _wait_for_template(template_arr, confidence, timeout_ms)++ if match is None:+ raise RuntimeError(+ f"Template not found on screen after {timeout_ms}ms: {template_path} "+ f"(confidence={confidence})"+ )++ cx, cy, score = match+ _mouse_click(cx, cy, button=button, double=double)++ return {+ "clicked_at": [cx, cy],+ "match_score": round(score, 4),+ "template": template_path,+ }++ def _click_relative(self, p: dict, context: BackendContext) -> dict:+ """Click at a fractional position within a named window."""+ title = p.get("window_title", "")+ x_pct = float(p.get("x_pct", 0.5))+ y_pct = float(p.get("y_pct", 0.5))+ button = p.get("button", "left")+ double = bool(p.get("double", False))++ if title:+ bounds = _get_window_bounds(title)+ if bounds is None:+ raise RuntimeError(+ f"Window not found: '{title}'. "+ "Make sure the application is open and the title matches."+ )+ cx = int(bounds["x"] + bounds["width"] * x_pct)+ cy = int(bounds["y"] + bounds["height"] * y_pct)+ else:+ # Relative to full screen+ _, monitor = _screenshot_as_array()+ cx = int(monitor["width"] * x_pct)+ cy = int(monitor["height"] * y_pct)++ _mouse_click(cx, cy, button=button, double=double)+ return {"clicked_at": [cx, cy], "x_pct": x_pct, "y_pct": y_pct}++ def _wait_image(self, p: dict, context: BackendContext) -> dict:+ """Wait until a template image appears on screen."""+ template_path = p.get("template", "")+ if not template_path or not Path(template_path).is_file():+ raise FileNotFoundError(f"Template image not found: '{template_path}'")++ confidence = float(p.get("confidence", 0.85))+ timeout_ms = int(p.get("timeout_ms", 10000))++ template_arr = _load_image_as_array(template_path)+ match = _wait_for_template(template_arr, confidence, timeout_ms)++ if match is None:+ raise RuntimeError(+ f"Template never appeared within {timeout_ms}ms: {template_path}"+ )++ cx, cy, score = match+ return {"found_at": [cx, cy], "match_score": round(score, 4)}++ def _type_text(self, p: dict, context: BackendContext) -> dict:+ """Type a string using keyboard injection."""+ text = p.get("text", "")+ interval_ms = int(p.get("interval_ms", 30))+ if not text:+ raise ValueError("type_text requires 'text' param.")++ _, keyboard_mod = _require_pynput()+ ctrl = keyboard_mod.Controller()++ import time as _time+ for char in text:+ ctrl.press(char)+ ctrl.release(char)+ if interval_ms > 0:+ _time.sleep(interval_ms / 1000.0)++ return {"typed": len(text), "text_preview": text[:40]}++ def _hotkey(self, p: dict, context: BackendContext) -> dict:+ """Send a keyboard shortcut (e.g. ctrl+shift+e)."""+ keys_str = p.get("keys", "")+ if not keys_str:+ raise ValueError("hotkey requires 'keys' param (e.g. 'ctrl+s').")++ _, keyboard_mod = _require_pynput()+ Key = keyboard_mod.Key+ ctrl = keyboard_mod.Controller()++ # Parse keys: ctrl+shift+e → [Key.ctrl, Key.shift, 'e']+ key_objects = []+ for k in keys_str.split("+"):+ k = k.strip().lower()+ # Map common names to pynput Key enum+ mapping = {+ "ctrl": Key.ctrl, "control": Key.ctrl,+ "shift": Key.shift,+ "alt": Key.alt,+ "cmd": Key.cmd, "super": Key.cmd, "win": Key.cmd,+ "enter": Key.enter, "return": Key.enter,+ "tab": Key.tab,+ "esc": Key.esc, "escape": Key.esc,+ "space": Key.space,+ "backspace": Key.backspace,+ "delete": Key.delete,+ "up": Key.up, "down": Key.down,+ "left": Key.left, "right": Key.right,+ "home": Key.home, "end": Key.end,+ "f1": Key.f1, "f2": Key.f2, "f3": Key.f3, "f4": Key.f4,+ "f5": Key.f5, "f6": Key.f6, "f7": Key.f7, "f8": Key.f8,+ "f9": Key.f9, "f10": Key.f10, "f11": Key.f11, "f12": Key.f12,+ }+ if k in mapping:+ key_objects.append(mapping[k])+ elif len(k) == 1:+ key_objects.append(k)+ else:+ raise ValueError(f"Unknown key name: '{k}'")++ # Press all, then release all in reverse+ for k in key_objects:+ ctrl.press(k)+ for k in reversed(key_objects):+ ctrl.release(k)++ return {"hotkey": keys_str}++ def _scroll(self, p: dict, context: BackendContext) -> dict:+ """Scroll at the position of a template image."""+ template_path = p.get("template", "")+ dx = int(p.get("dx", 0))+ dy = int(p.get("dy", -3)) # negative = scroll down+ timeout_ms = int(p.get("timeout_ms", 5000))+ confidence = float(p.get("confidence", 0.85))++ if template_path and Path(template_path).is_file():+ template_arr = _load_image_as_array(template_path)+ match = _wait_for_template(template_arr, confidence, timeout_ms)+ if match is None:+ raise RuntimeError(f"Template not found: {template_path}")+ cx, cy, _ = match+ else:+ # Scroll at current mouse position+ mouse_mod, _ = _require_pynput()+ pos = mouse_mod.Controller().position+ cx, cy = int(pos[0]), int(pos[1])++ mouse_mod, _ = _require_pynput()+ mouse_ctrl = mouse_mod.Controller()+ mouse_ctrl.position = (cx, cy)+ mouse_ctrl.scroll(dx, dy)++ return {"scrolled_at": [cx, cy], "dx": dx, "dy": dy}++ def _capture_region(self, p: dict, context: BackendContext) -> dict:+ """Screenshot a region of the screen and save as a template."""+ output_path = p.get("output", "")+ if not output_path:+ raise ValueError("capture_region requires 'output' param.")++ x = int(p.get("x", 0))+ y = int(p.get("y", 0))+ width = int(p.get("width", 100))+ height = int(p.get("height", 50))++ mss = _require_mss()+ Image = _require_pil()++ Path(output_path).parent.mkdir(parents=True, exist_ok=True)++ with mss.mss() as sct:+ region = {"left": x, "top": y, "width": width, "height": height}+ raw = sct.grab(region)+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")+ img.save(output_path)++ size = Path(output_path).stat().st_size+ return {+ "saved": output_path,+ "region": [x, y, width, height],+ "file_size": size,+ }+++# ── pynput mouse helper ───────────────────────────────────────────────────────++def _mouse_click(x: int, y: int, button: str = "left", double: bool = False):+ """Move mouse to (x, y) and click."""+ mouse_mod, _ = _require_pynput()+ Button = mouse_mod.Button+ ctrl = mouse_mod.Controller()++ btn_map = {+ "left": Button.left,+ "right": Button.right,+ "middle": Button.middle,+ }+ btn = btn_map.get(button.lower(), Button.left)++ ctrl.position = (x, y)+ time.sleep(0.05) # small settle delay+ ctrl.press(btn)+ ctrl.release(btn)+ if double:+ time.sleep(0.08)+ ctrl.press(btn)+ ctrl.release(btn)diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/gemini_assist.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/gemini_assist.pynew file mode 100644index 000000000..623c1dcb6--- /dev/null+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/gemini_assist.py@@ -0,0 +1,441 @@+"""GeminiAssist — use Gemini Vision to generate macro steps from screenshots.++This module is OPTIONAL. It requires:+ pip install google-generativeai++How it works:+ 1. Capture a screenshot of the current screen (or use a provided image)+ 2. Send the image + user goal to Gemini Vision with a strict system prompt+ 3. Gemini returns a JSON array of steps (constrained action space)+ 4. Steps are validated and written as a macro YAML file++The action space Gemini is allowed to produce:++ {"type": "click_image", "description": "...", "confidence": 0.85}+ {"type": "click_relative", "window_title": "...", "x_pct": 0.5, "y_pct": 0.1}+ {"type": "type_text", "text": "..."}+ {"type": "hotkey", "keys": "ctrl+s"}+ {"type": "wait_image", "description": "...", "timeout_ms": 5000}+ {"type": "wait_for_window","title_contains": "...", "timeout_ms": 5000}+ {"type": "menu_click", "app_name": "...", "menu_path": ["File", "Export"]}+ {"type": "scroll", "description": "...", "dy": -3}++Gemini is NOT allowed to:+ - Produce shell commands, Python code, or arbitrary actions+ - Use absolute pixel coordinates+ - Output anything other than the JSON array++The "description" field in click_image / wait_image / scroll tells the user+what template image to capture with 'macro record' or 'capture_region'.++Usage:+ cli-anything-openclaw macro define my_export --assist \+ --goal "Export the current diagram as PNG to /tmp/out.png" \+ --screenshot current # takes a fresh screenshot+ --screenshot /path/to/img.png # use existing image+ --api-key $GEMINI_API_KEY+"""++from __future__ import annotations++import base64+import json+import os+from pathlib import Path+from typing import Optional++try:+ import yaml+except ImportError:+ raise ImportError("PyYAML required: pip install PyYAML")+++# ── Strict system prompt ──────────────────────────────────────────────────────++_SYSTEM_PROMPT = """\+You are a GUI macro step generator. Given a screenshot and a user goal, \+output ONLY a valid JSON array of macro steps.++ALLOWED step types (use EXACTLY these schemas):++1. Click a UI element by visual description (template matching will be used):+ {"type": "click_image", "description": "<what the element looks like>", \+"confidence": 0.85, "timeout_ms": 5000}++2. Click at a fractional position within a named window:+ {"type": "click_relative", "window_title": "<partial title>", \+"x_pct": 0.0-1.0, "y_pct": 0.0-1.0}++3. Type text into the focused field:+ {"type": "type_text", "text": "<text to type>"}++4. Send a keyboard shortcut:+ {"type": "hotkey", "keys": "<key1+key2+...>"}++5. Wait for a visual element to appear:+ {"type": "wait_image", "description": "<what to wait for>", \+"timeout_ms": 5000}++6. Wait for a window with a certain title:+ {"type": "wait_for_window", "title_contains": "<partial title>", \+"timeout_ms": 5000}++7. Click a menu item by path:+ {"type": "menu_click", "app_name": "<app name>", \+"menu_path": ["Menu", "Submenu", "Item"]}++8. Scroll near a visual element:+ {"type": "scroll", "description": "<near what element>", "dy": -3}++STRICT RULES:+- Output RAW JSON ONLY. No markdown, no explanation, no code blocks.+- The output must be a JSON array: [step1, step2, ...]+- NEVER use absolute pixel coordinates (x, y numbers).+- NEVER output shell commands, Python, or any non-JSON content.+- NEVER invent step types not listed above.+- Prefer menu_click and hotkey over click_image when possible.+- For click_image: describe the element clearly so a human can find and \+photograph it.+- Keep the plan minimal: use the fewest steps that achieve the goal.+"""+++# ── Screenshot helpers ────────────────────────────────────────────────────────++def _take_screenshot() -> bytes:+ """Capture the current screen and return as PNG bytes."""+ try:+ import mss+ from PIL import Image+ import io+ with mss.mss() as sct:+ monitor = sct.monitors[1]+ raw = sct.grab(monitor)+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")+ buf = io.BytesIO()+ img.save(buf, format="PNG")+ return buf.getvalue()+ except ImportError:+ raise ImportError("mss and Pillow required: pip install mss Pillow")+++def _load_image_bytes(path: str) -> bytes:+ with open(path, "rb") as f:+ return f.read()+++def _image_to_base64(data: bytes) -> str:+ return base64.b64encode(data).decode("utf-8")+++# ── Step validation ───────────────────────────────────────────────────────────++_ALLOWED_TYPES = {+ "click_image", "click_relative", "type_text", "hotkey",+ "wait_image", "wait_for_window", "menu_click", "scroll",+}++_REQUIRED_FIELDS = {+ "click_image": {"type", "description"},+ "click_relative": {"type", "window_title", "x_pct", "y_pct"},+ "type_text": {"type", "text"},+ "hotkey": {"type", "keys"},+ "wait_image": {"type", "description"},+ "wait_for_window": {"type", "title_contains"},+ "menu_click": {"type", "app_name", "menu_path"},+ "scroll": {"type"},+}+++def _validate_steps(raw_steps: list) -> tuple[list[dict], list[str]]:+ """Validate and sanitize steps from Gemini output.++ Returns (valid_steps, error_messages).+ """+ valid = []+ errors = []++ for i, step in enumerate(raw_steps):+ if not isinstance(step, dict):+ errors.append(f"Step {i}: not a dict, skipped.")+ continue++ stype = step.get("type", "")+ if stype not in _ALLOWED_TYPES:+ errors.append(f"Step {i}: unknown type '{stype}', skipped.")+ continue++ required = _REQUIRED_FIELDS.get(stype, {"type"})+ missing = required - set(step.keys())+ if missing:+ errors.append(f"Step {i} ({stype}): missing fields {missing}, skipped.")+ continue++ # Reject any absolute coordinate fields+ for bad_field in ("x", "y", "px", "pixels"):+ if bad_field in step:+ errors.append(+ f"Step {i} ({stype}): absolute coordinate field '{bad_field}' rejected."+ )+ step.pop(bad_field)++ valid.append(step)++ return valid, errors+++# ── Step → YAML step dict conversion ─────────────────────────────────────────++def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:+ """Convert a validated Gemini step to a macro YAML step dict."""+ stype = step["type"]+ sid = f"step_{index:03d}_{stype}"++ if stype == "click_image":+ return {+ "id": sid,+ "backend": "visual_anchor",+ "action": "click_image",+ "params": {+ "template": f"templates/{index:03d}_{stype}.png",+ "confidence": step.get("confidence", 0.85),+ "timeout_ms": step.get("timeout_ms", 5000),+ "_template_description": step.get("description", ""),+ },+ "on_failure": "fail",+ "_gemini_description": step.get("description", ""),+ }+ elif stype == "click_relative":+ return {+ "id": sid,+ "backend": "visual_anchor",+ "action": "click_relative",+ "params": {+ "window_title": step["window_title"],+ "x_pct": step["x_pct"],+ "y_pct": step["y_pct"],+ },+ "on_failure": "fail",+ }+ elif stype == "type_text":+ return {+ "id": sid,+ "backend": "visual_anchor",+ "action": "type_text",+ "params": {"text": step["text"]},+ "on_failure": "fail",+ }+ elif stype == "hotkey":+ return {+ "id": sid,+ "backend": "visual_anchor",+ "action": "hotkey",+ "params": {"keys": step["keys"]},+ "on_failure": "fail",+ }+ elif stype == "wait_image":+ return {+ "id": sid,+ "backend": "visual_anchor",+ "action": "wait_image",+ "params": {+ "template": f"templates/{index:03d}_{stype}.png",+ "confidence": step.get("confidence", 0.85),+ "timeout_ms": step.get("timeout_ms", 10000),+ "_template_description": step.get("description", ""),+ },+ "on_failure": "fail",+ "_gemini_description": step.get("description", ""),+ }+ elif stype == "wait_for_window":+ return {+ "id": sid,+ "backend": "semantic_ui",+ "action": "wait_for_window",+ "params": {+ "title_contains": step["title_contains"],+ "timeout_ms": step.get("timeout_ms", 5000),+ },+ "on_failure": "fail",+ }+ elif stype == "menu_click":+ return {+ "id": sid,+ "backend": "semantic_ui",+ "action": "menu_click",+ "params": {+ "app_name": step["app_name"],+ "menu_path": step["menu_path"],+ },+ "on_failure": "fail",+ }+ elif stype == "scroll":+ return {+ "id": sid,+ "backend": "visual_anchor",+ "action": "scroll",+ "params": {+ "template": f"templates/{index:03d}_{stype}.png"+ if step.get("description") else "",+ "dy": step.get("dy", -3),+ "dx": step.get("dx", 0),+ "_template_description": step.get("description", ""),+ },+ "on_failure": "fail",+ }+ return {}+++# ── Main API ──────────────────────────────────────────────────────────────────++def generate_macro(+ goal: str,+ macro_name: str,+ screenshot_source: str = "current", # "current" | path to image file+ api_key: Optional[str] = None,+ model: str = "gemini-1.5-flash",+ output_path: Optional[str] = None,+) -> dict:+ """Generate a macro YAML from a user goal and screenshot using Gemini.++ Args:+ goal: Natural language description of what the macro should do.+ macro_name: Name for the generated macro.+ screenshot_source: "current" to take a fresh screenshot, or a+ file path to use an existing image.+ api_key: Gemini API key. Falls back to GEMINI_API_KEY env var.+ model: Gemini model to use.+ output_path: Where to write the YAML file. Defaults to+ <macro_name>.yaml in the current directory.++ Returns:+ dict with keys: yaml_path, steps_count, warnings, raw_steps+ """+ try:+ import google.generativeai as genai+ except ImportError:+ raise ImportError(+ "google-generativeai is required for Gemini assist.\n"+ " pip install google-generativeai"+ )++ # Resolve API key+ key = api_key or os.environ.get("GEMINI_API_KEY", "")+ if not key:+ raise ValueError(+ "Gemini API key required. Pass --api-key or set GEMINI_API_KEY env var.\n"+ "Get a key at: https://aistudio.google.com/app/apikey"+ )++ genai.configure(api_key=key)++ # Get screenshot+ if screenshot_source == "current":+ image_bytes = _take_screenshot()+ else:+ if not Path(screenshot_source).is_file():+ raise FileNotFoundError(f"Screenshot not found: {screenshot_source}")+ image_bytes = _load_image_bytes(screenshot_source)++ # Build prompt+ user_prompt = (+ f"Goal: {goal}\n\n"+ "Generate the minimal sequence of steps to achieve this goal. "+ "Output ONLY the JSON array, nothing else."+ )++ # Call Gemini+ gemini_model = genai.GenerativeModel(+ model_name=model,+ system_instruction=_SYSTEM_PROMPT,+ )++ import PIL.Image+ import io+ img = PIL.Image.open(io.BytesIO(image_bytes))++ response = gemini_model.generate_content([user_prompt, img])+ raw_text = response.text.strip()++ # Strip markdown code fences if model added them despite instructions+ if raw_text.startswith("```"):+ lines = raw_text.split("\n")+ raw_text = "\n".join(+ line for line in lines+ if not line.startswith("```")+ ).strip()++ # Parse JSON+ try:+ raw_steps = json.loads(raw_text)+ except json.JSONDecodeError as e:+ raise ValueError(+ f"Gemini returned invalid JSON: {e}\n"+ f"Raw response (first 500 chars):\n{raw_text[:500]}"+ )++ if not isinstance(raw_steps, list):+ raise ValueError(+ f"Gemini returned non-array JSON (expected list): {type(raw_steps)}"+ )++ # Validate+ valid_steps, warnings = _validate_steps(raw_steps)++ # Convert to YAML step dicts+ yaml_steps = [+ _gemini_step_to_yaml_step(s, i + 1)+ for i, s in enumerate(valid_steps)+ ]++ # Build macro dict+ macro = {+ "name": macro_name,+ "version": "1.0",+ "description": goal,+ "tags": ["generated", "gemini-assist"],+ "parameters": {},+ "preconditions": [],+ "steps": yaml_steps,+ "postconditions": [],+ "outputs": [],+ "agent_hints": {+ "danger_level": "moderate",+ "side_effects": ["gui_interaction"],+ "reversible": False,+ "generated_by": "gemini-assist",+ "model": model,+ },+ }++ # Add note about templates that need to be captured+ templates_needed = [+ {+ "step_id": s["id"],+ "template_path": s["params"].get("template", ""),+ "description": s.get("_gemini_description", ""),+ }+ for s in yaml_steps+ if s.get("params", {}).get("template") and s.get("_gemini_description")+ ]++ if templates_needed:+ macro["_templates_to_capture"] = templates_needed++ # Write YAML+ if output_path is None:+ output_path = f"{macro_name}.yaml"++ Path(output_path).parent.mkdir(parents=True, exist_ok=True)+ Path(output_path).write_text(+ yaml.dump(macro, allow_unicode=True, sort_keys=False, default_flow_style=False),+ encoding="utf-8",+ )++ return {+ "yaml_path": str(Path(output_path).resolve()),+ "steps_count": len(yaml_steps),+ "warnings": warnings,+ "raw_steps": raw_steps,+ "templates_to_capture": templates_needed,+ }diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.pyindex b9b2a6edf..74a1cb2ea 100644--- a/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.py+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.py@@ -169,7 +169,7 @@ class MacroDefinition:errors.append("Macro name is required.")if not self.steps:errors.append(f"Macro '{self.name}' has no steps.")- valid_backends = {"native_api", "file_transform", "semantic_ui", "gui_macro", "recovery"}+ valid_backends = {"native_api", "file_transform", "semantic_ui", "gui_macro", "recovery", "visual_anchor"}for i, step in enumerate(self.steps):if step.backend not in valid_backends:errors.append(diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/recorder.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/recorder.pynew file mode 100644index 000000000..378c3a79d--- /dev/null+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/recorder.py@@ -0,0 +1,404 @@+"""Macro recorder — record GUI interactions and generate macro YAML.++Usage:+ cli-anything-openclaw macro record my_workflow++What it does:+ 1. Starts listening for mouse clicks and keyboard events (pynput)+ 2. On each click: captures a small screenshot region around the click+ point and saves it as a template image+ 3. On each hotkey / type event: records the keystroke+ 4. When the user presses Ctrl+Alt+S (or sends SIGINT): stops recording+ and writes a macro YAML file++The generated macro uses the `visual_anchor` backend for click steps+(template images, not hardcoded coordinates) so it is robust to window+movement and minor layout changes.++Output layout:+ <macro_name>.yaml+ <macro_name>_templates/+ step_001_click.png+ step_002_click.png+ ...+"""++from __future__ import annotations++import os+import sys+import time+import threading+from dataclasses import dataclass, field+from pathlib import Path+from typing import Optional++try:+ import yaml+except ImportError:+ raise ImportError("PyYAML required: pip install PyYAML")+++# ── Step data classes ─────────────────────────────────────────────────────────++@dataclass+class RecordedStep:+ index: int+ kind: str # click | type | hotkey | scroll+ # click fields+ x: int = 0+ y: int = 0+ button: str = "left"+ double: bool = False+ template_path: str = "" # relative path to saved template png+ # type fields+ text: str = ""+ # hotkey fields+ keys: str = ""+ # scroll fields+ dx: int = 0+ dy: int = 0+ # timing+ timestamp: float = field(default_factory=time.time)++ def to_step_dict(self) -> dict:+ """Convert to a macro YAML step dict."""+ if self.kind == "click":+ params: dict = {+ "template": self.template_path,+ "confidence": 0.85,+ "timeout_ms": 5000,+ }+ if self.button != "left":+ params["button"] = self.button+ if self.double:+ params["double"] = True+ return {+ "id": f"step_{self.index:03d}_click",+ "backend": "visual_anchor",+ "action": "click_image",+ "params": params,+ "on_failure": "fail",+ }+ elif self.kind == "type":+ return {+ "id": f"step_{self.index:03d}_type",+ "backend": "visual_anchor",+ "action": "type_text",+ "params": {"text": self.text},+ "on_failure": "fail",+ }+ elif self.kind == "hotkey":+ return {+ "id": f"step_{self.index:03d}_hotkey",+ "backend": "visual_anchor",+ "action": "hotkey",+ "params": {"keys": self.keys},+ "on_failure": "fail",+ }+ elif self.kind == "scroll":+ return {+ "id": f"step_{self.index:03d}_scroll",+ "backend": "visual_anchor",+ "action": "scroll",+ "params": {+ "template": self.template_path or "",+ "dx": self.dx,+ "dy": self.dy,+ },+ "on_failure": "fail",+ }+ return {}+++# ── Template capture ──────────────────────────────────────────────────────────++_TEMPLATE_PADDING = 30 # pixels around click point to capture+++def _capture_template(x: int, y: int, output_path: str) -> bool:+ """Capture a small region around (x, y) and save as PNG template."""+ try:+ import mss+ from PIL import Image++ pad = _TEMPLATE_PADDING+ region = {+ "left": max(0, x - pad),+ "top": max(0, y - pad),+ "width": pad * 2,+ "height": pad * 2,+ }+ Path(output_path).parent.mkdir(parents=True, exist_ok=True)+ with mss.mss() as sct:+ raw = sct.grab(region)+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")+ img.save(output_path)+ return True+ except Exception as e:+ print(f"[recorder] Warning: could not capture template: {e}", file=sys.stderr)+ return False+++# ── Key accumulation helper ───────────────────────────────────────────────────++_MODIFIER_KEYS = frozenset([+ "ctrl_l", "ctrl_r", "shift", "shift_r", "alt_l", "alt_r",+ "alt_gr", "cmd", "cmd_r", "super_l", "super_r",+])++_KEY_NAME_MAP = {+ "ctrl_l": "ctrl", "ctrl_r": "ctrl",+ "shift_r": "shift",+ "alt_l": "alt", "alt_r": "alt", "alt_gr": "alt",+ "cmd_r": "cmd",+ "super_l": "super", "super_r": "super",+}+++def _key_to_str(key) -> str:+ """Convert a pynput Key or KeyCode to a string."""+ try:+ from pynput.keyboard import Key+ if isinstance(key, Key):+ name = key.name # e.g. "ctrl_l", "shift", "f5"+ return _KEY_NAME_MAP.get(name, name)+ # KeyCode+ if hasattr(key, "char") and key.char:+ return key.char+ if hasattr(key, "vk") and key.vk:+ return f"vk{key.vk}"+ except Exception:+ pass+ return str(key)+++# ── Recorder ─────────────────────────────────────────────────────────────────++class MacroRecorder:+ """Records mouse and keyboard events and converts them to macro steps."""++ STOP_HOTKEY = frozenset(["ctrl", "alt", "s"]) # Ctrl+Alt+S to stop++ def __init__(self, macro_name: str, output_dir: str = "."):+ self.macro_name = macro_name+ self.output_dir = Path(output_dir)+ self.templates_dir = self.output_dir / f"{macro_name}_templates"++ self._steps: list[RecordedStep] = []+ self._step_index = 0+ self._pressed_modifiers: set[str] = set()+ self._pending_chars: list[str] = []+ self._last_event_time = time.time()+ self._lock = threading.Lock()+ self._stop_event = threading.Event()++ # Double-click detection+ self._last_click_pos: Optional[tuple[int, int]] = None+ self._last_click_time: float = 0.0+ self._DOUBLE_CLICK_MS = 400++ def _next_index(self) -> int:+ self._step_index += 1+ return self._step_index++ def _flush_pending_chars(self):+ """Accumulate consecutive character presses into a single type step."""+ if self._pending_chars:+ text = "".join(self._pending_chars)+ step = RecordedStep(+ index=self._next_index(),+ kind="type",+ text=text,+ )+ self._steps.append(step)+ self._pending_chars.clear()++ # ── Mouse callbacks ───────────────────────────────────────────────────────++ def on_click(self, x: int, y: int, button, pressed: bool):+ if not pressed:+ return # only record press events++ with self._lock:+ self._flush_pending_chars()++ btn_str = button.name if hasattr(button, "name") else str(button)++ # Detect double click+ now = time.time()+ is_double = (+ self._last_click_pos == (x, y)+ and (now - self._last_click_time) * 1000 < self._DOUBLE_CLICK_MS+ )+ if is_double:+ # Upgrade last click step to double=True+ if self._steps and self._steps[-1].kind == "click":+ self._steps[-1].double = True+ self._last_click_pos = None+ return++ self._last_click_pos = (x, y)+ self._last_click_time = now++ # Save template+ idx = self._next_index()+ template_file = str(+ self.templates_dir / f"step_{idx:03d}_click.png"+ )+ captured = _capture_template(x, y, template_file)++ step = RecordedStep(+ index=idx,+ kind="click",+ x=x,+ y=y,+ button=btn_str,+ template_path=template_file if captured else "",+ )+ self._steps.append(step)+ print(f"[recorder] click #{idx} at ({x},{y}) btn={btn_str}", flush=True)++ def on_scroll(self, x: int, y: int, dx: int, dy: int):+ with self._lock:+ self._flush_pending_chars()+ idx = self._next_index()+ # Capture template near scroll position+ template_file = str(+ self.templates_dir / f"step_{idx:03d}_scroll.png"+ )+ captured = _capture_template(x, y, template_file)++ step = RecordedStep(+ index=idx,+ kind="scroll",+ x=x, y=y,+ dx=dx, dy=dy,+ template_path=template_file if captured else "",+ )+ self._steps.append(step)++ # ── Keyboard callbacks ────────────────────────────────────────────────────++ def on_key_press(self, key):+ key_str = _key_to_str(key)++ # Check stop hotkey+ if key_str.lower() in ("ctrl", "alt"):+ self._pressed_modifiers.add(key_str.lower())+ elif key_str.lower() == "s" and self._pressed_modifiers >= {"ctrl", "alt"}:+ print("\n[recorder] Stop hotkey detected (Ctrl+Alt+S). Stopping...", flush=True)+ self._stop_event.set()+ return False # stop listener++ if key_str in _MODIFIER_KEYS or _KEY_NAME_MAP.get(key_str, key_str) in _MODIFIER_KEYS:+ self._pressed_modifiers.add(_KEY_NAME_MAP.get(key_str, key_str))+ return++ # If modifiers are pressed, it's a hotkey combination+ active_mods = {_KEY_NAME_MAP.get(m, m) for m in self._pressed_modifiers}+ if active_mods:+ with self._lock:+ self._flush_pending_chars()+ combo = "+".join(sorted(active_mods) + [key_str])+ idx = self._next_index()+ step = RecordedStep(index=idx, kind="hotkey", keys=combo)+ self._steps.append(step)+ print(f"[recorder] hotkey #{idx}: {combo}", flush=True)+ else:+ # Regular character — accumulate+ if len(key_str) == 1:+ with self._lock:+ self._pending_chars.append(key_str)+ else:+ # Special key alone (enter, tab, backspace, etc.)+ with self._lock:+ self._flush_pending_chars()+ idx = self._next_index()+ step = RecordedStep(index=idx, kind="hotkey", keys=key_str)+ self._steps.append(step)++ def on_key_release(self, key):+ key_str = _key_to_str(key)+ normalized = _KEY_NAME_MAP.get(key_str, key_str)+ self._pressed_modifiers.discard(normalized)++ # ── Main record loop ──────────────────────────────────────────────────────++ def record(self, timeout_s: Optional[float] = None) -> list[RecordedStep]:+ """Start recording. Blocks until Ctrl+Alt+S or timeout_s seconds."""+ try:+ from pynput import mouse as mouse_mod, keyboard as kb_mod+ except ImportError:+ raise ImportError(+ "pynput is required for recording.\n"+ " pip install pynput"+ )++ self.templates_dir.mkdir(parents=True, exist_ok=True)++ print(+ f"[recorder] Recording '{self.macro_name}'. "+ "Press Ctrl+Alt+S to stop.",+ flush=True,+ )++ mouse_listener = mouse_mod.Listener(+ on_click=self.on_click,+ on_scroll=self.on_scroll,+ )+ kb_listener = kb_mod.Listener(+ on_press=self.on_key_press,+ on_release=self.on_key_release,+ )++ mouse_listener.start()+ kb_listener.start()++ try:+ self._stop_event.wait(timeout=timeout_s)+ except KeyboardInterrupt:+ pass+ finally:+ mouse_listener.stop()+ kb_listener.stop()++ with self._lock:+ self._flush_pending_chars()++ print(f"[recorder] Recorded {len(self._steps)} steps.", flush=True)+ return self._steps++ # ── YAML output ───────────────────────────────────────────────────────────++ def to_yaml(self) -> str:+ """Generate macro YAML from recorded steps."""+ steps = [s.to_step_dict() for s in self._steps if s.to_step_dict()]++ macro = {+ "name": self.macro_name,+ "version": "1.0",+ "description": f"Recorded macro: {self.macro_name}",+ "tags": ["recorded", "visual_anchor"],+ "parameters": {},+ "preconditions": [],+ "steps": steps,+ "postconditions": [],+ "outputs": [],+ "agent_hints": {+ "danger_level": "moderate",+ "side_effects": ["gui_interaction"],+ "reversible": False,+ "recorded": True,+ },+ }+ return yaml.dump(macro, allow_unicode=True, sort_keys=False, default_flow_style=False)++ def save(self, output_path: Optional[str] = None) -> str:+ """Write the generated YAML to a file. Returns the path."""+ if output_path is None:+ output_path = str(self.output_dir / f"{self.macro_name}.yaml")+ Path(output_path).parent.mkdir(parents=True, exist_ok=True)+ Path(output_path).write_text(self.to_yaml(), encoding="utf-8")+ print(f"[recorder] Saved macro to: {output_path}", flush=True)+ return output_pathdiff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.pyindex ccac3b0ce..cf198e744 100644--- a/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.py+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.py@@ -3,6 +3,7 @@Priority order (higher = preferred):native_api 100gui_macro 80+ visual_anchor 75file_transform 70semantic_ui 50recovery 10 (only via explicit backend: recovery, or auto-retry)@@ -19,6 +20,7 @@ from cli_anything.openclaw.backends.native_api import NativeAPIBackendfrom cli_anything.openclaw.backends.file_transform import FileTransformBackendfrom cli_anything.openclaw.backends.semantic_ui import SemanticUIBackendfrom cli_anything.openclaw.backends.gui_macro import GUIMacroBackend+from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackendfrom cli_anything.openclaw.backends.recovery import RecoveryBackendfrom cli_anything.openclaw.core.macro_model import MacroStep@@ -26,6 +28,7 @@ from cli_anything.openclaw.core.macro_model import MacroStep_BACKEND_PRIORITY: dict[str, int] = {"native_api": 100,"gui_macro": 80,+ "visual_anchor": 75,"file_transform": 70,"semantic_ui": 50,"recovery": 10,@@ -42,6 +45,7 @@ class RoutingEngine:"file_transform": FileTransformBackend(),"semantic_ui": SemanticUIBackend(),"gui_macro": GUIMacroBackend(),+ "visual_anchor": VisualAnchorBackend(),"recovery": self._recovery,}# Wire recovery with all other backends so it can delegatediff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py b/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.pyindex 1dba50274..48a062e7f 100644--- a/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py@@ -400,6 +400,169 @@ def macro_define(name, output):click.echo(template)[email protected]("record")[email protected]("name")[email protected]("--output-dir", "-d", default=".",+ help="Directory to write <name>.yaml and <name>_templates/.")[email protected]("--timeout", default=0, type=float,+ help="Auto-stop after N seconds (0 = wait for Ctrl+Alt+S).")+@handle_error+def macro_record(name, output_dir, timeout):+ """Record GUI interactions and generate a macro YAML.++ \b+ Start recording, perform your GUI workflow, then press Ctrl+Alt+S to stop.+ Each mouse click is saved with a screenshot template so the macro is+ robust to window position changes.++ Requires: pip install mss Pillow pynput+ """+ try:+ from cli_anything.openclaw.core.recorder import MacroRecorder+ except ImportError as e:+ click.echo(f"Error: {e}", err=True)+ sys.exit(1)++ recorder = MacroRecorder(macro_name=name, output_dir=output_dir)++ if not _json_output:+ click.echo(f"Recording '{name}'. Press Ctrl+Alt+S to stop...")++ try:+ recorder.record(timeout_s=timeout if timeout > 0 else None)+ yaml_path = recorder.save()+ except Exception as e:+ if _json_output:+ output({"error": str(e), "success": False})+ else:+ click.echo(f"Error during recording: {e}", err=True)+ if not _repl_mode:+ sys.exit(1)+ return++ if _json_output:+ output({+ "success": True,+ "yaml_path": yaml_path,+ "steps": len(recorder._steps),+ "templates_dir": str(recorder.templates_dir),+ })+ else:+ click.echo(f"✓ Saved {len(recorder._steps)} steps to: {yaml_path}")+ click.echo(f" Templates: {recorder.templates_dir}/")+ click.echo(+ "\n Next: register the macro in macro_definitions/manifest.yaml\n"+ " or run it directly with: macro run --macro-file " + yaml_path+ )++[email protected]("assist")[email protected]("name")[email protected]("--goal", "-g", required=True,+ help="Natural language goal (what the macro should do).")[email protected]("--screenshot", default="current",+ help="'current' to take a screenshot now, or path to an image file.")[email protected]("--output", "-o", default=None,+ help="Output YAML file path (default: <name>.yaml).")[email protected]("--api-key", default=None, envvar="GEMINI_API_KEY",+ help="Gemini API key (or set GEMINI_API_KEY env var).")[email protected]("--model", default="gemini-1.5-flash",+ help="Gemini model name.")+@handle_error+def macro_assist(name, goal, screenshot, output, api_key, model):+ """Generate a macro YAML from a screenshot using Gemini Vision (optional).++ \b+ Takes a screenshot, sends it to Gemini with your goal, and generates+ a macro YAML. Steps that require visual templates will include+ instructions for which template images to capture.++ Requires: pip install google-generativeai mss Pillow++ \b+ Example:+ macro assist export_png \\+ --goal "Export the current diagram as PNG to /tmp/out.png" \\+ --api-key $GEMINI_API_KEY+ """+ try:+ from cli_anything.openclaw.core.gemini_assist import generate_macro+ except ImportError as e:+ click.echo(f"Error: {e}", err=True)+ sys.exit(1)++ if not _json_output:+ click.echo(f"Sending screenshot to Gemini ({model})...")++ result = generate_macro(+ goal=goal,+ macro_name=name,+ screenshot_source=screenshot,+ api_key=api_key,+ model=model,+ output_path=output,+ )++ if _json_output:+ output(result)+ else:+ click.echo(f"✓ Generated {result['steps_count']} steps → {result['yaml_path']}")+ if result["warnings"]:+ for w in result["warnings"]:+ click.echo(f" ⚠ {w}")+ if result.get("templates_to_capture"):+ click.echo("\n Templates to capture (use 'macro capture-template'):")+ for t in result["templates_to_capture"]:+ click.echo(f" {t['template_path']}: {t['description']}")++[email protected]("capture-template")[email protected]("output_path")[email protected]("--x", type=int, required=True, help="Left edge of region.")[email protected]("--y", type=int, required=True, help="Top edge of region.")[email protected]("--width", type=int, required=True, help="Region width in pixels.")[email protected]("--height", type=int, required=True, help="Region height in pixels.")+@handle_error+def macro_capture_template(output_path, x, y, width, height):+ """Capture a screen region and save it as a template image.++ \b+ Use this to create the template PNG files that visual_anchor macros need.++ \b+ Example:+ macro capture-template templates/export_button.png \\+ --x 245 --y 110 --width 80 --height 30++ Requires: pip install mss Pillow+ """+ try:+ from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackend+ from cli_anything.openclaw.backends.base import BackendContext+ from cli_anything.openclaw.core.macro_model import MacroStep+ except ImportError as e:+ click.echo(f"Error: {e}", err=True)+ sys.exit(1)++ va = VisualAnchorBackend()+ step = MacroStep(id="capture", backend="visual_anchor", action="capture_region",+ params={"output": output_path, "x": x, "y": y,+ "width": width, "height": height})+ ctx = BackendContext(params={})+ result = va.execute(step, {}, ctx)++ if _json_output:+ output(result.to_dict())+ else:+ if result.success:+ click.echo(f"✓ Template saved: {result.output.get('saved')}")+ click.echo(f" Size: {result.output.get('file_size', 0)} bytes")+ else:+ click.echo(f"✗ {result.error}", err=True)+ if not _repl_mode:+ sys.exit(1)++# ── session group ─────────────────────────────────────────────────────────────@cli.group()diff --git a/openclaw-skill/agent-harness/setup.py b/openclaw-skill/agent-harness/setup.pyindex 69c962550..c3df342e0 100644--- a/openclaw-skill/agent-harness/setup.py+++ b/openclaw-skill/agent-harness/setup.py@@ -44,6 +44,24 @@ setup("pytest>=7.0.0","pytest-cov>=4.0.0",],+ "visual": [+ "mss>=9.0.0",+ "Pillow>=10.0.0",+ "numpy>=1.24.0",+ "pynput>=1.7.0",+ ],+ "gemini": [+ "google-generativeai>=0.8.0",+ "mss>=9.0.0",+ "Pillow>=10.0.0",+ ],+ "all": [+ "mss>=9.0.0",+ "Pillow>=10.0.0",+ "numpy>=1.24.0",+ "pynput>=1.7.0",+ "google-generativeai>=0.8.0",+ ],},entry_points={"console_scripts": [