Skip to content

"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/

Python

feat(openclaw): add visual_anchor backend, recorder, semantic_ui completion, gemini assist

haorui-harry37ae1604a7b9
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.py b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.py
index 46ff8bbe6..33e1c9b44 100644
--- a/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.py
+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/semantic_ui.py
@@ -1,15 +1,28 @@
-"""SemanticUIBackend — drive applications via accessibility and keyboard shortcuts.
+"""SemanticUIBackend — drive applications via accessibility APIs and keyboard shortcuts.
-This backend provides stubs for AT-SPI (Linux), Windows UIA, and macOS
-accessibility APIs. Full implementations require platform-specific libraries
-(pyatspi2, pywinauto, pyobjc-framework-Accessibility).
+Backends by platform:
+ Linux: AT-SPI via python3-pyatspi (apt install python3-pyatspi)
+ Fallback: xdotool for keyboard/shortcuts
+ macOS: ApplicationServices / Quartz via pyobjc
+ Fallback: osascript (AppleScript)
+ Windows: UI Automation via pywinauto (pip install pywinauto)
-Example macro step:
+Action space:
+
+ shortcut — send keyboard shortcut to focused window
+ type_text — type text into focused control
+ menu_click — activate a menu item by path
+ button_click — click a button by label/role
+ wait_for_window — wait for a window with given title to appear
+ focus_window — bring a window to foreground
+ get_controls — list interactive controls (for discovery)
+
+Example YAML steps:
- backend: semantic_ui
action: menu_click
params:
- menu_path: [File, Export As, PNG]
+ menu_path: [File, Export As, PNG Image]
- backend: semantic_ui
action: shortcut
@@ -21,16 +34,196 @@ Example macro step:
params:
title_contains: Export
timeout_ms: 5000
+
+ - backend: semantic_ui
+ action: button_click
+ params:
+ label: OK
+
+ - backend: semantic_ui
+ action: focus_window
+ params:
+ title_contains: Inkscape
+
+ - backend: semantic_ui
+ action: get_controls
+ params:
+ window_title: Inkscape
"""
from __future__ import annotations
import platform
+import shutil
+import subprocess
import time
+from typing import Optional
from cli_anything.openclaw.backends.base import Backend, BackendContext, StepResult
from cli_anything.openclaw.core.macro_model import MacroStep, substitute
+_SYSTEM = platform.system()
+
+
+# ── AT-SPI helpers (Linux) ────────────────────────────────────────────────────
+
+def _atspi_available() -> bool:
+ try:
+ import pyatspi # noqa: F401
+ return True
+ except ImportError:
+ return False
+
+
+def _atspi_find_app(name_fragment: str):
+ """Return the first AT-SPI application matching name_fragment."""
+ import pyatspi
+ desktop = pyatspi.Registry.getDesktop(0)
+ for app in desktop:
+ if app and name_fragment.lower() in (app.name or "").lower():
+ return app
+ return None
+
+
+def _atspi_find_control(root, role_name: str, label_fragment: str, max_depth: int = 20):
+ """BFS search for a control by role and label."""
+ import pyatspi
+ role_map = {
+ "button": pyatspi.ROLE_PUSH_BUTTON,
+ "menu": pyatspi.ROLE_MENU,
+ "menu_item": pyatspi.ROLE_MENU_ITEM,
+ "menu_bar": pyatspi.ROLE_MENU_BAR,
+ "text": pyatspi.ROLE_TEXT,
+ "combo_box": pyatspi.ROLE_COMBO_BOX,
+ "check_box": pyatspi.ROLE_CHECK_BOX,
+ "radio": pyatspi.ROLE_RADIO_BUTTON,
+ "list_item": pyatspi.ROLE_LIST_ITEM,
+ "dialog": pyatspi.ROLE_DIALOG,
+ "window": pyatspi.ROLE_FRAME,
+ }
+ target_role = role_map.get(role_name.lower())
+
+ from collections import deque
+ queue = deque([(root, 0)])
+ while queue:
+ node, depth = queue.popleft()
+ if depth > max_depth:
+ continue
+ try:
+ node_role = node.getRole()
+ node_name = node.name or ""
+ if (target_role is None or node_role == target_role):
+ if label_fragment.lower() in node_name.lower():
+ return node
+ for i in range(node.childCount):
+ child = node.getChildAtIndex(i)
+ if child:
+ queue.append((child, depth + 1))
+ except Exception:
+ continue
+ return None
+
+
+def _atspi_menu_path(app, menu_path: list[str]):
+ """Navigate a menu path and activate the final item."""
+ import pyatspi
+
+ # Find the menu bar
+ menu_bar = _atspi_find_control(app, "menu_bar", "", max_depth=3)
+ if menu_bar is None:
+ raise RuntimeError("AT-SPI: menu bar not found in application.")
+
+ current = menu_bar
+ for label in menu_path:
+ item = _atspi_find_control(current, "menu", label)
+ if item is None:
+ item = _atspi_find_control(current, "menu_item", label)
+ if item is None:
+ raise RuntimeError(f"AT-SPI: menu item '{label}' not found.")
+ # Activate / click
+ try:
+ action = item.queryAction()
+ for i in range(action.nActions):
+ if action.getName(i).lower() in ("click", "activate", "open"):
+ action.doAction(i)
+ break
+ except Exception:
+ pass
+ current = item
+ time.sleep(0.15)
+
+ return True
+
+
+# ── xdotool helpers (Linux fallback) ─────────────────────────────────────────
+
+def _xdotool_key(keys: str) -> None:
+ if not shutil.which("xdotool"):
+ raise RuntimeError("xdotool not found. Install with: apt install xdotool")
+ # ctrl+shift+e → ctrl+shift+e (xdotool accepts this format directly)
+ subprocess.run(["xdotool", "key", "--clearmodifiers", keys], check=True)
+
+
+def _xdotool_type(text: str) -> None:
+ if not shutil.which("xdotool"):
+ raise RuntimeError("xdotool not found. Install with: apt install xdotool")
+ subprocess.run(["xdotool", "type", "--clearmodifiers", "--delay", "30", text], check=True)
+
+
+def _xdotool_focus(title: str) -> None:
+ if not shutil.which("xdotool"):
+ raise RuntimeError("xdotool not found. Install with: apt install xdotool")
+ subprocess.run(
+ ["xdotool", "search", "--name", title, "windowfocus", "--sync"],
+ check=True,
+ )
+
+
+# ── osascript helpers (macOS) ─────────────────────────────────────────────────
+
+def _osascript(script: str) -> str:
+ r = subprocess.run(
+ ["osascript", "-e", script], capture_output=True, text=True
+ )
+ if r.returncode != 0:
+ raise RuntimeError(f"osascript failed: {r.stderr.strip()}")
+ return r.stdout.strip()
+
+
+def _macos_menu_click(app_name: str, menu_path: list[str]) -> None:
+ if len(menu_path) < 2:
+ raise ValueError("menu_path needs at least 2 elements (menu name + item).")
+ menu_name = menu_path[0]
+ items = menu_path[1:]
+ # Build nested AppleScript path
+ item_script = " of menu ".join(
+ [f'menu item "{i}"' for i in reversed(items)]
+ )
+ script = f"""
+ tell application "{app_name}"
+ activate
+ end tell
+ tell application "System Events"
+ tell process "{app_name}"
+ click {item_script} of menu "{menu_name}" of menu bar 1
+ end tell
+ end tell
+ """
+ _osascript(script)
+
+
+# ── pywinauto helpers (Windows) ───────────────────────────────────────────────
+
+def _win_find_app(title_fragment: str):
+ from pywinauto import Application, findwindows
+ handles = findwindows.find_windows(title_re=f".*{title_fragment}.*")
+ if not handles:
+ raise RuntimeError(f"Window not found: '{title_fragment}'")
+ app = Application().connect(handle=handles[0])
+ return app.window(handle=handles[0])
+
+
+# ── Backend ───────────────────────────────────────────────────────────────────
class SemanticUIBackend(Backend):
"""Drive applications through semantic (accessibility) controls."""
@@ -41,7 +234,7 @@ class SemanticUIBackend(Backend):
def execute(self, step: MacroStep, params: dict, context: BackendContext) -> StepResult:
t0 = time.time()
action = step.action
- step_params = substitute(step.params, params)
+ p = substitute(step.params, params)
if context.dry_run:
return StepResult(
@@ -52,37 +245,33 @@ class SemanticUIBackend(Backend):
)
dispatch = {
- "shortcut": self._shortcut,
- "menu_click": self._menu_click,
+ "shortcut": self._shortcut,
+ "type_text": self._type_text,
+ "menu_click": self._menu_click,
+ "button_click": self._button_click,
"wait_for_window": self._wait_for_window,
- "button_click": self._button_click,
- "type_text": self._type_text,
+ "focus_window": self._focus_window,
+ "get_controls": self._get_controls,
}
handler = dispatch.get(action)
if handler is None:
return StepResult(
success=False,
- error=f"SemanticUIBackend: unknown action '{action}'.",
+ error=f"SemanticUIBackend: unknown action '{action}'. "
+ f"Available: {sorted(dispatch)}",
backend_used=self.name,
duration_ms=(time.time() - t0) * 1000,
)
try:
- output = handler(step_params)
+ output = handler(p, context)
return StepResult(
success=True,
output=output or {},
backend_used=self.name,
duration_ms=(time.time() - t0) * 1000,
)
- except NotImplementedError as exc:
- return StepResult(
- success=False,
- error=str(exc),
- backend_used=self.name,
- duration_ms=(time.time() - t0) * 1000,
- )
except Exception as exc:
return StepResult(
success=False,
@@ -92,111 +281,298 @@ class SemanticUIBackend(Backend):
)
def is_available(self) -> bool:
- # Available when at least one accessibility library can be found
- if platform.system() == "Linux":
- try:
- import pyatspi # noqa: F401
- return True
- except ImportError:
- pass
- elif platform.system() == "Windows":
+ if _SYSTEM == "Linux":
+ return _atspi_available() or bool(shutil.which("xdotool"))
+ elif _SYSTEM == "Darwin":
+ return bool(shutil.which("osascript"))
+ elif _SYSTEM == "Windows":
try:
import pywinauto # noqa: F401
return True
except ImportError:
- pass
- elif platform.system() == "Darwin":
- try:
- import ApplicationServices # noqa: F401
- return True
- except ImportError:
- pass
+ return False
return False
- # ── Actions ──────────────────────────────────────────────────────────
+ # ── shortcut ─────────────────────────────────────────────────────────────
- def _shortcut(self, p: dict) -> dict:
- """Send a keyboard shortcut to the focused window."""
+ def _shortcut(self, p: dict, context: BackendContext) -> dict:
keys: str = p.get("keys", "")
if not keys:
- raise ValueError("shortcut action requires 'keys' param.")
-
- sys = platform.system()
- if sys == "Linux":
- return self._shortcut_xdotool(keys)
- elif sys == "Windows":
- return self._shortcut_win32(keys)
- elif sys == "Darwin":
- return self._shortcut_macos(keys)
- raise NotImplementedError(
- f"SemanticUIBackend.shortcut: not yet implemented for platform {sys}. "
- "Consider using native_api backend instead."
- )
+ raise ValueError("shortcut requires 'keys' param.")
- def _shortcut_xdotool(self, keys: str) -> dict:
- """Use xdotool to send keys on Linux."""
- import shutil
- if not shutil.which("xdotool"):
- raise NotImplementedError(
- "xdotool not found. Install with: apt install xdotool"
- )
- import subprocess
- # Convert from ctrl+shift+e → ctrl+shift+e (xdotool format)
- xdg_keys = keys.replace("+", " ")
- subprocess.run(["xdotool", "key", xdg_keys], check=True)
- return {"keys": keys, "method": "xdotool"}
+ if _SYSTEM == "Linux":
+ _xdotool_key(keys)
+ return {"keys": keys, "method": "xdotool"}
- def _shortcut_win32(self, keys: str) -> dict:
- raise NotImplementedError(
- "SemanticUIBackend.shortcut: Windows requires pywinauto. "
- "pip install pywinauto"
- )
+ elif _SYSTEM == "Darwin":
+ # Use pynput (cross-platform) or AppleScript key code
+ from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackend
+ from cli_anything.openclaw.core.macro_model import MacroStep as MS
+ va = VisualAnchorBackend()
+ step = MS(id="x", backend="visual_anchor", action="hotkey", params={"keys": keys})
+ result = va._hotkey({"keys": keys}, context)
+ return result
- def _shortcut_macos(self, keys: str) -> dict:
- raise NotImplementedError(
- "SemanticUIBackend.shortcut: macOS requires pyobjc. "
- "pip install pyobjc-framework-Quartz"
- )
+ elif _SYSTEM == "Windows":
+ import pywinauto.keyboard as kb
+ # Convert ctrl+s → {VK_CONTROL}s
+ kb.send_keys(keys.replace("+", ""))
+ return {"keys": keys, "method": "pywinauto"}
+
+ raise NotImplementedError(f"shortcut not implemented for {_SYSTEM}")
+
+ # ── type_text ─────────────────────────────────────────────────────────────
+
+ def _type_text(self, p: dict, context: BackendContext) -> dict:
+ text: str = p.get("text", "")
+ if not text:
+ raise ValueError("type_text requires 'text' param.")
+
+ if _SYSTEM == "Linux":
+ _xdotool_type(text)
+ return {"typed": len(text), "method": "xdotool"}
+
+ # macOS / Windows: fall through to visual_anchor type_text
+ from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackend
+ va = VisualAnchorBackend()
+ return va._type_text(p, context)
- def _menu_click(self, p: dict) -> dict:
- """Click a menu item by path."""
+ # ── menu_click ────────────────────────────────────────────────────────────
+
+ def _menu_click(self, p: dict, context: BackendContext) -> dict:
menu_path: list = p.get("menu_path", [])
+ app_name: str = p.get("app_name", "")
+
+ if not menu_path:
+ raise ValueError("menu_click requires 'menu_path' param (list of strings).")
+
+ if _SYSTEM == "Linux":
+ if _atspi_available():
+ if not app_name:
+ raise ValueError(
+ "menu_click on Linux AT-SPI requires 'app_name' param."
+ )
+ app = _atspi_find_app(app_name)
+ if app is None:
+ raise RuntimeError(f"AT-SPI: application '{app_name}' not found.")
+ _atspi_menu_path(app, menu_path)
+ return {"menu_path": menu_path, "method": "at-spi"}
+ else:
+ raise RuntimeError(
+ "menu_click on Linux requires AT-SPI.\n"
+ " apt install python3-pyatspi\n"
+ " Or use visual_anchor backend instead."
+ )
+
+ elif _SYSTEM == "Darwin":
+ if not app_name:
+ raise ValueError("menu_click on macOS requires 'app_name' param.")
+ _macos_menu_click(app_name, menu_path)
+ return {"menu_path": menu_path, "method": "osascript"}
+
+ elif _SYSTEM == "Windows":
+ if not app_name:
+ raise ValueError("menu_click on Windows requires 'app_name' param.")
+ win = _win_find_app(app_name)
+ # pywinauto menu navigation
+ menu = win.menu()
+ for item in menu_path:
+ menu = menu.item_by_path(item)
+ menu.click_input()
+ return {"menu_path": menu_path, "method": "pywinauto"}
+
+ raise NotImplementedError(f"menu_click not implemented for {_SYSTEM}")
+
+ # ── button_click ──────────────────────────────────────────────────────────
+
+ def _button_click(self, p: dict, context: BackendContext) -> dict:
+ label: str = p.get("label", "")
+ app_name: str = p.get("app_name", "")
+
+ if not label:
+ raise ValueError("button_click requires 'label' param.")
+
+ if _SYSTEM == "Linux" and _atspi_available():
+ if not app_name:
+ raise ValueError("button_click on Linux AT-SPI requires 'app_name'.")
+ app = _atspi_find_app(app_name)
+ if app is None:
+ raise RuntimeError(f"AT-SPI: application '{app_name}' not found.")
+ btn = _atspi_find_control(app, "button", label)
+ if btn is None:
+ raise RuntimeError(f"AT-SPI: button '{label}' not found in '{app_name}'.")
+ action = btn.queryAction()
+ for i in range(action.nActions):
+ if action.getName(i).lower() == "click":
+ action.doAction(i)
+ return {"clicked": label, "method": "at-spi"}
+ raise RuntimeError(f"AT-SPI: no click action on button '{label}'.")
+
+ elif _SYSTEM == "Darwin":
+ script = f"""
+ tell application "System Events"
+ click button "{label}" of front window of (first process whose frontmost is true)
+ end tell
+ """
+ _osascript(script)
+ return {"clicked": label, "method": "osascript"}
+
+ elif _SYSTEM == "Windows":
+ win = _win_find_app(app_name or "")
+ win.child_window(title=label, control_type="Button").click_input()
+ return {"clicked": label, "method": "pywinauto"}
+
raise NotImplementedError(
- f"SemanticUIBackend.menu_click: not yet implemented. "
- f"Menu path: {menu_path}. "
- "Use native_api or file_transform backends instead when possible."
+ f"button_click not fully implemented for {_SYSTEM} without AT-SPI.\n"
+ "Use visual_anchor backend as fallback."
)
- def _wait_for_window(self, p: dict) -> dict:
- """Wait for a window matching title criteria."""
- title_contains: str = p.get("title_contains", "")
+ # ── wait_for_window ───────────────────────────────────────────────────────
+
+ def _wait_for_window(self, p: dict, context: BackendContext) -> dict:
+ title: str = p.get("title_contains", "")
timeout_ms: int = int(p.get("timeout_ms", 5000))
- raise NotImplementedError(
- f"SemanticUIBackend.wait_for_window: not yet implemented. "
- f"Looking for: '{title_contains}'. "
- "Consider using postconditions (file_exists) to detect completion."
- )
+ poll_ms: int = int(p.get("poll_ms", 300))
- def _button_click(self, p: dict) -> dict:
- """Click a UI button by label."""
- raise NotImplementedError(
- f"SemanticUIBackend.button_click: not yet implemented. "
- "Use native_api or file_transform backends instead when possible."
+ if not title:
+ raise ValueError("wait_for_window requires 'title_contains' param.")
+
+ deadline = time.time() + timeout_ms / 1000.0
+
+ if _SYSTEM == "Linux":
+ while time.time() < deadline:
+ if shutil.which("wmctrl"):
+ r = subprocess.run(
+ ["wmctrl", "-l"], capture_output=True, text=True
+ )
+ if title.lower() in r.stdout.lower():
+ return {"found": title, "method": "wmctrl"}
+ elif shutil.which("xdotool"):
+ r = subprocess.run(
+ ["xdotool", "search", "--name", title],
+ capture_output=True, text=True,
+ )
+ if r.returncode == 0 and r.stdout.strip():
+ return {"found": title, "method": "xdotool"}
+ time.sleep(poll_ms / 1000.0)
+
+ elif _SYSTEM == "Darwin":
+ while time.time() < deadline:
+ script = f"""
+ tell application "System Events"
+ set ws to name of every window of every process
+ set found to false
+ repeat with wlist in ws
+ repeat with wname in wlist
+ if "{title}" is in (wname as text) then
+ set found to true
+ end if
+ end repeat
+ end repeat
+ return found
+ end tell
+ """
+ result = _osascript(script)
+ if result.lower() == "true":
+ return {"found": title, "method": "osascript"}
+ time.sleep(poll_ms / 1000.0)
+
+ elif _SYSTEM == "Windows":
+ import pywinauto.findwindows as fw
+ while time.time() < deadline:
+ try:
+ handles = fw.find_windows(title_re=f".*{title}.*")
+ if handles:
+ return {"found": title, "method": "pywinauto"}
+ except Exception:
+ pass
+ time.sleep(poll_ms / 1000.0)
+
+ raise TimeoutError(
+ f"wait_for_window: window containing '{title}' did not appear "
+ f"within {timeout_ms}ms."
)
- def _type_text(self, p: dict) -> dict:
- """Type text into the focused input field."""
- text: str = p.get("text", "")
- if not text:
- raise ValueError("type_text action requires 'text' param.")
- sys = platform.system()
- if sys == "Linux":
- import shutil
- if not shutil.which("xdotool"):
- raise NotImplementedError("xdotool not found. apt install xdotool")
- import subprocess
- subprocess.run(["xdotool", "type", "--clearmodifiers", text], check=True)
- return {"text": text, "method": "xdotool"}
+ # ── focus_window ──────────────────────────────────────────────────────────
+
+ def _focus_window(self, p: dict, context: BackendContext) -> dict:
+ title: str = p.get("title_contains", "")
+ if not title:
+ raise ValueError("focus_window requires 'title_contains' param.")
+
+ if _SYSTEM == "Linux":
+ if shutil.which("wmctrl"):
+ subprocess.run(["wmctrl", "-a", title], check=True)
+ return {"focused": title, "method": "wmctrl"}
+ _xdotool_focus(title)
+ return {"focused": title, "method": "xdotool"}
+
+ elif _SYSTEM == "Darwin":
+ _osascript(f'tell application "{title}" to activate')
+ return {"focused": title, "method": "osascript"}
+
+ elif _SYSTEM == "Windows":
+ win = _win_find_app(title)
+ win.set_focus()
+ return {"focused": title, "method": "pywinauto"}
+
+ raise NotImplementedError(f"focus_window not implemented for {_SYSTEM}")
+
+ # ── get_controls ──────────────────────────────────────────────────────────
+
+ def _get_controls(self, p: dict, context: BackendContext) -> dict:
+ """List interactive controls in a window (for macro authoring / discovery)."""
+ window_title: str = p.get("window_title", "")
+ max_depth: int = int(p.get("max_depth", 5))
+
+ if _SYSTEM == "Linux" and _atspi_available():
+ import pyatspi
+ app = _atspi_find_app(window_title) if window_title else None
+ root = app or pyatspi.Registry.getDesktop(0)
+ controls = []
+ interactive_roles = {
+ pyatspi.ROLE_PUSH_BUTTON,
+ pyatspi.ROLE_MENU,
+ pyatspi.ROLE_MENU_ITEM,
+ pyatspi.ROLE_TEXT,
+ pyatspi.ROLE_COMBO_BOX,
+ pyatspi.ROLE_CHECK_BOX,
+ pyatspi.ROLE_RADIO_BUTTON,
+ pyatspi.ROLE_TOGGLE_BUTTON,
+ }
+ from collections import deque
+ queue = deque([(root, 0)])
+ while queue:
+ node, depth = queue.popleft()
+ if depth > max_depth:
+ continue
+ try:
+ if node.getRole() in interactive_roles:
+ controls.append({
+ "role": node.getRoleName(),
+ "name": node.name,
+ })
+ for i in range(node.childCount):
+ child = node.getChildAtIndex(i)
+ if child:
+ queue.append((child, depth + 1))
+ except Exception:
+ continue
+ return {"controls": controls, "count": len(controls)}
+
+ elif _SYSTEM == "Windows":
+ win = _win_find_app(window_title)
+ controls = []
+ for ctrl in win.descendants():
+ try:
+ controls.append({
+ "role": ctrl.element_info.control_type,
+ "name": ctrl.element_info.name,
+ })
+ except Exception:
+ pass
+ return {"controls": controls, "count": len(controls)}
+
raise NotImplementedError(
- f"SemanticUIBackend.type_text: not yet implemented for {sys}."
+ f"get_controls not implemented for {_SYSTEM} without AT-SPI / pywinauto."
)
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/backends/visual_anchor.py b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/visual_anchor.py
new file mode 100644
index 000000000..cfecc158d
--- /dev/null
+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/backends/visual_anchor.py
@@ -0,0 +1,579 @@
+"""VisualAnchorBackend — find UI elements by image template and interact.
+
+Approach:
+ 1. Capture full screen with mss (pure Python, cross-platform)
+ 2. Find the template image inside the screenshot using numpy correlation
+ 3. Use pynput to click / type / scroll at the discovered coordinates
+
+This backend never uses hardcoded absolute coordinates in macro definitions.
+Instead, macros store small PNG templates of the UI elements they want to
+interact with, and coordinates are computed at runtime.
+
+Supported actions:
+
+ click_image — find template on screen and click its center
+ click_relative — click at (x_pct, y_pct) relative to a named window bounds
+ wait_image — wait until template appears on screen
+ type_text — type a string (keyboard injection, no coordinates needed)
+ hotkey — send a keyboard shortcut
+ scroll — scroll at the position of a template image
+ capture_region — screenshot a region and save it (for template creation)
+
+Example YAML steps:
+
+ - backend: visual_anchor
+ action: click_image
+ params:
+ template: templates/export_button.png
+ confidence: 0.85 # 0..1, lower = more tolerant
+ timeout_ms: 5000 # wait this long for the image to appear
+
+ - backend: visual_anchor
+ action: click_relative
+ params:
+ window_title: "Draw.io" # partial window title match
+ x_pct: 0.5 # 50% across the window
+ y_pct: 0.1 # 10% down the window
+
+ - backend: visual_anchor
+ action: type_text
+ params:
+ text: "output.png"
+ interval_ms: 30 # delay between key presses
+
+ - backend: visual_anchor
+ action: hotkey
+ params:
+ keys: ctrl+shift+e # + separated
+
+ - backend: visual_anchor
+ action: wait_image
+ params:
+ template: templates/dialog_ok.png
+ timeout_ms: 10000
+
+ - backend: visual_anchor
+ action: capture_region
+ params:
+ output: templates/my_button.png
+ x: 100
+ y: 200
+ width: 80
+ height: 30
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from pathlib import Path
+from typing import Optional
+
+from cli_anything.openclaw.backends.base import Backend, BackendContext, StepResult
+from cli_anything.openclaw.core.macro_model import MacroStep, substitute
+
+
+# ── lazy imports (only needed when backend is actually used) ──────────────────
+
+def _require_numpy():
+ try:
+ import numpy as np
+ return np
+ except ImportError:
+ raise ImportError(
+ "numpy is required for the visual_anchor backend.\n"
+ " pip install numpy"
+ )
+
+
+def _require_pil():
+ try:
+ from PIL import Image
+ return Image
+ except ImportError:
+ raise ImportError(
+ "Pillow is required for the visual_anchor backend.\n"
+ " pip install Pillow"
+ )
+
+
+def _require_mss():
+ try:
+ import mss
+ return mss
+ except ImportError:
+ raise ImportError(
+ "mss is required for screen capture.\n"
+ " pip install mss"
+ )
+
+
+def _require_pynput():
+ try:
+ from pynput import mouse as _m, keyboard as _k
+ return _m, _k
+ except ImportError:
+ raise ImportError(
+ "pynput is required for mouse/keyboard control.\n"
+ " pip install pynput"
+ )
+
+
+# ── template matching ─────────────────────────────────────────────────────────
+
+def _load_image_as_array(path: str):
+ """Load an image file as a numpy uint8 RGB array."""
+ np = _require_numpy()
+ Image = _require_pil()
+ img = Image.open(path).convert("RGB")
+ return np.array(img, dtype=np.uint8)
+
+
+def _screenshot_as_array():
+ """Capture the full screen and return as numpy RGB array."""
+ np = _require_numpy()
+ mss = _require_mss()
+ Image = _require_pil()
+ with mss.mss() as sct:
+ # Monitor 1 = first physical monitor (index 0 = all monitors combined)
+ monitor = sct.monitors[1]
+ raw = sct.grab(monitor)
+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
+ return np.array(img, dtype=np.uint8), monitor
+
+
+def _find_template(
+ screen: "np.ndarray",
+ template: "np.ndarray",
+ confidence: float = 0.85,
+ step: int = 1,
+) -> Optional[tuple[int, int, float]]:
+ """Find template in screen. Returns (center_x, center_y, score) or None.
+
+ score is 0..1 where 1 = perfect match.
+ Confidence threshold: only return match if score >= confidence.
+ """
+ np = _require_numpy()
+
+ sh, sw = screen.shape[:2]
+ th, tw = template.shape[:2]
+
+ if th > sh or tw > sw:
+ return None
+
+ screen_f = screen.astype(np.float32)
+ tmpl_f = template.astype(np.float32)
+ tmpl_norm = tmpl_f - tmpl_f.mean()
+ tmpl_std = tmpl_f.std()
+ if tmpl_std < 1e-6:
+ return None # blank template
+
+ best_score = -1.0
+ best_pos: Optional[tuple[int, int]] = None
+
+ for y in range(0, sh - th + 1, step):
+ for x in range(0, sw - tw + 1, step):
+ region = screen_f[y:y + th, x:x + tw]
+ region_norm = region - region.mean()
+ region_std = region.std()
+ if region_std < 1e-6:
+ continue
+ score = float(
+ (region_norm * tmpl_norm).sum()
+ / (th * tw * region_std * tmpl_std)
+ )
+ if score > best_score:
+ best_score = score
+ best_pos = (x + tw // 2, y + th // 2)
+
+ if best_pos is None or best_score < confidence:
+ return None
+
+ return (best_pos[0], best_pos[1], best_score)
+
+
+def _wait_for_template(
+ template_array: "np.ndarray",
+ confidence: float,
+ timeout_ms: int,
+ poll_ms: int = 300,
+) -> Optional[tuple[int, int, float]]:
+ """Poll until template found on screen or timeout. Returns match or None."""
+ deadline = time.time() + timeout_ms / 1000.0
+ while time.time() < deadline:
+ screen, _ = _screenshot_as_array()
+ result = _find_template(screen, template_array, confidence)
+ if result is not None:
+ return result
+ time.sleep(poll_ms / 1000.0)
+ return None
+
+
+# ── window bounds helper ──────────────────────────────────────────────────────
+
+def _get_window_bounds(title_fragment: str) -> Optional[dict]:
+ """Return {x, y, width, height} of the first window whose title contains
+ title_fragment. Works on Linux (xwininfo + wmctrl) and macOS (AppleScript).
+ Returns None if not found or not available.
+ """
+ import subprocess
+ import shutil
+ import platform
+
+ system = platform.system()
+
+ if system == "Linux":
+ # Try wmctrl first (most reliable)
+ if shutil.which("wmctrl"):
+ r = subprocess.run(
+ ["wmctrl", "-lG"], capture_output=True, text=True
+ )
+ for line in r.stdout.splitlines():
+ parts = line.split(None, 9)
+ if len(parts) >= 9 and title_fragment.lower() in parts[-1].lower():
+ try:
+ # wmctrl -lG: wid desktop x y w h host title
+ x, y, w, h = int(parts[2]), int(parts[3]), int(parts[4]), int(parts[5])
+ return {"x": x, "y": y, "width": w, "height": h}
+ except ValueError:
+ pass
+ # Fallback: xwininfo
+ if shutil.which("xwininfo"):
+ r = subprocess.run(
+ ["xwininfo", "-name", title_fragment],
+ capture_output=True, text=True
+ )
+ bounds = {}
+ for line in r.stdout.splitlines():
+ line = line.strip()
+ if "Absolute upper-left X:" in line:
+ bounds["x"] = int(line.split()[-1])
+ elif "Absolute upper-left Y:" in line:
+ bounds["y"] = int(line.split()[-1])
+ elif "Width:" in line:
+ bounds["width"] = int(line.split()[-1])
+ elif "Height:" in line:
+ bounds["height"] = int(line.split()[-1])
+ if len(bounds) == 4:
+ return bounds
+
+ elif system == "Darwin":
+ # macOS: use AppleScript to get window position
+ script = f"""
+ tell application "System Events"
+ set ws to every window of every process whose name contains "{title_fragment}"
+ if ws is not {{}} then
+ set w to item 1 of item 1 of ws
+ set p to position of w
+ set s to size of w
+ return (item 1 of p as text) & "," & (item 2 of p as text) & "," & (item 1 of s as text) & "," & (item 2 of s as text)
+ end if
+ end tell
+ """
+ r = subprocess.run(["osascript", "-e", script], capture_output=True, text=True)
+ if r.returncode == 0 and r.stdout.strip():
+ parts = r.stdout.strip().split(",")
+ if len(parts) == 4:
+ try:
+ return {
+ "x": int(parts[0]), "y": int(parts[1]),
+ "width": int(parts[2]), "height": int(parts[3])
+ }
+ except ValueError:
+ pass
+
+ return None
+
+
+# ── Backend ───────────────────────────────────────────────────────────────────
+
+class VisualAnchorBackend(Backend):
+ """Find UI elements by image template and interact with them."""
+
+ name = "visual_anchor"
+ priority = 75 # between file_transform(70) and gui_macro(80)
+
+ def execute(self, step: MacroStep, params: dict, context: BackendContext) -> StepResult:
+ t0 = time.time()
+ action = step.action
+ p = substitute(step.params, params)
+
+ if context.dry_run:
+ return StepResult(
+ success=True,
+ output={"dry_run": True, "action": action},
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ dispatch = {
+ "click_image": self._click_image,
+ "click_relative": self._click_relative,
+ "wait_image": self._wait_image,
+ "type_text": self._type_text,
+ "hotkey": self._hotkey,
+ "scroll": self._scroll,
+ "capture_region": self._capture_region,
+ }
+
+ handler = dispatch.get(action)
+ if handler is None:
+ return StepResult(
+ success=False,
+ error=f"VisualAnchorBackend: unknown action '{action}'. "
+ f"Available: {sorted(dispatch)}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ try:
+ output = handler(p, context)
+ return StepResult(
+ success=True,
+ output=output or {},
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+ except Exception as exc:
+ return StepResult(
+ success=False,
+ error=f"VisualAnchorBackend.{action}: {exc}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ def is_available(self) -> bool:
+ for pkg in ("mss", "numpy", "PIL", "pynput"):
+ try:
+ __import__(pkg if pkg != "PIL" else "PIL.Image")
+ except ImportError:
+ return False
+ return True
+
+ # ── Actions ──────────────────────────────────────────────────────────────
+
+ def _click_image(self, p: dict, context: BackendContext) -> dict:
+ """Find template on screen and click its center."""
+ template_path = p.get("template", "")
+ if not template_path or not Path(template_path).is_file():
+ raise FileNotFoundError(
+ f"Template image not found: '{template_path}'. "
+ "Use 'macro record' or 'capture_region' to create one."
+ )
+
+ confidence = float(p.get("confidence", 0.85))
+ timeout_ms = int(p.get("timeout_ms", 5000))
+ button = p.get("button", "left") # left | right | middle
+ double = bool(p.get("double", False))
+
+ template_arr = _load_image_as_array(template_path)
+ match = _wait_for_template(template_arr, confidence, timeout_ms)
+
+ if match is None:
+ raise RuntimeError(
+ f"Template not found on screen after {timeout_ms}ms: {template_path} "
+ f"(confidence={confidence})"
+ )
+
+ cx, cy, score = match
+ _mouse_click(cx, cy, button=button, double=double)
+
+ return {
+ "clicked_at": [cx, cy],
+ "match_score": round(score, 4),
+ "template": template_path,
+ }
+
+ def _click_relative(self, p: dict, context: BackendContext) -> dict:
+ """Click at a fractional position within a named window."""
+ title = p.get("window_title", "")
+ x_pct = float(p.get("x_pct", 0.5))
+ y_pct = float(p.get("y_pct", 0.5))
+ button = p.get("button", "left")
+ double = bool(p.get("double", False))
+
+ if title:
+ bounds = _get_window_bounds(title)
+ if bounds is None:
+ raise RuntimeError(
+ f"Window not found: '{title}'. "
+ "Make sure the application is open and the title matches."
+ )
+ cx = int(bounds["x"] + bounds["width"] * x_pct)
+ cy = int(bounds["y"] + bounds["height"] * y_pct)
+ else:
+ # Relative to full screen
+ _, monitor = _screenshot_as_array()
+ cx = int(monitor["width"] * x_pct)
+ cy = int(monitor["height"] * y_pct)
+
+ _mouse_click(cx, cy, button=button, double=double)
+ return {"clicked_at": [cx, cy], "x_pct": x_pct, "y_pct": y_pct}
+
+ def _wait_image(self, p: dict, context: BackendContext) -> dict:
+ """Wait until a template image appears on screen."""
+ template_path = p.get("template", "")
+ if not template_path or not Path(template_path).is_file():
+ raise FileNotFoundError(f"Template image not found: '{template_path}'")
+
+ confidence = float(p.get("confidence", 0.85))
+ timeout_ms = int(p.get("timeout_ms", 10000))
+
+ template_arr = _load_image_as_array(template_path)
+ match = _wait_for_template(template_arr, confidence, timeout_ms)
+
+ if match is None:
+ raise RuntimeError(
+ f"Template never appeared within {timeout_ms}ms: {template_path}"
+ )
+
+ cx, cy, score = match
+ return {"found_at": [cx, cy], "match_score": round(score, 4)}
+
+ def _type_text(self, p: dict, context: BackendContext) -> dict:
+ """Type a string using keyboard injection."""
+ text = p.get("text", "")
+ interval_ms = int(p.get("interval_ms", 30))
+ if not text:
+ raise ValueError("type_text requires 'text' param.")
+
+ _, keyboard_mod = _require_pynput()
+ ctrl = keyboard_mod.Controller()
+
+ import time as _time
+ for char in text:
+ ctrl.press(char)
+ ctrl.release(char)
+ if interval_ms > 0:
+ _time.sleep(interval_ms / 1000.0)
+
+ return {"typed": len(text), "text_preview": text[:40]}
+
+ def _hotkey(self, p: dict, context: BackendContext) -> dict:
+ """Send a keyboard shortcut (e.g. ctrl+shift+e)."""
+ keys_str = p.get("keys", "")
+ if not keys_str:
+ raise ValueError("hotkey requires 'keys' param (e.g. 'ctrl+s').")
+
+ _, keyboard_mod = _require_pynput()
+ Key = keyboard_mod.Key
+ ctrl = keyboard_mod.Controller()
+
+ # Parse keys: ctrl+shift+e → [Key.ctrl, Key.shift, 'e']
+ key_objects = []
+ for k in keys_str.split("+"):
+ k = k.strip().lower()
+ # Map common names to pynput Key enum
+ mapping = {
+ "ctrl": Key.ctrl, "control": Key.ctrl,
+ "shift": Key.shift,
+ "alt": Key.alt,
+ "cmd": Key.cmd, "super": Key.cmd, "win": Key.cmd,
+ "enter": Key.enter, "return": Key.enter,
+ "tab": Key.tab,
+ "esc": Key.esc, "escape": Key.esc,
+ "space": Key.space,
+ "backspace": Key.backspace,
+ "delete": Key.delete,
+ "up": Key.up, "down": Key.down,
+ "left": Key.left, "right": Key.right,
+ "home": Key.home, "end": Key.end,
+ "f1": Key.f1, "f2": Key.f2, "f3": Key.f3, "f4": Key.f4,
+ "f5": Key.f5, "f6": Key.f6, "f7": Key.f7, "f8": Key.f8,
+ "f9": Key.f9, "f10": Key.f10, "f11": Key.f11, "f12": Key.f12,
+ }
+ if k in mapping:
+ key_objects.append(mapping[k])
+ elif len(k) == 1:
+ key_objects.append(k)
+ else:
+ raise ValueError(f"Unknown key name: '{k}'")
+
+ # Press all, then release all in reverse
+ for k in key_objects:
+ ctrl.press(k)
+ for k in reversed(key_objects):
+ ctrl.release(k)
+
+ return {"hotkey": keys_str}
+
+ def _scroll(self, p: dict, context: BackendContext) -> dict:
+ """Scroll at the position of a template image."""
+ template_path = p.get("template", "")
+ dx = int(p.get("dx", 0))
+ dy = int(p.get("dy", -3)) # negative = scroll down
+ timeout_ms = int(p.get("timeout_ms", 5000))
+ confidence = float(p.get("confidence", 0.85))
+
+ if template_path and Path(template_path).is_file():
+ template_arr = _load_image_as_array(template_path)
+ match = _wait_for_template(template_arr, confidence, timeout_ms)
+ if match is None:
+ raise RuntimeError(f"Template not found: {template_path}")
+ cx, cy, _ = match
+ else:
+ # Scroll at current mouse position
+ mouse_mod, _ = _require_pynput()
+ pos = mouse_mod.Controller().position
+ cx, cy = int(pos[0]), int(pos[1])
+
+ mouse_mod, _ = _require_pynput()
+ mouse_ctrl = mouse_mod.Controller()
+ mouse_ctrl.position = (cx, cy)
+ mouse_ctrl.scroll(dx, dy)
+
+ return {"scrolled_at": [cx, cy], "dx": dx, "dy": dy}
+
+ def _capture_region(self, p: dict, context: BackendContext) -> dict:
+ """Screenshot a region of the screen and save as a template."""
+ output_path = p.get("output", "")
+ if not output_path:
+ raise ValueError("capture_region requires 'output' param.")
+
+ x = int(p.get("x", 0))
+ y = int(p.get("y", 0))
+ width = int(p.get("width", 100))
+ height = int(p.get("height", 50))
+
+ mss = _require_mss()
+ Image = _require_pil()
+
+ Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+
+ with mss.mss() as sct:
+ region = {"left": x, "top": y, "width": width, "height": height}
+ raw = sct.grab(region)
+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
+ img.save(output_path)
+
+ size = Path(output_path).stat().st_size
+ return {
+ "saved": output_path,
+ "region": [x, y, width, height],
+ "file_size": size,
+ }
+
+
+# ── pynput mouse helper ───────────────────────────────────────────────────────
+
+def _mouse_click(x: int, y: int, button: str = "left", double: bool = False):
+ """Move mouse to (x, y) and click."""
+ mouse_mod, _ = _require_pynput()
+ Button = mouse_mod.Button
+ ctrl = mouse_mod.Controller()
+
+ btn_map = {
+ "left": Button.left,
+ "right": Button.right,
+ "middle": Button.middle,
+ }
+ btn = btn_map.get(button.lower(), Button.left)
+
+ ctrl.position = (x, y)
+ time.sleep(0.05) # small settle delay
+ ctrl.press(btn)
+ ctrl.release(btn)
+ if double:
+ time.sleep(0.08)
+ ctrl.press(btn)
+ ctrl.release(btn)
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/gemini_assist.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/gemini_assist.py
new file mode 100644
index 000000000..623c1dcb6
--- /dev/null
+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/gemini_assist.py
@@ -0,0 +1,441 @@
+"""GeminiAssist — use Gemini Vision to generate macro steps from screenshots.
+
+This module is OPTIONAL. It requires:
+ pip install google-generativeai
+
+How it works:
+ 1. Capture a screenshot of the current screen (or use a provided image)
+ 2. Send the image + user goal to Gemini Vision with a strict system prompt
+ 3. Gemini returns a JSON array of steps (constrained action space)
+ 4. Steps are validated and written as a macro YAML file
+
+The action space Gemini is allowed to produce:
+
+ {"type": "click_image", "description": "...", "confidence": 0.85}
+ {"type": "click_relative", "window_title": "...", "x_pct": 0.5, "y_pct": 0.1}
+ {"type": "type_text", "text": "..."}
+ {"type": "hotkey", "keys": "ctrl+s"}
+ {"type": "wait_image", "description": "...", "timeout_ms": 5000}
+ {"type": "wait_for_window","title_contains": "...", "timeout_ms": 5000}
+ {"type": "menu_click", "app_name": "...", "menu_path": ["File", "Export"]}
+ {"type": "scroll", "description": "...", "dy": -3}
+
+Gemini is NOT allowed to:
+ - Produce shell commands, Python code, or arbitrary actions
+ - Use absolute pixel coordinates
+ - Output anything other than the JSON array
+
+The "description" field in click_image / wait_image / scroll tells the user
+what template image to capture with 'macro record' or 'capture_region'.
+
+Usage:
+ cli-anything-openclaw macro define my_export --assist \
+ --goal "Export the current diagram as PNG to /tmp/out.png" \
+ --screenshot current # takes a fresh screenshot
+ --screenshot /path/to/img.png # use existing image
+ --api-key $GEMINI_API_KEY
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+try:
+ import yaml
+except ImportError:
+ raise ImportError("PyYAML required: pip install PyYAML")
+
+
+# ── Strict system prompt ──────────────────────────────────────────────────────
+
+_SYSTEM_PROMPT = """\
+You are a GUI macro step generator. Given a screenshot and a user goal, \
+output ONLY a valid JSON array of macro steps.
+
+ALLOWED step types (use EXACTLY these schemas):
+
+1. Click a UI element by visual description (template matching will be used):
+ {"type": "click_image", "description": "<what the element looks like>", \
+"confidence": 0.85, "timeout_ms": 5000}
+
+2. Click at a fractional position within a named window:
+ {"type": "click_relative", "window_title": "<partial title>", \
+"x_pct": 0.0-1.0, "y_pct": 0.0-1.0}
+
+3. Type text into the focused field:
+ {"type": "type_text", "text": "<text to type>"}
+
+4. Send a keyboard shortcut:
+ {"type": "hotkey", "keys": "<key1+key2+...>"}
+
+5. Wait for a visual element to appear:
+ {"type": "wait_image", "description": "<what to wait for>", \
+"timeout_ms": 5000}
+
+6. Wait for a window with a certain title:
+ {"type": "wait_for_window", "title_contains": "<partial title>", \
+"timeout_ms": 5000}
+
+7. Click a menu item by path:
+ {"type": "menu_click", "app_name": "<app name>", \
+"menu_path": ["Menu", "Submenu", "Item"]}
+
+8. Scroll near a visual element:
+ {"type": "scroll", "description": "<near what element>", "dy": -3}
+
+STRICT RULES:
+- Output RAW JSON ONLY. No markdown, no explanation, no code blocks.
+- The output must be a JSON array: [step1, step2, ...]
+- NEVER use absolute pixel coordinates (x, y numbers).
+- NEVER output shell commands, Python, or any non-JSON content.
+- NEVER invent step types not listed above.
+- Prefer menu_click and hotkey over click_image when possible.
+- For click_image: describe the element clearly so a human can find and \
+photograph it.
+- Keep the plan minimal: use the fewest steps that achieve the goal.
+"""
+
+
+# ── Screenshot helpers ────────────────────────────────────────────────────────
+
+def _take_screenshot() -> bytes:
+ """Capture the current screen and return as PNG bytes."""
+ try:
+ import mss
+ from PIL import Image
+ import io
+ with mss.mss() as sct:
+ monitor = sct.monitors[1]
+ raw = sct.grab(monitor)
+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+ return buf.getvalue()
+ except ImportError:
+ raise ImportError("mss and Pillow required: pip install mss Pillow")
+
+
+def _load_image_bytes(path: str) -> bytes:
+ with open(path, "rb") as f:
+ return f.read()
+
+
+def _image_to_base64(data: bytes) -> str:
+ return base64.b64encode(data).decode("utf-8")
+
+
+# ── Step validation ───────────────────────────────────────────────────────────
+
+_ALLOWED_TYPES = {
+ "click_image", "click_relative", "type_text", "hotkey",
+ "wait_image", "wait_for_window", "menu_click", "scroll",
+}
+
+_REQUIRED_FIELDS = {
+ "click_image": {"type", "description"},
+ "click_relative": {"type", "window_title", "x_pct", "y_pct"},
+ "type_text": {"type", "text"},
+ "hotkey": {"type", "keys"},
+ "wait_image": {"type", "description"},
+ "wait_for_window": {"type", "title_contains"},
+ "menu_click": {"type", "app_name", "menu_path"},
+ "scroll": {"type"},
+}
+
+
+def _validate_steps(raw_steps: list) -> tuple[list[dict], list[str]]:
+ """Validate and sanitize steps from Gemini output.
+
+ Returns (valid_steps, error_messages).
+ """
+ valid = []
+ errors = []
+
+ for i, step in enumerate(raw_steps):
+ if not isinstance(step, dict):
+ errors.append(f"Step {i}: not a dict, skipped.")
+ continue
+
+ stype = step.get("type", "")
+ if stype not in _ALLOWED_TYPES:
+ errors.append(f"Step {i}: unknown type '{stype}', skipped.")
+ continue
+
+ required = _REQUIRED_FIELDS.get(stype, {"type"})
+ missing = required - set(step.keys())
+ if missing:
+ errors.append(f"Step {i} ({stype}): missing fields {missing}, skipped.")
+ continue
+
+ # Reject any absolute coordinate fields
+ for bad_field in ("x", "y", "px", "pixels"):
+ if bad_field in step:
+ errors.append(
+ f"Step {i} ({stype}): absolute coordinate field '{bad_field}' rejected."
+ )
+ step.pop(bad_field)
+
+ valid.append(step)
+
+ return valid, errors
+
+
+# ── Step → YAML step dict conversion ─────────────────────────────────────────
+
+def _gemini_step_to_yaml_step(step: dict, index: int) -> dict:
+ """Convert a validated Gemini step to a macro YAML step dict."""
+ stype = step["type"]
+ sid = f"step_{index:03d}_{stype}"
+
+ if stype == "click_image":
+ return {
+ "id": sid,
+ "backend": "visual_anchor",
+ "action": "click_image",
+ "params": {
+ "template": f"templates/{index:03d}_{stype}.png",
+ "confidence": step.get("confidence", 0.85),
+ "timeout_ms": step.get("timeout_ms", 5000),
+ "_template_description": step.get("description", ""),
+ },
+ "on_failure": "fail",
+ "_gemini_description": step.get("description", ""),
+ }
+ elif stype == "click_relative":
+ return {
+ "id": sid,
+ "backend": "visual_anchor",
+ "action": "click_relative",
+ "params": {
+ "window_title": step["window_title"],
+ "x_pct": step["x_pct"],
+ "y_pct": step["y_pct"],
+ },
+ "on_failure": "fail",
+ }
+ elif stype == "type_text":
+ return {
+ "id": sid,
+ "backend": "visual_anchor",
+ "action": "type_text",
+ "params": {"text": step["text"]},
+ "on_failure": "fail",
+ }
+ elif stype == "hotkey":
+ return {
+ "id": sid,
+ "backend": "visual_anchor",
+ "action": "hotkey",
+ "params": {"keys": step["keys"]},
+ "on_failure": "fail",
+ }
+ elif stype == "wait_image":
+ return {
+ "id": sid,
+ "backend": "visual_anchor",
+ "action": "wait_image",
+ "params": {
+ "template": f"templates/{index:03d}_{stype}.png",
+ "confidence": step.get("confidence", 0.85),
+ "timeout_ms": step.get("timeout_ms", 10000),
+ "_template_description": step.get("description", ""),
+ },
+ "on_failure": "fail",
+ "_gemini_description": step.get("description", ""),
+ }
+ elif stype == "wait_for_window":
+ return {
+ "id": sid,
+ "backend": "semantic_ui",
+ "action": "wait_for_window",
+ "params": {
+ "title_contains": step["title_contains"],
+ "timeout_ms": step.get("timeout_ms", 5000),
+ },
+ "on_failure": "fail",
+ }
+ elif stype == "menu_click":
+ return {
+ "id": sid,
+ "backend": "semantic_ui",
+ "action": "menu_click",
+ "params": {
+ "app_name": step["app_name"],
+ "menu_path": step["menu_path"],
+ },
+ "on_failure": "fail",
+ }
+ elif stype == "scroll":
+ return {
+ "id": sid,
+ "backend": "visual_anchor",
+ "action": "scroll",
+ "params": {
+ "template": f"templates/{index:03d}_{stype}.png"
+ if step.get("description") else "",
+ "dy": step.get("dy", -3),
+ "dx": step.get("dx", 0),
+ "_template_description": step.get("description", ""),
+ },
+ "on_failure": "fail",
+ }
+ return {}
+
+
+# ── Main API ──────────────────────────────────────────────────────────────────
+
+def generate_macro(
+ goal: str,
+ macro_name: str,
+ screenshot_source: str = "current", # "current" | path to image file
+ api_key: Optional[str] = None,
+ model: str = "gemini-1.5-flash",
+ output_path: Optional[str] = None,
+) -> dict:
+ """Generate a macro YAML from a user goal and screenshot using Gemini.
+
+ Args:
+ goal: Natural language description of what the macro should do.
+ macro_name: Name for the generated macro.
+ screenshot_source: "current" to take a fresh screenshot, or a
+ file path to use an existing image.
+ api_key: Gemini API key. Falls back to GEMINI_API_KEY env var.
+ model: Gemini model to use.
+ output_path: Where to write the YAML file. Defaults to
+ <macro_name>.yaml in the current directory.
+
+ Returns:
+ dict with keys: yaml_path, steps_count, warnings, raw_steps
+ """
+ try:
+ import google.generativeai as genai
+ except ImportError:
+ raise ImportError(
+ "google-generativeai is required for Gemini assist.\n"
+ " pip install google-generativeai"
+ )
+
+ # Resolve API key
+ key = api_key or os.environ.get("GEMINI_API_KEY", "")
+ if not key:
+ raise ValueError(
+ "Gemini API key required. Pass --api-key or set GEMINI_API_KEY env var.\n"
+ "Get a key at: https://aistudio.google.com/app/apikey"
+ )
+
+ genai.configure(api_key=key)
+
+ # Get screenshot
+ if screenshot_source == "current":
+ image_bytes = _take_screenshot()
+ else:
+ if not Path(screenshot_source).is_file():
+ raise FileNotFoundError(f"Screenshot not found: {screenshot_source}")
+ image_bytes = _load_image_bytes(screenshot_source)
+
+ # Build prompt
+ user_prompt = (
+ f"Goal: {goal}\n\n"
+ "Generate the minimal sequence of steps to achieve this goal. "
+ "Output ONLY the JSON array, nothing else."
+ )
+
+ # Call Gemini
+ gemini_model = genai.GenerativeModel(
+ model_name=model,
+ system_instruction=_SYSTEM_PROMPT,
+ )
+
+ import PIL.Image
+ import io
+ img = PIL.Image.open(io.BytesIO(image_bytes))
+
+ response = gemini_model.generate_content([user_prompt, img])
+ raw_text = response.text.strip()
+
+ # Strip markdown code fences if model added them despite instructions
+ if raw_text.startswith("```"):
+ lines = raw_text.split("\n")
+ raw_text = "\n".join(
+ line for line in lines
+ if not line.startswith("```")
+ ).strip()
+
+ # Parse JSON
+ try:
+ raw_steps = json.loads(raw_text)
+ except json.JSONDecodeError as e:
+ raise ValueError(
+ f"Gemini returned invalid JSON: {e}\n"
+ f"Raw response (first 500 chars):\n{raw_text[:500]}"
+ )
+
+ if not isinstance(raw_steps, list):
+ raise ValueError(
+ f"Gemini returned non-array JSON (expected list): {type(raw_steps)}"
+ )
+
+ # Validate
+ valid_steps, warnings = _validate_steps(raw_steps)
+
+ # Convert to YAML step dicts
+ yaml_steps = [
+ _gemini_step_to_yaml_step(s, i + 1)
+ for i, s in enumerate(valid_steps)
+ ]
+
+ # Build macro dict
+ macro = {
+ "name": macro_name,
+ "version": "1.0",
+ "description": goal,
+ "tags": ["generated", "gemini-assist"],
+ "parameters": {},
+ "preconditions": [],
+ "steps": yaml_steps,
+ "postconditions": [],
+ "outputs": [],
+ "agent_hints": {
+ "danger_level": "moderate",
+ "side_effects": ["gui_interaction"],
+ "reversible": False,
+ "generated_by": "gemini-assist",
+ "model": model,
+ },
+ }
+
+ # Add note about templates that need to be captured
+ templates_needed = [
+ {
+ "step_id": s["id"],
+ "template_path": s["params"].get("template", ""),
+ "description": s.get("_gemini_description", ""),
+ }
+ for s in yaml_steps
+ if s.get("params", {}).get("template") and s.get("_gemini_description")
+ ]
+
+ if templates_needed:
+ macro["_templates_to_capture"] = templates_needed
+
+ # Write YAML
+ if output_path is None:
+ output_path = f"{macro_name}.yaml"
+
+ Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+ Path(output_path).write_text(
+ yaml.dump(macro, allow_unicode=True, sort_keys=False, default_flow_style=False),
+ encoding="utf-8",
+ )
+
+ return {
+ "yaml_path": str(Path(output_path).resolve()),
+ "steps_count": len(yaml_steps),
+ "warnings": warnings,
+ "raw_steps": raw_steps,
+ "templates_to_capture": templates_needed,
+ }
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.py
index b9b2a6edf..74a1cb2ea 100644
--- a/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.py
+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/macro_model.py
@@ -169,7 +169,7 @@ class MacroDefinition:
errors.append("Macro name is required.")
if not self.steps:
errors.append(f"Macro '{self.name}' has no steps.")
- valid_backends = {"native_api", "file_transform", "semantic_ui", "gui_macro", "recovery"}
+ valid_backends = {"native_api", "file_transform", "semantic_ui", "gui_macro", "recovery", "visual_anchor"}
for i, step in enumerate(self.steps):
if step.backend not in valid_backends:
errors.append(
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/recorder.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/recorder.py
new file mode 100644
index 000000000..378c3a79d
--- /dev/null
+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/recorder.py
@@ -0,0 +1,404 @@
+"""Macro recorder — record GUI interactions and generate macro YAML.
+
+Usage:
+ cli-anything-openclaw macro record my_workflow
+
+What it does:
+ 1. Starts listening for mouse clicks and keyboard events (pynput)
+ 2. On each click: captures a small screenshot region around the click
+ point and saves it as a template image
+ 3. On each hotkey / type event: records the keystroke
+ 4. When the user presses Ctrl+Alt+S (or sends SIGINT): stops recording
+ and writes a macro YAML file
+
+The generated macro uses the `visual_anchor` backend for click steps
+(template images, not hardcoded coordinates) so it is robust to window
+movement and minor layout changes.
+
+Output layout:
+ <macro_name>.yaml
+ <macro_name>_templates/
+ step_001_click.png
+ step_002_click.png
+ ...
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import time
+import threading
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Optional
+
+try:
+ import yaml
+except ImportError:
+ raise ImportError("PyYAML required: pip install PyYAML")
+
+
+# ── Step data classes ─────────────────────────────────────────────────────────
+
+@dataclass
+class RecordedStep:
+ index: int
+ kind: str # click | type | hotkey | scroll
+ # click fields
+ x: int = 0
+ y: int = 0
+ button: str = "left"
+ double: bool = False
+ template_path: str = "" # relative path to saved template png
+ # type fields
+ text: str = ""
+ # hotkey fields
+ keys: str = ""
+ # scroll fields
+ dx: int = 0
+ dy: int = 0
+ # timing
+ timestamp: float = field(default_factory=time.time)
+
+ def to_step_dict(self) -> dict:
+ """Convert to a macro YAML step dict."""
+ if self.kind == "click":
+ params: dict = {
+ "template": self.template_path,
+ "confidence": 0.85,
+ "timeout_ms": 5000,
+ }
+ if self.button != "left":
+ params["button"] = self.button
+ if self.double:
+ params["double"] = True
+ return {
+ "id": f"step_{self.index:03d}_click",
+ "backend": "visual_anchor",
+ "action": "click_image",
+ "params": params,
+ "on_failure": "fail",
+ }
+ elif self.kind == "type":
+ return {
+ "id": f"step_{self.index:03d}_type",
+ "backend": "visual_anchor",
+ "action": "type_text",
+ "params": {"text": self.text},
+ "on_failure": "fail",
+ }
+ elif self.kind == "hotkey":
+ return {
+ "id": f"step_{self.index:03d}_hotkey",
+ "backend": "visual_anchor",
+ "action": "hotkey",
+ "params": {"keys": self.keys},
+ "on_failure": "fail",
+ }
+ elif self.kind == "scroll":
+ return {
+ "id": f"step_{self.index:03d}_scroll",
+ "backend": "visual_anchor",
+ "action": "scroll",
+ "params": {
+ "template": self.template_path or "",
+ "dx": self.dx,
+ "dy": self.dy,
+ },
+ "on_failure": "fail",
+ }
+ return {}
+
+
+# ── Template capture ──────────────────────────────────────────────────────────
+
+_TEMPLATE_PADDING = 30 # pixels around click point to capture
+
+
+def _capture_template(x: int, y: int, output_path: str) -> bool:
+ """Capture a small region around (x, y) and save as PNG template."""
+ try:
+ import mss
+ from PIL import Image
+
+ pad = _TEMPLATE_PADDING
+ region = {
+ "left": max(0, x - pad),
+ "top": max(0, y - pad),
+ "width": pad * 2,
+ "height": pad * 2,
+ }
+ Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+ with mss.mss() as sct:
+ raw = sct.grab(region)
+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
+ img.save(output_path)
+ return True
+ except Exception as e:
+ print(f"[recorder] Warning: could not capture template: {e}", file=sys.stderr)
+ return False
+
+
+# ── Key accumulation helper ───────────────────────────────────────────────────
+
+_MODIFIER_KEYS = frozenset([
+ "ctrl_l", "ctrl_r", "shift", "shift_r", "alt_l", "alt_r",
+ "alt_gr", "cmd", "cmd_r", "super_l", "super_r",
+])
+
+_KEY_NAME_MAP = {
+ "ctrl_l": "ctrl", "ctrl_r": "ctrl",
+ "shift_r": "shift",
+ "alt_l": "alt", "alt_r": "alt", "alt_gr": "alt",
+ "cmd_r": "cmd",
+ "super_l": "super", "super_r": "super",
+}
+
+
+def _key_to_str(key) -> str:
+ """Convert a pynput Key or KeyCode to a string."""
+ try:
+ from pynput.keyboard import Key
+ if isinstance(key, Key):
+ name = key.name # e.g. "ctrl_l", "shift", "f5"
+ return _KEY_NAME_MAP.get(name, name)
+ # KeyCode
+ if hasattr(key, "char") and key.char:
+ return key.char
+ if hasattr(key, "vk") and key.vk:
+ return f"vk{key.vk}"
+ except Exception:
+ pass
+ return str(key)
+
+
+# ── Recorder ─────────────────────────────────────────────────────────────────
+
+class MacroRecorder:
+ """Records mouse and keyboard events and converts them to macro steps."""
+
+ STOP_HOTKEY = frozenset(["ctrl", "alt", "s"]) # Ctrl+Alt+S to stop
+
+ def __init__(self, macro_name: str, output_dir: str = "."):
+ self.macro_name = macro_name
+ self.output_dir = Path(output_dir)
+ self.templates_dir = self.output_dir / f"{macro_name}_templates"
+
+ self._steps: list[RecordedStep] = []
+ self._step_index = 0
+ self._pressed_modifiers: set[str] = set()
+ self._pending_chars: list[str] = []
+ self._last_event_time = time.time()
+ self._lock = threading.Lock()
+ self._stop_event = threading.Event()
+
+ # Double-click detection
+ self._last_click_pos: Optional[tuple[int, int]] = None
+ self._last_click_time: float = 0.0
+ self._DOUBLE_CLICK_MS = 400
+
+ def _next_index(self) -> int:
+ self._step_index += 1
+ return self._step_index
+
+ def _flush_pending_chars(self):
+ """Accumulate consecutive character presses into a single type step."""
+ if self._pending_chars:
+ text = "".join(self._pending_chars)
+ step = RecordedStep(
+ index=self._next_index(),
+ kind="type",
+ text=text,
+ )
+ self._steps.append(step)
+ self._pending_chars.clear()
+
+ # ── Mouse callbacks ───────────────────────────────────────────────────────
+
+ def on_click(self, x: int, y: int, button, pressed: bool):
+ if not pressed:
+ return # only record press events
+
+ with self._lock:
+ self._flush_pending_chars()
+
+ btn_str = button.name if hasattr(button, "name") else str(button)
+
+ # Detect double click
+ now = time.time()
+ is_double = (
+ self._last_click_pos == (x, y)
+ and (now - self._last_click_time) * 1000 < self._DOUBLE_CLICK_MS
+ )
+ if is_double:
+ # Upgrade last click step to double=True
+ if self._steps and self._steps[-1].kind == "click":
+ self._steps[-1].double = True
+ self._last_click_pos = None
+ return
+
+ self._last_click_pos = (x, y)
+ self._last_click_time = now
+
+ # Save template
+ idx = self._next_index()
+ template_file = str(
+ self.templates_dir / f"step_{idx:03d}_click.png"
+ )
+ captured = _capture_template(x, y, template_file)
+
+ step = RecordedStep(
+ index=idx,
+ kind="click",
+ x=x,
+ y=y,
+ button=btn_str,
+ template_path=template_file if captured else "",
+ )
+ self._steps.append(step)
+ print(f"[recorder] click #{idx} at ({x},{y}) btn={btn_str}", flush=True)
+
+ def on_scroll(self, x: int, y: int, dx: int, dy: int):
+ with self._lock:
+ self._flush_pending_chars()
+ idx = self._next_index()
+ # Capture template near scroll position
+ template_file = str(
+ self.templates_dir / f"step_{idx:03d}_scroll.png"
+ )
+ captured = _capture_template(x, y, template_file)
+
+ step = RecordedStep(
+ index=idx,
+ kind="scroll",
+ x=x, y=y,
+ dx=dx, dy=dy,
+ template_path=template_file if captured else "",
+ )
+ self._steps.append(step)
+
+ # ── Keyboard callbacks ────────────────────────────────────────────────────
+
+ def on_key_press(self, key):
+ key_str = _key_to_str(key)
+
+ # Check stop hotkey
+ if key_str.lower() in ("ctrl", "alt"):
+ self._pressed_modifiers.add(key_str.lower())
+ elif key_str.lower() == "s" and self._pressed_modifiers >= {"ctrl", "alt"}:
+ print("\n[recorder] Stop hotkey detected (Ctrl+Alt+S). Stopping...", flush=True)
+ self._stop_event.set()
+ return False # stop listener
+
+ if key_str in _MODIFIER_KEYS or _KEY_NAME_MAP.get(key_str, key_str) in _MODIFIER_KEYS:
+ self._pressed_modifiers.add(_KEY_NAME_MAP.get(key_str, key_str))
+ return
+
+ # If modifiers are pressed, it's a hotkey combination
+ active_mods = {_KEY_NAME_MAP.get(m, m) for m in self._pressed_modifiers}
+ if active_mods:
+ with self._lock:
+ self._flush_pending_chars()
+ combo = "+".join(sorted(active_mods) + [key_str])
+ idx = self._next_index()
+ step = RecordedStep(index=idx, kind="hotkey", keys=combo)
+ self._steps.append(step)
+ print(f"[recorder] hotkey #{idx}: {combo}", flush=True)
+ else:
+ # Regular character — accumulate
+ if len(key_str) == 1:
+ with self._lock:
+ self._pending_chars.append(key_str)
+ else:
+ # Special key alone (enter, tab, backspace, etc.)
+ with self._lock:
+ self._flush_pending_chars()
+ idx = self._next_index()
+ step = RecordedStep(index=idx, kind="hotkey", keys=key_str)
+ self._steps.append(step)
+
+ def on_key_release(self, key):
+ key_str = _key_to_str(key)
+ normalized = _KEY_NAME_MAP.get(key_str, key_str)
+ self._pressed_modifiers.discard(normalized)
+
+ # ── Main record loop ──────────────────────────────────────────────────────
+
+ def record(self, timeout_s: Optional[float] = None) -> list[RecordedStep]:
+ """Start recording. Blocks until Ctrl+Alt+S or timeout_s seconds."""
+ try:
+ from pynput import mouse as mouse_mod, keyboard as kb_mod
+ except ImportError:
+ raise ImportError(
+ "pynput is required for recording.\n"
+ " pip install pynput"
+ )
+
+ self.templates_dir.mkdir(parents=True, exist_ok=True)
+
+ print(
+ f"[recorder] Recording '{self.macro_name}'. "
+ "Press Ctrl+Alt+S to stop.",
+ flush=True,
+ )
+
+ mouse_listener = mouse_mod.Listener(
+ on_click=self.on_click,
+ on_scroll=self.on_scroll,
+ )
+ kb_listener = kb_mod.Listener(
+ on_press=self.on_key_press,
+ on_release=self.on_key_release,
+ )
+
+ mouse_listener.start()
+ kb_listener.start()
+
+ try:
+ self._stop_event.wait(timeout=timeout_s)
+ except KeyboardInterrupt:
+ pass
+ finally:
+ mouse_listener.stop()
+ kb_listener.stop()
+
+ with self._lock:
+ self._flush_pending_chars()
+
+ print(f"[recorder] Recorded {len(self._steps)} steps.", flush=True)
+ return self._steps
+
+ # ── YAML output ───────────────────────────────────────────────────────────
+
+ def to_yaml(self) -> str:
+ """Generate macro YAML from recorded steps."""
+ steps = [s.to_step_dict() for s in self._steps if s.to_step_dict()]
+
+ macro = {
+ "name": self.macro_name,
+ "version": "1.0",
+ "description": f"Recorded macro: {self.macro_name}",
+ "tags": ["recorded", "visual_anchor"],
+ "parameters": {},
+ "preconditions": [],
+ "steps": steps,
+ "postconditions": [],
+ "outputs": [],
+ "agent_hints": {
+ "danger_level": "moderate",
+ "side_effects": ["gui_interaction"],
+ "reversible": False,
+ "recorded": True,
+ },
+ }
+ return yaml.dump(macro, allow_unicode=True, sort_keys=False, default_flow_style=False)
+
+ def save(self, output_path: Optional[str] = None) -> str:
+ """Write the generated YAML to a file. Returns the path."""
+ if output_path is None:
+ output_path = str(self.output_dir / f"{self.macro_name}.yaml")
+ Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+ Path(output_path).write_text(self.to_yaml(), encoding="utf-8")
+ print(f"[recorder] Saved macro to: {output_path}", flush=True)
+ return output_path
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.py b/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.py
index ccac3b0ce..cf198e744 100644
--- a/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.py
+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/core/routing.py
@@ -3,6 +3,7 @@
Priority order (higher = preferred):
native_api 100
gui_macro 80
+ visual_anchor 75
file_transform 70
semantic_ui 50
recovery 10 (only via explicit backend: recovery, or auto-retry)
@@ -19,6 +20,7 @@ from cli_anything.openclaw.backends.native_api import NativeAPIBackend
from cli_anything.openclaw.backends.file_transform import FileTransformBackend
from cli_anything.openclaw.backends.semantic_ui import SemanticUIBackend
from cli_anything.openclaw.backends.gui_macro import GUIMacroBackend
+from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackend
from cli_anything.openclaw.backends.recovery import RecoveryBackend
from cli_anything.openclaw.core.macro_model import MacroStep
@@ -26,6 +28,7 @@ from cli_anything.openclaw.core.macro_model import MacroStep
_BACKEND_PRIORITY: dict[str, int] = {
"native_api": 100,
"gui_macro": 80,
+ "visual_anchor": 75,
"file_transform": 70,
"semantic_ui": 50,
"recovery": 10,
@@ -42,6 +45,7 @@ class RoutingEngine:
"file_transform": FileTransformBackend(),
"semantic_ui": SemanticUIBackend(),
"gui_macro": GUIMacroBackend(),
+ "visual_anchor": VisualAnchorBackend(),
"recovery": self._recovery,
}
# Wire recovery with all other backends so it can delegate
diff --git a/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py b/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py
index 1dba50274..48a062e7f 100644
--- a/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py
+++ b/openclaw-skill/agent-harness/cli_anything/openclaw/openclaw_cli.py
@@ -400,6 +400,169 @@ def macro_define(name, output):
click.echo(template)
[email protected]("--output-dir", "-d", default=".",
+ help="Directory to write <name>.yaml and <name>_templates/.")
[email protected]("--timeout", default=0, type=float,
+ help="Auto-stop after N seconds (0 = wait for Ctrl+Alt+S).")
+@handle_error
+def macro_record(name, output_dir, timeout):
+ """Record GUI interactions and generate a macro YAML.
+
+ \b
+ Start recording, perform your GUI workflow, then press Ctrl+Alt+S to stop.
+ Each mouse click is saved with a screenshot template so the macro is
+ robust to window position changes.
+
+ Requires: pip install mss Pillow pynput
+ """
+ try:
+ from cli_anything.openclaw.core.recorder import MacroRecorder
+ except ImportError as e:
+ click.echo(f"Error: {e}", err=True)
+ sys.exit(1)
+
+ recorder = MacroRecorder(macro_name=name, output_dir=output_dir)
+
+ if not _json_output:
+ click.echo(f"Recording '{name}'. Press Ctrl+Alt+S to stop...")
+
+ try:
+ recorder.record(timeout_s=timeout if timeout > 0 else None)
+ yaml_path = recorder.save()
+ except Exception as e:
+ if _json_output:
+ output({"error": str(e), "success": False})
+ else:
+ click.echo(f"Error during recording: {e}", err=True)
+ if not _repl_mode:
+ sys.exit(1)
+ return
+
+ if _json_output:
+ output({
+ "success": True,
+ "yaml_path": yaml_path,
+ "steps": len(recorder._steps),
+ "templates_dir": str(recorder.templates_dir),
+ })
+ else:
+ click.echo(f"✓ Saved {len(recorder._steps)} steps to: {yaml_path}")
+ click.echo(f" Templates: {recorder.templates_dir}/")
+ click.echo(
+ "\n Next: register the macro in macro_definitions/manifest.yaml\n"
+ " or run it directly with: macro run --macro-file " + yaml_path
+ )
+
+
[email protected]("--goal", "-g", required=True,
+ help="Natural language goal (what the macro should do).")
[email protected]("--screenshot", default="current",
+ help="'current' to take a screenshot now, or path to an image file.")
[email protected]("--output", "-o", default=None,
+ help="Output YAML file path (default: <name>.yaml).")
[email protected]("--api-key", default=None, envvar="GEMINI_API_KEY",
+ help="Gemini API key (or set GEMINI_API_KEY env var).")
[email protected]("--model", default="gemini-1.5-flash",
+ help="Gemini model name.")
+@handle_error
+def macro_assist(name, goal, screenshot, output, api_key, model):
+ """Generate a macro YAML from a screenshot using Gemini Vision (optional).
+
+ \b
+ Takes a screenshot, sends it to Gemini with your goal, and generates
+ a macro YAML. Steps that require visual templates will include
+ instructions for which template images to capture.
+
+ Requires: pip install google-generativeai mss Pillow
+
+ \b
+ Example:
+ macro assist export_png \\
+ --goal "Export the current diagram as PNG to /tmp/out.png" \\
+ --api-key $GEMINI_API_KEY
+ """
+ try:
+ from cli_anything.openclaw.core.gemini_assist import generate_macro
+ except ImportError as e:
+ click.echo(f"Error: {e}", err=True)
+ sys.exit(1)
+
+ if not _json_output:
+ click.echo(f"Sending screenshot to Gemini ({model})...")
+
+ result = generate_macro(
+ goal=goal,
+ macro_name=name,
+ screenshot_source=screenshot,
+ api_key=api_key,
+ model=model,
+ output_path=output,
+ )
+
+ if _json_output:
+ output(result)
+ else:
+ click.echo(f"✓ Generated {result['steps_count']} steps → {result['yaml_path']}")
+ if result["warnings"]:
+ for w in result["warnings"]:
+ click.echo(f" ⚠ {w}")
+ if result.get("templates_to_capture"):
+ click.echo("\n Templates to capture (use 'macro capture-template'):")
+ for t in result["templates_to_capture"]:
+ click.echo(f" {t['template_path']}: {t['description']}")
+
+
[email protected]("capture-template")
[email protected]("output_path")
[email protected]("--x", type=int, required=True, help="Left edge of region.")
[email protected]("--y", type=int, required=True, help="Top edge of region.")
[email protected]("--width", type=int, required=True, help="Region width in pixels.")
[email protected]("--height", type=int, required=True, help="Region height in pixels.")
+@handle_error
+def macro_capture_template(output_path, x, y, width, height):
+ """Capture a screen region and save it as a template image.
+
+ \b
+ Use this to create the template PNG files that visual_anchor macros need.
+
+ \b
+ Example:
+ macro capture-template templates/export_button.png \\
+ --x 245 --y 110 --width 80 --height 30
+
+ Requires: pip install mss Pillow
+ """
+ try:
+ from cli_anything.openclaw.backends.visual_anchor import VisualAnchorBackend
+ from cli_anything.openclaw.backends.base import BackendContext
+ from cli_anything.openclaw.core.macro_model import MacroStep
+ except ImportError as e:
+ click.echo(f"Error: {e}", err=True)
+ sys.exit(1)
+
+ va = VisualAnchorBackend()
+ step = MacroStep(id="capture", backend="visual_anchor", action="capture_region",
+ params={"output": output_path, "x": x, "y": y,
+ "width": width, "height": height})
+ ctx = BackendContext(params={})
+ result = va.execute(step, {}, ctx)
+
+ if _json_output:
+ output(result.to_dict())
+ else:
+ if result.success:
+ click.echo(f"✓ Template saved: {result.output.get('saved')}")
+ click.echo(f" Size: {result.output.get('file_size', 0)} bytes")
+ else:
+ click.echo(f"✗ {result.error}", err=True)
+ if not _repl_mode:
+ sys.exit(1)
+
+
# ── session group ─────────────────────────────────────────────────────────────
@cli.group()
diff --git a/openclaw-skill/agent-harness/setup.py b/openclaw-skill/agent-harness/setup.py
index 69c962550..c3df342e0 100644
--- a/openclaw-skill/agent-harness/setup.py
+++ b/openclaw-skill/agent-harness/setup.py
@@ -44,6 +44,24 @@ setup(
"pytest>=7.0.0",
"pytest-cov>=4.0.0",
],
+ "visual": [
+ "mss>=9.0.0",
+ "Pillow>=10.0.0",
+ "numpy>=1.24.0",
+ "pynput>=1.7.0",
+ ],
+ "gemini": [
+ "google-generativeai>=0.8.0",
+ "mss>=9.0.0",
+ "Pillow>=10.0.0",
+ ],
+ "all": [
+ "mss>=9.0.0",
+ "Pillow>=10.0.0",
+ "numpy>=1.24.0",
+ "pynput>=1.7.0",
+ "google-generativeai>=0.8.0",
+ ],
},
entry_points={
"console_scripts": [