Skip to content

"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/

Python

feat(macrocli): add gui_agent instruct_with_refine, drag via xdotool, kolourpaint demo

haorui-harry960adb58fa30
diff --git a/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
index 361a62b98..e38ce75e6 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
+++ b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
@@ -57,17 +57,17 @@ from cli_anything.macrocli.core.macro_model import MacroStep, substitute
_SYSTEM_PROMPT = """\
You are a GUI automation agent. You will be shown:
1. A screenshot of the CURRENT screen state
-2. A screenshot of the TARGET end state
+2. A screenshot of the TARGET end state (optional)
3. A description of what needs to be accomplished
-Your job is to figure out what single action to take next to move
-from the current state toward the target state.
+Your job is to figure out what single action to take next.
OUTPUT FORMAT: Respond with ONLY a JSON object, one of:
{"action": "click", "x": <int>, "y": <int>, "button": "left"}
{"action": "double_click", "x": <int>, "y": <int>}
{"action": "right_click", "x": <int>, "y": <int>}
+ {"action": "drag", "from_x": <int>, "from_y": <int>, "to_x": <int>, "to_y": <int>, "duration_ms": 300}
{"action": "type", "text": "<string>"}
{"action": "hotkey", "keys": "<key1+key2+...>"}
{"action": "scroll", "x": <int>, "y": <int>, "dy": <int>}
@@ -78,6 +78,7 @@ Use {"action": "done"} ONLY when the current state matches the target state.
RULES:
- Output RAW JSON ONLY. No markdown, no explanation.
- Use pixel coordinates from the CURRENT screenshot.
+- For drag: from_x/from_y is where you start pressing, to_x/to_y is where you release.
- Prefer clicking on visible labeled controls over guessing coordinates.
- If the target state is already achieved, output {"action": "done"}.
- Never output any action not listed above.
@@ -172,6 +173,13 @@ def _execute_action(action_dict: dict, context: BackendContext) -> None:
ctrl.position = (x, y)
ctrl.scroll(0, dy)
+ elif action == "drag":
+ fx, fy = int(action_dict["from_x"]), int(action_dict["from_y"])
+ tx, ty = int(action_dict["to_x"]), int(action_dict["to_y"])
+ duration_ms = int(action_dict.get("duration_ms", 300))
+ from cli_anything.macrocli.backends.visual_anchor import _mouse_drag
+ _mouse_drag(fx, fy, tx, ty, duration_ms=duration_ms)
+
elif action == "done":
pass # caller checks for done
@@ -201,20 +209,22 @@ class GUIAgentBackend(Backend):
duration_ms=(time.time() - t0) * 1000,
)
- if step.action != "instruct":
+ if step.action == "instruct":
+ return self._instruct(p, context, t0)
+ elif step.action == "instruct_with_refine":
+ return self._instruct_with_refine(p, context, t0)
+ else:
return StepResult(
success=False,
error=f"GUIAgentBackend: unknown action '{step.action}'. "
- "Only 'instruct' is supported.",
+ "Supported: 'instruct', 'instruct_with_refine'.",
backend_used=self.name,
duration_ms=(time.time() - t0) * 1000,
)
- return self._instruct(p, context, t0)
-
def is_available(self) -> bool:
try:
- import google.generativeai # noqa: F401
+ import openai # noqa: F401
import mss # noqa: F401
return True
except ImportError:
@@ -223,154 +233,314 @@ class GUIAgentBackend(Backend):
def _instruct(
self, p: dict, context: BackendContext, t0: float
) -> StepResult:
+ """Execute exactly ONE action decided by the vision model.
+
+ The macro author is responsible for:
+ - focusing the target window before calling gui_agent
+ - writing multiple gui_agent steps if multiple actions are needed
+ - verifying the outcome via postconditions or subsequent steps
+
+ This step:
+ 1. Takes a screenshot
+ 2. Sends it + description + end_state_snapshot to the model
+ 3. Model returns one action (click/type/hotkey/scroll/done)
+ 4. Executes that action
+ 5. Returns success with the action taken
+ """
description: str = p.get("description", "")
end_state_desc: str = p.get("end_state_description", "")
snapshot_path: str = p.get("end_state_snapshot", "")
- max_steps: int = int(p.get("max_steps", 8))
- model_name: str = p.get("model", "gemini-1.5-flash")
- api_key: str = p.get("api_key", os.environ.get("GEMINI_API_KEY", ""))
+ window_title: str = p.get("window_title", "") # focus this window first
+ model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", "gemini-2.5-flash"))
+ api_key: str = p.get("api_key", os.environ.get("MACROCLI_API_KEY", ""))
+ base_url: str = p.get("base_url", os.environ.get("MACROCLI_BASE_URL", ""))
if not api_key:
return StepResult(
success=False,
- error="GUIAgentBackend: api_key required. "
- "Pass via params or set GEMINI_API_KEY env var.",
+ error="GUIAgentBackend: api_key required. Set MACROCLI_API_KEY env var.",
backend_used=self.name,
duration_ms=(time.time() - t0) * 1000,
)
try:
- import google.generativeai as genai
+ from openai import OpenAI
except ImportError:
return StepResult(
success=False,
- error="google-generativeai required: pip install google-generativeai",
+ error="openai required: pip install openai",
backend_used=self.name,
duration_ms=(time.time() - t0) * 1000,
)
- genai.configure(api_key=api_key)
- model = genai.GenerativeModel(
- model_name=model_name,
- system_instruction=_SYSTEM_PROMPT,
- )
+ client_kwargs = {"api_key": api_key}
+ if base_url:
+ client_kwargs["base_url"] = base_url
+ client = OpenAI(**client_kwargs)
+
+ def _call_model(messages: list, max_tokens: int = 1024) -> str:
+ resp = client.chat.completions.create(
+ model=model_name,
+ messages=messages,
+ max_tokens=max_tokens,
+ )
+ return resp.choices[0].message.content.strip()
+
+ def _extract_json(raw: str) -> dict:
+ """Extract JSON from model output robustly."""
+ if raw.startswith("```"):
+ raw = "\n".join(
+ l for l in raw.split("\n") if not l.startswith("```")
+ ).strip()
+ start = raw.find('{')
+ end = raw.rfind('}')
+ if start != -1 and end != -1:
+ raw = raw[start:end+1]
+ return json.loads(raw)
+
+ # Step 1: Focus the target window if specified
+ if window_title and not context.dry_run:
+ import shutil, subprocess
+ env = os.environ.copy()
+ if "DISPLAY" not in env:
+ env["DISPLAY"] = ":0"
+ if shutil.which("wmctrl"):
+ subprocess.run(["wmctrl", "-a", window_title],
+ capture_output=True, env=env)
+ elif shutil.which("xdotool"):
+ subprocess.run(
+ ["xdotool", "search", "--name", window_title,
+ "windowfocus", "--sync"],
+ capture_output=True, env=env
+ )
+ time.sleep(0.3)
- # Load end state snapshot if provided
+ if context.dry_run:
+ return StepResult(
+ success=True,
+ output={"dry_run": True, "description": description},
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ # Step 2: Take screenshot
+ current_b64 = _screenshot_b64()
+
+ # Step 3: Load end state snapshot if provided
end_state_b64: Optional[str] = None
if snapshot_path and Path(snapshot_path).is_file():
end_state_b64 = _file_to_b64(snapshot_path)
- actions_taken = []
- reached = False
-
- for step_num in range(max_steps):
- # Take current screenshot
- current_b64 = _screenshot_b64()
+ # Step 4: Build prompt
+ content = []
+ content.append({"type": "text", "text": "CURRENT SCREEN STATE:"})
+ content.append({
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{current_b64}"}
+ })
+ if end_state_b64:
+ content.append({"type": "text", "text": "TARGET END STATE:"})
+ content.append({
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{end_state_b64}"}
+ })
+ task_text = f"TASK: {description}"
+ if end_state_desc:
+ task_text += f"\nTARGET: {end_state_desc}"
+ task_text += "\nOutput ONE action as JSON only."
+ content.append({"type": "text", "text": task_text})
+
+ messages = [
+ {"role": "system", "content": _SYSTEM_PROMPT},
+ {"role": "user", "content": content},
+ ]
+
+ # Step 5: Ask model for one action
+ try:
+ raw = _call_model(messages)
+ except Exception as exc:
+ return StepResult(
+ success=False,
+ error=f"GUIAgentBackend: model error: {exc}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
- # Build prompt parts
- parts = []
+ try:
+ action_dict = _extract_json(raw)
+ except json.JSONDecodeError:
+ return StepResult(
+ success=False,
+ error=f"GUIAgentBackend: invalid JSON from model: {raw[:200]}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
- # Current state
- parts.append(f"CURRENT STATE (step {step_num + 1}/{max_steps}):")
- parts.append({
- "mime_type": "image/png",
- "data": current_b64,
- })
+ action_name = action_dict.get("action", "")
+ print(f"[gui_agent] action: {action_dict}", flush=True)
- # Target state
- if end_state_b64:
- parts.append("TARGET END STATE:")
- parts.append({
- "mime_type": "image/png",
- "data": end_state_b64,
- })
-
- # Task description
- task = f"TASK: {description}"
- if end_state_desc:
- task += f"\nEND STATE: {end_state_desc}"
- parts.append(task)
- parts.append("What single action should I take next? Output JSON only.")
-
- # Ask model
+ # Step 6: Execute the action (unless model says done)
+ if action_name != "done":
try:
- response = model.generate_content(parts)
- raw = response.text.strip()
+ _execute_action(action_dict, context)
except Exception as exc:
return StepResult(
success=False,
- error=f"GUIAgentBackend: model error at step {step_num+1}: {exc}",
+ error=f"GUIAgentBackend: action execution failed: {exc}",
backend_used=self.name,
duration_ms=(time.time() - t0) * 1000,
)
+ time.sleep(0.5)
+
+ return StepResult(
+ success=True,
+ output={
+ "action": action_dict,
+ "done": action_name == "done",
+ },
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ def _instruct_with_refine(
+ self, p: dict, context: BackendContext, t0: float
+ ) -> StepResult:
+ """Execute one action, then compare result vs end_state_snapshot,
+ and if needed undo and re-execute with a refined action.
+
+ Sends to model on refine:
+ - Screenshot BEFORE first action (original state)
+ - The first action that was taken
+ - Screenshot AFTER first action (current result)
+ - end_state_snapshot (target state)
+ - Request for corrected action
+
+ This allows the model to see exactly what went wrong and correct it.
+ """
+ snapshot_path: str = p.get("end_state_snapshot", "")
+
+ if not snapshot_path or not Path(snapshot_path).is_file():
+ # No end state snapshot → fall back to single instruct
+ return self._instruct(p, context, t0)
+
+ # ── Round 1: initial action ───────────────────────────────────────────
+ before_b64 = _screenshot_b64()
+ result1 = self._instruct(p, context, t0)
+
+ if not result1.success:
+ return result1
+
+ first_action = result1.output.get("action", {})
+ if first_action.get("action") == "done":
+ return result1
+
+ time.sleep(0.5)
+ after_b64 = _screenshot_b64()
+ end_state_b64 = _file_to_b64(snapshot_path)
+
+ # ── Round 2: compare and refine ───────────────────────────────────────
+ description: str = p.get("description", "")
+ end_state_desc: str = p.get("end_state_description", "")
+ model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", "gemini-2.5-flash"))
+ api_key: str = p.get("api_key", os.environ.get("MACROCLI_API_KEY", ""))
+ base_url: str = p.get("base_url", os.environ.get("MACROCLI_BASE_URL", ""))
+
+ from openai import OpenAI
+ client_kwargs = {"api_key": api_key}
+ if base_url:
+ client_kwargs["base_url"] = base_url
+ client = OpenAI(**client_kwargs)
+
+ def _call(messages):
+ resp = client.chat.completions.create(
+ model=model_name, messages=messages, max_tokens=1024,
+ )
+ return resp.choices[0].message.content.strip()
- # Strip markdown fences
+ def _extract_json(raw):
if raw.startswith("```"):
- raw = "\n".join(
- l for l in raw.split("\n") if not l.startswith("```")
- ).strip()
+ raw = "\n".join(l for l in raw.split("\n") if not l.startswith("```")).strip()
+ s, e = raw.find('{'), raw.rfind('}')
+ if s != -1 and e != -1:
+ raw = raw[s:e+1]
+ return json.loads(raw)
- # Parse action
- try:
- action_dict = json.loads(raw)
- except json.JSONDecodeError:
- return StepResult(
- success=False,
- error=f"GUIAgentBackend: invalid JSON from model: {raw[:200]}",
- backend_used=self.name,
- duration_ms=(time.time() - t0) * 1000,
- )
+ refine_prompt = f"""You are refining a GUI automation action.
- action_name = action_dict.get("action", "")
- actions_taken.append(action_dict)
+ORIGINAL TASK: {description}
+TARGET STATE: {end_state_desc}
- if action_name == "done":
- reached = True
- break
+WHAT HAPPENED:
+- First action taken: {json.dumps(first_action)}
- # Execute action
- try:
- _execute_action(action_dict, context)
- except Exception as exc:
- return StepResult(
- success=False,
- error=f"GUIAgentBackend: action execution failed: {exc}",
- backend_used=self.name,
- duration_ms=(time.time() - t0) * 1000,
- )
+Now compare these three screenshots:
+
+1. BEFORE (original state before any action):
+2. AFTER FIRST ACTION (current result):
+3. TARGET END STATE (what it should look like):
+
+The first action was not quite right. Looking at:
+- Where the rectangle was drawn vs where it should be
+- The difference between AFTER and TARGET
+
+Provide a corrected drag action with better coordinates.
+Output ONE JSON action only."""
+
+ content = [
+ {"type": "text", "text": refine_prompt},
+ {"type": "text", "text": "BEFORE:"},
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{before_b64}"}},
+ {"type": "text", "text": "AFTER FIRST ACTION:"},
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{after_b64}"}},
+ {"type": "text", "text": "TARGET END STATE:"},
+ {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{end_state_b64}"}},
+ ]
+
+ messages = [
+ {"role": "system", "content": _SYSTEM_PROMPT},
+ {"role": "user", "content": content},
+ ]
+
+ try:
+ raw = _call(messages)
+ refined_action = _extract_json(raw)
+ print(f"[gui_agent] refined action: {refined_action}", flush=True)
+ except Exception as exc:
+ # Refine failed, return original result
+ print(f"[gui_agent] refine failed: {exc}, keeping original", flush=True)
+ return result1
+
+ if refined_action.get("action") == "done":
+ return result1
+
+ # Undo the first action, then execute the refined one
+ import shutil, subprocess as sp
+ env = os.environ.copy()
+ if "DISPLAY" not in env:
+ env["DISPLAY"] = ":0"
+ if shutil.which("xdotool"):
+ sp.run(["xdotool", "key", "ctrl+z"], env=env)
+ time.sleep(0.3)
+
+ try:
+ _execute_action(refined_action, context)
+ except Exception as exc:
+ return StepResult(
+ success=False,
+ error=f"GUIAgentBackend: refine action failed: {exc}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
- time.sleep(0.5) # wait for UI to respond
-
- # After executing, check if end state reached
- if end_state_b64 and step_num < max_steps - 1:
- current_b64 = _screenshot_b64()
- check_model = genai.GenerativeModel(model_name=model_name)
- check_parts = [
- _CHECK_PROMPT,
- "CURRENT:", {"mime_type": "image/png", "data": current_b64},
- "TARGET:", {"mime_type": "image/png", "data": end_state_b64},
- ]
- try:
- check_resp = check_model.generate_content(check_parts)
- check_raw = check_resp.text.strip()
- if check_raw.startswith("```"):
- check_raw = "\n".join(
- l for l in check_raw.split("\n")
- if not l.startswith("```")
- ).strip()
- check_dict = json.loads(check_raw)
- if check_dict.get("reached"):
- reached = True
- break
- except Exception:
- pass # continue if check fails
+ time.sleep(0.5)
return StepResult(
- success=reached or len(actions_taken) > 0,
+ success=True,
output={
- "reached_end_state": reached,
- "steps_taken": len(actions_taken),
- "actions": actions_taken,
+ "action": refined_action,
+ "first_action": first_action,
+ "refined": True,
+ "done": False,
},
backend_used=self.name,
duration_ms=(time.time() - t0) * 1000,
diff --git a/macrocli/agent-harness/cli_anything/macrocli/backends/visual_anchor.py b/macrocli/agent-harness/cli_anything/macrocli/backends/visual_anchor.py
index cb4abb679..aa1958e71 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/backends/visual_anchor.py
+++ b/macrocli/agent-harness/cli_anything/macrocli/backends/visual_anchor.py
@@ -321,6 +321,8 @@ class VisualAnchorBackend(Backend):
"type_text": self._type_text,
"hotkey": self._hotkey,
"scroll": self._scroll,
+ "drag": self._drag,
+ "drag_relative": self._drag_relative,
"capture_region": self._capture_region,
}
@@ -532,6 +534,91 @@ class VisualAnchorBackend(Backend):
return {"scrolled_at": [cx, cy], "dx": dx, "dy": dy}
+ def _drag(self, p: dict, context: BackendContext) -> dict:
+ """Drag from one template image to another (or to absolute coords).
+
+ Params:
+ from_template: path to template image for drag start (optional)
+ to_template: path to template image for drag end (optional)
+ from_x / from_y: fallback absolute coords if no from_template
+ to_x / to_y: fallback absolute coords if no to_template
+ button: left | right | middle (default left)
+ duration_ms: how long to hold during drag (default 200)
+ confidence: template match threshold (default 0.85)
+ timeout_ms: how long to wait for templates (default 5000)
+ """
+ button = p.get("button", "left")
+ duration_ms = int(p.get("duration_ms", 200))
+ confidence = float(p.get("confidence", 0.85))
+ timeout_ms = int(p.get("timeout_ms", 5000))
+
+ # Resolve start position
+ from_tmpl = p.get("from_template", "")
+ if from_tmpl and Path(from_tmpl).is_file():
+ tmpl = _load_image_as_array(from_tmpl)
+ match = _wait_for_template(tmpl, confidence, timeout_ms)
+ if match is None:
+ raise RuntimeError(f"drag: from_template not found: {from_tmpl}")
+ fx, fy = match[0], match[1]
+ else:
+ fx = int(p.get("from_x", 0))
+ fy = int(p.get("from_y", 0))
+
+ # Resolve end position
+ to_tmpl = p.get("to_template", "")
+ if to_tmpl and Path(to_tmpl).is_file():
+ tmpl = _load_image_as_array(to_tmpl)
+ match = _wait_for_template(tmpl, confidence, timeout_ms)
+ if match is None:
+ raise RuntimeError(f"drag: to_template not found: {to_tmpl}")
+ tx, ty = match[0], match[1]
+ else:
+ tx = int(p.get("to_x", fx))
+ ty = int(p.get("to_y", fy))
+
+ _mouse_drag(fx, fy, tx, ty, button=button, duration_ms=duration_ms)
+ return {"dragged_from": [fx, fy], "dragged_to": [tx, ty]}
+
+ def _drag_relative(self, p: dict, context: BackendContext) -> dict:
+ """Drag within a window using fractional coordinates.
+
+ Params:
+ window_title: partial window title (uses focused window if empty)
+ from_x_pct: drag start x as fraction of window width
+ from_y_pct: drag start y as fraction of window height
+ to_x_pct: drag end x as fraction of window width
+ to_y_pct: drag end y as fraction of window height
+ button: left | right | middle (default left)
+ duration_ms: hold duration in ms (default 200)
+ """
+ title = p.get("window_title", "")
+ button = p.get("button", "left")
+ duration_ms = int(p.get("duration_ms", 200))
+
+ if title:
+ bounds = _get_window_bounds(title)
+ if bounds is None:
+ raise RuntimeError(f"drag_relative: window not found: '{title}'")
+ wx, wy = bounds["x"], bounds["y"]
+ ww, wh = bounds["width"], bounds["height"]
+ else:
+ _, monitor = _screenshot_as_array()
+ wx, wy = 0, 0
+ ww, wh = monitor["width"], monitor["height"]
+
+ fx = int(wx + ww * float(p.get("from_x_pct", 0.0)))
+ fy = int(wy + wh * float(p.get("from_y_pct", 0.0)))
+ tx = int(wx + ww * float(p.get("to_x_pct", 1.0)))
+ ty = int(wy + wh * float(p.get("to_y_pct", 1.0)))
+
+ _mouse_drag(fx, fy, tx, ty, button=button, duration_ms=duration_ms)
+ return {
+ "dragged_from": [fx, fy],
+ "dragged_to": [tx, ty],
+ "from_pct": [p.get("from_x_pct"), p.get("from_y_pct")],
+ "to_pct": [p.get("to_x_pct"), p.get("to_y_pct")],
+ }
+
def _capture_region(self, p: dict, context: BackendContext) -> dict:
"""Screenshot a region of the screen and save as a template."""
output_path = p.get("output", "")
@@ -562,7 +649,7 @@ class VisualAnchorBackend(Backend):
}
-# ── pynput mouse helper ───────────────────────────────────────────────────────
+# ── pynput mouse helpers ──────────────────────────────────────────────────────
def _mouse_click(x: int, y: int, button: str = "left", double: bool = False):
"""Move mouse to (x, y) and click."""
@@ -578,10 +665,66 @@ def _mouse_click(x: int, y: int, button: str = "left", double: bool = False):
btn = btn_map.get(button.lower(), Button.left)
ctrl.position = (x, y)
- time.sleep(0.05) # small settle delay
+ time.sleep(0.05)
ctrl.press(btn)
ctrl.release(btn)
if double:
time.sleep(0.08)
ctrl.press(btn)
ctrl.release(btn)
+
+
+def _mouse_drag(
+ fx: int, fy: int, tx: int, ty: int,
+ button: str = "left", duration_ms: int = 200
+):
+ """Press at (fx, fy), move to (tx, ty) over duration_ms, release.
+
+ Tries xdotool first (works with Qt5/KDE apps), falls back to pynput.
+ """
+ import shutil, subprocess, os
+
+ env = os.environ.copy()
+ if "DISPLAY" not in env:
+ env["DISPLAY"] = ":0"
+
+ if shutil.which("xdotool"):
+ # xdotool is more reliable with Qt5 apps
+ steps = max(5, duration_ms // 30)
+ subprocess.run(["xdotool", "mousemove", str(fx), str(fy)], env=env)
+ time.sleep(0.05)
+ subprocess.run(["xdotool", "mousedown", "1"], env=env)
+ time.sleep(0.05)
+ for i in range(1, steps + 1):
+ ix = int(fx + (tx - fx) * i / steps)
+ iy = int(fy + (ty - fy) * i / steps)
+ subprocess.run(["xdotool", "mousemove", str(ix), str(iy)], env=env)
+ time.sleep(duration_ms / 1000.0 / steps)
+ subprocess.run(["xdotool", "mousemove", str(tx), str(ty)], env=env)
+ time.sleep(0.05)
+ subprocess.run(["xdotool", "mouseup", "1"], env=env)
+ return
+
+ # Fallback: pynput
+ mouse_mod, _ = _require_pynput()
+ Button = mouse_mod.Button
+ ctrl = mouse_mod.Controller()
+ btn_map = {"left": Button.left, "right": Button.right, "middle": Button.middle}
+ btn = btn_map.get(button.lower(), Button.left)
+
+ ctrl.position = (fx, fy)
+ time.sleep(0.05)
+ ctrl.press(btn)
+ time.sleep(0.05)
+
+ steps = max(10, duration_ms // 20)
+ step_sleep = duration_ms / 1000.0 / steps
+ for i in range(1, steps + 1):
+ ix = int(fx + (tx - fx) * i / steps)
+ iy = int(fy + (ty - fy) * i / steps)
+ ctrl.position = (ix, iy)
+ time.sleep(step_sleep)
+
+ ctrl.position = (tx, ty)
+ time.sleep(0.05)
+ ctrl.release(btn)
diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml
new file mode 100644
index 000000000..6014027f9
--- /dev/null
+++ b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml
@@ -0,0 +1,187 @@
+name: kolourpaint_annotate
+version: "1.0"
+description: >
+ Open a screenshot in KolourPaint, let the AI agent identify and draw
+ a red rectangle around the area matching the annotation goal,
+ then save the annotated image.
+
+ This macro demonstrates the hybrid fixed+agent approach:
+ - Fixed steps handle application launch, tool selection, color selection, save
+ - The agent step handles the annotation position decision (purely visual)
+
+tags: [demo, kolourpaint, annotation, gui_agent, visual_anchor]
+
+parameters:
+ input_file:
+ type: string
+ required: true
+ description: Path to the screenshot PNG to annotate.
+ example: /tmp/screenshot.png
+ output_file:
+ type: string
+ required: true
+ description: Path to save the annotated PNG.
+ example: /tmp/annotated.png
+ annotation_goal:
+ type: string
+ required: false
+ default: "the error or problem in the image"
+ description: >
+ Natural language description of what to annotate.
+ The AI agent will decide where to draw the rectangle.
+ example: "the error message in red text"
+
+preconditions:
+ - file_exists: ${input_file}
+ - file_exists: /usr/bin/kolourpaint
+
+steps:
+ # ── Fixed: launch KolourPaint with the input file ─────────────────────────
+ - id: launch_kolourpaint
+ backend: native_api
+ action: start_process
+ params:
+ command: [kolourpaint, "${input_file}"]
+ log_file: /tmp/kolourpaint.log
+ env:
+ DISPLAY: ":99"
+ on_failure: fail
+
+ - id: wait_kolourpaint
+ backend: native_api
+ action: run_command
+ params:
+ command: [sleep, "2.5"]
+ on_failure: skip
+
+ # ── Fixed: select Rectangle tool (hotkey r) ───────────────────────────────
+ - id: select_rectangle_tool
+ backend: semantic_ui
+ action: focus_window
+ params:
+ title_contains: KolourPaint
+ on_failure: fail
+
+ - id: press_rectangle_hotkey
+ backend: visual_anchor
+ action: hotkey
+ params:
+ keys: r
+ on_failure: fail
+
+ # ── Fixed: select red color (click red swatch in color palette) ───────────
+ - id: select_red_color
+ backend: visual_anchor
+ action: click_relative
+ params:
+ window_title: KolourPaint
+ x_pct: 0.41
+ y_pct: 0.72
+ on_failure: fail
+
+ - id: pause_after_color
+ backend: native_api
+ action: run_command
+ params:
+ command: [sleep, "0.3"]
+ on_failure: skip
+
+ # ── Agent: decide WHERE to draw the rectangle ─────────────────────────────
+ # The agent looks at the screenshot content and decides the coordinates
+ # of the annotation rectangle based on the annotation_goal.
+ - id: agent_draw_annotation
+ backend: gui_agent
+ action: instruct
+ params:
+ window_title: KolourPaint
+ description: >
+ The Rectangle drawing tool is active and a color is selected in KolourPaint.
+ The image canvas occupies this exact screen region:
+ Top-left corner: (282, 95)
+ Top-right corner: (980, 95)
+ Bottom-left corner: (282, 715)
+ Bottom-right corner: (980, 715)
+ The canvas shows a screenshot. Find the area matching: ${annotation_goal}
+ Draw a rectangle around it using the drag action.
+ The drag coordinates MUST be within the canvas bounds above.
+ end_state_description: >
+ A rectangle is drawn around ${annotation_goal} on the canvas.
+ end_state_snapshot: snapshots/annotated_end_state.png
+ on_failure: fail
+
+ - id: pause_after_draw
+ backend: native_api
+ action: run_command
+ params:
+ command: [sleep, "0.5"]
+ on_failure: skip
+
+ # ── Fixed: save as output_file ────────────────────────────────────────────
+ - id: open_save_as
+ backend: visual_anchor
+ action: hotkey
+ params:
+ keys: ctrl+shift+s
+ on_failure: fail
+
+ - id: wait_save_dialog
+ backend: semantic_ui
+ action: wait_for_window
+ params:
+ title_contains: Save As
+ timeout_ms: 4000
+ on_failure: skip
+
+ - id: clear_filename
+ backend: visual_anchor
+ action: hotkey
+ params:
+ keys: ctrl+l
+ on_failure: skip
+
+ - id: clear_path_field
+ backend: visual_anchor
+ action: hotkey
+ params:
+ keys: ctrl+a
+ on_failure: skip
+
+ - id: type_output_path
+ backend: visual_anchor
+ action: type_text
+ params:
+ text: "${output_file}"
+ interval_ms: 30
+ on_failure: fail
+
+ - id: confirm_save
+ backend: visual_anchor
+ action: hotkey
+ params:
+ keys: enter
+ on_failure: fail
+
+ - id: wait_save_complete
+ backend: native_api
+ action: run_command
+ params:
+ command: [sleep, "1.0"]
+ on_failure: skip
+
+postconditions:
+ - file_exists: ${output_file}
+
+outputs:
+ - name: annotated_file
+ path: ${output_file}
+ description: Path to the annotated screenshot.
+
+agent_hints:
+ danger_level: safe
+ side_effects: [creates_file, opens_application]
+ reversible: true
+ estimated_duration_ms: 15000
+ note: >
+ The annotation position is decided by the AI agent based on the
+ screenshot content and annotation_goal. This cannot be done with
+ fixed coordinates because the target area varies per screenshot.
diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png
new file mode 100644
index 000000000..9459f653c
Binary files /dev/null and b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png differ
diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml
index 4fef107c4..c0d7e5852 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml
+++ b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml
@@ -21,3 +21,7 @@ macros:
- name: gedit_find_and_replace
path: demo/gedit_find_and_replace.yaml
version: "1.0"
+ # ── Demo macros (flameshot) ───────────────────────────────────
+ - name: flameshot_annotate
+ path: demo/flameshot_annotate.yaml
+ version: "1.0"