Skip to content

"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/

Python

feat(macrocli): add gui_agent backend and hybrid fixed+agent macro recording

haorui-harry6ecf72d9c9c4
diff --git a/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
new file mode 100644
index 000000000..361a62b98
--- /dev/null
+++ b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
@@ -0,0 +1,377 @@
+"""GUIAgentBackend — execute a macro step by letting a vision model
+(Gemini or Claude) look at the screen and decide what to do.
+
+This backend is used for steps that cannot be expressed as fixed
+coordinates or hotkeys because the interface state is unpredictable.
+The macro author provides:
+ - description: what needs to be accomplished in this step
+ - end_state_description: text description of the desired end state
+ - end_state_snapshot: screenshot of the desired end state (taken
+ by the macro author at recording time)
+
+At runtime the backend:
+ 1. Takes a screenshot of the current screen
+ 2. Sends current screenshot + end_state_snapshot + description to the model
+ 3. Model returns the next action (click x,y / type text / hotkey)
+ 4. Executes the action
+ 5. Takes another screenshot
+ 6. Asks model: "have we reached the end state?"
+ 7. Loops until end state reached or max_steps exceeded
+
+Supported models:
+ - gemini-1.5-flash (default, fastest)
+ - gemini-1.5-pro
+ - gemini-2.0-flash
+
+Example YAML step:
+
+ - id: select_png_format
+ backend: gui_agent
+ action: instruct
+ params:
+ description: >
+ The export dialog is open. Find the Format dropdown and
+ select PNG. Then ensure Resolution shows 300.
+ end_state_description: >
+ Format dropdown shows PNG, Resolution input shows 300.
+ end_state_snapshot: snapshots/step_003_end_state.png
+ max_steps: 8
+ model: gemini-1.5-flash
+ api_key: ${GEMINI_API_KEY}
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import os
+import time
+from pathlib import Path
+from typing import Optional
+
+from cli_anything.macrocli.backends.base import Backend, BackendContext, StepResult
+from cli_anything.macrocli.core.macro_model import MacroStep, substitute
+
+# ── Strict action space prompt ────────────────────────────────────────────────
+
+_SYSTEM_PROMPT = """\
+You are a GUI automation agent. You will be shown:
+1. A screenshot of the CURRENT screen state
+2. A screenshot of the TARGET end state
+3. A description of what needs to be accomplished
+
+Your job is to figure out what single action to take next to move
+from the current state toward the target state.
+
+OUTPUT FORMAT: Respond with ONLY a JSON object, one of:
+
+ {"action": "click", "x": <int>, "y": <int>, "button": "left"}
+ {"action": "double_click", "x": <int>, "y": <int>}
+ {"action": "right_click", "x": <int>, "y": <int>}
+ {"action": "type", "text": "<string>"}
+ {"action": "hotkey", "keys": "<key1+key2+...>"}
+ {"action": "scroll", "x": <int>, "y": <int>, "dy": <int>}
+ {"action": "done"}
+
+Use {"action": "done"} ONLY when the current state matches the target state.
+
+RULES:
+- Output RAW JSON ONLY. No markdown, no explanation.
+- Use pixel coordinates from the CURRENT screenshot.
+- Prefer clicking on visible labeled controls over guessing coordinates.
+- If the target state is already achieved, output {"action": "done"}.
+- Never output any action not listed above.
+"""
+
+_CHECK_PROMPT = """\
+Compare these two screenshots:
+1. CURRENT state
+2. TARGET end state
+
+Has the current state reached the target end state?
+Answer with ONLY: {"reached": true} or {"reached": false, "reason": "<brief reason>"}
+"""
+
+
+# ── Image helpers ─────────────────────────────────────────────────────────────
+
+def _screenshot_b64() -> str:
+ """Take a screenshot and return as base64 PNG string."""
+ try:
+ import mss
+ from PIL import Image
+ import io
+ with mss.mss() as sct:
+ monitor = sct.monitors[1]
+ raw = sct.grab(monitor)
+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+ return base64.b64encode(buf.getvalue()).decode("utf-8")
+ except ImportError:
+ raise ImportError("mss and Pillow required: pip install mss Pillow")
+
+
+def _file_to_b64(path: str) -> str:
+ with open(path, "rb") as f:
+ return base64.b64encode(f.read()).decode("utf-8")
+
+
+# ── Action executor ───────────────────────────────────────────────────────────
+
+def _execute_action(action_dict: dict, context: BackendContext) -> None:
+ """Execute a single action returned by the model."""
+ from cli_anything.macrocli.backends.visual_anchor import (
+ _mouse_click, _mouse_drag, _require_pynput
+ )
+
+ action = action_dict.get("action", "")
+
+ if action == "click":
+ x, y = int(action_dict["x"]), int(action_dict["y"])
+ _mouse_click(x, y, button=action_dict.get("button", "left"))
+
+ elif action == "double_click":
+ x, y = int(action_dict["x"]), int(action_dict["y"])
+ _mouse_click(x, y, double=True)
+
+ elif action == "right_click":
+ x, y = int(action_dict["x"]), int(action_dict["y"])
+ _mouse_click(x, y, button="right")
+
+ elif action == "type":
+ text = action_dict.get("text", "")
+ _, keyboard_mod = _require_pynput()
+ ctrl = keyboard_mod.Controller()
+ for char in text:
+ ctrl.press(char)
+ ctrl.release(char)
+ time.sleep(0.03)
+
+ elif action == "hotkey":
+ keys_str = action_dict.get("keys", "")
+ _, keyboard_mod = _require_pynput()
+ Key = keyboard_mod.Key
+ ctrl = keyboard_mod.Controller()
+ _KEY_MAP = {
+ "ctrl": Key.ctrl, "shift": Key.shift, "alt": Key.alt,
+ "enter": Key.enter, "tab": Key.tab, "esc": Key.esc,
+ "escape": Key.esc, "space": Key.space, "backspace": Key.backspace,
+ }
+ keys = [_KEY_MAP.get(k.lower(), k) for k in keys_str.split("+")]
+ for k in keys:
+ ctrl.press(k)
+ for k in reversed(keys):
+ ctrl.release(k)
+
+ elif action == "scroll":
+ x, y = int(action_dict["x"]), int(action_dict["y"])
+ dy = int(action_dict.get("dy", -3))
+ mouse_mod, _ = _require_pynput()
+ ctrl = mouse_mod.Controller()
+ ctrl.position = (x, y)
+ ctrl.scroll(0, dy)
+
+ elif action == "done":
+ pass # caller checks for done
+
+ else:
+ raise ValueError(f"GUIAgentBackend: unknown action '{action}'")
+
+
+# ── Backend ───────────────────────────────────────────────────────────────────
+
+class GUIAgentBackend(Backend):
+ """Execute GUI steps using a vision model (Gemini) to decide actions."""
+
+ name = "gui_agent"
+ priority = 60 # between semantic_ui(50) and file_transform(70)
+
+ def execute(
+ self, step: MacroStep, params: dict, context: BackendContext
+ ) -> StepResult:
+ t0 = time.time()
+ p = substitute(step.params, params)
+
+ if context.dry_run:
+ return StepResult(
+ success=True,
+ output={"dry_run": True, "action": step.action},
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ if step.action != "instruct":
+ return StepResult(
+ success=False,
+ error=f"GUIAgentBackend: unknown action '{step.action}'. "
+ "Only 'instruct' is supported.",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ return self._instruct(p, context, t0)
+
+ def is_available(self) -> bool:
+ try:
+ import google.generativeai # noqa: F401
+ import mss # noqa: F401
+ return True
+ except ImportError:
+ return False
+
+ def _instruct(
+ self, p: dict, context: BackendContext, t0: float
+ ) -> StepResult:
+ description: str = p.get("description", "")
+ end_state_desc: str = p.get("end_state_description", "")
+ snapshot_path: str = p.get("end_state_snapshot", "")
+ max_steps: int = int(p.get("max_steps", 8))
+ model_name: str = p.get("model", "gemini-1.5-flash")
+ api_key: str = p.get("api_key", os.environ.get("GEMINI_API_KEY", ""))
+
+ if not api_key:
+ return StepResult(
+ success=False,
+ error="GUIAgentBackend: api_key required. "
+ "Pass via params or set GEMINI_API_KEY env var.",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ try:
+ import google.generativeai as genai
+ except ImportError:
+ return StepResult(
+ success=False,
+ error="google-generativeai required: pip install google-generativeai",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ genai.configure(api_key=api_key)
+ model = genai.GenerativeModel(
+ model_name=model_name,
+ system_instruction=_SYSTEM_PROMPT,
+ )
+
+ # Load end state snapshot if provided
+ end_state_b64: Optional[str] = None
+ if snapshot_path and Path(snapshot_path).is_file():
+ end_state_b64 = _file_to_b64(snapshot_path)
+
+ actions_taken = []
+ reached = False
+
+ for step_num in range(max_steps):
+ # Take current screenshot
+ current_b64 = _screenshot_b64()
+
+ # Build prompt parts
+ parts = []
+
+ # Current state
+ parts.append(f"CURRENT STATE (step {step_num + 1}/{max_steps}):")
+ parts.append({
+ "mime_type": "image/png",
+ "data": current_b64,
+ })
+
+ # Target state
+ if end_state_b64:
+ parts.append("TARGET END STATE:")
+ parts.append({
+ "mime_type": "image/png",
+ "data": end_state_b64,
+ })
+
+ # Task description
+ task = f"TASK: {description}"
+ if end_state_desc:
+ task += f"\nEND STATE: {end_state_desc}"
+ parts.append(task)
+ parts.append("What single action should I take next? Output JSON only.")
+
+ # Ask model
+ try:
+ response = model.generate_content(parts)
+ raw = response.text.strip()
+ except Exception as exc:
+ return StepResult(
+ success=False,
+ error=f"GUIAgentBackend: model error at step {step_num+1}: {exc}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ # Strip markdown fences
+ if raw.startswith("```"):
+ raw = "\n".join(
+ l for l in raw.split("\n") if not l.startswith("```")
+ ).strip()
+
+ # Parse action
+ try:
+ action_dict = json.loads(raw)
+ except json.JSONDecodeError:
+ return StepResult(
+ success=False,
+ error=f"GUIAgentBackend: invalid JSON from model: {raw[:200]}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ action_name = action_dict.get("action", "")
+ actions_taken.append(action_dict)
+
+ if action_name == "done":
+ reached = True
+ break
+
+ # Execute action
+ try:
+ _execute_action(action_dict, context)
+ except Exception as exc:
+ return StepResult(
+ success=False,
+ error=f"GUIAgentBackend: action execution failed: {exc}",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
+ time.sleep(0.5) # wait for UI to respond
+
+ # After executing, check if end state reached
+ if end_state_b64 and step_num < max_steps - 1:
+ current_b64 = _screenshot_b64()
+ check_model = genai.GenerativeModel(model_name=model_name)
+ check_parts = [
+ _CHECK_PROMPT,
+ "CURRENT:", {"mime_type": "image/png", "data": current_b64},
+ "TARGET:", {"mime_type": "image/png", "data": end_state_b64},
+ ]
+ try:
+ check_resp = check_model.generate_content(check_parts)
+ check_raw = check_resp.text.strip()
+ if check_raw.startswith("```"):
+ check_raw = "\n".join(
+ l for l in check_raw.split("\n")
+ if not l.startswith("```")
+ ).strip()
+ check_dict = json.loads(check_raw)
+ if check_dict.get("reached"):
+ reached = True
+ break
+ except Exception:
+ pass # continue if check fails
+
+ return StepResult(
+ success=reached or len(actions_taken) > 0,
+ output={
+ "reached_end_state": reached,
+ "steps_taken": len(actions_taken),
+ "actions": actions_taken,
+ },
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
diff --git a/macrocli/agent-harness/cli_anything/macrocli/core/macro_model.py b/macrocli/agent-harness/cli_anything/macrocli/core/macro_model.py
index 74a1cb2ea..4d387666e 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/core/macro_model.py
+++ b/macrocli/agent-harness/cli_anything/macrocli/core/macro_model.py
@@ -169,7 +169,7 @@ class MacroDefinition:
errors.append("Macro name is required.")
if not self.steps:
errors.append(f"Macro '{self.name}' has no steps.")
- valid_backends = {"native_api", "file_transform", "semantic_ui", "gui_macro", "recovery", "visual_anchor"}
+ valid_backends = {"native_api", "file_transform", "semantic_ui", "gui_macro", "recovery", "visual_anchor", "gui_agent"}
for i, step in enumerate(self.steps):
if step.backend not in valid_backends:
errors.append(
diff --git a/macrocli/agent-harness/cli_anything/macrocli/core/recorder.py b/macrocli/agent-harness/cli_anything/macrocli/core/recorder.py
index 0285e65bc..331b166e0 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/core/recorder.py
+++ b/macrocli/agent-harness/cli_anything/macrocli/core/recorder.py
@@ -63,15 +63,34 @@ class RecordedStep:
dy: int = 0
# timing
timestamp: float = field(default_factory=time.time)
+ # agent step fields (set during post-recording review)
+ is_agent_step: bool = False
+ agent_description: str = ""
+ agent_end_state_description: str = ""
+ agent_end_state_snapshot: str = "" # relative path to snapshot png
def to_step_dict(self) -> dict:
"""Convert to a macro YAML step dict.
- Strategy for clicks:
- 1. If we have a window title → use click_relative (most robust)
- 2. If we have a good template → use click_image
- 3. Fallback → click_relative using screen percentages
+ If marked as agent step, emits a gui_agent/instruct step.
+ Otherwise uses visual_anchor with window-relative coords or template.
"""
+ # Agent step overrides everything
+ if self.is_agent_step:
+ params: dict = {
+ "description": self.agent_description,
+ "end_state_description": self.agent_end_state_description,
+ "max_steps": 8,
+ }
+ if self.agent_end_state_snapshot:
+ params["end_state_snapshot"] = self.agent_end_state_snapshot
+ return {
+ "id": f"step_{self.index:03d}_agent",
+ "backend": "gui_agent",
+ "action": "instruct",
+ "params": params,
+ "on_failure": "fail",
+ }
if self.kind == "click":
if self.window_title:
# Best case: window-relative fractional coordinates
@@ -561,6 +580,142 @@ class MacroRecorder:
print(f"[recorder] Saved macro to: {output_path}", flush=True)
return output_path
+ def save_as_package(
+ self,
+ output_dir: Optional[str] = None,
+ parameters: Optional[dict] = None,
+ ) -> str:
+ """Save as a macro package folder:
+ <macro_name>/
+ macro.yaml
+ snapshots/
+ step_XXX_end_state.png (copied from recorded locations)
+
+ Returns path to macro.yaml.
+ """
+ pkg_dir = Path(output_dir or self.output_dir) / self.macro_name
+ pkg_dir.mkdir(parents=True, exist_ok=True)
+ snapshots_dir = pkg_dir / "snapshots"
+ snapshots_dir.mkdir(exist_ok=True)
+
+ # Copy any end_state snapshots into the package and update paths
+ for step in self._steps:
+ if step.is_agent_step and step.agent_end_state_snapshot:
+ src = Path(step.agent_end_state_snapshot)
+ if src.is_file():
+ dst = snapshots_dir / src.name
+ import shutil
+ if src.resolve() != dst.resolve():
+ shutil.copy2(src, dst)
+ # Update path to be relative to macro.yaml
+ step.agent_end_state_snapshot = f"snapshots/{src.name}"
+
+ yaml_path = str(pkg_dir / "macro.yaml")
+ Path(yaml_path).write_text(
+ self.to_yaml(parameters=parameters), encoding="utf-8"
+ )
+ print(f"[recorder] Saved macro package to: {pkg_dir}/", flush=True)
+ return yaml_path
+
+ # ── Agent step review ─────────────────────────────────────────────────────
+
+ def interactive_agent_review(self, snapshots_dir: Optional[str] = None) -> None:
+ """Interactively review all steps and mark some as agent steps.
+
+ For each step the user can:
+ - Press Enter → keep as fixed step
+ - Type 'a' → mark as agent step, then provide description,
+ end_state_description, and take a snapshot
+
+ snapshots_dir: where to save end_state snapshots (default: output_dir/snapshots)
+ """
+ snap_dir = Path(snapshots_dir or self.output_dir / "snapshots")
+ snap_dir.mkdir(parents=True, exist_ok=True)
+
+ print()
+ print("─" * 60)
+ print(" Step Review — mark steps as 'fixed' or 'agent'")
+ print(" Enter = fixed (fast, deterministic)")
+ print(" a = agent step (Gemini decides at runtime)")
+ print("─" * 60)
+
+ for i, step in enumerate(self._steps):
+ # Build a human-readable description of the step
+ if step.kind == "click":
+ step_desc = f"click {step.window_title or 'screen'} ({step.x_pct:.2f}, {step.y_pct:.2f})"
+ elif step.kind == "type":
+ preview = step.text[:40] + "..." if len(step.text) > 40 else step.text
+ step_desc = f"type_text {preview!r}"
+ elif step.kind == "hotkey":
+ step_desc = f"hotkey {step.keys}"
+ elif step.kind == "scroll":
+ step_desc = f"scroll dy={step.dy}"
+ else:
+ step_desc = step.kind
+
+ print(f"\n [{i+1}/{len(self._steps)}] {step_desc}")
+
+ try:
+ choice = input(" → fixed or agent? [Enter=fixed / a=agent]: ").strip().lower()
+ except (EOFError, KeyboardInterrupt):
+ print()
+ break
+
+ if choice != "a":
+ continue
+
+ # Mark as agent step
+ step.is_agent_step = True
+
+ try:
+ step.agent_description = input(
+ " → Describe what this step needs to do:\n "
+ ).strip()
+ step.agent_end_state_description = input(
+ " → Describe the target end state (what should the screen show):\n "
+ ).strip()
+ except (EOFError, KeyboardInterrupt):
+ print()
+ break
+
+ # Capture end-state snapshot
+ print(" → Now manually operate the UI to reach the end state.")
+ try:
+ input(" Press Enter when ready to take snapshot...")
+ except (EOFError, KeyboardInterrupt):
+ print()
+ continue
+
+ snapshot_path = str(snap_dir / f"step_{step.index:03d}_end_state.png")
+ if self._capture_end_state_snapshot(snapshot_path):
+ step.agent_end_state_snapshot = snapshot_path
+ print(f" ✓ Snapshot saved: {snapshot_path}")
+ else:
+ print(" ⚠ Snapshot failed — no snapshot will be used for this step.")
+
+ print()
+ print("─" * 60)
+ agent_count = sum(1 for s in self._steps if s.is_agent_step)
+ fixed_count = len(self._steps) - agent_count
+ print(f" Review complete: {fixed_count} fixed, {agent_count} agent steps")
+ print("─" * 60)
+
+ def _capture_end_state_snapshot(self, output_path: str) -> bool:
+ """Capture the current screen and save as end-state snapshot."""
+ try:
+ import mss
+ from PIL import Image
+ Path(output_path).parent.mkdir(parents=True, exist_ok=True)
+ with mss.mss() as sct:
+ monitor = sct.monitors[1]
+ raw = sct.grab(monitor)
+ img = Image.frombytes("RGB", raw.size, raw.bgra, "raw", "BGRX")
+ img.save(output_path)
+ return True
+ except Exception as e:
+ print(f"[recorder] Snapshot failed: {e}", file=sys.stderr)
+ return False
+
# ── Parameterization ──────────────────────────────────────────────────────
def get_type_steps(self) -> list[tuple[int, "RecordedStep"]]:
diff --git a/macrocli/agent-harness/cli_anything/macrocli/core/routing.py b/macrocli/agent-harness/cli_anything/macrocli/core/routing.py
index 1658dbd6a..ff9a348d9 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/core/routing.py
+++ b/macrocli/agent-harness/cli_anything/macrocli/core/routing.py
@@ -5,6 +5,7 @@ Priority order (higher = preferred):
gui_macro 80
visual_anchor 75
file_transform 70
+ gui_agent 60
semantic_ui 50
recovery 10 (only via explicit backend: recovery, or auto-retry)
@@ -22,6 +23,7 @@ from cli_anything.macrocli.backends.semantic_ui import SemanticUIBackend
from cli_anything.macrocli.backends.gui_macro import GUIMacroBackend
from cli_anything.macrocli.backends.visual_anchor import VisualAnchorBackend
from cli_anything.macrocli.backends.recovery import RecoveryBackend
+from cli_anything.macrocli.backends.gui_agent import GUIAgentBackend
from cli_anything.macrocli.core.macro_model import MacroStep
@@ -30,6 +32,7 @@ _BACKEND_PRIORITY: dict[str, int] = {
"gui_macro": 80,
"visual_anchor": 75,
"file_transform": 70,
+ "gui_agent": 60,
"semantic_ui": 50,
"recovery": 10,
}
@@ -46,6 +49,7 @@ class RoutingEngine:
"semantic_ui": SemanticUIBackend(),
"gui_macro": GUIMacroBackend(),
"visual_anchor": VisualAnchorBackend(),
+ "gui_agent": GUIAgentBackend(),
"recovery": self._recovery,
}
# Wire recovery with all other backends so it can delegate
diff --git a/macrocli/agent-harness/cli_anything/macrocli/macrocli_cli.py b/macrocli/agent-harness/cli_anything/macrocli/macrocli_cli.py
index e47697310..322204560 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/macrocli_cli.py
+++ b/macrocli/agent-harness/cli_anything/macrocli/macrocli_cli.py
@@ -430,9 +430,12 @@ def macro_define(name, output):
@macro.command("record")
@click.argument("name")
@click.option("--output-dir", "-d", default=".",
- help="Directory to write <name>.yaml and <name>_templates/.")
+ help="Directory to write the macro package folder.")
@click.option("--timeout", default=0, type=float,
help="Auto-stop after N seconds (0 = wait for Ctrl+Alt+S).")
[email protected]("--agent-review", "do_agent_review", is_flag=True,
+ help="After recording, review each step and mark some as "
+ "agent steps with descriptions and end-state snapshots.")
@click.option("--parameterize", "do_parameterize", is_flag=True,
help="After recording, interactively choose which typed values "
"become CLI parameters.")
@@ -442,7 +445,140 @@ def macro_define(name, output):
@click.option("--api-key", default=None, envvar="GEMINI_API_KEY",
help="Gemini API key for --auto-parameterize.")
@handle_error
-def macro_record(name, output_dir, timeout, do_parameterize, do_auto_param, api_key):
+def macro_record(name, output_dir, timeout, do_agent_review,
+ do_parameterize, do_auto_param, api_key):
+ """Record GUI interactions and generate a macro YAML package.
+
+ \b
+ The macro is saved as a folder:
+ <name>/
+ macro.yaml
+ snapshots/ (end-state screenshots for agent steps)
+
+ \b
+ Examples:
+ # Basic recording
+ macro record my_export
+
+ # Record + mark agent steps interactively
+ macro record my_export --agent-review
+
+ # Record + agent review + parameterize typed values
+ macro record my_export --agent-review --parameterize
+
+ Requires: pip install mss Pillow pynput
+ """
+ try:
+ from cli_anything.macrocli.core.recorder import MacroRecorder
+ except ImportError as e:
+ click.echo(f"Error: {e}", err=True)
+ sys.exit(1)
+
+ if do_parameterize and do_auto_param:
+ click.echo(
+ "Error: --parameterize and --auto-parameterize are mutually exclusive.",
+ err=True,
+ )
+ sys.exit(1)
+
+ recorder = MacroRecorder(macro_name=name, output_dir=output_dir)
+
+ if not _json_output:
+ click.echo(f"Recording '{name}'. Press Ctrl+Alt+S to stop...")
+
+ try:
+ recorder.record(timeout_s=timeout if timeout > 0 else None)
+ except Exception as e:
+ if _json_output:
+ output({"error": str(e), "success": False})
+ else:
+ click.echo(f"Error during recording: {e}", err=True)
+ if not _repl_mode:
+ sys.exit(1)
+ return
+
+ # ── Agent step review ─────────────────────────────────────────────────────
+ if do_agent_review and recorder._steps:
+ snap_dir = Path(output_dir) / name / "snapshots"
+ recorder.interactive_agent_review(snapshots_dir=str(snap_dir))
+
+ # ── Parameterization phase ────────────────────────────────────────────────
+ parameters = None
+ type_steps = recorder.get_type_steps()
+
+ if do_auto_param and type_steps:
+ try:
+ from cli_anything.macrocli.core.parameterize import (
+ gemini_suggest_parameters,
+ interactive_parameterize,
+ )
+ if not _json_output:
+ click.echo(f"\nAsking Gemini to suggest parameters...")
+ suggestions = gemini_suggest_parameters(type_steps, api_key=api_key)
+ if suggestions and not _json_output:
+ click.echo(" Gemini suggestions:")
+ for idx, pname in suggestions.items():
+ step = recorder._steps[idx]
+ click.echo(f" step {idx+1} {step.text!r} → ${{{pname}}}")
+ click.echo()
+ confirmed = interactive_parameterize(
+ [(i, s) for i, s in type_steps if i in suggestions],
+ )
+ final = {**suggestions, **confirmed}
+ parameters = recorder.apply_parameterization(final)
+ elif not suggestions and not _json_output:
+ click.echo(" Gemini found no values to parameterize.")
+ except Exception as e:
+ click.echo(f" Warning: Gemini parameterization failed: {e}", err=True)
+ do_parameterize = True
+
+ if do_parameterize and type_steps:
+ from cli_anything.macrocli.core.parameterize import interactive_parameterize
+ assignments = interactive_parameterize(type_steps)
+ if assignments:
+ parameters = recorder.apply_parameterization(assignments)
+
+ # ── Save as package ───────────────────────────────────────────────────────
+ try:
+ yaml_path = recorder.save_as_package(
+ output_dir=output_dir,
+ parameters=parameters,
+ )
+ except Exception as e:
+ if _json_output:
+ output({"error": str(e), "success": False})
+ else:
+ click.echo(f"Error saving macro: {e}", err=True)
+ if not _repl_mode:
+ sys.exit(1)
+ return
+
+ pkg_dir = str(Path(output_dir) / name)
+ agent_count = sum(1 for s in recorder._steps if s.is_agent_step)
+
+ if _json_output:
+ output({
+ "success": True,
+ "yaml_path": yaml_path,
+ "package_dir": pkg_dir,
+ "steps": len(recorder._steps),
+ "agent_steps": agent_count,
+ "parameters": list((parameters or {}).keys()),
+ })
+ else:
+ click.echo(f"✓ Saved {len(recorder._steps)} steps to: {pkg_dir}/")
+ if agent_count:
+ click.echo(f" Agent steps: {agent_count} (will use Gemini at runtime)")
+ if parameters:
+ click.echo(f" Parameters: {', '.join(parameters.keys())}")
+ click.echo(
+ f"\n Run with:\n"
+ f" macro run {name} --macro-file {yaml_path}"
+ + (
+ "".join(f" --param {k}=<value>" for k in (parameters or {}))
+ if parameters else ""
+ )
+ )
"""Record GUI interactions and generate a macro YAML.
\b