Skip to content

"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/

Python

fix(macrocli): address remaining review blockers

haorui-harryfb767afe0af7
diff --git a/macrocli/agent-harness/cli_anything/macrocli/README.md b/macrocli/agent-harness/cli_anything/macrocli/README.md
index a0bd73eec..e52ccbe93 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/README.md
+++ b/macrocli/agent-harness/cli_anything/macrocli/README.md
@@ -13,6 +13,23 @@ pip install -e .
**Dependencies:** Python 3.10+, PyYAML, click, prompt-toolkit
+**Optional extras:**
+
+```bash
+pip install -e ".[visual]" # visual_anchor backend (mss, Pillow, numpy, pynput)
+pip install -e ".[gui_agent]" # gui_agent backend (openai, mss, Pillow)
+pip install -e ".[all]" # everything
+```
+
+The `gui_agent` backend uses the OpenAI SDK and is compatible with any
+OpenAI-compatible API. Configure via environment variables:
+
+| Variable | Description |
+|--------------------|---------------------------------------------|
+| `MACROCLI_MODEL` | Model name (required, e.g. `gpt-4o`) |
+| `MACROCLI_API_KEY` | API key for the provider |
+| `MACROCLI_BASE_URL`| Base URL (only needed for non-OpenAI hosts) |
+
## Usage
```bash
@@ -63,6 +80,8 @@ cli-anything-macrocli (CLI)
- `native_api` — subprocess / shell commands
- `file_transform` — XML, JSON, text file editing
- `semantic_ui` — accessibility controls + keyboard shortcuts
+- `visual_anchor` — template-matching click/type (requires `[visual]`)
+- `gui_agent` — vision-model-driven automation (requires `[gui_agent]`)
- `gui_macro` — precompiled coordinate-based replay
- `recovery` — retry + fallback orchestration
diff --git a/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
index e38ce75e6..cdef55ea2 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
+++ b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py
@@ -1,5 +1,5 @@
"""GUIAgentBackend — execute a macro step by letting a vision model
-(Gemini or Claude) look at the screen and decide what to do.
+look at the screen and decide what to do.
This backend is used for steps that cannot be expressed as fixed
coordinates or hotkeys because the interface state is unpredictable.
@@ -18,10 +18,15 @@ At runtime the backend:
6. Asks model: "have we reached the end state?"
7. Loops until end state reached or max_steps exceeded
-Supported models:
- - gemini-1.5-flash (default, fastest)
- - gemini-1.5-pro
- - gemini-2.0-flash
+The backend uses the OpenAI SDK, which is compatible with any
+OpenAI-compatible API provider (OpenAI, Azure, local vLLM, Ollama,
+LiteLLM, etc.). Configure model and endpoint via environment
+variables or per-step YAML params:
+
+ Environment variables:
+ MACROCLI_MODEL — model name (required, no default)
+ MACROCLI_API_KEY — API key
+ MACROCLI_BASE_URL — base URL for non-OpenAI providers
Example YAML step:
@@ -36,8 +41,9 @@ Example YAML step:
Format dropdown shows PNG, Resolution input shows 300.
end_state_snapshot: snapshots/step_003_end_state.png
max_steps: 8
- model: gemini-1.5-flash
- api_key: ${GEMINI_API_KEY}
+ model: ${MACROCLI_MODEL}
+ api_key: ${MACROCLI_API_KEY}
+ base_url: ${MACROCLI_BASE_URL}
"""
from __future__ import annotations
@@ -190,7 +196,7 @@ def _execute_action(action_dict: dict, context: BackendContext) -> None:
# ── Backend ───────────────────────────────────────────────────────────────────
class GUIAgentBackend(Backend):
- """Execute GUI steps using a vision model (Gemini) to decide actions."""
+ """Execute GUI steps using a vision model (OpenAI-compatible API) to decide actions."""
name = "gui_agent"
priority = 60 # between semantic_ui(50) and file_transform(70)
@@ -251,10 +257,18 @@ class GUIAgentBackend(Backend):
end_state_desc: str = p.get("end_state_description", "")
snapshot_path: str = p.get("end_state_snapshot", "")
window_title: str = p.get("window_title", "") # focus this window first
- model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", "gemini-2.5-flash"))
+ model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", ""))
api_key: str = p.get("api_key", os.environ.get("MACROCLI_API_KEY", ""))
base_url: str = p.get("base_url", os.environ.get("MACROCLI_BASE_URL", ""))
+ if not model_name:
+ return StepResult(
+ success=False,
+ error="GUIAgentBackend: model required. Set MACROCLI_MODEL env var or pass model in step params.",
+ backend_used=self.name,
+ duration_ms=(time.time() - t0) * 1000,
+ )
+
if not api_key:
return StepResult(
success=False,
@@ -441,7 +455,7 @@ class GUIAgentBackend(Backend):
# ── Round 2: compare and refine ───────────────────────────────────────
description: str = p.get("description", "")
end_state_desc: str = p.get("end_state_description", "")
- model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", "gemini-2.5-flash"))
+ model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", ""))
api_key: str = p.get("api_key", os.environ.get("MACROCLI_API_KEY", ""))
base_url: str = p.get("base_url", os.environ.get("MACROCLI_BASE_URL", ""))
diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml
deleted file mode 100644
index 6014027f9..000000000
--- a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml
+++ /dev/null
@@ -1,187 +0,0 @@
-name: kolourpaint_annotate
-version: "1.0"
-description: >
- Open a screenshot in KolourPaint, let the AI agent identify and draw
- a red rectangle around the area matching the annotation goal,
- then save the annotated image.
-
- This macro demonstrates the hybrid fixed+agent approach:
- - Fixed steps handle application launch, tool selection, color selection, save
- - The agent step handles the annotation position decision (purely visual)
-
-tags: [demo, kolourpaint, annotation, gui_agent, visual_anchor]
-
-parameters:
- input_file:
- type: string
- required: true
- description: Path to the screenshot PNG to annotate.
- example: /tmp/screenshot.png
- output_file:
- type: string
- required: true
- description: Path to save the annotated PNG.
- example: /tmp/annotated.png
- annotation_goal:
- type: string
- required: false
- default: "the error or problem in the image"
- description: >
- Natural language description of what to annotate.
- The AI agent will decide where to draw the rectangle.
- example: "the error message in red text"
-
-preconditions:
- - file_exists: ${input_file}
- - file_exists: /usr/bin/kolourpaint
-
-steps:
- # ── Fixed: launch KolourPaint with the input file ─────────────────────────
- - id: launch_kolourpaint
- backend: native_api
- action: start_process
- params:
- command: [kolourpaint, "${input_file}"]
- log_file: /tmp/kolourpaint.log
- env:
- DISPLAY: ":99"
- on_failure: fail
-
- - id: wait_kolourpaint
- backend: native_api
- action: run_command
- params:
- command: [sleep, "2.5"]
- on_failure: skip
-
- # ── Fixed: select Rectangle tool (hotkey r) ───────────────────────────────
- - id: select_rectangle_tool
- backend: semantic_ui
- action: focus_window
- params:
- title_contains: KolourPaint
- on_failure: fail
-
- - id: press_rectangle_hotkey
- backend: visual_anchor
- action: hotkey
- params:
- keys: r
- on_failure: fail
-
- # ── Fixed: select red color (click red swatch in color palette) ───────────
- - id: select_red_color
- backend: visual_anchor
- action: click_relative
- params:
- window_title: KolourPaint
- x_pct: 0.41
- y_pct: 0.72
- on_failure: fail
-
- - id: pause_after_color
- backend: native_api
- action: run_command
- params:
- command: [sleep, "0.3"]
- on_failure: skip
-
- # ── Agent: decide WHERE to draw the rectangle ─────────────────────────────
- # The agent looks at the screenshot content and decides the coordinates
- # of the annotation rectangle based on the annotation_goal.
- - id: agent_draw_annotation
- backend: gui_agent
- action: instruct
- params:
- window_title: KolourPaint
- description: >
- The Rectangle drawing tool is active and a color is selected in KolourPaint.
- The image canvas occupies this exact screen region:
- Top-left corner: (282, 95)
- Top-right corner: (980, 95)
- Bottom-left corner: (282, 715)
- Bottom-right corner: (980, 715)
- The canvas shows a screenshot. Find the area matching: ${annotation_goal}
- Draw a rectangle around it using the drag action.
- The drag coordinates MUST be within the canvas bounds above.
- end_state_description: >
- A rectangle is drawn around ${annotation_goal} on the canvas.
- end_state_snapshot: snapshots/annotated_end_state.png
- on_failure: fail
-
- - id: pause_after_draw
- backend: native_api
- action: run_command
- params:
- command: [sleep, "0.5"]
- on_failure: skip
-
- # ── Fixed: save as output_file ────────────────────────────────────────────
- - id: open_save_as
- backend: visual_anchor
- action: hotkey
- params:
- keys: ctrl+shift+s
- on_failure: fail
-
- - id: wait_save_dialog
- backend: semantic_ui
- action: wait_for_window
- params:
- title_contains: Save As
- timeout_ms: 4000
- on_failure: skip
-
- - id: clear_filename
- backend: visual_anchor
- action: hotkey
- params:
- keys: ctrl+l
- on_failure: skip
-
- - id: clear_path_field
- backend: visual_anchor
- action: hotkey
- params:
- keys: ctrl+a
- on_failure: skip
-
- - id: type_output_path
- backend: visual_anchor
- action: type_text
- params:
- text: "${output_file}"
- interval_ms: 30
- on_failure: fail
-
- - id: confirm_save
- backend: visual_anchor
- action: hotkey
- params:
- keys: enter
- on_failure: fail
-
- - id: wait_save_complete
- backend: native_api
- action: run_command
- params:
- command: [sleep, "1.0"]
- on_failure: skip
-
-postconditions:
- - file_exists: ${output_file}
-
-outputs:
- - name: annotated_file
- path: ${output_file}
- description: Path to the annotated screenshot.
-
-agent_hints:
- danger_level: safe
- side_effects: [creates_file, opens_application]
- reversible: true
- estimated_duration_ms: 15000
- note: >
- The annotation position is decided by the AI agent based on the
- screenshot content and annotation_goal. This cannot be done with
- fixed coordinates because the target area varies per screenshot.
diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png
deleted file mode 100644
index 9459f653c..000000000
Binary files a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png and /dev/null differ
diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml
index 27c0437b6..4fef107c4 100644
--- a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml
+++ b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml
@@ -21,11 +21,3 @@ macros:
- name: gedit_find_and_replace
path: demo/gedit_find_and_replace.yaml
version: "1.0"
- # ── Demo macros (flameshot) ───────────────────────────────────
- - name: flameshot_annotate
- path: demo/flameshot_annotate.yaml
- version: "1.0"
- # ── Demo macros (kolourpaint) ─────────────────────────────────
- - name: kolourpaint_annotate
- path: demo/kolourpaint_annotate.yaml
- version: "1.0"
diff --git a/macrocli/agent-harness/setup.py b/macrocli/agent-harness/setup.py
index cbbe44d87..c73064e2d 100644
--- a/macrocli/agent-harness/setup.py
+++ b/macrocli/agent-harness/setup.py
@@ -50,12 +50,7 @@ setup(
"numpy>=1.24.0",
"pynput>=1.7.0",
],
- "gemini": [
- "google-generativeai>=0.8.0",
- "mss>=9.0.0",
- "Pillow>=10.0.0",
- ],
- "openai": [
+ "gui_agent": [
"openai>=1.0.0",
"mss>=9.0.0",
"Pillow>=10.0.0",
@@ -65,7 +60,6 @@ setup(
"Pillow>=10.0.0",
"numpy>=1.24.0",
"pynput>=1.7.0",
- "google-generativeai>=0.8.0",
"openai>=1.0.0",
],
},
@@ -80,9 +74,6 @@ setup(
"macro_definitions/*.yaml",
"macro_definitions/examples/*.yaml",
"macro_definitions/demo/*.yaml",
- "macro_definitions/demo/**/*.png",
- "macro_definitions/demo/flameshot_templates/*.png",
- "macro_definitions/demo/snapshots/*.png",
],
},
include_package_data=True,