"CLI-Anything: Making ALL Software Agent-Native" -- CLI-Hub: https://clianything.cc/
Python
fix(macrocli): address remaining review blockers
haorui-harryfb767afe0af7
diff --git a/macrocli/agent-harness/cli_anything/macrocli/README.md b/macrocli/agent-harness/cli_anything/macrocli/README.mdindex a0bd73eec..e52ccbe93 100644--- a/macrocli/agent-harness/cli_anything/macrocli/README.md+++ b/macrocli/agent-harness/cli_anything/macrocli/README.md@@ -13,6 +13,23 @@ pip install -e .**Dependencies:** Python 3.10+, PyYAML, click, prompt-toolkit+**Optional extras:**++```bash+pip install -e ".[visual]" # visual_anchor backend (mss, Pillow, numpy, pynput)+pip install -e ".[gui_agent]" # gui_agent backend (openai, mss, Pillow)+pip install -e ".[all]" # everything+```++The `gui_agent` backend uses the OpenAI SDK and is compatible with any+OpenAI-compatible API. Configure via environment variables:++| Variable | Description |+|--------------------|---------------------------------------------|+| `MACROCLI_MODEL` | Model name (required, e.g. `gpt-4o`) |+| `MACROCLI_API_KEY` | API key for the provider |+| `MACROCLI_BASE_URL`| Base URL (only needed for non-OpenAI hosts) |+## Usage```bash@@ -63,6 +80,8 @@ cli-anything-macrocli (CLI)- `native_api` — subprocess / shell commands- `file_transform` — XML, JSON, text file editing- `semantic_ui` — accessibility controls + keyboard shortcuts+- `visual_anchor` — template-matching click/type (requires `[visual]`)+- `gui_agent` — vision-model-driven automation (requires `[gui_agent]`)- `gui_macro` — precompiled coordinate-based replay- `recovery` — retry + fallback orchestrationdiff --git a/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.pyindex e38ce75e6..cdef55ea2 100644--- a/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py+++ b/macrocli/agent-harness/cli_anything/macrocli/backends/gui_agent.py@@ -1,5 +1,5 @@"""GUIAgentBackend — execute a macro step by letting a vision model-(Gemini or Claude) look at the screen and decide what to do.+look at the screen and decide what to do.This backend is used for steps that cannot be expressed as fixedcoordinates or hotkeys because the interface state is unpredictable.@@ -18,10 +18,15 @@ At runtime the backend:6. Asks model: "have we reached the end state?"7. Loops until end state reached or max_steps exceeded-Supported models:- - gemini-1.5-flash (default, fastest)- - gemini-1.5-pro- - gemini-2.0-flash+The backend uses the OpenAI SDK, which is compatible with any+OpenAI-compatible API provider (OpenAI, Azure, local vLLM, Ollama,+LiteLLM, etc.). Configure model and endpoint via environment+variables or per-step YAML params:++ Environment variables:+ MACROCLI_MODEL — model name (required, no default)+ MACROCLI_API_KEY — API key+ MACROCLI_BASE_URL — base URL for non-OpenAI providersExample YAML step:@@ -36,8 +41,9 @@ Example YAML step:Format dropdown shows PNG, Resolution input shows 300.end_state_snapshot: snapshots/step_003_end_state.pngmax_steps: 8- model: gemini-1.5-flash- api_key: ${GEMINI_API_KEY}+ model: ${MACROCLI_MODEL}+ api_key: ${MACROCLI_API_KEY}+ base_url: ${MACROCLI_BASE_URL}"""from __future__ import annotations@@ -190,7 +196,7 @@ def _execute_action(action_dict: dict, context: BackendContext) -> None:# ── Backend ───────────────────────────────────────────────────────────────────class GUIAgentBackend(Backend):- """Execute GUI steps using a vision model (Gemini) to decide actions."""+ """Execute GUI steps using a vision model (OpenAI-compatible API) to decide actions."""name = "gui_agent"priority = 60 # between semantic_ui(50) and file_transform(70)@@ -251,10 +257,18 @@ class GUIAgentBackend(Backend):end_state_desc: str = p.get("end_state_description", "")snapshot_path: str = p.get("end_state_snapshot", "")window_title: str = p.get("window_title", "") # focus this window first- model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", "gemini-2.5-flash"))+ model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", ""))api_key: str = p.get("api_key", os.environ.get("MACROCLI_API_KEY", ""))base_url: str = p.get("base_url", os.environ.get("MACROCLI_BASE_URL", ""))+ if not model_name:+ return StepResult(+ success=False,+ error="GUIAgentBackend: model required. Set MACROCLI_MODEL env var or pass model in step params.",+ backend_used=self.name,+ duration_ms=(time.time() - t0) * 1000,+ )+if not api_key:return StepResult(success=False,@@ -441,7 +455,7 @@ class GUIAgentBackend(Backend):# ── Round 2: compare and refine ───────────────────────────────────────description: str = p.get("description", "")end_state_desc: str = p.get("end_state_description", "")- model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", "gemini-2.5-flash"))+ model_name: str = p.get("model", os.environ.get("MACROCLI_MODEL", ""))api_key: str = p.get("api_key", os.environ.get("MACROCLI_API_KEY", ""))base_url: str = p.get("base_url", os.environ.get("MACROCLI_BASE_URL", ""))diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yamldeleted file mode 100644index 6014027f9..000000000--- a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/kolourpaint_annotate.yaml+++ /dev/null@@ -1,187 +0,0 @@-name: kolourpaint_annotate-version: "1.0"-description: >- Open a screenshot in KolourPaint, let the AI agent identify and draw- a red rectangle around the area matching the annotation goal,- then save the annotated image.-- This macro demonstrates the hybrid fixed+agent approach:- - Fixed steps handle application launch, tool selection, color selection, save- - The agent step handles the annotation position decision (purely visual)--tags: [demo, kolourpaint, annotation, gui_agent, visual_anchor]--parameters:- input_file:- type: string- required: true- description: Path to the screenshot PNG to annotate.- example: /tmp/screenshot.png- output_file:- type: string- required: true- description: Path to save the annotated PNG.- example: /tmp/annotated.png- annotation_goal:- type: string- required: false- default: "the error or problem in the image"- description: >- Natural language description of what to annotate.- The AI agent will decide where to draw the rectangle.- example: "the error message in red text"--preconditions:- - file_exists: ${input_file}- - file_exists: /usr/bin/kolourpaint--steps:- # ── Fixed: launch KolourPaint with the input file ─────────────────────────- - id: launch_kolourpaint- backend: native_api- action: start_process- params:- command: [kolourpaint, "${input_file}"]- log_file: /tmp/kolourpaint.log- env:- DISPLAY: ":99"- on_failure: fail-- - id: wait_kolourpaint- backend: native_api- action: run_command- params:- command: [sleep, "2.5"]- on_failure: skip-- # ── Fixed: select Rectangle tool (hotkey r) ───────────────────────────────- - id: select_rectangle_tool- backend: semantic_ui- action: focus_window- params:- title_contains: KolourPaint- on_failure: fail-- - id: press_rectangle_hotkey- backend: visual_anchor- action: hotkey- params:- keys: r- on_failure: fail-- # ── Fixed: select red color (click red swatch in color palette) ───────────- - id: select_red_color- backend: visual_anchor- action: click_relative- params:- window_title: KolourPaint- x_pct: 0.41- y_pct: 0.72- on_failure: fail-- - id: pause_after_color- backend: native_api- action: run_command- params:- command: [sleep, "0.3"]- on_failure: skip-- # ── Agent: decide WHERE to draw the rectangle ─────────────────────────────- # The agent looks at the screenshot content and decides the coordinates- # of the annotation rectangle based on the annotation_goal.- - id: agent_draw_annotation- backend: gui_agent- action: instruct- params:- window_title: KolourPaint- description: >- The Rectangle drawing tool is active and a color is selected in KolourPaint.- The image canvas occupies this exact screen region:- Top-left corner: (282, 95)- Top-right corner: (980, 95)- Bottom-left corner: (282, 715)- Bottom-right corner: (980, 715)- The canvas shows a screenshot. Find the area matching: ${annotation_goal}- Draw a rectangle around it using the drag action.- The drag coordinates MUST be within the canvas bounds above.- end_state_description: >- A rectangle is drawn around ${annotation_goal} on the canvas.- end_state_snapshot: snapshots/annotated_end_state.png- on_failure: fail-- - id: pause_after_draw- backend: native_api- action: run_command- params:- command: [sleep, "0.5"]- on_failure: skip-- # ── Fixed: save as output_file ────────────────────────────────────────────- - id: open_save_as- backend: visual_anchor- action: hotkey- params:- keys: ctrl+shift+s- on_failure: fail-- - id: wait_save_dialog- backend: semantic_ui- action: wait_for_window- params:- title_contains: Save As- timeout_ms: 4000- on_failure: skip-- - id: clear_filename- backend: visual_anchor- action: hotkey- params:- keys: ctrl+l- on_failure: skip-- - id: clear_path_field- backend: visual_anchor- action: hotkey- params:- keys: ctrl+a- on_failure: skip-- - id: type_output_path- backend: visual_anchor- action: type_text- params:- text: "${output_file}"- interval_ms: 30- on_failure: fail-- - id: confirm_save- backend: visual_anchor- action: hotkey- params:- keys: enter- on_failure: fail-- - id: wait_save_complete- backend: native_api- action: run_command- params:- command: [sleep, "1.0"]- on_failure: skip--postconditions:- - file_exists: ${output_file}--outputs:- - name: annotated_file- path: ${output_file}- description: Path to the annotated screenshot.--agent_hints:- danger_level: safe- side_effects: [creates_file, opens_application]- reversible: true- estimated_duration_ms: 15000- note: >- The annotation position is decided by the AI agent based on the- screenshot content and annotation_goal. This cannot be done with- fixed coordinates because the target area varies per screenshot.diff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.pngdeleted file mode 100644index 9459f653c..000000000Binary files a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/demo/snapshots/kolourpaint_annotate_end_state.png and /dev/null differdiff --git a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yamlindex 27c0437b6..4fef107c4 100644--- a/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml+++ b/macrocli/agent-harness/cli_anything/macrocli/macro_definitions/manifest.yaml@@ -21,11 +21,3 @@ macros:- name: gedit_find_and_replacepath: demo/gedit_find_and_replace.yamlversion: "1.0"- # ── Demo macros (flameshot) ───────────────────────────────────- - name: flameshot_annotate- path: demo/flameshot_annotate.yaml- version: "1.0"- # ── Demo macros (kolourpaint) ─────────────────────────────────- - name: kolourpaint_annotate- path: demo/kolourpaint_annotate.yaml- version: "1.0"diff --git a/macrocli/agent-harness/setup.py b/macrocli/agent-harness/setup.pyindex cbbe44d87..c73064e2d 100644--- a/macrocli/agent-harness/setup.py+++ b/macrocli/agent-harness/setup.py@@ -50,12 +50,7 @@ setup("numpy>=1.24.0","pynput>=1.7.0",],- "gemini": [- "google-generativeai>=0.8.0",- "mss>=9.0.0",- "Pillow>=10.0.0",- ],- "openai": [+ "gui_agent": ["openai>=1.0.0","mss>=9.0.0","Pillow>=10.0.0",@@ -65,7 +60,6 @@ setup("Pillow>=10.0.0","numpy>=1.24.0","pynput>=1.7.0",- "google-generativeai>=0.8.0","openai>=1.0.0",],},@@ -80,9 +74,6 @@ setup("macro_definitions/*.yaml","macro_definitions/examples/*.yaml","macro_definitions/demo/*.yaml",- "macro_definitions/demo/**/*.png",- "macro_definitions/demo/flameshot_templates/*.png",- "macro_definitions/demo/snapshots/*.png",],},include_package_data=True,