Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

refactor(agent): fix fara implementation

[email protected]e94b0e850ef4
diff --git a/libs/python/agent/agent/loops/fara/__init__.py b/libs/python/agent/agent/loops/fara/__init__.py
new file mode 100644
index 0000000000..82a4ab9182
--- /dev/null
+++ b/libs/python/agent/agent/loops/fara/__init__.py
@@ -0,0 +1,8 @@
+"""
+FARA-7B agent loop implementation.
+Original implementation from Microsoft: https://github.com/microsoft/Fara
+"""
+
+from .config import FaraVlmConfig
+
+__all__ = ("FaraVlmConfig", )
diff --git a/libs/python/agent/agent/loops/fara.py b/libs/python/agent/agent/loops/fara/config.py
similarity index 71%
rename from libs/python/agent/agent/loops/fara.py
rename to libs/python/agent/agent/loops/fara/config.py
index eaa2d45ce7..ffcab1ad25 100644
--- a/libs/python/agent/agent/loops/fara.py
+++ b/libs/python/agent/agent/loops/fara/config.py
@@ -1,13 +1,8 @@
-"""
-Qwen3-VL agent loop implementation using litellm with function/tool calling.
-- Passes a ComputerUse tool schema to acompletion
-- Converts between Responses items and completion messages using helpers
-"""
+"""FARA VLM agent configuration."""
from __future__ import annotations
import json
-import re
from typing import Any, Dict, List, Optional, Tuple
import litellm
@@ -15,223 +10,20 @@ from litellm.responses.litellm_completion_transformation.transformation import (
LiteLLMCompletionResponsesConfig,
)
-from ..decorators import register_agent
-from ..loops.base import AsyncAgentConfig
-from ..responses import (
+from ...decorators import register_agent
+from ...loops.base import AsyncAgentConfig
+from ...responses import (
convert_completion_messages_to_responses_items,
convert_responses_items_to_completion_messages,
make_reasoning_item,
)
-from ..types import AgentCapability
-
-# ComputerUse tool schema (OpenAI function tool format)
-QWEN3_COMPUTER_TOOL: Dict[str, Any] = {
- "type": "function",
- "function": {
- "name": "computer",
- "description": (
- "Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
- "* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n"
- "* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\n"
- "* The screen's resolution is 1000x1000.\n"
- "* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n"
- "* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n"
- "* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges."
- ),
- "parameters": {
- "type": "object",
- "properties": {
- "action": {
- "description": "The action to perform.",
- "enum": [
- "key",
- "type",
- "mouse_move",
- "left_click",
- "left_click_drag",
- "right_click",
- "middle_click",
- "double_click",
- "triple_click",
- "scroll",
- "hscroll",
- "screenshot",
- "wait",
- # "terminate",
- # "answer",
- ],
- "type": "string",
- },
- "keys": {
- "description": "Required only by action=key.",
- "type": "array",
- "items": {"type": "string"},
- },
- "text": {
- "description": "Required only by action=type and action=answer.",
- "type": "string",
- },
- "coordinate": {
- "description": "(x, y): Pixel coordinates from top-left.",
- "type": "array",
- "items": {"type": ["number", "integer"]},
- "minItems": 2,
- "maxItems": 2,
- },
- "pixels": {
- "description": "Scroll amount. Positive=up, negative=down. For scroll/hscroll.",
- "type": "number",
- },
- "time": {
- "description": "Seconds to wait (action=wait).",
- "type": "number",
- },
- # "status": {
- # "description": "Task status (action=terminate).",
- # "type": "string",
- # "enum": ["success", "failure"],
- # },
- },
- "required": ["action"],
- },
- },
-}
-
-
-def _build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
- """Use qwen-agent NousFnCallPrompt to generate a system message embedding tool schema."""
- try:
- from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
- ContentItem as NousContentItem,
- )
- from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
- Message as NousMessage,
- )
- from qwen_agent.llm.fncall_prompts.nous_fncall_prompt import (
- NousFnCallPrompt,
- )
- except ImportError:
- raise ImportError(
- "qwen-agent not installed. Please install it with `pip install cua-agent[qwen]`."
- )
- msgs = NousFnCallPrompt().preprocess_fncall_messages(
- messages=[
- NousMessage(
- role="system", content=[NousContentItem(text="You are a helpful assistant.")]
- )
- ],
- functions=functions,
- lang="en",
- )
- sys = msgs[0].model_dump()
- # Convert qwen-agent structured content to OpenAI-style content list
- content = [{"type": "text", "text": c["text"]} for c in sys.get("content", [])]
- return {"role": "system", "content": content}
-
-
-def _parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
- """Extract JSON object within <tool_call>...</tool_call> from model text."""
- m = re.search(r"<tool_call>\s*(\{[\s\S]*?\})\s*</tool_call>", text)
- if not m:
- return None
- try:
- return json.loads(m.group(1))
- except Exception:
- return None
-
-
-async def _unnormalize_coordinate(args: Dict[str, Any], dims: Tuple[int, int]) -> Dict[str, Any]:
- """Coordinates appear in 0..1000 space, scale to actual screen size using dims if provided."""
- coord = args.get("coordinate")
- if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
- return args
- x, y = float(coord[0]), float(coord[1])
- width, height = float(dims[0]), float(dims[1])
- x_abs = max(0.0, min(width, (x / 1000.0) * width))
- y_abs = max(0.0, min(height, (y / 1000.0) * height))
- args = {**args, "coordinate": [round(x_abs), round(y_abs)]}
- return args
-
-
-def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
- """
- Convert Qwen computer tool arguments to the Computer Calls action schema.
-
- Qwen (example):
- {"action": "left_click", "coordinate": [114, 68]}
-
- Target (example):
- {"action": "left_click", "x": 114, "y": 68}
-
- Other mappings:
- - right_click, middle_click, double_click (triple_click -> double_click)
- - mouse_move -> { action: "move", x, y }
- - key -> { action: "keypress", keys: [...] }
- - type -> { action: "type", text }
- - scroll/hscroll -> { action: "scroll", scroll_x, scroll_y, x, y }
- - wait -> { action: "wait" }
- - terminate/answer are not direct UI actions; return None for now
- """
- if not isinstance(args, dict):
- return None
-
- action = args.get("action")
- if not isinstance(action, str):
- return None
-
- # Coordinates helper
- coord = args.get("coordinate")
- x = y = None
- if isinstance(coord, (list, tuple)) and len(coord) >= 2:
- try:
- x = int(round(float(coord[0])))
- y = int(round(float(coord[1])))
- except Exception:
- x = y = None
-
- # Map actions
- a = action.lower()
- if a in {"left_click", "right_click", "middle_click", "double_click"}:
- if x is None or y is None:
- return None
- return {"action": a, "x": x, "y": y}
- if a == "triple_click":
- # Approximate as double_click
- if x is None or y is None:
- return None
- return {"action": "double_click", "x": x, "y": y}
- if a == "mouse_move":
- if x is None or y is None:
- return None
- return {"action": "move", "x": x, "y": y}
- if a == "key":
- keys = args.get("keys")
- if isinstance(keys, list) and all(isinstance(k, str) for k in keys):
- return {"action": "keypress", "keys": keys}
- return None
- if a == "type":
- text = args.get("text")
- if isinstance(text, str):
- return {"action": "type", "text": text}
- return None
- if a in {"scroll", "hscroll"}:
- pixels = args.get("pixels") or 0
- try:
- pixels_val = int(round(float(pixels)))
- except Exception:
- pixels_val = 0
- scroll_x = pixels_val if a == "hscroll" else 0
- scroll_y = pixels_val if a == "scroll" else 0
- # Include cursor position if available (optional)
- out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
- if x is not None and y is not None:
- out.update({"x": x, "y": y})
- return out
- if a == "wait":
- return {"action": "wait"}
-
- # Non-UI or terminal actions: terminate/answer -> not mapped here
- return None
+from ...types import AgentCapability
+from .helpers import (
+ build_nous_system,
+ convert_qwen_tool_args_to_computer_action,
+ parse_tool_call_from_text,
+ unnormalize_coordinate,
+)
@register_agent(models=r"(?i).*fara-7b.*")
@@ -260,7 +52,7 @@ class FaraVlmConfig(AsyncAgentConfig):
# Build function schemas from tools array
function_schemas = []
if tools:
- from ..computers import is_agent_computer
+ from ...computers import is_agent_computer
for tool in tools:
tool_type = tool.get("type")
@@ -281,11 +73,10 @@ class FaraVlmConfig(AsyncAgentConfig):
function_schemas = [QWEN3_COMPUTER_TOOL["function"]]
# Prepend Nous-generated system if available
- nous_system = _build_nous_system(function_schemas)
+ nous_system = build_nous_system(function_schemas)
completion_messages = ([nous_system] if nous_system else []) + converted_msgs
# If there is no screenshot in the conversation, take one now and inject it.
- # Also record a pre_output_items assistant message to reflect action.
def _has_any_image(msgs: List[Dict[str, Any]]) -> bool:
for m in msgs:
content = m.get("content")
@@ -304,6 +95,7 @@ class FaraVlmConfig(AsyncAgentConfig):
screenshot_b64 = await computer_handler.screenshot()
if not screenshot_b64:
raise RuntimeError("Failed to capture screenshot from computer_handler.")
+
await _on_screenshot(screenshot_b64, "screenshot_before")
# Check if computer_handler has get_current_url method
@@ -315,23 +107,19 @@ class FaraVlmConfig(AsyncAgentConfig):
except Exception:
# If get_current_url fails, fall back to default text
pass
- else:
- print(computer_handler)
- print("HAS ATTR get_current_url", hasattr(computer_handler, "get_current_url"))
# Inject a user message with the screenshot so the model can see current context
- completion_messages.append(
- {
- "role": "user",
- "content": [
- {
- "type": "image_url",
- "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
- },
- {"type": "text", "text": screenshot_text},
- ],
- }
- )
+ screenshot_msg = {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": f"data:image/png;base64,{screenshot_b64}"},
+ },
+ {"type": "text", "text": screenshot_text},
+ ],
+ }
+ completion_messages.append(screenshot_msg)
# Smart-resize all screenshots and attach min/max pixel hints. Fail fast if deps missing.
# Also record the last resized width/height to unnormalize coordinates later.
@@ -413,7 +201,7 @@ class FaraVlmConfig(AsyncAgentConfig):
output_items.append(make_reasoning_item(reasoning_text))
# Priority 1: Try to parse tool call from content text (OpenRouter format)
- tool_call = _parse_tool_call_from_text(content_text)
+ tool_call = parse_tool_call_from_text(content_text)
if tool_call and isinstance(tool_call, dict):
fn_name = tool_call.get("name") or "computer"
@@ -423,11 +211,17 @@ class FaraVlmConfig(AsyncAgentConfig):
raise RuntimeError(
"No screenshots found to derive dimensions for coordinate unnormalization."
)
- args = await _unnormalize_coordinate(raw_args, (last_rw, last_rh))
+ args = await unnormalize_coordinate(raw_args, (last_rw, last_rh))
+
+ # Extract thoughts (text before <tool_call> tag)
+ thoughts = ""
+ if "<tool_call>" in content_text:
+ thoughts = content_text.split("<tool_call>")[0].strip()
# Build an OpenAI-style tool call so we can reuse the converter
fake_cm = {
"role": "assistant",
+ "content": thoughts, # Preserve thoughts before tool call
"tool_calls": [
{
"type": "function",
@@ -454,7 +248,7 @@ class FaraVlmConfig(AsyncAgentConfig):
# Unnormalize coordinates if present
if "coordinate" in args and last_rw is not None and last_rh is not None:
- args = await _unnormalize_coordinate(args, (last_rw, last_rh))
+ args = await unnormalize_coordinate(args, (last_rw, last_rh))
# Convert Qwen format to Computer Calls format if this is a computer tool
if fn_name == "computer":
@@ -487,6 +281,34 @@ class FaraVlmConfig(AsyncAgentConfig):
fake_cm = {"role": "assistant", "content": content_text}
output_items.extend(convert_completion_messages_to_responses_items([fake_cm]))
+ # Check if this is a terminate action - if so, add a final assistant message to stop the loop
+ has_terminate = False
+ for item in output_items:
+ if item.get("type") in ("computer_call", "function_call"):
+ try:
+ args = json.loads(item.get("arguments", "{}"))
+ if args.get("action") == "terminate":
+ has_terminate = True
+ break
+ except:
+ pass
+
+ # If terminate detected, ensure there's a final assistant message to exit the loop
+ # The generic agent loop checks if last item has role="assistant" to determine if it should continue
+ if has_terminate:
+ # Check if there's already an assistant message with the thoughts
+ has_assistant_message = any(
+ item.get("type") == "message" and item.get("role") == "assistant"
+ for item in output_items
+ )
+ if not has_assistant_message:
+ # Add a simple assistant message to signal completion
+ output_items.append({
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "output_text", "text": ""}]
+ })
+
# Prepend any pre_output_items (e.g., simulated screenshot-taking message)
return {"output": (pre_output_items + output_items), "usage": usage}
@@ -525,7 +347,7 @@ class FaraVlmConfig(AsyncAgentConfig):
}
# Build Nous system (lazy import inside helper already raises clear guidance if missing)
- nous_system = _build_nous_system([reduced_tool["function"]])
+ nous_system = build_nous_system([reduced_tool["function"]])
# Pre-process using smart_resize
min_pixels = 3136
@@ -577,10 +399,77 @@ class FaraVlmConfig(AsyncAgentConfig):
resp = response.model_dump() # type: ignore
choice = (resp.get("choices") or [{}])[0]
content_text = ((choice.get("message") or {}).get("content")) or ""
- tool_call = _parse_tool_call_from_text(content_text) or {}
+ tool_call = parse_tool_call_from_text(content_text) or {}
args = tool_call.get("arguments") or {}
- args = await _unnormalize_coordinate(args, (rh, rw))
+ args = await unnormalize_coordinate(args, (rh, rw))
coord = args.get("coordinate")
if isinstance(coord, (list, tuple)) and len(coord) >= 2:
return int(coord[0]), int(coord[1])
return None
+
+
+# ComputerUse tool schema (OpenAI function tool format)
+QWEN3_COMPUTER_TOOL: dict[str, Any] = {
+ "type": "function",
+ "function": {
+ "name": "computer",
+ "description": (
+ "Use a mouse and keyboard to interact with a computer, and take screenshots.\n"
+ "* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.\n"
+ "* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.\n"
+ "* The screen's resolution is 1000x1000.\n"
+ "* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.\n"
+ "* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.\n"
+ "* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges."
+ ),
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "action": {
+ "description": "The action to perform.",
+ "enum": [
+ "key",
+ "type",
+ "mouse_move",
+ "left_click",
+ "left_click_drag",
+ "right_click",
+ "middle_click",
+ "double_click",
+ "triple_click",
+ "scroll",
+ "hscroll",
+ "screenshot",
+ "wait",
+ ],
+ "type": "string",
+ },
+ "keys": {
+ "description": "Required only by action=key.",
+ "type": "array",
+ "items": {"type": "string"},
+ },
+ "text": {
+ "description": "Required only by action=type and action=answer.",
+ "type": "string",
+ },
+ "coordinate": {
+ "description": "(x, y): Pixel coordinates from top-left.",
+ "type": "array",
+ "items": {"type": ["number", "integer"]},
+ "minItems": 2,
+ "maxItems": 2,
+ },
+ "pixels": {
+ "description": "Scroll amount. Positive=up, negative=down. For scroll/hscroll.",
+ "type": "number",
+ },
+ "time": {
+ "description": "Seconds to wait (action=wait).",
+ "type": "number",
+ },
+ },
+ "required": ["action"],
+ },
+ },
+}
\ No newline at end of file
diff --git a/libs/python/agent/agent/loops/fara/helpers.py b/libs/python/agent/agent/loops/fara/helpers.py
new file mode 100644
index 0000000000..f6df45448d
--- /dev/null
+++ b/libs/python/agent/agent/loops/fara/helpers.py
@@ -0,0 +1,361 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/fncall_prompts/nous_fncall_prompt.py
+
+import copy
+import json
+import os
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
+
+from .schema import ContentItem, Message
+
+FN_CALL_TEMPLATE_QWEN = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+FN_CALL_TEMPLATE = """You are a web automation agent that performs actions on websites to fulfill user requests by calling various tools.
+* You should stop execution at Critical Points. A Critical Point would be encountered in tasks like 'Checkout', 'Book', 'Purchase', 'Call', 'Email', 'Order', etc where a binding transaction/agreement would require the user's permission/personal or sensitive information (name, email, credit card, address, payment information, resume, etc) in order to complete a transaction (purchase, reservation, sign-up etc), or to communicate in a way that a human would be expected to do (call, email, apply to a job, etc).
+* Solve the task as far as you can up until a Critical Point:
+ - For example, if the task is to "call a restaurant to make a reservation", you should not actually make the call but should navigate to the restaurant's page and find the phone number.
+ - Similarly, if the task is to "order new size 12 running shoes" you should not actually place the order but should instead search for the right shoes that meet the criteria and add them to the cart.
+ - Some tasks, like answering questions, may not encounter a Critical Point at all.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>"""
+
+
+SPECIAL_CODE_MODE = os.getenv("SPECIAL_CODE_MODE", "false").lower() == "true"
+CODE_TOOL_PATTERN = "code_interpreter"
+FN_CALL_TEMPLATE_WITH_CI = """# Tools
+
+You may call one or more functions to assist with the user query.
+
+You are provided with function signatures within <tools></tools> XML tags:
+<tools>
+{tool_descs}
+</tools>
+
+For each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:
+<tool_call>
+{{"name": <function-name>, "arguments": <args-json-object>}}
+</tool_call>
+For code parameters, use placeholders first, and then put the code within <code></code> XML tags, such as:
+<tool_call>
+{{"name": <function-name>, "arguments": {{"code": ""}}}}
+<code>
+Here is the code.
+</code>
+</tool_call>"""
+
+class NousFnCallPrompt:
+ def __init__(self, template_name: str = "default"):
+ """Initialize NousFnCallPrompt with a specific template.
+
+ Args:
+ template_name: Name of the template to use. Options:
+ "default", "qwen", "with_ci"
+ """
+ self.template_name = template_name
+ self.template_map = {
+ "default": FN_CALL_TEMPLATE,
+ "qwen": FN_CALL_TEMPLATE_QWEN,
+ "with_ci": FN_CALL_TEMPLATE_WITH_CI,
+ }
+
+ if template_name not in self.template_map:
+ raise ValueError(
+ f"Unknown template_name: {template_name}. "
+ f"Available options: {list(self.template_map.keys())}"
+ )
+
+ def preprocess_fncall_messages(
+ self,
+ messages: List[Message],
+ functions: List[dict],
+ lang: Literal["en", "zh"],
+ parallel_function_calls: bool = True,
+ function_choice: Union[Literal["auto"], str] = "auto",
+ ) -> List[Message]:
+ del lang # ignored
+ del parallel_function_calls # ignored
+ if function_choice != "auto":
+ raise NotImplementedError
+
+ ori_messages = messages
+
+ # Change function_call responses to plaintext responses:
+ messages = []
+ for msg in copy.deepcopy(ori_messages):
+ role, content, reasoning_content = (
+ msg.role,
+ msg.content,
+ msg.reasoning_content,
+ )
+ if role in ("system", "user"):
+ messages.append(msg)
+ elif role == "assistant":
+ content = content or []
+ fn_call = msg.function_call
+ if fn_call:
+ if (not SPECIAL_CODE_MODE) or (
+ CODE_TOOL_PATTERN not in fn_call.name
+ ):
+ fc = {
+ "name": fn_call.name,
+ "arguments": json.loads(fn_call.arguments),
+ }
+ fc = json.dumps(fc, ensure_ascii=False)
+ fc = f"<tool_call>\n{fc}\n</tool_call>"
+ else:
+ para = json.loads(fn_call.arguments)
+ code = para["code"]
+ para["code"] = ""
+ fc = {"name": fn_call.name, "arguments": para}
+ fc = json.dumps(fc, ensure_ascii=False)
+ fc = f"<tool_call>\n{fc}\n<code>\n{code}\n</code>\n</tool_call>"
+
+ content.append(ContentItem(text=fc))
+ if messages[-1].role == "assistant":
+ messages[-1].content.append(ContentItem(text="\n"))
+ messages[-1].content.extend(content)
+ else:
+ # TODO: Assuming there will only be one continuous reasoning_content here
+ messages.append(
+ Message(
+ role=role,
+ content=content,
+ reasoning_content=reasoning_content,
+ )
+ )
+ elif role == "function":
+ assert isinstance(content, list)
+ assert len(content) == 1
+ assert content[0].text
+ fc = f"<tool_response>\n{content[0].text}\n</tool_response>"
+ content = [ContentItem(text=fc)]
+ if messages[-1].role == "user":
+ messages[-1].content.append(ContentItem(text="\n"))
+ messages[-1].content.extend(content)
+ else:
+ messages.append(Message(role="user", content=content))
+ else:
+ raise TypeError
+
+ tool_descs = [{"type": "function", "function": f} for f in functions]
+ tool_names = [
+ function.get("name_for_model", function.get("name", ""))
+ for function in functions
+ ]
+ tool_descs = "\n".join([json.dumps(f, ensure_ascii=False) for f in tool_descs])
+
+ # Select template based on configuration
+ if SPECIAL_CODE_MODE and any([CODE_TOOL_PATTERN in x for x in tool_names]):
+ selected_template = FN_CALL_TEMPLATE_WITH_CI
+ else:
+ selected_template = self.template_map[self.template_name]
+
+ tool_system = selected_template.format(tool_descs=tool_descs)
+ if messages[0].role == "system":
+ messages[0].content.append(ContentItem(text="\n\n" + tool_system))
+ else:
+ messages = [
+ Message(role="system", content=[ContentItem(text=tool_system)])
+ ] + messages
+ return messages
+
+
+# Mainly for removing incomplete special tokens when streaming the output
+# This assumes that '<tool_call>\n{"name": "' is the special token for the NousFnCallPrompt
+def remove_incomplete_special_tokens(text: str) -> str:
+ if text in '<tool_call>\n{"name": "':
+ text = ""
+ return text
+
+
+def extract_fn(text: str):
+ fn_name, fn_args = "", ""
+ fn_name_s = '"name": "'
+ fn_name_e = '", "'
+ fn_args_s = '"arguments": '
+ i = text.find(fn_name_s)
+ k = text.find(fn_args_s)
+ if i > 0:
+ _text = text[i + len(fn_name_s) :]
+ j = _text.find(fn_name_e)
+ if j > -1:
+ fn_name = _text[:j]
+ if k > 0:
+ fn_args = text[k + len(fn_args_s) :]
+
+ if len(fn_args) > 5:
+ fn_args = fn_args[:-5]
+ else:
+ fn_args = ""
+ return fn_name, fn_args
+
+
+def build_nous_system(functions: List[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+ """Use original FARA NousFnCallPrompt to generate a system message embedding tool schema."""
+ from .schema import ContentItem as NousContentItem
+ from .schema import Message as NousMessage
+
+ msgs = NousFnCallPrompt().preprocess_fncall_messages(
+ messages=[
+ NousMessage(
+ role="system", content=[NousContentItem(text="You are a helpful assistant.")]
+ )
+ ],
+ functions=functions,
+ lang="en",
+ )
+ sys = msgs[0].model_dump()
+ # Convert structured content to OpenAI-style content list
+ content = [{"type": "text", "text": c["text"]} for c in sys.get("content", [])]
+ return {"role": "system", "content": content}
+
+
+def parse_tool_call_from_text(text: str) -> Optional[Dict[str, Any]]:
+ """Extract JSON object within <tool_call>...</tool_call> from model text.
+
+ Accepts both </tool_call> and <tool_call> as closing tags for robustness.
+ Handles nested braces in JSON objects.
+ """
+ # Find the opening tag
+ start_idx = text.find("<tool_call>")
+ if start_idx == -1:
+ return None
+
+ # Find the start of JSON (first '{' after opening tag)
+ json_start = text.find("{", start_idx)
+ if json_start == -1:
+ return None
+
+ # Extract JSON by counting braces
+ brace_count = 0
+ json_end = json_start
+ for i in range(json_start, len(text)):
+ if text[i] == '{':
+ brace_count += 1
+ elif text[i] == '}':
+ brace_count -= 1
+ if brace_count == 0:
+ json_end = i + 1
+ break
+
+ if brace_count != 0:
+ return None
+
+ json_str = text[json_start:json_end]
+ try:
+ return json.loads(json_str)
+ except Exception:
+ return None
+
+
+async def unnormalize_coordinate(args: Dict[str, Any], dims: Tuple[int, int]) -> Dict[str, Any]:
+ """Coordinates appear in 0..1000 space, scale to actual screen size using dims if provided."""
+ coord = args.get("coordinate")
+ if not coord or not isinstance(coord, (list, tuple)) or len(coord) < 2:
+ return args
+ x, y = float(coord[0]), float(coord[1])
+ width, height = float(dims[0]), float(dims[1])
+ x_abs = max(0.0, min(width, (x / 1000.0) * width))
+ y_abs = max(0.0, min(height, (y / 1000.0) * height))
+ args = {**args, "coordinate": [round(x_abs), round(y_abs)]}
+ return args
+
+
+def convert_qwen_tool_args_to_computer_action(args: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+ """
+ Convert Qwen computer tool arguments to the Computer Calls action schema.
+
+ Qwen (example):
+ {"action": "left_click", "coordinate": [114, 68]}
+
+ Target (example):
+ {"action": "left_click", "x": 114, "y": 68}
+
+ Other mappings:
+ - right_click, middle_click, double_click (triple_click -> double_click)
+ - mouse_move -> { action: "move", x, y }
+ - key -> { action: "keypress", keys: [...] }
+ - type -> { action: "type", text }
+ - scroll/hscroll -> { action: "scroll", scroll_x, scroll_y, x, y }
+ - wait -> { action: "wait" }
+ - terminate/answer are not direct UI actions; return None for now
+ """
+ if not isinstance(args, dict):
+ return None
+
+ action = args.get("action")
+ if not isinstance(action, str):
+ return None
+
+ # Coordinates helper
+ coord = args.get("coordinate")
+ x = y = None
+ if isinstance(coord, (list, tuple)) and len(coord) >= 2:
+ try:
+ x = int(round(float(coord[0])))
+ y = int(round(float(coord[1])))
+ except Exception:
+ x = y = None
+
+ # Map actions
+ a = action.lower()
+ if a in {"left_click", "right_click", "middle_click", "double_click"}:
+ if x is None or y is None:
+ return None
+ return {"action": a, "x": x, "y": y}
+ if a == "triple_click":
+ # Approximate as double_click
+ if x is None or y is None:
+ return None
+ return {"action": "double_click", "x": x, "y": y}
+ if a == "mouse_move":
+ if x is None or y is None:
+ return None
+ return {"action": "move", "x": x, "y": y}
+ if a == "key":
+ keys = args.get("keys")
+ if isinstance(keys, list) and all(isinstance(k, str) for k in keys):
+ return {"action": "keypress", "keys": keys}
+ return None
+ if a == "type":
+ text = args.get("text")
+ if isinstance(text, str):
+ return {"action": "type", "text": text}
+ return None
+ if a in {"scroll", "hscroll"}:
+ pixels = args.get("pixels") or 0
+ try:
+ pixels_val = int(round(float(pixels)))
+ except Exception:
+ pixels_val = 0
+ scroll_x = pixels_val if a == "hscroll" else 0
+ scroll_y = pixels_val if a == "scroll" else 0
+ # Include cursor position if available (optional)
+ out: Dict[str, Any] = {"action": "scroll", "scroll_x": scroll_x, "scroll_y": scroll_y}
+ if x is not None and y is not None:
+ out.update({"x": x, "y": y})
+ return out
+ if a == "wait":
+ return {"action": "wait"}
+
+ # Non-UI or terminal actions: terminate/answer -> not mapped here
+ return None
diff --git a/libs/python/agent/agent/loops/fara/schema.py b/libs/python/agent/agent/loops/fara/schema.py
new file mode 100644
index 0000000000..7ec8c6505c
--- /dev/null
+++ b/libs/python/agent/agent/loops/fara/schema.py
@@ -0,0 +1,149 @@
+# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/schema.py
+
+from typing import List, Literal, Optional, Tuple, Union
+
+from pydantic import BaseModel, field_validator, model_validator
+
+
+class BaseModelCompatibleDict(BaseModel):
+ def __getitem__(self, item):
+ return getattr(self, item)
+
+ def __setitem__(self, key, value):
+ setattr(self, key, value)
+
+ def model_dump(self, **kwargs):
+ if "exclude_none" not in kwargs:
+ kwargs["exclude_none"] = True
+ return super().model_dump(**kwargs)
+
+ def model_dump_json(self, **kwargs):
+ if "exclude_none" not in kwargs:
+ kwargs["exclude_none"] = True
+ return super().model_dump_json(**kwargs)
+
+ def get(self, key, default=None):
+ try:
+ value = getattr(self, key)
+ if value:
+ return value
+ else:
+ return default
+ except AttributeError:
+ return default
+
+ def __str__(self):
+ return f"{self.model_dump()}"
+
+
+class FunctionCall(BaseModelCompatibleDict):
+ name: str
+ arguments: str
+
+ def __init__(self, name: str, arguments: str):
+ super().__init__(name=name, arguments=arguments)
+
+ def __repr__(self):
+ return f"FunctionCall({self.model_dump()})"
+
+
+class ContentItem(BaseModelCompatibleDict):
+ text: Optional[str] = None
+ image: Optional[str] = None
+ file: Optional[str] = None
+ audio: Optional[Union[str, dict]] = None
+ video: Optional[Union[str, list]] = None
+
+ def __init__(
+ self,
+ text: Optional[str] = None,
+ image: Optional[str] = None,
+ file: Optional[str] = None,
+ audio: Optional[Union[str, dict]] = None,
+ video: Optional[Union[str, list]] = None,
+ ):
+ super().__init__(text=text, image=image, file=file, audio=audio, video=video)
+
+ @model_validator(mode="after")
+ def check_exclusivity(self):
+ provided_fields = 0
+ if self.text is not None:
+ provided_fields += 1
+ if self.image:
+ provided_fields += 1
+ if self.file:
+ provided_fields += 1
+ if self.audio:
+ provided_fields += 1
+ if self.video:
+ provided_fields += 1
+
+ if provided_fields != 1:
+ raise ValueError(
+ "Exactly one of 'text', 'image', 'file', 'audio', or 'video' must be provided."
+ )
+ return self
+
+ def __repr__(self):
+ return f"ContentItem({self.model_dump()})"
+
+ def get_type_and_value(
+ self,
+ ) -> Tuple[Literal["text", "image", "file", "audio", "video"], str]:
+ ((t, v),) = self.model_dump().items()
+ assert t in ("text", "image", "file", "audio", "video")
+ return t, v
+
+ @property
+ def type(self) -> Literal["text", "image", "file", "audio", "video"]:
+ t, _ = self.get_type_and_value()
+ return t
+
+ @property
+ def value(self) -> str:
+ _, v = self.get_type_and_value()
+ return v
+
+
+class Message(BaseModelCompatibleDict):
+ role: str
+ content: Union[str, List[ContentItem]]
+ reasoning_content: Optional[Union[str, List[ContentItem]]] = None
+ name: Optional[str] = None
+ function_call: Optional[FunctionCall] = None
+ extra: Optional[dict] = None
+
+ def __init__(
+ self,
+ role: str,
+ content: Union[str, List[ContentItem]],
+ reasoning_content: Optional[Union[str, List[ContentItem]]] = None,
+ name: Optional[str] = None,
+ function_call: Optional[FunctionCall] = None,
+ extra: Optional[dict] = None,
+ **kwargs,
+ ):
+ if content is None:
+ content = ""
+ if reasoning_content is None:
+ reasoning_content = ""
+ super().__init__(
+ role=role,
+ content=content,
+ reasoning_content=reasoning_content,
+ name=name,
+ function_call=function_call,
+ extra=extra,
+ )
+
+ def __repr__(self):
+ return f"Message({self.model_dump()})"
+
+ @field_validator("role")
+ def role_checker(cls, value: str) -> str:
+ values = ["system", "user", "assistant", "function"]
+ if value not in values:
+ raise ValueError(
+ f'{value} must be one of {",".join(values)}'
+ )
+ return value