Fara1.5 – A family of frontier computer use agent models
stage1
Hussein Mozannard58a16cb6a55
diff --git a/pyproject.toml b/pyproject.tomlindex 6566c44..257597a 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -32,6 +32,9 @@ dependencies = ["pyyaml","jsonschema","browserbase",+ "pydantic",+ "markitdown",+ "tiktoken",]@@ -43,7 +46,11 @@ Repository = "https://github.com/microsoft/fara"Issues = "https://github.com/microsoft/fara/issues"[project.optional-dependencies]-vllm = ["vllm>=0.10.0"]+# vllm==0.19.1 + transformers>=5.2.0 is the tested serving stack for the+# Qwen3.5-based Fara-1.5 models. nvidia-cutlass-dsl is capped below 4.6+# because 4.6.0.dev0 removed the `cute.core.ThrMma` alias that quack (a+# transitive vLLM dep) uses, crashing vLLM at startup.+vllm = ["vllm==0.19.1", "transformers>=5.2.0", "nvidia-cutlass-dsl==4.5.2"]lmstudio = ["lmstudio"]ollama = ["ollama"]@@ -80,6 +87,7 @@ include = ["src/fara/**/*.py","src/fara/**/*.yaml","src/fara/**/*.json",+ "src/fara/**/*.js",]exclude = ["**/__pycache__",diff --git a/src/fara/__init__.py b/src/fara/__init__.pyindex b639809..b4cbdb2 100644--- a/src/fara/__init__.py+++ b/src/fara/__init__.py@@ -1,4 +1,28 @@-from .fara_agent import FARA_ACTION_DEFINITIONS, FaraAgent+from .agents.fara.fara_qwen3_next import (+ FaraQwen3NextAgent,+ FaraQwen3NextAgentConfig,+)+from .agents.fara.fara_qwen3 import FaraQwen3Agent, FaraQwen3AgentConfig+from .environments.playwright import (+ PlaywrightEnvironment,+ PlaywrightEnvironmentConfig,+)+from .core.data_point import DataPoint, Task+from .core.run_context import RunContext+from .fara_7b import FARA_ACTION_DEFINITIONS, FaraAgentfrom .browser.playwright_controller import PlaywrightController-__all__ = ["FARA_ACTION_DEFINITIONS", "FaraAgent", "PlaywrightController"]+__all__ = [+ "FaraQwen3NextAgent",+ "FaraQwen3NextAgentConfig",+ "FaraQwen3Agent",+ "FaraQwen3AgentConfig",+ "PlaywrightEnvironment",+ "PlaywrightEnvironmentConfig",+ "DataPoint",+ "Task",+ "RunContext",+ "FARA_ACTION_DEFINITIONS",+ "FaraAgent",+ "PlaywrightController",+]diff --git a/src/fara/_prompts.py b/src/fara/_prompts.pyindex 09933f8..5400760 100644--- a/src/fara/_prompts.py+++ b/src/fara/_prompts.py@@ -1,22 +1,140 @@-import math+"""Fara-1.5 browser toolset and system prompt.-from typing import Union, Tuple+Ported from agento_train's FaraNextBrowserComputerUse toolset and the+composable fara-qwen3vl fn-call template (identity + critical points).+"""++from typing import Unionfrom .qwen_helpers.base_tool import BaseToolfrom .qwen_helpers.fncall_prompt import NousFnCallPrompt-from .qwen_helpers.schema import (- ContentItem,- Message,+from .qwen_helpers.utils import smart_resize++# ============================================================+# Model identities+# ============================================================+FARA_QWEN35_IDENTITY = """\+You are Fara, a computer use agent (CUA) specialized for web browsers. \+You are developed by Microsoft AI Frontiers. You assist users with \+completing and automating tasks that require the use of a web browser.++The model was trained in the timeframe of January - April 2026. You can \+effectively perform tasks even beyond this range by accessing the web \+browser and using the latest information on the live web. But your \+knowledge cutoff is limited to early 2026, so you may not be aware of \+events or developments that occurred after that time, without explicitly \+browsing and searching for latest information on the web.++This edition of the model was trained using SFT on top of \+Qwen3.5-9B, using a synthetic data mixture generated and \+developed by Microsoft AI Frontiers."""++FARA_QWEN3VL_IDENTITY = """\+You are Fara, a computer use agent (CUA) specialized for web browsers. \+You are developed by Microsoft AI Frontiers. You assist users with \+completing and automating tasks that require the use of a web browser.++The model was trained in the timeframe of January - March 2026. You can \+effectively perform tasks even beyond this range by accessing the web \+browser and using the latest information on the live web. But your \+knowledge cutoff is limited to early 2026, so you may not be aware of \+events or developments that occurred after that time, without explicitly \+browsing and searching for latest information on the web.++This edition of the model was trained using SFT on top of \+Qwen3-VL-8B-Instruct, using a synthetic data mixture generated and \+developed by Microsoft AI Frontiers."""++# ============================================================+# Critical points+# ============================================================+CRITICAL_POINTS_FARA_1_5 = """\+A critical point is a situation where we must pause and request information or confirmation from the user before \+proceeding. There are three types:++Case 1: Missing User Information — The task requires personal information that the user has not provided (e.g., email, \+phone number, address, payment details). Never fabricate or assume personal information. Fill in only what the user has \+explicitly provided, then pause and ask for any missing required fields. (e.g., form requires phone number but user \+only gave name and email -> fill name and email, then ask for phone number.) If the user has provided all required \+information, proceed without stopping.++Case 2: Underspecified Task — The task description is ambiguous or missing details needed to make a decision at the \+current step. Pause and ask for clarification. (e.g., user asks to book a flight but doesn't specify destination -> \+ask for destination.) If the user's instructions contain all information needed for the current decision, proceed \+without stopping.++Case 3: Irreversible Action — We are about to perform an action that cannot be undone (e.g., submitting a form, \+completing a purchase, sending a message, deleting data). If the user explicitly authorized the action (e.g., "submit \+the form", "complete the purchase", "you have my permission to submit") -> proceed without stopping. If the user did \+NOT explicitly authorize the action -> stop and ask for confirmation. (e.g., "fill out a form" with no mention of \+submitting -> fill the form, then ask before submitting; "fill out and submit a form" -> fill and submit without \+stopping.)++Only stop at a critical point if (1) required information is missing, (2) the task is ambiguous, OR (3) an irreversible \+action lacks explicit user authorization. If the user has provided all necessary information AND explicitly authorized \+the action, proceed without interruption."""++IDENTITY_REGISTRY = {+ "fara_qwen35": FARA_QWEN35_IDENTITY,+ "fara_qwen3vl": FARA_QWEN3VL_IDENTITY,+}++CRITICAL_POINTS_REGISTRY = {+ "fara-1.5": CRITICAL_POINTS_FARA_1_5,+}++# NOTE: {tool_descs} is the ONLY placeholder — it is filled by+# NousFnCallPrompt with the computer_use tool JSON schema (generated+# dynamically per-image for screen resolution). All other braces must be+# doubled to survive Python's str.format().+FN_CALL_FORMAT = (+ "You are provided with function signatures within <tools></tools> XML tags:\n"+ "<tools>\n"+ "{tool_descs}\n"+ "</tools>\n"+ "\n"+ "For each function call, return a json object with function name and arguments "+ "within <tool_call></tool_call> XML tags:\n"+ "<tool_call>\n"+ '{{"name": <function-name>, "arguments": <args-json-object>}}\n'+ "</tool_call>")-IMAGE_FACTOR = 28-MIN_PIXELS = 4 * 28 * 28-MAX_PIXELS = 16384 * 28 * 28-MAX_RATIO = 200+def build_fara_fn_call_template(identity: str, critical_points: str) -> str:+ """Compose a Fara fn-call template from independent identity + critical points.++ Args:+ identity: Key in IDENTITY_REGISTRY or a raw string (>= 20 chars).+ critical_points: Key in CRITICAL_POINTS_REGISTRY or a raw string (>= 20 chars).++ Returns:+ Fully composed template string with {tool_descs} placeholder.+ """+ if identity in IDENTITY_REGISTRY:+ identity = IDENTITY_REGISTRY[identity]+ elif not isinstance(identity, str) or len(identity) < 20:+ raise ValueError(+ f"Unknown identity key: {identity!r}. "+ f"Available keys: {list(IDENTITY_REGISTRY.keys())}. "+ f"Or pass a raw string (>= 20 chars)."+ )++ if critical_points in CRITICAL_POINTS_REGISTRY:+ critical_points = CRITICAL_POINTS_REGISTRY[critical_points]+ elif not isinstance(critical_points, str) or len(critical_points) < 20:+ raise ValueError(+ f"Unknown critical_points key: {critical_points!r}. "+ f"Available keys: {list(CRITICAL_POINTS_REGISTRY.keys())}. "+ f"Or pass a raw string (>= 20 chars)."+ )++ return identity + "\n\n" + critical_points + "\n\n" + FN_CALL_FORMAT+++class FaraBrowserComputerUse(BaseTool):+ """Schema-only computer_use tool definition for the Fara-1.5 browser action space."""-# @register_tool("computer_use")-class FaraComputerUse(BaseTool):name = "computer_use"@property@@ -28,11 +146,7 @@ Use a mouse and keyboard to interact with a computer, and take screenshots.* The screen's resolution is {self.display_width_px}x{self.display_height_px}.* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.-* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.-* When a separate scrollable container prominently overlays the webpage, if you want to scroll within it, you typically need to mouse_move() over it first and then scroll().-* If a popup window appears that you want to close, if left_click() on the 'X' or close button doesn't work, try key(keys=['Escape']) to close it.-* On some search bars, when you type(), you may need to press_enter=False and instead separately call left_click() on the search button to submit the search query. This is especially true of search bars that have auto-suggest popups for e.g. locations-* For calendar widgets, you usually need to left_click() on arrows to move between months and left_click() on dates to select them; type() is not typically used to input dates there.+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges.""".strip()parameters = {@@ -44,24 +158,38 @@ The action to perform. The available actions are:* `type`: Type a string of text on the keyboard.* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.* `left_click`: Click the left mouse button.+* `double_click`: Double-click the left mouse button.+* `right_click`: Click the right mouse button.+* `triple_click`: Triple-click the left mouse button (e.g. to select a line of text).+* `left_click_drag`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen.* `scroll`: Performs a scroll of the mouse scroll wheel.+* `hscroll`: Performs a horizontal scroll (mapped to regular scroll).* `visit_url`: Visit a specified URL.-* `web_search`: Perform a web search with a specified query.* `history_back`: Go back to the previous page in the browser history.+* `web_search`: Perform a web search with a specified query.+* `read_page_answer_question`: Read the current page content and answer a question about it.* `pause_and_memorize_fact`: Pause and memorize a fact for future reference.+* `ask_user_question`: Ask the user a clarifying question and wait for a response.* `wait`: Wait specified seconds for the change to happen.-* `terminate`: Terminate the current task and report its completion status.+* `terminate`: Terminate the current task and provide the final answer.""".strip(),"enum": ["key","type","mouse_move","left_click",+ "left_click_drag",+ "right_click",+ "double_click",+ "triple_click","scroll",+ "hscroll","visit_url",- "web_search","history_back",+ "web_search",+ "read_page_answer_question","pause_and_memorize_fact",+ "ask_user_question","wait","terminate",],@@ -75,20 +203,12 @@ The action to perform. The available actions are:"description": "Required only by `action=type`.","type": "string",},- "press_enter": {- "description": "Whether to press the Enter key after typing. Required only by `action=type`.",- "type": "boolean",- },- "delete_existing_text": {- "description": "Whether to delete existing text before typing. Required only by `action=type`.",- "type": "boolean",- },"coordinate": {- "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=left_click`, `action=mouse_move`, and `action=type`.",+ "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required by `action=left_click`, `action=double_click`, `action=right_click`, `action=triple_click`, `action=left_click_drag`, and `action=mouse_move`.","type": "array",},"pixels": {- "description": "The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll`.",+ "description": "The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll` and `action=hscroll`.","type": "number",},"url": {@@ -103,14 +223,17 @@ The action to perform. The available actions are:"description": "The fact to remember for the future. Required only by `action=pause_and_memorize_fact`.","type": "string",},+ "question": {+ "description": "The question to ask. Required by `action=read_page_answer_question` and `action=ask_user_question`.",+ "type": "string",+ },"time": {"description": "The seconds to wait. Required only by `action=wait`.","type": "number",},- "status": {- "description": "The status of the task. Required only by `action=terminate`.",+ "answer": {+ "description": "The final answer for the task. Required only by `action=terminate`.","type": "string",- "enum": ["success", "failure"],},},"required": ["action"],@@ -120,122 +243,43 @@ The action to perform. The available actions are:def __init__(self, cfg=None):self.display_width_px = cfg["display_width_px"]self.display_height_px = cfg["display_height_px"]- include_input_text_key_args = cfg.pop("include_input_text_key_args", False)- if not include_input_text_key_args:- self.parameters["properties"].pop("press_enter", None)- self.parameters["properties"].pop("delete_existing_text", None)super().__init__(cfg)def call(self, params: Union[str, dict], **kwargs):- params = self._verify_json_format_args(params)- action = params["action"]- if action == "key":- return self._key(params["text"])- elif action == "click":- return self._click(coordinate=params["coordinate"])- elif action == "long_press":- return self._long_press(- coordinate=params["coordinate"], time=params["time"]- )- elif action == "swipe":- return self._swipe(- coordinate=params["coordinate"], coordinate2=params["coordinate2"]- )- elif action == "type":- return self._type(params["text"])- elif action == "system_button":- return self._system_button(params["button"])- elif action == "open":- return self._open(params["text"])- elif action == "wait":- return self._wait(params["time"])- elif action == "terminate":- return self._terminate(params["status"])- else:- raise ValueError(f"Unknown action: {action}")-- def _key(self, text: str):- raise NotImplementedError()-- def _click(self, coordinate: Tuple[int, int]):- raise NotImplementedError()-- def _long_press(self, coordinate: Tuple[int, int], time: int):- raise NotImplementedError()-- def _swipe(self, coordinate: Tuple[int, int], coordinate2: Tuple[int, int]):- raise NotImplementedError()-- def _type(self, text: str):- raise NotImplementedError()-- def _system_button(self, button: str):- raise NotImplementedError()-- def _open(self, text: str):- raise NotImplementedError()-- def _wait(self, time: int):- raise NotImplementedError()-- def _terminate(self, status: str):- raise NotImplementedError()---def round_by_factor(number: int, factor: int) -> int:- """Returns the closest integer to 'number' that is divisible by 'factor'."""- return round(number / factor) * factor---def ceil_by_factor(number: int, factor: int) -> int:- """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""- return math.ceil(number / factor) * factor---def floor_by_factor(number: int, factor: int) -> int:- """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""- return math.floor(number / factor) * factor---def smart_resize(- height: int,- width: int,- factor: int = IMAGE_FACTOR,- min_pixels: int = MIN_PIXELS,- max_pixels: int = MAX_PIXELS,-) -> tuple[int, int]:- """- Rescales the image so that the following conditions are met:-- 1. Both dimensions (height and width) are divisible by 'factor'.-- 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].-- 3. The aspect ratio of the image is maintained as closely as possible.- """- if max(height, width) / min(height, width) > MAX_RATIO:- raise ValueError(- f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"- )- h_bar = max(factor, round_by_factor(height, factor))- w_bar = max(factor, round_by_factor(width, factor))- if h_bar * w_bar > max_pixels:- beta = math.sqrt((height * width) / max_pixels)- h_bar = floor_by_factor(height / beta, factor)- w_bar = floor_by_factor(width / beta, factor)- elif h_bar * w_bar < min_pixels:- beta = math.sqrt(min_pixels / (height * width))- h_bar = ceil_by_factor(height * beta, factor)- w_bar = ceil_by_factor(width * beta, factor)- return h_bar, w_bar+ raise NotImplementedError("FaraBrowserComputerUse is schema-only.")def get_computer_use_system_prompt(image,processor_im_cfg,+ mode="fara_next_browser",include_input_text_key_args=False,- fn_call_template="default",+ fn_call_template="fara-qwen3vl",+ randomize=False,+ display_size=None,+ identity="fara_qwen35",+ critical_points="fara-1.5",):+ """Build the Fara-1.5 system prompt for a screenshot.++ Args:+ image: PIL image of the current screenshot.+ processor_im_cfg: dict with patch_size, merge_size, min_pixels, max_pixels.+ mode: Tool-set mode. Only ``fara_next_browser`` is supported here (this+ repo ships the browser action space only).+ include_input_text_key_args, fn_call_template, randomize: accepted for+ signature compatibility with the training-time prompt builder;+ unused by the browser tool.+ display_size: Fixed coordinate-space size the model emits coordinates in;+ None uses the resized image dimensions.+ identity: Key in IDENTITY_REGISTRY or a raw identity string.+ critical_points: Key in CRITICAL_POINTS_REGISTRY or a raw string.+ """+ if mode != "fara_next_browser":+ raise NotImplementedError(+ f"computer_use_mode={mode!r} is not supported; this repo ships the "+ f"'fara_next_browser' action space only."+ )patch_size = processor_im_cfg["patch_size"]merge_size = processor_im_cfg["merge_size"]min_pixels = processor_im_cfg["min_pixels"]@@ -249,23 +293,19 @@ def get_computer_use_system_prompt(max_pixels=max_pixels,)- computer_use = FaraComputerUse(+ display_w = display_size if display_size is not None else resized_width+ display_h = display_size if display_size is not None else resized_height++ computer_use = FaraBrowserComputerUse(cfg={- "display_width_px": resized_width,- "display_height_px": resized_height,- "include_input_text_key_args": include_input_text_key_args,+ "display_width_px": display_w,+ "display_height_px": display_h,})- conversation = NousFnCallPrompt(- template_name=fn_call_template- ).preprocess_fncall_messages(- messages=[- Message(- role="system",- content=[ContentItem(text="You are a helpful assistant.")],- ),- ],+ template = build_fara_fn_call_template(identity, critical_points)+ conversation = NousFnCallPrompt(template=template).preprocess_fncall_messages(+ messages=[],functions=[computer_use.function],lang=None,)diff --git a/src/fara/agents/__init__.py b/src/fara/agents/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/fara/agents/captcha.py b/src/fara/agents/captcha.pynew file mode 100644index 0000000..903b40c--- /dev/null+++ b/src/fara/agents/captcha.py@@ -0,0 +1,28 @@+"""Shared captcha-waiting utility for browser-based agents."""++import asyncio+import logging++CAPTCHA_TIMEOUT = 60++logger = logging.getLogger(__name__)+++async def wait_for_captcha(env, timeout: int = CAPTCHA_TIMEOUT) -> bool:+ """Wait for captcha to be solved if environment supports it."""+ if not hasattr(env, "wait_for_captcha_resolution"):+ return True++ if hasattr(env, "_captcha_event") and env._captcha_event.is_set():+ return True++ logger.info(f"Waiting {timeout}s for captcha to finish...")+ try:+ await asyncio.wait_for(+ env.wait_for_captcha_resolution(),+ timeout=timeout,+ )+ return True+ except asyncio.TimeoutError:+ logger.warning(f"Captcha timeout after {timeout} seconds!")+ return Falsediff --git a/src/fara/agents/computer_agent/__init__.py b/src/fara/agents/computer_agent/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/fara/agents/computer_agent/utils.py b/src/fara/agents/computer_agent/utils.pynew file mode 100644index 0000000..545a556--- /dev/null+++ b/src/fara/agents/computer_agent/utils.py@@ -0,0 +1,137 @@+"""Shared utilities for agents."""++from __future__ import annotations++import json+import logging+from typing import Any, List++from ...clients.wrapper import ChatCompletionClient+from ...clients.messages import SystemMessage, UserMessage, ImageObj++logger = logging.getLogger(__name__)+++def compact_messages(+ messages: List[Any],+ tools: list,+ client: ChatCompletionClient,+ max_prompt_tokens: int,+ min_keep_rounds: int = 2,+) -> List[Any]:+ """Remove middle round triplets until token count is under max_prompt_tokens.++ Each round after the initial messages (system + first user) is a triplet:+ (assistant tool_call dict, tool result dict, UserMessage/user dict).+ Keeps the first two messages and the last ``min_keep_rounds`` rounds.+ Removes from the middle outward.++ Args:+ messages: The conversation message list (modified in place via filtering).+ tools: Tool definitions passed to the model.+ client: The ChatCompletionClient used for token counting.+ max_prompt_tokens: Target token budget. <=0 disables compaction.+ min_keep_rounds: Minimum recent rounds to always retain.+ """+ # TODO: this is not maybe the best compacting messages strategy+ if max_prompt_tokens <= 0:+ return messages+ token_count = client.count_tokens(messages, tools)+ if token_count <= max_prompt_tokens:+ return messages++ # Identify round triplet start indices (skip system[0] and initial user[1])+ round_starts = []+ idx = 2+ while idx + 2 < len(messages):+ msg = messages[idx]+ if isinstance(msg, dict) and msg.get("role") == "assistant":+ round_starts.append(idx)+ idx += 3+ else:+ idx += 1++ removable = (+ round_starts[:-min_keep_rounds] if len(round_starts) > min_keep_rounds else []+ )+ if not removable:+ return messages++ # Remove from the middle outward+ mid = len(removable) // 2+ removal_order = sorted(range(len(removable)), key=lambda i: abs(i - mid))++ for ri in removal_order:+ start = removable[ri]+ if start >= len(messages) or messages[start] is None:+ continue+ messages[start] = None+ messages[start + 1] = None+ messages[start + 2] = None+ compact = [m for m in messages if m is not None]+ token_count = client.count_tokens(compact, tools)+ logger.info(f"Compacted messages: {token_count} tokens")+ if token_count <= max_prompt_tokens:+ return compact++ return [m for m in messages if m is not None]+++def drop_old_screenshots(+ messages: List[Any],+ max_screenshots: int = 2,+) -> None:+ """Replace all but the latest ``max_screenshots`` image-bearing user messages+ with text-only summaries, in place.++ Looks for the ``screen_description`` field in the preceding assistant+ tool-call arguments to build a fallback label.+ """+ screenshot_indices = [+ j+ for j, m in enumerate(messages)+ if isinstance(m, UserMessage)+ and isinstance(m.content, list)+ and any(isinstance(c, ImageObj) for c in m.content)+ ]+ for j in screenshot_indices[:-max_screenshots]:+ screen_desc = ""+ if (+ j >= 2+ and isinstance(messages[j - 2], dict)+ and messages[j - 2].get("role") == "assistant"+ ):+ tool_calls = messages[j - 2].get("tool_calls", [])+ if tool_calls:+ prev_args = json.loads(tool_calls[0]["function"]["arguments"])+ if "screen_description" in prev_args:+ screen_desc = (+ f"[Previous screen: {prev_args['screen_description']}]"+ )+ text_parts = [c for c in messages[j].content if isinstance(c, str)]+ combined = " ".join(text_parts) if text_parts else ""+ fallback = screen_desc or "[screenshot removed]"+ messages[j] = UserMessage(+ content=f"{fallback} {combined}".strip() if combined else fallback+ )+++async def extract_from_page(+ markdown: str,+ question: str,+ client: ChatCompletionClient,+ max_chars: int = 20000,+) -> str:+ """Send page markdown to the model with a question, return concise answer."""+ if len(markdown) > max_chars:+ markdown = markdown[:max_chars] + "\n... [truncated]"+ extract_messages = [+ SystemMessage(+ content="You are given the markdown content of a web page. "+ "Answer the user's question based solely on this content. "+ "Be concise and extract only the relevant information."+ ),+ UserMessage(content=f"Page content:\n{markdown}\n\nQuestion: {question}"),+ ]+ result = await client.create(messages=extract_messages)+ return result.content.content or "Error: Extraction failed."diff --git a/src/fara/agents/coord_spaces.py b/src/fara/agents/coord_spaces.pynew file mode 100644index 0000000..03cb56f--- /dev/null+++ b/src/fara/agents/coord_spaces.py@@ -0,0 +1,24 @@+"""Model-display coordinate-space sizes shared across agents.++Fara and Qwen35 agents emit click coordinates in a normalized+model-display space rather than viewport pixels. Downstream consumers+(the inference loop's rescale, the prepro coord normalizer, the+failure-analysis viewport scaler) need to know that space's upper bound+to convert back to pixels.++Keeping this as a flat, dependency-free module under ``agents/`` (same+style as ``agents/key_mapping.py``) means any caller — including the+failure-analysis workflow — can import it without pulling in the+heavyweight agent implementations.+"""++# Fara family (FaraQwen3Agent, FaraQwen3NextAgent): emits coords in+# [0, FARA_DISPLAY_SIZE); the agent's inference loop rescales to+# viewport pixels via ``proc_coords`` before clicking.+FARA_DISPLAY_SIZE: int = 1000++# Qwen35Agent in default ``coordinate_type="relative"`` mode: emits+# coords in [0, QWEN35_DISPLAY_SIZE]. Note the off-by-one vs Fara — the+# divisor used in :func:`Qwen35Agent.adjust_coordinates` is 999, not+# 1000.+QWEN35_DISPLAY_SIZE: int = 999diff --git a/src/fara/agents/fara/__init__.py b/src/fara/agents/fara/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/fara/agents/fara/fara_qwen3.py b/src/fara/agents/fara/fara_qwen3.pynew file mode 100644index 0000000..5e9ffe8--- /dev/null+++ b/src/fara/agents/fara/fara_qwen3.py@@ -0,0 +1,769 @@+"""FaraQwen3Agent - Web automation agent using Qwen3 vision-language model."""++from __future__ import annotations++import copy+import io+import json+import ast+import logging+import shutil++import openai+from dataclasses import dataclass, field+from typing import Any, Dict, List, Tuple+from urllib.parse import quote_plus++from PIL import Image+from tenacity import (+ retry,+ retry_if_not_exception_type,+ stop_after_attempt,+ wait_exponential,+ before_sleep_log,+)++from ...clients.wrapper import ChatCompletionClient+from ...clients.create_utils import create_client_from_config+from ...clients.messages import (+ LLMMessage,+ SystemMessage,+ UserMessage,+ AssistantMessage,+ ImageObj,+ FunctionCall,+)++from ...core.agent import Agent, AgentConfig+from ...core.run_context import RunContext+from ...core.data_point import (+ Action as DPAction,+ AgentState,+ LLMConversation,+ LLMMessage as DPLLMMessage,+ SolverStatus,+ Outcome,+ ComputerObservation,+)+from ..captcha import wait_for_captcha+from ..coord_spaces import FARA_DISPLAY_SIZE+from .fara_types import WebSurferEvent+from .prompts import get_computer_use_system_prompt+from .utils import get_trimmed_url+++class FaraQwen3AgentConfig(AgentConfig):+ """Configuration for FaraAgent.++ Notable fields:+ extra_create_args: Extra kwargs merged on top of ``temperature: 0`` and+ forwarded to ``chat.completions.create``. vLLM-only keys are routed+ via ``extra_body``.+ auto_user_reply: When True, ``ask_user_question`` does NOT halt+ the run — a fixed dummy user response is injected and the agent+ continues. For eval / no-user-simulator settings only. Default False+ preserves ``task_workflow.py``'s ``WAITING_FOR_USER`` semantics for+ data_gen.+ captcha_timeout_limit: Number of consecutive captcha-wait timeouts+ tolerated within a single run before the per-step captcha-wait+ gate is auto-disabled for the rest of the run. Set to ``0`` to+ skip the gate entirely from the start (appropriate without a+ Browserbase cloud session, which is what triggers the captcha+ solver). Defaults to ``2``.+ raise_on_captcha_timeout: When True, restore the legacy behavior+ of raising ``RuntimeError`` on the first captcha-wait timeout.+ When False (default), the agent logs a warning and moves past+ the timed-out captcha, falling back to ``captcha_timeout_limit``+ to auto-disable the gate after repeated timeouts.+ """++ name: str = "fara"+ client_config: dict[str, Any] | None = None+ start_page: str = "about:blank"+ max_rounds: int = 10+ max_n_images: int = 3+ fn_call_template: str = "fara-qwen3vl"+ identity: str | None = "fara_qwen3vl"+ critical_points: str | None = "fara-1.5"+ computer_use_mode: str = "aurora"+ save_screenshots: bool = False+ animate_actions: bool = False+ single_tab_mode: bool = True+ include_input_text_key_args: bool = True+ viewport_width: int = 1440+ viewport_height: int = 900+ min_pixels: int = 3136+ max_pixels: int = 12845056+ extra_create_args: dict[str, Any] | None = None+ auto_user_reply: bool = False+ captcha_timeout_limit: int = 2+ raise_on_captcha_timeout: bool = True+ terminate_on_parse_error: bool = False+ image_token_estimate: int = 1500+ # When >0, drop oldest screenshots until the estimated prompt is <= this cap.+ image_budget_token_cap: int = 0+++@dataclass+class FaraQwen3AgentState:+ """Mutable state for FaraQwen3Agent that persists across resume calls."""++ chat_history: list[LLMMessage] = field(default_factory=list)+ facts: list[str] = field(default_factory=list)+ num_actions: int = 0+ current_step: int = 0+ mlm_width: int = 0+ mlm_height: int = 0+++class FaraQwen3Agent(Agent):+ """Web automation agent using vision-language models."""++ DEFAULT_START_PAGE = "https://www.bing.com/"+ DISPLAY_SIZE = FARA_DISPLAY_SIZE+ PATCH_SIZE = 16+ MERGE_SIZE = 2+ USER_MESSAGE = "Here is the next screenshot. Think about what to do next."+ MAX_URL_LENGTH = 100++ def __init__(+ self, config: FaraQwen3AgentConfig | dict[str, Any] | None = None, **kwargs: Any+ ):+ super().__init__(config, **kwargs)+ self.config: FaraQwen3AgentConfig+ if type(self) is FaraQwen3Agent:+ assert (+ self.config.computer_use_mode == "aurora"+ ), f"FaraQwen3Agent is aurora-v2 era; got computer_use_mode={self.config.computer_use_mode!r}"+ self.logger = logging.getLogger(__name__)+ self._client: ChatCompletionClient | None = None+ self._state: FaraQwen3AgentState | None = None+ self._pending_observation: str = ""+ self._os_type: str = "linux"+ self._output_dir = None+ # Captcha-wait disable: after `config.captcha_timeout_limit`+ # consecutive captcha-wait timeouts in a single run, stop waiting+ # for captchas at all. A limit of 0 disables the gate up front.+ self._captcha_timeouts: int = 0+ self._captcha_disabled: bool = self.config.captcha_timeout_limit <= 0++ @property+ def mlm_processor_im_cfg(self) -> dict[str, int]:+ return {+ "min_pixels": self.config.min_pixels,+ "max_pixels": self.config.max_pixels,+ "patch_size": self.PATCH_SIZE,+ "merge_size": self.MERGE_SIZE,+ }++ @classmethod+ def _get_config_class(cls) -> type[FaraQwen3AgentConfig]:+ return FaraQwen3AgentConfig++ async def initialize(self, run_context: RunContext) -> None:+ """Initialize the agent."""+ await super().initialize(run_context)+ self._state = FaraQwen3AgentState(+ mlm_width=self.config.viewport_width,+ mlm_height=self.config.viewport_height,+ )++ if self.config.client_config is not None:+ self._client = create_client_from_config(self.config.client_config)+ elif self.config.client is not None:+ self._client = self.config.client+ else:+ raise ValueError("Either client or client_config must be provided")++ async def run(+ self, run_context: RunContext, input: Any = None+ ) -> Tuple[str, List, List]:+ """Run the agent on a task.++ Supports resuming after WAITING_FOR_USER: if ``_state.chat_history``+ is already populated the fresh-start init is skipped and the user+ response is appended to the conversation.+ """+ env = run_context.environment+ traj = run_context.solver_log+ self._os_type = getattr(env, "os_type", "linux")+ self._output_dir = run_context.output_dir++ pending_user_response = ""+ if self._state.chat_history:+ # Continuation after WAITING_FOR_USER+ traj.status = SolverStatus.RUNNING+ pending_user_response = traj.get_last_user_message() or ""+ start_step = self._state.current_step+ else:+ # Fresh start+ task_instruction = run_context.task.instruction+ scaled_screenshot = await self._get_scaled_screenshot(env)+ await self._save_screenshot(+ env, run_context.output_dir, "screenshot_0_pre.png"+ )+ self._log_initial_observations(run_context, task_instruction)++ self._state.chat_history.append(+ UserMessage(+ content=[ImageObj.from_pil(scaled_screenshot), task_instruction],+ metadata={"is_original": True},+ )+ )+ start_step = 0++ all_actions = []+ all_observations = []+ final_answer = "<no_answer>"++ for step in range(start_step + 1, self.config.max_rounds + 1):+ is_first_round = step == 1++ if not self._captcha_disabled:+ if not await wait_for_captcha(env):+ if self.config.raise_on_captcha_timeout:+ raise RuntimeError(+ "Captcha timed out, unable to proceed with web surfing."+ )+ self._captcha_timeouts += 1+ self.logger.warning(+ "Captcha timed out at step %d (%d/%d); moving past "+ "unsolved captcha.",+ step,+ self._captcha_timeouts,+ self.config.captcha_timeout_limit,+ )+ if self._captcha_timeouts >= self.config.captcha_timeout_limit:+ self.logger.warning(+ "Captcha timed out %d times; disabling further "+ "captcha waits for the rest of this run.",+ self._captcha_timeouts,+ )+ self._captcha_disabled = True++ pre_screenshot_name = f"screenshot_{step}_pre.png"+ await self._save_screenshot(+ env, run_context.output_dir, pre_screenshot_name+ )+ current_url, page_context = await env.get_page_context()++ function_call, raw_response = await self._generate_model_call(+ env,+ is_first_round,+ scaled_screenshot if is_first_round else None,+ user_response=pending_user_response,+ )+ pending_user_response = ""+ all_actions.append(raw_response)++ action_args = function_call[0].arguments+ action_name = action_args.get("action", "unknown")+ thoughts = action_args.get("thoughts", "")++ self.logger.debug(+ f"\nThought #{step}: {thoughts}\nAction #{step}: executing tool '{action_name}' with arguments {json.dumps(action_args)}"+ )++ run_context.add_observation(+ ComputerObservation(+ screenshot_path=pre_screenshot_name,+ url=current_url,+ page_info=page_context,+ )+ )+ dp_action = self._log_action(+ run_context, action_name, action_args, raw_response+ )++ is_stop_action, action_description = await self._execute_action(+ env, function_call+ )+ all_observations.append(action_description)++ self.logger.debug(f"Observation#{step}: {action_description}")++ post_screenshot_name = f"screenshot_{step}_post.png"+ if is_stop_action:+ if run_context.output_dir:+ shutil.copyfile(+ run_context.output_dir / pre_screenshot_name,+ run_context.output_dir / post_screenshot_name,+ )+ page_context_after = page_context+ url_after = current_url+ else:+ await self._save_screenshot(+ env, run_context.output_dir, post_screenshot_name+ )+ url_after, page_context_after = await env.get_page_context()++ self._log_observation(+ run_context,+ action_description,+ post_screenshot_name,+ page_info=page_context_after,+ action_id=dp_action.id,+ url=url_after,+ )+ self._state.current_step = step+ run_context.checkpoint()++ if action_name == "ask_user_question":+ if self.config.auto_user_reply:+ pending_user_response = (+ "keep going so far as you don't make up information, "+ "you have my approval"+ )+ run_context.checkpoint()+ continue+ traj.status = SolverStatus.WAITING_FOR_USER+ run_context.checkpoint()+ return action_description, all_actions, all_observations++ if is_stop_action:+ final_answer = self._get_final_answer(thoughts, action_description)+ traj.outcome = Outcome(answer=final_answer)+ traj.status = SolverStatus.COMPLETE+ break++ run_context.checkpoint()+ if traj.status != SolverStatus.COMPLETE:+ final_answer = self._get_final_answer(thoughts, action_description)+ traj.outcome = Outcome(answer=final_answer)+ traj.status = SolverStatus.COMPLETE++ return final_answer, all_actions, all_observations++ async def close(self, run_context: RunContext) -> None:+ """Cleanup after the agent is done."""+ self._state = None+ self._client = None+ self._captcha_timeouts = 0+ self._captcha_disabled = self.config.captcha_timeout_limit <= 0+ await super().close(run_context)++ def _get_final_answer(self, thoughts: str, action_description: str) -> str:+ return thoughts++ def _get_observation_prefix(self) -> str:+ """Return text to prepend to the next user message. Override in subclasses."""+ obs = self._pending_observation+ self._pending_observation = ""+ return obs++ def _log_action(+ self,+ run_context: RunContext,+ name: str,+ args: dict,+ raw_response: str,+ ) -> DPAction:+ clean_args = {k: v for k, v in args.items() if k not in ("action", "thoughts")}+ thoughts = args.get("thoughts", "")+ action_summary = f"{name}({json.dumps(clean_args)})"+ nl_desc = f"{thoughts}\n{action_summary}" if thoughts else action_summary+ action = DPAction(+ action_name=name,+ content={"action": name, "arguments": dict(args)},+ action_nl_description=nl_desc,+ llm_conversation=LLMConversation(+ messages=[DPLLMMessage(raw_response=raw_response, reasoning=thoughts)]+ ),+ agent_state=AgentState(agent=self.name),+ )+ run_context.add_action(action)+ return action++ async def _save_screenshot(self, env, output_dir, filename: str) -> None:+ if output_dir:+ screenshot_bytes = await env.get_observation()+ output_dir.mkdir(parents=True, exist_ok=True)+ (output_dir / filename).write_bytes(screenshot_bytes)++ async def _get_scaled_screenshot(self, env) -> Image.Image:+ """Get current screenshot and scale it for the model."""+ screenshot_bytes = await env.get_observation()+ screenshot = Image.open(io.BytesIO(screenshot_bytes))+ _, scaled_screenshot = self._get_system_message(screenshot)+ return scaled_screenshot++ def _get_system_message(+ self, screenshot: Image.Image+ ) -> Tuple[List[SystemMessage], Image.Image]:+ system_prompt_info = get_computer_use_system_prompt(+ screenshot,+ self.mlm_processor_im_cfg,+ mode=self.config.computer_use_mode,+ include_input_text_key_args=self.config.include_input_text_key_args,+ fn_call_template=self.config.fn_call_template,+ randomize=False,+ display_size=self.DISPLAY_SIZE,+ identity=self.config.identity,+ critical_points=self.config.critical_points,+ )+ self._state.mlm_width, self._state.mlm_height = system_prompt_info["im_size"]+ scaled_screenshot = screenshot.resize(+ (self._state.mlm_width, self._state.mlm_height)+ )++ system_message = []+ for msg in system_prompt_info["conversation"]:+ tmp_content = ""+ for content in msg["content"]:+ tmp_content += content["text"]+ system_message.append(SystemMessage(content=tmp_content))++ return system_message, scaled_screenshot++ def _parse_thoughts_and_action(self, message: str) -> Tuple[str, Dict[str, Any]]:+ try:+ tmp = message.split("<tool_call>\n")+ thoughts = tmp[0].strip()+ action_text = tmp[1].split("\n</tool_call>")[0]+ try:+ action = json.loads(action_text)+ except json.decoder.JSONDecodeError:+ self.logger.error(f"Invalid action text: {action_text}")+ action = ast.literal_eval(action_text)+ return thoughts, action+ except Exception as e:+ self.logger.error(+ f"Error parsing thoughts and action: {message}", exc_info=True+ )+ if self.config.terminate_on_parse_error:+ self.logger.warning(+ "terminate_on_parse_error=true: ending trajectory with raw "+ "model response as final answer"+ )+ return message.strip(), {+ "name": "computer_use",+ "arguments": {+ "action": "terminate",+ "answer": message.strip(),+ },+ }+ raise e++ def convert_resized_coords_to_original(+ self, coords: List[float], rsz_w: int, rsz_h: int, og_w: int, og_h: int+ ) -> List[float]:+ scale_x = og_w / rsz_w+ scale_y = og_h / rsz_h+ return [coords[0] * scale_x, coords[1] * scale_y]++ def proc_coords(+ self,+ coords: List[float] | None,+ im_w: int,+ im_h: int,+ og_im_w: int | None = None,+ og_im_h: int | None = None,+ ) -> List[float] | None:+ if not coords:+ return coords+ if og_im_w is None:+ og_im_w = im_w+ if og_im_h is None:+ og_im_h = im_h+ tgt_x, tgt_y = coords+ return self.convert_resized_coords_to_original(+ [tgt_x, tgt_y], im_w, im_h, og_im_w, og_im_h+ )++ @retry(+ retry=retry_if_not_exception_type(openai.BadRequestError),+ stop=stop_after_attempt(5),+ wait=wait_exponential(multiplier=5.0, min=5.0, max=60),+ before_sleep=before_sleep_log(logging.getLogger(__name__), logging.WARNING),+ reraise=True,+ )+ async def _make_model_call(+ self,+ history: List[LLMMessage],+ extra_create_args: Dict[str, Any] | None = None,+ ) -> str:+ """Make a model call using the client."""+ if self._client is None:+ raise ValueError(+ "No client configured for FaraAgent. Call initialize() first."+ )++ result = await self._client.create(+ messages=history,+ extra_create_args=extra_create_args or {},+ )+ return result.content.content++ def remove_screenshot_from_message(self, msg: LLMMessage) -> LLMMessage | None:+ """Remove the screenshot from the message content."""+ if isinstance(msg.content, list):+ new_content = [c for c in msg.content if not isinstance(c, ImageObj)]+ msg.content = new_content+ return msg+ elif isinstance(msg.content, ImageObj):+ return None+ return msg++ def maybe_remove_old_screenshots(+ self, history: List[LLMMessage], includes_current: bool = False+ ) -> List[LLMMessage]:+ """Remove old screenshots from the chat history."""+ if self.config.max_n_images <= 0:+ return history++ max_n_images = (+ self.config.max_n_images+ if includes_current+ else self.config.max_n_images - 1+ )+ new_history: List[LLMMessage] = []+ n_images = 0++ for i in range(len(history) - 1, -1, -1):+ msg = history[i]+ meta = (msg.metadata or {}) if isinstance(msg, UserMessage) else {}+ preserve_text = meta.get("is_original", False) or meta.get(+ "is_user_response", False+ )++ if i == 0 and n_images >= max_n_images:+ msg = self.remove_screenshot_from_message(msg)+ if msg is None:+ continue++ if isinstance(msg.content, list):+ has_image = any(isinstance(c, ImageObj) for c in msg.content)+ if has_image:+ if n_images < max_n_images:+ new_history.append(msg)+ elif preserve_text:+ msg = self.remove_screenshot_from_message(msg)+ if msg is not None:+ raw = meta.get("user_response")+ if raw is not None:+ msg.content = [raw]+ new_history.append(msg)+ n_images += 1+ else:+ new_history.append(msg)+ elif isinstance(msg.content, ImageObj):+ if n_images < max_n_images:+ new_history.append(msg)+ n_images += 1+ else:+ new_history.append(msg)++ return new_history[::-1]++ def _estimate_prompt_tokens(self, history: List[LLMMessage]) -> int:+ total = 0+ for msg in history:+ items = msg.content if isinstance(msg.content, list) else [msg.content]+ for c in items:+ if isinstance(c, ImageObj):+ total += self.config.image_token_estimate+ elif isinstance(c, str):+ total += len(c) // 4 + 1+ return total++ @staticmethod+ def _msg_has_image(msg: LLMMessage) -> bool:+ c = msg.content+ if isinstance(c, list):+ return any(isinstance(x, ImageObj) for x in c)+ return isinstance(c, ImageObj)++ def _fit_images_to_budget(self, history: List[LLMMessage]) -> List[LLMMessage]:+ """Drop oldest screenshots (keeping the most recent) until the estimated+ prompt is within ``image_budget_token_cap``. Returns a new list; the+ original messages in ``chat_history`` are left unchanged."""+ cap = self.config.image_budget_token_cap+ if cap <= 0 or self._estimate_prompt_tokens(history) <= cap:+ return history+ out = list(history)+ img_idxs = [i for i, m in enumerate(out) if self._msg_has_image(m)]+ for i in img_idxs[:-1]:+ msg = copy.copy(out[i])+ msg.content = [x for x in msg.content if not isinstance(x, ImageObj)]+ out[i] = msg+ if self._estimate_prompt_tokens(out) <= cap:+ break+ return out++ async def _generate_model_call(+ self,+ env,+ is_first_round: bool,+ first_screenshot: Image.Image | None = None,+ user_response: str = "",+ ) -> Tuple[List[FunctionCall], str]:+ history = self.maybe_remove_old_screenshots(self._state.chat_history)++ screenshot_for_system = first_screenshot+ if not is_first_round:+ scaled_screenshot = await self._get_scaled_screenshot(env)+ screenshot_for_system = scaled_screenshot++ curr_url = (await env.get_page_context()).url+ if curr_url:+ trimmed_url = get_trimmed_url(curr_url, self.MAX_URL_LENGTH)+ url_prefix = f"Current URL: {trimmed_url}\n"+ else:+ url_prefix = ""+ observation_prefix = self._get_observation_prefix()+ if user_response:+ text_prompt = f"{url_prefix}{user_response}"+ elif observation_prefix:+ text_prompt = f"{url_prefix}{observation_prefix}\n{self.USER_MESSAGE}"+ else:+ text_prompt = f"{url_prefix}{self.USER_MESSAGE}"++ metadata = (+ {"is_user_response": True, "user_response": user_response}+ if user_response+ else None+ )+ curr_message = UserMessage(+ content=[ImageObj.from_pil(scaled_screenshot), text_prompt],+ metadata=metadata,+ )+ self._state.chat_history.append(curr_message)+ history.append(curr_message)++ system_message, _ = self._get_system_message(screenshot_for_system)+ history = system_message + history+ history = self._fit_images_to_budget(history)++ call_args: dict[str, Any] = {"temperature": 0}+ if self.config.extra_create_args:+ call_args.update(self.config.extra_create_args)+ message = await self._make_model_call(history, extra_create_args=call_args)++ self._state.chat_history.append(AssistantMessage(content=message))+ thoughts, action = self._parse_thoughts_and_action(message)+ action["arguments"]["thoughts"] = thoughts+ function_call = [FunctionCall(id="dummy", **action)]+ return function_call, message++ async def _execute_action(+ self, env, function_call: list[FunctionCall]+ ) -> tuple[bool, str]:+ """Execute an action on the environment."""+ name = function_call[0].name+ args = function_call[0].arguments+ action_description = ""++ self.logger.debug(+ WebSurferEvent(+ source="FaraAgent",+ url=(await env.get_page_context()).url,+ action=name,+ arguments=args,+ message=f"{name}( {json.dumps(args)} )",+ )+ )++ if "coordinate" in args:+ args["coordinate"] = self.proc_coords(+ args["coordinate"],+ self.DISPLAY_SIZE,+ self.DISPLAY_SIZE,+ self.config.viewport_width,+ self.config.viewport_height,+ )++ is_stop_action = False+ action_type = args["action"]++ if action_type == "visit_url":+ url = str(args["url"])+ action_description = f"I typed '{url}' into the browser address bar."+ if url.startswith(("https://", "http://", "file://", "about:")):+ await env.goto(url)+ elif " " in url:+ await env.goto(+ f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH"+ )+ else:+ await env.goto("https://" + url)++ elif action_type == "history_back":+ action_description = "I clicked the browser back button."+ await env.back()++ elif action_type == "web_search":+ query = args.get("query")+ action_description = f"I typed '{query}' into the browser search bar."+ encoded_query = quote_plus(query)+ await env.goto(f"https://www.bing.com/search?q={encoded_query}&FORM=QBLH")++ elif action_type == "scroll":+ pixels = int(args.get("pixels", 0))+ if pixels > 0:+ action_description = "I scrolled up one page in the browser."+ await env.scroll_up()+ elif pixels < 0:+ action_description = "I scrolled down one page in the browser."+ await env.scroll_down()++ elif action_type in ("keypress", "key"):+ keys = args.get("keys", [])+ action_description = f"I pressed the following keys: {keys}"+ await env.keypress(keys)++ elif action_type in ("hover", "mouse_move"):+ if "coordinate" in args:+ tgt_x, tgt_y = args["coordinate"]+ await env.hover(tgt_x, tgt_y)++ elif action_type in ("sleep", "wait"):+ duration = args.get("duration", args.get("time", 3.0))+ action_description = (+ "I am waiting a short period of time before taking further action."+ )+ await env.wait(duration)++ elif action_type in ("click", "left_click"):+ if "coordinate" in args:+ tgt_x, tgt_y = args["coordinate"]+ action_description = f"I clicked at coordinates ({tgt_x}, {tgt_y})."+ await env.click(tgt_x, tgt_y)++ elif action_type in ("input_text", "type"):+ text_value = args.get("text", args.get("text_value"))+ if text_value is None:+ raise ValueError(+ "input_text/type action requires 'text' or 'text_value' argument"+ )+ text_value = str(text_value)+ action_description = f"I typed '{text_value}'."+ press_enter = args.get("press_enter", True)+ delete_existing = args.get("delete_existing_text", False)++ if "coordinate" in args:+ tgt_x, tgt_y = args["coordinate"]+ await env.type_text(+ tgt_x,+ tgt_y,+ text_value,+ press_enter=press_enter,+ clear_first=delete_existing,+ )++ elif action_type == "pause_and_memorize_fact":+ fact = str(args.get("fact"))+ self._state.facts.append(fact)+ action_description = f"I memorized the following fact: {fact}"++ elif action_type in ("stop", "terminate"):+ action_description = args.get("thoughts", "Task terminated")+ is_stop_action = True++ else:+ raise ValueError(f"Unknown action: {action_type}")++ if not is_stop_action:+ await env.wait_for_load()++ self._state.num_actions += 1+ return is_stop_action, action_descriptiondiff --git a/src/fara/agents/fara/fara_qwen3_next.py b/src/fara/agents/fara/fara_qwen3_next.pynew file mode 100644index 0000000..9880c04--- /dev/null+++ b/src/fara/agents/fara/fara_qwen3_next.py@@ -0,0 +1,373 @@+"""FaraQwen3NextAgent - Extends FaraQwen3Agent with expanded tool sets."""++from __future__ import annotations++import json+import re+from typing import Any+from urllib.parse import quote_plus++from ...clients.messages import FunctionCall, ImageObj, SystemMessage, UserMessage++from ...core.run_context import RunContext+from .fara_qwen3 import FaraQwen3Agent, FaraQwen3AgentConfig+from .fara_types import WebSurferEvent+from .prompts import get_computer_use_system_prompt, TOOL_SET_TO_MODE+from ..computer_agent.utils import extract_from_page+from ..utils import (+ format_text_observation,+ save_and_truncate_command_output,+ wrap_command_with_tee,+)+++def extract_allowed_actions(system_prompt_text: str) -> frozenset[str]:+ """Extract the allowed action names from the tool schema enum in the system prompt."""+ match = re.search(r'"enum":\s*\[([^\]]+)\]', system_prompt_text)+ if not match:+ raise ValueError("Could not find action enum in system prompt")+ return frozenset(json.loads(f"[{match.group(1)}]"))+++class FaraQwen3NextAgentConfig(FaraQwen3AgentConfig):+ """Configuration for FaraQwen3NextAgent."""++ name: str = "fara_next"+ max_observation_chars: int = 1000+ computer_use_mode: str = "fara_next_browser"+ identity: str | None = "fara_qwen3vl"+ critical_points: str | None = "fara-1.5"+++class FaraQwen3NextAgent(FaraQwen3Agent):+ """Web/desktop automation agent with expanded tool sets (FaraNext).++ Extends FaraQwen3Agent with:+ - Dynamic tool mode selection (browser vs windows) via config.computer_use_mode+ - New GUI actions: double_click, right_click, triple_click, left_click_drag, hscroll+ - New non-GUI actions: read_page_answer_question, ask_user_question, run_command+ - Cursor position tracking for left_click_drag+ - press_enter defaults to False (model should use key(["Enter"]) explicitly)+ """++ _TEXT_OBSERVATION_ACTIONS: frozenset[str] = frozenset(+ {+ "run_command",+ "read_page_answer_question",+ }+ )++ _MODE_CANONICAL_IDENTITY: dict[str, tuple[str, ...]] = {+ "fara_next_browser": ("fara_qwen3vl", "fara_qwen35"),+ "fara_next_windows": ("fara_qwen3vl_windows", "fara_qwen35_windows"),+ "fara_next_windows_core": ("fara_qwen3vl_windows", "fara_qwen35_windows"),+ }++ def __init__(+ self,+ config: FaraQwen3NextAgentConfig | dict[str, Any] | None = None,+ **kwargs: Any,+ ):+ super().__init__(config, **kwargs)+ self.config: FaraQwen3NextAgentConfig+ self._allowed_actions: frozenset[str] = frozenset()+ self._cursor_x: float = 0.0+ self._cursor_y: float = 0.0+ self._pending_observation: str = ""++ @classmethod+ def _get_config_class(cls) -> type[FaraQwen3NextAgentConfig]:+ return FaraQwen3NextAgentConfig++ async def initialize(self, run_context: RunContext) -> None:+ await super().initialize(run_context)+ valid_modes = set(TOOL_SET_TO_MODE.values())+ if self.config.computer_use_mode not in valid_modes:+ raise ValueError(+ f"Unknown computer_use_mode '{self.config.computer_use_mode}'. "+ f"Known values: {sorted(valid_modes)}"+ )+ canonical_identities = self._MODE_CANONICAL_IDENTITY[+ self.config.computer_use_mode+ ]+ if (+ self.config.identity is not None+ and self.config.identity not in canonical_identities+ ):+ raise ValueError(+ f"identity={self.config.identity!r} does not match any canonical "+ f"identity in {list(canonical_identities)!r} for "+ f"computer_use_mode={self.config.computer_use_mode!r}. "+ f"Use one of {list(canonical_identities)!r} or set identity=None "+ f"to auto-detect."+ )++ def _get_system_message(self, screenshot):+ system_prompt_info = get_computer_use_system_prompt(+ screenshot,+ self.mlm_processor_im_cfg,+ mode=self.config.computer_use_mode,+ include_input_text_key_args=self.config.include_input_text_key_args,+ fn_call_template=self.config.fn_call_template,+ randomize=False,+ display_size=self.DISPLAY_SIZE,+ identity=self.config.identity,+ critical_points=self.config.critical_points,+ )+ self._state.mlm_width, self._state.mlm_height = system_prompt_info["im_size"]+ scaled_screenshot = screenshot.resize(+ (self._state.mlm_width, self._state.mlm_height)+ )++ system_message = []+ full_text = ""+ for msg in system_prompt_info["conversation"]:+ tmp_content = ""+ for content in msg["content"]:+ tmp_content += content["text"]+ system_message.append(SystemMessage(content=tmp_content))+ full_text += tmp_content++ self._allowed_actions = extract_allowed_actions(full_text)++ return system_message, scaled_screenshot++ def _get_final_answer(self, thoughts: str, action_description: str) -> str:+ return action_description++ def _get_observation_prefix(self) -> str:+ obs = self._pending_observation+ self._pending_observation = ""+ return obs++ def maybe_remove_old_screenshots(self, history, includes_current=False):+ """Override to also strip observation text from old messages in windows mode."""+ result = super().maybe_remove_old_screenshots(history, includes_current)+ if "windows" not in self.config.computer_use_mode:+ return result++ for msg in result:+ if not isinstance(msg, UserMessage):+ continue+ if (msg.metadata or {}).get("is_original", False):+ continue+ if isinstance(msg.content, list):+ has_image = any(isinstance(c, ImageObj) for c in msg.content)+ if not has_image:+ msg.content = [+ self.USER_MESSAGE if isinstance(c, str) else c+ for c in msg.content+ ]++ return result++ async def _execute_action(+ self, env, function_call: list[FunctionCall]+ ) -> tuple[bool, str]:+ """Execute an action. Only actions in the current schema are allowed."""+ name = function_call[0].name+ args = function_call[0].arguments+ action_type = args.get("action", "")++ if action_type not in self._allowed_actions:+ if self.config.terminate_on_parse_error:+ self.logger.warning(+ "terminate_on_parse_error=true: invalid action %r — "+ "ending trajectory with thoughts as final answer",+ action_type,+ )+ return True, args.get("thoughts", "") or args.get("answer", "")+ raise ValueError(+ f"Action '{action_type}' is not allowed in mode "+ f"'{self.config.computer_use_mode}'. Allowed: {sorted(self._allowed_actions)}"+ )++ url = (await env.get_page_context()).url+ self.logger.debug(+ WebSurferEvent(+ source="FaraQwen3NextAgent",+ url=url,+ action=name,+ arguments=args,+ message=f"{name}( {json.dumps(args)} )",+ )+ )++ if "coordinate" in args:+ args["coordinate"] = self._proc(args["coordinate"])++ is_stop_action, action_description = await self._dispatch_action(+ env, action_type, args+ )++ if action_type in self._TEXT_OBSERVATION_ACTIONS:+ self._pending_observation = format_text_observation(+ action_type,+ action_description,+ self.config.max_observation_chars,+ )++ if not is_stop_action:+ if hasattr(env, "wait_for_load"):+ await env.wait_for_load()++ self._state.num_actions += 1+ return is_stop_action, action_description++ async def _dispatch_action(+ self, env, action_type: str, args: dict+ ) -> tuple[bool, str]:+ """Dispatch action using ComputerEnvironment canonical methods."""++ # -- GUI actions --++ if action_type == "left_click":+ tgt_x, tgt_y = args["coordinate"]+ await env.left_click(tgt_x, tgt_y)+ self._cursor_x, self._cursor_y = tgt_x, tgt_y+ return False, f"I clicked at coordinates ({tgt_x}, {tgt_y})."++ elif action_type == "key":+ keys = args.get("keys", [])+ await env.key(keys)+ return False, f"I pressed the following keys: {keys}"++ elif action_type == "mouse_move":+ tgt_x, tgt_y = args["coordinate"]+ await env.mouse_move(tgt_x, tgt_y)+ self._cursor_x, self._cursor_y = tgt_x, tgt_y+ return False, f"I moved the cursor to ({tgt_x}, {tgt_y})."++ elif action_type == "type":+ text = str(args.get("text", args.get("text_value", "")))+ await env.type(text)+ return False, f"I typed '{text}'."++ elif action_type == "scroll":+ pixels = int(args.get("pixels", 0))+ pixels = int(pixels * self.config.viewport_height / self.DISPLAY_SIZE)+ await env.scroll(pixels)+ direction = "up" if pixels > 0 else "down"+ return False, f"I scrolled {direction}."++ elif action_type == "double_click":+ tgt_x, tgt_y = args["coordinate"]+ await env.double_click(tgt_x, tgt_y)+ self._cursor_x, self._cursor_y = tgt_x, tgt_y+ return False, f"I double-clicked at coordinates ({tgt_x}, {tgt_y})."++ elif action_type == "right_click":+ tgt_x, tgt_y = args["coordinate"]+ await env.right_click(tgt_x, tgt_y)+ self._cursor_x, self._cursor_y = tgt_x, tgt_y+ return False, f"I right-clicked at coordinates ({tgt_x}, {tgt_y})."++ elif action_type == "triple_click":+ tgt_x, tgt_y = args["coordinate"]+ await env.triple_click(tgt_x, tgt_y)+ self._cursor_x, self._cursor_y = tgt_x, tgt_y+ return False, f"I triple-clicked at coordinates ({tgt_x}, {tgt_y})."++ elif action_type == "left_click_drag":+ tgt_x, tgt_y = args["coordinate"]+ await env.left_click_drag(tgt_x, tgt_y)+ self._cursor_x, self._cursor_y = tgt_x, tgt_y+ return False, f"I dragged to ({tgt_x}, {tgt_y})."++ elif action_type == "hscroll":+ pixels = int(args.get("pixels", 0))+ pixels = int(pixels * self.config.viewport_width / self.DISPLAY_SIZE)+ await env.hscroll(pixels)+ return False, f"I scrolled horizontally by {pixels} pixels."++ # -- Browser-only actions --++ elif action_type == "visit_url":+ url = args.get("url", "")+ if url.startswith(("https://", "http://", "file://", "about:")):+ target = url+ elif " " in url:+ target = f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH"+ else:+ target = "https://" + url+ try:+ await env.goto_url(target)+ except Exception as e:+ msg = f"visit_url to {url} failed: {type(e).__name__}: {e}"+ self._pending_observation = msg+ return False, f"I tried to navigate to {url} but it failed: {e}"+ return False, f"I navigated to {url}."++ elif action_type == "history_back":+ await env.go_back()+ return False, "I clicked the browser back button."++ elif action_type == "web_search":+ query = args.get("query", "")+ await env.goto_url(+ f"https://www.bing.com/search?q={quote_plus(query)}&FORM=QBLH"+ )+ return False, f"I searched for '{query}'."++ elif action_type == "read_page_answer_question":+ question = str(args.get("question", ""))+ markdown = await env.get_page_markdown()+ answer = await extract_from_page(markdown, question, self._client)+ return False, f"I read the page to answer: {question}\nAnswer: {answer}"++ # -- Windows-only actions --++ elif action_type == "run_command":+ command = str(args.get("command", ""))+ step = self._state.current_step+ cmd = wrap_command_with_tee(command, step, self._os_type)+ result = await env.execute(cmd)+ if isinstance(result, dict):+ stdout = result.get("stdout", "")+ stderr = result.get("stderr", "")+ result = f"{stdout}\n{stderr}".strip() if stderr else stdout+ else:+ result = str(result)+ result = save_and_truncate_command_output(+ result,+ self._output_dir,+ step,+ max_chars=self.config.max_observation_chars,+ os_type=self._os_type,+ )+ return False, f"I executed command: {command}\nOutput: {result}"++ # -- Shared non-GUI actions --++ elif action_type == "ask_user_question":+ question = str(args.get("question", ""))+ return True, f"I asked the user: {question}"++ elif action_type == "wait":+ duration = args.get("time", args.get("duration", 3.0))+ await env.wait(duration)+ return False, f"I waited {duration}s."++ elif action_type == "pause_and_memorize_fact":+ fact = str(args.get("fact", ""))+ self._state.facts.append(fact)+ return False, f"I memorized the following fact: {fact}"++ elif action_type == "terminate":+ answer = args.get("answer")+ if answer is None:+ raise ValueError("terminate action requires 'answer' argument")+ return True, answer++ else:+ raise ValueError(f"Unknown action: {action_type}")++ def _proc(self, coordinate):+ """Shorthand for proc_coords with standard display/viewport args."""+ return self.proc_coords(+ coordinate,+ self.DISPLAY_SIZE,+ self.DISPLAY_SIZE,+ self.config.viewport_width,+ self.config.viewport_height,+ )diff --git a/src/fara/agents/fara/fara_types.py b/src/fara/agents/fara/fara_types.pynew file mode 100644index 0000000..b4379f1--- /dev/null+++ b/src/fara/agents/fara/fara_types.py@@ -0,0 +1,15 @@+"""Fara-specific types. Most types are imported from aztool.clients."""++from dataclasses import dataclass+from typing import Any, Dict+++@dataclass+class WebSurferEvent:+ """Event for logging web surfing actions."""++ source: str+ message: str+ url: str+ action: str | None = None+ arguments: Dict[str, Any] | None = Nonediff --git a/src/fara/agents/fara/prompts.py b/src/fara/agents/fara/prompts.pynew file mode 100644index 0000000..f536e23--- /dev/null+++ b/src/fara/agents/fara/prompts.py@@ -0,0 +1,20 @@+"""Fara prompt entry points.++``get_computer_use_system_prompt`` is re-exported from this repo's+``fara._prompts`` (which ships the FaraNext browser tool schema, the model+identities, the critical-points block, and the fn-call template). Only the+browser tool set is supported here.+"""++from ..._prompts import get_computer_use_system_prompt # noqa: F401++TOOL_SET_TO_MODE = {+ "BROWSER_TOOLS_CORE": "fara_next_browser",+ "GPT54_BROWSER_TOOLS_CORE": "fara_next_browser",+ "BROWSER_TOOLS_WITH_READ_PAGE": "fara_next_browser",+ "GPT54_BROWSER_TOOLS_WITH_READ_PAGE": "fara_next_browser",+ "WINDOWS_TOOLS_CORE": "fara_next_windows_core",+ "WINDOWS_TOOLS_WITH_RUN_COMMAND": "fara_next_windows",+ "GPT54_WINDOWS_TOOLS_CORE": "fara_next_windows_core",+ "GPT54_WINDOWS_TOOLS_WITH_RUN_COMMAND": "fara_next_windows",+}diff --git a/src/fara/agents/fara/utils.py b/src/fara/agents/fara/utils.pynew file mode 100644index 0000000..7ebe614--- /dev/null+++ b/src/fara/agents/fara/utils.py@@ -0,0 +1,9 @@+def strip_url_query(url):+ return url.split("?", 1)[0]+++def get_trimmed_url(url, max_len):+ trimmed_url = strip_url_query(url)+ if len(trimmed_url) > max_len:+ trimmed_url = trimmed_url[:max_len] + " ..."+ return trimmed_urldiff --git a/src/fara/agents/key_mapping.py b/src/fara/agents/key_mapping.pynew file mode 100644index 0000000..7a51800--- /dev/null+++ b/src/fara/agents/key_mapping.py@@ -0,0 +1,34 @@+"""CUA key-name mapping shared across agents and environments."""++CUA_KEY_TO_PLAYWRIGHT_KEY = {+ "/": "Divide",+ "\\": "Backslash",+ "alt": "Alt",+ "arrowdown": "ArrowDown",+ "arrowleft": "ArrowLeft",+ "arrowright": "ArrowRight",+ "arrowup": "ArrowUp",+ "down": "ArrowDown",+ "left": "ArrowLeft",+ "right": "ArrowRight",+ "up": "ArrowUp",+ "backspace": "Backspace",+ "capslock": "CapsLock",+ "cmd": "Meta",+ "ctrl": "Control",+ "delete": "Delete",+ "end": "End",+ "enter": "Enter",+ "esc": "Escape",+ "escape": "Escape",+ "home": "Home",+ "insert": "Insert",+ "option": "Alt",+ "pagedown": "PageDown",+ "pageup": "PageUp",+ "shift": "Shift",+ "space": " ",+ "super": "Meta",+ "tab": "Tab",+ "win": "Meta",+}diff --git a/src/fara/agents/utils.py b/src/fara/agents/utils.pynew file mode 100644index 0000000..b8bec8c--- /dev/null+++ b/src/fara/agents/utils.py@@ -0,0 +1,136 @@+"""Utilities shared across agent implementations."""++from __future__ import annotations++import logging+import os+from pathlib import Path++from PIL import Image++logger = logging.getLogger(__name__)++CMD_OUTPUT_DIR = "cmd_outputs"+CMD_OUTPUT_ENV_DIR_LINUX = "/tmp"+CMD_OUTPUT_ENV_DIR_WINDOWS = r"C:\Users\Docker\AppData\Local\Temp"+++def get_cmd_output_env_path(step: int, os_type: str = "linux") -> str:+ """Return the in-environment file path for a step's command output."""+ if os_type == "windows":+ return f"{CMD_OUTPUT_ENV_DIR_WINDOWS}\\cmd_output_step_{step}.txt"+ return f"{CMD_OUTPUT_ENV_DIR_LINUX}/cmd_output_step_{step}.txt"+++def wrap_command_with_tee(command: str, step: int, os_type: str = "linux") -> str:+ """Wrap a shell command so its combined stdout+stderr is teed to a file.++ The original command's output is still returned; a copy is saved inside+ the environment for the agent to read.++ On Linux (Bash): uses `(cmd) 2>&1 | tee <path>`+ On Windows (PowerShell): uses `& { cmd } 2>&1 | Tee-Object -FilePath <path>`+ """+ env_path = get_cmd_output_env_path(step, os_type)+ if os_type == "windows":+ return f"& {{ {command} }} 2>&1 | Tee-Object -FilePath '{env_path}'"+ return f"({command}) 2>&1 | tee {env_path}"+++def save_and_truncate_command_output(+ output: str,+ output_dir: Path | None,+ step: int,+ max_chars: int = 1000,+ os_type: str = "linux",+) -> str:+ """Save full output to host, return truncated version for LLM context.++ - If output <= max_chars (or max_chars <= 0): return output unchanged.+ - Otherwise: save full output to <output_dir>/cmd_outputs/step_<N>.txt+ and return first+last halves with a truncation notice + env file path.+ """+ if max_chars <= 0 or len(output) <= max_chars:+ return output++ # Save to host output_dir for post-run inspection+ if output_dir:+ host_dir = output_dir / CMD_OUTPUT_DIR+ os.makedirs(host_dir, exist_ok=True)+ host_path = host_dir / f"step_{step}.txt"+ host_path.write_text(output, encoding="utf-8")+ logger.debug("Saved full command output to %s", host_path)++ env_path = get_cmd_output_env_path(step, os_type)+ read_cmd = (+ f"Get-Content '{env_path}'" if os_type == "windows" else f"cat {env_path}"+ )+ half = max_chars // 2+ return (+ f"{output[:half]}\n"+ f"... [truncated {len(output) - max_chars} chars] ...\n"+ f"{output[-half:]}\n"+ f"Full output saved to: {env_path}\n"+ f"You can read it with: {read_cmd}"+ )+++def truncate_observation(text: str, max_chars: int = 1000) -> str:+ """Truncate text keeping first and last halves with a marker in between.++ Duplicated in agento_train/data/convo_builder.py (FaraNextConvoBuilder._truncate_observation)+ because agento_train cannot import from agento_next. Keep both in sync.+ """+ if len(text) <= max_chars:+ return text+ half = max_chars // 2+ return (+ text[:half]+ + f"\n... [truncated {len(text) - max_chars} chars] ...\n"+ + text[-half:]+ )+++def format_text_observation(action_name: str, text: str, max_chars: int = 1000) -> str:+ """Prepare text observations for the next model turn.++ ``run_command`` observations may already include a saved-file hint from+ ``save_and_truncate_command_output``. Preserve them as-is so that hint+ isn't lost to a second truncation pass. Other text observations still use+ the standard truncation helper.+ """+ if action_name == "run_command":+ return text+ return truncate_observation(text, max_chars)+++def get_screenshot_dimensions(screenshot: Image.Image) -> tuple[int, int]:+ """Return validated screenshot dimensions for prompt construction."""+ width, height = screenshot.size+ if width <= 0 or height <= 0:+ raise ValueError(f"Invalid screenshot size: {width}x{height}")+ return int(width), int(height)+++def build_screen_text(+ prefix_text: str,+ screenshot: Image.Image,+ url: str = "",+ page_context: str = "",+ context_summary: str = "",+) -> str:+ """Build the text block paired with a screenshot before a model call."""+ width, height = get_screenshot_dimensions(screenshot)+ parts = [+ prefix_text,+ f"Screenshot resolution: {width}x{height}",+ ]+ if url:+ parts.append(f"Current URL: {url}")+ if page_context:+ parts.append(page_context)++ text = "\n".join(parts)+ if context_summary:+ text = f"{text}\n\n{context_summary}"+ return textdiff --git a/src/fara/clients/__init__.py b/src/fara/clients/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/fara/clients/create_utils.py b/src/fara/clients/create_utils.pynew file mode 100644index 0000000..e96e069--- /dev/null+++ b/src/fara/clients/create_utils.py@@ -0,0 +1,14 @@+"""Client factory. Builds a ChatCompletionClient from an endpoint config dict."""++from typing import Any, Dict++from .wrapper import ChatCompletionClient+++def create_client_from_config(config: Dict[str, Any]) -> ChatCompletionClient:+ """Create a ChatCompletionClient from a ``{model, base_url, api_key}`` dict."""+ return ChatCompletionClient(+ model=config.get("model", "Fara1.5-9B"),+ base_url=config.get("base_url"),+ api_key=config.get("api_key"),+ )diff --git a/src/fara/clients/messages.py b/src/fara/clients/messages.pynew file mode 100644index 0000000..77c26ef--- /dev/null+++ b/src/fara/clients/messages.py@@ -0,0 +1,148 @@+import io+import base64+from dataclasses import dataclass, field+from typing import Any, List, Tuple, Dict, Optional+from PIL import Image++ToolSchema = Dict[str, Any]+Tool = ToolSchema+++@dataclass+class LLMMessage:+ content: str | List[Dict[str, Any]]+ source: str = "user"+ metadata: Optional[Dict[str, Any]] = None+++@dataclass+class SystemMessage(LLMMessage):+ source: str = "system"+++@dataclass+class UserMessage(LLMMessage):+ source: str = "user"+++@dataclass+class AssistantMessage(LLMMessage):+ source: str = "assistant"+++@dataclass+class ImageObj:+ """Image wrapper for handling screenshots and images"""++ image: Image.Image++ @classmethod+ def from_pil(cls, image: Image.Image) -> "ImageObj":+ return cls(image=image)++ def to_base64(self) -> str:+ """Convert PIL image to base64 string"""+ buffered = io.BytesIO()+ self.image.save(buffered, format="PNG")+ return base64.b64encode(buffered.getvalue()).decode("utf-8")++ def resize(self, size: Tuple[int, int]) -> Image.Image:+ """Resize the image"""+ return self.image.resize(size)+++@dataclass+class RequestUsage:+ """Usage statistics for a model request"""++ prompt_tokens: int = 0+ completion_tokens: int = 0+ reasoning_tokens: int = 0+ num_calls: int = 1++ def __add__(self, other: "RequestUsage") -> "RequestUsage":+ return RequestUsage(+ prompt_tokens=self.prompt_tokens + other.prompt_tokens,+ completion_tokens=self.completion_tokens + other.completion_tokens,+ reasoning_tokens=self.reasoning_tokens + other.reasoning_tokens,+ num_calls=self.num_calls + other.num_calls,+ )++ def get_cost(self, prompt_price: float, completion_price: float) -> float:+ """+ Calculate cost based on token usage and pricing.+ Args:+ prompt_price: Price per prompt token+ completion_price: Price per completion token+ """+ return (self.prompt_tokens * prompt_price) + (+ self.completion_tokens * completion_price+ )+++@dataclass+class ModelResponse:+ """Response from model call"""++ content: str+ usage: RequestUsage = field(default_factory=RequestUsage)+++@dataclass+class CreateResult:+ """Result from a chat completion request.++ Attributes:+ content: The message object from the API response.+ For OpenAI/Azure: access text via content.content, tool calls via content.tool_calls.+ For Anthropic: access text via content.content[0].text.+ usage: Token usage statistics for the request.+ finish_reason: The reason the model stopped generating ('stop', 'tool_calls', etc.).+ cached: Whether this result was returned from cache.+ """++ content: Any+ usage: RequestUsage = field(default_factory=RequestUsage)+ finish_reason: Optional[str] = None+ cached: bool = False+++@dataclass+class FunctionCall:+ """Represents a function call with arguments"""++ id: str+ name: str+ arguments: Dict[str, Any]+++def message_to_openai_format(message: LLMMessage) -> Dict[str, Any]:+ """Convert our LLMMessage to OpenAI API format"""+ role = (+ "system"+ if isinstance(message, SystemMessage)+ else "assistant"+ if isinstance(message, AssistantMessage)+ else "user"+ )++ # Handle multimodal content (text + images)+ if isinstance(message.content, list):+ content_parts = []+ for item in message.content:+ if isinstance(item, ImageObj):+ # Convert image to base64 data URL+ base64_image = item.to_base64()+ content_parts.append(+ {+ "type": "image_url",+ "image_url": {"url": f"data:image/png;base64,{base64_image}"},+ }+ )+ elif isinstance(item, str):+ content_parts.append({"type": "text", "text": item})+ elif isinstance(item, dict):+ content_parts.append(item)+ return {"role": role, "content": content_parts}+ else:+ return {"role": role, "content": message.content}diff --git a/src/fara/clients/wrapper.py b/src/fara/clients/wrapper.pynew file mode 100644index 0000000..edecfa1--- /dev/null+++ b/src/fara/clients/wrapper.py@@ -0,0 +1,52 @@+"""Minimal OpenAI-compatible chat completion client.++Replaces aztool's ChatCompletionClient with a thin wrapper over+``openai.AsyncOpenAI`` that exposes the same ``create(messages,+extra_create_args)`` interface the Fara agents expect, returning a+``CreateResult`` whose ``.content`` is the raw API message object+(``.content.content`` is the text).+"""++from typing import Any, Dict, List++from openai import AsyncOpenAI++from .messages import CreateResult, LLMMessage, RequestUsage, message_to_openai_format+++class ChatCompletionClient:+ """Chat completion client backed by an OpenAI-compatible endpoint."""++ def __init__(+ self,+ model: str,+ base_url: str | None = None,+ api_key: str | None = None,+ ):+ self.model = model+ self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)++ async def create(+ self,+ messages: List[LLMMessage],+ extra_create_args: Dict[str, Any] | None = None,+ ) -> CreateResult:+ request_params: Dict[str, Any] = {+ "model": self.model,+ "messages": [message_to_openai_format(m) for m in messages],+ }+ if extra_create_args:+ request_params.update(extra_create_args)++ response = await self._client.chat.completions.create(**request_params)+ usage = RequestUsage()+ if response.usage:+ usage = RequestUsage(+ prompt_tokens=response.usage.prompt_tokens,+ completion_tokens=response.usage.completion_tokens,+ )+ return CreateResult(+ content=response.choices[0].message,+ usage=usage,+ finish_reason=response.choices[0].finish_reason,+ )diff --git a/src/fara/core/__init__.py b/src/fara/core/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/fara/core/agent.py b/src/fara/core/agent.pynew file mode 100644index 0000000..72fca6c--- /dev/null+++ b/src/fara/core/agent.py@@ -0,0 +1,170 @@+"""Agent base class - replaces autogen dependency."""++from __future__ import annotations++import shutil+from abc import ABC, abstractmethod+from typing import Any++from pydantic import BaseModel, ConfigDict++from .data_point import ComputerObservation, ToolOutput, UserMessage as UserMessageObs+from .run_context import RunContext+++class AgentConfig(BaseModel):+ """Base configuration for agents."""++ model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)++ name: str = "agent"+ client: Any = None+ metadata: dict[str, Any] = {}+++class Agent(ABC):+ """+ Base class for all agents.++ Replaces autogen dependency with a simple interface:+ - initialize: setup the agent+ - run: execute the agent's logic+ - close: cleanup++ Agents are responsible for logging their actions to run_context.data_point.solver_log+ """++ def __init__(+ self, config: AgentConfig | dict[str, Any] | None = None, **kwargs: Any+ ):+ if config is None:+ config = self._get_config_class().model_validate(kwargs)+ elif isinstance(config, dict):+ config = self._get_config_class().model_validate(config)+ self.config = config+ self._initialized = False++ @classmethod+ def _get_config_class(cls) -> type[AgentConfig]:+ """Override in subclasses to return the specific config class."""+ return AgentConfig++ @property+ def name(self) -> str:+ return self.config.name++ @property+ def client(self) -> Any:+ return self.config.client++ async def initialize(self, run_context: RunContext) -> None:+ """+ Initialize the agent for a run.++ Override this to setup any resources needed.+ Called before run().++ Args:+ run_context: The shared run context+ """+ self._initialized = True++ @abstractmethod+ async def run(+ self,+ run_context: RunContext,+ input: Any = None,+ ) -> Any:+ """+ Execute the agent's main logic.++ Args:+ run_context: Shared context with environment and data point+ input: Optional input (e.g., Task for orchestrator)++ Returns:+ Agent-specific output+ """+ ...++ async def close(self, run_context: RunContext) -> None:+ """+ Cleanup after the agent is done.++ Override this to release any resources.+ Called after run() completes or on error.++ Args:+ run_context: The shared run context+ """+ self._initialized = False++ def _log_observation(+ self,+ run_context: RunContext,+ output: str,+ screenshot: str,+ page_info: str,+ error: str = "",+ action_id: str | None = None,+ url: str = "",+ ) -> None:+ """Log post-action observations (ToolOutput, ComputerObservation) to the solver log."""+ run_context.add_observation(+ ToolOutput(output=output, error=error, action_id=action_id)+ )+ if screenshot or page_info:+ run_context.add_observation(+ ComputerObservation(+ screenshot_path=screenshot,+ url=url,+ page_info=page_info,+ action_id=action_id,+ )+ )++ def _log_initial_observations(+ self,+ run_context: RunContext,+ task: str,+ screenshot_path: str = "screenshot_0_pre.png",+ ) -> None:+ """Log the initial task message and screenshot at the start of a fresh run."""+ run_context.add_observation(UserMessageObs(content=task))+ run_context.add_observation(+ ComputerObservation(screenshot_path=screenshot_path)+ )+ if run_context.output_dir:+ post_path = screenshot_path.replace("_pre.", "_post.")+ shutil.copyfile(+ run_context.output_dir / screenshot_path,+ run_context.output_dir / post_path,+ )++ def save_state(self, run_context: RunContext) -> None:+ """+ Save agent-specific state for resuming.++ Override to save any state needed to resume mid-run.++ Args:+ run_context: The shared run context+ """+ pass++ def load_state(self, run_context: RunContext) -> None:+ """+ Load agent-specific state when resuming.++ Override to restore state from a previous run.++ Args:+ run_context: The shared run context+ """+ pass++ async def __aenter__(self):+ return self++ async def __aexit__(self, exc_type, exc_val, exc_tb):+ passdiff --git a/src/fara/core/data_point.py b/src/fara/core/data_point.pynew file mode 100644index 0000000..911d426--- /dev/null+++ b/src/fara/core/data_point.py@@ -0,0 +1,717 @@+"""DataPoint - the output of a single data generation run."""++from typing import (+ Annotated,+ Any,+ Dict,+ Iterable,+ List,+ Literal,+ Optional,+ Sequence,+ Tuple,+ Type,+ TypeVar,+ Union,+)+from datetime import datetime+from enum import Enum+import json+import os+import uuid++from pydantic import BaseModel, Field, model_validator+++class Component(BaseModel):+ """Base class providing serialization/deserialization via pydantic."""++ def to_dict(self) -> Dict[str, Any]:+ return self.model_dump(mode="python")++ @classmethod+ def from_dict(cls, data: Dict[str, Any]) -> "Component":+ return cls.model_validate(data)++ def save(self, path: os.PathLike) -> None:+ with open(path, "w") as f:+ json.dump(self.to_dict(), f, indent=2)++ @classmethod+ def load(cls, path: os.PathLike) -> "Component":+ with open(path, "r") as f:+ return cls.from_dict(json.load(f))+++class SolverStatus(str, Enum):+ RUNNING = "running"+ COMPLETE = "complete"+ # Loop ended without a clean terminate. TIMED_OUT / MAX_ROUNDS name the two+ # structured cut-offs; ABORTED is the catch-all for any other abnormal end+ # (e.g. an agent crash). The cause is machine-checkable on the status alone,+ # without a free-text field. (LLM-call finish reasons live on+ # LLMMessage.finish_reason.)+ ABORTED = "aborted"+ TIMED_OUT = "timed_out"+ MAX_ROUNDS = "max_rounds"+ WAITING_FOR_USER = "waiting_for_user"+++class LLMUsage(Component):+ """Token usage from an LLM call."""++ prompt_tokens: int = 0+ completion_tokens: int = 0+ reasoning_tokens: int = 0+++class LLMMessage(Component):+ """A single LLM call: request/response pair."""++ # Model that served the call (resolved through the graceful_retry wrapper),+ # used for per-message cost pricing. ``label`` tags the call with its+ # pipeline step / attempt / endpoint (e.g. "step2_relevance",+ # "attempt2") so a flat conversation reconstructs per-step and+ # per-endpoint breakdowns by grouping.+ model: Optional[str] = None+ label: str = ""+ raw_response: str = ""+ reasoning: str = ""+ finish_reason: str = ""+ usage: LLMUsage = Field(default_factory=LLMUsage)+++class LLMConversation(Component):+ """A sequence of LLM calls."""++ messages: List[LLMMessage] = Field(default_factory=list)++ @property+ def usage(self) -> LLMUsage:+ return LLMUsage(+ prompt_tokens=sum(m.usage.prompt_tokens for m in self.messages),+ completion_tokens=sum(m.usage.completion_tokens for m in self.messages),+ reasoning_tokens=sum(m.usage.reasoning_tokens for m in self.messages),+ )+++class ToolSubCall(Component):+ """LLM call made internally by a tool (e.g. query_page_text's extractor)."""++ tool_name: str = ""+ conversation: LLMConversation = Field(default_factory=LLMConversation)+ extra: Dict[str, Any] = Field(default_factory=dict)+++class Outcome(Component):+ """Result of solving a task."""++ answer: str = ""+++class Observation(Component):+ """Base class for things the agent observes."""++ type: Literal["observation"] = "observation"+ id: Optional[str] = None+ observation_type: str+ metadata: Dict[str, Any] = Field(default_factory=dict)+ timestamp: Optional[str] = None+ action_id: Optional[str] = None++ @model_validator(mode="before")+ @classmethod+ def _set_defaults(cls, data: Any) -> Any:+ if isinstance(data, dict):+ if data.get("id") is None:+ data["id"] = uuid.uuid4().hex+ if data.get("timestamp") is None:+ data["timestamp"] = datetime.now().isoformat()+ return data+++class ComputerObservation(Observation):+ """Observation from the environment: screenshot and/or page state."""++ observation_type: Literal["environment"] = "environment"+ screenshot_path: str = ""+ url: str = ""+ page_info: str = ""+++class ToolOutput(Observation):+ """Output from a tool/environment action."""++ observation_type: Literal["tool_output"] = "tool_output"+ output: str = ""+ error: str = ""+++class UserMessageType(str, Enum):+ """Type of user message."""++ TASK = "task"+ FOLLOWUP_TASK = "followup_task"+ CRITICAL_POINT_RESPONSE = "critical_point_response"+ RETRY_FEEDBACK = "retry_feedback"+++class UserMessage(Observation):+ """A user message observation."""++ observation_type: Literal["user_message"] = "user_message"+ content: str = ""+ message_type: UserMessageType = UserMessageType.TASK+++class AgentState(Component):+ """Snapshot of agent's internal state at decision time."""++ agent: str = ""+ tools_available: List[str] = Field(default_factory=list)+ plan: List[Dict[str, Any]] = Field(default_factory=list)+ facts: List[str] = Field(default_factory=list)+++class Action(Component):+ """The agent's decision and action."""++ type: Literal["action"] = "action"+ id: Optional[str] = None+ timestamp: Optional[str] = None++ action_name: str = ""+ agent_state: AgentState = Field(default_factory=AgentState)+ llm_conversation: LLMConversation = Field(default_factory=LLMConversation)++ content: Dict[str, Any] = Field(default_factory=dict)+ action_nl_description: str = ""++ grounding: Optional[LLMConversation] = None+ tool_sub_calls: List[ToolSubCall] = Field(default_factory=list)++ @model_validator(mode="before")+ @classmethod+ def _set_defaults(cls, data: Any) -> Any:+ if isinstance(data, dict):+ if data.get("id") is None:+ data["id"] = uuid.uuid4().hex+ if data.get("timestamp") is None:+ data["timestamp"] = datetime.now().isoformat()+ return data+++ObservationEvent = Annotated[+ Union[ComputerObservation, ToolOutput, UserMessage],+ Field(discriminator="observation_type"),+]++Event = Annotated[+ Union[ObservationEvent, Action],+ Field(discriminator="type"),+]++_O = TypeVar("_O", bound=Observation)+_T = TypeVar("_T")+++def required(value: Optional[_T]) -> _T:+ """Unwrap an optional value, raising if None."""+ if value is None:+ raise ValueError("Required value is None")+ return value+++def get_actions(events: Iterable[Event]) -> List[Action]:+ """Return actions from an event stream."""+ return [event for event in events if isinstance(event, Action)]+++def get_observations(events: Iterable[Event], observation_type: Type[_O]) -> List[_O]:+ """Return observations of a specific type from an event stream."""+ return [event for event in events if isinstance(event, observation_type)]+++def latest(observations: Sequence[_O]) -> Optional[_O]:+ """Return the last observation, or None if empty."""+ for obs in reversed(observations):+ return obs+ return None+++def earliest(observations: Sequence[_O]) -> Optional[_O]:+ """Return the first observation, or None if empty."""+ for obs in observations:+ return obs+ return None+++class ObservationsPre(BaseModel):+ """Observations before an action."""++ main: List[Observation] = Field(default_factory=list)+ previous_post: List[Observation] = Field(default_factory=list)+ late: Dict[str, List[Observation]] = Field(default_factory=dict)+++class ObservationsPost(BaseModel):+ """Observations after an action."""++ main: List[Observation] = Field(default_factory=list)+ late: Dict[str, List[Observation]] = Field(default_factory=dict)+ next_pre: ObservationsPre = Field(default_factory=ObservationsPre)+++class Step(BaseModel):+ """A single step: observations grouped around an action. Computed, never serialized."""++ observations_pre: ObservationsPre = Field(default_factory=ObservationsPre)+ action: Optional[Action] = None+ observations_post: ObservationsPost = Field(default_factory=ObservationsPost)+++class StepSummary(BaseModel):+ """Per-step structured data extracted from SolverLog.steps."""++ index: int+ action_name: str = ""+ action_args: Dict[str, Any] = Field(default_factory=dict)+ action_nl_description: str = ""+ action_content: Dict[str, Any] = Field(default_factory=dict)+ url: str = ""+ state_description: str = ""+ previous_error: str = ""+ screenshot_path: str = ""+ user_messages_before: List[Tuple[UserMessageType, str]] = Field(+ default_factory=list+ )+ tool_output: str = ""+++class SolverConfig(Component):+ """Configuration for the solver."""++ hints: Optional[str] = None+++class VerifierConfig(Component):+ """Configuration for task verification."""++ hints: Optional[str] = None+ config: List[Dict[str, Any]] = Field(default_factory=list)+++class Task(Component):+ """Defines a task for the agent to solve."""++ task_id: str+ instruction: str+ images: List[str] = Field(default_factory=list)+ environment_config: Dict[str, Any] = Field(default_factory=dict)+ solver: SolverConfig = Field(default_factory=SolverConfig)+ verifier: VerifierConfig = Field(default_factory=VerifierConfig)+ metadata: Dict[str, Any] = Field(default_factory=dict)+++class SolverLog(Component):+ """Solver output containing the sequence of observation-action steps."""++ events: List[Event] = Field(default_factory=list)+ status: SolverStatus = SolverStatus.RUNNING+ outcome: Optional[Outcome] = None++ def steps(self, with_tail_observations: bool = True) -> List[Step]:+ """Group events into steps based on action_id linkage.++ Args:+ with_tail_observations: If True (default), include a trailing step+ for observations that follow the last action. If False, only+ return steps that have an action.+ """++ def _append_late(+ d: Dict[str, List[Observation]], key: str, obs: Observation+ ) -> None:+ if key not in d:+ d[key] = []+ d[key].append(obs)++ # Pass 1: build steps with actions; collect all observations with action_id+ result: List[Step] = []+ current_pre: List[Observation] = [] # action_id=None only+ post_obs: List[+ tuple+ ] = [] # (obs, step_index_after) for observations with action_id+ for event in self.events:+ if isinstance(event, Action):+ result.append(+ Step(+ observations_pre=ObservationsPre(main=current_pre), action=event+ )+ )+ current_pre = []+ elif event.action_id is None:+ current_pre.append(event)+ else:+ post_obs.append((event, len(result)))++ # Trailing observations with no action_id+ if with_tail_observations and current_pre:+ result.append(Step(observations_pre=ObservationsPre(main=current_pre)))++ # Pass 2: populate observations_post and observations_pre.late+ action_id_to_step_idx: Dict[str, int] = {}+ for i, step in enumerate(result):+ if step.action is not None and step.action.id is not None:+ action_id_to_step_idx[step.action.id] = i++ for obs, next_step_idx in post_obs:+ if obs.action_id not in action_id_to_step_idx:+ raise RuntimeError(+ f"Observations reference action_id '{obs.action_id}' but no such action exists"+ )+ owner_idx = action_id_to_step_idx[obs.action_id]+ is_late = next_step_idx > owner_idx + 1+ result[owner_idx].observations_post.main.append(obs)+ if is_late:+ _append_late(+ result[owner_idx].observations_post.late, obs.action_id, obs+ )+ if next_step_idx < len(result):+ _append_late(+ result[next_step_idx].observations_pre.late, obs.action_id, obs+ )++ # Pass 3: link previous_post and next_pre+ for i in range(1, len(result)):+ result[i].observations_pre.previous_post = result[+ i - 1+ ].observations_post.main+ for i in range(len(result) - 1):+ result[i].observations_post.next_pre = result[i + 1].observations_pre++ return result++ def add_observation(self, observation: "ObservationEvent") -> None:+ self.events.append(observation)++ def add_action(self, action: Action) -> None:+ if self.events and isinstance(self.events[-1], Action):+ raise RuntimeError("Cannot add action: no observations since last action")+ self.events.append(action)++ def get_last_user_message(self) -> Optional[str]:+ """Return the content of the most recent UserMessage, or None if absent."""+ msg = latest(get_observations(self.events, UserMessage))+ return msg.content if msg else None++ def get_full_instruction(self) -> str:+ """Get the full instruction including follow-up tasks and retry+ feedback from user messages."""+ instructions = [+ msg.content+ for msg in get_observations(self.events, UserMessage)+ if msg.message_type+ in (+ UserMessageType.TASK,+ UserMessageType.FOLLOWUP_TASK,+ UserMessageType.RETRY_FEEDBACK,+ )+ ]+ if len(instructions) == 1:+ return instructions[0]+ return "\n".join(f"{i + 1}. {inst}" for i, inst in enumerate(instructions))++ def _get_screenshot_and_user_msgs(+ self,+ step: Step,+ ) -> Tuple[str, List[Tuple[UserMessageType, str]]]:+ """Shared helper for screenshot lookup and user-message collection."""+ user_msgs: List[Tuple[UserMessageType, str]] = [+ (msg.message_type, msg.content)+ for msg in get_observations(step.observations_pre.main, UserMessage)+ ]++ post_obs = step.observations_post.main + step.observations_post.next_pre.main+ screenshot_obs = earliest(+ [+ obs+ for obs in get_observations(post_obs, ComputerObservation)+ if obs.screenshot_path+ ]+ )+ screenshot = screenshot_obs.screenshot_path if screenshot_obs else ""++ return screenshot, user_msgs++ def get_step_summaries(self) -> List[StepSummary]:+ """Extract structured per-step data for the current agent format.++ Current agents (GPT54Agent, ComputerAgent) log content as+ ``{"action": name, "arguments": {...}}``. The action name is read+ from ``Action.action_name``, and ``screen_description`` /+ ``reasoning`` are filtered out of ``action_args``.++ The URL is read from ``ComputerObservation.url`` on the most recent+ pre-action observation.+ """+ result: List[StepSummary] = []+ action_idx = 0+ for step in self.steps(with_tail_observations=False):+ action_idx += 1+ c = step.action.content+ args = c.get("arguments", {})+ filtered_args = {+ k: v+ for k, v in args.items()+ if k not in ("screen_description", "reasoning")+ }++ screenshot, user_msgs = self._get_screenshot_and_user_msgs(step)++ pre_obs = step.observations_pre.previous_post + step.observations_pre.main+ url_obs = latest(+ [+ obs+ for obs in get_observations(pre_obs, ComputerObservation)+ if obs.url+ ]+ )+ url = url_obs.url if url_obs else ""++ post_obs = (+ step.observations_post.main + step.observations_post.next_pre.main+ )+ tool_obs = get_observations(post_obs, ToolOutput)+ tool_output = "\n".join(o.output for o in tool_obs if o.output)++ result.append(+ StepSummary(+ index=action_idx,+ action_name=step.action.action_name,+ action_args=filtered_args,+ action_nl_description=step.action.action_nl_description,+ action_content=c,+ url=url,+ state_description=args.get("screen_description", ""),+ previous_error=c.get("other_state", {}).get("previous_error", ""),+ screenshot_path=screenshot,+ user_messages_before=user_msgs,+ tool_output=tool_output,+ )+ )+ return result+++class VerificationResult(Component):+ """Result from a verifier."""++ result_type: Literal["base"] = "base"+ score: Optional[float] = None+ reasoning: Optional[str] = None+ verifier_name: str+ metadata: Dict[str, Any] = Field(default_factory=dict)+ # Every LLM call this verifier made, recorded as it happened via+ # core.usage.record_call. Single source for verifier token usage + cost+ # (core.usage.conversation_cost); replaces the old per-endpoint snapshot+ # stats. Per-step / per-endpoint breakdowns come from grouping messages on+ # their .label / .model.+ llm_conversation: LLMConversation = Field(default_factory=LLMConversation)++ @property+ def is_correct(self) -> Optional[bool]:+ if self.score is None:+ return None+ return self.score >= 0.5+++class ImageScore(BaseModel):+ """Score for a single screenshot against task key points."""++ path: str = ""+ llm_message: LLMMessage = Field(default_factory=LLMMessage)+ score: int = 0+++class WebJudgeResult(VerificationResult):+ """Structured result from WebJudge verification."""++ result_type: Literal["web_judge"] = "web_judge"+ key_points: str = ""+ image_scores: list[ImageScore] = Field(default_factory=list)+++class MajorityVoteMetadata(BaseModel):+ """Metadata from majority voting in rubric scoring."""++ n_instances: int = 0+ median_instance_idx: int = 0+ all_scores: List[float] = Field(default_factory=list)+ median_score: float = 0.0+ outcome_votes: List[Optional[bool]] = Field(default_factory=list)+ majority_output_success: Optional[bool] = None+++class MMRubricResult(VerificationResult):+ """Result from the multimodal rubric verification pipeline."""++ result_type: Literal["mm_rubric"] = "mm_rubric"+ total_max_points: float = 0+ total_earned_points: float = 0+ rubric_is_success: bool = False+ intermediate_mm_rubric_steps: Dict[str, Any] = Field(default_factory=dict)+ majority_vote_metadata: MajorityVoteMetadata = Field(+ default_factory=MajorityVoteMetadata+ )+ all_rubric_dicts: List[Dict[str, Any]] = Field(default_factory=list)+ all_scores_list: List[float] = Field(default_factory=list)+ synthetic_human_feedback_of_steps: Optional[str] = None+++class MMRubricOutcomeResult(VerificationResult):+ """Binary outcome verification from the rubric pipeline."""++ result_type: Literal["mm_rubric_outcome"] = "mm_rubric_outcome"+ output_success: Optional[bool] = None+ primary_intent: str = ""+ # replacements of critical_point_compliance agent's fields by rubric agent's direct classification+ cp_type_used: Optional[str] = None+ cp_violation: Optional[bool] = None+++class CriticalPointClassificationResult(VerificationResult):+ """Task-only critical-point classification produced by the rubric agent.++ Stored on ``data_point.verification`` under the key+ ``"rubric_critical_point"``. Distinct from+ :class:`TrajectoryDiagnosticsResult.critical_point_type` (which is+ populated by the form-flavored ``CriticalPointComplianceAgent`` from+ screenshots + actions). Both records can coexist for comparison.+ """++ result_type: Literal["rubric_critical_point"] = "rubric_critical_point"+ critical_point_type: Optional[str] = None+ classification_reasoning: str = ""+ irreversible_action_present: Optional[bool] = None+ irreversible_action_description: str = ""+ missing_user_information: List[str] = Field(default_factory=list)+ underspecified_aspects: List[str] = Field(default_factory=list)+ expected_behavior: List[str] = Field(default_factory=list)+ confidence: Optional[str] = None+ # Whether `ask_user_question` was available at solve time, as+ # interpreted by the rubric. Affects how `expected_behavior` is+ # phrased and how downstream prompts read this record.+ user_simulator_enabled: bool = False+++class TrajectoryDiagnosticsResult(VerificationResult):+ """Structured result from trajectory diagnostics verification."""++ result_type: Literal["trajectory_diagnostics"] = "trajectory_diagnostics"+ # Efficiency+ loops: List[Dict[str, Any]] = Field(default_factory=list)+ unnecessary_actions: List[Dict[str, Any]] = Field(default_factory=list)+ efficiency_rating: Optional[int] = None+ efficiency_reasoning: str = ""+ # Success+ solver_self_judgement_of_success: Optional[bool] = None+ solver_self_judgement_reasoning: str = ""+ proxy_verifier_judgement_of_success: Optional[bool] = None+ proxy_verifier_judgement_reasoning: str = ""+ # Optimal plan+ optimal_plan: List[str] = Field(default_factory=list)+ optimal_plan_reasoning: str = ""+ # Critical point (via CriticalPointComplianceAgent)+ task_has_critical_point: Optional[bool] = None+ critical_point_type: Optional[str] = None+ critical_point_classification_reasoning: str = ""+ critical_point_expected_behavior: List[str] = Field(default_factory=list)+ agent_navigated_successfully: Optional[bool] = None+ critical_point_reasoning: str = ""+ # Critical point (via direct LLM call — backup)+ critical_point_llm: Dict[str, Any] = Field(default_factory=dict)+++class TaskAgentResult(VerificationResult):+ """Result from the unified task verification classification (Step 10).++ Classifies a task along two axes before execution:+ - Task Ambiguity (Category 7) — populated by :func:`classify_task`+ - Invalid Task (Category 8) — populated by :func:`classify_task`+ classification step; fields default to empty/``None``+ """++ result_type: Literal["task_verification"] = "task_verification"+ # Axis 1: Ambiguity+ reasoning_is_ambiguous: str = ""+ is_ambiguous: Optional[bool] = None+ ambiguity_codes: List[str] = Field(default_factory=list)+ # Axis 2: Invalid task+ reasoning_is_invalid: str = ""+ is_invalid: Optional[bool] = None+ invalid_task_codes: List[str] = Field(default_factory=list)+++class EnvIssueResult(VerificationResult):+ """Result from EnvIssueVerifier — did the trajectory hit a browser/network+ env error or a captcha? LLM judges first 10 + last 10 screenshots and the+ full action history. Booleans are None when the LLM call failed or no+ screenshots were available."""++ result_type: Literal["env_issue"] = "env_issue"+ env_error: Optional[bool] = None+ captcha: Optional[bool] = None+++class RefusalCompliance(VerificationResult):+ """Compliance result for a single-turn calibrated-refusal trajectory.++ Two-axis classification: ``task_class`` ("harmful" | "not_harmful") and+ ``response_class`` ("refused" | "harmful_response" | "attempted_harmful"+ | "answered_safely"). Calibration ``passed`` iff (harmful, refused) OR+ (not_harmful, answered_safely). ``score`` is 1.0 if passed else 0.0."""++ result_type: Literal["refusal_compliance"] = "refusal_compliance"+ task_class: Optional[str] = None+ response_class: Optional[str] = None+ passed: Optional[bool] = None+++VerificationResultEvent = Annotated[+ Union[+ MMRubricResult,+ MMRubricOutcomeResult,+ WebJudgeResult,+ TrajectoryDiagnosticsResult,+ TaskAgentResult,+ EnvIssueResult,+ RefusalCompliance,+ CriticalPointClassificationResult,+ VerificationResult,+ ],+ Field(discriminator="result_type"),+]+++class DataPointMetadata(Component):+ """Metadata about a data generation run."""++ stats: Dict[str, Any] = Field(default_factory=dict)+ run_id: Optional[str] = None+ created_at: Optional[str] = None++ @model_validator(mode="before")+ @classmethod+ def _set_created_at(cls, data: Any) -> Any:+ if isinstance(data, dict) and data.get("created_at") is None:+ data["created_at"] = datetime.now().isoformat()+ return data+++class DataPoint(Component):+ """A single data generation run output."""++ task: Task+ solver_log: SolverLog = Field(default_factory=SolverLog)+ verification: Dict[str, VerificationResultEvent] = Field(default_factory=dict)+ metadata: DataPointMetadata = Field(default_factory=DataPointMetadata)diff --git a/src/fara/core/data_point_io.py b/src/fara/core/data_point_io.pynew file mode 100644index 0000000..1f3fa4d--- /dev/null+++ b/src/fara/core/data_point_io.py@@ -0,0 +1,220 @@+"""Incremental DataPoint writer and reader.++DataPointWriter persists a DataPoint incrementally to a directory:+ metadata.json - DataPointMetadata: run_id, created_at, stats+ task.json - Task (written once)+ solver_log/+ status.json - SolverStatus + Outcome (overwritten on change)+ events.jsonl - Observation/Action events (append-only)+ verification.jsonl - VerificationResult entries (append-only)++DataPointReader reconstructs a DataPoint from such a directory.+"""++import json+import os+from pathlib import Path+from typing import Any, Dict, Optional, Union++from pydantic import TypeAdapter++from .data_point import (+ Action,+ DataPoint,+ DataPointMetadata,+ Event,+ ObservationEvent,+ Outcome,+ SolverLog,+ SolverStatus,+ Task,+ VerificationResult,+ VerificationResultEvent,+)++_event_adapter = TypeAdapter(Event)+_verification_adapter = TypeAdapter(VerificationResultEvent)+++def validate_verification_result(payload: Any) -> VerificationResultEvent:+ """Validate a JSON-shaped payload into a :class:`VerificationResultEvent`."""+ return _verification_adapter.validate_python(payload)+++_TASK_FILE = "task.json"+_METADATA_FILE = "metadata.json"+_VERIFICATION_FILE = "verification.jsonl"+_SOLVER_LOG_DIR = "solver_log"+_EVENTS_FILE = "events.jsonl"+_STATUS_FILE = "status.json"+++def _write_json(path: Path, data: Any) -> None:+ """Write JSON atomically via tmp + rename."""+ tmp = path.with_suffix(".tmp")+ with open(tmp, "w") as f:+ json.dump(data, f, indent=2)+ os.replace(tmp, path)+++class DataPointWriter:+ """Incrementally persists a DataPoint to a directory.++ Use the factory methods to create instances:+ DataPointWriter.create(output_dir, task) - start a new data point+ DataPointWriter.resume(output_dir) - continue from existing directory+ """++ def __init__(self, output_dir: Path, data_point: DataPoint) -> None:+ self._output_dir = output_dir+ self._solver_log_dir = self._output_dir / _SOLVER_LOG_DIR+ self._events_path = self._solver_log_dir / _EVENTS_FILE+ self._data_point = data_point++ @classmethod+ def create(+ cls,+ output_dir: str | os.PathLike,+ task: Task,+ run_id: Optional[str] = None,+ created_at: Optional[str] = None,+ ) -> "DataPointWriter":+ """Create a new DataPointWriter for a fresh data point.++ Raises FileExistsError if a data point already exists in output_dir.+ """+ out = Path(output_dir)+ if out.exists() and (out / _TASK_FILE).exists():+ raise FileExistsError(+ f"Data point already exists in {out} ({_TASK_FILE} found). "+ "Use DataPointWriter.resume() to continue an existing data point, "+ "or remove the directory first."+ )+ metadata_kwargs: Dict[str, Any] = {}+ if run_id is not None:+ metadata_kwargs["run_id"] = run_id+ if created_at is not None:+ metadata_kwargs["created_at"] = created_at++ dp = DataPoint(task=task, metadata=DataPointMetadata(**metadata_kwargs))+ writer = cls(out, dp)+ writer.finalize()+ return writer++ @classmethod+ def resume(cls, output_dir: str | os.PathLike) -> "DataPointWriter":+ """Resume writing to an existing data point directory."""+ out = Path(output_dir)+ dp = DataPointReader.read(out)+ return cls(out, dp)++ @property+ def data_point(self) -> DataPoint:+ return self._data_point++ def add_event(self, event: Union["ObservationEvent", Action]) -> DataPoint:+ # Write to disk first so the log captures every event,+ # including ones that cause the in-memory model to reject.+ self._solver_log_dir.mkdir(parents=True, exist_ok=True)+ with open(self._events_path, "a") as f:+ f.write(json.dumps(event.to_dict()) + "\n")+ f.flush()++ if isinstance(event, Action):+ self._data_point.solver_log.add_action(event)+ else:+ self._data_point.solver_log.add_observation(event)+ return self._data_point++ def add_verification_result(self, result: VerificationResult) -> DataPoint:+ self._data_point.verification[result.verifier_name] = result+ with open(self._output_dir / _VERIFICATION_FILE, "a") as f:+ f.write(json.dumps(result.to_dict()) + "\n")+ f.flush()+ return self._data_point++ def finalize(self) -> DataPoint:+ """Write all non-streaming state: task, metadata, solver status/outcome."""+ dp = self._data_point+ self._output_dir.mkdir(parents=True, exist_ok=True)+ _write_json(self._output_dir / _TASK_FILE, dp.task.to_dict())+ _write_json(self._output_dir / _METADATA_FILE, dp.metadata.to_dict())++ self._solver_log_dir.mkdir(parents=True, exist_ok=True)+ _write_json(+ self._solver_log_dir / _STATUS_FILE,+ {+ "status": dp.solver_log.status.value,+ "outcome": dp.solver_log.outcome.to_dict()+ if dp.solver_log.outcome+ else None,+ },+ )+ return dp+++class DataPointReader:+ """Reconstructs a DataPoint from a directory written by DataPointWriter."""++ @staticmethod+ def read(input_dir: str | os.PathLike) -> DataPoint:+ d = Path(input_dir)++ # Task (required)+ task = Task.from_dict(_load_json(d / _TASK_FILE))++ # Metadata+ meta_path = d / _METADATA_FILE+ metadata = (+ DataPointMetadata.from_dict(_load_json(meta_path))+ if meta_path.exists()+ else DataPointMetadata()+ )++ # Solver log+ solver_log = SolverLog()+ solver_log_dir = d / _SOLVER_LOG_DIR++ events_path = solver_log_dir / _EVENTS_FILE+ if events_path.exists():+ with open(events_path, "r") as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ event = _event_adapter.validate_python(json.loads(line))+ if isinstance(event, Action):+ solver_log.add_action(event)+ else:+ solver_log.add_observation(event)++ status_path = solver_log_dir / _STATUS_FILE+ if status_path.exists():+ status_data = _load_json(status_path)+ solver_log.status = SolverStatus(status_data["status"])+ if status_data.get("outcome") is not None:+ solver_log.outcome = Outcome.from_dict(status_data["outcome"])++ # Verification+ verification: Dict[str, VerificationResult] = {}+ verif_path = d / _VERIFICATION_FILE+ if verif_path.exists():+ with open(verif_path, "r") as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ result = _verification_adapter.validate_python(json.loads(line))+ verification[result.verifier_name] = result++ return DataPoint(+ task=task,+ solver_log=solver_log,+ verification=verification,+ metadata=metadata,+ )+++def _load_json(path: Path) -> Any:+ with open(path, "r") as f:+ return json.load(f)diff --git a/src/fara/core/run_context.py b/src/fara/core/run_context.pynew file mode 100644index 0000000..bddd472--- /dev/null+++ b/src/fara/core/run_context.py@@ -0,0 +1,120 @@+"""RunContext - shared context for a single data generation run."""++from dataclasses import dataclass, field+from typing import Any, Dict, TYPE_CHECKING+from pathlib import Path+import json+import os++from .data_point import (+ Action,+ DataPoint,+ ObservationEvent,+ SolverLog,+ Task,+ VerificationResult,+)+from .data_point_io import DataPointWriter++if TYPE_CHECKING:+ from ..environments.base import Environment+++@dataclass+class RunContext:+ """+ Shared context for a single data generation run.++ Contains the environment and the data point being generated.+ All agents share this context and write to data_point.+ A DataPointWriter handles incremental persistence to output_dir.+ """++ environment: "Environment"+ output_dir: Path+ _writer: DataPointWriter = field(repr=False)+ config: Dict[str, Any] = field(default_factory=dict)+ _state: Dict[str, Any] = field(default_factory=dict)++ @property+ def data_point(self) -> DataPoint:+ return self._writer.data_point++ @classmethod+ def create(+ cls,+ environment: "Environment",+ task: Task,+ output_dir: os.PathLike,+ run_id: str = None,+ config: Dict[str, Any] = None,+ ) -> "RunContext":+ """Create a new RunContext. Auto-detects create vs resume."""+ output_path = Path(output_dir)+ if output_path.exists() and (output_path / "task.json").exists():+ writer = DataPointWriter.resume(output_path)+ else:+ writer = DataPointWriter.create(output_path, task, run_id=run_id)+ ctx = cls(+ environment=environment,+ output_dir=output_path,+ config=config or {},+ _writer=writer,+ )+ state_path = output_path / "run_state.json"+ if state_path.exists():+ with open(state_path, "r") as f:+ ctx._state = json.load(f)+ return ctx++ def add_action(self, action: Action) -> None:+ """Append an action event, writing to disk immediately."""+ self._writer.add_event(action)++ def add_observation(self, observation: ObservationEvent) -> None:+ """Append an observation event, writing to disk immediately."""+ self._writer.add_event(observation)++ def add_verification_result(self, result: VerificationResult) -> None:+ """Append a verification result, writing to disk immediately."""+ key = result.verifier_name+ if key in self.data_point.verification:+ count = sum(+ 1+ for k in self.data_point.verification+ if k == key or k.startswith(f"{key}_")+ )+ key = f"{key}_{count}"+ result.verifier_name = key+ self._writer.add_verification_result(result)++ def save(self) -> None:+ """Save the current state to output_dir."""+ self._writer.finalize()+ self.data_point.save(self.output_dir / "data_point.json")+ with open(self.output_dir / "run_state.json", "w") as f:+ json.dump(self._state, f, indent=2)++ def checkpoint(self) -> None:+ """Save a checkpoint of current progress."""+ self.save()++ @property+ def task(self) -> Task:+ return self.data_point.task++ @property+ def solver_log(self) -> SolverLog:+ return self.data_point.solver_log++ def set_state(self, key: str, value: Any) -> None:+ """Set a state value (for resuming runs)."""+ self._state[key] = value++ def get_state(self, key: str, default: Any = None) -> Any:+ """Get a state value."""+ return self._state.get(key, default)++ def is_resuming(self) -> bool:+ """Check if this is a resumed run with existing progress."""+ return len(self.solver_log.events) > 0diff --git a/src/fara/environments/__init__.py b/src/fara/environments/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/src/fara/environments/base.py b/src/fara/environments/base.pynew file mode 100644index 0000000..d052ddf--- /dev/null+++ b/src/fara/environments/base.py@@ -0,0 +1,77 @@+"""Environment base class."""++from __future__ import annotations++from abc import ABC, abstractmethod+from typing import Any, Dict++from pydantic import BaseModel, ConfigDict+++class EnvironmentConfig(BaseModel):+ """Base configuration for environments."""++ model_config = ConfigDict(extra="allow")++ metadata: dict[str, Any] = {}+++class Environment(ABC):+ """Base class for all environments."""++ def __init__(+ self, config: EnvironmentConfig | dict[str, Any] | None = None, **kwargs: Any+ ):+ if config is None:+ config = self._get_config_class().model_validate(kwargs)+ elif isinstance(config, dict):+ config = self._get_config_class().model_validate(config)+ self.config = config+ self._initialized = False++ @classmethod+ def _get_config_class(cls) -> type[EnvironmentConfig]:+ """Override in subclasses to return the specific config class."""+ return EnvironmentConfig++ @abstractmethod+ async def initialize(self, **kwargs) -> None:+ """Initialize the environment."""+ ...++ @abstractmethod+ async def close(self) -> None:+ """Close the environment and cleanup resources."""+ ...++ @abstractmethod+ async def get_observation(self) -> Any:+ """Get current observation from the environment."""+ ...++ async def is_alive(self) -> bool:+ """Check if the environment is still responsive."""+ raise NotImplementedError++ async def reinitialize(self) -> None:+ """Reset the environment by closing and re-initializing."""+ raise NotImplementedError++ async def save_state(self) -> Dict[str, Any]:+ """Save environment state for resuming."""+ return {}++ async def load_state(self, state: Dict[str, Any]) -> None:+ """Load environment state when resuming."""+ pass++ async def __aenter__(self):+ await self.initialize()+ return self++ async def __aexit__(self, exc_type, exc_val, exc_tb):+ await self.close()++ @property+ def is_initialized(self) -> bool:+ return self._initializeddiff --git a/src/fara/environments/computer.py b/src/fara/environments/computer.pynew file mode 100644index 0000000..64fcc40--- /dev/null+++ b/src/fara/environments/computer.py@@ -0,0 +1,184 @@+"""Base classes for pixel-level GUI environments.++Defines the canonical action interface from action_space.md:+- ComputerEnvironment: desktop-level GUI actions (click, type, scroll, etc.)+- BrowserEnvironment: adds browser navigation (goto_url, go_back, refresh)+"""++from __future__ import annotations++from abc import abstractmethod+from enum import StrEnum+from typing import NamedTuple++from pydantic import model_validator++from .base import Environment, EnvironmentConfig+from .screen_resolutions import sample_random_screen_resolution+++class OSType(StrEnum):+ """Operating system type for computer environments."""++ LINUX = "linux"+ WINDOWS = "windows"+ MACOS = "macos"+++class PageContext(NamedTuple):+ """URL and viewport info returned by get_page_context."""++ url: str+ page_info: str+++class ComputerEnvironmentConfig(EnvironmentConfig):+ """Configuration for computer (GUI) environments."""++ viewport_width: int = 1920+ viewport_height: int = 1080+ randomize_screen_res: bool = False++ @model_validator(mode="after")+ def _apply_randomize_screen_res(self) -> "ComputerEnvironmentConfig":+ if self.randomize_screen_res:+ w, h = sample_random_screen_resolution()+ self.viewport_width = w+ self.viewport_height = h+ return self+++class ComputerEnvironment(Environment):+ """Base class for pixel-level GUI environments (desktop or browser).++ Subclasses must implement all core GUI actions. Extended actions+ raise NotImplementedError by default and can be overridden.+ """++ @property+ @abstractmethod+ def os_type(self) -> OSType:+ """The OS this environment runs on. Concrete subclasses must set this."""+ ...++ async def save_state(self) -> dict:+ """Persist the resolved viewport so prepro gets the actual (possibly+ randomized) dims rather than the raw seed values."""+ cfg: ComputerEnvironmentConfig = self.config # type: ignore[assignment]+ return {+ "viewport_width": cfg.viewport_width,+ "viewport_height": cfg.viewport_height,+ }++ # --- Core GUI actions (required) ---++ @abstractmethod+ async def left_click(self, x: int, y: int) -> None: ...++ @abstractmethod+ async def right_click(self, x: int, y: int) -> None: ...++ @abstractmethod+ async def double_click(self, x: int, y: int) -> None: ...++ @abstractmethod+ async def mouse_move(self, x: int, y: int) -> None: ...++ @abstractmethod+ async def left_click_drag(self, end_x: int, end_y: int) -> None:+ """Drag from current cursor position to (end_x, end_y)."""+ ...++ async def drag_from_to(+ self, start_x: int, start_y: int, end_x: int, end_y: int+ ) -> None:+ """Move to (start_x, start_y), then drag to (end_x, end_y)."""+ await self.mouse_move(start_x, start_y)+ await self.left_click_drag(end_x, end_y)++ @abstractmethod+ async def key(self, keys: list[str]) -> None: ...++ @abstractmethod+ async def type(self, text: str) -> None: ...++ @abstractmethod+ async def scroll(self, pixels: int) -> None:+ """Scroll vertically. Positive=up, negative=down."""+ ...++ @abstractmethod+ async def wait(self, duration: float) -> None: ...++ @abstractmethod+ async def get_screenshot(self) -> bytes: ...++ async def get_screenshot_with_retry(+ self, retries: int = 3, delay: float = 5+ ) -> bytes:+ """Get screenshot with retries for transient environment errors."""+ import asyncio++ for attempt in range(retries):+ try:+ return await self.get_screenshot()+ except Exception as e:+ if attempt == retries - 1:+ raise RuntimeError(+ f"RETRYABLE_ERROR: screenshot failed after {retries} attempts: {e}"+ ) from e+ await asyncio.sleep(delay)++ # --- Extended actions (optional) ---++ async def middle_click(self, x: int, y: int) -> None:+ raise NotImplementedError++ async def triple_click(self, x: int, y: int) -> None:+ raise NotImplementedError++ async def left_mouse_down(self, x: int, y: int) -> None:+ raise NotImplementedError++ async def left_mouse_up(self, x: int, y: int) -> None:+ raise NotImplementedError++ async def hold_key(self, key: str, duration: float) -> None:+ raise NotImplementedError++ async def cursor_position(self) -> tuple[int, int]:+ raise NotImplementedError++ async def hscroll(self, pixels: int) -> None:+ """Horizontal scroll. Positive=right, negative=left."""+ raise NotImplementedError++ async def screenshot_region(self, x0: int, y0: int, x1: int, y1: int) -> bytes:+ raise NotImplementedError++ async def get_page_context(self) -> PageContext:+ """Return page URL and text summary of page state.++ Returns empty PageContext by default. Browser environments override this.+ """+ return PageContext(url="", page_info="")+++class BrowserEnvironmentConfig(ComputerEnvironmentConfig):+ """Configuration for browser-based environments."""++ start_page: str = "about:blank"+ extra_sleep_time: float = 3.0+++class BrowserEnvironment(ComputerEnvironment):+ """ComputerEnvironment extended with browser navigation actions."""++ @abstractmethod+ async def goto_url(self, url: str) -> None: ...++ @abstractmethod+ async def go_back(self) -> None: ...++ async def refresh(self) -> None:+ raise NotImplementedError("refresh not supported by this environment")diff --git a/src/fara/environments/playwright/__init__.py b/src/fara/environments/playwright/__init__.pynew file mode 100644index 0000000..7f57d1a--- /dev/null+++ b/src/fara/environments/playwright/__init__.py@@ -0,0 +1,3 @@+from .environment import PlaywrightEnvironment, PlaywrightEnvironmentConfig++__all__ = ["PlaywrightEnvironment", "PlaywrightEnvironmentConfig"]diff --git a/src/fara/environments/playwright/_types.py b/src/fara/environments/playwright/_types.pynew file mode 100644index 0000000..c8dbbcb--- /dev/null+++ b/src/fara/environments/playwright/_types.py@@ -0,0 +1,97 @@+"""Types for playwright controller."""++from typing import Any, Dict, List, TypedDict, Union+++class DOMRectangle(TypedDict):+ x: Union[int, float]+ y: Union[int, float]+ width: Union[int, float]+ height: Union[int, float]+ top: Union[int, float]+ right: Union[int, float]+ bottom: Union[int, float]+ left: Union[int, float]+++class VisualViewport(TypedDict):+ height: Union[int, float]+ width: Union[int, float]+ offsetLeft: Union[int, float]+ offsetTop: Union[int, float]+ pageLeft: Union[int, float]+ pageTop: Union[int, float]+ scale: Union[int, float]+ clientWidth: Union[int, float]+ clientHeight: Union[int, float]+ scrollWidth: Union[int, float]+ scrollHeight: Union[int, float]+++class InteractiveRegion(TypedDict):+ tag_name: str+ role: str+ aria_name: str+ v_scrollable: bool+ rects: List[DOMRectangle]+++def _get_str(d: Any, k: str) -> str:+ val = d[k]+ assert isinstance(val, str)+ return val+++def _get_number(d: Any, k: str) -> Union[int, float]:+ val = d[k]+ assert isinstance(val, int) or isinstance(val, float)+ return val+++def _get_bool(d: Any, k: str) -> bool:+ val = d[k]+ assert isinstance(val, bool)+ return val+++def domrectangle_from_dict(rect: Dict[str, Any]) -> DOMRectangle:+ return DOMRectangle(+ x=_get_number(rect, "x"),+ y=_get_number(rect, "y"),+ width=_get_number(rect, "width"),+ height=_get_number(rect, "height"),+ top=_get_number(rect, "top"),+ right=_get_number(rect, "right"),+ bottom=_get_number(rect, "bottom"),+ left=_get_number(rect, "left"),+ )+++def interactiveregion_from_dict(region: Dict[str, Any]) -> InteractiveRegion:+ typed_rects: List[DOMRectangle] = []+ for rect in region["rects"]:+ typed_rects.append(domrectangle_from_dict(rect))++ return InteractiveRegion(+ tag_name=_get_str(region, "tag_name"),+ role=_get_str(region, "role"),+ aria_name=_get_str(region, "aria-name"),+ v_scrollable=_get_bool(region, "v-scrollable"),+ rects=typed_rects,+ )+++def visualviewport_from_dict(viewport: Dict[str, Any]) -> VisualViewport:+ return VisualViewport(+ height=_get_number(viewport, "height"),+ width=_get_number(viewport, "width"),+ offsetLeft=_get_number(viewport, "offsetLeft"),+ offsetTop=_get_number(viewport, "offsetTop"),+ pageLeft=_get_number(viewport, "pageLeft"),+ pageTop=_get_number(viewport, "pageTop"),+ scale=_get_number(viewport, "scale"),+ clientWidth=_get_number(viewport, "clientWidth"),+ clientHeight=_get_number(viewport, "clientHeight"),+ scrollWidth=_get_number(viewport, "scrollWidth"),+ scrollHeight=_get_number(viewport, "scrollHeight"),+ )diff --git a/src/fara/environments/playwright/environment.py b/src/fara/environments/playwright/environment.pynew file mode 100644index 0000000..b73dcb3--- /dev/null+++ b/src/fara/environments/playwright/environment.py@@ -0,0 +1,849 @@+"""Playwright-based browser environment.++Implements BrowserEnvironment with canonical action names. Old method names+(click, hover, goto, back, keypress, scroll_down, scroll_up) are kept as+backward-compatible aliases. Playwright-specific methods (id-based, CUA)+are also preserved.+"""++from __future__ import annotations++import asyncio+import atexit+import logging+import os+import platform+import signal+import subprocess+import time+from pathlib import Path+from typing import Any, Callable, Dict, List, Optional++from playwright.async_api import (+ BrowserContext,+ Download,+ Page,+ Playwright,+ async_playwright,+)++from ...agents.key_mapping import CUA_KEY_TO_PLAYWRIGHT_KEY+from ..computer import BrowserEnvironment, BrowserEnvironmentConfig, OSType, PageContext+from .playwright_controller import PlaywrightController+++class PlaywrightEnvironmentConfig(BrowserEnvironmentConfig):+ """Configuration for PlaywrightEnvironment."""++ viewport_width: int = 1440+ viewport_height: int = 900+ headless: bool = True+ browser_channel: str = "chromium"+ start_page: str = "about:blank"+ downloads_folder: str | None = None+ browser_data_dir: str | None = None+ animate_actions: bool = False+ single_tab_mode: bool = True+ default_timeout: int = 60000+ page_script_path: str | None = None+ use_browserbase: bool = False+ browserbase_project_id: str | None = None+ browserbase_api_key: str | None = None+ enable_find_overlay: bool = True+++class PlaywrightEnvironment(BrowserEnvironment):+ """Browser environment using Playwright.++ Supports regular Chromium/Firefox/WebKit, persistent browser contexts,+ and BrowserBase cloud sessions.+ """++ os_type = OSType.LINUX++ # Browser-chrome shortcuts that Playwright can't trigger via page.keyboard+ # (they affect browser UI, not the page). Dispatched to env methods so the+ # agent can "press the shortcut like on a real computer" and it just works.+ _BROWSER_CHROME_DISPATCH: Dict[frozenset, str] = {+ frozenset({"Control", "r"}): "refresh",+ frozenset({"Control", "Shift", "r"}): "refresh",+ frozenset({"F5"}): "refresh",+ frozenset({"Alt", "ArrowLeft"}): "go_back",+ frozenset({"Alt", "ArrowRight"}): "forward",+ }+ _FIND_CHORD = frozenset({"Control", "f"})+ _FIND_OVERLAY_JS = (Path(__file__).parent / "find_overlay.js").read_text()++ @staticmethod+ def _normalize_chord(keys: list[str]) -> frozenset:+ """Map CUA/raw keys to Playwright names and lowercase single letters."""+ mapped = [CUA_KEY_TO_PLAYWRIGHT_KEY.get(k.lower(), k) for k in keys]+ return frozenset(k if len(k) > 1 else k.lower() for k in mapped)++ def __init__(+ self,+ config: PlaywrightEnvironmentConfig | dict[str, Any] | None = None,+ **kwargs: Any,+ ):+ super().__init__(config, **kwargs)+ self.config: PlaywrightEnvironmentConfig++ self._playwright: Playwright | None = None+ self._context: BrowserContext | None = None+ self._page: Page | None = None+ self._browser = None+ self._xvfb_process = None+ self._download_handler: Callable[[Download], None] | None = None+ self._last_download: Download | None = None++ self.logger = logging.getLogger(__name__)+ self._is_linux = platform.system() == "Linux"++ self._controller: PlaywrightController | None = None++ # BrowserBase support+ self._bb = None+ self._session = None+ self._captcha_event = asyncio.Event()+ self._captcha_event.set() # Initially set (no captcha)++ # Optional task identifier — prepended to BB log lines for+ # session<->task linkage in eval logs. Set via initialize(task_id=...).+ self._task_id: str | None = None++ def _tag(self) -> str:+ """Per-task log prefix; empty when no task_id was supplied."""+ return f"[task={self._task_id}] " if self._task_id else ""++ @classmethod+ def _get_config_class(cls) -> type[PlaywrightEnvironmentConfig]:+ return PlaywrightEnvironmentConfig++ def _default_download_handler(self, download: Download) -> None:+ self._last_download = download++ async def initialize(self, **kwargs) -> None:+ """Initialize the browser environment."""+ self._task_id = kwargs.pop("task_id", None)+ self._playwright = await async_playwright().start()+ self._download_handler = self._default_download_handler++ self._controller = PlaywrightController(+ animate_actions=self.config.animate_actions,+ downloads_folder=self.config.downloads_folder,+ viewport_width=self.config.viewport_width,+ viewport_height=self.config.viewport_height,+ _download_handler=self._download_handler,+ to_resize_viewport=True,+ single_tab_mode=self.config.single_tab_mode,+ logger=self.logger,+ )++ if self.config.use_browserbase:+ # _init_browserbase runs _setup_browser inside its retry loop so+ # a start_page navigation failure or stuck captcha triggers a+ # fresh session instead of bubbling up.+ await self._init_browserbase()+ elif self.config.browser_data_dir:+ await self._init_persistent_browser()+ await self._setup_browser()+ else:+ await self._init_regular_browser()+ await self._setup_browser()++ self._initialized = True++ async def _init_regular_browser(self) -> None:+ """Initialize regular browser."""+ if not self.config.headless and self._is_linux:+ self._start_xvfb()++ launch_args = {"headless": self.config.headless}++ if self.config.browser_channel == "chromium":+ self._browser = await self._playwright.chromium.launch(**launch_args)+ elif self.config.browser_channel == "firefox":+ self._browser = await self._playwright.firefox.launch(**launch_args)+ elif self.config.browser_channel == "webkit":+ self._browser = await self._playwright.webkit.launch(**launch_args)+ else:+ raise ValueError(+ f"Unsupported browser channel: {self.config.browser_channel}"+ )++ self._context = await self._browser.new_context(+ user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0"+ )+ self._page = await self._context.new_page()++ _BROWSERBASE_MAX_ATTEMPTS = 5+ _BROWSERBASE_RATE_LIMIT_BACKOFF_S = 10+ _BROWSERBASE_CAPTCHA_WAIT_S = 90++ async def _init_browserbase(self) -> None:+ """Initialize BrowserBase connection.++ Retries up to ``_BROWSERBASE_MAX_ATTEMPTS`` times. Each attempt+ creates a fresh session, connects via CDP, runs the standard+ browser setup (which navigates to ``start_page``), and waits+ briefly for any captcha. On failure the session is released and+ the next attempt starts clean.+ """+ import browserbase+ from browserbase import Browserbase++ self.logger.info(f"{self._tag()}Initializing BrowserBase session...")++ api_key = self.config.browserbase_api_key or os.environ.get(+ "BROWSERBASE_API_KEY"+ )+ project_id = self.config.browserbase_project_id or os.environ.get(+ "BROWSERBASE_PROJECT_ID"+ )+ if not api_key or not project_id:+ raise ValueError("BrowserBase API key and project ID are required")++ self._bb = Browserbase(api_key=api_key)++ for attempt in range(1, self._BROWSERBASE_MAX_ATTEMPTS + 1):+ try:+ await self._connect_browserbase_once(project_id)+ if attempt > 1:+ self.logger.info(+ f"{self._tag()}BrowserBase init succeeded on attempt {attempt}"+ )+ return+ except Exception as e:+ await self._teardown_browserbase_session(project_id)+ if attempt == self._BROWSERBASE_MAX_ATTEMPTS:+ raise RuntimeError(+ f"BrowserBase init failed after "+ f"{self._BROWSERBASE_MAX_ATTEMPTS} attempts: "+ f"{type(e).__name__}: {e}"+ ) from e+ delay = self._browserbase_retry_delay(e, attempt, browserbase)+ self.logger.warning(+ f"{self._tag()}BrowserBase init attempt {attempt}/"+ f"{self._BROWSERBASE_MAX_ATTEMPTS} failed "+ f"({type(e).__name__}: {e}); retrying in {delay}s"+ )+ await asyncio.sleep(delay)++ async def _connect_browserbase_once(self, project_id: str) -> None:+ """Run one BrowserBase bring-up: session, CDP, setup, captcha wait."""+ # Clean slate: a prior attempt may have left _captcha_event cleared+ # if it saw "solving-started" but the session died before+ # "solving-finished" — the dead session can never set it.+ self._captcha_event.set()++ self._session = self._bb.sessions.create(+ project_id=project_id,+ proxies=True,+ browser_settings={"advanced_stealth": True, "enablePdfViewer": True},+ keep_alive=True,+ timeout=7200,+ api_timeout=3600,+ )+ assert self._session.id is not None+ assert (+ self._session.status == "RUNNING"+ ), f"Session status is {self._session.status}"++ self._browser = await self._playwright.chromium.connect_over_cdp(+ self._session.connect_url+ )+ self.logger.info(+ f"{self._tag()}Connected to BrowserBase: "+ f"https://browserbase.com/sessions/{self._session.id}"+ )++ self._context = self._browser.contexts[0]+ assert len(self._context.pages) == 1+ self._page = self._context.pages[0]++ # Context-level "console" already bubbles up page console messages+ # for every current and future page in the context, so we don't+ # also register on _page.+ self._context.on("console", self._handle_browserbase_console)++ # Navigate to start_page + wire handlers. Failures here trigger+ # a retry with a fresh session.+ await self._setup_browser()++ # Best-effort wait for a captcha triggered on start_page.+ # _captcha_event is already set when no captcha is active, so+ # this returns immediately in the common case.+ try:+ await asyncio.wait_for(+ self._captcha_event.wait(),+ timeout=self._BROWSERBASE_CAPTCHA_WAIT_S,+ )+ except asyncio.TimeoutError:+ self.logger.warning(+ f"{self._tag()}Captcha not resolved within "+ f"{self._BROWSERBASE_CAPTCHA_WAIT_S}s on start_page; proceeding"+ )++ def _handle_browserbase_console(self, msg) -> None:+ """Track BrowserBase captcha-solving lifecycle via console messages."""+ if msg.text == "browserbase-solving-started":+ self.logger.info(f"{self._tag()}Captcha Solving In Progress!")+ self._captcha_event.clear()+ elif msg.text == "browserbase-solving-finished":+ self.logger.info(f"{self._tag()}Captcha Solving Completed!")+ asyncio.create_task(self._resume_after_captcha())++ async def _resume_after_captcha(self) -> None:+ """Release the captcha event after a best-effort post-captcha settle."""+ try:+ await asyncio.sleep(3)+ try:+ # Prefer "domcontentloaded" over "networkidle" — the latter+ # can hang on sites with persistent network activity+ # (analytics, websockets). If captcha triggered a+ # navigation, this waits for the new DOM; otherwise it+ # returns immediately.+ await self._page.wait_for_load_state("domcontentloaded", timeout=10000)+ except Exception as e:+ self.logger.warning(+ f"Post-captcha wait_for_load_state failed "+ f"(continuing anyway): {e}"+ )+ finally:+ self._captcha_event.set()++ def _browserbase_retry_delay(self, exc: Exception, attempt: int, bb_module) -> int:+ """Pick a backoff delay for a failed BrowserBase init attempt."""+ if isinstance(exc, bb_module.RateLimitError):+ return self._BROWSERBASE_RATE_LIMIT_BACKOFF_S+ return 2 ** (attempt - 1)++ async def _teardown_browserbase_session(self, project_id: str) -> None:+ """Release a partially-built BrowserBase session and reset state."""+ browser_connected = self._browser is not None and self._browser.is_connected()+ if self._browser is not None:+ try:+ await self._browser.close()+ except Exception:+ pass+ self._browser = None+ self._context = None+ self._page = None+ if self._session is not None and self._bb is not None:+ session_id = self._session.id+ try:+ self._bb.sessions.update(+ self._session.id,+ status="REQUEST_RELEASE",+ project_id=project_id,+ )+ except Exception:+ pass+ # Single canonical terminal line per session — grep with [BB-END]+ # to reconstruct session<->task<->status without parsing tricks.+ self.logger.info(+ f"[BB-END] task={self._task_id} session={session_id} "+ f"browser_connected_at_teardown={browser_connected}"+ )+ self._session = None++ async def _init_persistent_browser(self) -> None:+ """Initialize persistent browser with data directory."""+ if not self.config.headless and self._is_linux:+ self._start_xvfb()++ launch_args = {"headless": self.config.headless}+ self._context = await self._playwright.chromium.launch_persistent_context(+ self.config.browser_data_dir, **launch_args+ )+ self._page = await self._context.new_page()++ # Hacky, brittle, undocumented: this cookie makes Bing serve search+ # results as same-tab links instead of target="_blank", which skips+ # the per-click popup-capture overhead on bing.com. Reverse-engineered+ # empirically; Microsoft can change it anytime. See PR #1114.+ _BING_SAME_TAB_COOKIE = {+ "name": "SRCHHPGUSR",+ "value": "EXLKNT=0",+ "domain": ".bing.com",+ "path": "/",+ "secure": True,+ }++ async def _setup_browser(self) -> None:+ """Set up common browser features."""+ self._context.set_default_timeout(self.config.default_timeout)++ await self._controller.on_new_page(self._page)++ if self._download_handler:+ self._page.on("download", self._download_handler)++ await self._page.set_viewport_size(+ {"width": self.config.viewport_width, "height": self.config.viewport_height}+ )++ if self.config.page_script_path and os.path.exists(+ self.config.page_script_path+ ):+ await self._page.add_init_script(path=self.config.page_script_path)++ # Inject Bing same-tab cookie before navigation so it applies to+ # the very first bing.com load. See _BING_SAME_TAB_COOKIE above.+ try:+ await self._context.add_cookies([self._BING_SAME_TAB_COOKIE])+ except Exception as e:+ self.logger.warning(f"Failed to inject Bing same-tab cookie: {e}")++ await self._page.goto(self.config.start_page, wait_until="commit")++ async def _swap_to_new_page(self, new_page: Page) -> None:+ """Promote a popup the agent's action just opened to the active page.++ Browsers foreground a newly opened tab on `target="_blank"` and+ `window.open()`, so the agent observing the new page matches what+ a real user would see. In ``single_tab_mode`` we also close the+ previous page to preserve the one-page worldview; otherwise we+ leave it on the context for a future multi-tab action space to+ switch back to.+ """+ old_page = self._page+ if new_page is old_page:+ return+ self._page = new_page+ if self.config.single_tab_mode and not old_page.is_closed():+ try:+ await old_page.close()+ except Exception as e:+ self.logger.warning(f"Error closing previous page after swap: {e}")++ def _start_xvfb(self) -> None:+ """Start Xvfb virtual display server."""+ display_num = 99+ self._xvfb_process = subprocess.Popen(+ ["Xvfb", f":{display_num}", "-screen", "0", "1280x1024x24", "-ac"],+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+ os.environ["DISPLAY"] = f":{display_num}"+ time.sleep(1)+ atexit.register(self._stop_xvfb)++ def _stop_xvfb(self) -> None:+ """Stop the Xvfb process."""+ if self._xvfb_process:+ self._xvfb_process.send_signal(signal.SIGTERM)+ self._xvfb_process.wait()+ self._xvfb_process = None++ async def wait_for_captcha_resolution(self) -> None:+ """Wait for captcha to be resolved if one is being solved."""+ await self._captcha_event.wait()++ async def close(self) -> None:+ """Close the browser and cleanup."""+ self.logger.info("Closing browser...")++ if self._page:+ await self._page.close()+ self._page = None++ if self._context:+ await self._context.close()+ self._context = None++ if self._browser:+ if self.config.use_browserbase and self._session and self._bb:+ project_id = self.config.browserbase_project_id or os.environ.get(+ "BROWSERBASE_PROJECT_ID"+ )+ session_id = self._session.id+ browser_connected = self._browser.is_connected()+ self._bb.sessions.update(+ self._session.id,+ status="REQUEST_RELEASE",+ project_id=project_id,+ )+ self.logger.info(+ f"[BB-END] task={self._task_id} session={session_id} "+ f"browser_connected_at_teardown={browser_connected}"+ )+ self._session = None+ await self._browser.close()+ self._browser = None++ if self._playwright:+ await self._playwright.stop()+ self._playwright = None++ if not self.config.headless:+ self._stop_xvfb()++ self._initialized = False++ async def get_observation(self) -> bytes:+ """Get screenshot of current page."""+ return await self.get_screenshot()++ # --- Core GUI actions (ComputerEnvironment interface) ---++ async def left_click(self, x: int, y: int) -> None:+ new_page = await self._controller.click_coords(self._page, x, y)+ if new_page is not None:+ await self._swap_to_new_page(new_page)++ async def right_click(self, x: int, y: int) -> None:+ new_page = await self._controller.cua_click(self._page, x, y, "right")+ if new_page is not None:+ await self._swap_to_new_page(new_page)++ async def double_click(self, x: int, y: int) -> None:+ new_page = await self._controller.cua_double_click(self._page, x, y)+ if new_page is not None:+ await self._swap_to_new_page(new_page)++ async def mouse_move(self, x: int, y: int) -> None:+ await self._controller.cua_move(self._page, x, y)++ async def left_click_drag(self, end_x: int, end_y: int) -> None:+ await self._page.mouse.down()+ await self._page.mouse.move(end_x, end_y)+ await self._page.mouse.up()++ async def key(self, keys: list[str]) -> None:+ chord = self._normalize_chord(keys)+ if self.config.enable_find_overlay and chord == self._FIND_CHORD:+ await self._page.evaluate(self._FIND_OVERLAY_JS)+ return+ handler = self._BROWSER_CHROME_DISPATCH.get(chord)+ if handler is not None:+ await getattr(self, handler)()+ return+ await self._controller.keypress(self._page, keys)++ async def type(self, text: str) -> None:+ await self._controller.cua_type(self._page, text)++ async def scroll(self, pixels: int) -> None:+ """Scroll vertically. Positive=up, negative=down."""+ await self._page.mouse.wheel(0, -pixels)++ async def wait(self, duration: float = 1.0) -> None:+ await self._controller.sleep(self._page, duration)++ async def get_screenshot(self, path: str | None = None) -> bytes:+ """Capture a screenshot of the current page."""+ return await self._controller.get_screenshot(self._page, path=path)++ # --- Browser navigation (BrowserEnvironment interface) ---++ async def goto_url(self, url: str) -> None:+ await self._controller.visit_page(self._page, url)++ async def go_back(self) -> None:+ await self._controller.back(self._page)++ async def refresh(self) -> None:+ await self._page.reload(wait_until="commit")++ # --- Extended actions ---++ async def middle_click(self, x: int, y: int) -> None:+ await self._page.mouse.click(x, y, button="middle")++ async def triple_click(self, x: int, y: int) -> None:+ await self._page.mouse.click(x, y, click_count=3)++ async def left_mouse_down(self, x: int, y: int) -> None:+ await self._page.mouse.move(x, y)+ await self._page.mouse.down()++ async def left_mouse_up(self, x: int, y: int) -> None:+ await self._page.mouse.move(x, y)+ await self._page.mouse.up()++ async def cursor_position(self) -> tuple[int, int]:+ pos = self._controller.last_cursor_position+ return (int(pos[0]), int(pos[1]))++ async def hscroll(self, pixels: int) -> None:+ """Horizontal scroll. Positive=right, negative=left."""+ await self._page.mouse.wheel(pixels, 0)++ # --- Backward-compat aliases ---++ async def click(self, x: float, y: float) -> Dict[str, Any]:+ """Click at coordinates."""+ await self.left_click(int(x), int(y))+ return {"success": True}++ async def hover(self, x: float, y: float) -> Dict[str, Any]:+ """Hover at coordinates."""+ await self._controller.hover_coords(self._page, x, y)+ return {"success": True}++ async def goto(self, url: str) -> Dict[str, Any]:+ """Navigate to a URL."""+ await self.goto_url(url)+ return {"success": True}++ async def back(self) -> Dict[str, Any]:+ """Go back in browser history."""+ await self.go_back()+ return {"success": True}++ async def forward(self) -> Dict[str, Any]:+ """Go forward in browser history."""+ await self._controller.cua_forward(self._page)+ return {"success": True}++ async def keypress(self, keys: List[str]) -> Dict[str, Any]:+ """Press a sequence of keys."""+ await self.key(keys)+ return {"success": True}++ async def scroll_down(self, amount: int = 400) -> Dict[str, Any]:+ """Scroll down the page."""+ await self._controller.page_down(self._page, amount=amount)+ return {"success": True}++ async def scroll_up(self, amount: int = 400) -> Dict[str, Any]:+ """Scroll up the page."""+ await self._controller.page_up(self._page, amount=amount)+ return {"success": True}++ # --- Playwright-specific methods (id-based, CUA, etc.) ---++ async def click_id(self, identifier: str) -> Dict[str, Any]:+ """Click on an element by its identifier."""+ new_page = await self._controller.click_id(self._page, identifier)+ if new_page is not None:+ await self._swap_to_new_page(new_page)+ return {"success": True}++ async def type_text(+ self,+ x: float,+ y: float,+ text: str,+ press_enter: bool = True,+ clear_first: bool = False,+ ) -> Dict[str, Any]:+ """Type text at coordinates."""+ new_page = await self._controller.fill_coords(+ self._page,+ x,+ y,+ text,+ press_enter=press_enter,+ delete_existing_text=clear_first,+ )+ if new_page is not None:+ await self._swap_to_new_page(new_page)+ return {"success": True}++ async def type_id(+ self,+ identifier: str,+ text: str,+ press_enter: bool = True,+ clear_first: bool = False,+ ) -> Dict[str, Any]:+ """Type text into an element by its identifier."""+ await self._controller.fill_id(+ self._page,+ identifier,+ text,+ press_enter=press_enter,+ delete_existing_text=clear_first,+ )+ return {"success": True}++ async def hover_id(self, identifier: str) -> Dict[str, Any]:+ """Hover over an element by its identifier."""+ await self._controller.hover_id(self._page, identifier)+ return {"success": True}++ async def scroll_id(self, identifier: str, direction: str) -> Dict[str, Any]:+ """Scroll within an element by its identifier."""+ await self._controller.scroll_id(self._page, identifier, direction)+ return {"success": True}++ async def wait_for_load(+ self, state: str = "domcontentloaded", timeout: Optional[int] = None+ ) -> Dict[str, Any]:+ """Wait for the page to reach a load state."""+ await self._controller.wait_for_load_state(+ self._page, state=state, timeout=timeout+ )+ if self.config.extra_sleep_time > 0:+ await asyncio.sleep(self.config.extra_sleep_time)+ return {"success": True}++ async def get_url(self) -> str:+ """Get the current page URL."""+ return await self._controller.get_page_url(self._page)++ async def get_title(self) -> str:+ """Get the current page title."""+ return await self._controller.get_page_title(self._page)++ async def get_text(self, n_lines: int = 100) -> str:+ """Get the visible text on the page."""+ return await self._controller.get_webpage_text(self._page, n_lines=n_lines)++ async def get_interactive_rects(self) -> Dict[str, Any]:+ """Get interactive element rectangles."""+ return await self._controller.get_interactive_rects(self._page)++ async def get_visual_viewport(self) -> Dict[str, Any]:+ """Get the visual viewport information."""+ return await self._controller.get_visual_viewport(self._page)++ async def get_page_metadata(self) -> Dict[str, Any]:+ """Get page metadata."""+ return await self._controller.get_page_metadata(self._page)++ async def get_focused_rect_id(self) -> str:+ """Get the ID of the currently focused element."""+ return await self._controller.get_focused_rect_id(self._page)++ async def get_page_markdown(self) -> str:+ """Get page content as markdown."""+ return await self._controller.get_page_markdown(self._page)++ async def get_page_context(self) -> PageContext:+ """Return current URL and viewport scroll position."""+ url = await self._controller.get_page_url(self._page)+ parts = [f"Current URL: {url}"]+ vp = await self._controller.get_visual_viewport(self._page)+ page_top = vp.get("pageTop", 0)+ viewport_h = vp.get("height", 0)+ scroll_h = vp.get("scrollHeight", 0)+ if scroll_h > 0:+ bottom = page_top + viewport_h+ pct = min(int(bottom / scroll_h * 100), 100)+ parts.append(+ f"Viewport: scrolled to {pct}% of page "+ f"({int(page_top)}-{int(bottom)}px of {int(scroll_h)}px)"+ )+ return PageContext(url=url, page_info="\n".join(parts))++ async def fill_coords(+ self,+ x: float,+ y: float,+ text: str,+ press_enter: bool = True,+ delete_existing_text: bool = False,+ ) -> Dict[str, Any]:+ """Fill text at coordinates."""+ new_page = await self._controller.fill_coords(+ self._page,+ x,+ y,+ text,+ press_enter=press_enter,+ delete_existing_text=delete_existing_text,+ )+ if new_page is not None:+ await self._swap_to_new_page(new_page)+ return {"success": True}++ async def fill_id(+ self,+ identifier: str,+ text: str,+ press_enter: bool = True,+ delete_existing_text: bool = False,+ ) -> Dict[str, Any]:+ """Fill text into an element by its identifier."""+ await self._controller.fill_id(+ self._page,+ identifier,+ text,+ press_enter=press_enter,+ delete_existing_text=delete_existing_text,+ )+ return {"success": True}++ async def select_option(self, identifier: str) -> Dict[str, Any]:+ """Select an option element by its identifier."""+ new_page = await self._controller.select_option(self._page, identifier)+ if new_page is not None:+ await self._swap_to_new_page(new_page)+ return {"success": True}++ async def cua_click(self, x: int, y: int, button: str = "left") -> Dict[str, Any]:+ """CUA click action."""+ new_page = await self._controller.cua_click(self._page, x, y, button)+ if new_page is not None:+ await self._swap_to_new_page(new_page)+ return {"success": True}++ async def cua_double_click(self, x: int, y: int) -> Dict[str, Any]:+ """CUA double click action."""+ new_page = await self._controller.cua_double_click(self._page, x, y)+ if new_page is not None:+ await self._swap_to_new_page(new_page)+ return {"success": True}++ async def cua_scroll(+ self, x: int, y: int, scroll_x: int, scroll_y: int+ ) -> Dict[str, Any]:+ """CUA scroll action."""+ await self._controller.cua_scroll(self._page, x, y, scroll_x, scroll_y)+ return {"success": True}++ async def cua_type(self, text: str) -> Dict[str, Any]:+ """CUA type action."""+ await self._controller.cua_type(self._page, text)+ return {"success": True}++ async def cua_wait(self, ms: int = 1000) -> Dict[str, Any]:+ """CUA wait action."""+ await self._controller.cua_wait(self._page, ms)+ return {"success": True}++ async def cua_move(self, x: int, y: int) -> Dict[str, Any]:+ """CUA move mouse action."""+ await self._controller.cua_move(self._page, x, y)+ return {"success": True}++ async def cua_drag(self, path: List[Dict[str, int]]) -> Dict[str, Any]:+ """CUA drag action along a path."""+ await self._controller.cua_drag(self._page, path)+ return {"success": True}++ async def evaluate(self, script: str) -> Any:+ """Evaluate JavaScript on the page."""+ return await self._page.evaluate(script)++ async def save_state(self) -> Dict[str, Any]:+ """Save environment state."""+ return {+ "url": self._page.url if self._page else None,+ "viewport_width": self.config.viewport_width,+ "viewport_height": self.config.viewport_height,+ }++ async def load_state(self, state: Dict[str, Any]) -> None:+ """Load environment state."""+ url = state.get("url")+ if url and self._page:+ await self._controller.visit_page(self._page, url)++ @property+ def page(self) -> Page | None:+ """Get the current page."""+ return self._page++ @property+ def context(self) -> BrowserContext | None:+ """Get the browser context."""+ return self._context++ @property+ def controller(self) -> PlaywrightController | None:+ """Get the playwright controller."""+ return self._controllerdiff --git a/src/fara/environments/playwright/find_overlay.js b/src/fara/environments/playwright/find_overlay.jsnew file mode 100644index 0000000..22191c8--- /dev/null+++ b/src/fara/environments/playwright/find_overlay.js@@ -0,0 +1,204 @@+// In-page find overlay injected by PlaywrightEnvironment on Ctrl+F.+// The input is focused so the agent's follow-up type() + keypress(["Enter"])+// flow into it naturally. Enter walks text nodes, highlights all matches,+// scrolls to the current one, and shows N/M. Repeated Enter advances and+// wraps. Esc removes the overlay and clears highlights.+(() => {+ const ID = '__agento_find__';+ const HL_CLASS = '__agento_find_hl__';+ const HL_ACTIVE = '__agento_find_hl_active__';++ const EXISTING = document.getElementById(ID);+ if (EXISTING) { EXISTING.querySelector('input').focus(); return; }++ // Inject highlight styles once.+ if (!document.getElementById(ID + '_style')) {+ const style = document.createElement('style');+ style.id = ID + '_style';+ style.textContent =+ '.' + HL_CLASS + '{background:#ffe066;color:#000;border-radius:2px;'+ + 'box-shadow:0 0 0 1px #f59f00}'+ + '.' + HL_ACTIVE + '{background:#ff922b;color:#000;'+ + 'box-shadow:0 0 0 2px #d9480f}'+ + '#' + ID + ' input::placeholder{color:#e8eaed;opacity:1}'+ + '@keyframes __agento_caret_blink{50%{opacity:0}}'+ + '#' + ID + ' input::placeholder{'+ + 'animation:__agento_caret_blink 1s step-end infinite}';+ document.head.appendChild(style);+ }++ const wrap = document.createElement('div');+ wrap.id = ID;+ wrap.style.cssText = [+ 'position:fixed', 'top:4px', 'right:8px', 'z-index:2147483647',+ 'background:#3c4043', 'color:#e8eaed',+ 'border-radius:4px', 'padding:4px 8px',+ 'font:13px system-ui,-apple-system,Segoe UI,sans-serif',+ 'box-shadow:0 2px 6px rgba(0,0,0,.25)',+ 'display:flex', 'align-items:center', 'gap:8px',+ ].join(';');++ const input = document.createElement('input');+ input.type = 'text';+ input.placeholder = '|';+ input.style.cssText = [+ 'width:180px', 'border:none', 'outline:none',+ 'background:transparent', 'color:#e8eaed',+ 'font:inherit', 'padding:2px 0',+ ].join(';');+ wrap.appendChild(input);++ const status = document.createElement('span');+ status.style.cssText = 'color:#9aa0a6;font-size:12px;min-width:32px;text-align:right';+ wrap.appendChild(status);++ const btnStyle = [+ 'background:rgba(255,255,255,0.08)', 'border:1px solid rgba(255,255,255,0.15)',+ 'color:#e8eaed', 'font:14px system-ui', 'cursor:pointer',+ 'padding:6px 10px', 'border-radius:4px', 'line-height:1',+ 'min-width:30px', 'min-height:28px', 'display:inline-flex',+ 'align-items:center', 'justify-content:center',+ ].join(';');+ const prevBtn = document.createElement('button');+ prevBtn.type = 'button';+ prevBtn.textContent = '\u25B2'; // up triangle+ prevBtn.title = 'Previous match (Shift+Enter)';+ prevBtn.style.cssText = btnStyle;+ wrap.appendChild(prevBtn);+ const nextBtn = document.createElement('button');+ nextBtn.type = 'button';+ nextBtn.textContent = '\u25BC'; // down triangle+ nextBtn.title = 'Next match (Enter)';+ nextBtn.style.cssText = btnStyle;+ wrap.appendChild(nextBtn);+ const closeBtn = document.createElement('button');+ closeBtn.type = 'button';+ closeBtn.textContent = '\u2715'; // x+ closeBtn.title = 'Close (Escape)';+ closeBtn.style.cssText = btnStyle;+ wrap.appendChild(closeBtn);++ document.body.appendChild(wrap);+ input.focus();++ let lastQuery = null;+ let ranges = []; // Range objects for all matches+ let spans = []; // highlight <span> elements matching ranges+ let cursor = -1;++ function clearHighlights() {+ for (const span of spans) {+ const parent = span.parentNode;+ if (!parent) continue;+ while (span.firstChild) parent.insertBefore(span.firstChild, span);+ parent.removeChild(span);+ parent.normalize();+ }+ spans = [];+ ranges = [];+ }++ function collectMatches(q) {+ const needle = q.toLowerCase();+ const hits = [];+ const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, {+ acceptNode: (n) => {+ if (!n.nodeValue) return NodeFilter.FILTER_REJECT;+ if (n.parentElement && n.parentElement.closest('#' + ID))+ return NodeFilter.FILTER_REJECT;+ if (n.parentElement && n.parentElement.closest('.' + HL_CLASS))+ return NodeFilter.FILTER_REJECT;+ return NodeFilter.FILTER_ACCEPT;+ }+ });+ let node;+ while ((node = walker.nextNode())) {+ const hay = node.nodeValue.toLowerCase();+ let i = 0;+ while ((i = hay.indexOf(needle, i)) !== -1) {+ hits.push({ node, offset: i });+ i += needle.length;+ }+ }+ return hits;+ }++ function highlightAll(hits, q) {+ // Process bottom-up per text node so earlier offsets remain valid.+ const byNode = new Map();+ for (const h of hits) {+ if (!byNode.has(h.node)) byNode.set(h.node, []);+ byNode.get(h.node).push(h.offset);+ }+ const newSpans = [];+ for (const [node, offsets] of byNode) {+ offsets.sort((a, b) => b - a);+ for (const off of offsets) {+ const range = document.createRange();+ range.setStart(node, off);+ range.setEnd(node, off + q.length);+ const span = document.createElement('span');+ span.className = HL_CLASS;+ range.surroundContents(span);+ newSpans.push(span);+ }+ }+ // Reorder in document order so cursor-advance matches visual reading.+ newSpans.sort((a, b) => {+ if (a === b) return 0;+ const pos = a.compareDocumentPosition(b);+ return pos & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1;+ });+ return newSpans;+ }++ function setActive(idx) {+ for (const s of spans) s.classList.remove(HL_ACTIVE);+ if (idx < 0 || idx >= spans.length) return;+ const span = spans[idx];+ span.classList.add(HL_ACTIVE);+ span.scrollIntoView({ block: 'center' });+ }++ function advance(direction) {+ const q = input.value;+ if (!q) {+ status.textContent = '';+ clearHighlights();+ return;+ }+ if (q !== lastQuery) {+ clearHighlights();+ lastQuery = q;+ const hits = collectMatches(q);+ if (hits.length === 0) {+ status.textContent = '0/0';+ return;+ }+ spans = highlightAll(hits, q);+ cursor = -1;+ }+ if (spans.length === 0) return;+ cursor = (cursor + direction + spans.length) % spans.length;+ setActive(cursor);+ status.textContent = (cursor + 1) + '/' + spans.length;+ }++ function close() {+ clearHighlights();+ wrap.remove();+ }++ input.addEventListener('keydown', (e) => {+ if (e.key === 'Enter') {+ e.preventDefault();+ advance(e.shiftKey ? -1 : 1);+ } else if (e.key === 'Escape') {+ e.preventDefault();+ close();+ }+ });+ prevBtn.addEventListener('click', () => { advance(-1); input.focus(); });+ nextBtn.addEventListener('click', () => { advance(1); input.focus(); });+ closeBtn.addEventListener('click', close);+})();diff --git a/src/fara/environments/playwright/page_script.js b/src/fara/environments/playwright/page_script.jsnew file mode 100644index 0000000..36cf56a--- /dev/null+++ b/src/fara/environments/playwright/page_script.js@@ -0,0 +1,610 @@+var MultimodalWebSurfer = MultimodalWebSurfer || (function() {+ let nextLabel = 10;++ let roleMapping = {+ "a": "link",+ "area": "link",+ "button": "button",+ "input, type=button": "button",+ "input, type=checkbox": "checkbox",+ "input, type=email": "textbox",+ "input, type=number": "spinbutton",+ "input, type=radio": "radio",+ "input, type=range": "slider",+ "input, type=reset": "button",+ "input, type=search": "searchbox",+ "input, type=submit": "button",+ "input, type=tel": "textbox",+ "input, type=text": "textbox",+ "input, type=url": "textbox",+ "search": "search",+ "select": "combobox",+ "option": "option",+ "textarea": "textbox"+ };++ let getCursor = function (elm) {+ return window.getComputedStyle(elm)["cursor"];+ };++ let isVisible = function (element) {+ return !!(element.offsetWidth || element.offsetHeight || element.getClientRects().length);+ };++ /**+ * Finds interactive elements in the regular DOM (excluding Shadow DOM)+ * Looks for elements that are:+ * 1. Standard interactive elements (inputs, buttons, links)+ * 2. Elements with ARIA roles indicating interactivity+ * 3. Elements with cursor styles suggesting interactivity+ *+ * @returns {Array} Array of DOM elements that are deemed interactive+ */+ let getInteractiveElementsNoShaddow = function () {+ let results = []+ let roles = ["scrollbar", "searchbox", "slider", "spinbutton", "switch", "tab", "treeitem", "button", "checkbox", "gridcell", "link", "menuitem", "menuitemcheckbox", "menuitemradio", "option", "progressbar", "radio", "textbox", "combobox", "menu", "tree", "treegrid", "grid", "listbox", "radiogroup", "widget"];+ let inertCursors = ["auto", "default", "none", "text", "vertical-text", "not-allowed", "no-drop"];++ // Get the main interactive elements+ let nodeList = document.querySelectorAll("input, select, textarea, button, [href], [onclick], [contenteditable], [tabindex]:not([tabindex='-1'])");+ for (let i = 0; i < nodeList.length; i++) { // Copy to something mutable+ // make sure not disabled and visible+ if (nodeList[i].disabled || !isVisible(nodeList[i])) {+ continue;+ }+ results.push(nodeList[i]);+ }++ // Anything not already included that has a suitable role+ nodeList = document.querySelectorAll("[role]");+ for (let i = 0; i < nodeList.length; i++) { // Copy to something mutable+ // make sure not disabled and visible+ if (nodeList[i].disabled || !isVisible(nodeList[i])) {+ continue;+ }+ if (results.indexOf(nodeList[i]) == -1) {+ let role = nodeList[i].getAttribute("role");+ if (roles.indexOf(role) > -1) {+ results.push(nodeList[i]);+ }+ }+ }++ // Any element that changes the cursor to something implying interactivity+ nodeList = document.querySelectorAll("*");+ for (let i = 0; i < nodeList.length; i++) {+ let node = nodeList[i];+ if (node.disabled || !isVisible(node)) {+ continue;+ }++ // Cursor is default, or does not suggest interactivity+ let cursor = getCursor(node);+ if (inertCursors.indexOf(cursor) >= 0) {+ continue;+ }++ // Move up to the first instance of this cursor change+ let parent = node.parentNode;+ while (parent && getCursor(parent) == cursor) {+ node = parent;+ parent = node.parentNode;+ }++ // Add the node if it is new+ if (results.indexOf(node) == -1) {+ results.push(node);+ }+ }++ return results;+ };++ /**+ * Recursively gathers elements matching specified roles from both regular DOM and Shadow DOM+ * @param {Array} roles - Array of role selectors to match+ * @param {Document|ShadowRoot} root - Root element to start search from+ * @returns {Array} Array of matching elements+ */+ function gatherAllElements(roles, root = document) {+ const elements = [];+ const stack = [root];+ const selector = roles.join(",");++ while (stack.length > 0) {+ const currentRoot = stack.pop();++ // Add elements at current level+ elements.push(...Array.from(currentRoot.querySelectorAll(selector)));++ // Add shadow roots to stack+ currentRoot.querySelectorAll("*").forEach(el => {+ if (el.shadowRoot && el.shadowRoot.mode === "open") {+ stack.push(el.shadowRoot);+ }+ });+ }++ return elements;+ }++ /**+ * Gets all interactive elements from both regular DOM and Shadow DOM+ * Filters elements to ensure they are visible and accessible+ * @returns {Array} Array of interactive elements+ */+ let getInteractiveElements = function () {+ // Get all elements that are interactive without the shadow DOM+ const interactive_roles = ["input", "option", "select", "textarea", "button", "href", "onclick", "contenteditable", "tabindex:not([tabindex='-1'])"];++ let results = [];++ let elements_no_shaddow = getInteractiveElementsNoShaddow();+ for (let i = 0; i < elements_no_shaddow.length; i++) {+ if (results.indexOf(elements_no_shaddow[i]) == -1) {+ // check if it has a rect+ let rects = elements_no_shaddow[i].getClientRects();+ for (const rect of rects) {+ let x = rect.left + rect.width / 2;+ let y = rect.top + rect.height / 2;+ if (isTopmost(elements_no_shaddow[i], x, y)) {+ results.push(elements_no_shaddow[i]);+ break;+ }+ }+ }+ }++ // From the shadow DOM get all interactive elements and options that are not in the no shadow list+ let elements_all = gatherAllElements(interactive_roles);++ // Filter and process interactive elements+ elements_all.forEach(element => {+ // if file, auto add+ if (element.tagName.toLowerCase() === "input" && element.getAttribute("type") == "file") {+ results.push(element);+ return;+ }+ // if option auto add+ if (element.tagName.toLowerCase() === "option") {+ results.push(element);+ return;+ }+ if (element.disabled || !isVisible(element)) {+ return;+ }++ if (interactive_roles.includes(element.tagName.toLowerCase())) {+ results.push(element);+ }+ });++ return results;+ };++ /**+ * Assigns unique identifiers to interactive elements+ * @param {Array} elements - Array of elements to label+ * @returns {Array} Updated array of interactive elements+ */+ let labelElements = function (elements) {+ for (let i = 0; i < elements.length; i++) {+ if (!elements[i].hasAttribute("__elementId")) {+ elements[i].setAttribute("__elementId", "" + (nextLabel++));+ }+ }+ return getInteractiveElements();+ };++ /**+ * Checks if an element is the topmost element at given coordinates+ * @param {Element} element - Element to check+ * @param {number} x - X coordinate+ * @param {number} y - Y coordinate+ * @returns {boolean} True if element is topmost at coordinates+ */+ let isTopmost = function (element, x, y) {+ let hit = document.elementFromPoint(x, y);++ // Hack to handle elements outside the viewport+ if (hit === null) {+ return true;+ }++ while (hit) {+ if (hit == element) return true;+ hit = hit.parentNode;+ }+ return false;+ };++ let getFocusedElementId = function () {+ let elm = document.activeElement;+ while (elm) {+ if (elm.hasAttribute && elm.hasAttribute("__elementId")) {+ return elm.getAttribute("__elementId");+ }+ elm = elm.parentNode;+ }+ return null;+ };++ let trimmedInnerText = function (element) {+ if (!element) {+ return "";+ }+ let text = element.innerText;+ if (!text) {+ return "";+ }+ return text.trim();+ };++ let getApproximateAriaName = function (element) {+ if (element.hasAttribute("aria-label")) {+ return element.getAttribute("aria-label");+ }++ // check if element has span that is called label and grab the inner text+ if (element.querySelector("span.label")) {+ return element.querySelector("span.label").innerText;+ }++ // Check for aria labels+ if (element.hasAttribute("aria-labelledby")) {+ let buffer = "";+ let ids = element.getAttribute("aria-labelledby").split(" ");+ for (let i = 0; i < ids.length; i++) {+ let label = document.getElementById(ids[i]);+ if (label) {+ buffer = buffer + " " + trimmedInnerText(label);+ }+ }+ return buffer.trim();+ }++ if (element.hasAttribute("aria-label")) {+ return element.getAttribute("aria-label");+ }++ // Check for labels+ if (element.hasAttribute("id")) {+ let label_id = element.getAttribute("id");+ let label = "";+ try {+ // Escape special characters in the ID+ let escaped_id = CSS.escape(label_id);+ let labels = document.querySelectorAll(`label[for="${escaped_id}"]`);+ for (let j = 0; j < labels.length; j++) {+ label += labels[j].innerText + " ";+ }+ label = label.trim();+ if (label != "") {+ return label;+ }+ } catch (e) {+ console.warn("Error finding label for element:", e);+ }+ }++ if (element.hasAttribute("name")) {+ return element.getAttribute("name");+ }++ if (element.parentElement && element.parentElement.tagName == "LABEL") {+ return element.parentElement.innerText;+ }++ // Check for alt text or titles+ if (element.hasAttribute("alt")) {+ return element.getAttribute("alt")+ }++ if (element.hasAttribute("title")) {+ return element.getAttribute("title")+ }++ return trimmedInnerText(element);+ };++ let getApproximateAriaRole = function (element) {+ let tag = element.tagName.toLowerCase();+ if (tag == "input" && element.hasAttribute("type")) {+ tag = tag + ", type=" + element.getAttribute("type");+ }++ if (element.hasAttribute("role")) {+ return [element.getAttribute("role"), tag];+ }+ else if (tag in roleMapping) {+ return [roleMapping[tag], tag];+ }+ else {+ return ["", tag];+ }+ };++ /**+ * Gets information about all interactive elements including their:+ * - Position and dimensions+ * - ARIA roles and names+ * - Tag names+ * - Scrollability+ *+ * @returns {Object} Map of element IDs to their properties+ */+ let getInteractiveRects = function () {+ let elements = labelElements(getInteractiveElements());+ let results = {};+ for (let i = 0; i < elements.length; i++) {+ let key = elements[i].getAttribute("__elementId");+ let rects = elements[i].getBoundingClientRect();++ // Skip options unless their select is focused+ if (elements[i].tagName.toLowerCase() === "option") {++ let select_focused = false;+ let select = elements[i].closest("select");+ if (select && select.hasAttribute("__elementId") &&+ getFocusedElementId() === select.getAttribute("__elementId")) {+ select_focused = true;+ }+ // check if option is visible without select being focused+ let option_visible = false;+ if (isVisible(elements[i])) {+ option_visible = true;+ }+ // check if select is expanded even if not focused+ let select_expanded = false;+ if (select && select.hasAttribute("open")) {+ select_expanded = true;+ }+ if (!(select_focused || option_visible || select_expanded)) {+ continue;+ }+ }++ let ariaRole = getApproximateAriaRole(elements[i]);+ let ariaName = getApproximateAriaName(elements[i]);+ let vScrollable = elements[i].scrollHeight - elements[i].clientHeight >= 1;++ let record = {+ "tag_name": ariaRole[1],+ "role": ariaRole[0],+ "aria-name": ariaName,+ "v-scrollable": vScrollable,+ "rects": []+ };++ if (rects.length > 0) {+ for (const rect of rects) {+ let x = rect.left + rect.width / 2;+ let y = rect.top + rect.height / 2;+ if (isTopmost(elements[i], x, y)) {+ record["rects"].push(JSON.parse(JSON.stringify(rect)));+ }+ }+ }+ else {+ record["rects"].push(JSON.parse(JSON.stringify(rects)));+ }++ results[key] = record;+ }+ return results;+ };++ /**+ * Gets current viewport information including dimensions and scroll positions+ * @returns {Object} Viewport properties+ */+ let getVisualViewport = function () {+ let vv = window.visualViewport;+ let de = document.documentElement;+ return {+ "height": vv ? vv.height : 0,+ "width": vv ? vv.width : 0,+ "offsetLeft": vv ? vv.offsetLeft : 0,+ "offsetTop": vv ? vv.offsetTop : 0,+ "pageLeft": vv ? vv.pageLeft : 0,+ "pageTop": vv ? vv.pageTop : 0,+ "scale": vv ? vv.scale : 0,+ "clientWidth": de ? de.clientWidth : 0,+ "clientHeight": de ? de.clientHeight : 0,+ "scrollWidth": de ? de.scrollWidth : 0,+ "scrollHeight": de ? de.scrollHeight : 0+ };+ };++ let _getMetaTags = function () {+ let meta = document.querySelectorAll("meta");+ let results = {};+ for (let i = 0; i < meta.length; i++) {+ let key = null;+ if (meta[i].hasAttribute("name")) {+ key = meta[i].getAttribute("name");+ }+ else if (meta[i].hasAttribute("property")) {+ key = meta[i].getAttribute("property");+ }+ else {+ continue;+ }+ if (meta[i].hasAttribute("content")) {+ results[key] = meta[i].getAttribute("content");+ }+ }+ return results;+ };++ let _getJsonLd = function () {+ let jsonld = [];+ let scripts = document.querySelectorAll('script[type="application/ld+json"]');+ for (let i = 0; i < scripts.length; i++) {+ jsonld.push(scripts[i].innerHTML.trim());+ }+ return jsonld;+ };++ // From: https://www.stevefenton.co.uk/blog/2022/12/parse-microdata-with-javascript/+ let _getMicrodata = function () {+ function sanitize(input) {+ return input.replace(/\s/gi, ' ').trim();+ }++ function addValue(information, name, value) {+ if (information[name]) {+ if (typeof information[name] === 'array') {+ information[name].push(value);+ } else {+ const arr = [];+ arr.push(information[name]);+ arr.push(value);+ information[name] = arr;+ }+ } else {+ information[name] = value;+ }+ }++ function traverseItem(item, information) {+ const children = item.children;++ for (let i = 0; i < children.length; i++) {+ const child = children[i];++ if (child.hasAttribute('itemscope')) {+ if (child.hasAttribute('itemprop')) {+ const itemProp = child.getAttribute('itemprop');+ const itemType = child.getAttribute('itemtype');++ const childInfo = {+ itemType: itemType+ };++ traverseItem(child, childInfo);++ itemProp.split(' ').forEach(propName => {+ addValue(information, propName, childInfo);+ });+ }++ } else if (child.hasAttribute('itemprop')) {+ const itemProp = child.getAttribute('itemprop');+ itemProp.split(' ').forEach(propName => {+ if (propName === 'url') {+ addValue(information, propName, child.href);+ } else {+ addValue(information, propName, sanitize(child.getAttribute("content") || child.content || child.textContent || child.src || ""));+ }+ });+ traverseItem(child, information);+ } else {+ traverseItem(child, information);+ }+ }+ }++ const microdata = [];++ document.querySelectorAll("[itemscope]").forEach(function (elem, i) {+ const itemType = elem.getAttribute('itemtype');+ const information = {+ itemType: itemType+ };+ traverseItem(elem, information);+ microdata.push(information);+ });++ return microdata;+ };++ let getPageMetadata = function () {+ let jsonld = _getJsonLd();+ let metaTags = _getMetaTags();+ let microdata = _getMicrodata();+ let results = {}+ if (jsonld.length > 0) {+ try {+ results["jsonld"] = JSON.parse(jsonld);+ }+ catch (e) {+ results["jsonld"] = jsonld;+ }+ }+ if (microdata.length > 0) {+ results["microdata"] = microdata;+ }+ for (let key in metaTags) {+ if (metaTags.hasOwnProperty(key)) {+ results["meta_tags"] = metaTags;+ break;+ }+ }+ return results;+ };++ /**+ * Extracts all visible text content from the viewport+ * Preserves basic formatting with newlines for block elements+ * @returns {string} Visible text content+ */+ let getVisibleText = function () {+ // Get the window's current viewport boundaries+ const viewportHeight = window.innerHeight || document.documentElement.clientHeight;+ const viewportWidth = window.innerWidth || document.documentElement.clientWidth;++ let textInView = "";+ const walker = document.createTreeWalker(+ document.body,+ NodeFilter.SHOW_TEXT,+ null,+ false+ );++ while (walker.nextNode()) {+ const textNode = walker.currentNode;+ // Create a range to retrieve bounding rectangles of the current text node+ const range = document.createRange();+ range.selectNodeContents(textNode);++ const rects = range.getClientRects();++ // Check if any rect is inside (or partially inside) the viewport+ for (const rect of rects) {+ const isVisible =+ rect.width > 0 &&+ rect.height > 0 &&+ rect.bottom >= 0 &&+ rect.right >= 0 &&+ rect.top <= viewportHeight &&+ rect.left <= viewportWidth;++ if (isVisible) {+ textInView += textNode.nodeValue.replace(/\s+/g, " ");+ // Is the parent a block element?+ if (textNode.parentNode) {+ const parent = textNode.parentNode;+ const style = window.getComputedStyle(parent);+ if (["inline", "hidden", "none"].indexOf(style.display) === -1) {+ textInView += "\n";+ }+ }+ break; // No need to check other rects once found visible+ }+ }+ }++ // Remove blank lines from textInView+ textInView = textInView.replace(/^\s*\n/gm, "").trim().replace(/\n+/g, "\n");+ return textInView;+ };++ // Public API+ return {+ getInteractiveRects: getInteractiveRects,+ getVisualViewport: getVisualViewport,+ getFocusedElementId: getFocusedElementId,+ getPageMetadata: getPageMetadata,+ };+})();\ No newline at end of filediff --git a/src/fara/environments/playwright/playwright_controller.py b/src/fara/environments/playwright/playwright_controller.pynew file mode 100644index 0000000..6b002fb--- /dev/null+++ b/src/fara/environments/playwright/playwright_controller.py@@ -0,0 +1,1187 @@+import asyncio+import base64+import os+import logging+import functools+from typing import (+ Any,+ Callable,+ Dict,+ List,+ Optional,+ Tuple,+ Union,+ TypeVar,+ Awaitable,+ cast,+)++from playwright._impl._errors import Error as PlaywrightError+from playwright._impl._errors import TimeoutError, TargetClosedError+from playwright.async_api import Download, Page+from playwright.async_api import TimeoutError as PlaywrightTimeoutError+from ._types import (+ InteractiveRegion,+ VisualViewport,+ interactiveregion_from_dict,+ visualviewport_from_dict,+)+from .webpage_text_utils import WebpageTextUtilsPlaywright++# Some of the Code for clicking coordinates and keypresses adapted from https://github.com/openai/openai-cua-sample-app/blob/main/computers/base_playwright.py+# Copyright 2025 OpenAI - MIT License+from ...agents.key_mapping import CUA_KEY_TO_PLAYWRIGHT_KEY++F = TypeVar("F", bound=Callable[..., Awaitable[Any]])+++def handle_target_closed(max_retries: int = 2, timeout_secs: int = 30):+ """+ Decorator to handle TargetClosedError and tunnel connection errors by attempting to recover the page.++ Args:+ max_retries: Maximum number of retry attempts+ timeout_secs: Timeout for page operations during recovery+ """++ def decorator(func: F) -> F:+ @functools.wraps(func)+ async def wrapper(*args, **kwargs):+ # Extract the page object - assume it's the first argument after self+ logger = args[0].logger+ page = None+ if len(args) >= 2 and hasattr(+ args[1], "url"+ ): # Check if second arg looks like a Page+ page = args[1]++ retries = 0+ last_error = None++ while retries <= max_retries:+ try:+ return await func(*args, **kwargs)+ except (TargetClosedError, PlaywrightError) as e:+ # Check if this is a tunnel connection error+ is_tunnel_error = "net::ERR_TUNNEL_CONNECTION_FAILED" in str(e)+ is_target_closed = isinstance(+ e, TargetClosedError+ ) or "Target page, context or browser has been closed" in str(e)++ if not (is_tunnel_error or is_target_closed):+ # Not an error we handle, re-raise+ raise e++ last_error = e+ retries += 1++ if retries > max_retries:+ raise e++ if page is None:+ # Can't recover without page reference+ raise e++ error_type = (+ "tunnel connection" if is_tunnel_error else "target closed"+ )+ logger.warning(+ f"{error_type} error in {func.__name__}, attempting recovery (retry {retries}/{max_retries})"+ )++ try:+ # Attempt to recover the page+ await _recover_page(page, timeout_secs, logger)+ # Small delay before retry+ await asyncio.sleep(0.5)+ except Exception as recovery_error:+ logger.error(f"Page recovery failed: {recovery_error}")+ # If recovery fails, raise the original error+ raise e from recovery_error++ # This shouldn't be reached, but just in case+ raise last_error++ return wrapper++ return decorator+++async def _recover_page(page: Page, timeout_secs: int = 30, logger=None) -> None:+ """+ Attempt to recover a closed page by reloading it.++ Args:+ page: The Playwright page object to recover+ timeout_secs: Timeout for recovery operations+ """+ logger = logger or logging.getLogger("playwright_controller")+ try:+ # First, try to check if the page is still responsive+ await page.evaluate("1", timeout=1000)+ # If we get here, the page is actually fine+ return+ except Exception:+ # Page is indeed problematic, attempt recovery+ pass++ # If the underlying browser is gone (BB session died, CDP dropped),+ # reload/goto on this page will both fail with the same TargetClosed —+ # ~60s wasted per recovery. Bail out early.+ browser = page.context.browser if page.context else None+ if browser is not None and not browser.is_connected():+ raise Exception(+ "playwright_controller._recover_page(): browser disconnected, recovery impossible"+ )++ try:+ # Stop any ongoing navigation+ await page.evaluate("window.stop()", timeout=2000)+ except Exception:+ # Ignore errors from window.stop()+ pass++ try:+ # Try to reload the page+ await page.reload(timeout=timeout_secs * 1000, wait_until="commit")+ logger.info("playwright_controller._recover_page(): Page recovery successful")+ except Exception as e:+ logger.error(f"playwright_controller._recover_page(): Page reload failed: {e}")++ # Try alternative recovery: navigate to current URL+ try:+ current_url = page.url+ if current_url and current_url != "about:blank":+ await page.goto(+ current_url,+ timeout=timeout_secs * 1000,+ wait_until="commit",+ )+ logger.info(+ "playwright_controller._recover_page(): Page recovery via goto successful"+ )+ else:+ raise Exception(+ "playwright_controller._recover_page(): No valid URL to navigate to"+ )+ except Exception as goto_error:+ raise Exception(+ f"playwright_controller._recover_page(): All recovery methods failed. Reload error: {e}, Goto error: {goto_error}"+ )+++# Enhanced version that can handle browser context recreation+def handle_target_closed_with_context(max_retries: int = 2, timeout_secs: int = 30):+ """+ Enhanced decorator that can also handle browser context recreation.+ Use this for critical operations where you have access to the browser context.+ """++ def decorator(func: F) -> F:+ @functools.wraps(func)+ async def wrapper(*args, **kwargs):+ logger = args[0].logger+ page = None+ if len(args) >= 2 and hasattr(args[1], "url"):+ page = args[1]++ retries = 0+ last_error = None++ while retries <= max_retries:+ try:+ return await func(*args, **kwargs)+ except (TargetClosedError, PlaywrightError) as e:+ # Check if this is a tunnel connection error+ is_tunnel_error = "net::ERR_TUNNEL_CONNECTION_FAILED" in str(e)+ is_target_closed = isinstance(+ e, TargetClosedError+ ) or "Target page, context or browser has been closed" in str(e)++ if not (is_tunnel_error or is_target_closed):+ # Not an error we handle, re-raise+ raise e++ last_error = e+ retries += 1++ if retries > max_retries:+ raise e++ if page is None:+ raise e++ error_type = (+ "tunnel connection" if is_tunnel_error else "target closed"+ )+ logger.warning(+ f"playwright_controller.handle_target_closed_with_context(): {error_type} error in {func.__name__}, attempting enhanced recovery (retry {retries}/{max_retries})"+ )++ try:+ # Check if the browser context is still alive+ context = page.context+ browser = context.browser++ if browser and not browser.is_connected():+ # Browser connection is lost - this is a more serious issue+ logger.error(+ "playwright_controller.handle_target_closed_with_context(): Browser connection lost - cannot recover automatically"+ )+ raise e++ # Try basic recovery first+ await _recover_page(page, timeout_secs)+ await asyncio.sleep(0.5)++ except Exception as recovery_error:+ logger.error(+ f"playwright_controller.handle_target_closed_with_context(): Enhanced page recovery failed: {recovery_error}"+ )+ raise e from recovery_error++ raise last_error++ return wrapper++ return decorator+++class PlaywrightController:+ def __init__(+ self,+ animate_actions: bool = False,+ downloads_folder: Optional[str] = None,+ viewport_width: int = 1440,+ viewport_height: int = 900,+ _download_handler: Optional[Callable[[Download], None]] = None,+ to_resize_viewport: bool = True,+ single_tab_mode: bool = False,+ sleep_after_action: int = 10,+ timeout_load: int = 1,+ logger=None,+ ) -> None:+ """+ A controller for Playwright to interact with web pages.+ animate_actions: If True, actions will be animated.+ downloads_folder: The folder to save downloads to.+ viewport_width: The width of the viewport.+ viewport_height: The height of the viewport.+ _download_handler: A handler for downloads.+ to_resize_viewport: If True, the viewport will be resized.+ single_tab_mode (bool): If True, forces navigation to happen in the same tab rather than opening new tabs/windows.++ """+ self.animate_actions = animate_actions+ self.downloads_folder = downloads_folder+ self.viewport_width = viewport_width+ self.viewport_height = viewport_height+ self._download_handler = _download_handler+ self.to_resize_viewport = to_resize_viewport+ self.single_tab_mode = single_tab_mode+ self._sleep_after_action = sleep_after_action+ self._timeout_load = timeout_load+ self.logger = logger or logging.getLogger("playwright_controller")++ # Set up the download handler++ self._page_script: str = ""+ self.last_cursor_position: Tuple[float, float] = (0.0, 0.0)+ self._text_utils = WebpageTextUtilsPlaywright()++ # Read page_script+ with open(+ os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"),+ "rt",+ ) as fh:+ self._page_script = fh.read()++ async def sleep(self, page: Page, duration: Union[int, float]) -> None:+ await asyncio.sleep(duration)++ @handle_target_closed()+ async def get_interactive_rects(self, page: Page) -> Dict[str, InteractiveRegion]:+ await self._ensure_page_ready(page)+ # Read the regions from the DOM+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = cast(+ Dict[str, Dict[str, Any]],+ await page.evaluate("MultimodalWebSurfer.getInteractiveRects();"),+ )++ # Convert the results into appropriate types+ assert isinstance(result, dict)+ typed_results: Dict[str, InteractiveRegion] = {}+ for k in result:+ assert isinstance(k, str)+ typed_results[k] = interactiveregion_from_dict(result[k])++ return typed_results++ @handle_target_closed()+ async def get_visual_viewport(self, page: Page) -> VisualViewport:+ await self._ensure_page_ready(page)+ try:+ await page.evaluate(self._page_script)+ except Exception:+ self.logger.warning("Failed to inject page script for visual viewport")+ return visualviewport_from_dict({})+ return visualviewport_from_dict(+ await page.evaluate("MultimodalWebSurfer.getVisualViewport();")+ )++ @handle_target_closed()+ async def get_focused_rect_id(self, page: Page) -> str:+ await self._ensure_page_ready(page)+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = await page.evaluate("MultimodalWebSurfer.getFocusedElementId();")+ return str(result)++ @handle_target_closed()+ async def get_page_metadata(self, page: Page) -> Dict[str, Any]:+ assert page is not None+ attempts = 3+ while attempts > 0:+ try:+ try:+ await self._ensure_page_ready(page)+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = await page.evaluate("MultimodalWebSurfer.getPageMetadata();")+ assert isinstance(result, dict)+ return cast(Dict[str, Any], result)+ except Exception as e:+ self.logger.error(+ f"Error: playwrite_controller.get_page_metadata: {str(e)},\nattempting again..."+ )+ attempts -= 1+ await asyncio.sleep(0.5)++ @handle_target_closed()+ async def on_new_page(self, page: Page) -> None:+ assert page is not None+ # bring page to front just in case+ await page.bring_to_front()+ page.on("download", self._download_handler) # type: ignore+ if self.to_resize_viewport and self.viewport_width and self.viewport_height:+ await page.set_viewport_size(+ {"width": self.viewport_width, "height": self.viewport_height}+ )++ await self.sleep(page, 0.2)+ await page.add_init_script(+ path=os.path.join(+ os.path.abspath(os.path.dirname(__file__)), "page_script.js"+ )+ )+ try:+ await page.wait_for_load_state("domcontentloaded", timeout=60000)+ except PlaywrightTimeoutError:+ self.logger.error("WARNING: Page load timeout, page might not be loaded")+ # stop page loading+ await page.evaluate("window.stop()")++ @handle_target_closed()+ async def _ensure_page_ready(self, page: Page) -> None:+ assert page is not None+ await self.on_new_page(page)++ @handle_target_closed()+ async def get_screenshot(self, page: Page, path: str | None = None) -> bytes:+ """+ Capture a screenshot of the current page.++ Args:+ page (Page): The Playwright page object.+ path (str, optional): The file path to save the screenshot. If None, the screenshot will be returned as bytes. Default: None+ """+ await self._ensure_page_ready(page)+ try:+ screenshot = await page.screenshot(path=path, timeout=15000)+ return screenshot+ except Exception:+ await page.evaluate("window.stop()")+ # try again+ screenshot = await page.screenshot(path=path, timeout=15000)+ return screenshot++ @handle_target_closed()+ async def back(self, page: Page) -> None:+ await self._ensure_page_ready(page)+ await page.go_back()++ @handle_target_closed()+ async def visit_page(self, page: Page, url: str) -> Tuple[bool, bool]:+ await self._ensure_page_ready(page)+ reset_prior_metadata_hash = False+ reset_last_download = False+ try:+ # Regular webpage+ await page.goto(url, wait_until="commit")+ reset_prior_metadata_hash = True+ except Exception as e_outer:+ # Downloaded file+ if self.downloads_folder and "net::ERR_ABORTED" in str(e_outer):+ async with page.expect_download() as download_info:+ try:+ await page.goto(url, wait_until="commit")+ except Exception as e_inner:+ if "net::ERR_ABORTED" in str(e_inner):+ pass+ else:+ raise e_inner+ download = await download_info.value+ fname = os.path.join(+ self.downloads_folder, download.suggested_filename+ )+ await download.save_as(fname)+ message = f"<body style=\"margin: 20px;\"><h1>Successfully downloaded '{download.suggested_filename}' to local path:<br><br>{fname}</h1></body>"+ await page.goto(+ "data:text/html;base64,"+ + base64.b64encode(message.encode("utf-8")).decode("utf-8"),+ wait_until="commit",+ )+ reset_last_download = True+ else:+ raise e_outer+ return reset_prior_metadata_hash, reset_last_download++ @handle_target_closed()+ async def page_down(+ self, page: Page, amount: int = 400, full_page: bool = False+ ) -> None:+ await self._ensure_page_ready(page)+ if full_page:+ await page.mouse.wheel(0, self.viewport_height - 50)+ else:+ await page.mouse.wheel(0, amount)++ @handle_target_closed()+ async def page_up(+ self, page: Page, amount: int = 400, full_page: bool = False+ ) -> None:+ await self._ensure_page_ready(page)+ if full_page:+ await page.mouse.wheel(0, -self.viewport_height + 50)+ else:+ await page.mouse.wheel(0, -amount)++ async def gradual_cursor_animation(+ self, page: Page, start_x: float, start_y: float, end_x: float, end_y: float+ ) -> None:+ # animation helper+ steps = 20+ for step in range(steps):+ x = start_x + (end_x - start_x) * (step / steps)+ y = start_y + (end_y - start_y) * (step / steps)+ # await page.mouse.move(x, y, steps=1)+ await page.evaluate(f"""+ (function() {{+ let cursor = document.getElementById('red-cursor');+ cursor.style.left = '{x}px';+ cursor.style.top = '{y}px';+ }})();+ """)+ await asyncio.sleep(0.05)++ self.last_cursor_position = (end_x, end_y)++ async def add_cursor_box(self, page: Page, identifier: str) -> None:+ # animation helper+ await page.evaluate(f"""+ (function() {{+ let elm = document.querySelector("[__elementId='{identifier}']");+ if (elm) {{+ elm.style.transition = 'border 0.3s ease-in-out';+ elm.style.border = '2px solid red';+ }}+ }})();+ """)+ await asyncio.sleep(0.3)++ # Create a red cursor+ await page.evaluate("""+ (function() {+ let cursor = document.createElement('div');+ cursor.id = 'red-cursor';+ cursor.style.width = '10px';+ cursor.style.height = '10px';+ cursor.style.backgroundColor = 'red';+ cursor.style.position = 'absolute';+ cursor.style.borderRadius = '50%';+ cursor.style.zIndex = '10000';+ document.body.appendChild(cursor);+ })();+ """)++ async def remove_cursor_box(self, page: Page, identifier: str) -> None:+ # Remove the highlight and cursor+ await page.evaluate(f"""+ (function() {{+ let elm = document.querySelector("[__elementId='{identifier}']");+ if (elm) {{+ elm.style.border = '';+ }}+ let cursor = document.getElementById('red-cursor');+ if (cursor) {{+ cursor.remove();+ }}+ }})();+ """)++ async def _click_with_popup_capture(+ self, page: Page, action: Callable[[], Awaitable[None]]+ ) -> Page | None:+ """Run a click-like action and capture any popup it opens.++ Returns the new page so the caller can promote it to the active+ page (see ``Environment._swap_to_new_page``). The env decides what+ to do with the old page (close it in ``single_tab_mode``, keep it+ otherwise). Returns ``None`` if no popup fires within the timeout.+ """+ new_page: Page | None = None+ try:+ async with page.expect_event("popup", timeout=1000) as page_info: # type: ignore+ await action()+ new_page = await page_info.value # type: ignore+ assert isinstance(new_page, Page)+ await self.on_new_page(new_page)+ except TimeoutError:+ pass+ return new_page++ @handle_target_closed()+ async def click_coords(self, page: Page, x: float, y: float) -> Page | None:+ await self._ensure_page_ready(page)++ if self.animate_actions:+ # Move cursor to the box slowly+ start_x, start_y = self.last_cursor_position+ await self.gradual_cursor_animation(page, start_x, start_y, x, y)+ await asyncio.sleep(0.1)++ return await self._click_with_popup_capture(+ page, lambda: page.mouse.click(x, y, delay=10)+ )++ @handle_target_closed()+ async def click_id(self, page: Page, identifier: str) -> Page | None:+ """+ Returns new page if a new page is opened, otherwise None.+ """+ new_page: Page | None = None+ await self._ensure_page_ready(page)+ selector = f"[__elementId='{identifier}']"+ try:+ # Wait for the element to be visible and scroll it into view+ await page.wait_for_selector(+ selector, state="visible", timeout=self._timeout_load * 1000+ )+ target = page.locator(selector)+ await target.scroll_into_view_if_needed()+ except TimeoutError:+ raise ValueError(+ f"Element with identifier {identifier} not found or not visible"+ )++ # Retrieve bounding box to determine the center for clicking+ box = await target.bounding_box()+ if not box:+ raise ValueError(+ f"Element with identifier {identifier} is not visible on the page."+ )+ center_x = box["x"] + box["width"] / 2+ center_y = box["y"] + box["height"] / 2++ # In single tab mode, override target attributes to avoid opening a new tab+ if self.single_tab_mode:+ await target.evaluate("""+ el => {+ // Remove target attribute from clicked element and all _blank links/forms+ el.removeAttribute('target');+ document.querySelectorAll('a[target=_blank], form[target=_blank]')+ .forEach(e => e.removeAttribute('target'));+ }+ """)++ download = None+ download_future: asyncio.Task[Download] | None = None++ # Start listening for a download event if downloads are enabled+ if self.downloads_folder:+ try:+ download_future = asyncio.create_task(+ page.wait_for_event( # type: ignore+ "download", timeout=500+ )+ )+ except Exception as e:+ self.logger.error(f"Failed to set up download listener: {e}")+ download_future = None++ async def perform_click() -> Optional[Page]:+ nonlocal download+ try:+ if self.single_tab_mode:+ await page.mouse.move(center_x, center_y, steps=1)+ await page.mouse.click(center_x, center_y)+ return None+ else:+ # Create a task to wait for a new page event+ new_page_promise: asyncio.Task[Page] = asyncio.create_task(+ page.context.wait_for_event( # type: ignore+ "page", timeout=self._timeout_load * 1000+ )+ )++ # Perform the click+ await page.mouse.move(center_x, center_y, steps=1)+ await page.mouse.click(center_x, center_y, delay=10)++ try:+ # Wait for the new page to open+ new_page = await new_page_promise+ await self.on_new_page(new_page)+ return new_page+ except TimeoutError:+ # No new page opened within timeout+ return None+ except Exception as e:+ raise e++ # Optionally animate the click+ if self.animate_actions:+ await self.add_cursor_box(page, identifier)+ start_x, start_y = self.last_cursor_position+ await self.gradual_cursor_animation(+ page, start_x, start_y, center_x, center_y+ )++ new_page = await perform_click()++ # Handle any download that occurred+ if download_future:+ try:+ if not download:+ # Use asyncio.wait_for with a reasonable timeout+ try:+ download = await asyncio.wait_for(+ download_future, timeout=self._timeout_load * 1000+ )+ except asyncio.TimeoutError:+ # No download occurred within the timeout period+ # logger.debug("No download detected within timeout period")+ pass++ if download:+ self.logger.info(+ f"Downloading {download.suggested_filename} to {self.downloads_folder}"+ )+ assert self.downloads_folder is not None+ fname = os.path.join(+ self.downloads_folder, download.suggested_filename+ )+ await download.save_as(fname)+ except Exception as e:+ self.logger.error(f"Error handling download: {e}")+ finally:+ if not download_future.done():+ download_future.cancel()++ if self.animate_actions:+ await self.remove_cursor_box(page, identifier)++ if new_page:+ await new_page.wait_for_load_state("domcontentloaded")+ if self._sleep_after_action > 0:+ await new_page.wait_for_timeout(self._sleep_after_action * 1000)+ else:+ await page.wait_for_load_state("domcontentloaded")+ if self._sleep_after_action > 0:+ await page.wait_for_timeout(self._sleep_after_action * 1000)+ # await asyncio.sleep(10)++ return new_page++ @handle_target_closed()+ async def select_option(self, page: Page, identifier: str) -> Optional[Page]:+ """+ Select an option element with the given identifier. If the element has a visible size,+ it will be clicked normally. Otherwise, it will be selected programmatically.++ Args:+ page (Page): The Playwright page object.+ identifier (str): The element identifier of the option to select.++ Returns:+ Optional[Page]: The Playwright page object of the newly focused tab.+ """+ await self._ensure_page_ready(page)+ new_page: Optional[Page] = None+ try:+ # Wait for element to be present+ await page.wait_for_selector(+ f"[__elementId='{identifier}']", state="attached"+ )++ try:+ # First try normal click if element is visible+ target = page.locator(f"[__elementId='{identifier}']").first+ # Get the bounding box to check element size+ box = await target.bounding_box()++ if box and box["width"] > 0 and box["height"] > 0:+ # Element has visible size - use normal click+ return await self.click_id(page, identifier)++ except PlaywrightError as e:+ if "strict mode violation" in str(e):+ # If multiple elements found, try clicking the first visible one+ elements = await page.locator(f"[__elementId='{identifier}']").all()+ for element in elements:+ try:+ if await element.is_visible():+ await element.click()+ return new_page+ except PlaywrightError:+ continue++ # If click didn't work, try programmatic selection+ # First check if it's a standard <option> element+ option_element = await page.evaluate(+ """+ (identifier) => {+ const elements = document.querySelectorAll(`[__elementId='${identifier}']`);+ for (const el of elements) {+ if (el.tagName.toLowerCase() === 'option') {+ return true;+ }+ }+ return false;+ }+ """,+ identifier,+ )++ if option_element:+ # Handle standard <select> dropdown+ await page.evaluate(+ """+ (identifier) => {+ const option = Array.from(document.querySelectorAll(`[__elementId='${identifier}']`))+ .find(el => el.tagName.toLowerCase() === 'option');+ if (!option) throw new Error('Option not found');+ const select = option.closest('select');+ if (select) {+ option.selected = true;+ select.dispatchEvent(new Event('change', { bubbles: true }));+ select.blur();+ }+ }+ """,+ identifier,+ )+ else:+ # Handle custom dropdown/combobox options+ await page.evaluate(+ """+ (identifier) => {+ const element = document.querySelector(`[__elementId='${identifier}']`);+ if (!element) throw new Error('Element not found');++ // Dispatch multiple events to ensure the selection is registered+ const events = ['mousedown', 'mouseup', 'click', 'change'];+ events.forEach(eventType => {+ element.dispatchEvent(new Event(eventType, { bubbles: true }));+ });++ // If element has aria-selected, set it+ if (element.hasAttribute('aria-selected')) {+ element.setAttribute('aria-selected', 'true');+ }++ // If element has a data-value, try to set it on the parent+ const value = element.getAttribute('data-value');+ if (value) {+ const parent = element.closest('[role="listbox"], [role="combobox"]');+ if (parent) {+ parent.setAttribute('data-value', value);+ }+ }+ }+ """,+ identifier,+ )++ # Optional sleep/pause after the action+ if self._sleep_after_action > 0:+ await page.wait_for_timeout(self._sleep_after_action * 1000)++ except PlaywrightTimeoutError:+ raise ValueError(+ f"No option found with identifier '{identifier}' within "+ f"{self._timeout_load} seconds."+ ) from None+ return new_page++ @handle_target_closed()+ async def hover_coords(self, page: Page, x: float, y: float) -> None:+ """+ Hovers the mouse over the specified coordinates.++ Args:+ page (Page): The Playwright page object.+ x (float): The x coordinate to hover over.+ y (float): The y coordinate to hover over.+ """+ await self._ensure_page_ready(page)++ if self.animate_actions:+ # Move cursor to the coordinates slowly+ start_x, start_y = self.last_cursor_position+ await self.gradual_cursor_animation(page, start_x, start_y, x, y)+ await asyncio.sleep(0.1)++ await page.mouse.move(x, y)++ @handle_target_closed()+ async def hover_id(self, page: Page, identifier: str) -> None:+ """+ Hovers the mouse over the target with the given id.+ """+ await self._ensure_page_ready(page)+ target = page.locator(f"[__elementId='{identifier}']")++ # See if it exists+ try:+ await target.wait_for(timeout=5000)+ except TimeoutError:+ raise RuntimeError(+ f"Tool use response is invalid: no such element to hover: {identifier}"+ )++ # Hover over it+ await target.scroll_into_view_if_needed()+ await asyncio.sleep(0.3)++ box = cast(Dict[str, Union[int, float]], await target.bounding_box())++ if self.animate_actions:+ await self.add_cursor_box(page, identifier)+ # Move cursor to the box slowly+ start_x, start_y = self.last_cursor_position+ end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2+ await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)+ await asyncio.sleep(0.1)+ await page.mouse.move(+ box["x"] + box["width"] / 2, box["y"] + box["height"] / 2+ )++ await self.remove_cursor_box(page, identifier)+ else:+ await page.mouse.move(+ box["x"] + box["width"] / 2, box["y"] + box["height"] / 2+ )++ async def _clear_field(self, page: Page) -> None:+ """Select all text in the focused field and delete it."""+ await page.keyboard.press("ControlOrMeta+A")+ await page.keyboard.press("Backspace")++ @handle_target_closed()+ async def fill_coords(+ self,+ page: Page,+ x: float,+ y: float,+ value: str,+ press_enter: bool = True,+ delete_existing_text: bool = False,+ ) -> Page | None:+ await self._ensure_page_ready(page)+ new_page: Page | None = None++ if self.animate_actions:+ # Move cursor to the box slowly+ start_x, start_y = self.last_cursor_position+ await self.gradual_cursor_animation(page, start_x, start_y, x, y)+ await asyncio.sleep(0.1)++ await page.mouse.click(x, y)++ if delete_existing_text:+ await self._clear_field(page)++ # Short strings (<100 chars) use 100ms delay per keystroke to avoid+ # dropping characters on fields with JS input masks (phone, credit card,+ # date). These masks have per-keystroke event handlers (onKeyDown/onChange)+ # that need time to reformat the value and reposition the cursor. See+ # https://github.com/microsoft/magentic-ui2.0/issues/279#issuecomment-4127277347+ # for an example of where this issue manifested itself.+ #+ # Long strings use 10ms since they are almost always plain text fields+ # with no input masks, and 100ms would be too slow (500 chars = 50s).+ if len(value) < 100:+ delay_typing_speed = 100+ else:+ delay_typing_speed = 10++ # If a PlaywrightError occurs mid-typing (e.g., element detach,+ # navigation), clear the field before retrying to avoid corrupted input+ # from a partial first attempt followed by a full retry.+ try:+ await page.keyboard.type(value, delay=delay_typing_speed)+ except PlaywrightError:+ await self._clear_field(page)+ await page.keyboard.type(value, delay=delay_typing_speed)++ if press_enter:+ await page.keyboard.press("Enter")++ # Wait briefly for a popup that may have opened during typing or Enter.+ # Some websites open a new tab/popup on form submission (e.g., forms+ # with target="_blank"). The listener starts after typing so the timeout+ # window isn't consumed by keystroke delivery.+ try:+ new_page = await page.wait_for_event("popup", timeout=2000)+ assert isinstance(new_page, Page)+ await self.on_new_page(new_page)+ except (TimeoutError, asyncio.TimeoutError):+ pass++ return new_page++ @handle_target_closed()+ async def fill_id(+ self,+ page: Page,+ identifier: str,+ value: str,+ press_enter: bool = True,+ delete_existing_text: bool = False,+ ) -> None:+ """+ Fill the element with the given identifier with the specified value.+ Works with text inputs, textareas, and comboboxes.+ Args:+ page (Page): The Playwright page object.+ identifier (str): The element identifier.+ value (str): The value to fill.+ press_enter (bool): Whether to press enter after filling.+ delete_existing_text (bool): Whether to delete existing text before filling.+ """+ await self._ensure_page_ready(page)+ await page.wait_for_selector(f"[__elementId='{identifier}']", state="visible")+ target = page.locator(f"[__elementId='{identifier}']")+ await target.scroll_into_view_if_needed()++ # See if it exists+ try:+ await target.wait_for(timeout=5000)+ except TimeoutError:+ raise RuntimeError(+ f"Tool use response is invalid: No such element to fill input_text into: {identifier}"+ ) from None++ # Fill it+ box = cast(Dict[str, Union[int, float]], await target.bounding_box())++ if self.single_tab_mode:+ # Remove target attributes to prevent new tabs+ await target.evaluate("""+ el => el.removeAttribute('target')+ // Remove 'target' on all <a> tags+ for (const a of document.querySelectorAll('a[target=_blank]')) {+ a.removeAttribute('target');+ }+ // Remove 'target' on all <form> tags+ for (const frm of document.querySelectorAll('form[target=_blank]')) {+ frm.removeAttribute('target');+ }+ """)++ page = await self.fill_coords(+ page,+ float(box["x"] + box["width"] / 2),+ float(box["y"] + box["height"] / 2),+ value,+ press_enter,+ delete_existing_text,+ )++ @handle_target_closed()+ async def scroll_id(self, page: Page, identifier: str, direction: str) -> None:+ await self._ensure_page_ready(page)+ await page.evaluate(+ f"""+ (function() {{+ let elm = document.querySelector("[__elementId='{identifier}']");+ if (elm) {{+ if ("{direction}" == "up") {{+ elm.scrollTop = Math.max(0, elm.scrollTop - elm.clientHeight);+ }} else if ("{direction}" == "down") {{+ elm.scrollTop = Math.min(elm.scrollHeight - elm.clientHeight, elm.scrollTop + elm.clientHeight);+ }} else if ("{direction}" == "left") {{+ elm.scrollLeft = Math.max(0, elm.scrollLeft - elm.clientWidth);+ }} else if ("{direction}" == "right") {{+ elm.scrollLeft = Math.min(elm.scrollWidth - elm.clientWidth, elm.scrollLeft + elm.clientWidth);+ }}+ }}+ }})();+ """+ )++ async def keypress(self, page: Page, keys: list[str], key_mapping=None) -> None:+ """+ Press specified keys in sequence.++ Args:+ page (Page): The Playwright page object+ keys (List[str]): List of keys to press+ """+ await self._ensure_page_ready(page)+ key_mapping = key_mapping or CUA_KEY_TO_PLAYWRIGHT_KEY+ mapped_keys = [key_mapping.get(key.lower(), key) for key in keys]+ pressed: list[str] = []+ try:+ for key in mapped_keys:+ await page.keyboard.down(key)+ pressed.append(key)+ if self.animate_actions:+ await asyncio.sleep(0.05)+ except Exception as e:+ msg = str(e)+ if "Unknown key" in msg:+ bad = next(+ (k for k in keys if key_mapping.get(k.lower(), k) not in pressed),+ keys[-1],+ )+ raise ValueError(+ f"Unknown key: {bad!r}. keypress only accepts single characters "+ f"or named keys (Enter, Tab, ArrowDown, Control, ...). "+ f"Use type() for text input."+ ) from None+ raise RuntimeError(+ f"I tried to keypress(keys={keys}), but I got an error: {e}"+ ) from None+ finally:+ for key in reversed(pressed):+ try:+ await page.keyboard.up(key)+ if self.animate_actions:+ await asyncio.sleep(0.05)+ except Exception:+ continue # cleanup must not mask the primary failure++ @handle_target_closed()+ async def cua_click(+ self, page: Page, x: int, y: int, button: str = "left"+ ) -> Page | None:+ match button:+ case "back":+ await self.back(page)+ return None+ case "forward":+ await self.cua_forward(page)+ return None+ case "wheel":+ await page.mouse.wheel(x, y)+ return None+ case _:+ await self._ensure_page_ready(page)+ button_type = "right" if button == "right" else "left"+ return await self._click_with_popup_capture(+ page, lambda: page.mouse.click(x, y, button=button_type)+ )++ @handle_target_closed()+ async def cua_double_click(self, page: Page, x: int, y: int) -> Page | None:+ await self._ensure_page_ready(page)+ return await self._click_with_popup_capture(+ page, lambda: page.mouse.dblclick(x, y)+ )++ @handle_target_closed()+ async def cua_scroll(+ self, page: Page, x: int, y: int, scroll_x: int, scroll_y: int+ ) -> None:+ await page.mouse.move(x, y)+ await page.evaluate(f"window.scrollBy({scroll_x}, {scroll_y})")++ @handle_target_closed()+ async def cua_type(self, page: Page, text: str) -> None:+ await page.keyboard.type(text)++ async def cua_wait(self, page: Page, ms: int = 1000) -> None:+ await asyncio.sleep(ms / 1000.0)++ @handle_target_closed()+ async def cua_move(self, page: Page, x: int, y: int) -> None:+ await page.mouse.move(x, y)++ @handle_target_closed()+ async def cua_drag(self, page: Page, path: List[Dict[str, int]]) -> None:+ if not path:+ return+ await page.mouse.move(path[0]["x"], path[0]["y"])+ await page.mouse.down()+ for point in path[1:]:+ await page.mouse.move(point["x"], point["y"])+ await page.mouse.up()++ @handle_target_closed()+ async def cua_forward(self, page: Page) -> None:+ await self._ensure_page_ready(page)+ await page.go_forward()++ @handle_target_closed()+ async def get_webpage_text(self, page: Page, n_lines: int = 100) -> str:+ """+ page: playwright page object+ n_lines: number of lines to return from the page innertext+ return: text in the first n_lines of the page+ """+ await self._ensure_page_ready(page)+ try:+ text_in_viewport = await page.evaluate("""() => {+ return document.body.innerText;+ }""")+ text_in_viewport = "\n".join(text_in_viewport.split("\n")[:n_lines])+ # remove empty lines+ text_in_viewport = "\n".join(+ [line for line in text_in_viewport.split("\n") if line.strip()]+ )+ assert isinstance(text_in_viewport, str)+ return text_in_viewport+ except Exception:+ return ""++ async def get_page_markdown(self, page: Page) -> str:+ await self._ensure_page_ready(page)+ return await self._text_utils.get_page_markdown(page)++ @handle_target_closed()+ async def get_page_url(self, page: Page) -> str:+ """Get the current page URL."""+ await self._ensure_page_ready(page)+ return page.url++ @handle_target_closed()+ async def get_page_title(self, page: Page) -> str:+ """Get the current page title."""+ await self._ensure_page_ready(page)+ return await page.title()++ @handle_target_closed()+ async def wait_for_load_state(+ self, page: Page, state: str = "domcontentloaded", timeout: Optional[int] = None+ ) -> None:+ """Wait for the page to reach a specific load state."""+ await page.wait_for_load_state(state, timeout=timeout)diff --git a/src/fara/environments/playwright/webpage_text_utils.py b/src/fara/environments/playwright/webpage_text_utils.pynew file mode 100644index 0000000..7ccf6d9--- /dev/null+++ b/src/fara/environments/playwright/webpage_text_utils.py@@ -0,0 +1,169 @@+"""Utility class for extracting text/markdown from web pages via Playwright."""++import io+import logging+import os+import tempfile+from typing import Any, Optional++import tiktoken+from playwright.async_api import Page++logger = logging.getLogger(__name__)+++class WebpageTextUtilsPlaywright:+ def __init__(self):+ self._markdown_converter: Optional[Any] = None+ self._page_script: str = ""++ with open(+ os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"),+ "rt",+ encoding="utf-8",+ ) as fh:+ self._page_script = fh.read()++ async def get_all_webpage_text(self, page: Page, n_lines: int = 50) -> str:+ """Retrieve the text content of the web page.++ Args:+ page: The Playwright page object.+ n_lines: The number of lines to return from the page inner text.++ Returns:+ The text content of the page.+ """+ try:+ text_in_viewport = await page.evaluate("""() => {+ return document.body.innerText;+ }""")+ text_in_viewport = "\n".join(text_in_viewport.split("\n")[:n_lines])+ text_in_viewport = "\n".join(+ [line for line in text_in_viewport.split("\n") if line.strip()]+ )+ assert isinstance(text_in_viewport, str)+ return text_in_viewport+ except Exception:+ return ""++ async def get_visible_text(self, page: Page) -> str:+ """Retrieve the text content of the browser viewport (approximately).++ Args:+ page: The Playwright page object.++ Returns:+ The text content of the page.+ """+ try:+ await page.evaluate(self._page_script)+ except Exception:+ pass+ result = await page.evaluate("WebSurfer.getVisibleText();")+ assert isinstance(result, str)+ return result++ async def get_page_markdown(self, page: Page, max_tokens: int = -1) -> str:+ """Retrieve the markdown content of the web page.+ Automatically detects and handles PDF content.++ Args:+ page: The Playwright page object.+ max_tokens: The maximum number of tokens to return. -1 means no limit.++ Returns:+ The markdown content of the page or extracted PDF content.+ """+ is_pdf = await self._is_pdf_page(page)++ if is_pdf:+ pdf_content = await self._extract_pdf_content(page)+ if max_tokens == -1:+ return pdf_content+ tokenizer = tiktoken.encoding_for_model("gpt-4o")+ tokens = tokenizer.encode(pdf_content)+ return tokenizer.decode(tokens[:max_tokens])++ if self._markdown_converter is None:+ from markitdown import MarkItDown++ self._markdown_converter = MarkItDown()+ html = await page.evaluate("document.documentElement.outerHTML;")+ res = self._markdown_converter.convert_stream(+ io.BytesIO(html.encode("utf-8")), file_extension=".html", url=page.url+ )+ text_content = res.text_content++ if max_tokens == -1:+ return text_content+ tokenizer = tiktoken.encoding_for_model("gpt-4o")+ tokens = tokenizer.encode(text_content)+ return tokenizer.decode(tokens[:max_tokens])++ async def _is_pdf_page(self, page: Page) -> bool:+ """Check if the current page is a PDF document."""+ if page.url.lower().endswith(".pdf"):+ return True++ result = await page.evaluate("""() => {+ if (document.contentType === 'application/pdf') return true;+ if (document.querySelector('embed[type="application/pdf"]') ||+ document.querySelector('object[type="application/pdf"]')) return true;+ if (window.PDFViewerApplication || document.querySelector('#viewer.pdfViewer')) return true;+ return false;+ }""")++ return result++ async def _extract_pdf_content(self, page: Page) -> str:+ """Extract text content from a PDF page."""+ url = page.url++ try:+ browser_text = await self._extract_pdf_browser(page)+ if browser_text and len(browser_text) > 100:+ return browser_text++ logger.info("Using MarkItDown for better PDF extraction...")+ pdf_buffer = await page.context.request.get(url)+ pdf_data = await pdf_buffer.body()++ with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as temp_file:+ temp_file_path = temp_file.name+ temp_file.write(pdf_data)++ if self._markdown_converter is None:+ from markitdown import MarkItDown++ self._markdown_converter = MarkItDown()+ result = self._markdown_converter.convert(temp_file_path)++ os.unlink(temp_file_path)++ return result.text_content++ except Exception as e:+ logger.error(f"Error extracting PDF content: {str(e)}")+ return f"Error extracting PDF content: {str(e)}"++ async def _extract_pdf_browser(self, page: Page) -> str:+ """Extract text content from a PDF page using browser methods."""+ pdf_text = await page.evaluate("""() => {+ if (window.PDFViewerApplication) {+ const textContent = document.querySelectorAll('.textLayer div');+ if (textContent.length > 0) {+ return Array.from(textContent).map(div => div.textContent).join('\\n');+ }+ }+ const textElements = Array.from(document.querySelectorAll('p, span, div'))+ .filter(el => {+ const style = window.getComputedStyle(el);+ return style.display !== 'none' &&+ style.visibility !== 'hidden' &&+ el.textContent.trim() !== '';+ });+ return textElements.map(el => el.textContent).join('\\n');+ }""")++ return pdf_textdiff --git a/src/fara/environments/screen_resolutions.py b/src/fara/environments/screen_resolutions.pynew file mode 100644index 0000000..c970518--- /dev/null+++ b/src/fara/environments/screen_resolutions.py@@ -0,0 +1,209 @@+"""Popular desktop screen resolutions used for `randomize_screen_res`.++`weight` is the sampling probability (in %) used by+``sample_random_screen_resolution``; the 12 resolutions below are chosen+and weighted to span Windows (native + DPI-scaled), macOS (MBP "Looks+Like" modes), Linux-flagship 16:10, Surface 3:2, and ultrawide layouts.+Weights do not need to sum to exactly 100 — ``random.choices`` normalizes+them. The `aspect`, `scenario`, and `sources` fields are documentation+only.+"""++from __future__ import annotations++import random+from typing import NamedTuple+++class ScreenResolution(NamedTuple):+ width: int+ height: int+ aspect: str+ scenario: str+ weight: float+ sources: tuple[str, ...]+++POPULAR_SCREEN_RESOLUTIONS: list[ScreenResolution] = [+ ScreenResolution(+ width=1920,+ height=1080,+ aspect="16:9",+ scenario="Windows laptop/desktop dominant baseline — Full HD at 100% scaling",+ weight=32.0,+ sources=(+ "https://gs.statcounter.com/screen-resolution-stats/desktop",+ "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",+ ),+ ),+ ScreenResolution(+ width=1536,+ height=864,+ aspect="16:9",+ scenario=(+ "Windows 1920x1080 @125% scaling as seen by DPI-Unaware pyautogui — "+ "the default 'scaled FHD laptop' view"+ ),+ weight=14.0,+ sources=(+ "https://gs.statcounter.com/screen-resolution-stats/desktop",+ "https://learn.microsoft.com/en-us/windows/win32/hidpi/high-dpi-desktop-application-development-on-windows",+ ),+ ),+ ScreenResolution(+ width=1366,+ height=768,+ aspect="16:9",+ scenario=(+ "Budget Windows laptops native; sub-$300 Chromebooks (K-12 fleet); "+ "Surface Laptop SE"+ ),+ weight=8.0,+ sources=(+ "https://gs.statcounter.com/screen-resolution-stats/desktop",+ "https://www.aboutchromebooks.com/chromebook-hardware-statistics/",+ "https://www.starryhope.com/chromebooks/chromebook-comparison-chart/",+ ),+ ),+ ScreenResolution(+ width=2560,+ height=1440,+ aspect="16:9",+ scenario=(+ "QHD desktop monitor standard for Windows; also macOS Studio Display "+ "default 'Looks Like' and 4K Windows panels @ 150%"+ ),+ weight=9.0,+ sources=(+ "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",+ "https://www.apple.com/studio-display/specs/",+ "https://gs.statcounter.com/screen-resolution-stats/desktop",+ ),+ ),+ ScreenResolution(+ width=1280,+ height=720,+ aspect="16:9",+ scenario=(+ "Windows 1920x1080 @150% scaling as seen by DPI-Unaware pyautogui; "+ "also small/netbook panels"+ ),+ weight=5.0,+ sources=(+ "https://gs.statcounter.com/screen-resolution-stats/desktop",+ "https://github.com/asweigart/pyautogui/issues/33",+ ),+ ),+ ScreenResolution(+ width=3840,+ height=2160,+ aspect="16:9",+ scenario=(+ "4K monitor and 4K laptop @ 100% scaling — Windows enthusiasts, "+ "some Linux dev workstations"+ ),+ weight=5.0,+ sources=(+ "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",+ "https://tech-docs.system76.com/models/oryp10/README.html",+ ),+ ),+ ScreenResolution(+ width=1504,+ height=1000,+ aspect="3:2",+ scenario=(+ 'Surface Laptop 13.5" (2256x1504 @150% default, DPI-Unaware view); '+ 'Surface Book 3 13.5" (3000x2000 @200% = 1500x1000)'+ ),+ weight=4.0,+ sources=(+ "https://surfacetip.com/surface-display-comparison/",+ "https://glossary.surfacetip.com/definition/2256x1504/",+ ),+ ),+ ScreenResolution(+ width=3440,+ height=1440,+ aspect="21:9",+ scenario=(+ "Ultrawide gaming/productivity monitor — almost exclusively Windows "+ "desktop enthusiasts; fundamentally different layout regime "+ "(side-by-side apps are the default)"+ ),+ weight=3.0,+ sources=(+ "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",+ ),+ ),+ ScreenResolution(+ width=1512,+ height=982,+ aspect="~3:2",+ scenario=(+ "macOS 14\" MacBook Pro M1 Pro -> M5 Max default 'Looks Like' "+ "(pixel-doubled from 3024x1964 native) — the developer-laptop default"+ ),+ weight=7.0,+ sources=(+ "https://www.apple.com/macbook-pro/specs/",+ "https://9to5mac.com/2021/10/19/new-macbook-pro-screen-resolution-options/",+ "https://github.com/asweigart/pyautogui/issues/699",+ ),+ ),+ ScreenResolution(+ width=1728,+ height=1117,+ aspect="~3:2",+ scenario=(+ "macOS 16\" MacBook Pro M1 Pro -> M5 Max default 'Looks Like' "+ "(pixel-doubled from 3456x2234 native)"+ ),+ weight=5.0,+ sources=(+ "https://www.apple.com/macbook-pro/specs/",+ "https://9to5mac.com/2021/10/19/new-macbook-pro-screen-resolution-options/",+ ),+ ),+ ScreenResolution(+ width=1440,+ height=900,+ aspect="16:10",+ scenario=(+ 'Legacy 13" MBP/MBA (Intel/M1 era, 2560x1600 panel @2x); also older '+ "Windows WXGA+ business monitors"+ ),+ weight=3.0,+ sources=(+ "https://gs.statcounter.com/screen-resolution-stats/desktop",+ "https://support.apple.com/en-us/111883",+ ),+ ),+ ScreenResolution(+ width=1920,+ height=1200,+ aspect="16:10",+ scenario=(+ "Linux laptop flagship: ThinkPad X1C/T14/P14s base, Dell XPS 13 "+ "Developer Edition base, System76 Lemur Pro/Pangolin/Oryx Pro, "+ "Framework 13, Lenovo/HP Chromebook Plus 14 (2025-26)"+ ),+ weight=5.0,+ sources=(+ "https://psref.lenovo.com/syspool/Sys/PDF/ThinkPad/ThinkPad_X1_Carbon_Gen_12/ThinkPad_X1_Carbon_Gen_12_Spec.pdf",+ "https://www.dell.com/support/manuals/en-us/xps-13-9340-laptop/xps-13-9340_owners_manual/display",+ "https://system76.com/laptops/lemur-pro",+ "https://news.lenovo.com/pressroom/press-releases/next-generation-hybrid-ai-chromebook-plus-14-10/",+ ),+ ),+]+++def sample_random_screen_resolution(+ rng: random.Random | None = None,+) -> tuple[int, int]:+ """Sample a (width, height) from POPULAR_SCREEN_RESOLUTIONS weighted by `weight`."""+ r = rng or random+ resolutions = [(s.width, s.height) for s in POPULAR_SCREEN_RESOLUTIONS]+ weights = [s.weight for s in POPULAR_SCREEN_RESOLUTIONS]+ return r.choices(resolutions, weights=weights, k=1)[0]diff --git a/src/fara/fara_7b/__init__.py b/src/fara/fara_7b/__init__.pynew file mode 100644index 0000000..dc7b761--- /dev/null+++ b/src/fara/fara_7b/__init__.py@@ -0,0 +1,5 @@+"""Fara-7B (previous generation) agent and prompts."""++from .fara_agent import FARA_ACTION_DEFINITIONS, FaraAgent++__all__ = ["FARA_ACTION_DEFINITIONS", "FaraAgent"]diff --git a/src/fara/fara_7b/_prompts.py b/src/fara/fara_7b/_prompts.pynew file mode 100644index 0000000..f43cef1--- /dev/null+++ b/src/fara/fara_7b/_prompts.py@@ -0,0 +1,222 @@+from typing import Union, Tuple++from ..qwen_helpers.base_tool import BaseTool+from ..qwen_helpers.fncall_prompt import NousFnCallPrompt+from ..qwen_helpers.schema import (+ ContentItem,+ Message,+)+from ..qwen_helpers.utils import smart_resize+++# @register_tool("computer_use")+class FaraComputerUse(BaseTool):+ name = "computer_use"++ @property+ def description(self):+ return f"""+Use a mouse and keyboard to interact with a computer, and take screenshots.+* This is an interface to a desktop GUI. You do not have access to a terminal or applications menu. You must click on desktop icons to start applications.+* Some applications may take time to start or process actions, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you click on Firefox and a window doesn't open, try wait and taking another screenshot.+* The screen's resolution is {self.display_width_px}x{self.display_height_px}.+* Whenever you intend to move the cursor to click on an element like an icon, you should consult a screenshot to determine the coordinates of the element before moving the cursor.+* If you tried clicking on a program or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click.+* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.+* When a separate scrollable container prominently overlays the webpage, if you want to scroll within it, you typically need to mouse_move() over it first and then scroll().+* If a popup window appears that you want to close, if left_click() on the 'X' or close button doesn't work, try key(keys=['Escape']) to close it.+* On some search bars, when you type(), you may need to press_enter=False and instead separately call left_click() on the search button to submit the search query. This is especially true of search bars that have auto-suggest popups for e.g. locations+* For calendar widgets, you usually need to left_click() on arrows to move between months and left_click() on dates to select them; type() is not typically used to input dates there.+""".strip()++ parameters = {+ "properties": {+ "action": {+ "description": """+The action to perform. The available actions are:+* `key`: Performs key down presses on the arguments passed in order, then performs key releases in reverse order. Includes "Enter", "Alt", "Shift", "Tab", "Control", "Backspace", "Delete", "Escape", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "PageDown", "PageUp", "Shift", etc.+* `type`: Type a string of text on the keyboard.+* `mouse_move`: Move the cursor to a specified (x, y) pixel coordinate on the screen.+* `left_click`: Click the left mouse button.+* `scroll`: Performs a scroll of the mouse scroll wheel.+* `visit_url`: Visit a specified URL.+* `web_search`: Perform a web search with a specified query.+* `history_back`: Go back to the previous page in the browser history.+* `pause_and_memorize_fact`: Pause and memorize a fact for future reference.+* `wait`: Wait specified seconds for the change to happen.+* `terminate`: Terminate the current task and report its completion status.+""".strip(),+ "enum": [+ "key",+ "type",+ "mouse_move",+ "left_click",+ "scroll",+ "visit_url",+ "web_search",+ "history_back",+ "pause_and_memorize_fact",+ "wait",+ "terminate",+ ],+ "type": "string",+ },+ "keys": {+ "description": "Required only by `action=key`.",+ "type": "array",+ },+ "text": {+ "description": "Required only by `action=type`.",+ "type": "string",+ },+ "press_enter": {+ "description": "Whether to press the Enter key after typing. Required only by `action=type`.",+ "type": "boolean",+ },+ "delete_existing_text": {+ "description": "Whether to delete existing text before typing. Required only by `action=type`.",+ "type": "boolean",+ },+ "coordinate": {+ "description": "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to. Required only by `action=left_click`, `action=mouse_move`, and `action=type`.",+ "type": "array",+ },+ "pixels": {+ "description": "The amount of scrolling to perform. Positive values scroll up, negative values scroll down. Required only by `action=scroll`.",+ "type": "number",+ },+ "url": {+ "description": "The URL to visit. Required only by `action=visit_url`.",+ "type": "string",+ },+ "query": {+ "description": "The query to search for. Required only by `action=web_search`.",+ "type": "string",+ },+ "fact": {+ "description": "The fact to remember for the future. Required only by `action=pause_and_memorize_fact`.",+ "type": "string",+ },+ "time": {+ "description": "The seconds to wait. Required only by `action=wait`.",+ "type": "number",+ },+ "status": {+ "description": "The status of the task. Required only by `action=terminate`.",+ "type": "string",+ "enum": ["success", "failure"],+ },+ },+ "required": ["action"],+ "type": "object",+ }++ def __init__(self, cfg=None):+ self.display_width_px = cfg["display_width_px"]+ self.display_height_px = cfg["display_height_px"]+ include_input_text_key_args = cfg.pop("include_input_text_key_args", False)+ if not include_input_text_key_args:+ self.parameters["properties"].pop("press_enter", None)+ self.parameters["properties"].pop("delete_existing_text", None)+ super().__init__(cfg)++ def call(self, params: Union[str, dict], **kwargs):+ params = self._verify_json_format_args(params)+ action = params["action"]+ if action == "key":+ return self._key(params["text"])+ elif action == "click":+ return self._click(coordinate=params["coordinate"])+ elif action == "long_press":+ return self._long_press(+ coordinate=params["coordinate"], time=params["time"]+ )+ elif action == "swipe":+ return self._swipe(+ coordinate=params["coordinate"], coordinate2=params["coordinate2"]+ )+ elif action == "type":+ return self._type(params["text"])+ elif action == "system_button":+ return self._system_button(params["button"])+ elif action == "open":+ return self._open(params["text"])+ elif action == "wait":+ return self._wait(params["time"])+ elif action == "terminate":+ return self._terminate(params["status"])+ else:+ raise ValueError(f"Unknown action: {action}")++ def _key(self, text: str):+ raise NotImplementedError()++ def _click(self, coordinate: Tuple[int, int]):+ raise NotImplementedError()++ def _long_press(self, coordinate: Tuple[int, int], time: int):+ raise NotImplementedError()++ def _swipe(self, coordinate: Tuple[int, int], coordinate2: Tuple[int, int]):+ raise NotImplementedError()++ def _type(self, text: str):+ raise NotImplementedError()++ def _system_button(self, button: str):+ raise NotImplementedError()++ def _open(self, text: str):+ raise NotImplementedError()++ def _wait(self, time: int):+ raise NotImplementedError()++ def _terminate(self, status: str):+ raise NotImplementedError()+++def get_computer_use_system_prompt(+ image,+ processor_im_cfg,+ include_input_text_key_args=False,+ fn_call_template="default",+):+ patch_size = processor_im_cfg["patch_size"]+ merge_size = processor_im_cfg["merge_size"]+ min_pixels = processor_im_cfg["min_pixels"]+ max_pixels = processor_im_cfg["max_pixels"]++ resized_height, resized_width = smart_resize(+ image.height,+ image.width,+ factor=patch_size * merge_size,+ min_pixels=min_pixels,+ max_pixels=max_pixels,+ )++ computer_use = FaraComputerUse(+ cfg={+ "display_width_px": resized_width,+ "display_height_px": resized_height,+ "include_input_text_key_args": include_input_text_key_args,+ }+ )++ conversation = NousFnCallPrompt(+ template_name=fn_call_template+ ).preprocess_fncall_messages(+ messages=[+ Message(+ role="system",+ content=[ContentItem(text="You are a helpful assistant.")],+ ),+ ],+ functions=[computer_use.function],+ lang=None,+ )++ return {+ "conversation": [msg.model_dump() for msg in conversation],+ "im_size": (resized_width, resized_height),+ }diff --git a/src/fara/fara_agent.py b/src/fara/fara_7b/fara_agent.pysimilarity index 99%rename from src/fara/fara_agent.pyrename to src/fara/fara_7b/fara_agent.pyindex 6fe00c9..5465b9c 100644--- a/src/fara/fara_agent.py+++ b/src/fara/fara_7b/fara_agent.py@@ -13,9 +13,9 @@ from tenacity import retry, stop_after_attempt, wait_exponential, before_sleep_lfrom playwright.async_api import Downloadfrom playwright.async_api import BrowserContextimport asyncio-from .browser.playwright_controller import PlaywrightController+from ..browser.playwright_controller import PlaywrightControllerfrom ._prompts import get_computer_use_system_prompt-from .fara_types import (+from ..fara_types import (LLMMessage,SystemMessage,UserMessage,@@ -26,7 +26,7 @@ from .fara_types import (message_to_openai_format,WebSurferEvent,)-from .utils import get_trimmed_url+from ..utils import get_trimmed_url# Canonical fara-1.0 action space mapping {action_name: set(arg_names)}.diff --git a/src/fara/qwen_helpers/fncall_prompt.py b/src/fara/qwen_helpers/fncall_prompt.pyindex 344b92e..83236a8 100644--- a/src/fara/qwen_helpers/fncall_prompt.py+++ b/src/fara/qwen_helpers/fncall_prompt.py@@ -10,12 +10,14 @@ from .schema import ASSISTANT, FUNCTION, SYSTEM, USER, ContentItem, Messageclass NousFnCallPrompt:- def __init__(self, template_name: str = "default"):+ def __init__(self, template_name: str = "default", template: str = None):"""Initialize NousFnCallPrompt with a specific template.Args:template_name: Name of the template to use. Options:"default", "qwen", "with_ci"+ template: Pre-built template string. If provided, overrides+ template_name lookup. Must contain {tool_descs} placeholder."""self.template_name = template_nameself.template_map = {@@ -23,8 +25,9 @@ class NousFnCallPrompt:"qwen": FN_CALL_TEMPLATE_QWEN,"with_ci": FN_CALL_TEMPLATE_WITH_CI,}+ self._custom_template = template- if template_name not in self.template_map:+ if template is None and template_name not in self.template_map:raise ValueError(f"Unknown template_name: {template_name}. "f"Available options: {list(self.template_map.keys())}"@@ -113,11 +116,13 @@ class NousFnCallPrompt:# Select template based on configurationif SPECIAL_CODE_MODE and any([CODE_TOOL_PATTERN in x for x in tool_names]):selected_template = FN_CALL_TEMPLATE_WITH_CI+ elif self._custom_template is not None:+ selected_template = self._custom_templateelse:selected_template = self.template_map[self.template_name]tool_system = selected_template.format(tool_descs=tool_descs)- if messages[0].role == SYSTEM:+ if len(messages) > 0 and messages[0].role == SYSTEM:messages[0].content.append(ContentItem(text="\n\n" + tool_system))else:messages = [diff --git a/src/fara/qwen_helpers/utils.py b/src/fara/qwen_helpers/utils.pyindex b67e442..3cc689f 100644--- a/src/fara/qwen_helpers/utils.py+++ b/src/fara/qwen_helpers/utils.py@@ -1,6 +1,7 @@# Source: https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/utils/utils.pyimport json+import mathimport refrom typing import Any@@ -8,6 +9,59 @@ from typing import AnyCHINESE_CHAR_RE = re.compile(r"[\u4e00-\u9fff]")+IMAGE_FACTOR = 28+MIN_PIXELS = 4 * 28 * 28+MAX_PIXELS = 16384 * 28 * 28+MAX_RATIO = 200+++def round_by_factor(number: int, factor: int) -> int:+ """Returns the closest integer to 'number' that is divisible by 'factor'."""+ return round(number / factor) * factor+++def ceil_by_factor(number: int, factor: int) -> int:+ """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""+ return math.ceil(number / factor) * factor+++def floor_by_factor(number: int, factor: int) -> int:+ """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""+ return math.floor(number / factor) * factor+++def smart_resize(+ height: int,+ width: int,+ factor: int = IMAGE_FACTOR,+ min_pixels: int = MIN_PIXELS,+ max_pixels: int = MAX_PIXELS,+) -> tuple[int, int]:+ """+ Rescales the image so that the following conditions are met:++ 1. Both dimensions (height and width) are divisible by 'factor'.++ 2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].++ 3. The aspect ratio of the image is maintained as closely as possible.+ """+ if max(height, width) / min(height, width) > MAX_RATIO:+ raise ValueError(+ f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"+ )+ h_bar = max(factor, round_by_factor(height, factor))+ w_bar = max(factor, round_by_factor(width, factor))+ if h_bar * w_bar > max_pixels:+ beta = math.sqrt((height * width) / max_pixels)+ h_bar = floor_by_factor(height / beta, factor)+ w_bar = floor_by_factor(width / beta, factor)+ elif h_bar * w_bar < min_pixels:+ beta = math.sqrt(min_pixels / (height * width))+ h_bar = ceil_by_factor(height * beta, factor)+ w_bar = ceil_by_factor(width * beta, factor)+ return h_bar, w_bar+def has_chinese_chars(data: Any) -> bool:text = f"{data}"diff --git a/src/fara/run_fara.py b/src/fara/run_fara.pyindex 9525828..367ff96 100755--- a/src/fara/run_fara.py+++ b/src/fara/run_fara.py@@ -1,42 +1,151 @@import asyncioimport argparseimport os-from .fara_agent import FaraAgent-from .browser.browser_bb import BrowserBB+import uuidimport loggingfrom typing import Dictfrom pathlib import Pathimport json+from .agents.fara.fara_qwen3_next import FaraQwen3NextAgent, FaraQwen3NextAgentConfig+from .environments.playwright import PlaywrightEnvironment+from .core.run_context import RunContext+from .core.data_point import Task, SolverStatus, UserMessage, UserMessageType+from .fara_7b import FaraAgent+from .browser.browser_bb import BrowserBB+-# Configure logging to only show logs from fara.fara_agent+# Configure logging to only show logs from the fara agentslogging.basicConfig(level=logging.CRITICAL,format="%(message)s",)-# Enable INFO level only for fara.fara_agent-fara_agent_logger = logging.getLogger("fara.fara_agent")-fara_agent_logger.setLevel(logging.INFO)--# Add a handler to ensure fara_agent logs are shown-handler = logging.StreamHandler()-handler.setLevel(logging.INFO)-handler.setFormatter(logging.Formatter("%(message)s"))-fara_agent_logger.addHandler(handler)-fara_agent_logger.propagate = False # Don't propagate to root logger+for _logger_name in ("fara.agents.fara.fara_qwen3", "fara.fara_7b.fara_agent"):+ _agent_logger = logging.getLogger(_logger_name)+ _agent_logger.setLevel(logging.INFO)+ _handler = logging.StreamHandler()+ _handler.setLevel(logging.INFO)+ _handler.setFormatter(logging.Formatter("%(message)s"))+ _agent_logger.addHandler(_handler)+ _agent_logger.propagate = False # Don't propagate to root loggerlogger = logging.getLogger(__name__)DEFAULT_ENDPOINT_CONFIG = {+ "model": "Fara1.5-9B",+ "base_url": "http://localhost:5000/v1",+ "api_key": "not-needed",+}++DEFAULT_FARA_7B_ENDPOINT_CONFIG = {"model": "microsoft/Fara-7B","base_url": "http://localhost:5000/v1","api_key": "not-needed",}-async def run_fara_agent(+async def run_fara15_agent(+ initial_task: str = None,+ endpoint_config: Dict[str, str] = None,+ start_page: str = "https://www.bing.com/",+ headless: bool = True,+ output_folder: str = None,+ save_screenshots: bool = True,+ max_rounds: int = 100,+ use_browser_base: bool = False,+) -> None:+ """Interactive loop for the Fara-1.5 agent (FaraQwen3NextAgent)."""+ output_root = Path(output_folder or "./fara_runs")++ print("Initializing Browser...")+ env = PlaywrightEnvironment(+ viewport_width=1440,+ viewport_height=900,+ headless=headless,+ browser_channel="chromium",+ start_page=start_page,+ single_tab_mode=True,+ use_browserbase=use_browser_base,+ )+ await env.initialize()+ print("Browser Running... Starting Fara-1.5 Agent...")++ agent = FaraQwen3NextAgent(+ FaraQwen3NextAgentConfig(+ client_config=endpoint_config,+ max_rounds=max_rounds,+ identity="fara_qwen35",+ critical_points="fara-1.5",+ save_screenshots=save_screenshots,+ )+ )++ try:+ task_text = initial_task+ first_round = True+ run_idx = 0++ while True:+ if task_text is None:+ if first_round:+ task_text = input("Enter task: ").strip()+ else:+ task_text = input(+ "\nEnter another task (or press Enter to exit): "+ ).strip()+ if not task_text:+ print("Exiting...")+ break++ print("##########################################")+ print(f"Task: {task_text}")+ print("##########################################")++ run_idx += 1+ output_dir = output_root / f"task_{run_idx}_{uuid.uuid4().hex[:8]}"+ task = Task(task_id=uuid.uuid4().hex, instruction=task_text)+ run_context = RunContext.create(+ environment=env, task=task, output_dir=output_dir+ )+ await agent.initialize(run_context)++ try:+ print("Running Fara...\n")+ final_answer, _, _ = await agent.run(run_context)++ # Critical-point resume loop: agent paused to ask the user.+ while (+ run_context.solver_log.status == SolverStatus.WAITING_FOR_USER+ ):+ print(f"\nFara asks: {final_answer}")+ reply = input("Your response (Enter to abandon): ").strip()+ if not reply:+ break+ run_context.add_observation(+ UserMessage(+ content=reply,+ message_type=UserMessageType.CRITICAL_POINT_RESPONSE,+ )+ )+ final_answer, _, _ = await agent.run(run_context)++ if run_context.solver_log.status == SolverStatus.COMPLETE:+ print(f"\nFinal Answer: {final_answer}")+ print(f"Trajectory saved to: {output_dir}")+ except Exception as e:+ print(f"Error occurred: {e}")+ finally:+ await agent.close(run_context)++ task_text = None+ first_round = False+ finally:+ await env.close()+++async def run_fara7b_agent(initial_task: str = None,endpoint_config: Dict[str, str] = None,start_page: str = "https://www.bing.com/",@@ -45,8 +154,8 @@ async def run_fara_agent(save_screenshots: bool = True,max_rounds: int = 100,use_browser_base: bool = False,-):- # Initialize browser manager+) -> None:+ """Interactive loop for the previous-generation Fara-7B agent."""print("Initializing Browser...")browser_manager = BrowserBB(headless=headless,@@ -62,7 +171,7 @@ async def run_fara_agent(use_browser_base=use_browser_base,logger=logger,)- print("Browser Running... Starting Fara Agent...")+ print("Browser Running... Starting Fara-7B Agent...")agent = FaraAgent(browser_manager=browser_manager,@@ -75,11 +184,8 @@ async def run_fara_agent(try:await agent.initialize()-- # Interactive looptask = initial_taskfirst_round = True-while True:if task is None:if first_round:@@ -88,7 +194,6 @@ async def run_fara_agent(task = input("\nEnter another task (or press Enter to exit): ").strip()-if not task:print("Exiting...")break@@ -96,18 +201,15 @@ async def run_fara_agent(print("##########################################")print(f"Task: {task}")print("##########################################")-try:print("Running Fara...\n")- final_answer, all_actions, all_observations = await agent.run(task)+ final_answer, _, _ = await agent.run(task)print(f"\nFinal Answer: {final_answer}")except Exception as e:print(f"Error occurred: {e}")task = Nonefirst_round = False-finally:- # Close the agent and browserawait agent.close()@@ -132,10 +234,10 @@ def main():help="Run the browser in headful mode (show GUI, default is headless)",)parser.add_argument(- "--downloads_folder",+ "--output_folder",type=str,default=None,- help="Folder to save screenshots and downloads",+ help="Folder to save screenshots, downloads and the trajectory data_point.json",)parser.add_argument("--save_screenshots",@@ -177,6 +279,12 @@ def main():default=None,help="Model name to use (overrides endpoint_config)",)+ parser.add_argument(+ "--fara-7b",+ action="store_true",+ dest="fara_7b",+ help="Run the previous-generation Fara-7B agent instead of Fara-1.5",+ )args = parser.parse_args()@@ -188,7 +296,9 @@ def main():"BROWSERBASE_PROJECT_ID"), "BROWSERBASE_API_KEY and BROWSERBASE_PROJECT_ID environment variables must be set to use browserbase"- endpoint_config = DEFAULT_ENDPOINT_CONFIG+ endpoint_config = (+ DEFAULT_FARA_7B_ENDPOINT_CONFIG if args.fara_7b else DEFAULT_ENDPOINT_CONFIG+ )if args.endpoint_config:with open(args.endpoint_config, "r") as f:endpoint_config = json.load(f)@@ -197,7 +307,6 @@ def main():and "base_url" in endpoint_configand "model" in endpoint_config), "endpoint_config file must contain api_key, base_url, and model fields"- # Override with command-line arguments if providedif args.api_key:endpoint_config["api_key"] = args.api_keyif args.base_url:@@ -205,18 +314,32 @@ def main():if args.model:endpoint_config["model"] = args.model- asyncio.run(- run_fara_agent(- initial_task=args.task,- endpoint_config=endpoint_config,- start_page=args.start_page,- headless=not args.headful,- downloads_folder=args.downloads_folder,- save_screenshots=args.save_screenshots,- max_rounds=args.max_rounds,- use_browser_base=args.browserbase,+ if args.fara_7b:+ asyncio.run(+ run_fara7b_agent(+ initial_task=args.task,+ endpoint_config=endpoint_config,+ start_page=args.start_page,+ headless=not args.headful,+ downloads_folder=args.output_folder,+ save_screenshots=args.save_screenshots,+ max_rounds=args.max_rounds,+ use_browser_base=args.browserbase,+ )+ )+ else:+ asyncio.run(+ run_fara15_agent(+ initial_task=args.task,+ endpoint_config=endpoint_config,+ start_page=args.start_page,+ headless=not args.headful,+ output_folder=args.output_folder,+ save_screenshots=args.save_screenshots,+ max_rounds=args.max_rounds,+ use_browser_base=args.browserbase,+ ))- )if __name__ == "__main__":diff --git a/src/fara/vllm/requirements.txt b/src/fara/vllm/requirements.txtindex f78d6a8..7dfc26d 100644--- a/src/fara/vllm/requirements.txt+++ b/src/fara/vllm/requirements.txt@@ -1,4 +1,7 @@uvicornfastapi-vllm==0.10.0-transformers==4.55.0\ No newline at end of file+vllm==0.19.1+transformers>=5.2.0+# 4.6.0.dev0 removed the `cute.core.ThrMma` alias that quack (a transitive+# vLLM dep) uses, crashing vLLM at startup. Unpin once resolved upstream.+nvidia-cutlass-dsl==4.5.2diff --git a/tests/test_fara15.py b/tests/test_fara15.pynew file mode 100644index 0000000..31b87c8--- /dev/null+++ b/tests/test_fara15.py@@ -0,0 +1,143 @@+"""Tests for the Fara-1.5 agent port: prompts, action space, parsing, DataPoint."""++import json++import pytest+from PIL import Image++from fara import (+ FARA15_ACTION_DEFINITIONS,+ FARA_ACTION_DEFINITIONS,+ Fara15Agent,+ FaraAgent,+)+from fara._prompts import (+ CRITICAL_POINTS_REGISTRY,+ FaraBrowserComputerUse,+ IDENTITY_REGISTRY,+ get_computer_use_system_prompt,+)+from fara.data_point import (+ Action,+ ComputerObservation,+ DataPoint,+ SolverStatus,+ Task,+ ToolOutput,+)+++def build_prompt(**kwargs):+ image = Image.new("RGB", (1440, 900))+ return get_computer_use_system_prompt(+ image, Fara15Agent.MLM_PROCESSOR_IM_CFG, display_size=1000, **kwargs+ )+++def prompt_text(out):+ return "".join(c["text"] for msg in out["conversation"] for c in msg["content"])+++def test_action_definitions_match_tool_schema():+ schema_actions = FaraBrowserComputerUse.parameters["properties"]["action"]["enum"]+ assert set(schema_actions) == set(FARA15_ACTION_DEFINITIONS)+ # Every non-action argument referenced by the definitions exists in the schema+ schema_args = set(FaraBrowserComputerUse.parameters["properties"]) - {"action"}+ for action, arg_names in FARA15_ACTION_DEFINITIONS.items():+ assert arg_names <= schema_args, action+++def test_system_prompt_contents():+ out = build_prompt()+ text = prompt_text(out)+ assert len(out["conversation"]) == 1+ assert out["conversation"][0]["role"] == "system"+ assert text.startswith("You are Fara, a computer use agent")+ assert "Qwen3.5-9B" in text+ assert "A critical point is a situation" in text+ assert "The screen's resolution is 1000x1000" in text+ assert "<tool_call>" in text+ for action in FARA15_ACTION_DEFINITIONS:+ assert f"`{action}`" in text+ # 1440x900 resized for patch 16 / merge 2 (factor 32)+ assert out["im_size"] == (1440, 896)+++def test_system_prompt_identity_variants():+ out = build_prompt(identity="fara_qwen3vl")+ assert "Qwen3-VL-8B-Instruct" in prompt_text(out)+ with pytest.raises(ValueError):+ build_prompt(identity="bogus")+ with pytest.raises(ValueError):+ build_prompt(critical_points="bogus")+ assert set(IDENTITY_REGISTRY) == {"fara_qwen35", "fara_qwen3vl"}+ assert set(CRITICAL_POINTS_REGISTRY) == {"fara-1.5"}+++def test_fara15_action_space_is_superset_of_fara7b():+ # fara-1.0 space minus renamed/removed args is contained in fara-1.5+ assert set(FARA_ACTION_DEFINITIONS) - {"terminate"} <= set(+ FARA15_ACTION_DEFINITIONS+ )+ # terminate changed: status enum -> free-form answer+ assert FARA15_ACTION_DEFINITIONS["terminate"] == {"answer"}+++def test_parse_thoughts_and_action():+ agent = Fara15Agent.__new__(Fara15Agent)+ import logging++ agent.logger = logging.getLogger("test")+ action_json = json.dumps(+ {+ "name": "computer_use",+ "arguments": {"action": "left_click", "coordinate": [500, 500]},+ }+ )+ message = f"I should click the button.\n<tool_call>\n{action_json}\n</tool_call>"+ thoughts, action = agent._parse_thoughts_and_action(message)+ assert thoughts == "I should click the button."+ assert action["arguments"]["action"] == "left_click"+++def test_proc_coords_scales_from_display_space_to_viewport():+ agent = Fara15Agent.__new__(Fara15Agent)+ agent.viewport_width = 1440+ agent.viewport_height = 900+ assert agent.proc_coords([500, 500]) == [720.0, 450.0]+ assert agent.proc_coords([1000, 1000]) == [1440.0, 900.0]+ assert agent.proc_coords(None) is None+++def test_data_point_roundtrip(tmp_path):+ dp = DataPoint(task=Task(task_id="t1", instruction="find a hotel"))+ dp.solver_log.add_observation(+ ComputerObservation(screenshot_path="screenshot0.png", url="https://bing.com")+ )+ action = Action(+ action_name="left_click",+ content={"action": "left_click", "arguments": {"coordinate": [1, 2]}},+ )+ dp.solver_log.add_action(action)+ dp.solver_log.add_observation(+ ToolOutput(output="I clicked at coordinates (1, 2).", action_id=action.id)+ )+ dp.solver_log.status = SolverStatus.COMPLETE++ path = tmp_path / "data_point.json"+ dp.save(path)+ loaded = DataPoint.load(path)+ assert loaded.task.instruction == "find a hotel"+ assert loaded.solver_log.status == SolverStatus.COMPLETE+ steps = loaded.solver_log.steps()+ assert len(steps) == 1+ assert steps[0].action.action_name == "left_click"+ assert steps[0].observations_post.main[0].output == "I clicked at coordinates (1, 2)."+++def test_fara7b_agent_still_importable():+ assert FaraAgent.__name__ == "FaraAgent"+ assert "terminate" in FARA_ACTION_DEFINITIONS+ from fara.fara_7b import FaraAgent as Fara7B++ assert Fara7B is FaraAgent