Skip to content

armgit / fara

Public Imported from GitHub
0

Fara1.5 – A family of frontier computer use agent models

comments

Hussein Mozannar8e33bdec69ec
diff --git a/src/fara/_prompts.py b/src/fara/_prompts.py
index 5400760..4526b5b 100644
--- a/src/fara/_prompts.py
+++ b/src/fara/_prompts.py
@@ -10,9 +10,6 @@ from .qwen_helpers.base_tool import BaseTool
from .qwen_helpers.fncall_prompt import NousFnCallPrompt
from .qwen_helpers.utils import smart_resize
-# ============================================================
-# Model identities
-# ============================================================
FARA_QWEN35_IDENTITY = """\
You are Fara, a computer use agent (CUA) specialized for web browsers. \
You are developed by Microsoft AI Frontiers. You assist users with \
@@ -45,9 +42,6 @@ This edition of the model was trained using SFT on top of \
Qwen3-VL-8B-Instruct, using a synthetic data mixture generated and \
developed by Microsoft AI Frontiers."""
-# ============================================================
-# Critical points
-# ============================================================
CRITICAL_POINTS_FARA_1_5 = """\
A critical point is a situation where we must pause and request information or confirmation from the user before \
proceeding. There are three types:
@@ -83,10 +77,6 @@ CRITICAL_POINTS_REGISTRY = {
"fara-1.5": CRITICAL_POINTS_FARA_1_5,
}
-# NOTE: {tool_descs} is the ONLY placeholder — it is filled by
-# NousFnCallPrompt with the computer_use tool JSON schema (generated
-# dynamically per-image for screen resolution). All other braces must be
-# doubled to survive Python's str.format().
FN_CALL_FORMAT = (
"You are provided with function signatures within <tools></tools> XML tags:\n"
"<tools>\n"
diff --git a/src/fara/agents/computer_agent/utils.py b/src/fara/agents/computer_agent/utils.py
index 545a556..35963fc 100644
--- a/src/fara/agents/computer_agent/utils.py
+++ b/src/fara/agents/computer_agent/utils.py
@@ -33,14 +33,12 @@ def compact_messages(
max_prompt_tokens: Target token budget. <=0 disables compaction.
min_keep_rounds: Minimum recent rounds to always retain.
"""
- # TODO: this is not maybe the best compacting messages strategy
if max_prompt_tokens <= 0:
return messages
token_count = client.count_tokens(messages, tools)
if token_count <= max_prompt_tokens:
return messages
- # Identify round triplet start indices (skip system[0] and initial user[1])
round_starts = []
idx = 2
while idx + 2 < len(messages):
@@ -57,7 +55,6 @@ def compact_messages(
if not removable:
return messages
- # Remove from the middle outward
mid = len(removable) // 2
removal_order = sorted(range(len(removable)), key=lambda i: abs(i - mid))
diff --git a/src/fara/agents/coord_spaces.py b/src/fara/agents/coord_spaces.py
index 03cb56f..9cd96e5 100644
--- a/src/fara/agents/coord_spaces.py
+++ b/src/fara/agents/coord_spaces.py
@@ -12,13 +12,6 @@ failure-analysis workflow — can import it without pulling in the
heavyweight agent implementations.
"""
-# Fara family (FaraQwen3Agent, FaraQwen3NextAgent): emits coords in
-# [0, FARA_DISPLAY_SIZE); the agent's inference loop rescales to
-# viewport pixels via ``proc_coords`` before clicking.
FARA_DISPLAY_SIZE: int = 1000
-# Qwen35Agent in default ``coordinate_type="relative"`` mode: emits
-# coords in [0, QWEN35_DISPLAY_SIZE]. Note the off-by-one vs Fara — the
-# divisor used in :func:`Qwen35Agent.adjust_coordinates` is 999, not
-# 1000.
QWEN35_DISPLAY_SIZE: int = 999
diff --git a/src/fara/agents/fara/fara_qwen3.py b/src/fara/agents/fara/fara_qwen3.py
index 5e9ffe8..10fd5a5 100644
--- a/src/fara/agents/fara/fara_qwen3.py
+++ b/src/fara/agents/fara/fara_qwen3.py
@@ -100,7 +100,6 @@ class FaraQwen3AgentConfig(AgentConfig):
raise_on_captcha_timeout: bool = True
terminate_on_parse_error: bool = False
image_token_estimate: int = 1500
- # When >0, drop oldest screenshots until the estimated prompt is <= this cap.
image_budget_token_cap: int = 0
@@ -141,9 +140,6 @@ class FaraQwen3Agent(Agent):
self._pending_observation: str = ""
self._os_type: str = "linux"
self._output_dir = None
- # Captcha-wait disable: after `config.captcha_timeout_limit`
- # consecutive captcha-wait timeouts in a single run, stop waiting
- # for captchas at all. A limit of 0 disables the gate up front.
self._captcha_timeouts: int = 0
self._captcha_disabled: bool = self.config.captcha_timeout_limit <= 0
@@ -191,12 +187,10 @@ class FaraQwen3Agent(Agent):
pending_user_response = ""
if self._state.chat_history:
- # Continuation after WAITING_FOR_USER
traj.status = SolverStatus.RUNNING
pending_user_response = traj.get_last_user_message() or ""
start_step = self._state.current_step
else:
- # Fresh start
task_instruction = run_context.task.instruction
scaled_screenshot = await self._get_scaled_screenshot(env)
await self._save_screenshot(
diff --git a/src/fara/agents/fara/fara_qwen3_next.py b/src/fara/agents/fara/fara_qwen3_next.py
index 9880c04..802821a 100644
--- a/src/fara/agents/fara/fara_qwen3_next.py
+++ b/src/fara/agents/fara/fara_qwen3_next.py
@@ -219,7 +219,6 @@ class FaraQwen3NextAgent(FaraQwen3Agent):
) -> tuple[bool, str]:
"""Dispatch action using ComputerEnvironment canonical methods."""
- # -- GUI actions --
if action_type == "left_click":
tgt_x, tgt_y = args["coordinate"]
@@ -280,7 +279,6 @@ class FaraQwen3NextAgent(FaraQwen3Agent):
await env.hscroll(pixels)
return False, f"I scrolled horizontally by {pixels} pixels."
- # -- Browser-only actions --
elif action_type == "visit_url":
url = args.get("url", "")
@@ -315,7 +313,6 @@ class FaraQwen3NextAgent(FaraQwen3Agent):
answer = await extract_from_page(markdown, question, self._client)
return False, f"I read the page to answer: {question}\nAnswer: {answer}"
- # -- Windows-only actions --
elif action_type == "run_command":
command = str(args.get("command", ""))
@@ -337,7 +334,6 @@ class FaraQwen3NextAgent(FaraQwen3Agent):
)
return False, f"I executed command: {command}\nOutput: {result}"
- # -- Shared non-GUI actions --
elif action_type == "ask_user_question":
question = str(args.get("question", ""))
diff --git a/src/fara/agents/fara/prompts.py b/src/fara/agents/fara/prompts.py
index f536e23..e64f2e3 100644
--- a/src/fara/agents/fara/prompts.py
+++ b/src/fara/agents/fara/prompts.py
@@ -6,7 +6,9 @@ identities, the critical-points block, and the fn-call template). Only the
browser tool set is supported here.
"""
-from ..._prompts import get_computer_use_system_prompt # noqa: F401
+from ..._prompts import get_computer_use_system_prompt
+
+__all__ = ["get_computer_use_system_prompt", "TOOL_SET_TO_MODE"]
TOOL_SET_TO_MODE = {
"BROWSER_TOOLS_CORE": "fara_next_browser",
diff --git a/src/fara/agents/utils.py b/src/fara/agents/utils.py
index b8bec8c..a1503fa 100644
--- a/src/fara/agents/utils.py
+++ b/src/fara/agents/utils.py
@@ -53,7 +53,6 @@ def save_and_truncate_command_output(
if max_chars <= 0 or len(output) <= max_chars:
return output
- # Save to host output_dir for post-run inspection
if output_dir:
host_dir = output_dir / CMD_OUTPUT_DIR
os.makedirs(host_dir, exist_ok=True)
diff --git a/src/fara/clients/messages.py b/src/fara/clients/messages.py
index 77c26ef..2cdd093 100644
--- a/src/fara/clients/messages.py
+++ b/src/fara/clients/messages.py
@@ -126,12 +126,10 @@ def message_to_openai_format(message: LLMMessage) -> Dict[str, Any]:
else "user"
)
- # Handle multimodal content (text + images)
if isinstance(message.content, list):
content_parts = []
for item in message.content:
if isinstance(item, ImageObj):
- # Convert image to base64 data URL
base64_image = item.to_base64()
content_parts.append(
{
diff --git a/src/fara/core/data_point.py b/src/fara/core/data_point.py
index 8028550..b688e57 100644
--- a/src/fara/core/data_point.py
+++ b/src/fara/core/data_point.py
@@ -46,11 +46,6 @@ class Component(BaseModel):
class SolverStatus(str, Enum):
RUNNING = "running"
COMPLETE = "complete"
- # Loop ended without a clean terminate. TIMED_OUT / MAX_ROUNDS name the two
- # structured cut-offs; ABORTED is the catch-all for any other abnormal end
- # (e.g. an agent crash). The cause is machine-checkable on the status alone,
- # without a free-text field. (LLM-call finish reasons live on
- # LLMMessage.finish_reason.)
ABORTED = "aborted"
TIMED_OUT = "timed_out"
MAX_ROUNDS = "max_rounds"
@@ -68,11 +63,6 @@ class LLMUsage(Component):
class LLMMessage(Component):
"""A single LLM call: request/response pair."""
- # Model that served the call (resolved through the graceful_retry wrapper),
- # used for per-message cost pricing. ``label`` tags the call with its
- # pipeline step / attempt / endpoint (e.g. "step2_relevance",
- # "attempt2") so a flat conversation reconstructs per-step and
- # per-endpoint breakdowns by grouping.
model: Optional[str] = None
label: str = ""
raw_response: str = ""
@@ -336,12 +326,11 @@ class SolverLog(Component):
d[key] = []
d[key].append(obs)
- # Pass 1: build steps with actions; collect all observations with action_id
result: List[Step] = []
- current_pre: List[Observation] = [] # action_id=None only
+ current_pre: List[Observation] = []
post_obs: List[
tuple
- ] = [] # (obs, step_index_after) for observations with action_id
+ ] = []
for event in self.events:
if isinstance(event, Action):
result.append(
@@ -355,11 +344,9 @@ class SolverLog(Component):
else:
post_obs.append((event, len(result)))
- # Trailing observations with no action_id
if with_tail_observations and current_pre:
result.append(Step(observations_pre=ObservationsPre(main=current_pre)))
- # Pass 2: populate observations_post and observations_pre.late
action_id_to_step_idx: Dict[str, int] = {}
for i, step in enumerate(result):
if step.action is not None and step.action.id is not None:
@@ -382,7 +369,6 @@ class SolverLog(Component):
result[next_step_idx].observations_pre.late, obs.action_id, obs
)
- # Pass 3: link previous_post and next_pre
for i in range(1, len(result)):
result[i].observations_pre.previous_post = result[
i - 1
@@ -511,11 +497,6 @@ class VerificationResult(Component):
reasoning: Optional[str] = None
verifier_name: str
metadata: Dict[str, Any] = Field(default_factory=dict)
- # Every LLM call this verifier made, recorded as it happened via
- # core.usage.record_call. Single source for verifier token usage + cost
- # (core.usage.conversation_cost); replaces the old per-endpoint snapshot
- # stats. Per-step / per-endpoint breakdowns come from grouping messages on
- # their .label / .model.
llm_conversation: LLMConversation = Field(default_factory=LLMConversation)
@property
@@ -525,9 +506,6 @@ class VerificationResult(Component):
return self.score >= 0.5
-# The solver/agent only ever produces the generic VerificationResult; the
-# specialized verifier-pipeline result schemas from agento_next are not part
-# of this inference repo.
VerificationResultEvent = VerificationResult
diff --git a/src/fara/core/data_point_io.py b/src/fara/core/data_point_io.py
index 1f3fa4d..a1c9755 100644
--- a/src/fara/core/data_point_io.py
+++ b/src/fara/core/data_point_io.py
@@ -113,8 +113,6 @@ class DataPointWriter:
return self._data_point
def add_event(self, event: Union["ObservationEvent", Action]) -> DataPoint:
- # Write to disk first so the log captures every event,
- # including ones that cause the in-memory model to reject.
self._solver_log_dir.mkdir(parents=True, exist_ok=True)
with open(self._events_path, "a") as f:
f.write(json.dumps(event.to_dict()) + "\n")
@@ -160,10 +158,8 @@ class DataPointReader:
def read(input_dir: str | os.PathLike) -> DataPoint:
d = Path(input_dir)
- # Task (required)
task = Task.from_dict(_load_json(d / _TASK_FILE))
- # Metadata
meta_path = d / _METADATA_FILE
metadata = (
DataPointMetadata.from_dict(_load_json(meta_path))
@@ -171,7 +167,6 @@ class DataPointReader:
else DataPointMetadata()
)
- # Solver log
solver_log = SolverLog()
solver_log_dir = d / _SOLVER_LOG_DIR
@@ -195,7 +190,6 @@ class DataPointReader:
if status_data.get("outcome") is not None:
solver_log.outcome = Outcome.from_dict(status_data["outcome"])
- # Verification
verification: Dict[str, VerificationResult] = {}
verif_path = d / _VERIFICATION_FILE
if verif_path.exists():
diff --git a/src/fara/environments/computer.py b/src/fara/environments/computer.py
index e5cc80e..18f3eff 100644
--- a/src/fara/environments/computer.py
+++ b/src/fara/environments/computer.py
@@ -52,13 +52,12 @@ class ComputerEnvironment(Environment):
async def save_state(self) -> dict:
"""Persist the resolved viewport so prepro gets the actual (possibly
randomized) dims rather than the raw seed values."""
- cfg: ComputerEnvironmentConfig = self.config # type: ignore[assignment]
+ cfg: ComputerEnvironmentConfig = self.config
return {
"viewport_width": cfg.viewport_width,
"viewport_height": cfg.viewport_height,
}
- # --- Core GUI actions (required) ---
@abstractmethod
async def left_click(self, x: int, y: int) -> None: ...
@@ -117,7 +116,6 @@ class ComputerEnvironment(Environment):
) from e
await asyncio.sleep(delay)
- # --- Extended actions (optional) ---
async def middle_click(self, x: int, y: int) -> None:
raise NotImplementedError
diff --git a/src/fara/environments/playwright/environment.py b/src/fara/environments/playwright/environment.py
index b73dcb3..d89f922 100644
--- a/src/fara/environments/playwright/environment.py
+++ b/src/fara/environments/playwright/environment.py
@@ -61,9 +61,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
os_type = OSType.LINUX
- # Browser-chrome shortcuts that Playwright can't trigger via page.keyboard
- # (they affect browser UI, not the page). Dispatched to env methods so the
- # agent can "press the shortcut like on a real computer" and it just works.
_BROWSER_CHROME_DISPATCH: Dict[frozenset, str] = {
frozenset({"Control", "r"}): "refresh",
frozenset({"Control", "Shift", "r"}): "refresh",
@@ -101,14 +98,11 @@ class PlaywrightEnvironment(BrowserEnvironment):
self._controller: PlaywrightController | None = None
- # BrowserBase support
self._bb = None
self._session = None
self._captcha_event = asyncio.Event()
- self._captcha_event.set() # Initially set (no captcha)
+ self._captcha_event.set()
- # Optional task identifier — prepended to BB log lines for
- # session<->task linkage in eval logs. Set via initialize(task_id=...).
self._task_id: str | None = None
def _tag(self) -> str:
@@ -140,9 +134,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
)
if self.config.use_browserbase:
- # _init_browserbase runs _setup_browser inside its retry loop so
- # a start_page navigation failure or stuck captcha triggers a
- # fresh session instead of bubbling up.
await self._init_browserbase()
elif self.config.browser_data_dir:
await self._init_persistent_browser()
@@ -231,9 +222,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
async def _connect_browserbase_once(self, project_id: str) -> None:
"""Run one BrowserBase bring-up: session, CDP, setup, captcha wait."""
- # Clean slate: a prior attempt may have left _captcha_event cleared
- # if it saw "solving-started" but the session died before
- # "solving-finished" — the dead session can never set it.
self._captcha_event.set()
self._session = self._bb.sessions.create(
@@ -261,18 +249,10 @@ class PlaywrightEnvironment(BrowserEnvironment):
assert len(self._context.pages) == 1
self._page = self._context.pages[0]
- # Context-level "console" already bubbles up page console messages
- # for every current and future page in the context, so we don't
- # also register on _page.
self._context.on("console", self._handle_browserbase_console)
- # Navigate to start_page + wire handlers. Failures here trigger
- # a retry with a fresh session.
await self._setup_browser()
- # Best-effort wait for a captcha triggered on start_page.
- # _captcha_event is already set when no captcha is active, so
- # this returns immediately in the common case.
try:
await asyncio.wait_for(
self._captcha_event.wait(),
@@ -298,11 +278,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
try:
await asyncio.sleep(3)
try:
- # Prefer "domcontentloaded" over "networkidle" — the latter
- # can hang on sites with persistent network activity
- # (analytics, websockets). If captcha triggered a
- # navigation, this waits for the new DOM; otherwise it
- # returns immediately.
await self._page.wait_for_load_state("domcontentloaded", timeout=10000)
except Exception as e:
self.logger.warning(
@@ -339,8 +314,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
)
except Exception:
pass
- # Single canonical terminal line per session — grep with [BB-END]
- # to reconstruct session<->task<->status without parsing tricks.
self.logger.info(
f"[BB-END] task={self._task_id} session={session_id} "
f"browser_connected_at_teardown={browser_connected}"
@@ -358,10 +331,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
)
self._page = await self._context.new_page()
- # Hacky, brittle, undocumented: this cookie makes Bing serve search
- # results as same-tab links instead of target="_blank", which skips
- # the per-click popup-capture overhead on bing.com. Reverse-engineered
- # empirically; Microsoft can change it anytime. See PR #1114.
_BING_SAME_TAB_COOKIE = {
"name": "SRCHHPGUSR",
"value": "EXLKNT=0",
@@ -388,8 +357,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
):
await self._page.add_init_script(path=self.config.page_script_path)
- # Inject Bing same-tab cookie before navigation so it applies to
- # the very first bing.com load. See _BING_SAME_TAB_COOKIE above.
try:
await self._context.add_cookies([self._BING_SAME_TAB_COOKIE])
except Exception as e:
@@ -485,7 +452,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
"""Get screenshot of current page."""
return await self.get_screenshot()
- # --- Core GUI actions (ComputerEnvironment interface) ---
async def left_click(self, x: int, y: int) -> None:
new_page = await self._controller.click_coords(self._page, x, y)
@@ -535,7 +501,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
"""Capture a screenshot of the current page."""
return await self._controller.get_screenshot(self._page, path=path)
- # --- Browser navigation (BrowserEnvironment interface) ---
async def goto_url(self, url: str) -> None:
await self._controller.visit_page(self._page, url)
@@ -546,7 +511,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
async def refresh(self) -> None:
await self._page.reload(wait_until="commit")
- # --- Extended actions ---
async def middle_click(self, x: int, y: int) -> None:
await self._page.mouse.click(x, y, button="middle")
@@ -570,7 +534,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
"""Horizontal scroll. Positive=right, negative=left."""
await self._page.mouse.wheel(pixels, 0)
- # --- Backward-compat aliases ---
async def click(self, x: float, y: float) -> Dict[str, Any]:
"""Click at coordinates."""
@@ -612,7 +575,6 @@ class PlaywrightEnvironment(BrowserEnvironment):
await self._controller.page_up(self._page, amount=amount)
return {"success": True}
- # --- Playwright-specific methods (id-based, CUA, etc.) ---
async def click_id(self, identifier: str) -> Dict[str, Any]:
"""Click on an element by its identifier."""
diff --git a/src/fara/environments/playwright/playwright_controller.py b/src/fara/environments/playwright/playwright_controller.py
index 6b002fb..f98e783 100644
--- a/src/fara/environments/playwright/playwright_controller.py
+++ b/src/fara/environments/playwright/playwright_controller.py
@@ -47,12 +47,11 @@ def handle_target_closed(max_retries: int = 2, timeout_secs: int = 30):
def decorator(func: F) -> F:
@functools.wraps(func)
async def wrapper(*args, **kwargs):
- # Extract the page object - assume it's the first argument after self
logger = args[0].logger
page = None
if len(args) >= 2 and hasattr(
args[1], "url"
- ): # Check if second arg looks like a Page
+ ):
page = args[1]
retries = 0
@@ -62,14 +61,12 @@ def handle_target_closed(max_retries: int = 2, timeout_secs: int = 30):
try:
return await func(*args, **kwargs)
except (TargetClosedError, PlaywrightError) as e:
- # Check if this is a tunnel connection error
is_tunnel_error = "net::ERR_TUNNEL_CONNECTION_FAILED" in str(e)
is_target_closed = isinstance(
e, TargetClosedError
) or "Target page, context or browser has been closed" in str(e)
if not (is_tunnel_error or is_target_closed):
- # Not an error we handle, re-raise
raise e
last_error = e
@@ -79,7 +76,6 @@ def handle_target_closed(max_retries: int = 2, timeout_secs: int = 30):
raise e
if page is None:
- # Can't recover without page reference
raise e
error_type = (
@@ -90,16 +86,12 @@ def handle_target_closed(max_retries: int = 2, timeout_secs: int = 30):
)
try:
- # Attempt to recover the page
await _recover_page(page, timeout_secs, logger)
- # Small delay before retry
await asyncio.sleep(0.5)
except Exception as recovery_error:
logger.error(f"Page recovery failed: {recovery_error}")
- # If recovery fails, raise the original error
raise e from recovery_error
- # This shouldn't be reached, but just in case
raise last_error
return wrapper
@@ -117,17 +109,11 @@ async def _recover_page(page: Page, timeout_secs: int = 30, logger=None) -> None
"""
logger = logger or logging.getLogger("playwright_controller")
try:
- # First, try to check if the page is still responsive
await page.evaluate("1", timeout=1000)
- # If we get here, the page is actually fine
return
except Exception:
- # Page is indeed problematic, attempt recovery
pass
- # If the underlying browser is gone (BB session died, CDP dropped),
- # reload/goto on this page will both fail with the same TargetClosed —
- # ~60s wasted per recovery. Bail out early.
browser = page.context.browser if page.context else None
if browser is not None and not browser.is_connected():
raise Exception(
@@ -135,20 +121,16 @@ async def _recover_page(page: Page, timeout_secs: int = 30, logger=None) -> None
)
try:
- # Stop any ongoing navigation
await page.evaluate("window.stop()", timeout=2000)
except Exception:
- # Ignore errors from window.stop()
pass
try:
- # Try to reload the page
await page.reload(timeout=timeout_secs * 1000, wait_until="commit")
logger.info("playwright_controller._recover_page(): Page recovery successful")
except Exception as e:
logger.error(f"playwright_controller._recover_page(): Page reload failed: {e}")
- # Try alternative recovery: navigate to current URL
try:
current_url = page.url
if current_url and current_url != "about:blank":
@@ -170,7 +152,6 @@ async def _recover_page(page: Page, timeout_secs: int = 30, logger=None) -> None
)
-# Enhanced version that can handle browser context recreation
def handle_target_closed_with_context(max_retries: int = 2, timeout_secs: int = 30):
"""
Enhanced decorator that can also handle browser context recreation.
@@ -192,14 +173,12 @@ def handle_target_closed_with_context(max_retries: int = 2, timeout_secs: int =
try:
return await func(*args, **kwargs)
except (TargetClosedError, PlaywrightError) as e:
- # Check if this is a tunnel connection error
is_tunnel_error = "net::ERR_TUNNEL_CONNECTION_FAILED" in str(e)
is_target_closed = isinstance(
e, TargetClosedError
) or "Target page, context or browser has been closed" in str(e)
if not (is_tunnel_error or is_target_closed):
- # Not an error we handle, re-raise
raise e
last_error = e
@@ -219,18 +198,15 @@ def handle_target_closed_with_context(max_retries: int = 2, timeout_secs: int =
)
try:
- # Check if the browser context is still alive
context = page.context
browser = context.browser
if browser and not browser.is_connected():
- # Browser connection is lost - this is a more serious issue
logger.error(
"playwright_controller.handle_target_closed_with_context(): Browser connection lost - cannot recover automatically"
)
raise e
- # Try basic recovery first
await _recover_page(page, timeout_secs)
await asyncio.sleep(0.5)
@@ -283,13 +259,11 @@ class PlaywrightController:
self._timeout_load = timeout_load
self.logger = logger or logging.getLogger("playwright_controller")
- # Set up the download handler
self._page_script: str = ""
self.last_cursor_position: Tuple[float, float] = (0.0, 0.0)
self._text_utils = WebpageTextUtilsPlaywright()
- # Read page_script
with open(
os.path.join(os.path.abspath(os.path.dirname(__file__)), "page_script.js"),
"rt",
@@ -302,7 +276,6 @@ class PlaywrightController:
@handle_target_closed()
async def get_interactive_rects(self, page: Page) -> Dict[str, InteractiveRegion]:
await self._ensure_page_ready(page)
- # Read the regions from the DOM
try:
await page.evaluate(self._page_script)
except Exception:
@@ -312,7 +285,6 @@ class PlaywrightController:
await page.evaluate("MultimodalWebSurfer.getInteractiveRects();"),
)
- # Convert the results into appropriate types
assert isinstance(result, dict)
typed_results: Dict[str, InteractiveRegion] = {}
for k in result:
@@ -367,9 +339,8 @@ class PlaywrightController:
@handle_target_closed()
async def on_new_page(self, page: Page) -> None:
assert page is not None
- # bring page to front just in case
await page.bring_to_front()
- page.on("download", self._download_handler) # type: ignore
+ page.on("download", self._download_handler)
if self.to_resize_viewport and self.viewport_width and self.viewport_height:
await page.set_viewport_size(
{"width": self.viewport_width, "height": self.viewport_height}
@@ -385,7 +356,6 @@ class PlaywrightController:
await page.wait_for_load_state("domcontentloaded", timeout=60000)
except PlaywrightTimeoutError:
self.logger.error("WARNING: Page load timeout, page might not be loaded")
- # stop page loading
await page.evaluate("window.stop()")
@handle_target_closed()
@@ -408,7 +378,6 @@ class PlaywrightController:
return screenshot
except Exception:
await page.evaluate("window.stop()")
- # try again
screenshot = await page.screenshot(path=path, timeout=15000)
return screenshot
@@ -423,11 +392,9 @@ class PlaywrightController:
reset_prior_metadata_hash = False
reset_last_download = False
try:
- # Regular webpage
await page.goto(url, wait_until="commit")
reset_prior_metadata_hash = True
except Exception as e_outer:
- # Downloaded file
if self.downloads_folder and "net::ERR_ABORTED" in str(e_outer):
async with page.expect_download() as download_info:
try:
@@ -476,12 +443,10 @@ class PlaywrightController:
async def gradual_cursor_animation(
self, page: Page, start_x: float, start_y: float, end_x: float, end_y: float
) -> None:
- # animation helper
steps = 20
for step in range(steps):
x = start_x + (end_x - start_x) * (step / steps)
y = start_y + (end_y - start_y) * (step / steps)
- # await page.mouse.move(x, y, steps=1)
await page.evaluate(f"""
(function() {{
let cursor = document.getElementById('red-cursor');
@@ -494,7 +459,6 @@ class PlaywrightController:
self.last_cursor_position = (end_x, end_y)
async def add_cursor_box(self, page: Page, identifier: str) -> None:
- # animation helper
await page.evaluate(f"""
(function() {{
let elm = document.querySelector("[__elementId='{identifier}']");
@@ -506,7 +470,6 @@ class PlaywrightController:
""")
await asyncio.sleep(0.3)
- # Create a red cursor
await page.evaluate("""
(function() {
let cursor = document.createElement('div');
@@ -522,7 +485,6 @@ class PlaywrightController:
""")
async def remove_cursor_box(self, page: Page, identifier: str) -> None:
- # Remove the highlight and cursor
await page.evaluate(f"""
(function() {{
let elm = document.querySelector("[__elementId='{identifier}']");
@@ -548,9 +510,9 @@ class PlaywrightController:
"""
new_page: Page | None = None
try:
- async with page.expect_event("popup", timeout=1000) as page_info: # type: ignore
+ async with page.expect_event("popup", timeout=1000) as page_info:
await action()
- new_page = await page_info.value # type: ignore
+ new_page = await page_info.value
assert isinstance(new_page, Page)
await self.on_new_page(new_page)
except TimeoutError:
@@ -562,7 +524,6 @@ class PlaywrightController:
await self._ensure_page_ready(page)
if self.animate_actions:
- # Move cursor to the box slowly
start_x, start_y = self.last_cursor_position
await self.gradual_cursor_animation(page, start_x, start_y, x, y)
await asyncio.sleep(0.1)
@@ -580,7 +541,6 @@ class PlaywrightController:
await self._ensure_page_ready(page)
selector = f"[__elementId='{identifier}']"
try:
- # Wait for the element to be visible and scroll it into view
await page.wait_for_selector(
selector, state="visible", timeout=self._timeout_load * 1000
)
@@ -591,7 +551,6 @@ class PlaywrightController:
f"Element with identifier {identifier} not found or not visible"
)
- # Retrieve bounding box to determine the center for clicking
box = await target.bounding_box()
if not box:
raise ValueError(
@@ -600,7 +559,6 @@ class PlaywrightController:
center_x = box["x"] + box["width"] / 2
center_y = box["y"] + box["height"] / 2
- # In single tab mode, override target attributes to avoid opening a new tab
if self.single_tab_mode:
await target.evaluate("""
el => {
@@ -614,11 +572,10 @@ class PlaywrightController:
download = None
download_future: asyncio.Task[Download] | None = None
- # Start listening for a download event if downloads are enabled
if self.downloads_folder:
try:
download_future = asyncio.create_task(
- page.wait_for_event( # type: ignore
+ page.wait_for_event(
"download", timeout=500
)
)
@@ -634,29 +591,24 @@ class PlaywrightController:
await page.mouse.click(center_x, center_y)
return None
else:
- # Create a task to wait for a new page event
new_page_promise: asyncio.Task[Page] = asyncio.create_task(
- page.context.wait_for_event( # type: ignore
+ page.context.wait_for_event(
"page", timeout=self._timeout_load * 1000
)
)
- # Perform the click
await page.mouse.move(center_x, center_y, steps=1)
await page.mouse.click(center_x, center_y, delay=10)
try:
- # Wait for the new page to open
new_page = await new_page_promise
await self.on_new_page(new_page)
return new_page
except TimeoutError:
- # No new page opened within timeout
return None
except Exception as e:
raise e
- # Optionally animate the click
if self.animate_actions:
await self.add_cursor_box(page, identifier)
start_x, start_y = self.last_cursor_position
@@ -666,18 +618,14 @@ class PlaywrightController:
new_page = await perform_click()
- # Handle any download that occurred
if download_future:
try:
if not download:
- # Use asyncio.wait_for with a reasonable timeout
try:
download = await asyncio.wait_for(
download_future, timeout=self._timeout_load * 1000
)
except asyncio.TimeoutError:
- # No download occurred within the timeout period
- # logger.debug("No download detected within timeout period")
pass
if download:
@@ -706,7 +654,6 @@ class PlaywrightController:
await page.wait_for_load_state("domcontentloaded")
if self._sleep_after_action > 0:
await page.wait_for_timeout(self._sleep_after_action * 1000)
- # await asyncio.sleep(10)
return new_page
@@ -726,24 +673,19 @@ class PlaywrightController:
await self._ensure_page_ready(page)
new_page: Optional[Page] = None
try:
- # Wait for element to be present
await page.wait_for_selector(
f"[__elementId='{identifier}']", state="attached"
)
try:
- # First try normal click if element is visible
target = page.locator(f"[__elementId='{identifier}']").first
- # Get the bounding box to check element size
box = await target.bounding_box()
if box and box["width"] > 0 and box["height"] > 0:
- # Element has visible size - use normal click
return await self.click_id(page, identifier)
except PlaywrightError as e:
if "strict mode violation" in str(e):
- # If multiple elements found, try clicking the first visible one
elements = await page.locator(f"[__elementId='{identifier}']").all()
for element in elements:
try:
@@ -753,8 +695,6 @@ class PlaywrightController:
except PlaywrightError:
continue
- # If click didn't work, try programmatic selection
- # First check if it's a standard <option> element
option_element = await page.evaluate(
"""
(identifier) => {
@@ -771,7 +711,6 @@ class PlaywrightController:
)
if option_element:
- # Handle standard <select> dropdown
await page.evaluate(
"""
(identifier) => {
@@ -789,7 +728,6 @@ class PlaywrightController:
identifier,
)
else:
- # Handle custom dropdown/combobox options
await page.evaluate(
"""
(identifier) => {
@@ -820,7 +758,6 @@ class PlaywrightController:
identifier,
)
- # Optional sleep/pause after the action
if self._sleep_after_action > 0:
await page.wait_for_timeout(self._sleep_after_action * 1000)
@@ -844,7 +781,6 @@ class PlaywrightController:
await self._ensure_page_ready(page)
if self.animate_actions:
- # Move cursor to the coordinates slowly
start_x, start_y = self.last_cursor_position
await self.gradual_cursor_animation(page, start_x, start_y, x, y)
await asyncio.sleep(0.1)
@@ -859,7 +795,6 @@ class PlaywrightController:
await self._ensure_page_ready(page)
target = page.locator(f"[__elementId='{identifier}']")
- # See if it exists
try:
await target.wait_for(timeout=5000)
except TimeoutError:
@@ -867,7 +802,6 @@ class PlaywrightController:
f"Tool use response is invalid: no such element to hover: {identifier}"
)
- # Hover over it
await target.scroll_into_view_if_needed()
await asyncio.sleep(0.3)
@@ -875,7 +809,6 @@ class PlaywrightController:
if self.animate_actions:
await self.add_cursor_box(page, identifier)
- # Move cursor to the box slowly
start_x, start_y = self.last_cursor_position
end_x, end_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2
await self.gradual_cursor_animation(page, start_x, start_y, end_x, end_y)
@@ -909,7 +842,6 @@ class PlaywrightController:
new_page: Page | None = None
if self.animate_actions:
- # Move cursor to the box slowly
start_x, start_y = self.last_cursor_position
await self.gradual_cursor_animation(page, start_x, start_y, x, y)
await asyncio.sleep(0.1)
@@ -919,23 +851,11 @@ class PlaywrightController:
if delete_existing_text:
await self._clear_field(page)
- # Short strings (<100 chars) use 100ms delay per keystroke to avoid
- # dropping characters on fields with JS input masks (phone, credit card,
- # date). These masks have per-keystroke event handlers (onKeyDown/onChange)
- # that need time to reformat the value and reposition the cursor. See
- # https://github.com/microsoft/magentic-ui2.0/issues/279#issuecomment-4127277347
- # for an example of where this issue manifested itself.
- #
- # Long strings use 10ms since they are almost always plain text fields
- # with no input masks, and 100ms would be too slow (500 chars = 50s).
if len(value) < 100:
delay_typing_speed = 100
else:
delay_typing_speed = 10
- # If a PlaywrightError occurs mid-typing (e.g., element detach,
- # navigation), clear the field before retrying to avoid corrupted input
- # from a partial first attempt followed by a full retry.
try:
await page.keyboard.type(value, delay=delay_typing_speed)
except PlaywrightError:
@@ -945,10 +865,6 @@ class PlaywrightController:
if press_enter:
await page.keyboard.press("Enter")
- # Wait briefly for a popup that may have opened during typing or Enter.
- # Some websites open a new tab/popup on form submission (e.g., forms
- # with target="_blank"). The listener starts after typing so the timeout
- # window isn't consumed by keystroke delivery.
try:
new_page = await page.wait_for_event("popup", timeout=2000)
assert isinstance(new_page, Page)
@@ -982,7 +898,6 @@ class PlaywrightController:
target = page.locator(f"[__elementId='{identifier}']")
await target.scroll_into_view_if_needed()
- # See if it exists
try:
await target.wait_for(timeout=5000)
except TimeoutError:
@@ -990,11 +905,9 @@ class PlaywrightController:
f"Tool use response is invalid: No such element to fill input_text into: {identifier}"
) from None
- # Fill it
box = cast(Dict[str, Union[int, float]], await target.bounding_box())
if self.single_tab_mode:
- # Remove target attributes to prevent new tabs
await target.evaluate("""
el => el.removeAttribute('target')
// Remove 'target' on all <a> tags
@@ -1078,7 +991,7 @@ class PlaywrightController:
if self.animate_actions:
await asyncio.sleep(0.05)
except Exception:
- continue # cleanup must not mask the primary failure
+ continue
@handle_target_closed()
async def cua_click(
@@ -1154,7 +1067,6 @@ class PlaywrightController:
return document.body.innerText;
}""")
text_in_viewport = "\n".join(text_in_viewport.split("\n")[:n_lines])
- # remove empty lines
text_in_viewport = "\n".join(
[line for line in text_in_viewport.split("\n") if line.strip()]
)
diff --git a/src/fara/fara_7b/_prompts.py b/src/fara/fara_7b/_prompts.py
index f43cef1..850e6e9 100644
--- a/src/fara/fara_7b/_prompts.py
+++ b/src/fara/fara_7b/_prompts.py
@@ -9,7 +9,6 @@ from ..qwen_helpers.schema import (
from ..qwen_helpers.utils import smart_resize
-# @register_tool("computer_use")
class FaraComputerUse(BaseTool):
name = "computer_use"
diff --git a/src/fara/fara_7b/fara_agent.py b/src/fara/fara_7b/fara_agent.py
index 5465b9c..cff6fd3 100644
--- a/src/fara/fara_7b/fara_agent.py
+++ b/src/fara/fara_7b/fara_agent.py
@@ -29,12 +29,6 @@ from ..fara_types import (
from ..utils import get_trimmed_url
-# Canonical fara-1.0 action space mapping {action_name: set(arg_names)}.
-# Source of truth: ``FaraComputerUseTool.parameters`` in ``_prompts.py``.
-# Kept here so downstream callers (the rubric verifier, evaluation
-# scripts, training code, etc.) have a single place to import from
-# instead of duplicating the schema. Update both this dict AND the tool
-# parameter schema together if the action space changes.
FARA_ACTION_DEFINITIONS: Dict[str, set] = {
"key": {"keys"},
"type": {"text", "coordinate", "press_enter", "delete_existing_text"},
@@ -111,7 +105,6 @@ class FaraAgent:
self._download_handler = _download_handler
self.did_initialize = False
- # OpenAI client will be initialized in initialize()
self._openai_client: AsyncOpenAI | None = None
self._chat_history: List[LLMMessage] = []
@@ -121,16 +114,13 @@ class FaraAgent:
self._last_download = None
self._prior_metadata_hash = None
- # Initialize OpenAI client
self._openai_client = AsyncOpenAI(
api_key=self.client_config.get("api_key"),
base_url=self.client_config.get("base_url"),
)
- # Set up download handler
self.browser_manager.set_download_handler(self._download_handler)
- # Initialize browser
await self.browser_manager.init(self.start_page)
self.did_initialize = True
@@ -160,19 +150,18 @@ class FaraAgent:
async def wait_for_captcha_with_timeout(
self, timeout_seconds=300
- ): # 5 minutes default
+ ):
"""Wait for captcha to be solved with timeout"""
try:
await asyncio.wait_for(
self.browser_manager.wait_for_captcha_resolution(),
timeout=timeout_seconds,
)
- return True # Captcha solved in time
+ return True
except asyncio.TimeoutError:
self.logger.warning(f"Captcha timeout after {timeout_seconds} seconds!")
- # Force resume execution
self.browser_manager._captcha_event.set()
- return False # Captcha timed out
+ return False
@retry(
stop=stop_after_attempt(5),
@@ -240,13 +229,11 @@ class FaraAgent:
)
if i == 0 and n_images >= max_n_images:
- # First message is always the task so we keep it and remove the screenshot if necessary
msg = self.remove_screenshot_from_message(msg)
if msg is None:
continue
if isinstance(msg.content, list):
- # Check if the message contains an image. Assumes 1 image per message.
has_image = False
for c in msg.content:
if isinstance(c, ImageObj):
@@ -256,7 +243,6 @@ class FaraAgent:
if n_images < max_n_images:
new_history.append(msg)
elif is_original_user_message:
- # Original user message but over limit: keep text, remove image
msg = self.remove_screenshot_from_message(msg)
if msg is not None:
new_history.append(msg)
@@ -351,13 +337,10 @@ class FaraAgent:
async def run(self, user_message: str) -> Tuple:
"""Run the agent with a user message."""
- # Initialize if not already done
await self.initialize()
- # Ensure page is ready after initialization
assert self._page is not None, "Page should be initialized"
- # Get initial screenshot and add user message with image to chat history
scaled_screenshot = await self._get_scaled_screenshot()
if self.save_screenshots:
@@ -426,7 +409,6 @@ class FaraAgent:
screenshot_for_system = first_screenshot
if not is_first_round:
- # Get screenshot and add new user message for subsequent rounds
scaled_screenshot = await self._get_scaled_screenshot()
screenshot_for_system = scaled_screenshot
@@ -441,7 +423,6 @@ class FaraAgent:
self._chat_history.append(curr_message)
history.append(curr_message)
- # Generate system message using the screenshot
system_message, _ = self._get_system_message(screenshot_for_system)
history = system_message + history
response = await self._make_model_call(
@@ -487,13 +468,11 @@ class FaraAgent:
if args["action"] == "visit_url":
url = str(args["url"])
action_description = f"I typed '{url}' into the browser address bar."
- # Check if the argument starts with a known protocol
if url.startswith(("https://", "http://", "file://", "about:")):
(
reset_prior_metadata,
reset_last_download,
) = await self._playwright_controller.visit_page(self._page, url)
- # If the argument contains a space, treat it as a search query
elif " " in url:
(
reset_prior_metadata,
@@ -502,7 +481,6 @@ class FaraAgent:
self._page,
f"https://www.bing.com/search?q={quote_plus(url)}&FORM=QBLH",
)
- # Otherwise, prefix with https://
else:
(
reset_prior_metadata,
@@ -601,7 +579,6 @@ class FaraAgent:
await self._playwright_controller.wait_for_load_state(self._page)
await self._playwright_controller.sleep(self._page, 3)
- # Get new screenshot after action
self._num_actions += 1
if self.save_screenshots:
new_screenshot = await self._playwright_controller.get_screenshot(
diff --git a/src/fara/fara_types.py b/src/fara/fara_types.py
index 0f1e3a2..5173494 100644
--- a/src/fara/fara_types.py
+++ b/src/fara/fara_types.py
@@ -86,12 +86,10 @@ def message_to_openai_format(message: LLMMessage) -> Dict[str, Any]:
else "user"
)
- # Handle multimodal content (text + images)
if isinstance(message.content, list):
content_parts = []
for item in message.content:
if isinstance(item, ImageObj):
- # Convert image to base64 data URL
base64_image = item.to_base64()
content_parts.append(
{
@@ -102,11 +100,9 @@ def message_to_openai_format(message: LLMMessage) -> Dict[str, Any]:
elif isinstance(item, str):
content_parts.append({"type": "text", "text": item})
elif isinstance(item, dict):
- # Already in proper format
content_parts.append(item)
return {"role": role, "content": content_parts}
else:
- # Simple text content
return {"role": role, "content": message.content}
diff --git a/src/fara/qwen_helpers/base_tool.py b/src/fara/qwen_helpers/base_tool.py
index 6fc8fc3..8ff63c5 100644
--- a/src/fara/qwen_helpers/base_tool.py
+++ b/src/fara/qwen_helpers/base_tool.py
@@ -126,13 +126,11 @@ class BaseTool(ABC):
return params_json
@property
- def function(self) -> dict: # Bad naming. It should be `function_info`.
+ def function(self) -> dict:
return {
- # 'name_for_human': self.name_for_human,
"name": self.name,
"description": self.description,
"parameters": self.parameters,
- # 'args_format': self.args_format
}
@property
diff --git a/src/fara/qwen_helpers/fncall_prompt.py b/src/fara/qwen_helpers/fncall_prompt.py
index 83236a8..5b09d29 100644
--- a/src/fara/qwen_helpers/fncall_prompt.py
+++ b/src/fara/qwen_helpers/fncall_prompt.py
@@ -41,14 +41,13 @@ class NousFnCallPrompt:
parallel_function_calls: bool = True,
function_choice: Union[Literal["auto"], str] = "auto",
) -> List[Message]:
- del lang # ignored
- del parallel_function_calls # ignored
+ del lang
+ del parallel_function_calls
if function_choice != "auto":
raise NotImplementedError
ori_messages = messages
- # Change function_call responses to plaintext responses:
messages = []
for msg in copy.deepcopy(ori_messages):
role, content, reasoning_content = (
@@ -84,7 +83,6 @@ class NousFnCallPrompt:
messages[-1].content.append(ContentItem(text="\n"))
messages[-1].content.extend(content)
else:
- # TODO: Assuming there will only be one continuous reasoning_content here
messages.append(
Message(
role=role,
@@ -113,7 +111,6 @@ class NousFnCallPrompt:
]
tool_descs = "\n".join([json.dumps(f, ensure_ascii=False) for f in tool_descs])
- # Select template based on configuration
if SPECIAL_CODE_MODE and any([CODE_TOOL_PATTERN in x for x in tool_names]):
selected_template = FN_CALL_TEMPLATE_WITH_CI
elif self._custom_template is not None:
@@ -187,8 +184,6 @@ Here is the code.
</tool_call>"""
-# Mainly for removing incomplete special tokens when streaming the output
-# This assumes that '<tool_call>\n{"name": "' is the special token for the NousFnCallPrompt
def remove_incomplete_special_tokens(text: str) -> str:
if text in '<tool_call>\n{"name": "':
text = ""
diff --git a/src/fara/run_fara.py b/src/fara/run_fara.py
index 367ff96..a919d3d 100755
--- a/src/fara/run_fara.py
+++ b/src/fara/run_fara.py
@@ -15,7 +15,6 @@ from .fara_7b import FaraAgent
from .browser.browser_bb import BrowserBB
-# Configure logging to only show logs from the fara agents
logging.basicConfig(
level=logging.CRITICAL,
format="%(message)s",
@@ -28,7 +27,7 @@ for _logger_name in ("fara.agents.fara.fara_qwen3", "fara.fara_7b.fara_agent"):
_handler.setLevel(logging.INFO)
_handler.setFormatter(logging.Formatter("%(message)s"))
_agent_logger.addHandler(_handler)
- _agent_logger.propagate = False # Don't propagate to root logger
+ _agent_logger.propagate = False
logger = logging.getLogger(__name__)
@@ -115,7 +114,6 @@ async def run_fara15_agent(
print("Running Fara...\n")
final_answer, _, _ = await agent.run(run_context)
- # Critical-point resume loop: agent paused to ask the user.
while (
run_context.solver_log.status == SolverStatus.WAITING_FOR_USER
):