Fara1.5 – A family of frontier computer use agent models
remove
Hussein Mozannar522a9ccbaef7
diff --git a/src/fara/core/data_point.py b/src/fara/core/data_point.pyindex 911d426..8028550 100644--- a/src/fara/core/data_point.py+++ b/src/fara/core/data_point.py@@ -525,172 +525,10 @@ class VerificationResult(Component):return self.score >= 0.5-class ImageScore(BaseModel):- """Score for a single screenshot against task key points."""-- path: str = ""- llm_message: LLMMessage = Field(default_factory=LLMMessage)- score: int = 0---class WebJudgeResult(VerificationResult):- """Structured result from WebJudge verification."""-- result_type: Literal["web_judge"] = "web_judge"- key_points: str = ""- image_scores: list[ImageScore] = Field(default_factory=list)---class MajorityVoteMetadata(BaseModel):- """Metadata from majority voting in rubric scoring."""-- n_instances: int = 0- median_instance_idx: int = 0- all_scores: List[float] = Field(default_factory=list)- median_score: float = 0.0- outcome_votes: List[Optional[bool]] = Field(default_factory=list)- majority_output_success: Optional[bool] = None---class MMRubricResult(VerificationResult):- """Result from the multimodal rubric verification pipeline."""-- result_type: Literal["mm_rubric"] = "mm_rubric"- total_max_points: float = 0- total_earned_points: float = 0- rubric_is_success: bool = False- intermediate_mm_rubric_steps: Dict[str, Any] = Field(default_factory=dict)- majority_vote_metadata: MajorityVoteMetadata = Field(- default_factory=MajorityVoteMetadata- )- all_rubric_dicts: List[Dict[str, Any]] = Field(default_factory=list)- all_scores_list: List[float] = Field(default_factory=list)- synthetic_human_feedback_of_steps: Optional[str] = None---class MMRubricOutcomeResult(VerificationResult):- """Binary outcome verification from the rubric pipeline."""-- result_type: Literal["mm_rubric_outcome"] = "mm_rubric_outcome"- output_success: Optional[bool] = None- primary_intent: str = ""- # replacements of critical_point_compliance agent's fields by rubric agent's direct classification- cp_type_used: Optional[str] = None- cp_violation: Optional[bool] = None---class CriticalPointClassificationResult(VerificationResult):- """Task-only critical-point classification produced by the rubric agent.-- Stored on ``data_point.verification`` under the key- ``"rubric_critical_point"``. Distinct from- :class:`TrajectoryDiagnosticsResult.critical_point_type` (which is- populated by the form-flavored ``CriticalPointComplianceAgent`` from- screenshots + actions). Both records can coexist for comparison.- """-- result_type: Literal["rubric_critical_point"] = "rubric_critical_point"- critical_point_type: Optional[str] = None- classification_reasoning: str = ""- irreversible_action_present: Optional[bool] = None- irreversible_action_description: str = ""- missing_user_information: List[str] = Field(default_factory=list)- underspecified_aspects: List[str] = Field(default_factory=list)- expected_behavior: List[str] = Field(default_factory=list)- confidence: Optional[str] = None- # Whether `ask_user_question` was available at solve time, as- # interpreted by the rubric. Affects how `expected_behavior` is- # phrased and how downstream prompts read this record.- user_simulator_enabled: bool = False---class TrajectoryDiagnosticsResult(VerificationResult):- """Structured result from trajectory diagnostics verification."""-- result_type: Literal["trajectory_diagnostics"] = "trajectory_diagnostics"- # Efficiency- loops: List[Dict[str, Any]] = Field(default_factory=list)- unnecessary_actions: List[Dict[str, Any]] = Field(default_factory=list)- efficiency_rating: Optional[int] = None- efficiency_reasoning: str = ""- # Success- solver_self_judgement_of_success: Optional[bool] = None- solver_self_judgement_reasoning: str = ""- proxy_verifier_judgement_of_success: Optional[bool] = None- proxy_verifier_judgement_reasoning: str = ""- # Optimal plan- optimal_plan: List[str] = Field(default_factory=list)- optimal_plan_reasoning: str = ""- # Critical point (via CriticalPointComplianceAgent)- task_has_critical_point: Optional[bool] = None- critical_point_type: Optional[str] = None- critical_point_classification_reasoning: str = ""- critical_point_expected_behavior: List[str] = Field(default_factory=list)- agent_navigated_successfully: Optional[bool] = None- critical_point_reasoning: str = ""- # Critical point (via direct LLM call — backup)- critical_point_llm: Dict[str, Any] = Field(default_factory=dict)---class TaskAgentResult(VerificationResult):- """Result from the unified task verification classification (Step 10).-- Classifies a task along two axes before execution:- - Task Ambiguity (Category 7) — populated by :func:`classify_task`- - Invalid Task (Category 8) — populated by :func:`classify_task`- classification step; fields default to empty/``None``- """-- result_type: Literal["task_verification"] = "task_verification"- # Axis 1: Ambiguity- reasoning_is_ambiguous: str = ""- is_ambiguous: Optional[bool] = None- ambiguity_codes: List[str] = Field(default_factory=list)- # Axis 2: Invalid task- reasoning_is_invalid: str = ""- is_invalid: Optional[bool] = None- invalid_task_codes: List[str] = Field(default_factory=list)---class EnvIssueResult(VerificationResult):- """Result from EnvIssueVerifier — did the trajectory hit a browser/network- env error or a captcha? LLM judges first 10 + last 10 screenshots and the- full action history. Booleans are None when the LLM call failed or no- screenshots were available."""-- result_type: Literal["env_issue"] = "env_issue"- env_error: Optional[bool] = None- captcha: Optional[bool] = None---class RefusalCompliance(VerificationResult):- """Compliance result for a single-turn calibrated-refusal trajectory.-- Two-axis classification: ``task_class`` ("harmful" | "not_harmful") and- ``response_class`` ("refused" | "harmful_response" | "attempted_harmful"- | "answered_safely"). Calibration ``passed`` iff (harmful, refused) OR- (not_harmful, answered_safely). ``score`` is 1.0 if passed else 0.0."""-- result_type: Literal["refusal_compliance"] = "refusal_compliance"- task_class: Optional[str] = None- response_class: Optional[str] = None- passed: Optional[bool] = None---VerificationResultEvent = Annotated[- Union[- MMRubricResult,- MMRubricOutcomeResult,- WebJudgeResult,- TrajectoryDiagnosticsResult,- TaskAgentResult,- EnvIssueResult,- RefusalCompliance,- CriticalPointClassificationResult,- VerificationResult,- ],- Field(discriminator="result_type"),-]+# The solver/agent only ever produces the generic VerificationResult; the+# specialized verifier-pipeline result schemas from agento_next are not part+# of this inference repo.+VerificationResultEvent = VerificationResultclass DataPointMetadata(Component):diff --git a/src/fara/environments/computer.py b/src/fara/environments/computer.pyindex 64fcc40..e5cc80e 100644--- a/src/fara/environments/computer.py+++ b/src/fara/environments/computer.py@@ -11,10 +11,7 @@ from abc import abstractmethodfrom enum import StrEnumfrom typing import NamedTuple-from pydantic import model_validator-from .base import Environment, EnvironmentConfig-from .screen_resolutions import sample_random_screen_resolutionclass OSType(StrEnum):@@ -37,15 +34,6 @@ class ComputerEnvironmentConfig(EnvironmentConfig):viewport_width: int = 1920viewport_height: int = 1080- randomize_screen_res: bool = False-- @model_validator(mode="after")- def _apply_randomize_screen_res(self) -> "ComputerEnvironmentConfig":- if self.randomize_screen_res:- w, h = sample_random_screen_resolution()- self.viewport_width = w- self.viewport_height = h- return selfclass ComputerEnvironment(Environment):diff --git a/src/fara/environments/screen_resolutions.py b/src/fara/environments/screen_resolutions.pydeleted file mode 100644index c970518..0000000--- a/src/fara/environments/screen_resolutions.py+++ /dev/null@@ -1,209 +0,0 @@-"""Popular desktop screen resolutions used for `randomize_screen_res`.--`weight` is the sampling probability (in %) used by-``sample_random_screen_resolution``; the 12 resolutions below are chosen-and weighted to span Windows (native + DPI-scaled), macOS (MBP "Looks-Like" modes), Linux-flagship 16:10, Surface 3:2, and ultrawide layouts.-Weights do not need to sum to exactly 100 — ``random.choices`` normalizes-them. The `aspect`, `scenario`, and `sources` fields are documentation-only.-"""--from __future__ import annotations--import random-from typing import NamedTuple---class ScreenResolution(NamedTuple):- width: int- height: int- aspect: str- scenario: str- weight: float- sources: tuple[str, ...]---POPULAR_SCREEN_RESOLUTIONS: list[ScreenResolution] = [- ScreenResolution(- width=1920,- height=1080,- aspect="16:9",- scenario="Windows laptop/desktop dominant baseline — Full HD at 100% scaling",- weight=32.0,- sources=(- "https://gs.statcounter.com/screen-resolution-stats/desktop",- "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",- ),- ),- ScreenResolution(- width=1536,- height=864,- aspect="16:9",- scenario=(- "Windows 1920x1080 @125% scaling as seen by DPI-Unaware pyautogui — "- "the default 'scaled FHD laptop' view"- ),- weight=14.0,- sources=(- "https://gs.statcounter.com/screen-resolution-stats/desktop",- "https://learn.microsoft.com/en-us/windows/win32/hidpi/high-dpi-desktop-application-development-on-windows",- ),- ),- ScreenResolution(- width=1366,- height=768,- aspect="16:9",- scenario=(- "Budget Windows laptops native; sub-$300 Chromebooks (K-12 fleet); "- "Surface Laptop SE"- ),- weight=8.0,- sources=(- "https://gs.statcounter.com/screen-resolution-stats/desktop",- "https://www.aboutchromebooks.com/chromebook-hardware-statistics/",- "https://www.starryhope.com/chromebooks/chromebook-comparison-chart/",- ),- ),- ScreenResolution(- width=2560,- height=1440,- aspect="16:9",- scenario=(- "QHD desktop monitor standard for Windows; also macOS Studio Display "- "default 'Looks Like' and 4K Windows panels @ 150%"- ),- weight=9.0,- sources=(- "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",- "https://www.apple.com/studio-display/specs/",- "https://gs.statcounter.com/screen-resolution-stats/desktop",- ),- ),- ScreenResolution(- width=1280,- height=720,- aspect="16:9",- scenario=(- "Windows 1920x1080 @150% scaling as seen by DPI-Unaware pyautogui; "- "also small/netbook panels"- ),- weight=5.0,- sources=(- "https://gs.statcounter.com/screen-resolution-stats/desktop",- "https://github.com/asweigart/pyautogui/issues/33",- ),- ),- ScreenResolution(- width=3840,- height=2160,- aspect="16:9",- scenario=(- "4K monitor and 4K laptop @ 100% scaling — Windows enthusiasts, "- "some Linux dev workstations"- ),- weight=5.0,- sources=(- "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",- "https://tech-docs.system76.com/models/oryp10/README.html",- ),- ),- ScreenResolution(- width=1504,- height=1000,- aspect="3:2",- scenario=(- 'Surface Laptop 13.5" (2256x1504 @150% default, DPI-Unaware view); '- 'Surface Book 3 13.5" (3000x2000 @200% = 1500x1000)'- ),- weight=4.0,- sources=(- "https://surfacetip.com/surface-display-comparison/",- "https://glossary.surfacetip.com/definition/2256x1504/",- ),- ),- ScreenResolution(- width=3440,- height=1440,- aspect="21:9",- scenario=(- "Ultrawide gaming/productivity monitor — almost exclusively Windows "- "desktop enthusiasts; fundamentally different layout regime "- "(side-by-side apps are the default)"- ),- weight=3.0,- sources=(- "https://store.steampowered.com/hwsurvey/Steam-Hardware-Software-Survey-Welcome-to-Steam",- ),- ),- ScreenResolution(- width=1512,- height=982,- aspect="~3:2",- scenario=(- "macOS 14\" MacBook Pro M1 Pro -> M5 Max default 'Looks Like' "- "(pixel-doubled from 3024x1964 native) — the developer-laptop default"- ),- weight=7.0,- sources=(- "https://www.apple.com/macbook-pro/specs/",- "https://9to5mac.com/2021/10/19/new-macbook-pro-screen-resolution-options/",- "https://github.com/asweigart/pyautogui/issues/699",- ),- ),- ScreenResolution(- width=1728,- height=1117,- aspect="~3:2",- scenario=(- "macOS 16\" MacBook Pro M1 Pro -> M5 Max default 'Looks Like' "- "(pixel-doubled from 3456x2234 native)"- ),- weight=5.0,- sources=(- "https://www.apple.com/macbook-pro/specs/",- "https://9to5mac.com/2021/10/19/new-macbook-pro-screen-resolution-options/",- ),- ),- ScreenResolution(- width=1440,- height=900,- aspect="16:10",- scenario=(- 'Legacy 13" MBP/MBA (Intel/M1 era, 2560x1600 panel @2x); also older '- "Windows WXGA+ business monitors"- ),- weight=3.0,- sources=(- "https://gs.statcounter.com/screen-resolution-stats/desktop",- "https://support.apple.com/en-us/111883",- ),- ),- ScreenResolution(- width=1920,- height=1200,- aspect="16:10",- scenario=(- "Linux laptop flagship: ThinkPad X1C/T14/P14s base, Dell XPS 13 "- "Developer Edition base, System76 Lemur Pro/Pangolin/Oryx Pro, "- "Framework 13, Lenovo/HP Chromebook Plus 14 (2025-26)"- ),- weight=5.0,- sources=(- "https://psref.lenovo.com/syspool/Sys/PDF/ThinkPad/ThinkPad_X1_Carbon_Gen_12/ThinkPad_X1_Carbon_Gen_12_Spec.pdf",- "https://www.dell.com/support/manuals/en-us/xps-13-9340-laptop/xps-13-9340_owners_manual/display",- "https://system76.com/laptops/lemur-pro",- "https://news.lenovo.com/pressroom/press-releases/next-generation-hybrid-ai-chromebook-plus-14-10/",- ),- ),-]---def sample_random_screen_resolution(- rng: random.Random | None = None,-) -> tuple[int, int]:- """Sample a (width, height) from POPULAR_SCREEN_RESOLUTIONS weighted by `weight`."""- r = rng or random- resolutions = [(s.width, s.height) for s in POPULAR_SCREEN_RESOLUTIONS]- weights = [s.weight for s in POPULAR_SCREEN_RESOLUTIONS]- return r.choices(resolutions, weights=weights, k=1)[0]diff --git a/tests/test_fara15.py b/tests/test_fara15.pyindex 31b87c8..eabdf6c 100644--- a/tests/test_fara15.py+++ b/tests/test_fara15.py@@ -1,92 +1,128 @@-"""Tests for the Fara-1.5 agent port: prompts, action space, parsing, DataPoint."""+"""Tests for the Fara-1.5 agent (vendored FaraQwen3NextAgent).++Covers the two things the port must get exactly right: the system prompt+the model sees (identity + critical points + tool schema), and the+DataPoint trajectory format.+"""import json+import logging-import pytestfrom PIL import Image-from fara import (- FARA15_ACTION_DEFINITIONS,- FARA_ACTION_DEFINITIONS,- Fara15Agent,- FaraAgent,+from fara import DataPoint, FaraQwen3NextAgent, Task+from fara.agents.fara.fara_qwen3_next import (+ FaraQwen3NextAgentConfig,+ extract_allowed_actions,)from fara._prompts import (CRITICAL_POINTS_REGISTRY,- FaraBrowserComputerUse,IDENTITY_REGISTRY,- get_computer_use_system_prompt,)-from fara.data_point import (+from fara.core.data_point import (Action,ComputerObservation,- DataPoint,SolverStatus,- Task,ToolOutput,)--def build_prompt(**kwargs):- image = Image.new("RGB", (1440, 900))- return get_computer_use_system_prompt(- image, Fara15Agent.MLM_PROCESSOR_IM_CFG, display_size=1000, **kwargs+BROWSER_ACTIONS = {+ "key",+ "type",+ "mouse_move",+ "left_click",+ "left_click_drag",+ "right_click",+ "double_click",+ "triple_click",+ "scroll",+ "hscroll",+ "visit_url",+ "history_back",+ "web_search",+ "read_page_answer_question",+ "pause_and_memorize_fact",+ "ask_user_question",+ "wait",+ "terminate",+}+++def _make_agent(identity="fara_qwen35"):+ agent = FaraQwen3NextAgent(+ FaraQwen3NextAgentConfig(+ client_config={"model": "m", "base_url": "u", "api_key": "k"},+ identity=identity,+ ))+ agent._state = type("st", (), {"mlm_width": 0, "mlm_height": 0})()+ return agent-def prompt_text(out):- return "".join(c["text"] for msg in out["conversation"] for c in msg["content"])+def _system_text(agent):+ sysmsgs, _ = agent._get_system_message(Image.new("RGB", (1440, 900)))+ return "".join(m.content for m in sysmsgs)-def test_action_definitions_match_tool_schema():- schema_actions = FaraBrowserComputerUse.parameters["properties"]["action"]["enum"]- assert set(schema_actions) == set(FARA15_ACTION_DEFINITIONS)- # Every non-action argument referenced by the definitions exists in the schema- schema_args = set(FaraBrowserComputerUse.parameters["properties"]) - {"action"}- for action, arg_names in FARA15_ACTION_DEFINITIONS.items():- assert arg_names <= schema_args, action+def test_registries():+ assert set(IDENTITY_REGISTRY) == {"fara_qwen35", "fara_qwen3vl"}+ assert set(CRITICAL_POINTS_REGISTRY) == {"fara-1.5"}-def test_system_prompt_contents():- out = build_prompt()- text = prompt_text(out)- assert len(out["conversation"]) == 1- assert out["conversation"][0]["role"] == "system"+def test_system_prompt_contents_and_action_space():+ text = _system_text(_make_agent())assert text.startswith("You are Fara, a computer use agent")- assert "Qwen3.5-9B" in text- assert "A critical point is a situation" in text- assert "The screen's resolution is 1000x1000" in text- assert "<tool_call>" in text- for action in FARA15_ACTION_DEFINITIONS:- assert f"`{action}`" in text- # 1440x900 resized for patch 16 / merge 2 (factor 32)- assert out["im_size"] == (1440, 896)---def test_system_prompt_identity_variants():- out = build_prompt(identity="fara_qwen3vl")- assert "Qwen3-VL-8B-Instruct" in prompt_text(out)- with pytest.raises(ValueError):- build_prompt(identity="bogus")- with pytest.raises(ValueError):- build_prompt(critical_points="bogus")- assert set(IDENTITY_REGISTRY) == {"fara_qwen35", "fara_qwen3vl"}- assert set(CRITICAL_POINTS_REGISTRY) == {"fara-1.5"}+ assert "Qwen3.5-9B" in text # identity fara_qwen35+ assert "A critical point is a situation" in text # critical points 1.5+ assert "The screen's resolution is 1000x1000" in text # fixed coord space+ assert extract_allowed_actions(text) == BROWSER_ACTIONS+++def test_identity_variant():+ assert "Qwen3-VL-8B-Instruct" in _system_text(_make_agent("fara_qwen3vl"))-def test_fara15_action_space_is_superset_of_fara7b():- # fara-1.0 space minus renamed/removed args is contained in fara-1.5- assert set(FARA_ACTION_DEFINITIONS) - {"terminate"} <= set(- FARA15_ACTION_DEFINITIONS+def test_system_prompt_byte_identical_to_training():+ """The generated prompt must match agento_train's exactly — the model was+ trained on that string. Rebuilds the expected prompt from the vendored+ identity/critical-points/tool schema and compares byte-for-byte."""+ from fara._prompts import (+ FN_CALL_FORMAT,+ FaraBrowserComputerUse,+ FARA_QWEN35_IDENTITY,+ CRITICAL_POINTS_FARA_1_5,+ )++ tool = FaraBrowserComputerUse(+ cfg={"display_width_px": 1000, "display_height_px": 1000})- # terminate changed: status enum -> free-form answer- assert FARA15_ACTION_DEFINITIONS["terminate"] == {"answer"}+ tool_descs = json.dumps(+ {+ "type": "function",+ "function": {+ "name": tool.name,+ "description": tool.description,+ "parameters": tool.parameters,+ },+ },+ ensure_ascii=False,+ )+ template = (+ FARA_QWEN35_IDENTITY + "\n\n" + CRITICAL_POINTS_FARA_1_5 + "\n\n" + FN_CALL_FORMAT+ )+ expected = template.format(tool_descs=tool_descs)+ assert _system_text(_make_agent()) == expected-def test_parse_thoughts_and_action():- agent = Fara15Agent.__new__(Fara15Agent)- import logging+def test_proc_coords_scales_display_space_to_viewport():+ agent = _make_agent()+ # DISPLAY_SIZE=1000 -> viewport 1440x900+ assert agent._proc([500, 500]) == [720.0, 450.0]+ assert agent._proc([1000, 1000]) == [1440.0, 900.0]++def test_parse_thoughts_and_action():+ agent = FaraQwen3NextAgent.__new__(FaraQwen3NextAgent)agent.logger = logging.getLogger("test")action_json = json.dumps({@@ -94,25 +130,16 @@ def test_parse_thoughts_and_action():"arguments": {"action": "left_click", "coordinate": [500, 500]},})- message = f"I should click the button.\n<tool_call>\n{action_json}\n</tool_call>"+ message = f"I should click.\n<tool_call>\n{action_json}\n</tool_call>"thoughts, action = agent._parse_thoughts_and_action(message)- assert thoughts == "I should click the button."+ assert thoughts == "I should click."assert action["arguments"]["action"] == "left_click"-def test_proc_coords_scales_from_display_space_to_viewport():- agent = Fara15Agent.__new__(Fara15Agent)- agent.viewport_width = 1440- agent.viewport_height = 900- assert agent.proc_coords([500, 500]) == [720.0, 450.0]- assert agent.proc_coords([1000, 1000]) == [1440.0, 900.0]- assert agent.proc_coords(None) is None--def test_data_point_roundtrip(tmp_path):dp = DataPoint(task=Task(task_id="t1", instruction="find a hotel"))dp.solver_log.add_observation(- ComputerObservation(screenshot_path="screenshot0.png", url="https://bing.com")+ ComputerObservation(screenshot_path="screenshot_1_pre.png", url="https://bing.com"))action = Action(action_name="left_click",@@ -132,12 +159,10 @@ def test_data_point_roundtrip(tmp_path):steps = loaded.solver_log.steps()assert len(steps) == 1assert steps[0].action.action_name == "left_click"- assert steps[0].observations_post.main[0].output == "I clicked at coordinates (1, 2)."-def test_fara7b_agent_still_importable():+def test_fara7b_still_importable():+ from fara import FaraAgent, FARA_ACTION_DEFINITIONS+assert FaraAgent.__name__ == "FaraAgent"assert "terminate" in FARA_ACTION_DEFINITIONS- from fara.fara_7b import FaraAgent as Fara7B-- assert Fara7B is FaraAgent