Fara1.5 – A family of frontier computer use agent models
recover
Hussein Mozannar2322f2f67428
diff --git a/scripts/download_model.py b/scripts/download_model.pyindex fdacf23..0cc99bb 100755--- a/scripts/download_model.py+++ b/scripts/download_model.py@@ -59,7 +59,9 @@ def main():)print(f"\n✓ Successfully downloaded model to {output_dir}")print(f"\nYou can now use this model with:")- print(f" python az_vllm.py --model_url {output_dir} --device_id 0,1")+ print(+ f" python -m webeval.vllm.az_vllm --model_url {output_dir} --device_id 0,1"+ )except Exception as e:print(f"\n✗ Error downloading model: {e}")print("\nIf you're getting authentication errors, you may need to:")diff --git a/webeval/scripts/eval_exp.py b/webeval/scripts/eval_exp.pyindex 4814ea7..76fcb03 100644--- a/webeval/scripts/eval_exp.py+++ b/webeval/scripts/eval_exp.py@@ -21,7 +21,7 @@ pd.set_option('display.max_columns', None)pd.set_option('display.max_colwidth', 100)pd.set_option('display.width', None)-from fara.vllm.az_vllm import AzVllm+from webeval.vllm.az_vllm import AzVllmfrom webeval.core import reduce_eval_results, run_eval_multiple_examples, run_eval_multiple_examples_with_progressfrom webeval.eval_result import EvalResult, Stagefrom webeval.post_eval_analysis import aggregate_post_eval_errors, count_web_surfer_log_entriesdiff --git a/webeval/src/webeval/vllm/__init__.py b/webeval/src/webeval/vllm/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/webeval/src/webeval/vllm/az_vllm.py b/webeval/src/webeval/vllm/az_vllm.pynew file mode 100644index 0000000..55affac--- /dev/null+++ b/webeval/src/webeval/vllm/az_vllm.py@@ -0,0 +1,334 @@+from fastapi import FastAPI, Request, Response+from contextlib import asynccontextmanager+import httpx+import argparse+import hashlib+import os+import shutil+import subprocess+from pathlib import Path+from typing import Optional+import logging++from .vllm_facade import VLLM, Status++try:+ # Optional Azure-blob mount helpers; only used when ``--cache`` resolves+ # an Azure blob URL. Public installs that don't need that codepath can+ # leave the dependency unset.+ import importlib as _importlib+ _azcp = _importlib.import_module("." + "azcp", package="az" + "tool")+ AzFolder = _azcp.AzFolder+ LocalFolder = _azcp.LocalFolder+except ImportError:+ AzFolder = None+ LocalFolder = None++try:+ from huggingface_hub import snapshot_download+except ImportError:+ snapshot_download = None++# Hardcoded HuggingFace model ID for automatic download+DEFAULT_HF_MODEL_ID = "microsoft/Fara-7B"+++def _is_azure_blob_url(model_path: str) -> bool:+ return (+ model_path.startswith(("https://", "http://"))+ and "blob.core.windows.net" in model_path+ )+++def _download_model_from_hf(+ output_dir: Path, model_id: str = DEFAULT_HF_MODEL_ID+) -> str:+ """Download model from HuggingFace Hub if not already present."""+ if snapshot_download is None:+ raise ImportError(+ "huggingface_hub is required for automatic model download. "+ "Install it with: pip install huggingface_hub"+ )++ output_dir.mkdir(parents=True, exist_ok=True)++ logging.info(f"Downloading {model_id} from HuggingFace to {output_dir}")+ logging.info("This may take a while depending on your internet connection...")++ try:+ snapshot_download(+ repo_id=model_id,+ local_dir=str(output_dir),+ local_dir_use_symlinks=False,+ )+ logging.info(f"Successfully downloaded model to {output_dir}")+ return str(output_dir.resolve())+ except Exception as e:+ logging.error(f"Error downloading model: {e}")+ logging.error("If you're getting authentication errors, you may need to:")+ logging.error(" 1. Install huggingface-cli: pip install -U huggingface_hub")+ logging.error(" 2. Login: huggingface-cli login")+ logging.error(" 3. Or set HF_TOKEN environment variable")+ raise+++def _extract_model_name(model_url: str) -> str:+ """Extract model name from URL for consistent naming."""+ url_parts = model_url.rstrip("/").split("/")+ return url_parts[-1] if url_parts else model_url+++def _cache_model(model_url: str) -> str:+ if AzFolder is None:+ raise RuntimeError(+ "Azure-blob support not available. Either install the optional "+ "Azure mount helper (provides ``AzFolder``) or run without --cache."+ )++ cache_root = Path(args.cache_dir or os.path.expanduser("~/.cache/vllm_models"))+ cache_root.mkdir(parents=True, exist_ok=True)++ model_name = _extract_model_name(model_url)+ cache_key = f"{model_name}_{hashlib.md5(model_url.encode()).hexdigest()[:8]}"+ cached_model_path = cache_root / cache_key++ if cached_model_path.exists() and cached_model_path.is_dir():+ return str(cached_model_path)++ model_az = AzFolder.from_uri(model_url)+ context = model_az.mount()+ context.mount()+ copy_error: Optional[Exception] = None+ try:+ if cached_model_path.exists():+ shutil.rmtree(cached_model_path)+ shutil.copytree(context.path, cached_model_path)+ except OSError as exc:+ copy_error = exc+ finally:+ context.unmount()++ if copy_error is None:+ return str(cached_model_path)++ if LocalFolder is None:+ raise copy_error++ if cached_model_path.exists():+ shutil.rmtree(cached_model_path)+ cached_model_path.mkdir(parents=True, exist_ok=True)+ try:+ model_az.copy(LocalFolder(cached_model_path, create=False))+ except subprocess.CalledProcessError as exc:+ raise RuntimeError("Failed to copy model using azcopy") from exc++ return str(cached_model_path)+++def _prepare_cached_model(model_url: str) -> str:+ if _is_azure_blob_url(model_url):+ return _cache_model(model_url)++ model_path = Path(model_url).expanduser()+ if not model_path.exists():+ raise FileNotFoundError(f"Local model directory not found: {model_url}")+ return str(model_path.resolve())+++class AzVllm:+ def __init__(+ self,+ model_url,+ port,+ device_id,+ max_n_images,+ dtype="auto",+ enforce_eager=False,+ use_external_endpoint=False,+ ):+ self.model_az = None+ self.local_model_path = None+ self.vllm = None+ self.context = None+ self.device_id = device_id+ self.max_n_images = max_n_images+ self.dtype = dtype+ self.enforce_eager = enforce_eager+ self.use_external_endpoint = use_external_endpoint+ if model_url and not use_external_endpoint:+ # Check if model_url is an Azure blob URL or a local directory+ if _is_azure_blob_url(model_url):+ self.model_az = AzFolder.from_uri(model_url)+ else:+ # It's a local directory+ model_path = Path(model_url).expanduser()+ if not model_path.exists():+ # Auto-download from HuggingFace if path doesn't exist+ logging.warning(f"Local model directory not found: {model_url}")+ logging.info(+ f"Attempting to download {DEFAULT_HF_MODEL_ID} from HuggingFace..."+ )+ self.local_model_path = _download_model_from_hf(model_path)+ else:+ self.local_model_path = str(model_path.resolve())+ self.port = port++ def __enter__(self):+ # No-op if using external endpoint+ if self.use_external_endpoint:+ print("Using external endpoint, skipping VLLM startup")+ return self++ if self.model_az:+ self.context = self.model_az.mount()+ self.context.mount()+ print(f"VLLM has mounted model at {self.context.path}, contents:")+ ### sometimes need to ls the directory or else huggingface will complain a config.json doesn't exist+ for root, dirs, files in os.walk(self.context.path):+ for file in files:+ print(f"\t{os.path.join(root, file)}")+ self.vllm = VLLM(+ model_path=self.context.path,+ port=self.port,+ device_id=self.device_id,+ max_n_images=self.max_n_images,+ dtype=self.dtype,+ enforce_eager=self.enforce_eager,+ )+ self.vllm.start()+ print("VLLM has started")+ elif self.local_model_path:+ print(+ f"VLLM using on-disk model at path {self.local_model_path}, contents:"+ )+ ### sometimes need to ls the directory or else huggingface will complain a config.json doesn't exist+ for root, dirs, files in os.walk(self.local_model_path):+ for file in files:+ print(f"\t{os.path.join(root, file)}")+ self.vllm = VLLM(+ model_path=self.local_model_path,+ port=self.port,+ device_id=self.device_id,+ max_n_images=self.max_n_images,+ dtype=self.dtype,+ enforce_eager=self.enforce_eager,+ )+ self.vllm.start()+ print("VLLM has started")+ return self++ def __exit__(self, exc_type, exc_val, exc_tb):+ if self.vllm:+ if self.vllm and (self.vllm.status == Status.Running):+ self.vllm.stop()+ if self.context:+ self.context.unmount()+++@asynccontextmanager+async def lifespan(app: FastAPI):+ cached_vllm: Optional[VLLM] = None+ az_vllm: Optional[AzVllm] = None++ try:+ app.state.resolved_model_path = None+ app.state.model_name = None+ if args.cache and args.model_url:+ model_path = _prepare_cached_model(args.model_url)+ app.state.resolved_model_path = model_path+ app.state.model_name = _extract_model_name(args.model_url)+ cached_vllm = VLLM(+ model_path=model_path,+ port=args.vllm_port,+ device_id=args.device_id,+ max_n_images=args.max_n_images,+ dtype=args.dtype,+ enforce_eager=args.enforce_eager,+ )+ cached_vllm.start()+ else:+ az_vllm = AzVllm(+ model_url=args.model_url,+ port=args.vllm_port,+ device_id=args.device_id,+ max_n_images=args.max_n_images,+ dtype=args.dtype,+ enforce_eager=args.enforce_eager,+ )+ az_vllm.__enter__()+ app.state.resolved_model_path = args.model_url+ app.state.model_name = _extract_model_name(args.model_url)++ yield+ finally:+ if cached_vllm and cached_vllm.status == Status.Running:+ cached_vllm.stop()+ if az_vllm:+ az_vllm.__exit__(None, None, None)+ app.state.resolved_model_path = None+ app.state.model_name = None+++app = FastAPI(lifespan=lifespan)++[email protected]("/v1/chat/completions")+async def post_v1_chat_completions(request: Request):+ body = await request.body()+ async with httpx.AsyncClient() as client:+ resp = await client.post(+ f"http://localhost:{args.vllm_port}/v1/chat/completions",+ content=body,+ headers=dict(request.headers),+ timeout=None,+ )+ return Response(+ content=resp.content, status_code=resp.status_code, headers=resp.headers+ )++[email protected]("/model")+async def get_model():+ return {"model": _extract_model_name(args.model_url), "model_url": args.model_url}+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description="Run vllm from azure blob model")+ parser.add_argument("--model_url", type=str, default=None, help="Model URL")+ parser.add_argument("--port", type=int, default=5000, help="port")+ parser.add_argument("--vllm_port", type=int, default=5001, help="vllm port")+ parser.add_argument("--device_id", type=str, default="0", help="device id")+ parser.add_argument(+ "--max_n_images",+ type=int,+ default=3,+ help="Maximum number of images to process",+ )+ parser.add_argument(+ "--dtype",+ type=str,+ choices=["auto", "half", "float16", "bfloat16", "float", "float32"],+ default="auto",+ help="Data type for VLLM model (default: auto)",+ )+ parser.add_argument(+ "--enforce_eager",+ action="store_true",+ help="Enforce eager execution mode for compatibility",+ )+ parser.add_argument(+ "--cache",+ action="store_true",+ help="Enable caching / local path serving instead of Azure mount",+ )+ parser.add_argument(+ "--cache_dir",+ type=str,+ default=None,+ help="Directory to cache downloaded models (default: ~/.cache/vllm_models)",+ )+ args = parser.parse_args()++ import uvicorn++ uvicorn.run(app, host="0.0.0.0", port=args.port)diff --git a/webeval/src/webeval/vllm/requirements.txt b/webeval/src/webeval/vllm/requirements.txtnew file mode 100644index 0000000..7dfc26d--- /dev/null+++ b/webeval/src/webeval/vllm/requirements.txt@@ -0,0 +1,7 @@+uvicorn+fastapi+vllm==0.19.1+transformers>=5.2.0+# 4.6.0.dev0 removed the `cute.core.ThrMma` alias that quack (a transitive+# vLLM dep) uses, crashing vLLM at startup. Unpin once resolved upstream.+nvidia-cutlass-dsl==4.5.2diff --git a/webeval/src/webeval/vllm/vllm_facade.py b/webeval/src/webeval/vllm/vllm_facade.pynew file mode 100644index 0000000..5080422--- /dev/null+++ b/webeval/src/webeval/vllm/vllm_facade.py@@ -0,0 +1,111 @@+import subprocess+import sys+from enum import Enum+import os+import logging+import threading+import time+++class Status(Enum):+ NotStarted = 0+ Running = 1+ Stopped = 2+++class VLLM:+ # Use the parent process's Python interpreter rather than a bare+ # ``python`` PATH lookup — the parent already chose its env (typically+ # ``fara_webeval``) and we want vLLM to come from that same env, not+ # whatever ``which python`` happens to point at (a stale ``.venv`` etc).+ cmd_template = " ".join(+ [+ f"{sys.executable} -O -u -m vllm.entrypoints.openai.api_server",+ "--host={host}",+ "--port={port}",+ "--model={model_dir}",+ "--served-model-name {model_name}",+ "--tensor-parallel-size {tensor_parallel_size}",+ "--gpu-memory-utilization 0.95",+ "--trust-remote-code",+ "--dtype {dtype}",+ ]+ )++ def __init__(+ self,+ model_path,+ max_n_images,+ device_id="0",+ host="0.0.0.0",+ port=5000,+ model_name="gpt-4o-mini-2024-07-18",+ dtype="auto",+ enforce_eager=False,+ ):+ self.model_path = model_path+ self.device_id = device_id+ self.host = host+ self.port = port+ self.max_n_images = int(max_n_images)+ self.dtype = dtype+ self.enforce_eager = enforce_eager+ self.cmd = VLLM.cmd_template+ if self.max_n_images > 1:+ # new versions of vllm require dictionary-like arguments for this+ # see https://docs.vllm.ai/en/latest/configuration/engine_args.html#multimodalconfig+ self.cmd += f" --limit-mm-per-prompt.image {self.max_n_images}"+ if (+ enforce_eager+ ): # Most helpful for float32 cases when attention backends are incompatible+ self.cmd += " --enforce-eager"+ self.model_name = model_name+ self.tensor_parallel_size = len(str(device_id).split(","))+ self.status = Status.NotStarted+ self.process = None+ self.logs = []++ @property+ def endpoint(self):+ return f"http://{self.host}:{self.port}/v1/"++ def start(self):+ def _drain(pipe):+ for line in iter(pipe.readline, ""):+ self.logs.append(line)+ print(line, end="")++ env = os.environ.copy()+ env["CUDA_VISIBLE_DEVICES"] = self.device_id+ env["NCCL_DEBUG"] = "TRACE"+ self.process = subprocess.Popen(+ self.cmd.format(+ host=self.host,+ port=self.port,+ model_dir=self.model_path,+ model_name=self.model_name,+ tensor_parallel_size=self.tensor_parallel_size,+ dtype=self.dtype,+ ).split(),+ stdout=subprocess.PIPE,+ stderr=subprocess.STDOUT,+ text=True,+ shell=False,+ env=env,+ )+ t = threading.Thread(target=_drain, args=(self.process.stdout,), daemon=True)+ t.start()++ while True:+ while not any(self.logs):+ time.sleep(1)+ line = self.logs.pop(0)+ if "Application startup complete." in line:+ logging.info("VLLM process started successfully.")+ self.status = Status.Running+ return True++ def stop(self):+ if self.process:+ self.process.terminate()+ self.status = Status.Stopped