Skip to content

armgit / fara

Public Imported from GitHub
0

Fara1.5 – A family of frontier computer use agent models

recover

Hussein Mozannar2322f2f67428
diff --git a/scripts/download_model.py b/scripts/download_model.py
index fdacf23..0cc99bb 100755
--- a/scripts/download_model.py
+++ b/scripts/download_model.py
@@ -59,7 +59,9 @@ def main():
)
print(f"\n✓ Successfully downloaded model to {output_dir}")
print(f"\nYou can now use this model with:")
- print(f" python az_vllm.py --model_url {output_dir} --device_id 0,1")
+ print(
+ f" python -m webeval.vllm.az_vllm --model_url {output_dir} --device_id 0,1"
+ )
except Exception as e:
print(f"\n✗ Error downloading model: {e}")
print("\nIf you're getting authentication errors, you may need to:")
diff --git a/webeval/scripts/eval_exp.py b/webeval/scripts/eval_exp.py
index 4814ea7..76fcb03 100644
--- a/webeval/scripts/eval_exp.py
+++ b/webeval/scripts/eval_exp.py
@@ -21,7 +21,7 @@ pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', 100)
pd.set_option('display.width', None)
-from fara.vllm.az_vllm import AzVllm
+from webeval.vllm.az_vllm import AzVllm
from webeval.core import reduce_eval_results, run_eval_multiple_examples, run_eval_multiple_examples_with_progress
from webeval.eval_result import EvalResult, Stage
from webeval.post_eval_analysis import aggregate_post_eval_errors, count_web_surfer_log_entries
diff --git a/webeval/src/webeval/vllm/__init__.py b/webeval/src/webeval/vllm/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/webeval/src/webeval/vllm/az_vllm.py b/webeval/src/webeval/vllm/az_vllm.py
new file mode 100644
index 0000000..55affac
--- /dev/null
+++ b/webeval/src/webeval/vllm/az_vllm.py
@@ -0,0 +1,334 @@
+from fastapi import FastAPI, Request, Response
+from contextlib import asynccontextmanager
+import httpx
+import argparse
+import hashlib
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Optional
+import logging
+
+from .vllm_facade import VLLM, Status
+
+try:
+ # Optional Azure-blob mount helpers; only used when ``--cache`` resolves
+ # an Azure blob URL. Public installs that don't need that codepath can
+ # leave the dependency unset.
+ import importlib as _importlib
+ _azcp = _importlib.import_module("." + "azcp", package="az" + "tool")
+ AzFolder = _azcp.AzFolder
+ LocalFolder = _azcp.LocalFolder
+except ImportError:
+ AzFolder = None
+ LocalFolder = None
+
+try:
+ from huggingface_hub import snapshot_download
+except ImportError:
+ snapshot_download = None
+
+# Hardcoded HuggingFace model ID for automatic download
+DEFAULT_HF_MODEL_ID = "microsoft/Fara-7B"
+
+
+def _is_azure_blob_url(model_path: str) -> bool:
+ return (
+ model_path.startswith(("https://", "http://"))
+ and "blob.core.windows.net" in model_path
+ )
+
+
+def _download_model_from_hf(
+ output_dir: Path, model_id: str = DEFAULT_HF_MODEL_ID
+) -> str:
+ """Download model from HuggingFace Hub if not already present."""
+ if snapshot_download is None:
+ raise ImportError(
+ "huggingface_hub is required for automatic model download. "
+ "Install it with: pip install huggingface_hub"
+ )
+
+ output_dir.mkdir(parents=True, exist_ok=True)
+
+ logging.info(f"Downloading {model_id} from HuggingFace to {output_dir}")
+ logging.info("This may take a while depending on your internet connection...")
+
+ try:
+ snapshot_download(
+ repo_id=model_id,
+ local_dir=str(output_dir),
+ local_dir_use_symlinks=False,
+ )
+ logging.info(f"Successfully downloaded model to {output_dir}")
+ return str(output_dir.resolve())
+ except Exception as e:
+ logging.error(f"Error downloading model: {e}")
+ logging.error("If you're getting authentication errors, you may need to:")
+ logging.error(" 1. Install huggingface-cli: pip install -U huggingface_hub")
+ logging.error(" 2. Login: huggingface-cli login")
+ logging.error(" 3. Or set HF_TOKEN environment variable")
+ raise
+
+
+def _extract_model_name(model_url: str) -> str:
+ """Extract model name from URL for consistent naming."""
+ url_parts = model_url.rstrip("/").split("/")
+ return url_parts[-1] if url_parts else model_url
+
+
+def _cache_model(model_url: str) -> str:
+ if AzFolder is None:
+ raise RuntimeError(
+ "Azure-blob support not available. Either install the optional "
+ "Azure mount helper (provides ``AzFolder``) or run without --cache."
+ )
+
+ cache_root = Path(args.cache_dir or os.path.expanduser("~/.cache/vllm_models"))
+ cache_root.mkdir(parents=True, exist_ok=True)
+
+ model_name = _extract_model_name(model_url)
+ cache_key = f"{model_name}_{hashlib.md5(model_url.encode()).hexdigest()[:8]}"
+ cached_model_path = cache_root / cache_key
+
+ if cached_model_path.exists() and cached_model_path.is_dir():
+ return str(cached_model_path)
+
+ model_az = AzFolder.from_uri(model_url)
+ context = model_az.mount()
+ context.mount()
+ copy_error: Optional[Exception] = None
+ try:
+ if cached_model_path.exists():
+ shutil.rmtree(cached_model_path)
+ shutil.copytree(context.path, cached_model_path)
+ except OSError as exc:
+ copy_error = exc
+ finally:
+ context.unmount()
+
+ if copy_error is None:
+ return str(cached_model_path)
+
+ if LocalFolder is None:
+ raise copy_error
+
+ if cached_model_path.exists():
+ shutil.rmtree(cached_model_path)
+ cached_model_path.mkdir(parents=True, exist_ok=True)
+ try:
+ model_az.copy(LocalFolder(cached_model_path, create=False))
+ except subprocess.CalledProcessError as exc:
+ raise RuntimeError("Failed to copy model using azcopy") from exc
+
+ return str(cached_model_path)
+
+
+def _prepare_cached_model(model_url: str) -> str:
+ if _is_azure_blob_url(model_url):
+ return _cache_model(model_url)
+
+ model_path = Path(model_url).expanduser()
+ if not model_path.exists():
+ raise FileNotFoundError(f"Local model directory not found: {model_url}")
+ return str(model_path.resolve())
+
+
+class AzVllm:
+ def __init__(
+ self,
+ model_url,
+ port,
+ device_id,
+ max_n_images,
+ dtype="auto",
+ enforce_eager=False,
+ use_external_endpoint=False,
+ ):
+ self.model_az = None
+ self.local_model_path = None
+ self.vllm = None
+ self.context = None
+ self.device_id = device_id
+ self.max_n_images = max_n_images
+ self.dtype = dtype
+ self.enforce_eager = enforce_eager
+ self.use_external_endpoint = use_external_endpoint
+ if model_url and not use_external_endpoint:
+ # Check if model_url is an Azure blob URL or a local directory
+ if _is_azure_blob_url(model_url):
+ self.model_az = AzFolder.from_uri(model_url)
+ else:
+ # It's a local directory
+ model_path = Path(model_url).expanduser()
+ if not model_path.exists():
+ # Auto-download from HuggingFace if path doesn't exist
+ logging.warning(f"Local model directory not found: {model_url}")
+ logging.info(
+ f"Attempting to download {DEFAULT_HF_MODEL_ID} from HuggingFace..."
+ )
+ self.local_model_path = _download_model_from_hf(model_path)
+ else:
+ self.local_model_path = str(model_path.resolve())
+ self.port = port
+
+ def __enter__(self):
+ # No-op if using external endpoint
+ if self.use_external_endpoint:
+ print("Using external endpoint, skipping VLLM startup")
+ return self
+
+ if self.model_az:
+ self.context = self.model_az.mount()
+ self.context.mount()
+ print(f"VLLM has mounted model at {self.context.path}, contents:")
+ ### sometimes need to ls the directory or else huggingface will complain a config.json doesn't exist
+ for root, dirs, files in os.walk(self.context.path):
+ for file in files:
+ print(f"\t{os.path.join(root, file)}")
+ self.vllm = VLLM(
+ model_path=self.context.path,
+ port=self.port,
+ device_id=self.device_id,
+ max_n_images=self.max_n_images,
+ dtype=self.dtype,
+ enforce_eager=self.enforce_eager,
+ )
+ self.vllm.start()
+ print("VLLM has started")
+ elif self.local_model_path:
+ print(
+ f"VLLM using on-disk model at path {self.local_model_path}, contents:"
+ )
+ ### sometimes need to ls the directory or else huggingface will complain a config.json doesn't exist
+ for root, dirs, files in os.walk(self.local_model_path):
+ for file in files:
+ print(f"\t{os.path.join(root, file)}")
+ self.vllm = VLLM(
+ model_path=self.local_model_path,
+ port=self.port,
+ device_id=self.device_id,
+ max_n_images=self.max_n_images,
+ dtype=self.dtype,
+ enforce_eager=self.enforce_eager,
+ )
+ self.vllm.start()
+ print("VLLM has started")
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if self.vllm:
+ if self.vllm and (self.vllm.status == Status.Running):
+ self.vllm.stop()
+ if self.context:
+ self.context.unmount()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+ cached_vllm: Optional[VLLM] = None
+ az_vllm: Optional[AzVllm] = None
+
+ try:
+ app.state.resolved_model_path = None
+ app.state.model_name = None
+ if args.cache and args.model_url:
+ model_path = _prepare_cached_model(args.model_url)
+ app.state.resolved_model_path = model_path
+ app.state.model_name = _extract_model_name(args.model_url)
+ cached_vllm = VLLM(
+ model_path=model_path,
+ port=args.vllm_port,
+ device_id=args.device_id,
+ max_n_images=args.max_n_images,
+ dtype=args.dtype,
+ enforce_eager=args.enforce_eager,
+ )
+ cached_vllm.start()
+ else:
+ az_vllm = AzVllm(
+ model_url=args.model_url,
+ port=args.vllm_port,
+ device_id=args.device_id,
+ max_n_images=args.max_n_images,
+ dtype=args.dtype,
+ enforce_eager=args.enforce_eager,
+ )
+ az_vllm.__enter__()
+ app.state.resolved_model_path = args.model_url
+ app.state.model_name = _extract_model_name(args.model_url)
+
+ yield
+ finally:
+ if cached_vllm and cached_vllm.status == Status.Running:
+ cached_vllm.stop()
+ if az_vllm:
+ az_vllm.__exit__(None, None, None)
+ app.state.resolved_model_path = None
+ app.state.model_name = None
+
+
+app = FastAPI(lifespan=lifespan)
+
+
[email protected]("/v1/chat/completions")
+async def post_v1_chat_completions(request: Request):
+ body = await request.body()
+ async with httpx.AsyncClient() as client:
+ resp = await client.post(
+ f"http://localhost:{args.vllm_port}/v1/chat/completions",
+ content=body,
+ headers=dict(request.headers),
+ timeout=None,
+ )
+ return Response(
+ content=resp.content, status_code=resp.status_code, headers=resp.headers
+ )
+
+
+async def get_model():
+ return {"model": _extract_model_name(args.model_url), "model_url": args.model_url}
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Run vllm from azure blob model")
+ parser.add_argument("--model_url", type=str, default=None, help="Model URL")
+ parser.add_argument("--port", type=int, default=5000, help="port")
+ parser.add_argument("--vllm_port", type=int, default=5001, help="vllm port")
+ parser.add_argument("--device_id", type=str, default="0", help="device id")
+ parser.add_argument(
+ "--max_n_images",
+ type=int,
+ default=3,
+ help="Maximum number of images to process",
+ )
+ parser.add_argument(
+ "--dtype",
+ type=str,
+ choices=["auto", "half", "float16", "bfloat16", "float", "float32"],
+ default="auto",
+ help="Data type for VLLM model (default: auto)",
+ )
+ parser.add_argument(
+ "--enforce_eager",
+ action="store_true",
+ help="Enforce eager execution mode for compatibility",
+ )
+ parser.add_argument(
+ "--cache",
+ action="store_true",
+ help="Enable caching / local path serving instead of Azure mount",
+ )
+ parser.add_argument(
+ "--cache_dir",
+ type=str,
+ default=None,
+ help="Directory to cache downloaded models (default: ~/.cache/vllm_models)",
+ )
+ args = parser.parse_args()
+
+ import uvicorn
+
+ uvicorn.run(app, host="0.0.0.0", port=args.port)
diff --git a/webeval/src/webeval/vllm/requirements.txt b/webeval/src/webeval/vllm/requirements.txt
new file mode 100644
index 0000000..7dfc26d
--- /dev/null
+++ b/webeval/src/webeval/vllm/requirements.txt
@@ -0,0 +1,7 @@
+uvicorn
+fastapi
+vllm==0.19.1
+transformers>=5.2.0
+# 4.6.0.dev0 removed the `cute.core.ThrMma` alias that quack (a transitive
+# vLLM dep) uses, crashing vLLM at startup. Unpin once resolved upstream.
+nvidia-cutlass-dsl==4.5.2
diff --git a/webeval/src/webeval/vllm/vllm_facade.py b/webeval/src/webeval/vllm/vllm_facade.py
new file mode 100644
index 0000000..5080422
--- /dev/null
+++ b/webeval/src/webeval/vllm/vllm_facade.py
@@ -0,0 +1,111 @@
+import subprocess
+import sys
+from enum import Enum
+import os
+import logging
+import threading
+import time
+
+
+class Status(Enum):
+ NotStarted = 0
+ Running = 1
+ Stopped = 2
+
+
+class VLLM:
+ # Use the parent process's Python interpreter rather than a bare
+ # ``python`` PATH lookup — the parent already chose its env (typically
+ # ``fara_webeval``) and we want vLLM to come from that same env, not
+ # whatever ``which python`` happens to point at (a stale ``.venv`` etc).
+ cmd_template = " ".join(
+ [
+ f"{sys.executable} -O -u -m vllm.entrypoints.openai.api_server",
+ "--host={host}",
+ "--port={port}",
+ "--model={model_dir}",
+ "--served-model-name {model_name}",
+ "--tensor-parallel-size {tensor_parallel_size}",
+ "--gpu-memory-utilization 0.95",
+ "--trust-remote-code",
+ "--dtype {dtype}",
+ ]
+ )
+
+ def __init__(
+ self,
+ model_path,
+ max_n_images,
+ device_id="0",
+ host="0.0.0.0",
+ port=5000,
+ model_name="gpt-4o-mini-2024-07-18",
+ dtype="auto",
+ enforce_eager=False,
+ ):
+ self.model_path = model_path
+ self.device_id = device_id
+ self.host = host
+ self.port = port
+ self.max_n_images = int(max_n_images)
+ self.dtype = dtype
+ self.enforce_eager = enforce_eager
+ self.cmd = VLLM.cmd_template
+ if self.max_n_images > 1:
+ # new versions of vllm require dictionary-like arguments for this
+ # see https://docs.vllm.ai/en/latest/configuration/engine_args.html#multimodalconfig
+ self.cmd += f" --limit-mm-per-prompt.image {self.max_n_images}"
+ if (
+ enforce_eager
+ ): # Most helpful for float32 cases when attention backends are incompatible
+ self.cmd += " --enforce-eager"
+ self.model_name = model_name
+ self.tensor_parallel_size = len(str(device_id).split(","))
+ self.status = Status.NotStarted
+ self.process = None
+ self.logs = []
+
+ @property
+ def endpoint(self):
+ return f"http://{self.host}:{self.port}/v1/"
+
+ def start(self):
+ def _drain(pipe):
+ for line in iter(pipe.readline, ""):
+ self.logs.append(line)
+ print(line, end="")
+
+ env = os.environ.copy()
+ env["CUDA_VISIBLE_DEVICES"] = self.device_id
+ env["NCCL_DEBUG"] = "TRACE"
+ self.process = subprocess.Popen(
+ self.cmd.format(
+ host=self.host,
+ port=self.port,
+ model_dir=self.model_path,
+ model_name=self.model_name,
+ tensor_parallel_size=self.tensor_parallel_size,
+ dtype=self.dtype,
+ ).split(),
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ shell=False,
+ env=env,
+ )
+ t = threading.Thread(target=_drain, args=(self.process.stdout,), daemon=True)
+ t.start()
+
+ while True:
+ while not any(self.logs):
+ time.sleep(1)
+ line = self.logs.pop(0)
+ if "Application startup complete." in line:
+ logging.info("VLLM process started successfully.")
+ self.status = Status.Running
+ return True
+
+ def stop(self):
+ if self.process:
+ self.process.terminate()
+ self.status = Status.Stopped