Skip to content

armgit / fara

Public Imported from GitHub
0

Fara1.5 – A family of frontier computer use agent models

revert back az vllm

Hussein Mozannar1d0e1451501b
diff --git a/src/fara/vllm/__init__.py b/src/fara/vllm/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/fara/vllm/az_vllm.py b/src/fara/vllm/az_vllm.py
new file mode 100644
index 0000000..198f1fe
--- /dev/null
+++ b/src/fara/vllm/az_vllm.py
@@ -0,0 +1,239 @@
+from fastapi import FastAPI, Request, Response
+from contextlib import asynccontextmanager
+import httpx
+import argparse
+import hashlib
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from typing import Optional
+
+from vllm_facade import VLLM, Status
+
+try:
+ from aztool.azcp import AzFolder, LocalFolder
+except ImportError: # keep old behaviour when aztool missing
+ AzFolder = None
+ LocalFolder = None
+
+
+def _is_azure_blob_url(model_path: str) -> bool:
+ return model_path.startswith(("https://", "http://")) and "blob.core.windows.net" in model_path
+
+
+def _extract_model_name(model_url: str) -> str:
+ """Extract model name from URL for consistent naming."""
+ url_parts = model_url.rstrip('/').split('/')
+ return url_parts[-1] if url_parts else model_url
+
+
+def _cache_model(model_url: str) -> str:
+ if AzFolder is None:
+ raise RuntimeError("Azure support not available. Install aztool or run without --cache.")
+
+ cache_root = Path(args.cache_dir or os.path.expanduser("~/.cache/vllm_models"))
+ cache_root.mkdir(parents=True, exist_ok=True)
+
+ model_name = _extract_model_name(model_url)
+ cache_key = f"{model_name}_{hashlib.md5(model_url.encode()).hexdigest()[:8]}"
+ cached_model_path = cache_root / cache_key
+
+ if cached_model_path.exists() and cached_model_path.is_dir():
+ return str(cached_model_path)
+
+ model_az = AzFolder.from_uri(model_url)
+ context = model_az.mount()
+ context.mount()
+ copy_error: Optional[Exception] = None
+ try:
+ if cached_model_path.exists():
+ shutil.rmtree(cached_model_path)
+ shutil.copytree(context.path, cached_model_path)
+ except OSError as exc:
+ copy_error = exc
+ finally:
+ context.unmount()
+
+ if copy_error is None:
+ return str(cached_model_path)
+
+ if LocalFolder is None:
+ raise copy_error
+
+ if cached_model_path.exists():
+ shutil.rmtree(cached_model_path)
+ cached_model_path.mkdir(parents=True, exist_ok=True)
+ try:
+ model_az.copy(LocalFolder(cached_model_path, create=False))
+ except subprocess.CalledProcessError as exc:
+ raise RuntimeError("Failed to copy model using azcopy") from exc
+
+ return str(cached_model_path)
+
+
+def _prepare_cached_model(model_url: str) -> str:
+ if _is_azure_blob_url(model_url):
+ return _cache_model(model_url)
+
+ model_path = Path(model_url).expanduser()
+ if not model_path.exists():
+ raise FileNotFoundError(f"Local model directory not found: {model_url}")
+ return str(model_path.resolve())
+
+class AzVllm:
+ def __init__(self, model_url, port, device_id, max_n_images, dtype='auto', enforce_eager=False, use_external_endpoint=False):
+ self.model_az = None
+ self.local_model_path = None
+ self.vllm = None
+ self.context = None
+ self.device_id = device_id
+ self.max_n_images = max_n_images
+ self.dtype = dtype
+ self.enforce_eager = enforce_eager
+ self.use_external_endpoint = use_external_endpoint
+ if model_url and not use_external_endpoint:
+ # Check if model_url is an Azure blob URL or a local directory
+ if _is_azure_blob_url(model_url):
+ self.model_az = AzFolder.from_uri(model_url)
+ else:
+ # It's a local directory
+ model_path = Path(model_url).expanduser()
+ if not model_path.exists():
+ raise FileNotFoundError(f"Local model directory not found: {model_url}")
+ self.local_model_path = str(model_path.resolve())
+ self.port = port
+
+ def __enter__(self):
+ # No-op if using external endpoint
+ if self.use_external_endpoint:
+ print('Using external endpoint, skipping VLLM startup')
+ return self
+
+ if self.model_az:
+ self.context = self.model_az.mount()
+ self.context.mount()
+ print(f"VLLM has mounted model at {self.context.path}, contents:")
+ ### sometimes need to ls the directory or else huggingface will complain a config.json doesn't exist
+ for root, dirs, files in os.walk(self.context.path):
+ for file in files:
+ print(f"\t{os.path.join(root, file)}")
+ self.vllm = VLLM(
+ model_path = self.context.path,
+ port = self.port,
+ device_id = self.device_id,
+ max_n_images = self.max_n_images,
+ dtype = self.dtype,
+ enforce_eager = self.enforce_eager
+ )
+ self.vllm.start()
+ print('VLLM has started')
+ elif self.local_model_path:
+ print(f"VLLM using on-disk model at path {self.local_model_path}, contents:")
+ ### sometimes need to ls the directory or else huggingface will complain a config.json doesn't exist
+ for root, dirs, files in os.walk(self.local_model_path):
+ for file in files:
+ print(f"\t{os.path.join(root, file)}")
+ self.vllm = VLLM(
+ model_path = self.local_model_path,
+ port = self.port,
+ device_id = self.device_id,
+ max_n_images = self.max_n_images,
+ dtype = self.dtype,
+ enforce_eager = self.enforce_eager
+ )
+ self.vllm.start()
+ print('VLLM has started')
+ return self
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ if self.vllm:
+ if self.vllm and (self.vllm.status == Status.Running):
+ self.vllm.stop()
+ if self.context:
+ self.context.unmount()
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+ cached_vllm: Optional[VLLM] = None
+ az_vllm: Optional[AzVllm] = None
+
+ try:
+ app.state.resolved_model_path = None
+ app.state.model_name = None
+ if args.cache and args.model_url:
+ model_path = _prepare_cached_model(args.model_url)
+ app.state.resolved_model_path = model_path
+ app.state.model_name = _extract_model_name(args.model_url)
+ cached_vllm = VLLM(
+ model_path=model_path,
+ port=args.vllm_port,
+ device_id=args.device_id,
+ max_n_images=args.max_n_images,
+ dtype=args.dtype,
+ enforce_eager=args.enforce_eager
+ )
+ cached_vllm.start()
+ else:
+ az_vllm = AzVllm(
+ model_url = args.model_url,
+ port = args.vllm_port,
+ device_id = args.device_id,
+ max_n_images = args.max_n_images,
+ dtype = args.dtype,
+ enforce_eager = args.enforce_eager)
+ az_vllm.__enter__()
+ app.state.resolved_model_path = args.model_url
+ app.state.model_name = _extract_model_name(args.model_url)
+
+ yield
+ finally:
+ if cached_vllm and cached_vllm.status == Status.Running:
+ cached_vllm.stop()
+ if az_vllm:
+ az_vllm.__exit__(None, None, None)
+ app.state.resolved_model_path = None
+ app.state.model_name = None
+
+
+app = FastAPI(lifespan = lifespan)
+
+
[email protected]("/v1/chat/completions")
+async def post_v1_chat_completions(request: Request):
+ body = await request.body()
+ async with httpx.AsyncClient() as client:
+ resp = await client.post(
+ f'http://localhost:{args.vllm_port}/v1/chat/completions',
+ content=body,
+ headers=dict(request.headers),
+ timeout=None
+ )
+ return Response(
+ content=resp.content,
+ status_code=resp.status_code,
+ headers=resp.headers
+ )
+
+
+async def get_model():
+
+ return {"model": _extract_model_name(args.model_url), "model_url": args.model_url}
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Run vllm from azure blob model")
+ parser.add_argument("--model_url", type=str, default=None, help="Model URL")
+ parser.add_argument("--port", type=int, default=5000, help="port")
+ parser.add_argument("--vllm_port", type=int, default=5001, help="vllm port")
+ parser.add_argument("--device_id", type=str, default="0", help="device id")
+ parser.add_argument("--max_n_images", type=int, default=3, help="Maximum number of images to process")
+ parser.add_argument('--dtype', type=str, choices=['auto', 'half', 'float16', 'bfloat16', 'float', 'float32'], default='auto', help='Data type for VLLM model (default: auto)')
+ parser.add_argument('--enforce_eager', action='store_true', help='Enforce eager execution mode for compatibility')
+ parser.add_argument('--cache', action='store_true', help='Enable caching / local path serving instead of Azure mount')
+ parser.add_argument('--cache_dir', type=str, default=None, help='Directory to cache downloaded models (default: ~/.cache/vllm_models)')
+ args = parser.parse_args()
+
+ import uvicorn
+ uvicorn.run(app, host="0.0.0.0", port=args.port)
\ No newline at end of file
diff --git a/src/fara/vllm/requirements.txt b/src/fara/vllm/requirements.txt
new file mode 100644
index 0000000..f78d6a8
--- /dev/null
+++ b/src/fara/vllm/requirements.txt
@@ -0,0 +1,4 @@
+uvicorn
+fastapi
+vllm==0.10.0
+transformers==4.55.0
\ No newline at end of file
diff --git a/src/fara/vllm/vllm_facade.py b/src/fara/vllm/vllm_facade.py
new file mode 100644
index 0000000..73cb8d4
--- /dev/null
+++ b/src/fara/vllm/vllm_facade.py
@@ -0,0 +1,96 @@
+import subprocess
+from enum import Enum
+import os
+import logging
+import threading
+import time
+
+class Status(Enum):
+ NotStarted = 0
+ Running = 1
+ Stopped = 2
+
+class VLLM:
+ cmd_template = ' '.join([
+ "python -O -u -m vllm.entrypoints.openai.api_server",
+ "--host={host}",
+ "--port={port}",
+ "--model={model_dir}",
+ "--served-model-name {model_name}",
+ "--tensor-parallel-size {tensor_parallel_size}",
+ "--gpu-memory-utilization 0.95",
+ "--trust-remote-code",
+ "--dtype {dtype}"
+ ])
+ def __init__(self,
+ model_path,
+ max_n_images,
+ device_id = "0",
+ host = "0.0.0.0",
+ port = 5000,
+ model_name = "gpt-4o-mini-2024-07-18",
+ dtype = "auto",
+ enforce_eager = False):
+ self.model_path = model_path
+ self.device_id = device_id
+ self.host = host
+ self.port = port
+ self.max_n_images = int(max_n_images)
+ self.dtype = dtype
+ self.enforce_eager = enforce_eager
+ self.cmd = VLLM.cmd_template
+ if self.max_n_images > 1:
+ # new versions of vllm require dictionary-like arguments for this
+ # see https://docs.vllm.ai/en/latest/configuration/engine_args.html#multimodalconfig
+ self.cmd += f" --limit-mm-per-prompt.image {self.max_n_images}"
+ if enforce_eager: # Most helpful for float32 cases when attention backends are incompatible
+ self.cmd += " --enforce-eager"
+ self.model_name = model_name
+ self.tensor_parallel_size = len(str(device_id).split(','))
+ self.status = Status.NotStarted
+ self.process = None
+ self.logs = []
+
+ @property
+ def endpoint(self):
+ return f"http://{self.host}:{self.port}/v1/"
+
+ def start(self):
+ def _drain(pipe):
+ for line in iter(pipe.readline, ''):
+ self.logs.append(line)
+ print(line, end='')
+ env = os.environ.copy()
+ env['CUDA_VISIBLE_DEVICES'] = self.device_id
+ env['NCCL_DEBUG'] = "TRACE"
+ self.process = subprocess.Popen(
+ self.cmd.format(
+ host=self.host,
+ port=self.port,
+ model_dir=self.model_path,
+ model_name=self.model_name,
+ tensor_parallel_size=self.tensor_parallel_size,
+ dtype=self.dtype
+ ).split(),
+ stdout = subprocess.PIPE,
+ stderr = subprocess.STDOUT,
+ text = True,
+ shell = False,
+ env = env
+ )
+ t = threading.Thread(target=_drain, args=(self.process.stdout,), daemon=True)
+ t.start()
+
+ while True:
+ while not any(self.logs):
+ time.sleep(1)
+ line = self.logs.pop(0)
+ if "Application startup complete." in line:
+ logging.info("VLLM process started successfully.")
+ self.status = Status.Running
+ return True
+
+ def stop(self):
+ if self.process:
+ self.process.terminate()
+ self.status = Status.Stopped
\ No newline at end of file