Personal AI, On Personal Devices
Python
feat(engine): add NVIDIA NIM inference engine support (#291)
Anubhav Chaturvedi4780499ae29d
diff --git a/src/openjarvis/engine/__init__.py b/src/openjarvis/engine/__init__.pyindex d575da8f..484bf511 100644--- a/src/openjarvis/engine/__init__.py+++ b/src/openjarvis/engine/__init__.py@@ -5,6 +5,7 @@ from __future__ import annotationsimport importlib# Import engine modules to trigger @EngineRegistry.register() decorators+import openjarvis.engine.nim # noqa: F401import openjarvis.engine.ollama # noqa: F401import openjarvis.engine.openai_compat_engines # noqa: F401from openjarvis.engine._base import (diff --git a/src/openjarvis/engine/nim.py b/src/openjarvis/engine/nim.pynew file mode 100644index 00000000..31da6279--- /dev/null+++ b/src/openjarvis/engine/nim.py@@ -0,0 +1,283 @@+"""NVIDIA NIM inference engine — self-hosted inference microservices."""++from __future__ import annotations++import json+import logging+import os+from collections.abc import AsyncIterator, Sequence+from typing import Any, Dict, List++import httpx++from openjarvis.core.registry import EngineRegistry+from openjarvis.core.types import Message+from openjarvis.engine._base import (+ EngineConnectionError,+ InferenceEngine,+ estimate_prompt_tokens,+ messages_to_dicts,+)+from openjarvis.engine._stubs import StreamChunk++logger = logging.getLogger(__name__)++[email protected]("nim")+class NIMEngine(InferenceEngine):+ """NVIDIA NIM inference engine.++ Supports local and remote NIM deployments with OpenAI-compatible API.+ Requires NIM_API_KEY for NVIDIA-hosted NIM endpoints, optional for self-hosted.+ """++ engine_id = "nim"+ _default_host = "https://integrate.api.nvidia.com"+ _api_prefix = "/v1"++ def __init__(self, host: str | None = None, *, timeout: float = 600.0) -> None:+ env_host = os.environ.get("NIM_HOST")+ self._host = (host or env_host or self._default_host).rstrip("/")++ api_key = os.environ.get("NIM_API_KEY", "")+ self._api_key = api_key if api_key else None++ headers = {}+ if self._api_key:+ headers["Authorization"] = f"Bearer {self._api_key}"++ self._timeout = timeout+ self._client = httpx.Client(+ base_url=self._host,+ timeout=timeout,+ headers=headers,+ )++ def _get_headers(self) -> Dict[str, str]:+ """Get headers for requests, including auth if configured."""+ headers = {}+ if self._api_key:+ headers["Authorization"] = f"Bearer {self._api_key}"+ return headers++ def generate(+ self,+ messages: Sequence[Message],+ *,+ model: str,+ temperature: float = 0.7,+ max_tokens: int = 1024,+ **kwargs: Any,+ ) -> Dict[str, Any]:+ payload: Dict[str, Any] = {+ "model": model,+ "messages": messages_to_dicts(messages),+ "temperature": temperature,+ "max_tokens": max_tokens,+ "stream": False,+ **kwargs,+ }+ if "tools" in payload and "tool_choice" not in payload:+ payload["tool_choice"] = "auto"+ try:+ url = f"{self._api_prefix}/chat/completions"+ resp = self._client.post(url, json=payload, headers=self._get_headers())+ resp.raise_for_status()+ except (httpx.ConnectError, httpx.TimeoutException) as exc:+ raise EngineConnectionError(+ f"NIM engine not reachable at {self._host}"+ ) from exc+ data = resp.json()+ choices = data.get("choices", [])+ if not choices:+ return {+ "content": "",+ "usage": data.get("usage", {}),+ "model": data.get("model", model),+ "finish_reason": "error",+ }+ choice = choices[0]+ usage = data.get("usage", {})+ reported_prompt = usage.get("prompt_tokens", 0)+ estimated_prompt = estimate_prompt_tokens(messages)+ prompt_tokens = max(reported_prompt, estimated_prompt)+ completion_tokens = usage.get("completion_tokens", 0)+ result: Dict[str, Any] = {+ "content": choice["message"].get("content") or "",+ "usage": {+ "prompt_tokens": prompt_tokens,+ "prompt_tokens_evaluated": reported_prompt or prompt_tokens,+ "completion_tokens": completion_tokens,+ "total_tokens": prompt_tokens + completion_tokens,+ },+ "model": data.get("model", model),+ "finish_reason": choice.get("finish_reason", "stop"),+ }+ raw_tool_calls = choice["message"].get("tool_calls", [])+ if raw_tool_calls:+ result["tool_calls"] = [+ {+ "id": tc.get("id", ""),+ "name": tc.get("function", {}).get("name", ""),+ "arguments": tc.get("function", {}).get("arguments", "{}"),+ }+ for tc in raw_tool_calls+ ]+ return result++ async def stream(+ self,+ messages: Sequence[Message],+ *,+ model: str,+ temperature: float = 0.7,+ max_tokens: int = 1024,+ **kwargs: Any,+ ) -> AsyncIterator[str]:+ payload: Dict[str, Any] = {+ "model": model,+ "messages": messages_to_dicts(messages),+ "temperature": temperature,+ "max_tokens": max_tokens,+ "stream": True,+ **kwargs,+ }+ if "tools" in payload and "tool_choice" not in payload:+ payload["tool_choice"] = "auto"+ try:+ url = f"{self._api_prefix}/chat/completions"+ async with httpx.AsyncClient(+ base_url=self._host,+ timeout=self._timeout,+ headers=self._get_headers(),+ ) as client:+ async with client.stream("POST", url, json=payload) as resp:+ resp.raise_for_status()+ async for line in resp.aiter_lines():+ if not line.startswith("data:"):+ continue+ data_str = line[len("data:") :].strip()+ if data_str == "[DONE]":+ break+ try:+ chunk = json.loads(data_str)+ except json.JSONDecodeError:+ continue+ choices = chunk.get("choices", [])+ if not choices:+ continue+ delta = choices[0].get("delta", {})+ content = delta.get("content")+ if content:+ yield content+ except (httpx.ConnectError, httpx.TimeoutException) as exc:+ raise EngineConnectionError(+ f"NIM engine not reachable at {self._host}"+ ) from exc++ async def stream_full(+ self,+ messages: Sequence[Message],+ *,+ model: str,+ temperature: float = 0.7,+ max_tokens: int = 1024,+ **kwargs: Any,+ ) -> AsyncIterator["StreamChunk"]:+ """Yield StreamChunks with content, tool_calls, and finish_reason."""+ msg_dicts = messages_to_dicts(messages)+ payload: Dict[str, Any] = {+ "model": model,+ "messages": msg_dicts,+ "temperature": temperature,+ "max_tokens": max_tokens,+ "stream": True,+ **kwargs,+ }+ if "tools" in payload and "tool_choice" not in payload:+ payload["tool_choice"] = "auto"+ try:+ url = f"{self._api_prefix}/chat/completions"+ async with httpx.AsyncClient(+ base_url=self._host,+ timeout=self._timeout,+ headers=self._get_headers(),+ ) as client:+ async with client.stream("POST", url, json=payload) as resp:+ resp.raise_for_status()+ async for line in resp.aiter_lines():+ if not line.startswith("data:"):+ continue+ data_str = line[len("data:") :].strip()+ if data_str == "[DONE]":+ break+ try:+ chunk = json.loads(data_str)+ except json.JSONDecodeError:+ continue+ choices = chunk.get("choices", [])+ if not choices:+ continue+ choice = choices[0]+ delta = choice.get("delta", {})+ finish = choice.get("finish_reason")+ content = delta.get("content")+ tool_calls = delta.get("tool_calls")+ usage = chunk.get("usage")++ if content or tool_calls or finish or usage:+ yield StreamChunk(+ content=content,+ tool_calls=tool_calls,+ finish_reason=finish,+ usage=usage,+ )+ except (httpx.ConnectError, httpx.TimeoutException) as exc:+ raise EngineConnectionError(+ f"NIM engine not reachable at {self._host}"+ ) from exc++ def list_models(self) -> List[str]:+ try:+ resp = self._client.get(+ f"{self._api_prefix}/models",+ headers=self._get_headers(),+ )+ resp.raise_for_status()+ except (+ httpx.ConnectError,+ httpx.TimeoutException,+ httpx.HTTPStatusError,+ ) as exc:+ logger.warning("Failed to list models from NIM at %s: %s", self._host, exc)+ return []+ data = resp.json()+ return [m["id"] for m in data.get("data", [])]++ def health(self) -> bool:+ try:+ resp = self._client.get(+ f"{self._api_prefix}/health/ready",+ timeout=2.0,+ headers=self._get_headers(),+ )+ if resp.status_code == 200:+ return True+ except Exception:+ pass+ try:+ resp = self._client.get(+ f"{self._api_prefix}/models",+ timeout=2.0,+ headers=self._get_headers(),+ )+ return resp.status_code == 200+ except Exception as exc:+ logger.debug("NIM health check failed at %s: %s", self._host, exc)+ return False++ def close(self) -> None:+ self._client.close()++ def __repr__(self) -> str:+ return f"NIMEngine(host={self._host})"diff --git a/tests/engine/test_nim.py b/tests/engine/test_nim.pynew file mode 100644index 00000000..4158ad2b--- /dev/null+++ b/tests/engine/test_nim.py@@ -0,0 +1,135 @@+"""Tests for the NVIDIA NIM inference engine."""++from __future__ import annotations++import json+from unittest.mock import MagicMock++import httpx+import pytest++from openjarvis.core.types import Message, Role+from openjarvis.engine._base import EngineConnectionError+from openjarvis.engine.nim import NIMEngine++_HOST = "https://nim.test"+_CHAT_URL = f"{_HOST}/v1/chat/completions"+++def _message() -> list[Message]:+ return [Message(role=Role.USER, content="Hello")]+++def test_generate_preserves_tools_on_unsupported_request(respx_mock) -> None:+ """A 400 must not be retried after silently removing requested tools."""+ route = respx_mock.post(_CHAT_URL).mock(+ return_value=httpx.Response(400, json={"detail": "tools unsupported"})+ )+ engine = NIMEngine(host=_HOST)++ with pytest.raises(httpx.HTTPStatusError):+ engine.generate(+ _message(),+ model="nim-model",+ tools=[{"type": "function", "function": {"name": "lookup"}}],+ )++ assert route.call_count == 1+ payload = json.loads(route.calls[0].request.content)+ assert payload["tools"][0]["function"]["name"] == "lookup"+ assert payload["tool_choice"] == "auto"+ engine.close()+++async def test_stream_uses_async_http_and_sends_auth(respx_mock, monkeypatch) -> None:+ monkeypatch.setenv("NIM_API_KEY", "secret-token")+ body = "\n".join(+ [+ 'data: {"choices":[{"delta":{"content":"Hello"}}]}',+ 'data: {"choices":[{"delta":{"content":" world"}}]}',+ "data: [DONE]",+ ]+ )+ route = respx_mock.post(_CHAT_URL).mock(return_value=httpx.Response(200, text=body))+ engine = NIMEngine(host=_HOST)+ engine._client.stream = MagicMock( # type: ignore[method-assign]+ side_effect=AssertionError("sync client used from async stream")+ )++ chunks = [chunk async for chunk in engine.stream(_message(), model="nim-model")]++ assert chunks == ["Hello", " world"]+ assert route.calls[0].request.headers["Authorization"] == "Bearer secret-token"+ engine._client.close()+++async def test_stream_full_parses_tool_and_usage_chunks(respx_mock) -> None:+ body = "\n".join(+ [+ "data: "+ + json.dumps(+ {+ "choices": [+ {+ "delta": {+ "tool_calls": [+ {+ "index": 0,+ "function": {+ "name": "lookup",+ "arguments": "{}",+ },+ }+ ]+ },+ "finish_reason": None,+ }+ ]+ }+ ),+ "data: "+ + json.dumps(+ {+ "choices": [{"delta": {}, "finish_reason": "tool_calls"}],+ "usage": {"total_tokens": 9},+ }+ ),+ "data: [DONE]",+ ]+ )+ respx_mock.post(_CHAT_URL).mock(return_value=httpx.Response(200, text=body))+ engine = NIMEngine(host=_HOST)++ chunks = [+ chunk async for chunk in engine.stream_full(_message(), model="nim-model")+ ]++ assert chunks[0].tool_calls is not None+ assert chunks[0].tool_calls[0]["function"]["name"] == "lookup"+ assert chunks[1].finish_reason == "tool_calls"+ assert chunks[1].usage == {"total_tokens": 9}+ engine.close()+++def test_generate_maps_connection_errors(respx_mock) -> None:+ respx_mock.post(_CHAT_URL).mock(side_effect=httpx.ConnectError("offline"))+ engine = NIMEngine(host=_HOST)++ with pytest.raises(EngineConnectionError, match="not reachable"):+ engine.generate(_message(), model="nim-model")++ engine.close()+++def test_health_falls_back_to_models(respx_mock) -> None:+ respx_mock.get(f"{_HOST}/v1/health/ready").mock(return_value=httpx.Response(404))+ models = respx_mock.get(f"{_HOST}/v1/models").mock(+ return_value=httpx.Response(200, json={"data": []})+ )+ engine = NIMEngine(host=_HOST)++ assert engine.health() is True+ assert models.called+ engine.close()