Skip to content

Personal AI, On Personal Devices

Python

feat(engine): add NVIDIA NIM inference engine support (#291)

Anubhav Chaturvedi4780499ae29d
diff --git a/src/openjarvis/engine/__init__.py b/src/openjarvis/engine/__init__.py
index d575da8f..484bf511 100644
--- a/src/openjarvis/engine/__init__.py
+++ b/src/openjarvis/engine/__init__.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import importlib
# Import engine modules to trigger @EngineRegistry.register() decorators
+import openjarvis.engine.nim # noqa: F401
import openjarvis.engine.ollama # noqa: F401
import openjarvis.engine.openai_compat_engines # noqa: F401
from openjarvis.engine._base import (
diff --git a/src/openjarvis/engine/nim.py b/src/openjarvis/engine/nim.py
new file mode 100644
index 00000000..31da6279
--- /dev/null
+++ b/src/openjarvis/engine/nim.py
@@ -0,0 +1,283 @@
+"""NVIDIA NIM inference engine — self-hosted inference microservices."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from collections.abc import AsyncIterator, Sequence
+from typing import Any, Dict, List
+
+import httpx
+
+from openjarvis.core.registry import EngineRegistry
+from openjarvis.core.types import Message
+from openjarvis.engine._base import (
+ EngineConnectionError,
+ InferenceEngine,
+ estimate_prompt_tokens,
+ messages_to_dicts,
+)
+from openjarvis.engine._stubs import StreamChunk
+
+logger = logging.getLogger(__name__)
+
+
+class NIMEngine(InferenceEngine):
+ """NVIDIA NIM inference engine.
+
+ Supports local and remote NIM deployments with OpenAI-compatible API.
+ Requires NIM_API_KEY for NVIDIA-hosted NIM endpoints, optional for self-hosted.
+ """
+
+ engine_id = "nim"
+ _default_host = "https://integrate.api.nvidia.com"
+ _api_prefix = "/v1"
+
+ def __init__(self, host: str | None = None, *, timeout: float = 600.0) -> None:
+ env_host = os.environ.get("NIM_HOST")
+ self._host = (host or env_host or self._default_host).rstrip("/")
+
+ api_key = os.environ.get("NIM_API_KEY", "")
+ self._api_key = api_key if api_key else None
+
+ headers = {}
+ if self._api_key:
+ headers["Authorization"] = f"Bearer {self._api_key}"
+
+ self._timeout = timeout
+ self._client = httpx.Client(
+ base_url=self._host,
+ timeout=timeout,
+ headers=headers,
+ )
+
+ def _get_headers(self) -> Dict[str, str]:
+ """Get headers for requests, including auth if configured."""
+ headers = {}
+ if self._api_key:
+ headers["Authorization"] = f"Bearer {self._api_key}"
+ return headers
+
+ def generate(
+ self,
+ messages: Sequence[Message],
+ *,
+ model: str,
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+ **kwargs: Any,
+ ) -> Dict[str, Any]:
+ payload: Dict[str, Any] = {
+ "model": model,
+ "messages": messages_to_dicts(messages),
+ "temperature": temperature,
+ "max_tokens": max_tokens,
+ "stream": False,
+ **kwargs,
+ }
+ if "tools" in payload and "tool_choice" not in payload:
+ payload["tool_choice"] = "auto"
+ try:
+ url = f"{self._api_prefix}/chat/completions"
+ resp = self._client.post(url, json=payload, headers=self._get_headers())
+ resp.raise_for_status()
+ except (httpx.ConnectError, httpx.TimeoutException) as exc:
+ raise EngineConnectionError(
+ f"NIM engine not reachable at {self._host}"
+ ) from exc
+ data = resp.json()
+ choices = data.get("choices", [])
+ if not choices:
+ return {
+ "content": "",
+ "usage": data.get("usage", {}),
+ "model": data.get("model", model),
+ "finish_reason": "error",
+ }
+ choice = choices[0]
+ usage = data.get("usage", {})
+ reported_prompt = usage.get("prompt_tokens", 0)
+ estimated_prompt = estimate_prompt_tokens(messages)
+ prompt_tokens = max(reported_prompt, estimated_prompt)
+ completion_tokens = usage.get("completion_tokens", 0)
+ result: Dict[str, Any] = {
+ "content": choice["message"].get("content") or "",
+ "usage": {
+ "prompt_tokens": prompt_tokens,
+ "prompt_tokens_evaluated": reported_prompt or prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": prompt_tokens + completion_tokens,
+ },
+ "model": data.get("model", model),
+ "finish_reason": choice.get("finish_reason", "stop"),
+ }
+ raw_tool_calls = choice["message"].get("tool_calls", [])
+ if raw_tool_calls:
+ result["tool_calls"] = [
+ {
+ "id": tc.get("id", ""),
+ "name": tc.get("function", {}).get("name", ""),
+ "arguments": tc.get("function", {}).get("arguments", "{}"),
+ }
+ for tc in raw_tool_calls
+ ]
+ return result
+
+ async def stream(
+ self,
+ messages: Sequence[Message],
+ *,
+ model: str,
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+ **kwargs: Any,
+ ) -> AsyncIterator[str]:
+ payload: Dict[str, Any] = {
+ "model": model,
+ "messages": messages_to_dicts(messages),
+ "temperature": temperature,
+ "max_tokens": max_tokens,
+ "stream": True,
+ **kwargs,
+ }
+ if "tools" in payload and "tool_choice" not in payload:
+ payload["tool_choice"] = "auto"
+ try:
+ url = f"{self._api_prefix}/chat/completions"
+ async with httpx.AsyncClient(
+ base_url=self._host,
+ timeout=self._timeout,
+ headers=self._get_headers(),
+ ) as client:
+ async with client.stream("POST", url, json=payload) as resp:
+ resp.raise_for_status()
+ async for line in resp.aiter_lines():
+ if not line.startswith("data:"):
+ continue
+ data_str = line[len("data:") :].strip()
+ if data_str == "[DONE]":
+ break
+ try:
+ chunk = json.loads(data_str)
+ except json.JSONDecodeError:
+ continue
+ choices = chunk.get("choices", [])
+ if not choices:
+ continue
+ delta = choices[0].get("delta", {})
+ content = delta.get("content")
+ if content:
+ yield content
+ except (httpx.ConnectError, httpx.TimeoutException) as exc:
+ raise EngineConnectionError(
+ f"NIM engine not reachable at {self._host}"
+ ) from exc
+
+ async def stream_full(
+ self,
+ messages: Sequence[Message],
+ *,
+ model: str,
+ temperature: float = 0.7,
+ max_tokens: int = 1024,
+ **kwargs: Any,
+ ) -> AsyncIterator["StreamChunk"]:
+ """Yield StreamChunks with content, tool_calls, and finish_reason."""
+ msg_dicts = messages_to_dicts(messages)
+ payload: Dict[str, Any] = {
+ "model": model,
+ "messages": msg_dicts,
+ "temperature": temperature,
+ "max_tokens": max_tokens,
+ "stream": True,
+ **kwargs,
+ }
+ if "tools" in payload and "tool_choice" not in payload:
+ payload["tool_choice"] = "auto"
+ try:
+ url = f"{self._api_prefix}/chat/completions"
+ async with httpx.AsyncClient(
+ base_url=self._host,
+ timeout=self._timeout,
+ headers=self._get_headers(),
+ ) as client:
+ async with client.stream("POST", url, json=payload) as resp:
+ resp.raise_for_status()
+ async for line in resp.aiter_lines():
+ if not line.startswith("data:"):
+ continue
+ data_str = line[len("data:") :].strip()
+ if data_str == "[DONE]":
+ break
+ try:
+ chunk = json.loads(data_str)
+ except json.JSONDecodeError:
+ continue
+ choices = chunk.get("choices", [])
+ if not choices:
+ continue
+ choice = choices[0]
+ delta = choice.get("delta", {})
+ finish = choice.get("finish_reason")
+ content = delta.get("content")
+ tool_calls = delta.get("tool_calls")
+ usage = chunk.get("usage")
+
+ if content or tool_calls or finish or usage:
+ yield StreamChunk(
+ content=content,
+ tool_calls=tool_calls,
+ finish_reason=finish,
+ usage=usage,
+ )
+ except (httpx.ConnectError, httpx.TimeoutException) as exc:
+ raise EngineConnectionError(
+ f"NIM engine not reachable at {self._host}"
+ ) from exc
+
+ def list_models(self) -> List[str]:
+ try:
+ resp = self._client.get(
+ f"{self._api_prefix}/models",
+ headers=self._get_headers(),
+ )
+ resp.raise_for_status()
+ except (
+ httpx.ConnectError,
+ httpx.TimeoutException,
+ httpx.HTTPStatusError,
+ ) as exc:
+ logger.warning("Failed to list models from NIM at %s: %s", self._host, exc)
+ return []
+ data = resp.json()
+ return [m["id"] for m in data.get("data", [])]
+
+ def health(self) -> bool:
+ try:
+ resp = self._client.get(
+ f"{self._api_prefix}/health/ready",
+ timeout=2.0,
+ headers=self._get_headers(),
+ )
+ if resp.status_code == 200:
+ return True
+ except Exception:
+ pass
+ try:
+ resp = self._client.get(
+ f"{self._api_prefix}/models",
+ timeout=2.0,
+ headers=self._get_headers(),
+ )
+ return resp.status_code == 200
+ except Exception as exc:
+ logger.debug("NIM health check failed at %s: %s", self._host, exc)
+ return False
+
+ def close(self) -> None:
+ self._client.close()
+
+ def __repr__(self) -> str:
+ return f"NIMEngine(host={self._host})"
diff --git a/tests/engine/test_nim.py b/tests/engine/test_nim.py
new file mode 100644
index 00000000..4158ad2b
--- /dev/null
+++ b/tests/engine/test_nim.py
@@ -0,0 +1,135 @@
+"""Tests for the NVIDIA NIM inference engine."""
+
+from __future__ import annotations
+
+import json
+from unittest.mock import MagicMock
+
+import httpx
+import pytest
+
+from openjarvis.core.types import Message, Role
+from openjarvis.engine._base import EngineConnectionError
+from openjarvis.engine.nim import NIMEngine
+
+_HOST = "https://nim.test"
+_CHAT_URL = f"{_HOST}/v1/chat/completions"
+
+
+def _message() -> list[Message]:
+ return [Message(role=Role.USER, content="Hello")]
+
+
+def test_generate_preserves_tools_on_unsupported_request(respx_mock) -> None:
+ """A 400 must not be retried after silently removing requested tools."""
+ route = respx_mock.post(_CHAT_URL).mock(
+ return_value=httpx.Response(400, json={"detail": "tools unsupported"})
+ )
+ engine = NIMEngine(host=_HOST)
+
+ with pytest.raises(httpx.HTTPStatusError):
+ engine.generate(
+ _message(),
+ model="nim-model",
+ tools=[{"type": "function", "function": {"name": "lookup"}}],
+ )
+
+ assert route.call_count == 1
+ payload = json.loads(route.calls[0].request.content)
+ assert payload["tools"][0]["function"]["name"] == "lookup"
+ assert payload["tool_choice"] == "auto"
+ engine.close()
+
+
+async def test_stream_uses_async_http_and_sends_auth(respx_mock, monkeypatch) -> None:
+ monkeypatch.setenv("NIM_API_KEY", "secret-token")
+ body = "\n".join(
+ [
+ 'data: {"choices":[{"delta":{"content":"Hello"}}]}',
+ 'data: {"choices":[{"delta":{"content":" world"}}]}',
+ "data: [DONE]",
+ ]
+ )
+ route = respx_mock.post(_CHAT_URL).mock(return_value=httpx.Response(200, text=body))
+ engine = NIMEngine(host=_HOST)
+ engine._client.stream = MagicMock( # type: ignore[method-assign]
+ side_effect=AssertionError("sync client used from async stream")
+ )
+
+ chunks = [chunk async for chunk in engine.stream(_message(), model="nim-model")]
+
+ assert chunks == ["Hello", " world"]
+ assert route.calls[0].request.headers["Authorization"] == "Bearer secret-token"
+ engine._client.close()
+
+
+async def test_stream_full_parses_tool_and_usage_chunks(respx_mock) -> None:
+ body = "\n".join(
+ [
+ "data: "
+ + json.dumps(
+ {
+ "choices": [
+ {
+ "delta": {
+ "tool_calls": [
+ {
+ "index": 0,
+ "function": {
+ "name": "lookup",
+ "arguments": "{}",
+ },
+ }
+ ]
+ },
+ "finish_reason": None,
+ }
+ ]
+ }
+ ),
+ "data: "
+ + json.dumps(
+ {
+ "choices": [{"delta": {}, "finish_reason": "tool_calls"}],
+ "usage": {"total_tokens": 9},
+ }
+ ),
+ "data: [DONE]",
+ ]
+ )
+ respx_mock.post(_CHAT_URL).mock(return_value=httpx.Response(200, text=body))
+ engine = NIMEngine(host=_HOST)
+
+ chunks = [
+ chunk async for chunk in engine.stream_full(_message(), model="nim-model")
+ ]
+
+ assert chunks[0].tool_calls is not None
+ assert chunks[0].tool_calls[0]["function"]["name"] == "lookup"
+ assert chunks[1].finish_reason == "tool_calls"
+ assert chunks[1].usage == {"total_tokens": 9}
+ engine.close()
+
+
+def test_generate_maps_connection_errors(respx_mock) -> None:
+ respx_mock.post(_CHAT_URL).mock(side_effect=httpx.ConnectError("offline"))
+ engine = NIMEngine(host=_HOST)
+
+ with pytest.raises(EngineConnectionError, match="not reachable"):
+ engine.generate(_message(), model="nim-model")
+
+ engine.close()
+
+
+def test_health_falls_back_to_models(respx_mock) -> None:
+ respx_mock.get(f"{_HOST}/v1/health/ready").mock(return_value=httpx.Response(404))
+ models = respx_mock.get(f"{_HOST}/v1/models").mock(
+ return_value=httpx.Response(200, json={"data": []})
+ )
+ engine = NIMEngine(host=_HOST)
+
+ assert engine.health() is True
+ assert models.called
+ engine.close()