Skip to content

Personal AI, On Personal Devices

Python

fix: resolve MLX model ids before inference (#936)

Dustin Z.a92c2792997b
diff --git a/src/openjarvis/cli/doctor_cmd.py b/src/openjarvis/cli/doctor_cmd.py
index c3fc1c90..1e7ebfac 100644
--- a/src/openjarvis/cli/doctor_cmd.py
+++ b/src/openjarvis/cli/doctor_cmd.py
@@ -169,6 +169,7 @@ def _check_default_model() -> CheckResult:
from openjarvis.core.registry import EngineRegistry
from openjarvis.engine import _discovery
+ from openjarvis.intelligence.model_catalog import resolve_model_id_for_engine
preferred = config.intelligence.preferred_engine or config.engine.default
check_order = []
@@ -181,7 +182,8 @@ def _check_default_model() -> CheckResult:
engine = _discovery._make_engine(key, config)
if engine.health():
models = engine.list_models()
- if default_model in models:
+ resolved_model = resolve_model_id_for_engine(default_model, key)
+ if resolved_model in models:
return CheckResult(
"Default model",
"ok",
diff --git a/src/openjarvis/engine/_openai_compat.py b/src/openjarvis/engine/_openai_compat.py
index 507b6d83..db57eb44 100644
--- a/src/openjarvis/engine/_openai_compat.py
+++ b/src/openjarvis/engine/_openai_compat.py
@@ -22,6 +22,7 @@ from openjarvis.engine._http_async import (
AsyncHTTPEngineMixin,
)
from openjarvis.engine._stubs import StreamChunk
+from openjarvis.intelligence.model_catalog import resolve_model_id_for_engine
logger = logging.getLogger(__name__)
@@ -75,6 +76,9 @@ class _OpenAICompatibleEngine(AsyncHTTPEngineMixin, InferenceEngine):
# -- InferenceEngine interface ------------------------------------------
+ def _resolve_model_id(self, model: str) -> str:
+ return resolve_model_id_for_engine(model, self.engine_id)
+
def generate(
self,
messages: Sequence[Message],
@@ -85,7 +89,7 @@ class _OpenAICompatibleEngine(AsyncHTTPEngineMixin, InferenceEngine):
**kwargs: Any,
) -> Dict[str, Any]:
payload: Dict[str, Any] = {
- "model": model,
+ "model": self._resolve_model_id(model),
"messages": messages_to_dicts(messages),
"temperature": temperature,
"max_tokens": max_tokens,
@@ -177,7 +181,7 @@ class _OpenAICompatibleEngine(AsyncHTTPEngineMixin, InferenceEngine):
**kwargs: Any,
) -> AsyncIterator[str]:
payload: Dict[str, Any] = {
- "model": model,
+ "model": self._resolve_model_id(model),
"messages": messages_to_dicts(messages),
"temperature": temperature,
"max_tokens": max_tokens,
@@ -241,7 +245,7 @@ class _OpenAICompatibleEngine(AsyncHTTPEngineMixin, InferenceEngine):
"""Yield StreamChunks with content, tool_calls, and finish_reason."""
msg_dicts = messages_to_dicts(messages)
payload: Dict[str, Any] = {
- "model": model,
+ "model": self._resolve_model_id(model),
"messages": msg_dicts,
"temperature": temperature,
"max_tokens": max_tokens,
diff --git a/src/openjarvis/intelligence/__init__.py b/src/openjarvis/intelligence/__init__.py
index f59283e7..4fe53723 100644
--- a/src/openjarvis/intelligence/__init__.py
+++ b/src/openjarvis/intelligence/__init__.py
@@ -6,6 +6,12 @@ from openjarvis.intelligence.model_catalog import (
BUILTIN_MODELS,
merge_discovered_models,
register_builtin_models,
+ resolve_model_id_for_engine,
)
-__all__ = ["BUILTIN_MODELS", "merge_discovered_models", "register_builtin_models"]
+__all__ = [
+ "BUILTIN_MODELS",
+ "merge_discovered_models",
+ "register_builtin_models",
+ "resolve_model_id_for_engine",
+]
diff --git a/src/openjarvis/intelligence/model_catalog.py b/src/openjarvis/intelligence/model_catalog.py
index ea010ea6..8630cf09 100644
--- a/src/openjarvis/intelligence/model_catalog.py
+++ b/src/openjarvis/intelligence/model_catalog.py
@@ -1067,6 +1067,20 @@ BUILTIN_MODELS: List[ModelSpec] = [
]
+def resolve_model_id_for_engine(model_id: str, engine_id: str) -> str:
+ """Resolve a catalog model ID when an engine has a distinct runtime ID."""
+ if engine_id != "mlx":
+ return model_id
+
+ for spec in BUILTIN_MODELS:
+ if spec.model_id == model_id and engine_id in spec.supported_engines:
+ mlx_repo = spec.metadata.get("mlx_repo")
+ if isinstance(mlx_repo, str) and mlx_repo:
+ return mlx_repo
+ break
+ return model_id
+
+
def register_builtin_models() -> None:
"""Populate ``ModelRegistry`` with well-known models."""
for spec in BUILTIN_MODELS:
@@ -1088,4 +1102,9 @@ def merge_discovered_models(engine_key: str, model_ids: List[str]) -> None:
ModelRegistry.register_value(model_id, spec)
-__all__ = ["BUILTIN_MODELS", "merge_discovered_models", "register_builtin_models"]
+__all__ = [
+ "BUILTIN_MODELS",
+ "merge_discovered_models",
+ "register_builtin_models",
+ "resolve_model_id_for_engine",
+]
diff --git a/tests/cli/test_doctor_cmd.py b/tests/cli/test_doctor_cmd.py
index dcc26941..70b97d75 100644
--- a/tests/cli/test_doctor_cmd.py
+++ b/tests/cli/test_doctor_cmd.py
@@ -17,6 +17,7 @@ from openjarvis.cli.doctor_cmd import (
_check_python_version,
_check_speech_backend,
)
+from openjarvis.core.registry import EngineRegistry
class TestDoctorHelp:
@@ -151,6 +152,30 @@ class TestCheckDefaultModel:
assert result.status == "ok"
assert "auto" in result.message.lower()
+ def test_mlx_catalog_model_is_checked_by_repo_id(self) -> None:
+ mock_config = MagicMock()
+ mock_config.intelligence.default_model = "qwen3.5:4b"
+ mock_config.intelligence.preferred_engine = "mlx"
+ mock_config.engine.default = "mlx"
+ mock_engine = MagicMock()
+ mock_engine.health.return_value = True
+ mock_engine.list_models.return_value = ["mlx-community/Qwen3.5-4B-OptiQ-4bit"]
+
+ with (
+ patch("openjarvis.cli.doctor_cmd.load_config", return_value=mock_config),
+ patch("openjarvis.cli.doctor_cmd._ensure_engines_imported"),
+ patch.object(EngineRegistry, "keys", return_value=["mlx"]),
+ patch(
+ "openjarvis.engine._discovery._make_engine",
+ return_value=mock_engine,
+ ),
+ ):
+ result = _check_default_model()
+
+ assert result.status == "ok"
+ assert "qwen3.5:4b" in result.message
+ assert "mlx" in result.message
+
class TestCheckSpeechBackend:
def test_check_speech_backend_ready(self) -> None:
diff --git a/tests/engine/test_openai_compat.py b/tests/engine/test_openai_compat.py
index 3df1a283..d96f22b8 100644
--- a/tests/engine/test_openai_compat.py
+++ b/tests/engine/test_openai_compat.py
@@ -2,6 +2,8 @@
from __future__ import annotations
+import json
+
import httpx
import pytest
@@ -17,7 +19,7 @@ from openjarvis.core.registry import EngineRegistry
from openjarvis.core.types import Message, Role
from openjarvis.engine._base import EngineConnectionError
from openjarvis.engine._openai_compat import EngineContextLengthError
-from openjarvis.engine.openai_compat_engines import VLLMEngine
+from openjarvis.engine.openai_compat_engines import MLXEngine, VLLMEngine
# respx-backed tests exercise the SYNC client paths (generate/list_models/health)
# and skip cleanly when respx is absent; the async stream/timeout/disconnect tests
@@ -42,6 +44,68 @@ def engine() -> VLLMEngine:
return VLLMEngine(host="http://testhost:8000")
+class TestEngineModelIdResolution:
+ def test_generate_resolves_mlx_model_id(self) -> None:
+ seen: dict = {}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.update(json.loads(request.content))
+ return httpx.Response(
+ 200,
+ json={"choices": [{"message": {"content": "ok"}}]},
+ )
+
+ engine = MLXEngine(host="http://testhost:8080")
+ engine._client.close()
+ engine._client = httpx.Client(
+ base_url="http://testhost:8080", transport=httpx.MockTransport(handler)
+ )
+ try:
+ engine.generate([Message(role=Role.USER, content="hi")], model="qwen3.5:4b")
+ finally:
+ engine.close()
+ assert seen["model"] == "mlx-community/Qwen3.5-4B-OptiQ-4bit"
+
+ @pytest.mark.asyncio
+ async def test_stream_resolves_mlx_model_id(self) -> None:
+ seen: dict = {}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.update(json.loads(request.content))
+ return httpx.Response(200, text="data: [DONE]\n")
+
+ engine = MLXEngine(host="http://testhost:8080")
+ engine._async_transport = httpx.MockTransport(handler)
+ async for _ in engine.stream(
+ [Message(role=Role.USER, content="hi")], model="qwen3.5:4b"
+ ):
+ pass
+ engine.close()
+ assert seen["model"] == "mlx-community/Qwen3.5-4B-OptiQ-4bit"
+
+ @pytest.mark.asyncio
+ async def test_stream_full_resolves_mlx_model_id(self) -> None:
+ seen: dict = {}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen.update(json.loads(request.content))
+ return httpx.Response(200, text="data: [DONE]\n")
+
+ engine = MLXEngine(host="http://testhost:8080")
+ engine._async_transport = httpx.MockTransport(handler)
+ async for _ in engine.stream_full(
+ [Message(role=Role.USER, content="hi")], model="qwen3.5:4b"
+ ):
+ pass
+ engine.close()
+ assert seen["model"] == "mlx-community/Qwen3.5-4B-OptiQ-4bit"
+
+ def test_other_openai_compatible_engine_is_unchanged(self) -> None:
+ engine = VLLMEngine(host="http://testhost:8000")
+ assert engine._resolve_model_id("qwen3.5:4b") == "qwen3.5:4b"
+ engine.close()
+
+
@requires_respx
class TestOpenAICompatGenerate:
def test_generate_returns_content(self, engine: VLLMEngine) -> None:
diff --git a/tests/intelligence/test_model_catalog.py b/tests/intelligence/test_model_catalog.py
index 4130e72f..16eaf9ba 100644
--- a/tests/intelligence/test_model_catalog.py
+++ b/tests/intelligence/test_model_catalog.py
@@ -8,6 +8,7 @@ from openjarvis.intelligence.model_catalog import (
BUILTIN_MODELS,
merge_discovered_models,
register_builtin_models,
+ resolve_model_id_for_engine,
)
@@ -47,3 +48,16 @@ class TestMergeDiscoveredModels:
original = ModelRegistry.get("qwen3:8b")
merge_discovered_models("ollama", ["qwen3:8b"])
assert ModelRegistry.get("qwen3:8b").name == original.name
+
+
+class TestResolveModelIdForEngine:
+ def test_resolves_mlx_repo(self) -> None:
+ assert resolve_model_id_for_engine("qwen3.5:4b", "mlx") == (
+ "mlx-community/Qwen3.5-4B-OptiQ-4bit"
+ )
+
+ def test_unknown_model_is_unchanged(self) -> None:
+ assert resolve_model_id_for_engine("custom/model", "mlx") == "custom/model"
+
+ def test_non_mlx_engine_is_unchanged(self) -> None:
+ assert resolve_model_id_for_engine("qwen3.5:4b", "ollama") == "qwen3.5:4b"