Personal AI, On Personal Devices
Python
feat: hardware-aware model recommendation in jarvis init (#188)
Robby Manihani35b8991cc20b
diff --git a/src/openjarvis/cli/init_cmd.py b/src/openjarvis/cli/init_cmd.pyindex 8f801ae2..56280e58 100644--- a/src/openjarvis/cli/init_cmd.py+++ b/src/openjarvis/cli/init_cmd.py@@ -505,14 +505,23 @@ sources = ["hackernews", "news_rss"]else:spec = find_model_spec(model)size_gb = estimated_download_gb(spec.parameter_count_b) if spec else 0+ from openjarvis.core.config import _available_memory_gb++ avail = _available_memory_gb(hw)console.print(- f"\n [bold]Recommended model:[/bold] {model} (~{size_gb:.1f} GB estimated)"+ f"\n [bold]Recommended model:[/bold] {model} (~{size_gb:.1f} GB)"+ f" [dim](selected for {avail:.0f} GB available memory)[/dim]")if not no_download and spec:- prompt = f" Download {model} (~{size_gb:.1f} GB estimated) now?"+ prompt = f" Download {model} (~{size_gb:.1f} GB) now?"if click.confirm(prompt, default=True):_do_download(selected_engine, model, spec, console)+ else:+ console.print(+ f"\n Skipped. Download later with:\n"+ f" [bold]jarvis model pull {model}[/bold]"+ )if not skip_scan:_quick_privacy_check(console)diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.pyindex daafdcaf..35ae4e1c 100644--- a/src/openjarvis/core/config.py+++ b/src/openjarvis/core/config.py@@ -228,40 +228,67 @@ def recommend_engine(hw: HardwareInfo) -> str:return "llamacpp"+def _available_memory_gb(hw: HardwareInfo) -> float:+ """Return usable memory in GB for model loading."""+ gpu = hw.gpu+ if gpu and gpu.vram_gb > 0:+ return gpu.vram_gb * max(gpu.count, 1) * 0.9+ if hw.ram_gb > 0:+ return (hw.ram_gb - 4) * 0.8+ return 0.0+++# Explicit tier table: (max_ram_gb, model_id).+# Walked in order — first tier where available_gb <= max_ram is chosen.+_MODEL_TIERS = [+ (8, "qwen3:1.7b"),+ (16, "qwen3:4b"),+ (32, "qwen3:8b"),+ (64, "qwen3:14b"),+]+_MODEL_TIER_FALLBACK = "qwen3:14b"++def recommend_model(hw: HardwareInfo, engine: str) -> str:- """Suggest the largest Qwen3.5 model that fits the detected hardware.+ """Suggest a Qwen3 dense model that fits the detected hardware.- Uses llmfit-style VRAM estimation: Q4_K_M quantization is ~0.5 bytes/param- with 10% overhead. For MoE models Ollama loads full model weights, so we- use ``parameter_count_b`` (total), not ``active_parameter_count_b``.+ Uses an explicit tier table mapping available memory to model size.+ Falls back to scanning the full catalog if the tiered model is not+ compatible with the selected engine."""from openjarvis.intelligence.model_catalog import BUILTIN_MODELS- # Determine available memory in GB- gpu = hw.gpu- if gpu and gpu.vram_gb > 0:- available_gb = gpu.vram_gb * max(gpu.count, 1) * 0.9- elif hw.ram_gb > 0:- available_gb = (hw.ram_gb - 4) * 0.8- else:+ available_gb = _available_memory_gb(hw)+ if available_gb <= 0:return ""- # Filter Qwen3.5 models compatible with the chosen engine+ # Build a lookup for quick engine-compatibility checks+ catalog = {spec.model_id: spec for spec in BUILTIN_MODELS}++ # Try explicit tier mapping first+ model_id = _MODEL_TIER_FALLBACK+ for max_ram, tier_model in _MODEL_TIERS:+ if available_gb <= max_ram:+ model_id = tier_model+ break++ spec = catalog.get(model_id)+ if spec and engine in spec.supported_engines:+ return model_id++ # Fallback: scan all Qwen3 dense models for engine compatibilitycandidates = [- spec- for spec in BUILTIN_MODELS- if spec.provider == "alibaba"- and spec.model_id.startswith("qwen3.5:")- and engine in spec.supported_engines+ s+ for s in BUILTIN_MODELS+ if s.provider == "alibaba"+ and s.model_id.startswith("qwen3:")+ and engine in s.supported_engines]-- # Sort by parameter count descending — pick the largest that fitscandidates.sort(key=lambda s: s.parameter_count_b, reverse=True)-- for spec in candidates:- estimated_gb = spec.parameter_count_b * 0.5 * 1.1+ for s in candidates:+ estimated_gb = s.parameter_count_b * 0.5 * 1.1if estimated_gb <= available_gb:- return spec.model_id+ return s.model_idreturn ""diff --git a/src/openjarvis/intelligence/model_catalog.py b/src/openjarvis/intelligence/model_catalog.pyindex c98dd781..29bf3667 100644--- a/src/openjarvis/intelligence/model_catalog.py+++ b/src/openjarvis/intelligence/model_catalog.py@@ -11,18 +11,79 @@ BUILTIN_MODELS: List[ModelSpec] = [# -----------------------------------------------------------------------# Local models — Dense# -----------------------------------------------------------------------+ ModelSpec(+ model_id="qwen3:0.6b",+ name="Qwen3 0.6B",+ parameter_count_b=0.6,+ context_length=40960,+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),+ provider="alibaba",+ metadata={+ "architecture": "dense",+ "hf_repo": "Qwen/Qwen3-0.6B",+ },+ ),+ ModelSpec(+ model_id="qwen3:1.7b",+ name="Qwen3 1.7B",+ parameter_count_b=1.7,+ context_length=40960,+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),+ provider="alibaba",+ metadata={+ "architecture": "dense",+ "hf_repo": "Qwen/Qwen3-1.7B",+ },+ ),+ ModelSpec(+ model_id="qwen3:4b",+ name="Qwen3 4B",+ parameter_count_b=4.0,+ context_length=262144,+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),+ provider="alibaba",+ metadata={+ "architecture": "dense",+ "hf_repo": "Qwen/Qwen3-4B",+ },+ ),ModelSpec(model_id="qwen3:8b",name="Qwen3 8B",parameter_count_b=8.2,context_length=32768,- supported_engines=("vllm", "ollama", "llamacpp", "sglang", "lemonade"),+ supported_engines=("vllm", "ollama", "llamacpp", "sglang", "mlx", "lemonade"),provider="alibaba",metadata={"architecture": "dense","hf_repo": "Qwen/Qwen3-8B",},),+ ModelSpec(+ model_id="qwen3:14b",+ name="Qwen3 14B",+ parameter_count_b=14.0,+ context_length=40960,+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "lemonade"),+ provider="alibaba",+ metadata={+ "architecture": "dense",+ "hf_repo": "Qwen/Qwen3-14B",+ },+ ),+ ModelSpec(+ model_id="qwen3:30b",+ name="Qwen3 30B",+ parameter_count_b=30.0,+ context_length=262144,+ min_vram_gb=18.0,+ supported_engines=("ollama", "vllm", "sglang"),+ provider="alibaba",+ metadata={+ "architecture": "dense",+ "hf_repo": "Qwen/Qwen3-30B",+ },+ ),ModelSpec(model_id="qwen3:32b",name="Qwen3 32B",diff --git a/tests/core/test_recommend_model.py b/tests/core/test_recommend_model.pyindex 1a5cc5aa..456f7d3f 100644--- a/tests/core/test_recommend_model.py+++ b/tests/core/test_recommend_model.py@@ -5,98 +5,88 @@ from __future__ import annotationsfrom openjarvis.core.config import GpuInfo, HardwareInfo, recommend_model+class TestRecommendModelTiers:+ """Tier-based model recommendation (Qwen3 dense)."""++ def test_8gb_ram_picks_qwen3_1_7b(self) -> None:+ hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None)+ result = recommend_model(hw, "llamacpp")+ # available = (8 - 4) * 0.8 = 3.2 GB → ≤8 tier → qwen3:1.7b+ assert result == "qwen3:1.7b"++ def test_16gb_ram_picks_qwen3_4b(self) -> None:+ hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None)+ result = recommend_model(hw, "llamacpp")+ # available = (16 - 4) * 0.8 = 9.6 GB → ≤16 tier → qwen3:4b+ assert result == "qwen3:4b"++ def test_32gb_ram_picks_qwen3_8b(self) -> None:+ hw = HardwareInfo(platform="linux", ram_gb=32.0, gpu=None)+ result = recommend_model(hw, "llamacpp")+ # available = (32 - 4) * 0.8 = 22.4 GB → ≤32 tier → qwen3:8b+ assert result == "qwen3:8b"++ def test_64gb_ram_picks_qwen3_14b(self) -> None:+ hw = HardwareInfo(platform="linux", ram_gb=64.0, gpu=None)+ result = recommend_model(hw, "llamacpp")+ # available = (64 - 4) * 0.8 = 48 GB → >32 → qwen3:14b+ assert result == "qwen3:14b"++class TestRecommendModelGpu:"""GPU-based model recommendation."""- def test_24gb_gpu_picks_qwen35_35b(self) -> None:+ def test_24gb_gpu_picks_qwen3_14b(self) -> None:hw = HardwareInfo(platform="linux",ram_gb=64.0,gpu=GpuInfo(vendor="nvidia", name="RTX 4090", vram_gb=24.0, count=1),)result = recommend_model(hw, "ollama")- # 35B * 0.5 * 1.1 = 19.25 GB; available = 24 * 0.9 = 21.6 → fits- assert result == "qwen3.5:35b"+ # available = 24 * 0.9 = 21.6 GB → ≤32 tier → qwen3:8b+ # Actually 21.6 ≤ 32, so tier is qwen3:8b+ assert result == "qwen3:8b"- def test_8gb_gpu_picks_qwen35_14b(self) -> None:+ def test_8gb_gpu_picks_qwen3_1_7b(self) -> None:hw = HardwareInfo(platform="linux",ram_gb=32.0,gpu=GpuInfo(vendor="nvidia", name="RTX 3070", vram_gb=8.0, count=1),)result = recommend_model(hw, "ollama")- # 14B * 0.5 * 1.1 = 7.7 GB; available = 8 * 0.9 = 7.2 → too big- # 8B * 0.5 * 1.1 = 4.4 GB; available = 7.2 → fits- assert result == "qwen3.5:9b"+ # available = 8 * 0.9 = 7.2 GB → ≤8 tier → qwen3:1.7b+ assert result == "qwen3:1.7b"- def test_4gb_gpu_picks_qwen35_4b(self) -> None:+ def test_4gb_gpu_picks_qwen3_1_7b(self) -> None:hw = HardwareInfo(platform="linux",ram_gb=16.0,gpu=GpuInfo(vendor="nvidia", name="GTX 1650", vram_gb=4.0, count=1),)result = recommend_model(hw, "ollama")- # 4B * 0.5 * 1.1 = 2.2 GB; available = 4 * 0.9 = 3.6 → fits- assert result == "qwen3.5:4b"-- def test_2gb_gpu_picks_qwen35_3b(self) -> None:- hw = HardwareInfo(- platform="linux",- ram_gb=8.0,- gpu=GpuInfo(vendor="nvidia", name="GTX 750", vram_gb=2.0, count=1),- )- result = recommend_model(hw, "ollama")- # 3B * 0.5 * 1.1 = 1.65 GB; available = 2 * 0.9 = 1.8 → fits- assert result == "qwen3.5:2b"+ # available = 4 * 0.9 = 3.6 GB → ≤8 tier → qwen3:1.7b+ assert result == "qwen3:1.7b"- def test_multi_gpu_picks_larger_model(self) -> None:+ def test_multi_gpu_picks_qwen3_14b(self) -> None:hw = HardwareInfo(platform="linux",ram_gb=256.0,gpu=GpuInfo(vendor="nvidia", name="A100", vram_gb=80.0, count=2),)result = recommend_model(hw, "vllm")- # available = 80 * 2 * 0.9 = 144 GB- # 397B * 0.5 * 1.1 = 218.35 → too big- # 122B * 0.5 * 1.1 = 67.1 → fits- assert result == "qwen3.5:122b"+ # available = 80 * 2 * 0.9 = 144 GB → >64 → qwen3:14b (tier fallback)+ assert result == "qwen3:14b"- def test_huge_vram_picks_397b(self) -> None:+ def test_huge_vram_falls_back_to_scan(self) -> None:hw = HardwareInfo(platform="linux",ram_gb=512.0,gpu=GpuInfo(vendor="nvidia", name="H100", vram_gb=80.0, count=4),)result = recommend_model(hw, "vllm")- # available = 80 * 4 * 0.9 = 288 GB- # 397B * 0.5 * 1.1 = 218.35 → fits- assert result == "qwen3.5:397b"---class TestRecommendModelCpuOnly:- """CPU-only model recommendation."""-- def test_cpu_only_16gb_ram(self) -> None:- hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None)- result = recommend_model(hw, "llamacpp")- # available = (16 - 4) * 0.8 = 9.6 GB- # 27B * 0.5 * 1.1 = 14.85 → too big- # 9B * 0.5 * 1.1 = 4.95 → fits- assert result == "qwen3.5:9b"-- def test_cpu_only_8gb_ram(self) -> None:- hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None)- result = recommend_model(hw, "llamacpp")- # available = (8 - 4) * 0.8 = 3.2 GB- # 8B * 0.5 * 1.1 = 4.4 → too big- # 4B * 0.5 * 1.1 = 2.2 → fits- assert result == "qwen3.5:4b"-- def test_cpu_only_4gb_ram(self) -> None:- hw = HardwareInfo(platform="linux", ram_gb=4.0, gpu=None)- result = recommend_model(hw, "llamacpp")- # available = (4 - 4) * 0.8 = 0 → nothing fits- assert result == ""+ # available = 288 GB → tier gives qwen3:14b, but scan finds larger+ # Tier fallback is qwen3:14b — it's valid for vllm+ assert result == "qwen3:14b"class TestRecommendModelEdgeCases:@@ -106,16 +96,18 @@ class TestRecommendModelEdgeCases:hw = HardwareInfo(platform="linux", ram_gb=0.0, gpu=None)assert recommend_model(hw, "ollama") == ""- def test_engine_filter(self) -> None:- """397b is not supported on ollama, only vllm/sglang."""- hw = HardwareInfo(- platform="linux",- ram_gb=512.0,- gpu=GpuInfo(vendor="nvidia", name="H100", vram_gb=80.0, count=4),- )- # With ollama, 397b is excluded (only vllm, sglang)- result = recommend_model(hw, "ollama")- assert result == "qwen3.5:122b"+ def test_4gb_ram_picks_qwen3_1_7b(self) -> None:+ hw = HardwareInfo(platform="linux", ram_gb=6.0, gpu=None)+ result = recommend_model(hw, "llamacpp")+ # available = (6 - 4) * 0.8 = 1.6 GB → ≤8 tier → qwen3:1.7b+ # 1.7 * 0.5 * 1.1 = 0.935 → fits in 1.6+ assert result == "qwen3:1.7b"++ def test_very_low_ram_returns_empty(self) -> None:+ hw = HardwareInfo(platform="linux", ram_gb=4.0, gpu=None)+ result = recommend_model(hw, "llamacpp")+ # available = (4 - 4) * 0.8 = 0 → nothing+ assert result == ""class TestRecommendModelMlx:@@ -128,7 +120,9 @@ class TestRecommendModelMlx:gpu=GpuInfo(vendor="apple", name="Apple M1", vram_gb=8.0, count=1),)result = recommend_model(hw, "mlx")- assert result == "qwen3.5:9b"+ # available = 8 * 0.9 = 7.2 GB → ≤8 tier → qwen3:1.7b+ # But qwen3:1.7b supports mlx → good+ assert result == "qwen3:1.7b"def test_apple_silicon_16gb_mlx(self) -> None:hw = HardwareInfo(@@ -137,10 +131,8 @@ class TestRecommendModelMlx:gpu=GpuInfo(vendor="apple", name="Apple M2", vram_gb=16.0, count=1),)result = recommend_model(hw, "mlx")- # available = 16 * 0.9 = 14.4 GB- # 27B * 0.5 * 1.1 = 14.85 → too big- # 9B * 0.5 * 1.1 = 4.95 → fits- assert result == "qwen3.5:9b"+ # available = 16 * 0.9 = 14.4 GB → ≤16 tier → qwen3:4b+ assert result == "qwen3:4b"def test_apple_silicon_32gb_mlx(self) -> None:hw = HardwareInfo(@@ -149,7 +141,8 @@ class TestRecommendModelMlx:gpu=GpuInfo(vendor="apple", name="Apple M2 Pro", vram_gb=32.0, count=1),)result = recommend_model(hw, "mlx")- assert result == "qwen3.5:27b"+ # available = 32 * 0.9 = 28.8 GB → ≤32 tier → qwen3:8b+ assert result == "qwen3:8b"def test_apple_silicon_64gb_mlx(self) -> None:hw = HardwareInfo(@@ -158,4 +151,6 @@ class TestRecommendModelMlx:gpu=GpuInfo(vendor="apple", name="Apple M2 Max", vram_gb=64.0, count=1),)result = recommend_model(hw, "mlx")- assert result == "qwen3.5:27b"+ # available = 64 * 0.9 = 57.6 GB → ≤64 tier → qwen3:14b+ # qwen3:14b doesn't support mlx → fallback scan finds qwen3:8b+ assert result == "qwen3:8b"