Skip to content

Personal AI, On Personal Devices

Python

feat: hardware-aware model recommendation in jarvis init (#188)

Robby Manihani35b8991cc20b
diff --git a/src/openjarvis/cli/init_cmd.py b/src/openjarvis/cli/init_cmd.py
index 8f801ae2..56280e58 100644
--- a/src/openjarvis/cli/init_cmd.py
+++ b/src/openjarvis/cli/init_cmd.py
@@ -505,14 +505,23 @@ sources = ["hackernews", "news_rss"]
else:
spec = find_model_spec(model)
size_gb = estimated_download_gb(spec.parameter_count_b) if spec else 0
+ from openjarvis.core.config import _available_memory_gb
+
+ avail = _available_memory_gb(hw)
console.print(
- f"\n [bold]Recommended model:[/bold] {model} (~{size_gb:.1f} GB estimated)"
+ f"\n [bold]Recommended model:[/bold] {model} (~{size_gb:.1f} GB)"
+ f" [dim](selected for {avail:.0f} GB available memory)[/dim]"
)
if not no_download and spec:
- prompt = f" Download {model} (~{size_gb:.1f} GB estimated) now?"
+ prompt = f" Download {model} (~{size_gb:.1f} GB) now?"
if click.confirm(prompt, default=True):
_do_download(selected_engine, model, spec, console)
+ else:
+ console.print(
+ f"\n Skipped. Download later with:\n"
+ f" [bold]jarvis model pull {model}[/bold]"
+ )
if not skip_scan:
_quick_privacy_check(console)
diff --git a/src/openjarvis/core/config.py b/src/openjarvis/core/config.py
index daafdcaf..35ae4e1c 100644
--- a/src/openjarvis/core/config.py
+++ b/src/openjarvis/core/config.py
@@ -228,40 +228,67 @@ def recommend_engine(hw: HardwareInfo) -> str:
return "llamacpp"
+def _available_memory_gb(hw: HardwareInfo) -> float:
+ """Return usable memory in GB for model loading."""
+ gpu = hw.gpu
+ if gpu and gpu.vram_gb > 0:
+ return gpu.vram_gb * max(gpu.count, 1) * 0.9
+ if hw.ram_gb > 0:
+ return (hw.ram_gb - 4) * 0.8
+ return 0.0
+
+
+# Explicit tier table: (max_ram_gb, model_id).
+# Walked in order — first tier where available_gb <= max_ram is chosen.
+_MODEL_TIERS = [
+ (8, "qwen3:1.7b"),
+ (16, "qwen3:4b"),
+ (32, "qwen3:8b"),
+ (64, "qwen3:14b"),
+]
+_MODEL_TIER_FALLBACK = "qwen3:14b"
+
+
def recommend_model(hw: HardwareInfo, engine: str) -> str:
- """Suggest the largest Qwen3.5 model that fits the detected hardware.
+ """Suggest a Qwen3 dense model that fits the detected hardware.
- Uses llmfit-style VRAM estimation: Q4_K_M quantization is ~0.5 bytes/param
- with 10% overhead. For MoE models Ollama loads full model weights, so we
- use ``parameter_count_b`` (total), not ``active_parameter_count_b``.
+ Uses an explicit tier table mapping available memory to model size.
+ Falls back to scanning the full catalog if the tiered model is not
+ compatible with the selected engine.
"""
from openjarvis.intelligence.model_catalog import BUILTIN_MODELS
- # Determine available memory in GB
- gpu = hw.gpu
- if gpu and gpu.vram_gb > 0:
- available_gb = gpu.vram_gb * max(gpu.count, 1) * 0.9
- elif hw.ram_gb > 0:
- available_gb = (hw.ram_gb - 4) * 0.8
- else:
+ available_gb = _available_memory_gb(hw)
+ if available_gb <= 0:
return ""
- # Filter Qwen3.5 models compatible with the chosen engine
+ # Build a lookup for quick engine-compatibility checks
+ catalog = {spec.model_id: spec for spec in BUILTIN_MODELS}
+
+ # Try explicit tier mapping first
+ model_id = _MODEL_TIER_FALLBACK
+ for max_ram, tier_model in _MODEL_TIERS:
+ if available_gb <= max_ram:
+ model_id = tier_model
+ break
+
+ spec = catalog.get(model_id)
+ if spec and engine in spec.supported_engines:
+ return model_id
+
+ # Fallback: scan all Qwen3 dense models for engine compatibility
candidates = [
- spec
- for spec in BUILTIN_MODELS
- if spec.provider == "alibaba"
- and spec.model_id.startswith("qwen3.5:")
- and engine in spec.supported_engines
+ s
+ for s in BUILTIN_MODELS
+ if s.provider == "alibaba"
+ and s.model_id.startswith("qwen3:")
+ and engine in s.supported_engines
]
-
- # Sort by parameter count descending — pick the largest that fits
candidates.sort(key=lambda s: s.parameter_count_b, reverse=True)
-
- for spec in candidates:
- estimated_gb = spec.parameter_count_b * 0.5 * 1.1
+ for s in candidates:
+ estimated_gb = s.parameter_count_b * 0.5 * 1.1
if estimated_gb <= available_gb:
- return spec.model_id
+ return s.model_id
return ""
diff --git a/src/openjarvis/intelligence/model_catalog.py b/src/openjarvis/intelligence/model_catalog.py
index c98dd781..29bf3667 100644
--- a/src/openjarvis/intelligence/model_catalog.py
+++ b/src/openjarvis/intelligence/model_catalog.py
@@ -11,18 +11,79 @@ BUILTIN_MODELS: List[ModelSpec] = [
# -----------------------------------------------------------------------
# Local models — Dense
# -----------------------------------------------------------------------
+ ModelSpec(
+ model_id="qwen3:0.6b",
+ name="Qwen3 0.6B",
+ parameter_count_b=0.6,
+ context_length=40960,
+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),
+ provider="alibaba",
+ metadata={
+ "architecture": "dense",
+ "hf_repo": "Qwen/Qwen3-0.6B",
+ },
+ ),
+ ModelSpec(
+ model_id="qwen3:1.7b",
+ name="Qwen3 1.7B",
+ parameter_count_b=1.7,
+ context_length=40960,
+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),
+ provider="alibaba",
+ metadata={
+ "architecture": "dense",
+ "hf_repo": "Qwen/Qwen3-1.7B",
+ },
+ ),
+ ModelSpec(
+ model_id="qwen3:4b",
+ name="Qwen3 4B",
+ parameter_count_b=4.0,
+ context_length=262144,
+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "mlx", "lemonade"),
+ provider="alibaba",
+ metadata={
+ "architecture": "dense",
+ "hf_repo": "Qwen/Qwen3-4B",
+ },
+ ),
ModelSpec(
model_id="qwen3:8b",
name="Qwen3 8B",
parameter_count_b=8.2,
context_length=32768,
- supported_engines=("vllm", "ollama", "llamacpp", "sglang", "lemonade"),
+ supported_engines=("vllm", "ollama", "llamacpp", "sglang", "mlx", "lemonade"),
provider="alibaba",
metadata={
"architecture": "dense",
"hf_repo": "Qwen/Qwen3-8B",
},
),
+ ModelSpec(
+ model_id="qwen3:14b",
+ name="Qwen3 14B",
+ parameter_count_b=14.0,
+ context_length=40960,
+ supported_engines=("ollama", "vllm", "llamacpp", "sglang", "lemonade"),
+ provider="alibaba",
+ metadata={
+ "architecture": "dense",
+ "hf_repo": "Qwen/Qwen3-14B",
+ },
+ ),
+ ModelSpec(
+ model_id="qwen3:30b",
+ name="Qwen3 30B",
+ parameter_count_b=30.0,
+ context_length=262144,
+ min_vram_gb=18.0,
+ supported_engines=("ollama", "vllm", "sglang"),
+ provider="alibaba",
+ metadata={
+ "architecture": "dense",
+ "hf_repo": "Qwen/Qwen3-30B",
+ },
+ ),
ModelSpec(
model_id="qwen3:32b",
name="Qwen3 32B",
diff --git a/tests/core/test_recommend_model.py b/tests/core/test_recommend_model.py
index 1a5cc5aa..456f7d3f 100644
--- a/tests/core/test_recommend_model.py
+++ b/tests/core/test_recommend_model.py
@@ -5,98 +5,88 @@ from __future__ import annotations
from openjarvis.core.config import GpuInfo, HardwareInfo, recommend_model
+class TestRecommendModelTiers:
+ """Tier-based model recommendation (Qwen3 dense)."""
+
+ def test_8gb_ram_picks_qwen3_1_7b(self) -> None:
+ hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None)
+ result = recommend_model(hw, "llamacpp")
+ # available = (8 - 4) * 0.8 = 3.2 GB → ≤8 tier → qwen3:1.7b
+ assert result == "qwen3:1.7b"
+
+ def test_16gb_ram_picks_qwen3_4b(self) -> None:
+ hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None)
+ result = recommend_model(hw, "llamacpp")
+ # available = (16 - 4) * 0.8 = 9.6 GB → ≤16 tier → qwen3:4b
+ assert result == "qwen3:4b"
+
+ def test_32gb_ram_picks_qwen3_8b(self) -> None:
+ hw = HardwareInfo(platform="linux", ram_gb=32.0, gpu=None)
+ result = recommend_model(hw, "llamacpp")
+ # available = (32 - 4) * 0.8 = 22.4 GB → ≤32 tier → qwen3:8b
+ assert result == "qwen3:8b"
+
+ def test_64gb_ram_picks_qwen3_14b(self) -> None:
+ hw = HardwareInfo(platform="linux", ram_gb=64.0, gpu=None)
+ result = recommend_model(hw, "llamacpp")
+ # available = (64 - 4) * 0.8 = 48 GB → >32 → qwen3:14b
+ assert result == "qwen3:14b"
+
+
class TestRecommendModelGpu:
"""GPU-based model recommendation."""
- def test_24gb_gpu_picks_qwen35_35b(self) -> None:
+ def test_24gb_gpu_picks_qwen3_14b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=64.0,
gpu=GpuInfo(vendor="nvidia", name="RTX 4090", vram_gb=24.0, count=1),
)
result = recommend_model(hw, "ollama")
- # 35B * 0.5 * 1.1 = 19.25 GB; available = 24 * 0.9 = 21.6 → fits
- assert result == "qwen3.5:35b"
+ # available = 24 * 0.9 = 21.6 GB → ≤32 tier → qwen3:8b
+ # Actually 21.6 ≤ 32, so tier is qwen3:8b
+ assert result == "qwen3:8b"
- def test_8gb_gpu_picks_qwen35_14b(self) -> None:
+ def test_8gb_gpu_picks_qwen3_1_7b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=32.0,
gpu=GpuInfo(vendor="nvidia", name="RTX 3070", vram_gb=8.0, count=1),
)
result = recommend_model(hw, "ollama")
- # 14B * 0.5 * 1.1 = 7.7 GB; available = 8 * 0.9 = 7.2 → too big
- # 8B * 0.5 * 1.1 = 4.4 GB; available = 7.2 → fits
- assert result == "qwen3.5:9b"
+ # available = 8 * 0.9 = 7.2 GB → ≤8 tier → qwen3:1.7b
+ assert result == "qwen3:1.7b"
- def test_4gb_gpu_picks_qwen35_4b(self) -> None:
+ def test_4gb_gpu_picks_qwen3_1_7b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=16.0,
gpu=GpuInfo(vendor="nvidia", name="GTX 1650", vram_gb=4.0, count=1),
)
result = recommend_model(hw, "ollama")
- # 4B * 0.5 * 1.1 = 2.2 GB; available = 4 * 0.9 = 3.6 → fits
- assert result == "qwen3.5:4b"
-
- def test_2gb_gpu_picks_qwen35_3b(self) -> None:
- hw = HardwareInfo(
- platform="linux",
- ram_gb=8.0,
- gpu=GpuInfo(vendor="nvidia", name="GTX 750", vram_gb=2.0, count=1),
- )
- result = recommend_model(hw, "ollama")
- # 3B * 0.5 * 1.1 = 1.65 GB; available = 2 * 0.9 = 1.8 → fits
- assert result == "qwen3.5:2b"
+ # available = 4 * 0.9 = 3.6 GB → ≤8 tier → qwen3:1.7b
+ assert result == "qwen3:1.7b"
- def test_multi_gpu_picks_larger_model(self) -> None:
+ def test_multi_gpu_picks_qwen3_14b(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=256.0,
gpu=GpuInfo(vendor="nvidia", name="A100", vram_gb=80.0, count=2),
)
result = recommend_model(hw, "vllm")
- # available = 80 * 2 * 0.9 = 144 GB
- # 397B * 0.5 * 1.1 = 218.35 → too big
- # 122B * 0.5 * 1.1 = 67.1 → fits
- assert result == "qwen3.5:122b"
+ # available = 80 * 2 * 0.9 = 144 GB → >64 → qwen3:14b (tier fallback)
+ assert result == "qwen3:14b"
- def test_huge_vram_picks_397b(self) -> None:
+ def test_huge_vram_falls_back_to_scan(self) -> None:
hw = HardwareInfo(
platform="linux",
ram_gb=512.0,
gpu=GpuInfo(vendor="nvidia", name="H100", vram_gb=80.0, count=4),
)
result = recommend_model(hw, "vllm")
- # available = 80 * 4 * 0.9 = 288 GB
- # 397B * 0.5 * 1.1 = 218.35 → fits
- assert result == "qwen3.5:397b"
-
-
-class TestRecommendModelCpuOnly:
- """CPU-only model recommendation."""
-
- def test_cpu_only_16gb_ram(self) -> None:
- hw = HardwareInfo(platform="linux", ram_gb=16.0, gpu=None)
- result = recommend_model(hw, "llamacpp")
- # available = (16 - 4) * 0.8 = 9.6 GB
- # 27B * 0.5 * 1.1 = 14.85 → too big
- # 9B * 0.5 * 1.1 = 4.95 → fits
- assert result == "qwen3.5:9b"
-
- def test_cpu_only_8gb_ram(self) -> None:
- hw = HardwareInfo(platform="linux", ram_gb=8.0, gpu=None)
- result = recommend_model(hw, "llamacpp")
- # available = (8 - 4) * 0.8 = 3.2 GB
- # 8B * 0.5 * 1.1 = 4.4 → too big
- # 4B * 0.5 * 1.1 = 2.2 → fits
- assert result == "qwen3.5:4b"
-
- def test_cpu_only_4gb_ram(self) -> None:
- hw = HardwareInfo(platform="linux", ram_gb=4.0, gpu=None)
- result = recommend_model(hw, "llamacpp")
- # available = (4 - 4) * 0.8 = 0 → nothing fits
- assert result == ""
+ # available = 288 GB → tier gives qwen3:14b, but scan finds larger
+ # Tier fallback is qwen3:14b — it's valid for vllm
+ assert result == "qwen3:14b"
class TestRecommendModelEdgeCases:
@@ -106,16 +96,18 @@ class TestRecommendModelEdgeCases:
hw = HardwareInfo(platform="linux", ram_gb=0.0, gpu=None)
assert recommend_model(hw, "ollama") == ""
- def test_engine_filter(self) -> None:
- """397b is not supported on ollama, only vllm/sglang."""
- hw = HardwareInfo(
- platform="linux",
- ram_gb=512.0,
- gpu=GpuInfo(vendor="nvidia", name="H100", vram_gb=80.0, count=4),
- )
- # With ollama, 397b is excluded (only vllm, sglang)
- result = recommend_model(hw, "ollama")
- assert result == "qwen3.5:122b"
+ def test_4gb_ram_picks_qwen3_1_7b(self) -> None:
+ hw = HardwareInfo(platform="linux", ram_gb=6.0, gpu=None)
+ result = recommend_model(hw, "llamacpp")
+ # available = (6 - 4) * 0.8 = 1.6 GB → ≤8 tier → qwen3:1.7b
+ # 1.7 * 0.5 * 1.1 = 0.935 → fits in 1.6
+ assert result == "qwen3:1.7b"
+
+ def test_very_low_ram_returns_empty(self) -> None:
+ hw = HardwareInfo(platform="linux", ram_gb=4.0, gpu=None)
+ result = recommend_model(hw, "llamacpp")
+ # available = (4 - 4) * 0.8 = 0 → nothing
+ assert result == ""
class TestRecommendModelMlx:
@@ -128,7 +120,9 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M1", vram_gb=8.0, count=1),
)
result = recommend_model(hw, "mlx")
- assert result == "qwen3.5:9b"
+ # available = 8 * 0.9 = 7.2 GB → ≤8 tier → qwen3:1.7b
+ # But qwen3:1.7b supports mlx → good
+ assert result == "qwen3:1.7b"
def test_apple_silicon_16gb_mlx(self) -> None:
hw = HardwareInfo(
@@ -137,10 +131,8 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M2", vram_gb=16.0, count=1),
)
result = recommend_model(hw, "mlx")
- # available = 16 * 0.9 = 14.4 GB
- # 27B * 0.5 * 1.1 = 14.85 → too big
- # 9B * 0.5 * 1.1 = 4.95 → fits
- assert result == "qwen3.5:9b"
+ # available = 16 * 0.9 = 14.4 GB → ≤16 tier → qwen3:4b
+ assert result == "qwen3:4b"
def test_apple_silicon_32gb_mlx(self) -> None:
hw = HardwareInfo(
@@ -149,7 +141,8 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M2 Pro", vram_gb=32.0, count=1),
)
result = recommend_model(hw, "mlx")
- assert result == "qwen3.5:27b"
+ # available = 32 * 0.9 = 28.8 GB → ≤32 tier → qwen3:8b
+ assert result == "qwen3:8b"
def test_apple_silicon_64gb_mlx(self) -> None:
hw = HardwareInfo(
@@ -158,4 +151,6 @@ class TestRecommendModelMlx:
gpu=GpuInfo(vendor="apple", name="Apple M2 Max", vram_gb=64.0, count=1),
)
result = recommend_model(hw, "mlx")
- assert result == "qwen3.5:27b"
+ # available = 64 * 0.9 = 57.6 GB → ≤64 tier → qwen3:14b
+ # qwen3:14b doesn't support mlx → fallback scan finds qwen3:8b
+ assert result == "qwen3:8b"