Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio (Jetson): size llama-server context from system RAM when nvidia-smi reports [N/A] (#13286)

Daniel Hanb2925e79054d
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 1cd7ee352b1..0ea20519050 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -14516,6 +14516,7 @@ class LlamaCppBackend:
if LlamaCppBackend._is_vulkan_backend(binary):
return LlamaCppBackend._get_gpu_free_memory_vulkan(binary)
# ── NVIDIA via nvidia-smi ────────────────────────────────────
+ unsized_ids: list[int] = []
try:
from utils.hardware import gpu_query
@@ -14547,6 +14548,9 @@ class LlamaCppBackend:
idx = int(parts[0])
free_mib = int(parts[1])
except ValueError:
+ # Jetson answers "[N/A]": no dedicated VRAM (#4862).
+ if parts[0].isdigit() and (allowed is None or int(parts[0]) in allowed):
+ unsized_ids.append(int(parts[0]))
continue
# Total parsed separately: a two-column line or a non-integer
# total ("N/A" on MIG/vGPU) keeps the GPU at total 0 (fit uses
@@ -14584,6 +14588,43 @@ class LlamaCppBackend:
return rocm_gpus
# ── Torch fallback (covers AMD ROCm and missing nvidia-smi) ──
+ gpus = LlamaCppBackend._get_gpu_memory_torch(binary, for_llama_server)
+ if gpus or not (for_llama_server and unsized_ids):
+ return gpus
+ return LlamaCppBackend._tegra_unified_memory_rows(unsized_ids)
+
+ @staticmethod
+ def _tegra_unified_memory_rows(gpu_ids: list[int]) -> list[tuple[int, int, int]]:
+ """Price a Jetson's GPU from system RAM when nvidia-smi answers ``[N/A]`` and
+ torch cannot see it (a generic aarch64 wheel), so the context is still capped
+ (#4862). llama-server only: it runs on JetPack's CUDA, torch-backed callers do not."""
+ from utils.tegra import is_tegra
+
+ if not is_tegra():
+ return []
+ avail = LlamaCppBackend._available_system_memory_mib()
+ if avail is None:
+ return []
+ cgroup_mib = LlamaCppBackend._cgroup_available_memory_mib()
+ # Same policy as the torch integrated branch: a cgroup-bound pool publishes no total.
+ total = (
+ 0
+ if cgroup_mib is not None and cgroup_mib <= avail
+ else LlamaCppBackend._total_system_memory_mib() or 0
+ )
+ free = _apply_igpu_host_reserve_mib(avail, True)
+ LlamaCppBackend._GPU_IDS_ARE_PCI_INDICES = True
+ # Torch never saw these, so record them as integrated or the pool is credited twice.
+ LlamaCppBackend._INTEGRATED_CUDA_IDS[LlamaCppBackend._integrated_cuda_mask_key()] = set(
+ gpu_ids
+ )
+ return [(idx, free, total) for idx in sorted(gpu_ids)]
+
+ @staticmethod
+ def _get_gpu_memory_torch(
+ binary: Optional[str], for_llama_server: bool
+ ) -> list[tuple[int, int, int]]:
+ """The torch arm of ``_get_gpu_memory``."""
try:
import torch
@@ -16057,6 +16098,11 @@ class LlamaCppBackend:
# unified" question, and a discrete card next to an APU still has a pool of
# its own, so re-ask per row rather than forfeit a fit that is really there.
shared |= {idx for idx, _free in rows if self._amd_apu_wants_unified_memory([idx])}
+ # Integrated CUDA SoCs (Jetson, DGX Spark) too. Cache only: _get_gpu_memory filled it.
+ _integrated = LlamaCppBackend._INTEGRATED_CUDA_IDS.get(
+ LlamaCppBackend._integrated_cuda_mask_key(), ()
+ )
+ shared |= {idx for idx, _free in rows if idx in _integrated}
# --no-kv-offload puts the WHOLE cache in host RAM whatever the layer placement
# says (llama-kv-cache.cpp upgrades a layer's buffer type only inside
# `if (offload)`), so free VRAM may not pay for it. Resolved off the same
diff --git a/studio/backend/tests/test_load_mode_fit_matrix.py b/studio/backend/tests/test_load_mode_fit_matrix.py
index 3c89443f5a8..ebc1a9078a9 100644
--- a/studio/backend/tests/test_load_mode_fit_matrix.py
+++ b/studio/backend/tests/test_load_mode_fit_matrix.py
@@ -2409,3 +2409,12 @@ def test_the_reserve_holds_for_a_placement_the_fitter_will_not_move():
assert _placement_is_fitter_proof(["-ngl", "auto"], env = {}) is False
assert _placement_is_fitter_proof(None, gpu_layers = 0, env = {}) is True
assert _placement_is_fitter_proof(None, env = {}) is False
+
+
[email protected]("platform", ["linux", "windows", "wsl"])
+def test_an_integrated_cuda_soc_is_never_counted_twice(platform, monkeypatch):
+ """A Jetson / DGX Spark row is the host pool too (#4862): 40 GiB does not fit 32 GiB twice."""
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ assert _mode(platform, "nvidia_discrete", 40 * GIB, 32 * 1024, monkeypatch) == FIT_MODE
+ LlamaCppBackend._INTEGRATED_CUDA_IDS[LlamaCppBackend._integrated_cuda_mask_key()] = {0}
+ assert _mode(platform, "nvidia_discrete", 40 * GIB, 32 * 1024, monkeypatch) is None
diff --git a/studio/backend/tests/test_tegra_na_gpu_memory.py b/studio/backend/tests/test_tegra_na_gpu_memory.py
new file mode 100644
index 00000000000..49b1bf391d7
--- /dev/null
+++ b/studio/backend/tests/test_tegra_na_gpu_memory.py
@@ -0,0 +1,108 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A Jetson whose nvidia-smi answers "[N/A]" for memory and whose torch cannot see the GPU is
+priced from system RAM for llama-server, so the context is still capped (#4862)."""
+
+import importlib.util
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+from core.inference.llama_cpp import LlamaCppBackend, _apply_igpu_host_reserve_mib
+from utils import tegra
+from utils.hardware import gpu_query
+
+_spec = importlib.util.spec_from_file_location(
+ "_auto_offload_matrix_for_tegra",
+ Path(__file__).resolve().parent / "test_auto_offload_ctx_platform_matrix.py",
+)
+M = importlib.util.module_from_spec(_spec)
+sys.modules[_spec.name] = M # dataclasses resolves annotations through sys.modules
+_spec.loader.exec_module(M)
+
+TOTAL_MIB = 62_841
+AVAIL_MIB = 58_000
+NATIVE_CTX = 262_144
+
+
+def _probe(
+ monkeypatch,
+ *,
+ jetson = True,
+ smi_free = "[N/A]",
+ for_llama_server = True,
+ cgroup = None,
+):
+ for name in ("CUDA_VISIBLE_DEVICES", "HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES"):
+ monkeypatch.delenv(name, raising = False)
+ monkeypatch.setattr(LlamaCppBackend, "_INTEGRATED_CUDA_IDS", {})
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda _b = None: False))
+ monkeypatch.setattr(
+ gpu_query,
+ "run_nvidia_smi",
+ lambda argv, **_kw: subprocess.CompletedProcess(
+ argv, 0, f"0, {smi_free}, {smi_free}\n", ""
+ ),
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_get_gpu_memory_nvml", staticmethod(lambda: []))
+ monkeypatch.setattr(
+ LlamaCppBackend, "_get_gpu_memory_amd_smi", staticmethod(lambda *_a, **_k: [])
+ )
+ # A generic aarch64 torch wheel cannot drive the Tegra GPU.
+ monkeypatch.setitem(sys.modules, "torch", None)
+ monkeypatch.setattr(tegra, "is_tegra", lambda: jetson)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_available_system_memory_mib", staticmethod(lambda: AVAIL_MIB)
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_total_system_memory_mib", staticmethod(lambda: TOTAL_MIB)
+ )
+ monkeypatch.setattr(
+ LlamaCppBackend, "_cgroup_available_memory_mib", staticmethod(lambda: cgroup)
+ )
+ return LlamaCppBackend._get_gpu_memory("/fake/llama-server", for_llama_server = for_llama_server)
+
+
+def test_jetson_na_is_priced_from_system_ram(monkeypatch):
+ assert _probe(monkeypatch) == [(0, _apply_igpu_host_reserve_mib(AVAIL_MIB, True), TOTAL_MIB)]
+ # One shared pool: it must not also be credited as discrete VRAM.
+ assert LlamaCppBackend._integrated_cuda_unified_memory([0])
+
+
+def test_cgroup_bound_jetson_publishes_no_total(monkeypatch):
+ assert _probe(monkeypatch, cgroup = 30_000) == [
+ (0, _apply_igpu_host_reserve_mib(AVAIL_MIB, True), 0)
+ ]
+
+
+ "kwargs",
+ [
+ {"jetson": False}, # a non-Jetson [N/A] row (MIG / vGPU) keeps today's answer
+ {"for_llama_server": False}, # torch-backed callers cannot use this GPU
+ ],
+)
+def test_other_hosts_and_callers_are_unchanged(monkeypatch, kwargs):
+ assert _probe(monkeypatch, **kwargs) == []
+ assert not LlamaCppBackend._integrated_cuda_unified_memory([0])
+
+
+def test_jetson_na_caps_native_context(tmp_path, monkeypatch):
+ rows = _probe(monkeypatch)
+ monkeypatch.undo()
+ linux = next(p for p in M.PLATFORMS if p[0] == "linux")
+ backend, gguf = M.cell_backend(
+ tmp_path,
+ monkeypatch,
+ linux,
+ M.Accelerator("jetson", False, tuple(rows)),
+ native_ctx = NATIVE_CTX,
+ )
+ backend._get_gguf_size_bytes = lambda _p: 16 << 30
+ backend._estimate_kv_cache_bytes = lambda ctx, *_a, **_k: int(ctx * 0.25 * M.MIB)
+ ctx = int(M._flag(M._launch(backend, gguf, n_ctx = 0)["cmd"], "-c", "--ctx-size"))
+ # 16 GiB of weights plus 64 GiB of KV at native cannot fit a 64 GB pool.
+ assert 0 < ctx < NATIVE_CTX