LLM inference in C/C++
C++ ggml
llama: disable lazy tensor loading by default on iGPUs (#28326)
Ruben Ortlamf3f1a8f2760f
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex ffedf89e67..0adc07449b 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1422,6 +1422,18 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {}}+ // resolve AUTO on systems without mmap support (e.g. iGPUs): fall back to OFF; see #28160+ if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO) {+ for (const auto & dev : devices) {+ ggml_backend_dev_props props;+ ggml_backend_dev_get_props(dev.dev, &props);+ if (!props.caps.mmap_support) {+ ml.lazy.mode = LLAMA_LAZY_MODE_OFF;+ break;+ }+ }+ }+const char * load_mode_name = params.load_mode == LLAMA_LOAD_MODE_AUTO? llama_load_mode_name(ml.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE): llama_load_mode_name(params.load_mode);