Skip to content

LLM inference in C/C++

C++ ggml

llama: disable lazy tensor loading by default on iGPUs (#28326)

Ruben Ortlamf3f1a8f2760f
diff --git a/src/llama-model.cpp b/src/llama-model.cpp
index ffedf89e67..0adc07449b 100644
--- a/src/llama-model.cpp
+++ b/src/llama-model.cpp
@@ -1422,6 +1422,18 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
}
}
+ // resolve AUTO on systems without mmap support (e.g. iGPUs): fall back to OFF; see #28160
+ if (ml.lazy.mode == LLAMA_LAZY_MODE_AUTO) {
+ for (const auto & dev : devices) {
+ ggml_backend_dev_props props;
+ ggml_backend_dev_get_props(dev.dev, &props);
+ if (!props.caps.mmap_support) {
+ ml.lazy.mode = LLAMA_LAZY_MODE_OFF;
+ break;
+ }
+ }
+ }
+
const char * load_mode_name = params.load_mode == LLAMA_LOAD_MODE_AUTO
? llama_load_mode_name(ml.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE)
: llama_load_mode_name(params.load_mode);