LLM inference in C/C++
C++ ggml
llama: add default load-mode auto, which avoids mmap on iGPUs (#26081)
Ruben Ortlam153d324bcf86
diff --git a/common/arg.cpp b/common/arg.cppindex c37d5cd0aa..cb314eee70 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2605,14 +2605,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex).set_env("LLAMA_ARG_DIO"));add_opt(common_arg({"-lm", "--load-mode"}, "MODE",- "model loading mode (default: mmap)\n"+ "model loading mode (default: auto)\n"+ "- auto: mmap, unless a device does not support it\n""- none: no special loading mode\n""- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n""- mlock: force system to keep model in RAM rather than swapping or compressing\n""- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n""- dio: use DirectIO if available\n",[](common_params & params, const std::string & value) {- /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }+ /**/ if (value == "auto") { params.load_mode = LLAMA_LOAD_MODE_AUTO; }+ else if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }else if (value == "mmap+mlock") { params.load_mode = LLAMA_LOAD_MODE_MMAP_MLOCK; }diff --git a/common/common.h b/common/common.hindex 878534dccf..d485d4fb41 100644--- a/common/common.h+++ b/common/common.h@@ -473,7 +473,7 @@ struct common_params {std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs- enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the model+ enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the modelcommon_cpu_params cpuparams;common_cpu_params cpuparams_batch;diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.hindex 2924fdbe98..cc3f8cd36e 100644--- a/ggml/include/ggml-backend.h+++ b/ggml/include/ggml-backend.h@@ -154,6 +154,8 @@ extern "C" {bool buffer_from_host_ptr;// event synchronizationbool events;+ // mmap is supported for loading+ bool mmap_support;};// all the device propertiesdiff --git a/ggml/src/ggml-backend-meta.cpp b/ggml/src/ggml-backend-meta.cppindex a5a3a58ad0..7654ea1f30 100644--- a/ggml/src/ggml-backend-meta.cpp+++ b/ggml/src/ggml-backend-meta.cpp@@ -132,6 +132,7 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_back/* .host_buffer = */ false, // Not implemented./* .buffer_from_host_ptr = */ false, // Not implemented./* .events = */ false, // Not implemented.+ /* .mmap_support = */ true,};for (ggml_backend_dev_t simple_dev : meta_dev_ctx->simple_devs) {ggml_backend_dev_props tmp_props;@@ -140,6 +141,7 @@ static void ggml_backend_meta_device_get_props(ggml_backend_dev_t dev, ggml_backprops->caps.host_buffer = props->caps.host_buffer && tmp_props.caps.host_buffer;props->caps.buffer_from_host_ptr = props->caps.buffer_from_host_ptr && tmp_props.caps.buffer_from_host_ptr;props->caps.events = props->caps.events && tmp_props.caps.events;+ props->caps.mmap_support = props->caps.mmap_support && tmp_props.caps.mmap_support;}}diff --git a/ggml/src/ggml-blas/ggml-blas.cpp b/ggml/src/ggml-blas/ggml-blas.cppindex 9745fa29f5..e4b5bd2547 100644--- a/ggml/src/ggml-blas/ggml-blas.cpp+++ b/ggml/src/ggml-blas/ggml-blas.cpp@@ -367,6 +367,7 @@ static void ggml_backend_blas_device_get_props(ggml_backend_dev_t dev, struct gg/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ true,/* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-cann/ggml-cann.cpp b/ggml/src/ggml-cann/ggml-cann.cppindex 5f51ea3bb3..ffa361af4e 100644--- a/ggml/src/ggml-cann/ggml-cann.cpp+++ b/ggml/src/ggml-cann/ggml-cann.cpp@@ -2815,6 +2815,7 @@ static void ggml_backend_cann_device_get_props(ggml_backend_dev_t dev, ggml_back/* .host_buffer = */ host_buffer,/* .buffer_from_host_ptr = */ false,/* .events = */ true,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cppindex 16cc5116c5..c0c9aa3cf0 100644--- a/ggml/src/ggml-cpu/ggml-cpu.cpp+++ b/ggml/src/ggml-cpu/ggml-cpu.cpp@@ -397,6 +397,7 @@ static void ggml_backend_cpu_device_get_props(ggml_backend_dev_t dev, struct ggm/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ true,/* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cuindex 05e8d7f736..1d4f4dfbd2 100644--- a/ggml/src/ggml-cuda/ggml-cuda.cu+++ b/ggml/src/ggml-cuda/ggml-cuda.cu@@ -4801,6 +4801,7 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back/* .host_buffer = */ host_buffer,/* .buffer_from_host_ptr = */ false,/* .events = */ events,+ /* .mmap_support = */ props->type != GGML_BACKEND_DEVICE_TYPE_IGPU,};}diff --git a/ggml/src/ggml-et/ggml-et.cpp b/ggml/src/ggml-et/ggml-et.cppindex b302090956..e8482f7346 100644--- a/ggml/src/ggml-et/ggml-et.cpp+++ b/ggml/src/ggml-et/ggml-et.cpp@@ -1646,6 +1646,7 @@ static void ggml_backend_et_device_get_props(ggml_backend_dev_t dev, struct ggml/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ false,/* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cppindex bdb8af0820..f80c60a500 100644--- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp+++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp@@ -3930,6 +3930,7 @@ static void ggml_backend_hexagon_device_get_props(ggml_backend_dev_t dev, struct/* .host_buffer = */ (bool) opt_hostbuf,/* .buffer_from_host_ptr = */ false,/* .events = */ false,+ /* .mmap_support = */ false,};}diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cppindex a1003b3acf..ef3c92f271 100644--- a/ggml/src/ggml-metal/ggml-metal.cpp+++ b/ggml/src/ggml-metal/ggml-metal.cpp@@ -681,6 +681,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ true,/* .events = */ true,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-opencl/ggml-opencl.cpp b/ggml/src/ggml-opencl/ggml-opencl.cppindex 9874ffe8d2..19dca4ced0 100644--- a/ggml/src/ggml-opencl/ggml-opencl.cpp+++ b/ggml/src/ggml-opencl/ggml-opencl.cpp@@ -10795,6 +10795,7 @@ static void ggml_backend_opencl_device_get_props(ggml_backend_dev_t dev, struct/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ false,/* .events = */ false,+ /* .mmap_support = */ false,};}diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cppindex 0e7501fefe..dfe80e6c86 100644--- a/ggml/src/ggml-openvino/ggml-openvino.cpp+++ b/ggml/src/ggml-openvino/ggml-openvino.cpp@@ -763,6 +763,7 @@ static void ggml_backend_openvino_device_get_props(ggml_backend_dev_t dev, ggml_/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ false,/* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-rpc/ggml-rpc.cpp b/ggml/src/ggml-rpc/ggml-rpc.cppindex 17c53a5f04..e9de0d0aa9 100644--- a/ggml/src/ggml-rpc/ggml-rpc.cpp+++ b/ggml/src/ggml-rpc/ggml-rpc.cpp@@ -1881,6 +1881,7 @@ static void ggml_backend_rpc_device_get_props(ggml_backend_dev_t dev, struct ggm/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ false,/* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cppindex 18d58782eb..8a19f648bf 100644--- a/ggml/src/ggml-sycl/ggml-sycl.cpp+++ b/ggml/src/ggml-sycl/ggml-sycl.cpp@@ -5649,6 +5649,7 @@ static void ggml_backend_sycl_device_get_props(ggml_backend_dev_t dev, ggml_back/* .host_buffer = */ host_buffer,/* .buffer_from_host_ptr = */ false,/* .events = */ events,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp b/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cppindex c7acb8b51c..87872df1c7 100644--- a/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp+++ b/ggml/src/ggml-virtgpu/backend/backend-dispatched-device.cpp@@ -111,6 +111,7 @@ uint32_t backend_device_get_props(apir_encoder * enc, apir_decoder * dec, virgl_apir_encode_bool_t(enc, &props.caps.host_buffer);apir_encode_bool_t(enc, &props.caps.buffer_from_host_ptr);apir_encode_bool_t(enc, &props.caps.events);+ apir_encode_bool_t(enc, &props.caps.mmap_support);return 0;}diff --git a/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h b/ggml/src/ggml-virtgpu/backend/shared/api_remoting.hindex 6bf97e8a3a..a5ef3ea476 100644--- a/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h+++ b/ggml/src/ggml-virtgpu/backend/shared/api_remoting.h@@ -7,7 +7,7 @@#include <cstdint>#define APIR_PROTOCOL_MAJOR 0-#define APIR_PROTOCOL_MINOR 1+#define APIR_PROTOCOL_MINOR 2#define APIR_HANDSHAKE_MAGIC 0xab1ediff --git a/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp b/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cppindex 8fa20ff43b..d5bdc993b4 100644--- a/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp+++ b/ggml/src/ggml-virtgpu/ggml-backend-buffer-type.cpp@@ -11,9 +11,9 @@ static ggml_backend_buffer_t ggml_backend_remoting_buffer_type_alloc_buffer(ggmlcontext->gpu = gpu;- bool async__unused, host_buffer__unused, events__unused;+ bool async__unused, host_buffer__unused, events__unused, mmap_support__unused;bool buffer_from_host_ptr;- apir_device_get_props(gpu, &async__unused, &host_buffer__unused, &buffer_from_host_ptr, &events__unused);+ apir_device_get_props(gpu, &async__unused, &host_buffer__unused, &buffer_from_host_ptr, &events__unused, &mmap_support__unused);if (buffer_from_host_ptr) {context->apir_context = apir_device_buffer_from_ptr(gpu, size, size);diff --git a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp b/ggml/src/ggml-virtgpu/ggml-backend-device.cppindex a978812cd9..987ce9dd11 100644--- a/ggml/src/ggml-virtgpu/ggml-backend-device.cpp+++ b/ggml/src/ggml-virtgpu/ggml-backend-device.cpp@@ -65,7 +65,7 @@ static void ggml_backend_remoting_device_get_props(ggml_backend_dev_t dev, ggml_virtgpu * gpu = DEV_TO_GPU(dev);apir_device_get_props(gpu, &props->caps.async, &props->caps.host_buffer, &props->caps.buffer_from_host_ptr,- &props->caps.events);+ &props->caps.events, &props->caps.mmap_support);props->caps.buffer_from_host_ptr = false;props->caps.async = false;diff --git a/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp b/ggml/src/ggml-virtgpu/virtgpu-forward-device.cppindex 9f513c138d..864264f213 100644--- a/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp+++ b/ggml/src/ggml-virtgpu/virtgpu-forward-device.cpp@@ -144,7 +144,8 @@ void apir_device_get_props(virtgpu * gpu,bool * async,bool * host_buffer,bool * buffer_from_host_ptr,- bool * events) {+ bool * events,+ bool * mmap_support) {apir_encoder * encoder;apir_decoder * decoder;ApirForwardReturnCode ret;@@ -157,6 +158,7 @@ void apir_device_get_props(virtgpu * gpu,apir_decode_bool_t(decoder, host_buffer);apir_decode_bool_t(decoder, buffer_from_host_ptr);apir_decode_bool_t(decoder, events);+ apir_decode_bool_t(decoder, mmap_support);remote_call_finish(gpu, encoder, decoder);diff --git a/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h b/ggml/src/ggml-virtgpu/virtgpu-forward.gen.hindex 44b0ad1ffa..da28aa5f90 100644--- a/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h+++ b/ggml/src/ggml-virtgpu/virtgpu-forward.gen.h@@ -13,7 +13,8 @@ void apir_device_get_props(struct virtgpu * gpu,bool * async,bool * host_buffer,bool * buffer_from_host_ptr,- bool * events);+ bool * events,+ bool * mmap_support);apir_buffer_context_t apir_device_buffer_from_ptr(struct virtgpu * gpu, size_t size, size_t max_tensor_size);/* buffer-type */diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cppindex a923755f9e..45fa97f812 100644--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp@@ -17891,6 +17891,7 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml/* .host_buffer = */ true,/* .buffer_from_host_ptr = */ false,/* .events = */ true,+ /* .mmap_support = */ !ctx->is_integrated_gpu,};}diff --git a/ggml/src/ggml-webgpu/ggml-webgpu.cpp b/ggml/src/ggml-webgpu/ggml-webgpu.cppindex 98c7162478..6741752b36 100644--- a/ggml/src/ggml-webgpu/ggml-webgpu.cpp+++ b/ggml/src/ggml-webgpu/ggml-webgpu.cpp@@ -3942,6 +3942,7 @@ static void ggml_backend_webgpu_device_get_props(ggml_backend_dev_t dev, struct/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ false,/* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-zdnn/ggml-zdnn.cpp b/ggml/src/ggml-zdnn/ggml-zdnn.cppindex 639b818d12..4007ac9dfc 100644--- a/ggml/src/ggml-zdnn/ggml-zdnn.cpp+++ b/ggml/src/ggml-zdnn/ggml-zdnn.cpp@@ -487,7 +487,8 @@ static void ggml_backend_zdnn_device_get_props(ggml_backend_dev_t dev, ggml_back/* .async = */ false,/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ false,- /* .events = */ false+ /* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/ggml/src/ggml-zendnn/ggml-zendnn.cpp b/ggml/src/ggml-zendnn/ggml-zendnn.cppindex e6a9b51b79..ec7ce23314 100644--- a/ggml/src/ggml-zendnn/ggml-zendnn.cpp+++ b/ggml/src/ggml-zendnn/ggml-zendnn.cpp@@ -654,7 +654,8 @@ static void ggml_backend_zendnn_device_get_props(ggml_backend_dev_t dev, struct/* .async = */ false,/* .host_buffer = */ false,/* .buffer_from_host_ptr = */ true,- /* .events = */ false+ /* .events = */ false,+ /* .mmap_support = */ true,};}diff --git a/include/llama.h b/include/llama.hindex 5a0b66dc25..bfef0e1d11 100644--- a/include/llama.h+++ b/include/llama.h@@ -203,11 +203,12 @@ extern "C" {};enum llama_load_mode {- LLAMA_LOAD_MODE_NONE = 0, // no special loading mode- LLAMA_LOAD_MODE_MMAP = 1, // memory map the model- LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing- LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing- LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available+ LLAMA_LOAD_MODE_AUTO = -1, // auto-detect based on device capabilities+ LLAMA_LOAD_MODE_NONE = 0, // no special loading mode+ LLAMA_LOAD_MODE_MMAP = 1, // memory map the model+ LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing+ LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing+ LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available};LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 3d50f8a1cb..51ba054396 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -543,7 +543,7 @@ llama_model_loader::llama_model_loader(tensor_buft_overrides = param_tensor_buft_overrides_p;- this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK;+ this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK || load_mode == LLAMA_LOAD_MODE_AUTO;this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO;if (!fname.empty()) {diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 3bf3a22f26..cc1917b6da 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1271,8 +1271,23 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {this->ml = &ml; // to be used by create_tensor() and load_arch_tensors()+ if (ml.use_mmap && params.load_mode == LLAMA_LOAD_MODE_AUTO) {+ for (const auto & dev : devices) {+ ggml_backend_dev_props props;+ ggml_backend_dev_get_props(dev.dev, &props);+ if (!props.caps.mmap_support) {+ ml.use_mmap = false;+ break;+ }+ }+ }++ const char * load_mode_name = params.load_mode == LLAMA_LOAD_MODE_AUTO+ ? llama_load_mode_name(ml.use_mmap ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE)+ : llama_load_mode_name(params.load_mode);+LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (load_mode = %s)\n",- __func__, llama_load_mode_name(params.load_mode));+ __func__, load_mode_name);// build a list of buffer types for the CPU and GPU devicespimpl->cpu_buft_list = make_cpu_buft_list(devices, params.use_extra_bufts, params.no_host);@@ -2452,7 +2467,7 @@ llama_model_params llama_model_default_params() {/*.tensor_buft_overrides =*/ nullptr,/*.n_gpu_layers =*/ -1,/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,- /*.load_mode =*/ LLAMA_LOAD_MODE_MMAP,+ /*.load_mode =*/ LLAMA_LOAD_MODE_AUTO,/*.main_gpu =*/ 0,/*.tensor_split =*/ nullptr,/*.progress_callback =*/ nullptr,diff --git a/src/llama.cpp b/src/llama.cppindex d6e0bbfefa..94c8f60e0c 100644--- a/src/llama.cpp+++ b/src/llama.cpp@@ -48,6 +48,8 @@ const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_tyconst char * llama_load_mode_name(enum llama_load_mode load_mode) {switch (load_mode) {+ case LLAMA_LOAD_MODE_AUTO:+ return "auto";case LLAMA_LOAD_MODE_NONE:return "none";case LLAMA_LOAD_MODE_MMAP:@@ -63,11 +65,12 @@ const char * llama_load_mode_name(enum llama_load_mode load_mode) {}enum llama_load_mode llama_load_mode_from_str(const char * str) {- if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }- if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }- if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }+ if (std::strcmp(str, "auto") == 0) { return LLAMA_LOAD_MODE_AUTO; }+ if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }+ if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }+ if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }if (std::strcmp(str, "mmap+mlock") == 0) { return LLAMA_LOAD_MODE_MMAP_MLOCK; }- if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }+ if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }throw std::invalid_argument(std::string("unknown load mode: ") + str);}diff --git a/tools/cli/README.md b/tools/cli/README.mdindex b42b2e5343..880c4a5408 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -58,7 +58,7 @@| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |-| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |diff --git a/tools/completion/README.md b/tools/completion/README.mdindex 552a0c6abf..c2e52ac066 100644--- a/tools/completion/README.md+++ b/tools/completion/README.md@@ -141,7 +141,7 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |-| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cppindex c17a27b540..7c495afe20 100644--- a/tools/llama-bench/llama-bench.cpp+++ b/tools/llama-bench/llama-bench.cpp@@ -384,7 +384,7 @@ static const cmd_params cmd_params_defaults = {/* n_gpu_layers */ { -1 },/* n_cpu_moe */ { 0 },/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },- /* load_mode */ { LLAMA_LOAD_MODE_MMAP },+ /* load_mode */ { LLAMA_LOAD_MODE_AUTO },/* main_gpu */ { 0 },/* no_kv_offload */ { false },/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },@@ -459,7 +459,7 @@ static void print_usage(int /* argc */, char ** argv) {printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");- printf(" -lm, --load-mode <none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());+ printf(" -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());@@ -764,7 +764,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {std::vector<llama_load_mode> modes;for (const auto & m : p) {llama_load_mode mode;- if (m == "none") {+ if (m == "auto") {+ mode = LLAMA_LOAD_MODE_AUTO;+ } else if (m == "none") {mode = LLAMA_LOAD_MODE_NONE;} else if (m == "mmap") {mode = LLAMA_LOAD_MODE_MMAP;diff --git a/tools/server/README.md b/tools/server/README.mdindex 6927caddbb..a2ab872b4c 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -75,7 +75,7 @@ For the full list of features, please refer to [server's changelog](https://gith| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |-| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |