LLM inference in C/C++
C++ ggml
args: refactor mlock/mmap/directio into load-mode (#20834)
Aaron Teoe6dd0e29a675
diff --git a/common/arg.cpp b/common/arg.cppindex 1f5b5a8abf..a287b907d4 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -5,6 +5,7 @@#include "common.h"#include "download.h"#include "json-schema-to-grammar.h"+#include "llama.h"#include "log.h"#include "sampling.h"#include "speculative.h"@@ -785,6 +786,17 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_contextarg.c_str(), e.what(), opt.to_string().c_str()));}}++ // TODO: remove this check after deprecating --mmap|mlock|dio+ auto has_arg = [&](std::initializer_list<const char *> names) {+ return std::any_of(names.begin(), names.end(), [&](const char * name) {+ return seen_args.count(name);+ });+ };+ if (has_arg({"-lm", "--load-mode"}) &&+ has_arg({"--mlock", "--mmap", "--no-mmap", "-dio", "--direct-io", "-ndio", "--no-direct-io"})) {+ LOG_WRN("DEPRECATED: `--load-mode` and `--mlock`/`--mmap`/`--direct-io` should not be combined; only the last flag on the command line will take effect\n");+ }};// parse all CLI args now, so that -hf is available below for remote preset resolution@@ -2495,27 +2507,45 @@ common_params_context common_params_parser_init(common_params & params, llama_ex}add_opt(common_arg({"--mlock"},- "force system to keep model in RAM rather than swapping or compressing",+ "DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing",[](common_params & params) {- params.use_mlock = true;+ LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");+ params.load_mode = LLAMA_LOAD_MODE_MLOCK;}).set_env("LLAMA_ARG_MLOCK"));add_opt(common_arg({"--mmap"},{"--no-mmap"},- string_format("whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: %s)", params.use_mmap ? "enabled" : "disabled"),+ "DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)",[](common_params & params, bool value) {- params.use_mmap = value;+ LOG_WRN("DEPRECATED: --mmap and --no-mmap are deprecated. use --load-mode mmap instead\n");+ params.load_mode = value ? LLAMA_LOAD_MODE_MMAP : LLAMA_LOAD_MODE_NONE;}).set_env("LLAMA_ARG_MMAP"));add_opt(common_arg({"-dio", "--direct-io"},{"-ndio", "--no-direct-io"},- string_format("use DirectIO if available. (default: %s)", params.use_direct_io ? "enabled" : "disabled"),+ "DEPRECATED in favor of `--load-mode`: use DirectIO if available",[](common_params & params, bool value) {- params.use_direct_io = value;+ LOG_WRN("DEPRECATED: --direct-io and --no-direct-io are deprecated. use --load-mode dio instead\n");+ params.load_mode = value ? LLAMA_LOAD_MODE_DIRECT_IO : LLAMA_LOAD_MODE_NONE;}).set_env("LLAMA_ARG_DIO"));+ add_opt(common_arg(+ {"-lm", "--load-mode"}, "MODE",+ "model loading mode (default: mmap)\n"+ "- none: no special loading mode\n"+ "- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n"+ "- mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"+ "- dio: use DirectIO if available\n",+ [](common_params & params, const std::string & value) {+ /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }+ else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }+ else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }+ else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }+ else { throw std::invalid_argument("invalid value"); }+ }+ ).set_env("LLAMA_ARG_LOAD_MODE"));add_opt(common_arg({"--numa"}, "TYPE","attempt optimizations that help on some NUMA systems\n"diff --git a/common/common.cpp b/common/common.cppindex 8f13217ab4..a68766cbbb 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1558,10 +1558,8 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {mparams.n_gpu_layers = params.n_gpu_layers;mparams.main_gpu = params.main_gpu;mparams.split_mode = params.split_mode;+ mparams.load_mode = params.load_mode;mparams.tensor_split = params.tensor_split;- mparams.use_mmap = params.use_mmap;- mparams.use_direct_io = params.use_direct_io;- mparams.use_mlock = params.use_mlock;mparams.check_tensors = params.check_tensors;mparams.use_extra_bufts = !params.no_extra_bufts;mparams.no_host = params.no_host;diff --git a/common/common.h b/common/common.hindex bffc1767a7..cb1827f5dd 100644--- a/common/common.h+++ b/common/common.h@@ -6,6 +6,7 @@#include "ggml-opt.h"#include "ggml.h"+#include "llama.h"#include <set>#include <sstream>@@ -482,6 +483,7 @@ struct common_params {std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs+ enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the modelcommon_cpu_params cpuparams;common_cpu_params cpuparams_batch;@@ -572,9 +574,6 @@ struct common_params {bool kv_unified = false; // enable unified KV cachebool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix- bool use_mmap = true; // enable mmap to use filesystem cache- bool use_direct_io = false; // read from disk without buffering- bool use_mlock = false; // use mlock to keep model in memorybool verbose_prompt = false; // print prompt tokens before generationbool display_prompt = true; // print prompt before generationbool no_kv_offload = false; // disable KV offloadingdiff --git a/common/fit.cpp b/common/fit.cppindex afbf0b10f3..c79221cb00 100644--- a/common/fit.cpp+++ b/common/fit.cpp@@ -54,8 +54,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(llama_model_params mparams_copy = *mparams;mparams_copy.no_alloc = true;- mparams_copy.use_mmap = false;- mparams_copy.use_mlock = false;+ mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;llama_model * model = llama_model_load_from_file(path_model, mparams_copy);if (model == nullptr) {diff --git a/examples/diffusion/diffusion-cli.cpp b/examples/diffusion/diffusion-cli.cppindex 86ebbf88c9..d58d22eff5 100644--- a/examples/diffusion/diffusion-cli.cpp+++ b/examples/diffusion/diffusion-cli.cpp@@ -117,9 +117,7 @@ int main(int argc, char ** argv) {llama_model_params model_params = llama_model_default_params();model_params.n_gpu_layers = params.n_gpu_layers;model_params.devices = params.devices.data();- model_params.use_mmap = params.use_mmap;- model_params.use_direct_io = params.use_direct_io;- model_params.use_mlock = params.use_mlock;+ model_params.load_mode = params.load_mode;model_params.check_tensors = params.check_tensors;llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);diff --git a/examples/training/finetune.cpp b/examples/training/finetune.cppindex 0a75ac110c..44b2843918 100644--- a/examples/training/finetune.cpp+++ b/examples/training/finetune.cpp@@ -26,10 +26,9 @@ int main(int argc, char ** argv) {return 1;}- if (params.use_mmap) {- LOG_INF("%s: force disabling memory mapping because it would result in-read-only pointers to the weights\n",- __func__);- params.use_mmap = false;+ if (params.load_mode != LLAMA_LOAD_MODE_NONE) {+ LOG_INF("%s: forcing load_mode = none to enable writable pointers to the weights\n", __func__);+ params.load_mode = LLAMA_LOAD_MODE_NONE;}if (params.cache_type_k != GGML_TYPE_F32) {LOG_INF("%s: force changing k cache type to f32 due to a lack of f16 support for OUT_PROD\n", __func__);diff --git a/include/llama.h b/include/llama.hindex a311ac2023..9fab693170 100644--- a/include/llama.h+++ b/include/llama.h@@ -202,6 +202,16 @@ extern "C" {LLAMA_SPLIT_MODE_TENSOR = 3,};+ enum llama_load_mode {+ LLAMA_LOAD_MODE_NONE = 0, // no special loading mode+ LLAMA_LOAD_MODE_MMAP = 1, // memory map the model+ LLAMA_LOAD_MODE_MLOCK = 2, // mmap + force system to keep model in RAM rather than swapping or compressing+ LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available+ };++ LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);+ LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);+enum llama_context_type {LLAMA_CONTEXT_TYPE_DEFAULT = 0,LLAMA_CONTEXT_TYPE_MTP = 1,@@ -301,6 +311,7 @@ extern "C" {int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layersenum llama_split_mode split_mode; // how to split the model across multiple GPUs+ enum llama_load_mode load_mode; // how to load the model// the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONEint32_t main_gpu;@@ -321,9 +332,6 @@ extern "C" {// Keep the booleans together to avoid misalignment during copy-by-value.bool vocab_only; // only load the vocabulary, no weights- bool use_mmap; // use mmap if possible- bool use_direct_io; // use direct io, takes precedence over use_mmap when supported- bool use_mlock; // force system to keep model in RAMbool check_tensors; // validate model tensor databool use_extra_bufts; // use extra buffer types (used for weight repacking)bool no_host; // bypass host buffer allowing extra buffers to be useddiff --git a/scripts/compare-llama-bench.py b/scripts/compare-llama-bench.pyindex 5a6cc7dbb1..e5f26b5a41 100755--- a/scripts/compare-llama-bench.py+++ b/scripts/compare-llama-bench.py@@ -28,7 +28,7 @@ LLAMA_BENCH_DB_FIELDS = ["model_type", "model_size", "model_n_params", "n_batch", "n_ubatch", "n_threads","cpu_mask", "cpu_strict", "poll", "type_k", "type_v", "n_gpu_layers","split_mode", "main_gpu", "no_kv_offload", "flash_attn", "tensor_split", "tensor_buft_overrides",- "use_mmap", "embeddings", "no_op_offload", "n_prompt", "n_gen", "n_depth",+ "load_mode", "embeddings", "no_op_offload", "n_prompt", "n_gen", "n_depth","test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts", "n_cpu_moe","fit_target", "fit_min_ctx"]@@ -38,7 +38,7 @@ LLAMA_BENCH_DB_TYPES = ["TEXT", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER","TEXT", "INTEGER", "INTEGER", "TEXT", "TEXT", "INTEGER","TEXT", "INTEGER", "INTEGER", "INTEGER", "TEXT", "TEXT",- "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER",+ "TEXT", "INTEGER", "INTEGER", "INTEGER", "INTEGER", "INTEGER","TEXT", "INTEGER", "INTEGER", "REAL", "REAL", "INTEGER","INTEGER", "INTEGER"]@@ -63,7 +63,7 @@ assert len(TEST_BACKEND_OPS_DB_FIELDS) == len(TEST_BACKEND_OPS_DB_TYPES)LLAMA_BENCH_KEY_PROPERTIES = ["cpu_info", "gpu_info", "backends", "n_gpu_layers", "n_cpu_moe", "tensor_buft_overrides", "model_filename", "model_type","n_batch", "n_ubatch", "embeddings", "cpu_mask", "cpu_strict", "poll", "n_threads", "type_k", "type_v",- "use_mmap", "no_kv_offload", "split_mode", "main_gpu", "tensor_split", "flash_attn", "n_prompt", "n_gen", "n_depth",+ "load_mode", "no_kv_offload", "split_mode", "main_gpu", "tensor_split", "flash_attn", "n_prompt", "n_gen", "n_depth","fit_target", "fit_min_ctx"]@@ -73,7 +73,7 @@ TEST_BACKEND_OPS_KEY_PROPERTIES = []# Properties that are boolean and are converted to Yes/No for the table:-LLAMA_BENCH_BOOL_PROPERTIES = ["embeddings", "cpu_strict", "use_mmap", "no_kv_offload", "flash_attn"]+LLAMA_BENCH_BOOL_PROPERTIES = ["embeddings", "cpu_strict", "no_kv_offload", "flash_attn"]TEST_BACKEND_OPS_BOOL_PROPERTIES = ["supported", "passed"]# Header names for the table (llama-bench):@@ -82,7 +82,7 @@ LLAMA_BENCH_PRETTY_NAMES = {"tensor_buft_overrides": "Tensor overrides", "model_filename": "File", "model_type": "Model", "model_size": "Model size [GiB]","model_n_params": "Num. of par.", "n_batch": "Batch size", "n_ubatch": "Microbatch size", "embeddings": "Embeddings","cpu_mask": "CPU mask", "cpu_strict": "CPU strict", "poll": "Poll", "n_threads": "Threads", "type_k": "K type", "type_v": "V type",- "use_mmap": "Use mmap", "no_kv_offload": "NKVO", "split_mode": "Split mode", "main_gpu": "Main GPU", "tensor_split": "Tensor split",+ "load_mode": "Load mode", "no_kv_offload": "NKVO", "split_mode": "Split mode", "main_gpu": "Main GPU", "tensor_split": "Tensor split","flash_attn": "FlashAttention",}diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 28f8bb7934..43447f57d3 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -4,6 +4,7 @@#include "ggml.h"#include "gguf.h"#include "llama-hparams.h"+#include "llama.h"#include <algorithm>#include <array>@@ -522,8 +523,7 @@ llama_model_loader::llama_model_loader(const std::string & fname,std::vector<std::string> & splits,FILE * file,- bool use_mmap,- bool use_direct_io,+ llama_load_mode load_mode,bool check_tensors,bool no_alloc,const llama_model_kv_override * param_overrides_p,@@ -542,6 +542,9 @@ llama_model_loader::llama_model_loader(tensor_buft_overrides = param_tensor_buft_overrides_p;+ this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MLOCK;+ this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO;+if (!fname.empty()) {// Load the main GGUFstruct ggml_context * ctx = NULL;@@ -562,20 +565,6 @@ llama_model_loader::llama_model_loader(files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));contexts.emplace_back(ctx);- if (use_mmap && use_direct_io) {- if (files.back()->has_direct_io()) {- LLAMA_LOG_WARN("%s: direct I/O is enabled, disabling mmap\n", __func__);- use_mmap = false;- } else {- LLAMA_LOG_WARN("%s: direct I/O is not available, using mmap\n", __func__);- use_direct_io = false;-- // reopen file using std::fopen for mmap- files.pop_back();- files.emplace_back(new llama_file(fname.c_str(), "rb", false));- }- }-// Save tensors data offset of the main file.// For subsidiary files, `meta` tensor data offset must not be used,// so we build a unified tensors index for weights.@@ -816,13 +805,11 @@ llama_model_loader::llama_model_loader(}}- if (!llama_mmap::SUPPORTED) {+ if (this->use_mmap && !llama_mmap::SUPPORTED) {LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);- use_mmap = false;+ this->use_mmap = false;}- this->use_mmap = use_mmap;- this->use_direct_io = use_direct_io;this->check_tensors = check_tensors;this->no_alloc = no_alloc;}diff --git a/src/llama-model-loader.h b/src/llama-model-loader.hindex c476026d3e..75a3652d06 100644--- a/src/llama-model-loader.h+++ b/src/llama-model-loader.h@@ -126,8 +126,7 @@ struct llama_model_loader {const std::string & fname,std::vector<std::string> & splits, // optional, only need if the split does not follow naming schemeFILE * file,- bool use_mmap,- bool use_direct_io,+ llama_load_mode load_mode,bool check_tensors,bool no_alloc,const llama_model_kv_override * param_overrides_p,diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 7e5bab26c5..c9b290e99f 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -16,6 +16,7 @@#include "llama-memory-hybrid-iswa.h"#include "llama-memory-recurrent.h"+#include "llama.h"#include "models/models.h"#include "ggml.h"@@ -1243,7 +1244,7 @@ void llama_model_base::load_vocab(llama_model_loader & ml) {bool llama_model_base::load_tensors(llama_model_loader & ml) {const auto & split_mode = params.split_mode;- const auto & use_mlock = params.use_mlock;+ const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK;const auto & tensor_split = params.tensor_split;const int n_layer_all = hparams.n_layer_all;@@ -1253,8 +1254,8 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {this->ml = &ml; // to be used by create_tensor() and load_arch_tensors()- LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (mmap = %s, direct_io = %s)\n",- __func__, ml.use_mmap ? "true" : "false", ml.use_direct_io ? "true" : "false");+ LLAMA_LOG_INFO("%s: loading model tensors, this can take a while... (load_mode = %s)\n",+ __func__, llama_load_mode_name(params.load_mode));// build a list of buffer types for the CPU and GPU devicespimpl->cpu_buft_list = make_cpu_buft_list(devices, params.use_extra_bufts, params.no_host);@@ -2318,15 +2319,13 @@ llama_model_params llama_model_default_params() {/*.tensor_buft_overrides =*/ nullptr,/*.n_gpu_layers =*/ -1,/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,+ /*.load_mode =*/ LLAMA_LOAD_MODE_MMAP,/*.main_gpu =*/ 0,/*.tensor_split =*/ nullptr,/*.progress_callback =*/ nullptr,/*.progress_callback_user_data =*/ nullptr,/*.kv_overrides =*/ nullptr,/*.vocab_only =*/ false,- /*.use_mmap =*/ true,- /*.use_direct_io =*/ false,- /*.use_mlock =*/ false,/*.check_tensors =*/ false,/*.use_extra_bufts =*/ true,/*.no_host =*/ false,diff --git a/src/llama-quant.cpp b/src/llama-quant.cppindex b187f5de96..afecb76946 100644--- a/src/llama-quant.cpp+++ b/src/llama-quant.cpp@@ -2,6 +2,7 @@#include "llama-model.h"#include "llama-model-loader.h"#include "llama-ext.h"+#include "llama.h"#include <algorithm>#include <cmath>@@ -876,15 +877,15 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std::// mmap consistently increases speed on Linux, and also increases speed on Windows with// hot cache. It may cause a slowdown on macOS, possibly related to free memory.#if defined(__linux__) || defined(_WIN32)- constexpr bool use_mmap = true;+ constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP;#else- constexpr bool use_mmap = false;+ constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_NONE;#endifconst llama_model_kv_override * kv_overrides = params->kv_overrides;std::vector<std::string> splits = {};llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr,- fname_inp, splits, /*file*/ nullptr, use_mmap, /*use_direct_io*/ false, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);+ fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, kv_overrides, nullptr);ml.init_mappings(false); // no prefetchingauto mparams = llama_model_default_params();diff --git a/src/llama.cpp b/src/llama.cppindex 0de6048f28..11ac9656d9 100644--- a/src/llama.cpp+++ b/src/llama.cpp@@ -46,6 +46,28 @@ const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_tyGGML_ABORT("fatal error");}+const char * llama_load_mode_name(enum llama_load_mode load_mode) {+ switch (load_mode) {+ case LLAMA_LOAD_MODE_NONE:+ return "none";+ case LLAMA_LOAD_MODE_MMAP:+ return "mmap";+ case LLAMA_LOAD_MODE_MLOCK:+ return "mlock";+ case LLAMA_LOAD_MODE_DIRECT_IO:+ return "dio";+ }+ GGML_ABORT("fatal error");+}++enum llama_load_mode llama_load_mode_from_str(const char * str) {+ if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }+ if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }+ if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }+ if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }+ throw std::invalid_argument(std::string("unknown load mode: ") + str);+}+struct llama_sampler_chain_params llama_sampler_chain_default_params() {struct llama_sampler_chain_params result = {/*.no_perf =*/ true,@@ -279,7 +301,7 @@ static bool llama_prepare_model_devices(const llama_model_params & params, llamastatic std::pair<int, llama_model *> llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud,const std::string & fname, std::vector<std::string> & splits, FILE * file, llama_model_params & params) {try {- llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.use_mmap, params.use_direct_io,+ llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,params.check_tensors, params.no_alloc, params.kv_overrides, params.tensor_buft_overrides);ml.print_info();@@ -412,7 +434,7 @@ struct llama_model * llama_model_init_from_user(GGML_ASSERT(metadata != nullptr);std::string path_model;std::vector<std::string> splits = {};- params.use_mmap = false;+ params.load_mode = LLAMA_LOAD_MODE_NONE;params.use_extra_bufts = false;return llama_model_load_from_file_impl(metadata, set_tensor_data, set_tensor_data_ud, path_model, splits, /*file*/ nullptr, params);}diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cppindex e83ee85dd4..000ecd9aaa 100644--- a/tests/test-arg-parser.cpp+++ b/tests/test-arg-parser.cpp@@ -1,6 +1,7 @@#include "arg.h"#include "common.h"#include "download.h"+#include "llama.h"#include <string>#include <vector>@@ -102,11 +103,9 @@ static void test(void) {argv = {"binary_name", "--draft", "123"};assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));- // negated arg- argv = {"binary_name", "--no-mmap"};+ argv = {"binary_name", "-lm", "hello"};assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));-printf("test-arg-parser: test valid usage\n\n");argv = {"binary_name", "-m", "model_file.gguf"};@@ -132,6 +131,22 @@ static void test(void) {assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));assert(params.speculative.draft.n_max == 123);+ argv = {"binary_name", "-lm", "none"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_NONE);++ argv = {"binary_name", "-lm", "mmap"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);++ argv = {"binary_name", "-lm", "mlock"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);++ argv = {"binary_name", "-lm", "dio"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);+// multi-value args (CSV)argv = {"binary_name", "--lora", "file1.gguf,\"file2,2.gguf\",\"file3\"\"3\"\".gguf\",file4\".gguf"};assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));@@ -158,13 +173,32 @@ static void test(void) {assert(params.model.path == "blah.gguf");assert(params.cpuparams.n_threads == 1010);+ setenv("LLAMA_ARG_LOAD_MODE", "blah", true);+ argv = {"binary_name"};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));++ setenv("LLAMA_ARG_LOAD_MODE", "mmap", true);+ argv = {"binary_name"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP);++ setenv("LLAMA_ARG_LOAD_MODE", "mlock", true);+ argv = {"binary_name"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);++ setenv("LLAMA_ARG_LOAD_MODE", "dio", true);+ argv = {"binary_name"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);+printf("test-arg-parser: test negated environment variables\n\n");- setenv("LLAMA_ARG_MMAP", "0", true);+ setenv("LLAMA_ARG_LOAD_MODE", "none", true);setenv("LLAMA_ARG_NO_PERF", "1", true); // legacy formatargv = {"binary_name"};assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));- assert(params.use_mmap == false);+ assert(params.load_mode == LLAMA_LOAD_MODE_NONE);assert(params.no_perf == true);printf("test-arg-parser: test environment variables being overwritten\n\n");diff --git a/tests/test-model-load-cancel.cpp b/tests/test-model-load-cancel.cppindex 9095826fa9..ecc3022711 100644--- a/tests/test-model-load-cancel.cpp+++ b/tests/test-model-load-cancel.cpp@@ -16,7 +16,7 @@ int main(int argc, char *argv[] ) {llama_backend_init();auto params = llama_model_params{};- params.use_mmap = false;+ params.load_mode = LLAMA_LOAD_MODE_NONE;params.progress_callback = [](float progress, void * ctx){(void) ctx;return progress > 0.50;diff --git a/tests/test-quantize-stats.cpp b/tests/test-quantize-stats.cppindex e53a7b3553..c655575340 100644--- a/tests/test-quantize-stats.cpp+++ b/tests/test-quantize-stats.cpp@@ -312,7 +312,7 @@ int main(int argc, char ** argv) {{auto mparams = llama_model_default_params();- mparams.use_mlock = false;+ mparams.load_mode = LLAMA_LOAD_MODE_NONE;model = llama_model_load_from_file(params.model.c_str(), mparams);diff --git a/tools/cli/README.md b/tools/cli/README.mdindex 89be3a5b36..6ee447b073 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -55,9 +55,10 @@| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) || `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |-| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |-| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |-| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |+| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |+| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |+| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |diff --git a/tools/completion/README.md b/tools/completion/README.mdindex 9873dd24e3..17f7cd7659 100644--- a/tools/completion/README.md+++ b/tools/completion/README.md@@ -138,9 +138,10 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) || `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |-| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |-| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |-| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |+| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |+| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |+| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cppindex 0756893881..29ad352d0c 100644--- a/tools/llama-bench/llama-bench.cpp+++ b/tools/llama-bench/llama-bench.cpp@@ -26,6 +26,7 @@#include "fit.h"#include "ggml.h"#include "llama.h"+#include "log.h"#ifdef _WIN32# define WIN32_LEAN_AND_MEAN@@ -339,14 +340,13 @@ struct cmd_params {std::vector<int> n_gpu_layers;std::vector<int> n_cpu_moe;std::vector<llama_split_mode> split_mode;+ std::vector<llama_load_mode> load_mode;std::vector<int> main_gpu;std::vector<bool> no_kv_offload;std::vector<llama_flash_attn_type> flash_attn;std::vector<std::vector<ggml_backend_dev_t>> devices;std::vector<std::vector<float>> tensor_split;std::vector<std::vector<llama_model_tensor_buft_override>> tensor_buft_overrides;- std::vector<bool> use_mmap;- std::vector<bool> use_direct_io;std::vector<bool> embeddings;std::vector<bool> no_op_offload;std::vector<bool> no_host;@@ -384,14 +384,13 @@ static const cmd_params cmd_params_defaults = {/* n_gpu_layers */ { -1 },/* n_cpu_moe */ { 0 },/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },+ /* load_mode */ { LLAMA_LOAD_MODE_MMAP },/* main_gpu */ { 0 },/* no_kv_offload */ { false },/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },/* devices */ { {} },/* tensor_split */ { std::vector<float>(llama_max_devices(), 0.0f) },/* tensor_buft_overrides*/ { std::vector<llama_model_tensor_buft_override>{ { nullptr, nullptr } } },- /* use_mmap */ { true },- /* use_direct_io */ { false },/* embeddings */ { false },/* no_op_offload */ { false },/* no_host */ { false },@@ -460,8 +459,9 @@ static void print_usage(int /* argc */, char ** argv) {printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");- printf(" -mmp, --mmap <0|1> (default: %s)\n", join(cmd_params_defaults.use_mmap, ",").c_str());- printf(" -dio, --direct-io <0|1> (default: %s)\n", join(cmd_params_defaults.use_direct_io, ",").c_str());+ printf(" -lm, --load-mode <none|mmap|mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());+ printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");+ printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n");printf(" -ot --override-tensor <tensor name pattern>=<buffer type>;...\n");@@ -769,6 +769,34 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {break;}params.split_mode.insert(params.split_mode.end(), modes.begin(), modes.end());+ } else if (arg == "-lm" || arg == "--load-mode") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ auto p = string_split<std::string>(argv[i], split_delim);++ std::vector<llama_load_mode> modes;+ for (const auto & m : p) {+ llama_load_mode mode;+ if (m == "none") {+ mode = LLAMA_LOAD_MODE_NONE;+ } else if (m == "mmap") {+ mode = LLAMA_LOAD_MODE_MMAP;+ } else if (m == "mlock") {+ mode = LLAMA_LOAD_MODE_MLOCK;+ } else if (m == "dio") {+ mode = LLAMA_LOAD_MODE_DIRECT_IO;+ } else {+ invalid_param = true;+ break;+ }+ modes.push_back(mode);+ }+ if (invalid_param) {+ break;+ }+ params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());} else if (arg == "-mg" || arg == "--main-gpu") {if (++i >= argc) {invalid_param = true;@@ -829,15 +857,39 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {invalid_param = true;break;}+ LOG_WRN("DEPRECATED: -mmp and --mmap are deprecated in favour of --load-mode. Please use --load-mode mmap instead.");auto p = string_split<bool>(argv[i], split_delim);- params.use_mmap.insert(params.use_mmap.end(), p.begin(), p.end());++ std::vector<llama_load_mode> modes;+ for (const auto & m : p) {+ llama_load_mode mode;+ if (m) {+ mode = LLAMA_LOAD_MODE_MMAP;+ } else {+ mode = LLAMA_LOAD_MODE_NONE;+ }+ modes.push_back(mode);+ }+ params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());} else if (arg == "-dio" || arg == "--direct-io") {if (++i >= argc) {invalid_param = true;break;}+ LOG_WRN("DEPRECATED: -dio and --direct-io are deprecated in favour of --load-mode. Please use --load-mode dio instead.");auto p = string_split<bool>(argv[i], split_delim);- params.use_direct_io.insert(params.use_direct_io.end(), p.begin(), p.end());++ std::vector<llama_load_mode> modes;+ for (const auto & m : p) {+ llama_load_mode mode;+ if (m) {+ mode = LLAMA_LOAD_MODE_DIRECT_IO;+ } else {+ mode = LLAMA_LOAD_MODE_NONE;+ }+ modes.push_back(mode);+ }+ params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());} else if (arg == "-embd" || arg == "--embeddings") {if (++i >= argc) {invalid_param = true;@@ -1093,6 +1145,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {if (params.split_mode.empty()) {params.split_mode = cmd_params_defaults.split_mode;}+ if (params.load_mode.empty()) {+ params.load_mode = cmd_params_defaults.load_mode;+ }if (params.main_gpu.empty()) {params.main_gpu = cmd_params_defaults.main_gpu;}@@ -1111,12 +1166,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {if (params.tensor_buft_overrides.empty()) {params.tensor_buft_overrides = cmd_params_defaults.tensor_buft_overrides;}- if (params.use_mmap.empty()) {- params.use_mmap = cmd_params_defaults.use_mmap;- }- if (params.use_direct_io.empty()) {- params.use_direct_io = cmd_params_defaults.use_direct_io;- }if (params.embeddings.empty()) {params.embeddings = cmd_params_defaults.embeddings;}@@ -1164,14 +1213,13 @@ struct cmd_params_instance {int n_gpu_layers;int n_cpu_moe;llama_split_mode split_mode;+ llama_load_mode load_mode;int main_gpu;bool no_kv_offload;llama_flash_attn_type flash_attn;std::vector<ggml_backend_dev_t> devices;std::vector<float> tensor_split;std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;- bool use_mmap;- bool use_direct_io;bool embeddings;bool no_op_offload;bool no_host;@@ -1186,10 +1234,9 @@ struct cmd_params_instance {mparams.devices = const_cast<ggml_backend_dev_t *>(devices.data());}mparams.split_mode = split_mode;+ mparams.load_mode = load_mode;mparams.main_gpu = main_gpu;mparams.tensor_split = tensor_split.data();- mparams.use_mmap = use_mmap;- mparams.use_direct_io = use_direct_io;mparams.no_host = no_host;if (n_cpu_moe <= 0) {@@ -1235,9 +1282,7 @@ struct cmd_params_instance {return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&split_mode == other.split_mode &&main_gpu == other.main_gpu && tensor_split == other.tensor_split &&- use_mmap == other.use_mmap && use_direct_io == other.use_direct_io &&- devices == other.devices &&- no_host == other.no_host &&+ load_mode == other.load_mode && devices == other.devices && no_host == other.no_host &&vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);}@@ -1270,12 +1315,11 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_paramfor (const auto & nl : params.n_gpu_layers)for (const auto & ncmoe : params.n_cpu_moe)for (const auto & sm : params.split_mode)+ for (const auto & lm : params.load_mode)for (const auto & mg : params.main_gpu)for (const auto & devs : params.devices)for (const auto & ts : params.tensor_split)for (const auto & ot : params.tensor_buft_overrides)- for (const auto & mmp : params.use_mmap)- for (const auto & dio : params.use_direct_io)for (const auto & noh : params.no_host)for (const auto & embd : params.embeddings)for (const auto & nopo : params.no_op_offload)@@ -1295,34 +1339,33 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_paramcontinue;}cmd_params_instance instance = {- /* .model = */ m,- /* .n_prompt = */ n_prompt,- /* .n_gen = */ 0,- /* .n_depth = */ nd,- /* .n_batch = */ nb,- /* .n_ubatch = */ nub,- /* .type_k = */ tk,- /* .type_v = */ tv,- /* .n_threads = */ nt,- /* .cpu_mask = */ cm,- /* .cpu_strict = */ cs,- /* .poll = */ pl,- /* .n_gpu_layers = */ nl,- /* .n_cpu_moe = */ ncmoe,- /* .split_mode = */ sm,- /* .main_gpu = */ mg,- /* .no_kv_offload= */ nkvo,- /* .flash_attn = */ fa,- /* .devices = */ devs,- /* .tensor_split = */ ts,+ /* .model = */ m,+ /* .n_prompt = */ n_prompt,+ /* .n_gen = */ 0,+ /* .n_depth = */ nd,+ /* .n_batch = */ nb,+ /* .n_ubatch = */ nub,+ /* .type_k = */ tk,+ /* .type_v = */ tv,+ /* .n_threads = */ nt,+ /* .cpu_mask = */ cm,+ /* .cpu_strict = */ cs,+ /* .poll = */ pl,+ /* .n_gpu_layers = */ nl,+ /* .n_cpu_moe = */ ncmoe,+ /* .split_mode = */ sm,+ /* .load_mode = */ lm,+ /* .main_gpu = */ mg,+ /* .no_kv_offload = */ nkvo,+ /* .flash_attn = */ fa,+ /* .devices = */ devs,+ /* .tensor_split = */ ts,/* .tensor_buft_overrides = */ ot,- /* .use_mmap = */ mmp,- /* .use_direct_io= */ dio,- /* .embeddings = */ embd,- /* .no_op_offload= */ nopo,- /* .no_host = */ noh,- /* .fit_target = */ fpt,- /* .fit_min_ctx = */ fpc,+ /* .embeddings = */ embd,+ /* .no_op_offload = */ nopo,+ /* .no_host = */ noh,+ /* .fit_target = */ fpt,+ /* .fit_min_ctx = */ fpc,};instances.push_back(instance);}@@ -1332,34 +1375,33 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_paramcontinue;}cmd_params_instance instance = {- /* .model = */ m,- /* .n_prompt = */ 0,- /* .n_gen = */ n_gen,- /* .n_depth = */ nd,- /* .n_batch = */ nb,- /* .n_ubatch = */ nub,- /* .type_k = */ tk,- /* .type_v = */ tv,- /* .n_threads = */ nt,- /* .cpu_mask = */ cm,- /* .cpu_strict = */ cs,- /* .poll = */ pl,- /* .n_gpu_layers = */ nl,- /* .n_cpu_moe = */ ncmoe,- /* .split_mode = */ sm,- /* .main_gpu = */ mg,- /* .no_kv_offload= */ nkvo,- /* .flash_attn = */ fa,- /* .devices = */ devs,- /* .tensor_split = */ ts,+ /* .model = */ m,+ /* .n_prompt = */ 0,+ /* .n_gen = */ n_gen,+ /* .n_depth = */ nd,+ /* .n_batch = */ nb,+ /* .n_ubatch = */ nub,+ /* .type_k = */ tk,+ /* .type_v = */ tv,+ /* .n_threads = */ nt,+ /* .cpu_mask = */ cm,+ /* .cpu_strict = */ cs,+ /* .poll = */ pl,+ /* .n_gpu_layers = */ nl,+ /* .n_cpu_moe = */ ncmoe,+ /* .split_mode = */ sm,+ /* .load_mode = */ lm,+ /* .main_gpu = */ mg,+ /* .no_kv_offload = */ nkvo,+ /* .flash_attn = */ fa,+ /* .devices = */ devs,+ /* .tensor_split = */ ts,/* .tensor_buft_overrides = */ ot,- /* .use_mmap = */ mmp,- /* .use_direct_io= */ dio,- /* .embeddings = */ embd,- /* .no_op_offload= */ nopo,- /* .no_host = */ noh,- /* .fit_target = */ fpt,- /* .fit_min_ctx = */ fpc,+ /* .embeddings = */ embd,+ /* .no_op_offload = */ nopo,+ /* .no_host = */ noh,+ /* .fit_target = */ fpt,+ /* .fit_min_ctx = */ fpc,};instances.push_back(instance);}@@ -1369,34 +1411,33 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_paramcontinue;}cmd_params_instance instance = {- /* .model = */ m,- /* .n_prompt = */ n_pg.first,- /* .n_gen = */ n_pg.second,- /* .n_depth = */ nd,- /* .n_batch = */ nb,- /* .n_ubatch = */ nub,- /* .type_k = */ tk,- /* .type_v = */ tv,- /* .n_threads = */ nt,- /* .cpu_mask = */ cm,- /* .cpu_strict = */ cs,- /* .poll = */ pl,- /* .n_gpu_layers = */ nl,- /* .n_cpu_moe = */ ncmoe,- /* .split_mode = */ sm,- /* .main_gpu = */ mg,- /* .no_kv_offload= */ nkvo,- /* .flash_attn = */ fa,- /* .devices = */ devs,- /* .tensor_split = */ ts,+ /* .model = */ m,+ /* .n_prompt = */ n_pg.first,+ /* .n_gen = */ n_pg.second,+ /* .n_depth = */ nd,+ /* .n_batch = */ nb,+ /* .n_ubatch = */ nub,+ /* .type_k = */ tk,+ /* .type_v = */ tv,+ /* .n_threads = */ nt,+ /* .cpu_mask = */ cm,+ /* .cpu_strict = */ cs,+ /* .poll = */ pl,+ /* .n_gpu_layers = */ nl,+ /* .n_cpu_moe = */ ncmoe,+ /* .split_mode = */ sm,+ /* .load_mode = */ lm,+ /* .main_gpu = */ mg,+ /* .no_kv_offload = */ nkvo,+ /* .flash_attn = */ fa,+ /* .devices = */ devs,+ /* .tensor_split = */ ts,/* .tensor_buft_overrides = */ ot,- /* .use_mmap = */ mmp,- /* .use_direct_io= */ dio,- /* .embeddings = */ embd,- /* .no_op_offload= */ nopo,- /* .no_host = */ noh,- /* .fit_target = */ fpt,- /* .fit_min_ctx = */ fpc,+ /* .embeddings = */ embd,+ /* .no_op_offload = */ nopo,+ /* .no_host = */ noh,+ /* .fit_target = */ fpt,+ /* .fit_min_ctx = */ fpc,};instances.push_back(instance);}@@ -1426,14 +1467,13 @@ struct test {int n_gpu_layers;int n_cpu_moe;llama_split_mode split_mode;+ llama_load_mode load_mode;int main_gpu;bool no_kv_offload;llama_flash_attn_type flash_attn;std::vector<ggml_backend_dev_t> devices;std::vector<float> tensor_split;std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;- bool use_mmap;- bool use_direct_io;bool embeddings;bool no_op_offload;bool no_host;@@ -1466,14 +1506,13 @@ struct test {n_gpu_layers = inst.n_gpu_layers;n_cpu_moe = inst.n_cpu_moe;split_mode = inst.split_mode;+ load_mode = inst.load_mode;main_gpu = inst.main_gpu;no_kv_offload = inst.no_kv_offload;flash_attn = inst.flash_attn;devices = inst.devices;tensor_split = inst.tensor_split;tensor_buft_overrides = inst.tensor_buft_overrides;- use_mmap = inst.use_mmap;- use_direct_io = inst.use_direct_io;embeddings = inst.embeddings;no_op_offload = inst.no_op_offload;no_host = inst.no_host;@@ -1535,8 +1574,8 @@ struct test {"n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll","type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode","main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",- "tensor_buft_overrides", "use_mmap", "use_direct_io", "embeddings",- "no_op_offload", "no_host", "fit_target", "fit_min_ctx",+ "tensor_buft_overrides", "load_mode", "embeddings",+ "no_op_offload", "no_host", "fit_target", "fit_min_ctx","n_prompt", "n_gen", "n_depth","test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts"};@@ -1554,12 +1593,15 @@ struct test {return INT;}if (field == "f16_kv" || field == "no_kv_offload" || field == "cpu_strict" ||- field == "use_mmap" || field == "use_direct_io" || field == "embeddings" || field == "no_host") {+ field == "embeddings" || field == "no_host") {return BOOL;}if (field == "avg_ts" || field == "stddev_ts") {return FLOAT;}+ if (field == "load_mode") {+ return STRING;+ }return STRING;}@@ -1626,8 +1668,7 @@ struct test {devices_to_string(devices),tensor_split_str,tensor_buft_overrides_str,- std::to_string(use_mmap),- std::to_string(use_direct_io),+ llama_load_mode_name(load_mode),std::to_string(embeddings),std::to_string(no_op_offload),std::to_string(no_host),@@ -1806,18 +1847,15 @@ struct markdown_printer : public printer {if (field == "split_mode") {return 6;}+ if (field == "load_mode") {+ return 10;+ }if (field == "flash_attn") {return 3;}if (field == "devices") {return -12;}- if (field == "use_mmap") {- return 4;- }- if (field == "use_direct_io") {- return 3;- }if (field == "test") {return 15;}@@ -1852,11 +1890,8 @@ struct markdown_printer : public printer {if (field == "flash_attn") {return "fa";}- if (field == "use_mmap") {- return "mmap";- }- if (field == "use_direct_io") {- return "dio";+ if (field == "load_mode") {+ return "lm";}if (field == "embeddings") {return "embd";@@ -1945,11 +1980,8 @@ struct markdown_printer : public printer {if (params.tensor_buft_overrides.size() > 1 || !vec_vec_tensor_buft_override_equal(params.tensor_buft_overrides, cmd_params_defaults.tensor_buft_overrides)) {fields.emplace_back("tensor_buft_overrides");}- if (params.use_mmap.size() > 1 || params.use_mmap != cmd_params_defaults.use_mmap) {- fields.emplace_back("use_mmap");- }- if (params.use_direct_io.size() > 1 || params.use_direct_io != cmd_params_defaults.use_direct_io) {- fields.emplace_back("use_direct_io");+ if (params.load_mode.size() > 1 || params.load_mode != cmd_params_defaults.load_mode) {+ fields.emplace_back("load_mode");}if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {fields.emplace_back("embeddings");diff --git a/tools/server/README.md b/tools/server/README.mdindex 1941dcea20..e6b27a5576 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -72,9 +72,10 @@ For the full list of features, please refer to [server's changelog](https://gith| `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) || `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |-| `--mlock` | force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |-| `--mmap, --no-mmap` | whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock) (default: enabled)<br/>(env: LLAMA_ARG_MMAP) |-| `-dio, --direct-io, -ndio, --no-direct-io` | use DirectIO if available. (default: disabled)<br/>(env: LLAMA_ARG_DIO) |+| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |+| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) |+| `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |