LLM inference in C/C++
C++ ggml
args: add `-lm mlock` where it mlocks but doesnt mmap (#26135)
Aaron Teoad256ded30b5
diff --git a/common/arg.cpp b/common/arg.cppindex 9753441313..84c19e0632 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2508,7 +2508,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex}add_opt(common_arg({"--mlock"},- "DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing",+ "DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing",[](common_params & params) {LOG_WRN("DEPRECATED: --mlock is deprecated. use --load-mode mlock instead\n");params.load_mode = LLAMA_LOAD_MODE_MLOCK;@@ -2537,13 +2537,15 @@ common_params_context common_params_parser_init(common_params & params, llama_ex"model loading mode (default: mmap)\n""- none: no special loading mode\n""- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n"- "- mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n"+ "- mlock: force system to keep model in RAM rather than swapping or compressing\n"+ "- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n""- dio: use DirectIO if available\n",[](common_params & params, const std::string & value) {- /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }- else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }- else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }- else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }+ /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; }+ else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; }+ else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; }+ else if (value == "mmap+mlock") { params.load_mode = LLAMA_LOAD_MODE_MMAP_MLOCK; }+ else if (value == "dio") { params.load_mode = LLAMA_LOAD_MODE_DIRECT_IO; }else { throw std::invalid_argument("invalid value"); }}).set_env("LLAMA_ARG_LOAD_MODE"));diff --git a/include/llama.h b/include/llama.hindex 9fab693170..3c6d22be89 100644--- a/include/llama.h+++ b/include/llama.h@@ -203,10 +203,11 @@ extern "C" {};enum llama_load_mode {- LLAMA_LOAD_MODE_NONE = 0, // no special loading mode- LLAMA_LOAD_MODE_MMAP = 1, // memory map the model- LLAMA_LOAD_MODE_MLOCK = 2, // mmap + force system to keep model in RAM rather than swapping or compressing- LLAMA_LOAD_MODE_DIRECT_IO = 3, // use direct I/O if available+ LLAMA_LOAD_MODE_NONE = 0, // no special loading mode+ LLAMA_LOAD_MODE_MMAP = 1, // memory map the model+ LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing+ LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing+ LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available};LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 43447f57d3..510586e96c 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -542,7 +542,7 @@ llama_model_loader::llama_model_loader(tensor_buft_overrides = param_tensor_buft_overrides_p;- this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MLOCK;+ this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK;this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO;if (!fname.empty()) {diff --git a/src/llama-model.cpp b/src/llama-model.cppindex 5179692108..074acbe1fa 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1249,7 +1249,7 @@ void llama_model_base::load_vocab(llama_model_loader & ml) {bool llama_model_base::load_tensors(llama_model_loader & ml) {const auto & split_mode = params.split_mode;- const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK;+ const bool use_mlock = params.load_mode == LLAMA_LOAD_MODE_MLOCK || params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK;const auto & tensor_split = params.tensor_split;const int n_layer_all = hparams.n_layer_all;diff --git a/src/llama.cpp b/src/llama.cppindex 11ac9656d9..d22e4c81a9 100644--- a/src/llama.cpp+++ b/src/llama.cpp@@ -54,6 +54,8 @@ const char * llama_load_mode_name(enum llama_load_mode load_mode) {return "mmap";case LLAMA_LOAD_MODE_MLOCK:return "mlock";+ case LLAMA_LOAD_MODE_MMAP_MLOCK:+ return "mmap+mlock";case LLAMA_LOAD_MODE_DIRECT_IO:return "dio";}@@ -61,10 +63,11 @@ const char * llama_load_mode_name(enum llama_load_mode load_mode) {}enum llama_load_mode llama_load_mode_from_str(const char * str) {- if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }- if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }- if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }- if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }+ if (std::strcmp(str, "none") == 0) { return LLAMA_LOAD_MODE_NONE; }+ if (std::strcmp(str, "mmap") == 0) { return LLAMA_LOAD_MODE_MMAP; }+ if (std::strcmp(str, "mlock") == 0) { return LLAMA_LOAD_MODE_MLOCK; }+ if (std::strcmp(str, "mmap+mlock") == 0) { return LLAMA_LOAD_MODE_MMAP_MLOCK; }+ if (std::strcmp(str, "dio") == 0) { return LLAMA_LOAD_MODE_DIRECT_IO; }throw std::invalid_argument(std::string("unknown load mode: ") + str);}diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cppindex 000ecd9aaa..1d3584f903 100644--- a/tests/test-arg-parser.cpp+++ b/tests/test-arg-parser.cpp@@ -143,6 +143,10 @@ static void test(void) {assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);+ argv = {"binary_name", "-lm", "mmap+mlock"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);+argv = {"binary_name", "-lm", "dio"};assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));assert(params.load_mode == LLAMA_LOAD_MODE_DIRECT_IO);@@ -187,6 +191,11 @@ static void test(void) {assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));assert(params.load_mode == LLAMA_LOAD_MODE_MLOCK);+ setenv("LLAMA_ARG_LOAD_MODE", "mmap+mlock", true);+ argv = {"binary_name"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ assert(params.load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK);+setenv("LLAMA_ARG_LOAD_MODE", "dio", true);argv = {"binary_name"};assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));diff --git a/tools/cli/README.md b/tools/cli/README.mdindex 6ee447b073..972ea04dc7 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -55,10 +55,10 @@| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) || `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |-| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |+| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |-| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |diff --git a/tools/completion/README.md b/tools/completion/README.mdindex 17f7cd7659..bce71d68d9 100644--- a/tools/completion/README.md+++ b/tools/completion/README.md@@ -138,10 +138,10 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1| `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `-np, --parallel N` | number of parallel sequences to decode (default: 1)<br/>(env: LLAMA_ARG_N_PARALLEL) || `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |-| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |+| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |-| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cppindex 29ad352d0c..dc1c7caf4a 100644--- a/tools/llama-bench/llama-bench.cpp+++ b/tools/llama-bench/llama-bench.cpp@@ -429,45 +429,45 @@ static void print_usage(int /* argc */, char ** argv) {}printf("\n");printf("test parameters:\n");- printf(" -m, --model <filename> (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());- printf(" -hf, -hfr, --hf-repo <user>/<model>[:quant] Hugging Face model repository; quant is optional, case-insensitive\n");- printf(" default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n");- printf(" example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M\n");- printf(" (default: unused)\n");- printf(" -hff, --hf-file <file> Hugging Face model file. If specified, it will override the quant in --hf-repo\n");- printf(" (default: unused)\n");- printf(" -hft, --hf-token <token> Hugging Face access token\n");- printf(" (default: value from HF_TOKEN environment variable)\n");- printf(" --offline Offline mode: forces use of cache, prevents network access\n");- printf(" (default: disabled)\n");- printf(" -p, --n-prompt <n> (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str());- printf(" -n, --n-gen <n> (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());- printf(" -pg <pp,tg> (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());- printf(" -d, --n-depth <n> (default: %s)\n", join(cmd_params_defaults.n_depth, ",").c_str());- printf(" -b, --batch-size <n> (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str());- printf(" -ub, --ubatch-size <n> (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str());- printf(" -ctk, --cache-type-k <t> (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());- printf(" -ctv, --cache-type-v <t> (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());- printf(" -t, --threads <n> (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str());- printf(" -C, --cpu-mask <hex,hex> (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str());- printf(" --cpu-strict <0|1> (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str());- printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());- printf(" -ngl, --n-gpu-layers <n> (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str());- printf(" -ncmoe, --n-cpu-moe <n> (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str());- printf(" -sm, --split-mode <none|layer|row|tensor> (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());- printf(" -mg, --main-gpu <i> (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str());- printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());- printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());- printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");- printf(" -lm, --load-mode <none|mmap|mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());- printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");- printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");- printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());- printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n");+ printf(" -m, --model <filename> (default: %s)\n", join(cmd_params_defaults.model, ",").c_str());+ printf(" -hf, -hfr, --hf-repo <user>/<model>[:quant] Hugging Face model repository; quant is optional, case-insensitive\n");+ printf(" default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n");+ printf(" example: ggml-org/GLM-4.7-Flash-GGUF:Q4_K_M\n");+ printf(" (default: unused)\n");+ printf(" -hff, --hf-file <file> Hugging Face model file. If specified, it will override the quant in --hf-repo\n");+ printf(" (default: unused)\n");+ printf(" -hft, --hf-token <token> Hugging Face access token\n");+ printf(" (default: value from HF_TOKEN environment variable)\n");+ printf(" --offline Offline mode: forces use of cache, prevents network access\n");+ printf(" (default: disabled)\n");+ printf(" -p, --n-prompt <n> (default: %s)\n", join(cmd_params_defaults.n_prompt, ",").c_str());+ printf(" -n, --n-gen <n> (default: %s)\n", join(cmd_params_defaults.n_gen, ",").c_str());+ printf(" -pg <pp,tg> (default: %s)\n", join(transform_to_str(cmd_params_defaults.n_pg, pair_str), ",").c_str());+ printf(" -d, --n-depth <n> (default: %s)\n", join(cmd_params_defaults.n_depth, ",").c_str());+ printf(" -b, --batch-size <n> (default: %s)\n", join(cmd_params_defaults.n_batch, ",").c_str());+ printf(" -ub, --ubatch-size <n> (default: %s)\n", join(cmd_params_defaults.n_ubatch, ",").c_str());+ printf(" -ctk, --cache-type-k <t> (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_k, ggml_type_name), ",").c_str());+ printf(" -ctv, --cache-type-v <t> (default: %s)\n", join(transform_to_str(cmd_params_defaults.type_v, ggml_type_name), ",").c_str());+ printf(" -t, --threads <n> (default: %s)\n", join(cmd_params_defaults.n_threads, ",").c_str());+ printf(" -C, --cpu-mask <hex,hex> (default: %s)\n", join(cmd_params_defaults.cpu_mask, ",").c_str());+ printf(" --cpu-strict <0|1> (default: %s)\n", join(cmd_params_defaults.cpu_strict, ",").c_str());+ printf(" --poll <0...100> (default: %s)\n", join(cmd_params_defaults.poll, ",").c_str());+ printf(" -ngl, --n-gpu-layers <n> (default: %s)\n", join(cmd_params_defaults.n_gpu_layers, ",").c_str());+ printf(" -ncmoe, --n-cpu-moe <n> (default: %s)\n", join(cmd_params_defaults.n_cpu_moe, ",").c_str());+ printf(" -sm, --split-mode <none|layer|row|tensor> (default: %s)\n", join(transform_to_str(cmd_params_defaults.split_mode, split_mode_str), ",").c_str());+ printf(" -mg, --main-gpu <i> (default: %s)\n", join(cmd_params_defaults.main_gpu, ",").c_str());+ printf(" -nkvo, --no-kv-offload <0|1> (default: %s)\n", join(cmd_params_defaults.no_kv_offload, ",").c_str());+ printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());+ printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");+ printf(" -lm, --load-mode <none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());+ printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");+ printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");+ printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());+ printf(" -ts, --tensor-split <ts0/ts1/..> (default: 0)\n");printf(" -ot --override-tensor <tensor name pattern>=<buffer type>;...\n");- printf(" (default: disabled)\n");- printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");- printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str());+ printf(" (default: disabled)\n");+ printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");+ printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str());printf("\n");printf("Multiple values can be given for each parameter by separating them with ','\n"@@ -785,6 +785,8 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {mode = LLAMA_LOAD_MODE_MMAP;} else if (m == "mlock") {mode = LLAMA_LOAD_MODE_MLOCK;+ } else if (m == "mmap+mlock") {+ mode = LLAMA_LOAD_MODE_MMAP_MLOCK;} else if (m == "dio") {mode = LLAMA_LOAD_MODE_DIRECT_IO;} else {diff --git a/tools/server/README.md b/tools/server/README.mdindex d345654554..25aacf9f51 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -72,10 +72,10 @@ For the full list of features, please refer to [server's changelog](https://gith| `-ctv, --cache-type-v TYPE` | KV cache data type for V<br/>allowed values: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1<br/>(default: f16)<br/>(env: LLAMA_ARG_CACHE_TYPE_V) || `-dt, --defrag-thold N` | KV cache defragmentation threshold (DEPRECATED)<br/>(env: LLAMA_ARG_DEFRAG_THOLD) || `--rpc SERVERS` | comma-separated list of RPC servers (host:port)<br/>(env: LLAMA_ARG_RPC) |-| `--mlock` | DEPRECATED in favor of `--load-mode`: mmap + force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) |+| `--mlock` | DEPRECATED in favor of `--load-mode`: force system to keep model in RAM rather than swapping or compressing<br/>(env: LLAMA_ARG_MLOCK) || `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) |-| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `-lm, --load-mode MODE` | model loading mode (default: mmap)<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit |