LLM inference in C/C++
C++ ggml
llama: model_loader: add TENSOR_READ_LAZY (#27794)
Xuan-Son Nguyenfac889fb38fd
diff --git a/common/arg.cpp b/common/arg.cppindex 403cc2b781..f1e2bf6908 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2720,6 +2720,19 @@ common_params_context common_params_parser_init(common_params & params, llama_exelse { throw std::invalid_argument("invalid value"); }}).set_env("LLAMA_ARG_LOAD_MODE"));+ add_opt(common_arg(+ {"--tensor-read-lazy"}, "MODE",+ "on-demand reading of certain tensors, for example per-layer embeddings (default: auto)\n"+ "- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)\n"+ "- auto: on, but only for tensors larger than 4 GiB\n"+ "- off: always keep them resident",+ [](common_params & params, const std::string & value) {+ /**/ if (value == "on") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON; }+ else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }+ else if (value == "off") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF; }+ else { throw std::invalid_argument("invalid value"); }+ }+ ).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));add_opt(common_arg({"--numa"}, "TYPE","attempt optimizations that help on some NUMA systems\n"diff --git a/common/common.cpp b/common/common.cppindex 3d54bd6002..347e8e9fc4 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1688,6 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {mparams.main_gpu = params.main_gpu;mparams.split_mode = params.split_mode;mparams.load_mode = params.load_mode;+ mparams.tensor_read_lazy = params.tensor_read_lazy;mparams.tensor_split = params.tensor_split;mparams.check_tensors = params.check_tensors;mparams.use_extra_bufts = !params.no_extra_bufts;diff --git a/common/common.h b/common/common.hindex 1cfb01e1ef..82fed22092 100644--- a/common/common.h+++ b/common/common.h@@ -483,6 +483,8 @@ struct common_params {enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUsenum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model+ enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch+common_cpu_params cpuparams;common_cpu_params cpuparams_batch;diff --git a/include/llama.h b/include/llama.hindex 84313cfd74..b8020a7fce 100644--- a/include/llama.h+++ b/include/llama.h@@ -214,6 +214,12 @@ extern "C" {LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);+ enum llama_tensor_read_lazy {+ LLAMA_TENSOR_READ_LAZY_OFF = 0, // always read the whole tensor up front+ LLAMA_TENSOR_READ_LAZY_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)+ LLAMA_TENSOR_READ_LAZY_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)+ };+enum llama_context_type {LLAMA_CONTEXT_TYPE_DEFAULT = 0,LLAMA_CONTEXT_TYPE_MTP = 1,@@ -315,6 +321,8 @@ extern "C" {enum llama_split_mode split_mode; // how to split the model across multiple GPUsenum llama_load_mode load_mode; // how to load the model+ enum llama_tensor_read_lazy tensor_read_lazy; // on-demand reading of tensors marked by the arch+// the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONEint32_t main_gpu;diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cppindex ed572da7fb..4d183cbc9c 100644--- a/src/llama-mmap.cpp+++ b/src/llama-mmap.cpp@@ -438,11 +438,34 @@ void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }// llama_mmap+#if defined(_POSIX_MAPPED_FILES) || defined(_WIN32)+// merge `ranges` and return their complement within [0, limit)+static llama_mmap::ranges ranges_complement(llama_mmap::ranges ranges, size_t limit) {+ llama_mmap::ranges res;+ std::sort(ranges.begin(), ranges.end());++ size_t pos = 0;+ for (const auto & range : ranges) {+ const size_t beg = std::min(range.first, limit);+ const size_t end = std::min(range.second, limit);+ if (beg > pos) {+ res.emplace_back(pos, beg);+ }+ pos = std::max(pos, end);+ }+ if (pos < limit) {+ res.emplace_back(pos, limit);+ }++ return res;+}+#endif+struct llama_mmap::impl {#ifdef _POSIX_MAPPED_FILESstd::vector<std::pair<size_t, size_t>> mapped_fragments;- impl(struct llama_file * file, size_t prefetch, bool numa) {+ impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {size = file->size();int fd = file->file_id();int flags = MAP_SHARED;@@ -452,19 +475,35 @@ struct llama_mmap::impl {LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",strerror(errno));}- if (prefetch) { flags |= MAP_POPULATE; }+ // MAP_POPULATE would fault in the lazy ranges too+ if (prefetch && lazy_ranges.empty()) { flags |= MAP_POPULATE; }#endifaddr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);if (addr == MAP_FAILED) {throw std::runtime_error(format("mmap failed: %s", strerror(errno)));}+ // page-aligned madvise over [beg, end), clamped to the file+ auto advise = [&](size_t beg, size_t end, int advice, const char * name) {+ const size_t page_size = sysconf(_SC_PAGESIZE);+ beg = beg & ~(page_size - 1);+ end = std::min((end + page_size - 1) & ~(page_size - 1), file->size());+ if (beg >= end) {+ return;+ }+ if (posix_madvise((char *) addr + beg, end - beg, advice)) {+ LLAMA_LOG_WARN("warning: posix_madvise(.., %s) failed: %s\n", name, strerror(errno));+ }+ };+if (prefetch > 0) {- if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {- LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",- strerror(errno));+ for (const auto & range : ranges_complement(lazy_ranges, std::min(file->size(), prefetch))) {+ advise(range.first, range.second, POSIX_MADV_WILLNEED, "POSIX_MADV_WILLNEED");}}+ for (const auto & range : lazy_ranges) {+ advise(range.first, range.second, POSIX_MADV_RANDOM, "POSIX_MADV_RANDOM");+ }if (numa) {if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",@@ -533,7 +572,7 @@ struct llama_mmap::impl {#elif defined(_WIN32)HANDLE hMapping = nullptr;- impl(struct llama_file * file, size_t prefetch, bool numa) {+ impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {GGML_UNUSED(numa);size = file->size();@@ -563,10 +602,15 @@ struct llama_mmap::impl {pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");if (pPrefetchVirtualMemory) {- WIN32_MEMORY_RANGE_ENTRY range;- range.VirtualAddress = addr;- range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);- if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {+ std::vector<WIN32_MEMORY_RANGE_ENTRY> entries;+ for (const auto & range : ranges_complement(lazy_ranges, std::min(size, prefetch))) {+ WIN32_MEMORY_RANGE_ENTRY entry;+ entry.VirtualAddress = (char *) addr + range.first;+ entry.NumberOfBytes = (SIZE_T) (range.second - range.first);+ entries.push_back(entry);+ }+ if (!entries.empty() &&+ !pPrefetchVirtualMemory(GetCurrentProcess(), (ULONG_PTR) entries.size(), entries.data(), 0)) {LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",llama_format_win_err(GetLastError()).c_str());}@@ -597,10 +641,11 @@ struct llama_mmap::impl {}}#else- impl(struct llama_file * file, size_t prefetch, bool numa) {+ impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) {GGML_UNUSED(file);GGML_UNUSED(prefetch);GGML_UNUSED(numa);+ GGML_UNUSED(lazy_ranges);throw std::runtime_error("mmap not supported");}@@ -617,7 +662,8 @@ struct llama_mmap::impl {size_t size;};-llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}+llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa,+ const ranges & lazy_ranges) : pimpl(std::make_unique<impl>(file, prefetch, numa, lazy_ranges)) {}llama_mmap::~llama_mmap() = default;size_t llama_mmap::size() const { return pimpl->size; }diff --git a/src/llama-mmap.h b/src/llama-mmap.hindex b7d5c61e95..cc28c8a73f 100644--- a/src/llama-mmap.h+++ b/src/llama-mmap.h@@ -2,6 +2,7 @@#include <cstdint>#include <memory>+#include <utility>#include <vector>#include <cstdio>@@ -41,8 +42,12 @@ private:};struct llama_mmap {+ // list of [first, last) byte ranges within a file+ using ranges = std::vector<std::pair<size_t, size_t>>;+llama_mmap(const llama_mmap &) = delete;- llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false);+ llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false,+ const ranges & lazy_ranges = {});~llama_mmap();size_t size() const;diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex 9b22cb05f2..2f1a09a19a 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -1282,6 +1282,18 @@ struct ggml_tensor * llama_model_loader::create_tensor(return NULL;}+ if ((flags & TENSOR_READ_LAZY) && use_mmap && tensor_read_lazy != LLAMA_TENSOR_READ_LAZY_OFF) {+ // in auto mode, small tensors are cheap enough to keep resident+ constexpr size_t auto_lazy_min_size = 4ull * 1024 * 1024 * 1024;+ if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) {+ const auto & w = require_weight(tn.str().c_str());+ lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur));++ LLAMA_LOG_INFO("%s: tensor %s (size = %zu MiB) lazy read enabled\n",+ __func__, tn.str().c_str(), ggml_nbytes(cur)/1024/1024);+ }+ }+ggml_tensor t_meta = *cur;if (flags & TENSOR_ALLOW_RESHAPE) {for (size_t dim = 0; dim < GGML_MAX_DIMS; dim++) {@@ -1349,7 +1361,9 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmapsif (use_mmap) {mappings.reserve(files.size());mmaps_used.reserve(files.size());- for (const auto & file : files) {+ for (uint32_t idx = 0; idx < files.size(); idx++) {+ const auto & file = files[idx];+bool is_numa = false;auto * dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);@@ -1361,7 +1375,11 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps}}- std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa);+ const auto it_lazy = lazy_tensor_ranges.find(idx);+ static const llama_mmap::ranges no_lazy_ranges;++ std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa,+ it_lazy != lazy_tensor_ranges.end() ? it_lazy->second : no_lazy_ranges);mmaps_used.emplace_back(mapping->size(), 0);if (mlock_mmaps) {std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());diff --git a/src/llama-model-loader.h b/src/llama-model-loader.hindex e9fe3592d4..ec5b692463 100644--- a/src/llama-model-loader.h+++ b/src/llama-model-loader.h@@ -68,6 +68,7 @@ struct llama_model_loader {static const int TENSOR_SKIP = 1 << 2;static const int TENSOR_SKIP_IF_VIRTUAL = 1 << 3;static const int TENSOR_ALLOW_RESHAPE = 1 << 4;+ static const int TENSOR_READ_LAZY = 1 << 5; // read rows on demand instead of loading whole tensor; requires mmap for nowint n_kv = 0;int n_tensors = 0;@@ -82,12 +83,18 @@ struct llama_model_loader {bool no_alloc;bool load_mtp;+ // set by the caller before the create_tensor() calls+ enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;+llama_files files;llama_ftype ftype;llama_fver fver;llama_mmaps mappings;+ // byte ranges of TENSOR_READ_LAZY tensors, per file index+ std::map<uint32_t, llama_mmap::ranges> lazy_tensor_ranges;+std::map<std::string, llama_tensor_weight, weight_name_comparer> weights_map;std::unordered_map<std::string, llama_model_kv_override> kv_overrides;const llama_model_tensor_buft_override * tensor_buft_overrides;diff --git a/src/llama-model.cpp b/src/llama-model.cppindex c34700ff56..ad50093e41 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -2631,6 +2631,7 @@ llama_model_params llama_model_default_params() {/*.n_gpu_layers =*/ -1,/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,/*.load_mode =*/ LLAMA_LOAD_MODE_AUTO,+ /*.tensor_read_lazy =*/ LLAMA_TENSOR_READ_LAZY_AUTO,/*.main_gpu =*/ 0,/*.tensor_split =*/ nullptr,/*.progress_callback =*/ nullptr,@@ -3067,7 +3068,8 @@ llama_model_base::llama_model_base(const struct llama_model_params & params) : lTENSOR_NOT_REQUIRED (llama_model_loader::TENSOR_NOT_REQUIRED),TENSOR_SKIP (llama_model_loader::TENSOR_SKIP),TENSOR_SKIP_IF_VIRTUAL(llama_model_loader::TENSOR_SKIP_IF_VIRTUAL),- TENSOR_ALLOW_RESHAPE (llama_model_loader::TENSOR_ALLOW_RESHAPE) {}+ TENSOR_ALLOW_RESHAPE (llama_model_loader::TENSOR_ALLOW_RESHAPE),+ TENSOR_READ_LAZY (llama_model_loader::TENSOR_READ_LAZY) {}ggml_tensor * llama_model_base::create_tensor(const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {GGML_ASSERT(ml != nullptr);diff --git a/src/llama-model.h b/src/llama-model.hindex 44bd967575..25898ad4af 100644--- a/src/llama-model.h+++ b/src/llama-model.h@@ -756,6 +756,7 @@ struct llama_model_base : public llama_model {const int TENSOR_SKIP;const int TENSOR_SKIP_IF_VIRTUAL;const int TENSOR_ALLOW_RESHAPE;+ const int TENSOR_READ_LAZY;explicit llama_model_base(const llama_model_params & params);virtual ~llama_model_base() = default;diff --git a/src/llama.cpp b/src/llama.cppindex 1609fec88d..9c841ee352 100644--- a/src/llama.cpp+++ b/src/llama.cpp@@ -318,6 +318,8 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * metallama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);+ ml.tensor_read_lazy = params.tensor_read_lazy;+ml.print_info();std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));diff --git a/src/models/gemma4.cpp b/src/models/gemma4.cppindex e44f423bdb..aa518c6df5 100644--- a/src/models/gemma4.cpp+++ b/src/models/gemma4.cpp@@ -50,7 +50,7 @@ void llama_model_gemma4::load_arch_tensors(llama_model_loader &) {tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);if (n_embd_per_layer > 0) {- per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {n_embd_per_layer * n_layer, n_vocab}, 0);+ per_layer_tok_embd = create_tensor(tn(LLM_TENSOR_PER_LAYER_TOKEN_EMBD, "weight"), {n_embd_per_layer * n_layer, n_vocab}, TENSOR_READ_LAZY);per_layer_model_proj = create_tensor(tn(LLM_TENSOR_PER_LAYER_MODEL_PROJ, "weight", 0), {n_embd, n_embd_per_layer * n_layer}, 0);per_layer_proj_norm = create_tensor(tn(LLM_TENSOR_PER_LAYER_PROJ_NORM, "weight", 0), {n_embd_per_layer}, 0);}diff --git a/tools/cli/README.md b/tools/cli/README.mdindex e663cfa3b2..163ee4fbaf 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -59,12 +59,14 @@| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) || `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `--tensor-read-lazy MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_TENSOR_READ_LAZY) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit || `-ot, --override-tensor <tensor name pattern>=<buffer type>,...` | override tensor buffer type<br/>(env: LLAMA_ARG_OVERRIDE_TENSOR) || `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU<br/>(env: LLAMA_ARG_CPU_MOE) || `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU<br/>(env: LLAMA_ARG_N_CPU_MOE) |+| `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU<br/>(dense models; for MoE expert weights use --n-cpu-moe)<br/>(env: LLAMA_ARG_N_CPU_FFN) || `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS) || `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:<br/>- none: use one GPU only<br/>- layer (default): split layers and KV across GPUs (pipelined)<br/>- row: split weight across GPUs by rows (parallelized)<br/>- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)<br/>(env: LLAMA_ARG_SPLIT_MODE) || `-ts, --tensor-split N0,N1,N2,...` | fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1<br/>(env: LLAMA_ARG_TENSOR_SPLIT) |@@ -154,7 +156,6 @@| `-sysf, --system-prompt-file FNAME` | a file containing the system prompt (default: none) || `-r, --reverse-prompt PROMPT` | halt generation at PROMPT, return control in interactive mode || `-sp, --special` | special tokens output enabled (default: false) |-| `-cnv, --conversation, -no-cnv, --no-conversation` | whether to run in conversation mode:<br/>- does not print special tokens and suffix/prefix<br/>- interactive mode is also enabled<br/>(default: auto enabled if chat template is available) || `-st, --single-turn` | run conversation for a single turn only, then exit when done<br/>will not be interactive if first turn is predefined with --prompt<br/>(default: false) || `-mli, --multiline-input` | allows you to write or paste multiple lines without ending each in '\' || `--warmup, --no-warmup` | whether to perform warmup with an empty run (default: enabled) |diff --git a/tools/completion/README.md b/tools/completion/README.mdindex 833687dcad..0cd86bac70 100644--- a/tools/completion/README.md+++ b/tools/completion/README.md@@ -142,12 +142,14 @@ llama-completion.exe -m models\gemma-1.1-7b-it.Q4_K_M.gguf --ignore-eos -n -1| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) || `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `--tensor-read-lazy MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_TENSOR_READ_LAZY) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit || `-ot, --override-tensor <tensor name pattern>=<buffer type>,...` | override tensor buffer type<br/>(env: LLAMA_ARG_OVERRIDE_TENSOR) || `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU<br/>(env: LLAMA_ARG_CPU_MOE) || `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU<br/>(env: LLAMA_ARG_N_CPU_MOE) |+| `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU<br/>(dense models; for MoE expert weights use --n-cpu-moe)<br/>(env: LLAMA_ARG_N_CPU_FFN) || `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS) || `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:<br/>- none: use one GPU only<br/>- layer (default): split layers and KV across GPUs (pipelined)<br/>- row: split weight across GPUs by rows (parallelized)<br/>- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)<br/>(env: LLAMA_ARG_SPLIT_MODE) || `-ts, --tensor-split N0,N1,N2,...` | fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1<br/>(env: LLAMA_ARG_TENSOR_SPLIT) |diff --git a/tools/server/README.md b/tools/server/README.mdindex 6fd27f1389..07e58fe916 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -76,12 +76,14 @@ For the full list of features, please refer to [server's changelog](https://gith| `--mmap, --no-mmap` | DEPRECATED in favor of `--load-mode`: whether to memory-map model. (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>(env: LLAMA_ARG_MMAP) || `-dio, --direct-io, -ndio, --no-direct-io` | DEPRECATED in favor of `--load-mode`: use DirectIO if available<br/>(env: LLAMA_ARG_DIO) || `-lm, --load-mode MODE` | model loading mode (default: auto)<br/>- auto: mmap, unless a device does not support it<br/>- none: no special loading mode<br/>- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)<br/>- mlock: force system to keep model in RAM rather than swapping or compressing<br/>- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing<br/>- dio: use DirectIO if available<br/><br/>(env: LLAMA_ARG_LOAD_MODE) |+| `--tensor-read-lazy MODE` | on-demand reading of certain tensors, for example per-layer embeddings (default: auto)<br/>- on: read the rows of such tensors from disk on demand instead of keeping them resident (requires mmap)<br/>- auto: on, but only for tensors larger than 4 GiB<br/>- off: always keep them resident<br/>(env: LLAMA_ARG_TENSOR_READ_LAZY) || `--numa TYPE` | attempt optimizations that help on some NUMA systems<br/>- distribute: spread execution evenly over all nodes<br/>- isolate: only spawn threads on CPUs on the node that execution started on<br/>- numactl: use the CPU map provided by numactl<br/>if run without this previously, it is recommended to drop the system page cache before using this<br/>see https://github.com/ggml-org/llama.cpp/issues/1437<br/>(env: LLAMA_ARG_NUMA) || `-dev, --device <dev1,dev2,..>` | comma-separated list of devices to use for offloading (none = don't offload)<br/>use --list-devices to see a list of available devices<br/>(env: LLAMA_ARG_DEVICE) || `--list-devices` | print list of available devices and exit || `-ot, --override-tensor <tensor name pattern>=<buffer type>,...` | override tensor buffer type<br/>(env: LLAMA_ARG_OVERRIDE_TENSOR) || `-cmoe, --cpu-moe` | keep all Mixture of Experts (MoE) weights in the CPU<br/>(env: LLAMA_ARG_CPU_MOE) || `-ncmoe, --n-cpu-moe N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU<br/>(env: LLAMA_ARG_N_CPU_MOE) |+| `-ncffn, --n-cpu-ffn N` | keep the dense FFN weights of the first N layers in the CPU<br/>(dense models; for MoE expert weights use --n-cpu-moe)<br/>(env: LLAMA_ARG_N_CPU_FFN) || `-ngl, --gpu-layers, --n-gpu-layers N` | max. number of layers to store in VRAM, either an exact number, 'auto', or 'all' (default: auto)<br/>(env: LLAMA_ARG_N_GPU_LAYERS) || `-sm, --split-mode {none,layer,row,tensor}` | how to split the model across multiple GPUs, one of:<br/>- none: use one GPU only<br/>- layer (default): split layers and KV across GPUs (pipelined)<br/>- row: split weight across GPUs by rows (parallelized)<br/>- tensor: split weights and KV across GPUs (parallelized, EXPERIMENTAL)<br/>(env: LLAMA_ARG_SPLIT_MODE) || `-ts, --tensor-split N0,N1,N2,...` | fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1<br/>(env: LLAMA_ARG_TENSOR_SPLIT) |