LLM inference in C/C++
C++ ggml
bench: add --tensor-read-lazy (#27881)
Xuan-Son Nguyen50f068ffffc3
diff --git a/common/arg.cpp b/common/arg.cppindex e346863e51..4469612cd5 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -2735,9 +2735,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex"- auto: on, but only for tensors larger than 4 GiB\n""- off: always keep them resident",[](common_params & params, const std::string & value) {- /**/ if (value == "on") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_ON; }- else if (value == "auto") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; }- else if (value == "off") { params.tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF; }+ /**/ if (value == "on") { params.lazy_mode = LLAMA_LAZY_MODE_ON; }+ else if (value == "auto") { params.lazy_mode = LLAMA_LAZY_MODE_AUTO; }+ else if (value == "off") { params.lazy_mode = LLAMA_LAZY_MODE_OFF; }else { throw std::invalid_argument("invalid value"); }}).set_env("LLAMA_ARG_TENSOR_READ_LAZY"));diff --git a/common/common.cpp b/common/common.cppindex 347e8e9fc4..d162a38800 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1688,7 +1688,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {mparams.main_gpu = params.main_gpu;mparams.split_mode = params.split_mode;mparams.load_mode = params.load_mode;- mparams.tensor_read_lazy = params.tensor_read_lazy;+ mparams.lazy_mode = params.lazy_mode;mparams.tensor_split = params.tensor_split;mparams.check_tensors = params.check_tensors;mparams.use_extra_bufts = !params.no_extra_bufts;diff --git a/common/common.h b/common/common.hindex a333f702ac..4e9448bb10 100644--- a/common/common.h+++ b/common/common.h@@ -483,7 +483,7 @@ struct common_params {enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUsenum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model- enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_AUTO; // on-demand reading of tensors marked by the arch+ enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the archcommon_cpu_params cpuparams;common_cpu_params cpuparams_batch;diff --git a/include/llama.h b/include/llama.hindex 49a758db26..ef7a012c43 100644--- a/include/llama.h+++ b/include/llama.h@@ -214,10 +214,10 @@ extern "C" {LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);- enum llama_tensor_read_lazy {- LLAMA_TENSOR_READ_LAZY_OFF = 0, // always read the whole tensor up front- LLAMA_TENSOR_READ_LAZY_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)- LLAMA_TENSOR_READ_LAZY_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)+ enum llama_lazy_mode {+ LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front+ LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)+ LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)};enum llama_context_type {@@ -321,7 +321,7 @@ extern "C" {enum llama_split_mode split_mode; // how to split the model across multiple GPUsenum llama_load_mode load_mode; // how to load the model- enum llama_tensor_read_lazy tensor_read_lazy; // on-demand reading of tensors marked by the arch+ enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch// the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONEint32_t main_gpu;diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cppindex d9241022cf..1b1f852a01 100644--- a/src/llama-model-loader.cpp+++ b/src/llama-model-loader.cpp@@ -1287,10 +1287,10 @@ struct ggml_tensor * llama_model_loader::create_tensor(return NULL;}- if ((flags & TENSOR_READ_LAZY) && use_mmap && tensor_read_lazy != LLAMA_TENSOR_READ_LAZY_OFF) {+ if ((flags & TENSOR_READ_LAZY) && use_mmap && lazy_mode != LLAMA_LAZY_MODE_OFF) {// in auto mode, small tensors are cheap enough to keep residentconstexpr size_t auto_lazy_min_size = 4ull * 1024 * 1024 * 1024;- if (tensor_read_lazy == LLAMA_TENSOR_READ_LAZY_ON || ggml_nbytes(cur) > auto_lazy_min_size) {+ if (lazy_mode == LLAMA_LAZY_MODE_ON || ggml_nbytes(cur) > auto_lazy_min_size) {const auto & w = require_weight(tn.str().c_str());lazy_tensor_ranges[w.idx].emplace_back(w.offs, w.offs + ggml_nbytes(cur));diff --git a/src/llama-model-loader.h b/src/llama-model-loader.hindex 407260e990..20f7442538 100644--- a/src/llama-model-loader.h+++ b/src/llama-model-loader.h@@ -84,7 +84,7 @@ struct llama_model_loader {bool load_mtp;// set by the caller before the create_tensor() calls- enum llama_tensor_read_lazy tensor_read_lazy = LLAMA_TENSOR_READ_LAZY_OFF;+ enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_OFF;llama_files files;llama_ftype ftype;diff --git a/src/llama-model.cpp b/src/llama-model.cppindex fc83658dd7..65a6702cef 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -2681,7 +2681,7 @@ llama_model_params llama_model_default_params() {/*.n_gpu_layers =*/ -1,/*.split_mode =*/ LLAMA_SPLIT_MODE_LAYER,/*.load_mode =*/ LLAMA_LOAD_MODE_AUTO,- /*.tensor_read_lazy =*/ LLAMA_TENSOR_READ_LAZY_AUTO,+ /*.lazy_mode =*/ LLAMA_LAZY_MODE_AUTO,/*.main_gpu =*/ 0,/*.tensor_split =*/ nullptr,/*.progress_callback =*/ nullptr,diff --git a/src/llama.cpp b/src/llama.cppindex 9c841ee352..6ec5d315dc 100644--- a/src/llama.cpp+++ b/src/llama.cpp@@ -318,7 +318,7 @@ static std::pair<int, llama_model *> llama_model_load(struct gguf_context * metallama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode,params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides);- ml.tensor_read_lazy = params.tensor_read_lazy;+ ml.lazy_mode = params.lazy_mode;ml.print_info();std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, params));diff --git a/tools/llama-bench/README.md b/tools/llama-bench/README.mdindex 42cb14859f..a1404d2e33 100644--- a/tools/llama-bench/README.md+++ b/tools/llama-bench/README.md@@ -67,6 +67,7 @@ test parameters:-nkvo, --no-kv-offload <0|1> (default: 0)-fa, --flash-attn <on|off|auto> (default: auto)-dev, --device <dev0/dev1/...> (default: auto)+ --tensor-read-lazy <on|auto|off> (default: auto)-mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)-dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)-embd, --embeddings <0|1> (default: 0)diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cppindex bc14d15c79..1b4bbde4d4 100644--- a/tools/llama-bench/llama-bench.cpp+++ b/tools/llama-bench/llama-bench.cpp@@ -271,6 +271,19 @@ static const char * split_mode_str(llama_split_mode mode) {}}+static const char * lazy_mode_str(llama_lazy_mode mode) {+ switch (mode) {+ case LLAMA_LAZY_MODE_OFF:+ return "off";+ case LLAMA_LAZY_MODE_AUTO:+ return "auto";+ case LLAMA_LAZY_MODE_ON:+ return "on";+ default:+ GGML_ABORT("invalid tensor read lazy mode");+ }+}+static std::string pair_str(const std::pair<int, int> & p) {static char buf[32];snprintf(buf, sizeof(buf), "%d,%d", p.first, p.second);@@ -341,6 +354,7 @@ struct cmd_params {std::vector<int> n_cpu_moe;std::vector<llama_split_mode> split_mode;std::vector<llama_load_mode> load_mode;+ std::vector<llama_lazy_mode> lazy_mode;std::vector<int> main_gpu;std::vector<bool> no_kv_offload;std::vector<llama_flash_attn_type> flash_attn;@@ -385,6 +399,7 @@ static const cmd_params cmd_params_defaults = {/* n_cpu_moe */ { 0 },/* split_mode */ { LLAMA_SPLIT_MODE_LAYER },/* load_mode */ { LLAMA_LOAD_MODE_AUTO },+ /* lazy_mode */ { LLAMA_LAZY_MODE_AUTO },/* main_gpu */ { 0 },/* no_kv_offload */ { false },/* flash_attn */ { LLAMA_FLASH_ATTN_TYPE_AUTO },@@ -460,6 +475,7 @@ static void print_usage(int /* argc */, char ** argv) {printf(" -fa, --flash-attn <on|off|auto> (default: %s)\n", join(transform_to_str(cmd_params_defaults.flash_attn, llama_flash_attn_type_name), ",").c_str());printf(" -dev, --device <dev0/dev1/...> (default: auto)\n");printf(" -lm, --load-mode <auto|none|mmap|mlock|mmap+mlock|dio> (default: %s)\n", join(transform_to_str(cmd_params_defaults.load_mode, llama_load_mode_name), ",").c_str());+ printf(" --tensor-read-lazy <on|auto|off> (default: %s)\n", join(transform_to_str(cmd_params_defaults.lazy_mode, lazy_mode_str), ",").c_str());printf(" -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");printf(" -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode)\n");printf(" -embd, --embeddings <0|1> (default: %s)\n", join(cmd_params_defaults.embeddings, ",").c_str());@@ -786,6 +802,32 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {break;}params.load_mode.insert(params.load_mode.end(), modes.begin(), modes.end());+ } else if (arg == "--tensor-read-lazy") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ auto p = string_split<std::string>(argv[i], split_delim);++ std::vector<llama_lazy_mode> modes;+ for (const auto & m : p) {+ llama_lazy_mode mode;+ if (m == "on") {+ mode = LLAMA_LAZY_MODE_ON;+ } else if (m == "auto") {+ mode = LLAMA_LAZY_MODE_AUTO;+ } else if (m == "off") {+ mode = LLAMA_LAZY_MODE_OFF;+ } else {+ invalid_param = true;+ break;+ }+ modes.push_back(mode);+ }+ if (invalid_param) {+ break;+ }+ params.lazy_mode.insert(params.lazy_mode.end(), modes.begin(), modes.end());} else if (arg == "-mg" || arg == "--main-gpu") {if (++i >= argc) {invalid_param = true;@@ -1137,6 +1179,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {if (params.load_mode.empty()) {params.load_mode = cmd_params_defaults.load_mode;}+ if (params.lazy_mode.empty()) {+ params.lazy_mode = cmd_params_defaults.lazy_mode;+ }if (params.main_gpu.empty()) {params.main_gpu = cmd_params_defaults.main_gpu;}@@ -1203,6 +1248,7 @@ struct cmd_params_instance {int n_cpu_moe;llama_split_mode split_mode;llama_load_mode load_mode;+ llama_lazy_mode lazy_mode;int main_gpu;bool no_kv_offload;llama_flash_attn_type flash_attn;@@ -1224,6 +1270,7 @@ struct cmd_params_instance {}mparams.split_mode = split_mode;mparams.load_mode = load_mode;+ mparams.lazy_mode = lazy_mode;mparams.main_gpu = main_gpu;mparams.tensor_split = tensor_split.data();mparams.no_host = no_host;@@ -1271,7 +1318,8 @@ struct cmd_params_instance {return model == other.model && n_gpu_layers == other.n_gpu_layers && n_cpu_moe == other.n_cpu_moe &&split_mode == other.split_mode &&main_gpu == other.main_gpu && tensor_split == other.tensor_split &&- load_mode == other.load_mode && devices == other.devices && no_host == other.no_host &&+ load_mode == other.load_mode && lazy_mode == other.lazy_mode &&+ devices == other.devices && no_host == other.no_host &&vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);}@@ -1305,6 +1353,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_paramfor (const auto & ncmoe : params.n_cpu_moe)for (const auto & sm : params.split_mode)for (const auto & lm : params.load_mode)+ for (const auto & lzm : params.lazy_mode)for (const auto & mg : params.main_gpu)for (const auto & devs : params.devices)for (const auto & ts : params.tensor_split)@@ -1344,6 +1393,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param/* .n_cpu_moe = */ ncmoe,/* .split_mode = */ sm,/* .load_mode = */ lm,+ /* .lazy_mode = */ lzm,/* .main_gpu = */ mg,/* .no_kv_offload = */ nkvo,/* .flash_attn = */ fa,@@ -1380,6 +1430,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param/* .n_cpu_moe = */ ncmoe,/* .split_mode = */ sm,/* .load_mode = */ lm,+ /* .lazy_mode = */ lzm,/* .main_gpu = */ mg,/* .no_kv_offload = */ nkvo,/* .flash_attn = */ fa,@@ -1416,6 +1467,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param/* .n_cpu_moe = */ ncmoe,/* .split_mode = */ sm,/* .load_mode = */ lm,+ /* .lazy_mode = */ lzm,/* .main_gpu = */ mg,/* .no_kv_offload = */ nkvo,/* .flash_attn = */ fa,@@ -1457,6 +1509,7 @@ struct test {int n_cpu_moe;llama_split_mode split_mode;llama_load_mode load_mode;+ llama_lazy_mode lazy_mode;int main_gpu;bool no_kv_offload;llama_flash_attn_type flash_attn;@@ -1496,6 +1549,7 @@ struct test {n_cpu_moe = inst.n_cpu_moe;split_mode = inst.split_mode;load_mode = inst.load_mode;+ lazy_mode = inst.lazy_mode;main_gpu = inst.main_gpu;no_kv_offload = inst.no_kv_offload;flash_attn = inst.flash_attn;@@ -1563,7 +1617,8 @@ struct test {"n_ubatch", "n_threads", "cpu_mask", "cpu_strict", "poll","type_k", "type_v", "n_gpu_layers", "n_cpu_moe", "split_mode","main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split",- "tensor_buft_overrides", "load_mode", "embeddings",+ "tensor_buft_overrides", "load_mode", "lazy_mode",+ "embeddings","no_op_offload", "no_host", "fit_target", "fit_min_ctx","n_prompt", "n_gen", "n_depth","test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts"@@ -1588,7 +1643,7 @@ struct test {if (field == "avg_ts" || field == "stddev_ts") {return FLOAT;}- if (field == "load_mode") {+ if (field == "load_mode" || field == "lazy_mode") {return STRING;}return STRING;@@ -1658,6 +1713,7 @@ struct test {tensor_split_str,tensor_buft_overrides_str,llama_load_mode_name(load_mode),+ lazy_mode_str(lazy_mode),std::to_string(embeddings),std::to_string(no_op_offload),std::to_string(no_host),@@ -1972,6 +2028,9 @@ struct markdown_printer : public printer {if (params.load_mode.size() > 1 || params.load_mode != cmd_params_defaults.load_mode) {fields.emplace_back("load_mode");}+ if (params.lazy_mode.size() > 1 || params.lazy_mode != cmd_params_defaults.lazy_mode) {+ fields.emplace_back("lazy_mode");+ }if (params.embeddings.size() > 1 || params.embeddings != cmd_params_defaults.embeddings) {fields.emplace_back("embeddings");}