LLM inference in C/C++
C++ ggml
polished Readme and llama-bench (#28968)
Vladislav965f89794fd9
diff --git a/tools/llama-bench/README.md b/tools/llama-bench/README.mdindex 8adc56514d..512a2c0248 100644--- a/tools/llama-bench/README.md+++ b/tools/llama-bench/README.md@@ -76,6 +76,7 @@ test parameters:(default: disabled)-nopo, --no-op-offload <0|1> (default: 0)--no-host <0|1> (default: 0)+ --repack <0|1> (default: 1)Multiple values can be given for each parameter by separating them with ','or by specifying the parameter multiple times. Ranges can be given as@@ -210,7 +211,7 @@ $ ./llama-bench -o csv``````csv-build_commit,build_number,cpu_info,gpu_info,backends,model_filename,model_type,model_size,model_n_params,n_batch,n_ubatch,n_threads,cpu_mask,cpu_strict,poll,type_k,type_v,n_gpu_layers,n_cpu_moe,split_mode,main_gpu,no_kv_offload,flash_attn,devices,tensor_split,tensor_buft_overrides,use_mmap,use_direct_io,embeddings,no_op_offload,no_host,fit_target,fit_min_ctx,n_prompt,n_gen,n_depth,test_time,avg_ns,stddev_ns,avg_ts,stddev_ts+build_commit,build_number,cpu_info,gpu_info,backends,model_filename,model_type,model_size,model_n_params,n_batch,n_ubatch,n_threads,cpu_mask,cpu_strict,poll,type_k,type_v,n_gpu_layers,n_cpu_moe,split_mode,main_gpu,no_kv_offload,flash_attn,devices,tensor_split,tensor_buft_overrides,use_mmap,use_direct_io,embeddings,no_op_offload,no_host,repack,fit_target,fit_min_ctx,n_prompt,n_gen,n_depth,test_time,avg_ns,stddev_ns,avg_ts,stddev_ts"8cf427ff","5163","AMD Ryzen 7 7800X3D 8-Core Processor","NVIDIA GeForce RTX 4080","CUDA","models/Qwen2.5-7B-Instruct-Q4_K_M.gguf","qwen2 7B Q4_K - Medium","4677120000","7615616512","2048","512","8","0x0","0","50","f16","f16","-1","0","layer","0","0","-1","auto","0.00","none","1","0","0","0","0","0","0","512","0","0","2025-04-24T11:57:09Z","70285660","982040","7285.676949","100.064434""8cf427ff","5163","AMD Ryzen 7 7800X3D 8-Core Processor","NVIDIA GeForce RTX 4080","CUDA","models/Qwen2.5-7B-Instruct-Q4_K_M.gguf","qwen2 7B Q4_K - Medium","4677120000","7615616512","2048","512","8","0x0","0","50","f16","f16","-1","0","layer","0","0","-1","auto","0.00","none","1","0","0","0","0","0","0","0","128","0","2025-04-24T11:57:10Z","1067431600","3834831","119.915244","0.430617"```@@ -255,6 +256,7 @@ $ ./llama-bench -o json"embeddings": false,"no_op_offload": 0,"no_host": false,+ "repack": true,"fit_target": 0,"fit_min_ctx": 0,"n_prompt": 512,@@ -300,6 +302,7 @@ $ ./llama-bench -o json"embeddings": false,"no_op_offload": 0,"no_host": false,+ "repack": true,"fit_target": 0,"fit_min_ctx": 0,"n_prompt": 0,@@ -324,8 +327,8 @@ $ ./llama-bench -o jsonl``````json lines-{"build_commit": "8cf427ff", "build_number": 5163, "cpu_info": "AMD Ryzen 7 7800X3D 8-Core Processor", "gpu_info": "NVIDIA GeForce RTX 4080", "backends": "CUDA", "model_filename": "models/Qwen2.5-7B-Instruct-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": -1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "use_mmap": true, "use_direct_io": false, "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2025-04-24T11:59:33Z", "avg_ns": 70497220, "stddev_ns": 883196, "avg_ts": 7263.609157, "stddev_ts": 90.940578, "samples_ns": [ 71551000, 71222800, 70364100, 69439100, 69909100 ],"samples_ts": [ 7155.74, 7188.71, 7276.44, 7373.37, 7323.8 ]}-{"build_commit": "8cf427ff", "build_number": 5163, "cpu_info": "AMD Ryzen 7 7800X3D 8-Core Processor", "gpu_info": "NVIDIA GeForce RTX 4080", "backends": "CUDA", "model_filename": "models/Qwen2.5-7B-Instruct-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": -1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "use_mmap": true, "use_direct_io": false, "embeddings": false, "no_op_offload": 0, "no_host": false, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2025-04-24T11:59:33Z", "avg_ns": 1068078400, "stddev_ns": 6279455, "avg_ts": 119.844681, "stddev_ts": 0.699739, "samples_ns": [ 1066331700, 1064864900, 1079042600, 1063328400, 1066824400 ],"samples_ts": [ 120.038, 120.203, 118.624, 120.377, 119.982 ]}+{"build_commit": "8cf427ff", "build_number": 5163, "cpu_info": "AMD Ryzen 7 7800X3D 8-Core Processor", "gpu_info": "NVIDIA GeForce RTX 4080", "backends": "CUDA", "model_filename": "models/Qwen2.5-7B-Instruct-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": -1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "use_mmap": true, "use_direct_io": false, "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 512, "n_gen": 0, "n_depth": 0, "test_time": "2025-04-24T11:59:33Z", "avg_ns": 70497220, "stddev_ns": 883196, "avg_ts": 7263.609157, "stddev_ts": 90.940578, "samples_ns": [ 71551000, 71222800, 70364100, 69439100, 69909100 ],"samples_ts": [ 7155.74, 7188.71, 7276.44, 7373.37, 7323.8 ]}+{"build_commit": "8cf427ff", "build_number": 5163, "cpu_info": "AMD Ryzen 7 7800X3D 8-Core Processor", "gpu_info": "NVIDIA GeForce RTX 4080", "backends": "CUDA", "model_filename": "models/Qwen2.5-7B-Instruct-Q4_K_M.gguf", "model_type": "qwen2 7B Q4_K - Medium", "model_size": 4677120000, "model_n_params": 7615616512, "n_batch": 2048, "n_ubatch": 512, "n_threads": 8, "cpu_mask": "0x0", "cpu_strict": false, "poll": 50, "type_k": "f16", "type_v": "f16", "n_gpu_layers": -1, "n_cpu_moe": 0, "split_mode": "layer", "main_gpu": 0, "no_kv_offload": false, "flash_attn": -1, "devices": "auto", "tensor_split": "0.00", "tensor_buft_overrides": "none", "use_mmap": true, "use_direct_io": false, "embeddings": false, "no_op_offload": 0, "no_host": false, "repack": true, "fit_target": 0, "fit_min_ctx": 0, "n_prompt": 0, "n_gen": 128, "n_depth": 0, "test_time": "2025-04-24T11:59:33Z", "avg_ns": 1068078400, "stddev_ns": 6279455, "avg_ts": 119.844681, "stddev_ts": 0.699739, "samples_ns": [ 1066331700, 1064864900, 1079042600, 1063328400, 1066824400 ],"samples_ts": [ 120.038, 120.203, 118.624, 120.377, 119.982 ]}```@@ -370,6 +373,7 @@ CREATE TABLE IF NOT EXISTS llama_bench (embeddings INTEGER,no_op_offload INTEGER,no_host INTEGER,+ repack INTEGER,fit_target INTEGER,fit_min_ctx INTEGER,n_prompt INTEGER,@@ -382,6 +386,6 @@ CREATE TABLE IF NOT EXISTS llama_bench (stddev_ts REAL);-INSERT INTO llama_bench (build_commit, build_number, cpu_info, gpu_info, backends, model_filename, model_type, model_size, model_n_params, n_batch, n_ubatch, n_threads, cpu_mask, cpu_strict, poll, type_k, type_v, n_gpu_layers, n_cpu_moe, split_mode, main_gpu, no_kv_offload, flash_attn, devices, tensor_split, tensor_buft_overrides, use_mmap, use_direct_io, embeddings, no_op_offload, no_host, fit_target, fit_min_ctx, n_prompt, n_gen, n_depth, test_time, avg_ns, stddev_ns, avg_ts, stddev_ts) VALUES ('8cf427ff', '5163', 'AMD Ryzen 7 7800X3D 8-Core Processor', 'NVIDIA GeForce RTX 4080', 'CUDA', 'models/Qwen2.5-7B-Instruct-Q4_K_M.gguf', 'qwen2 7B Q4_K - Medium', '4677120000', '7615616512', '2048', '512', '8', '0x0', '0', '50', 'f16', 'f16', '-1', '0', 'layer', '0', '0', '-1', 'auto', '0.00', 'none', '1', '0', '0', '0', '0', '0', '0', '512', '0', '0', '2025-04-24T12:00:08Z', '69905000', '519516', '7324.546977', '54.032613');-INSERT INTO llama_bench (build_commit, build_number, cpu_info, gpu_info, backends, model_filename, model_type, model_size, model_n_params, n_batch, n_ubatch, n_threads, cpu_mask, cpu_strict, poll, type_k, type_v, n_gpu_layers, n_cpu_moe, split_mode, main_gpu, no_kv_offload, flash_attn, devices, tensor_split, tensor_buft_overrides, use_mmap, use_direct_io, embeddings, no_op_offload, no_host, fit_target, fit_min_ctx, n_prompt, n_gen, n_depth, test_time, avg_ns, stddev_ns, avg_ts, stddev_ts) VALUES ('8cf427ff', '5163', 'AMD Ryzen 7 7800X3D 8-Core Processor', 'NVIDIA GeForce RTX 4080', 'CUDA', 'models/Qwen2.5-7B-Instruct-Q4_K_M.gguf', 'qwen2 7B Q4_K - Medium', '4677120000', '7615616512', '2048', '512', '8', '0x0', '0', '50', 'f16', 'f16', '-1', '0', 'layer', '0', '0', '-1', 'auto', '0.00', 'none', '1', '0', '0', '0', '0', '0', '0', '0', '128', '0', '2025-04-24T12:00:09Z', '1063608780', '4464130', '120.346696', '0.504647');+INSERT INTO llama_bench (build_commit, build_number, cpu_info, gpu_info, backends, model_filename, model_type, model_size, model_n_params, n_batch, n_ubatch, n_threads, cpu_mask, cpu_strict, poll, type_k, type_v, n_gpu_layers, n_cpu_moe, split_mode, main_gpu, no_kv_offload, flash_attn, devices, tensor_split, tensor_buft_overrides, use_mmap, use_direct_io, embeddings, no_op_offload, no_host, repack, fit_target, fit_min_ctx, n_prompt, n_gen, n_depth, test_time, avg_ns, stddev_ns, avg_ts, stddev_ts) VALUES ('8cf427ff', '5163', 'AMD Ryzen 7 7800X3D 8-Core Processor', 'NVIDIA GeForce RTX 4080', 'CUDA', 'models/Qwen2.5-7B-Instruct-Q4_K_M.gguf', 'qwen2 7B Q4_K - Medium', '4677120000', '7615616512', '2048', '512', '8', '0x0', '0', '50', 'f16', 'f16', '-1', '0', 'layer', '0', '0', '-1', 'auto', '0.00', 'none', '1', '0', '0', '0', '0', '1', '0', '0', '512', '0', '0', '2025-04-24T12:00:08Z', '69905000', '519516', '7324.546977', '54.032613');+INSERT INTO llama_bench (build_commit, build_number, cpu_info, gpu_info, backends, model_filename, model_type, model_size, model_n_params, n_batch, n_ubatch, n_threads, cpu_mask, cpu_strict, poll, type_k, type_v, n_gpu_layers, n_cpu_moe, split_mode, main_gpu, no_kv_offload, flash_attn, devices, tensor_split, tensor_buft_overrides, use_mmap, use_direct_io, embeddings, no_op_offload, no_host, repack, fit_target, fit_min_ctx, n_prompt, n_gen, n_depth, test_time, avg_ns, stddev_ns, avg_ts, stddev_ts) VALUES ('8cf427ff', '5163', 'AMD Ryzen 7 7800X3D 8-Core Processor', 'NVIDIA GeForce RTX 4080', 'CUDA', 'models/Qwen2.5-7B-Instruct-Q4_K_M.gguf', 'qwen2 7B Q4_K - Medium', '4677120000', '7615616512', '2048', '512', '8', '0x0', '0', '50', 'f16', 'f16', '-1', '0', 'layer', '0', '0', '-1', 'auto', '0.00', 'none', '1', '0', '0', '0', '0', '1', '0', '0', '0', '128', '0', '2025-04-24T12:00:09Z', '1063608780', '4464130', '120.346696', '0.504647');```diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cppindex 6541bb5078..6e7eab7f0f 100644--- a/tools/llama-bench/llama-bench.cpp+++ b/tools/llama-bench/llama-bench.cpp@@ -364,6 +364,7 @@ struct cmd_params {std::vector<bool> embeddings;std::vector<bool> no_op_offload;std::vector<bool> no_host;+ std::vector<bool> repack;std::vector<size_t> fit_params_target;std::vector<uint32_t> fit_params_min_ctx;ggml_numa_strategy numa;@@ -409,6 +410,7 @@ static const cmd_params cmd_params_defaults = {/* embeddings */ { false },/* no_op_offload */ { false },/* no_host */ { false },+ /* repack */ { llama_model_default_params().use_extra_bufts },/* fit_params_target */ { 0 },/* fit_params_min_ctx */ { 0 },/* numa */ GGML_NUMA_STRATEGY_DISABLED,@@ -483,6 +485,7 @@ static void print_usage(int /* argc */, char ** argv) {printf(" (default: disabled)\n");printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");printf(" --no-host <0|1> (default: %s)\n", join(cmd_params_defaults.no_host, ",").c_str());+ printf(" --repack <0|1> (default: %s)\n", join(cmd_params_defaults.repack, ",").c_str());printf("\n");printf("Multiple values can be given for each parameter by separating them with ','\n"@@ -906,6 +909,13 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {}auto p = string_split<bool>(argv[i], split_delim);params.no_host.insert(params.no_host.end(), p.begin(), p.end());+ } else if (arg == "--repack") {+ if (++i >= argc) {+ invalid_param = true;+ break;+ }+ auto p = string_split<bool>(argv[i], split_delim);+ params.repack.insert(params.repack.end(), p.begin(), p.end());} else if (arg == "-ts" || arg == "--tensor-split") {if (++i >= argc) {invalid_param = true;@@ -1173,6 +1183,9 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {if (params.no_host.empty()) {params.no_host = cmd_params_defaults.no_host;}+ if (params.repack.empty()) {+ params.repack = cmd_params_defaults.repack;+ }if (params.n_threads.empty()) {params.n_threads = cmd_params_defaults.n_threads;}@@ -1222,6 +1235,7 @@ struct cmd_params_instance {bool embeddings;bool no_op_offload;bool no_host;+ bool repack;size_t fit_target;uint32_t fit_min_ctx;@@ -1238,6 +1252,7 @@ struct cmd_params_instance {mparams.main_gpu = main_gpu;mparams.tensor_split = tensor_split.data();mparams.no_host = no_host;+ mparams.use_extra_bufts = repack;if (n_cpu_moe <= 0) {if (tensor_buft_overrides.empty()) {@@ -1283,7 +1298,7 @@ struct cmd_params_instance {split_mode == other.split_mode &&main_gpu == other.main_gpu && tensor_split == other.tensor_split &&load_mode == other.load_mode && lazy_mode == other.lazy_mode &&- devices == other.devices && no_host == other.no_host &&+ devices == other.devices && no_host == other.no_host && repack == other.repack &&vec_tensor_buft_override_equal(tensor_buft_overrides, other.tensor_buft_overrides);}@@ -1323,6 +1338,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_paramfor (const auto & ts : params.tensor_split)for (const auto & ot : params.tensor_buft_overrides)for (const auto & noh : params.no_host)+ for (const auto & rpk : params.repack)for (const auto & embd : params.embeddings)for (const auto & nopo : params.no_op_offload)for (const auto & nb : params.n_batch)@@ -1367,6 +1383,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param/* .embeddings = */ embd,/* .no_op_offload = */ nopo,/* .no_host = */ noh,+ /* .repack = */ rpk,/* .fit_target = */ fpt,/* .fit_min_ctx = */ fpc,};@@ -1404,6 +1421,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param/* .embeddings = */ embd,/* .no_op_offload = */ nopo,/* .no_host = */ noh,+ /* .repack = */ rpk,/* .fit_target = */ fpt,/* .fit_min_ctx = */ fpc,};@@ -1441,6 +1459,7 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param/* .embeddings = */ embd,/* .no_op_offload = */ nopo,/* .no_host = */ noh,+ /* .repack = */ rpk,/* .fit_target = */ fpt,/* .fit_min_ctx = */ fpc,};@@ -1483,6 +1502,7 @@ struct test {bool embeddings;bool no_op_offload;bool no_host;+ bool repack;size_t fit_target;uint32_t fit_min_ctx;int n_prompt;@@ -1523,6 +1543,7 @@ struct test {embeddings = inst.embeddings;no_op_offload = inst.no_op_offload;no_host = inst.no_host;+ repack = inst.repack;fit_target = inst.fit_target;fit_min_ctx = inst.fit_min_ctx;n_prompt = inst.n_prompt;@@ -1583,7 +1604,7 @@ struct test {"main_gpu", "no_kv_offload", "flash_attn", "devices", "tensor_split","tensor_buft_overrides", "load_mode", "lazy_mode","embeddings",- "no_op_offload", "no_host", "fit_target", "fit_min_ctx",+ "no_op_offload", "no_host", "repack", "fit_target", "fit_min_ctx","n_prompt", "n_gen", "n_depth","test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts"};@@ -1601,7 +1622,7 @@ struct test {return INT;}if (field == "f16_kv" || field == "no_kv_offload" || field == "cpu_strict" ||- field == "embeddings" || field == "no_host") {+ field == "embeddings" || field == "no_host" || field == "repack") {return BOOL;}if (field == "avg_ts" || field == "stddev_ts") {@@ -1681,6 +1702,7 @@ struct test {std::to_string(embeddings),std::to_string(no_op_offload),std::to_string(no_host),+ std::to_string(repack),std::to_string(fit_target),std::to_string(fit_min_ctx),std::to_string(n_prompt),@@ -1874,6 +1896,9 @@ struct markdown_printer : public printer {if (field == "no_host") {return 4;}+ if (field == "repack") {+ return 3;+ }int width = std::max((int) field.length(), 10);@@ -1911,6 +1936,9 @@ struct markdown_printer : public printer {if (field == "no_host") {return "noh";}+ if (field == "repack") {+ return "rpk";+ }if (field == "devices") {return "dev";}@@ -2004,6 +2032,9 @@ struct markdown_printer : public printer {if (params.no_host.size() > 1 || params.no_host != cmd_params_defaults.no_host) {fields.emplace_back("no_host");}+ if (params.repack.size() > 1 || params.repack != cmd_params_defaults.repack) {+ fields.emplace_back("repack");+ }if (params.fit_params_target.size() > 1 || params.fit_params_target != cmd_params_defaults.fit_params_target) {fields.emplace_back("fit_target");}