LLM inference in C/C++
C++ ggml
sampler: reduce the size of the probe (#29285)
Max Krasnyansky441df11f65ea
diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cppindex 61d28ad8a8..b797c50bef 100644--- a/src/llama-sampler.cpp+++ b/src/llama-sampler.cpp@@ -644,7 +644,7 @@ static bool llama_sampler_backend_support(return true;}- auto probe = llama_sampler_backend_probe_graph(smpl, 1024*1024, GGML_DEFAULT_GRAPH_SIZE, true);+ auto probe = llama_sampler_backend_probe_graph(smpl, 128*1024, GGML_DEFAULT_GRAPH_SIZE, true);for (int i = 0; i < ggml_graph_n_nodes(probe.gf); i++) {struct ggml_tensor * op = ggml_graph_node(probe.gf, i);@@ -764,7 +764,7 @@ static bool llama_sampler_chain_backend_init(res = res && cur_prefix;}- auto probe = llama_sampler_backend_probe_graph(smpl, 1024*1024, GGML_DEFAULT_GRAPH_SIZE, false);+ auto probe = llama_sampler_backend_probe_graph(smpl, 128*1024, GGML_DEFAULT_GRAPH_SIZE, false);chain->n_nodes = llama_sampler_backend_probe_n_nodes(probe);return res;