LLM inference in C/C++
C++ ggml
model: split layers evenly when no device reports free memory (#29677)
Pascaldf18bee11e96
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex aeab7b75b3..923c2c0a2b 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1645,7 +1645,7 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {splits[i] = split_sum;}for (size_t i = 0; i < n_devices(); ++i) {- splits[i] /= split_sum;+ splits[i] = split_sum != 0.0f ? splits[i] / split_sum : float(i + 1) / n_devices();}const int i_gpu_start = std::max(n_layer_all + 1 - n_gpu_layers, 0);