LLM inference in C/C++
C++ ggml
Fix stale tensor-split params for draft models (#24814)
Al G2da668617612
diff --git a/src/llama-model.cpp b/src/llama-model.cppindex e07f6e9863..8d68cff45b 100644--- a/src/llama-model.cpp+++ b/src/llama-model.cpp@@ -1012,9 +1012,17 @@ struct llama_model::impl {std::vector<layer_dev> dev_layer;bool has_tensor_overrides;++ std::vector<float> tensor_split_owned;};llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {+ if (params.tensor_split != nullptr) {+ // llama_model_params stores tensor_split as a borrowed pointer, but the model+ // may need it later for tensor-parallel KV-cache split metadata.+ pimpl->tensor_split_owned.assign(params.tensor_split, params.tensor_split + llama_max_devices());+ this->params.tensor_split = pimpl->tensor_split_owned.data();+ }pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;}