LLM inference in C/C++
C++ ggml
fit : avoid including llama-ext.h in fit.h (#24506)
Georgi Gerganov02182fc5b9b0
diff --git a/common/fit.h b/common/fit.hindex 643d342009..a6ba71b89e 100644--- a/common/fit.h+++ b/common/fit.h@@ -1,9 +1,7 @@#pragma once#include "ggml.h"-#include "ggml-backend.h"#include "llama.h"-#include "../src/llama-ext.h"#include <vector>@@ -18,31 +16,35 @@ enum common_params_fit_status {// - this function is NOT thread safe because it modifies the global llama logger state// - only parameters that have the same value as in llama_default_model_params are modified// with the exception of the context size which is modified if and only if equal to 0-enum common_params_fit_status common_fit_params(- const char * path_model,- struct llama_model_params * mparams,- struct llama_context_params * cparams,- float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements- struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements- size_t * margins, // margins of memory to leave per device in bytes- uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use- enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log+common_params_fit_status common_fit_params(+ const char * path_model,+ llama_model_params * mparams,+ llama_context_params * cparams,+ float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements+ llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements+ size_t * margins, // margins of memory to leave per device in bytes+ uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use+ ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log// print estimated memory to stdoutvoid common_fit_print(- const char * path_model,- struct llama_model_params * mparams,- struct llama_context_params * cparams);+ const char * path_model,+ llama_model_params * mparams,+ llama_context_params * cparams);-void common_memory_breakdown_print(const struct llama_context * ctx);+void common_memory_breakdown_print(const llama_context * ctx);++// TODO: convert this to common_device_memory_data that wraps llama_device_memory_data+// add API for accessing the internal `llama-ext.h` information+struct llama_device_memory_data;// Load a model + context with no_alloc and return the per-device memory breakdown.std::vector<llama_device_memory_data> common_get_device_memory_data(- const char * path_model,- const struct llama_model_params * mparams,- const struct llama_context_params * cparams,- std::vector<ggml_backend_dev_t> & devs,- uint32_t & hp_ngl,- uint32_t & hp_n_ctx_train,- uint32_t & hp_n_expert,- enum ggml_log_level log_level);+ const char * path_model,+ const llama_model_params * mparams,+ const llama_context_params * cparams,+ std::vector<ggml_backend_dev_t> & devs,+ uint32_t & hp_ngl,+ uint32_t & hp_n_ctx_train,+ uint32_t & hp_n_expert,+ ggml_log_level log_level);diff --git a/src/llama-ext.h b/src/llama-ext.hindex b744af5286..8b5679b690 100644--- a/src/llama-ext.h+++ b/src/llama-ext.h@@ -2,6 +2,7 @@// this is a staging header for new llama.cpp API// breaking changes and C++ are allowed. everything here should be considered WIP+// try as much as possible to not include this header in the rest of the codebase#include "llama.h"