LLM inference in C/C++
C++ ggml
context : autoscale n_ctx_train when yarn scaling specified (#28030)
Sigbjørn Skjæretdfc29b64eb88
diff --git a/src/llama-context.cpp b/src/llama-context.cppindex a920c42311..f286bd1da2 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -125,8 +125,9 @@ llama_context::llama_context(cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);embd_layer_inp.resize(hparams.n_layer() + 1);- cparams.ctx_type = params.ctx_type;- cparams.pooling_type = params.pooling_type;+ cparams.ctx_type = params.ctx_type;+ cparams.rope_scaling_type = params.rope_scaling_type;+ cparams.pooling_type = params.pooling_type;cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;@@ -160,17 +161,16 @@ llama_context::llama_context(}}- auto rope_scaling_type = params.rope_scaling_type;- if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {- rope_scaling_type = hparams.rope_scaling_type_train;+ if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {+ cparams.rope_scaling_type = hparams.rope_scaling_type_train;}- if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {+ if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none}if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'- cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;+ cparams.yarn_ext_factor = cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;}if (cparams.yarn_ext_factor != 0) {@@ -3734,6 +3734,14 @@ llama_context * llama_init_from_model(try {auto * ctx = new llama_context(*model, params);+ const auto & cparams = ctx->get_cparams();++ if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN && cparams.rope_freq_scale != model->hparams.rope_freq_scale_train) {+ LLAMA_LOG_INFO("%s: custom YaRN scaling detected, re-adjusting n_ctx_train(%u)...\n", __func__, model->hparams.n_ctx_train);+ model->hparams.n_ctx_train = cparams.n_ctx_orig_yarn / cparams.rope_freq_scale;+ LLAMA_LOG_INFO("%s: n_ctx_train adjusted to %u\n", __func__, model->hparams.n_ctx_train);+ }+return ctx;} catch (const std::exception & err) {LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what());diff --git a/src/llama-cparams.h b/src/llama-cparams.hindex 574ce95920..b592de18c7 100644--- a/src/llama-cparams.h+++ b/src/llama-cparams.h@@ -57,6 +57,7 @@ struct llama_cparams {std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layerenum llama_context_type ctx_type;+ enum llama_rope_scaling_type rope_scaling_type;enum llama_pooling_type pooling_type;ggml_backend_sched_eval_callback cb_eval;