Skip to content

LLM inference in C/C++

C++ ggml

context : autoscale n_ctx_train when yarn scaling specified (#28030)

Sigbjørn Skjæretdfc29b64eb88
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index a920c42311..f286bd1da2 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -125,8 +125,9 @@ llama_context::llama_context(
cparams.embeddings_layer_inp.resize(hparams.n_layer() + 1, false);
embd_layer_inp.resize(hparams.n_layer() + 1);
- cparams.ctx_type = params.ctx_type;
- cparams.pooling_type = params.pooling_type;
+ cparams.ctx_type = params.ctx_type;
+ cparams.rope_scaling_type = params.rope_scaling_type;
+ cparams.pooling_type = params.pooling_type;
cparams.n_ctx = params.n_ctx == 0 ? hparams.n_ctx_train : params.n_ctx;
cparams.rope_freq_base = params.rope_freq_base == 0.0f ? hparams.rope_freq_base_train : params.rope_freq_base;
@@ -160,17 +161,16 @@ llama_context::llama_context(
}
}
- auto rope_scaling_type = params.rope_scaling_type;
- if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
- rope_scaling_type = hparams.rope_scaling_type_train;
+ if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED) {
+ cparams.rope_scaling_type = hparams.rope_scaling_type_train;
}
- if (rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
+ if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_NONE) {
cparams.rope_freq_scale = 1.0f; // never scale if scaling type is none
}
if (cparams.yarn_ext_factor < 0.0f) { // negative indicates 'not set'
- cparams.yarn_ext_factor = rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
+ cparams.yarn_ext_factor = cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN ? 1.0f : 0.0f;
}
if (cparams.yarn_ext_factor != 0) {
@@ -3734,6 +3734,14 @@ llama_context * llama_init_from_model(
try {
auto * ctx = new llama_context(*model, params);
+ const auto & cparams = ctx->get_cparams();
+
+ if (cparams.rope_scaling_type == LLAMA_ROPE_SCALING_TYPE_YARN && cparams.rope_freq_scale != model->hparams.rope_freq_scale_train) {
+ LLAMA_LOG_INFO("%s: custom YaRN scaling detected, re-adjusting n_ctx_train(%u)...\n", __func__, model->hparams.n_ctx_train);
+ model->hparams.n_ctx_train = cparams.n_ctx_orig_yarn / cparams.rope_freq_scale;
+ LLAMA_LOG_INFO("%s: n_ctx_train adjusted to %u\n", __func__, model->hparams.n_ctx_train);
+ }
+
return ctx;
} catch (const std::exception & err) {
LLAMA_LOG_ERROR("%s: failed to initialize the context: %s\n", __func__, err.what());
diff --git a/src/llama-cparams.h b/src/llama-cparams.h
index 574ce95920..b592de18c7 100644
--- a/src/llama-cparams.h
+++ b/src/llama-cparams.h
@@ -57,6 +57,7 @@ struct llama_cparams {
std::vector<bool> embeddings_layer_inp; // [n_layer()] extract input embeddings for layer
enum llama_context_type ctx_type;
+ enum llama_rope_scaling_type rope_scaling_type;
enum llama_pooling_type pooling_type;
ggml_backend_sched_eval_callback cb_eval;