LLM inference in C/C++
C++ ggml
speculative : fix n_outputs_max and remove draft-simple auto-enable (#23988)
Georgi Gerganov5dcb71166686
diff --git a/.github/workflows/server.yml b/.github/workflows/server.ymlindex f42b30d5cd..5a02cc15ad 100644--- a/.github/workflows/server.yml+++ b/.github/workflows/server.yml@@ -102,7 +102,6 @@ jobs:- name: Testsid: server_integration_tests- if: ${{ !github.event.pull_request }}run: |cd tools/server/testspytest -v -x -m "not slow"@@ -116,7 +115,6 @@ jobs:- name: Tests (Backend sampling)id: server_integration_tests_backend_sampling- if: ${{ !github.event.pull_request }}run: |cd tools/server/testsexport LLAMA_ARG_BACKEND_SAMPLING=1@@ -169,7 +167,6 @@ jobs:- name: Testsid: server_integration_tests- if: ${{ !github.event.pull_request }}run: |cd tools/server/tests$env:PYTHONIOENCODING = ":replace"diff --git a/common/arg.cpp b/common/arg.cppindex e0f6c60660..2df446d9d6 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -1041,11 +1041,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex// we define here to make sure it's included in llama-gen-docsif (ex == LLAMA_EXAMPLE_COMPLETION) {params.use_jinja = false; // disable jinja by default-} else if (ex == LLAMA_EXAMPLE_MTMD) {params.use_jinja = false; // disable jinja by defaultparams.sampling.temp = 0.2; // lower temp by default for better quality-} else if (ex == LLAMA_EXAMPLE_SERVER) {params.n_parallel = -1; // auto by default}@@ -1066,7 +1064,6 @@ common_params_context common_params_parser_init(common_params & params, llama_exsampler_type_names.pop_back(); // remove last semicolon}-/*** filter options by example* rules:@@ -1080,7 +1077,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex}};-add_opt(common_arg({"-h", "--help", "--usage"},"print usage and exit",diff --git a/common/speculative.cpp b/common/speculative.cppindex 253a5ececb..73830fda6c 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -1317,6 +1317,40 @@ static uint32_t common_get_enabled_speculative_configs(const std::vector<common_return result;}+int32_t common_speculative_n_max(const common_params_speculative * spec) {+ int32_t n_max = 0;++ for (const auto type : spec->types) {+ switch (type) {+ case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE:+ case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:+ case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:+ n_max = std::max(n_max, std::max(0, spec->draft.n_max));+ break;+ case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:+ n_max = std::max(n_max, (int32_t) spec->ngram_simple.size_m);+ break;+ case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:+ n_max = std::max(n_max, (int32_t) spec->ngram_map_k.size_m);+ break;+ case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:+ n_max = std::max(n_max, (int32_t) spec->ngram_map_k4v.size_m);+ break;+ case COMMON_SPECULATIVE_TYPE_NGRAM_MOD:+ n_max = std::max(n_max, std::max(0, spec->ngram_mod.n_max));+ break;+ case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:+ n_max = std::max(n_max, (int32_t) 8);+ break;+ case COMMON_SPECULATIVE_TYPE_NONE:+ case COMMON_SPECULATIVE_TYPE_COUNT:+ break;+ }+ }++ return n_max;+}+// initialization of the speculative decoding system//common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) {@@ -1325,8 +1359,6 @@ common_speculative * common_speculative_init(common_params_speculative & params,{uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types);- bool has_draft_model_path = !params.draft.mparams.path.empty();-bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE));bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;@@ -1359,16 +1391,6 @@ common_speculative * common_speculative_init(common_params_speculative & params,if (has_ngram_cache) {configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params));}- if (has_draft_simple) {- if (!has_draft_model_path) {- LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__);- has_draft_simple = false;- }- } else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) {- LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__);- has_draft_simple = true;- }-if (has_draft_simple) {configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params));}diff --git a/common/speculative.h b/common/speculative.hindex f24bac79ed..deba7dac72 100644--- a/common/speculative.h+++ b/common/speculative.h@@ -20,6 +20,9 @@ enum common_speculative_type common_speculative_type_from_name(const std::string// convert type to stringstd::string common_speculative_type_to_str(enum common_speculative_type type);+// return the max number of draft tokens based on the speculative parameters+int32_t common_speculative_n_max(const common_params_speculative * spec);+common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq);void common_speculative_free(common_speculative * spec);diff --git a/src/llama-context.cpp b/src/llama-context.cppindex a7dfb24924..946c2d0ef3 100644--- a/src/llama-context.cpp+++ b/src/llama-context.cpp@@ -229,6 +229,7 @@ llama_context::llama_context(LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq);+ LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max);if (cparams.n_ctx_seq < hparams.n_ctx_train) {LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex a5a6882545..44fca83c6b 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -45,35 +45,7 @@ static uint32_t server_n_outputs_max(const common_params & params) {return n_batch;}- uint32_t n_outputs_per_seq = 1;-- for (const auto type : params.speculative.types) {- switch (type) {- case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE:- case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:- case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + std::max(0, params.speculative.draft.n_max));- break;- case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_simple.size_m);- break;- case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_map_k.size_m);- break;- case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_map_k4v.size_m);- break;- case COMMON_SPECULATIVE_TYPE_NGRAM_MOD:- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + std::max(0, params.speculative.ngram_mod.n_max));- break;- case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + 8);- break;- case COMMON_SPECULATIVE_TYPE_NONE:- case COMMON_SPECULATIVE_TYPE_COUNT:- break;- }- }+ const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(¶ms.speculative);const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq;@@ -862,9 +834,7 @@ private:measure_model_bytes = false;}- if (!has_draft) {- params_dft.n_outputs_max = params_base.n_parallel;- }+ params_dft.n_outputs_max = params_base.n_parallel;auto mparams_dft = common_model_params_to_llama(params_dft);auto cparams_dft = common_context_params_to_llama(params_dft);