Skip to content

LLM inference in C/C++

C++ ggml

speculative : fix n_outputs_max and remove draft-simple auto-enable (#23988)

Georgi Gerganov5dcb71166686
diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml
index f42b30d5cd..5a02cc15ad 100644
--- a/.github/workflows/server.yml
+++ b/.github/workflows/server.yml
@@ -102,7 +102,6 @@ jobs:
- name: Tests
id: server_integration_tests
- if: ${{ !github.event.pull_request }}
run: |
cd tools/server/tests
pytest -v -x -m "not slow"
@@ -116,7 +115,6 @@ jobs:
- name: Tests (Backend sampling)
id: server_integration_tests_backend_sampling
- if: ${{ !github.event.pull_request }}
run: |
cd tools/server/tests
export LLAMA_ARG_BACKEND_SAMPLING=1
@@ -169,7 +167,6 @@ jobs:
- name: Tests
id: server_integration_tests
- if: ${{ !github.event.pull_request }}
run: |
cd tools/server/tests
$env:PYTHONIOENCODING = ":replace"
diff --git a/common/arg.cpp b/common/arg.cpp
index e0f6c60660..2df446d9d6 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -1041,11 +1041,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
// we define here to make sure it's included in llama-gen-docs
if (ex == LLAMA_EXAMPLE_COMPLETION) {
params.use_jinja = false; // disable jinja by default
-
} else if (ex == LLAMA_EXAMPLE_MTMD) {
params.use_jinja = false; // disable jinja by default
params.sampling.temp = 0.2; // lower temp by default for better quality
-
} else if (ex == LLAMA_EXAMPLE_SERVER) {
params.n_parallel = -1; // auto by default
}
@@ -1066,7 +1064,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
sampler_type_names.pop_back(); // remove last semicolon
}
-
/**
* filter options by example
* rules:
@@ -1080,7 +1077,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
}
};
-
add_opt(common_arg(
{"-h", "--help", "--usage"},
"print usage and exit",
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 253a5ececb..73830fda6c 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -1317,6 +1317,40 @@ static uint32_t common_get_enabled_speculative_configs(const std::vector<common_
return result;
}
+int32_t common_speculative_n_max(const common_params_speculative * spec) {
+ int32_t n_max = 0;
+
+ for (const auto type : spec->types) {
+ switch (type) {
+ case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE:
+ case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:
+ case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:
+ n_max = std::max(n_max, std::max(0, spec->draft.n_max));
+ break;
+ case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
+ n_max = std::max(n_max, (int32_t) spec->ngram_simple.size_m);
+ break;
+ case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
+ n_max = std::max(n_max, (int32_t) spec->ngram_map_k.size_m);
+ break;
+ case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:
+ n_max = std::max(n_max, (int32_t) spec->ngram_map_k4v.size_m);
+ break;
+ case COMMON_SPECULATIVE_TYPE_NGRAM_MOD:
+ n_max = std::max(n_max, std::max(0, spec->ngram_mod.n_max));
+ break;
+ case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
+ n_max = std::max(n_max, (int32_t) 8);
+ break;
+ case COMMON_SPECULATIVE_TYPE_NONE:
+ case COMMON_SPECULATIVE_TYPE_COUNT:
+ break;
+ }
+ }
+
+ return n_max;
+}
+
// initialization of the speculative decoding system
//
common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) {
@@ -1325,8 +1359,6 @@ common_speculative * common_speculative_init(common_params_speculative & params,
{
uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types);
- bool has_draft_model_path = !params.draft.mparams.path.empty();
-
bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE));
bool has_draft_eagle3 = false; // TODO PR-18039: if params.speculative.eagle3
bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr;
@@ -1359,16 +1391,6 @@ common_speculative * common_speculative_init(common_params_speculative & params,
if (has_ngram_cache) {
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params));
}
- if (has_draft_simple) {
- if (!has_draft_model_path) {
- LOG_WRN("%s: draft model is not specified - cannot use 'draft' type\n", __func__);
- has_draft_simple = false;
- }
- } else if (has_draft_model_path && !has_mtp && !has_draft_eagle3) {
- LOG_WRN("%s: draft model is specified but 'draft' speculative type is not explicitly enabled - enabling it\n", __func__);
- has_draft_simple = true;
- }
-
if (has_draft_simple) {
configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params));
}
diff --git a/common/speculative.h b/common/speculative.h
index f24bac79ed..deba7dac72 100644
--- a/common/speculative.h
+++ b/common/speculative.h
@@ -20,6 +20,9 @@ enum common_speculative_type common_speculative_type_from_name(const std::string
// convert type to string
std::string common_speculative_type_to_str(enum common_speculative_type type);
+// return the max number of draft tokens based on the speculative parameters
+int32_t common_speculative_n_max(const common_params_speculative * spec);
+
common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq);
void common_speculative_free(common_speculative * spec);
diff --git a/src/llama-context.cpp b/src/llama-context.cpp
index a7dfb24924..946c2d0ef3 100644
--- a/src/llama-context.cpp
+++ b/src/llama-context.cpp
@@ -229,6 +229,7 @@ llama_context::llama_context(
LLAMA_LOG_INFO("%s: freq_base = %.1f\n", __func__, cparams.rope_freq_base);
LLAMA_LOG_INFO("%s: freq_scale = %g\n", __func__, cparams.rope_freq_scale);
LLAMA_LOG_INFO("%s: n_rs_seq = %u\n", __func__, cparams.n_rs_seq);
+ LLAMA_LOG_INFO("%s: n_outputs_max = %u\n", __func__, cparams.n_outputs_max);
if (cparams.n_ctx_seq < hparams.n_ctx_train) {
LLAMA_LOG_WARN("%s: n_ctx_seq (%u) < n_ctx_train (%u) -- the full capacity of the model will not be utilized\n",
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index a5a6882545..44fca83c6b 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -45,35 +45,7 @@ static uint32_t server_n_outputs_max(const common_params & params) {
return n_batch;
}
- uint32_t n_outputs_per_seq = 1;
-
- for (const auto type : params.speculative.types) {
- switch (type) {
- case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE:
- case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3:
- case COMMON_SPECULATIVE_TYPE_DRAFT_MTP:
- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + std::max(0, params.speculative.draft.n_max));
- break;
- case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE:
- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_simple.size_m);
- break;
- case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K:
- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_map_k.size_m);
- break;
- case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V:
- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + params.speculative.ngram_map_k4v.size_m);
- break;
- case COMMON_SPECULATIVE_TYPE_NGRAM_MOD:
- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + std::max(0, params.speculative.ngram_mod.n_max));
- break;
- case COMMON_SPECULATIVE_TYPE_NGRAM_CACHE:
- n_outputs_per_seq = std::max<uint32_t>(n_outputs_per_seq, 1 + 8);
- break;
- case COMMON_SPECULATIVE_TYPE_NONE:
- case COMMON_SPECULATIVE_TYPE_COUNT:
- break;
- }
- }
+ const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(&params.speculative);
const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq;
@@ -862,9 +834,7 @@ private:
measure_model_bytes = false;
}
- if (!has_draft) {
- params_dft.n_outputs_max = params_base.n_parallel;
- }
+ params_dft.n_outputs_max = params_base.n_parallel;
auto mparams_dft = common_model_params_to_llama(params_dft);
auto cparams_dft = common_context_params_to_llama(params_dft);