LLM inference in C/C++
C++ ggml
spec: Add benchmark-only synthetic speculative acceptance options (#27711)
Gaurav Garg2bb9bddafad4
diff --git a/common/arg.cpp b/common/arg.cppindex dd92e4720b..403cc2b781 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -4132,6 +4132,38 @@ common_params_context common_params_parser_init(common_params & params, llama_exparams.speculative.draft.n_min = value;}).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));+ add_opt(common_arg(+ {"--spec-synth-len"}, "L",+ "target mean synthetic acceptance length, including the target token (benchmarking only)",+ [](common_params & params, const std::string & value) {+ const std::string text = string_strip(value);+ size_t pos = 0;+ const double length = std::stod(text, &pos);+ if (pos != text.size() || length == -1.0) {+ throw std::invalid_argument("invalid value");+ }+ params.speculative.synth_len = length;+ }+ ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN"));+ add_opt(common_arg(+ {"--spec-synth-rates"}, "P0,P1,...",+ "comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)",+ [](common_params & params, const std::string & value) {+ const auto values = string_split<std::string>(value, ',');+ std::vector<double> rates;+ rates.reserve(values.size());+ for (const auto & raw : values) {+ const std::string text = string_strip(raw);+ size_t pos = 0;+ const double rate = std::stod(text, &pos);+ if (pos != text.size()) {+ throw std::invalid_argument("invalid value");+ }+ rates.push_back(rate);+ }+ params.speculative.synth_rates = std::move(rates);+ }+ ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES"));add_opt(common_arg({"--spec-draft-p-split", "--draft-p-split"}, "P",diff --git a/common/common.h b/common/common.hindex 9593e10d9a..1cfb01e1ef 100644--- a/common/common.h+++ b/common/common.h@@ -370,6 +370,9 @@ struct common_params_speculative_ngram_cache {struct common_params_speculative {std::vector<enum common_speculative_type> types = { COMMON_SPECULATIVE_TYPE_NONE };+ double synth_len = -1.0;+ std::vector<double> synth_rates;+// used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft modelcommon_params_speculative_draft draft;@@ -384,6 +387,10 @@ struct common_params_speculative {return !draft.mparams.empty();}+ bool has_synth() const {+ return synth_len != -1.0 || !synth_rates.empty();+ }+uint32_t need_n_rs_seq() const {bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;diff --git a/common/speculative.cpp b/common/speculative.cppindex 4eef2212e7..393a73bc39 100644--- a/common/speculative.cpp+++ b/common/speculative.cpp@@ -14,6 +14,7 @@#include <algorithm>#include <cassert>+#include <cmath>#include <cstring>#include <iomanip>#include <map>@@ -138,6 +139,7 @@ struct common_speculative_impl {const common_speculative_type type;uint32_t n_seq;+ int32_t n_max; // maximum draft length after implementation-specific limitssize_t n_call_begin = 0; // number of times this implementation was called for refresh.size_t n_call_draft = 0; // number of times this implementation was called for generation.@@ -157,7 +159,7 @@ struct common_speculative_impl {int64_t t_draft_us = 0; // total time spent in generating drafts in this implementation in microseconds.int64_t t_accept_us = 0; // total time spent in accumulation of this implementation in microseconds.- common_speculative_impl(common_speculative_type type, uint32_t n_seq) : type(type), n_seq(n_seq) {}+ common_speculative_impl(common_speculative_type type, uint32_t n_seq, int32_t n_max) : type(type), n_seq(n_seq), n_max(n_max) {}virtual ~common_speculative_impl() = default;@@ -182,7 +184,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {std::vector<common_sampler_ptr> smpls;common_speculative_impl_draft_simple(const common_params_speculative & params, uint32_t n_seq)- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq)+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq, params.draft.n_max), params(params.draft){auto * ctx_dft = this->params.ctx_dft;@@ -452,7 +454,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {std::vector<float> g_embd_buf;common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq)- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq)+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq, params.draft.n_max), params(params.draft){SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n");@@ -937,7 +939,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)- : common_speculative_impl(type, n_seq)+ : common_speculative_impl(type, n_seq, params.draft.n_max), params(params.draft), is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK){@@ -983,6 +985,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {this->params.n_max = std::min(this->params.n_max, n_draft_max);this->params.n_min = std::min(this->params.n_min, n_draft_max);}+ this->n_max = this->params.n_max;batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq);@@ -1315,7 +1318,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {std::vector<std::vector<float>> chain_h;common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq)+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max), params(params.draft){auto * ctx_tgt = this->params.ctx_tgt;@@ -1382,6 +1385,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {c.reserve((size_t) (this->params.n_max + 1) * n_embd);}}+ this->n_max = this->params.n_max;pending_h.assign(n_seq, std::vector<float>(n_embd, 0.0f));@@ -1726,7 +1730,7 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl {common_speculative_impl_ngram_simple(const common_params_speculative & params, uint32_t n_seq,common_ngram_simple_config config)- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq)+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq, params.ngram_simple.size_m), params(params.ngram_simple), config(config){@@ -1770,7 +1774,7 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl {const common_ngram_map & config,uint32_t n_seq): common_speculative_impl(config.key_only ? COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K- : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq)+ : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq, config.size_value){for (uint32_t i = 0; i < n_seq; i++) {this->config.push_back(config);@@ -1841,7 +1845,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl {common_speculative_impl_ngram_mod(const common_params_speculative & params,uint32_t n_seq)- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq)+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq, params.ngram_mod.n_max), params(params.ngram_mod), mod(params.ngram_mod.n_match, 4*1024*1024), verbose(std::getenv("LLAMA_TRACE") != nullptr) {@@ -2017,7 +2021,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {const std::string & path_dynamic,bool save_dynamic,bool save_static)- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq)+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq, n_draft), params(params.ngram_cache), n_draft(n_draft), save_dynamic(save_dynamic)@@ -2138,6 +2142,8 @@ struct common_speculative {// which implementaion was used for a given seq_idstd::vector<common_speculative_impl *> impl_last;++ std::vector<double> synth_probs;};static common_ngram_map get_common_ngram_map(@@ -2316,6 +2322,101 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {return n_max;}+int32_t common_speculative_n_max(const common_speculative * spec) {+ int32_t n_max = 0;++ if (spec == nullptr) {+ return n_max;+ }++ for (const auto & impl : spec->impls) {+ n_max = std::max(n_max, std::max(0, impl->n_max));+ }++ return n_max;+}++std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max) {+ const bool has_length = spec->synth_len != -1.0;+ const bool has_rates = !spec->synth_rates.empty();++ if (!has_length && !has_rates) {+ return {};+ }+ if (has_length && has_rates) {+ throw std::invalid_argument("synthetic acceptance length and rates are mutually exclusive");+ }++ if (n_max <= 0) {+ throw std::invalid_argument("synthetic acceptance requires at least one speculative token");+ }++ if (has_rates) {+ const auto & rates = spec->synth_rates;+ if (rates.size() != (size_t) n_max) {+ throw std::invalid_argument(string_format(+ "synthetic acceptance rates must contain %d values, got %zu", n_max, rates.size()));+ }++ for (size_t i = 0; i < rates.size(); ++i) {+ if (!std::isfinite(rates[i]) || rates[i] < 0.0 || rates[i] > 1.0) {+ throw std::invalid_argument("synthetic acceptance rates must be finite and within [0, 1]");+ }+ if (i > 0 && rates[i] > rates[i - 1]) {+ throw std::invalid_argument("synthetic acceptance rates must be monotonically non-increasing");+ }+ }++ return rates;+ }++ const double length = spec->synth_len;+ const double length_max = (double) n_max + 1.0;+ if (!std::isfinite(length) || length < 1.0 || length > length_max) {+ throw std::invalid_argument(string_format(+ "synthetic acceptance length must be finite and within [1, %.0f]", length_max));+ }++ double p = 0.0;+ if (length == length_max) {+ p = 1.0;+ } else if (length > 1.0) {+ double p_min = 0.0;+ double p_max = 1.0;+ for (int i = 0; i < 32; ++i) {+ const double p_mid = 0.5 * (p_min + p_max);+ double sum = 0.0;+ double term = p_mid;+ for (int32_t j = 0; j < n_max; ++j) {+ sum += term;+ term *= p_mid;+ }++ if (sum < length - 1.0) {+ p_min = p_mid;+ } else {+ p_max = p_mid;+ }+ }+ p = 0.5 * (p_min + p_max);+ }++ std::vector<double> rates;+ rates.reserve(n_max);+ double rate = p;+ for (int32_t i = 0; i < n_max; ++i) {+ rates.push_back(rate);+ rate *= p;+ }++ return rates;+}++const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec) {+ GGML_ASSERT(spec);+ return spec->synth_probs;+}+common_params common_base_params_to_speculative(const common_params & params) {const bool has_draft = params.speculative.has_dft();@@ -2568,13 +2669,39 @@ common_speculative * common_speculative_init(common_params_speculative & params,return nullptr;}- auto * result = new common_speculative {- /* .dparams = */ common_speculative_draft_params_vec(n_seq),- /* .impls = */ std::move(impls),- /* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr)- };+ common_speculative_ptr result(new common_speculative {+ /* .dparams = */ common_speculative_draft_params_vec(n_seq),+ /* .impls = */ std::move(impls),+ /* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr),+ /* .synth_probs = */ {},+ });- return result;+ const int32_t n_max_configured = common_speculative_n_max(¶ms);+ const int32_t n_max_effective = common_speculative_n_max(result.get());+ const auto rates = common_speculative_synth_rates_resolve(¶ms, n_max_effective);++ std::vector<std::string> rates_str;+ rates_str.reserve(rates.size());+ result->synth_probs.reserve(rates.size());+ double rate_prev = 1.0;+ double acceptance_length = 1.0;+ for (const double rate : rates) {+ result->synth_probs.push_back(rate_prev > 0.0 ? rate / rate_prev : 0.0);+ rates_str.push_back(string_format("%.6g", rate));+ rate_prev = rate;+ acceptance_length += rate;+ }+ if (!result->synth_probs.empty()) {+ SPC_WRN("%s", "synthetic speculative acceptance is enabled for benchmarking; generated output is not valid\n");+ if (n_max_effective != n_max_configured) {+ SPC_WRN("synthetic acceptance draft limit was reduced from %d to %d by the initialized speculative implementations\n",+ n_max_configured, n_max_effective);+ }+ SPC_INF("synthetic acceptance: n_max = %zu, mean length = %.6f, rates = [%s]\n",+ rates.size(), acceptance_length, string_join(rates_str, ", ").c_str());+ }++ return result.release();}void common_speculative_free(common_speculative * spec) {diff --git a/common/speculative.h b/common/speculative.hindex 12ae31b7de..22505891f7 100644--- a/common/speculative.h+++ b/common/speculative.h@@ -26,6 +26,15 @@ std::string common_speculative_type_to_str(enum common_speculative_type type);// return the max number of draft tokens based on the speculative parametersint32_t common_speculative_n_max(const common_params_speculative * spec);+// return the max number of draft tokens from the initialized implementations+int32_t common_speculative_n_max(const common_speculative * spec);++// validate and resolve the unconditional synthetic acceptance rates+std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max);++// return the conditional synthetic acceptance probabilities+const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec);+common_params common_base_params_to_speculative(const common_params & params);struct common_speculative_output_limits {diff --git a/docs/speculative.md b/docs/speculative.mdindex 0f9f8a3d97..ffb1e34c7f 100644--- a/docs/speculative.md+++ b/docs/speculative.md@@ -212,6 +212,15 @@ Use `--backend-sampling` to run supported target-model samplers on the model bacUnsupported samplers and device layouts fall back to CPU sampling. Tensor split mode does not support backend sampling. A fixed seed produces repeatable random draws, but stochastic CPU and backend sampling can still select different tokens because floating-point operations can differ between implementations and devices. Use greedy sampling when exact output matching is required.+### Synthetic Acceptance++`llama-server` and `llama-cli` can replace normal speculative verification with synthetic decisions for benchmarking. The generated output is not valid model output because accepted draft tokens do not have to match the target model.++Use exactly one of these options:++- `--spec-synth-rates P0,P1,...` sets unconditional per-position acceptance probabilities. Entry `i` is the probability that the first `i+1` draft tokens are all accepted. The number of entries must match the effective maximum draft length. Values must be finite, within `[0, 1]`, and monotonically non-increasing.+- `--spec-synth-len L` sets the target mean acceptance length, including the target token. For `K` maximum draft tokens, `L` must be within `[1, K+1]`. The server finds a constant conditional probability `p` such that `p + p^2 + ... + p^K = L - 1`, then uses unconditional rates `[p, p^2, ..., p^K]`.+### General Speculative Parameters```diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cppindex ba58f852eb..e0907631ab 100644--- a/tests/test-arg-parser.cpp+++ b/tests/test-arg-parser.cpp@@ -4,6 +4,7 @@#include "llama.h"#include "speculative.h"+#include <cmath>#include <limits>#include <string>#include <vector>@@ -34,6 +35,62 @@ static void test(void) {std::numeric_limits<int32_t>::max(),std::numeric_limits<int32_t>::max());+ {+ common_params_speculative spec;+ spec.synth_len = 3.4;++ auto assert_invalid = [](const common_params_speculative & value, int32_t n_max) {+ try {+ common_speculative_synth_rates_resolve(&value, n_max);+ assert(false);+ } catch (const std::invalid_argument &) {+ }+ };++ const auto rates = common_speculative_synth_rates_resolve(&spec, 4);+ assert(rates.size() == 4);+ assert(std::abs(rates[0] - 0.80581) < 1e-5);+ assert(std::abs(rates[1] - 0.64933) < 1e-5);+ assert(std::abs(rates[2] - 0.52323) < 1e-5);+ assert(std::abs(rates[3] - 0.42163) < 1e-5);+ assert(std::abs(1.0 + rates[0] + rates[1] + rates[2] + rates[3] - 3.4) < 1e-8);++ spec.synth_len = 1.0;+ assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({0.0, 0.0, 0.0, 0.0}));++ spec.synth_len = 5.0;+ assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({1.0, 1.0, 1.0, 1.0}));++ spec.synth_len = 5.1;+ assert_invalid(spec, 4);++ spec.synth_len = std::numeric_limits<double>::quiet_NaN();+ assert_invalid(spec, 4);++ spec.synth_len = 0.0;+ assert_invalid(spec, 4);++ spec.synth_len = -1.0;+ spec.synth_rates = {0.8, 0.6, 0.4};+ assert_invalid(spec, 4);++ spec.synth_rates = {0.8, 0.6, 0.4, 0.2};+ assert(common_speculative_synth_rates_resolve(&spec, 4) == spec.synth_rates);++ spec.synth_rates = {0.8, 0.9, 0.4, 0.2};+ assert_invalid(spec, 4);++ spec.synth_rates = {0.8, std::numeric_limits<double>::quiet_NaN(), 0.4, 0.2};+ assert_invalid(spec, 4);++ spec.synth_rates = {0.8, 0.6, 0.4, -0.2};+ assert_invalid(spec, 4);++ spec.synth_rates = {0.8, 0.6, 0.4, 0.2};+ spec.synth_len = 3.0;+ assert_invalid(spec, 4);+ }+{common_params base;base.n_parallel = 4;@@ -197,6 +254,26 @@ static void test(void) {assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));assert(params.speculative.draft.n_max == 123);+ {+ common_params synth_params;+ argv = {"binary_name", "--spec-synth-len", "3.4"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));+ assert(synth_params.speculative.synth_len == 3.4);+ }++ {+ common_params synth_params;+ argv = {"binary_name", "--spec-synth-rates", "0.8,0.6,0.2"};+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));+ assert(synth_params.speculative.synth_rates == std::vector<double>({0.8, 0.6, 0.2}));+ }++ {+ common_params synth_params;+ argv = {"binary_name", "--spec-synth-len", "3.4x"};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));+ }+argv = {"binary_name", "-lm", "none"};assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));assert(params.load_mode == LLAMA_LOAD_MODE_NONE);diff --git a/tools/cli/README.md b/tools/cli/README.mdindex 0fba70a90e..e663cfa3b2 100644--- a/tools/cli/README.md+++ b/tools/cli/README.md@@ -200,6 +200,8 @@| `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) || `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) || `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |+| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_LEN) |+| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_RATES) || `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) || `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) || `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) |diff --git a/tools/server/README.md b/tools/server/README.mdindex f49cdb2729..6fd27f1389 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -259,6 +259,8 @@ For the full list of features, please refer to [server's changelog](https://gith| `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) || `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) || `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |+| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_LEN) |+| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_RATES) || `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) || `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) || `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) |diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 9fdfcae563..e6c991f7cf 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -23,6 +23,7 @@#include <exception>#include <memory>#include <filesystem>+#include <random>#include <utility>#include <fstream>@@ -51,6 +52,50 @@ static common_speculative_output_limits server_output_limits(const common_paramsreturn result;}+// synthetic draft verification for benchmarking - accept draft tokens at random instead of by match with the target+// on replay the draft was already accepted before a context checkpoint restore, so repeat the same decisions+static std::vector<llama_token> server_sample_and_accept_synth(+ common_sampler * smpl,+ llama_context * ctx,+ const std::vector<int32_t> & idxs,+ const llama_tokens & draft,+ const std::vector<double> & synth_probs,+ std::mt19937 & rng,+ bool is_replay) {+ GGML_ASSERT(idxs.size() == draft.size() + 1);+ GGML_ASSERT(synth_probs.size() >= draft.size());++ std::vector<llama_token> result;+ result.reserve(idxs.size());++ const llama_vocab * vocab = llama_model_get_vocab(llama_get_model(ctx));+ std::uniform_real_distribution<double> dist(0.0, 1.0);+ for (size_t i = 0; i < draft.size(); ++i) {+ const llama_token id = common_sampler_sample(smpl, ctx, idxs[i]);+ const bool accept = is_replay || dist(rng) < synth_probs[i];+ // do not accept a drafted EOG token - it would end the generation early+ // on replay the last token is from the target and can be EOG, so skip this check+ if (accept && (is_replay || !llama_vocab_is_eog(vocab, draft[i]))) {+ // synthetic draft tokens do not advance grammar or reasoning state+ // the last replay token is from the target and must advance both+ const bool is_replay_target = is_replay && i + 1 == draft.size();+ common_sampler_accept(smpl, draft[i], is_replay_target);+ result.push_back(draft[i]);+ continue;+ }++ common_sampler_accept(smpl, id, true);+ result.push_back(id);+ return result;+ }++ const llama_token id = common_sampler_sample(smpl, ctx, idxs[draft.size()]);+ common_sampler_accept(smpl, id, true);+ result.push_back(id);++ return result;+}+// state diagram: https://github.com/ggml-org/llama.cpp/pull/9283enum slot_state {SLOT_STATE_IDLE,@@ -211,6 +256,7 @@ struct server_slot {std::vector<int32_t> spec_i_batch;common_prompt_checkpoint spec_ckpt;bool spec_is_replay = false;+ std::mt19937 spec_synth_rng;// TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837@@ -1194,6 +1240,9 @@ private:spec.reset(common_speculative_init(params_base.speculative, params_base.n_parallel));} catch (const std::exception & e) {SRV_ERR("failed to initialize speculative decoding context: %s\n", e.what());+ if (params_base.speculative.has_synth()) {+ return false;+ }}}@@ -1209,6 +1258,11 @@ private:model_dft = nullptr;}+ if (!spec && params_base.speculative.has_synth()) {+ SRV_ERR("%s", "synthetic acceptance requires an initialized speculative decoding context\n");+ return false;+ }+for (int i = 0; i < params_base.n_parallel; i++) {server_slot & slot = slots[i];@@ -1717,6 +1771,13 @@ private:SLT_TRC(slot, "sampler chain: %s\n", common_sampler_print(slot.smpl.get()).c_str());SLT_TRC(slot, "sampler params: \n%s\n", task.params.sampling.print().c_str());++ if (spec && !common_speculative_get_synth_probs(spec.get()).empty()) {+ const uint32_t seed = task.params.sampling.seed == LLAMA_DEFAULT_SEED+ ? std::random_device{}()+ : task.params.sampling.seed;+ slot.spec_synth_rng.seed(seed);+ }} else {slot.smpl.reset();}@@ -3802,7 +3863,12 @@ private:common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);- auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);+ const auto & synth_probs = common_speculative_get_synth_probs(spec.get());+ auto accepted = synth_probs.empty()+ ? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft)+ : server_sample_and_accept_synth(+ slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft,+ synth_probs, slot.spec_synth_rng, slot.spec_is_replay);slot.spec_i_batch.clear();GGML_ASSERT(accepted.size() >= 1);@@ -3868,7 +3934,7 @@ private:auto & n_accepted_per_pos = slot.n_accepted_per_pos;if (n_accepted_per_pos.empty()) {- n_accepted_per_pos.resize(common_speculative_n_max(¶ms_base.speculative), 0);+ n_accepted_per_pos.resize(common_speculative_n_max(spec.get()), 0);}for (size_t i = 0; i < n_accepted && i < n_accepted_per_pos.size(); ++i) {n_accepted_per_pos[i]++;diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.pyindex 5837195006..22b523954e 100644--- a/tools/server/tests/unit/test_speculative.py+++ b/tools/server/tests/unit/test_speculative.py@@ -52,6 +52,18 @@ def test_with_and_without_draft():assert tokens_no_draft == tokens_draft+ server.stop()+ create_server()+ assert server.spec_draft_n_max is not None+ server.spec_synth_rates = [0.0] * server.spec_draft_n_max+ server.start()+ res = server.make_request("POST", "/completion", data=request)++ assert res.status_code == 200+ assert res.body["timings"]["draft_n"] > 0+ assert res.body["timings"]["draft_n_accepted"] == 0+ assert res.body["tokens"] == tokens_no_draft+def test_different_draft_min_draft_max():global server@@ -80,6 +92,66 @@ def test_different_draft_min_draft_max():last_content = res.body["content"]+def test_synth_is_deterministic():+ global server+ assert server.spec_draft_n_max is not None+ server.spec_synth_rates = [0.75 ** (i + 1) for i in range(server.spec_draft_n_max)]+ server.start()++ request = {+ "prompt": "I believe the meaning of life is",+ "temperature": 0.2,+ "top_k": 5,+ "seed": 4242,+ "n_predict": 32,+ }+ responses = [server.make_request("POST", "/completion", data=request) for _ in range(2)]++ for res in responses:+ assert res.status_code == 200+ assert res.body["timings"]["draft_n"] > 0+ assert responses[0].body["timings"]["draft_n"] == responses[1].body["timings"]["draft_n"]+ assert responses[0].body["timings"]["draft_n_accepted"] == responses[1].body["timings"]["draft_n_accepted"]+++def test_synth_ignores_target_tokens():+ global server+ assert server.spec_draft_n_max is not None+ server.spec_synth_rates = [1.0] * server.spec_draft_n_max+ server.start()++ res = server.make_request("POST", "/completion", data={+ "prompt": "I believe the meaning of life is",+ "temperature": 0.0,+ "seed": 4242,+ "n_predict": 32,+ })++ assert res.status_code == 200+ assert res.body["timings"]["draft_n"] > 0+ assert res.body["timings"]["draft_n_accepted"] == res.body["timings"]["draft_n"]++ res = server.make_request("POST", "/completion", data={+ "prompt": "I believe the meaning of life is",+ "temperature": 0.0,+ "seed": 4242,+ "n_predict": 6,+ "grammar": 'root ::= "a"{5,5}',+ })+ assert res.status_code == 200, res.body++ res = server.make_request("POST", "/completion", data={+ "prompt": "Respond with only: OK",+ "temperature": 0.0,+ "seed": 4242,+ "n_predict": 64,+ "ignore_eos": True,+ })+ assert res.status_code == 200, res.body+ assert res.body["tokens_predicted"] == 64+ assert res.body["stop_type"] == "limit"++def test_slot_ctx_not_exceeded():global serverserver.n_ctx = 256diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.pyindex a0d2dfa3c5..5a4f31a53e 100644--- a/tools/server/tests/utils.py+++ b/tools/server/tests/utils.py@@ -99,6 +99,8 @@ class ServerProcess:spec_type: str | None = Nonespec_draft_n_min: int | None = Nonespec_draft_n_max: int | None = None+ spec_synth_len: float | None = None+ spec_synth_rates: List[float] | None = Noneno_ui: bool | None = Nonejinja: bool | None = Nonereasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None@@ -245,6 +247,11 @@ class ServerProcess:server_args.extend(["--spec-draft-n-max", self.spec_draft_n_max])if self.spec_draft_n_min:server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min])+ if self.spec_synth_len is not None:+ server_args.extend(["--spec-synth-len", self.spec_synth_len])+ if self.spec_synth_rates is not None:+ rates = ",".join(str(rate) for rate in self.spec_synth_rates)+ server_args.extend(["--spec-synth-rates", rates])if self.no_ui:server_args.append("--no-ui")if self.no_models_autoload: