Skip to content

LLM inference in C/C++

C++ ggml

spec: Add benchmark-only synthetic speculative acceptance options (#27711)

Gaurav Garg2bb9bddafad4
diff --git a/common/arg.cpp b/common/arg.cpp
index dd92e4720b..403cc2b781 100644
--- a/common/arg.cpp
+++ b/common/arg.cpp
@@ -4132,6 +4132,38 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.speculative.draft.n_min = value;
}
).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MIN"));
+ add_opt(common_arg(
+ {"--spec-synth-len"}, "L",
+ "target mean synthetic acceptance length, including the target token (benchmarking only)",
+ [](common_params & params, const std::string & value) {
+ const std::string text = string_strip(value);
+ size_t pos = 0;
+ const double length = std::stod(text, &pos);
+ if (pos != text.size() || length == -1.0) {
+ throw std::invalid_argument("invalid value");
+ }
+ params.speculative.synth_len = length;
+ }
+ ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_LEN"));
+ add_opt(common_arg(
+ {"--spec-synth-rates"}, "P0,P1,...",
+ "comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)",
+ [](common_params & params, const std::string & value) {
+ const auto values = string_split<std::string>(value, ',');
+ std::vector<double> rates;
+ rates.reserve(values.size());
+ for (const auto & raw : values) {
+ const std::string text = string_strip(raw);
+ size_t pos = 0;
+ const double rate = std::stod(text, &pos);
+ if (pos != text.size()) {
+ throw std::invalid_argument("invalid value");
+ }
+ rates.push_back(rate);
+ }
+ params.speculative.synth_rates = std::move(rates);
+ }
+ ).set_spec().set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_SYNTH_RATES"));
add_opt(common_arg(
{"--spec-draft-p-split", "--draft-p-split"}, "P",
diff --git a/common/common.h b/common/common.h
index 9593e10d9a..1cfb01e1ef 100644
--- a/common/common.h
+++ b/common/common.h
@@ -370,6 +370,9 @@ struct common_params_speculative_ngram_cache {
struct common_params_speculative {
std::vector<enum common_speculative_type> types = { COMMON_SPECULATIVE_TYPE_NONE };
+ double synth_len = -1.0;
+ std::vector<double> synth_rates;
+
// used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model
common_params_speculative_draft draft;
@@ -384,6 +387,10 @@ struct common_params_speculative {
return !draft.mparams.empty();
}
+ bool has_synth() const {
+ return synth_len != -1.0 || !synth_rates.empty();
+ }
+
uint32_t need_n_rs_seq() const {
bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {
return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;
diff --git a/common/speculative.cpp b/common/speculative.cpp
index 4eef2212e7..393a73bc39 100644
--- a/common/speculative.cpp
+++ b/common/speculative.cpp
@@ -14,6 +14,7 @@
#include <algorithm>
#include <cassert>
+#include <cmath>
#include <cstring>
#include <iomanip>
#include <map>
@@ -138,6 +139,7 @@ struct common_speculative_impl {
const common_speculative_type type;
uint32_t n_seq;
+ int32_t n_max; // maximum draft length after implementation-specific limits
size_t n_call_begin = 0; // number of times this implementation was called for refresh.
size_t n_call_draft = 0; // number of times this implementation was called for generation.
@@ -157,7 +159,7 @@ struct common_speculative_impl {
int64_t t_draft_us = 0; // total time spent in generating drafts in this implementation in microseconds.
int64_t t_accept_us = 0; // total time spent in accumulation of this implementation in microseconds.
- common_speculative_impl(common_speculative_type type, uint32_t n_seq) : type(type), n_seq(n_seq) {}
+ common_speculative_impl(common_speculative_type type, uint32_t n_seq, int32_t n_max) : type(type), n_seq(n_seq), n_max(n_max) {}
virtual ~common_speculative_impl() = default;
@@ -182,7 +184,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl {
std::vector<common_sampler_ptr> smpls;
common_speculative_impl_draft_simple(const common_params_speculative & params, uint32_t n_seq)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq)
+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, n_seq, params.draft.n_max)
, params(params.draft)
{
auto * ctx_dft = this->params.ctx_dft;
@@ -452,7 +454,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl {
std::vector<float> g_embd_buf;
common_speculative_impl_draft_eagle3(const common_params_speculative & params, uint32_t n_seq)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq)
+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq, params.draft.n_max)
, params(params.draft)
{
SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n");
@@ -937,7 +939,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq,
common_speculative_type type = COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)
- : common_speculative_impl(type, n_seq)
+ : common_speculative_impl(type, n_seq, params.draft.n_max)
, params(params.draft)
, is_dspark(type == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)
{
@@ -983,6 +985,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl {
this->params.n_max = std::min(this->params.n_max, n_draft_max);
this->params.n_min = std::min(this->params.n_min, n_draft_max);
}
+ this->n_max = this->params.n_max;
batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq);
batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq);
@@ -1315,7 +1318,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
std::vector<std::vector<float>> chain_h;
common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq)
+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq, params.draft.n_max)
, params(params.draft)
{
auto * ctx_tgt = this->params.ctx_tgt;
@@ -1382,6 +1385,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
c.reserve((size_t) (this->params.n_max + 1) * n_embd);
}
}
+ this->n_max = this->params.n_max;
pending_h.assign(n_seq, std::vector<float>(n_embd, 0.0f));
@@ -1726,7 +1730,7 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl {
common_speculative_impl_ngram_simple(
const common_params_speculative & params, uint32_t n_seq,
common_ngram_simple_config config)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq)
+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, n_seq, params.ngram_simple.size_m)
, params(params.ngram_simple)
, config(config)
{
@@ -1770,7 +1774,7 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl {
const common_ngram_map & config,
uint32_t n_seq)
: common_speculative_impl(config.key_only ? COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K
- : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq)
+ : COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, n_seq, config.size_value)
{
for (uint32_t i = 0; i < n_seq; i++) {
this->config.push_back(config);
@@ -1841,7 +1845,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl {
common_speculative_impl_ngram_mod(
const common_params_speculative & params,
uint32_t n_seq)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq)
+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, n_seq, params.ngram_mod.n_max)
, params(params.ngram_mod)
, mod(params.ngram_mod.n_match, 4*1024*1024)
, verbose(std::getenv("LLAMA_TRACE") != nullptr) {
@@ -2017,7 +2021,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl {
const std::string & path_dynamic,
bool save_dynamic,
bool save_static)
- : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq)
+ : common_speculative_impl(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, n_seq, n_draft)
, params(params.ngram_cache)
, n_draft(n_draft)
, save_dynamic(save_dynamic)
@@ -2138,6 +2142,8 @@ struct common_speculative {
// which implementaion was used for a given seq_id
std::vector<common_speculative_impl *> impl_last;
+
+ std::vector<double> synth_probs;
};
static common_ngram_map get_common_ngram_map(
@@ -2316,6 +2322,101 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) {
return n_max;
}
+int32_t common_speculative_n_max(const common_speculative * spec) {
+ int32_t n_max = 0;
+
+ if (spec == nullptr) {
+ return n_max;
+ }
+
+ for (const auto & impl : spec->impls) {
+ n_max = std::max(n_max, std::max(0, impl->n_max));
+ }
+
+ return n_max;
+}
+
+std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max) {
+ const bool has_length = spec->synth_len != -1.0;
+ const bool has_rates = !spec->synth_rates.empty();
+
+ if (!has_length && !has_rates) {
+ return {};
+ }
+ if (has_length && has_rates) {
+ throw std::invalid_argument("synthetic acceptance length and rates are mutually exclusive");
+ }
+
+ if (n_max <= 0) {
+ throw std::invalid_argument("synthetic acceptance requires at least one speculative token");
+ }
+
+ if (has_rates) {
+ const auto & rates = spec->synth_rates;
+ if (rates.size() != (size_t) n_max) {
+ throw std::invalid_argument(string_format(
+ "synthetic acceptance rates must contain %d values, got %zu", n_max, rates.size()));
+ }
+
+ for (size_t i = 0; i < rates.size(); ++i) {
+ if (!std::isfinite(rates[i]) || rates[i] < 0.0 || rates[i] > 1.0) {
+ throw std::invalid_argument("synthetic acceptance rates must be finite and within [0, 1]");
+ }
+ if (i > 0 && rates[i] > rates[i - 1]) {
+ throw std::invalid_argument("synthetic acceptance rates must be monotonically non-increasing");
+ }
+ }
+
+ return rates;
+ }
+
+ const double length = spec->synth_len;
+ const double length_max = (double) n_max + 1.0;
+ if (!std::isfinite(length) || length < 1.0 || length > length_max) {
+ throw std::invalid_argument(string_format(
+ "synthetic acceptance length must be finite and within [1, %.0f]", length_max));
+ }
+
+ double p = 0.0;
+ if (length == length_max) {
+ p = 1.0;
+ } else if (length > 1.0) {
+ double p_min = 0.0;
+ double p_max = 1.0;
+ for (int i = 0; i < 32; ++i) {
+ const double p_mid = 0.5 * (p_min + p_max);
+ double sum = 0.0;
+ double term = p_mid;
+ for (int32_t j = 0; j < n_max; ++j) {
+ sum += term;
+ term *= p_mid;
+ }
+
+ if (sum < length - 1.0) {
+ p_min = p_mid;
+ } else {
+ p_max = p_mid;
+ }
+ }
+ p = 0.5 * (p_min + p_max);
+ }
+
+ std::vector<double> rates;
+ rates.reserve(n_max);
+ double rate = p;
+ for (int32_t i = 0; i < n_max; ++i) {
+ rates.push_back(rate);
+ rate *= p;
+ }
+
+ return rates;
+}
+
+const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec) {
+ GGML_ASSERT(spec);
+ return spec->synth_probs;
+}
+
common_params common_base_params_to_speculative(const common_params & params) {
const bool has_draft = params.speculative.has_dft();
@@ -2568,13 +2669,39 @@ common_speculative * common_speculative_init(common_params_speculative & params,
return nullptr;
}
- auto * result = new common_speculative {
- /* .dparams = */ common_speculative_draft_params_vec(n_seq),
- /* .impls = */ std::move(impls),
- /* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr)
- };
+ common_speculative_ptr result(new common_speculative {
+ /* .dparams = */ common_speculative_draft_params_vec(n_seq),
+ /* .impls = */ std::move(impls),
+ /* .impl_last = */ std::vector<common_speculative_impl *>(n_seq, nullptr),
+ /* .synth_probs = */ {},
+ });
- return result;
+ const int32_t n_max_configured = common_speculative_n_max(&params);
+ const int32_t n_max_effective = common_speculative_n_max(result.get());
+ const auto rates = common_speculative_synth_rates_resolve(&params, n_max_effective);
+
+ std::vector<std::string> rates_str;
+ rates_str.reserve(rates.size());
+ result->synth_probs.reserve(rates.size());
+ double rate_prev = 1.0;
+ double acceptance_length = 1.0;
+ for (const double rate : rates) {
+ result->synth_probs.push_back(rate_prev > 0.0 ? rate / rate_prev : 0.0);
+ rates_str.push_back(string_format("%.6g", rate));
+ rate_prev = rate;
+ acceptance_length += rate;
+ }
+ if (!result->synth_probs.empty()) {
+ SPC_WRN("%s", "synthetic speculative acceptance is enabled for benchmarking; generated output is not valid\n");
+ if (n_max_effective != n_max_configured) {
+ SPC_WRN("synthetic acceptance draft limit was reduced from %d to %d by the initialized speculative implementations\n",
+ n_max_configured, n_max_effective);
+ }
+ SPC_INF("synthetic acceptance: n_max = %zu, mean length = %.6f, rates = [%s]\n",
+ rates.size(), acceptance_length, string_join(rates_str, ", ").c_str());
+ }
+
+ return result.release();
}
void common_speculative_free(common_speculative * spec) {
diff --git a/common/speculative.h b/common/speculative.h
index 12ae31b7de..22505891f7 100644
--- a/common/speculative.h
+++ b/common/speculative.h
@@ -26,6 +26,15 @@ std::string common_speculative_type_to_str(enum common_speculative_type type);
// return the max number of draft tokens based on the speculative parameters
int32_t common_speculative_n_max(const common_params_speculative * spec);
+// return the max number of draft tokens from the initialized implementations
+int32_t common_speculative_n_max(const common_speculative * spec);
+
+// validate and resolve the unconditional synthetic acceptance rates
+std::vector<double> common_speculative_synth_rates_resolve(const common_params_speculative * spec, int32_t n_max);
+
+// return the conditional synthetic acceptance probabilities
+const std::vector<double> & common_speculative_get_synth_probs(const common_speculative * spec);
+
common_params common_base_params_to_speculative(const common_params & params);
struct common_speculative_output_limits {
diff --git a/docs/speculative.md b/docs/speculative.md
index 0f9f8a3d97..ffb1e34c7f 100644
--- a/docs/speculative.md
+++ b/docs/speculative.md
@@ -212,6 +212,15 @@ Use `--backend-sampling` to run supported target-model samplers on the model bac
Unsupported samplers and device layouts fall back to CPU sampling. Tensor split mode does not support backend sampling. A fixed seed produces repeatable random draws, but stochastic CPU and backend sampling can still select different tokens because floating-point operations can differ between implementations and devices. Use greedy sampling when exact output matching is required.
+### Synthetic Acceptance
+
+`llama-server` and `llama-cli` can replace normal speculative verification with synthetic decisions for benchmarking. The generated output is not valid model output because accepted draft tokens do not have to match the target model.
+
+Use exactly one of these options:
+
+- `--spec-synth-rates P0,P1,...` sets unconditional per-position acceptance probabilities. Entry `i` is the probability that the first `i+1` draft tokens are all accepted. The number of entries must match the effective maximum draft length. Values must be finite, within `[0, 1]`, and monotonically non-increasing.
+- `--spec-synth-len L` sets the target mean acceptance length, including the target token. For `K` maximum draft tokens, `L` must be within `[1, K+1]`. The server finds a constant conditional probability `p` such that `p + p^2 + ... + p^K = L - 1`, then uses unconditional rates `[p, p^2, ..., p^K]`.
+
### General Speculative Parameters
```
diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp
index ba58f852eb..e0907631ab 100644
--- a/tests/test-arg-parser.cpp
+++ b/tests/test-arg-parser.cpp
@@ -4,6 +4,7 @@
#include "llama.h"
#include "speculative.h"
+#include <cmath>
#include <limits>
#include <string>
#include <vector>
@@ -34,6 +35,62 @@ static void test(void) {
std::numeric_limits<int32_t>::max(),
std::numeric_limits<int32_t>::max());
+ {
+ common_params_speculative spec;
+ spec.synth_len = 3.4;
+
+ auto assert_invalid = [](const common_params_speculative & value, int32_t n_max) {
+ try {
+ common_speculative_synth_rates_resolve(&value, n_max);
+ assert(false);
+ } catch (const std::invalid_argument &) {
+ }
+ };
+
+ const auto rates = common_speculative_synth_rates_resolve(&spec, 4);
+ assert(rates.size() == 4);
+ assert(std::abs(rates[0] - 0.80581) < 1e-5);
+ assert(std::abs(rates[1] - 0.64933) < 1e-5);
+ assert(std::abs(rates[2] - 0.52323) < 1e-5);
+ assert(std::abs(rates[3] - 0.42163) < 1e-5);
+ assert(std::abs(1.0 + rates[0] + rates[1] + rates[2] + rates[3] - 3.4) < 1e-8);
+
+ spec.synth_len = 1.0;
+ assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({0.0, 0.0, 0.0, 0.0}));
+
+ spec.synth_len = 5.0;
+ assert(common_speculative_synth_rates_resolve(&spec, 4) == std::vector<double>({1.0, 1.0, 1.0, 1.0}));
+
+ spec.synth_len = 5.1;
+ assert_invalid(spec, 4);
+
+ spec.synth_len = std::numeric_limits<double>::quiet_NaN();
+ assert_invalid(spec, 4);
+
+ spec.synth_len = 0.0;
+ assert_invalid(spec, 4);
+
+ spec.synth_len = -1.0;
+ spec.synth_rates = {0.8, 0.6, 0.4};
+ assert_invalid(spec, 4);
+
+ spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
+ assert(common_speculative_synth_rates_resolve(&spec, 4) == spec.synth_rates);
+
+ spec.synth_rates = {0.8, 0.9, 0.4, 0.2};
+ assert_invalid(spec, 4);
+
+ spec.synth_rates = {0.8, std::numeric_limits<double>::quiet_NaN(), 0.4, 0.2};
+ assert_invalid(spec, 4);
+
+ spec.synth_rates = {0.8, 0.6, 0.4, -0.2};
+ assert_invalid(spec, 4);
+
+ spec.synth_rates = {0.8, 0.6, 0.4, 0.2};
+ spec.synth_len = 3.0;
+ assert_invalid(spec, 4);
+ }
+
{
common_params base;
base.n_parallel = 4;
@@ -197,6 +254,26 @@ static void test(void) {
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_SPECULATIVE));
assert(params.speculative.draft.n_max == 123);
+ {
+ common_params synth_params;
+ argv = {"binary_name", "--spec-synth-len", "3.4"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
+ assert(synth_params.speculative.synth_len == 3.4);
+ }
+
+ {
+ common_params synth_params;
+ argv = {"binary_name", "--spec-synth-rates", "0.8,0.6,0.2"};
+ assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
+ assert(synth_params.speculative.synth_rates == std::vector<double>({0.8, 0.6, 0.2}));
+ }
+
+ {
+ common_params synth_params;
+ argv = {"binary_name", "--spec-synth-len", "3.4x"};
+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), synth_params, LLAMA_EXAMPLE_SERVER));
+ }
+
argv = {"binary_name", "-lm", "none"};
assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));
assert(params.load_mode == LLAMA_LOAD_MODE_NONE);
diff --git a/tools/cli/README.md b/tools/cli/README.md
index 0fba70a90e..e663cfa3b2 100644
--- a/tools/cli/README.md
+++ b/tools/cli/README.md
@@ -200,6 +200,8 @@
| `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) |
| `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
| `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |
+| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_LEN) |
+| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_RATES) |
| `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) |
| `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
| `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) |
diff --git a/tools/server/README.md b/tools/server/README.md
index f49cdb2729..6fd27f1389 100644
--- a/tools/server/README.md
+++ b/tools/server/README.md
@@ -259,6 +259,8 @@ For the full list of features, please refer to [server's changelog](https://gith
| `--spec-draft-n-cpu-moe, --spec-draft-ncmoe, -ncmoed, --n-cpu-moe-draft N` | keep the Mixture of Experts (MoE) weights of the first N layers in the CPU for the draft model<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_CPU_MOE) |
| `--spec-draft-n-max N` | number of tokens to draft for speculative decoding (default: 3)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MAX) |
| `--spec-draft-n-min N` | minimum number of draft tokens to use for speculative decoding (default: 0)<br/>(env: LLAMA_ARG_SPEC_DRAFT_N_MIN) |
+| `--spec-synth-len L` | target mean synthetic acceptance length, including the target token (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_LEN) |
+| `--spec-synth-rates P0,P1,...` | comma-separated unconditional per-position synthetic acceptance probabilities (benchmarking only)<br/>(env: LLAMA_ARG_SPEC_SYNTH_RATES) |
| `--spec-draft-p-split, --draft-p-split P` | speculative decoding split probability (default: 0.10)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_SPLIT) |
| `--spec-draft-p-min, --draft-p-min P` | minimum speculative decoding probability (greedy) (default: 0.00)<br/>(env: LLAMA_ARG_SPEC_DRAFT_P_MIN) |
| `--spec-draft-backend-sampling, --no-spec-draft-backend-sampling` | offload draft sampling to the backend (default: enabled)<br/>(env: LLAMA_ARG_SPEC_DRAFT_BACKEND_SAMPLING) |
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 9fdfcae563..e6c991f7cf 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -23,6 +23,7 @@
#include <exception>
#include <memory>
#include <filesystem>
+#include <random>
#include <utility>
#include <fstream>
@@ -51,6 +52,50 @@ static common_speculative_output_limits server_output_limits(const common_params
return result;
}
+// synthetic draft verification for benchmarking - accept draft tokens at random instead of by match with the target
+// on replay the draft was already accepted before a context checkpoint restore, so repeat the same decisions
+static std::vector<llama_token> server_sample_and_accept_synth(
+ common_sampler * smpl,
+ llama_context * ctx,
+ const std::vector<int32_t> & idxs,
+ const llama_tokens & draft,
+ const std::vector<double> & synth_probs,
+ std::mt19937 & rng,
+ bool is_replay) {
+ GGML_ASSERT(idxs.size() == draft.size() + 1);
+ GGML_ASSERT(synth_probs.size() >= draft.size());
+
+ std::vector<llama_token> result;
+ result.reserve(idxs.size());
+
+ const llama_vocab * vocab = llama_model_get_vocab(llama_get_model(ctx));
+ std::uniform_real_distribution<double> dist(0.0, 1.0);
+ for (size_t i = 0; i < draft.size(); ++i) {
+ const llama_token id = common_sampler_sample(smpl, ctx, idxs[i]);
+ const bool accept = is_replay || dist(rng) < synth_probs[i];
+ // do not accept a drafted EOG token - it would end the generation early
+ // on replay the last token is from the target and can be EOG, so skip this check
+ if (accept && (is_replay || !llama_vocab_is_eog(vocab, draft[i]))) {
+ // synthetic draft tokens do not advance grammar or reasoning state
+ // the last replay token is from the target and must advance both
+ const bool is_replay_target = is_replay && i + 1 == draft.size();
+ common_sampler_accept(smpl, draft[i], is_replay_target);
+ result.push_back(draft[i]);
+ continue;
+ }
+
+ common_sampler_accept(smpl, id, true);
+ result.push_back(id);
+ return result;
+ }
+
+ const llama_token id = common_sampler_sample(smpl, ctx, idxs[draft.size()]);
+ common_sampler_accept(smpl, id, true);
+ result.push_back(id);
+
+ return result;
+}
+
// state diagram: https://github.com/ggml-org/llama.cpp/pull/9283
enum slot_state {
SLOT_STATE_IDLE,
@@ -211,6 +256,7 @@ struct server_slot {
std::vector<int32_t> spec_i_batch;
common_prompt_checkpoint spec_ckpt;
bool spec_is_replay = false;
+ std::mt19937 spec_synth_rng;
// TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state
// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837
@@ -1194,6 +1240,9 @@ private:
spec.reset(common_speculative_init(params_base.speculative, params_base.n_parallel));
} catch (const std::exception & e) {
SRV_ERR("failed to initialize speculative decoding context: %s\n", e.what());
+ if (params_base.speculative.has_synth()) {
+ return false;
+ }
}
}
@@ -1209,6 +1258,11 @@ private:
model_dft = nullptr;
}
+ if (!spec && params_base.speculative.has_synth()) {
+ SRV_ERR("%s", "synthetic acceptance requires an initialized speculative decoding context\n");
+ return false;
+ }
+
for (int i = 0; i < params_base.n_parallel; i++) {
server_slot & slot = slots[i];
@@ -1717,6 +1771,13 @@ private:
SLT_TRC(slot, "sampler chain: %s\n", common_sampler_print(slot.smpl.get()).c_str());
SLT_TRC(slot, "sampler params: \n%s\n", task.params.sampling.print().c_str());
+
+ if (spec && !common_speculative_get_synth_probs(spec.get()).empty()) {
+ const uint32_t seed = task.params.sampling.seed == LLAMA_DEFAULT_SEED
+ ? std::random_device{}()
+ : task.params.sampling.seed;
+ slot.spec_synth_rng.seed(seed);
+ }
} else {
slot.smpl.reset();
}
@@ -3802,7 +3863,12 @@ private:
common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);
- auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft);
+ const auto & synth_probs = common_speculative_get_synth_probs(spec.get());
+ auto accepted = synth_probs.empty()
+ ? common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft)
+ : server_sample_and_accept_synth(
+ slot.smpl.get(), slot.ctx_tgt, slot.spec_i_batch, slot.spec_draft,
+ synth_probs, slot.spec_synth_rng, slot.spec_is_replay);
slot.spec_i_batch.clear();
GGML_ASSERT(accepted.size() >= 1);
@@ -3868,7 +3934,7 @@ private:
auto & n_accepted_per_pos = slot.n_accepted_per_pos;
if (n_accepted_per_pos.empty()) {
- n_accepted_per_pos.resize(common_speculative_n_max(&params_base.speculative), 0);
+ n_accepted_per_pos.resize(common_speculative_n_max(spec.get()), 0);
}
for (size_t i = 0; i < n_accepted && i < n_accepted_per_pos.size(); ++i) {
n_accepted_per_pos[i]++;
diff --git a/tools/server/tests/unit/test_speculative.py b/tools/server/tests/unit/test_speculative.py
index 5837195006..22b523954e 100644
--- a/tools/server/tests/unit/test_speculative.py
+++ b/tools/server/tests/unit/test_speculative.py
@@ -52,6 +52,18 @@ def test_with_and_without_draft():
assert tokens_no_draft == tokens_draft
+ server.stop()
+ create_server()
+ assert server.spec_draft_n_max is not None
+ server.spec_synth_rates = [0.0] * server.spec_draft_n_max
+ server.start()
+ res = server.make_request("POST", "/completion", data=request)
+
+ assert res.status_code == 200
+ assert res.body["timings"]["draft_n"] > 0
+ assert res.body["timings"]["draft_n_accepted"] == 0
+ assert res.body["tokens"] == tokens_no_draft
+
def test_different_draft_min_draft_max():
global server
@@ -80,6 +92,66 @@ def test_different_draft_min_draft_max():
last_content = res.body["content"]
+def test_synth_is_deterministic():
+ global server
+ assert server.spec_draft_n_max is not None
+ server.spec_synth_rates = [0.75 ** (i + 1) for i in range(server.spec_draft_n_max)]
+ server.start()
+
+ request = {
+ "prompt": "I believe the meaning of life is",
+ "temperature": 0.2,
+ "top_k": 5,
+ "seed": 4242,
+ "n_predict": 32,
+ }
+ responses = [server.make_request("POST", "/completion", data=request) for _ in range(2)]
+
+ for res in responses:
+ assert res.status_code == 200
+ assert res.body["timings"]["draft_n"] > 0
+ assert responses[0].body["timings"]["draft_n"] == responses[1].body["timings"]["draft_n"]
+ assert responses[0].body["timings"]["draft_n_accepted"] == responses[1].body["timings"]["draft_n_accepted"]
+
+
+def test_synth_ignores_target_tokens():
+ global server
+ assert server.spec_draft_n_max is not None
+ server.spec_synth_rates = [1.0] * server.spec_draft_n_max
+ server.start()
+
+ res = server.make_request("POST", "/completion", data={
+ "prompt": "I believe the meaning of life is",
+ "temperature": 0.0,
+ "seed": 4242,
+ "n_predict": 32,
+ })
+
+ assert res.status_code == 200
+ assert res.body["timings"]["draft_n"] > 0
+ assert res.body["timings"]["draft_n_accepted"] == res.body["timings"]["draft_n"]
+
+ res = server.make_request("POST", "/completion", data={
+ "prompt": "I believe the meaning of life is",
+ "temperature": 0.0,
+ "seed": 4242,
+ "n_predict": 6,
+ "grammar": 'root ::= "a"{5,5}',
+ })
+ assert res.status_code == 200, res.body
+
+ res = server.make_request("POST", "/completion", data={
+ "prompt": "Respond with only: OK",
+ "temperature": 0.0,
+ "seed": 4242,
+ "n_predict": 64,
+ "ignore_eos": True,
+ })
+ assert res.status_code == 200, res.body
+ assert res.body["tokens_predicted"] == 64
+ assert res.body["stop_type"] == "limit"
+
+
def test_slot_ctx_not_exceeded():
global server
server.n_ctx = 256
diff --git a/tools/server/tests/utils.py b/tools/server/tests/utils.py
index a0d2dfa3c5..5a4f31a53e 100644
--- a/tools/server/tests/utils.py
+++ b/tools/server/tests/utils.py
@@ -99,6 +99,8 @@ class ServerProcess:
spec_type: str | None = None
spec_draft_n_min: int | None = None
spec_draft_n_max: int | None = None
+ spec_synth_len: float | None = None
+ spec_synth_rates: List[float] | None = None
no_ui: bool | None = None
jinja: bool | None = None
reasoning_format: Literal['deepseek', 'none', 'nothink'] | None = None
@@ -245,6 +247,11 @@ class ServerProcess:
server_args.extend(["--spec-draft-n-max", self.spec_draft_n_max])
if self.spec_draft_n_min:
server_args.extend(["--spec-draft-n-min", self.spec_draft_n_min])
+ if self.spec_synth_len is not None:
+ server_args.extend(["--spec-synth-len", self.spec_synth_len])
+ if self.spec_synth_rates is not None:
+ rates = ",".join(str(rate) for rate in self.spec_synth_rates)
+ server_args.extend(["--spec-synth-rates", rates])
if self.no_ui:
server_args.append("--no-ui")
if self.no_models_autoload: