LLM inference in C/C++
C++ ggml
CUDA: Add backend sampler for penalties sampler (#25262)
Konrad Moren96278e39fc83
diff --git a/common/arg.cpp b/common/arg.cppindex 772422f682..305938fcb2 100644--- a/common/arg.cpp+++ b/common/arg.cpp@@ -27,6 +27,7 @@#include <algorithm>#include <cinttypes>#include <climits>+#include <cmath>#include <cstdarg>#include <filesystem>#include <fstream>@@ -2036,7 +2037,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex{"--repeat-penalty"}, "N",string_format("penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)", (double)params.sampling.penalty_repeat),[](common_params & params, const std::string & value) {- params.sampling.penalty_repeat = std::stof(value);+ const float penalty_repeat = std::stof(value);+ if (!std::isfinite(penalty_repeat) ||+ penalty_repeat <= 0.0f ||+ !std::isfinite(1.0f/penalty_repeat)) {+ throw std::runtime_error("error: repeat-penalty must be finite and greater than 0\n");+ }+ params.sampling.penalty_repeat = penalty_repeat;params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT;}).set_sampling());@@ -2044,14 +2051,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex{"--presence-penalty"}, "N",string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present),[](common_params & params, const std::string & value) {- params.sampling.penalty_present = std::stof(value);+ const float penalty_present = std::stof(value);+ if (!std::isfinite(penalty_present)) {+ throw std::runtime_error("error: presence-penalty must be finite\n");+ }+ params.sampling.penalty_present = penalty_present;}).set_sampling());add_opt(common_arg({"--frequency-penalty"}, "N",string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq),[](common_params & params, const std::string & value) {- params.sampling.penalty_freq = std::stof(value);+ const float penalty_freq = std::stof(value);+ if (!std::isfinite(penalty_freq)) {+ throw std::runtime_error("error: frequency-penalty must be finite\n");+ }+ params.sampling.penalty_freq = penalty_freq;}).set_sampling());add_opt(common_arg(diff --git a/common/common.cpp b/common/common.cppindex ff27d392fb..c941fd505a 100644--- a/common/common.cpp+++ b/common/common.cpp@@ -1299,8 +1299,9 @@ common_init_result::common_init_result(common_params & params, bool model_only)pimpl->samplers.resize(cparams.n_seq_max);pimpl->samplers_seq_config.resize(cparams.n_seq_max);+ const int32_t n_ctx = cparams.n_ctx > 0 ? (int32_t) cparams.n_ctx : llama_model_n_ctx_train(model);for (int i = 0; i < (int) cparams.n_seq_max; ++i) {- pimpl->samplers[i].reset(common_sampler_init(model, params.sampling));+ pimpl->samplers[i].reset(common_sampler_init(model, params.sampling, n_ctx));pimpl->samplers_seq_config[i] = { i, common_sampler_get(pimpl->samplers[i].get()) };}diff --git a/common/sampling.cpp b/common/sampling.cppindex 256ac161e2..5698c0263b 100644--- a/common/sampling.cpp+++ b/common/sampling.cpp@@ -184,9 +184,26 @@ std::string common_params_sampling::print() const {return std::string(result);}-struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params) {- const llama_vocab * vocab = llama_model_get_vocab(model);+struct common_sampler * common_sampler_init(+ const struct llama_model * model,+ struct common_params_sampling & params,+ int32_t n_ctx) {+ if (!std::isfinite(params.penalty_repeat) ||+ params.penalty_repeat <= 0.0f ||+ !std::isfinite(1.0f/params.penalty_repeat)) {+ throw std::invalid_argument("penalty_repeat must be finite and greater than 0");+ }+ if (!std::isfinite(params.penalty_freq)) {+ throw std::invalid_argument("penalty_freq must be finite");+ }+ if (!std::isfinite(params.penalty_present)) {+ throw std::invalid_argument("penalty_present must be finite");+ }+ if (params.penalty_last_n == -1) {+ params.penalty_last_n = n_ctx > 0 ? n_ctx : llama_model_n_ctx_train(model);+ }+ const llama_vocab * vocab = llama_model_get_vocab(model);llama_sampler_chain_params lparams = llama_sampler_chain_default_params();lparams.no_perf = params.no_perf;diff --git a/common/sampling.h b/common/sampling.hindex 4191988bb8..91e2cea787 100644--- a/common/sampling.h+++ b/common/sampling.h@@ -37,7 +37,10 @@ struct common_sampler;// llama_sampler API overloads// note: can mutate params in some cases-struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params);+struct common_sampler * common_sampler_init(+ const struct llama_model * model,+ struct common_params_sampling & params,+ int32_t n_ctx = 0);void common_sampler_free(struct common_sampler * gsmpl);diff --git a/include/llama.h b/include/llama.hindex 6e53e22972..f2d7e38858 100644--- a/include/llama.h+++ b/include/llama.h@@ -1256,6 +1256,7 @@ extern "C" {struct ggml_tensor * probs;struct ggml_tensor * sampled;struct ggml_tensor * candidates;+ int64_t n_vocab;};// user code can implement the interface below in order to create custom llama_sampler@@ -1425,9 +1426,9 @@ extern "C" {/// NOTE: Avoid using on the full vocabulary as searching for repeated tokens can become slow. For example, apply top-k or top-p sampling first.LLAMA_API struct llama_sampler * llama_sampler_init_penalties(int32_t penalty_last_n, // last n tokens to penalize (0 = disable penalty, -1 = context size)- float penalty_repeat, // 1.0 = disabled- float penalty_freq, // 0.0 = disabled- float penalty_present); // 0.0 = disabled+ float penalty_repeat, // must be > 0.0, 1.0 = disabled+ float penalty_freq, // must be finite, 0.0 = disabled+ float penalty_present); // must be finite, 0.0 = disabled/// @details DRY sampler, designed by p-e-w, as described in: https://github.com/oobabooga/text-generation-webui/pull/5677, porting Koboldcpp implementation authored by pi6am: https://github.com/LostRuins/koboldcpp/pull/982LLAMA_API struct llama_sampler * llama_sampler_init_dry(diff --git a/src/llama-graph.cpp b/src/llama-graph.cppindex e12a8cdc2a..1a35692300 100644--- a/src/llama-graph.cpp+++ b/src/llama-graph.cpp@@ -3620,6 +3620,7 @@ void llm_graph_context::build_sampling() const {/*.probs =*/ nullptr,/*.sampled =*/ nullptr,/*.candidates =*/ nullptr,+ /*.n_vocab =*/ logits_seq->ne[0],};assert(sampler->iface->backend_apply);diff --git a/src/llama-sampler.cpp b/src/llama-sampler.cppindex a9cb6bee5f..b2f1abe737 100644--- a/src/llama-sampler.cpp+++ b/src/llama-sampler.cpp@@ -589,6 +589,7 @@ static bool llama_sampler_backend_support(/*.probs = */ nullptr,/*.sampled = */ nullptr,/*.candidates = */ ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n),+ /*.n_vocab = */ n,};ggml_cgraph * gf = ggml_new_graph(ctx);@@ -2638,7 +2639,7 @@ struct llama_sampler * llama_sampler_init_grammar_lazy_patterns(// penalties-struct llama_sampler_penalties {+struct llama_sampler_penalties : public llama_sampler_backend {const int32_t penalty_last_n;const float penalty_repeat;const float penalty_freq;@@ -2648,10 +2649,49 @@ struct llama_sampler_penalties {// a frequency map to count token occurrencesstd::unordered_map<llama_token, int> token_count;++ // backend graph inputs+ ggml_tensor * inp_token_ids = nullptr;+ ggml_tensor * inp_counts = nullptr;++ // backend helpers+ int32_t n_vocab = 0;+ int32_t n_max = 0;+ bool has_candidates = false;++ std::vector<int32_t> host_token_ids;+ std::vector<int32_t> host_counts;++ static bool is_disabled(+ int32_t penalty_last_n,+ float penalty_repeat,+ float penalty_freq,+ float penalty_present) {+ return penalty_last_n == 0 ||+ (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f);+ }++ bool is_disabled() const {+ return is_disabled(penalty_last_n, penalty_repeat, penalty_freq, penalty_present);+ }++ llama_sampler_penalties(+ int32_t penalty_last_n,+ float penalty_repeat,+ float penalty_freq,+ float penalty_present)+ : llama_sampler_backend("penalties")+ , penalty_last_n (penalty_last_n)+ , penalty_repeat (penalty_repeat)+ , penalty_freq (penalty_freq)+ , penalty_present (penalty_present)+ , prev (penalty_last_n) {+ }};-static const char * llama_sampler_penalties_name(const struct llama_sampler * /*smpl*/) {- return "penalties";+static const char * llama_sampler_penalties_name(const struct llama_sampler * smpl) {+ auto * ctx = (llama_sampler_penalties *) smpl->ctx;+ return ctx->get_name();}static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_token token) {@@ -2688,8 +2728,7 @@ static void llama_sampler_penalties_accept(struct llama_sampler * smpl, llama_tostatic void llama_sampler_penalties_apply(struct llama_sampler * smpl, llama_token_data_array * cur_p) {auto * ctx = (llama_sampler_penalties *) smpl->ctx;- if ((ctx->penalty_last_n == 0) ||- (ctx->penalty_repeat == 1.0f && ctx->penalty_freq == 0.0f && ctx->penalty_present == 0.0f)) {+ if (ctx->is_disabled()) {return;}@@ -2736,7 +2775,8 @@ static struct llama_sampler * llama_sampler_penalties_clone(const struct llama_s{auto * result_ctx = (llama_sampler_penalties *) result->ctx;- result_ctx->prev = ctx->prev;+ result_ctx->prev = ctx->prev;+ result_ctx->token_count = ctx->token_count;}return result;@@ -2746,6 +2786,171 @@ static void llama_sampler_penalties_free(struct llama_sampler * smpl) {delete (llama_sampler_penalties *) smpl->ctx;}+static bool llama_sampler_penalties_backend_init(+ struct llama_sampler * smpl,+ ggml_backend_buffer_type_t buft) {+ auto * sctx = (llama_sampler_penalties *) smpl->ctx;++ const bool res = llama_sampler_backend_support(smpl, buft);++ sctx->init(res);++ return res;+}++static void llama_sampler_penalties_backend_apply(+ struct llama_sampler * smpl,+ struct ggml_context * ctx,+ struct ggml_cgraph * gf,+ struct llama_sampler_data * data) {+ GGML_UNUSED(gf);++ auto * sctx = (llama_sampler_penalties *) smpl->ctx;++ if (sctx->is_disabled()) {+ return;+ }++ GGML_ASSERT(data->n_vocab > 0 && data->n_vocab <= INT32_MAX);++ sctx->has_candidates = data->candidates != nullptr;+ sctx->n_vocab = (int32_t) data->n_vocab;+ sctx->n_max = std::min(sctx->penalty_last_n, sctx->n_vocab);++ sctx->inp_token_ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sctx->n_max);+ ggml_set_name(sctx->inp_token_ids, "penalties_token_ids");+ ggml_set_input(sctx->inp_token_ids);++ sctx->inp_counts = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, sctx->n_max);+ ggml_set_name(sctx->inp_counts, "penalties_counts");+ ggml_set_input(sctx->inp_counts);++ if ((int32_t) sctx->host_token_ids.size() != sctx->n_max) {+ sctx->host_token_ids.assign(sctx->n_max, 0);+ sctx->host_counts.assign(sctx->n_max, 0);+ }++ // flatten+ ggml_tensor * logits = ggml_reshape_1d(ctx, data->logits, ggml_nelements(data->logits));+ ggml_tensor * gathered = logits;+ ggml_tensor * counts_f32 = ggml_cast(ctx, sctx->inp_counts, GGML_TYPE_F32);++ if (sctx->has_candidates) {+ ggml_tensor * candidates = ggml_reshape_1d(+ ctx, data->candidates, ggml_nelements(data->candidates));+ const int64_t n_candidates = candidates->ne[0];+ GGML_ASSERT(n_candidates == ggml_nelements(logits));++ ggml_tensor * counts_rows = ggml_fill(+ ctx, ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1, sctx->n_vocab), 0.0f);+ ggml_tensor * scatter_rows = ggml_reshape_2d(ctx, counts_f32, 1, sctx->n_max);+ counts_rows = ggml_set_rows(ctx, counts_rows, scatter_rows, sctx->inp_token_ids);+ counts_f32 = ggml_get_rows(ctx, counts_rows, candidates);+ counts_f32 = ggml_reshape_1d(ctx, counts_f32, n_candidates);+ } else {+ ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, ggml_nelements(logits));+ gathered = ggml_get_rows(ctx, logits_rows, sctx->inp_token_ids);+ gathered = ggml_reshape_1d(ctx, gathered, sctx->n_max);+ }++ ggml_tensor * active_mask = ggml_step(ctx, counts_f32);+ ggml_tensor * inactive_mask = ggml_sub(ctx, ggml_fill(ctx, active_mask, 1.0f), active_mask);++ ggml_tensor * penalized = gathered;++ if (sctx->penalty_repeat != 1.0f) {+ ggml_tensor * pos_mask = ggml_step(ctx, penalized);+ ggml_tensor * neg_mask = ggml_sub(ctx, ggml_fill(ctx, pos_mask, 1.0f), pos_mask);++ ggml_tensor * pos_scale = ggml_scale(ctx, pos_mask, 1.0f/sctx->penalty_repeat);+ ggml_tensor * neg_scale = ggml_scale(ctx, neg_mask, sctx->penalty_repeat);+ ggml_tensor * repeat_scale = ggml_add(ctx, pos_scale, neg_scale);++ // scale inactive entries with 1 to avoid -INF * 0 = NaN for values masked by top-p+ repeat_scale = ggml_mul(ctx, repeat_scale, active_mask);+ repeat_scale = ggml_add(ctx, repeat_scale, inactive_mask);+ penalized = ggml_mul(ctx, gathered, repeat_scale);+ }++ if (sctx->penalty_freq != 0.0f) {+ ggml_tensor * penalty_freq = ggml_scale(ctx, counts_f32, sctx->penalty_freq);+ penalized = ggml_sub(ctx, penalized, penalty_freq);+ }++ if (sctx->penalty_present != 0.0f) {+ ggml_tensor * penalty_present = ggml_scale(ctx, active_mask, sctx->penalty_present);+ penalized = ggml_sub(ctx, penalized, penalty_present);+ }++ if (sctx->has_candidates) {+ data->logits = penalized;+ } else {+ ggml_tensor * logits_rows = ggml_reshape_2d(ctx, logits, 1, ggml_nelements(logits));+ ggml_tensor * scatter_rows = ggml_reshape_2d(ctx, penalized, 1, sctx->n_max);+ logits_rows = ggml_set_rows(ctx, logits_rows, scatter_rows, sctx->inp_token_ids);+ data->logits = ggml_reshape_1d(ctx, logits_rows, ggml_nelements(logits));+ }+}++static void llama_sampler_penalties_backend_set_input(struct llama_sampler * smpl) {+ auto * sctx = (llama_sampler_penalties *) smpl->ctx;++ if (!sctx->inp_token_ids || !sctx->inp_counts || sctx->n_max <= 0 || sctx->n_vocab <= 0) {+ return;+ }++ if (sctx->is_disabled()) {+ return;+ }++ // fill active entries from the map+ int32_t n_active = 0;++ for (const auto & it : sctx->token_count) {+ GGML_ASSERT(n_active < sctx->n_max);+ sctx->host_token_ids[n_active] = it.first;+ sctx->host_counts [n_active] = it.second;+ ++n_active;+ }++ // Sorting is required because backend_apply uses ggml_set_rows (a scatter-back operation)+ std::vector<std::pair<int32_t, int32_t>> entries;+ entries.reserve(n_active);+ for (int32_t i = 0; i < n_active; ++i) {+ entries.emplace_back(sctx->host_token_ids[i], sctx->host_counts[i]);+ }+ std::sort(entries.begin(), entries.end(), [](const auto & a, const auto & b) {+ return a.first < b.first;+ });+ for (int32_t i = 0; i < n_active; ++i) {+ sctx->host_token_ids[i] = entries[i].first;+ sctx->host_counts [i] = entries[i].second;+ }++ // Padding: Finds a filler token id that is not present in token_count.+ // Use it to do padding for the arrays, it avoids resizing every time.+ // The arrays must always have exactly n_max entries (the GPU tensor is a fixed size).+ int32_t filler = 0;+ if (n_active < sctx->n_max) {+ while (sctx->token_count.find(filler) != sctx->token_count.end()) {+ ++filler;+ }+ GGML_ASSERT(filler < sctx->n_vocab);+ }++ // Fill the rest of the arrays with the filler token id and count 0.+ // Inactive slots are padded with a unique dummy token ID (count = 0).+ // The uniqueness matters because ggml_set_rows with duplicate indices can produce non-deterministic or incorrect results.+ // Using a filler token with count 0 that isn't in the active set is safe, because the active_mask step in backend_apply filters them out via ggml_step(counts_f32)+ for (int32_t i = n_active; i < sctx->n_max; ++i) {+ sctx->host_token_ids[i] = filler;+ sctx->host_counts [i] = 0;+ }++ ggml_backend_tensor_set(sctx->inp_token_ids, sctx->host_token_ids.data(), 0, sctx->n_max * sizeof(int32_t));+ ggml_backend_tensor_set(sctx->inp_counts, sctx->host_counts.data(), 0, sctx->n_max * sizeof(int32_t));+}+static struct llama_sampler_i llama_sampler_penalties_i = {/* .name = */ llama_sampler_penalties_name,/* .accept = */ llama_sampler_penalties_accept,@@ -2753,10 +2958,10 @@ static struct llama_sampler_i llama_sampler_penalties_i = {/* .reset = */ llama_sampler_penalties_reset,/* .clone = */ llama_sampler_penalties_clone,/* .free = */ llama_sampler_penalties_free,- /* .backend_init = */ nullptr,+ /* .backend_init = */ llama_sampler_penalties_backend_init,/* .backend_accept = */ nullptr,- /* .backend_apply = */ nullptr,- /* .backend_set_input = */ nullptr,+ /* .backend_apply = */ llama_sampler_penalties_backend_apply,+ /* .backend_set_input = */ llama_sampler_penalties_backend_set_input,};struct llama_sampler * llama_sampler_init_penalties(@@ -2766,22 +2971,18 @@ struct llama_sampler * llama_sampler_init_penalties(float penalty_present) {penalty_last_n = std::max(penalty_last_n, 0);- const bool is_empty = (penalty_last_n == 0 || (penalty_repeat == 1.0f && penalty_freq == 0.0f && penalty_present == 0.0f));-- if (is_empty) {+ if (llama_sampler_penalties::is_disabled(+ penalty_last_n, penalty_repeat, penalty_freq, penalty_present)) {return llama_sampler_init_empty("?penalties");}return llama_sampler_init(/* .iface = */ &llama_sampler_penalties_i,- /* .ctx = */ new llama_sampler_penalties {- /* .penalty_last_n = */ penalty_last_n,- /* .penalty_repeat = */ penalty_repeat,- /* .penalty_freq = */ penalty_freq,- /* .penalty_present = */ penalty_present,- /* .prev = */ ring_buffer<llama_token>(penalty_last_n),- /* .token_count = */ {},- }+ /* .ctx = */ new llama_sampler_penalties(+ penalty_last_n,+ penalty_repeat,+ penalty_freq,+ penalty_present));}diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cppindex 1d3584f903..fd5adb740e 100644--- a/tests/test-arg-parser.cpp+++ b/tests/test-arg-parser.cpp@@ -99,6 +99,34 @@ static void test(void) {argv = {"binary_name", "-sm", "hello"};assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON));+ {+ common_params penalty_params;++ argv = {"binary_name", "--repeat-penalty", "0"};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));++ argv = {"binary_name", "--repeat-penalty", "-1"};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));++ argv = {"binary_name", "--repeat-penalty", "nan"};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));++ argv = {"binary_name", "--repeat-penalty", "inf"};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));++ argv = {"binary_name", "--repeat-penalty", "-inf"};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));++ const char * penalty_options[] = {"--frequency-penalty", "--presence-penalty"};+ const char * nonfinite_values[] = {"nan", "inf", "-inf"};+ for (const char * option : penalty_options) {+ for (const char * value : nonfinite_values) {+ argv = {"binary_name", option, value};+ assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), penalty_params, LLAMA_EXAMPLE_COMMON));+ }+ }+ }+// non-existence arg in specific example (--draft cannot be used outside llama-speculative)argv = {"binary_name", "--draft", "123"};assert(false == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_EMBEDDING));diff --git a/tests/test-backend-sampler.cpp b/tests/test-backend-sampler.cppindex c24076e313..1a46468ba2 100644--- a/tests/test-backend-sampler.cpp+++ b/tests/test-backend-sampler.cpp@@ -8,12 +8,15 @@#endif#include <algorithm>+#include <cmath>#include <cstdlib>#include <cstring>#include <fstream>+#include <functional>#include <map>#include <string>#include <unordered_map>+#include <unordered_set>#include <vector>struct test_args {@@ -761,6 +764,563 @@ static void test_backend_logit_bias_sampling(const test_params & params) {printf("backend logit bias sampling test PASSED\n");}+static void accept_prompt(llama_sampler * smpl, const llama_vocab * vocab, const std::string & prompt) {+ const llama_token bos = llama_vocab_bos(vocab);+ if (bos != LLAMA_TOKEN_NULL) {+ llama_sampler_accept(smpl, bos);+ }++ std::vector<llama_token> tokens(64);+ int32_t n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(),+ tokens.data(), (int32_t) tokens.size(), false, false);+ if (n_tokens < 0) {+ tokens.resize(-n_tokens);+ n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(),+ tokens.data(), (int32_t) tokens.size(), false, false);+ }++ for (int32_t i = 0; i < n_tokens; ++i) {+ llama_sampler_accept(smpl, tokens[i]);+ }+}++static std::vector<float> decode_raw_logits(const test_params & params, const std::string & prompt) {+ const int seq_id = 0;+ const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(params.model.get()));+ std::vector<llama_sampler_seq_config> empty_configs;+ test_context ctx(params, empty_configs);++ GGML_ASSERT(ctx.decode({{ seq_id, prompt }}));++ float * logits = llama_get_logits_ith(ctx.ctx.get(), ctx.idx_for_seq(seq_id));+ GGML_ASSERT(logits != nullptr);+ return std::vector<float>(logits, logits + n_vocab);+}++static std::vector<llama_token_data> apply_cpu_sampler(+ const std::vector<float> & raw_logits,+ llama_sampler * sampler) {+ std::vector<llama_token_data> data;+ data.reserve(raw_logits.size());+ for (llama_token token = 0; token < (llama_token) raw_logits.size(); ++token) {+ data.push_back({ token, raw_logits[token], 0.0f });+ }++ llama_token_data_array cur_p = { data.data(), data.size(), -1, false };+ llama_sampler_apply(sampler, &cur_p);+ data.resize(cur_p.size);+ return data;+}++using sampler_setup_fn = std::function<void(llama_sampler *)>;+using sampler_init_fn = std::function<llama_sampler *()>;++enum class penalties_position {+ before_filter,+ after_filter,+};++static void add_filter_and_penalties(+ llama_sampler * chain,+ const sampler_init_fn & init_filter,+ int32_t penalty_last_n,+ float penalty_repeat,+ float penalty_freq,+ float penalty_present,+ penalties_position position) {+ const auto add_penalties = [&]() {+ llama_sampler_chain_add(chain, llama_sampler_init_penalties(+ penalty_last_n, penalty_repeat, penalty_freq, penalty_present));+ };++ if (position == penalties_position::before_filter) {+ add_penalties();+ llama_sampler_chain_add(chain, init_filter());+ } else {+ llama_sampler_chain_add(chain, init_filter());+ add_penalties();+ }+}++static llama_sampler_ptr make_sampler_chain(+ const sampler_setup_fn & add_samplers,+ const sampler_setup_fn & accept_history) {+ llama_sampler_ptr chain(llama_sampler_chain_init(llama_sampler_chain_default_params()));+ add_samplers(chain.get());+ accept_history(chain.get());+ return chain;+}++struct backend_sampler_output {+ std::vector<float> logits;+ std::vector<llama_token> candidates;+};++static backend_sampler_output run_backend_sampler(+ const test_params & params,+ const std::string & prompt,+ llama_sampler * sampler) {+ const int seq_id = 0;+ std::vector<llama_sampler_seq_config> configs = {{ seq_id, sampler }};+ test_context ctx(params, configs);++ GGML_ASSERT(ctx.decode({{ seq_id, prompt }}));+ llama_synchronize(ctx.ctx.get());++ const int32_t idx = ctx.idx_for_seq(seq_id);+ const uint32_t n_logits = llama_get_sampled_logits_count_ith(ctx.ctx.get(), idx);+ const uint32_t n_candidates = llama_get_sampled_candidates_count_ith(ctx.ctx.get(), idx);+ float * logits = llama_get_sampled_logits_ith(ctx.ctx.get(), idx);+ llama_token * candidates = llama_get_sampled_candidates_ith(ctx.ctx.get(), idx);+ GGML_ASSERT(logits != nullptr);++ backend_sampler_output result;+ result.logits.assign(logits, logits + n_logits);+ result.candidates.resize(n_logits);++ if (n_candidates == 0) {+ for (uint32_t i = 0; i < n_logits; ++i) {+ result.candidates[i] = (llama_token) i;+ }+ } else {+ GGML_ASSERT(candidates != nullptr);+ GGML_ASSERT(n_candidates == n_logits);+ std::memcpy(result.candidates.data(), candidates, n_candidates * sizeof(llama_token));+ }++ return result;+}++struct sampler_comparison_output {+ std::vector<llama_token_data> expected;+ backend_sampler_output actual;+};++static sampler_comparison_output run_sampler_comparison(+ const test_params & params,+ const std::string & prompt,+ const std::vector<float> & raw_logits,+ const sampler_setup_fn & add_samplers,+ const sampler_setup_fn & accept_history) {+ llama_sampler_ptr cpu_chain = make_sampler_chain(add_samplers, accept_history);+ llama_sampler_ptr backend_chain = make_sampler_chain(add_samplers, accept_history);+ return {+ apply_cpu_sampler(raw_logits, cpu_chain.get()),+ run_backend_sampler(params, prompt, backend_chain.get()),+ };+}++static std::unordered_map<llama_token, float> map_logits(const std::vector<llama_token_data> & data) {+ std::unordered_map<llama_token, float> result;+ result.reserve(data.size());+ for (const auto & item : data) {+ result[item.id] = item.logit;+ }+ return result;+}++struct sampler_comparison_stats {+ int n_mismatch = 0;+ int n_masked = 0;+ float max_diff = 0.0f;+};++static sampler_comparison_stats compare_sampler_outputs(+ const char * name,+ const std::unordered_map<llama_token, float> & expected,+ const backend_sampler_output & actual,+ bool allow_extra_candidates = false) {+ GGML_ASSERT(actual.logits.size() == actual.candidates.size());++ sampler_comparison_stats result;+ std::unordered_set<llama_token> seen;+ seen.reserve(actual.candidates.size());++ for (size_t i = 0; i < actual.logits.size(); ++i) {+ const llama_token token = actual.candidates[i];+ const float logit = actual.logits[i];+ if (!seen.insert(token).second || std::isnan(logit)) {+ if (result.n_mismatch < 5) {+ printf("%s token %d has invalid backend output\n", name, token);+ }+ ++result.n_mismatch;+ continue;+ }++ const auto it = expected.find(token);+ if (it == expected.end()) {+ if (std::isinf(logit) && logit < 0.0f) {+ ++result.n_masked;+ } else if (!allow_extra_candidates) {+ if (result.n_mismatch < 5) {+ printf("%s token %d was not masked\n", name, token);+ }+ ++result.n_mismatch;+ }+ continue;+ }++ const float diff = fabsf(it->second - logit);+ result.max_diff = std::max(result.max_diff, diff);+ if (!std::isfinite(logit) || diff > 1e-3f) {+ if (result.n_mismatch < 5) {+ printf("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n",+ name, token, it->second, logit, diff);+ }+ ++result.n_mismatch;+ }+ }++ for (const auto & item : expected) {+ if (seen.find(item.first) == seen.end()) {+ if (result.n_mismatch < 5) {+ printf("%s missing backend token %d\n", name, item.first);+ }+ ++result.n_mismatch;+ }+ }++ printf("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n",+ name, result.max_diff, result.n_masked, result.n_mismatch);+ return result;+}++static float find_backend_logit(const backend_sampler_output & output, llama_token token) {+ for (size_t i = 0; i < output.candidates.size(); ++i) {+ if (output.candidates[i] == token) {+ return output.logits[i];+ }+ }+ GGML_ABORT("backend token not found");+}++static sampler_comparison_output run_penalties_comparison(+ const test_params & params,+ int32_t penalty_last_n,+ float penalty_repeat,+ float penalty_freq,+ float penalty_present,+ const std::string & prompt,+ const std::function<void(llama_sampler *)> & extra_accept = {}) {+ const auto * vocab = llama_model_get_vocab(params.model.get());+ const std::vector<float> raw_logits = decode_raw_logits(params, prompt);+ const auto add_samplers = [&](llama_sampler * chain) {+ llama_sampler_chain_add(chain, llama_sampler_init_penalties(+ penalty_last_n, penalty_repeat, penalty_freq, penalty_present));+ };+ const auto accept_history = [&](llama_sampler * chain) {+ accept_prompt(chain, vocab, prompt);+ if (extra_accept) {+ extra_accept(chain);+ }+ };++ return run_sampler_comparison(+ params, prompt, raw_logits, add_samplers, accept_history);+}++static void compare_penalties_logits(+ const test_params & params,+ int32_t penalty_last_n,+ float penalty_repeat,+ float penalty_freq,+ float penalty_present,+ const std::string & prompt,+ const std::function<void(llama_sampler *)> & extra_accept = {}) {+ const sampler_comparison_output output = run_penalties_comparison(+ params, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, prompt, extra_accept);++ GGML_ASSERT(output.expected.size() == output.actual.logits.size());++ const sampler_comparison_stats stats = compare_sampler_outputs(+ "penalties", map_logits(output.expected), output.actual);+ GGML_ASSERT(stats.n_masked == 0);+ GGML_ASSERT(stats.n_mismatch == 0);+}++static void test_penalty_parameter_values(const test_params & params) {+ struct penalty_test_case {+ const char * name;+ float repeat;+ float frequency;+ float presence;+ };++ const penalty_test_case cases[] = {+ { "frequency -1", 1.0f, -1.0f, 0.0f },+ { "frequency 0", 1.0f, 0.0f, 0.0f },+ { "frequency 1", 1.0f, 1.0f, 0.0f },+ { "presence -1", 1.0f, 0.0f, -1.0f },+ { "presence 0", 1.0f, 0.0f, 0.0f },+ { "presence 1", 1.0f, 0.0f, 1.0f },+ { "repeat 1", 1.0f, 0.0f, 0.0f },+ };++ int n_failed = 0;+ for (const auto & test : cases) {+ const sampler_comparison_output output = run_penalties_comparison(+ params, 64, test.repeat, test.frequency, test.presence, "Hello Hello world");+ GGML_ASSERT(output.expected.size() == output.actual.logits.size());+ const sampler_comparison_stats stats = compare_sampler_outputs(+ test.name, map_logits(output.expected), output.actual);+ n_failed += stats.n_mismatch != 0;+ }++ GGML_ASSERT(n_failed == 0);+}++static void compare_top_k_penalties_logits(+ const test_params & params,+ int32_t k,+ int32_t penalty_last_n,+ float penalty_repeat,+ float penalty_freq,+ float penalty_present,+ const std::string & prompt,+ penalties_position position) {+ const auto * vocab = llama_model_get_vocab(params.model.get());+ const std::vector<float> raw_logits = decode_raw_logits(params, prompt);+ const int n_vocab = (int) raw_logits.size();++ GGML_ASSERT(n_vocab > k);++ const sampler_init_fn init_top_k = [k]() {+ return llama_sampler_init_top_k(k);+ };+ llama_sampler_ptr top_k(init_top_k());+ const std::vector<llama_token_data> top_k_data = apply_cpu_sampler(raw_logits, top_k.get());+ GGML_ASSERT(top_k_data.size() == (size_t) k);+ const llama_token retained_history_token = top_k_data[0].id;++ llama_token excluded_history_token = LLAMA_TOKEN_NULL;+ for (llama_token token = 0; token < n_vocab; ++token) {+ const auto it = std::find_if(top_k_data.begin(), top_k_data.end(), [token](const llama_token_data & data) {+ return data.id == token;+ });+ if (it == top_k_data.end()) {+ excluded_history_token = token;+ break;+ }+ }+ GGML_ASSERT(excluded_history_token != LLAMA_TOKEN_NULL);++ const auto add_samplers = [&](llama_sampler * chain) {+ add_filter_and_penalties(chain, init_top_k,+ penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position);+ };++ auto accept_history = [&](llama_sampler * smpl) {+ accept_prompt(smpl, vocab, prompt);+ llama_sampler_accept(smpl, excluded_history_token);+ llama_sampler_accept(smpl, excluded_history_token);+ llama_sampler_accept(smpl, retained_history_token);+ llama_sampler_accept(smpl, retained_history_token);+ };++ const sampler_comparison_output output = run_sampler_comparison(+ params, prompt, raw_logits, add_samplers, accept_history);++ GGML_ASSERT(output.expected.size() == (size_t) k);+ GGML_ASSERT(output.actual.logits.size() == (size_t) k);++ const std::unordered_map<llama_token, float> expected_logits = map_logits(output.expected);++ if (position == penalties_position::after_filter) {+ GGML_ASSERT(expected_logits.find(retained_history_token) != expected_logits.end());+ GGML_ASSERT(fabsf(expected_logits.at(retained_history_token) - raw_logits[retained_history_token]) > 1e-6f);+ GGML_ASSERT(expected_logits.find(excluded_history_token) == expected_logits.end());+ GGML_ASSERT(std::find(output.actual.candidates.begin(), output.actual.candidates.end(),+ excluded_history_token) == output.actual.candidates.end());+ } else {+ const std::unordered_map<llama_token, float> unpenalized_logits = map_logits(top_k_data);+ bool changed = false;+ for (const auto & item : expected_logits) {+ const auto it = unpenalized_logits.find(item.first);+ if (it == unpenalized_logits.end() || fabsf(it->second - item.second) > 1e-6f) {+ changed = true;+ break;+ }+ }+ GGML_ASSERT(changed);+ }++ const char * name = position == penalties_position::before_filter+ ? "penalties top-k"+ : "top-k penalties";+ const sampler_comparison_stats stats = compare_sampler_outputs(+ name, expected_logits, output.actual);+ GGML_ASSERT(stats.n_masked == 0);+ GGML_ASSERT(stats.n_mismatch == 0);+}++static void compare_masking_penalties_logits(+ const test_params & params,+ const char * filter_name,+ const sampler_init_fn & init_filter,+ int32_t penalty_last_n,+ float penalty_repeat,+ float penalty_freq,+ float penalty_present,+ const std::string & prompt,+ penalties_position position,+ bool allow_extra_candidates,+ bool add_history = true) {+ const auto * vocab = llama_model_get_vocab(params.model.get());+ const std::vector<float> raw_logits = decode_raw_logits(params, prompt);+ const int n_vocab = (int) raw_logits.size();+ llama_sampler_ptr filter(init_filter());+ const std::vector<llama_token_data> filtered_data = apply_cpu_sampler(raw_logits, filter.get());+ GGML_ASSERT(!filtered_data.empty());+ GGML_ASSERT(filtered_data.size() < (size_t) n_vocab);++ const llama_token penalized_token = filtered_data[0].id;+ std::unordered_set<llama_token> retained_tokens;+ retained_tokens.reserve(filtered_data.size());+ for (const auto & data : filtered_data) {+ retained_tokens.insert(data.id);+ }++ llama_token masked_token = LLAMA_TOKEN_NULL;+ for (llama_token token = 0; token < n_vocab; ++token) {+ if (retained_tokens.find(token) == retained_tokens.end()) {+ masked_token = token;+ break;+ }+ }+ GGML_ASSERT(masked_token != LLAMA_TOKEN_NULL);++ const auto add_samplers = [&](llama_sampler * chain) {+ add_filter_and_penalties(chain, init_filter,+ penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position);+ };+ auto accept_history = [&](llama_sampler * smpl) {+ if (!add_history) {+ return;+ }+ accept_prompt(smpl, vocab, prompt);+ llama_sampler_accept(smpl, penalized_token);+ llama_sampler_accept(smpl, penalized_token);+ llama_sampler_accept(smpl, masked_token);+ llama_sampler_accept(smpl, masked_token);+ };++ const sampler_comparison_output output = run_sampler_comparison(+ params, prompt, raw_logits, add_samplers, accept_history);++ GGML_ASSERT(output.actual.logits.size() == (size_t) n_vocab);++ const std::unordered_map<llama_token, float> expected_logits = map_logits(output.expected);++ GGML_ASSERT(expected_logits.find(masked_token) == expected_logits.end());+ if (add_history) {+ if (position == penalties_position::after_filter) {+ GGML_ASSERT(expected_logits.find(penalized_token) != expected_logits.end());+ GGML_ASSERT(fabsf(expected_logits.at(penalized_token) - raw_logits[penalized_token]) > 1e-6f);+ } else {+ llama_sampler_ptr penalties(llama_sampler_init_penalties(+ penalty_last_n, penalty_repeat, penalty_freq, penalty_present));+ accept_history(penalties.get());+ const std::unordered_map<llama_token, float> penalized_logits =+ map_logits(apply_cpu_sampler(raw_logits, penalties.get()));+ GGML_ASSERT(fabsf(penalized_logits.at(penalized_token) - raw_logits[penalized_token]) > 1e-6f);+ }+ }++ const std::string name = position == penalties_position::before_filter+ ? "penalties " + std::string(filter_name)+ : std::string(filter_name) + " penalties";+ const sampler_comparison_stats stats = compare_sampler_outputs(+ name.c_str(), expected_logits, output.actual, allow_extra_candidates);+ const float masked_logit = find_backend_logit(output.actual, masked_token);+ GGML_ASSERT(stats.n_masked > 0);+ GGML_ASSERT(std::isinf(masked_logit) && masked_logit < 0.0f);+ GGML_ASSERT(stats.n_mismatch == 0);+}++static void test_backend_penalties_sampling(const test_params & params) {+ printf("Testing backend penalties (repeat + freq + presence)\n");+ compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello Hello world");++ printf("Testing backend penalties with penalty_last_n > 64\n");+ const auto * vocab = llama_model_get_vocab(params.model.get());+ std::vector<llama_token> tokens(8);+ int32_t n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false);+ if (n_tok < 0) {+ tokens.resize(-n_tok);+ n_tok = llama_tokenize(vocab, "a", 1, tokens.data(), (int32_t) tokens.size(), false, false);+ }+ GGML_ASSERT(n_tok > 0);+ const llama_token tok = tokens[0];++ compare_penalties_logits(params, 80, 1.15f, 0.1f, 0.05f, "a", [tok](llama_sampler * smpl) {+ // accept_prompt already accepted BOS + one 'a'; fill the ring to n=80+ for (int i = 0; i < 78; ++i) {+ llama_sampler_accept(smpl, tok);+ }+ });++ printf("Testing backend penalties without filler entries\n");+ compare_penalties_logits(params, 64, 1.1f, 0.5f, 0.25f, "Hello", [](llama_sampler * smpl) {+ for (llama_token token = 0; token < 64; ++token) {+ llama_sampler_accept(smpl, token);+ }+ });++ printf("Testing backend top-k followed by penalties\n");+ compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello",+ penalties_position::after_filter);++ printf("Testing backend penalties followed by top-k\n");+ compare_top_k_penalties_logits(params, 8, 64, 1.1f, 0.5f, 0.25f, "Hello",+ penalties_position::before_filter);++ printf("Testing backend top-p followed by penalties\n");+ compare_masking_penalties_logits(params, "top-p", []() {+ return llama_sampler_init_top_p(0.9f, 0);+ }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true);++ printf("Testing backend top-p followed by penalties with a large history window\n");+ compare_masking_penalties_logits(params, "top-p large-window", []() {+ return llama_sampler_init_top_p(0.9f, 0);+ }, 4096, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true);++ printf("Testing backend penalties followed by top-p\n");+ compare_masking_penalties_logits(params, "top-p", []() {+ return llama_sampler_init_top_p(0.9f, 0);+ }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, true);++ printf("Testing backend min-p followed by penalties\n");+ compare_masking_penalties_logits(params, "min-p", []() {+ return llama_sampler_init_min_p(0.1f, 0);+ }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, false);++ printf("Testing backend penalties followed by min-p\n");+ compare_masking_penalties_logits(params, "min-p", []() {+ return llama_sampler_init_min_p(0.1f, 0);+ }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::before_filter, false);++ printf("Testing backend top-p followed by penalties with empty history\n");+ compare_masking_penalties_logits(params, "top-p empty", []() {+ return llama_sampler_init_top_p(0.9f, 0);+ }, 64, 1.1f, 0.5f, 0.25f, "Hello", penalties_position::after_filter, true, false);++ printf("Testing backend top-p followed by individual penalties\n");+ compare_masking_penalties_logits(params, "top-p repeat", []() {+ return llama_sampler_init_top_p(0.9f, 0);+ }, 64, 1.1f, 0.0f, 0.0f, "Hello", penalties_position::after_filter, true);+ compare_masking_penalties_logits(params, "top-p frequency", []() {+ return llama_sampler_init_top_p(0.9f, 0);+ }, 64, 1.0f, 0.5f, 0.0f, "Hello", penalties_position::after_filter, true);+ compare_masking_penalties_logits(params, "top-p presence", []() {+ return llama_sampler_init_top_p(0.9f, 0);+ }, 64, 1.0f, 0.0f, 0.25f, "Hello", penalties_position::after_filter, true);++ printf("Testing backend penalty parameter values\n");+ test_penalty_parameter_values(params);++ printf("backend penalties sampling test PASSED\n");+}+// This test verifies that it is possible to have two different backend samplers,// one that uses the backend dist sampler, and another that uses CPU dist sampler.static void test_backend_mixed_sampling(const test_params & params) {@@ -1014,6 +1574,7 @@ struct backend_test_case {static const backend_test_case BACKEND_TESTS[] = {{ "greedy", test_backend_greedy_sampling, true },{ "logit_bias", test_backend_logit_bias_sampling, true },+ { "penalties", test_backend_penalties_sampling, true },{ "temp", test_backend_temp_sampling, true },{ "temp_ext", test_backend_temp_ext_sampling, true },{ "top_k", test_backend_top_k_sampling, true },diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 4655b518e2..5d2798cc14 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -1807,7 +1807,8 @@ private:// initialize samplersif (task.need_sampling()) {try {- slot.smpl.reset(common_sampler_init(model_tgt, task.params.sampling));+ slot.smpl.reset(common_sampler_init(+ model_tgt, task.params.sampling, (int32_t) llama_n_ctx(ctx_tgt)));} catch (std::exception & e) {std::string err_msg = std::string("Failed to initialize samplers: ") + e.what();send_error(task, err_msg, ERROR_TYPE_INVALID_REQUEST);