LLM inference in C/C++
C++ ggml
server : accumulate generated text and tokens as parse input (#29876)
Aldehir Rojas18b5f8b1862e
diff --git a/common/chat.cpp b/common/chat.cppindex 1c1533325c..2c795c281f 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -1452,14 +1452,70 @@ common_chat_params common_chat_templates_apply(const struct common_chat_templatecommon_chat_templates_apply_legacy(tmpls, inputs);}-common_chat_msg common_chat_parse(const std::string & input,+void common_chat_input::append(const std::string & piece, llama_token token) {+ if (piece.empty()) {+ return;+ }+ tokens.push_back(token);+ tokens.resize(tokens.size() + piece.size() - 1, LLAMA_TOKEN_NULL);+ text += piece;+}++void common_chat_input::append(const common_chat_input & chunk) {+ tokens.insert(tokens.end(), chunk.tokens.begin(), chunk.tokens.end());+ text += chunk.text;+}++void common_chat_input::truncate(size_t pos) {+ if (pos < text.size()) {+ text.erase(pos);+ tokens.resize(pos);+ }+}++common_chat_input common_chat_input::substr(size_t pos, size_t n) const {+ common_chat_input out;+ out.text = text.substr(pos, n);+ out.tokens.assign(tokens.begin() + pos, tokens.begin() + pos + out.size());+ return out;+}++void common_chat_input::prepend(const std::string & prefix) {+ tokens.insert(tokens.begin(), prefix.size(), LLAMA_TOKEN_NULL);+ text = prefix + text;+}++void common_chat_input::prepend(const common_chat_input & prefix) {+ tokens.insert(tokens.begin(), prefix.tokens.begin(), prefix.tokens.end());+ text = prefix.text + text;+}++common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text) {+ common_chat_input input;+ auto tokens = common_tokenize(vocab, text, false, true);+ for (size_t i = 0; i < tokens.size(); i++) {+ std::string piece = common_token_to_piece(vocab, tokens[i], true);+ if (i == 0 && std::isspace(piece[0]) && !std::isspace(text[0])) {+ // Some tokenizers will add a space before the first special token, need to exclude+ continue;+ }+ input.append(piece, tokens[i]);+ }+ if (input.text != text) {+ // the pieces do not give back the same text, keep the text without tokens+ return common_chat_input(text);+ }+ return input;+}++common_chat_msg common_chat_parse(const common_chat_input & input,bool is_partial,const common_chat_parser_params & params) {return common_chat_peg_parse(params.parser, input, is_partial, params);}common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser,- const std::string & input,+ const common_chat_input & input,bool is_partial,const common_chat_parser_params & params) {const common_peg_arena & parser = src_parser.empty() ?@@ -1470,18 +1526,17 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parsLOG_DBG("No parser definition detected, assuming pure content parser.");}- const std::string effective_input = params.generation_prompt.empty()- ? input- : params.generation_prompt + input;+ common_chat_input effective_input = input;+ effective_input.prepend(params.generation_prompt);- //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.c_str());+ //LOG_DBG("Parsing PEG input with format %s: %s\n", common_chat_format_name(params.format), effective_input.text.c_str());common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_LENIENT;if (params.debug) {flags |= COMMON_PEG_PARSE_FLAG_DEBUG;}- common_peg_parse_context ctx(effective_input, flags);+ common_peg_parse_context ctx(std::move(effective_input.text), std::move(effective_input.tokens), flags);auto result = parser.parse(ctx);if (result.fail()) {@@ -1507,8 +1562,8 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars}return msg;}- LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), effective_input.substr(result.end).c_str());- LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), effective_input.c_str());+ LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), ctx.input.substr(result.end).c_str());+ LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), ctx.input.c_str());throw std::runtime_error(std::string("The model produced output that does not match the expected ") + common_chat_format_name(params.format) + " format");}diff --git a/common/chat.h b/common/chat.hindex 0e1423a5a3..4509a04cc1 100644--- a/common/chat.h+++ b/common/chat.h@@ -282,6 +282,31 @@ struct common_chat_params {common_chat_msg_delimiters message_delimiters;};+struct common_chat_input {+ std::string text;+ std::vector<llama_token> tokens;++ common_chat_input() = default;++ // plain text, with no tokens+ explicit common_chat_input(std::string text) : text(std::move(text)), tokens(this->text.size(), LLAMA_TOKEN_NULL) {}++ size_t size() const { return text.size(); }+ bool empty() const { return text.empty(); }++ void append(const std::string & piece, llama_token token);+ void append(const common_chat_input & chunk);++ void prepend(const std::string & prefix);+ void prepend(const common_chat_input & prefix);++ void truncate(size_t pos);++ common_chat_input substr(size_t pos, size_t n = std::string::npos) const;+};++common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const std::string & text);+// per-message parsing syntax// should be derived from common_chat_paramsstruct common_chat_parser_params {@@ -289,7 +314,7 @@ struct common_chat_parser_params {common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"// Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)bool reasoning_in_content = false;- std::string generation_prompt;+ common_chat_input generation_prompt;bool parse_tool_calls = true;bool is_continuation = false;bool echo = false; // Include assistant prefilled msg in output@@ -298,7 +323,7 @@ struct common_chat_parser_params {common_chat_parser_params() = default;common_chat_parser_params(const common_chat_params & chat_params) {format = chat_params.format;- generation_prompt = chat_params.generation_prompt;+ generation_prompt = common_chat_input(chat_params.generation_prompt);}};@@ -337,8 +362,8 @@ std::string common_chat_format_example(const struct common_chat_templates *const std::map<std::string, std::string> & chat_template_kwargs);const char * common_chat_format_name(common_chat_format format);-common_chat_msg common_chat_parse(const std::string & input, bool is_partial, const common_chat_parser_params & params);-common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const std::string & input, bool is_partial, const common_chat_parser_params & params);+common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);+common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);// used by arg and serverconst char * common_reasoning_format_name(common_reasoning_format format);diff --git a/common/peg-parser.h b/common/peg-parser.hindex 8883259457..816d007b06 100644--- a/common/peg-parser.h+++ b/common/peg-parser.h@@ -2,6 +2,7 @@#include "json-schema.h"#include "json.h"+#include "llama.h"#include <memory>#include <set>@@ -182,7 +183,8 @@ inline common_peg_parse_flags operator~(common_peg_parse_flags a) {}struct common_peg_parse_context {- std::string input;+ std::string input; // [h, e, l, l, o, _, w, o, r, l, d]+ std::vector<llama_token> tokens; // [id, -1, -1, -1, -1, id, -1, -1, -1, -1, -1]common_peg_parse_flags flags;common_peg_ast_arena ast;@@ -194,6 +196,11 @@ struct common_peg_parse_context {common_peg_parse_context(const std::string & input, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE): input(input), flags(flags), parse_depth(0) {}+ common_peg_parse_context(std::string input, std::vector<llama_token> tokens, common_peg_parse_flags flags = COMMON_PEG_PARSE_FLAG_NONE)+ : input(std::move(input)), tokens(std::move(tokens)), flags(flags), parse_depth(0) {+ GGML_ASSERT(this->tokens.empty() || this->tokens.size() == this->input.size());+ }+bool is_lenient() const { return flags & COMMON_PEG_PARSE_FLAG_LENIENT; }bool is_debug() const { return flags & COMMON_PEG_PARSE_FLAG_DEBUG; }};diff --git a/tests/test-chat.cpp b/tests/test-chat.cppindex 9f96e1084e..e31c093b0a 100644--- a/tests/test-chat.cpp+++ b/tests/test-chat.cpp@@ -1105,7 +1105,7 @@ struct make_peg_parser {common_chat_msg parse(const std::string & msg, bool is_partial) const {common_chat_parser_params parser_params(params_);parser_params.debug = detailed_debug_;- return common_chat_peg_parse(arena_, msg, is_partial, parser_params);+ return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params);}};@@ -4643,7 +4643,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {bool got_out_of_range = false;std::string error_msg;try {- common_chat_peg_parse(arena, bad_input, /*is_partial=*/false, pp);+ common_chat_peg_parse(arena, common_chat_input(bad_input), /*is_partial=*/false, pp);} catch (const std::out_of_range & e) {got_out_of_range = true;error_msg = e.what();diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex d1af984e67..23835e1858 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -267,7 +267,7 @@ struct server_slot {bool spec_is_replay = false;std::mt19937 spec_synth_rng;- // TODO: move members that belong to the task (such as `generated_text`, `has_new_line`) to task_results_state+ // TODO: move members that belong to the task (such as `generated`, `has_new_line`) to task_results_state// see https://github.com/ggml-org/llama.cpp/pull/18283#issuecomment-3710175837std::unique_ptr<const server_task> task;std::unique_ptr<const server_task> task_prev; // used for debugging@@ -285,7 +285,7 @@ struct server_slot {size_t last_nl_pos = 0;- std::string generated_text;+ common_chat_input generated;std::string debug_generated_text;llama_tokens generated_tokens;size_t n_sent_text = 0; // number of sent text character (i.e. handle partial UTF-8 on streaming)@@ -381,7 +381,7 @@ struct server_slot {spec_is_replay = false;last_nl_pos = 0;- generated_text = "";+ generated = {};has_new_line = false;truncated = false;stop = STOP_TYPE_NONE;@@ -750,7 +750,7 @@ struct server_slot {if (!only_metrics) {res["prompt"] = ptask->tokens.detokenize(ctx_tgt, true);- res["generated"] = generated_text.empty() ? debug_generated_text : generated_text;+ res["generated"] = generated.empty() ? debug_generated_text : generated.text;}}@@ -1968,28 +1968,26 @@ private:const std::string token_str = result.text_to_send;slot.sampled = result.tok;- slot.generated_text += token_str;+ slot.generated.append(token_str, result.tok);if (slot.task->params.return_tokens) {slot.generated_tokens.push_back(result.tok);}slot.has_next_token = true;// check if there is incomplete UTF-8 character at the end- bool incomplete = validate_utf8(slot.generated_text) < slot.generated_text.size();+ bool incomplete = validate_utf8(slot.generated.text) < slot.generated.text.size();// search stop word and delete itif (!incomplete) {- size_t pos = std::min(slot.n_sent_text, slot.generated_text.size());+ size_t pos = std::min(slot.n_sent_text, slot.generated.text.size());- const std::string str_test = slot.generated_text.substr(pos);+ const std::string str_test = slot.generated.text.substr(pos);bool send_text = true;size_t stop_pos = slot.find_stopping_strings(str_test, token_str.size(), true);if (stop_pos != std::string::npos) {- slot.generated_text.erase(- slot.generated_text.begin() + pos + stop_pos,- slot.generated_text.end());- pos = std::min(slot.n_sent_text, slot.generated_text.size());+ slot.generated.truncate(pos + stop_pos);+ pos = std::min(slot.n_sent_text, slot.generated.text.size());} else if (slot.has_next_token && !llama_vocab_is_eog(vocab, result.tok) ) {stop_pos = slot.find_stopping_strings(str_test, token_str.size(), false);send_text = stop_pos == std::string::npos;@@ -1998,7 +1996,7 @@ private:// check if there is any token to predictif (send_text) {// no send the stop word in the response- result.text_to_send = slot.generated_text.substr(pos, std::string::npos);+ result.text_to_send = slot.generated.text.substr(pos, std::string::npos);slot.n_sent_text += result.text_to_send.size();// add the token to slot queue and cache} else {@@ -2042,17 +2040,17 @@ private:size_t pos = slot.last_nl_pos;int n_indent = 0;- while (pos < slot.generated_text.size() && (slot.generated_text[pos] == ' ' || slot.generated_text[pos] == '\t')) {+ while (pos < slot.generated.text.size() && (slot.generated.text[pos] == ' ' || slot.generated.text[pos] == '\t')) {n_indent++;pos++;}- if (pos < slot.generated_text.size() && n_indent < slot.task->params.n_indent) {+ if (pos < slot.generated.text.size() && n_indent < slot.task->params.n_indent) {slot.stop = STOP_TYPE_LIMIT;slot.has_next_token = false;// cut the last line- slot.generated_text.erase(pos, std::string::npos);+ slot.generated.truncate(pos);SLT_DBG(slot, "stopped by indentation limit, n_gen = %d, n_indent = %d\n", (int) slot.stats.n_gen, n_indent);}@@ -2060,7 +2058,7 @@ private:// find the next new line{- const size_t pos = slot.generated_text.find('\n', slot.last_nl_pos);+ const size_t pos = slot.generated.text.find('\n', slot.last_nl_pos);if (pos != std::string::npos) {slot.last_nl_pos = pos + 1;@@ -2192,7 +2190,7 @@ private:if (is_begin) {res->is_begin = true;} else {- res->content = tkn.text_to_send;+ res->content = slot.generated.substr(slot.n_sent_text - tkn.text_to_send.size(), tkn.text_to_send.size());res->tokens.assign(1, tkn.tok);}@@ -2229,15 +2227,15 @@ private:// keep copy of last generated text for debugging purposesif (slots_debug) {- slot.debug_generated_text = slot.generated_text;+ slot.debug_generated_text = slot.generated.text;}// in stream mode, content and tokens are already in last partial chunkif (slot.task->params.stream) {- res->content = "";+ res->content = {};res->tokens = llama_tokens{};} else {- res->content = std::move(slot.generated_text);+ res->content = std::move(slot.generated);res->tokens = std::move(slot.generated_tokens);}res->stats = slot.stats;diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cppindex 27ecafb7a5..c7d0bc4de2 100644--- a/tools/server/server-schema.cpp+++ b/tools/server/server-schema.cpp@@ -311,8 +311,14 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &->set_desc("Generation prompt appended to the chat template output")->set_handler([&](field_eval_context & ctx, const json & data) {std::string s = data.at("generation_prompt").get<std::string>();- ctx.params.chat_parser_params.generation_prompt = s;ctx.params.sampling.generation_prompt = s;++ if (ctx.vocab == nullptr) {+ ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);+ return;+ }++ ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);}));add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex a5c33c056b..21c1fe4e10 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -76,7 +76,7 @@ json task_params::to_json(bool only_metrics) const {{"chat_format", common_chat_format_name(chat_parser_params.format)},{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},{"reasoning_in_content", chat_parser_params.reasoning_in_content},- {"generation_prompt", chat_parser_params.generation_prompt},+ {"generation_prompt", chat_parser_params.generation_prompt.text},{"samplers", samplers},{"speculative.types", common_speculative_type_name_str(speculative.types)},{"timings_per_token", timings_per_token},@@ -135,7 +135,7 @@ json task_params::to_json(bool only_metrics) const {{"chat_format", common_chat_format_name(chat_parser_params.format)},{"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},{"reasoning_in_content", chat_parser_params.reasoning_in_content},- {"generation_prompt", chat_parser_params.generation_prompt},+ {"generation_prompt", chat_parser_params.generation_prompt.text},{"samplers", samplers},{"speculative.types", common_speculative_type_name_str(speculative.types)},{"timings_per_token", timings_per_token},@@ -155,20 +155,20 @@ task_result_state::task_result_state(const common_chat_parser_params & chat_pars, oai_resp_message_id("msg_" + random_string()) {if (chat_parser_params.is_continuation && !chat_parser_params.echo) {// initialize chat_msg to avoid emitting a delta containing the assistant prefill- chat_msg = common_chat_parse("", true, chat_parser_params);+ chat_msg = common_chat_parse(generated_input, true, chat_parser_params);}}common_chat_msg task_result_state::update_chat_msg(- const std::string & text_added,+ const common_chat_input & added,bool is_partial,std::vector<common_chat_msg_diff> & diffs,bool filter_tool_calls) {- generated_text += text_added;+ generated_input.append(added);auto msg_prv_copy = chat_msg;- //SRV_DBG("Parsing chat message: %s\n", generated_text.c_str());+ //SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());auto new_msg = common_chat_parse(- generated_text,+ generated_input,is_partial,chat_parser_params);if (!new_msg.empty()) {@@ -340,7 +340,7 @@ json server_task_result_cmpl_final::to_json() {json server_task_result_cmpl_final::to_json_non_oaicompat() {json res = json {{"index", index},- {"content", content},+ {"content", content.text},{"tokens", tokens},{"id_slot", id_slot},{"stop", true},@@ -386,7 +386,7 @@ json server_task_result_cmpl_final::to_json_oaicompat() {json res = json {{"choices", json::array({json{- {"text", content},+ {"text", content.text},{"index", index},{"logprobs", logprobs},{"finish_reason", finish_reason},@@ -418,7 +418,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_chat() {msg = oaicompat_msg;} else {msg.role = "assistant";- msg.content = content;+ msg.content = content.text;}if (stop == STOP_TYPE_WORD || stop == STOP_TYPE_EOS) {finish_reason = msg.tool_calls.empty() ? "stop" : "tool_calls";@@ -531,7 +531,7 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {msg = oaicompat_msg;} else {msg.role = "assistant";- msg.content = content;+ msg.content = content.text;}std::vector<json> output;@@ -741,7 +741,7 @@ json server_task_result_cmpl_final::to_json_anthropic() {msg = oaicompat_msg;} else {msg.role = "assistant";- msg.content = content;+ msg.content = content.text;}// thinking block comes first (Anthropic extended thinking format)@@ -1050,7 +1050,7 @@ json server_task_result_cmpl_partial::to_json_non_oaicompat() {// non-OAI-compat JSONjson res = json {{"index", index},- {"content", content},+ {"content", content.text},{"tokens", tokens},{"stop", false},{"id_slot", id_slot},@@ -1081,7 +1081,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat() {json res = json {{"choices", json::array({json{- {"text", content},+ {"text", content.text},{"index", index},{"logprobs", logprobs},{"finish_reason", nullptr},@@ -1316,7 +1316,7 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {json server_task_result_cmpl_partial::to_json_oaicompat_asr() {json event = json {{"type", "transcript.text.delta"},- {"delta", content},+ {"delta", content.text},};return event;}diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex 752595b0fb..552c03b075 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -108,7 +108,7 @@ struct task_result_state {std::vector<common_chat_msg_diff> diffs;common_chat_parser_params chat_parser_params;common_chat_msg chat_msg;- std::string generated_text; // append new chunks of generated text here+ common_chat_input generated_input; // append new chunks of generated text herestd::vector<std::string> generated_tool_call_ids;std::unordered_set<size_t> sent_tool_call_names;@@ -128,7 +128,7 @@ struct task_result_state {// parse partial tool calls and update the internal statecommon_chat_msg update_chat_msg(- const std::string & text_added,+ const common_chat_input & added,bool is_partial,std::vector<common_chat_msg_diff> & diffs,bool filter_tool_calls = false);@@ -350,7 +350,7 @@ struct completion_token_output {};struct server_task_result_cmpl_final : server_task_result {- std::string content;+ common_chat_input content;llama_tokens tokens;bool stream;@@ -425,8 +425,8 @@ struct server_task_result_cmpl_final : server_task_result {};struct server_task_result_cmpl_partial : server_task_result {- std::string content;- llama_tokens tokens;+ common_chat_input content;+ llama_tokens tokens;int32_t n_decoded;int32_t n_prompt_tokens;