LLM inference in C/C++
C++ ggml
chat : refactor API (#30210)
Aldehir Rojas8b543610254a
diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cppindex 9e3b6d6407..1112738af1 100644--- a/common/chat-auto-parser-generator.cpp+++ b/common/chat-auto-parser-generator.cpp@@ -61,8 +61,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &data.prompt += data.generation_prompt;}- auto parser = autoparser.build_parser(inputs, parser_generation_prompt);- data.parser = parser.save();+ data.parser = autoparser.build_parser(inputs, parser_generation_prompt);// Build grammar if tools are presentbool has_tools =@@ -78,7 +77,7 @@ common_chat_params peg_generator::generate_parser(const common_chat_template &if (include_grammar) {data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});// Set grammar triggers based on tool section markers (fall back to per-call markers)diff --git a/common/chat.cpp b/common/chat.cppindex c8f156a70e..87f1e99a2b 100644--- a/common/chat.cpp+++ b/common/chat.cpp@@ -9,6 +9,7 @@#include "json.h"#include "log.h"#include "parsers/parsers.h"+#include "sampling.h"#include "jinja/value.h"#include "jinja/runtime.h"@@ -112,38 +113,6 @@ const char * common_chat_role_to_string(common_chat_role role) {return "";}-json common_chat_msg_delimiters::to_json() const {- json result = json::array();- for (const auto & d : delimiters) {- result.push_back({- { "role", common_chat_role_to_string(d.role) },- { "delimiter", d.delimiter },- });- }- return result;-}--common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) {- common_chat_msg_delimiters result;-- if (!delimiters.is_array()) {- return result;- }-- result.delimiters.reserve(delimiters.size());- for (const auto & d : delimiters) {- if (!d.is_object()) {- continue;- }- result.delimiters.push_back({- common_chat_role_from_string(d.value("role", std::string())),- d.value("delimiter", std::string()),- });- }-- return result;-}-void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) {for (auto & d : delimiters) {d.tokens = common_tokenize(vocab, d.delimiter, false, true);@@ -620,8 +589,11 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too}common_chat_continuation common_chat_continuation_parse(const common_json & value) {- if (value.is_boolean() && value.get<bool>()) {- return COMMON_CHAT_CONTINUATION_AUTO;+ if (value.is_null()) {+ return COMMON_CHAT_CONTINUATION_NONE;+ }+ if (value.is_boolean()) {+ return value.get<bool>() ? COMMON_CHAT_CONTINUATION_AUTO : COMMON_CHAT_CONTINUATION_NONE;}if (value.is_string()) {auto value_str = value.get<std::string>();@@ -632,7 +604,7 @@ common_chat_continuation common_chat_continuation_parse(const common_json & valureturn COMMON_CHAT_CONTINUATION_CONTENT;}}- return COMMON_CHAT_CONTINUATION_NONE;+ throw std::invalid_argument("Invalid continue_final_message: expected a boolean, \"content\" or \"reasoning_content\"");}bool common_chat_verify_template(const std::string & tmpl, bool use_jinja) {@@ -1087,41 +1059,55 @@ static json common_chat_extra_context() {return ctx;}-std::optional<common_chat_params> common_chat_try_specialized_template(- const common_chat_template & tmpl,- const std::string & src,- autoparser::generation_params & params) {+static common_chat_params common_chat_params_init_lfm2_tokens(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {+ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ true);+}++static common_chat_params common_chat_params_init_lfm2_5(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {+ return common_chat_params_init_lfm2(tmpl, inputs, /* tool_list_tokens = */ false);+}++// Older gemma4 templates need their tool responses rewritten before rendering+static common_chat_params common_chat_params_init_gemma4_legacy(const common_chat_template & tmpl, const autoparser::generation_params & inputs) {+ auto adjusted = inputs;+ workaround::convert_tool_responses_gemma4(adjusted.messages);+ return common_chat_params_init_gemma4(tmpl, adjusted);+}++// Pick the dedicated handler for a template from its source, or null for the autoparser.+// Order matters: the first match wins, and later checks assume the earlier ones did not match.+static common_chat_params_init_fn common_chat_template_detect_params_init(const std::string & src) {// Ministral/Mistral Large 3 - uses special reasoning structure fixes, can't use autoparser// Note: Mistral Small 3.2 uses [CALL_ID] which Ministral doesn't have, so we can distinguish themif (src.find("[SYSTEM_PROMPT]") != std::string::npos && src.find("[TOOL_CALLS]") != std::string::npos &&src.find("[ARGS]") != std::string::npos && src.find("[CALL_ID]") == std::string::npos) {LOG_DBG("Using specialized template: Ministral/Magistral Large 3\n");- return common_chat_params_init_ministral_3(tmpl, params);+ return common_chat_params_init_ministral_3;}// LLM-jp-4.1 - GPT-OSS dialect (spaces after special tokens, <|end|>-separated parallel calls)if (src.find("chat_format=llm-jp-harmony-v1") != std::string::npos) {LOG_DBG("Using specialized template: LLM-jp Harmony v1\n");- return common_chat_params_init_llm_jp_harmony(tmpl, params);+ return common_chat_params_init_llm_jp_harmony;}// GPT-OSS - has unique channel-based structure that needs dedicated handlerif (src.find("<|channel|>") != std::string::npos) {LOG_DBG("Using specialized template: GPT-OSS\n");- return common_chat_params_init_gpt_oss(tmpl, params);+ return common_chat_params_init_gpt_oss;}// Muse Glimmer format using " to=<recipient>" recipients and <|eom|>/<|eot|> message terminators.if (src.find("<atem:function_calls>") != std::string::npos && src.find("<|eom|>") != std::string::npos) {LOG_DBG("Using specialized template: Muse Glimmer\n");- return common_chat_params_init_muse_glimmer(tmpl, params);+ return common_chat_params_init_muse_glimmer;}// Functionary v3.2 - uses recipient-based format with >>>recipient\n{content}// Detection: template has ">>>all" for content and ">>>" prefix for tool callsif (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) {LOG_DBG("Using specialized template: Functionary v3.2\n");- return common_chat_params_init_functionary_v3_2(tmpl, params);+ return common_chat_params_init_functionary_v3_2;}// Kimi K2 Thinking - uses unique tool call ID format: functions.<name>:<index>@@ -1129,14 +1115,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(if (src.find("<|tool_calls_section_begin|>") != std::string::npos &&src.find("<|tool_call_begin|>") != std::string::npos) {LOG_DBG("Using specialized template: Kimi K2 Thinking\n");- return common_chat_params_init_kimi_k2(tmpl, params);+ return common_chat_params_init_kimi_k2;}// Kimi K3 - the <|open|>/<|close|>/<|end_of_msg|> markers are unique to itif (src.find("<|open|>") != std::string::npos && src.find("<|close|>") != std::string::npos &&src.find("<|end_of_msg|>") != std::string::npos) {LOG_DBG("Using specialized template: Kimi K3\n");- return common_chat_params_init_kimi_k3(tmpl, params);+ return common_chat_params_init_kimi_k3;}// K2 Horizon - <|ifm|im_start|> turns, <ifm|think*> reasoning picked by reasoning_effort and@@ -1144,7 +1130,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(if (src.find("<|ifm|im_start|>") != std::string::npos &&src.find("<ifm|tool_calls>") != std::string::npos) {LOG_DBG("Using specialized template: K2 Horizon\n");- return common_chat_params_init_k2_horizon(tmpl, params);+ return common_chat_params_init_k2_horizon;}// Ling 3.0 / Bailing V3 - <role>X</role> sections with <arg_key>/<arg_value> tagged@@ -1152,7 +1138,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(if (src.find("<role>ASSISTANT</role>") != std::string::npos &&src.find("<arg_key>") != std::string::npos) {LOG_DBG("Using specialized template: Ling 3.0 (Bailing V3)\n");- return common_chat_params_init_ling3(tmpl, params);+ return common_chat_params_init_ling3;}// Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and@@ -1161,19 +1147,19 @@ std::optional<common_chat_params> common_chat_try_specialized_template(if (src.find("<|START_TEXT|>") != std::string::npos &&src.find("<|START_ACTION|>") != std::string::npos) {LOG_DBG("Using specialized template: Cohere2 MoE\n");- return common_chat_params_init_cohere2moe(tmpl, params);+ return common_chat_params_init_cohere2moe;}if (is_lfm2_template(src)) {LOG_DBG("Using specialized template: LFM2\n");- return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);+ return common_chat_params_init_lfm2_tokens;}// LFM2.5 format detection: template uses plain "List of tools: [...]" with no special tokensif (src.find("List of tools: [") != std::string::npos &&src.find("<|tool_list_start|>") == std::string::npos) {LOG_DBG("Using specialized template: LFM2.5\n");- return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ false);+ return common_chat_params_init_lfm2_5;}// GigaChatV3 format detection@@ -1181,7 +1167,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(src.find("<|message_sep|>") != std::string::npos &&src.find("<|function_call|>") == std::string::npos) {LOG_DBG("Using specialized template: GigaChatV3\n");- return common_chat_params_init_gigachat_v3(tmpl, params);+ return common_chat_params_init_gigachat_v3;}// MiniMax-M3: the namespace token "]<]minimax[>[" collides with the autoparser's@@ -1190,7 +1176,7 @@ std::optional<common_chat_params> common_chat_try_specialized_template(src.find("<tool_call>") != std::string::npos &&src.find("<invoke name=") != std::string::npos) {LOG_DBG("Using specialized template: MiniMax-M3\n");- return common_chat_params_init_minimax_m3(tmpl, params);+ return common_chat_params_init_minimax_m3;}// DeepSeek V3.2/V4 format detection: template defines dsml_token and uses it for tool calls.@@ -1201,18 +1187,18 @@ std::optional<common_chat_params> common_chat_try_specialized_template((src.find("function_calls") != std::string::npos ||src.find("tool_calls") != std::string::npos)) {LOG_DBG("Using specialized template: DeepSeek V3.2/V4\n");- return common_chat_params_init_deepseek_v3_2(tmpl, params);+ return common_chat_params_init_deepseek_v3_2;}// Gemma4 format detectionif (src.find("'<|tool_call>call:'") != std::string::npos) {+ LOG_DBG("Using specialized template: Gemma4\n");if (src.find("{#- OpenAI Chat Completions:") == std::string::npos) {- // apply workarounds if using the older gemma4 templatesLOG_WRN("%s: detected an outdated gemma4 chat template, applying compatibility workarounds. ""Consider updating to the official template.\n", __func__);- workaround::convert_tool_responses_gemma4(params.messages);+ return common_chat_params_init_gemma4_legacy;}- return common_chat_params_init_gemma4(tmpl, params);+ return common_chat_params_init_gemma4;}// MiniCPM5 - XML tool calls with <function name="..."><param name="...">...</param></function>@@ -1220,14 +1206,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(src.find("<function name=\"") != std::string::npos &&src.find("<param name=\"") != std::string::npos) {LOG_DBG("Using specialized template: MiniCPM5\n");- return common_chat_params_init_minicpm5(tmpl, params);+ return common_chat_params_init_minicpm5;}// TranslateGemma - user content must follow a custom schema with language codesif (src.find("[source_lang_code]") != std::string::npos &&src.find("[target_lang_code]") != std::string::npos) {LOG_DBG("Using specialized template: TranslateGemma\n");- return common_chat_params_init_translate_gemma(tmpl, params);+ return common_chat_params_init_translate_gemma;}// Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash@@ -1237,10 +1223,51 @@ std::optional<common_chat_params> common_chat_try_specialized_template(// Exclude models that don't use \n between tagssrc.find("'<tool_call><function=' ~ tool_call.name ~ '>'") == std::string::npos) {LOG_DBG("Using specialized template: Qwen3-Coder\n");- return common_chat_params_init_qwen3_coder(tmpl, params);+ return common_chat_params_init_qwen3_coder;+ }++ return nullptr;+}++common_chat_template::common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {+ jinja::lexer lexer;+ auto lexer_res = lexer.tokenize(src);+ this->prog = jinja::parse_from_tokens(lexer_res);++ this->src = lexer_res.source;+ this->bos_tok = bos_token;+ this->eos_tok = eos_token;++ this->caps = jinja::caps_get(prog);+ // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());++ this->params_init = common_chat_template_detect_params_init(this->src);+ if (this->params_init) {+ return;+ }++ // The analysis depends only on the template, so run it once here instead of on every apply.+ // A failure is kept for apply to report, so a bad template still loads like it did before.+ try {+ analysis = std::make_unique<autoparser::autoparser>();+ analysis->analyze_template(*this);+ } catch (const std::exception & e) {+ analysis.reset();+ analysis_error = e.what();}+}++common_chat_template::~common_chat_template() = default;+common_chat_template::common_chat_template(common_chat_template &&) = default;+common_chat_template & common_chat_template::operator=(common_chat_template &&) = default;- return std::nullopt;+std::optional<common_chat_params> common_chat_try_specialized_template(+ const common_chat_template & tmpl,+ const autoparser::generation_params & params) {+ if (!tmpl.params_init) {+ return std::nullopt;+ }+ return tmpl.params_init(tmpl, params);}static common_chat_params common_chat_templates_apply_jinja(const struct common_chat_templates * tmpls,@@ -1342,21 +1369,23 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_data.prompt = common_chat_template_direct_apply_impl(tmpl, params_copy);data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, params);data.format = COMMON_CHAT_FORMAT_PEG_NATIVE;- auto parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {+ data.parser = build_chat_peg_parser([&data](common_chat_peg_builder &p) {return p.literal(data.generation_prompt) << p.content(p.rest());});- data.parser = parser.save();return data;}- if (auto result = common_chat_try_specialized_template(tmpl, src, params)) {+ if (auto result = common_chat_try_specialized_template(tmpl, params)) {return *result;}+ if (!tmpl.analysis) {+ throw std::invalid_argument("Unable to generate parser for this template. Automatic parser generation failed: " + tmpl.analysis_error);+ }+try {LOG_DBG("%s: using differential autoparser\n", __func__);- struct autoparser::autoparser autoparser;- autoparser.analyze_template(tmpl);+ const auto & autoparser = *tmpl.analysis;auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser);common_chat_msg_delimiters delimiters;@@ -1377,8 +1406,7 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_auto_params.thinking_end_tags = {std::move(end_tag)};}}- common_peg_arena arena;- arena.load(auto_params.parser);+ const auto & arena = auto_params.parser;LOG_DBG("%s: generated parser:\n%s\n\nparser generation prompt: %s\n", __func__, arena.dump(arena.root()).c_str(), auto_params.generation_prompt.c_str());return auto_params;} catch (const std::exception & e) {@@ -1525,9 +1553,10 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parsconst common_chat_input & input,bool is_partial,const common_chat_parser_params & params) {- const common_peg_arena & parser = src_parser.empty() ?- build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); }) :- src_parser;+ // both branches must be lvalues, a temporary here would copy the arena on every call+ static const common_peg_arena content_only =+ build_chat_peg_parser([](common_chat_peg_builder & p) { return p.content(p.rest()) + p.end(); });+ const common_peg_arena & parser = src_parser.empty() ? content_only : src_parser;if (src_parser.empty()) {LOG_DBG("No parser definition detected, assuming pure content parser.");@@ -1598,6 +1627,84 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parsreturn msg;}+common_chat_session::common_chat_session(const common_chat_templates * tmpls,+ const llama_vocab * vocab,+ const common_chat_templates_inputs & inputs,+ const common_chat_session_params & params) {+ auto applied = common_chat_templates_apply(tmpls, inputs);++ templated = true;+ prompt_text = std::move(applied.prompt);+ result.role = "assistant";++ grammar_text = std::move(applied.grammar);+ grammar_lazy = applied.grammar_lazy;+ stops = std::move(applied.additional_stops);+ generation_prompt_text = applied.generation_prompt;+ thinking_start = std::move(applied.thinking_start_tag);+ thinking_ends = std::move(applied.thinking_end_tags);++ parser_params.format = applied.format;+ parser_params.generation_prompt = vocab ? common_chat_input_tokenize(vocab, applied.generation_prompt)+ : common_chat_input(applied.generation_prompt);+ parser_params.debug = params.debug;+ parser_params.parser = std::move(applied.parser);++ delimiters = std::move(applied.message_delimiters);++ if (vocab) {+ common_params_sampling resolved;+ resolved.grammar_lazy = applied.grammar_lazy;+ common_sampling_add_preserved_tokens(resolved, vocab, applied.preserved_tokens);+ common_sampling_add_grammar_triggers(resolved, vocab, std::move(applied.grammar_triggers));+ preserved_tokens = std::move(resolved.preserved_tokens);+ grammar_triggers = std::move(resolved.grammar_triggers);++ delimiters.tokenize(vocab);+ } else {+ grammar_triggers = std::move(applied.grammar_triggers);+ }++ if (inputs.continue_final_message != COMMON_CHAT_CONTINUATION_NONE && !params.echo) {+ // start from the prefill so it is not emitted as part of the first delta+ result = common_chat_parse(input, true, parser_params);+ }+}++void common_chat_session::apply_sampling(common_params_sampling & sampling) const {+ if (!templated) {+ return;+ }+ if (!grammar_text.empty()) {+ sampling.grammar = {COMMON_GRAMMAR_TYPE_TOOL_CALLS, grammar_text};+ }+ sampling.grammar_lazy = grammar_lazy;+ sampling.generation_prompt = generation_prompt_text;+ sampling.preserved_tokens.insert(preserved_tokens.begin(), preserved_tokens.end());+ sampling.grammar_triggers.insert(sampling.grammar_triggers.end(), grammar_triggers.begin(), grammar_triggers.end());+}++const common_chat_msg & common_chat_session::feed(const common_chat_input & chunk) {+ GGML_ASSERT(!finished && "feed() after finish()");+ input.append(chunk);+ auto msg = common_chat_parse(input, true, parser_params);+ if (!msg.empty()) {+ result = std::move(msg);+ }+ return result;+}++const common_chat_msg & common_chat_session::finish(const common_chat_input & chunk) {+ GGML_ASSERT(!finished && "finish() called twice");+ finished = true;+ input.append(chunk);+ auto msg = common_chat_parse(input, false, parser_params);+ if (!msg.empty()) {+ result = std::move(msg);+ }+ return result;+}+std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {GGML_ASSERT(chat_templates != nullptr);GGML_ASSERT(chat_templates->template_default != nullptr);diff --git a/common/chat.h b/common/chat.hindex 4509a04cc1..7753349242 100644--- a/common/chat.h+++ b/common/chat.h@@ -22,8 +22,16 @@ struct common_chat_templates;namespace autoparser {struct generation_params;+struct autoparser;} // namespace autoparser+struct common_chat_params;+struct common_chat_template;++// Builds the prompt and parser for a template that has a dedicated handler (see common/parsers)+using common_chat_params_init_fn = common_chat_params (*)(const common_chat_template & tmpl,+ const autoparser::generation_params & inputs);+struct common_chat_tool_call {std::string name;std::string arguments;@@ -54,19 +62,20 @@ struct common_chat_template {std::string eos_tok;std::string src;chat_template_caps caps;+ // Dedicated handler picked once from the source, null when the differential autoparser is used+ common_chat_params_init_fn params_init = nullptr;- common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token) {- jinja::lexer lexer;- auto lexer_res = lexer.tokenize(src);- this->prog = jinja::parse_from_tokens(lexer_res);+ // Differential analysis, run once here when there is no dedicated handler. Null when there+ // is one, or when the analysis failed, in which case analysis_error says why.+ std::unique_ptr<autoparser::autoparser> analysis;+ std::string analysis_error;- this->src = lexer_res.source;- this->bos_tok = bos_token;- this->eos_tok = eos_token;+ common_chat_template(const std::string & src, const std::string & bos_token, const std::string & eos_token);- this->caps = jinja::caps_get(prog);- // LOG_INF("%s: caps:\n%s\n", __func__, this->caps.to_string().c_str());- }+ // autoparser is incomplete here, so these are defined where it is complete+ ~common_chat_template();+ common_chat_template(common_chat_template &&);+ common_chat_template & operator=(common_chat_template &&);const std::string & source() const { return src; }const std::string & bos_token() const { return bos_tok; }@@ -209,8 +218,6 @@ struct common_chat_msg_delimiters {// split tokens into message spans. skips maps a start index to a length of a region to jump over without matchingcommon_chat_msg_spans split(const llama_tokens & tokens, const std::map<size_t, size_t> & skips = {}) const;-- common_json to_json() const;};struct common_chat_tool {@@ -278,7 +285,7 @@ struct common_chat_params {std::vector<common_grammar_trigger> grammar_triggers;std::vector<std::string> preserved_tokens;std::vector<std::string> additional_stops;- std::string parser;+ common_peg_arena parser;common_chat_msg_delimiters message_delimiters;};@@ -310,16 +317,10 @@ common_chat_input common_chat_input_tokenize(const llama_vocab * vocab, const st// per-message parsing syntax// should be derived from common_chat_paramsstruct common_chat_parser_params {- common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;- common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE; // TODO: refactor this to "bool parse_reasoning"- // Whether reasoning_content should be inlined in the content (e.g. for reasoning_format=deepseek in stream mode)- bool reasoning_in_content = false;- common_chat_input generation_prompt;- bool parse_tool_calls = true;- bool is_continuation = false;- bool echo = false; // Include assistant prefilled msg in output- bool debug = false; // Enable debug output for PEG parser- common_peg_arena parser = {};+ common_chat_format format = COMMON_CHAT_FORMAT_CONTENT_ONLY;+ common_chat_input generation_prompt;+ bool debug = false; // Enable debug output for PEG parser+ common_peg_arena parser = {};common_chat_parser_params() = default;common_chat_parser_params(const common_chat_params & chat_params) {format = chat_params.format;@@ -365,6 +366,60 @@ const char * common_chat_format_name(common_chat_format format);common_chat_msg common_chat_parse(const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);common_chat_msg common_chat_peg_parse(const common_peg_arena & src_parser, const common_chat_input & input, bool is_partial, const common_chat_parser_params & params);+struct common_chat_session_params {+ bool echo = false; // include the assistant prefill in the output when continuing a message+ bool debug = false; // enable debug output for the PEG parser+};++class common_chat_session {+ public:+ common_chat_session() { result.role = "assistant"; }++ common_chat_session(const common_chat_templates * tmpls,+ const llama_vocab * vocab,+ const common_chat_templates_inputs & inputs,+ const common_chat_session_params & params = {});++ const std::string & prompt() const { return prompt_text; }+ common_chat_format format() const { return parser_params.format; }+ const common_chat_msg & msg() const { return result; }+ const common_peg_arena & parser() const { return parser_params.parser; }++ const std::string & grammar() const { return grammar_text; }+ const std::string & generation_prompt() const { return generation_prompt_text; }+ const std::string & thinking_start_tag() const { return thinking_start; }+ const std::vector<std::string> & thinking_end_tags() const { return thinking_ends; }+ const std::vector<std::string> & additional_stops() const { return stops; }++ const common_chat_msg_delimiters & message_delimiters() const { return delimiters; }++ void apply_sampling(common_params_sampling & sampling) const;++ bool has_template() const { return templated; }++ const common_chat_msg & feed(const common_chat_input & chunk);++ const common_chat_msg & finish(const common_chat_input & chunk = {});++ private:+ std::string prompt_text;+ std::string grammar_text;+ bool grammar_lazy = false;+ std::vector<common_grammar_trigger> grammar_triggers;+ std::set<llama_token> preserved_tokens;+ std::vector<std::string> stops;+ std::string generation_prompt_text;+ std::string thinking_start;+ std::vector<std::string> thinking_ends;++ common_chat_parser_params parser_params;+ common_chat_msg_delimiters delimiters;+ common_chat_input input;+ common_chat_msg result;+ bool templated = false;+ bool finished = false;+};+// used by arg and serverconst char * common_reasoning_format_name(common_reasoning_format format);common_reasoning_format common_reasoning_format_from_name(const std::string & format);@@ -401,8 +456,7 @@ std::string common_chat_template_generation_prompt(std::optional<common_chat_params> common_chat_try_specialized_template(const common_chat_template & tmpl,- const std::string & src,- autoparser::generation_params & params);+ const autoparser::generation_params & params);// specialized per-task preset@@ -412,5 +466,3 @@ struct common_chat_prompt_preset {};common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);--common_chat_msg_delimiters common_chat_msg_delimiters_parse(const common_json & delimiters);diff --git a/common/parsers/cohere2moe.cpp b/common/parsers/cohere2moe.cppindex 59595368dc..dde2da929d 100644--- a/common/parsers/cohere2moe.cpp+++ b/common/parsers/cohere2moe.cpp@@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_templatedata.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.literal(GEN_PREFIX);auto end = p.end();@@ -124,12 +124,10 @@ common_chat_params common_chat_params_init_cohere2moe(const common_chat_templatereturn generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !has_response_format && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cppindex 88122eaab7..61b2b66c6b 100644--- a/common/parsers/deepseek.cpp+++ b/common/parsers/deepseek.cpp@@ -145,7 +145,7 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templbool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED;bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.literal(GEN_PROMPT);auto end = p.end();@@ -256,12 +256,10 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templgeneration_prompt + reasoning + content_before_tools + tool_calls + end;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = has_tools && !require_tools;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/functionary-v3-2.cpp b/common/parsers/functionary-v3-2.cppindex 56f7643f29..3a1eff3d5f 100644--- a/common/parsers/functionary-v3-2.cpp+++ b/common/parsers/functionary-v3-2.cpp@@ -21,7 +21,7 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_tedata.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {// Functionary v3.2 format:// - Normal content: >>>all\n{content}// - Tool calls: >>>function_name\n{json_args}@@ -76,13 +76,11 @@ common_chat_params common_chat_params_init_functionary_v3_2(const common_chat_tereturn generation_prompt + ret;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});// Grammar trigger for when the model starts outputting a tool calldiff --git a/common/parsers/gemma4.cpp b/common/parsers/gemma4.cppindex 2223089da7..5ba2648890 100644--- a/common/parsers/gemma4.cpp+++ b/common/parsers/gemma4.cpp@@ -198,7 +198,7 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto start = p.rule("start", p.optional(p.literal("<|turn>model\n")));if (extract_reasoning) {@@ -290,12 +290,10 @@ common_chat_params common_chat_params_init_gemma4(const common_chat_template &return start + p.one_or_more(message);});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/gigachat-v3.cpp b/common/parsers/gigachat-v3.cppindex 9499425474..0d20f83f7d 100644--- a/common/parsers/gigachat-v3.cpp+++ b/common/parsers/gigachat-v3.cpp@@ -25,7 +25,7 @@ common_chat_params common_chat_params_init_gigachat_v3(auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE;const auto *tool_call_start_prefix = "<|message_sep|>\n\nfunction call<|role_sep|>\n";- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto ret = p.eps();if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) {// Build a choice of all available tools@@ -60,13 +60,11 @@ common_chat_params common_chat_params_init_gigachat_v3(return p.literal("assistant<|role_sep|>\n") + ret;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/gpt-oss.cpp b/common/parsers/gpt-oss.cppindex ed50524a99..b91c61d2ae 100644--- a/common/parsers/gpt-oss.cpp+++ b/common/parsers/gpt-oss.cpp@@ -45,8 +45,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &data.thinking_start_tag = "<|channel|>analysis<|message|>";data.thinking_end_tags = {"<|end|>"};- // These special tokens are required to parse properly, so we include them- // even if parse_tool_calls is false.+ // These special tokens are required to parse properlydata.preserved_tokens = {"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",};@@ -68,7 +67,7 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto start = p.rule("start", p.literal("<|start|>assistant"));auto end = p.rule("end", p.literal("<|end|>"));auto content = p.rule("message-content", p.until("<|end|>"));@@ -138,12 +137,10 @@ common_chat_params common_chat_params_init_gpt_oss(const common_chat_template &return p.zero_or_more(start + any) + start + (final_msg | unsolicited);});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/k2-horizon.cpp b/common/parsers/k2-horizon.cppindex bd64f68900..b73734fbdc 100644--- a/common/parsers/k2-horizon.cpp+++ b/common/parsers/k2-horizon.cpp@@ -77,7 +77,7 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_templatedata.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.literal(GEN_PREFIX);auto think_end = p.choice();@@ -174,12 +174,10 @@ common_chat_params common_chat_params_init_k2_horizon(const common_chat_templatereturn generation_prompt + (reasoning << content << tool_calls);});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED);data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});if (data.grammar_lazy) {diff --git a/common/parsers/kimi-k2.cpp b/common/parsers/kimi-k2.cppindex 9992968a5a..5e9d49ef9d 100644--- a/common/parsers/kimi-k2.cpp+++ b/common/parsers/kimi-k2.cpp@@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &data.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {// Kimi K2 Thinking format:// - Reasoning: <think>{reasoning}</think>// - Content: text after reasoning@@ -111,12 +111,10 @@ common_chat_params common_chat_params_init_kimi_k2(const common_chat_template &return generation_prompt + reasoning + content_before_tools + tool_calls + end;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/kimi-k3.cpp b/common/parsers/kimi-k3.cppindex cf153f6b5a..cee1c16716 100644--- a/common/parsers/kimi-k3.cpp+++ b/common/parsers/kimi-k3.cpp@@ -66,7 +66,7 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &data.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto end = p.end();auto start = p.optional(p.literal(MSG_START));@@ -151,12 +151,10 @@ common_chat_params common_chat_params_init_kimi_k3(const common_chat_template &return start + reasoning + response + tools + trailer + end;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/lfm2.cpp b/common/parsers/lfm2.cppindex 280788509e..8659298607 100644--- a/common/parsers/lfm2.cpp+++ b/common/parsers/lfm2.cpp@@ -64,7 +64,7 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &data.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.literal(GEN_PROMPT);auto end = p.end();@@ -93,12 +93,10 @@ common_chat_params common_chat_params_init_lfm2(const common_chat_template &return generation_prompt + reasoning + content + tool_calls + end;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/ling3.cpp b/common/parsers/ling3.cppindex 67d44d47ad..5d8deb9af0 100644--- a/common/parsers/ling3.cpp+++ b/common/parsers/ling3.cpp@@ -80,7 +80,7 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template &auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto end = p.end();// the effective parse input is generation_prompt + model output, so the@@ -185,12 +185,10 @@ common_chat_params common_chat_params_init_ling3(const common_chat_template &return opener + reasoning + content + tools + tail + end;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !has_response_format && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/llm-jp-harmony.cpp b/common/parsers/llm-jp-harmony.cppindex cafb82c128..aebceb1260 100644--- a/common/parsers/llm-jp-harmony.cpp+++ b/common/parsers/llm-jp-harmony.cpp@@ -48,8 +48,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_tempdata.thinking_start_tag = "<|channel|>analysis<|message|>";data.thinking_end_tags = {"<|end|>"};- // These special tokens are required to parse properly, so we include them- // even if parse_tool_calls is false.+ // These special tokens are required to parse properlydata.preserved_tokens = {"<|channel|>", "<|constrain|>", "<|message|>", "<|start|>", "<|end|>",};@@ -71,7 +70,7 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_tempauto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {// tokenizer space after special tokens; not p.space() since GBNF `space` allows one space onlyauto sp = p.chars("[ ]", 0, -1);auto channel_tag = p.literal("<|channel|>") + sp;@@ -144,12 +143,10 @@ common_chat_params common_chat_params_init_llm_jp_harmony(const common_chat_tempreturn p.zero_or_more(start + any) + start + final_msg;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/minicpm5.cpp b/common/parsers/minicpm5.cppindex 06f4cc8c83..45fad952bd 100644--- a/common/parsers/minicpm5.cpp+++ b/common/parsers/minicpm5.cpp@@ -46,7 +46,7 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &data.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.literal("<|im_start|>assistant\n");auto reasoning = p.eps();@@ -113,12 +113,10 @@ common_chat_params common_chat_params_init_minicpm5(const common_chat_template &return generation_prompt + reasoning + p.content(p.rest()) + p.end();});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/minimax-m3.cpp b/common/parsers/minimax-m3.cppindex c38ab3e238..65a023959a 100644--- a/common/parsers/minimax-m3.cpp+++ b/common/parsers/minimax-m3.cpp@@ -56,7 +56,7 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_templatedata.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.prefix(GEN_PROMPT, THINK_START);auto end = p.end();@@ -213,12 +213,10 @@ common_chat_params common_chat_params_init_minimax_m3(const common_chat_templatereturn generation_prompt + reasoning + content_before_tools + tool_calls + end;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/ministral3.cpp b/common/parsers/ministral3.cppindex 9cc5248b5f..83089d7111 100644--- a/common/parsers/ministral3.cpp+++ b/common/parsers/ministral3.cpp@@ -72,7 +72,7 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templatdata.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.eps();auto reasoning =extract_reasoning ? p.optional("[THINK]" + p.reasoning(p.until("[/THINK]")) + "[/THINK]") : p.eps();@@ -108,13 +108,11 @@ common_chat_params common_chat_params_init_ministral_3(const common_chat_templatreturn generation_prompt + (reasoning << p.content(p.rest()));});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {diff --git a/common/parsers/muse-glimmer.cpp b/common/parsers/muse-glimmer.cppindex 6c73037012..f75af1ceea 100644--- a/common/parsers/muse-glimmer.cpp+++ b/common/parsers/muse-glimmer.cpp@@ -48,7 +48,7 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templa// Constrained grammar whenever tools are offered or a response format is requested.auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE);- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto start = p.rule("start", p.literal("<|start|>assistant"));if (!extract_reasoning && !include_grammar) {@@ -131,12 +131,10 @@ common_chat_params common_chat_params_init_muse_glimmer(const common_chat_templareturn p.zero_or_more(start + analysis) + start + final_msg;});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED));data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});data.grammar_triggers = {{ COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,diff --git a/common/parsers/qwen3-coder.cpp b/common/parsers/qwen3-coder.cppindex 323b3da317..006ec6f987 100644--- a/common/parsers/qwen3-coder.cpp+++ b/common/parsers/qwen3-coder.cpp@@ -71,7 +71,7 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templat});}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {auto generation_prompt = p.literal(GEN_PREFIX);auto reasoning = p.eps();@@ -174,13 +174,11 @@ common_chat_params common_chat_params_init_qwen3_coder(const common_chat_templatreturn generation_prompt + (reasoning << p.content(p.rest()));});- data.parser = parser.save();-if (include_grammar) {data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO;data.grammar = build_grammar([&](const common_grammar_builder & builder) {- parser.build_grammar(builder, data.grammar_lazy);+ data.parser.build_grammar(builder, data.grammar_lazy);});if (data.grammar_lazy) {diff --git a/common/parsers/translate-gemma.cpp b/common/parsers/translate-gemma.cppindex c527780641..45b740e908 100644--- a/common/parsers/translate-gemma.cpp+++ b/common/parsers/translate-gemma.cpp@@ -54,10 +54,9 @@ common_chat_params common_chat_params_init_translate_gemma(data.prompt += data.generation_prompt;}- auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {+ data.parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {return p.literal(data.generation_prompt) << p.content(p.rest());});- data.parser = parser.save();return data;}diff --git a/common/peg-parser.cpp b/common/peg-parser.cppindex 75a908a281..8033f7e815 100644--- a/common/peg-parser.cpp+++ b/common/peg-parser.cpp@@ -1814,309 +1814,6 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo}}-static common_json serialize_parser_variant(const common_peg_parser_variant & variant) {- using json = common_json;-- return std::visit([](const auto & p) -> json {- using T = std::decay_t<decltype(p)>;-- if constexpr (std::is_same_v<T, common_peg_epsilon_parser>) {- return json{{"type", "epsilon"}};- } else if constexpr (std::is_same_v<T, common_peg_start_parser>) {- return json{{"type", "start"}};- } else if constexpr (std::is_same_v<T, common_peg_end_parser>) {- return json{{"type", "end"}};- } else if constexpr (std::is_same_v<T, common_peg_literal_parser>) {- return json{{"type", "literal"}, {"literal", p.literal}};- } else if constexpr (std::is_same_v<T, common_peg_sequence_parser>) {- return json{{"type", "sequence"}, {"children", p.children}};- } else if constexpr (std::is_same_v<T, common_peg_choice_parser>) {- return json{{"type", "choice"}, {"children", p.children}};- } else if constexpr (std::is_same_v<T, common_peg_repetition_parser>) {- return json{- {"type", "repetition"},- {"child", p.child},- {"min_count", p.min_count},- {"max_count", p.max_count}- };- } else if constexpr (std::is_same_v<T, common_peg_and_parser>) {- return json{{"type", "and"}, {"child", p.child}};- } else if constexpr (std::is_same_v<T, common_peg_not_parser>) {- return json{{"type", "not"}, {"child", p.child}};- } else if constexpr (std::is_same_v<T, common_peg_any_parser>) {- return json{{"type", "any"}};- } else if constexpr (std::is_same_v<T, common_peg_space_parser>) {- return json{{"type", "space"}};- } else if constexpr (std::is_same_v<T, common_peg_chars_parser>) {- json ranges = json::array();- for (const auto & range : p.ranges) {- ranges.push_back({{"start", range.start}, {"end", range.end}});- }- return json{- {"type", "chars"},- {"pattern", p.pattern},- {"ranges", ranges},- {"negated", p.negated},- {"min_count", p.min_count},- {"max_count", p.max_count}- };- } else if constexpr (std::is_same_v<T, common_peg_string_parser>) {- return json{{"type", "string"}, {"delimiter", std::string(1, p.delimiter)}};- } else if constexpr (std::is_same_v<T, common_peg_until_parser>) {- return json{{"type", "until"}, {"delimiters", p.delimiters}};- } else if constexpr (std::is_same_v<T, common_peg_schema_parser>) {- return json{- {"type", "schema"},- {"child", p.child},- {"name", p.name},- {"raw", p.raw}- };- } else if constexpr (std::is_same_v<T, common_peg_rule_parser>) {- return json{- {"type", "rule"},- {"name", p.name},- {"child", p.child},- {"trigger", p.trigger}- };- } else if constexpr (std::is_same_v<T, common_peg_ref_parser>) {- return json{{"type", "ref"}, {"name", p.name}};- } else if constexpr (std::is_same_v<T, common_peg_atomic_parser>) {- return json{{"type", "atomic"}, {"child", p.child}};- } else if constexpr (std::is_same_v<T, common_peg_tag_parser>) {- return json{- {"type", "tag"},- {"child", p.child},- {"tag", p.tag}- };- } else if constexpr (std::is_same_v<T, common_peg_gbnf_parser>) {- return json{{"type", "gbnf"}, {"child", p.child}, {"grammar", p.grammar}};- } else if constexpr (std::is_same_v<T, common_peg_ac_parser>) {- return json{{"type", "ac"}, {"child", p.child}, {"delimiters", p.delimiters}};- }- }, variant);-}--common_json common_peg_arena::to_json() const {- auto parsers = common_json::array();- for (const auto & parser : parsers_) {- parsers.push_back(serialize_parser_variant(parser));- }- return common_json{- {"parsers", parsers},- {"rules", rules_},- {"root", root_}- };-}--static common_peg_parser_variant deserialize_parser_variant(const common_json & j) {- if (!j.contains("type") || !j["type"].is_string()) {- throw std::runtime_error("Parser variant JSON missing or invalid 'type' field");- }-- std::string type = j["type"];-- if (type == "epsilon") {- return common_peg_epsilon_parser{};- }- if (type == "start") {- return common_peg_start_parser{};- }- if (type == "end") {- return common_peg_end_parser{};- }- if (type == "literal") {- if (!j.contains("literal") || !j["literal"].is_string()) {- throw std::runtime_error("literal parser missing or invalid 'literal' field");- }- return common_peg_literal_parser{j["literal"]};- }- if (type == "sequence") {- if (!j.contains("children") || !j["children"].is_array()) {- throw std::runtime_error("sequence parser missing or invalid 'children' field");- }- return common_peg_sequence_parser{j["children"].get<std::vector<common_peg_parser_id>>()};- }- if (type == "choice") {- if (!j.contains("children") || !j["children"].is_array()) {- throw std::runtime_error("choice parser missing or invalid 'children' field");- }- return common_peg_choice_parser{j["children"].get<std::vector<common_peg_parser_id>>()};- }- if (type == "repetition") {- if (!j.contains("child") || !j.contains("min_count") || !j.contains("max_count")) {- throw std::runtime_error("repetition parser missing required fields");- }- return common_peg_repetition_parser{- j["child"].get<common_peg_parser_id>(),- j["min_count"].get<int>(),- j["max_count"].get<int>()- };- }- if (type == "and") {- if (!j.contains("child")) {- throw std::runtime_error("and parser missing 'child' field");- }- return common_peg_and_parser{j["child"].get<common_peg_parser_id>()};- }- if (type == "not") {- if (!j.contains("child")) {- throw std::runtime_error("not parser missing 'child' field");- }- return common_peg_not_parser{j["child"].get<common_peg_parser_id>()};- }- if (type == "any") {- return common_peg_any_parser{};- }- if (type == "space") {- return common_peg_space_parser{};- }- if (type == "chars") {- if (!j.contains("pattern") || !j.contains("ranges") || !j.contains("negated") ||- !j.contains("min_count") || !j.contains("max_count")) {- throw std::runtime_error("chars parser missing required fields");- }- common_peg_chars_parser parser;- parser.pattern = j["pattern"];- parser.negated = j["negated"].get<bool>();- parser.min_count = j["min_count"].get<int>();- parser.max_count = j["max_count"].get<int>();- for (const auto & range_json : j["ranges"]) {- if (!range_json.contains("start") || !range_json.contains("end")) {- throw std::runtime_error("char_range missing 'start' or 'end' field");- }- parser.ranges.push_back({- range_json["start"].get<uint32_t>(),- range_json["end"].get<uint32_t>()- });- }- return parser;- }- if (type == "string") {- if (!j.contains("delimiter")) {- throw std::runtime_error("string parser missing delimiter field.");- }- std::string delimiter = j["delimiter"];- if (delimiter.empty()) {- throw std::runtime_error("string parser delimiter is empty.");- }- return common_peg_string_parser{delimiter[0]};- }- if (type == "until") {- if (!j.contains("delimiters") || !j["delimiters"].is_array()) {- throw std::runtime_error("until parser missing or invalid 'delimiters' field");- }- return common_peg_until_parser{j["delimiters"].get<std::vector<std::string>>()};- }- if (type == "schema") {- if (!j.contains("child") || !j.contains("name") || !j.contains("raw")) {- throw std::runtime_error("schema parser missing required fields");- }- common_peg_schema_parser parser;- parser.child = j["child"].get<common_peg_parser_id>();- parser.name = j["name"];- parser.raw = j["raw"].get<bool>();- return parser;- }- if (type == "rule") {- if (!j.contains("name") || !j.contains("child") || !j.contains("trigger")) {- throw std::runtime_error("rule parser missing required fields");- }- return common_peg_rule_parser{- j["name"].get<std::string>(),- j["child"].get<common_peg_parser_id>(),- j["trigger"].get<bool>()- };- }- if (type == "ref") {- if (!j.contains("name") || !j["name"].is_string()) {- throw std::runtime_error("ref parser missing or invalid 'name' field");- }- return common_peg_ref_parser{j["name"]};- }- if (type == "atomic") {- if (!j.contains("child")) {- throw std::runtime_error("tag parser missing required fields");- }- return common_peg_atomic_parser{- j["child"].get<common_peg_parser_id>(),- };- }- if (type == "tag") {- if (!j.contains("child") || !j.contains("tag")) {- throw std::runtime_error("tag parser missing required fields");- }- return common_peg_tag_parser{- j["child"].get<common_peg_parser_id>(),- j["tag"].get<std::string>(),- };- }-- if (type == "gbnf") {- if (!j.contains("child") || !j.contains("grammar")) {- throw std::runtime_error("gbnf parser missing required fields");- }- return common_peg_gbnf_parser{- j["child"].get<common_peg_parser_id>(),- j["grammar"].get<std::string>(),- };- }-- if (type == "ac") {- if (!j.contains("child") || !j.contains("delimiters") || !j["delimiters"].is_array() || j["delimiters"].empty()) {- throw std::runtime_error("ac parser requires 'child' and a non-empty 'delimiters' array");- }- return common_peg_ac_parser{- j["child"].get<common_peg_parser_id>(),- j["delimiters"].get<std::vector<std::string>>(),- };- }-- throw std::runtime_error("Unknown parser type: " + type);-}--common_peg_arena common_peg_arena::from_json(const common_json & j) {- if (!j.contains("parsers") || !j["parsers"].is_array()) {- throw std::runtime_error("JSON missing or invalid 'parsers' array");- }- if (!j.contains("rules") || !j["rules"].is_object()) {- throw std::runtime_error("JSON missing or invalid 'rules' object");- }- if (!j.contains("root")) {- throw std::runtime_error("JSON missing 'root' field");- }-- common_peg_arena arena;-- const auto & parsers_json = j["parsers"];- arena.parsers_.reserve(parsers_json.size());- for (const auto & parser_json : parsers_json) {- arena.parsers_.push_back(deserialize_parser_variant(parser_json));- }-- arena.rules_ = j["rules"].get<std::unordered_map<std::string, common_peg_parser_id>>();-- for (const auto & [name, id] : arena.rules_) {- if (id >= arena.parsers_.size()) {- throw std::runtime_error("Rule '" + name + "' references invalid parser ID: " + std::to_string(id));- }- }-- arena.root_ = j["root"].get<common_peg_parser_id>();- if (arena.root_ != COMMON_PEG_INVALID_PARSER_ID && arena.root_ >= arena.parsers_.size()) {- throw std::runtime_error("Root references invalid parser ID: " + std::to_string(arena.root_));- }-- return arena;-}--std::string common_peg_arena::save() const {- return to_json().dump();-}--void common_peg_arena::load(const std::string & data) {- *this = from_json(common_json::parse(data));-}-common_peg_arena build_peg_parser(const std::function<common_peg_parser(common_peg_parser_builder & builder)> & fn) {common_peg_parser_builder builder;builder.set_root(fn(builder));diff --git a/common/peg-parser.h b/common/peg-parser.hindex 816d007b06..9255d6eb97 100644--- a/common/peg-parser.h+++ b/common/peg-parser.h@@ -357,12 +357,6 @@ class common_peg_arena {std::string dump(common_peg_parser_id id) const;- common_json to_json() const;- static common_peg_arena from_json(const common_json & j);-- std::string save() const;- void load(const std::string & data);-friend class common_peg_parser_builder;private:diff --git a/common/sampling.cpp b/common/sampling.cppindex 6cc8872f23..9e16693fbb 100644--- a/common/sampling.cpp+++ b/common/sampling.cpp@@ -1050,3 +1050,41 @@ std::vector<common_sampler_type> common_sampler_types_from_chars(const std::strireturn samplers;}++void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector<std::string> & tokens) {+ GGML_ASSERT(vocab != nullptr);+ for (const auto & t : tokens) {+ auto ids = common_tokenize(vocab, t, false, true);+ if (ids.size() == 1) {+ sampling.preserved_tokens.insert(ids[0]);+ }+ }+}++void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector<common_grammar_trigger> triggers) {+ GGML_ASSERT(vocab != nullptr);+ for (auto & trigger : triggers) {+ if (trigger.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {+ const auto & word = trigger.value;+ auto ids = common_tokenize(vocab, word, false, true);+ if (ids.size() == 1) {+ auto token = ids[0];+ if (std::find(sampling.preserved_tokens.begin(), sampling.preserved_tokens.end(), (llama_token) token) == sampling.preserved_tokens.end()) {+ throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);+ }+ common_grammar_trigger token_trigger;+ token_trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;+ token_trigger.value = word;+ token_trigger.token = token;+ sampling.grammar_triggers.push_back(std::move(token_trigger));+ } else {+ sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});+ }+ } else {+ sampling.grammar_triggers.push_back(std::move(trigger));+ }+ }+ if (sampling.grammar_lazy && sampling.grammar_triggers.empty()) {+ throw std::runtime_error("Error: no triggers set for lazy grammar!");+ }+}diff --git a/common/sampling.h b/common/sampling.hindex 7ebae3df82..f19bd27c40 100644--- a/common/sampling.h+++ b/common/sampling.h@@ -118,6 +118,12 @@ std::string common_sampler_type_to_str(enum common_sampler_type cnstr);std::vector<enum common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names);std::vector<enum common_sampler_type> common_sampler_types_from_chars(const std::string & chars);+// add the strings that are a single token in the vocab to the preserved tokens+void common_sampling_add_preserved_tokens(common_params_sampling & sampling, const llama_vocab * vocab, const std::vector<std::string> & tokens);++// add grammar triggers, a trigger word that is a single token becomes a token trigger and must be a preserved token+void common_sampling_add_grammar_triggers(common_params_sampling & sampling, const llama_vocab * vocab, std::vector<common_grammar_trigger> triggers);+llama_sampler * llama_sampler_init_llg(const llama_vocab * vocab,const char * grammar_kind, const char * grammar_data);diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txtindex c44189cebd..5f9bd037f3 100644--- a/tests/CMakeLists.txt+++ b/tests/CMakeLists.txt@@ -245,7 +245,6 @@ llama_build_and_test(peg-parser/test-basic.cpppeg-parser/test-gbnf-generation.cpppeg-parser/test-json-parser.cpp- peg-parser/test-json-serialization.cpppeg-parser/test-python-dict-parser.cpppeg-parser/test-unicode.cpppeg-parser/tests.hdiff --git a/tests/peg-parser/test-json-serialization.cpp b/tests/peg-parser/test-json-serialization.cppdeleted file mode 100644index da63a23bf2..0000000000--- a/tests/peg-parser/test-json-serialization.cpp+++ /dev/null@@ -1,28 +0,0 @@-#include "tests.h"--void test_json_serialization(testing &t) {- auto original = build_peg_parser([](common_peg_parser_builder & p) {- return "<tool_call>" + p.json() + "</tool_call>";- });-- auto json_serialized = original.to_json().dump();-- t.test("compare before/after", [&](testing &t) {- auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));-- // Test complex JSON- std::string input = R"({"name": "test", "values": [1, 2, 3], "nested": {"a": true}})";- common_peg_parse_context ctx1(input);- common_peg_parse_context ctx2(input);-- auto result1 = original.parse(ctx1);- auto result2 = deserialized.parse(ctx2);-- t.assert_equal("both_succeed", result1.success(), result2.success());- t.assert_equal("same_end_pos", result1.end, result2.end);- });-- t.bench("deserialize", [&]() {- auto deserialized = common_peg_arena::from_json(common_json::parse(json_serialized));- }, 100);-}diff --git a/tests/peg-parser/tests.h b/tests/peg-parser/tests.hindex 00e81815b6..287da16d18 100644--- a/tests/peg-parser/tests.h+++ b/tests/peg-parser/tests.h@@ -21,5 +21,4 @@ void test_basic(testing &t);void test_json_parser(testing &t);void test_gbnf_generation(testing &t);void test_unicode(testing &t);-void test_json_serialization(testing &t);void test_python_dict_parser(testing &t);diff --git a/tests/test-chat-auto-parser.cpp b/tests/test-chat-auto-parser.cppindex 5aa9482512..1c641b329f 100644--- a/tests/test-chat-auto-parser.cpp+++ b/tests/test-chat-auto-parser.cpp@@ -450,7 +450,7 @@ static int debug_single_template(const debug_options & opts) {generation_params params = prepare_debug_params(opts, tools);common_chat_params parser_data;if (std::optional<common_chat_params> spec_tmpl =- common_chat_try_specialized_template(chat_template, template_source, params)) {+ common_chat_try_specialized_template(chat_template, params)) {LOG_ERR("\n");LOG_ERR("This template uses a specialized parser, analysis results will not be available.\n");parser_data = *spec_tmpl;@@ -484,8 +484,7 @@ static int debug_single_template(const debug_options & opts) {if (!std::empty(parser_data.parser)) {LOG_ERR("\n=== Generated Parser ===\n");- common_peg_arena arena;- arena.load(parser_data.parser);+ const common_peg_arena & arena = parser_data.parser;LOG_ERR("%s\n", arena.dump(arena.root()).c_str());LOG_ERR("\n=== Generated Grammar ===\n");diff --git a/tests/test-chat.cpp b/tests/test-chat.cppindex 41e4948c49..09f5936929 100644--- a/tests/test-chat.cpp+++ b/tests/test-chat.cpp@@ -1090,26 +1090,6 @@ struct peg_test_case {std::vector<std::string> expect_rules;};-struct make_peg_parser {- common_chat_params params_;- common_peg_arena arena_;- bool detailed_debug_;-- make_peg_parser(common_chat_templates * tmpls,- const common_chat_templates_inputs & inputs,- bool detailed_debug = false) {- detailed_debug_ = detailed_debug;- params_ = common_chat_templates_apply(tmpls, inputs);- arena_.load(params_.parser);- }-- common_chat_msg parse(const std::string & msg, bool is_partial) const {- common_chat_parser_params parser_params(params_);- parser_params.debug = detailed_debug_;- return common_chat_peg_parse(arena_, common_chat_input(msg), is_partial, parser_params);- }-};-// Global template filter for --template flagstatic std::string g_template_filter;@@ -1161,14 +1141,19 @@ static void test_peg_parser(common_chat_templates * tmpls,tc.expect.role = "assistant";}- auto parser = make_peg_parser(tmpls, tc.params, detailed_debug);+ common_chat_session_params session_params;+ session_params.debug = detailed_debug;+ common_chat_session session(tmpls, nullptr, tc.params, session_params);+ const auto & parser = session.parser();+ common_params_sampling sampling;+ session.apply_sampling(sampling);if (detailed_debug) {- LOG_DBG("Using parser: \n%s\n", parser.arena_.dump(parser.arena_.root()).c_str());- LOG_DBG("Generation prompt: '%s'\n", parser.params_.generation_prompt.c_str());+ LOG_DBG("Using parser: \n%s\n", parser.dump(parser.root()).c_str());+ LOG_DBG("Generation prompt: '%s'\n", session.generation_prompt().c_str());}for (const auto & rule : tc.expect_rules) {- if (!parser.arena_.has_rule(rule)) {+ if (!parser.has_rule(rule)) {LOG_ERR("Missing rule: %s\n", rule.c_str());common_log_flush(common_log_main());throw std::runtime_error("Test failed");@@ -1179,12 +1164,14 @@ static void test_peg_parser(common_chat_templates * tmpls,common_chat_msg msg_prev;msg_accum.role = msg_prev.role = "assistant";+ size_t fed = 0;for (size_t i = 1; i <= tc.input.size(); ++i) {auto is_partial = i < tc.input.size() || tc.is_partial;// Use UTF-8 safe truncation to avoid corrupting multi-byte characterssize_t safe_len = utf8_truncate_safe_len(std::string_view(tc.input).substr(0, i));- std::string prefix = tc.input.substr(0, safe_len);- common_chat_msg msg_current = parser.parse(prefix, is_partial);+ common_chat_input chunk(tc.input.substr(fed, safe_len - fed));+ fed = safe_len;+ const common_chat_msg & msg_current = is_partial ? session.feed(chunk) : session.finish(chunk);for (const auto & diff : common_chat_msg_diff::compute_diffs(msg_prev, msg_current)) {if (!diff.reasoning_content_delta.empty()) {@@ -1222,30 +1209,33 @@ static void test_peg_parser(common_chat_templates * tmpls,}if (!tc.is_partial) {- assert_msg_equals(tc.expect, parser.parse(tc.input, false), true);+ if (tc.input.empty()) {+ session.finish();+ }+ assert_msg_equals(tc.expect, session.msg(), true);}assert_msg_equals(tc.expect, msg_accum, true);// A response format must be enforced by an eager grammarif (!tc.params.json_schema.empty()) {- if (parser.params_.grammar.empty()) {+ if (session.grammar().empty()) {throw std::runtime_error("json_schema is set but no grammar was produced");}- if (parser.params_.grammar_lazy) {+ if (sampling.grammar_lazy) {throw std::runtime_error("json_schema is set but the grammar is lazy");}}// Test grammar if present in params- if (!parser.params_.grammar.empty()) {- auto grammar = build_grammar(parser.params_.grammar);+ if (!session.grammar().empty()) {+ auto grammar = build_grammar(session.grammar());if (!grammar) {- throw std::runtime_error("Failed to build grammar: " + parser.params_.grammar);+ throw std::runtime_error("Failed to build grammar: " + session.grammar());}// In production, grammar triggers match against the full generated text// including the generation prompt. All positions are in full_input coordinates.- const auto & gen_prompt = parser.params_.generation_prompt;+ const auto & gen_prompt = sampling.generation_prompt;std::string full_input = gen_prompt + tc.input;// Determine whether the reasoning-budget sampler path applies: tool-call grammar@@ -1253,9 +1243,9 @@ static void test_peg_parser(common_chat_templates * tmpls,// budget sampler inhibits grammar application while inside thinking blocks —// triggers inside <think>...</think> are suppressed.bool use_reasoning_budget_path = false;- if (parser.params_.grammar_lazy && !parser.params_.thinking_end_tags.empty()) {+ if (sampling.grammar_lazy && !session.thinking_end_tags().empty()) {use_reasoning_budget_path = true;- for (const auto & trigger : parser.params_.grammar_triggers) {+ for (const auto & trigger : sampling.grammar_triggers) {if (trigger.type != COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {use_reasoning_budget_path = false;break;@@ -1270,8 +1260,8 @@ static void test_peg_parser(common_chat_templates * tmpls,// Reasoning-budget path: simulate thinking-aware trigger detection.// Walk through full_input tracking thinking state; only match triggers// when outside thinking blocks.- const auto & think_start = parser.params_.thinking_start_tag;- const auto & think_ends = parser.params_.thinking_end_tags;+ const auto & think_start = session.thinking_start_tag();+ const auto & think_ends = session.thinking_end_tags();bool in_thinking = false;for (size_t i = 0; i < full_input.size(); ++i) {@@ -1292,7 +1282,7 @@ static void test_peg_parser(common_chat_templates * tmpls,continue;}// Outside thinking — check if any trigger word starts here- for (const auto & trigger : parser.params_.grammar_triggers) {+ for (const auto & trigger : sampling.grammar_triggers) {if (full_input.compare(i, trigger.value.size(), trigger.value) == 0) {if (earliest_trigger_pos == std::string::npos || i < earliest_trigger_pos) {earliest_trigger_pos = i;@@ -1314,7 +1304,7 @@ static void test_peg_parser(common_chat_templates * tmpls,if (!use_reasoning_budget_path) {// Legacy path: find triggers without thinking-awareness- for (const auto & trigger : parser.params_.grammar_triggers) {+ for (const auto & trigger : sampling.grammar_triggers) {size_t pos = std::string::npos;std::smatch match;switch (trigger.type) {@@ -1370,16 +1360,16 @@ static void test_peg_parser(common_chat_templates * tmpls,// If the test expects tool calls and the grammar is lazy, the trigger must fire.// Otherwise the grammar would never activate in production and tool calls wouldn't// be constrained. A silent skip here would hide broken triggers.- if (parser.params_.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial+ if (sampling.grammar_lazy && !tc.expect.tool_calls.empty() && !tc.is_partial&& earliest_trigger_pos == std::string::npos) {std::string trigger_desc;- for (const auto & trigger : parser.params_.grammar_triggers) {+ for (const auto & trigger : sampling.grammar_triggers) {trigger_desc += "\n [type=" + std::to_string(trigger.type) + "] " + trigger.value;}throw std::runtime_error("Grammar trigger did not fire, but test expects tool calls (lazy grammar).\n"">>> Input: " + full_input + "\n"- ">>> Triggers (" + std::to_string(parser.params_.grammar_triggers.size()) + "):" + trigger_desc);+ ">>> Triggers (" + std::to_string(sampling.grammar_triggers.size()) + "):" + trigger_desc);}// Determine the constrained portion of input to test against grammar.@@ -1392,7 +1382,7 @@ static void test_peg_parser(common_chat_templates * tmpls,auto constrain_from = std::max(earliest_trigger_pos, gen_prompt.size());constrained = full_input.substr(constrain_from);grammar_triggered = true;- } else if (!parser.params_.grammar_lazy) {+ } else if (!sampling.grammar_lazy) {// For non-lazy grammars, the entire input should matchgrammar_triggered = true;}@@ -1411,7 +1401,7 @@ static void test_peg_parser(common_chat_templates * tmpls,std::to_string(result.matched_codepoints) + " codepoints): " +(result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) +"\n\n>>> Expected next: " + result.expected_description +- "\n\n>>> Grammar: " + parser.params_.grammar;+ "\n\n>>> Grammar: " + session.grammar();} else {error_msg ="Grammar match failed:\n\n"@@ -1422,7 +1412,7 @@ static void test_peg_parser(common_chat_templates * tmpls,(result.matched_prefix.size() > 100 ? result.matched_prefix.substr(0, 100) + "..." : result.matched_prefix) +"\n\n>>> Failing character: " + result.failing_char +"\n\n>>> Expected: " + result.expected_description +- "\n\n>>> Grammar: " + parser.params_.grammar;+ "\n\n>>> Grammar: " + session.grammar();}throw std::runtime_error(error_msg);}@@ -1437,7 +1427,7 @@ static void test_peg_parser(common_chat_templates * tmpls,// Start from tc.expect but copy tool call arguments from the actual parser// output, which preserves original JSON formatting (e.g. {"arg1":1} vs {"arg1": 1}).auto reconstruction_msg = tc.expect;- auto parsed_msg = parser.parse(tc.input, false);+ const auto & parsed_msg = session.msg();for (size_t i = 0; i < reconstruction_msg.tool_calls.size() && i < parsed_msg.tool_calls.size(); i++) {reconstruction_msg.tool_calls[i].arguments = parsed_msg.tool_calls[i].arguments;}@@ -1446,7 +1436,7 @@ static void test_peg_parser(common_chat_templates * tmpls,reconstruction_inputs.add_generation_prompt = false;auto reconstruction_params = common_chat_templates_apply(tmpls, reconstruction_inputs);- std::string expected_text = parser.params_.prompt + tc.input;+ std::string expected_text = session.prompt() + tc.input;bool match = reconstruction_params.prompt == expected_text ||(reconstruction_params.prompt.size() > expected_text.size() &&reconstruction_params.prompt.compare(0, expected_text.size(), expected_text) == 0);@@ -4644,8 +4634,7 @@ static void test_template_output_peg_parsers(bool detailed_debug) {inputs.messages = { msg };auto params = common_chat_templates_apply(tmpls.get(), inputs);- common_peg_arena arena;- arena.load(params.parser);+ const common_peg_arena & arena = params.parser;common_chat_parser_params pp(params);// generation_prompt is non-empty for thinking models, so result.end@@ -7791,6 +7780,77 @@ static void test_deepseek_v4_tool_result_ordering() {}}+static void test_chat_session() {+ LOG_DBG("%s\n", __func__);+ auto tmpls = read_templates("models/templates/Qwen3.5-4B.jinja");++ common_chat_templates_inputs inputs;+ inputs.messages = { message_user };+ inputs.tools = { special_function_tool };+ inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;++ const std::string output =+ "I'm\nthinking\n</think>\n\n"+ "<tool_call>\n"+ "<function=special_function>\n"+ "<parameter=arg1>\n1\n</parameter>\n"+ "</function>\n"+ "</tool_call>";++ // the session renders the prompt and parses the output fed to it in small chunks+ {+ common_chat_session session(tmpls.get(), nullptr, inputs);++ assert_contains(session.prompt(), "\"name\": \"special_function\"");+ assert_contains(session.prompt(), "<|im_start|>user\nHey there!<|im_end|>\n<|im_start|>assistant\n<think>\n");+ assert_equals(false, session.grammar().empty());+ assert_equals(std::string("<|im_start|>assistant\n<think>\n"), session.generation_prompt());++ const std::string thinking = "I'm\nthinking\n</think>\n\n";+ for (size_t i = 0; i < thinking.size(); i += 3) {+ session.feed(common_chat_input(thinking.substr(i, 3)));+ }+ assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n"), session.msg());++ for (size_t i = thinking.size(); i < output.size(); i += 3) {+ session.feed(common_chat_input(output.substr(i, 3)));+ }+ assert_msg_equals(simple_assist_msg("", "I'm\nthinking\n", "special_function", "{\"arg1\":1}"), session.finish());+ }++ // a copy does not see what is fed to the original+ {+ common_chat_session a(tmpls.get(), nullptr, inputs);+ common_chat_session b = a;+ a.feed(common_chat_input(output));+ assert_equals(true, b.msg().empty());+ b.feed(common_chat_input("I'm\nthinking\n</think>\n\nHello"));+ assert_equals(std::string("Hello"), b.msg().content);+ assert_equals(std::string("special_function"), a.finish().tool_calls.at(0).name);+ }++ // a continued message starts from the prefill, unless it is echoed+ {+ common_chat_templates_inputs cont;+ cont.messages = { message_user, message_assist_prefill_content };+ cont.add_generation_prompt = false;+ cont.continue_final_message = COMMON_CHAT_CONTINUATION_CONTENT;+ cont.reasoning_format = COMMON_REASONING_FORMAT_AUTO;++ common_chat_session session(tmpls.get(), nullptr, cont);+ assert_equals(std::string("<|im_start|>assistant\n<think>\nI'm thinking\n</think>\n\nHello, "),+ session.generation_prompt());+ assert_msg_equals(simple_assist_msg("Hello, ", "I'm thinking\n"), session.msg());+ session.feed(common_chat_input("world!"));+ assert_equals(std::string("Hello, world!"), session.msg().content);++ common_chat_session_params echo;+ echo.echo = true;+ common_chat_session echoed(tmpls.get(), nullptr, cont, echo);+ assert_equals(true, echoed.msg().empty());+ }+}+static void test_reasoning_budget_tokens_per_request() {LOG_DBG("%s\n", __func__);// Use Qwen3 template which has <think>...</think> reasoning markers.@@ -7811,7 +7871,8 @@ static void test_reasoning_budget_tokens_per_request() {{"reasoning_budget_tokens", 0},};std::vector<raw_buffer> out_files;- auto llama_params = oaicompat_chat_params_parse(body, opt, out_files);+ common_chat_session out_session;+ auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session);// The per-request value must win over the server default (-1).if (!llama_params.contains("reasoning_budget_tokens")) {@@ -7844,7 +7905,8 @@ static void test_reasoning_budget_message_per_request() {{"reasoning_budget_message", per_request_message},};std::vector<raw_buffer> out_files;- auto llama_params = oaicompat_chat_params_parse(body, opt, out_files);+ common_chat_session out_session;+ auto llama_params = oaicompat_chat_params_parse(nullptr, body, opt, out_files, out_session);// The per-request value must win over the server default.if (!llama_params.contains("reasoning_budget_message")) {@@ -8035,6 +8097,7 @@ int main(int argc, char ** argv) {test_deepseek_v4_tool_result_ordering();test_template_generation_prompt();test_reasoning_effort_caps();+ test_chat_session();test_reasoning_budget_tokens_per_request();test_reasoning_budget_message_per_request();test_template_output_peg_parsers(detailed_debug);diff --git a/tests/test-peg-parser.cpp b/tests/test-peg-parser.cppindex 7d22d77612..863bfe8a0f 100644--- a/tests/test-peg-parser.cpp+++ b/tests/test-peg-parser.cpp@@ -19,7 +19,6 @@ int main(int argc, char *argv[]) {t.test("unicode", test_unicode);t.test("json", test_json_parser);t.test("gbnf", test_gbnf_generation);- t.test("serialization", test_json_serialization);t.test("python-dict", test_python_dict_parser);return t.summary();diff --git a/tools/server/README.md b/tools/server/README.mdindex 41dca6152a..8fe260a32e 100644--- a/tools/server/README.md+++ b/tools/server/README.md@@ -1353,10 +1353,6 @@ The `response_format` parameter supports both plain JSON output (e.g. `{"type":`reasoning_control`: Arms realtime reasoning control for this completion so it can be ended early via `/v1/chat/completions/control`. Defaults to `false`.-`generation_prompt`: The generation prompt that was prefilled in by the template. Prepended to model output before parsing.--`parse_tool_calls`: Whether to parse the generated tool call.-`parallel_tool_calls` : Whether to enable parallel/multiple tool calls (only supported on some models, verification is based on jinja template).For multimodal input (typed content, `messages[i].content[j]`):diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cppindex d364ba8985..5f77d43d05 100644--- a/tools/server/server-common.cpp+++ b/tools/server/server-common.cpp@@ -1266,9 +1266,11 @@ server_tokens tokenize_oai_content_array(const llama_vocab * vocab, mtmd_context// used by /chat/completions endpointjson oaicompat_chat_params_parse(+ const llama_vocab * vocab,json & body, /* openai api json semantics */const server_chat_params & opt,- std::vector<raw_buffer> & out_files)+ std::vector<raw_buffer> & out_files,+ common_chat_session & out_session){json llama_params;@@ -1393,7 +1395,6 @@ json oaicompat_chat_params_parse(if (body.contains("grammar")) {throw std::invalid_argument("Cannot use custom grammar constraints with tools.");}- llama_params["parse_tool_calls"] = true;}// merge the template args provided from command line with the args provided in the user request@@ -1427,31 +1428,11 @@ json oaicompat_chat_params_parse(inputs.force_pure_content = opt.force_pure_content;// Apply chat template to the list of messages- auto chat_params = common_chat_templates_apply(opt.tmpls.get(), inputs);+ common_chat_session_params session_params;+ session_params.echo = json_value(body, "echo", false);+ out_session = common_chat_session(opt.tmpls.get(), vocab, inputs, session_params);- llama_params["chat_format"] = static_cast<int>(chat_params.format);- llama_params["prompt"] = chat_params.prompt;- if (!chat_params.grammar.empty()) {- llama_params["grammar"] = chat_params.grammar;- llama_params["grammar_type"] = std::string("tool_calls");- }- llama_params["grammar_lazy"] = chat_params.grammar_lazy;- auto grammar_triggers = json::array();- for (const auto & trigger : chat_params.grammar_triggers) {- server_grammar_trigger ct(trigger);- grammar_triggers.push_back(ct.to_json());- }- llama_params["grammar_triggers"] = grammar_triggers;- llama_params["preserved_tokens"] = chat_params.preserved_tokens;- llama_params["generation_prompt"] = chat_params.generation_prompt;- for (const auto & stop : chat_params.additional_stops) {- llama_params["stop"].push_back(stop);- }- if (!chat_params.parser.empty()) {- llama_params["chat_parser"] = chat_params.parser;- }-- llama_params["message_delimiters"] = chat_params.message_delimiters.to_json();+ llama_params["prompt"] = out_session.prompt();// Reasoning budget: pass parameters through to sampling layer{@@ -1461,10 +1442,10 @@ json oaicompat_chat_params_parse(reasoning_budget = opt.reasoning_budget;}- if (!chat_params.thinking_end_tags.empty()) {+ if (!out_session.thinking_end_tags().empty()) {llama_params["reasoning_budget_tokens"] = reasoning_budget;- llama_params["reasoning_budget_start_tag"] = chat_params.thinking_start_tag;- llama_params["reasoning_budget_end_tags"] = chat_params.thinking_end_tags;+ llama_params["reasoning_budget_start_tag"] = out_session.thinking_start_tag();+ llama_params["reasoning_budget_end_tags"] = out_session.thinking_end_tags();llama_params["reasoning_budget_message"] = json_value(body, "reasoning_budget_message", opt.reasoning_budget_message);llama_params["reasoning_control"] = json_value(body, "reasoning_control", false);}@@ -1491,6 +1472,15 @@ json oaicompat_chat_params_parse(}}+ // the session owns these, the server applies them with server_task::apply_chat_session()+ for (const char * key : { "grammar_lazy", "grammar_triggers", "preserved_tokens" }) {+ llama_params.erase(key);+ }+ if (!out_session.grammar().empty()) {+ llama_params.erase("grammar");+ llama_params.erase("json_schema");+ }+return llama_params;}diff --git a/tools/server/server-common.h b/tools/server/server-common.hindex 6165d871cc..5bd2ed57ab 100644--- a/tools/server/server-common.h+++ b/tools/server/server-common.h@@ -355,9 +355,11 @@ json oaicompat_completion_params_parse(const json & body);// used by /chat/completions endpointjson oaicompat_chat_params_parse(+ const llama_vocab * vocab,json & body, /* openai api json semantics */const server_chat_params & opt,- std::vector<raw_buffer> & out_files);+ std::vector<raw_buffer> & out_files,+ common_chat_session & out_session);// used by /embeddings endpoint, content has the same format as a chat message content arrayserver_tokens tokenize_oai_content_array(diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cppindex 1e3021d4ac..9a9c0c90d5 100644--- a/tools/server/server-context.cpp+++ b/tools/server/server-context.cpp@@ -4767,7 +4767,8 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(server_task_type type,const json & data,const std::vector<raw_buffer> & files,- task_response_type res_type) {+ task_response_type res_type,+ const common_chat_session & chat_session) {GGML_ASSERT(type == SERVER_TASK_TYPE_COMPLETION || type == SERVER_TASK_TYPE_INFILL);auto res = create_response();@@ -4809,11 +4810,6 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(// tasks.reserve(inputs.size()); // TODO: this is inaccurate due to child tasks- // message delimiters for checkpointing- json delims = json_value(data, "message_delimiters", json::array());- auto delimiters = common_chat_msg_delimiters_parse(delims);- delimiters.tokenize(ctx_server.vocab);-for (size_t i = 0; i < inputs.size(); i++) {server_task task = server_task(type);@@ -4826,7 +4822,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(meta->logit_bias_eog,data);- task.params.message_spans = task.tokens.find_message_spans(delimiters);+ task.apply_chat_session(chat_session);task.id_slot = json_value(data, "id_slot", -1);sse_ping_interval = task.params.sse_ping_interval;@@ -4847,7 +4843,7 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(tasks.push_back(std::move(task));}- rd.post_tasks(std::move(tasks));+ rd.post_tasks(std::move(tasks), chat_session);} catch (const std::exception & e) {res->error(format_error_response(e.what(), ERROR_TYPE_INVALID_REQUEST));return res;@@ -5444,16 +5440,20 @@ void server_routes::init_routes() {auto res = create_response();std::vector<raw_buffer> files;json body = json::parse(req.body);+ common_chat_session session;json body_parsed = oaicompat_chat_params_parse(+ ctx_server.vocab,body,meta->chat_params,- files);+ files,+ session);return handle_completions_impl(req,SERVER_TASK_TYPE_COMPLETION,body_parsed,files,- TASK_RESPONSE_TYPE_OAI_CHAT);+ TASK_RESPONSE_TYPE_OAI_CHAT,+ session);};this->post_chat_completions_tok = [this](const server_http_req & req) {@@ -5503,16 +5503,20 @@ void server_routes::init_routes() {json body = server_chat_convert_responses_to_chatcmpl(json::parse(req.body));SRV_DBG("%s\n", "Request converted: OpenAI Responses -> OpenAI Chat Completions");SRV_DBG("converted request: %s\n", body.dump().c_str());+ common_chat_session session;json body_parsed = oaicompat_chat_params_parse(+ ctx_server.vocab,body,meta->chat_params,- files);+ files,+ session);return handle_completions_impl(req,SERVER_TASK_TYPE_COMPLETION,body_parsed,files,- TASK_RESPONSE_TYPE_OAI_RESP);+ TASK_RESPONSE_TYPE_OAI_RESP,+ session);};this->post_responses_tok_oai = [this](const server_http_req & req) {@@ -5535,16 +5539,20 @@ void server_routes::init_routes() {files);SRV_DBG("%s\n", "Request converted: OpenAI Transcriptions -> OpenAI Chat Completions");SRV_DBG("converted request: %s\n", body.dump().c_str());+ common_chat_session session;json body_parsed = oaicompat_chat_params_parse(+ ctx_server.vocab,body,meta->chat_params,- files);+ files,+ session);return handle_completions_impl(req,SERVER_TASK_TYPE_COMPLETION,body_parsed,files,- TASK_RESPONSE_TYPE_OAI_ASR);+ TASK_RESPONSE_TYPE_OAI_ASR,+ session);};this->post_anthropic_messages = [this](const server_http_req & req) {@@ -5553,16 +5561,20 @@ void server_routes::init_routes() {json body = server_chat_convert_anthropic_to_oai(json::parse(req.body));SRV_DBG("%s\n", "Request converted: Anthropic -> OpenAI Chat Completions");SRV_DBG("converted request: %s\n", body.dump().c_str());+ common_chat_session session;json body_parsed = oaicompat_chat_params_parse(+ ctx_server.vocab,body,meta->chat_params,- files);+ files,+ session);return handle_completions_impl(req,SERVER_TASK_TYPE_COMPLETION,body_parsed,files,- TASK_RESPONSE_TYPE_ANTHROPIC);+ TASK_RESPONSE_TYPE_ANTHROPIC,+ session);};this->post_anthropic_count_tokens = [this](const server_http_req & req) {@@ -5573,11 +5585,14 @@ void server_routes::init_routes() {this->post_apply_template = [this](const server_http_req & req) {auto res = create_response();std::vector<raw_buffer> files; // dummy, unused+ common_chat_session session; // dummy, unusedjson body = json::parse(req.body);json data = oaicompat_chat_params_parse(+ ctx_server.vocab,body,meta->chat_params,- files);+ files,+ session);res->ok({{ "prompt", std::move(data.at("prompt")) }});return res;};@@ -6129,10 +6144,13 @@ std::unique_ptr<server_res_generator> server_routes::handle_count_tokens(const sreturn res;}+ common_chat_session session; // dummy, unusedjson body_parsed = oaicompat_chat_params_parse(+ ctx_server.vocab,body,meta->chat_params,- files);+ files,+ session);json prompt = body_parsed.at("prompt");// SRV_DBG("prompt = %s\n", prompt.dump().c_str());diff --git a/tools/server/server-context.h b/tools/server/server-context.hindex 7fe0035315..ed23f7d4e6 100644--- a/tools/server/server-context.h+++ b/tools/server/server-context.h@@ -166,7 +166,8 @@ private:server_task_type type,const json & data,const std::vector<raw_buffer> & files,- task_response_type res_type);+ task_response_type res_type,+ const common_chat_session & chat_session = {});std::unique_ptr<server_res_generator> handle_slots_save(const server_http_req & req, int id_slot);std::unique_ptr<server_res_generator> handle_slots_restore(const server_http_req & req, int id_slot);std::unique_ptr<server_res_generator> handle_slots_erase(const server_http_req &, int id_slot);diff --git a/tools/server/server-queue.cpp b/tools/server/server-queue.cppindex 78169e9a5d..3352c3cb5d 100644--- a/tools/server/server-queue.cpp+++ b/tools/server/server-queue.cpp@@ -517,23 +517,23 @@ void server_response_reader::post_task(server_task && task, bool front) {GGML_ASSERT(!task.is_parent() && "not supported, use post_tasks() instead");task.index = 0;id_tasks.insert(task.id);- states.push_back(task.create_state());+ states.push_back(task_result_state());queue_results.add_waiting_task_id(task.id);queue_tasks.post(std::move(task), front);}-void server_response_reader::post_tasks(std::vector<server_task> && tasks, bool front) {+void server_response_reader::post_tasks(std::vector<server_task> && tasks, const common_chat_session & session, bool front) {GGML_ASSERT(id_tasks.empty() && "post_tasks() can only be called once per reader");id_tasks = server_task::get_list_id(tasks);states.reserve(tasks.size());size_t index = 0;for (auto & task : tasks) {task.index = index++;- states.push_back(task.create_state());+ states.push_back(task_result_state(session));// for child tasksfor (auto & child_task : task.child_tasks) {child_task.index = index++;- states.push_back(child_task.create_state());+ states.push_back(task_result_state(session));}}GGML_ASSERT(states.size() == id_tasks.size());diff --git a/tools/server/server-queue.h b/tools/server/server-queue.hindex e17733a743..692255e4f6 100644--- a/tools/server/server-queue.h+++ b/tools/server/server-queue.h@@ -225,7 +225,7 @@ struct server_response_reader {// if front = true, the task will be posted to the front of the queue (high priority)void post_task(server_task && task, bool front = false);- void post_tasks(std::vector<server_task> && tasks, bool front = false);+ void post_tasks(std::vector<server_task> && tasks, const common_chat_session & session = {}, bool front = false);bool has_next() const;// return nullptr if should_stop() is true before receiving a resultdiff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cppindex c7d0bc4de2..54e126aa0c 100644--- a/tools/server/server-schema.cpp+++ b/tools/server/server-schema.cpp@@ -291,54 +291,23 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &// Chat parser params//- // TODO: change this to string field instead- add((new field_json("chat_format"))- ->set_desc("Chat format used internally by the server")- ->set_handler([&](field_eval_context & ctx, const json & data) {- ctx.params.chat_parser_params.format = static_cast<common_chat_format>(data.at("chat_format").get<int>());- SRV_TRC("chat format: %s\n", common_chat_format_name(ctx.params.chat_parser_params.format));- }));-add((new field_str("reasoning_format"))->set_desc("Reasoning format for chain-of-thought models")->set_handler([&](field_eval_context & ctx, const json & data) {- auto reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());- ctx.params.chat_parser_params.reasoning_format = reasoning_format;- ctx.params.chat_parser_params.reasoning_in_content = ctx.params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);- }));-- add((new field_str("generation_prompt"))- ->set_desc("Generation prompt appended to the chat template output")- ->set_handler([&](field_eval_context & ctx, const json & data) {- std::string s = data.at("generation_prompt").get<std::string>();- ctx.params.sampling.generation_prompt = s;-- if (ctx.vocab == nullptr) {- ctx.params.chat_parser_params.generation_prompt = common_chat_input(s);- return;- }-- ctx.params.chat_parser_params.generation_prompt = common_chat_input_tokenize(ctx.vocab, s);- }));-- add((new field_bool("parse_tool_calls", params.chat_parser_params.parse_tool_calls))- ->set_desc("Whether to parse tool calls from the generated output"));-- add((new field_str("chat_parser"))- ->set_desc("Chat parser configuration string")- ->set_handler([&](field_eval_context & ctx, const json & data) {- ctx.params.chat_parser_params.parser.load(data.at("chat_parser").get<std::string>());+ ctx.params.reasoning_format = common_reasoning_format_from_name(data.at("reasoning_format").get<std::string>());}));add((new field_json("continue_final_message"))->set_desc("Whether to continue the final message of the chat template")- ->set_handler([&](field_eval_context & ctx, const json & data) {- auto continuation = common_chat_continuation_parse(data.at("continue_final_message"));- ctx.params.chat_parser_params.is_continuation = continuation != COMMON_CHAT_CONTINUATION_NONE;+ ->set_handler([&](field_eval_context &, const json & data) {+ common_chat_continuation_parse(data.at("continue_final_message"));}));- add((new field_bool("echo", params.chat_parser_params.echo))- ->set_desc("Whether to echo the input tokens in the output"));+ add((new field_json("echo"))+ ->set_desc("Whether to include the continued assistant message in the output")+ ->set_handler([&](field_eval_context &, const json & data) {+ data.at("echo").get<bool>();+ }));//// Token-level fields (require vocab)@@ -347,44 +316,17 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &add((new field_json("preserved_tokens"))->set_desc("List of token strings that must not be split during tokenization")->set_handler([&](field_eval_context & ctx, const json & data) {- GGML_ASSERT(ctx.vocab != nullptr);- for (const auto & t : data.at("preserved_tokens")) {- auto ids = common_tokenize(ctx.vocab, t.get<std::string>(), false, true);- if (ids.size() == 1) {- ctx.params.sampling.preserved_tokens.insert(ids[0]);- }- }+ common_sampling_add_preserved_tokens(ctx.params.sampling, ctx.vocab, data.at("preserved_tokens").get<std::vector<std::string>>());}));add((new field_json("grammar_triggers"))->set_desc("List of strings or patterns that trigger grammar-constrained generation")->set_handler([&](field_eval_context & ctx, const json & data) {- GGML_ASSERT(ctx.vocab != nullptr);+ std::vector<common_grammar_trigger> triggers;for (const auto & t : data.at("grammar_triggers")) {- server_grammar_trigger ct(t);- if (ct.value.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {- const auto & word = ct.value.value;- auto ids = common_tokenize(ctx.vocab, word, false, true);- if (ids.size() == 1) {- auto token = ids[0];- if (std::find(ctx.params.sampling.preserved_tokens.begin(), ctx.params.sampling.preserved_tokens.end(), (llama_token) token) == ctx.params.sampling.preserved_tokens.end()) {- throw std::runtime_error("Grammar trigger word should be marked as preserved token: " + word);- }- common_grammar_trigger trigger;- trigger.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;- trigger.value = word;- trigger.token = token;- ctx.params.sampling.grammar_triggers.push_back(std::move(trigger));- } else {- ctx.params.sampling.grammar_triggers.push_back({COMMON_GRAMMAR_TRIGGER_TYPE_WORD, word});- }- } else {- ctx.params.sampling.grammar_triggers.emplace_back(std::move(ct.value));- }- }- if (ctx.params.sampling.grammar_lazy && ctx.params.sampling.grammar_triggers.empty()) {- throw std::runtime_error("Error: no triggers set for lazy grammar!");+ triggers.push_back(server_grammar_trigger(t).value);}+ common_sampling_add_grammar_triggers(ctx.params.sampling, ctx.vocab, std::move(triggers));}));add((new field_bool("reasoning_control", params.sampling.reasoning_control))@@ -542,7 +484,7 @@ task_params eval_llama_cmpl_schema(// enabling this will output extra debug information in the HTTP responses from the serverparams.verbose = params_base.verbosity > 9;- params.chat_parser_params.reasoning_format = params_base.reasoning_format;+ params.reasoning_format = params_base.reasoning_format;// create context and schemafield_eval_context ctx(params);@@ -556,13 +498,6 @@ task_params eval_llama_cmpl_schema(f->eval(ctx, data);}- // post-processing- {- // if "reasoning_format" is not provided, its handler will not be called, we will need to handle it here- auto reasoning_format = params.chat_parser_params.reasoning_format;- params.chat_parser_params.reasoning_in_content = params.stream && (reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY);- }-// debugging{auto budget = params.sampling.reasoning_budget_tokens;diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex 21c1fe4e10..3f30da2cd6 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -73,10 +73,10 @@ json task_params::to_json(bool only_metrics) const {{"stream", stream},{"n_probs", sampling.n_probs},{"min_keep", sampling.min_keep},- {"chat_format", common_chat_format_name(chat_parser_params.format)},- {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},- {"reasoning_in_content", chat_parser_params.reasoning_in_content},- {"generation_prompt", chat_parser_params.generation_prompt.text},+ {"chat_format", common_chat_format_name(chat_format)},+ {"reasoning_format", common_reasoning_format_name(reasoning_format)},+ {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY},+ {"generation_prompt", sampling.generation_prompt},{"samplers", samplers},{"speculative.types", common_speculative_type_name_str(speculative.types)},{"timings_per_token", timings_per_token},@@ -132,10 +132,10 @@ json task_params::to_json(bool only_metrics) const {{"grammar_lazy", sampling.grammar_lazy},{"grammar_triggers", grammar_triggers},{"preserved_tokens", sampling.preserved_tokens},- {"chat_format", common_chat_format_name(chat_parser_params.format)},- {"reasoning_format", common_reasoning_format_name(chat_parser_params.reasoning_format)},- {"reasoning_in_content", chat_parser_params.reasoning_in_content},- {"generation_prompt", chat_parser_params.generation_prompt.text},+ {"chat_format", common_chat_format_name(chat_format)},+ {"reasoning_format", common_reasoning_format_name(reasoning_format)},+ {"reasoning_in_content", stream && reasoning_format == COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY},+ {"generation_prompt", sampling.generation_prompt},{"samplers", samplers},{"speculative.types", common_speculative_type_name_str(speculative.types)},{"timings_per_token", timings_per_token},@@ -148,15 +148,12 @@ json task_params::to_json(bool only_metrics) const {//// task_result_state//-task_result_state::task_result_state(const common_chat_parser_params & chat_parser_params)- : chat_parser_params(chat_parser_params)+task_result_state::task_result_state(common_chat_session session)+ : chat_session(std::move(session))+ , chat_msg(chat_session.msg()), oai_resp_id("resp_" + random_string()), oai_resp_reasoning_id("rs_" + random_string()), oai_resp_message_id("msg_" + random_string()) {- if (chat_parser_params.is_continuation && !chat_parser_params.echo) {- // initialize chat_msg to avoid emitting a delta containing the assistant prefill- chat_msg = common_chat_parse(generated_input, true, chat_parser_params);- }}common_chat_msg task_result_state::update_chat_msg(@@ -164,13 +161,8 @@ common_chat_msg task_result_state::update_chat_msg(bool is_partial,std::vector<common_chat_msg_diff> & diffs,bool filter_tool_calls) {- generated_input.append(added);auto msg_prv_copy = chat_msg;- //SRV_DBG("Parsing chat message: %s\n", generated_input.text.c_str());- auto new_msg = common_chat_parse(- generated_input,- is_partial,- chat_parser_params);+ auto new_msg = is_partial ? chat_session.feed(added) : chat_session.finish(added);if (!new_msg.empty()) {new_msg.set_tool_call_ids(generated_tool_call_ids, gen_tool_call_id);chat_msg = new_msg;diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex 0cbc5f3535..b8d6a3b670 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -89,8 +89,9 @@ struct task_params {std::string control_action;std::string control_cmpl_id;- // per-request parameters for chat parsing- common_chat_parser_params chat_parser_params;+ // reported in generation_settings, parsing itself is owned by the chat session+ common_chat_format chat_format = COMMON_CHAT_FORMAT_CONTENT_ONLY;+ common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_NONE;// message spans for checkpointingcommon_chat_msg_spans message_spans;@@ -106,9 +107,8 @@ struct task_params {struct task_result_state {// tracking diffs for partial tool callsstd::vector<common_chat_msg_diff> diffs;- common_chat_parser_params chat_parser_params;+ common_chat_session chat_session; // owns all parsing for this generationcommon_chat_msg chat_msg;- common_chat_input generated_input; // append new chunks of generated text herestd::vector<std::string> generated_tool_call_ids;std::unordered_set<size_t> sent_tool_call_names;@@ -124,7 +124,7 @@ struct task_result_state {const std::string oai_resp_message_id;std::string oai_resp_fc_id; // function call ID for current args delta- task_result_state(const common_chat_parser_params & chat_parser_params);+ task_result_state(common_chat_session session = {});// parse partial tool calls and update the internal statecommon_chat_msg update_chat_msg(@@ -258,6 +258,17 @@ struct server_task {return ids;}+ void apply_chat_session(const common_chat_session & session) {+ if (!session.has_template()) {+ return;+ }++ session.apply_sampling(params.sampling);+ params.chat_format = session.format();+ params.antiprompt.insert(params.antiprompt.end(), session.additional_stops().begin(), session.additional_stops().end());+ params.message_spans = tokens.find_message_spans(session.message_delimiters());+ }+void add_child(int id_parent, int id_child) {server_task copy;@@ -277,12 +288,6 @@ struct server_task {child_tasks.push_back(std::move(copy));}- // the task will be moved into queue, then onto slots- // however, the state must be kept by caller (e.g., HTTP thread)- task_result_state create_state() const {- return task_result_state(params.chat_parser_params);- }-bool is_parent() const {return child_tasks.size() > 0;}