Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
diff --git a/src/model.cpp b/src/model.cpp
--- a/agent.cpp/src/model.cpp
+++ b/agent.cpp/src/model.cpp
@@ -1,9 +1,14 @@
#include "model.h"
#include "chat.h"
+#include "sampling.h" // common_sampler (llamafile)
+#include "log.h" // LOG_WRN (llamafile)
+#include <nlohmann/json.hpp> // heuristic tool-call recovery (llamafile)
#include "error.h"
#include <algorithm>
#include <cstdio>
#include <fstream>
+#include <optional>
+#include <random> // new_tool_call_id (llamafile)
#include <sstream>
namespace agent_cpp {
@@ -78,7 +83,7 @@ Model::create_with_weights(std::shared_ptr<ModelWeights> weights,
Model::~Model()
{
if (sampler_ != nullptr) {
- llama_sampler_free(sampler_);
+ common_sampler_free(sampler_);
}
if (ctx_ != nullptr) {
llama_free(ctx_);
@@ -93,7 +98,6 @@ Model::Model(Model&& other) noexcept
: weights_(std::move(other.weights_))
, ctx_(other.ctx_)
, sampler_(other.sampler_)
- , grammar_sampler_(other.grammar_sampler_)
, loras_(std::move(other.loras_))
, processed_tokens_(std::move(other.processed_tokens_))
, n_past_(other.n_past_)
@@ -101,7 +105,6 @@ Model::Model(Model&& other) noexcept
{
other.ctx_ = nullptr;
other.sampler_ = nullptr;
- other.grammar_sampler_ = nullptr;
other.n_past_ = 0;
}
@@ -110,7 +113,7 @@ Model::operator=(Model&& other) noexcept
{
if (this != &other) {
if (sampler_ != nullptr) {
- llama_sampler_free(sampler_);
+ common_sampler_free(sampler_);
}
if (ctx_ != nullptr) {
llama_free(ctx_);
@@ -122,7 +125,6 @@ Model::operator=(Model&& other) noexcept
weights_ = std::move(other.weights_);
ctx_ = other.ctx_;
sampler_ = other.sampler_;
- grammar_sampler_ = other.grammar_sampler_;
loras_ = std::move(other.loras_);
processed_tokens_ = std::move(other.processed_tokens_);
n_past_ = other.n_past_;
@@ -130,12 +132,45 @@ Model::operator=(Model&& other) noexcept
other.ctx_ = nullptr;
other.sampler_ = nullptr;
- other.grammar_sampler_ = nullptr;
other.n_past_ = 0;
}
return *this;
}
+namespace {
+
+/// common_sampler_init() throws (it never returns nullptr) on a bad grammar
+/// or a rejected grammar prefill; surface that as a library error.
+common_sampler*
+init_sampler(const llama_model* model,
+ common_params_sampling& params,
+ const char* what)
+{
+ try {
+ return common_sampler_init(model, params);
+ } catch (const std::exception& e) {
+ throw ModelError(std::string(what) + ": " + e.what());
+ }
+}
+
+/// Sampling settings from ModelConfig, including the user's GBNF grammar.
+common_params_sampling
+sampling_params(const ModelConfig& config)
+{
+ common_params_sampling sp;
+ sp.temp = config.temp;
+ sp.top_k = config.top_k;
+ sp.top_p = config.top_p;
+ sp.min_p = config.min_p;
+ sp.seed = config.seed;
+ if (!config.grammar.empty()) {
+ sp.grammar = { COMMON_GRAMMAR_TYPE_USER, config.grammar };
+ }
+ return sp;
+}
+
+}
+
void
Model::initialize_context(const ModelConfig& model_config)
{
@@ -173,57 +208,37 @@ Model::initialize_context(const ModelConfig& model_config)
}
}
- sampler_ = llama_sampler_chain_init(llama_sampler_chain_default_params());
-
- if (!model_config.grammar.empty()) {
- llama_sampler* grammar_sampler =
- llama_sampler_init_grammar(weights_->get_vocab(),
- model_config.grammar.c_str(),
- model_config.grammar_root.c_str());
- if (grammar_sampler == nullptr) {
- llama_sampler_free(sampler_);
- sampler_ = nullptr;
- throw ModelError("failed to parse GBNF grammar (root rule '" +
- model_config.grammar_root + "')");
- }
- // Add grammar before the rest of the sampler chain
- llama_sampler_chain_add(sampler_, grammar_sampler);
- // Keep a non-owning reference so we can reset this sampler between
- // turns
- grammar_sampler_ = grammar_sampler;
- }
-
- llama_sampler_chain_add(sampler_,
- llama_sampler_init_top_k(model_config.top_k));
- llama_sampler_chain_add(sampler_,
- llama_sampler_init_top_p(model_config.top_p, 1));
- llama_sampler_chain_add(sampler_,
- llama_sampler_init_min_p(model_config.min_p, 1));
- llama_sampler_chain_add(sampler_,
- llama_sampler_init_temp(model_config.temp));
- llama_sampler_chain_add(sampler_,
- llama_sampler_init_dist(model_config.seed));
+ // llamafile: grammar-aware common_sampler instead of a raw chain. The
+ // user's GBNF grammar (if any) rides along; generate() re-inits the
+ // sampler per call, with the user grammar or the chat template's
+ // tool-call grammar. Note: common_grammar has no root-rule field, so the
+ // grammar's start rule must be "root" (ModelConfig::grammar_root is
+ // ignored).
+ common_params_sampling sp = sampling_params(model_config);
+ sampler_ = init_sampler(weights_->get_model(), sp,
+ "failed to initialize sampler (bad GBNF grammar?)");
}
std::vector<llama_token>
Model::tokenize(const std::string& prompt) const
{
const llama_vocab* vocab = weights_->get_vocab();
- // Use processed_tokens to determine if this is the first tokenization
- // This is important for cache loading: even if KV cache memory is
- // populated, we need IS_FIRST=true if we're tokenizing a full prompt from
- // scratch to ensure consistent BOS token handling for prefix matching
- const bool IS_FIRST = processed_tokens_.empty();
+ // llamafile: every caller tokenizes a full rendered prompt from scratch
+ // (the chat template strips its BOS text when the vocab adds one), so
+ // always add special tokens. Keying this on processed_tokens_ dropped
+ // BOS on every call after the first, which broke the KV prefix match at
+ // token 0 and re-decoded the whole conversation without BOS.
+ const bool ADD_SPECIAL = true;
const int N_PROMPT_TOKENS = -llama_tokenize(
- vocab, prompt.c_str(), prompt.size(), nullptr, 0, IS_FIRST, true);
+ vocab, prompt.c_str(), prompt.size(), nullptr, 0, ADD_SPECIAL, true);
std::vector<llama_token> prompt_tokens(N_PROMPT_TOKENS);
if (llama_tokenize(vocab,
prompt.c_str(),
prompt.size(),
prompt_tokens.data(),
prompt_tokens.size(),
- IS_FIRST,
+ ADD_SPECIAL,
true) < 0) {
return {}; // Return empty vector on failure
}
@@ -232,19 +247,6 @@ Model::tokenize(const std::string& prompt) const
namespace {
-/// Deserializes the PEG parser derived from the chat template. An empty
-/// definition yields an empty arena, which common_chat_peg_parse() treats as a
-/// pure-content parser.
-common_peg_arena
-load_parser(const std::string& parser_definition)
-{
- common_peg_arena arena;
- if (!parser_definition.empty()) {
- arena.load(parser_definition);
- }
- return arena;
-}
-
common_chat_msg
parse_with_parser(const common_peg_arena& parser,
const common_chat_params& params,
@@ -259,12 +261,14 @@ parse_with_parser(const common_peg_arena& parser,
// The parser is derived from the chat template together with the
// generation prompt it was built against, and parsing needs both
// (ggml-org/llama.cpp#18675). Without them tool calls stay as raw text.
- syntax.generation_prompt = params.generation_prompt;
- syntax.parse_tool_calls = true;
+ // llamafile: since llama.cpp v0.8.0 tool calls are always parsed, and
+ // reasoning extraction is built into the parser at apply time
+ // (inputs.reasoning_format), so there are no parse-time switches left.
+ syntax.generation_prompt = common_chat_input(params.generation_prompt);
try {
auto parsed_msg =
- common_chat_peg_parse(parser, response, false, syntax);
+ common_chat_peg_parse(parser, common_chat_input(response), false, syntax);
parsed_msg.role = "assistant";
return parsed_msg;
} catch (const std::exception& e) {
@@ -276,6 +280,62 @@ parse_with_parser(const common_peg_arena& parser,
}
}
+/// llamafile: moves a leading reasoning block out of msg.content into
+/// msg.reasoning_content, using the template's thinking tags. For raw text
+/// the parser refused. An unclosed block is all reasoning.
+void
+split_reasoning(const common_chat_params& params, common_chat_msg& msg)
+{
+ const std::string& start = params.thinking_start_tag;
+ if (start.empty() || params.thinking_end_tags.empty()) {
+ return;
+ }
+ std::string text = msg.content;
+ size_t begin = text.find_first_not_of(" \t\r\n");
+ if (begin != std::string::npos &&
+ text.compare(begin, start.size(), start) == 0) {
+ text.erase(0, begin + start.size());
+ } else {
+ // The generation prompt may have opened the block already.
+ std::string gp = params.generation_prompt;
+ gp.erase(gp.find_last_not_of(" \t\r\n") + 1);
+ if (gp.size() < start.size() ||
+ gp.compare(gp.size() - start.size(), start.size(), start) != 0) {
+ return;
+ }
+ }
+ // The first end tag closes the block; others (e.g. "<tool_call>" for
+ // Qwen3-Coder) also end reasoning but belong to the content.
+ size_t cut = std::string::npos;
+ size_t skip = 0;
+ for (size_t k = 0; k < params.thinking_end_tags.size(); k++) {
+ const std::string& tag = params.thinking_end_tags[k];
+ size_t p = tag.empty() ? std::string::npos : text.find(tag);
+ if (p < cut) {
+ cut = p;
+ skip = k == 0 ? tag.size() : 0;
+ }
+ }
+ msg.reasoning_content = text.substr(0, cut);
+ msg.content = cut == std::string::npos ? "" : text.substr(cut + skip);
+ for (std::string* s : { &msg.reasoning_content, &msg.content }) {
+ s->erase(0, s->find_first_not_of(" \t\r\n"));
+ }
+}
+
+/// llamafile: an id for a tool call the chat parser left without one.
+std::string
+new_tool_call_id()
+{
+ static std::mt19937 rng{ std::random_device{}() };
+ static const char hex[] = "0123456789abcdef";
+ std::string id = "call_";
+ for (int i = 0; i < 8; ++i) {
+ id += hex[rng() % 16];
+ }
+ return id;
+}
+
}
common_chat_msg
@@ -288,28 +348,213 @@ Model::generate(const std::vector<common_chat_msg>& messages,
inputs.tools = tools;
inputs.tool_choice = COMMON_CHAT_TOOL_CHOICE_AUTO;
inputs.add_generation_prompt = true;
- inputs.enable_thinking = false;
+ inputs.enable_thinking = config_.enable_thinking; // llamafile: was false
+ // llamafile: build the template's parser with reasoning rules so any
+ // <think>...</think> the model emits anyway is extracted into
+ // reasoning_content.
+ inputs.reasoning_format = COMMON_REASONING_FORMAT_AUTO;
auto params =
common_chat_templates_apply(weights_->get_templates(), inputs);
+ // llamafile: when the chat template provides a tool-calling grammar
+ // (or preserved tokens / triggers), re-init the sampler with it so
+ // generation is constrained to syntactically valid tool calls. Mirrors
+ // llama-server's wiring. An explicit user grammar in ModelConfig takes
+ // precedence; it is rebuilt every call because common_sampler_reset()
+ // resets the chain but not the grammar, and a grammar that completed on
+ // the last turn would only allow EOG on this one.
+ if (!config_.grammar.empty()) {
+ common_params_sampling sp = sampling_params(config_);
+ common_sampler* sampler = init_sampler(
+ weights_->get_model(), sp,
+ "failed to initialize sampler (bad GBNF grammar?)");
+ common_sampler_free(sampler_);
+ sampler_ = sampler;
+ } else if (!params.grammar.empty() || !params.grammar_triggers.empty() ||
+ !params.preserved_tokens.empty()) {
+ common_params_sampling sp = sampling_params(config_);
+ sp.generation_prompt = params.generation_prompt;
+ sp.grammar = { COMMON_GRAMMAR_TYPE_TOOL_CALLS, params.grammar };
+ sp.grammar_lazy = params.grammar_lazy;
+
+ const llama_vocab* vocab = weights_->get_vocab();
+
+ // Tell the sampler where reasoning starts and ends: a lazy grammar
+ // must not fire on a tool-call tag the model writes inside <think>.
+ // With these set, common_sampler keeps the grammar dormant until the
+ // reasoning block closes. Mirrors server-common.cpp.
+ if (!params.thinking_end_tags.empty()) {
+ sp.reasoning_budget_start =
+ common_tokenize(vocab, params.thinking_start_tag,
+ /*add_special=*/false,
+ /*parse_special=*/true);
+ for (const auto& tag : params.thinking_end_tags) {
+ if (!tag.empty()) {
+ sp.reasoning_budget_end.push_back(
+ common_tokenize(vocab, tag,
+ /*add_special=*/false,
+ /*parse_special=*/true));
+ }
+ }
+ }
+
+ // Resolve preserved-token strings to single token ids.
+ for (const auto& t : params.preserved_tokens) {
+ auto ids = common_tokenize(vocab, t,
+ /*add_special=*/false,
+ /*parse_special=*/true);
+ if (ids.size() == 1) {
+ sp.preserved_tokens.insert(ids[0]);
+ }
+ }
+
+ // WORD-type triggers become TOKEN-type when the word is a single
+ // preserved token. Mirrors server-task.cpp.
+ for (const auto& t : params.grammar_triggers) {
+ if (t.type == COMMON_GRAMMAR_TRIGGER_TYPE_WORD) {
+ auto ids = common_tokenize(vocab, t.value,
+ /*add_special=*/false,
+ /*parse_special=*/true);
+ if (ids.size() == 1 &&
+ sp.preserved_tokens.count(ids[0])) {
+ common_grammar_trigger trig;
+ trig.type = COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN;
+ trig.value = t.value;
+ trig.token = ids[0];
+ sp.grammar_triggers.push_back(std::move(trig));
+ } else {
+ sp.grammar_triggers.push_back(t);
+ }
+ } else {
+ sp.grammar_triggers.push_back(t);
+ }
+ }
+
+ // Swap only once the new sampler exists: if init throws, sampler_
+ // must still be the live one that ~Model frees.
+ common_sampler* sampler = init_sampler(
+ weights_->get_model(), sp, "failed to initialize tool-call sampler");
+ common_sampler_free(sampler_);
+ sampler_ = sampler;
+ }
+
// Tokenize the prompt
- std::vector<llama_token> prompt_tokens = tokenize(params.prompt);
+ std::vector<llama_token> prompt_tokens = tokenize(params.prompt.str());
if (prompt_tokens.empty()) {
throw ModelError("failed to tokenize prompt");
}
std::string response = generate_from_tokens(prompt_tokens, callback);
- // The PEG parser only changes when the rendered template changes, so keep
- // the deserialized arena around instead of re-parsing it every turn.
- if (parser_source_ != params.parser) {
- parser_arena_ = load_parser(params.parser);
- parser_source_ = params.parser;
+ common_chat_msg parsed_msg;
+ std::optional<ModelError> parse_error;
+ try {
+ parsed_msg = parse_with_parser(
+ params.parser, params, response, config_.chat_format);
+ } catch (const ModelError& e) {
+ // llamafile: a parse refusal must not kill the agent loop. Fall
+ // back to content-only, with the reasoning split off so it is
+ // neither returned as the answer nor scanned for tool calls; the
+ // recovery scan below may still lift tool calls out of the rest.
+ LOG_WRN("[agent.cpp] %s; using heuristic recovery\n", e.what());
+ // common_log prints from a worker thread; flush so the warning
+ // lands before a confirmation prompt for a call recovered here.
+ common_log_flush(common_log_main());
+ parse_error = e;
+ parsed_msg.content = response;
+ split_reasoning(params, parsed_msg);
}
-
- return parse_with_parser(
- parser_arena_, params, response, config_.chat_format);
+ // A reply that ends inside a reasoning block it never closed, with
+ // nothing else, holds the model's answer in that block: small models
+ // do this after a tool result. Take it as the content, so a tool call
+ // in it is recovered below like one the parser left in the content.
+ bool unclosed_reasoning = false;
+ if (parsed_msg.tool_calls.empty() && parsed_msg.content.empty() &&
+ !parsed_msg.reasoning_content.empty() &&
+ std::none_of(params.thinking_end_tags.begin(),
+ params.thinking_end_tags.end(),
+ [&](const std::string& tag) {
+ return !tag.empty() &&
+ response.find(tag) != std::string::npos;
+ })) {
+ parsed_msg.content = std::move(parsed_msg.reasoning_content);
+ parsed_msg.reasoning_content.clear();
+ unclosed_reasoning = true;
+ }
+ // llamafile heuristic recovery: lift <tool_call>{json}</tool_call>
+ // blocks out of the content. Only when the parser failed, has no
+ // tool-call format of its own (no grammar), or read the text as
+ // reasoning (above): after a clean parse by a tool-aware parser, such
+ // text in the content is prose, not a call.
+ if (parsed_msg.tool_calls.empty() &&
+ (parse_error || params.grammar.empty() || unclosed_reasoning)) {
+ const std::string text = parsed_msg.content;
+ const std::string open = "<tool_call>";
+ const std::string close = "</tool_call>";
+ std::string content_clean;
+ size_t i = 0;
+ while (i < text.size()) {
+ auto a = text.find(open, i);
+ if (a == std::string::npos) {
+ content_clean.append(text, i, std::string::npos);
+ break;
+ }
+ content_clean.append(text, i, a - i);
+ auto b = text.find(close, a + open.size());
+ if (b == std::string::npos) break;
+ std::string payload =
+ text.substr(a + open.size(), b - (a + open.size()));
+ size_t s = payload.find_first_not_of(" \t\r\n");
+ size_t e2 = payload.find_last_not_of(" \t\r\n");
+ if (s != std::string::npos && e2 != std::string::npos) {
+ payload = payload.substr(s, e2 - s + 1);
+ }
+ try {
+ // ordered_json keeps the keys in the order the model wrote
+ // them, so the next prompt renders this call exactly as it
+ // was generated and reuses its tokens from the KV cache.
+ auto j = nlohmann::ordered_json::parse(payload);
+ common_chat_tool_call call;
+ call.name = j.value("name", std::string{});
+ // Calls to parameterless tools often omit "arguments", and
+ // some models say "parameters"; the tool loop needs JSON.
+ auto args = nlohmann::ordered_json::object();
+ if (j.contains("arguments")) {
+ args = j["arguments"];
+ } else if (j.contains("parameters")) {
+ args = j["parameters"];
+ }
+ call.arguments =
+ args.is_string() ? args.get<std::string>() : args.dump();
+ if (!call.name.empty()) {
+ parsed_msg.tool_calls.push_back(std::move(call));
+ }
+ } catch (...) {
+ content_clean.append(open).append(payload).append(close);
+ }
+ i = b + close.size();
+ }
+ if (!parsed_msg.tool_calls.empty()) {
+ parsed_msg.content = content_clean;
+ }
+ }
+ // A tool call the parser refused and recovery could not lift is not an
+ // answer: report the parse error rather than return the raw markup.
+ if (parse_error && parsed_msg.tool_calls.empty() &&
+ parsed_msg.content.find("<tool_call>") != std::string::npos) {
+ throw *parse_error;
+ }
+ // llamafile: every tool call gets an id, as llama-server gives them, so
+ // the tool result can name the call it answers. Some templates' parsers
+ // leave ids empty, and so does the recovery above.
+ for (auto& call : parsed_msg.tool_calls) {
+ if (call.id.empty()) {
+ call.id = new_tool_call_id();
+ }
+ }
+ parsed_msg.role = "assistant";
+ return parsed_msg;
}
common_chat_msg
@@ -317,8 +562,7 @@ parse_response(const common_chat_params& params,
const std::string& response,
std::optional<common_chat_format> format_override)
{
- return parse_with_parser(
- load_parser(params.parser), params, response, format_override);
+ return parse_with_parser(params.parser, params, response, format_override);
}
std::string
@@ -338,10 +582,16 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,
common_prefix++;
}
- // If tokens diverged, clear KV cache from divergence point onwards
+ // If tokens diverged, clear KV cache from divergence point onwards.
+ // llamafile: recurrent/hybrid models cannot rewind their state to a
+ // mid-sequence position — llama_memory_seq_rm refuses and returns
+ // false. Fall back to dropping the whole memory and re-decoding.
if (common_prefix < processed_tokens_.size()) {
llama_memory_t mem = llama_get_memory(ctx_);
- llama_memory_seq_rm(mem, 0, common_prefix, -1);
+ if (!llama_memory_seq_rm(mem, 0, common_prefix, -1)) {
+ llama_memory_clear(mem, /*data=*/true);
+ common_prefix = 0;
+ }
processed_tokens_.resize(common_prefix);
n_past_ = common_prefix;
}
@@ -370,15 +620,15 @@ Model::generate_from_tokens(const std::vector<llama_token>& all_tokens,
i += batch_size;
}
- // Reset the grammar sampler before each turn so a finished grammar does
- // not force EOS on the next call.
- if (grammar_sampler_ != nullptr) {
- llama_sampler_reset(grammar_sampler_);
- }
+ // Reset the sampler chain before each turn. This leaves the grammar
+ // alone; generate() rebuilds the sampler when a grammar must restart.
+ common_sampler_reset(sampler_);
llama_token new_token_id{};
while (true) {
- new_token_id = llama_sampler_sample(sampler_, ctx_, -1);
+ new_token_id = common_sampler_sample(sampler_, ctx_, -1);
+ common_sampler_accept(sampler_, new_token_id,
+ /*accept_grammar=*/true);
if (llama_vocab_is_eog(vocab, new_token_id)) {
break;