Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Re-apply grammar parser memory bug fix 22aba95a1
Justine Tunney3fe045feef36
diff --git a/llama.cpp/grammar-parser.cpp b/llama.cpp/grammar-parser.cppindex bce28784..57463bd4 100644--- a/llama.cpp/grammar-parser.cpp+++ b/llama.cpp/grammar-parser.cpp@@ -145,6 +145,9 @@ namespace grammar_parser {pos++;last_sym_start = out_elements.size();while (*pos != '"') {+ if (!*pos) { // [jart] don't sync until upstream fixes bug+ throw std::runtime_error("unexpected end of input");+ }auto char_pair = parse_char(pos);pos = char_pair.second;out_elements.push_back({LLAMA_GRETYPE_CHAR, char_pair.first});@@ -159,6 +162,9 @@ namespace grammar_parser {}last_sym_start = out_elements.size();while (*pos != ']') {+ if (!*pos) { // [jart] don't sync until upstream fixes bug+ throw std::runtime_error("unexpected end of input");+ }auto char_pair = parse_char(pos);pos = char_pair.second;enum llama_gretype type = last_sym_start < out_elements.size()@@ -167,6 +173,9 @@ namespace grammar_parser {out_elements.push_back({type, char_pair.first});if (pos[0] == '-' && pos[1] != ']') {+ if (pos[1]) { // [jart] don't sync until upstream fixes bug+ throw std::runtime_error("unexpected end of input");+ }auto endchar_pair = parse_char(pos + 1);pos = endchar_pair.second;out_elements.push_back({LLAMA_GRETYPE_CHAR_RNG_UPPER, endchar_pair.first});diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex d9abcd81..cdedcdf5 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -866,12 +866,12 @@ struct llama_server_context}}- if (slot->ctx_sampling != nullptr)- {+ if (slot->ctx_sampling != nullptr) {llama_sampling_free(slot->ctx_sampling);}slot->ctx_sampling = llama_sampling_init(slot->sparams);- if (!slot->ctx_sampling) { // [jart] fixes crash+ if (slot->ctx_sampling == nullptr) {+ // for now, the only error that may happen here is invalid grammarLOG_TEE("%s: failed to initialize sampling subsystem\n", __func__);return false;}