Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Allow --grammar to be used on --image prompts
Justine Tunney1cd334ffa81f
diff --git a/README.md b/README.mdindex 093a72d8..80a22f1e 100644--- a/README.md+++ b/README.md@@ -329,6 +329,24 @@ llamafile --temp 0 \--silent-prompt 2>/dev/null```+It's possible to use BNF grammar to enforce the output is predictable+and safe to use in your shell script. The simplest grammar would be+`--grammar 'root ::= "yes" | "no"'` to force the LLM to only print to+standard output either `"yes\n"` or `"no\n"`. Another example is if you+wanted to write a script to rename all your image files, you could say:++```sh+llamafile --temp 0 \+ --image ~/Pictures/lemurs.jpg \+ -m llava-v1.5-7b-Q4_K.gguf \+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \+ --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \+ -p $'### User: What do you see?\n### Assistant: ' \+ --silent-prompt 2>/dev/null |+ sed -e's/ /_/' -e's/$/.jpg/'+a_baby_monkey_on_the_back_of_a_mother.jpg+```+Here's an example of how to run llama.cpp's built-in HTTP server. Thisexample uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp'srecently-added support for image inputs.diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafileindex bf6ab964..4efc2136 100644--- a/llama.cpp/README.llamafile+++ b/llama.cpp/README.llamafile@@ -33,6 +33,7 @@ LOCAL MODIFICATIONS- Introduce pledge() SECCOMP sandboxing to improve security- Introduce ggml_interrupt() to gracefully handle signals- Remove log callback pointer API from Metal GPU module+ - Allow --grammar to be used on --image prompts- Use runtime dispatching for matmul quants- Remove operating system #ifdef statements- Introduce --silent-prompt flag to maindiff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cppindex acdc94fa..e30f8c01 100644--- a/llama.cpp/llava/llava-cli.cpp+++ b/llama.cpp/llava/llava-cli.cpp@@ -42,73 +42,11 @@ static bool eval_string(struct llama_context * ctx_llama, const char* str, int nreturn true;}-// TODO: use common/sampling.h-static llama_token sample_id(llama_context * ctx_llama, gpt_params & params) {- auto & sparams = params.sparams;-- // out of user input, sample next token- const float temp = sparams.temp;- const int32_t top_k = sparams.top_k <= 0 ? llama_n_vocab(llama_get_model(ctx_llama)) : sparams.top_k;- const float top_p = sparams.top_p;- const float tfs_z = sparams.tfs_z;- const float typical_p = sparams.typical_p;- // const int32_t repeat_last_n = sparams.repeat_last_n < 0 ? n_ctx : sparams.repeat_last_n;- // const float repeat_penalty = sparams.repeat_penalty;- // const float alpha_presence = sparams.presence_penalty;- // const float alpha_frequency = sparams.frequency_penalty;- const int mirostat = sparams.mirostat;- const float mirostat_tau = sparams.mirostat_tau;- const float mirostat_eta = sparams.mirostat_eta;- // const bool penalize_nl = sparams.penalize_nl;-- llama_token id = 0;- {- auto logits = llama_get_logits(ctx_llama);- auto n_vocab = llama_n_vocab(llama_get_model(ctx_llama));-- // Apply params.logit_bias map- for (auto it = sparams.logit_bias.begin(); it != sparams.logit_bias.end(); it++) {- logits[it->first] += it->second;- }-- std::vector<llama_token_data> candidates;- candidates.reserve(n_vocab);- for (llama_token token_id = 0; token_id < n_vocab; token_id++) {- candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});- }-- llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };-- if (temp <= 0) {- // Greedy sampling- id = llama_sample_token_greedy(ctx_llama, &candidates_p);- } else {- if (mirostat == 1) {- static float mirostat_mu = 2.0f * mirostat_tau;- const int mirostat_m = 100;- llama_sample_temp(ctx_llama, &candidates_p, temp);- id = llama_sample_token_mirostat(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);- } else if (mirostat == 2) {- static float mirostat_mu = 2.0f * mirostat_tau;- llama_sample_temp(ctx_llama, &candidates_p, temp);- id = llama_sample_token_mirostat_v2(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);- } else {- // Temperature sampling- llama_sample_top_k(ctx_llama, &candidates_p, top_k, 1);- llama_sample_tail_free(ctx_llama, &candidates_p, tfs_z, 1);- llama_sample_typical(ctx_llama, &candidates_p, typical_p, 1);- llama_sample_top_p(ctx_llama, &candidates_p, top_p, 1);- llama_sample_temp(ctx_llama, &candidates_p, temp);- id = llama_sample_token(ctx_llama, &candidates_p);- }- }- }-- return id;-}--static const char * sample(struct llama_context * ctx_llama, gpt_params & params, int * n_past) {- int id = sample_id(ctx_llama, params);+static const char * sample(struct llama_sampling_context * ctx_sampling,+ struct llama_context * ctx_llama,+ int * n_past) {+ const llama_token id = llama_sampling_sample(ctx_sampling, ctx_llama, NULL);+ llama_sampling_accept(ctx_sampling, ctx_llama, id, true);static std::string ret;if (id == llama_token_eos(llama_get_model(ctx_llama))) {ret = "</s>";@@ -221,14 +159,17 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_fprintf(stderr, "\n");+ struct llama_sampling_context * ctx_sampling = llama_sampling_init(params->sparams);+for (int i = 0; i < max_tgt_len; i++) {- const char * tmp = sample(ctx_llava->ctx_llama, *params, &n_past);+ const char * tmp = sample(ctx_sampling, ctx_llava->ctx_llama, &n_past);if (strcmp(tmp, "</s>") == 0) break;printf("%s", tmp);fflush(stdout);}+ llama_sampling_free(ctx_sampling);printf("\n");}