Skip to content

Distribute and run LLMs with a single file.

Allow --grammar to be used on --image prompts

Justine Tunney1cd334ffa81f
diff --git a/README.md b/README.md
index 093a72d8..80a22f1e 100644
--- a/README.md
+++ b/README.md
@@ -329,6 +329,24 @@ llamafile --temp 0 \
--silent-prompt 2>/dev/null
```
+It's possible to use BNF grammar to enforce the output is predictable
+and safe to use in your shell script. The simplest grammar would be
+`--grammar 'root ::= "yes" | "no"'` to force the LLM to only print to
+standard output either `"yes\n"` or `"no\n"`. Another example is if you
+wanted to write a script to rename all your image files, you could say:
+
+```sh
+llamafile --temp 0 \
+ --image ~/Pictures/lemurs.jpg \
+ -m llava-v1.5-7b-Q4_K.gguf \
+ --mmproj llava-v1.5-7b-mmproj-Q4_0.gguf \
+ --grammar 'root ::= [a-z]+ (" " [a-z]+)+' \
+ -p $'### User: What do you see?\n### Assistant: ' \
+ --silent-prompt 2>/dev/null |
+ sed -e's/ /_/' -e's/$/.jpg/'
+a_baby_monkey_on_the_back_of_a_mother.jpg
+```
+
Here's an example of how to run llama.cpp's built-in HTTP server. This
example uses LLaVA v1.5-7B, a multimodal LLM that works with llama.cpp's
recently-added support for image inputs.
diff --git a/llama.cpp/README.llamafile b/llama.cpp/README.llamafile
index bf6ab964..4efc2136 100644
--- a/llama.cpp/README.llamafile
+++ b/llama.cpp/README.llamafile
@@ -33,6 +33,7 @@ LOCAL MODIFICATIONS
- Introduce pledge() SECCOMP sandboxing to improve security
- Introduce ggml_interrupt() to gracefully handle signals
- Remove log callback pointer API from Metal GPU module
+ - Allow --grammar to be used on --image prompts
- Use runtime dispatching for matmul quants
- Remove operating system #ifdef statements
- Introduce --silent-prompt flag to main
diff --git a/llama.cpp/llava/llava-cli.cpp b/llama.cpp/llava/llava-cli.cpp
index acdc94fa..e30f8c01 100644
--- a/llama.cpp/llava/llava-cli.cpp
+++ b/llama.cpp/llava/llava-cli.cpp
@@ -42,73 +42,11 @@ static bool eval_string(struct llama_context * ctx_llama, const char* str, int n
return true;
}
-// TODO: use common/sampling.h
-static llama_token sample_id(llama_context * ctx_llama, gpt_params & params) {
- auto & sparams = params.sparams;
-
- // out of user input, sample next token
- const float temp = sparams.temp;
- const int32_t top_k = sparams.top_k <= 0 ? llama_n_vocab(llama_get_model(ctx_llama)) : sparams.top_k;
- const float top_p = sparams.top_p;
- const float tfs_z = sparams.tfs_z;
- const float typical_p = sparams.typical_p;
- // const int32_t repeat_last_n = sparams.repeat_last_n < 0 ? n_ctx : sparams.repeat_last_n;
- // const float repeat_penalty = sparams.repeat_penalty;
- // const float alpha_presence = sparams.presence_penalty;
- // const float alpha_frequency = sparams.frequency_penalty;
- const int mirostat = sparams.mirostat;
- const float mirostat_tau = sparams.mirostat_tau;
- const float mirostat_eta = sparams.mirostat_eta;
- // const bool penalize_nl = sparams.penalize_nl;
-
- llama_token id = 0;
- {
- auto logits = llama_get_logits(ctx_llama);
- auto n_vocab = llama_n_vocab(llama_get_model(ctx_llama));
-
- // Apply params.logit_bias map
- for (auto it = sparams.logit_bias.begin(); it != sparams.logit_bias.end(); it++) {
- logits[it->first] += it->second;
- }
-
- std::vector<llama_token_data> candidates;
- candidates.reserve(n_vocab);
- for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
- candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
- }
-
- llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
-
- if (temp <= 0) {
- // Greedy sampling
- id = llama_sample_token_greedy(ctx_llama, &candidates_p);
- } else {
- if (mirostat == 1) {
- static float mirostat_mu = 2.0f * mirostat_tau;
- const int mirostat_m = 100;
- llama_sample_temp(ctx_llama, &candidates_p, temp);
- id = llama_sample_token_mirostat(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, mirostat_m, &mirostat_mu);
- } else if (mirostat == 2) {
- static float mirostat_mu = 2.0f * mirostat_tau;
- llama_sample_temp(ctx_llama, &candidates_p, temp);
- id = llama_sample_token_mirostat_v2(ctx_llama, &candidates_p, mirostat_tau, mirostat_eta, &mirostat_mu);
- } else {
- // Temperature sampling
- llama_sample_top_k(ctx_llama, &candidates_p, top_k, 1);
- llama_sample_tail_free(ctx_llama, &candidates_p, tfs_z, 1);
- llama_sample_typical(ctx_llama, &candidates_p, typical_p, 1);
- llama_sample_top_p(ctx_llama, &candidates_p, top_p, 1);
- llama_sample_temp(ctx_llama, &candidates_p, temp);
- id = llama_sample_token(ctx_llama, &candidates_p);
- }
- }
- }
-
- return id;
-}
-
-static const char * sample(struct llama_context * ctx_llama, gpt_params & params, int * n_past) {
- int id = sample_id(ctx_llama, params);
+static const char * sample(struct llama_sampling_context * ctx_sampling,
+ struct llama_context * ctx_llama,
+ int * n_past) {
+ const llama_token id = llama_sampling_sample(ctx_sampling, ctx_llama, NULL);
+ llama_sampling_accept(ctx_sampling, ctx_llama, id, true);
static std::string ret;
if (id == llama_token_eos(llama_get_model(ctx_llama))) {
ret = "</s>";
@@ -221,14 +159,17 @@ static void process_prompt(struct llava_context * ctx_llava, struct llava_image_
fprintf(stderr, "\n");
+ struct llama_sampling_context * ctx_sampling = llama_sampling_init(params->sparams);
+
for (int i = 0; i < max_tgt_len; i++) {
- const char * tmp = sample(ctx_llava->ctx_llama, *params, &n_past);
+ const char * tmp = sample(ctx_sampling, ctx_llava->ctx_llama, &n_past);
if (strcmp(tmp, "</s>") == 0) break;
printf("%s", tmp);
fflush(stdout);
}
+ llama_sampling_free(ctx_sampling);
printf("\n");
}