Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Introduce /upload and /forget commands to chatbot
Justine Tunneyd25c07745f67
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex c4086d74..75ee12bc 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -168,6 +168,8 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {FLAGS_READY = true;params.n_gpu_layers = llamafile_gpu_layers(params.n_gpu_layers);+ FLAG_threads = params.n_threads; // [jart]+ FLAG_threads_batch = params.n_threads_batch; // [jart]return true;}@@ -275,7 +277,6 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paif (params.n_threads <= 0) {params.n_threads = MIN(cpu_get_num_math(), 20); // [jart]}- FLAG_threads = params.n_threads; // [jart]return true;}if (arg == "-tb" || arg == "--threads-batch") {@@ -284,7 +285,6 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paif (params.n_threads_batch <= 0) {params.n_threads_batch = cpu_get_num_math(); // [jart]}- FLAG_threads_batch = params.n_threads_batch; // [jart]return true;}if (arg == "-td" || arg == "--threads-draft") {@@ -691,9 +691,10 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paparams.control_vector_layer_end = std::stoi(argv[i]);return true;}- if (arg == "--mmproj") {+ if (arg == "-mm" || arg == "--mmproj") { // [jart]CHECK_ARGparams.mmproj = argv[i];+ FLAG_mmproj = argv[i]; // [jart]return true;}if (arg == "--image") {diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 181ebaf2..fd998513 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -194,8 +194,9 @@ struct gpt_params {bool warmup = true; // warmup runbool check_tensors = false; // validate tensor data- std::string cache_type_k = X86_HAVE(AVX512_BF16) ? "bf16" : "f16"; // KV cache data type for the K [jart]- std::string cache_type_v = X86_HAVE(AVX512_BF16) ? "bf16" : "f16"; // KV cache data type for the V [jart]+ // [jart] warning: rope only supports f32 and f16+ std::string cache_type_k = "f16"; // KV cache data type for the K+ std::string cache_type_v = "f16"; // KV cache data type for the V// multimodal models (see examples/llava)std::string mmproj = ""; // path to multimodal projectordiff --git a/llama.cpp/ggml-cuda.cu b/llama.cpp/ggml-cuda.cuindex c41990ff..3ca5a410 100644--- a/llama.cpp/ggml-cuda.cu+++ b/llama.cpp/ggml-cuda.cu@@ -13341,6 +13341,7 @@ void ggml_cuda_op_rope(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {float * dst_d = (float *)dst->data;cudaStream_t stream = ctx.stream();+ // TODO[jart]: support bf16GGML_ASSERT(ggml_is_contiguous(src0));GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);GGML_ASSERT( dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);diff --git a/llama.cpp/ggml.c b/llama.cpp/ggml.cindex 0c22ee1e..9ebc1762 100644--- a/llama.cpp/ggml.c+++ b/llama.cpp/ggml.c@@ -13043,6 +13043,7 @@ static void ggml_compute_forward_rope(const struct ggml_tensor * src0 = dst->src[0];+ // TODO[jart]: support bf16switch (src0->type) {case GGML_TYPE_F16:{@@ -13067,6 +13068,7 @@ static void ggml_compute_forward_rope_back(const struct ggml_tensor * src0 = dst->src[0];+ // TODO[jart]: support bf16switch (src0->type) {case GGML_TYPE_F16:{diff --git a/llama.cpp/main/README.md b/llama.cpp/main/README.mdindex e381ea4c..d45d0f37 100644--- a/llama.cpp/main/README.md+++ b/llama.cpp/main/README.md@@ -83,30 +83,58 @@ The `main` program provides several ways to interact with the LLaMA models using## Interaction-The `main` program offers a seamless way to interact with LLaMA models, allowing users to engage in real-time conversations or provide instructions for specific tasks. The interactive mode can be triggered using various options, including `--interactive`, `--interactive-first`, and `--instruct`.--In interactive mode, users can participate in text generation by injecting their input during the process. Users can press `Ctrl+C` at any time to interject and type their input, followed by pressing `Return` to submit it to the LLaMA model. To submit additional lines without finalizing input, users can end the current line with a backslash (`\`) and continue typing.+The `main` program offers a seamless way to interact with LLaMA models,+allowing users to engage in real-time conversations or provide+instructions for specific tasks. The interactive mode can be triggered+using various options, including `--interactive`, `--interactive-first`,+and `--instruct`.++In interactive mode, users can participate in text generation by+injecting their input during the process. Users can press `Ctrl+C` at+any time to interject and type their input, followed by pressing+`Return` to submit it to the LLaMA model. To submit additional lines+without finalizing input, users can end the current line with a+backslash (`\`) and continue typing.### Interaction Options-- `-i, --interactive`: Run the program in interactive mode, allowing users to engage in real-time conversations or provide specific instructions to the model.-- `--interactive-first`: Run the program in interactive mode and immediately wait for user input before starting the text generation.-- `-ins, --instruct`: Run the program in instruction mode, which is specifically designed to work with Alpaca models that excel in completing tasks based on user instructions.-- `--color`: Enable colorized output to differentiate visually distinguishing between prompts, user input, and generated text.--By understanding and utilizing these interaction options, you can create engaging and dynamic experiences with the LLaMA models, tailoring the text generation process to your specific needs.+- `-i, --interactive`: Run the program in interactive mode, allowing+ users to engage in real-time conversations or provide specific+ instructions to the model.+- `--interactive-first`: Run the program in interactive mode and+ immediately wait for user input before starting the text generation.+- `-ins, --instruct`: Run the program in instruction mode, which is+ specifically designed to work with Alpaca models that excel in+ completing tasks based on user instructions.+- `--color`: Enable colorized output to differentiate visually+ distinguishing between prompts, user input, and generated text.++By understanding and utilizing these interaction options, you can create+engaging and dynamic experiences with the LLaMA models, tailoring the+text generation process to your specific needs.### Reverse Prompts-Reverse prompts are a powerful way to create a chat-like experience with a LLaMA model by pausing the text generation when specific text strings are encountered:+Reverse prompts are a powerful way to create a chat-like experience with+a LLaMA model by pausing the text generation when specific text strings+are encountered:-- `-r PROMPT, --reverse-prompt PROMPT`: Specify one or multiple reverse prompts to pause text generation and switch to interactive mode. For example, `-r "User:"` can be used to jump back into the conversation whenever it's the user's turn to speak. This helps create a more interactive and conversational experience. However, the reverse prompt doesn't work when it ends with a space.+- `-r PROMPT, --reverse-prompt PROMPT`: Specify one or multiple+ reverse prompts to pause text generation and switch to interactive+ mode. For example, `-r "User:"` can be used to jump back into the+ conversation whenever it's the user's turn to speak. This helps+ create a more interactive and conversational experience. However,+ the reverse prompt doesn't work when it ends with a space.-To overcome this limitation, you can use the `--in-prefix` flag to add a space or any other characters after the reverse prompt.+To overcome this limitation, you can use the `--in-prefix` flag to add a+space or any other characters after the reverse prompt.### In-Prefix-The `--in-prefix` flag is used to add a prefix to your input, primarily, this is used to insert a space after the reverse prompt. Here's an example of how to use the `--in-prefix` flag in conjunction with the `--reverse-prompt` flag:+The `--in-prefix` flag is used to add a prefix to your input, primarily,+this is used to insert a space after the reverse prompt. Here's an+example of how to use the `--in-prefix` flag in conjunction with the+`--reverse-prompt` flag:```sh./main -r "User:" --in-prefix " "@@ -114,7 +142,11 @@ The `--in-prefix` flag is used to add a prefix to your input, primarily, this is### In-Suffix-The `--in-suffix` flag is used to add a suffix after your input. This is useful for adding an "Assistant:" prompt after the user's input. It's added after the new-line character (`\n`) that's automatically added to the end of the user's input. Here's an example of how to use the `--in-suffix` flag in conjunction with the `--reverse-prompt` flag:+The `--in-suffix` flag is used to add a suffix after your input. This is+useful for adding an "Assistant:" prompt after the user's input. It's+added after the new-line character (`\n`) that's automatically added to+the end of the user's input. Here's an example of how to use the+`--in-suffix` flag in conjunction with the `--reverse-prompt` flag:```sh./main -r "User:" --in-prefix " " --in-suffix "Assistant:"@@ -122,13 +154,21 @@ The `--in-suffix` flag is used to add a suffix after your input. This is useful### Instruction Mode-Instruction mode is particularly useful when working with Alpaca models, which are designed to follow user instructions for specific tasks:+Instruction mode is particularly useful when working with Alpaca models,+which are designed to follow user instructions for specific tasks:-- `-ins, --instruct`: Enable instruction mode to leverage the capabilities of Alpaca models in completing tasks based on user-provided instructions.+- `-ins, --instruct`: Enable instruction mode to leverage the+ capabilities of Alpaca models in completing tasks based on+ user-provided instructions.-Technical detail: the user's input is internally prefixed with the reverse prompt (or `### Instruction:` as the default), and followed by `### Response:` (except if you just press Return without any input, to keep generating a longer response).+Technical detail: the user's input is internally prefixed with the+reverse prompt (or `### Instruction:` as the default), and followed by+`### Response:` (except if you just press Return without any input, to+keep generating a longer response).-By understanding and utilizing these interaction options, you can create engaging and dynamic experiences with the LLaMA models, tailoring the text generation process to your specific needs.+By understanding and utilizing these interaction options, you can create+engaging and dynamic experiences with the LLaMA models, tailoring the+text generation process to your specific needs.## Context Management@@ -146,7 +186,11 @@ used by the LLaMA models during text generation. A larger context sizehelps the model to better comprehend and generate responses for longerinput or conversations.-- `-c N, --ctx-size N`: Set the size of the prompt context (default: 512). The LLaMA models were built with a context of 2048, which will yield the best results on longer input/inference. However, increasing the context size beyond 2048 may lead to unpredictable results.+- `-c N, --ctx-size N`: Set the size of the prompt context (default:+ 512). The LLaMA models were built with a context of 2048, which will+ yield the best results on longer input/inference. However,+ increasing the context size beyond 2048 may lead to unpredictable+ results.### Extended Context Size@@ -156,7 +200,8 @@ For example, if the original pre-trained model have a context lengthThat is a scaling factor of 8, and should work by setting the above`--ctx-size` to 32768 (32k) and `--rope-scale` to 8.-- `--rope-scale N`: Where N is the linear scaling factor used by the fine-tuned model.+- `--rope-scale N`: Where N is the linear scaling factor used by the+ fine-tuned model.### Keep Prompt@@ -169,7 +214,10 @@ instruction or conversation topic is maintained.value is set to 0 (meaning no tokens are kept). Use `-1` to retainall tokens from the initial prompt.-By utilizing context management options like `--ctx-size` and `--keep`, you can maintain a more coherent and consistent interaction with the LLaMA models, ensuring that the generated text remains relevant to the original prompt or conversation.+By utilizing context management options like `--ctx-size` and `--keep`,+you can maintain a more coherent and consistent interaction with the+LLaMA models, ensuring that the generated text remains relevant to the+original prompt or conversation.## Generation Flags@@ -181,101 +229,210 @@ best settings for your specific use case.### Number of Tokens to Predict-- `-n N, --n-predict N`: Set the number of tokens to predict when generating text (default: 128, -1 = infinity, -2 = until context filled)--The `--n-predict` option controls the number of tokens the model generates in response to the input prompt. By adjusting this value, you can influence the length of the generated text. A higher value will result in longer text, while a lower value will produce shorter text.--A value of -1 will enable infinite text generation, even though we have a finite context window. When the context window is full, some of the earlier tokens (half of the tokens after `--n-keep`) will be discarded. The context must then be re-evaluated before generation can resume. On large models and/or large context windows, this will result in significant pause in output.--If the pause is undesirable, a value of -2 will stop generation immediately when the context is filled.--It is important to note that the generated text may be shorter than the specified number of tokens if an End-of-Sequence (EOS) token or a reverse prompt is encountered. In interactive mode text generation will pause and control will be returned to the user. In non-interactive mode, the program will end. In both cases, the text generation may stop before reaching the specified `n-predict` value. If you want the model to keep going without ever producing End-of-Sequence on its own, you can use the `--ignore-eos` parameter.+- `-n N, --n-predict N`: Set the number of tokens to predict when+ generating text (default: 128, -1 = infinity, -2 = until context+ filled)++The `--n-predict` option controls the number of tokens the model+generates in response to the input prompt. By adjusting this value, you+can influence the length of the generated text. A higher value will+result in longer text, while a lower value will produce shorter text.++A value of -1 will enable infinite text generation, even though we have+a finite context window. When the context window is full, some of the+earlier tokens (half of the tokens after `--n-keep`) will be discarded.+The context must then be re-evaluated before generation can resume. On+large models and/or large context windows, this will result in+significant pause in output.++If the pause is undesirable, a value of -2 will stop generation+immediately when the context is filled.++It is important to note that the generated text may be shorter than the+specified number of tokens if an End-of-Sequence (EOS) token or a+reverse prompt is encountered. In interactive mode text generation will+pause and control will be returned to the user. In non-interactive mode,+the program will end. In both cases, the text generation may stop before+reaching the specified `n-predict` value. If you want the model to keep+going without ever producing End-of-Sequence on its own, you can use the+`--ignore-eos` parameter.### Temperature- `--temp N`: Adjust the randomness of the generated text (default: 0.8).-Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The default value is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.+Temperature is a hyperparameter that controls the randomness of the+generated text. It affects the probability distribution of the model's+output tokens. A higher temperature (e.g., 1.5) makes the output more+random and creative, while a lower temperature (e.g., 0.5) makes the+output more focused, deterministic, and conservative. The default value+is 0.8, which provides a balance between randomness and determinism. At+the extreme, a temperature of 0 will always pick the most likely next+token, leading to identical outputs in each run.Example usage: `--temp 0.5`### Repeat Penalty-- `--repeat-penalty N`: Control the repetition of token sequences in the generated text (default: 1.1).-- `--repeat-last-n N`: Last n tokens to consider for penalizing repetition (default: 64, 0 = disabled, -1 = ctx-size).-- `--no-penalize-nl`: Disable penalization for newline tokens when applying the repeat penalty.--The `repeat-penalty` option helps prevent the model from generating repetitive or monotonous text. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. The default value is 1.1.--The `repeat-last-n` option controls the number of tokens in the history to consider for penalizing repetition. A larger value will look further back in the generated text to prevent repetitions, while a smaller value will only consider recent tokens. A value of 0 disables the penalty, and a value of -1 sets the number of tokens considered equal to the context size (`ctx-size`).--Use the `--no-penalize-nl` option to disable newline penalization when applying the repeat penalty. This option is particularly useful for generating chat conversations, dialogues, code, poetry, or any text where newline tokens play a significant role in structure and formatting. Disabling newline penalization helps maintain the natural flow and intended formatting in these specific use cases.+- `--repeat-penalty N`: Control the repetition of token sequences in+ the generated text (default: 1.1).+- `--repeat-last-n N`: Last n tokens to consider for penalizing+ repetition (default: 64, 0 = disabled, -1 = ctx-size).+- `--no-penalize-nl`: Disable penalization for newline tokens when+ applying the repeat penalty.++The `repeat-penalty` option helps prevent the model from generating+repetitive or monotonous text. A higher value (e.g., 1.5) will penalize+repetitions more strongly, while a lower value (e.g., 0.9) will be more+lenient. The default value is 1.1.++The `repeat-last-n` option controls the number of tokens in the history+to consider for penalizing repetition. A larger value will look further+back in the generated text to prevent repetitions, while a smaller value+will only consider recent tokens. A value of 0 disables the penalty, and+a value of -1 sets the number of tokens considered equal to the context+size (`ctx-size`).++Use the `--no-penalize-nl` option to disable newline penalization when+applying the repeat penalty. This option is particularly useful for+generating chat conversations, dialogues, code, poetry, or any text+where newline tokens play a significant role in structure and+formatting. Disabling newline penalization helps maintain the natural+flow and intended formatting in these specific use cases.Example usage: `--repeat-penalty 1.15 --repeat-last-n 128 --no-penalize-nl`### Top-K Sampling-- `--top-k N`: Limit the next token selection to the K most probable tokens (default: 40).+- `--top-k N`: Limit the next token selection to the K most probable+ tokens (default: 40).-Top-k sampling is a text generation method that selects the next token only from the top k most likely tokens predicted by the model. It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit the diversity of the output. A higher value for top-k (e.g., 100) will consider more tokens and lead to more diverse text, while a lower value (e.g., 10) will focus on the most probable tokens and generate more conservative text. The default value is 40.+Top-k sampling is a text generation method that selects the next token+only from the top k most likely tokens predicted by the model. It helps+reduce the risk of generating low-probability or nonsensical tokens, but+it may also limit the diversity of the output. A higher value for top-k+(e.g., 100) will consider more tokens and lead to more diverse text,+while a lower value (e.g., 10) will focus on the most probable tokens+and generate more conservative text. The default value is 40.Example usage: `--top-k 30`### Top-P Sampling-- `--top-p N`: Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P (default: 0.9).+- `--top-p N`: Limit the next token selection to a subset of tokens+ with a cumulative probability above a threshold P (default: 0.9).-Top-p sampling, also known as nucleus sampling, is another text generation method that selects the next token from a subset of tokens that together have a cumulative probability of at least p. This method provides a balance between diversity and quality by considering both the probabilities of tokens and the number of tokens to sample from. A higher value for top-p (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. The default value is 0.9.+Top-p sampling, also known as nucleus sampling, is another text+generation method that selects the next token from a subset of tokens+that together have a cumulative probability of at least p. This method+provides a balance between diversity and quality by considering both the+probabilities of tokens and the number of tokens to sample from. A+higher value for top-p (e.g., 0.95) will lead to more diverse text,+while a lower value (e.g., 0.5) will generate more focused and+conservative text. The default value is 0.9.Example usage: `--top-p 0.95`### Min P Sampling-- `--min-p N`: Sets a minimum base probability threshold for token selection (default: 0.05).+- `--min-p N`: Sets a minimum base probability threshold for token+ selection (default: 0.05).-The Min-P sampling method was designed as an alternative to Top-P, and aims to ensure a balance of quality and variety. The parameter *p* represents the minimum probability for a token to be considered, relative to the probability of the most likely token. For example, with *p*=0.05 and the most likely token having a probability of 0.9, logits with a value less than 0.045 are filtered out.+The Min-P sampling method was designed as an alternative to Top-P, and+aims to ensure a balance of quality and variety. The parameter *p*+represents the minimum probability for a token to be considered,+relative to the probability of the most likely token. For example, with+*p*=0.05 and the most likely token having a probability of 0.9, logits+with a value less than 0.045 are filtered out.Example usage: `--min-p 0.05`### Tail Free Sampling (TFS)-- `--tfs N`: Enable tail free sampling with parameter z (default: 1.0, 1.0 = disabled).--Tail free sampling (TFS) is a text generation technique that aims to reduce the impact of less likely tokens, which may be less relevant, less coherent, or nonsensical, on the output. Similar to Top-P it tries to determine the bulk of the most likely tokens dynamically. But TFS filters out logits based on the second derivative of their probabilities. Adding tokens is stopped after the sum of the second derivatives reaches the parameter z. In short: TFS looks how quickly the probabilities of the tokens decrease and cuts off the tail of unlikely tokens using the parameter z. Typical values for z are in the range of 0.9 to 0.95. A value of 1.0 would include all tokens, and thus disables the effect of TFS.+- `--tfs N`: Enable tail free sampling with parameter z (default: 1.0,+ 1.0 = disabled).++Tail free sampling (TFS) is a text generation technique that aims to+reduce the impact of less likely tokens, which may be less relevant,+less coherent, or nonsensical, on the output. Similar to Top-P it tries+to determine the bulk of the most likely tokens dynamically. But TFS+filters out logits based on the second derivative of their+probabilities. Adding tokens is stopped after the sum of the second+derivatives reaches the parameter z. In short: TFS looks how quickly the+probabilities of the tokens decrease and cuts off the tail of unlikely+tokens using the parameter z. Typical values for z are in the range of+0.9 to 0.95. A value of 1.0 would include all tokens, and thus disables+the effect of TFS.Example usage: `--tfs 0.95`### Locally Typical Sampling-- `--typical N`: Enable locally typical sampling with parameter p (default: 1.0, 1.0 = disabled).+- `--typical N`: Enable locally typical sampling with parameter p+ (default: 1.0, 1.0 = disabled).-Locally typical sampling promotes the generation of contextually coherent and diverse text by sampling tokens that are typical or expected based on the surrounding context. By setting the parameter p between 0 and 1, you can control the balance between producing text that is locally coherent and diverse. A value closer to 1 will promote more contextually coherent tokens, while a value closer to 0 will promote more diverse tokens. A value equal to 1 disables locally typical sampling.+Locally typical sampling promotes the generation of contextually+coherent and diverse text by sampling tokens that are typical or+expected based on the surrounding context. By setting the parameter p+between 0 and 1, you can control the balance between producing text that+is locally coherent and diverse. A value closer to 1 will promote more+contextually coherent tokens, while a value closer to 0 will promote+more diverse tokens. A value equal to 1 disables locally typical+sampling.Example usage: `--typical 0.9`### Mirostat Sampling-- `--mirostat N`: Enable Mirostat sampling, controlling perplexity during text generation (default: 0, 0 = disabled, 1 = Mirostat, 2 = Mirostat 2.0).-- `--mirostat-lr N`: Set the Mirostat learning rate, parameter eta (default: 0.1).-- `--mirostat-ent N`: Set the Mirostat target entropy, parameter tau (default: 5.0).--Mirostat is an algorithm that actively maintains the quality of generated text within a desired range during text generation. It aims to strike a balance between coherence and diversity, avoiding low-quality output caused by excessive repetition (boredom traps) or incoherence (confusion traps).--The `--mirostat-lr` option sets the Mirostat learning rate (eta). The learning rate influences how quickly the algorithm responds to feedback from the generated text. A lower learning rate will result in slower adjustments, while a higher learning rate will make the algorithm more responsive. The default value is `0.1`.--The `--mirostat-ent` option sets the Mirostat target entropy (tau), which represents the desired perplexity value for the generated text. Adjusting the target entropy allows you to control the balance between coherence and diversity in the generated text. A lower value will result in more focused and coherent text, while a higher value will lead to more diverse and potentially less coherent text. The default value is `5.0`.+- `--mirostat N`: Enable Mirostat sampling, controlling perplexity+ during text generation (default: 0, 0 = disabled, 1 = Mirostat, 2 =+ Mirostat 2.0).+- `--mirostat-lr N`: Set the Mirostat learning rate, parameter eta+ (default: 0.1).+- `--mirostat-ent N`: Set the Mirostat target entropy, parameter tau+ (default: 5.0).++Mirostat is an algorithm that actively maintains the quality of+generated text within a desired range during text generation. It aims to+strike a balance between coherence and diversity, avoiding low-quality+output caused by excessive repetition (boredom traps) or incoherence+(confusion traps).++The `--mirostat-lr` option sets the Mirostat learning rate (eta). The+learning rate influences how quickly the algorithm responds to feedback+from the generated text. A lower learning rate will result in slower+adjustments, while a higher learning rate will make the algorithm more+responsive. The default value is `0.1`.++The `--mirostat-ent` option sets the Mirostat target entropy (tau),+which represents the desired perplexity value for the generated text.+Adjusting the target entropy allows you to control the balance between+coherence and diversity in the generated text. A lower value will result+in more focused and coherent text, while a higher value will lead to+more diverse and potentially less coherent text. The default value is+`5.0`.Example usage: `--mirostat 2 --mirostat-lr 0.05 --mirostat-ent 3.0`### Logit Bias-- `-l TOKEN_ID(+/-)BIAS, --logit-bias TOKEN_ID(+/-)BIAS`: Modify the likelihood of a token appearing in the generated text completion.+- `-l TOKEN_ID(+/-)BIAS, --logit-bias TOKEN_ID(+/-)BIAS`: Modify the+ likelihood of a token appearing in the generated text completion.-The logit bias option allows you to manually adjust the likelihood of specific tokens appearing in the generated text. By providing a token ID and a positive or negative bias value, you can increase or decrease the probability of that token being generated.+The logit bias option allows you to manually adjust the likelihood of+specific tokens appearing in the generated text. By providing a token ID+and a positive or negative bias value, you can increase or decrease the+probability of that token being generated.-For example, use `--logit-bias 15043+1` to increase the likelihood of the token 'Hello', or `--logit-bias 15043-1` to decrease its likelihood. Using a value of negative infinity, `--logit-bias 15043-inf` ensures that the token `Hello` is never produced.+For example, use `--logit-bias 15043+1` to increase the likelihood of+the token 'Hello', or `--logit-bias 15043-1` to decrease its likelihood.+Using a value of negative infinity, `--logit-bias 15043-inf` ensures+that the token `Hello` is never produced.-A more practical use case might be to prevent the generation of `\code{begin}` and `\code{end}` by setting the `\` token (29905) to negative infinity with `-l 29905-inf`. (This is due to the prevalence of LaTeX codes that show up in LLaMA model inference.)+A more practical use case might be to prevent the generation of+`\code{begin}` and `\code{end}` by setting the `\` token (29905) to+negative infinity with `-l 29905-inf`. (This is due to the prevalence of+LaTeX codes that show up in LLaMA model inference.)Example usage: `--logit-bias 29905-inf`@@ -287,7 +444,10 @@ The RNG seed is used to initialize the random number generator that influences t## Performance Tuning and Memory Options-These options help improve the performance and memory usage of the LLaMA models. By adjusting these settings, you can fine-tune the model's behavior to better suit your system's capabilities and achieve optimal performance for your specific use case.+These options help improve the performance and memory usage of the LLaMA+models. By adjusting these settings, you can fine-tune the model's+behavior to better suit your system's capabilities and achieve optimal+performance for your specific use case.### Number of Threads@@ -312,7 +472,11 @@ These options help improve the performance and memory usage of the LLaMA models.### Batch Size-- `-b N, --batch-size N`: Set the batch size for prompt processing (default: 512). This large batch size benefits users who have BLAS installed and enabled it during the build. If you don't have BLAS enabled ("BLAS=0"), you can use a smaller number, such as 8, to see the prompt progress as it's evaluated in some situations.+- `-b N, --batch-size N`: Set the batch size for prompt processing+ (default: 512). This large batch size benefits users who have BLAS+ installed and enabled it during the build. If you don't have BLAS+ enabled ("BLAS=0"), you can use a smaller number, such as 8, to see+ the prompt progress as it's evaluated in some situations.### Prompt Cachingdiff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index 391b64f3..20775dc0 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -147,7 +147,33 @@ flag:.PpDefault: empty.It Fl n Ar N , Fl Fl n-predict Ar N-Number of tokens to predict.+Sets number of tokens to predict when generating text.+.Pp+This option controls the number of tokens the model generates in+response to the input prompt. By adjusting this value, you can influence+the length of the generated text. A higher value will result in longer+text, while a lower value will produce shorter text.+.Pp+A value of -1 will enable infinite text generation, even though we have+a finite context window. When the context window is full, some of the+earlier tokens (half of the tokens after+.Fl Fl n-keep )+will be discarded. The context must then be re-evaluated before+generation can resume. On large models and/or large context windows,+this will result in significant pause in output.+.Pp+If the pause is undesirable, a value of -2 will stop generation+immediately when the context is filled.+.Pp+It is important to note that the generated text may be shorter than the+specified number of tokens if an End-of-Sequence (EOS) token or a+reverse prompt is encountered. In interactive mode text generation will+pause and control will be returned to the user. In non-interactive mode,+the program will end. In both cases, the text generation may stop before+reaching the specified `n-predict` value. If you want the model to keep+going without ever producing End-of-Sequence on its own, you can use the+.Fl Fl ignore-eos+parameter..Pp.Bl -dash -compact.It@@ -164,56 +190,91 @@ mode, this value sets a hard limit on how long your conversation can be.The default is 8192 tokens. If this value is zero, then it'll be set tothe maximum context size the model allows..It Fl b Ar N , Fl Fl batch-size Ar N-Batch size for prompt processing.+Set batch size for prompt processing..PpDefault: 2048.It Fl Fl top-k Ar N-Top-k sampling.+Limits next token selection to K most probable tokens..Pp-.Bl -dash -compact-.It-0 = disabled-.El+Top-k sampling is a text generation method that selects the next token+only from the top k most likely tokens predicted by the model. It helps+reduce the risk of generating low-probability or nonsensical tokens, but+it may also limit the diversity of the output. A higher value for top-k+(e.g., 100) will consider more tokens and lead to more diverse text,+while a lower value (e.g., 10) will focus on the most probable tokens+and generate more conservative text..PpDefault: 40.It Fl Fl top-p Ar N-Top-p sampling.-.Pp-.Bl -dash -compact-.It-1.0 = disabled-.El+Limits next token selection to a subset of tokens with a cumulative+probability above a threshold P.+.Pp+Top-p sampling, also known as nucleus sampling, is another text+generation method that selects the next token from a subset of tokens+that together have a cumulative probability of at least p. This method+provides a balance between diversity and quality by considering both the+probabilities of tokens and the number of tokens to sample from. A+higher value for top-p (e.g., 0.95) will lead to more diverse text,+while a lower value (e.g., 0.5) will generate more focused and+conservative text..PpDefault: 0.9.It Fl Fl min-p Ar N-Min-p sampling.+Sets minimum base probability threshold for token selection..Pp-.Bl -dash -compact-.It-0.0 = disabled-.El+The Min-P sampling method was designed as an alternative to Top-P, and+aims to ensure a balance of quality and variety. The parameter p+represents the minimum probability for a token to be considered,+relative to the probability of the most likely token. For example, with+p=0.05 and the most likely token having a probability of 0.9, logits+with a value less than 0.045 are filtered out..Pp-Default: 0.1+Default: 0.05.It Fl Fl tfs Ar N-Tail free sampling, parameter z.-.Pp-.Bl -dash -compact-.It-1.0 = disabled-.El-.Pp-Default: 1.0+Enables tail free sampling with parameter z.+.Pp+Tail free sampling (TFS) is a text generation technique that aims to+reduce the impact of less likely tokens, which may be less relevant,+less coherent, or nonsensical, on the output. Similar to Top-P it tries+to determine the bulk of the most likely tokens dynamically. But TFS+filters out logits based on the second derivative of their+probabilities. Adding tokens is stopped after the sum of the second+derivatives reaches the parameter z. In short: TFS looks how quickly the+probabilities of the tokens decrease and cuts off the tail of unlikely+tokens using the parameter z. Typical values for z are in the range of+0.9 to 0.95. A value of 1.0 would include all tokens, and thus disables+the effect of TFS.+.Pp+Default: 1.0 (which means disabled).It Fl Fl typical Ar N-Locally typical sampling, parameter p.+Enables locally typical sampling with parameter p.+.Pp+Locally typical sampling promotes the generation of contextually+coherent and diverse text by sampling tokens that are typical or+expected based on the surrounding context. By setting the parameter p+between 0 and 1, you can control the balance between producing text that+is locally coherent and diverse. A value closer to 1 will promote more+contextually coherent tokens, while a value closer to 0 will promote+more diverse tokens. A value equal to 1 disables locally typical+sampling.+.Pp+Default: 1.0 (which means disabled)+.It Fl Fl repeat-penalty Ar N+Controls repetition of token sequences in generated text..Pp-.Bl -dash -compact-.It-1.0 = disabled-.El+This can help prevent the model from generating repetitive or monotonous+text. A higher value (e.g., 1.5) will penalize repetitions more+strongly, while a lower value (e.g., 0.9) will be more lenient..Pp-Default: 1.0+Default: 1.1.It Fl Fl repeat-last-n Ar N-Last n tokens to consider for penalize.+Last n tokens to consider for penalizing repetition.+.Pp+This controls the number of tokens in the history to consider for+penalizing repetition. A larger value will look further back in the+generated text to prevent repetitions, while a smaller value will only+consider recent tokens. A value of 0 disables the penalty, and a value+of -1 sets the number of tokens considered equal to the context size..Pp.Bl -dash -compact.It@@ -223,15 +284,15 @@ Last n tokens to consider for penalize..El.PpDefault: 64-.It Fl Fl repeat-penalty Ar N-Penalize repeat sequence of tokens.-.Pp-.Bl -dash -compact-.It-1.0 = disabled-.El-.Pp-Default: 1.1+.It Fl Fl no-penalize-nl+Disables penalization of newline tokens when applying the repeat+penalty.+.Pp+This option is particularly useful for generating chat conversations,+dialogues, code, poetry, or any text where newline tokens play a+significant role in structure and formatting. Disabling newline+penalization helps maintain the natural flow and intended formatting in+these specific use cases..It Fl Fl presence-penalty Ar NRepeat alpha presence penalty..Pp@@ -251,7 +312,16 @@ Repeat alpha frequency penalty..PpDefault: 0.0.It Fl Fl mirostat Ar N-Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally Typical samplers are ignored if used..+Use Mirostat sampling.+.Pp+Mirostat is an algorithm that actively maintains the quality of+generated text within a desired range during text generation. It aims to+strike a balance between coherence and diversity, avoiding low-quality+output caused by excessive repetition (boredom traps) or incoherence+(confusion traps).+.Pp+Using Mirostat causes the Top K, Nucleus, Tail Free and Locally Typical+samplers parameter to be ignored if used..Pp.Bl -dash -compact.It@@ -264,11 +334,22 @@ Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally Typical samplers ar.PpDefault: 0.It Fl Fl mirostat-lr Ar N-Mirostat learning rate, parameter eta.+Sets the Mirostat learning rate (eta).+.Pp+The learning rate influences how quickly the algorithm responds to+feedback from the generated text. A lower learning rate will result in+slower adjustments, while a higher learning rate will make the algorithm+more responsive..PpDefault: 0.1.It Fl Fl mirostat-ent Ar N-Mirostat target entropy, parameter tau.+Sets the Mirostat target entropy (tau).+.Pp+This represents the desired perplexity value for the generated text.+Adjusting the target entropy allows you to control the balance between+coherence and diversity in the generated text. A lower value will result+in more focused and coherent text, while a higher value will lead to+more diverse and potentially less coherent text..PpDefault: 5.0.It Fl l Ar TOKEN_ID(+/-)BIAS , Fl Fl logit-bias Ar TOKEN_ID(+/-)BIAS@@ -352,10 +433,17 @@ Default: 32.0.It Fl Fl ignore-eosIgnore end of stream token and continue generating (implies.Fl Fl logit-bias Ar 2-inf )-.It Fl Fl no-penalize-nl-Do not penalize newline token..It Fl Fl temp Ar N-Temperature.+Adjust the randomness of the generated text.+.Pp+Temperature is a hyperparameter that controls the randomness of the+generated text. It affects the probability distribution of the model's+output tokens. A higher temperature (e.g., 1.5) makes the output more+random and creative, while a lower temperature (e.g., 0.5) makes the+output more focused, deterministic, and conservative. The default value+is 0.8, which provides a balance between randomness and determinism. At+the extreme, a temperature of 0 will always pick the most likely next+token, leading to identical outputs in each run..PpDefault: 0.8.It Fl Fl logits-all@@ -525,10 +613,6 @@ Run in chatml mode (use with ChatML-compatible models)Verbose print of the KV cache..It Fl nkvo , Fl Fl no-kv-offloadDisable KV offload.-.It Fl ctk Ar TYPE , Fl Fl cache-type-k Ar TYPE-KV cache data type for K.-.It Fl ctv Ar TYPE , Fl Fl cache-type-v Ar TYPE-KV cache data type for V..It Fl gan Ar N , Fl Fl grp-attn-n Ar NGroup-attention factor..Ppdiff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.ascindex 55467b6d..c6d87fb1 100644--- a/llama.cpp/main/main.1.asc+++ b/llama.cpp/main/main.1.asc@@ -106,7 +106,34 @@Default: empty[1m-n [4m[22mN[24m, [1m--n-predict [4m[22mN[0m- Number of tokens to predict.+ Sets number of tokens to predict when generating text.++ This option controls the number of tokens the model generates+ in response to the input prompt. By adjusting this value, you+ can influence the length of the generated text. A higher value+ will result in longer text, while a lower value will produce+ shorter text.++ A value of -1 will enable infinite text generation, even though+ we have a finite context window. When the context window is+ full, some of the earlier tokens (half of the tokens after+ [1m--n-keep[22m) will be discarded. The context must then be re-evalu‐+ ated before generation can resume. On large models and/or large+ context windows, this will result in significant pause in out‐+ put.++ If the pause is undesirable, a value of -2 will stop generation+ immediately when the context is filled.++ It is important to note that the generated text may be shorter+ than the specified number of tokens if an End-of-Sequence (EOS)+ token or a reverse prompt is encountered. In interactive mode+ text generation will pause and control will be returned to the+ user. In non-interactive mode, the program will end. In both+ cases, the text generation may stop before reaching the speci‐+ fied `n-predict` value. If you want the model to keep going+ without ever producing End-of-Sequence on its own, you can use+ the [1m--ignore-eos [22mparameter.[1m- [22m-1 = infinity[1m- [22m-2 = until context filled@@ -120,59 +147,118 @@be set to the maximum context size the model allows.[1m-b [4m[22mN[24m, [1m--batch-size [4m[22mN[0m- Batch size for prompt processing.+ Set batch size for prompt processing.Default: 2048[1m--top-k [4m[22mN[0m- Top-k sampling.+ Limits next token selection to K most probable tokens.- [1m- [22m0 = disabled+ Top-k sampling is a text generation method that selects the+ next token only from the top k most likely tokens predicted by+ the model. It helps reduce the risk of generating low-probabil‐+ ity or nonsensical tokens, but it may also limit the diversity+ of the output. A higher value for top-k (e.g., 100) will con‐+ sider more tokens and lead to more diverse text, while a lower+ value (e.g., 10) will focus on the most probable tokens and+ generate more conservative text.Default: 40[1m--top-p [4m[22mN[0m- Top-p sampling.-- [1m- [22m1.0 = disabled+ Limits next token selection to a subset of tokens with a cumu‐+ lative probability above a threshold P.++ Top-p sampling, also known as nucleus sampling, is another text+ generation method that selects the next token from a subset of+ tokens that together have a cumulative probability of at least+ p. This method provides a balance between diversity and quality+ by considering both the probabilities of tokens and the number+ of tokens to sample from. A higher value for top-p (e.g., 0.95)+ will lead to more diverse text, while a lower value (e.g., 0.5)+ will generate more focused and conservative text.Default: 0.9[1m--min-p [4m[22mN[0m- Min-p sampling.+ Sets minimum base probability threshold for token selection.- [1m- [22m0.0 = disabled+ The Min-P sampling method was designed as an alternative to+ Top-P, and aims to ensure a balance of quality and variety. The+ parameter p represents the minimum probability for a token to+ be considered, relative to the probability of the most likely+ token. For example, with p=0.05 and the most likely token hav‐+ ing a probability of 0.9, logits with a value less than 0.045+ are filtered out.- Default: 0.1+ Default: 0.05[1m--tfs [4m[22mN[0m- Tail free sampling, parameter z.+ Enables tail free sampling with parameter z.++ Tail free sampling (TFS) is a text generation technique that+ aims to reduce the impact of less likely tokens, which may be+ less relevant, less coherent, or nonsensical, on the output.+ Similar to Top-P it tries to determine the bulk of the most+ likely tokens dynamically. But TFS filters out logits based on+ the second derivative of their probabilities. Adding tokens is+ stopped after the sum of the second derivatives reaches the pa‐+ rameter z. In short: TFS looks how quickly the probabilities of+ the tokens decrease and cuts off the tail of unlikely tokens+ using the parameter z. Typical values for z are in the range of+ 0.9 to 0.95. A value of 1.0 would include all tokens, and thus+ disables the effect of TFS.++ Default: 1.0 (which means disabled)- [1m- [22m1.0 = disabled+ [1m--typical [4m[22mN[0m+ Enables locally typical sampling with parameter p.- Default: 1.0+ Locally typical sampling promotes the generation of contextu‐+ ally coherent and diverse text by sampling tokens that are typ‐+ ical or expected based on the surrounding context. By setting+ the parameter p between 0 and 1, you can control the balance+ between producing text that is locally coherent and diverse. A+ value closer to 1 will promote more contextually coherent to‐+ kens, while a value closer to 0 will promote more diverse to‐+ kens. A value equal to 1 disables locally typical sampling.- [1m--typical [4m[22mN[0m- Locally typical sampling, parameter p.+ Default: 1.0 (which means disabled)- [1m- [22m1.0 = disabled+ [1m--repeat-penalty [4m[22mN[0m+ Controls repetition of token sequences in generated text.- Default: 1.0+ This can help prevent the model from generating repetitive or+ monotonous text. A higher value (e.g., 1.5) will penalize repe‐+ titions more strongly, while a lower value (e.g., 0.9) will be+ more lenient.++ Default: 1.1[1m--repeat-last-n [4m[22mN[0m- Last n tokens to consider for penalize.+ Last n tokens to consider for penalizing repetition.++ This controls the number of tokens in the history to consider+ for penalizing repetition. A larger value will look further+ back in the generated text to prevent repetitions, while a+ smaller value will only consider recent tokens. A value of 0+ disables the penalty, and a value of -1 sets the number of to‐+ kens considered equal to the context size.[1m- [22m0 = disabled[1m- [22m-1 = ctx_sizeDefault: 64- [1m--repeat-penalty [4m[22mN[0m- Penalize repeat sequence of tokens.-- [1m- [22m1.0 = disabled+ [1m--no-penalize-nl[0m+ Disables penalization of newline tokens when applying the re‐+ peat penalty.- Default: 1.1+ This option is particularly useful for generating chat conver‐+ sations, dialogues, code, poetry, or any text where newline to‐+ kens play a significant role in structure and formatting. Dis‐+ abling newline penalization helps maintain the natural flow and+ intended formatting in these specific use cases.[1m--presence-penalty [4m[22mN[0mRepeat alpha presence penalty.@@ -189,8 +275,16 @@Default: 0.0[1m--mirostat [4m[22mN[0m- Use Mirostat sampling. Top K, Nucleus, Tail Free and Locally- Typical samplers are ignored if used..+ Use Mirostat sampling.++ Mirostat is an algorithm that actively maintains the quality of+ generated text within a desired range during text generation.+ It aims to strike a balance between coherence and diversity,+ avoiding low-quality output caused by excessive repetition+ (boredom traps) or incoherence (confusion traps).++ Using Mirostat causes the Top K, Nucleus, Tail Free and Locally+ Typical samplers parameter to be ignored if used.[1m- [22m0 = disabled[1m- [22m1 = Mirostat@@ -199,12 +293,24 @@Default: 0[1m--mirostat-lr [4m[22mN[0m- Mirostat learning rate, parameter eta.+ Sets the Mirostat learning rate (eta).++ The learning rate influences how quickly the algorithm responds+ to feedback from the generated text. A lower learning rate will+ result in slower adjustments, while a higher learning rate will+ make the algorithm more responsive.Default: 0.1[1m--mirostat-ent [4m[22mN[0m- Mirostat target entropy, parameter tau.+ Sets the Mirostat target entropy (tau).++ This represents the desired perplexity value for the generated+ text. Adjusting the target entropy allows you to control the+ balance between coherence and diversity in the generated text.+ A lower value will result in more focused and coherent text,+ while a higher value will lead to more diverse and potentially+ less coherent text.Default: 5.0@@ -290,11 +396,18 @@Ignore end of stream token and continue generating (implies[1m--logit-bias [4m[22m2-inf[24m)- [1m--no-penalize-nl[0m- Do not penalize newline token.-[1m--temp [4m[22mN[0m- Temperature.+ Adjust the randomness of the generated text.++ Temperature is a hyperparameter that controls the randomness of+ the generated text. It affects the probability distribution of+ the model's output tokens. A higher temperature (e.g., 1.5)+ makes the output more random and creative, while a lower tem‐+ perature (e.g., 0.5) makes the output more focused, determinis‐+ tic, and conservative. The default value is 0.8, which provides+ a balance between randomness and determinism. At the extreme, a+ temperature of 0 will always pick the most likely next token,+ leading to identical outputs in each run.Default: 0.8@@ -481,12 +594,6 @@[1m-nkvo[22m, [1m--no-kv-offload[0mDisable KV offload.- [1m-ctk [4m[22mTYPE[24m, [1m--cache-type-k [4m[22mTYPE[0m- KV cache data type for K.-- [1m-ctv [4m[22mTYPE[24m, [1m--cache-type-v [4m[22mTYPE[0m- KV cache data type for V.-[1m-gan [4m[22mN[24m, [1m--grp-attn-n [4m[22mN[0mGroup-attention factor.diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cppindex 53bdd4e7..0a69f223 100644--- a/llamafile/chatbot.cpp+++ b/llamafile/chatbot.cpp@@ -18,16 +18,20 @@#include <assert.h>#include <cosmo.h>#include <ctype.h>+#include <glob.h>#include <math.h>#include <signal.h>#include <sstream>#include <stdio.h>#include <string>+#include <sys/stat.h>#include <vector>#include "llama.cpp/common.h"#include "llama.cpp/ggml-cuda.h"#include "llama.cpp/llama.h"+#include "llama.cpp/llava/clip.h"+#include "llama.cpp/llava/llava.h"#include "llama.cpp/server/server.h"#include "llamafile/bestline.h"#include "llamafile/chatbot.h"@@ -36,6 +40,10 @@#include "llamafile/llama.h"#include "llamafile/llamafile.h"#include "llamafile/string.h"+#include "llamafile/xterm.h"+#include "stb/stb_image.h"++#define IMAGE_PLACEHOLDER_TOKEN -31337enum Role {ROLE_USER,@@ -51,6 +59,8 @@ struct ServerArgs {static bool g_manual_mode;static bool g_said_something;static int g_system_prompt_tokens;+static gpt_params g_params;+static clip_ctx *g_clip;static llama_model *g_model;static llama_context *g_ctx;static std::vector<llama_pos> g_undo;@@ -88,11 +98,34 @@ static void fix_stacks(void) {fix_stack(&g_stack);}+static std::vector<llama_pos> adjust_stack(llama_pos erase_begin, llama_pos erase_end,+ const std::vector<llama_pos> &stack) {+ std::vector<llama_pos> builder;+ for (llama_pos pos : stack) {+ if (erase_begin <= pos && pos < erase_end)+ continue;+ if (pos >= erase_end)+ pos -= erase_end - erase_begin;+ builder.push_back(pos);+ }+ return builder;+}++static void adjust_stacks(llama_pos erase_begin, llama_pos erase_end) {+ g_undo = adjust_stack(erase_begin, erase_end, g_undo);+ g_stack = adjust_stack(erase_begin, erase_end, g_stack);+}+static void record_undo(void) {if (g_undo.empty() || g_undo.back() != tokens_used())g_undo.push_back(tokens_used());}+static bool is_directory(const char *path) {+ struct stat st;+ return !stat(path, &st) && S_ISDIR(st.st_mode);+}+static const char *get_role_name(enum Role role) {switch (role) {case ROLE_USER:@@ -145,6 +178,22 @@ static enum Role cycle_role(enum Role role) {}}+static bool is_image_file(const char *path) {+ int width, height, channels;+ stbi_set_flip_vertically_on_load(false); // force full image decode+ unsigned char *data = stbi_load(path, &width, &height, &channels, 0);+ if (data != nullptr) {+ stbi_image_free(data);+ return true;+ } else {+ return false;+ }+}++static bool has_binary(const std::string_view s) {+ return s.find('\0') != std::string_view::npos;+}+static std::string describe_compute(void) {if (llama_n_gpu_layers(g_model) > 0 && llamafile_has_gpu()) {if (llamafile_has_metal()) {@@ -164,15 +213,57 @@ static std::string describe_compute(void) {}}+static std::string token_to_piece(const struct llama_context *ctx, llama_token token,+ bool special) {+ if (token == IMAGE_PLACEHOLDER_TOKEN)+ return "⁑";+ return llama_token_to_piece(ctx, token, special);+}++static std::string describe_token(llama_token token) {+ if (token == llama_token_bos(g_model))+ return "§";+ if (token == llama_token_eos(g_model))+ return "∎";+ if (token == llama_token_cls(g_model))+ return "⌘";+ if (token == llama_token_sep(g_model))+ return "⋯";+ if (token == llama_token_pad(g_model))+ return "␣";+ if (token == llama_token_nl(g_model))+ return "↵";+ if (llama_token_is_eog(g_model, token))+ return "⌟";+ if (llama_token_is_control(g_model, token))+ return "∷";+ std::string s = token_to_piece(g_ctx, token, DONT_RENDER_SPECIAL_TOKENS);+ if (s.empty())+ return "↯";+ return s;+}++static std::string describe_erasure(llama_pos begin, llama_pos end) {+ unassert(begin <= end);+ unassert(end <= tokens_used());+ std::string description;+ llama_pos pos = begin;+ while (pos < end && description.size() < 63)+ description += describe_token(g_history[pos++]);+ if (!description.empty() && pos < end)+ description += " ...";+ description = lf::collapse(description);+ if (pos == end && description.empty())+ description = "<absolute end>";+ return description;+}+static std::string describe_position(llama_pos pos) {unassert(pos <= tokens_used());std::string description;- while (pos > 0 && description.size() < 63) {- std::string piece =- llama_token_to_piece(g_ctx, g_history[--pos], DONT_RENDER_SPECIAL_TOKENS);- description = piece + description;- }- if (pos > 0)+ while (pos > 0 && description.size() < 63)+ description = describe_token(g_history[--pos]) + description;+ if (!description.empty() && pos > 0)description = std::string("... ") + description;description = lf::collapse(description);if (!pos && description.empty())@@ -180,14 +271,131 @@ static std::string describe_position(llama_pos pos) {return description;}+static void err(const char *fmt, ...) {+ va_list ap;+ va_start(ap, fmt);+ fputs(BRIGHT_RED, stderr);+ vfprintf(stderr, fmt, ap);+ fputs(RESET "\n", stderr);+ va_end(ap);+}++static void print_ephemeral(const std::string_view &description) {+ fprintf(stderr, " " BRIGHT_BLACK "%.*s" UNFOREGROUND "\r", (int)description.size(),+ description.data());+}++static void clear_ephemeral(void) {+ fprintf(stderr, CLEAR_FORWARD);+}++static bool out_of_context(int extra) {+ clear_ephemeral();+ err("error: ran out of context window at %d tokens\n"+ "consider passing `-c %d` at startup for the maximum\n"+ "you can free up more space using /forget or /clear",+ tokens_used() + extra, llama_n_ctx_train(g_model));+ return false;+}++static bool eval_tokens(std::vector<llama_token> tokens) {+ int N = (int)tokens.size();+ if (tokens_used() + N > llama_n_ctx(g_ctx))+ return out_of_context(N);+ for (int i = 0; i < N; i += g_params.n_batch) {+ if (g_got_sigint) {+ g_got_sigint = false;+ if (N > g_params.n_batch)+ clear_ephemeral();+ return false;+ }+ if (N > g_params.n_batch)+ print_ephemeral(lf::format("loading prompt %d%%...", (int)((double)i / N * 100)));+ int n_eval = (int)tokens.size() - i;+ if (n_eval > g_params.n_batch)+ n_eval = g_params.n_batch;+ if (llama_decode(g_ctx, llama_batch_get_one(&tokens[i], n_eval, tokens_used(), 0)))+ return out_of_context(n_eval);+ g_history.insert(g_history.end(), tokens.begin() + i, tokens.begin() + i + n_eval);+ }+ if (N > g_params.n_batch)+ clear_ephemeral();+ return true;+}++static bool eval_id(int id) {+ std::vector<llama_token> tokens;+ tokens.push_back(id);+ return eval_tokens(tokens);+}++static bool eval_string(const std::string &str, bool add_special, bool parse_special) {+ return eval_tokens(llama_tokenize(g_model, str, add_special, parse_special));+}++static bool eval_image_embed(const struct llava_image_embed *image_embed) {+ int N = image_embed->n_image_pos;+ if (tokens_used() + N > llama_n_ctx(g_ctx))+ return out_of_context(N);+ int n_embd = llama_n_embd(llama_get_model(g_ctx));+ for (int i = 0; i < N; i += g_params.n_batch) {+ if (g_got_sigint) {+ g_got_sigint = false;+ if (N > g_params.n_batch)+ clear_ephemeral();+ return false;+ }+ if (N > g_params.n_batch)+ print_ephemeral(lf::format("loading image %d%%...", (int)((double)i / N * 100)));+ int n_eval = N - i;+ if (n_eval > g_params.n_batch)+ n_eval = g_params.n_batch;+ llama_batch batch = {+ .n_tokens = n_eval,+ .embd = image_embed->embed + i * n_embd,+ .all_pos_0 = tokens_used(),+ .all_pos_1 = 1,+ .all_seq_id = 0,+ };+ if (llama_decode(g_ctx, batch))+ return out_of_context(n_eval);+ for (int i = 0; i < n_eval; ++i)+ g_history.push_back(IMAGE_PLACEHOLDER_TOKEN);+ }+ if (N > g_params.n_batch)+ clear_ephemeral();+ return true;+}++static bool eval_image(const char *image_path) {+ unassert(g_clip);+ llava_image_embed *image_embed;+ print_ephemeral("analyzing image...");+ image_embed = llava_image_embed_make_with_filename(g_clip, FLAG_threads_batch, image_path);+ clear_ephemeral();+ if (!image_embed) {+ err("%s: failed to load image", image_path);+ return false;+ }+ bool ok = eval_image_embed(image_embed);+ llava_image_embed_free(image_embed);+ return ok;+}++static void rewind(int pos) {+ unassert(pos <= tokens_used());+ llama_kv_cache_seq_rm(g_ctx, 0, pos, -1);+ g_history.resize(pos);+}+static void on_manual(const std::vector<std::string> &args) {if (args.size() == 1) {g_manual_mode = !g_manual_mode;} else if (args.size() == 2 && (args[1] == "on" || args[1] == "off")) {g_manual_mode = args[1] == "on";} else {- fprintf(stderr, BRIGHT_RED "error: bad /manual command" RESET "\n"- "usage: /manual [on|off]\n");+ err("error: bad /manual command" RESET "\n"+ "usage: /manual [on|off]");return;}fprintf(stderr, FAINT "manual mode %s" RESET "\n", g_manual_mode ? "enabled" : "disabled");@@ -205,8 +413,7 @@ static void on_context(const std::vector<std::string> &args) {}static void on_clear(const std::vector<std::string> &args) {- llama_kv_cache_seq_rm(g_ctx, 0, g_system_prompt_tokens, -1);- g_history.resize(g_system_prompt_tokens);+ rewind(g_system_prompt_tokens);g_stack.clear();fix_stacks();}@@ -223,13 +430,12 @@ static void on_push(const std::vector<std::string> &args) {static void on_pop(const std::vector<std::string> &args) {if (g_stack.empty()) {- fprintf(stderr, BRIGHT_RED "error: context length stack is empty" RESET "\n");+ err("error: context length stack is empty");return;}printf(BOLD "%12d" RESET " restored " FAINT "(%s)" RESET "\n", g_stack.back(),describe_position(g_stack.back()).c_str());- llama_kv_cache_seq_rm(g_ctx, 0, g_stack.back(), -1);- g_history.resize(g_stack.back());+ rewind(g_stack.back());g_stack.pop_back();fix_stacks();print_stack();@@ -239,17 +445,36 @@ static void on_undo(const std::vector<std::string> &args) {while (!g_undo.empty() && g_undo.back() == tokens_used())g_undo.pop_back();if (g_undo.empty()) {- fprintf(stderr, BRIGHT_RED "error: no further undo actions possible" RESET "\n");+ err("error: no further undo actions possible");return;}printf(FAINT "restoring conversation to: %s" RESET "\n",describe_position(g_undo.back()).c_str());- llama_kv_cache_seq_rm(g_ctx, 0, g_undo.back(), -1);- g_history.resize(g_undo.back());+ rewind(g_undo.back());g_undo.pop_back();fix_stacks();}+static void on_forget(const std::vector<std::string> &args) {+ if (g_undo.size() < 2) {+ err("error: nothing left to forget");+ return;+ }+ int erase_count;+ llama_pos erase_begin = g_undo[1];+ llama_pos erase_end = g_undo.size() > 2 ? g_undo[2] : tokens_used();+ if (!(erase_count = erase_end - erase_begin)) {+ err("error: nothing left to forget");+ return;+ }+ printf(FAINT "forgetting: %s" RESET "\n", describe_erasure(erase_begin, erase_end).c_str());+ llama_kv_cache_seq_rm(g_ctx, 0, erase_begin, erase_end);+ llama_kv_cache_seq_add(g_ctx, 0, erase_end, -1, -erase_count);+ g_history.erase(g_history.begin() + erase_begin, //+ g_history.begin() + erase_end);+ adjust_stacks(erase_begin, erase_end);+}+static void on_stack(const std::vector<std::string> &args) {if (g_stack.empty()) {printf(FAINT "stack is currently empty (try using /push)" RESET "\n");@@ -274,7 +499,7 @@ static void on_dump(const std::vector<std::string> &args) {}std::string s;for (auto id : g_history)- s += llama_token_to_piece(g_ctx, id, RENDER_SPECIAL_TOKENS);+ s += token_to_piece(g_ctx, id, RENDER_SPECIAL_TOKENS);if (!s.empty() && s[s.size() - 1] != '\n')s += '\n';write(fd, s.data(), s.size());@@ -282,32 +507,157 @@ static void on_dump(const std::vector<std::string> &args) {close(fd);}+static void on_upload(const std::vector<std::string> &args) {+ if (args.size() < 2) {+ err("error: missing file path" RESET "\n"+ "usage: /upload PATH");+ return;+ }+ if (args.size() > 2) {+ err("error: too many arguments" RESET "\n"+ "usage: /upload PATH");+ return;+ }+ const char *path = args[1].c_str();+ struct stat st;+ if (stat(path, &st) || !S_ISREG(st.st_mode)) {+ err("%s: file does not exist", path);+ return;+ }+ int tokens_used_before = tokens_used();+ if (is_image_file(path)) {+ if (!g_clip) {+ err("%s: need --mmproj model to process image", path);+ return;+ }+ std::vector<llama_chat_msg> chat = {+ {get_role_name(g_role), lf::format("/upload %s", path)},+ };+ if (!eval_string(llama_chat_apply_template(g_model, g_params.chat_template, chat,+ DONT_ADD_ASSISTANT),+ DONT_ADD_SPECIAL, PARSE_SPECIAL)) {+ rewind(tokens_used_before);+ return;+ }+ if (!eval_image(path)) {+ rewind(tokens_used_before);+ return;+ }+ print_image(1, path, 80);+ return;+ } else {+ std::string markdown;+ markdown += "- **Filename**: `";+ markdown += path;+ markdown += "`\n- **Last modified**: ";+ markdown += lf::iso8601(st.st_mtim);+ markdown += "\n\n``````";+ markdown += lf::extname(path);+ markdown += '\n';+ if (!lf::slurp(&markdown, path)) {+ err("%s: failed to slurp file", path);+ return;+ }+ if (markdown.back() != '\n')+ markdown += '\n';+ markdown += "``````";+ if (has_binary(markdown)) {+ err("%s: binary file type not supported", path);+ return;+ }+ std::vector<llama_chat_msg> chat = {{"system", markdown}};+ if (!eval_string(llama_chat_apply_template(g_model, g_params.chat_template, chat,+ DONT_ADD_ASSISTANT),+ DONT_ADD_SPECIAL, PARSE_SPECIAL)) {+ rewind(tokens_used_before);+ return;+ }+ }+}++static const char *on_hint_impl(const char *line) {+ if (!*line && g_manual_mode)+ return get_role_name(g_role);+ if (!*line && !g_manual_mode && !g_said_something)+ return "say something (or type /help for help)";+ static const char *const kHints[] = {+ "/clear", //+ "/context", //+ "/dump", //+ "/exit", //+ "/forget", //+ "/help", //+ "/manual", //+ "/pop", //+ "/push", //+ "/stack", //+ "/stats", //+ "/undo", //+ "/upload", //+ };+ int z = strlen(line);+ int n = sizeof(kHints) / sizeof(kHints[0]);+ int l = 0;+ int r = n - 1;+ int i = -1;+ while (l <= r) {+ int m = (l & r) + ((l ^ r) >> 1); // floor((a+b)/2)+ int c = strncmp(line, kHints[m], z);+ if (!c) {+ i = m;+ r = m - 1;+ } else if (c < 0) {+ r = m - 1;+ } else {+ l = m + 1;+ }+ }+ if (i == -1 || (i + 1 < n && !strncmp(line, kHints[i + 1], z)))+ return "";+ return kHints[i] + z;+}+static char *on_hint(const char *line, const char **ansi1, const char **ansi2) {*ansi1 = FAINT;*ansi2 = UNBOLD;- if (!*line && g_manual_mode)- return strdup(get_role_name(g_role));- else if (!*line && !g_manual_mode && !g_said_something)- return strdup("say something (or type /help for help)");- return strdup("");+ return strdup(on_hint_impl(line));}static void on_completion(const char *line, int pos, bestlineCompletions *comp) {- static const char *const kCompletions[] = {- "/clear", // usage: /clear- "/context", // usage: /context- "/dump", // usage: /dump [FILE]- "/exit", // usage: /exit- "/help", // usage: /help [COMMAND]- "/manual", // usage: /manual [on|off]- "/pop", // usage: /pop- "/push", // usage: /push- "/stack", // usage: /stack- "/stats", // usage: /stats- };- for (int i = 0; i < sizeof(kCompletions) / sizeof(*kCompletions); ++i)- if (startswith(kCompletions[i], line))- bestlineAddCompletion(comp, kCompletions[i]);+ if (startswith(line, "/upload ")) {+ std::string pattern(line + strlen("/upload "));+ pattern += '*';+ glob_t gl;+ if (!glob(pattern.c_str(), GLOB_TILDE, 0, &gl)) {+ for (size_t i = 0; i < gl.gl_pathc; ++i) {+ std::string completion = "/upload ";+ completion += gl.gl_pathv[i];+ if (is_directory(gl.gl_pathv[i]))+ completion += '/';+ bestlineAddCompletion(comp, completion.c_str());+ }+ globfree(&gl);+ }+ } else {+ static const char *const kCompletions[] = {+ "/clear", // usage: /clear+ "/context", // usage: /context+ "/dump", // usage: /dump [FILE]+ "/exit", // usage: /exit+ "/forget", // usage: /forget+ "/help", // usage: /help [COMMAND]+ "/manual", // usage: /manual [on|off]+ "/pop", // usage: /pop+ "/push", // usage: /push+ "/stack", // usage: /stack+ "/stats", // usage: /stats+ "/undo", // usage: /undo+ "/upload", // usage: /upload FILE+ };+ for (int i = 0; i < sizeof(kCompletions) / sizeof(*kCompletions); ++i)+ if (startswith(kCompletions[i], line))+ bestlineAddCompletion(comp, kCompletions[i]);+ }}// handle irc style commands like: `/arg0 arg1 arg2`@@ -344,10 +694,14 @@ static bool handle_command(const char *command) {on_pop(args);} else if (args[0] == "undo") {on_undo(args);+ } else if (args[0] == "forget") {+ on_forget(args);} else if (args[0] == "stack") {on_stack(args);+ } else if (args[0] == "upload") {+ on_upload(args);} else {- fprintf(stderr, BRIGHT_RED "%s: unrecognized command" RESET "\n", args[0].c_str());+ err("%s: unrecognized command", args[0].c_str());}return true;}@@ -373,52 +727,6 @@ static void print_logo(const char16_t *s) {}}-static void print_ephemeral(const std::string_view &description) {- fprintf(stderr, " " BRIGHT_BLACK "%.*s" UNFOREGROUND "\r", (int)description.size(),- description.data());-}--static void clear_ephemeral(void) {- fprintf(stderr, CLEAR_FORWARD);-}--static void die_out_of_context(int extra) {- fprintf(stderr,- "\n" BRIGHT_RED- "error: ran out of context window at %d tokens; you can use the maximum "- "context window size by passing the flag `-c %d` to llamafile." UNFOREGROUND "\n",- tokens_used() + extra, llama_n_ctx_train(g_model));- exit(1);-}--static void eval_tokens(std::vector<llama_token> tokens, int n_batch) {- int N = (int)tokens.size();- if (tokens_used() + N > llama_n_ctx(g_ctx))- die_out_of_context(N);- for (int i = 0; i < N; i += n_batch) {- if (N > n_batch)- print_ephemeral(lf::format("loading prompt %d%%...", (int)((double)i / N * 100)));- int n_eval = (int)tokens.size() - i;- if (n_eval > n_batch)- n_eval = n_batch;- if (llama_decode(g_ctx, llama_batch_get_one(&tokens[i], n_eval, tokens_used(), 0)))- die_out_of_context(n_eval);- g_history.insert(g_history.end(), tokens.begin() + i, tokens.begin() + i + n_eval);- }- if (N > n_batch)- clear_ephemeral();-}--static void eval_id(int id) {- std::vector<llama_token> tokens;- tokens.push_back(id);- eval_tokens(tokens, 1);-}--static void eval_string(const std::string &str, int n_batch, bool add_special, bool parse_special) {- eval_tokens(llama_tokenize(g_model, str, add_special, parse_special), n_batch);-}-static void on_server_listening(const char *host, int port) {pthread_mutex_lock(&g_lock);g_listen_url = lf::format("http://%s:%d/", host, port);@@ -457,31 +765,29 @@ int chatbot_main(int argc, char **argv) {print_ephemeral("loading backend...");llama_backend_init();- gpt_params params;- params.n_batch = 512; // for better progress indication- params.sparams.temp = 0; // don't believe in randomness by default- params.prompt = "A chat between a curious human and an artificial intelligence assistant. "- "The assistant gives helpful, detailed, and polite answers to the "- "human's questions.";- if (!gpt_params_parse(argc, argv, params)) { // also loads gpu module+ g_params.n_batch = 256; // for better progress indication+ g_params.sparams.temp = 0; // don't believe in randomness by default+ g_params.prompt = "A chat between a curious human and an artificial intelligence assistant. "+ "The assistant gives helpful, detailed, and polite answers to the "+ "human's questions.";+ if (!gpt_params_parse(argc, argv, g_params)) { // also loads gpu modulefprintf(stderr, "error: failed to parse flags\n");exit(1);}clear_ephemeral();print_ephemeral("loading model...");- llama_model_params model_params = llama_model_default_params();- model_params.n_gpu_layers = llamafile_gpu_layers(35);- g_model = llama_load_model_from_file(params.model.c_str(), model_params);+ llama_model_params model_params = llama_model_params_from_gpt_params(g_params);+ g_model = llama_load_model_from_file(g_params.model.c_str(), model_params);if (g_model == NULL) {clear_ephemeral();- fprintf(stderr, "%s: failed to load model\n", params.model.c_str());+ fprintf(stderr, "%s: failed to load model\n", g_params.model.c_str());exit(2);}- if (params.n_ctx <= 0 || params.n_ctx > llama_n_ctx_train(g_model))- params.n_ctx = llama_n_ctx_train(g_model);- if (params.n_ctx < params.n_batch)- params.n_batch = params.n_ctx;+ if (g_params.n_ctx <= 0 || g_params.n_ctx > llama_n_ctx_train(g_model))+ g_params.n_ctx = llama_n_ctx_train(g_model);+ if (g_params.n_ctx < g_params.n_batch)+ g_params.n_batch = g_params.n_ctx;clear_ephemeral();bool want_server = !llamafile_has(argv, "--chat");@@ -502,43 +808,54 @@ int chatbot_main(int argc, char **argv) {printf(BOLD "software" UNBOLD ": llamafile " LLAMAFILE_VERSION_STRING "\n" //BOLD "model" UNBOLD ": %s\n",- lf::basename(params.model).c_str());+ lf::basename(g_params.model).c_str());printf(BOLD "compute" UNBOLD ": %s\n", describe_compute().c_str());if (want_server)printf(BOLD "server" UNBOLD ": %s\n", g_listen_url.c_str());printf("\n");print_ephemeral("initializing context...");- llama_context_params ctx_params = llama_context_params_from_gpt_params(params);+ llama_context_params ctx_params = llama_context_params_from_gpt_params(g_params);g_ctx = llama_new_context_with_model(g_model, ctx_params);- if (g_ctx == NULL) {- clear_ephemeral();+ clear_ephemeral();+ if (!g_ctx) {fprintf(stderr, "error: failed to initialize context\n");exit(3);}- clear_ephemeral();if (llama_model_has_encoder(g_model))fprintf(stderr, "warning: this model has an encoder\n");+ if (FLAG_mmproj) {+ print_ephemeral("initializing vision model...");+ g_clip = clip_model_load(FLAG_mmproj, 0);+ clear_ephemeral();+ if (!g_clip) {+ fprintf(stderr, "%s: failed to initialize clip image model\n", FLAG_mmproj);+ exit(4);+ }+ }+// setup conversationif (llama_should_add_bos_token(g_model))- eval_id(llama_token_bos(g_model));+ if (!eval_id(llama_token_bos(g_model)))+ exit(6);record_undo();// setup system prompt- std::vector<llama_chat_msg> chat = {{"system", params.prompt}};+ std::vector<llama_chat_msg> chat = {{"system", g_params.prompt}};std::string msg =- llama_chat_apply_template(g_model, params.chat_template, chat, DONT_ADD_ASSISTANT);- eval_string(msg, params.n_batch, DONT_ADD_SPECIAL, PARSE_SPECIAL);+ llama_chat_apply_template(g_model, g_params.chat_template, chat, DONT_ADD_ASSISTANT);+ if (!eval_string(msg, DONT_ADD_SPECIAL, PARSE_SPECIAL))+ exit(6);g_system_prompt_tokens = tokens_used();clear_ephemeral();- printf("%s\n", params.special ? msg.c_str() : params.prompt.c_str());+ printf("%s\n", g_params.special ? msg.c_str() : g_params.prompt.c_str());// perform important setupHighlightMarkdown highlighter;ColorBleeder bleeder(&highlighter);- struct llama_sampling_context *sampler = llama_sampling_init(params.sparams);+ struct llama_sampling_context *sampler = llama_sampling_init(g_params.sparams);signal(SIGINT, on_sigint);// run chatbot@@ -570,9 +887,14 @@ int chatbot_main(int argc, char **argv) {continue;}bool add_assi = !g_manual_mode;+ int tokens_used_before = tokens_used();std::vector<llama_chat_msg> chat = {{get_role_name(g_role), line}};- std::string msg = llama_chat_apply_template(g_model, params.chat_template, chat, add_assi);- eval_string(msg, params.n_batch, DONT_ADD_SPECIAL, PARSE_SPECIAL);+ std::string msg =+ llama_chat_apply_template(g_model, g_params.chat_template, chat, add_assi);+ if (!eval_string(msg, DONT_ADD_SPECIAL, PARSE_SPECIAL)) {+ rewind(tokens_used_before);+ continue;+ }if (g_manual_mode) {g_role = get_next_role(g_role);free(line);@@ -585,11 +907,12 @@ int chatbot_main(int argc, char **argv) {}llama_token id = llama_sampling_sample(sampler, g_ctx, NULL);llama_sampling_accept(sampler, g_ctx, id, APPLY_GRAMMAR);- eval_id(id);+ if (!eval_id(id))+ break;if (llama_token_is_eog(g_model, id))break;std::string s;- bleeder.feed(&s, llama_token_to_piece(g_ctx, id, params.special));+ bleeder.feed(&s, token_to_piece(g_ctx, id, g_params.special));printf("%s", s.c_str());fflush(stdout);}@@ -600,6 +923,12 @@ int chatbot_main(int argc, char **argv) {printf("%s\n", s.c_str());}+ if (g_clip) {+ print_ephemeral("freeing vision model...");+ clip_free(g_clip);+ clear_ephemeral();+ }+print_ephemeral("freeing context...");llama_sampling_free(sampler);llama_free(g_ctx);diff --git a/llamafile/chatbot_help.cpp b/llamafile/chatbot_help.cppindex f1ca2f7a..c0381d0e 100644--- a/llamafile/chatbot_help.cpp+++ b/llamafile/chatbot_help.cpp@@ -27,6 +27,7 @@ void on_help(const std::vector<std::string> &args) {/context print context window usage\n\/dump [FILE] print or save context window to file\n\/exit end program\n\+ /forget erase oldest message from context\n\/help [COMMAND] show help\n\/manual [on|off] toggle manual role mode\n\/pop restore context window size\n\@@ -34,6 +35,7 @@ void on_help(const std::vector<std::string> &args) {/stack prints context window stack\n\/stats print performance metrics\n\/undo erases last message in conversation\n\+ /upload FILE share image or text file with assistant\n\");} else if (args[1] == "context") {fprintf(stderr, "\@@ -117,6 +119,25 @@ usage: /undo" RESET "\n\erases last exchange in conversation. in the normal mode, this includes\n\what the assistant last said, as well as the question that was asked. in\n\manual mode, this will erase only the last chat message.\n\+");+ } else if (args[1] == "upload") {+ fprintf(stderr, "\+usage: /upload FILE" RESET "\n\+shares file from local hard drive with assistant. if this is a text file\n\+then a markdown system prompt is generated and added to the conversation\n\+history that gives the assistant readonly access to the file content and\n\+metadata. files with nul characters in them are currently not supported.\n\+image files (png/jpg/gif/bmp/hdr/psd/tga/pgm/ppm) may be uploaded if you\n\+specified a clip vision model (eg. LLaVA) earlier with the --mmproj flag\n\+");+ } else if (args[1] == "forget") {+ fprintf(stderr, "\+usage: /forget" RESET "\n\+erase oldest chat message from context window. if you run out of context\n\+window, then this command can help you free up space. the oldest message\n\+excludes the original system prompt, with is preserved. this command may\n\+be run multiple times to erase multiple messages. there's also the /undo\n\+command which deletes the most recent chat message instead.\n\");} else {fprintf(stderr, BRIGHT_RED "%s: unknown command" RESET "\n", args[1].c_str());diff --git a/llamafile/flags.cpp b/llamafile/flags.cppindex f85af82d..f43a6c30 100644--- a/llamafile/flags.cpp+++ b/llamafile/flags.cpp@@ -17,7 +17,6 @@#include "flags.h"#include "debug.h"-#include "llama.cpp/cores.h"#include "llamafile.h"#include "trust.h"@@ -31,7 +30,9 @@#include <time.h>#include <unistd.h>+#include "llama.cpp/cores.h"#include "llama.cpp/llama.h"+#include "llamafile/macros.h"bool FLAGS_READY = false;bool FLAG_ascii = false;@@ -49,6 +50,7 @@ bool FLAG_unsecure = false;const char *FLAG_file = nullptr;const char *FLAG_ip_header = nullptr;const char *FLAG_listen = "127.0.0.1:8080";+const char *FLAG_mmproj = nullptr;const char *FLAG_model = nullptr;const char *FLAG_prompt = nullptr;const char *FLAG_url_prefix = "";@@ -67,8 +69,8 @@ int FLAG_n_gpu_layers = -1;int FLAG_seed = LLAMA_DEFAULT_SEED;int FLAG_slots = 1;int FLAG_split_mode = LLAMA_SPLIT_MODE_LAYER;-int FLAG_threads;-int FLAG_threads_batch;+int FLAG_threads = MIN(cpu_get_num_math(), 20);+int FLAG_threads_batch = cpu_get_num_math();int FLAG_token_burst = 100;int FLAG_token_cidr = 24;int FLAG_ubatch = 512;@@ -116,7 +118,6 @@ static wontreturn void unknown(const char *flag) {void llamafile_get_flags(int argc, char **argv) {bool program_supports_gpu = FLAG_gpu != LLAMAFILE_GPU_DISABLE;- FLAG_threads = cpu_get_num_math();for (int i = 1; i < argc;) {const char *flag = argv[i++];@@ -288,6 +289,13 @@ void llamafile_get_flags(int argc, char **argv) {continue;}+ if (!strcmp(flag, "-mm") || !strcmp(flag, "--mmproj")) {+ if (i == argc)+ missing("--mmproj");+ FLAG_mmproj = argv[i++];+ continue;+ }+if (!strcmp(flag, "-f") || !strcmp(flag, "--file")) {if (i == argc)missing("--file");diff --git a/llamafile/is_rgb_terminal.cpp b/llamafile/is_rgb_terminal.cppnew file mode 100644index 00000000..9bf387cc--- /dev/null+++ b/llamafile/is_rgb_terminal.cpp@@ -0,0 +1,48 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "xterm.h"++#include <cosmo.h>+#include <stdlib.h>+#include <string.h>++/**+ * Detect if terminal supports 24-bit color.+ */+bool is_rgb_terminal(void) {++ if (IsWindows())+ return true;++ const char *colorterm = getenv("COLORTERM");+ if (colorterm && (!strcasecmp(colorterm, "truecolor") || //+ !strcasecmp(colorterm, "24bit"))) {+ return true;+ }++ const char *term = getenv("TERM");+ if (term && (strstr(term, "24bit") || //+ strstr(term, "truecolor") || //+ strstr(term, "iterm") || //+ strstr(term, "kitty") || //+ strstr(term, "wezterm"))) {+ return true;+ }++ return false;+}diff --git a/llamafile/iso8601.cpp b/llamafile/iso8601.cppnew file mode 100644index 00000000..d2179fb8--- /dev/null+++ b/llamafile/iso8601.cpp@@ -0,0 +1,46 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "string.h"++#include <assert.h>+#include <stdio.h>+#include <stdlib.h>+#include <string>+#include <time.h>++namespace lf {++/**+ * Turns timestamp into string.+ */+std::string iso8601(struct timespec ts) {+ struct tm tm;+ if (!localtime_r(&ts.tv_sec, &tm))+ if (!gmtime_r(&ts.tv_sec, &tm))+ abort();+ char res[256];+ char *ptr = res;+ char *end = res + sizeof(res);+ ptr += strftime(ptr, end - ptr, "%Y-%m-%d %H:%M:%S", &tm);+ ptr += snprintf(ptr, end - ptr, ".%09ld", ts.tv_nsec);+ ptr += strftime(ptr, end - ptr, "%z %Z", &tm);+ npassert(ptr + 1 <= end);+ return res;+}++} // namespace lfdiff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex c94fa750..77ddcfc9 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -23,6 +23,7 @@ extern bool FLAG_unsecure;extern const char *FLAG_file;extern const char *FLAG_ip_header;extern const char *FLAG_listen;+extern const char *FLAG_mmproj;extern const char *FLAG_model;extern const char *FLAG_prompt;extern const char *FLAG_url_prefix;diff --git a/llamafile/print_image.cpp b/llamafile/print_image.cppnew file mode 100644index 00000000..c51ecfce--- /dev/null+++ b/llamafile/print_image.cpp@@ -0,0 +1,81 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "xterm.h"++#include <string>+#include <termios.h>+#include <unistd.h>++#include "llamafile/macros.h"+#include "stb/stb_image.h"+#include "stb/stb_image_resize.h"++/**+ * Prints image to terminal.+ */+int print_image(int fd, const char *path, int max_width) {++ // load image+ int width, height, channels;+ unsigned char *img = stbi_load(path, &width, &height, &channels, 3);+ if (!img)+ return -1;++ // get terminal info+ bool use_rgb = is_rgb_terminal();+ struct winsize ws = {24, 80};+ tcgetwinsize(fd, &ws);++ // calculate new dimensions preserving aspect ratio+ int xn = MIN(max_width, ws.ws_col);+ int yn = (height * xn) / width; // *2 because we use half blocks+ yn = (yn + 1) & -2; // round up to even number++ // resize image+ unsigned char *resized = new unsigned char[xn * yn * 3];+ stbir_resize_uint8(img, width, height, 0, resized, xn, yn, 0, 3);+ stbi_image_free(img);++ // convert image to string using half blocks+ std::string s;+ for (int y = 0; y < yn; y += 2) {+ for (int x = 0; x < xn; ++x) {+ int upr = resized[((y + 0) * xn + x) * 3 + 0];+ int upg = resized[((y + 0) * xn + x) * 3 + 1];+ int upb = resized[((y + 0) * xn + x) * 3 + 2];+ int lor = resized[((y + 1) * xn + x) * 3 + 0];+ int log = resized[((y + 1) * xn + x) * 3 + 1];+ int lob = resized[((y + 1) * xn + x) * 3 + 2];+ char buf[48];+ if (use_rgb) {+ s.append(buf, snprintf(buf, sizeof(buf), "\033[38;2;%d;%d;%dm\033[48;2;%d;%d;%dm▀",+ upr, upg, upb, lor, log, lob));+ } else {+ int upx = rgb2xterm256((upr << 16) | (upg << 8) | upb);+ int lox = rgb2xterm256((lor << 16) | (log << 8) | lob);+ s.append(buf, snprintf(buf, sizeof(buf), "\033[38;5;%dm\033[48;5;%dm▀", upx, lox));+ }+ }+ s += "\033[0m\n";+ }++ // write image to terminal+ int rc = write(fd, s.data(), s.size());+ delete[] resized;+ return rc;+}diff --git a/llamafile/server/main.1 b/llamafile/server/main.1index 702c73e5..ebe6777e 100644--- a/llamafile/server/main.1+++ b/llamafile/server/main.1@@ -28,6 +28,8 @@ Path of GGUF model weights. Each server process is currently limited toserving only one model. If you need to host multiple models, then it'srecommended that you run multiple instances of llamafiler behind areverse proxy such as NGINX or Redbean.+.It Fl mm Ar FNAME , Fl Fl mmproj Ar FNAME+Path of vision model weights..It Fl l Ar HOSTPORT , Fl Fl listen Ar HOSTPORTSpecifies the local [HOST:]PORT on which the HTTP server should listen.By default this is 0.0.0.0:8080 which means llamafiler will bind to portdiff --git a/llamafile/server/main.1.asc b/llamafile/server/main.1.ascindex 91e23ab6..4fe4017a 100644--- a/llamafile/server/main.1.asc+++ b/llamafile/server/main.1.asc@@ -29,6 +29,9 @@models, then it's recommended that you run multiple instancesof llamafiler behind a reverse proxy such as NGINX or Redbean.+ [1m-mm [4m[22mFNAME[24m, [1m--mmproj [4m[22mFNAME[0m+ Path of vision model weights.+[1m-l [4m[22mHOSTPORT[24m, [1m--listen [4m[22mHOSTPORT[0mSpecifies the local [HOST:]PORT on which the HTTP server shouldlisten. By default this is 0.0.0.0:8080 which means llamafilerdiff --git a/llamafile/server/slot.cpp b/llamafile/server/slot.cppindex 1d550086..0f3bb75f 100644--- a/llamafile/server/slot.cpp+++ b/llamafile/server/slot.cpp@@ -85,8 +85,8 @@ Slot::start()cparams.yarn_orig_ctx = 0;cparams.defrag_thold = -1;cparams.offload_kqv = true;- cparams.type_k = X86_HAVE(AVX512_BF16) ? GGML_TYPE_BF16 : GGML_TYPE_F16;- cparams.type_v = X86_HAVE(AVX512_BF16) ? GGML_TYPE_BF16 : GGML_TYPE_F16;+ cparams.type_k = GGML_TYPE_F16;+ cparams.type_v = GGML_TYPE_F16;cparams.flash_attn = FLAG_flash_attn;system_fingerprint_ = generate_system_fingerprint(&cparams);if (!(ctx_ = llama_new_context_with_model(model_, cparams)))diff --git a/llamafile/slurp.cpp b/llamafile/slurp.cppnew file mode 100644index 00000000..84747d2e--- /dev/null+++ b/llamafile/slurp.cpp@@ -0,0 +1,57 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "string.h"++#include <fcntl.h>+#include <unistd.h>++namespace lf {++/**+ * Reads entire file into memory.+ */+ssize_t slurp(std::string *r, const char *path) {+ int fd;+ if ((fd = open(path, O_RDONLY)) == -1)+ return -1;+ size_t toto = 0;+ size_t orig = r->size();+ for (;;) {+ size_t want = 16384;+ size_t size = r->size();+ r->resize(size + want);+ ssize_t rc;+ if ((rc = read(fd, r->data() + size, want)) == -1) {+ r->resize(orig);+ close(fd);+ return -1;+ }+ size_t got = rc;+ r->resize(size + got);+ toto += got;+ if (!got)+ break;+ }+ if (close(fd)) {+ r->resize(orig);+ return -1;+ }+ return toto;+}++} // namespace lfdiff --git a/llamafile/string.h b/llamafile/string.hindex 89a4e935..70667011 100644--- a/llamafile/string.h+++ b/llamafile/string.h@@ -17,10 +17,12 @@#pragma once#include <string>+#include <time.h>#include <vector>namespace lf {+ssize_t slurp(std::string *, const char *);std::string basename(const std::string_view);std::string collapse(const std::string_view);std::string dirname(const std::string_view);@@ -28,6 +30,7 @@ std::string format(const char *, ...) __attribute__((format(printf, 1, 2)));std::string join(const std::vector<std::string> &, const std::string_view &);std::string resolve(const std::string_view, const std::string_view);std::string tolower(const std::string_view);+std::string iso8601(struct timespec);std::string_view extname(const std::string_view &);void append_wchar(std::string *, wchar_t);diff --git a/llamafile/xterm.h b/llamafile/xterm.hindex e3f02f84..6dd90520 100644--- a/llamafile/xterm.h+++ b/llamafile/xterm.h@@ -17,4 +17,6 @@#pragma once+bool is_rgb_terminal(void);+int print_image(int, const char *, int);int rgb2xterm256(int);