Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Use hack to measure prompt tok/sec in web gui
Justine Tunneya54815e67a0d
diff --git a/llama.cpp/server/public/index.html b/llama.cpp/server/public/index.htmlindex 4c40e8dc..4c37b7bb 100644--- a/llama.cpp/server/public/index.html+++ b/llama.cpp/server/public/index.html@@ -1140,7 +1140,7 @@}return html`<span>- ${llamaStats.value.tokens_evaluated} tokens evaluated (${llamaStats.value.tokens_cached} cached), ${llamaStats.value.timings.prompt_per_token_ms.toFixed()} ms per token, ${llamaStats.value.timings.prompt_per_second.toFixed(2)} tokens per second+ prompt evaluation speed is ${llamaStats.value.timings.prompt_per_second_jart.toFixed(2)} prompt tokens evaluated per second</span>`}@@ -1288,9 +1288,9 @@</section><footer>- <p><${ModelProcessingInfo} /></p><p><${ModelGenerationInfo} /></p>- <p>Powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a>, <a href="https://ggml.ai">ggml.ai</a>, and <a href="https://github.com/Mozilla-Ocho/llamafile/">llamafile</a></p>+ <p><${ModelProcessingInfo} /></p>+ <p>powered by <a href="https://github.com/ggerganov/llama.cpp">llama.cpp</a>, <a href="https://ggml.ai">ggml.ai</a>, and <a href="https://github.com/Mozilla-Ocho/llamafile/">llamafile</a></p></footer></div>`;diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 345b92c7..3b89492d 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -7,6 +7,7 @@#include "llama.cpp/stb_image.h"#include "utils.h"#include "oai.h"+#include "llamafile/micros.h"#include "macsandbox.h"// increase max payload length to allow use of larger context size@@ -24,6 +25,8 @@#include <tool/args/args.h>#include <libc/dce.h>+double g_prompt_per_second_jart;+using json = nlohmann::json;struct server_params@@ -288,6 +291,7 @@ struct llama_client_slot{"prompt_ms", t_prompt_processing},{"prompt_per_token_ms", t_prompt_processing / num_prompt_tokens_processed},{"prompt_per_second", 1e3 / t_prompt_processing * num_prompt_tokens_processed},+ {"prompt_per_second_jart", g_prompt_per_second_jart},{"predicted_n", n_decoded},{"predicted_ms", t_token_generation},@@ -896,6 +900,8 @@ struct llama_server_contextllama_batch_add(batch, system_tokens[i], i, { 0 }, false);}+ long t1 = micros();+for (int32_t i = 0; i < (int32_t) batch.n_tokens; i += params.n_batch){const int32_t n_tokens = std::min(params.n_batch, (int32_t) (batch.n_tokens - i));@@ -916,6 +922,12 @@ struct llama_server_context}}+ long t2 = micros();+ int n_tokens = batch.n_tokens;+ g_prompt_per_second_jart = 1e6 / (t2 - t1) * n_tokens;+ LOG_TEE("evaluated %d prompt tokens in %ld us at %g tok/sec\n",+ n_tokens, t2 - t1, g_prompt_per_second_jart);+// assign the system KV cache to all parallel sequencesfor (int32_t i = 1; i < params.n_parallel; ++i){@@ -1363,6 +1375,8 @@ struct llama_server_context{slot_image &img = slot.images[image_idx];+ long t1 = micros();+// process prefix promptfor (int32_t i = 0; i < (int32_t) batch.n_tokens; i += n_batch){@@ -1404,6 +1418,12 @@ struct llama_server_context}image_idx++;+ long t2 = micros();+ int n_tokens = batch.n_tokens + img.image_tokens;+ g_prompt_per_second_jart = 1e6 / (t2 - t1) * n_tokens;+ LOG_TEE("evaluated %d image tokens in %ld us at %g tok/sec\n",+ n_tokens, t2 - t1, g_prompt_per_second_jart);+llama_batch_clear(batch);// append prefix of next image