Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Improve the chatbot
Justine Tunneyd484f6e3958a
diff --git a/llama.cpp/llama-bench/llama-bench.cpp b/llama.cpp/llama-bench/llama-bench.cppindex 1b70f4c6..07757109 100644--- a/llama.cpp/llama-bench/llama-bench.cpp+++ b/llama.cpp/llama-bench/llama-bench.cpp@@ -22,7 +22,6 @@#include <cosmo.h>#include <libgen.h>#include <sys/stat.h>-#include <sys/auxv.h>#include <libc/intrin/x86.h>#include "llama.cpp/cores.h"#include <libc/sysv/consts/hwcap.h>@@ -33,6 +32,7 @@#include "llama.cpp/common.h"#include "llama.cpp/ggml-cuda.h"#include "llamafile/llamafile.h"+#include "llamafile/compute.h"// utilsstatic uint64_t get_time_ns() {@@ -93,92 +93,6 @@ static T stdev(const std::vector<T> & v) {return stdev;}-#ifdef __x86_64__-static void cpuid(unsigned leaf, unsigned subleaf, unsigned *info) {- asm("movq\t%%rbx,%%rsi\n\t"- "cpuid\n\t"- "xchgq\t%%rbx,%%rsi"- : "=a"(info[0]), "=S"(info[1]), "=c"(info[2]), "=d"(info[3])- : "0"(leaf), "2"(subleaf));-}-#endif // __x86_64__--static std::string get_cpu_info() { // [jart]- std::string id;--#ifdef __x86_64__- union { // [jart]- char str[64];- unsigned reg[16];- } u = {0};- cpuid(0x80000002, 0, u.reg + 0*4);- cpuid(0x80000003, 0, u.reg + 1*4);- cpuid(0x80000004, 0, u.reg + 2*4);- int len = strlen(u.str);- while (len > 0 && u.str[len - 1] == ' ')- u.str[--len] = 0;- id = u.str;-#else- if (IsLinux()) {- FILE * f = fopen("/proc/cpuinfo", "r");- if (f) {- char buf[1024];- while (fgets(buf, sizeof(buf), f)) {- if (!strncmp(buf, "model name", 10) ||- startswith(buf, "Model\t\t:")) { // e.g. raspi- char * p = strchr(buf, ':');- if (p) {- p++;- while (std::isspace(*p)) {- p++;- }- while (std::isspace(p[strlen(p) - 1])) {- p[strlen(p) - 1] = '\0';- }- id = p;- break;- }- }- }- fclose(f);- }- }- if (IsXnu()) {- char cpu_name[128] = {0};- size_t size = sizeof(cpu_name);- if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1) {- id = cpu_name;- }- }-#endif- id = replace_all(id, " 96-Cores", "");- id = replace_all(id, "(TM)", "");- id = replace_all(id, "(R)", "");-- std::string march;-#ifdef __x86_64__- if (__cpu_march(__cpu_model.__cpu_subtype))- march = __cpu_march(__cpu_model.__cpu_subtype);-#else- long hwcap = getauxval(AT_HWCAP);- if (hwcap & HWCAP_ASIMDHP)- march += "+fp16";- if (hwcap & HWCAP_ASIMDDP)- march += "+dotprod";-#endif-- if (!march.empty()) {- bool empty = id.empty();- if (!empty)- id += " (";- id += march;- if (!empty)- id += ")";- }-- return id;-}-static std::string get_gpu_info() {std::string id;#ifdef GGML_USE_CUDA@@ -984,7 +898,7 @@ const bool test::metal = false; // !!ggml_cpu_has_metal(); // [jarconst bool test::gpu_blas = false; // !!ggml_cpu_has_gpublas(); // [jart]const bool test::blas = false; // !!ggml_cpu_has_blas(); // [jart]const bool test::sycl = false; // !!ggml_cpu_has_sycl(); // [jart]-const std::string test::cpu_info = get_cpu_info();+const std::string test::cpu_info = llamafile_describe_cpu();const std::string test::gpu_info = ""; //get_gpu_info(); // [jart]struct printer {diff --git a/llama.cpp/llama.cpp b/llama.cpp/llama.cppindex 8872e1b6..7b870c7c 100644--- a/llama.cpp/llama.cpp+++ b/llama.cpp/llama.cpp@@ -17256,6 +17256,10 @@ int32_t llama_n_layer(const struct llama_model * model) {return model->hparams.n_layer;}+int32_t llama_n_gpu_layers(const struct llama_model * model) { // [jart]+ return model->n_gpu_layers;+}+float llama_rope_freq_scale_train(const struct llama_model * model) {return model->hparams.rope_freq_scale_train;}diff --git a/llama.cpp/llama.h b/llama.cpp/llama.hindex 562e7fe2..181fca09 100644--- a/llama.cpp/llama.h+++ b/llama.cpp/llama.h@@ -473,6 +473,8 @@ extern "C" {LLAMA_API int32_t llama_n_embd (const struct llama_model * model);LLAMA_API int32_t llama_n_layer (const struct llama_model * model);+ LLAMA_API int32_t llama_n_gpu_layers(const struct llama_model * model); // [jart]+// Get the model's RoPE frequency scaling factorLLAMA_API float llama_rope_freq_scale_train(const struct llama_model * model);diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cppindex 984524f5..526e8437 100644--- a/llamafile/chatbot.cpp+++ b/llamafile/chatbot.cpp@@ -26,32 +26,47 @@#include <vector>#include "llama.cpp/common.h"+#include "llama.cpp/ggml-cuda.h"#include "llama.cpp/llama.h"#include "llama.cpp/server/server.h"#include "llamafile/bestline.h"+#include "llamafile/compute.h"#include "llamafile/highlight.h"#include "llamafile/llamafile.h"+#define RESET "\e[0m"#define BOLD "\e[1m"#define FAINT "\e[2m"#define UNBOLD "\e[22m"#define RED "\e[31m"#define GREEN "\e[32m"#define MAGENTA "\e[35m"+#define YELLOW "\e[33m"+#define CYAN "\e[36m"#define UNFOREGROUND "\e[39m"#define BRIGHT_BLACK "\e[90m"#define BRIGHT_RED "\e[91m"#define BRIGHT_GREEN "\e[92m"#define CLEAR_FORWARD "\e[K"+enum Role {+ ROLE_USER,+ ROLE_ASSISTANT,+ ROLE_SYSTEM,+};+struct ServerArgs {int argc;char **argv;};static int n_past;+static bool g_once;+static bool g_manual;static llama_model *g_model;static llama_context *g_ctx;+static std::vector<llama_token> g_history;+static enum Role g_role = ROLE_USER;static pthread_cond_t g_cond = PTHREAD_COND_INITIALIZER;static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER;static std::string g_listen_url;@@ -69,6 +84,58 @@ static bool is_empty(const char *s) {return true;}+static const char *get_role_name(enum Role role) {+ switch (role) {+ case ROLE_USER:+ return "user";+ case ROLE_ASSISTANT:+ return "assistant";+ case ROLE_SYSTEM:+ return "system";+ default:+ __builtin_unreachable();+ }+}++static const char *get_role_color(enum Role role) {+ switch (role) {+ case ROLE_USER:+ return GREEN;+ case ROLE_ASSISTANT:+ return MAGENTA;+ case ROLE_SYSTEM:+ return YELLOW;+ default:+ __builtin_unreachable();+ }+}++static enum Role get_next_role(enum Role role) {+ switch (role) {+ case ROLE_USER:+ return ROLE_ASSISTANT;+ case ROLE_ASSISTANT:+ return ROLE_USER;+ case ROLE_SYSTEM:+ return ROLE_USER;+ default:+ __builtin_unreachable();+ }+}++static enum Role cycle_role(enum Role role) {+ switch (role) {+ case ROLE_USER:+ return ROLE_ASSISTANT;+ case ROLE_ASSISTANT:+ return ROLE_SYSTEM;+ case ROLE_SYSTEM:+ return ROLE_USER;+ default:+ __builtin_unreachable();+ }+}+static std::string basename(const std::string_view path) {size_t i, e;if ((e = path.size())) {@@ -98,11 +165,167 @@ __attribute__((format(printf, 1, 2))) static std::string format(const char *fmt,return res;}+static std::string join(const std::vector<std::string> &vec, const std::string_view &delim) {+ std::string result;+ for (size_t i = 0; i < vec.size(); i++) {+ result += vec[i];+ if (i < vec.size() - 1)+ result += delim;+ }+ return result;+}++static std::string describe_compute(void) {+ if (llama_n_gpu_layers(g_model) > 0 && llamafile_has_gpu()) {+ if (llamafile_has_metal()) {+ return "Apple Metal GPU";+ } else {+ std::vector<std::string> vec;+ int count = ggml_backend_cuda_get_device_count();+ for (int i = 0; i < count; i++) {+ char buf[128];+ ggml_backend_cuda_get_device_description(i, buf, sizeof(buf));+ vec.emplace_back(buf);+ }+ return join(vec, ", ");+ }+ } else {+ return llamafile_describe_cpu();+ }+}++static void on_help(const std::vector<std::string> &args) {+ if (args.size() == 1) {+ fprintf(stderr, "\+" BOLD "available commands" RESET "\n\+ /context\n\+ /dump [FILE]\n\+ /exit\n\+ /help [COMMAND]\n\+ /manual [on|off]\n\+ /stats\n\+");+ } else if (args[1] == "context") {+ fprintf(stderr, "\+usage: /context" RESET "\n\+prints information about context window usage. this helps you know how\n\+soon you're going to run out of tokens for the current conversation.\n\+");+ } else if (args[1] == "dump") {+ fprintf(stderr, "\+" BOLD "usage: /dump [FILE]" RESET "\n\+dumps raw tokens for current conversation history. special tokens are\n\+printed in the a model specific chat syntax. this is useful for seeing\n\+specifically what data is being evaluated by the model. by default it\n\+will be printed to the terminal. if a FILE argument is specified, then\n\+the raw conversation history will be written to that filename.\n\+");+ } else if (args[1] == "exit") {+ fprintf(stderr, "\+" BOLD "usage: /exit" RESET "\n\+this command will cause the process to exit. it is essentially the same\n\+as typing ctrl-d which signals an eof condition. it also does the same\n\+thing as typing ctrl-c when the >>> user input prompt is displayed.\n\+");+ } else if (args[1] == "manual") {+ fprintf(stderr, "\+" BOLD "usage: /manual [on|off]" RESET "\n\+puts the chatbot in manual mode. this is useful if you want to inject\n\+a response as the model rather than the user. it's also possible to add\n\+additional system prompts to the conversation history. when the manual\n\+mode is activated, a hint is displayed next to the '>>>' indicating\n\+the current role, which can be 'user', 'assistant', or 'system'. if\n\+enter is pressed on an empty line, then llamafile will cycle between\n\+all three roles. when /manual is specified without an argument, it will\n\+toggle manual mode. otherwise an 'on' or 'off' argument is supplied.\n\+");+ } else if (args[1] == "help") {+ fprintf(stderr, "\+" BOLD "usage: /help [COMMAND]" RESET "\n\+shows help on how to issue commands to your llamafile. if no argument is\n\+specified, then a synopsis of all available commands will be printed. if\n\+a specific command name is given (e.g. /help dump) then documentation on\n\+the usage of that specific command will be printed.\n\+");+ } else if (args[1] == "stats") {+ fprintf(stderr, "\+" BOLD "usage: /stats" RESET "\n\+prints performance statistics for current session. this includes prompt\n\+evaluation time in tokens per second, which indicates prefill speed, or\n\+how quickly llamafile is able to read text. the 'eval time' statistic\n\+gives you prediction or token generation speed, in tokens per second,\n\+which tells you how quickly llamafile is able to write text.\n\+");+ } else {+ fprintf(stderr, BRIGHT_RED "%s: unknown command" RESET "\n", args[1].c_str());+ }+}++static void on_manual(const std::vector<std::string> &args) {+ if (args.size() == 1) {+ g_manual = !g_manual;+ } else if (args.size() == 2 && (args[1] == "on" || args[1] == "off")) {+ g_manual = args[1] == "on";+ } else {+ fprintf(stderr, BRIGHT_RED "error: bad /manual command" RESET "\n"+ "usage: /manual [on|off]\n");+ }+ fprintf(stderr, FAINT "manual mode %s" RESET "\n", g_manual ? "enabled" : "disabled");+ if (!g_manual)+ g_role = ROLE_USER;+}++static void on_context(const std::vector<std::string> &args) {+ int configured_context = llama_n_ctx(g_ctx);+ int max_context = llama_n_ctx_train(g_model);+ printf("%d out of %d context tokens used (%d tokens remaining)\n", n_past, configured_context,+ configured_context - n_past);+ if (configured_context < max_context)+ printf("use the `-c %d` flag at startup for maximum context\n", max_context);+}++static void on_stats(const std::vector<std::string> &args) {+ FLAG_log_disable = false;+ llama_print_timings(g_ctx);+ FLAG_log_disable = true;+}++static void on_dump(const std::vector<std::string> &args) {+ int fd = 1;+ if (args.size() >= 2) {+ if ((fd = creat(args[1].c_str(), 0644)) == -1) {+ perror(args[1].c_str());+ return;+ }+ }+ std::string s;+ for (auto id : g_history)+ s += llama_token_to_piece(g_ctx, id, true);+ if (!s.empty() && s[s.size() - 1] != '\n')+ s += '\n';+ write(fd, s.data(), s.size());+ if (args.size() >= 2)+ close(fd);+}++static char *on_hint(const char *line, const char **ansi1, const char **ansi2) {+ *ansi1 = FAINT;+ *ansi2 = UNBOLD;+ if (!*line && g_manual)+ return strdup(get_role_name(g_role));+ else if (!*line && !g_manual && !g_once)+ return strdup("say something (or type /help for help)");+ return strdup("");+}+static void on_completion(const char *line, int pos, bestlineCompletions *comp) {static const char *const kCompletions[] = {- "/context", //- "/exit", //- "/stats", //+ "/context", // usage: /context+ "/dump", // usage: /dump [FILE]+ "/exit", // usage: /exit+ "/help", // usage: /help [COMMAND]+ "/manual", // usage: /manual [on|off]+ "/stats", // usage: /stats};for (int i = 0; i < sizeof(kCompletions) / sizeof(*kCompletions); ++i)if (startswith(kCompletions[i], line))@@ -118,21 +341,20 @@ static bool handle_command(const char *command) {std::string arg;while (iss >> arg)args.push_back(arg);- if (args[0] == "exit") {+ if (args[0] == "exit" || args[0] == "bye") {exit(0);+ } else if (args[0] == "help" || args[0] == "?") {+ on_help(args);} else if (args[0] == "stats") {- FLAG_log_disable = false;- llama_print_timings(g_ctx);- FLAG_log_disable = true;+ on_stats(args);} else if (args[0] == "context") {- int configured_context = llama_n_ctx(g_ctx);- int max_context = llama_n_ctx_train(g_model);- printf("%d out of %d context tokens used (%d tokens remaining)\n", n_past,- configured_context, configured_context - n_past);- if (configured_context < max_context)- printf("use the `-c %d` flag at startup for maximum context\n", max_context);+ on_context(args);+ } else if (args[0] == "manual") {+ on_manual(args);+ } else if (args[0] == "dump") {+ on_dump(args);} else {- printf("%s: unrecognized command\n", args[0].c_str());+ fprintf(stderr, BRIGHT_RED "%s: unrecognized command" RESET "\n", args[0].c_str());}return true;}@@ -190,6 +412,7 @@ static void eval_tokens(std::vector<llama_token> tokens, int n_batch) {n_eval = n_batch;if (llama_decode(g_ctx, llama_batch_get_one(&tokens[i], n_eval, n_past, 0)))die_out_of_context();+ g_history.insert(g_history.end(), tokens.begin() + i, tokens.begin() + i + n_eval);n_past += n_eval;}}@@ -276,6 +499,7 @@ int chatbot_main(int argc, char **argv) {printf(BOLD "software" UNBOLD ": llamafile " LLAMAFILE_VERSION_STRING "\n" //BOLD "model" UNBOLD ": %s\n",basename(params.model).c_str());+ printf(BOLD "compute" UNBOLD ": %s\n", describe_compute().c_str());if (want_server)printf(BOLD "server" UNBOLD ": %s\n", g_listen_url.c_str());printf("\n");@@ -311,8 +535,10 @@ int chatbot_main(int argc, char **argv) {// run chatbotfor (;;) {bestlineLlamaMode(true);+ bestlineSetHintsCallback(on_hint);+ bestlineSetFreeHintsCallback(free);bestlineSetCompletionCallback(on_completion);- write(1, GREEN, strlen(GREEN));+ write(1, get_role_color(g_role), strlen(get_role_color(g_role)));char *line = bestlineWithHistory(">>> ", "llamafile");write(1, UNFOREGROUND, strlen(UNFOREGROUND));if (!line) {@@ -321,6 +547,10 @@ int chatbot_main(int argc, char **argv) {break;}if (is_empty(line)) {+ if (g_manual) {+ g_role = cycle_role(g_role);+ write(1, "\033[F", 3);+ }free(line);continue;}@@ -328,9 +558,16 @@ int chatbot_main(int argc, char **argv) {free(line);continue;}- std::vector<llama_chat_msg> chat = {{"user", line}};- std::string msg = llama_chat_apply_template(g_model, params.chat_template, chat, true);+ g_once = true;+ bool add_assi = !g_manual;+ std::vector<llama_chat_msg> chat = {{get_role_name(g_role), line}};+ std::string msg = llama_chat_apply_template(g_model, params.chat_template, chat, add_assi);eval_string(msg, params.n_batch, false, true);+ if (g_manual) {+ g_role = get_next_role(g_role);+ free(line);+ continue;+ }while (!g_got_sigint) {llama_token id = llama_sampling_sample(sampler, g_ctx, NULL);llama_sampling_accept(sampler, g_ctx, id, true);diff --git a/llamafile/compute.cpp b/llamafile/compute.cppnew file mode 100644index 00000000..51b1b869--- /dev/null+++ b/llamafile/compute.cpp@@ -0,0 +1,110 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "compute.h"++#include <cosmo.h>+#include <libc/intrin/x86.h>+#include <sys/auxv.h>++#include "llama.cpp/string.h"++#ifdef __x86_64__+static void cpuid(unsigned leaf, unsigned subleaf, unsigned *info) {+ asm("movq\t%%rbx,%%rsi\n\t"+ "cpuid\n\t"+ "xchgq\t%%rbx,%%rsi"+ : "=a"(info[0]), "=S"(info[1]), "=c"(info[2]), "=d"(info[3])+ : "0"(leaf), "2"(subleaf));+}+#endif // __x86_64__++/**+ * Returns string describing host CPU.+ */+std::string llamafile_describe_cpu() {+ std::string id;++#ifdef __x86_64__+ union {+ char str[64];+ unsigned reg[16];+ } u = {0};+ cpuid(0x80000002, 0, u.reg + 0 * 4);+ cpuid(0x80000003, 0, u.reg + 1 * 4);+ cpuid(0x80000004, 0, u.reg + 2 * 4);+ int len = strlen(u.str);+ while (len > 0 && u.str[len - 1] == ' ')+ u.str[--len] = 0;+ id = u.str;+#else+ if (IsLinux()) {+ FILE *f = fopen("/proc/cpuinfo", "r");+ if (f) {+ char buf[1024];+ while (fgets(buf, sizeof(buf), f)) {+ if (!strncmp(buf, "model name", 10) ||+ startswith(buf, "Model\t\t:")) { // e.g. raspi+ char *p = strchr(buf, ':');+ if (p) {+ p++;+ while (std::isspace(*p))+ p++;+ while (std::isspace(p[strlen(p) - 1]))+ p[strlen(p) - 1] = '\0';+ id = p;+ break;+ }+ }+ }+ fclose(f);+ }+ }+ if (IsXnu()) {+ char cpu_name[128] = {0};+ size_t size = sizeof(cpu_name);+ if (sysctlbyname("machdep.cpu.brand_string", cpu_name, &size, NULL, 0) != -1)+ id = cpu_name;+ }+#endif+ id = replace_all(id, " 96-Cores", "");+ id = replace_all(id, "(TM)", "");+ id = replace_all(id, "(R)", "");++ std::string march;+#ifdef __x86_64__+ if (__cpu_march(__cpu_model.__cpu_subtype))+ march = __cpu_march(__cpu_model.__cpu_subtype);+#else+ long hwcap = getauxval(AT_HWCAP);+ if (hwcap & HWCAP_ASIMDHP)+ march += "+fp16";+ if (hwcap & HWCAP_ASIMDDP)+ march += "+dotprod";+#endif++ if (!march.empty()) {+ bool empty = id.empty();+ if (!empty)+ id += " (";+ id += march;+ if (!empty)+ id += ")";+ }++ return id;+}diff --git a/llamafile/compute.h b/llamafile/compute.hnew file mode 100644index 00000000..a8225098--- /dev/null+++ b/llamafile/compute.h@@ -0,0 +1,21 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <string>++std::string llamafile_describe_cpu();diff --git a/llamafile/cuda.c b/llamafile/cuda.cindex b62c5e21..d1f8df7e 100644--- a/llamafile/cuda.c+++ b/llamafile/cuda.c@@ -115,6 +115,7 @@ static struct Cuda {typeof(ggml_backend_cuda_get_device_count) *GGML_CALL get_device_count;typeof(ggml_backend_cuda_unregister_host_buffer) *GGML_CALL unreg_host_buf;typeof(ggml_backend_cuda_register_host_buffer) *GGML_CALL register_host_buffer;+ typeof(ggml_backend_cuda_get_device_description) *GGML_CALL get_description;} ggml_cuda;static const char *Dlerror(void) {@@ -731,6 +732,7 @@ static bool link_cuda_dso(const char *dso, const char *dir) {ok &= !!(ggml_cuda.get_device_count = imp(lib, "ggml_backend_cuda_get_device_count"));ok &= !!(ggml_cuda.unreg_host_buf = imp(lib, "ggml_backend_cuda_unregister_host_buffer"));ok &= !!(ggml_cuda.register_host_buffer = imp(lib, "ggml_backend_cuda_register_host_buffer"));+ ok &= !!(ggml_cuda.get_description = imp(lib, "ggml_backend_cuda_get_device_description"));if (!ok) {tinylog(__func__, ": error: not all cuda symbols could be imported\n", NULL);cosmo_dlclose(lib);@@ -1030,3 +1032,10 @@ GGML_CALL bool ggml_backend_cuda_register_host_buffer(void *buffer, size_t size)return false;return ggml_cuda.register_host_buffer(buffer, size);}++GGML_CALL void ggml_backend_cuda_get_device_description(int device, char *description,+ size_t description_size) {+ if (!llamafile_has_cuda())+ return;+ return ggml_cuda.get_description(device, description, description_size);+}