Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Create ollama inspired cli chatbot
Justine Tunneya384fd75d0f0
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mkindex 1fa70499..bc51c3a5 100644--- a/llamafile/BUILD.mk+++ b/llamafile/BUILD.mk@@ -33,21 +33,26 @@ o/$(MODE)/llamafile/zipalign: \o/$(MODE)/llamafile/zipcheck: \o/$(MODE)/llamafile/zipcheck.o \- o/$(MODE)/llamafile/zip.o+ o/$(MODE)/llamafile/zip.o \o/$(MODE)/llamafile/simple: \o/$(MODE)/llamafile/simple.o \- o/$(MODE)/llama.cpp/llama.cpp.a+ o/$(MODE)/llama.cpp/llama.cpp.a \++o/$(MODE)/llamafile/chatbot: \+ o/$(MODE)/llamafile/chatbot.o \+ o/$(MODE)/llama.cpp/llama.cpp.a \o/$(MODE)/llamafile/tokenize: \o/$(MODE)/llamafile/tokenize.o \- o/$(MODE)/llama.cpp/llama.cpp.a+ o/$(MODE)/llama.cpp/llama.cpp.a \.PHONY: o/$(MODE)/llamafileo/$(MODE)/llamafile: \$(LLAMAFILE_OBJS) \o/$(MODE)/llamafile/server \o/$(MODE)/llamafile/simple \+ o/$(MODE)/llamafile/chatbot \o/$(MODE)/llamafile/zipalign \o/$(MODE)/llamafile/zipcheck \o/$(MODE)/llamafile/tokenize \diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cppnew file mode 100644index 00000000..0cb988cc--- /dev/null+++ b/llamafile/chatbot.cpp@@ -0,0 +1,217 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include <assert.h>+#include <cosmo.h>+#include <math.h>+#include <signal.h>+#include <stdio.h>+#include <string>+#include <vector>++#include "llama.cpp/common.h"+#include "llama.cpp/llama.h"+#include "llamafile/bestline.h"+#include "llamafile/llamafile.h"++static sig_atomic_t g_got_sigint;++static void on_sigint(int sig) {+ g_got_sigint = 1;+}++static std::string basename(const std::string_view path) noexcept {+ size_t i, e;+ if ((e = path.size())) {+ while (e > 1 && path[e - 1] == '/')+ --e;+ i = e - 1;+ while (i && path[i - 1] != '/')+ --i;+ return std::string(path.substr(i, e - i));+ } else {+ return ".";+ }+}++std::string apply_template(const struct llama_model *model, const char *tmpl,+ const struct llama_chat_message *chat, size_t n_msg, bool add_ass) {++ // allocate buffer+ int alloc_size = 0;+ for (size_t i = 0; i < n_msg; ++i)+ alloc_size += (strlen(chat[i].role) + strlen(chat[i].content)) * 2;+ std::string buf;+ buf.resize(alloc_size);++ // run the first time to get the total output length+ int res = llama_chat_apply_template(model, tmpl, chat, n_msg, add_ass, &buf[0], buf.size());++ // error: chat template is not supported+ bool fallback = false;+ if (res < 0) {+ if (tmpl != nullptr) {+ // if the custom "tmpl" is not supported, we throw an error+ // this is a bit redundant (for good), since we're not sure if user validated the custom+ // template with llama_chat_verify_template()+ throw std::runtime_error("this custom template is not supported");+ } else {+ // If the built-in template is not supported, we default to chatml+ res = llama_chat_apply_template(nullptr, "chatml", chat, n_msg, add_ass, &buf[0],+ buf.size());+ fallback = true;+ }+ }++ // if it turns out that our buffer is too small, we resize it+ if ((size_t)res > buf.size()) {+ buf.resize(res);+ res = llama_chat_apply_template(fallback ? nullptr : model, fallback ? "chatml" : tmpl,+ chat, n_msg, add_ass, &buf[0], buf.size());+ assert(0 <= res && res <= buf.size());+ }++ buf.resize(res);+ return buf;+}++static bool eval_tokens(struct llama_context *ctx_llama, std::vector<llama_token> tokens,+ int n_batch, int *n_past) {+ int N = (int)tokens.size();+ for (int i = 0; i < N; i += n_batch) {+ int n_eval = (int)tokens.size() - i;+ if (n_eval > n_batch)+ n_eval = n_batch;+ if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0)))+ return false; // probably ran out of context+ *n_past += n_eval;+ }+ return true;+}++static bool eval_id(struct llama_context *ctx_llama, int id, int *n_past) {+ std::vector<llama_token> tokens;+ tokens.push_back(id);+ return eval_tokens(ctx_llama, tokens, 1, n_past);+}++static bool eval_string(struct llama_context *ctx_llama, const char *str, int n_batch, int *n_past,+ bool add_special, bool parse_special) {+ std::string str2 = str;+ std::vector<llama_token> embd_inp =+ ::llama_tokenize(ctx_llama, str2, add_special, parse_special);+ return eval_tokens(ctx_llama, embd_inp, n_batch, n_past);+}++static void print_ephemeral(const char *description) {+ fprintf(stderr, " \e[90m%s\e[39m\r", description);+}++static void clear_ephemeral(void) {+ fprintf(stderr, "\e[K");+}++int main(int argc, char **argv) {+ llamafile_check_cpu();+ ShowCrashReports();+ log_disable();++ gpt_params params;+ params.n_ctx = 8192; // make default context more reasonable++ if (!gpt_params_parse(argc, argv, params))+ return 1;++ printf("\n\+██╗ ██╗ █████╗ ███╗ ███╗ █████╗ ███████╗██╗██╗ ███████╗\n\+██║ ██║ ██╔══██╗████╗ ████║██╔══██╗██╔════╝██║██║ ██╔════╝\n\+██║ ██║ ███████║██╔████╔██║███████║█████╗ ██║██║ █████╗\n\+██║ ██║ ██╔══██║██║╚██╔╝██║██╔══██║██╔══╝ ██║██║ ██╔══╝\n\+███████╗███████╗██║ ██║██║ ╚═╝ ██║██║ ██║██║ ██║███████╗███████╗\n\+╚══════╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚══════╝\n\+\e[1msoftware\e[22m: llamafile " LLAMAFILE_VERSION_STRING "\n\+\e[1mmodel\e[22m: %s\n\n",+ basename(params.model).c_str());++ print_ephemeral("initializing backend...");+ llama_backend_init();+ clear_ephemeral();++ print_ephemeral("initializing model...");+ llama_model_params model_params = llama_model_default_params();+ model_params.n_gpu_layers = llamafile_gpu_layers(35);+ llama_model *model = llama_load_model_from_file(params.model.c_str(), model_params);+ if (model == NULL)+ return 2;+ clear_ephemeral();++ print_ephemeral("initializing context...");+ llama_context_params ctx_params = llama_context_params_from_gpt_params(params);+ llama_context *ctx = llama_new_context_with_model(model, ctx_params);+ if (ctx == NULL)+ return 3;+ clear_ephemeral();++ if (params.prompt.empty()) {+ params.prompt =+ "A chat between a curious human and an artificial intelligence assistant. The "+ "assistant gives helpful, detailed, and polite answers to the human's questions.";+ }++ print_ephemeral("loading prompt...");+ int n_past = 0;+ bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));+ llama_chat_message chat = {"system", params.prompt.c_str()};+ std::string msg = apply_template(model, nullptr, &chat, 1, false);+ eval_string(ctx, msg.c_str(), params.n_batch, &n_past, true, true);+ clear_ephemeral();+ printf("%s\n", params.prompt.c_str());++ // perform important setup+ struct llama_sampling_context *ctx_sampling = llama_sampling_init(params.sparams);+ signal(SIGINT, on_sigint);++ // run chatbot+ char *line;+ bestlineLlamaMode(true);+ while ((line = bestlineWithHistory(">>> ", "llamafile"))) {+ size_t buflen = strlen(line) * 2 + 1024;+ llama_chat_message chat = {"user", line};+ std::string msg = apply_template(model, nullptr, &chat, 1, true);+ eval_string(ctx, msg.c_str(), params.n_batch, &n_past, false, true);+ while (!g_got_sigint) {+ llama_token id = llama_sampling_sample(ctx_sampling, ctx, NULL);+ llama_sampling_accept(ctx_sampling, ctx, id, true);+ if (llama_token_is_eog(model, id))+ break;+ printf("%s", llama_token_to_piece(ctx, id).c_str(), false);+ fflush(stdout);+ if (!eval_id(ctx, id, &n_past)) {+ fprintf(stderr, "[out of context]\n");+ exit(1);+ }+ }+ g_got_sigint = 0;+ printf("\n");+ free(line);+ }++ llama_sampling_free(ctx_sampling);+ llama_free(ctx);+ llama_free_model(model);+ llama_backend_free();+}