Skip to content

Distribute and run LLMs with a single file.

Create ollama inspired cli chatbot

Justine Tunneya384fd75d0f0
diff --git a/llamafile/BUILD.mk b/llamafile/BUILD.mk
index 1fa70499..bc51c3a5 100644
--- a/llamafile/BUILD.mk
+++ b/llamafile/BUILD.mk
@@ -33,21 +33,26 @@ o/$(MODE)/llamafile/zipalign: \
o/$(MODE)/llamafile/zipcheck: \
o/$(MODE)/llamafile/zipcheck.o \
- o/$(MODE)/llamafile/zip.o
+ o/$(MODE)/llamafile/zip.o \
o/$(MODE)/llamafile/simple: \
o/$(MODE)/llamafile/simple.o \
- o/$(MODE)/llama.cpp/llama.cpp.a
+ o/$(MODE)/llama.cpp/llama.cpp.a \
+
+o/$(MODE)/llamafile/chatbot: \
+ o/$(MODE)/llamafile/chatbot.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
o/$(MODE)/llamafile/tokenize: \
o/$(MODE)/llamafile/tokenize.o \
- o/$(MODE)/llama.cpp/llama.cpp.a
+ o/$(MODE)/llama.cpp/llama.cpp.a \
.PHONY: o/$(MODE)/llamafile
o/$(MODE)/llamafile: \
$(LLAMAFILE_OBJS) \
o/$(MODE)/llamafile/server \
o/$(MODE)/llamafile/simple \
+ o/$(MODE)/llamafile/chatbot \
o/$(MODE)/llamafile/zipalign \
o/$(MODE)/llamafile/zipcheck \
o/$(MODE)/llamafile/tokenize \
diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cpp
new file mode 100644
index 00000000..0cb988cc
--- /dev/null
+++ b/llamafile/chatbot.cpp
@@ -0,0 +1,217 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <assert.h>
+#include <cosmo.h>
+#include <math.h>
+#include <signal.h>
+#include <stdio.h>
+#include <string>
+#include <vector>
+
+#include "llama.cpp/common.h"
+#include "llama.cpp/llama.h"
+#include "llamafile/bestline.h"
+#include "llamafile/llamafile.h"
+
+static sig_atomic_t g_got_sigint;
+
+static void on_sigint(int sig) {
+ g_got_sigint = 1;
+}
+
+static std::string basename(const std::string_view path) noexcept {
+ size_t i, e;
+ if ((e = path.size())) {
+ while (e > 1 && path[e - 1] == '/')
+ --e;
+ i = e - 1;
+ while (i && path[i - 1] != '/')
+ --i;
+ return std::string(path.substr(i, e - i));
+ } else {
+ return ".";
+ }
+}
+
+std::string apply_template(const struct llama_model *model, const char *tmpl,
+ const struct llama_chat_message *chat, size_t n_msg, bool add_ass) {
+
+ // allocate buffer
+ int alloc_size = 0;
+ for (size_t i = 0; i < n_msg; ++i)
+ alloc_size += (strlen(chat[i].role) + strlen(chat[i].content)) * 2;
+ std::string buf;
+ buf.resize(alloc_size);
+
+ // run the first time to get the total output length
+ int res = llama_chat_apply_template(model, tmpl, chat, n_msg, add_ass, &buf[0], buf.size());
+
+ // error: chat template is not supported
+ bool fallback = false;
+ if (res < 0) {
+ if (tmpl != nullptr) {
+ // if the custom "tmpl" is not supported, we throw an error
+ // this is a bit redundant (for good), since we're not sure if user validated the custom
+ // template with llama_chat_verify_template()
+ throw std::runtime_error("this custom template is not supported");
+ } else {
+ // If the built-in template is not supported, we default to chatml
+ res = llama_chat_apply_template(nullptr, "chatml", chat, n_msg, add_ass, &buf[0],
+ buf.size());
+ fallback = true;
+ }
+ }
+
+ // if it turns out that our buffer is too small, we resize it
+ if ((size_t)res > buf.size()) {
+ buf.resize(res);
+ res = llama_chat_apply_template(fallback ? nullptr : model, fallback ? "chatml" : tmpl,
+ chat, n_msg, add_ass, &buf[0], buf.size());
+ assert(0 <= res && res <= buf.size());
+ }
+
+ buf.resize(res);
+ return buf;
+}
+
+static bool eval_tokens(struct llama_context *ctx_llama, std::vector<llama_token> tokens,
+ int n_batch, int *n_past) {
+ int N = (int)tokens.size();
+ for (int i = 0; i < N; i += n_batch) {
+ int n_eval = (int)tokens.size() - i;
+ if (n_eval > n_batch)
+ n_eval = n_batch;
+ if (llama_decode(ctx_llama, llama_batch_get_one(&tokens[i], n_eval, *n_past, 0)))
+ return false; // probably ran out of context
+ *n_past += n_eval;
+ }
+ return true;
+}
+
+static bool eval_id(struct llama_context *ctx_llama, int id, int *n_past) {
+ std::vector<llama_token> tokens;
+ tokens.push_back(id);
+ return eval_tokens(ctx_llama, tokens, 1, n_past);
+}
+
+static bool eval_string(struct llama_context *ctx_llama, const char *str, int n_batch, int *n_past,
+ bool add_special, bool parse_special) {
+ std::string str2 = str;
+ std::vector<llama_token> embd_inp =
+ ::llama_tokenize(ctx_llama, str2, add_special, parse_special);
+ return eval_tokens(ctx_llama, embd_inp, n_batch, n_past);
+}
+
+static void print_ephemeral(const char *description) {
+ fprintf(stderr, " \e[90m%s\e[39m\r", description);
+}
+
+static void clear_ephemeral(void) {
+ fprintf(stderr, "\e[K");
+}
+
+int main(int argc, char **argv) {
+ llamafile_check_cpu();
+ ShowCrashReports();
+ log_disable();
+
+ gpt_params params;
+ params.n_ctx = 8192; // make default context more reasonable
+
+ if (!gpt_params_parse(argc, argv, params))
+ return 1;
+
+ printf("\n\
+██╗ ██╗ █████╗ ███╗ ███╗ █████╗ ███████╗██╗██╗ ███████╗\n\
+██║ ██║ ██╔══██╗████╗ ████║██╔══██╗██╔════╝██║██║ ██╔════╝\n\
+██║ ██║ ███████║██╔████╔██║███████║█████╗ ██║██║ █████╗\n\
+██║ ██║ ██╔══██║██║╚██╔╝██║██╔══██║██╔══╝ ██║██║ ██╔══╝\n\
+███████╗███████╗██║ ██║██║ ╚═╝ ██║██║ ██║██║ ██║███████╗███████╗\n\
+╚══════╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚══════╝\n\
+\e[1msoftware\e[22m: llamafile " LLAMAFILE_VERSION_STRING "\n\
+\e[1mmodel\e[22m: %s\n\n",
+ basename(params.model).c_str());
+
+ print_ephemeral("initializing backend...");
+ llama_backend_init();
+ clear_ephemeral();
+
+ print_ephemeral("initializing model...");
+ llama_model_params model_params = llama_model_default_params();
+ model_params.n_gpu_layers = llamafile_gpu_layers(35);
+ llama_model *model = llama_load_model_from_file(params.model.c_str(), model_params);
+ if (model == NULL)
+ return 2;
+ clear_ephemeral();
+
+ print_ephemeral("initializing context...");
+ llama_context_params ctx_params = llama_context_params_from_gpt_params(params);
+ llama_context *ctx = llama_new_context_with_model(model, ctx_params);
+ if (ctx == NULL)
+ return 3;
+ clear_ephemeral();
+
+ if (params.prompt.empty()) {
+ params.prompt =
+ "A chat between a curious human and an artificial intelligence assistant. The "
+ "assistant gives helpful, detailed, and polite answers to the human's questions.";
+ }
+
+ print_ephemeral("loading prompt...");
+ int n_past = 0;
+ bool add_bos = llama_should_add_bos_token(llama_get_model(ctx));
+ llama_chat_message chat = {"system", params.prompt.c_str()};
+ std::string msg = apply_template(model, nullptr, &chat, 1, false);
+ eval_string(ctx, msg.c_str(), params.n_batch, &n_past, true, true);
+ clear_ephemeral();
+ printf("%s\n", params.prompt.c_str());
+
+ // perform important setup
+ struct llama_sampling_context *ctx_sampling = llama_sampling_init(params.sparams);
+ signal(SIGINT, on_sigint);
+
+ // run chatbot
+ char *line;
+ bestlineLlamaMode(true);
+ while ((line = bestlineWithHistory(">>> ", "llamafile"))) {
+ size_t buflen = strlen(line) * 2 + 1024;
+ llama_chat_message chat = {"user", line};
+ std::string msg = apply_template(model, nullptr, &chat, 1, true);
+ eval_string(ctx, msg.c_str(), params.n_batch, &n_past, false, true);
+ while (!g_got_sigint) {
+ llama_token id = llama_sampling_sample(ctx_sampling, ctx, NULL);
+ llama_sampling_accept(ctx_sampling, ctx, id, true);
+ if (llama_token_is_eog(model, id))
+ break;
+ printf("%s", llama_token_to_piece(ctx, id).c_str(), false);
+ fflush(stdout);
+ if (!eval_id(ctx, id, &n_past)) {
+ fprintf(stderr, "[out of context]\n");
+ exit(1);
+ }
+ }
+ g_got_sigint = 0;
+ printf("\n");
+ free(line);
+ }
+
+ llama_sampling_free(ctx_sampling);
+ llama_free(ctx);
+ llama_free_model(model);
+ llama_backend_free();
+}