Skip to content

Distribute and run LLMs with a single file.

Refactor chatbot codebase into more files

Justine Tunneya03b47e127e6
diff --git a/llama.cpp/main/main.cpp b/llama.cpp/main/main.cpp
index 85cb95fd..12fbca74 100644
--- a/llama.cpp/main/main.cpp
+++ b/llama.cpp/main/main.cpp
@@ -18,6 +18,7 @@
#include <cosmo.h>
#include "llamafile/version.h"
+#include "llamafile/chatbot.h"
#include "llama.cpp/llama.h"
#include "llama.cpp/string.h"
#include "llama.cpp/common.h"
@@ -202,8 +203,7 @@ int main(int argc, char ** argv) {
!llamafile_has(argv, "-p") &&
!llamafile_has(argv, "-f") &&
!llamafile_has(argv, "--random-prompt"))) {
- int chatbot_main(int, char **);
- return chatbot_main(argc, argv);
+ return lf::chatbot::main(argc, argv);
}
if (prog == EMBEDDING) {
diff --git a/llamafile/chatbot.cpp b/llamafile/chatbot.cpp
deleted file mode 100644
index 68e734ab..00000000
--- a/llamafile/chatbot.cpp
+++ /dev/null
@@ -1,958 +0,0 @@
-// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
-// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
-//
-// Copyright 2024 Mozilla Foundation
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "chatbot.h"
-
-#include <assert.h>
-#include <cosmo.h>
-#include <ctype.h>
-#include <glob.h>
-#include <math.h>
-#include <signal.h>
-#include <sstream>
-#include <stdio.h>
-#include <string>
-#include <sys/stat.h>
-#include <vector>
-
-#include "llama.cpp/common.h"
-#include "llama.cpp/ggml-cuda.h"
-#include "llama.cpp/llama.h"
-#include "llama.cpp/llava/clip.h"
-#include "llama.cpp/llava/llava.h"
-#include "llama.cpp/server/server.h"
-#include "llamafile/bestline.h"
-#include "llamafile/compute.h"
-#include "llamafile/datauri.h"
-#include "llamafile/highlight.h"
-#include "llamafile/image.h"
-#include "llamafile/llama.h"
-#include "llamafile/llamafile.h"
-#include "llamafile/string.h"
-#include "llamafile/xterm.h"
-
-#define IMAGE_PLACEHOLDER_TOKEN -31337
-
-enum Role {
- ROLE_USER,
- ROLE_ASSISTANT,
- ROLE_SYSTEM,
-};
-
-struct ServerArgs {
- int argc;
- char **argv;
-};
-
-static bool g_manual_mode;
-static bool g_has_ephemeral;
-static bool g_said_something;
-static char g_last_printed_char;
-static int g_system_prompt_tokens;
-static gpt_params g_params;
-static clip_ctx *g_clip;
-static llama_model *g_model;
-static llama_context *g_ctx;
-static std::vector<llama_pos> g_undo;
-static std::vector<llama_pos> g_stack;
-static std::vector<llama_token> g_history;
-static enum Role g_role = ROLE_USER;
-static pthread_cond_t g_cond = PTHREAD_COND_INITIALIZER;
-static pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER;
-static std::string g_listen_url;
-static volatile sig_atomic_t g_got_sigint;
-
-static void on_sigint(int sig) {
- g_got_sigint = 1;
-}
-
-static int tokens_used(void) {
- return g_history.size();
-}
-
-static bool is_empty(const char *s) {
- int c;
- while ((c = *s++))
- if (!isspace(c))
- return false;
- return true;
-}
-
-static void fix_stack(std::vector<llama_pos> *stack) {
- while (!stack->empty() && stack->back() > tokens_used())
- stack->pop_back();
-}
-
-static void fix_stacks(void) {
- fix_stack(&g_undo);
- fix_stack(&g_stack);
- g_system_prompt_tokens = MIN(g_system_prompt_tokens, tokens_used());
-}
-
-static std::vector<llama_pos> adjust_stack(llama_pos erase_begin, llama_pos erase_end,
- const std::vector<llama_pos> &stack) {
- std::vector<llama_pos> builder;
- for (llama_pos pos : stack) {
- if (erase_begin <= pos && pos < erase_end)
- continue;
- if (pos >= erase_end)
- pos -= erase_end - erase_begin;
- builder.push_back(pos);
- }
- return builder;
-}
-
-static void adjust_stacks(llama_pos erase_begin, llama_pos erase_end) {
- g_undo = adjust_stack(erase_begin, erase_end, g_undo);
- g_stack = adjust_stack(erase_begin, erase_end, g_stack);
-}
-
-static void record_undo(void) {
- if (g_undo.empty() || g_undo.back() != tokens_used())
- g_undo.push_back(tokens_used());
-}
-
-static bool is_directory(const char *path) {
- struct stat st;
- return !stat(path, &st) && S_ISDIR(st.st_mode);
-}
-
-static const char *get_role_name(enum Role role) {
- switch (role) {
- case ROLE_USER:
- return "user";
- case ROLE_ASSISTANT:
- return "assistant";
- case ROLE_SYSTEM:
- return "system";
- default:
- __builtin_unreachable();
- }
-}
-
-static const char *get_role_color(enum Role role) {
- switch (role) {
- case ROLE_USER:
- return GREEN;
- case ROLE_ASSISTANT:
- return MAGENTA;
- case ROLE_SYSTEM:
- return YELLOW;
- default:
- __builtin_unreachable();
- }
-}
-
-static enum Role get_next_role(enum Role role) {
- switch (role) {
- case ROLE_USER:
- return ROLE_ASSISTANT;
- case ROLE_ASSISTANT:
- return ROLE_USER;
- case ROLE_SYSTEM:
- return ROLE_USER;
- default:
- __builtin_unreachable();
- }
-}
-
-static enum Role cycle_role(enum Role role) {
- switch (role) {
- case ROLE_USER:
- return ROLE_ASSISTANT;
- case ROLE_ASSISTANT:
- return ROLE_SYSTEM;
- case ROLE_SYSTEM:
- return ROLE_USER;
- default:
- __builtin_unreachable();
- }
-}
-
-static bool has_binary(const std::string_view s) {
- return s.find('\0') != std::string_view::npos;
-}
-
-static std::string describe_compute(void) {
- if (llama_n_gpu_layers(g_model) > 0 && llamafile_has_gpu()) {
- if (llamafile_has_metal()) {
- return "Apple Metal GPU";
- } else {
- std::vector<std::string> vec;
- int count = ggml_backend_cuda_get_device_count();
- for (int i = 0; i < count; i++) {
- char buf[128];
- ggml_backend_cuda_get_device_description(i, buf, sizeof(buf));
- vec.emplace_back(buf);
- }
- return lf::join(vec, ", ");
- }
- } else {
- return llamafile_describe_cpu();
- }
-}
-
-static std::string token_to_piece(const struct llama_context *ctx, llama_token token,
- bool special) {
- if (token == IMAGE_PLACEHOLDER_TOKEN)
- return "⁑";
- return llama_token_to_piece(ctx, token, special);
-}
-
-static std::string describe_token(llama_token token) {
- if (token == llama_token_bos(g_model))
- return "§";
- if (token == llama_token_eos(g_model))
- return "∎";
- if (token == llama_token_cls(g_model))
- return "⌘";
- if (token == llama_token_sep(g_model))
- return "⋯";
- if (token == llama_token_pad(g_model))
- return "␣";
- if (token == llama_token_nl(g_model))
- return "↵";
- if (llama_token_is_eog(g_model, token))
- return "⌟";
- if (llama_token_is_control(g_model, token))
- return "∷";
- std::string s = token_to_piece(g_ctx, token, DONT_RENDER_SPECIAL_TOKENS);
- if (s.empty())
- return "↯";
- return s;
-}
-
-static std::string describe_erasure(llama_pos begin, llama_pos end) {
- unassert(begin <= end);
- unassert(end <= tokens_used());
- std::string description;
- llama_pos pos = begin;
- while (pos < end && description.size() < 63)
- description += describe_token(g_history[pos++]);
- if (!description.empty() && pos < end)
- description += " ...";
- description = lf::collapse(description);
- if (pos == end && description.empty())
- description = "<absolute end>";
- return description;
-}
-
-static std::string describe_position(llama_pos pos) {
- unassert(pos <= tokens_used());
- std::string description;
- while (pos > 0 && description.size() < 63)
- description = describe_token(g_history[--pos]) + description;
- if (!description.empty() && pos > 0)
- description = std::string("... ") + description;
- description = lf::collapse(description);
- if (!pos && description.empty())
- description = "<absolute beginning>";
- return description;
-}
-
-static void print(const std::string_view &s) {
- for (char c : s) {
- g_last_printed_char = c;
- fputc(c, stdout);
- if (c == '\n')
- g_has_ephemeral = false;
- }
-}
-
-static void ensure_newline() {
- if (g_last_printed_char != '\n')
- print("\n");
-}
-
-static void print_ephemeral(const std::string_view &description) {
- fprintf(stderr, " " BRIGHT_BLACK "%.*s" UNFOREGROUND "\r", (int)description.size(),
- description.data());
- g_has_ephemeral = true;
-}
-
-static void clear_ephemeral(void) {
- if (g_has_ephemeral) {
- fprintf(stderr, CLEAR_FORWARD);
- g_has_ephemeral = false;
- }
-}
-
-static void err(const char *fmt, ...) {
- va_list ap;
- clear_ephemeral();
- ensure_newline();
- va_start(ap, fmt);
- fputs(BRIGHT_RED, stderr);
- vfprintf(stderr, fmt, ap);
- fputs(RESET "\n", stderr);
- va_end(ap);
-}
-
-static bool out_of_context(int extra) {
- err("error: ran out of context window at %d tokens\n"
- "consider passing `-c %d` at startup for the maximum\n"
- "you can free up more space using /forget or /clear",
- tokens_used() + extra, llama_n_ctx_train(g_model));
- return false;
-}
-
-static bool eval_tokens(std::vector<llama_token> tokens) {
- int N = (int)tokens.size();
- if (tokens_used() + N > llama_n_ctx(g_ctx))
- return out_of_context(N);
- for (int i = 0; i < N; i += g_params.n_batch) {
- if (g_got_sigint) {
- g_got_sigint = false;
- clear_ephemeral();
- return false;
- }
- if (N > g_params.n_batch)
- print_ephemeral(lf::format("loading prompt %d%%...", (int)((double)i / N * 100)));
- int n_eval = (int)tokens.size() - i;
- if (n_eval > g_params.n_batch)
- n_eval = g_params.n_batch;
- if (llama_decode(g_ctx, llama_batch_get_one(&tokens[i], n_eval, tokens_used(), 0)))
- return out_of_context(n_eval);
- g_history.insert(g_history.end(), tokens.begin() + i, tokens.begin() + i + n_eval);
- }
- clear_ephemeral();
- // this function is what computes /stats. we need to call it now
- // since llama_decode() kicks the can down the road to functions
- // like llama_sampling_sample(). that is bad because the chatbot
- // returns control to the repl rather than sampling when loading
- // system and image prompts.
- llama_synchronize(g_ctx);
- return true;
-}
-
-static bool eval_image_embed(const struct llava_image_embed *image_embed) {
- int N = image_embed->n_image_pos;
- if (tokens_used() + N > llama_n_ctx(g_ctx))
- return out_of_context(N);
- int n_embd = llama_n_embd(llama_get_model(g_ctx));
- for (int i = 0; i < N; i += g_params.n_batch) {
- if (g_got_sigint) {
- g_got_sigint = false;
- clear_ephemeral();
- return false;
- }
- if (N > g_params.n_batch)
- print_ephemeral(lf::format("loading image %d%%...", (int)((double)i / N * 100)));
- int n_eval = N - i;
- if (n_eval > g_params.n_batch)
- n_eval = g_params.n_batch;
- llama_batch batch = {
- .n_tokens = n_eval,
- .embd = image_embed->embed + i * n_embd,
- .all_pos_0 = tokens_used(),
- .all_pos_1 = 1,
- .all_seq_id = 0,
- };
- if (llama_decode(g_ctx, batch))
- return out_of_context(n_eval);
- for (int i = 0; i < n_eval; ++i)
- g_history.push_back(IMAGE_PLACEHOLDER_TOKEN);
- }
- clear_ephemeral();
- return true;
-}
-
-static bool eval_image(const std::string_view binary) {
- unassert(g_clip);
- llava_image_embed *image_embed;
- print_ephemeral("analyzing image...");
- image_embed = llava_image_embed_make_with_bytes(
- g_clip, FLAG_threads_batch, (const unsigned char *)binary.data(), binary.size());
- clear_ephemeral();
- if (!image_embed) {
- err("failed to load image");
- return false;
- }
- bool ok = eval_image_embed(image_embed);
- llava_image_embed_free(image_embed);
- return ok;
-}
-
-static bool eval_token(int id) {
- return eval_tokens({id});
-}
-
-static bool eval_plain_text(const std::string &str, bool add_special, bool parse_special) {
- return eval_tokens(llama_tokenize(g_model, str, add_special, parse_special));
-}
-
-static bool eval_string(std::string_view s, bool add_special, bool parse_special) {
- size_t i = 0;
- for (;;) {
- size_t pos = s.find("data:", i);
- if (pos == std::string_view::npos)
- return eval_plain_text(std::string(s), add_special, parse_special);
- DataUri uri;
- size_t end = uri.parse(s.substr(pos + 5));
- if (end == std::string_view::npos) {
- i = pos + 5;
- continue;
- }
- std::string image = uri.decode();
- if (!lf::is_image(image)) {
- i = pos + 5;
- continue;
- }
- if (!eval_plain_text(std::string(s.substr(0, pos)), add_special, parse_special))
- return false;
- if (!eval_image(image))
- return false;
- s = s.substr(pos + 5 + end);
- i = 0;
- }
-}
-
-static void rewind(int pos) {
- unassert(pos <= tokens_used());
- llama_kv_cache_seq_rm(g_ctx, 0, pos, -1);
- g_history.resize(pos);
-}
-
-static void on_manual(const std::vector<std::string> &args) {
- if (args.size() == 1) {
- g_manual_mode = !g_manual_mode;
- } else if (args.size() == 2 && (args[1] == "on" || args[1] == "off")) {
- g_manual_mode = args[1] == "on";
- } else {
- err("error: bad /manual command\n"
- "usage: /manual [on|off]");
- return;
- }
- fprintf(stderr, FAINT "manual mode %s" RESET "\n", g_manual_mode ? "enabled" : "disabled");
- if (!g_manual_mode)
- g_role = ROLE_USER;
-}
-
-static void on_context(const std::vector<std::string> &args) {
- int configured_context = llama_n_ctx(g_ctx);
- int max_context = llama_n_ctx_train(g_model);
- printf("%d out of %d context tokens used (%d tokens remaining)\n", tokens_used(),
- configured_context, configured_context - tokens_used());
- if (configured_context < max_context)
- printf("use the `-c %d` flag at startup for maximum context\n", max_context);
-}
-
-static void on_clear(const std::vector<std::string> &args) {
- rewind(g_system_prompt_tokens);
- g_stack.clear();
- fix_stacks();
-}
-
-static void print_stack(void) {
- for (size_t i = g_stack.size(); i--;)
- printf("%12d " FAINT "(%s)" RESET "\n", g_stack[i], describe_position(g_stack[i]).c_str());
-}
-
-static void on_push(const std::vector<std::string> &args) {
- g_stack.push_back(tokens_used());
- print_stack();
-}
-
-static void on_pop(const std::vector<std::string> &args) {
- if (g_stack.empty()) {
- err("error: context length stack is empty");
- return;
- }
- printf(BOLD "%12d" RESET " restored " FAINT "(%s)" RESET "\n", g_stack.back(),
- describe_position(g_stack.back()).c_str());
- rewind(g_stack.back());
- g_stack.pop_back();
- fix_stacks();
- print_stack();
-}
-
-static void on_undo(const std::vector<std::string> &args) {
- while (!g_undo.empty() && g_undo.back() == tokens_used())
- g_undo.pop_back();
- if (g_undo.empty()) {
- err("error: no further undo actions possible");
- return;
- }
- printf(FAINT "restoring conversation to: %s" RESET "\n",
- describe_position(g_undo.back()).c_str());
- rewind(g_undo.back());
- g_undo.pop_back();
- fix_stacks();
-}
-
-static void on_forget(const std::vector<std::string> &args) {
- if (g_undo.size() < 2) {
- err("error: nothing left to forget");
- return;
- }
- int erase_count;
- llama_pos erase_begin = g_undo[1];
- llama_pos erase_end = g_undo.size() > 2 ? g_undo[2] : tokens_used();
- if (!(erase_count = erase_end - erase_begin)) {
- err("error: nothing left to forget");
- return;
- }
- printf(FAINT "forgetting: %s" RESET "\n", describe_erasure(erase_begin, erase_end).c_str());
- llama_kv_cache_seq_rm(g_ctx, 0, erase_begin, erase_end);
- llama_kv_cache_seq_add(g_ctx, 0, erase_end, -1, -erase_count);
- g_history.erase(g_history.begin() + erase_begin, //
- g_history.begin() + erase_end);
- adjust_stacks(erase_begin, erase_end);
- fix_stacks();
-}
-
-static void on_stack(const std::vector<std::string> &args) {
- if (g_stack.empty()) {
- printf(FAINT "stack is currently empty (try using /push)" RESET "\n");
- return;
- }
- print_stack();
-}
-
-static void on_stats(const std::vector<std::string> &args) {
- FLAG_log_disable = false;
- llama_print_timings(g_ctx);
- FLAG_log_disable = true;
-}
-
-static void on_dump(const std::vector<std::string> &args) {
- int fd = 1;
- if (args.size() >= 2) {
- if ((fd = creat(args[1].c_str(), 0644)) == -1) {
- perror(args[1].c_str());
- return;
- }
- }
- std::string s;
- for (auto id : g_history)
- s += token_to_piece(g_ctx, id, RENDER_SPECIAL_TOKENS);
- if (!s.empty() && s[s.size() - 1] != '\n')
- s += '\n';
- write(fd, s.data(), s.size());
- if (args.size() >= 2)
- close(fd);
-}
-
-static void on_upload(const std::vector<std::string> &args) {
- if (args.size() < 2) {
- err("error: missing file path" RESET "\n"
- "usage: /upload PATH");
- return;
- }
- if (args.size() > 2) {
- err("error: too many arguments" RESET "\n"
- "usage: /upload PATH");
- return;
- }
- const char *path = args[1].c_str();
- struct stat st;
- if (stat(path, &st) || !S_ISREG(st.st_mode)) {
- err("%s: file does not exist", path);
- return;
- }
- int tokens_used_before = tokens_used();
- std::string content;
- if (!lf::slurp(&content, path)) {
- err("%s: failed to slurp file", path);
- return;
- }
- std::string markdown;
- markdown += "- **Filename**: `";
- markdown += path;
- markdown += "`\n- **Last modified**: ";
- markdown += lf::iso8601(st.st_mtim);
- markdown += "\n\n";
- if (lf::is_image(content)) {
- if (!g_clip) {
- err("%s: need --mmproj model to process images", path);
- return;
- }
- lf::print_image(1, content, 80);
- lf::convert_image_to_uri(&markdown, content);
- } else {
- if (has_binary(content)) {
- err("%s: binary file type not supported", path);
- return;
- }
- markdown += "``````";
- markdown += lf::extname(path);
- markdown += '\n';
- markdown += content;
- if (markdown.back() != '\n')
- markdown += '\n';
- markdown += "``````";
- }
- std::vector<llama_chat_msg> chat = {{"system", std::move(markdown)}};
- if (!eval_string(
- llama_chat_apply_template(g_model, g_params.chat_template, chat, DONT_ADD_ASSISTANT),
- DONT_ADD_SPECIAL, PARSE_SPECIAL)) {
- rewind(tokens_used_before);
- return;
- }
- llama_synchronize(g_ctx);
-}
-
-static const char *on_hint_impl(const char *line) {
- if (!*line && g_manual_mode)
- return get_role_name(g_role);
- if (!*line && !g_manual_mode && !g_said_something)
- return "say something (or type /help for help)";
- static const char *const kHints[] = {
- "/clear", //
- "/context", //
- "/dump", //
- "/exit", //
- "/forget", //
- "/help", //
- "/manual", //
- "/pop", //
- "/push", //
- "/stack", //
- "/stats", //
- "/undo", //
- "/upload", //
- };
- int z = strlen(line);
- int n = sizeof(kHints) / sizeof(kHints[0]);
- int l = 0;
- int r = n - 1;
- int i = -1;
- while (l <= r) {
- int m = (l & r) + ((l ^ r) >> 1); // floor((a+b)/2)
- int c = strncmp(line, kHints[m], z);
- if (!c) {
- i = m;
- r = m - 1;
- } else if (c < 0) {
- r = m - 1;
- } else {
- l = m + 1;
- }
- }
- if (i == -1 || (i + 1 < n && !strncmp(line, kHints[i + 1], z)))
- return "";
- return kHints[i] + z;
-}
-
-static char *on_hint(const char *line, const char **ansi1, const char **ansi2) {
- *ansi1 = FAINT;
- *ansi2 = UNBOLD;
- return strdup(on_hint_impl(line));
-}
-
-static void on_completion(const char *line, int pos, bestlineCompletions *comp) {
- if (startswith(line, "/upload ")) {
- std::string pattern(line + strlen("/upload "));
- pattern += '*';
- glob_t gl;
- if (!glob(pattern.c_str(), GLOB_TILDE, 0, &gl)) {
- for (size_t i = 0; i < gl.gl_pathc; ++i) {
- std::string completion = "/upload ";
- completion += gl.gl_pathv[i];
- if (is_directory(gl.gl_pathv[i]))
- completion += '/';
- bestlineAddCompletion(comp, completion.c_str());
- }
- globfree(&gl);
- }
- } else {
- static const char *const kCompletions[] = {
- "/clear", // usage: /clear
- "/context", // usage: /context
- "/dump", // usage: /dump [FILE]
- "/exit", // usage: /exit
- "/forget", // usage: /forget
- "/help", // usage: /help [COMMAND]
- "/manual", // usage: /manual [on|off]
- "/pop", // usage: /pop
- "/push", // usage: /push
- "/stack", // usage: /stack
- "/stats", // usage: /stats
- "/undo", // usage: /undo
- "/upload", // usage: /upload FILE
- };
- for (int i = 0; i < sizeof(kCompletions) / sizeof(*kCompletions); ++i)
- if (startswith(kCompletions[i], line))
- bestlineAddCompletion(comp, kCompletions[i]);
- }
-}
-
-// handle irc style commands like: `/arg0 arg1 arg2`
-static bool handle_command(const char *command) {
- if (!strcmp(command, "/?")) {
- const std::vector<std::string> args = {"?"};
- chatbot_help(args);
- return true;
- }
- if (!(command[0] == '/' && std::isalpha(command[1])))
- return false;
- std::vector<std::string> args;
- std::istringstream iss(command + 1);
- std::string arg;
- while (iss >> arg)
- args.push_back(arg);
- if (args[0] == "exit" || args[0] == "bye") {
- exit(0);
- } else if (args[0] == "help") {
- chatbot_help(args);
- } else if (args[0] == "stats") {
- on_stats(args);
- } else if (args[0] == "context") {
- on_context(args);
- } else if (args[0] == "manual") {
- on_manual(args);
- } else if (args[0] == "clear") {
- on_clear(args);
- } else if (args[0] == "dump") {
- on_dump(args);
- } else if (args[0] == "push") {
- on_push(args);
- } else if (args[0] == "pop") {
- on_pop(args);
- } else if (args[0] == "undo") {
- on_undo(args);
- } else if (args[0] == "forget") {
- on_forget(args);
- } else if (args[0] == "stack") {
- on_stack(args);
- } else if (args[0] == "upload") {
- on_upload(args);
- } else {
- err("%s: unrecognized command", args[0].c_str());
- }
- return true;
-}
-
-static void on_server_listening(const char *host, int port) {
- pthread_mutex_lock(&g_lock);
- g_listen_url = lf::format("http://%s:%d/", host, port);
- pthread_cond_signal(&g_cond);
- pthread_mutex_unlock(&g_lock);
-}
-
-static void *server_thread(void *arg) {
- ServerArgs *sargs = (ServerArgs *)arg;
- server_log_json = false;
- g_server_background_mode = true;
- g_server_force_llama_model = g_model;
- g_server_on_listening = on_server_listening;
- exit(server_cli(sargs->argc, sargs->argv));
-}
-
-int chatbot_main(int argc, char **argv) {
-
- // print logo
- chatbot_logo(argv);
-
- // disable llamafile gpu initialization log messages
- if (!llamafile_has(argv, "--verbose"))
- FLAG_log_disable = true;
-
- // override defaults for some flags
- g_params.n_batch = 256; // for better progress indication
- g_params.sparams.temp = 0; // don't believe in randomness by default
- g_params.prompt = "A chat between a curious human and an artificial intelligence assistant. "
- "The assistant gives helpful, detailed, and polite answers to the "
- "human's questions.";
-
- // parse flags (sadly initializes gpu support as side-effect)
- print_ephemeral("loading backend...");
- llama_backend_init();
- if (!gpt_params_parse(argc, argv, g_params)) { // also loads gpu module
- fprintf(stderr, "error: failed to parse flags\n");
- exit(1);
- }
- clear_ephemeral();
-
- // setup logging
- FLAG_log_disable = false;
- if (!g_params.verbosity)
- log_disable();
-
- print_ephemeral("loading model...");
- llama_model_params model_params = llama_model_params_from_gpt_params(g_params);
- g_model = llama_load_model_from_file(g_params.model.c_str(), model_params);
- if (g_model == NULL) {
- clear_ephemeral();
- fprintf(stderr, "%s: failed to load model\n", g_params.model.c_str());
- exit(2);
- }
- if (g_params.n_ctx <= 0 || g_params.n_ctx > llama_n_ctx_train(g_model))
- g_params.n_ctx = llama_n_ctx_train(g_model);
- if (g_params.n_ctx < g_params.n_batch)
- g_params.n_batch = g_params.n_ctx;
- clear_ephemeral();
-
- bool want_server = !llamafile_has(argv, "--chat");
- if (want_server) {
- print_ephemeral("launching server...");
- pthread_t thread;
- pthread_attr_t attr;
- ServerArgs sargs = {argc, argv};
- pthread_mutex_lock(&g_lock);
- pthread_attr_init(&attr);
- pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
- pthread_create(&thread, &attr, server_thread, &sargs);
- pthread_attr_destroy(&attr);
- pthread_cond_wait(&g_cond, &g_lock);
- pthread_mutex_unlock(&g_lock);
- clear_ephemeral();
- }
-
- if (!FLAG_nologo) {
- printf(BOLD "software" UNBOLD ": llamafile " LLAMAFILE_VERSION_STRING "\n" //
- BOLD "model" UNBOLD ": %s\n",
- lf::basename(g_params.model).c_str());
- printf(BOLD "compute" UNBOLD ": %s\n", describe_compute().c_str());
- if (want_server)
- printf(BOLD "server" UNBOLD ": %s\n", g_listen_url.c_str());
- printf("\n");
- }
-
- print_ephemeral("initializing context...");
- llama_context_params ctx_params = llama_context_params_from_gpt_params(g_params);
- g_ctx = llama_new_context_with_model(g_model, ctx_params);
- clear_ephemeral();
- if (!g_ctx) {
- fprintf(stderr, "error: failed to initialize context\n");
- exit(3);
- }
-
- if (llama_model_has_encoder(g_model))
- fprintf(stderr, "warning: this model has an encoder\n");
-
- if (FLAG_mmproj) {
- print_ephemeral("initializing vision model...");
- g_clip = clip_model_load(FLAG_mmproj, g_params.verbosity);
- clear_ephemeral();
- if (!g_clip) {
- fprintf(stderr, "%s: failed to initialize clip image model\n", FLAG_mmproj);
- exit(4);
- }
- }
-
- // setup conversation
- if (llama_should_add_bos_token(g_model))
- if (!eval_token(llama_token_bos(g_model)))
- exit(6);
- record_undo();
-
- // setup system prompt
- std::vector<llama_chat_msg> chat = {{"system", g_params.prompt}};
- std::string msg =
- llama_chat_apply_template(g_model, g_params.chat_template, chat, DONT_ADD_ASSISTANT);
- if (!eval_string(msg, DONT_ADD_SPECIAL, PARSE_SPECIAL))
- exit(6);
- llama_synchronize(g_ctx);
- g_system_prompt_tokens = tokens_used();
- clear_ephemeral();
- if (g_params.display_prompt)
- printf("%s\n", g_params.special ? msg.c_str() : g_params.prompt.c_str());
-
- // perform important setup
- HighlightMarkdown highlighter;
- ColorBleeder bleeder(&highlighter);
- struct llama_sampling_context *sampler = llama_sampling_init(g_params.sparams);
- signal(SIGINT, on_sigint);
-
- // run chatbot
- for (;;) {
- record_undo();
- bestlineLlamaMode(true);
- bestlineSetHintsCallback(on_hint);
- bestlineSetFreeHintsCallback(free);
- bestlineSetCompletionCallback(on_completion);
- write(1, get_role_color(g_role), strlen(get_role_color(g_role)));
- char *line = bestlineWithHistory(">>> ", "llamafile");
- write(1, UNFOREGROUND, strlen(UNFOREGROUND));
- if (!line) {
- if (g_got_sigint)
- ensure_newline();
- break;
- }
- if (is_empty(line)) {
- if (g_manual_mode) {
- g_role = cycle_role(g_role);
- write(1, "\033[F", 3);
- }
- free(line);
- continue;
- }
- g_said_something = true;
- if (handle_command(line)) {
- free(line);
- continue;
- }
- bool add_assi = !g_manual_mode;
- int tokens_used_before = tokens_used();
- std::vector<llama_chat_msg> chat = {{get_role_name(g_role), line}};
- std::string msg =
- llama_chat_apply_template(g_model, g_params.chat_template, chat, add_assi);
- if (!eval_string(msg, DONT_ADD_SPECIAL, PARSE_SPECIAL)) {
- rewind(tokens_used_before);
- continue;
- }
- if (g_manual_mode) {
- g_role = get_next_role(g_role);
- free(line);
- continue;
- }
- for (;;) {
- if (g_got_sigint) {
- eval_token(llamafile_token_eot(g_model));
- break;
- }
- llama_token id = llama_sampling_sample(sampler, g_ctx, NULL);
- llama_sampling_accept(sampler, g_ctx, id, APPLY_GRAMMAR);
- if (!eval_token(id))
- break;
- if (llama_token_is_eog(g_model, id))
- break;
- std::string s;
- bleeder.feed(&s, token_to_piece(g_ctx, id, g_params.special));
- print(s);
- fflush(stdout);
- }
- g_got_sigint = 0;
- free(line);
- std::string s;
- bleeder.flush(&s);
- print(s);
- ensure_newline();
- }
-
- if (g_clip) {
- print_ephemeral("freeing vision model...");
- clip_free(g_clip);
- clear_ephemeral();
- }
-
- print_ephemeral("freeing context...");
- llama_sampling_free(sampler);
- llama_free(g_ctx);
- clear_ephemeral();
-
- print_ephemeral("freeing model...");
- llama_free_model(g_model);
- clear_ephemeral();
-
- print_ephemeral("freeing backend...");
- llama_backend_free();
- clear_ephemeral();
-
- return 0;
-}
diff --git a/llamafile/chatbot.h b/llamafile/chatbot.h
index c92b35a9..d61bbee3 100644
--- a/llamafile/chatbot.h
+++ b/llamafile/chatbot.h
@@ -17,23 +17,78 @@
#pragma once
#include <__fwd/string.h>
+#include <__fwd/string_view.h>
#include <__fwd/vector.h>
+#include <signal.h>
-#define RESET "\e[0m"
-#define BOLD "\e[1m"
-#define FAINT "\e[2m"
-#define UNBOLD "\e[22m"
-#define RED "\e[31m"
-#define GREEN "\e[32m"
-#define MAGENTA "\e[35m"
-#define YELLOW "\e[33m"
-#define CYAN "\e[36m"
-#define UNFOREGROUND "\e[39m"
-#define BRIGHT_BLACK "\e[90m"
-#define BRIGHT_RED "\e[91m"
-#define BRIGHT_GREEN "\e[92m"
-#define CLEAR_FORWARD "\e[K"
-
-int chatbot_main(int, char **);
-void chatbot_help(const std::vector<std::string> &);
-void chatbot_logo(char **);
+struct bestlineCompletions;
+struct clip_ctx;
+struct gpt_params;
+struct llama_context;
+struct llama_model;
+
+namespace lf {
+namespace chatbot {
+
+enum Role {
+ ROLE_USER,
+ ROLE_ASSISTANT,
+ ROLE_SYSTEM,
+};
+
+enum SpecialToken {
+ IMAGE_PLACEHOLDER_TOKEN = -31337,
+};
+
+extern bool g_manual_mode;
+extern bool g_said_something;
+extern char g_last_printed_char;
+extern clip_ctx *g_clip;
+extern enum Role g_role;
+extern gpt_params g_params;
+extern int g_system_prompt_tokens;
+extern llama_context *g_ctx;
+extern llama_model *g_model;
+extern std::vector<int> g_history;
+extern volatile sig_atomic_t g_got_sigint;
+
+int main(int, char **);
+
+bool eval_string(std::string_view, bool, bool);
+bool eval_token(int);
+bool eval_tokens(std::vector<int>);
+bool handle_command(const char *);
+bool out_of_context(int);
+char *on_hint(const char *, const char **, const char **);
+const char *get_role_color(enum Role);
+const char *get_role_name(enum Role);
+enum Role cycle_role(enum Role);
+enum Role get_next_role(enum Role);
+int tokens_used(void);
+std::string token_to_piece(const llama_context *, int, bool);
+void adjust_stacks(int, int);
+void clear_ephemeral(void);
+void ensure_newline();
+void err(const char *, ...);
+void fix_stacks(void);
+void logo(char **);
+void on_clear(const std::vector<std::string> &);
+void on_completion(const char *, int, bestlineCompletions *);
+void on_context(const std::vector<std::string> &);
+void on_dump(const std::vector<std::string> &);
+void on_forget(const std::vector<std::string> &);
+void on_help(const std::vector<std::string> &);
+void on_manual(const std::vector<std::string> &);
+void on_pop(const std::vector<std::string> &);
+void on_push(const std::vector<std::string> &);
+void on_stack(const std::vector<std::string> &);
+void on_undo(const std::vector<std::string> &);
+void on_upload(const std::vector<std::string> &);
+void print(const std::string_view &);
+void print_ephemeral(const std::string_view &);
+void record_undo(void);
+void repl();
+void rewind(int);
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_comm.cpp b/llamafile/chatbot_comm.cpp
new file mode 100644
index 00000000..b3f186c5
--- /dev/null
+++ b/llamafile/chatbot_comm.cpp
@@ -0,0 +1,83 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <cassert>
+#include <sstream>
+#include <vector>
+
+#include "llama.cpp/llama.h"
+#include "llamafile/llamafile.h"
+
+namespace lf {
+namespace chatbot {
+
+void on_stats(const std::vector<std::string> &args) {
+ FLAG_log_disable = false;
+ llama_print_timings(g_ctx);
+ FLAG_log_disable = true;
+}
+
+// handle irc style commands like: `/arg0 arg1 arg2`
+bool handle_command(const char *command) {
+ if (!strcmp(command, "/?")) {
+ const std::vector<std::string> args = {"?"};
+ on_help(args);
+ return true;
+ }
+ if (!(command[0] == '/' && std::isalpha(command[1])))
+ return false;
+ std::vector<std::string> args;
+ std::istringstream iss(command + 1);
+ std::string arg;
+ while (iss >> arg)
+ args.push_back(arg);
+ if (args[0] == "exit" || args[0] == "bye") {
+ exit(0);
+ } else if (args[0] == "help") {
+ on_help(args);
+ } else if (args[0] == "stats") {
+ on_stats(args);
+ } else if (args[0] == "context") {
+ on_context(args);
+ } else if (args[0] == "manual") {
+ on_manual(args);
+ } else if (args[0] == "clear") {
+ on_clear(args);
+ } else if (args[0] == "dump") {
+ on_dump(args);
+ } else if (args[0] == "push") {
+ on_push(args);
+ } else if (args[0] == "pop") {
+ on_pop(args);
+ } else if (args[0] == "undo") {
+ on_undo(args);
+ } else if (args[0] == "forget") {
+ on_forget(args);
+ } else if (args[0] == "stack") {
+ on_stack(args);
+ } else if (args[0] == "upload") {
+ on_upload(args);
+ } else {
+ err("%s: unrecognized command", args[0].c_str());
+ }
+ return true;
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_comp.cpp b/llamafile/chatbot_comp.cpp
new file mode 100644
index 00000000..e3aa81b6
--- /dev/null
+++ b/llamafile/chatbot_comp.cpp
@@ -0,0 +1,72 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <glob.h>
+#include <string>
+#include <sys/stat.h>
+
+#include "llamafile/bestline.h"
+
+namespace lf {
+namespace chatbot {
+
+static bool is_directory(const char *path) {
+ struct stat st;
+ return !stat(path, &st) && S_ISDIR(st.st_mode);
+}
+
+void on_completion(const char *line, int pos, bestlineCompletions *comp) {
+ if (startswith(line, "/upload ")) {
+ std::string pattern(line + strlen("/upload "));
+ pattern += '*';
+ glob_t gl;
+ if (!glob(pattern.c_str(), GLOB_TILDE, 0, &gl)) {
+ for (size_t i = 0; i < gl.gl_pathc; ++i) {
+ std::string completion = "/upload ";
+ completion += gl.gl_pathv[i];
+ if (is_directory(gl.gl_pathv[i]))
+ completion += '/';
+ bestlineAddCompletion(comp, completion.c_str());
+ }
+ globfree(&gl);
+ }
+ } else {
+ static const char *const kCompletions[] = {
+ "/clear", // usage: /clear
+ "/context", // usage: /context
+ "/dump", // usage: /dump [FILE]
+ "/exit", // usage: /exit
+ "/forget", // usage: /forget
+ "/help", // usage: /help [COMMAND]
+ "/manual", // usage: /manual [on|off]
+ "/pop", // usage: /pop
+ "/push", // usage: /push
+ "/stack", // usage: /stack
+ "/stats", // usage: /stats
+ "/undo", // usage: /undo
+ "/upload", // usage: /upload FILE
+ };
+ for (int i = 0; i < sizeof(kCompletions) / sizeof(*kCompletions); ++i)
+ if (startswith(kCompletions[i], line))
+ bestlineAddCompletion(comp, kCompletions[i]);
+ }
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_eval.cpp b/llamafile/chatbot_eval.cpp
new file mode 100644
index 00000000..49f8b5f0
--- /dev/null
+++ b/llamafile/chatbot_eval.cpp
@@ -0,0 +1,146 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <cassert>
+#include <string>
+#include <vector>
+
+#include "llama.cpp/common.h"
+#include "llama.cpp/llama.h"
+#include "llama.cpp/llava/llava.h"
+#include "llamafile/datauri.h"
+#include "llamafile/image.h"
+#include "llamafile/string.h"
+
+namespace lf {
+namespace chatbot {
+
+bool eval_tokens(std::vector<llama_token> tokens) {
+ int N = (int)tokens.size();
+ if (tokens_used() + N > llama_n_ctx(g_ctx))
+ return out_of_context(N);
+ for (int i = 0; i < N; i += g_params.n_batch) {
+ if (g_got_sigint) {
+ g_got_sigint = false;
+ clear_ephemeral();
+ return false;
+ }
+ if (N > g_params.n_batch)
+ print_ephemeral(format("loading prompt %d%%...", (int)((double)i / N * 100)));
+ int n_eval = (int)tokens.size() - i;
+ if (n_eval > g_params.n_batch)
+ n_eval = g_params.n_batch;
+ if (llama_decode(g_ctx, llama_batch_get_one(&tokens[i], n_eval, tokens_used(), 0)))
+ return out_of_context(n_eval);
+ g_history.insert(g_history.end(), tokens.begin() + i, tokens.begin() + i + n_eval);
+ }
+ clear_ephemeral();
+ // this function is what computes /stats. we need to call it now
+ // since llama_decode() kicks the can down the road to functions
+ // like llama_sampling_sample(). that is bad because the chatbot
+ // returns control to the repl rather than sampling when loading
+ // system and image prompts.
+ llama_synchronize(g_ctx);
+ return true;
+}
+
+bool eval_image_embed(const struct llava_image_embed *image_embed) {
+ int N = image_embed->n_image_pos;
+ if (tokens_used() + N > llama_n_ctx(g_ctx))
+ return out_of_context(N);
+ int n_embd = llama_n_embd(llama_get_model(g_ctx));
+ for (int i = 0; i < N; i += g_params.n_batch) {
+ if (g_got_sigint) {
+ g_got_sigint = false;
+ clear_ephemeral();
+ return false;
+ }
+ if (N > g_params.n_batch)
+ print_ephemeral(format("loading image %d%%...", (int)((double)i / N * 100)));
+ int n_eval = N - i;
+ if (n_eval > g_params.n_batch)
+ n_eval = g_params.n_batch;
+ llama_batch batch = {
+ .n_tokens = n_eval,
+ .embd = image_embed->embed + i * n_embd,
+ .all_pos_0 = tokens_used(),
+ .all_pos_1 = 1,
+ .all_seq_id = 0,
+ };
+ if (llama_decode(g_ctx, batch))
+ return out_of_context(n_eval);
+ for (int i = 0; i < n_eval; ++i)
+ g_history.push_back(IMAGE_PLACEHOLDER_TOKEN);
+ }
+ clear_ephemeral();
+ return true;
+}
+
+bool eval_image(const std::string_view binary) {
+ unassert(g_clip);
+ llava_image_embed *image_embed;
+ print_ephemeral("analyzing image...");
+ image_embed = llava_image_embed_make_with_bytes(
+ g_clip, FLAG_threads_batch, (const unsigned char *)binary.data(), binary.size());
+ clear_ephemeral();
+ if (!image_embed) {
+ err("failed to load image");
+ return false;
+ }
+ bool ok = eval_image_embed(image_embed);
+ llava_image_embed_free(image_embed);
+ return ok;
+}
+
+bool eval_token(int id) {
+ return eval_tokens({id});
+}
+
+bool eval_plain_text(const std::string &str, bool add_special, bool parse_special) {
+ return eval_tokens(llama_tokenize(g_model, str, add_special, parse_special));
+}
+
+bool eval_string(std::string_view s, bool add_special, bool parse_special) {
+ size_t i = 0;
+ for (;;) {
+ size_t pos = s.find("data:", i);
+ if (pos == std::string_view::npos)
+ return eval_plain_text(std::string(s), add_special, parse_special);
+ DataUri uri;
+ size_t end = uri.parse(s.substr(pos + 5));
+ if (end == std::string_view::npos) {
+ i = pos + 5;
+ continue;
+ }
+ std::string image = uri.decode();
+ if (!is_image(image)) {
+ i = pos + 5;
+ continue;
+ }
+ if (!eval_plain_text(std::string(s.substr(0, pos)), add_special, parse_special))
+ return false;
+ if (!eval_image(image))
+ return false;
+ s = s.substr(pos + 5 + end);
+ i = 0;
+ }
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_file.cpp b/llamafile/chatbot_file.cpp
new file mode 100644
index 00000000..eedf2b28
--- /dev/null
+++ b/llamafile/chatbot_file.cpp
@@ -0,0 +1,97 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <string>
+#include <sys/stat.h>
+#include <vector>
+
+#include "llama.cpp/common.h"
+#include "llamafile/color.h"
+#include "llamafile/image.h"
+#include "llamafile/llama.h"
+#include "llamafile/string.h"
+
+namespace lf {
+namespace chatbot {
+
+static bool has_binary(const std::string_view s) {
+ return s.find('\0') != std::string_view::npos;
+}
+
+void on_upload(const std::vector<std::string> &args) {
+ if (args.size() < 2) {
+ err("error: missing file path" RESET "\n"
+ "usage: /upload PATH");
+ return;
+ }
+ if (args.size() > 2) {
+ err("error: too many arguments" RESET "\n"
+ "usage: /upload PATH");
+ return;
+ }
+ const char *path = args[1].c_str();
+ struct stat st;
+ if (stat(path, &st) || !S_ISREG(st.st_mode)) {
+ err("%s: file does not exist", path);
+ return;
+ }
+ int tokens_used_before = tokens_used();
+ std::string content;
+ if (!slurp(&content, path)) {
+ err("%s: failed to slurp file", path);
+ return;
+ }
+ std::string markdown;
+ markdown += "- **Filename**: `";
+ markdown += path;
+ markdown += "`\n- **Last modified**: ";
+ markdown += iso8601(st.st_mtim);
+ markdown += "\n\n";
+ if (is_image(content)) {
+ if (!g_clip) {
+ err("%s: need --mmproj model to process images", path);
+ return;
+ }
+ print_image(1, content, 80);
+ convert_image_to_uri(&markdown, content);
+ } else {
+ if (has_binary(content)) {
+ err("%s: binary file type not supported", path);
+ return;
+ }
+ markdown += "``````";
+ markdown += extname(path);
+ markdown += '\n';
+ markdown += content;
+ if (markdown.back() != '\n')
+ markdown += '\n';
+ markdown += "``````";
+ }
+ std::vector<llama_chat_msg> chat = {{"system", std::move(markdown)}};
+ if (!eval_string(
+ llama_chat_apply_template(g_model, g_params.chat_template, chat, DONT_ADD_ASSISTANT),
+ DONT_ADD_SPECIAL, PARSE_SPECIAL)) {
+ rewind(tokens_used_before);
+ return;
+ }
+ llama_synchronize(g_ctx);
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_help.cpp b/llamafile/chatbot_help.cpp
index 721e37d0..258a1274 100644
--- a/llamafile/chatbot_help.cpp
+++ b/llamafile/chatbot_help.cpp
@@ -19,7 +19,12 @@
#include <vector>
-void chatbot_help(const std::vector<std::string> &args) {
+#include "llamafile/color.h"
+
+namespace lf {
+namespace chatbot {
+
+void on_help(const std::vector<std::string> &args) {
if (args.size() == 1) {
fprintf(stderr, "\
" BOLD "available commands" RESET "\n\
@@ -145,3 +150,6 @@ command which deletes the most recent chat message instead.\n\
fprintf(stderr, BRIGHT_RED "%s: unknown command" RESET "\n", args[1].c_str());
}
}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_hint.cpp b/llamafile/chatbot_hint.cpp
new file mode 100644
index 00000000..88e29df0
--- /dev/null
+++ b/llamafile/chatbot_hint.cpp
@@ -0,0 +1,76 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <cstring>
+
+#include "llamafile/color.h"
+
+namespace lf {
+namespace chatbot {
+
+static const char *on_hint_impl(const char *line) {
+ if (!*line && g_manual_mode)
+ return get_role_name(g_role);
+ if (!*line && !g_manual_mode && !g_said_something)
+ return "say something (or type /help for help)";
+ static const char *const kHints[] = {
+ "/clear", //
+ "/context", //
+ "/dump", //
+ "/exit", //
+ "/forget", //
+ "/help", //
+ "/manual", //
+ "/pop", //
+ "/push", //
+ "/stack", //
+ "/stats", //
+ "/undo", //
+ "/upload", //
+ };
+ int z = strlen(line);
+ int n = sizeof(kHints) / sizeof(kHints[0]);
+ int l = 0;
+ int r = n - 1;
+ int i = -1;
+ while (l <= r) {
+ int m = (l & r) + ((l ^ r) >> 1); // floor((a+b)/2)
+ int c = strncmp(line, kHints[m], z);
+ if (!c) {
+ i = m;
+ r = m - 1;
+ } else if (c < 0) {
+ r = m - 1;
+ } else {
+ l = m + 1;
+ }
+ }
+ if (i == -1 || (i + 1 < n && !strncmp(line, kHints[i + 1], z)))
+ return "";
+ return kHints[i] + z;
+}
+
+char *on_hint(const char *line, const char **ansi1, const char **ansi2) {
+ *ansi1 = FAINT;
+ *ansi2 = UNBOLD;
+ return strdup(on_hint_impl(line));
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_hist.cpp b/llamafile/chatbot_hist.cpp
new file mode 100644
index 00000000..7bca4d3d
--- /dev/null
+++ b/llamafile/chatbot_hist.cpp
@@ -0,0 +1,301 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <cassert>
+#include <vector>
+
+#include "llama.cpp/llama.h"
+#include "llamafile/color.h"
+#include "llamafile/llama.h"
+#include "llamafile/macros.h"
+#include "llamafile/string.h"
+
+namespace lf {
+namespace chatbot {
+
+bool g_manual_mode;
+enum Role g_role = ROLE_USER;
+int g_system_prompt_tokens;
+std::vector<llama_pos> g_stack;
+std::vector<llama_pos> g_undo;
+std::vector<llama_token> g_history;
+
+const char *get_role_name(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return "user";
+ case ROLE_ASSISTANT:
+ return "assistant";
+ case ROLE_SYSTEM:
+ return "system";
+ default:
+ __builtin_unreachable();
+ }
+}
+
+const char *get_role_color(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return GREEN;
+ case ROLE_ASSISTANT:
+ return MAGENTA;
+ case ROLE_SYSTEM:
+ return YELLOW;
+ default:
+ __builtin_unreachable();
+ }
+}
+
+enum Role get_next_role(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return ROLE_ASSISTANT;
+ case ROLE_ASSISTANT:
+ return ROLE_USER;
+ case ROLE_SYSTEM:
+ return ROLE_USER;
+ default:
+ __builtin_unreachable();
+ }
+}
+
+enum Role cycle_role(enum Role role) {
+ switch (role) {
+ case ROLE_USER:
+ return ROLE_ASSISTANT;
+ case ROLE_ASSISTANT:
+ return ROLE_SYSTEM;
+ case ROLE_SYSTEM:
+ return ROLE_USER;
+ default:
+ __builtin_unreachable();
+ }
+}
+
+int tokens_used(void) {
+ return g_history.size();
+}
+
+std::string describe_token(llama_token token) {
+ if (token == llama_token_bos(g_model))
+ return "§";
+ if (token == llama_token_eos(g_model))
+ return "∎";
+ if (token == llama_token_cls(g_model))
+ return "⌘";
+ if (token == llama_token_sep(g_model))
+ return "⋯";
+ if (token == llama_token_pad(g_model))
+ return "␣";
+ if (token == llama_token_nl(g_model))
+ return "↵";
+ if (llama_token_is_eog(g_model, token))
+ return "⌟";
+ if (llama_token_is_control(g_model, token))
+ return "∷";
+ std::string s = token_to_piece(g_ctx, token, DONT_RENDER_SPECIAL_TOKENS);
+ if (s.empty())
+ return "↯";
+ return s;
+}
+
+std::string describe_erasure(llama_pos begin, llama_pos end) {
+ unassert(begin <= end);
+ unassert(end <= tokens_used());
+ std::string description;
+ llama_pos pos = begin;
+ while (pos < end && description.size() < 63)
+ description += describe_token(g_history[pos++]);
+ if (!description.empty() && pos < end)
+ description += " ...";
+ description = collapse(description);
+ if (pos == end && description.empty())
+ description = "<absolute end>";
+ return description;
+}
+
+std::string describe_position(llama_pos pos) {
+ unassert(pos <= tokens_used());
+ std::string description;
+ while (pos > 0 && description.size() < 63)
+ description = describe_token(g_history[--pos]) + description;
+ if (!description.empty() && pos > 0)
+ description = std::string("... ") + description;
+ description = collapse(description);
+ if (!pos && description.empty())
+ description = "<absolute beginning>";
+ return description;
+}
+
+static void fix_stack(std::vector<llama_pos> *stack) {
+ while (!stack->empty() && stack->back() > tokens_used())
+ stack->pop_back();
+}
+
+void fix_stacks(void) {
+ fix_stack(&g_undo);
+ fix_stack(&g_stack);
+ g_system_prompt_tokens = MIN(g_system_prompt_tokens, tokens_used());
+}
+
+static std::vector<llama_pos> adjust_stack(llama_pos erase_begin, llama_pos erase_end,
+ const std::vector<llama_pos> &stack) {
+ std::vector<llama_pos> builder;
+ for (llama_pos pos : stack) {
+ if (erase_begin <= pos && pos < erase_end)
+ continue;
+ if (pos >= erase_end)
+ pos -= erase_end - erase_begin;
+ builder.push_back(pos);
+ }
+ return builder;
+}
+
+void adjust_stacks(llama_pos erase_begin, llama_pos erase_end) {
+ g_undo = adjust_stack(erase_begin, erase_end, g_undo);
+ g_stack = adjust_stack(erase_begin, erase_end, g_stack);
+}
+
+void record_undo(void) {
+ if (g_undo.empty() || g_undo.back() != tokens_used())
+ g_undo.push_back(tokens_used());
+}
+
+void on_undo(const std::vector<std::string> &args) {
+ while (!g_undo.empty() && g_undo.back() == tokens_used())
+ g_undo.pop_back();
+ if (g_undo.empty()) {
+ err("error: no further undo actions possible");
+ return;
+ }
+ printf(FAINT "restoring conversation to: %s" RESET "\n",
+ describe_position(g_undo.back()).c_str());
+ rewind(g_undo.back());
+ g_undo.pop_back();
+ fix_stacks();
+}
+
+void on_forget(const std::vector<std::string> &args) {
+ if (g_undo.size() < 2) {
+ err("error: nothing left to forget");
+ return;
+ }
+ int erase_count;
+ llama_pos erase_begin = g_undo[1];
+ llama_pos erase_end = g_undo.size() > 2 ? g_undo[2] : tokens_used();
+ if (!(erase_count = erase_end - erase_begin)) {
+ err("error: nothing left to forget");
+ return;
+ }
+ printf(FAINT "forgetting: %s" RESET "\n", describe_erasure(erase_begin, erase_end).c_str());
+ llama_kv_cache_seq_rm(g_ctx, 0, erase_begin, erase_end);
+ llama_kv_cache_seq_add(g_ctx, 0, erase_end, -1, -erase_count);
+ g_history.erase(g_history.begin() + erase_begin, //
+ g_history.begin() + erase_end);
+ adjust_stacks(erase_begin, erase_end);
+ fix_stacks();
+}
+
+void rewind(int pos) {
+ unassert(pos <= tokens_used());
+ llama_kv_cache_seq_rm(g_ctx, 0, pos, -1);
+ g_history.resize(pos);
+}
+
+void on_manual(const std::vector<std::string> &args) {
+ if (args.size() == 1) {
+ g_manual_mode = !g_manual_mode;
+ } else if (args.size() == 2 && (args[1] == "on" || args[1] == "off")) {
+ g_manual_mode = args[1] == "on";
+ } else {
+ err("error: bad /manual command\n"
+ "usage: /manual [on|off]");
+ return;
+ }
+ fprintf(stderr, FAINT "manual mode %s" RESET "\n", g_manual_mode ? "enabled" : "disabled");
+ if (!g_manual_mode)
+ g_role = ROLE_USER;
+}
+
+void on_context(const std::vector<std::string> &args) {
+ int configured_context = llama_n_ctx(g_ctx);
+ int max_context = llama_n_ctx_train(g_model);
+ printf("%d out of %d context tokens used (%d tokens remaining)\n", tokens_used(),
+ configured_context, configured_context - tokens_used());
+ if (configured_context < max_context)
+ printf("use the `-c %d` flag at startup for maximum context\n", max_context);
+}
+
+void on_clear(const std::vector<std::string> &args) {
+ rewind(g_system_prompt_tokens);
+ g_stack.clear();
+ fix_stacks();
+}
+
+void print_stack(void) {
+ for (size_t i = g_stack.size(); i--;)
+ printf("%12d " FAINT "(%s)" RESET "\n", g_stack[i], describe_position(g_stack[i]).c_str());
+}
+
+void on_push(const std::vector<std::string> &args) {
+ g_stack.push_back(tokens_used());
+ print_stack();
+}
+
+void on_pop(const std::vector<std::string> &args) {
+ if (g_stack.empty()) {
+ err("error: context length stack is empty");
+ return;
+ }
+ printf(BOLD "%12d" RESET " restored " FAINT "(%s)" RESET "\n", g_stack.back(),
+ describe_position(g_stack.back()).c_str());
+ rewind(g_stack.back());
+ g_stack.pop_back();
+ fix_stacks();
+ print_stack();
+}
+
+void on_stack(const std::vector<std::string> &args) {
+ if (g_stack.empty()) {
+ printf(FAINT "stack is currently empty (try using /push)" RESET "\n");
+ return;
+ }
+ print_stack();
+}
+
+void on_dump(const std::vector<std::string> &args) {
+ int fd = 1;
+ if (args.size() >= 2) {
+ if ((fd = creat(args[1].c_str(), 0644)) == -1) {
+ perror(args[1].c_str());
+ return;
+ }
+ }
+ std::string s;
+ for (auto id : g_history)
+ s += token_to_piece(g_ctx, id, RENDER_SPECIAL_TOKENS);
+ if (!s.empty() && s[s.size() - 1] != '\n')
+ s += '\n';
+ write(fd, s.data(), s.size());
+ if (args.size() >= 2)
+ close(fd);
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_logo.cpp b/llamafile/chatbot_logo.cpp
index 0c5ae382..e3abe8c2 100644
--- a/llamafile/chatbot_logo.cpp
+++ b/llamafile/chatbot_logo.cpp
@@ -17,9 +17,13 @@
#include "chatbot.h"
-#include <stdio.h>
+#include <cstdio>
-#include "llamafile.h"
+#include "llamafile/color.h"
+#include "llamafile/llamafile.h"
+
+namespace lf {
+namespace chatbot {
static void print_logo(const char16_t *s) {
for (int i = 0; s[i]; ++i) {
@@ -42,7 +46,7 @@ static void print_logo(const char16_t *s) {
}
}
-void chatbot_logo(char **argv) {
+void logo(char **argv) {
if (llamafile_has(argv, "--nologo"))
return;
if (llamafile_has(argv, "--ascii")) {
@@ -62,3 +66,6 @@ void chatbot_logo(char **argv) {
╚══════╝╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚══════╝\n");
}
}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cpp
new file mode 100644
index 00000000..61e1c1e9
--- /dev/null
+++ b/llamafile/chatbot_main.cpp
@@ -0,0 +1,207 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <cosmo.h>
+#include <cstdio>
+#include <signal.h>
+#include <string>
+#include <vector>
+
+#include "llama.cpp/common.h"
+#include "llama.cpp/ggml-cuda.h"
+#include "llama.cpp/llava/clip.h"
+#include "llama.cpp/server/server.h"
+#include "llamafile/color.h"
+#include "llamafile/compute.h"
+#include "llamafile/string.h"
+
+namespace lf {
+namespace chatbot {
+
+struct ServerArgs {
+ int argc;
+ char **argv;
+};
+
+gpt_params g_params;
+clip_ctx *g_clip;
+llama_model *g_model;
+llama_context *g_ctx;
+pthread_cond_t g_cond = PTHREAD_COND_INITIALIZER;
+pthread_mutex_t g_lock = PTHREAD_MUTEX_INITIALIZER;
+std::string g_listen_url;
+
+std::string describe_compute(void) {
+ if (llama_n_gpu_layers(g_model) > 0 && llamafile_has_gpu()) {
+ if (llamafile_has_metal()) {
+ return "Apple Metal GPU";
+ } else {
+ std::vector<std::string> vec;
+ int count = ggml_backend_cuda_get_device_count();
+ for (int i = 0; i < count; i++) {
+ char buf[128];
+ ggml_backend_cuda_get_device_description(i, buf, sizeof(buf));
+ vec.emplace_back(buf);
+ }
+ return join(vec, ", ");
+ }
+ } else {
+ return llamafile_describe_cpu();
+ }
+}
+
+std::string token_to_piece(const struct llama_context *ctx, llama_token token, bool special) {
+ if (token == IMAGE_PLACEHOLDER_TOKEN)
+ return "⁑";
+ return llama_token_to_piece(ctx, token, special);
+}
+
+void on_server_listening(const char *host, int port) {
+ pthread_mutex_lock(&g_lock);
+ g_listen_url = format("http://%s:%d/", host, port);
+ pthread_cond_signal(&g_cond);
+ pthread_mutex_unlock(&g_lock);
+}
+
+void *server_thread(void *arg) {
+ ServerArgs *sargs = (ServerArgs *)arg;
+ server_log_json = false;
+ g_server_background_mode = true;
+ g_server_force_llama_model = g_model;
+ g_server_on_listening = on_server_listening;
+ exit(server_cli(sargs->argc, sargs->argv));
+}
+
+int main(int argc, char **argv) {
+
+ // print logo
+ logo(argv);
+
+ // disable llamafile gpu initialization log messages
+ if (!llamafile_has(argv, "--verbose"))
+ FLAG_log_disable = true;
+
+ // override defaults for some flags
+ g_params.n_batch = 256; // for better progress indication
+ g_params.sparams.temp = 0; // don't believe in randomness by default
+ g_params.prompt = "A chat between a curious human and an artificial intelligence assistant. "
+ "The assistant gives helpful, detailed, and polite answers to the "
+ "human's questions.";
+
+ // parse flags (sadly initializes gpu support as side-effect)
+ print_ephemeral("loading backend...");
+ llama_backend_init();
+ if (!gpt_params_parse(argc, argv, g_params)) { // also loads gpu module
+ fprintf(stderr, "error: failed to parse flags\n");
+ exit(1);
+ }
+ clear_ephemeral();
+
+ // setup logging
+ FLAG_log_disable = false;
+ if (!g_params.verbosity)
+ log_disable();
+
+ print_ephemeral("loading model...");
+ llama_model_params model_params = llama_model_params_from_gpt_params(g_params);
+ g_model = llama_load_model_from_file(g_params.model.c_str(), model_params);
+ if (g_model == NULL) {
+ clear_ephemeral();
+ fprintf(stderr, "%s: failed to load model\n", g_params.model.c_str());
+ exit(2);
+ }
+ if (g_params.n_ctx <= 0 || g_params.n_ctx > llama_n_ctx_train(g_model))
+ g_params.n_ctx = llama_n_ctx_train(g_model);
+ if (g_params.n_ctx < g_params.n_batch)
+ g_params.n_batch = g_params.n_ctx;
+ clear_ephemeral();
+
+ bool want_server = !llamafile_has(argv, "--chat");
+ if (want_server) {
+ print_ephemeral("launching server...");
+ pthread_t thread;
+ pthread_attr_t attr;
+ ServerArgs sargs = {argc, argv};
+ pthread_mutex_lock(&g_lock);
+ pthread_attr_init(&attr);
+ pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
+ pthread_create(&thread, &attr, server_thread, &sargs);
+ pthread_attr_destroy(&attr);
+ pthread_cond_wait(&g_cond, &g_lock);
+ pthread_mutex_unlock(&g_lock);
+ clear_ephemeral();
+ }
+
+ if (!FLAG_nologo) {
+ printf(BOLD "software" UNBOLD ": llamafile " LLAMAFILE_VERSION_STRING "\n" //
+ BOLD "model" UNBOLD ": %s\n",
+ basename(g_params.model).c_str());
+ printf(BOLD "compute" UNBOLD ": %s\n", describe_compute().c_str());
+ if (want_server)
+ printf(BOLD "server" UNBOLD ": %s\n", g_listen_url.c_str());
+ printf("\n");
+ }
+
+ print_ephemeral("initializing context...");
+ llama_context_params ctx_params = llama_context_params_from_gpt_params(g_params);
+ g_ctx = llama_new_context_with_model(g_model, ctx_params);
+ clear_ephemeral();
+ if (!g_ctx) {
+ fprintf(stderr, "error: failed to initialize context\n");
+ exit(3);
+ }
+
+ if (llama_model_has_encoder(g_model))
+ fprintf(stderr, "warning: this model has an encoder\n");
+
+ if (FLAG_mmproj) {
+ print_ephemeral("initializing vision model...");
+ g_clip = clip_model_load(FLAG_mmproj, g_params.verbosity);
+ clear_ephemeral();
+ if (!g_clip) {
+ fprintf(stderr, "%s: failed to initialize clip image model\n", FLAG_mmproj);
+ exit(4);
+ }
+ }
+
+ repl();
+
+ if (g_clip) {
+ print_ephemeral("freeing vision model...");
+ clip_free(g_clip);
+ clear_ephemeral();
+ }
+
+ print_ephemeral("freeing context...");
+ llama_free(g_ctx);
+ clear_ephemeral();
+
+ print_ephemeral("freeing model...");
+ llama_free_model(g_model);
+ clear_ephemeral();
+
+ print_ephemeral("freeing backend...");
+ llama_backend_free();
+ clear_ephemeral();
+
+ return 0;
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/chatbot_repl.cpp b/llamafile/chatbot_repl.cpp
new file mode 100644
index 00000000..f42a7ed2
--- /dev/null
+++ b/llamafile/chatbot_repl.cpp
@@ -0,0 +1,194 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "chatbot.h"
+
+#include <cctype>
+#include <csignal>
+#include <cstdio>
+#include <string_view>
+
+#include "llama.cpp/common.h"
+#include "llama.cpp/llama.h"
+#include "llamafile/bestline.h"
+#include "llamafile/color.h"
+#include "llamafile/highlight.h"
+#include "llamafile/llama.h"
+
+namespace lf {
+namespace chatbot {
+
+bool g_has_ephemeral;
+bool g_said_something;
+char g_last_printed_char;
+volatile sig_atomic_t g_got_sigint;
+
+void on_sigint(int sig) {
+ g_got_sigint = 1;
+}
+
+bool is_empty(const char *s) {
+ int c;
+ while ((c = *s++))
+ if (!isspace(c))
+ return false;
+ return true;
+}
+
+void print(const std::string_view &s) {
+ for (char c : s) {
+ g_last_printed_char = c;
+ fputc(c, stdout);
+ if (c == '\n')
+ g_has_ephemeral = false;
+ }
+}
+
+void ensure_newline() {
+ if (g_last_printed_char != '\n')
+ print("\n");
+}
+
+void err(const char *fmt, ...) {
+ va_list ap;
+ clear_ephemeral();
+ ensure_newline();
+ va_start(ap, fmt);
+ fputs(BRIGHT_RED, stderr);
+ vfprintf(stderr, fmt, ap);
+ fputs(RESET "\n", stderr);
+ va_end(ap);
+}
+
+void print_ephemeral(const std::string_view &description) {
+ fprintf(stderr, " " BRIGHT_BLACK "%.*s" UNFOREGROUND "\r", (int)description.size(),
+ description.data());
+ g_has_ephemeral = true;
+}
+
+void clear_ephemeral(void) {
+ if (g_has_ephemeral) {
+ fprintf(stderr, CLEAR_FORWARD);
+ g_has_ephemeral = false;
+ }
+}
+
+bool out_of_context(int extra) {
+ err("error: ran out of context window at %d tokens\n"
+ "consider passing `-c %d` at startup for the maximum\n"
+ "you can free up more space using /forget or /clear",
+ tokens_used() + extra, llama_n_ctx_train(g_model));
+ return false;
+}
+
+void repl() {
+
+ // setup conversation
+ if (llama_should_add_bos_token(g_model))
+ eval_token(llama_token_bos(g_model));
+ record_undo();
+
+ // setup system prompt
+ std::vector<llama_chat_msg> chat = {{"system", g_params.prompt}};
+ std::string msg =
+ llama_chat_apply_template(g_model, g_params.chat_template, chat, DONT_ADD_ASSISTANT);
+ if (!eval_string(msg, DONT_ADD_SPECIAL, PARSE_SPECIAL))
+ exit(6);
+ llama_synchronize(g_ctx);
+ g_system_prompt_tokens = tokens_used();
+ clear_ephemeral();
+ if (g_params.display_prompt)
+ printf("%s\n", g_params.special ? msg.c_str() : g_params.prompt.c_str());
+
+ // perform important setup
+ HighlightMarkdown highlighter;
+ ColorBleeder bleeder(&highlighter);
+ llama_sampling_context *sampler = llama_sampling_init(g_params.sparams);
+ signal(SIGINT, on_sigint);
+
+ // run chatbot
+ for (;;) {
+ record_undo();
+ bestlineLlamaMode(true);
+ bestlineSetHintsCallback(on_hint);
+ bestlineSetFreeHintsCallback(free);
+ bestlineSetCompletionCallback(on_completion);
+ write(1, get_role_color(g_role), strlen(get_role_color(g_role)));
+ char *line = bestlineWithHistory(">>> ", "llamafile");
+ write(1, UNFOREGROUND, strlen(UNFOREGROUND));
+ if (!line) {
+ if (g_got_sigint)
+ ensure_newline();
+ break;
+ }
+ if (is_empty(line)) {
+ if (g_manual_mode) {
+ g_role = cycle_role(g_role);
+ write(1, "\033[F", 3);
+ }
+ free(line);
+ continue;
+ }
+ g_said_something = true;
+ if (handle_command(line)) {
+ free(line);
+ continue;
+ }
+ bool add_assi = !g_manual_mode;
+ int tokens_used_before = tokens_used();
+ std::vector<llama_chat_msg> chat = {{get_role_name(g_role), line}};
+ std::string msg =
+ llama_chat_apply_template(g_model, g_params.chat_template, chat, add_assi);
+ if (!eval_string(msg, DONT_ADD_SPECIAL, PARSE_SPECIAL)) {
+ rewind(tokens_used_before);
+ continue;
+ }
+ if (g_manual_mode) {
+ g_role = get_next_role(g_role);
+ free(line);
+ continue;
+ }
+ for (;;) {
+ if (g_got_sigint) {
+ eval_token(llamafile_token_eot(g_model));
+ break;
+ }
+ llama_token id = llama_sampling_sample(sampler, g_ctx, NULL);
+ llama_sampling_accept(sampler, g_ctx, id, APPLY_GRAMMAR);
+ if (!eval_token(id))
+ break;
+ if (llama_token_is_eog(g_model, id))
+ break;
+ std::string s;
+ bleeder.feed(&s, token_to_piece(g_ctx, id, g_params.special));
+ print(s);
+ fflush(stdout);
+ }
+ g_got_sigint = 0;
+ free(line);
+ std::string s;
+ bleeder.flush(&s);
+ print(s);
+ ensure_newline();
+ }
+
+ // cleanup resources
+ llama_sampling_free(sampler);
+}
+
+} // namespace chatbot
+} // namespace lf
diff --git a/llamafile/color.h b/llamafile/color.h
new file mode 100644
index 00000000..284e217f
--- /dev/null
+++ b/llamafile/color.h
@@ -0,0 +1,33 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#define RESET "\e[0m"
+#define BOLD "\e[1m"
+#define FAINT "\e[2m"
+#define UNBOLD "\e[22m"
+#define RED "\e[31m"
+#define GREEN "\e[32m"
+#define MAGENTA "\e[35m"
+#define YELLOW "\e[33m"
+#define CYAN "\e[36m"
+#define UNFOREGROUND "\e[39m"
+#define BRIGHT_BLACK "\e[90m"
+#define BRIGHT_RED "\e[91m"
+#define BRIGHT_GREEN "\e[92m"
+#define CLEAR_FORWARD "\e[K"