Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add vision model support to new server
Justine Tunneybb917bdbe71d
diff --git a/llamafile/chatbot_eval.cpp b/llamafile/chatbot_eval.cppindex 49f8b5f0..397f8cb4 100644--- a/llamafile/chatbot_eval.cpp+++ b/llamafile/chatbot_eval.cpp@@ -16,17 +16,17 @@// limitations under the License.#include "chatbot.h"--#include <cassert>-#include <string>-#include <vector>-+#include "llama.cpp/base64.h"#include "llama.cpp/common.h"#include "llama.cpp/llama.h"#include "llama.cpp/llava/llava.h"#include "llamafile/datauri.h"#include "llamafile/image.h"+#include "llamafile/llama.h"#include "llamafile/string.h"+#include <cassert>+#include <string>+#include <vector>namespace lf {namespace chatbot {@@ -60,7 +60,7 @@ bool eval_tokens(std::vector<llama_token> tokens) {return true;}-bool eval_image_embed(const struct llava_image_embed *image_embed) {+bool eval_image_embed(const llava_image_embed *image_embed) {int N = image_embed->n_image_pos;if (tokens_used() + N > llama_n_ctx(g_ctx))return out_of_context(N);@@ -113,7 +113,7 @@ bool eval_token(int id) {}bool eval_plain_text(const std::string &str, bool add_special, bool parse_special) {- return eval_tokens(llama_tokenize(g_model, str, add_special, parse_special));+ return eval_tokens(llamafile_tokenize(g_model, str, add_special, parse_special));}bool eval_string(std::string_view s, bool add_special, bool parse_special) {@@ -122,22 +122,26 @@ bool eval_string(std::string_view s, bool add_special, bool parse_special) {size_t pos = s.find("data:", i);if (pos == std::string_view::npos)return eval_plain_text(std::string(s), add_special, parse_special);+ i = pos + 5;DataUri uri;size_t end = uri.parse(s.substr(pos + 5));- if (end == std::string_view::npos) {- i = pos + 5;+ if (end == std::string_view::npos)continue;- }- std::string image = uri.decode();- if (!is_image(image)) {- i = pos + 5;+ if (!uri.mime.starts_with("image/"))+ continue;+ std::string image;+ try {+ image = uri.decode();+ } catch (const base64_error &e) {continue;}+ if (!is_image(image))+ continue;if (!eval_plain_text(std::string(s.substr(0, pos)), add_special, parse_special))return false;if (!eval_image(image))return false;- s = s.substr(pos + 5 + end);+ s = s.substr(i + end);i = 0;}}diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cppindex 61e1c1e9..9a985b9f 100644--- a/llamafile/chatbot_main.cpp+++ b/llamafile/chatbot_main.cpp@@ -29,6 +29,7 @@#include "llama.cpp/server/server.h"#include "llamafile/color.h"#include "llamafile/compute.h"+#include "llamafile/llama.h"#include "llamafile/string.h"namespace lf {@@ -69,7 +70,7 @@ std::string describe_compute(void) {std::string token_to_piece(const struct llama_context *ctx, llama_token token, bool special) {if (token == IMAGE_PLACEHOLDER_TOKEN)return "⁑";- return llama_token_to_piece(ctx, token, special);+ return llamafile_token_to_piece(ctx, token, special);}void on_server_listening(const char *host, int port) {diff --git a/llamafile/flags.cpp b/llamafile/flags.cppindex 3266107e..55b1ff4b 100644--- a/llamafile/flags.cpp+++ b/llamafile/flags.cpp@@ -64,7 +64,7 @@ int FLAG_batch = 2048;int FLAG_ctx_size = 8192;int FLAG_flash_attn = false;int FLAG_gpu = 0;-int FLAG_http_ibuf_size = 1024 * 1024;+int FLAG_http_ibuf_size = 5 * 1024 * 1024;int FLAG_http_obuf_size = 1024 * 1024;int FLAG_keepalive = 5;int FLAG_main_gpu = 0;diff --git a/llamafile/llama.cpp b/llamafile/llama.cppindex 24628403..58b8904f 100644--- a/llamafile/llama.cpp+++ b/llamafile/llama.cpp@@ -16,6 +16,10 @@// limitations under the License.#include "llama.h"+#include "llama.cpp/llama.h"+#include <cassert>+#include <string>+#include <vector>int llamafile_token_eot(llama_model *model) {llama_token eot = llama_token_eot(model);@@ -23,3 +27,37 @@ int llamafile_token_eot(llama_model *model) {return eot;return llama_token_eos(model);}++std::string llamafile_token_to_piece(const llama_context *ctx, llama_token token, bool special) {+ std::string piece;+ piece.resize(piece.capacity());+ const int n_chars =+ llama_token_to_piece(llama_get_model(ctx), token, &piece[0], piece.size(), 0, special);+ if (n_chars < 0) {+ piece.resize(-n_chars);+ int check =+ llama_token_to_piece(llama_get_model(ctx), token, &piece[0], piece.size(), 0, special);+ unassert(check == -n_chars);+ } else {+ piece.resize(n_chars);+ }+ return piece;+}++std::vector<llama_token> llamafile_tokenize(const struct llama_model *model,+ const std::string_view &text, bool add_special,+ bool parse_special) {+ int n_tokens = text.size() + 2 * add_special;+ std::vector<llama_token> result(n_tokens);+ n_tokens = llama_tokenize(model, text.data(), text.size(), result.data(), result.size(),+ add_special, parse_special);+ if (n_tokens < 0) {+ result.resize(-n_tokens);+ int check = llama_tokenize(model, text.data(), text.size(), result.data(), result.size(),+ add_special, parse_special);+ unassert(check == -n_tokens);+ } else {+ result.resize(n_tokens);+ }+ return result;+}diff --git a/llamafile/llama.h b/llamafile/llama.hindex 429ac0ce..53da0eb9 100644--- a/llamafile/llama.h+++ b/llamafile/llama.h@@ -16,7 +16,8 @@// limitations under the License.#pragma once-#include "llama.cpp/llama.h"+#include <__fwd/string_view.h>+#include <__fwd/vector.h>// Many llama.cpp APIs take boolean parameters at the end. Please favor// passing these constants as arguments instead, for better readability@@ -36,4 +37,10 @@#define RENDER_SPECIAL_TOKENS true#define DONT_RENDER_SPECIAL_TOKENS false+struct llama_model;+struct llama_context;+int llamafile_token_eot(llama_model *);++std::string llamafile_token_to_piece(const llama_context *, int, bool);+std::vector<int> llamafile_tokenize(const llama_model *, const std::string_view &, bool, bool);diff --git a/llamafile/server/BUILD.mk b/llamafile/server/BUILD.mkindex f170bd8b..54aa5eb0 100644--- a/llamafile/server/BUILD.mk+++ b/llamafile/server/BUILD.mk@@ -12,17 +12,17 @@ LLAMAFILE_SERVER_ASSETS = $(wildcard llamafile/server/www/*)$(LLAMAFILE_SERVER_OBJS): private CCFLAGS += -g-o/$(MODE)/llamafile/server/server.a: \+o/$(MODE)/llamafile/server/server.a: \$(filter-out %_test.o,$(LLAMAFILE_SERVER_OBJS))-o/$(MODE)/llamafile/server/main: \- o/$(MODE)/llamafile/server/main.o \- o/$(MODE)/llamafile/server/main.1.asc.zip.o \- o/$(MODE)/llamafile/server/server.a \- o/$(MODE)/llama.cpp/llama.cpp.a \- o/$(MODE)/llama.cpp/llava/llava.a \- o/$(MODE)/double-conversion/double-conversion.a \- o/$(MODE)/stb/stb.a \+o/$(MODE)/llamafile/server/main: \+ o/$(MODE)/llamafile/server/main.o \+ o/$(MODE)/llamafile/server/main.1.asc.zip.o \+ o/$(MODE)/llamafile/server/server.a \+ o/$(MODE)/llama.cpp/llama.cpp.a \+ o/$(MODE)/llama.cpp/llava/llava.a \+ o/$(MODE)/double-conversion/double-conversion.a \+ o/$(MODE)/stb/stb.a \$(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)# turn /zip/llamafile/server/www/...@@ -31,24 +31,37 @@ $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o): private ZIPOBJ_FLAGS += -C2$(LLAMAFILE_SERVER_OBJS): llamafile/server/BUILD.mk-o/$(MODE)/llamafile/server/fastjson_test: \- o/$(MODE)/llamafile/server/fastjson_test.o \- o/$(MODE)/llamafile/server/fastjson.o \- o/$(MODE)/double-conversion/double-conversion.a \+o/$(MODE)/llamafile/server/atom_test: \+ o/$(MODE)/llamafile/server/atom_test.o \+ o/$(MODE)/llamafile/server/atom.o \+ o/$(MODE)/llamafile/server/image.o \-o/$(MODE)/llamafile/server/json_test: \- o/$(MODE)/llamafile/server/json_test.o \- o/$(MODE)/llamafile/server/json.o \- o/$(MODE)/llamafile/server/hextoint.o \- o/$(MODE)/double-conversion/double-conversion.a \+o/$(MODE)/llamafile/server/image_test: \+ o/$(MODE)/llamafile/server/image_test.o \+ o/$(MODE)/llamafile/server/image.o \-o/$(MODE)/llamafile/server/tokenbucket_test: \- o/$(MODE)/llamafile/server/tokenbucket_test.o \- o/$(MODE)/llamafile/server/tokenbucket.o \- o/$(MODE)/llamafile/server/log.o \- o/$(MODE)/llama.cpp/llama.cpp.a \+o/$(MODE)/llamafile/server/fastjson_test: \+ o/$(MODE)/llamafile/server/fastjson_test.o \+ o/$(MODE)/llamafile/server/fastjson.o \+ o/$(MODE)/double-conversion/double-conversion.a \++o/$(MODE)/llamafile/server/json_test: \+ o/$(MODE)/llamafile/server/json_test.o \+ o/$(MODE)/llamafile/server/json.o \+ o/$(MODE)/llamafile/server/hextoint.o \+ o/$(MODE)/double-conversion/double-conversion.a \++o/$(MODE)/llamafile/server/tokenbucket_test: \+ o/$(MODE)/llamafile/server/tokenbucket_test.o \+ o/$(MODE)/llamafile/server/tokenbucket.o \+ o/$(MODE)/llamafile/server/log.o \+ o/$(MODE)/llama.cpp/llama.cpp.a \.PHONY: o/$(MODE)/llamafile/server-o/$(MODE)/llamafile/server: \- o/$(MODE)/llamafile/server/main \- o/$(MODE)/llamafile/server/json_test.runs \+o/$(MODE)/llamafile/server: \+ o/$(MODE)/llamafile/server/main \+ o/$(MODE)/llamafile/server/atom_test.runs \+ o/$(MODE)/llamafile/server/fastjson_test.runs \+ o/$(MODE)/llamafile/server/image_test.runs \+ o/$(MODE)/llamafile/server/json_test.runs \+ o/$(MODE)/llamafile/server/tokenbucket_test.runs \diff --git a/llamafile/server/atob.cpp b/llamafile/server/atob.cppindex e8a1b738..8d125127 100644--- a/llamafile/server/atob.cpp+++ b/llamafile/server/atob.cpp@@ -16,7 +16,6 @@// limitations under the License.#include "utils.h"-#include <string_view>booldiff --git a/llamafile/server/atom.cpp b/llamafile/server/atom.cppnew file mode 100644index 00000000..da9f9a9e--- /dev/null+++ b/llamafile/server/atom.cpp@@ -0,0 +1,141 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "atom.h"+#include "image.h"+#include <cassert>++/**+ * @fileoverview Context window content container.+ *+ * The `Atom` class is used for tracking what's inside a context window.+ * Each individual atom can hold one of two values. The first is a token+ * which is a short piece of text, subjective to each `llama_model`. The+ * second is an `Image` which uses a separate evaluation process.+ */++Atom::Atom(int token) : word_(1ull << 56 | (unsigned)token)+{+}++Atom::Atom(Image* image) : word_(2ull << 56 | (uintptr_t)image)+{+ unassert(image != nullptr);+}++Atom::Atom(Atom&& other)+{+ word_ = other.word_;+ other.word_ = 0;+}++Atom::Atom(const Atom& other)+{+ if (!other.is_image()) {+ word_ = other.word_;+ return;+ }+ Image* image = new Image(other.image());+ word_ = 2ull << 56 | (uintptr_t)image;+}++Atom::~Atom()+{+ if (is_image())+ delete (Image*)(word_ & 0x00ffffffffffffff);+}++bool+Atom::empty() const+{+ return !word_;+}++bool+Atom::is_token() const+{+ return (word_ >> 56) == 1;+}++bool+Atom::is_image() const+{+ return (word_ >> 56) == 2;+}++int+Atom::token() const+{+ unassert(is_token());+ return word_;+}++const Image&+Atom::image() const+{+ unassert(is_image());+ return *(Image*)(word_ & 0x00ffffffffffffff);+}++int+Atom::ctx_used() const+{+ switch (word_ >> 56) {+ case 1:+ return 1;+ case 2:+ return image().ctx_used();+ default:+ return 0;+ }+}++bool+operator<(const Atom& lhs, const Atom& rhs)+{+ if (lhs.empty())+ return !rhs.empty();+ if (rhs.empty())+ return false;+ if (!lhs.is_image()) {+ if (!rhs.is_image())+ return lhs.token() < rhs.token();+ return true;+ } else {+ if (rhs.is_image())+ return lhs.image() < rhs.image();+ return false;+ }+}++bool+operator==(const Atom& lhs, const Atom& rhs)+{+ if (lhs.empty())+ return rhs.empty();+ if (rhs.empty())+ return false;+ if (lhs.is_token()) {+ if (!rhs.is_token())+ return false;+ return lhs.token() == rhs.token();+ } else {+ if (!rhs.is_image())+ return false;+ return lhs.image() == rhs.image();+ }+}diff --git a/llamafile/server/atom.h b/llamafile/server/atom.hnew file mode 100644index 00000000..444020d4--- /dev/null+++ b/llamafile/server/atom.h@@ -0,0 +1,47 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <__fwd/string.h>++class Image;++class Atom+{+ public:+ Atom() = default;+ explicit Atom(int);+ explicit Atom(Image*);+ Atom(const Atom&);+ Atom(Atom&&);+ ~Atom();+ int token() const;+ bool empty() const;+ int ctx_used() const;+ bool is_token() const;+ bool is_image() const;+ const Image& image() const;++ private:+ uint64_t word_ = 0;+};++bool+operator<(const Atom&, const Atom&);++bool+operator==(const Atom&, const Atom&);diff --git a/llamafile/server/atom_test.cpp b/llamafile/server/atom_test.cppnew file mode 100644index 00000000..d9e61c00--- /dev/null+++ b/llamafile/server/atom_test.cpp@@ -0,0 +1,109 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "atom.h"+#include "image.h"+#include <cstdlib>++static const Atom atoms[] = {+ Atom(),+ Atom(-1),+ Atom(0),+ Atom(1),+ Atom(new Image("hello", 1)),+ Atom(new Image("there", 1)),+};++const size_t n = sizeof(atoms) / sizeof(atoms[0]);++void+test_atom_operator_lt()+{+ // irreflexivity: no element can be less than itself+ for (size_t i = 0; i < n; ++i)+ if (atoms[i] < atoms[i])+ exit(1);++ // asymmetry: if x < y then !(y < x)+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ if (atoms[i] < atoms[j] && atoms[j] < atoms[i])+ exit(2);++ // transitivity: If x < y and y < z then x < z+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ for (size_t k = 0; k < n; ++k)+ if (atoms[i] < atoms[j] && atoms[j] < atoms[k])+ if (!(atoms[i] < atoms[k]))+ exit(3);++ // transitivity of incomparability+ // if x ≈ y (neither < the other) and y ≈ z, then x ≈ z+ for (size_t i = 0; i < n; ++i) {+ for (size_t j = 0; j < n; ++j) {+ for (size_t k = 0; k < n; ++k) {+ bool i_equiv_j =+ !((atoms[i] < atoms[j]) || (atoms[j] < atoms[i]));+ bool j_equiv_k =+ !((atoms[j] < atoms[k]) || (atoms[k] < atoms[j]));+ bool i_equiv_k =+ !((atoms[i] < atoms[k]) || (atoms[k] < atoms[i]));+ if (i_equiv_j && j_equiv_k && !i_equiv_k)+ exit(4);+ }+ }+ }+}++void+test_atom_operator_eq()+{+ // reflexivity: every element must equal itself+ for (size_t i = 0; i < n; ++i)+ if (!(atoms[i] == atoms[i]))+ exit(5);++ // symmetry: if x == y then y == x+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ if ((atoms[i] == atoms[j]) != (atoms[j] == atoms[i]))+ exit(6);++ // transitivity: if x == y and y == z then x == z+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ for (size_t k = 0; k < n; ++k)+ if (atoms[i] == atoms[j] && atoms[j] == atoms[k])+ if (!(atoms[i] == atoms[k]))+ exit(7);++ // consistency with operator<: if x == y then !(x < y) && !(y < x)+ // this ensures == and < agree on their notion of equivalence+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ if (atoms[i] == atoms[j])+ if (atoms[i] < atoms[j] || atoms[j] < atoms[i])+ exit(8);+}++int+main()+{+ test_atom_operator_lt();+ test_atom_operator_eq();+}diff --git a/llamafile/server/atomize.cpp b/llamafile/server/atomize.cppnew file mode 100644index 00000000..a4c4fe35--- /dev/null+++ b/llamafile/server/atomize.cpp@@ -0,0 +1,73 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "atom.h"+#include "llama.cpp/base64.h"+#include "llamafile/chatbot.h"+#include "llamafile/datauri.h"+#include "llamafile/image.h"+#include "llamafile/llama.h"+#include "llamafile/server/image.h"+#include <string>+#include <vector>++static void+append_tokens(const llama_model* model,+ std::vector<Atom>* result,+ const std::string_view& s,+ bool parse_special)+{+ std::vector<int> tokens = llamafile_tokenize(+ model, std::string(s), DONT_ADD_SPECIAL, parse_special);+ for (int token : tokens)+ result->emplace_back(token);+}++void+atomize(const llama_model* model,+ std::vector<Atom>* result,+ std::string_view s,+ bool parse_special)+{+ size_t i = 0;+ for (;;) {+ size_t pos = s.find("data:", i);+ if (pos == std::string_view::npos) {+ append_tokens(model, result, s, parse_special);+ return;+ }+ i = pos + 5;+ DataUri uri;+ size_t end = uri.parse(s.substr(i));+ if (end == std::string_view::npos)+ continue;+ if (!uri.mime.starts_with("image/"))+ continue;+ std::string image;+ try {+ image = uri.decode();+ } catch (const base64_error& e) {+ continue;+ }+ if (!lf::get_image_type(image))+ continue;+ append_tokens(model, result, s.substr(0, pos), parse_special);+ result->emplace_back(new Image(image, -1));+ s = s.substr(i + end);+ i = 0;+ }+}diff --git a/llamafile/server/buffer.cpp b/llamafile/server/buffer.cppindex fa0d0f00..ecd41226 100644--- a/llamafile/server/buffer.cpp+++ b/llamafile/server/buffer.cpp@@ -16,7 +16,6 @@// limitations under the License.#include "buffer.h"-#include <sys/mman.h>#include <unistd.h>diff --git a/llamafile/server/doc/v1_chat_completions.md b/llamafile/server/doc/v1_chat_completions.mdindex fce4af08..cee23faa 100644--- a/llamafile/server/doc/v1_chat_completions.md+++ b/llamafile/server/doc/v1_chat_completions.md@@ -2,6 +2,19 @@The `/v1/chat/completions` endpoint lets you build a chatbot.+The client is responsible for maintaining the history of a conversation.+The conversation is divided into messages, each having text content, and+a role name that describes the speaker of that text. Each time a request+is made to the API, the full conversation history is provided, and the+server will generate content for the next message in its history, as the+role of the assistant.++This endpoint supports the following features:++1. Automatic context window caching across requests+2. Streaming responses to give you generated tokens in real time+3. Image uploads via data: URIs+## Request URIs- `/v1/chat/completions` (OpenAI API compatible)@@ -43,7 +56,9 @@ The `/v1/chat/completions` endpoint lets you build a chatbot.- `content`: `string`- The content field holds text the system, user, or assistant said.+ The content field holds text the system, user, or assistant said. It+ is recommended that this be treated as markdown. This field may+ contain images which should be embedded as data: URIs.The completion endpoint will then generate at least one additionalchat message. If the client wishes to request another completion, then@@ -160,6 +175,54 @@ unsupported:- `function_call`- `parallel_tool_calls`+## Context Caching++It may seem surprising that the entire conversation history should be+uploaded each time a new message from the assistant is desired. This is+fast because the server has a fixed number of slots which are used for+caching context windows. When an HTTP client sends a chat completion+request, a prefix search is made for a slot that best matches the+supplied conversation history. Once a slot is chosen, unrelated content+(possibly from another user's conversation) is deleted, the common+prefix is preserved, and the remaining portions are prefilled. If all+slots are in use, then the server handler waits for one to be free.++## Image Uploads++If a vision model was specified by passing the `--mmproj` flag, then+each slot will additionally maintain a vision model context, which is+used to encode embeddings for any data: URIs holding valid images.++The data URI must conform to RFC2397. For example, a 1x1 transparent+pixel could be encoded as:++ data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==++Images can be included by any role. Multiple images may be embedded+within a single message content. The server will evaluate images in the+same order as any surrounding text, e.g.++```markdown++<img src="data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==">+```++The supported image file formats are:++- PNG+- JPEG+- GIF+- BMP+- TGA+- HDR+- PGM+- PPM+- PIC++The declared mime type of the data URI must begin with `image/`. Its+subtype is largely ignored, since the file type is inferred from the+binary payload.+## See Also- [LLaMAfiler Documentation Index](index.md)diff --git a/llamafile/server/embedding.cpp b/llamafile/server/embedding.cppindex 8ac915fc..94d927c4 100644--- a/llamafile/server/embedding.cpp+++ b/llamafile/server/embedding.cpp@@ -23,12 +23,11 @@#include <vector>#include "llama.cpp/llama.h"--#include "cleanup.h"-#include "fastjson.h"-#include "json.h"-#include "log.h"-#include "utils.h"+#include "llamafile/server/cleanup.h"+#include "llamafile/server/fastjson.h"+#include "llamafile/server/json.h"+#include "llamafile/server/log.h"+#include "llamafile/server/utils.h"struct EmbeddingParams{diff --git a/llamafile/server/fastjson_test.cpp b/llamafile/server/fastjson_test.cppindex 5e01282e..d63d3fb1 100644--- a/llamafile/server/fastjson_test.cpp+++ b/llamafile/server/fastjson_test.cpp@@ -22,52 +22,50 @@#include <stdio.h>#include <string>-using namespace ctl;--string+std::stringencode_json(int x) noexcept{char buf[12];return { buf, (size_t)(encode_json(buf, x) - buf) };}-string+std::stringencode_json(long x) noexcept{char buf[21];return { buf, (size_t)(encode_json(buf, x) - buf) };}-string+std::stringencode_json(double x) noexcept{char buf[128];return { buf, (size_t)(encode_json(buf, x) - buf) };}-string+std::stringencode_json(unsigned x) noexcept{char buf[128];return { buf, (size_t)(encode_json(buf, x) - buf) };}-string+std::stringencode_json(unsigned long x) noexcept{char buf[128];return { buf, (size_t)(encode_json(buf, x) - buf) };}-string-encode_json(const string_view x) noexcept+std::string+encode_json(const std::string_view x) noexcept{char buf[128];return { buf, (size_t)(encode_json(buf, x) - buf) };}-string-encode_js_string_literal(const string_view x) noexcept+std::string+encode_js_string_literal(const std::string_view x) noexcept{char buf[256]; // this isn't secure (no guard page)return { buf, (size_t)(encode_json(buf, x) - buf) };@@ -94,7 +92,7 @@ main(int argc, char* argv[])if (encode_json("") != "\"\"")return 8;- if (encode_json(string_view("\0\1", 2)) != "\"\\u0000\\u0001\"")+ if (encode_json(std::string_view("\0\1", 2)) != "\"\\u0000\\u0001\"")return 9;if (encode_json("\n\"\\\t") != "\"\\n\\\"\\\\\\t\"")return 10;diff --git a/llamafile/server/image.cpp b/llamafile/server/image.cppnew file mode 100644index 00000000..be684429--- /dev/null+++ b/llamafile/server/image.cpp@@ -0,0 +1,70 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "image.h"+#include "llamafile/llamafile.h"+#include <cassert>+#include <utility>++Image::~Image() = default;++Image::Image(const Image& old) : Image(old.bytes_, old.ctx_used_)+{+}++Image::Image(const std::string_view& bytes, int ctx_used)+ : bytes_(bytes), ctx_used_(ctx_used)+{+}++const std::string&+Image::bytes() const+{+ return bytes_;+}++int+Image::ctx_used() const+{+ // there are two use cases for image+ //+ // 1. when the http client asks for their chat messages to be+ // tokenized, ctx_used should be set to -1, because we don't want+ // to pay the price of the expensive clip image encode operation.+ // since ctx_used isn't considered by comparators, this is a fine+ // object state that's useful for searching data structures.+ //+ // 2. when an atom is actually evaluated by the model, if it's an+ // image atom it'll be recreated with ctx_used specified. this is+ // what slot does when maintaining its history, since it's the+ // only thing that cares right now about context usage.+ //+ unassert(ctx_used_ > 0);+ return ctx_used_;+}++bool+operator<(const Image& lhs, const Image& rhs)+{+ return lhs.bytes() < rhs.bytes();+}++bool+operator==(const Image& lhs, const Image& rhs)+{+ return lhs.bytes() == rhs.bytes();+}diff --git a/llamafile/server/image.h b/llamafile/server/image.hnew file mode 100644index 00000000..547444c6--- /dev/null+++ b/llamafile/server/image.h@@ -0,0 +1,39 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#pragma once+#include <string>++class Image+{+ public:+ ~Image();+ Image(const Image&);+ Image(const std::string_view&, int);+ const std::string& bytes() const;+ int ctx_used() const;++ private:+ std::string bytes_;+ int ctx_used_;+};++bool+operator<(const Image&, const Image&);++bool+operator==(const Image&, const Image&);diff --git a/llamafile/server/image_test.cpp b/llamafile/server/image_test.cppnew file mode 100644index 00000000..6f0982fc--- /dev/null+++ b/llamafile/server/image_test.cpp@@ -0,0 +1,107 @@+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi+//+// Copyright 2024 Mozilla Foundation+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++#include "atom.h"+#include "image.h"+#include <cstdlib>++static const Image images[] = {+ Image("", 0),+ Image("hello", 1),+ Image("there", 1),+ Image("there", 2),+};++const size_t n = sizeof(images) / sizeof(images[0]);++void+test_image_operator_lt()+{+ // irreflexivity: no element can be less than itself+ for (size_t i = 0; i < n; ++i)+ if (images[i] < images[i])+ exit(1);++ // asymmetry: if x < y then !(y < x)+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ if (images[i] < images[j] && images[j] < images[i])+ exit(2);++ // transitivity: If x < y and y < z then x < z+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ for (size_t k = 0; k < n; ++k)+ if (images[i] < images[j] && images[j] < images[k])+ if (!(images[i] < images[k]))+ exit(3);++ // transitivity of incomparability+ // if x ≈ y (neither < the other) and y ≈ z, then x ≈ z+ for (size_t i = 0; i < n; ++i) {+ for (size_t j = 0; j < n; ++j) {+ for (size_t k = 0; k < n; ++k) {+ bool i_equiv_j =+ !((images[i] < images[j]) || (images[j] < images[i]));+ bool j_equiv_k =+ !((images[j] < images[k]) || (images[k] < images[j]));+ bool i_equiv_k =+ !((images[i] < images[k]) || (images[k] < images[i]));+ if (i_equiv_j && j_equiv_k && !i_equiv_k)+ exit(4);+ }+ }+ }+}++void+test_image_operator_eq()+{+ // reflexivity: every element must equal itself+ for (size_t i = 0; i < n; ++i)+ if (!(images[i] == images[i]))+ exit(5);++ // symmetry: if x == y then y == x+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ if ((images[i] == images[j]) != (images[j] == images[i]))+ exit(6);++ // transitivity: if x == y and y == z then x == z+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ for (size_t k = 0; k < n; ++k)+ if (images[i] == images[j] && images[j] == images[k])+ if (!(images[i] == images[k]))+ exit(7);++ // consistency with operator<: if x == y then !(x < y) && !(y < x)+ // this ensures == and < agree on their notion of equivalence+ for (size_t i = 0; i < n; ++i)+ for (size_t j = 0; j < n; ++j)+ if (images[i] == images[j])+ if (images[i] < images[j] || images[j] < images[i])+ exit(8);+}++int+main()+{+ test_image_operator_lt();+ test_image_operator_eq();+}diff --git a/llamafile/server/log.cpp b/llamafile/server/log.cppindex b7834540..a6f07530 100644--- a/llamafile/server/log.cpp+++ b/llamafile/server/log.cpp@@ -16,12 +16,10 @@// limitations under the License.#include "log.h"-+#include "time.h"#include <pthread.h>#include <string.h>-#include "time.h"-static thread_local char g_thread_name[128];const char*diff --git a/llamafile/server/main.cpp b/llamafile/server/main.cppindex 7dd5f925..cbe87c10 100644--- a/llamafile/server/main.cpp+++ b/llamafile/server/main.cpp@@ -21,16 +21,15 @@#include "llama.cpp/llama.h"#include "llamafile/llamafile.h"#include "llamafile/pool.h"+#include "llamafile/server/log.h"+#include "llamafile/server/server.h"+#include "llamafile/server/signals.h"+#include "llamafile/server/slots.h"+#include "llamafile/server/time.h"+#include "llamafile/server/tokenbucket.h"+#include "llamafile/server/utils.h"#include "llamafile/version.h"-#include "log.h"-#include "server.h"-#include "signals.h"-#include "slots.h"-#include "time.h"-#include "tokenbucket.h"-#include "utils.h"-Server* g_server;intdiff --git a/llamafile/server/server.cpp b/llamafile/server/server.cppindex 108a9d7e..32d06bbf 100644--- a/llamafile/server/server.cpp+++ b/llamafile/server/server.cpp@@ -28,10 +28,9 @@#include "llamafile/crash.h"#include "llamafile/llamafile.h"--#include "log.h"-#include "server.h"-#include "worker.h"+#include "llamafile/server/log.h"+#include "llamafile/server/server.h"+#include "llamafile/server/worker.h"Server::Server(int fd, Slots* slots, llama_model* model): fd(fd), slots_(slots), model_(model)diff --git a/llamafile/server/signals.cpp b/llamafile/server/signals.cppindex 1dd9da96..09bf00d8 100644--- a/llamafile/server/signals.cpp+++ b/llamafile/server/signals.cpp@@ -21,11 +21,10 @@#include <ucontext.h>#include "llamafile/crash.h"+#include "llamafile/server/log.h"+#include "llamafile/server/server.h"#include "llamafile/threadlocal.h"-#include "log.h"-#include "server.h"-static struct{struct sigaction sigint; // ctrl-cdiff --git a/llamafile/server/slot.cpp b/llamafile/server/slot.cppindex 0f3bb75f..a58eb9d9 100644--- a/llamafile/server/slot.cpp+++ b/llamafile/server/slot.cpp@@ -16,18 +16,23 @@// limitations under the License.#include "slot.h"-#include "llama.cpp/common.h"+#include "llama.cpp/llava/clip.h"+#include "llama.cpp/llava/llava.h"+#include "llamafile/image.h"#include "llamafile/llama.h"#include "llamafile/llamafile.h"#include "llamafile/macros.h"+#include "llamafile/server/atom.h"+#include "llamafile/server/image.h"+#include "llamafile/server/log.h"#include "llamafile/vector.h"#include "llamafile/version.h"-#include "log.h"+#include <algorithm>#include <cassert>#include <cosmo.h>static int-ctx_size(llama_model* model)+choose_ctx_size(llama_model* model){int n_ctx_train = llama_n_ctx_train(model);if (FLAG_ctx_size <= 0 || FLAG_ctx_size > n_ctx_train)@@ -49,6 +54,27 @@ generate_system_fingerprint(const llama_context_params* cparams)return b;}+const char*+Slot::describe_error(int err)+{+ switch (err) {+ case uninitialized:+ return "uninitialized";+ case out_of_context:+ return "out_of_context";+ case no_vision_model:+ return "no_vision_model";+ case decode_token_failed:+ return "decode_token_failed";+ case decode_image_failed:+ return "decode_image_failed";+ case encode_image_failed:+ return "encode_image_failed";+ default:+ return "bad_error_code";+ }+}+Slot::Slot(llama_model* model) : model_(model){}@@ -57,6 +83,8 @@ Slot::~Slot(){if (ctx_)llama_free(ctx_);+ if (clip_ctx_)+ clip_free(clip_ctx_);}bool@@ -68,7 +96,7 @@ Slot::start()cparams.embeddings_only = false;cparams.logits_all = false;cparams.seed = 12345;- cparams.n_ctx = ctx_size(model_);+ cparams.n_ctx = choose_ctx_size(model_);cparams.n_batch = FLAG_batch;cparams.n_ubatch = FLAG_ubatch;cparams.n_seq_max = 1;@@ -91,78 +119,183 @@ Slot::start()system_fingerprint_ = generate_system_fingerprint(&cparams);if (!(ctx_ = llama_new_context_with_model(model_, cparams)))return false;+ if (FLAG_mmproj)+ if (!(clip_ctx_ = clip_model_load(FLAG_mmproj, FLAG_verbose)))+ return false;return true;}int-Slot::n_ctx()+Slot::ctx_size() const{return llama_n_ctx(ctx_);}-bool-Slot::eval_token(int id)+int+Slot::ctx_used() const{- std::vector<int> tokens;- tokens.push_back(id);- return eval_tokens(tokens);+ int tokens = 0;+ for (size_t i = 0; i < history_.size(); ++i)+ tokens += history_[i].ctx_used();+ return tokens;}-bool-Slot::eval_tokens(std::vector<int> tokens)+int+Slot::eval_token(int token){- unassert(ctx_);- if (history_.size() + tokens.size() > n_ctx())- return false;- for (int i = 0; i < tokens.size(); i += FLAG_batch) {- int n_eval = (int)tokens.size() - i;+ return eval_tokens({ token });+}++int+Slot::eval_tokens(const std::vector<int>& tokens)+{+ if (!ctx_)+ return uninitialized;+ if (tokens.empty())+ return 0;+ int N = tokens.size();+ int used = ctx_used();+ if (used + N > ctx_size())+ return out_of_context;+ std::vector<int> toks(tokens); // TODO(jart): is copying really needed?+ for (int i = 0; i < N; i += FLAG_batch) {+ int n_eval = N - i;if (n_eval > FLAG_batch)n_eval = FLAG_batch;- if (llama_decode(- ctx_,- llama_batch_get_one(&tokens[i], n_eval, history_.size(), 0)))- return false;- history_.insert(- history_.end(), tokens.begin() + i, tokens.begin() + i + n_eval);+ if (llama_decode(ctx_,+ { .n_tokens = n_eval,+ .token = &toks[i],+ .all_pos_0 = used,+ .all_pos_1 = 1 }))+ return decode_token_failed;+ for (int j = 0; j < n_eval; ++j)+ history_.emplace_back(toks[i + j]);+ used += n_eval;}last_used_ = timespec_real();- return true;+ return N;}-bool-Slot::prefill(const std::vector<int>& tokens)+int+Slot::eval_image(const std::string_view& bytes)+{+ if (!ctx_)+ return uninitialized;+ if (!clip_ctx_)+ return no_vision_model;+ llava_image_embed* image_embed =+ llava_image_embed_make_with_bytes(clip_ctx_,+ FLAG_threads_batch,+ (const unsigned char*)bytes.data(),+ bytes.size());+ if (!image_embed)+ return encode_image_failed;+ int used = ctx_used();+ int N = image_embed->n_image_pos;+ if (used + N > ctx_size()) {+ llava_image_embed_free(image_embed);+ return out_of_context;+ }+ int n_embd = llama_n_embd(llama_get_model(ctx_));+ for (int i = 0; i < N; i += FLAG_batch) {+ int n_eval = N - i;+ if (n_eval > FLAG_batch)+ n_eval = FLAG_batch;+ if (llama_decode(ctx_,+ { .n_tokens = n_eval,+ .embd = image_embed->embed + i * n_embd,+ .all_pos_0 = used,+ .all_pos_1 = 1 })) {+ llava_image_embed_free(image_embed);+ return decode_image_failed;+ }+ used += n_eval;+ }+ llava_image_embed_free(image_embed);+ history_.emplace_back(new Image(bytes, N));+ last_used_ = timespec_real();+ return N;+}++int+Slot::eval_atoms(const std::vector<Atom>& atoms){- unassert(ctx_);- size_t erase_count = 0;- size_t reuse_count = lf::vector_common_prefix_length(tokens, history_);+ int rc;+ int token_count = 0;+ std::vector<int> tokens;+ for (const Atom& atom : atoms) {+ if (atom.is_token()) {+ tokens.emplace_back(atom.token());+ } else if (atom.is_image()) {+ if ((rc = eval_tokens(tokens)) < 0)+ return rc;+ token_count += rc;+ tokens.clear();+ if ((rc = eval_image(atom.image().bytes())) < 0)+ return rc;+ token_count += rc;+ }+ }+ if ((rc = eval_tokens(tokens)) < 0)+ return rc;+ token_count += rc;+ return token_count;+}++int+Slot::prefill(const std::vector<Atom>& atoms)+{+ if (!ctx_)+ return uninitialized;+ int used_tokens = ctx_used();+ int reuse_atoms = 0;+ int reuse_tokens = 0;+ int erase_tokens = 0;+ int n = std::min(atoms.size(), history_.size());+ for (int i = 0; i < n && atoms[i] == history_[i]; ++i) {+ reuse_tokens += history_[i].ctx_used();+ reuse_atoms += 1;+ }// xxx: ensure we prefill at least one token (prevents badness)- if (reuse_count >= 1)- reuse_count -= 1;- if (history_.size() > reuse_count) {- erase_count = history_.size() - reuse_count;- if (llama_kv_cache_seq_rm(ctx_, 0, reuse_count, -1)) {- history_.resize(reuse_count);+ if (reuse_tokens >= 1) {+ reuse_atoms -= 1;+ reuse_tokens -= history_[reuse_atoms].ctx_used();+ }+ if (used_tokens > reuse_tokens) {+ erase_tokens = used_tokens - reuse_tokens;+ if (llama_kv_cache_seq_rm(ctx_, 0, reuse_tokens, -1)) {+ history_.resize(reuse_atoms);} else {SLOG("failed to remove tokens from KV cache");llama_kv_cache_clear(ctx_);- reuse_count = 0;- erase_count = history_.size();+ reuse_atoms = 0;+ reuse_tokens = 0;+ erase_tokens = used_tokens;history_.clear();}}- std::vector<int> new_tokens(tokens.begin() + reuse_count, tokens.end());- SLOG("prefilling %zu tokens (after removing %zu and reusing %zu)",- new_tokens.size(),- erase_count,- reuse_count);- return eval_tokens(new_tokens);+ std::vector<Atom> new_atoms(atoms.begin() + reuse_atoms, atoms.end());+ int rc;+ if ((rc = eval_atoms(new_atoms)) < 0)+ return rc;+ int token_count = reuse_tokens + rc;+ SLOG("prefilled %zu tokens (after removing %zu and reusing %zu)",+ token_count,+ erase_tokens,+ reuse_tokens);+ return token_count;}-std::string-Slot::dump()+void+Slot::dump(std::string* result){- std::string r;- for (size_t i = 0; i < history_.size(); ++i)- r += llama_token_to_piece(ctx_, history_[i], RENDER_SPECIAL_TOKENS);- return r;+ for (size_t i = 0; i < history_.size(); ++i) {+ if (history_[i].is_token()) {+ llama_token token = history_[i].token();+ *result +=+ llamafile_token_to_piece(ctx_, token, RENDER_SPECIAL_TOKENS);+ } else if (history_[i].is_image()) {+ lf::convert_image_to_uri(result, history_[i].image().bytes());+ }+ }}diff --git a/llamafile/server/slot.h b/llamafile/server/slot.hindex ffab94de..0e019070 100644--- a/llamafile/server/slot.h+++ b/llamafile/server/slot.h@@ -20,23 +20,43 @@#include <time.h>#include <vector>-struct llama_model;+struct Atom;+struct Image;struct llama_context;+struct llama_model;+struct clip_ctx;struct Slot{+ enum+ {+ uninitialized = -4096,+ out_of_context,+ no_vision_model,+ decode_token_failed,+ decode_image_failed,+ encode_image_failed,+ };++ static const char* describe_error(int);+llama_model* model_;+ clip_ctx* clip_ctx_ = nullptr;llama_context* ctx_ = nullptr;timespec last_used_ = timespec_zero;- std::vector<int> history_;+ std::vector<Atom> history_;std::string system_fingerprint_;- Slot(llama_model*);~Slot();- int n_ctx();+ explicit Slot(llama_model*);+ int ctx_size() const;+ int ctx_used() const;bool start();- bool eval_token(int);- bool eval_tokens(std::vector<int>);- bool prefill(const std::vector<int>&);- std::string dump();+ int eval_token(int);+ int eval_image(const std::string_view&);+ int eval_tokens(const std::vector<int>&);+ int eval_atoms(const std::vector<Atom>&);+ int prefill(const std::vector<Atom>&);+ void tokenize(std::vector<Atom>*, std::string_view, bool);+ void dump(std::string*);};diff --git a/llamafile/server/slot_entry.cpp b/llamafile/server/slot_entry.cppindex de8aab9b..a3dc2932 100644--- a/llamafile/server/slot_entry.cpp+++ b/llamafile/server/slot_entry.cpp@@ -16,6 +16,7 @@// limitations under the License.#include "slot_entry.h"+#include "atom.h"#include "slot.h"#include <assert.h>@@ -24,7 +25,7 @@ SlotEntry::SlotEntry(Slot* slot) : slot_(slot), key_(nullptr)unassert(slot);}-SlotEntry::SlotEntry(const std::vector<int>* key) : slot_(nullptr), key_(key)+SlotEntry::SlotEntry(const std::vector<Atom>* key) : slot_(nullptr), key_(key){unassert(key);}@@ -39,7 +40,7 @@ SlotEntry::slot() constreturn slot_;}-const std::vector<int>*+const std::vector<Atom>*SlotEntry::key() const{return key_ ? key_ : &slot_->history_;diff --git a/llamafile/server/slot_entry.h b/llamafile/server/slot_entry.hindex b6d884ba..e30bed0d 100644--- a/llamafile/server/slot_entry.h+++ b/llamafile/server/slot_entry.h@@ -19,19 +19,20 @@#include <vector>struct Slot;+struct Atom;class SlotEntry{public:explicit SlotEntry(Slot*);- explicit SlotEntry(const std::vector<int>*);+ explicit SlotEntry(const std::vector<Atom>*);~SlotEntry();Slot* slot() const;- const std::vector<int>* key() const;+ const std::vector<Atom>* key() const;private:Slot* slot_;- const std::vector<int>* key_;+ const std::vector<Atom>* key_;};booldiff --git a/llamafile/server/slots.cpp b/llamafile/server/slots.cppindex 4d8e7293..56c4bd7d 100644--- a/llamafile/server/slots.cpp+++ b/llamafile/server/slots.cpp@@ -16,11 +16,11 @@// limitations under the License.#include "slots.h"+#include "llamafile/server/atom.h"#include "llamafile/server/log.h"+#include "llamafile/server/slot.h"+#include "llamafile/server/slot_entry.h"#include "llamafile/vector.h"-#include "log.h"-#include "slot.h"-#include "slot_entry.h"#include <assert.h>Slots::Slots(llama_model* model) : model_(model)@@ -62,7 +62,7 @@ Slots::start(int count)}Slot*-Slots::take(const std::vector<int>& prefix)+Slots::take(const std::vector<Atom>& prefix){pthread_mutex_lock(&lock_);for (;;) {diff --git a/llamafile/server/slots.h b/llamafile/server/slots.hindex 00cb4f4b..d7020761 100644--- a/llamafile/server/slots.h+++ b/llamafile/server/slots.h@@ -20,6 +20,7 @@#include <set>#include <vector>+class Atom;struct Slot;class SlotEntry;struct llama_model;@@ -32,9 +33,10 @@ struct Slotspthread_mutex_t lock_;pthread_cond_t cond_;- Slots(llama_model*);+ explicit Slots(llama_model*);~Slots();int start(int);- Slot* take(const std::vector<int>&);+ void tokenize(std::vector<Atom>*, std::string_view, bool);+ Slot* take(const std::vector<Atom>&);void give(Slot*);};diff --git a/llamafile/server/slotz.cpp b/llamafile/server/slotz.cppindex 362b7c68..f00b0a77 100644--- a/llamafile/server/slotz.cpp+++ b/llamafile/server/slotz.cpp@@ -32,7 +32,8 @@ Client::slotz()if (id >= worker_->server_->slots_->all_slots_.size())return send_error(404);Slot* slot = worker_->server_->slots_->all_slots_[id];- std::string dump = slot->dump();+ std::string dump;+ slot->dump(&dump);char* p = append_http_response_message(obuf_.p, 200);p = stpcpy(p, "Content-Type: text/plain\r\n");return send_response(obuf_.p, p, dump);diff --git a/llamafile/server/time.cpp b/llamafile/server/time.cppindex 2ac0c9d8..1ef3a741 100644--- a/llamafile/server/time.cpp+++ b/llamafile/server/time.cpp@@ -23,8 +23,7 @@#include <unistd.h>#include "llamafile/crash.h"--#include "log.h"+#include "llamafile/server/log.h"//// lockless implementation of gmtime_r() and localtime_r()diff --git a/llamafile/server/tokenbucket.cpp b/llamafile/server/tokenbucket.cppindex 1e5d17bf..c04d5f44 100644--- a/llamafile/server/tokenbucket.cpp+++ b/llamafile/server/tokenbucket.cpp@@ -16,15 +16,13 @@// limitations under the License.#include "tokenbucket.h"-+#include "llamafile/llamafile.h"+#include "llamafile/server/log.h"#include <assert.h>#include <pthread.h>#include <signal.h>#include <stdatomic.h>-#include "llamafile/llamafile.h"-#include "llamafile/server/log.h"-union TokenBucket{atomic_uint_fast64_t* w;diff --git a/llamafile/server/tokenize.cpp b/llamafile/server/tokenize.cppindex 175d4615..d32a732d 100644--- a/llamafile/server/tokenize.cpp+++ b/llamafile/server/tokenize.cpp@@ -16,21 +16,18 @@// limitations under the License.#include "client.h"-+#include "llama.cpp/llama.h"+#include "llamafile/server/cleanup.h"+#include "llamafile/server/fastjson.h"+#include "llamafile/server/json.h"+#include "llamafile/server/log.h"+#include "llamafile/server/signals.h"+#include "llamafile/server/utils.h"#include <string.h>#include <sys/resource.h>#include <utility>#include <vector>-#include "llama.cpp/llama.h"--#include "cleanup.h"-#include "fastjson.h"-#include "json.h"-#include "log.h"-#include "signals.h"-#include "utils.h"-struct TokenizeParams{bool add_special;diff --git a/llamafile/server/utils.h b/llamafile/server/utils.hindex e5bbdf0e..30205316 100644--- a/llamafile/server/utils.h+++ b/llamafile/server/utils.h@@ -18,8 +18,12 @@#pragma once#include <__fwd/string.h>#include <__fwd/string_view.h>+#include <__fwd/vector.h>#include <optional>+class Atom;+struct llama_model;+extern const signed char kHexToInt[256];bool@@ -27,3 +31,9 @@ atob(std::string_view, bool);std::string_viewor_empty(std::optional<std::string_view> x);++void+atomize(const llama_model* model,+ std::vector<Atom>* result,+ std::string_view s,+ bool parse_special);diff --git a/llamafile/server/v1_chat_completions.cpp b/llamafile/server/v1_chat_completions.cppindex a9657fcf..d880658b 100644--- a/llamafile/server/v1_chat_completions.cpp+++ b/llamafile/server/v1_chat_completions.cpp@@ -16,28 +16,26 @@// limitations under the License.#include "client.h"--#include <math.h>-#include <string.h>-#include <sys/resource.h>-#include <vector>-#include "llama.cpp/llama.h"#include "llama.cpp/sampling.h"#include "llamafile/llama.h"#include "llamafile/macros.h"+#include "llamafile/server/atom.h"+#include "llamafile/server/cleanup.h"+#include "llamafile/server/fastjson.h"+#include "llamafile/server/json.h"+#include "llamafile/server/log.h"+#include "llamafile/server/server.h"+#include "llamafile/server/slot.h"+#include "llamafile/server/slots.h"+#include "llamafile/server/utils.h"+#include "llamafile/server/worker.h"#include "llamafile/string.h"#include "llamafile/vector.h"--#include "cleanup.h"-#include "fastjson.h"-#include "json.h"-#include "log.h"-#include "server.h"-#include "slot.h"-#include "slots.h"-#include "utils.h"-#include "worker.h"+#include <math.h>+#include <string.h>+#include <sys/resource.h>+#include <vector>struct V1ChatCompletionParams{@@ -51,16 +49,16 @@ struct V1ChatCompletionParamsstd::string user;std::string model;std::vector<llama_chat_msg> messages;- std::vector<std::vector<int>> stop;+ std::vector<std::vector<Atom>> stop;std::string grammar;void add_stop(llama_model* model, const std::string& text){- stop.emplace_back(- llama_tokenize(model, text, DONT_ADD_SPECIAL, DONT_PARSE_SPECIAL));+ stop.emplace_back();+ atomize(model, &stop.back(), text, DONT_PARSE_SPECIAL);}- bool should_stop(const std::vector<int>& history)+ bool should_stop(const std::vector<Atom>& history){for (const auto& suffix : stop)if (lf::vector_ends_with(history, suffix))@@ -72,7 +70,7 @@ struct V1ChatCompletionParamsstruct V1ChatCompletionState{std::string prompt;- std::vector<int> tokens;+ std::vector<Atom> atoms;std::string piece;};@@ -452,27 +450,19 @@ Client::v1_chat_completions()V1ChatCompletionResponse* response = new V1ChatCompletionResponse;defer_cleanup(cleanup_response, response);+ // add bos token if it's needed+ if (llama_should_add_bos_token(model_))+ state->atoms.emplace_back(llama_token_bos(model_));+// turn text into tokensstate->prompt =llama_chat_apply_template(model_, "", params->messages, ADD_ASSISTANT);- state->tokens =- llama_tokenize(model_, state->prompt, DONT_ADD_SPECIAL, PARSE_SPECIAL);-- // add bos token if it's needed- if (llama_should_add_bos_token(model_)) {- llama_token bos = llama_token_bos(model_);- if (state->tokens.empty() || state->tokens[0] != bos)- state->tokens.insert(state->tokens.begin(), bos);- }+ atomize(model_, &state->atoms, state->prompt, PARSE_SPECIAL);// find appropriate slot- slot_ = worker_->server_->slots_->take(state->tokens);+ slot_ = worker_->server_->slots_->take(state->atoms);defer_cleanup(cleanup_slot, this);- // sanity check- if (state->tokens.size() + 1 > slot_->n_ctx())- return send_error(400, "prompt too big for model context size");-// init samplingllama_sampling_context* sampler = create_sampler(params);if (!sampler)@@ -480,9 +470,10 @@ Client::v1_chat_completions()defer_cleanup(cleanup_sampler, sampler);// prefill time- if (!slot_->prefill(state->tokens)) {- SLOG("slot prefill failed");- return send_error(500, "llama_decode prefill failed");+ int prompt_tokens = 0;+ if ((prompt_tokens = slot_->prefill(state->atoms)) < 0) {+ SLOG("slot prefill failed: %s", Slot::describe_error(prompt_tokens));+ return send_error(500, Slot::describe_error(prompt_tokens));}// setup response json@@ -539,7 +530,7 @@ Client::v1_chat_completions()break;}state->piece =- llama_token_to_piece(slot_->ctx_, id, DONT_RENDER_SPECIAL_TOKENS);+ llamafile_token_to_piece(slot_->ctx_, id, DONT_RENDER_SPECIAL_TOKENS);if (!state->piece.empty()) {if (params->stream) {char* p = append_http_response_message(obuf_.p, 200);@@ -571,9 +562,9 @@ Client::v1_chat_completions()} else {Json& usage = response->json["usage"];usage.setObject();- usage["prompt_tokens"].setLong(state->tokens.size());+ usage["prompt_tokens"].setLong(prompt_tokens);usage["completion_tokens"].setLong(completion_tokens);- usage["total_tokens"].setLong(completion_tokens + state->tokens.size());+ usage["total_tokens"].setLong(completion_tokens + prompt_tokens);choice["message"].setObject();choice["message"]["role"].setString("assistant");choice["message"]["content"].setString(std::move(response->content));diff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cppindex bf73e381..1ae3240c 100644--- a/llamafile/server/worker.cpp+++ b/llamafile/server/worker.cpp@@ -16,20 +16,17 @@// limitations under the License.#include "worker.h"--#include <assert.h>-#include <exception>-#include <pthread.h>-#include "llamafile/llamafile.h"+#include "llamafile/server/client.h"+#include "llamafile/server/log.h"+#include "llamafile/server/server.h"+#include "llamafile/server/signals.h"+#include "llamafile/server/tokenbucket.h"#include "llamafile/threadlocal.h"#include "llamafile/trust.h"--#include "client.h"-#include "log.h"-#include "server.h"-#include "signals.h"-#include "tokenbucket.h"+#include <assert.h>+#include <exception>+#include <pthread.h>Worker::Worker(Server* server, llama_model* model): server_(server), client_(model)diff --git a/llamafile/server/www/chatbot.css b/llamafile/server/www/chatbot.cssindex 4fbea750..e23fba4d 100644--- a/llamafile/server/www/chatbot.css+++ b/llamafile/server/www/chatbot.css@@ -108,6 +108,11 @@ ul li:first-child {margin-right: auto;}+.message img {+ max-width: 100%;+ height: auto;+}+.chat-input-container {padding: 1rem;border-top: 1px solid #e9ecef;@@ -124,7 +129,7 @@ ul li:first-child {outline: none;resize: none;max-height: 200px;- min-height: 44px;+ min-height: 50px;line-height: 1.5;font-family: inherit;overflow-y: auto;@@ -132,6 +137,11 @@ ul li:first-child {scrollbar-color: #cbd5e1 transparent; /* Firefox */}+.chat-input.drag-over {+ border-color: #0366d6;+ background-color: #f6f8fa;+}+/* Chrome, Edge, and Safari */.chat-input::-webkit-scrollbar {width: 8px;diff --git a/llamafile/server/www/chatbot.js b/llamafile/server/www/chatbot.jsindex 72b675e1..13e534b2 100644--- a/llamafile/server/www/chatbot.js+++ b/llamafile/server/www/chatbot.js@@ -22,6 +22,7 @@ const stopButton = document.getElementById("stop-button");let abortController = null;let streamingMessageContent = [];+let uploadedFiles = [];let chatHistory = [{@@ -32,6 +33,10 @@ let chatHistory = [},];+function generateId() {+ return Date.now().toString(36) + Math.random().toString(36).substring(2);+}+function createMessageElement(content, role) {const messageDiv = document.createElement("div");messageDiv.classList.add("message", role);@@ -117,8 +122,16 @@ function stopMessage() {}}+function fixUploads(str) {+ str = uploadedFiles.reduce(+ (text, [from, to]) => text.replaceAll(from, to),+ str);+ uploadedFiles.length = 0;+ return str;+}+async function sendMessage() {- const message = chatInput.value.trim();+ const message = fixUploads(chatInput.value.trim());if (!message) return;// disable input while processing@@ -153,22 +166,22 @@ async function sendMessage() {}),signal: abortController.signal});-- if (!response.ok)- throw new Error(`HTTP error! status: ${response.status}`);-- // handle the stream- await handleChatStream(response);-- // update chat history with response- const lastMessage = streamingMessageContent.join("");- chatHistory.push({ role: "assistant", content: lastMessage });-+ if (response.ok) {+ await handleChatStream(response);+ const lastMessage = streamingMessageContent.join("");+ chatHistory.push({ role: "assistant", content: lastMessage });+ } else {+ console.error("sendMessage() failed due to server error", response);+ chatMessages.appendChild(createMessageElement(+ `Server replied with error code ${response.status} ${response.statusText}`,+ "system"));+ cleanupAfterMessage();+ }} catch (error) {if (error.name !== "AbortError") {- console.error("Error:", error);+ console.error("sendMessage() failed due to unexpected exception", error);const errorMessage = createMessageElement(- "Sorry, there was an error processing your request.",+ "There was an error processing your request.","system");chatMessages.appendChild(errorMessage);}@@ -176,24 +189,88 @@ async function sendMessage() {}}-// setup chat window-chatMessages.innerHTML = "";-for (let i = 0; i < chatHistory.length; i++) {- chatMessages.appendChild(createMessageElement(chatHistory[i].content,- chatHistory[i].role));- scrollToBottom();+function onDragBegin(e) {+ e.preventDefault();+ e.stopPropagation();+ chatInput.classList.add('drag-over');+}++function onDragEnd(e) {+ e.preventDefault();+ e.stopPropagation();+ chatInput.classList.remove('drag-over');}-// setup events-sendButton.addEventListener("click", sendMessage);-stopButton.addEventListener("click", stopMessage);-chatInput.addEventListener("input", onChatInput);-chatInput.addEventListener("keydown", function(e) {+function onDrop(e) {+ const files = e.dataTransfer.files;+ [...files].forEach(onFile);+}++function onPaste(e) {+ const items = e.clipboardData.items;+ for (let item of items) {+ if (item.type.startsWith('image/')) {+ e.preventDefault();+ onFile(item.getAsFile());+ return;+ }+ }+}++function onFile(file) {+ if (!file.type.startsWith('image/')) {+ console.warn('Only image files are supported');+ return;+ }+ if (file.size > 1 * 1024 * 1024) {+ console.warn('Image is larger than 1mb');+ return;+ }+ const reader = new FileReader();+ reader.onloadend = function() {+ const description = file.name;+ const realDataUri = reader.result;+ const fakeDataUri = 'data:,placeholder/' + generateId();+ uploadedFiles.push([fakeDataUri, realDataUri]);+ insertText(chatInput, ``);+ };+ reader.readAsDataURL(file);+}++function insertText(elem, text) {+ const pos = elem.selectionStart;+ elem.value = elem.value.slice(0, pos) + text + elem.value.slice(pos);+ const newPos = pos + text.length;+ elem.setSelectionRange(newPos, newPos);+ elem.focus();+ elem.dispatchEvent(new Event('input'));+}++function onKeyDown(e) {if (e.key === "Enter" && !e.shiftKey) {e.preventDefault();sendMessage();}-});+}++function chatbot() {+ chatMessages.innerHTML = "";+ for (let i = 0; i < chatHistory.length; i++) {+ chatMessages.appendChild(createMessageElement(chatHistory[i].content,+ chatHistory[i].role));+ scrollToBottom();+ }+ sendButton.addEventListener("click", sendMessage);+ stopButton.addEventListener("click", stopMessage);+ chatInput.addEventListener("input", onChatInput);+ chatInput.addEventListener("keydown", onKeyDown);+ document.body.addEventListener('dragenter', onDragBegin, false);+ document.body.addEventListener('dragover', onDragBegin, false);+ document.body.addEventListener('dragleave', onDragEnd, false);+ document.body.addEventListener('drop', onDragEnd, false);+ document.body.addEventListener('drop', onDrop, false);+ document.body.addEventListener('paste', onPaste);+ chatInput.focus();+}-// focus input-chatInput.focus();+chatbot();diff --git a/llamafile/server/www/render_markdown.js b/llamafile/server/www/render_markdown.jsindex ee539b10..07629899 100644--- a/llamafile/server/www/render_markdown.js+++ b/llamafile/server/www/render_markdown.js@@ -48,6 +48,7 @@ class RenderMarkdown extends Highlighter {static OLDCODE = 33;static BLOCKQUOTE = 34;static TICK_TICK_TICK = 35;+ static EXCLAIM = 36;static STYLE_SPAN = 16;static STYLE_STRONG = 16;@@ -100,6 +101,7 @@ class RenderMarkdown extends Highlighter {this.bqline = false;this.tick1 = 0;this.tick2 = 0;+ this.is_image = false;}inside(style) {@@ -270,7 +272,12 @@ class RenderMarkdown extends Highlighter {} else if (c == '~') {this.state = RenderMarkdown.TILDE;this.got();+ } else if (c == '!') {+ this.is_image = true;+ this.state = RenderMarkdown.EXCLAIM;+ this.got();} else if (c == '[') {+ this.is_image = false;this.state = RenderMarkdown.LSB;this.got();} else if (c == '<') {@@ -694,6 +701,15 @@ class RenderMarkdown extends Highlighter {this.state = RenderMarkdown.LAB;break;+ case RenderMarkdown.EXCLAIM:+ if (c == '[') {+ this.state = RenderMarkdown.LSB;+ } else {+ this.append('!');+ this.epsilon(RenderMarkdown.NORMAL);+ }+ break;+case RenderMarkdown.LSB:if (c == ']') {this.state = RenderMarkdown.LSB_RSB;@@ -723,9 +739,15 @@ class RenderMarkdown extends Highlighter {case RenderMarkdown.LSB_RSB_LPAREN:if (c == ')') {- let a = this.push("A", "");- a.innerText = this.text;- a.href = this.href;+ if (this.is_image) {+ let a = this.push("IMG", "");+ a.alt = this.text;+ a.src = this.href;+ } else {+ let a = this.push("A", "");+ a.innerText = this.text;+ a.href = this.href;+ }this.pop();this.href = '';this.text = '';