Skip to content

Distribute and run LLMs with a single file.

Add vision model support to new server

Justine Tunneybb917bdbe71d
diff --git a/llamafile/chatbot_eval.cpp b/llamafile/chatbot_eval.cpp
index 49f8b5f0..397f8cb4 100644
--- a/llamafile/chatbot_eval.cpp
+++ b/llamafile/chatbot_eval.cpp
@@ -16,17 +16,17 @@
// limitations under the License.
#include "chatbot.h"
-
-#include <cassert>
-#include <string>
-#include <vector>
-
+#include "llama.cpp/base64.h"
#include "llama.cpp/common.h"
#include "llama.cpp/llama.h"
#include "llama.cpp/llava/llava.h"
#include "llamafile/datauri.h"
#include "llamafile/image.h"
+#include "llamafile/llama.h"
#include "llamafile/string.h"
+#include <cassert>
+#include <string>
+#include <vector>
namespace lf {
namespace chatbot {
@@ -60,7 +60,7 @@ bool eval_tokens(std::vector<llama_token> tokens) {
return true;
}
-bool eval_image_embed(const struct llava_image_embed *image_embed) {
+bool eval_image_embed(const llava_image_embed *image_embed) {
int N = image_embed->n_image_pos;
if (tokens_used() + N > llama_n_ctx(g_ctx))
return out_of_context(N);
@@ -113,7 +113,7 @@ bool eval_token(int id) {
}
bool eval_plain_text(const std::string &str, bool add_special, bool parse_special) {
- return eval_tokens(llama_tokenize(g_model, str, add_special, parse_special));
+ return eval_tokens(llamafile_tokenize(g_model, str, add_special, parse_special));
}
bool eval_string(std::string_view s, bool add_special, bool parse_special) {
@@ -122,22 +122,26 @@ bool eval_string(std::string_view s, bool add_special, bool parse_special) {
size_t pos = s.find("data:", i);
if (pos == std::string_view::npos)
return eval_plain_text(std::string(s), add_special, parse_special);
+ i = pos + 5;
DataUri uri;
size_t end = uri.parse(s.substr(pos + 5));
- if (end == std::string_view::npos) {
- i = pos + 5;
+ if (end == std::string_view::npos)
continue;
- }
- std::string image = uri.decode();
- if (!is_image(image)) {
- i = pos + 5;
+ if (!uri.mime.starts_with("image/"))
+ continue;
+ std::string image;
+ try {
+ image = uri.decode();
+ } catch (const base64_error &e) {
continue;
}
+ if (!is_image(image))
+ continue;
if (!eval_plain_text(std::string(s.substr(0, pos)), add_special, parse_special))
return false;
if (!eval_image(image))
return false;
- s = s.substr(pos + 5 + end);
+ s = s.substr(i + end);
i = 0;
}
}
diff --git a/llamafile/chatbot_main.cpp b/llamafile/chatbot_main.cpp
index 61e1c1e9..9a985b9f 100644
--- a/llamafile/chatbot_main.cpp
+++ b/llamafile/chatbot_main.cpp
@@ -29,6 +29,7 @@
#include "llama.cpp/server/server.h"
#include "llamafile/color.h"
#include "llamafile/compute.h"
+#include "llamafile/llama.h"
#include "llamafile/string.h"
namespace lf {
@@ -69,7 +70,7 @@ std::string describe_compute(void) {
std::string token_to_piece(const struct llama_context *ctx, llama_token token, bool special) {
if (token == IMAGE_PLACEHOLDER_TOKEN)
return "⁑";
- return llama_token_to_piece(ctx, token, special);
+ return llamafile_token_to_piece(ctx, token, special);
}
void on_server_listening(const char *host, int port) {
diff --git a/llamafile/flags.cpp b/llamafile/flags.cpp
index 3266107e..55b1ff4b 100644
--- a/llamafile/flags.cpp
+++ b/llamafile/flags.cpp
@@ -64,7 +64,7 @@ int FLAG_batch = 2048;
int FLAG_ctx_size = 8192;
int FLAG_flash_attn = false;
int FLAG_gpu = 0;
-int FLAG_http_ibuf_size = 1024 * 1024;
+int FLAG_http_ibuf_size = 5 * 1024 * 1024;
int FLAG_http_obuf_size = 1024 * 1024;
int FLAG_keepalive = 5;
int FLAG_main_gpu = 0;
diff --git a/llamafile/llama.cpp b/llamafile/llama.cpp
index 24628403..58b8904f 100644
--- a/llamafile/llama.cpp
+++ b/llamafile/llama.cpp
@@ -16,6 +16,10 @@
// limitations under the License.
#include "llama.h"
+#include "llama.cpp/llama.h"
+#include <cassert>
+#include <string>
+#include <vector>
int llamafile_token_eot(llama_model *model) {
llama_token eot = llama_token_eot(model);
@@ -23,3 +27,37 @@ int llamafile_token_eot(llama_model *model) {
return eot;
return llama_token_eos(model);
}
+
+std::string llamafile_token_to_piece(const llama_context *ctx, llama_token token, bool special) {
+ std::string piece;
+ piece.resize(piece.capacity());
+ const int n_chars =
+ llama_token_to_piece(llama_get_model(ctx), token, &piece[0], piece.size(), 0, special);
+ if (n_chars < 0) {
+ piece.resize(-n_chars);
+ int check =
+ llama_token_to_piece(llama_get_model(ctx), token, &piece[0], piece.size(), 0, special);
+ unassert(check == -n_chars);
+ } else {
+ piece.resize(n_chars);
+ }
+ return piece;
+}
+
+std::vector<llama_token> llamafile_tokenize(const struct llama_model *model,
+ const std::string_view &text, bool add_special,
+ bool parse_special) {
+ int n_tokens = text.size() + 2 * add_special;
+ std::vector<llama_token> result(n_tokens);
+ n_tokens = llama_tokenize(model, text.data(), text.size(), result.data(), result.size(),
+ add_special, parse_special);
+ if (n_tokens < 0) {
+ result.resize(-n_tokens);
+ int check = llama_tokenize(model, text.data(), text.size(), result.data(), result.size(),
+ add_special, parse_special);
+ unassert(check == -n_tokens);
+ } else {
+ result.resize(n_tokens);
+ }
+ return result;
+}
diff --git a/llamafile/llama.h b/llamafile/llama.h
index 429ac0ce..53da0eb9 100644
--- a/llamafile/llama.h
+++ b/llamafile/llama.h
@@ -16,7 +16,8 @@
// limitations under the License.
#pragma once
-#include "llama.cpp/llama.h"
+#include <__fwd/string_view.h>
+#include <__fwd/vector.h>
// Many llama.cpp APIs take boolean parameters at the end. Please favor
// passing these constants as arguments instead, for better readability
@@ -36,4 +37,10 @@
#define RENDER_SPECIAL_TOKENS true
#define DONT_RENDER_SPECIAL_TOKENS false
+struct llama_model;
+struct llama_context;
+
int llamafile_token_eot(llama_model *);
+
+std::string llamafile_token_to_piece(const llama_context *, int, bool);
+std::vector<int> llamafile_tokenize(const llama_model *, const std::string_view &, bool, bool);
diff --git a/llamafile/server/BUILD.mk b/llamafile/server/BUILD.mk
index f170bd8b..54aa5eb0 100644
--- a/llamafile/server/BUILD.mk
+++ b/llamafile/server/BUILD.mk
@@ -12,17 +12,17 @@ LLAMAFILE_SERVER_ASSETS = $(wildcard llamafile/server/www/*)
$(LLAMAFILE_SERVER_OBJS): private CCFLAGS += -g
-o/$(MODE)/llamafile/server/server.a: \
+o/$(MODE)/llamafile/server/server.a: \
$(filter-out %_test.o,$(LLAMAFILE_SERVER_OBJS))
-o/$(MODE)/llamafile/server/main: \
- o/$(MODE)/llamafile/server/main.o \
- o/$(MODE)/llamafile/server/main.1.asc.zip.o \
- o/$(MODE)/llamafile/server/server.a \
- o/$(MODE)/llama.cpp/llama.cpp.a \
- o/$(MODE)/llama.cpp/llava/llava.a \
- o/$(MODE)/double-conversion/double-conversion.a \
- o/$(MODE)/stb/stb.a \
+o/$(MODE)/llamafile/server/main: \
+ o/$(MODE)/llamafile/server/main.o \
+ o/$(MODE)/llamafile/server/main.1.asc.zip.o \
+ o/$(MODE)/llamafile/server/server.a \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
+ o/$(MODE)/llama.cpp/llava/llava.a \
+ o/$(MODE)/double-conversion/double-conversion.a \
+ o/$(MODE)/stb/stb.a \
$(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o)
# turn /zip/llamafile/server/www/...
@@ -31,24 +31,37 @@ $(LLAMAFILE_SERVER_ASSETS:%=o/$(MODE)/%.zip.o): private ZIPOBJ_FLAGS += -C2
$(LLAMAFILE_SERVER_OBJS): llamafile/server/BUILD.mk
-o/$(MODE)/llamafile/server/fastjson_test: \
- o/$(MODE)/llamafile/server/fastjson_test.o \
- o/$(MODE)/llamafile/server/fastjson.o \
- o/$(MODE)/double-conversion/double-conversion.a \
+o/$(MODE)/llamafile/server/atom_test: \
+ o/$(MODE)/llamafile/server/atom_test.o \
+ o/$(MODE)/llamafile/server/atom.o \
+ o/$(MODE)/llamafile/server/image.o \
-o/$(MODE)/llamafile/server/json_test: \
- o/$(MODE)/llamafile/server/json_test.o \
- o/$(MODE)/llamafile/server/json.o \
- o/$(MODE)/llamafile/server/hextoint.o \
- o/$(MODE)/double-conversion/double-conversion.a \
+o/$(MODE)/llamafile/server/image_test: \
+ o/$(MODE)/llamafile/server/image_test.o \
+ o/$(MODE)/llamafile/server/image.o \
-o/$(MODE)/llamafile/server/tokenbucket_test: \
- o/$(MODE)/llamafile/server/tokenbucket_test.o \
- o/$(MODE)/llamafile/server/tokenbucket.o \
- o/$(MODE)/llamafile/server/log.o \
- o/$(MODE)/llama.cpp/llama.cpp.a \
+o/$(MODE)/llamafile/server/fastjson_test: \
+ o/$(MODE)/llamafile/server/fastjson_test.o \
+ o/$(MODE)/llamafile/server/fastjson.o \
+ o/$(MODE)/double-conversion/double-conversion.a \
+
+o/$(MODE)/llamafile/server/json_test: \
+ o/$(MODE)/llamafile/server/json_test.o \
+ o/$(MODE)/llamafile/server/json.o \
+ o/$(MODE)/llamafile/server/hextoint.o \
+ o/$(MODE)/double-conversion/double-conversion.a \
+
+o/$(MODE)/llamafile/server/tokenbucket_test: \
+ o/$(MODE)/llamafile/server/tokenbucket_test.o \
+ o/$(MODE)/llamafile/server/tokenbucket.o \
+ o/$(MODE)/llamafile/server/log.o \
+ o/$(MODE)/llama.cpp/llama.cpp.a \
.PHONY: o/$(MODE)/llamafile/server
-o/$(MODE)/llamafile/server: \
- o/$(MODE)/llamafile/server/main \
- o/$(MODE)/llamafile/server/json_test.runs \
+o/$(MODE)/llamafile/server: \
+ o/$(MODE)/llamafile/server/main \
+ o/$(MODE)/llamafile/server/atom_test.runs \
+ o/$(MODE)/llamafile/server/fastjson_test.runs \
+ o/$(MODE)/llamafile/server/image_test.runs \
+ o/$(MODE)/llamafile/server/json_test.runs \
+ o/$(MODE)/llamafile/server/tokenbucket_test.runs \
diff --git a/llamafile/server/atob.cpp b/llamafile/server/atob.cpp
index e8a1b738..8d125127 100644
--- a/llamafile/server/atob.cpp
+++ b/llamafile/server/atob.cpp
@@ -16,7 +16,6 @@
// limitations under the License.
#include "utils.h"
-
#include <string_view>
bool
diff --git a/llamafile/server/atom.cpp b/llamafile/server/atom.cpp
new file mode 100644
index 00000000..da9f9a9e
--- /dev/null
+++ b/llamafile/server/atom.cpp
@@ -0,0 +1,141 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "atom.h"
+#include "image.h"
+#include <cassert>
+
+/**
+ * @fileoverview Context window content container.
+ *
+ * The `Atom` class is used for tracking what's inside a context window.
+ * Each individual atom can hold one of two values. The first is a token
+ * which is a short piece of text, subjective to each `llama_model`. The
+ * second is an `Image` which uses a separate evaluation process.
+ */
+
+Atom::Atom(int token) : word_(1ull << 56 | (unsigned)token)
+{
+}
+
+Atom::Atom(Image* image) : word_(2ull << 56 | (uintptr_t)image)
+{
+ unassert(image != nullptr);
+}
+
+Atom::Atom(Atom&& other)
+{
+ word_ = other.word_;
+ other.word_ = 0;
+}
+
+Atom::Atom(const Atom& other)
+{
+ if (!other.is_image()) {
+ word_ = other.word_;
+ return;
+ }
+ Image* image = new Image(other.image());
+ word_ = 2ull << 56 | (uintptr_t)image;
+}
+
+Atom::~Atom()
+{
+ if (is_image())
+ delete (Image*)(word_ & 0x00ffffffffffffff);
+}
+
+bool
+Atom::empty() const
+{
+ return !word_;
+}
+
+bool
+Atom::is_token() const
+{
+ return (word_ >> 56) == 1;
+}
+
+bool
+Atom::is_image() const
+{
+ return (word_ >> 56) == 2;
+}
+
+int
+Atom::token() const
+{
+ unassert(is_token());
+ return word_;
+}
+
+const Image&
+Atom::image() const
+{
+ unassert(is_image());
+ return *(Image*)(word_ & 0x00ffffffffffffff);
+}
+
+int
+Atom::ctx_used() const
+{
+ switch (word_ >> 56) {
+ case 1:
+ return 1;
+ case 2:
+ return image().ctx_used();
+ default:
+ return 0;
+ }
+}
+
+bool
+operator<(const Atom& lhs, const Atom& rhs)
+{
+ if (lhs.empty())
+ return !rhs.empty();
+ if (rhs.empty())
+ return false;
+ if (!lhs.is_image()) {
+ if (!rhs.is_image())
+ return lhs.token() < rhs.token();
+ return true;
+ } else {
+ if (rhs.is_image())
+ return lhs.image() < rhs.image();
+ return false;
+ }
+}
+
+bool
+operator==(const Atom& lhs, const Atom& rhs)
+{
+ if (lhs.empty())
+ return rhs.empty();
+ if (rhs.empty())
+ return false;
+ if (lhs.is_token()) {
+ if (!rhs.is_token())
+ return false;
+ return lhs.token() == rhs.token();
+ } else {
+ if (!rhs.is_image())
+ return false;
+ return lhs.image() == rhs.image();
+ }
+}
diff --git a/llamafile/server/atom.h b/llamafile/server/atom.h
new file mode 100644
index 00000000..444020d4
--- /dev/null
+++ b/llamafile/server/atom.h
@@ -0,0 +1,47 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <__fwd/string.h>
+
+class Image;
+
+class Atom
+{
+ public:
+ Atom() = default;
+ explicit Atom(int);
+ explicit Atom(Image*);
+ Atom(const Atom&);
+ Atom(Atom&&);
+ ~Atom();
+ int token() const;
+ bool empty() const;
+ int ctx_used() const;
+ bool is_token() const;
+ bool is_image() const;
+ const Image& image() const;
+
+ private:
+ uint64_t word_ = 0;
+};
+
+bool
+operator<(const Atom&, const Atom&);
+
+bool
+operator==(const Atom&, const Atom&);
diff --git a/llamafile/server/atom_test.cpp b/llamafile/server/atom_test.cpp
new file mode 100644
index 00000000..d9e61c00
--- /dev/null
+++ b/llamafile/server/atom_test.cpp
@@ -0,0 +1,109 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "atom.h"
+#include "image.h"
+#include <cstdlib>
+
+static const Atom atoms[] = {
+ Atom(),
+ Atom(-1),
+ Atom(0),
+ Atom(1),
+ Atom(new Image("hello", 1)),
+ Atom(new Image("there", 1)),
+};
+
+const size_t n = sizeof(atoms) / sizeof(atoms[0]);
+
+void
+test_atom_operator_lt()
+{
+ // irreflexivity: no element can be less than itself
+ for (size_t i = 0; i < n; ++i)
+ if (atoms[i] < atoms[i])
+ exit(1);
+
+ // asymmetry: if x < y then !(y < x)
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ if (atoms[i] < atoms[j] && atoms[j] < atoms[i])
+ exit(2);
+
+ // transitivity: If x < y and y < z then x < z
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ for (size_t k = 0; k < n; ++k)
+ if (atoms[i] < atoms[j] && atoms[j] < atoms[k])
+ if (!(atoms[i] < atoms[k]))
+ exit(3);
+
+ // transitivity of incomparability
+ // if x ≈ y (neither < the other) and y ≈ z, then x ≈ z
+ for (size_t i = 0; i < n; ++i) {
+ for (size_t j = 0; j < n; ++j) {
+ for (size_t k = 0; k < n; ++k) {
+ bool i_equiv_j =
+ !((atoms[i] < atoms[j]) || (atoms[j] < atoms[i]));
+ bool j_equiv_k =
+ !((atoms[j] < atoms[k]) || (atoms[k] < atoms[j]));
+ bool i_equiv_k =
+ !((atoms[i] < atoms[k]) || (atoms[k] < atoms[i]));
+ if (i_equiv_j && j_equiv_k && !i_equiv_k)
+ exit(4);
+ }
+ }
+ }
+}
+
+void
+test_atom_operator_eq()
+{
+ // reflexivity: every element must equal itself
+ for (size_t i = 0; i < n; ++i)
+ if (!(atoms[i] == atoms[i]))
+ exit(5);
+
+ // symmetry: if x == y then y == x
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ if ((atoms[i] == atoms[j]) != (atoms[j] == atoms[i]))
+ exit(6);
+
+ // transitivity: if x == y and y == z then x == z
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ for (size_t k = 0; k < n; ++k)
+ if (atoms[i] == atoms[j] && atoms[j] == atoms[k])
+ if (!(atoms[i] == atoms[k]))
+ exit(7);
+
+ // consistency with operator<: if x == y then !(x < y) && !(y < x)
+ // this ensures == and < agree on their notion of equivalence
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ if (atoms[i] == atoms[j])
+ if (atoms[i] < atoms[j] || atoms[j] < atoms[i])
+ exit(8);
+}
+
+int
+main()
+{
+ test_atom_operator_lt();
+ test_atom_operator_eq();
+}
diff --git a/llamafile/server/atomize.cpp b/llamafile/server/atomize.cpp
new file mode 100644
index 00000000..a4c4fe35
--- /dev/null
+++ b/llamafile/server/atomize.cpp
@@ -0,0 +1,73 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "atom.h"
+#include "llama.cpp/base64.h"
+#include "llamafile/chatbot.h"
+#include "llamafile/datauri.h"
+#include "llamafile/image.h"
+#include "llamafile/llama.h"
+#include "llamafile/server/image.h"
+#include <string>
+#include <vector>
+
+static void
+append_tokens(const llama_model* model,
+ std::vector<Atom>* result,
+ const std::string_view& s,
+ bool parse_special)
+{
+ std::vector<int> tokens = llamafile_tokenize(
+ model, std::string(s), DONT_ADD_SPECIAL, parse_special);
+ for (int token : tokens)
+ result->emplace_back(token);
+}
+
+void
+atomize(const llama_model* model,
+ std::vector<Atom>* result,
+ std::string_view s,
+ bool parse_special)
+{
+ size_t i = 0;
+ for (;;) {
+ size_t pos = s.find("data:", i);
+ if (pos == std::string_view::npos) {
+ append_tokens(model, result, s, parse_special);
+ return;
+ }
+ i = pos + 5;
+ DataUri uri;
+ size_t end = uri.parse(s.substr(i));
+ if (end == std::string_view::npos)
+ continue;
+ if (!uri.mime.starts_with("image/"))
+ continue;
+ std::string image;
+ try {
+ image = uri.decode();
+ } catch (const base64_error& e) {
+ continue;
+ }
+ if (!lf::get_image_type(image))
+ continue;
+ append_tokens(model, result, s.substr(0, pos), parse_special);
+ result->emplace_back(new Image(image, -1));
+ s = s.substr(i + end);
+ i = 0;
+ }
+}
diff --git a/llamafile/server/buffer.cpp b/llamafile/server/buffer.cpp
index fa0d0f00..ecd41226 100644
--- a/llamafile/server/buffer.cpp
+++ b/llamafile/server/buffer.cpp
@@ -16,7 +16,6 @@
// limitations under the License.
#include "buffer.h"
-
#include <sys/mman.h>
#include <unistd.h>
diff --git a/llamafile/server/doc/v1_chat_completions.md b/llamafile/server/doc/v1_chat_completions.md
index fce4af08..cee23faa 100644
--- a/llamafile/server/doc/v1_chat_completions.md
+++ b/llamafile/server/doc/v1_chat_completions.md
@@ -2,6 +2,19 @@
The `/v1/chat/completions` endpoint lets you build a chatbot.
+The client is responsible for maintaining the history of a conversation.
+The conversation is divided into messages, each having text content, and
+a role name that describes the speaker of that text. Each time a request
+is made to the API, the full conversation history is provided, and the
+server will generate content for the next message in its history, as the
+role of the assistant.
+
+This endpoint supports the following features:
+
+1. Automatic context window caching across requests
+2. Streaming responses to give you generated tokens in real time
+3. Image uploads via data: URIs
+
## Request URIs
- `/v1/chat/completions` (OpenAI API compatible)
@@ -43,7 +56,9 @@ The `/v1/chat/completions` endpoint lets you build a chatbot.
- `content`: `string`
- The content field holds text the system, user, or assistant said.
+ The content field holds text the system, user, or assistant said. It
+ is recommended that this be treated as markdown. This field may
+ contain images which should be embedded as data: URIs.
The completion endpoint will then generate at least one additional
chat message. If the client wishes to request another completion, then
@@ -160,6 +175,54 @@ unsupported:
- `function_call`
- `parallel_tool_calls`
+## Context Caching
+
+It may seem surprising that the entire conversation history should be
+uploaded each time a new message from the assistant is desired. This is
+fast because the server has a fixed number of slots which are used for
+caching context windows. When an HTTP client sends a chat completion
+request, a prefix search is made for a slot that best matches the
+supplied conversation history. Once a slot is chosen, unrelated content
+(possibly from another user's conversation) is deleted, the common
+prefix is preserved, and the remaining portions are prefilled. If all
+slots are in use, then the server handler waits for one to be free.
+
+## Image Uploads
+
+If a vision model was specified by passing the `--mmproj` flag, then
+each slot will additionally maintain a vision model context, which is
+used to encode embeddings for any data: URIs holding valid images.
+
+The data URI must conform to RFC2397. For example, a 1x1 transparent
+pixel could be encoded as:
+
+ data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==
+
+Images can be included by any role. Multiple images may be embedded
+within a single message content. The server will evaluate images in the
+same order as any surrounding text, e.g.
+
+```markdown
+![1x1.gif](data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==)
+<img src="data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==">
+```
+
+The supported image file formats are:
+
+- PNG
+- JPEG
+- GIF
+- BMP
+- TGA
+- HDR
+- PGM
+- PPM
+- PIC
+
+The declared mime type of the data URI must begin with `image/`. Its
+subtype is largely ignored, since the file type is inferred from the
+binary payload.
+
## See Also
- [LLaMAfiler Documentation Index](index.md)
diff --git a/llamafile/server/embedding.cpp b/llamafile/server/embedding.cpp
index 8ac915fc..94d927c4 100644
--- a/llamafile/server/embedding.cpp
+++ b/llamafile/server/embedding.cpp
@@ -23,12 +23,11 @@
#include <vector>
#include "llama.cpp/llama.h"
-
-#include "cleanup.h"
-#include "fastjson.h"
-#include "json.h"
-#include "log.h"
-#include "utils.h"
+#include "llamafile/server/cleanup.h"
+#include "llamafile/server/fastjson.h"
+#include "llamafile/server/json.h"
+#include "llamafile/server/log.h"
+#include "llamafile/server/utils.h"
struct EmbeddingParams
{
diff --git a/llamafile/server/fastjson_test.cpp b/llamafile/server/fastjson_test.cpp
index 5e01282e..d63d3fb1 100644
--- a/llamafile/server/fastjson_test.cpp
+++ b/llamafile/server/fastjson_test.cpp
@@ -22,52 +22,50 @@
#include <stdio.h>
#include <string>
-using namespace ctl;
-
-string
+std::string
encode_json(int x) noexcept
{
char buf[12];
return { buf, (size_t)(encode_json(buf, x) - buf) };
}
-string
+std::string
encode_json(long x) noexcept
{
char buf[21];
return { buf, (size_t)(encode_json(buf, x) - buf) };
}
-string
+std::string
encode_json(double x) noexcept
{
char buf[128];
return { buf, (size_t)(encode_json(buf, x) - buf) };
}
-string
+std::string
encode_json(unsigned x) noexcept
{
char buf[128];
return { buf, (size_t)(encode_json(buf, x) - buf) };
}
-string
+std::string
encode_json(unsigned long x) noexcept
{
char buf[128];
return { buf, (size_t)(encode_json(buf, x) - buf) };
}
-string
-encode_json(const string_view x) noexcept
+std::string
+encode_json(const std::string_view x) noexcept
{
char buf[128];
return { buf, (size_t)(encode_json(buf, x) - buf) };
}
-string
-encode_js_string_literal(const string_view x) noexcept
+std::string
+encode_js_string_literal(const std::string_view x) noexcept
{
char buf[256]; // this isn't secure (no guard page)
return { buf, (size_t)(encode_json(buf, x) - buf) };
@@ -94,7 +92,7 @@ main(int argc, char* argv[])
if (encode_json("") != "\"\"")
return 8;
- if (encode_json(string_view("\0\1", 2)) != "\"\\u0000\\u0001\"")
+ if (encode_json(std::string_view("\0\1", 2)) != "\"\\u0000\\u0001\"")
return 9;
if (encode_json("\n\"\\\t") != "\"\\n\\\"\\\\\\t\"")
return 10;
diff --git a/llamafile/server/image.cpp b/llamafile/server/image.cpp
new file mode 100644
index 00000000..be684429
--- /dev/null
+++ b/llamafile/server/image.cpp
@@ -0,0 +1,70 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "image.h"
+#include "llamafile/llamafile.h"
+#include <cassert>
+#include <utility>
+
+Image::~Image() = default;
+
+Image::Image(const Image& old) : Image(old.bytes_, old.ctx_used_)
+{
+}
+
+Image::Image(const std::string_view& bytes, int ctx_used)
+ : bytes_(bytes), ctx_used_(ctx_used)
+{
+}
+
+const std::string&
+Image::bytes() const
+{
+ return bytes_;
+}
+
+int
+Image::ctx_used() const
+{
+ // there are two use cases for image
+ //
+ // 1. when the http client asks for their chat messages to be
+ // tokenized, ctx_used should be set to -1, because we don't want
+ // to pay the price of the expensive clip image encode operation.
+ // since ctx_used isn't considered by comparators, this is a fine
+ // object state that's useful for searching data structures.
+ //
+ // 2. when an atom is actually evaluated by the model, if it's an
+ // image atom it'll be recreated with ctx_used specified. this is
+ // what slot does when maintaining its history, since it's the
+ // only thing that cares right now about context usage.
+ //
+ unassert(ctx_used_ > 0);
+ return ctx_used_;
+}
+
+bool
+operator<(const Image& lhs, const Image& rhs)
+{
+ return lhs.bytes() < rhs.bytes();
+}
+
+bool
+operator==(const Image& lhs, const Image& rhs)
+{
+ return lhs.bytes() == rhs.bytes();
+}
diff --git a/llamafile/server/image.h b/llamafile/server/image.h
new file mode 100644
index 00000000..547444c6
--- /dev/null
+++ b/llamafile/server/image.h
@@ -0,0 +1,39 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <string>
+
+class Image
+{
+ public:
+ ~Image();
+ Image(const Image&);
+ Image(const std::string_view&, int);
+ const std::string& bytes() const;
+ int ctx_used() const;
+
+ private:
+ std::string bytes_;
+ int ctx_used_;
+};
+
+bool
+operator<(const Image&, const Image&);
+
+bool
+operator==(const Image&, const Image&);
diff --git a/llamafile/server/image_test.cpp b/llamafile/server/image_test.cpp
new file mode 100644
index 00000000..6f0982fc
--- /dev/null
+++ b/llamafile/server/image_test.cpp
@@ -0,0 +1,107 @@
+// -*- mode:c++;indent-tabs-mode:nil;c-basic-offset:4;coding:utf-8 -*-
+// vi: set et ft=cpp ts=4 sts=4 sw=4 fenc=utf-8 :vi
+//
+// Copyright 2024 Mozilla Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "atom.h"
+#include "image.h"
+#include <cstdlib>
+
+static const Image images[] = {
+ Image("", 0),
+ Image("hello", 1),
+ Image("there", 1),
+ Image("there", 2),
+};
+
+const size_t n = sizeof(images) / sizeof(images[0]);
+
+void
+test_image_operator_lt()
+{
+ // irreflexivity: no element can be less than itself
+ for (size_t i = 0; i < n; ++i)
+ if (images[i] < images[i])
+ exit(1);
+
+ // asymmetry: if x < y then !(y < x)
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ if (images[i] < images[j] && images[j] < images[i])
+ exit(2);
+
+ // transitivity: If x < y and y < z then x < z
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ for (size_t k = 0; k < n; ++k)
+ if (images[i] < images[j] && images[j] < images[k])
+ if (!(images[i] < images[k]))
+ exit(3);
+
+ // transitivity of incomparability
+ // if x ≈ y (neither < the other) and y ≈ z, then x ≈ z
+ for (size_t i = 0; i < n; ++i) {
+ for (size_t j = 0; j < n; ++j) {
+ for (size_t k = 0; k < n; ++k) {
+ bool i_equiv_j =
+ !((images[i] < images[j]) || (images[j] < images[i]));
+ bool j_equiv_k =
+ !((images[j] < images[k]) || (images[k] < images[j]));
+ bool i_equiv_k =
+ !((images[i] < images[k]) || (images[k] < images[i]));
+ if (i_equiv_j && j_equiv_k && !i_equiv_k)
+ exit(4);
+ }
+ }
+ }
+}
+
+void
+test_image_operator_eq()
+{
+ // reflexivity: every element must equal itself
+ for (size_t i = 0; i < n; ++i)
+ if (!(images[i] == images[i]))
+ exit(5);
+
+ // symmetry: if x == y then y == x
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ if ((images[i] == images[j]) != (images[j] == images[i]))
+ exit(6);
+
+ // transitivity: if x == y and y == z then x == z
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ for (size_t k = 0; k < n; ++k)
+ if (images[i] == images[j] && images[j] == images[k])
+ if (!(images[i] == images[k]))
+ exit(7);
+
+ // consistency with operator<: if x == y then !(x < y) && !(y < x)
+ // this ensures == and < agree on their notion of equivalence
+ for (size_t i = 0; i < n; ++i)
+ for (size_t j = 0; j < n; ++j)
+ if (images[i] == images[j])
+ if (images[i] < images[j] || images[j] < images[i])
+ exit(8);
+}
+
+int
+main()
+{
+ test_image_operator_lt();
+ test_image_operator_eq();
+}
diff --git a/llamafile/server/log.cpp b/llamafile/server/log.cpp
index b7834540..a6f07530 100644
--- a/llamafile/server/log.cpp
+++ b/llamafile/server/log.cpp
@@ -16,12 +16,10 @@
// limitations under the License.
#include "log.h"
-
+#include "time.h"
#include <pthread.h>
#include <string.h>
-#include "time.h"
-
static thread_local char g_thread_name[128];
const char*
diff --git a/llamafile/server/main.cpp b/llamafile/server/main.cpp
index 7dd5f925..cbe87c10 100644
--- a/llamafile/server/main.cpp
+++ b/llamafile/server/main.cpp
@@ -21,16 +21,15 @@
#include "llama.cpp/llama.h"
#include "llamafile/llamafile.h"
#include "llamafile/pool.h"
+#include "llamafile/server/log.h"
+#include "llamafile/server/server.h"
+#include "llamafile/server/signals.h"
+#include "llamafile/server/slots.h"
+#include "llamafile/server/time.h"
+#include "llamafile/server/tokenbucket.h"
+#include "llamafile/server/utils.h"
#include "llamafile/version.h"
-#include "log.h"
-#include "server.h"
-#include "signals.h"
-#include "slots.h"
-#include "time.h"
-#include "tokenbucket.h"
-#include "utils.h"
-
Server* g_server;
int
diff --git a/llamafile/server/server.cpp b/llamafile/server/server.cpp
index 108a9d7e..32d06bbf 100644
--- a/llamafile/server/server.cpp
+++ b/llamafile/server/server.cpp
@@ -28,10 +28,9 @@
#include "llamafile/crash.h"
#include "llamafile/llamafile.h"
-
-#include "log.h"
-#include "server.h"
-#include "worker.h"
+#include "llamafile/server/log.h"
+#include "llamafile/server/server.h"
+#include "llamafile/server/worker.h"
Server::Server(int fd, Slots* slots, llama_model* model)
: fd(fd), slots_(slots), model_(model)
diff --git a/llamafile/server/signals.cpp b/llamafile/server/signals.cpp
index 1dd9da96..09bf00d8 100644
--- a/llamafile/server/signals.cpp
+++ b/llamafile/server/signals.cpp
@@ -21,11 +21,10 @@
#include <ucontext.h>
#include "llamafile/crash.h"
+#include "llamafile/server/log.h"
+#include "llamafile/server/server.h"
#include "llamafile/threadlocal.h"
-#include "log.h"
-#include "server.h"
-
static struct
{
struct sigaction sigint; // ctrl-c
diff --git a/llamafile/server/slot.cpp b/llamafile/server/slot.cpp
index 0f3bb75f..a58eb9d9 100644
--- a/llamafile/server/slot.cpp
+++ b/llamafile/server/slot.cpp
@@ -16,18 +16,23 @@
// limitations under the License.
#include "slot.h"
-#include "llama.cpp/common.h"
+#include "llama.cpp/llava/clip.h"
+#include "llama.cpp/llava/llava.h"
+#include "llamafile/image.h"
#include "llamafile/llama.h"
#include "llamafile/llamafile.h"
#include "llamafile/macros.h"
+#include "llamafile/server/atom.h"
+#include "llamafile/server/image.h"
+#include "llamafile/server/log.h"
#include "llamafile/vector.h"
#include "llamafile/version.h"
-#include "log.h"
+#include <algorithm>
#include <cassert>
#include <cosmo.h>
static int
-ctx_size(llama_model* model)
+choose_ctx_size(llama_model* model)
{
int n_ctx_train = llama_n_ctx_train(model);
if (FLAG_ctx_size <= 0 || FLAG_ctx_size > n_ctx_train)
@@ -49,6 +54,27 @@ generate_system_fingerprint(const llama_context_params* cparams)
return b;
}
+const char*
+Slot::describe_error(int err)
+{
+ switch (err) {
+ case uninitialized:
+ return "uninitialized";
+ case out_of_context:
+ return "out_of_context";
+ case no_vision_model:
+ return "no_vision_model";
+ case decode_token_failed:
+ return "decode_token_failed";
+ case decode_image_failed:
+ return "decode_image_failed";
+ case encode_image_failed:
+ return "encode_image_failed";
+ default:
+ return "bad_error_code";
+ }
+}
+
Slot::Slot(llama_model* model) : model_(model)
{
}
@@ -57,6 +83,8 @@ Slot::~Slot()
{
if (ctx_)
llama_free(ctx_);
+ if (clip_ctx_)
+ clip_free(clip_ctx_);
}
bool
@@ -68,7 +96,7 @@ Slot::start()
cparams.embeddings_only = false;
cparams.logits_all = false;
cparams.seed = 12345;
- cparams.n_ctx = ctx_size(model_);
+ cparams.n_ctx = choose_ctx_size(model_);
cparams.n_batch = FLAG_batch;
cparams.n_ubatch = FLAG_ubatch;
cparams.n_seq_max = 1;
@@ -91,78 +119,183 @@ Slot::start()
system_fingerprint_ = generate_system_fingerprint(&cparams);
if (!(ctx_ = llama_new_context_with_model(model_, cparams)))
return false;
+ if (FLAG_mmproj)
+ if (!(clip_ctx_ = clip_model_load(FLAG_mmproj, FLAG_verbose)))
+ return false;
return true;
}
int
-Slot::n_ctx()
+Slot::ctx_size() const
{
return llama_n_ctx(ctx_);
}
-bool
-Slot::eval_token(int id)
+int
+Slot::ctx_used() const
{
- std::vector<int> tokens;
- tokens.push_back(id);
- return eval_tokens(tokens);
+ int tokens = 0;
+ for (size_t i = 0; i < history_.size(); ++i)
+ tokens += history_[i].ctx_used();
+ return tokens;
}
-bool
-Slot::eval_tokens(std::vector<int> tokens)
+int
+Slot::eval_token(int token)
{
- unassert(ctx_);
- if (history_.size() + tokens.size() > n_ctx())
- return false;
- for (int i = 0; i < tokens.size(); i += FLAG_batch) {
- int n_eval = (int)tokens.size() - i;
+ return eval_tokens({ token });
+}
+
+int
+Slot::eval_tokens(const std::vector<int>& tokens)
+{
+ if (!ctx_)
+ return uninitialized;
+ if (tokens.empty())
+ return 0;
+ int N = tokens.size();
+ int used = ctx_used();
+ if (used + N > ctx_size())
+ return out_of_context;
+ std::vector<int> toks(tokens); // TODO(jart): is copying really needed?
+ for (int i = 0; i < N; i += FLAG_batch) {
+ int n_eval = N - i;
if (n_eval > FLAG_batch)
n_eval = FLAG_batch;
- if (llama_decode(
- ctx_,
- llama_batch_get_one(&tokens[i], n_eval, history_.size(), 0)))
- return false;
- history_.insert(
- history_.end(), tokens.begin() + i, tokens.begin() + i + n_eval);
+ if (llama_decode(ctx_,
+ { .n_tokens = n_eval,
+ .token = &toks[i],
+ .all_pos_0 = used,
+ .all_pos_1 = 1 }))
+ return decode_token_failed;
+ for (int j = 0; j < n_eval; ++j)
+ history_.emplace_back(toks[i + j]);
+ used += n_eval;
}
last_used_ = timespec_real();
- return true;
+ return N;
}
-bool
-Slot::prefill(const std::vector<int>& tokens)
+int
+Slot::eval_image(const std::string_view& bytes)
+{
+ if (!ctx_)
+ return uninitialized;
+ if (!clip_ctx_)
+ return no_vision_model;
+ llava_image_embed* image_embed =
+ llava_image_embed_make_with_bytes(clip_ctx_,
+ FLAG_threads_batch,
+ (const unsigned char*)bytes.data(),
+ bytes.size());
+ if (!image_embed)
+ return encode_image_failed;
+ int used = ctx_used();
+ int N = image_embed->n_image_pos;
+ if (used + N > ctx_size()) {
+ llava_image_embed_free(image_embed);
+ return out_of_context;
+ }
+ int n_embd = llama_n_embd(llama_get_model(ctx_));
+ for (int i = 0; i < N; i += FLAG_batch) {
+ int n_eval = N - i;
+ if (n_eval > FLAG_batch)
+ n_eval = FLAG_batch;
+ if (llama_decode(ctx_,
+ { .n_tokens = n_eval,
+ .embd = image_embed->embed + i * n_embd,
+ .all_pos_0 = used,
+ .all_pos_1 = 1 })) {
+ llava_image_embed_free(image_embed);
+ return decode_image_failed;
+ }
+ used += n_eval;
+ }
+ llava_image_embed_free(image_embed);
+ history_.emplace_back(new Image(bytes, N));
+ last_used_ = timespec_real();
+ return N;
+}
+
+int
+Slot::eval_atoms(const std::vector<Atom>& atoms)
{
- unassert(ctx_);
- size_t erase_count = 0;
- size_t reuse_count = lf::vector_common_prefix_length(tokens, history_);
+ int rc;
+ int token_count = 0;
+ std::vector<int> tokens;
+ for (const Atom& atom : atoms) {
+ if (atom.is_token()) {
+ tokens.emplace_back(atom.token());
+ } else if (atom.is_image()) {
+ if ((rc = eval_tokens(tokens)) < 0)
+ return rc;
+ token_count += rc;
+ tokens.clear();
+ if ((rc = eval_image(atom.image().bytes())) < 0)
+ return rc;
+ token_count += rc;
+ }
+ }
+ if ((rc = eval_tokens(tokens)) < 0)
+ return rc;
+ token_count += rc;
+ return token_count;
+}
+
+int
+Slot::prefill(const std::vector<Atom>& atoms)
+{
+ if (!ctx_)
+ return uninitialized;
+ int used_tokens = ctx_used();
+ int reuse_atoms = 0;
+ int reuse_tokens = 0;
+ int erase_tokens = 0;
+ int n = std::min(atoms.size(), history_.size());
+ for (int i = 0; i < n && atoms[i] == history_[i]; ++i) {
+ reuse_tokens += history_[i].ctx_used();
+ reuse_atoms += 1;
+ }
// xxx: ensure we prefill at least one token (prevents badness)
- if (reuse_count >= 1)
- reuse_count -= 1;
- if (history_.size() > reuse_count) {
- erase_count = history_.size() - reuse_count;
- if (llama_kv_cache_seq_rm(ctx_, 0, reuse_count, -1)) {
- history_.resize(reuse_count);
+ if (reuse_tokens >= 1) {
+ reuse_atoms -= 1;
+ reuse_tokens -= history_[reuse_atoms].ctx_used();
+ }
+ if (used_tokens > reuse_tokens) {
+ erase_tokens = used_tokens - reuse_tokens;
+ if (llama_kv_cache_seq_rm(ctx_, 0, reuse_tokens, -1)) {
+ history_.resize(reuse_atoms);
} else {
SLOG("failed to remove tokens from KV cache");
llama_kv_cache_clear(ctx_);
- reuse_count = 0;
- erase_count = history_.size();
+ reuse_atoms = 0;
+ reuse_tokens = 0;
+ erase_tokens = used_tokens;
history_.clear();
}
}
- std::vector<int> new_tokens(tokens.begin() + reuse_count, tokens.end());
- SLOG("prefilling %zu tokens (after removing %zu and reusing %zu)",
- new_tokens.size(),
- erase_count,
- reuse_count);
- return eval_tokens(new_tokens);
+ std::vector<Atom> new_atoms(atoms.begin() + reuse_atoms, atoms.end());
+ int rc;
+ if ((rc = eval_atoms(new_atoms)) < 0)
+ return rc;
+ int token_count = reuse_tokens + rc;
+ SLOG("prefilled %zu tokens (after removing %zu and reusing %zu)",
+ token_count,
+ erase_tokens,
+ reuse_tokens);
+ return token_count;
}
-std::string
-Slot::dump()
+void
+Slot::dump(std::string* result)
{
- std::string r;
- for (size_t i = 0; i < history_.size(); ++i)
- r += llama_token_to_piece(ctx_, history_[i], RENDER_SPECIAL_TOKENS);
- return r;
+ for (size_t i = 0; i < history_.size(); ++i) {
+ if (history_[i].is_token()) {
+ llama_token token = history_[i].token();
+ *result +=
+ llamafile_token_to_piece(ctx_, token, RENDER_SPECIAL_TOKENS);
+ } else if (history_[i].is_image()) {
+ lf::convert_image_to_uri(result, history_[i].image().bytes());
+ }
+ }
}
diff --git a/llamafile/server/slot.h b/llamafile/server/slot.h
index ffab94de..0e019070 100644
--- a/llamafile/server/slot.h
+++ b/llamafile/server/slot.h
@@ -20,23 +20,43 @@
#include <time.h>
#include <vector>
-struct llama_model;
+struct Atom;
+struct Image;
struct llama_context;
+struct llama_model;
+struct clip_ctx;
struct Slot
{
+ enum
+ {
+ uninitialized = -4096,
+ out_of_context,
+ no_vision_model,
+ decode_token_failed,
+ decode_image_failed,
+ encode_image_failed,
+ };
+
+ static const char* describe_error(int);
+
llama_model* model_;
+ clip_ctx* clip_ctx_ = nullptr;
llama_context* ctx_ = nullptr;
timespec last_used_ = timespec_zero;
- std::vector<int> history_;
+ std::vector<Atom> history_;
std::string system_fingerprint_;
- Slot(llama_model*);
~Slot();
- int n_ctx();
+ explicit Slot(llama_model*);
+ int ctx_size() const;
+ int ctx_used() const;
bool start();
- bool eval_token(int);
- bool eval_tokens(std::vector<int>);
- bool prefill(const std::vector<int>&);
- std::string dump();
+ int eval_token(int);
+ int eval_image(const std::string_view&);
+ int eval_tokens(const std::vector<int>&);
+ int eval_atoms(const std::vector<Atom>&);
+ int prefill(const std::vector<Atom>&);
+ void tokenize(std::vector<Atom>*, std::string_view, bool);
+ void dump(std::string*);
};
diff --git a/llamafile/server/slot_entry.cpp b/llamafile/server/slot_entry.cpp
index de8aab9b..a3dc2932 100644
--- a/llamafile/server/slot_entry.cpp
+++ b/llamafile/server/slot_entry.cpp
@@ -16,6 +16,7 @@
// limitations under the License.
#include "slot_entry.h"
+#include "atom.h"
#include "slot.h"
#include <assert.h>
@@ -24,7 +25,7 @@ SlotEntry::SlotEntry(Slot* slot) : slot_(slot), key_(nullptr)
unassert(slot);
}
-SlotEntry::SlotEntry(const std::vector<int>* key) : slot_(nullptr), key_(key)
+SlotEntry::SlotEntry(const std::vector<Atom>* key) : slot_(nullptr), key_(key)
{
unassert(key);
}
@@ -39,7 +40,7 @@ SlotEntry::slot() const
return slot_;
}
-const std::vector<int>*
+const std::vector<Atom>*
SlotEntry::key() const
{
return key_ ? key_ : &slot_->history_;
diff --git a/llamafile/server/slot_entry.h b/llamafile/server/slot_entry.h
index b6d884ba..e30bed0d 100644
--- a/llamafile/server/slot_entry.h
+++ b/llamafile/server/slot_entry.h
@@ -19,19 +19,20 @@
#include <vector>
struct Slot;
+struct Atom;
class SlotEntry
{
public:
explicit SlotEntry(Slot*);
- explicit SlotEntry(const std::vector<int>*);
+ explicit SlotEntry(const std::vector<Atom>*);
~SlotEntry();
Slot* slot() const;
- const std::vector<int>* key() const;
+ const std::vector<Atom>* key() const;
private:
Slot* slot_;
- const std::vector<int>* key_;
+ const std::vector<Atom>* key_;
};
bool
diff --git a/llamafile/server/slots.cpp b/llamafile/server/slots.cpp
index 4d8e7293..56c4bd7d 100644
--- a/llamafile/server/slots.cpp
+++ b/llamafile/server/slots.cpp
@@ -16,11 +16,11 @@
// limitations under the License.
#include "slots.h"
+#include "llamafile/server/atom.h"
#include "llamafile/server/log.h"
+#include "llamafile/server/slot.h"
+#include "llamafile/server/slot_entry.h"
#include "llamafile/vector.h"
-#include "log.h"
-#include "slot.h"
-#include "slot_entry.h"
#include <assert.h>
Slots::Slots(llama_model* model) : model_(model)
@@ -62,7 +62,7 @@ Slots::start(int count)
}
Slot*
-Slots::take(const std::vector<int>& prefix)
+Slots::take(const std::vector<Atom>& prefix)
{
pthread_mutex_lock(&lock_);
for (;;) {
diff --git a/llamafile/server/slots.h b/llamafile/server/slots.h
index 00cb4f4b..d7020761 100644
--- a/llamafile/server/slots.h
+++ b/llamafile/server/slots.h
@@ -20,6 +20,7 @@
#include <set>
#include <vector>
+class Atom;
struct Slot;
class SlotEntry;
struct llama_model;
@@ -32,9 +33,10 @@ struct Slots
pthread_mutex_t lock_;
pthread_cond_t cond_;
- Slots(llama_model*);
+ explicit Slots(llama_model*);
~Slots();
int start(int);
- Slot* take(const std::vector<int>&);
+ void tokenize(std::vector<Atom>*, std::string_view, bool);
+ Slot* take(const std::vector<Atom>&);
void give(Slot*);
};
diff --git a/llamafile/server/slotz.cpp b/llamafile/server/slotz.cpp
index 362b7c68..f00b0a77 100644
--- a/llamafile/server/slotz.cpp
+++ b/llamafile/server/slotz.cpp
@@ -32,7 +32,8 @@ Client::slotz()
if (id >= worker_->server_->slots_->all_slots_.size())
return send_error(404);
Slot* slot = worker_->server_->slots_->all_slots_[id];
- std::string dump = slot->dump();
+ std::string dump;
+ slot->dump(&dump);
char* p = append_http_response_message(obuf_.p, 200);
p = stpcpy(p, "Content-Type: text/plain\r\n");
return send_response(obuf_.p, p, dump);
diff --git a/llamafile/server/time.cpp b/llamafile/server/time.cpp
index 2ac0c9d8..1ef3a741 100644
--- a/llamafile/server/time.cpp
+++ b/llamafile/server/time.cpp
@@ -23,8 +23,7 @@
#include <unistd.h>
#include "llamafile/crash.h"
-
-#include "log.h"
+#include "llamafile/server/log.h"
//
// lockless implementation of gmtime_r() and localtime_r()
diff --git a/llamafile/server/tokenbucket.cpp b/llamafile/server/tokenbucket.cpp
index 1e5d17bf..c04d5f44 100644
--- a/llamafile/server/tokenbucket.cpp
+++ b/llamafile/server/tokenbucket.cpp
@@ -16,15 +16,13 @@
// limitations under the License.
#include "tokenbucket.h"
-
+#include "llamafile/llamafile.h"
+#include "llamafile/server/log.h"
#include <assert.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
-#include "llamafile/llamafile.h"
-#include "llamafile/server/log.h"
-
union TokenBucket
{
atomic_uint_fast64_t* w;
diff --git a/llamafile/server/tokenize.cpp b/llamafile/server/tokenize.cpp
index 175d4615..d32a732d 100644
--- a/llamafile/server/tokenize.cpp
+++ b/llamafile/server/tokenize.cpp
@@ -16,21 +16,18 @@
// limitations under the License.
#include "client.h"
-
+#include "llama.cpp/llama.h"
+#include "llamafile/server/cleanup.h"
+#include "llamafile/server/fastjson.h"
+#include "llamafile/server/json.h"
+#include "llamafile/server/log.h"
+#include "llamafile/server/signals.h"
+#include "llamafile/server/utils.h"
#include <string.h>
#include <sys/resource.h>
#include <utility>
#include <vector>
-#include "llama.cpp/llama.h"
-
-#include "cleanup.h"
-#include "fastjson.h"
-#include "json.h"
-#include "log.h"
-#include "signals.h"
-#include "utils.h"
-
struct TokenizeParams
{
bool add_special;
diff --git a/llamafile/server/utils.h b/llamafile/server/utils.h
index e5bbdf0e..30205316 100644
--- a/llamafile/server/utils.h
+++ b/llamafile/server/utils.h
@@ -18,8 +18,12 @@
#pragma once
#include <__fwd/string.h>
#include <__fwd/string_view.h>
+#include <__fwd/vector.h>
#include <optional>
+class Atom;
+struct llama_model;
+
extern const signed char kHexToInt[256];
bool
@@ -27,3 +31,9 @@ atob(std::string_view, bool);
std::string_view
or_empty(std::optional<std::string_view> x);
+
+void
+atomize(const llama_model* model,
+ std::vector<Atom>* result,
+ std::string_view s,
+ bool parse_special);
diff --git a/llamafile/server/v1_chat_completions.cpp b/llamafile/server/v1_chat_completions.cpp
index a9657fcf..d880658b 100644
--- a/llamafile/server/v1_chat_completions.cpp
+++ b/llamafile/server/v1_chat_completions.cpp
@@ -16,28 +16,26 @@
// limitations under the License.
#include "client.h"
-
-#include <math.h>
-#include <string.h>
-#include <sys/resource.h>
-#include <vector>
-
#include "llama.cpp/llama.h"
#include "llama.cpp/sampling.h"
#include "llamafile/llama.h"
#include "llamafile/macros.h"
+#include "llamafile/server/atom.h"
+#include "llamafile/server/cleanup.h"
+#include "llamafile/server/fastjson.h"
+#include "llamafile/server/json.h"
+#include "llamafile/server/log.h"
+#include "llamafile/server/server.h"
+#include "llamafile/server/slot.h"
+#include "llamafile/server/slots.h"
+#include "llamafile/server/utils.h"
+#include "llamafile/server/worker.h"
#include "llamafile/string.h"
#include "llamafile/vector.h"
-
-#include "cleanup.h"
-#include "fastjson.h"
-#include "json.h"
-#include "log.h"
-#include "server.h"
-#include "slot.h"
-#include "slots.h"
-#include "utils.h"
-#include "worker.h"
+#include <math.h>
+#include <string.h>
+#include <sys/resource.h>
+#include <vector>
struct V1ChatCompletionParams
{
@@ -51,16 +49,16 @@ struct V1ChatCompletionParams
std::string user;
std::string model;
std::vector<llama_chat_msg> messages;
- std::vector<std::vector<int>> stop;
+ std::vector<std::vector<Atom>> stop;
std::string grammar;
void add_stop(llama_model* model, const std::string& text)
{
- stop.emplace_back(
- llama_tokenize(model, text, DONT_ADD_SPECIAL, DONT_PARSE_SPECIAL));
+ stop.emplace_back();
+ atomize(model, &stop.back(), text, DONT_PARSE_SPECIAL);
}
- bool should_stop(const std::vector<int>& history)
+ bool should_stop(const std::vector<Atom>& history)
{
for (const auto& suffix : stop)
if (lf::vector_ends_with(history, suffix))
@@ -72,7 +70,7 @@ struct V1ChatCompletionParams
struct V1ChatCompletionState
{
std::string prompt;
- std::vector<int> tokens;
+ std::vector<Atom> atoms;
std::string piece;
};
@@ -452,27 +450,19 @@ Client::v1_chat_completions()
V1ChatCompletionResponse* response = new V1ChatCompletionResponse;
defer_cleanup(cleanup_response, response);
+ // add bos token if it's needed
+ if (llama_should_add_bos_token(model_))
+ state->atoms.emplace_back(llama_token_bos(model_));
+
// turn text into tokens
state->prompt =
llama_chat_apply_template(model_, "", params->messages, ADD_ASSISTANT);
- state->tokens =
- llama_tokenize(model_, state->prompt, DONT_ADD_SPECIAL, PARSE_SPECIAL);
-
- // add bos token if it's needed
- if (llama_should_add_bos_token(model_)) {
- llama_token bos = llama_token_bos(model_);
- if (state->tokens.empty() || state->tokens[0] != bos)
- state->tokens.insert(state->tokens.begin(), bos);
- }
+ atomize(model_, &state->atoms, state->prompt, PARSE_SPECIAL);
// find appropriate slot
- slot_ = worker_->server_->slots_->take(state->tokens);
+ slot_ = worker_->server_->slots_->take(state->atoms);
defer_cleanup(cleanup_slot, this);
- // sanity check
- if (state->tokens.size() + 1 > slot_->n_ctx())
- return send_error(400, "prompt too big for model context size");
-
// init sampling
llama_sampling_context* sampler = create_sampler(params);
if (!sampler)
@@ -480,9 +470,10 @@ Client::v1_chat_completions()
defer_cleanup(cleanup_sampler, sampler);
// prefill time
- if (!slot_->prefill(state->tokens)) {
- SLOG("slot prefill failed");
- return send_error(500, "llama_decode prefill failed");
+ int prompt_tokens = 0;
+ if ((prompt_tokens = slot_->prefill(state->atoms)) < 0) {
+ SLOG("slot prefill failed: %s", Slot::describe_error(prompt_tokens));
+ return send_error(500, Slot::describe_error(prompt_tokens));
}
// setup response json
@@ -539,7 +530,7 @@ Client::v1_chat_completions()
break;
}
state->piece =
- llama_token_to_piece(slot_->ctx_, id, DONT_RENDER_SPECIAL_TOKENS);
+ llamafile_token_to_piece(slot_->ctx_, id, DONT_RENDER_SPECIAL_TOKENS);
if (!state->piece.empty()) {
if (params->stream) {
char* p = append_http_response_message(obuf_.p, 200);
@@ -571,9 +562,9 @@ Client::v1_chat_completions()
} else {
Json& usage = response->json["usage"];
usage.setObject();
- usage["prompt_tokens"].setLong(state->tokens.size());
+ usage["prompt_tokens"].setLong(prompt_tokens);
usage["completion_tokens"].setLong(completion_tokens);
- usage["total_tokens"].setLong(completion_tokens + state->tokens.size());
+ usage["total_tokens"].setLong(completion_tokens + prompt_tokens);
choice["message"].setObject();
choice["message"]["role"].setString("assistant");
choice["message"]["content"].setString(std::move(response->content));
diff --git a/llamafile/server/worker.cpp b/llamafile/server/worker.cpp
index bf73e381..1ae3240c 100644
--- a/llamafile/server/worker.cpp
+++ b/llamafile/server/worker.cpp
@@ -16,20 +16,17 @@
// limitations under the License.
#include "worker.h"
-
-#include <assert.h>
-#include <exception>
-#include <pthread.h>
-
#include "llamafile/llamafile.h"
+#include "llamafile/server/client.h"
+#include "llamafile/server/log.h"
+#include "llamafile/server/server.h"
+#include "llamafile/server/signals.h"
+#include "llamafile/server/tokenbucket.h"
#include "llamafile/threadlocal.h"
#include "llamafile/trust.h"
-
-#include "client.h"
-#include "log.h"
-#include "server.h"
-#include "signals.h"
-#include "tokenbucket.h"
+#include <assert.h>
+#include <exception>
+#include <pthread.h>
Worker::Worker(Server* server, llama_model* model)
: server_(server), client_(model)
diff --git a/llamafile/server/www/chatbot.css b/llamafile/server/www/chatbot.css
index 4fbea750..e23fba4d 100644
--- a/llamafile/server/www/chatbot.css
+++ b/llamafile/server/www/chatbot.css
@@ -108,6 +108,11 @@ ul li:first-child {
margin-right: auto;
}
+.message img {
+ max-width: 100%;
+ height: auto;
+}
+
.chat-input-container {
padding: 1rem;
border-top: 1px solid #e9ecef;
@@ -124,7 +129,7 @@ ul li:first-child {
outline: none;
resize: none;
max-height: 200px;
- min-height: 44px;
+ min-height: 50px;
line-height: 1.5;
font-family: inherit;
overflow-y: auto;
@@ -132,6 +137,11 @@ ul li:first-child {
scrollbar-color: #cbd5e1 transparent; /* Firefox */
}
+.chat-input.drag-over {
+ border-color: #0366d6;
+ background-color: #f6f8fa;
+}
+
/* Chrome, Edge, and Safari */
.chat-input::-webkit-scrollbar {
width: 8px;
diff --git a/llamafile/server/www/chatbot.js b/llamafile/server/www/chatbot.js
index 72b675e1..13e534b2 100644
--- a/llamafile/server/www/chatbot.js
+++ b/llamafile/server/www/chatbot.js
@@ -22,6 +22,7 @@ const stopButton = document.getElementById("stop-button");
let abortController = null;
let streamingMessageContent = [];
+let uploadedFiles = [];
let chatHistory = [
{
@@ -32,6 +33,10 @@ let chatHistory = [
},
];
+function generateId() {
+ return Date.now().toString(36) + Math.random().toString(36).substring(2);
+}
+
function createMessageElement(content, role) {
const messageDiv = document.createElement("div");
messageDiv.classList.add("message", role);
@@ -117,8 +122,16 @@ function stopMessage() {
}
}
+function fixUploads(str) {
+ str = uploadedFiles.reduce(
+ (text, [from, to]) => text.replaceAll(from, to),
+ str);
+ uploadedFiles.length = 0;
+ return str;
+}
+
async function sendMessage() {
- const message = chatInput.value.trim();
+ const message = fixUploads(chatInput.value.trim());
if (!message) return;
// disable input while processing
@@ -153,22 +166,22 @@ async function sendMessage() {
}),
signal: abortController.signal
});
-
- if (!response.ok)
- throw new Error(`HTTP error! status: ${response.status}`);
-
- // handle the stream
- await handleChatStream(response);
-
- // update chat history with response
- const lastMessage = streamingMessageContent.join("");
- chatHistory.push({ role: "assistant", content: lastMessage });
-
+ if (response.ok) {
+ await handleChatStream(response);
+ const lastMessage = streamingMessageContent.join("");
+ chatHistory.push({ role: "assistant", content: lastMessage });
+ } else {
+ console.error("sendMessage() failed due to server error", response);
+ chatMessages.appendChild(createMessageElement(
+ `Server replied with error code ${response.status} ${response.statusText}`,
+ "system"));
+ cleanupAfterMessage();
+ }
} catch (error) {
if (error.name !== "AbortError") {
- console.error("Error:", error);
+ console.error("sendMessage() failed due to unexpected exception", error);
const errorMessage = createMessageElement(
- "Sorry, there was an error processing your request.",
+ "There was an error processing your request.",
"system");
chatMessages.appendChild(errorMessage);
}
@@ -176,24 +189,88 @@ async function sendMessage() {
}
}
-// setup chat window
-chatMessages.innerHTML = "";
-for (let i = 0; i < chatHistory.length; i++) {
- chatMessages.appendChild(createMessageElement(chatHistory[i].content,
- chatHistory[i].role));
- scrollToBottom();
+function onDragBegin(e) {
+ e.preventDefault();
+ e.stopPropagation();
+ chatInput.classList.add('drag-over');
+}
+
+function onDragEnd(e) {
+ e.preventDefault();
+ e.stopPropagation();
+ chatInput.classList.remove('drag-over');
}
-// setup events
-sendButton.addEventListener("click", sendMessage);
-stopButton.addEventListener("click", stopMessage);
-chatInput.addEventListener("input", onChatInput);
-chatInput.addEventListener("keydown", function(e) {
+function onDrop(e) {
+ const files = e.dataTransfer.files;
+ [...files].forEach(onFile);
+}
+
+function onPaste(e) {
+ const items = e.clipboardData.items;
+ for (let item of items) {
+ if (item.type.startsWith('image/')) {
+ e.preventDefault();
+ onFile(item.getAsFile());
+ return;
+ }
+ }
+}
+
+function onFile(file) {
+ if (!file.type.startsWith('image/')) {
+ console.warn('Only image files are supported');
+ return;
+ }
+ if (file.size > 1 * 1024 * 1024) {
+ console.warn('Image is larger than 1mb');
+ return;
+ }
+ const reader = new FileReader();
+ reader.onloadend = function() {
+ const description = file.name;
+ const realDataUri = reader.result;
+ const fakeDataUri = 'data:,placeholder/' + generateId();
+ uploadedFiles.push([fakeDataUri, realDataUri]);
+ insertText(chatInput, `![${description}](${fakeDataUri})`);
+ };
+ reader.readAsDataURL(file);
+}
+
+function insertText(elem, text) {
+ const pos = elem.selectionStart;
+ elem.value = elem.value.slice(0, pos) + text + elem.value.slice(pos);
+ const newPos = pos + text.length;
+ elem.setSelectionRange(newPos, newPos);
+ elem.focus();
+ elem.dispatchEvent(new Event('input'));
+}
+
+function onKeyDown(e) {
if (e.key === "Enter" && !e.shiftKey) {
e.preventDefault();
sendMessage();
}
-});
+}
+
+function chatbot() {
+ chatMessages.innerHTML = "";
+ for (let i = 0; i < chatHistory.length; i++) {
+ chatMessages.appendChild(createMessageElement(chatHistory[i].content,
+ chatHistory[i].role));
+ scrollToBottom();
+ }
+ sendButton.addEventListener("click", sendMessage);
+ stopButton.addEventListener("click", stopMessage);
+ chatInput.addEventListener("input", onChatInput);
+ chatInput.addEventListener("keydown", onKeyDown);
+ document.body.addEventListener('dragenter', onDragBegin, false);
+ document.body.addEventListener('dragover', onDragBegin, false);
+ document.body.addEventListener('dragleave', onDragEnd, false);
+ document.body.addEventListener('drop', onDragEnd, false);
+ document.body.addEventListener('drop', onDrop, false);
+ document.body.addEventListener('paste', onPaste);
+ chatInput.focus();
+}
-// focus input
-chatInput.focus();
+chatbot();
diff --git a/llamafile/server/www/render_markdown.js b/llamafile/server/www/render_markdown.js
index ee539b10..07629899 100644
--- a/llamafile/server/www/render_markdown.js
+++ b/llamafile/server/www/render_markdown.js
@@ -48,6 +48,7 @@ class RenderMarkdown extends Highlighter {
static OLDCODE = 33;
static BLOCKQUOTE = 34;
static TICK_TICK_TICK = 35;
+ static EXCLAIM = 36;
static STYLE_SPAN = 16;
static STYLE_STRONG = 16;
@@ -100,6 +101,7 @@ class RenderMarkdown extends Highlighter {
this.bqline = false;
this.tick1 = 0;
this.tick2 = 0;
+ this.is_image = false;
}
inside(style) {
@@ -270,7 +272,12 @@ class RenderMarkdown extends Highlighter {
} else if (c == '~') {
this.state = RenderMarkdown.TILDE;
this.got();
+ } else if (c == '!') {
+ this.is_image = true;
+ this.state = RenderMarkdown.EXCLAIM;
+ this.got();
} else if (c == '[') {
+ this.is_image = false;
this.state = RenderMarkdown.LSB;
this.got();
} else if (c == '<') {
@@ -694,6 +701,15 @@ class RenderMarkdown extends Highlighter {
this.state = RenderMarkdown.LAB;
break;
+ case RenderMarkdown.EXCLAIM:
+ if (c == '[') {
+ this.state = RenderMarkdown.LSB;
+ } else {
+ this.append('!');
+ this.epsilon(RenderMarkdown.NORMAL);
+ }
+ break;
+
case RenderMarkdown.LSB:
if (c == ']') {
this.state = RenderMarkdown.LSB_RSB;
@@ -723,9 +739,15 @@ class RenderMarkdown extends Highlighter {
case RenderMarkdown.LSB_RSB_LPAREN:
if (c == ')') {
- let a = this.push("A", "");
- a.innerText = this.text;
- a.href = this.href;
+ if (this.is_image) {
+ let a = this.push("IMG", "");
+ a.alt = this.text;
+ a.src = this.href;
+ } else {
+ let a = this.push("A", "");
+ a.innerText = this.text;
+ a.href = this.href;
+ }
this.pop();
this.href = '';
this.text = '';