Skip to content

Distribute and run LLMs with a single file.

Forget old messages when running out of context

Justine Tunney08e7a21b6894
diff --git a/llamafile/flags.cpp b/llamafile/flags.cpp
index 7bd7cee4..390e9ec2 100644
--- a/llamafile/flags.cpp
+++ b/llamafile/flags.cpp
@@ -67,6 +67,7 @@ const char *FLAG_www_root = "/zip/www";
double FLAG_token_rate = 1;
float FLAG_frequency_penalty = 0;
float FLAG_presence_penalty = 0;
+float FLAG_reserve_tokens = .15;
float FLAG_temperature = .8;
float FLAG_top_p = .95;
int FLAG_batch = 256;
@@ -463,6 +464,24 @@ void llamafile_get_flags(int argc, char **argv) {
continue;
}
+ if (!strcmp(flag, "--reserve-tokens")) {
+ if (i == argc)
+ missing("--reserve-tokens");
+ const char *s = argv[i++];
+ if (strchr(s, '.')) {
+ float f = atof(s);
+ if (!(0 < f && f <= 1))
+ error("--reserve-tokens FLOAT must be on the interval (0,1]");
+ FLAG_reserve_tokens = f;
+ } else {
+ int n = atoi(s);
+ if (!(1 <= n && n <= 100))
+ error("--reserve-tokens INT must be between 1 and 100");
+ FLAG_reserve_tokens = n / 100.;
+ }
+ continue;
+ }
+
//////////////////////////////////////////////////////////////////////
// cpu flags
diff --git a/llamafile/llamafile.h b/llamafile/llamafile.h
index 15f34a7b..caf04f2d 100644
--- a/llamafile/llamafile.h
+++ b/llamafile/llamafile.h
@@ -49,6 +49,7 @@ extern int FLAG_http_obuf_size;
extern int FLAG_keepalive;
extern int FLAG_main_gpu;
extern int FLAG_n_gpu_layers;
+extern float FLAG_reserve_tokens;
extern int FLAG_slots;
extern int FLAG_split_mode;
extern int FLAG_threads;
diff --git a/llamafile/server/atomize.cpp b/llamafile/server/atomize.cpp
index 1cd8f339..a8039114 100644
--- a/llamafile/server/atomize.cpp
+++ b/llamafile/server/atomize.cpp
@@ -76,5 +76,21 @@ atomize(const llama_model* model,
}
}
+// having multiple images in the context window is janky right now, so
+// let's erase old images from the chat history until we find out more
+std::vector<Atom>
+remove_old_image_atoms(const std::vector<Atom>& atoms)
+{
+ int last_image_idx = -1;
+ for (int i = 0; i < atoms.size(); ++i)
+ if (atoms[i].is_image())
+ last_image_idx = i;
+ std::vector<Atom> result;
+ for (int i = 0; i < atoms.size(); i++)
+ if (!atoms[i].is_image() || i == last_image_idx)
+ result.emplace_back(atoms[i]);
+ return result;
+}
+
} // namespace server
} // namespace lf
diff --git a/llamafile/server/main.1 b/llamafile/server/main.1
index 58a59a43..d9679ca3 100644
--- a/llamafile/server/main.1
+++ b/llamafile/server/main.1
@@ -169,6 +169,12 @@ write-ahead log.
PRAGMA journal_mode=WAL;
PRAGMA synchronous=NORMAL;
.Ed
+.It Fl Fl reserve-tokens
+Percent of context window to reserve for predicted tokens. When the
+server runs out of context window, old chat messages will be forgotten
+until this percent of the context is empty. The default is 15%. If this
+is specified as a floating point number, e.g. 0.15, then it'll be
+multiplied by 100 to get the percent.
.El
.Sh EXAMPLE
Here's an example of how you might start this server:
diff --git a/llamafile/server/main.1.asc b/llamafile/server/main.1.asc
index c2db481e..fd8b41b8 100644
--- a/llamafile/server/main.1.asc
+++ b/llamafile/server/main.1.asc
@@ -1,227 +1,231 @@
-LLAMAFILER(1) General Commands Manual LLAMAFILER(1)
-
-NAME
- llamafiler — fast reliable embedding server
-
-SYNOPSIS
- llamafiler -m model.gguf [flags...]
-
-DESCRIPTION
- llamafiler is a brand new HTTP server for Large Language Models (LLMs).
- To date, its development has been focused on doing fewer things really
- well, and right now that's serving embeddings. It offers you 3.4x the
- performance, stronger security, client request prioritization, request
- preemption, as well as request isolation that helps ensure software
- bugs won't cause the whole server to crash.
-
-OPTIONS
- The following options are available:
-
- --version
- Print version and exit.
-
- -h, --help
- Show help message and exit.
-
- -m FNAME, --model FNAME
- Path of GGUF model weights. Each server process is currently
- limited to serving only one model. If you need to host multiple
- models, then it's recommended that you run multiple instances
- of llamafiler behind a reverse proxy such as NGINX or Redbean.
-
- -mm FNAME, --mmproj FNAME
- Path of vision model weights.
-
- --db FILE
- Specifies path of sqlite3 database.
-
- The default is ~/.llamafile/llamafile.sqlite3
-
- -l HOSTPORT, --listen HOSTPORT
- Specifies the local [HOST:]PORT on which the HTTP server should
- listen. By default this is 0.0.0.0:8080 which means llamafiler
- will bind to port 8080 on every locally available IPv4 network
- interface. This option may currently only be specified once.
-
- -c TOKENS, --ctx-size TOKENS
- Specifies context size. This specifies how long a completion
- can get before it runs out of space. It defaults to 8k which
- means 8192 tokens. Many models support a larger context size,
- like 128k, but that'll need much more RAM or VRAM per slot. If
- this value is larger than the trained context size of the
- model, it'll be tuned down to the maximum. If this value is 0
- or negative, the maximum number of tokens will be used.
-
- -s COUNT, --slots COUNT
- Specifies how many slots to maintain. This defaults to 1. Slots
- are used by chat completions requests. When such a request
- comes in, the client needs to take control of a slot. When the
- completion is finished, the slot is relinquished back to the
- server. HTTP clients will wait for a slot to be relinquished if
- none are available. Tuning this parameter to nicely fit avail‐
- able RAM or VRAM can help you manage your server resources, and
- control how much completion parallelism can happen. Please
- note that --ctx-size has a strong influence on how many slots
- can be created.
-
- -p TEXT, --prompt TEXT, --system-prompt TEXT
- Specifies system prompt. This value is passed along to the web
- frontend.
-
- --no-display-prompt
- Hide system prompt from web user interface.
-
- --nologo
- Hide llamafile logo icon from web ui.
-
- --url-prefix URLPREFIX
- Specifies a URL prefix (subdirectory) under which the HTTP
- server will make the API accessible, e.g. /lamafiler. Useful
- when running llamafiler behind a reverse proxy such as NGINX or
- Redbean. By default, this is set to / (root).
-
- -w N, --workers N
- Number of HTTP client handling threads.
-
- --trust CIDR
- Adds a network to the trusted network list. This argument is
- specified in the form IPV4/MASKBITS, e.g. 192.168.0.0/24. By
- default, all clients are untrusted, which means they're subject
- to token bucket throttling, and additional security precautions
- that may cause request handling to go slightly slower. There‐
- fore this flag is important to use if you want to accurately
- benchmark llamafiler, since the server will otherwise see the
- benchmark as a DDOS and deprioritize its traffic accordingly.
-
- --ip-header STR
- If this flag is passed a value, e.g. X-Forwarded-For, then any
- trusted may send this header to your llamafile server to let it
- know what the true effective client IPv4 address actually is.
- After this happens the default security restrictions, e.g. to‐
- ken bucket, will be measured and applied against that IPv4 ad‐
- dress and its adjacent networks.
-
- --token-rate N
- Specifies how many times per second a token is dropped in each
- bucket. This setting is used to define a limitation on how
- many TCP connects and HTTP messages each chunk of the IPv4 ad‐
- dress space is permitted to send to llamafiler over a sustained
- period of time. The default token rate is 1, which means that,
- on a long enough timeline, a class-C network will be depriori‐
- tized if it sends more than one request per second. No real
- penalty actually applies though until the server runs out of
- resources, e.g. HTTP request workers.
-
- --token-burst N
- Specifies how many HTTP requests and TCP connects a given slice
- of the IPv4 address space is permitted to send within a short
- period of time, before token bucket restrictions kick in, and
- cause the client to be deprioritized. By default, this value is
- set to 100. It may be tuned to any value between 1 and 127 in‐
- clusive.
-
- --token-cidr N
- Specifies IPv4 address space granularity of token bucket algo‐
- rithm, in network bits. By default, this value is set to 24
- which means individual IPv4 addresses are viewed as being rep‐
- resentative members of a class-C network, or in other words,
- each group of 256 IPv4 addresses is lumped together. If one IP
- in the group does something bad, then bad things happen to all
- the other IPv4 addresses in that granule. This number may be
- set to any integer between 3 and 32 inclusive. Specifying a
- higher number will trade away system memory to increase network
- specificity. For example, using 32 means that 4 billion indi‐
- vidual token buckets will be created. By default, a background
- thread drops one token in each bucket every second, so that
- could potentially be a lot of busy work. A value of three means
- that everyone on the Internet who talks to your server will
- have to fight over only eight token buckets in total.
-
- --unsecure
- Disables sandboxing. By default, llamafiler puts itself in a
- SECCOMP BPF sandbox, so that even if your server gets hacked in
- the worst possible way (some kind of C++ memory bug) then
- there's very little damage an attacker will be able to do. This
- works by restricting system calls using Cosmopolitan Libc's im‐
- plementation of pledge() which is currently only supported on
- Linux (other OSes will simply be unsecured by default). The
- pledge security policy that's used by default is "stdio anet"
- which means that only relatively harmless system calls like
- read(), write(), and accept() are allowed once the server has
- finished initializing. It's not possible for remotely executed
- code to do things like launch subprocesses, read or write to
- the filesystem, or initiate a new connection to a server.
-
- -k N, --keepalive N
- Specifies the TCP keepalive interval in seconds. This value is
- passed along to both TCP_KEEPIDLE and TCP_KEEPINTVL if they're
- supported by the host operating system. If this value is
- greater than 0, then the the SO_KEEPALIVE and TCP_NODELAY op‐
- tions are enabled on network sockets, if supported by the host
- operating system. The default keepalive is 5.
-
- --http-obuf-size N
- Size of HTTP output buffer size, in bytes. Default is 1048576.
-
- --http-ibuf-size N
- Size of HTTP input buffer size, in bytes. Default is 1048576.
-
- --chat-template NAME
- Specifies or overrides chat template for model.
-
- Normally the GGUF metadata tokenizer.chat_template will specify
- this value for instruct models. This flag may be used to either
- override the chat template, or specify one when the GGUF meta‐
- data field is absent, which effectively forces the web ui to
- enable chatbot mode.
-
- Supported chat template names are: chatml, llama2, llama3, mis‐
- tral (alias for llama2), phi3, zephyr, monarch, gemma, gemma2
- (alias for gemma), orion, openchat, vicuna, vicuna-orca,
- deepseek, command-r, chatglm3, chatglm4, minicpm, deepseek2, or
- exaone3.
-
- It is also possible to pass the jinja2 template itself to this
- argument. Since llamafiler doesn't currently support jinja2, a
- heuristic will be used to guess which of the above templates
- the template represents.
-
- --completion-mode
- Forces web ui to operate in completion mode, rather than chat
- mode. Normally the web ui chooses its mode based on the GGUF
- metadata. Base models normally don't define tokenizer.chat_tem‐
- plate whereas instruct models do. If it's a base model, then
- the web ui will automatically use completion mode only, without
- needing to specify this flag. This flag is useful in cases
- where a prompt template is defined by the gguf, but it is de‐
- sirable for the chat interface to be disabled.
-
- --db-startup-sql
- Specifies SQL code that should be executed whenever connecting
- to the SQLite database. The default is the following code,
- which enables the write-ahead log.
-
- PRAGMA journal_mode=WAL;
- PRAGMA synchronous=NORMAL;
-
-EXAMPLE
- Here's an example of how you might start this server:
-
- llamafiler -m all-MiniLM-L6-v2.F32.gguf
-
- Here's how to send a tokenization request:
-
- curl -v http://127.0.0.1:8080/tokenize?prompt=hello+world
-
- Here's how to send a embedding request:
-
- curl -v http://127.0.0.1:8080/embedding?content=hello+world
-
-DOCUMENTATION
- Read our Markdown documentation for additional help and tutorials. See
- llamafile/server/doc/index.md in the source repository on GitHub.
-
-SEE ALSO
- llamafile(1), whisperfile(1)
-
-Mozilla Ocho November 30, 2024 LLAMAFILER(1)
+LLAMAFILER(1) General Commands Manual LLAMAFILER(1)
+
+NNAAMMEE
+ llllaammaaffiilleerr - fast reliable embedding server
+
+SSYYNNOOPPSSIISS
+ llllaammaaffiilleerr --mm _m_o_d_e_l_._g_g_u_f [flags...]
+
+DDEESSCCRRIIPPTTIIOONN
+ llllaammaaffiilleerr is a brand new HTTP server for Large Language Models (LLMs). To
+ date, its development has been focused on doing fewer things really well,
+ and right now that's serving embeddings. It offers you 3.4x the
+ performance, stronger security, client request prioritization, request
+ preemption, as well as request isolation that helps ensure software bugs
+ won't cause the whole server to crash.
+
+OOPPTTIIOONNSS
+ The following options are available:
+
+ ----vveerrssiioonn
+ Print version and exit.
+
+ --hh, ----hheellpp
+ Show help message and exit.
+
+ --mm _F_N_A_M_E, ----mmooddeell _F_N_A_M_E
+ Path of GGUF model weights. Each server process is currently
+ limited to serving only one model. If you need to host multiple
+ models, then it's recommended that you run multiple instances of
+ llamafiler behind a reverse proxy such as NGINX or Redbean.
+
+ --mmmm _F_N_A_M_E, ----mmmmpprroojj _F_N_A_M_E
+ Path of vision model weights.
+
+ ----ddbb _F_I_L_E
+ Specifies path of sqlite3 database.
+
+ The default is _~_/_._l_l_a_m_a_f_i_l_e_/_l_l_a_m_a_f_i_l_e_._s_q_l_i_t_e_3
+
+ --ll _H_O_S_T_P_O_R_T, ----lliisstteenn _H_O_S_T_P_O_R_T
+ Specifies the local [HOST:]PORT on which the HTTP server should
+ listen. By default this is 0.0.0.0:8080 which means llamafiler
+ will bind to port 8080 on every locally available IPv4 network
+ interface. This option may currently only be specified once.
+
+ --cc _T_O_K_E_N_S, ----ccttxx--ssiizzee _T_O_K_E_N_S
+ Specifies context size. This specifies how long a completion can
+ get before it runs out of space. It defaults to 8k which means 8192
+ tokens. Many models support a larger context size, like 128k, but
+ that'll need much more RAM or VRAM per slot. If this value is
+ larger than the trained context size of the model, it'll be tuned
+ down to the maximum. If this value is 0 or negative, the maximum
+ number of tokens will be used.
+
+ --ss _C_O_U_N_T, ----sslloottss _C_O_U_N_T
+ Specifies how many slots to maintain. This defaults to 1. Slots are
+ used by chat completions requests. When such a request comes in,
+ the client needs to take control of a slot. When the completion is
+ finished, the slot is relinquished back to the server. HTTP clients
+ will wait for a slot to be relinquished if none are available.
+ Tuning this parameter to nicely fit available RAM or VRAM can help
+ you manage your server resources, and control how much completion
+ parallelism can happen. Please note that ----ccttxx--ssiizzee has a strong
+ influence on how many slots can be created.
+
+ --pp _T_E_X_T, ----pprroommpptt _T_E_X_T, ----ssyysstteemm--pprroommpptt _T_E_X_T
+ Specifies system prompt. This value is passed along to the web
+ frontend.
+
+ ----nnoo--ddiissppllaayy--pprroommpptt
+ Hide system prompt from web user interface.
+
+ ----nnoollooggoo
+ Hide llamafile logo icon from web ui.
+
+ ----uurrll--pprreeffiixx _U_R_L_P_R_E_F_I_X
+ Specifies a URL prefix (subdirectory) under which the HTTP server
+ will make the API accessible, e.g. /lamafiler. Useful when running
+ llamafiler behind a reverse proxy such as NGINX or Redbean. By
+ default, this is set to / (root).
+
+ --ww _N, ----wwoorrkkeerrss _N
+ Number of HTTP client handling threads.
+
+ ----ttrruusstt _C_I_D_R
+ Adds a network to the trusted network list. This argument is
+ specified in the form IPV4/MASKBITS, e.g. 192.168.0.0/24. By
+ default, all clients are untrusted, which means they're subject to
+ token bucket throttling, and additional security precautions that
+ may cause request handling to go slightly slower. Therefore this
+ flag is important to use if you want to accurately benchmark
+ llamafiler, since the server will otherwise see the benchmark as a
+ DDOS and deprioritize its traffic accordingly.
+
+ ----iipp--hheeaaddeerr _S_T_R
+ If this flag is passed a value, e.g. X-Forwarded-For, then any
+ trusted may send this header to your llamafile server to let it
+ know what the true effective client IPv4 address actually is. After
+ this happens the default security restrictions, e.g. token bucket,
+ will be measured and applied against that IPv4 address and its
+ adjacent networks.
+
+ ----ttookkeenn--rraattee _N
+ Specifies how many times per second a token is dropped in each
+ bucket. This setting is used to define a limitation on how many
+ TCP connects and HTTP messages each chunk of the IPv4 address space
+ is permitted to send to llamafiler over a sustained period of time.
+ The default token rate is 1, which means that, on a long enough
+ timeline, a class-C network will be deprioritized if it sends more
+ than one request per second. No real penalty actually applies
+ though until the server runs out of resources, e.g. HTTP request
+ workers.
+
+ ----ttookkeenn--bbuurrsstt _N
+ Specifies how many HTTP requests and TCP connects a given slice of
+ the IPv4 address space is permitted to send within a short period
+ of time, before token bucket restrictions kick in, and cause the
+ client to be deprioritized. By default, this value is set to 100.
+ It may be tuned to any value between 1 and 127 inclusive.
+
+ ----ttookkeenn--cciiddrr _N
+ Specifies IPv4 address space granularity of token bucket algorithm,
+ in network bits. By default, this value is set to 24 which means
+ individual IPv4 addresses are viewed as being representative
+ members of a class-C network, or in other words, each group of 256
+ IPv4 addresses is lumped together. If one IP in the group does
+ something bad, then bad things happen to all the other IPv4
+ addresses in that granule. This number may be set to any integer
+ between 3 and 32 inclusive. Specifying a higher number will trade
+ away system memory to increase network specificity. For example,
+ using 32 means that 4 billion individual token buckets will be
+ created. By default, a background thread drops one token in each
+ bucket every second, so that could potentially be a lot of busy
+ work. A value of three means that everyone on the Internet who
+ talks to your server will have to fight over only eight token
+ buckets in total.
+
+ ----uunnsseeccuurree
+ Disables sandboxing. By default, llamafiler puts itself in a
+ SECCOMP BPF sandbox, so that even if your server gets hacked in the
+ worst possible way (some kind of C++ memory bug) then there's very
+ little damage an attacker will be able to do. This works by
+ restricting system calls using Cosmopolitan Libc's implementation
+ of pledge() which is currently only supported on Linux (other OSes
+ will simply be unsecured by default). The pledge security policy
+ that's used by default is "stdio anet" which means that only
+ relatively harmless system calls like read(), write(), and accept()
+ are allowed once the server has finished initializing. It's not
+ possible for remotely executed code to do things like launch
+ subprocesses, read or write to the filesystem, or initiate a new
+ connection to a server.
+
+ --kk _N, ----kkeeeeppaalliivvee _N
+ Specifies the TCP keepalive interval in seconds. This value is
+ passed along to both TCP_KEEPIDLE and TCP_KEEPINTVL if they're
+ supported by the host operating system. If this value is greater
+ than 0, then the the SO_KEEPALIVE and TCP_NODELAY options are
+ enabled on network sockets, if supported by the host operating
+ system. The default keepalive is 5.
+
+ ----hhttttpp--oobbuuff--ssiizzee _N
+ Size of HTTP output buffer size, in bytes. Default is 1048576.
+
+ ----hhttttpp--iibbuuff--ssiizzee _N
+ Size of HTTP input buffer size, in bytes. Default is 1048576.
+
+ ----cchhaatt--tteemmppllaattee _N_A_M_E
+ Specifies or overrides chat template for model.
+
+ Normally the GGUF metadata tokenizer.chat_template will specify
+ this value for instruct models. This flag may be used to either
+ override the chat template, or specify one when the GGUF metadata
+ field is absent, which effectively forces the web ui to enable
+ chatbot mode.
+
+ Supported chat template names are: chatml, llama2, llama3, mistral
+ (alias for llama2), phi3, zephyr, monarch, gemma, gemma2 (alias for
+ gemma), orion, openchat, vicuna, vicuna-orca, deepseek, command-r,
+ chatglm3, chatglm4, minicpm, deepseek2, or exaone3.
+
+ It is also possible to pass the jinja2 template itself to this
+ argument. Since llamafiler doesn't currently support jinja2, a
+ heuristic will be used to guess which of the above templates the
+ template represents.
+
+ ----ccoommpplleettiioonn--mmooddee
+ Forces web ui to operate in completion mode, rather than chat mode.
+ Normally the web ui chooses its mode based on the GGUF metadata.
+ Base models normally don't define tokenizer.chat_template whereas
+ instruct models do. If it's a base model, then the web ui will
+ automatically use completion mode only, without needing to specify
+ this flag. This flag is useful in cases where a prompt template is
+ defined by the gguf, but it is desirable for the chat interface to
+ be disabled.
+
+ ----ddbb--ssttaarrttuupp--ssqqll
+ Specifies SQL code that should be executed whenever connecting to
+ the SQLite database. The default is the following code, which
+ enables the write-ahead log.
+
+ PRAGMA journal_mode=WAL;
+ PRAGMA synchronous=NORMAL;
+
+ ----rreesseerrvvee--ttookkeennss
+ Percent of context window to reserve for predicted tokens. When the
+ server runs out of context window, old chat messages will be
+ forgotten until this percent of the context is empty. The default
+ is 15%. If this is specified as a floating point number, e.g. 0.15,
+ then it'll be multiplied by 100 to get the percent.
+
+EEXXAAMMPPLLEE
+ Here's an example of how you might start this server:
+
+ llamafiler -m all-MiniLM-L6-v2.F32.gguf
+
+ Here's how to send a tokenization request:
+
+ curl -v http://127.0.0.1:8080/tokenize?prompt=hello+world
+
+ Here's how to send a embedding request:
+
+ curl -v http://127.0.0.1:8080/embedding?content=hello+world
+
+DDOOCCUUMMEENNTTAATTIIOONN
+ Read our Markdown documentation for additional help and tutorials. See
+ llamafile/server/doc/index.md in the source repository on GitHub.
+
+SSEEEE AALLSSOO
+ llamafile(1), whisperfile(1)
+
+Mozilla Ocho November 30, 2024 Mozilla Ocho
diff --git a/llamafile/server/slot.cpp b/llamafile/server/slot.cpp
index fee1ec80..78a9d9d2 100644
--- a/llamafile/server/slot.cpp
+++ b/llamafile/server/slot.cpp
@@ -57,22 +57,6 @@ generate_system_fingerprint(const llama_context_params* cparams)
return b;
}
-// having multiple images in the context window is janky right now, so
-// let's erase old images from the chat history until we find out more
-static std::vector<Atom>
-remove_old_image_atoms(const std::vector<Atom>& atoms)
-{
- int last_image_idx = -1;
- for (int i = 0; i < atoms.size(); ++i)
- if (atoms[i].is_image())
- last_image_idx = i;
- std::vector<Atom> result;
- for (int i = 0; i < atoms.size(); i++)
- if (!atoms[i].is_image() || i == last_image_idx)
- result.emplace_back(atoms[i]);
- return result;
-}
-
const char*
Slot::describe_error(int err)
{
@@ -255,11 +239,14 @@ Slot::eval_atoms(const std::vector<Atom>& atoms,
if (atom.is_token()) {
total_work += 1;
} else if (atom.is_image()) {
- llava_image_embed* image_embed = llava_image_embed_make_with_bytes(
- clip_ctx_,
- FLAG_threads_batch,
- (const unsigned char*)atom.image().bytes().data(),
- atom.image().bytes().size());
+ if (!clip_ctx_)
+ return no_vision_model;
+ llava_image_embed* image_embed =
+ llava_image_embed_make_with_bytes(
+ clip_ctx_,
+ FLAG_threads_batch,
+ (const unsigned char*)atom.image().bytes().data(),
+ atom.image().bytes().size());
if (image_embed) {
total_work += image_embed->n_image_pos;
llava_image_embed_free(image_embed);
@@ -299,11 +286,10 @@ Slot::eval_atoms(const std::vector<Atom>& atoms,
}
int
-Slot::prefill(const std::vector<Atom>& atoms_, const ProgressCallback& progress)
+Slot::prefill(const std::vector<Atom>& atoms, const ProgressCallback& progress)
{
if (!ctx_)
return uninitialized;
- std::vector<Atom> atoms = remove_old_image_atoms(atoms_);
int used_tokens = ctx_used();
int reuse_atoms = 0;
int reuse_tokens = 0;
diff --git a/llamafile/server/utils.h b/llamafile/server/utils.h
index 346be6a9..4ee1cea2 100644
--- a/llamafile/server/utils.h
+++ b/llamafile/server/utils.h
@@ -44,5 +44,8 @@ atomize(const llama_model* model,
std::string_view s,
bool parse_special);
+std::vector<Atom>
+remove_old_image_atoms(const std::vector<Atom>&);
+
} // namespace server
} // namespace lf
diff --git a/llamafile/server/v1_chat_completions.cpp b/llamafile/server/v1_chat_completions.cpp
index 8b5cf490..ab8aedf6 100644
--- a/llamafile/server/v1_chat_completions.cpp
+++ b/llamafile/server/v1_chat_completions.cpp
@@ -32,6 +32,7 @@
#include "llamafile/server/worker.h"
#include "llamafile/string.h"
#include "llamafile/vector.h"
+#include <cassert>
#include <cmath>
#include <cstring>
#include <sys/resource.h>
@@ -163,6 +164,33 @@ make_event(const Json& json)
return s;
}
+static int
+has_images(const std::vector<Atom>& atoms)
+{
+ for (const Atom& atom : atoms)
+ if (atom.is_image())
+ return true;
+ return false;
+}
+
+static int
+count_tokens(const std::vector<Atom>& atoms)
+{
+ int n = 0;
+ for (const Atom& atom : atoms)
+ n += atom.ctx_used();
+ return n;
+}
+
+static int
+count_bytes(const std::vector<llama_chat_msg>& messages)
+{
+ int n = 0;
+ for (const llama_chat_msg& message : messages)
+ n += message.content.size();
+ return n;
+}
+
bool
Client::get_v1_chat_completions_params(V1ChatCompletionParams* params)
{
@@ -226,6 +254,8 @@ Client::get_v1_chat_completions_params(V1ChatCompletionParams* params)
return send_error(400, "message role not system user assistant");
if (!message["content"].isString())
return send_error(400, "message must have string content");
+ if (message["content"].getString().empty())
+ return send_error(400, "message must not have empty content");
params->messages.emplace_back(message["role"].getString(),
message["content"].getString());
}
@@ -456,18 +486,71 @@ Client::v1_chat_completions()
V1ChatCompletionResponse* response = new V1ChatCompletionResponse;
defer_cleanup(cleanup_response, response);
- // add bos token if it's needed
- if (llama_should_add_bos_token(model_))
- state->atoms.emplace_back(llama_token_bos(model_));
+ // turn prompt into atom array that'll fit in context window
+ for (;;) {
+ // add bos token if it's needed
+ if (llama_should_add_bos_token(model_))
+ state->atoms.emplace_back(llama_token_bos(model_));
+
+ // turn text into tokens
+ state->prompt = llama_chat_apply_template(
+ model_, FLAG_chat_template, params->messages, ADD_ASSISTANT);
+ atomize(model_, &state->atoms, state->prompt, PARSE_SPECIAL);
+
+ // we don't support multiple images yet
+ state->atoms = remove_old_image_atoms(state->atoms);
+
+ // acquire best slot
+ if (!slot_) {
+ slot_ = worker_->server_->slots_->take(state->atoms);
+ defer_cleanup(cleanup_slot, this);
+ }
- // turn text into tokens
- state->prompt = llama_chat_apply_template(
- model_, FLAG_chat_template, params->messages, ADD_ASSISTANT);
- atomize(model_, &state->atoms, state->prompt, PARSE_SPECIAL);
+ // check if image uploading is supported
+ if (!slot_->clip_ctx_ && has_images(state->atoms))
+ return send_error(400, "no_vision_model");
- // find appropriate slot
- slot_ = worker_->server_->slots_->take(state->atoms);
- defer_cleanup(cleanup_slot, this);
+ // check if we have enough context
+ int space = slot_->ctx_size();
+ int avail = space - space * FLAG_reserve_tokens;
+ int need = count_tokens(state->atoms);
+ unassert(avail > 0);
+ if (need <= avail)
+ break;
+
+ // calling atomize() again will append
+ state->atoms.clear();
+
+ // forget old messages to clear up space in context window
+ //
+ // to avoid calling tokenize quadratically, we use a crude byte
+ // count heuristic to determine how many messages to drop. this
+ // is needed since the client sends the whole history each time
+ unassert(!params->messages.empty());
+ int keep_msgs = params->messages[0].role == "system";
+ int max_forget_msgs = (int)params->messages.size() - (keep_msgs + 1);
+ if (max_forget_msgs <= 0) {
+ SLOG("ran out of chat messages to forget");
+ return send_error(400, "out_of_context_due_to_long_message");
+ }
+ int bytes = count_bytes(params->messages);
+ double percent_to_remember = (double)avail / need;
+ int bytes_to_delete = bytes * (1 - percent_to_remember);
+ auto first = params->messages.begin();
+ if (keep_msgs)
+ ++first;
+ auto last = first;
+ int bytes_deleted = 0;
+ int forgotten_msgs = 0;
+ do {
+ bytes_deleted += last->content.size();
+ ++forgotten_msgs;
+ ++last;
+ } while (bytes_deleted < bytes_to_delete &&
+ forgotten_msgs < max_forget_msgs);
+ params->messages.erase(first, last);
+ SLOG("forgot %d old messages", forgotten_msgs);
+ }
// init sampling
llama_sampling_context* sampler = create_sampler(params);
@@ -517,7 +600,12 @@ Client::v1_chat_completions()
if (prompt_tokens < 0) {
SLOG("slot prefill failed: %s", Slot::describe_error(prompt_tokens));
- return send_error(500, Slot::describe_error(prompt_tokens));
+ if (!params->stream) {
+ return send_error(500, Slot::describe_error(prompt_tokens));
+ } else {
+ close_connection_ = true;
+ return false;
+ }
}
// initialize response
@@ -541,7 +629,7 @@ Client::v1_chat_completions()
llama_token id = llama_sampling_sample(sampler, slot_->ctx_, NULL);
llama_sampling_accept(sampler, slot_->ctx_, id, APPLY_GRAMMAR);
++completion_tokens;
- if (!slot_->eval_token(id)) {
+ if (slot_->eval_token(id) < 0) {
SLOG("ran out of context window");
break;
}
diff --git a/llamafile/server/v1_completions.cpp b/llamafile/server/v1_completions.cpp
index ac9e4678..5e110420 100644
--- a/llamafile/server/v1_completions.cpp
+++ b/llamafile/server/v1_completions.cpp
@@ -402,6 +402,9 @@ Client::v1_completions()
// turn text into tokens
atomize(model_, &state->atoms, params->prompt, PARSE_SPECIAL);
+ // we don't support multiple images yet
+ state->atoms = remove_old_image_atoms(state->atoms);
+
// find appropriate slot
slot_ = worker_->server_->slots_->take(state->atoms);
defer_cleanup(cleanup_slot, this);
@@ -456,7 +459,7 @@ Client::v1_completions()
llama_token id = llama_sampling_sample(sampler, slot_->ctx_, NULL);
llama_sampling_accept(sampler, slot_->ctx_, id, DONT_APPLY_GRAMMAR);
++completion_tokens;
- if (!slot_->eval_token(id)) {
+ if (slot_->eval_token(id) < 0) {
SLOG("ran out of context window");
break;
}