Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Forget old messages when running out of context
Justine Tunney08e7a21b6894
diff --git a/llamafile/flags.cpp b/llamafile/flags.cppindex 7bd7cee4..390e9ec2 100644--- a/llamafile/flags.cpp+++ b/llamafile/flags.cpp@@ -67,6 +67,7 @@ const char *FLAG_www_root = "/zip/www";double FLAG_token_rate = 1;float FLAG_frequency_penalty = 0;float FLAG_presence_penalty = 0;+float FLAG_reserve_tokens = .15;float FLAG_temperature = .8;float FLAG_top_p = .95;int FLAG_batch = 256;@@ -463,6 +464,24 @@ void llamafile_get_flags(int argc, char **argv) {continue;}+ if (!strcmp(flag, "--reserve-tokens")) {+ if (i == argc)+ missing("--reserve-tokens");+ const char *s = argv[i++];+ if (strchr(s, '.')) {+ float f = atof(s);+ if (!(0 < f && f <= 1))+ error("--reserve-tokens FLOAT must be on the interval (0,1]");+ FLAG_reserve_tokens = f;+ } else {+ int n = atoi(s);+ if (!(1 <= n && n <= 100))+ error("--reserve-tokens INT must be between 1 and 100");+ FLAG_reserve_tokens = n / 100.;+ }+ continue;+ }+//////////////////////////////////////////////////////////////////////// cpu flagsdiff --git a/llamafile/llamafile.h b/llamafile/llamafile.hindex 15f34a7b..caf04f2d 100644--- a/llamafile/llamafile.h+++ b/llamafile/llamafile.h@@ -49,6 +49,7 @@ extern int FLAG_http_obuf_size;extern int FLAG_keepalive;extern int FLAG_main_gpu;extern int FLAG_n_gpu_layers;+extern float FLAG_reserve_tokens;extern int FLAG_slots;extern int FLAG_split_mode;extern int FLAG_threads;diff --git a/llamafile/server/atomize.cpp b/llamafile/server/atomize.cppindex 1cd8f339..a8039114 100644--- a/llamafile/server/atomize.cpp+++ b/llamafile/server/atomize.cpp@@ -76,5 +76,21 @@ atomize(const llama_model* model,}}+// having multiple images in the context window is janky right now, so+// let's erase old images from the chat history until we find out more+std::vector<Atom>+remove_old_image_atoms(const std::vector<Atom>& atoms)+{+ int last_image_idx = -1;+ for (int i = 0; i < atoms.size(); ++i)+ if (atoms[i].is_image())+ last_image_idx = i;+ std::vector<Atom> result;+ for (int i = 0; i < atoms.size(); i++)+ if (!atoms[i].is_image() || i == last_image_idx)+ result.emplace_back(atoms[i]);+ return result;+}+} // namespace server} // namespace lfdiff --git a/llamafile/server/main.1 b/llamafile/server/main.1index 58a59a43..d9679ca3 100644--- a/llamafile/server/main.1+++ b/llamafile/server/main.1@@ -169,6 +169,12 @@ write-ahead log.PRAGMA journal_mode=WAL;PRAGMA synchronous=NORMAL;.Ed+.It Fl Fl reserve-tokens+Percent of context window to reserve for predicted tokens. When the+server runs out of context window, old chat messages will be forgotten+until this percent of the context is empty. The default is 15%. If this+is specified as a floating point number, e.g. 0.15, then it'll be+multiplied by 100 to get the percent..El.Sh EXAMPLEHere's an example of how you might start this server:diff --git a/llamafile/server/main.1.asc b/llamafile/server/main.1.ascindex c2db481e..fd8b41b8 100644--- a/llamafile/server/main.1.asc+++ b/llamafile/server/main.1.asc@@ -1,227 +1,231 @@-[4mLLAMAFILER[24m(1) General Commands Manual [4mLLAMAFILER[24m(1)--[1mNAME[0m- llamafiler — fast reliable embedding server--[1mSYNOPSIS[0m- [1mllamafiler -m [4m[22mmodel.gguf[24m [flags...]--[1mDESCRIPTION[0m- [1mllamafiler [22mis a brand new HTTP server for Large Language Models (LLMs).- To date, its development has been focused on doing fewer things really- well, and right now that's serving embeddings. It offers you 3.4x the- performance, stronger security, client request prioritization, request- preemption, as well as request isolation that helps ensure software- bugs won't cause the whole server to crash.--[1mOPTIONS[0m- The following options are available:-- [1m--version[0m- Print version and exit.-- [1m-h[22m, [1m--help[0m- Show help message and exit.-- [1m-m [4m[22mFNAME[24m, [1m--model [4m[22mFNAME[0m- Path of GGUF model weights. Each server process is currently- limited to serving only one model. If you need to host multiple- models, then it's recommended that you run multiple instances- of llamafiler behind a reverse proxy such as NGINX or Redbean.-- [1m-mm [4m[22mFNAME[24m, [1m--mmproj [4m[22mFNAME[0m- Path of vision model weights.-- [1m--db [4m[22mFILE[0m- Specifies path of sqlite3 database.-- The default is [4m~/.llamafile/llamafile.sqlite3[0m-- [1m-l [4m[22mHOSTPORT[24m, [1m--listen [4m[22mHOSTPORT[0m- Specifies the local [HOST:]PORT on which the HTTP server should- listen. By default this is 0.0.0.0:8080 which means llamafiler- will bind to port 8080 on every locally available IPv4 network- interface. This option may currently only be specified once.-- [1m-c [4m[22mTOKENS[24m, [1m--ctx-size [4m[22mTOKENS[0m- Specifies context size. This specifies how long a completion- can get before it runs out of space. It defaults to 8k which- means 8192 tokens. Many models support a larger context size,- like 128k, but that'll need much more RAM or VRAM per slot. If- this value is larger than the trained context size of the- model, it'll be tuned down to the maximum. If this value is 0- or negative, the maximum number of tokens will be used.-- [1m-s [4m[22mCOUNT[24m, [1m--slots [4m[22mCOUNT[0m- Specifies how many slots to maintain. This defaults to 1. Slots- are used by chat completions requests. When such a request- comes in, the client needs to take control of a slot. When the- completion is finished, the slot is relinquished back to the- server. HTTP clients will wait for a slot to be relinquished if- none are available. Tuning this parameter to nicely fit avail‐- able RAM or VRAM can help you manage your server resources, and- control how much completion parallelism can happen. Please- note that [1m--ctx-size [22mhas a strong influence on how many slots- can be created.-- [1m-p [4m[22mTEXT[24m, [1m--prompt [4m[22mTEXT[24m, [1m--system-prompt [4m[22mTEXT[0m- Specifies system prompt. This value is passed along to the web- frontend.-- [1m--no-display-prompt[0m- Hide system prompt from web user interface.-- [1m--nologo[0m- Hide llamafile logo icon from web ui.-- [1m--url-prefix [4m[22mURLPREFIX[0m- Specifies a URL prefix (subdirectory) under which the HTTP- server will make the API accessible, e.g. /lamafiler. Useful- when running llamafiler behind a reverse proxy such as NGINX or- Redbean. By default, this is set to / (root).-- [1m-w [4m[22mN[24m, [1m--workers [4m[22mN[0m- Number of HTTP client handling threads.-- [1m--trust [4m[22mCIDR[0m- Adds a network to the trusted network list. This argument is- specified in the form IPV4/MASKBITS, e.g. 192.168.0.0/24. By- default, all clients are untrusted, which means they're subject- to token bucket throttling, and additional security precautions- that may cause request handling to go slightly slower. There‐- fore this flag is important to use if you want to accurately- benchmark llamafiler, since the server will otherwise see the- benchmark as a DDOS and deprioritize its traffic accordingly.-- [1m--ip-header [4m[22mSTR[0m- If this flag is passed a value, e.g. X-Forwarded-For, then any- trusted may send this header to your llamafile server to let it- know what the true effective client IPv4 address actually is.- After this happens the default security restrictions, e.g. to‐- ken bucket, will be measured and applied against that IPv4 ad‐- dress and its adjacent networks.-- [1m--token-rate [4m[22mN[0m- Specifies how many times per second a token is dropped in each- bucket. This setting is used to define a limitation on how- many TCP connects and HTTP messages each chunk of the IPv4 ad‐- dress space is permitted to send to llamafiler over a sustained- period of time. The default token rate is 1, which means that,- on a long enough timeline, a class-C network will be depriori‐- tized if it sends more than one request per second. No real- penalty actually applies though until the server runs out of- resources, e.g. HTTP request workers.-- [1m--token-burst [4m[22mN[0m- Specifies how many HTTP requests and TCP connects a given slice- of the IPv4 address space is permitted to send within a short- period of time, before token bucket restrictions kick in, and- cause the client to be deprioritized. By default, this value is- set to 100. It may be tuned to any value between 1 and 127 in‐- clusive.-- [1m--token-cidr [4m[22mN[0m- Specifies IPv4 address space granularity of token bucket algo‐- rithm, in network bits. By default, this value is set to 24- which means individual IPv4 addresses are viewed as being rep‐- resentative members of a class-C network, or in other words,- each group of 256 IPv4 addresses is lumped together. If one IP- in the group does something bad, then bad things happen to all- the other IPv4 addresses in that granule. This number may be- set to any integer between 3 and 32 inclusive. Specifying a- higher number will trade away system memory to increase network- specificity. For example, using 32 means that 4 billion indi‐- vidual token buckets will be created. By default, a background- thread drops one token in each bucket every second, so that- could potentially be a lot of busy work. A value of three means- that everyone on the Internet who talks to your server will- have to fight over only eight token buckets in total.-- [1m--unsecure[0m- Disables sandboxing. By default, llamafiler puts itself in a- SECCOMP BPF sandbox, so that even if your server gets hacked in- the worst possible way (some kind of C++ memory bug) then- there's very little damage an attacker will be able to do. This- works by restricting system calls using Cosmopolitan Libc's im‐- plementation of pledge() which is currently only supported on- Linux (other OSes will simply be unsecured by default). The- pledge security policy that's used by default is "stdio anet"- which means that only relatively harmless system calls like- read(), write(), and accept() are allowed once the server has- finished initializing. It's not possible for remotely executed- code to do things like launch subprocesses, read or write to- the filesystem, or initiate a new connection to a server.-- [1m-k [4m[22mN[24m, [1m--keepalive [4m[22mN[0m- Specifies the TCP keepalive interval in seconds. This value is- passed along to both TCP_KEEPIDLE and TCP_KEEPINTVL if they're- supported by the host operating system. If this value is- greater than 0, then the the SO_KEEPALIVE and TCP_NODELAY op‐- tions are enabled on network sockets, if supported by the host- operating system. The default keepalive is 5.-- [1m--http-obuf-size [4m[22mN[0m- Size of HTTP output buffer size, in bytes. Default is 1048576.-- [1m--http-ibuf-size [4m[22mN[0m- Size of HTTP input buffer size, in bytes. Default is 1048576.-- [1m--chat-template [4m[22mNAME[0m- Specifies or overrides chat template for model.-- Normally the GGUF metadata tokenizer.chat_template will specify- this value for instruct models. This flag may be used to either- override the chat template, or specify one when the GGUF meta‐- data field is absent, which effectively forces the web ui to- enable chatbot mode.-- Supported chat template names are: chatml, llama2, llama3, mis‐- tral (alias for llama2), phi3, zephyr, monarch, gemma, gemma2- (alias for gemma), orion, openchat, vicuna, vicuna-orca,- deepseek, command-r, chatglm3, chatglm4, minicpm, deepseek2, or- exaone3.-- It is also possible to pass the jinja2 template itself to this- argument. Since llamafiler doesn't currently support jinja2, a- heuristic will be used to guess which of the above templates- the template represents.-- [1m--completion-mode[0m- Forces web ui to operate in completion mode, rather than chat- mode. Normally the web ui chooses its mode based on the GGUF- metadata. Base models normally don't define tokenizer.chat_tem‐- plate whereas instruct models do. If it's a base model, then- the web ui will automatically use completion mode only, without- needing to specify this flag. This flag is useful in cases- where a prompt template is defined by the gguf, but it is de‐- sirable for the chat interface to be disabled.-- [1m--db-startup-sql[0m- Specifies SQL code that should be executed whenever connecting- to the SQLite database. The default is the following code,- which enables the write-ahead log.-- PRAGMA journal_mode=WAL;- PRAGMA synchronous=NORMAL;--[1mEXAMPLE[0m- Here's an example of how you might start this server:-- [1mllamafiler -m all-MiniLM-L6-v2.F32.gguf[0m-- Here's how to send a tokenization request:-- [1mcurl -v http://127.0.0.1:8080/tokenize?prompt=hello+world[0m-- Here's how to send a embedding request:-- [1mcurl -v http://127.0.0.1:8080/embedding?content=hello+world[0m--[1mDOCUMENTATION[0m- Read our Markdown documentation for additional help and tutorials. See- llamafile/server/doc/index.md in the source repository on GitHub.--[1mSEE ALSO[0m- [4mllamafile[24m(1), [4mwhisperfile[24m(1)--Mozilla Ocho November 30, 2024 [4mLLAMAFILER[24m(1)+LLAMAFILER(1) General Commands Manual LLAMAFILER(1)++NNAAMMEE+ llllaammaaffiilleerr - fast reliable embedding server++SSYYNNOOPPSSIISS+ llllaammaaffiilleerr --mm _m_o_d_e_l_._g_g_u_f [flags...]++DDEESSCCRRIIPPTTIIOONN+ llllaammaaffiilleerr is a brand new HTTP server for Large Language Models (LLMs). To+ date, its development has been focused on doing fewer things really well,+ and right now that's serving embeddings. It offers you 3.4x the+ performance, stronger security, client request prioritization, request+ preemption, as well as request isolation that helps ensure software bugs+ won't cause the whole server to crash.++OOPPTTIIOONNSS+ The following options are available:++ ----vveerrssiioonn+ Print version and exit.++ --hh, ----hheellpp+ Show help message and exit.++ --mm _F_N_A_M_E, ----mmooddeell _F_N_A_M_E+ Path of GGUF model weights. Each server process is currently+ limited to serving only one model. If you need to host multiple+ models, then it's recommended that you run multiple instances of+ llamafiler behind a reverse proxy such as NGINX or Redbean.++ --mmmm _F_N_A_M_E, ----mmmmpprroojj _F_N_A_M_E+ Path of vision model weights.++ ----ddbb _F_I_L_E+ Specifies path of sqlite3 database.++ The default is _~_/_._l_l_a_m_a_f_i_l_e_/_l_l_a_m_a_f_i_l_e_._s_q_l_i_t_e_3++ --ll _H_O_S_T_P_O_R_T, ----lliisstteenn _H_O_S_T_P_O_R_T+ Specifies the local [HOST:]PORT on which the HTTP server should+ listen. By default this is 0.0.0.0:8080 which means llamafiler+ will bind to port 8080 on every locally available IPv4 network+ interface. This option may currently only be specified once.++ --cc _T_O_K_E_N_S, ----ccttxx--ssiizzee _T_O_K_E_N_S+ Specifies context size. This specifies how long a completion can+ get before it runs out of space. It defaults to 8k which means 8192+ tokens. Many models support a larger context size, like 128k, but+ that'll need much more RAM or VRAM per slot. If this value is+ larger than the trained context size of the model, it'll be tuned+ down to the maximum. If this value is 0 or negative, the maximum+ number of tokens will be used.++ --ss _C_O_U_N_T, ----sslloottss _C_O_U_N_T+ Specifies how many slots to maintain. This defaults to 1. Slots are+ used by chat completions requests. When such a request comes in,+ the client needs to take control of a slot. When the completion is+ finished, the slot is relinquished back to the server. HTTP clients+ will wait for a slot to be relinquished if none are available.+ Tuning this parameter to nicely fit available RAM or VRAM can help+ you manage your server resources, and control how much completion+ parallelism can happen. Please note that ----ccttxx--ssiizzee has a strong+ influence on how many slots can be created.++ --pp _T_E_X_T, ----pprroommpptt _T_E_X_T, ----ssyysstteemm--pprroommpptt _T_E_X_T+ Specifies system prompt. This value is passed along to the web+ frontend.++ ----nnoo--ddiissppllaayy--pprroommpptt+ Hide system prompt from web user interface.++ ----nnoollooggoo+ Hide llamafile logo icon from web ui.++ ----uurrll--pprreeffiixx _U_R_L_P_R_E_F_I_X+ Specifies a URL prefix (subdirectory) under which the HTTP server+ will make the API accessible, e.g. /lamafiler. Useful when running+ llamafiler behind a reverse proxy such as NGINX or Redbean. By+ default, this is set to / (root).++ --ww _N, ----wwoorrkkeerrss _N+ Number of HTTP client handling threads.++ ----ttrruusstt _C_I_D_R+ Adds a network to the trusted network list. This argument is+ specified in the form IPV4/MASKBITS, e.g. 192.168.0.0/24. By+ default, all clients are untrusted, which means they're subject to+ token bucket throttling, and additional security precautions that+ may cause request handling to go slightly slower. Therefore this+ flag is important to use if you want to accurately benchmark+ llamafiler, since the server will otherwise see the benchmark as a+ DDOS and deprioritize its traffic accordingly.++ ----iipp--hheeaaddeerr _S_T_R+ If this flag is passed a value, e.g. X-Forwarded-For, then any+ trusted may send this header to your llamafile server to let it+ know what the true effective client IPv4 address actually is. After+ this happens the default security restrictions, e.g. token bucket,+ will be measured and applied against that IPv4 address and its+ adjacent networks.++ ----ttookkeenn--rraattee _N+ Specifies how many times per second a token is dropped in each+ bucket. This setting is used to define a limitation on how many+ TCP connects and HTTP messages each chunk of the IPv4 address space+ is permitted to send to llamafiler over a sustained period of time.+ The default token rate is 1, which means that, on a long enough+ timeline, a class-C network will be deprioritized if it sends more+ than one request per second. No real penalty actually applies+ though until the server runs out of resources, e.g. HTTP request+ workers.++ ----ttookkeenn--bbuurrsstt _N+ Specifies how many HTTP requests and TCP connects a given slice of+ the IPv4 address space is permitted to send within a short period+ of time, before token bucket restrictions kick in, and cause the+ client to be deprioritized. By default, this value is set to 100.+ It may be tuned to any value between 1 and 127 inclusive.++ ----ttookkeenn--cciiddrr _N+ Specifies IPv4 address space granularity of token bucket algorithm,+ in network bits. By default, this value is set to 24 which means+ individual IPv4 addresses are viewed as being representative+ members of a class-C network, or in other words, each group of 256+ IPv4 addresses is lumped together. If one IP in the group does+ something bad, then bad things happen to all the other IPv4+ addresses in that granule. This number may be set to any integer+ between 3 and 32 inclusive. Specifying a higher number will trade+ away system memory to increase network specificity. For example,+ using 32 means that 4 billion individual token buckets will be+ created. By default, a background thread drops one token in each+ bucket every second, so that could potentially be a lot of busy+ work. A value of three means that everyone on the Internet who+ talks to your server will have to fight over only eight token+ buckets in total.++ ----uunnsseeccuurree+ Disables sandboxing. By default, llamafiler puts itself in a+ SECCOMP BPF sandbox, so that even if your server gets hacked in the+ worst possible way (some kind of C++ memory bug) then there's very+ little damage an attacker will be able to do. This works by+ restricting system calls using Cosmopolitan Libc's implementation+ of pledge() which is currently only supported on Linux (other OSes+ will simply be unsecured by default). The pledge security policy+ that's used by default is "stdio anet" which means that only+ relatively harmless system calls like read(), write(), and accept()+ are allowed once the server has finished initializing. It's not+ possible for remotely executed code to do things like launch+ subprocesses, read or write to the filesystem, or initiate a new+ connection to a server.++ --kk _N, ----kkeeeeppaalliivvee _N+ Specifies the TCP keepalive interval in seconds. This value is+ passed along to both TCP_KEEPIDLE and TCP_KEEPINTVL if they're+ supported by the host operating system. If this value is greater+ than 0, then the the SO_KEEPALIVE and TCP_NODELAY options are+ enabled on network sockets, if supported by the host operating+ system. The default keepalive is 5.++ ----hhttttpp--oobbuuff--ssiizzee _N+ Size of HTTP output buffer size, in bytes. Default is 1048576.++ ----hhttttpp--iibbuuff--ssiizzee _N+ Size of HTTP input buffer size, in bytes. Default is 1048576.++ ----cchhaatt--tteemmppllaattee _N_A_M_E+ Specifies or overrides chat template for model.++ Normally the GGUF metadata tokenizer.chat_template will specify+ this value for instruct models. This flag may be used to either+ override the chat template, or specify one when the GGUF metadata+ field is absent, which effectively forces the web ui to enable+ chatbot mode.++ Supported chat template names are: chatml, llama2, llama3, mistral+ (alias for llama2), phi3, zephyr, monarch, gemma, gemma2 (alias for+ gemma), orion, openchat, vicuna, vicuna-orca, deepseek, command-r,+ chatglm3, chatglm4, minicpm, deepseek2, or exaone3.++ It is also possible to pass the jinja2 template itself to this+ argument. Since llamafiler doesn't currently support jinja2, a+ heuristic will be used to guess which of the above templates the+ template represents.++ ----ccoommpplleettiioonn--mmooddee+ Forces web ui to operate in completion mode, rather than chat mode.+ Normally the web ui chooses its mode based on the GGUF metadata.+ Base models normally don't define tokenizer.chat_template whereas+ instruct models do. If it's a base model, then the web ui will+ automatically use completion mode only, without needing to specify+ this flag. This flag is useful in cases where a prompt template is+ defined by the gguf, but it is desirable for the chat interface to+ be disabled.++ ----ddbb--ssttaarrttuupp--ssqqll+ Specifies SQL code that should be executed whenever connecting to+ the SQLite database. The default is the following code, which+ enables the write-ahead log.++ PRAGMA journal_mode=WAL;+ PRAGMA synchronous=NORMAL;++ ----rreesseerrvvee--ttookkeennss+ Percent of context window to reserve for predicted tokens. When the+ server runs out of context window, old chat messages will be+ forgotten until this percent of the context is empty. The default+ is 15%. If this is specified as a floating point number, e.g. 0.15,+ then it'll be multiplied by 100 to get the percent.++EEXXAAMMPPLLEE+ Here's an example of how you might start this server:++ llamafiler -m all-MiniLM-L6-v2.F32.gguf++ Here's how to send a tokenization request:++ curl -v http://127.0.0.1:8080/tokenize?prompt=hello+world++ Here's how to send a embedding request:++ curl -v http://127.0.0.1:8080/embedding?content=hello+world++DDOOCCUUMMEENNTTAATTIIOONN+ Read our Markdown documentation for additional help and tutorials. See+ llamafile/server/doc/index.md in the source repository on GitHub.++SSEEEE AALLSSOO+ llamafile(1), whisperfile(1)++Mozilla Ocho November 30, 2024 Mozilla Ochodiff --git a/llamafile/server/slot.cpp b/llamafile/server/slot.cppindex fee1ec80..78a9d9d2 100644--- a/llamafile/server/slot.cpp+++ b/llamafile/server/slot.cpp@@ -57,22 +57,6 @@ generate_system_fingerprint(const llama_context_params* cparams)return b;}-// having multiple images in the context window is janky right now, so-// let's erase old images from the chat history until we find out more-static std::vector<Atom>-remove_old_image_atoms(const std::vector<Atom>& atoms)-{- int last_image_idx = -1;- for (int i = 0; i < atoms.size(); ++i)- if (atoms[i].is_image())- last_image_idx = i;- std::vector<Atom> result;- for (int i = 0; i < atoms.size(); i++)- if (!atoms[i].is_image() || i == last_image_idx)- result.emplace_back(atoms[i]);- return result;-}-const char*Slot::describe_error(int err){@@ -255,11 +239,14 @@ Slot::eval_atoms(const std::vector<Atom>& atoms,if (atom.is_token()) {total_work += 1;} else if (atom.is_image()) {- llava_image_embed* image_embed = llava_image_embed_make_with_bytes(- clip_ctx_,- FLAG_threads_batch,- (const unsigned char*)atom.image().bytes().data(),- atom.image().bytes().size());+ if (!clip_ctx_)+ return no_vision_model;+ llava_image_embed* image_embed =+ llava_image_embed_make_with_bytes(+ clip_ctx_,+ FLAG_threads_batch,+ (const unsigned char*)atom.image().bytes().data(),+ atom.image().bytes().size());if (image_embed) {total_work += image_embed->n_image_pos;llava_image_embed_free(image_embed);@@ -299,11 +286,10 @@ Slot::eval_atoms(const std::vector<Atom>& atoms,}int-Slot::prefill(const std::vector<Atom>& atoms_, const ProgressCallback& progress)+Slot::prefill(const std::vector<Atom>& atoms, const ProgressCallback& progress){if (!ctx_)return uninitialized;- std::vector<Atom> atoms = remove_old_image_atoms(atoms_);int used_tokens = ctx_used();int reuse_atoms = 0;int reuse_tokens = 0;diff --git a/llamafile/server/utils.h b/llamafile/server/utils.hindex 346be6a9..4ee1cea2 100644--- a/llamafile/server/utils.h+++ b/llamafile/server/utils.h@@ -44,5 +44,8 @@ atomize(const llama_model* model,std::string_view s,bool parse_special);+std::vector<Atom>+remove_old_image_atoms(const std::vector<Atom>&);+} // namespace server} // namespace lfdiff --git a/llamafile/server/v1_chat_completions.cpp b/llamafile/server/v1_chat_completions.cppindex 8b5cf490..ab8aedf6 100644--- a/llamafile/server/v1_chat_completions.cpp+++ b/llamafile/server/v1_chat_completions.cpp@@ -32,6 +32,7 @@#include "llamafile/server/worker.h"#include "llamafile/string.h"#include "llamafile/vector.h"+#include <cassert>#include <cmath>#include <cstring>#include <sys/resource.h>@@ -163,6 +164,33 @@ make_event(const Json& json)return s;}+static int+has_images(const std::vector<Atom>& atoms)+{+ for (const Atom& atom : atoms)+ if (atom.is_image())+ return true;+ return false;+}++static int+count_tokens(const std::vector<Atom>& atoms)+{+ int n = 0;+ for (const Atom& atom : atoms)+ n += atom.ctx_used();+ return n;+}++static int+count_bytes(const std::vector<llama_chat_msg>& messages)+{+ int n = 0;+ for (const llama_chat_msg& message : messages)+ n += message.content.size();+ return n;+}+boolClient::get_v1_chat_completions_params(V1ChatCompletionParams* params){@@ -226,6 +254,8 @@ Client::get_v1_chat_completions_params(V1ChatCompletionParams* params)return send_error(400, "message role not system user assistant");if (!message["content"].isString())return send_error(400, "message must have string content");+ if (message["content"].getString().empty())+ return send_error(400, "message must not have empty content");params->messages.emplace_back(message["role"].getString(),message["content"].getString());}@@ -456,18 +486,71 @@ Client::v1_chat_completions()V1ChatCompletionResponse* response = new V1ChatCompletionResponse;defer_cleanup(cleanup_response, response);- // add bos token if it's needed- if (llama_should_add_bos_token(model_))- state->atoms.emplace_back(llama_token_bos(model_));+ // turn prompt into atom array that'll fit in context window+ for (;;) {+ // add bos token if it's needed+ if (llama_should_add_bos_token(model_))+ state->atoms.emplace_back(llama_token_bos(model_));++ // turn text into tokens+ state->prompt = llama_chat_apply_template(+ model_, FLAG_chat_template, params->messages, ADD_ASSISTANT);+ atomize(model_, &state->atoms, state->prompt, PARSE_SPECIAL);++ // we don't support multiple images yet+ state->atoms = remove_old_image_atoms(state->atoms);++ // acquire best slot+ if (!slot_) {+ slot_ = worker_->server_->slots_->take(state->atoms);+ defer_cleanup(cleanup_slot, this);+ }- // turn text into tokens- state->prompt = llama_chat_apply_template(- model_, FLAG_chat_template, params->messages, ADD_ASSISTANT);- atomize(model_, &state->atoms, state->prompt, PARSE_SPECIAL);+ // check if image uploading is supported+ if (!slot_->clip_ctx_ && has_images(state->atoms))+ return send_error(400, "no_vision_model");- // find appropriate slot- slot_ = worker_->server_->slots_->take(state->atoms);- defer_cleanup(cleanup_slot, this);+ // check if we have enough context+ int space = slot_->ctx_size();+ int avail = space - space * FLAG_reserve_tokens;+ int need = count_tokens(state->atoms);+ unassert(avail > 0);+ if (need <= avail)+ break;++ // calling atomize() again will append+ state->atoms.clear();++ // forget old messages to clear up space in context window+ //+ // to avoid calling tokenize quadratically, we use a crude byte+ // count heuristic to determine how many messages to drop. this+ // is needed since the client sends the whole history each time+ unassert(!params->messages.empty());+ int keep_msgs = params->messages[0].role == "system";+ int max_forget_msgs = (int)params->messages.size() - (keep_msgs + 1);+ if (max_forget_msgs <= 0) {+ SLOG("ran out of chat messages to forget");+ return send_error(400, "out_of_context_due_to_long_message");+ }+ int bytes = count_bytes(params->messages);+ double percent_to_remember = (double)avail / need;+ int bytes_to_delete = bytes * (1 - percent_to_remember);+ auto first = params->messages.begin();+ if (keep_msgs)+ ++first;+ auto last = first;+ int bytes_deleted = 0;+ int forgotten_msgs = 0;+ do {+ bytes_deleted += last->content.size();+ ++forgotten_msgs;+ ++last;+ } while (bytes_deleted < bytes_to_delete &&+ forgotten_msgs < max_forget_msgs);+ params->messages.erase(first, last);+ SLOG("forgot %d old messages", forgotten_msgs);+ }// init samplingllama_sampling_context* sampler = create_sampler(params);@@ -517,7 +600,12 @@ Client::v1_chat_completions()if (prompt_tokens < 0) {SLOG("slot prefill failed: %s", Slot::describe_error(prompt_tokens));- return send_error(500, Slot::describe_error(prompt_tokens));+ if (!params->stream) {+ return send_error(500, Slot::describe_error(prompt_tokens));+ } else {+ close_connection_ = true;+ return false;+ }}// initialize response@@ -541,7 +629,7 @@ Client::v1_chat_completions()llama_token id = llama_sampling_sample(sampler, slot_->ctx_, NULL);llama_sampling_accept(sampler, slot_->ctx_, id, APPLY_GRAMMAR);++completion_tokens;- if (!slot_->eval_token(id)) {+ if (slot_->eval_token(id) < 0) {SLOG("ran out of context window");break;}diff --git a/llamafile/server/v1_completions.cpp b/llamafile/server/v1_completions.cppindex ac9e4678..5e110420 100644--- a/llamafile/server/v1_completions.cpp+++ b/llamafile/server/v1_completions.cpp@@ -402,6 +402,9 @@ Client::v1_completions()// turn text into tokensatomize(model_, &state->atoms, params->prompt, PARSE_SPECIAL);+ // we don't support multiple images yet+ state->atoms = remove_old_image_atoms(state->atoms);+// find appropriate slotslot_ = worker_->server_->slots_->take(state->atoms);defer_cleanup(cleanup_slot, this);@@ -456,7 +459,7 @@ Client::v1_completions()llama_token id = llama_sampling_sample(sampler, slot_->ctx_, NULL);llama_sampling_accept(sampler, slot_->ctx_, id, DONT_APPLY_GRAMMAR);++completion_tokens;- if (!slot_->eval_token(id)) {+ if (slot_->eval_token(id) < 0) {SLOG("ran out of context window");break;}