Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add OpenAI /v1/embeddings to new llamafiler server
Justine Tunneybe94c1ff317e
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex 6b690c89..d78fbc82 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -2141,7 +2141,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" - distribute: spread execution evenly over all nodes\n");printf(" - isolate: only spawn threads on CPUs on the node that execution started on\n");printf(" - numactl: use the CPU map provided my numactl\n");- if (llama_supports_gpu_offload()) {+ // if (llama_supports_gpu_offload()) { // [jart] prevent init errorprintf(" -ngl N, --n-gpu-layers N\n");printf(" number of layers to store in VRAM\n");printf(" -sm SPLIT_MODE, --split-mode SPLIT_MODE\n");@@ -2153,7 +2153,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" fraction of the model to offload to each GPU, comma-separated list of proportions, e.g. 3,1\n");printf(" -mg i, --main-gpu i the GPU to use for the model (with split-mode = none),\n");printf(" or for intermediate results and KV (with split-mode = row)\n");- }+ // } // [jart] prevent init errorprintf(" -m FNAME, --model FNAME\n");printf(" model path (default: %s)\n", params.model.c_str());printf(" -a ALIAS, --alias ALIAS\n");@@ -2431,13 +2431,13 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,invalid_param = true;break;}- if (llama_supports_gpu_offload()) {+ // if (llama_supports_gpu_offload()) { // [jart] prevent init errorparams.n_gpu_layers = std::stoi(argv[i]);- } else {- LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "- "See main README.md for information on enabling GPU BLAS support",- {{"n_gpu_layers", params.n_gpu_layers}});- }+ // } else {+ // LOG_WARNING("Not compiled with GPU offload support, --n-gpu-layers option will be ignored. "+ // "See main README.md for information on enabling GPU BLAS support",+ // {{"n_gpu_layers", params.n_gpu_layers}});+ // }}else if (arg == "--split-mode" || arg == "-sm"){@@ -2580,7 +2580,10 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}else if (arg == "--embedding"){- params.embedding = true;+ fprintf(stderr, "error: the --embedding endpoint is no longer supported in the the standard llamafile --server. "+ "please use our new llamafiler command, which gives you a 4x faster embedding server. this is our new "+ "server for llamafile that, once feature complete, will replace this one entirely.\n");+ exit(1);}else if (arg == "-cb" || arg == "--cont-batching"){@@ -2694,12 +2697,14 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,FLAGS_READY = true;+#if 0// [jart] setting `embeddings = true` on the clip model causes a// llama_get_logits_ith() fail crash due to how this param// due to `const bool has_logits = !cparams.embeddings;` from// llama.cpp interacting strangely with this parameter.if (params.mmproj.empty())params.embedding = true; // [jart] #243 always enable embedding mode+#endifparams.n_gpu_layers = llamafile_gpu_layers(params.n_gpu_layers);@@ -3512,6 +3517,23 @@ int server_cli(int argc, char **argv)svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res){++ // TODO(jart): something llama.cpp did upstream causes+ // logits to no longer be saved when we+ // enable embedding mode. let's use this as+ // an opportunity to nudge people into using+ // the newer better server, which is now+ // production worthy and recommended for+ // /embedding serving. it's compatible with+ // the existing http api.+ if (1) {+ fprintf(stderr, "warning: the --embedding endpoint is no longer supported in the the standard llamafile --server. "+ "please use our new llamafiler command, which gives you a 4x faster embedding server. this is our new "+ "server for llamafile that, once feature complete, will replace this one entirely.\n");+ res.status = 503;+ return res.set_content("Service Unavailable", "text/plain; charset=utf-8");+ }+res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));const json body = json::parse(req.body);json prompt;@@ -3548,6 +3570,23 @@ int server_cli(int argc, char **argv)svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res){++ // TODO(jart): something llama.cpp did upstream causes+ // logits to no longer be saved when we+ // enable embedding mode. let's use this as+ // an opportunity to nudge people into using+ // the newer better server, which is now+ // production worthy and recommended for+ // /embedding serving. it's compatible with+ // the existing http api.+ if (1) {+ fprintf(stderr, "warning: the --embedding endpoint is no longer supported in the the standard llamafile --server. "+ "please use our new llamafiler command, which gives you a 4x faster embedding server. this is our new "+ "server for llamafile that, once feature complete, will replace this one entirely.\n");+ res.status = 503;+ return res.set_content("Service Unavailable", "text/plain; charset=utf-8");+ }+res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));const json body = json::parse(req.body);diff --git a/llamafile/flags.cpp b/llamafile/flags.cppindex 9adb53e9..05858acb 100644--- a/llamafile/flags.cpp+++ b/llamafile/flags.cpp@@ -84,6 +84,12 @@ static wontreturn void bad(const char *flag) {exit(1);}+static wontreturn void nogpu(const char *flag) {+ tinyprint(2, program_invocation_name, ": ", flag, " was passed but ",+ program_invocation_short_name, " doesn't support GPU mode yet.\n", NULL);+ exit(1);+}+static wontreturn void missing(const char *flag) {tinyprint(2, program_invocation_name, ": ", flag, " missing argument\n", NULL);exit(1);@@ -100,6 +106,7 @@ static wontreturn void unknown(const char *flag) {}void llamafile_get_flags(int argc, char **argv) {+ bool program_supports_gpu = FLAG_gpu != LLAMAFILE_GPU_DISABLE;FLAG_threads = cpu_get_num_math();for (int i = 1; i < argc;) {const char *flag = argv[i++];@@ -320,16 +327,22 @@ void llamafile_get_flags(int argc, char **argv) {// gpu flagsif (!strcmp(flag, "--tinyblas")) {+ if (!program_supports_gpu)+ nogpu("--tinyblas");FLAG_tinyblas = true;continue;}if (!strcmp(flag, "--nocompile")) {+ if (!program_supports_gpu)+ nogpu("--nocompile");FLAG_nocompile = true;continue;}if (!strcmp(flag, "--recompile")) {+ if (!program_supports_gpu)+ nogpu("--recompile");FLAG_recompile = true;continue;}@@ -346,6 +359,8 @@ void llamafile_get_flags(int argc, char **argv) {if (!strcmp(flag, "-ngl") || //!strcmp(flag, "--gpu-layers") || //!strcmp(flag, "--n-gpu-layers")) {+ if (!program_supports_gpu)+ nogpu("--n-gpu-layers");if (i == argc)missing("--n-gpu-layers");FLAG_n_gpu_layers = atoi(argv[i++]);@@ -355,6 +370,8 @@ void llamafile_get_flags(int argc, char **argv) {}if (!strcmp(flag, "-mg") || !strcmp(flag, "--main-gpu")) {+ if (!program_supports_gpu)+ nogpu("--main-gpu");if (i == argc)missing("--main-gpu");FLAG_main_gpu = atoi(argv[i++]);@@ -362,6 +379,8 @@ void llamafile_get_flags(int argc, char **argv) {}if (!strcmp(flag, "-sm") || !strcmp(flag, "--split-mode")) {+ if (!program_supports_gpu)+ nogpu("--split-mode");if (i == argc)missing("--split-mode");const char *value = argv[i];diff --git a/llamafile/gpu.c b/llamafile/gpu.cindex 757805f8..e00ed03b 100644--- a/llamafile/gpu.c+++ b/llamafile/gpu.c@@ -53,6 +53,9 @@ bool llamafile_has_gpu(void) {*/int llamafile_gpu_layers(int n_gpu_layers) {+ if (FLAG_gpu == LLAMAFILE_GPU_DISABLE)+ return 0;+// if user explicitly passed `--gpu KIND` but didn't specify `-ngl// LAYERS` then assume the user wants their model fully offloaded.if (n_gpu_layers < 0 && FLAG_gpu > 0)diff --git a/llamafile/server/client.cpp b/llamafile/server/client.cppindex a28691ca..7a5f016c 100644--- a/llamafile/server/client.cpp+++ b/llamafile/server/client.cpp@@ -462,6 +462,8 @@ Client::dispatcher()return tokenize();if (path() == "/embedding")return embedding();+ if (path() == "/v1/embeddings")+ return embedding();if (path() == "/completion")return completion();return send_error(404);diff --git a/llamafile/server/doc/embedding.md b/llamafile/server/doc/embedding.mdindex 25623be6..728e4b7f 100644--- a/llamafile/server/doc/embedding.md+++ b/llamafile/server/doc/embedding.md@@ -14,7 +14,8 @@ classification, or content recommendation systems.## Request URIs-- `/embedding`+- `/embedding` (llama.cpp compatible)+- `/v1/embeddings` (OpenAI compatible)## Request Methods@@ -52,6 +53,12 @@ classification, or content recommendation systems.`tokens_provided`. The `/tokenize` endpoint may also be used to checkbeforehand how the model chops up strings and into how many pieces.+- `input` (string) is an alias for `content`, which is provided for+ OpenAI API compatibility.++- `prompt` (string) is an alias for `content`, which is provided for+ consistency with the `/tokenize` endpoint.+- `add_special` (bool; default: true) may be specified to indicate ifthe tokenizer should insert special tokens automatically. What tokensget inserted, depends on the model architecture. For example,diff --git a/llamafile/server/embedding.cpp b/llamafile/server/embedding.cppindex c5596642..ae9bd46d 100644--- a/llamafile/server/embedding.cpp+++ b/llamafile/server/embedding.cpp@@ -37,6 +37,7 @@ struct EmbeddingParamsbool parse_special;ctl::string_view prompt;ctl::string content;+ ctl::string model;};void@@ -78,10 +79,29 @@ Client::get_embedding_params(EmbeddingParams* params){params->add_special = atob(or_empty(param("add_special")), true);params->parse_special = atob(or_empty(param("parse_special")), false);++ // try obtaining prompt (or its aliases) from request-urictl::optional<ctl::string_view> prompt = param("content");+ if (!prompt.has_value()) {+ ctl::optional<ctl::string_view> prompt2 = param("prompt");+ if (prompt2.has_value()) {+ prompt = ctl::move(prompt2);+ } else {+ ctl::optional<ctl::string_view> prompt3 = param("input");+ if (prompt3.has_value()) {+ prompt = ctl::move(prompt3);+ }+ }+ }+if (prompt.has_value()) {+ // [simple mode] if the prompt was supplied in the request-uri+ // then we don't bother looking for a json body.params->prompt = prompt.value();} else if (HasHeader(kHttpContentType)) {+ // [standard mode] if the prompt wasn't specified as a+ // request-uri parameter, then it must be in the+ // http message body.if (IsMimeType(HeaderData(kHttpContentType),HeaderLength(kHttpContentType),"text/plain")) {@@ -94,14 +114,21 @@ Client::get_embedding_params(EmbeddingParams* params)return send_error(400, Json::StatusToString(json.first));if (!json.second.isObject())return send_error(400, "JSON body must be an object");- if (!json.second["content"].isString())- return send_error(400, "JSON missing \"content\" key");- params->content = ctl::move(json.second["content"].getString());+ if (json.second["content"].isString())+ params->content = ctl::move(json.second["content"].getString());+ else if (json.second["prompt"].isString())+ params->content = ctl::move(json.second["prompt"].getString());+ else if (json.second["input"].isString())+ params->content = ctl::move(json.second["input"].getString());+ else+ return send_error(400, "JSON missing content/prompt/input key");params->prompt = params->content;if (json.second["add_special"].isBool())params->add_special = json.second["add_special"].getBool();if (json.second["parse_special"].isBool())params->parse_special = json.second["parse_special"].getBool();+ if (json.second["model"].isString())+ params->model = ctl::move(json.second["model"].getString());} else {return send_error(501, "Content Type Not Implemented");}@@ -207,21 +234,68 @@ Client::embedding()embd, embeddings->data() + batch->seq_id[i][0] * n_embd, n_embd);}+ // determine how output json should look+ bool in_openai_mode = path() == "/v1/embeddings";+// serialize tokens to jsonchar* p = obuf.p;p = stpcpy(p, "{\n");- p = stpcpy(p, " \"add_special\": ");- p = encode_bool(p, params->add_special);- p = stpcpy(p, ",\n");- p = stpcpy(p, " \"parse_special\": ");- p = encode_bool(p, params->parse_special);- p = stpcpy(p, ",\n");- p = stpcpy(p, " \"tokens_provided\": ");- p = encode_json(p, toks->size());- p = stpcpy(p, ",\n");- p = stpcpy(p, " \"tokens_used\": ");- p = encode_json(p, count);- p = stpcpy(p, ",\n");++ // Here's what an OpenAI /v1/embedding response looks like:+ //+ // {+ // "object": "list",+ // "data": [+ // {+ // "object": "embedding",+ // "index": 0,+ // "embedding": [+ // -0.006929283495992422,+ // -0.005336422007530928,+ // ... (omitted for spacing)+ // -4.547132266452536e-05,+ // -0.024047505110502243+ // ],+ // }+ // ],+ // "model": "text-embedding-3-small",+ // "usage": {+ // "prompt_tokens": 5,+ // "total_tokens": 5+ // }+ // }+ //++ if (in_openai_mode) {+ p = stpcpy(p, " \"object\": \"list\",\n");+ p = stpcpy(p, " \"model\": ");+ p = encode_json(p, params->model);+ p = stpcpy(p, ",\n");+ p = stpcpy(p, " \"usage\": {\n");+ p = stpcpy(p, " \"prompt_tokens\": ");+ p = encode_json(p, count);+ p = stpcpy(p, ",\n");+ p = stpcpy(p, " \"total_tokens\": ");+ p = encode_json(p, toks->size());+ p = stpcpy(p, "\n },\n");+ p = stpcpy(p, " \"data\": [{\n");+ p = stpcpy(p, " \"object\": \"embedding\",\n");+ p = stpcpy(p, " \"index\": 0,\n");+ } else {+ p = stpcpy(p, " \"add_special\": ");+ p = encode_bool(p, params->add_special);+ p = stpcpy(p, ",\n");+ p = stpcpy(p, " \"parse_special\": ");+ p = encode_bool(p, params->parse_special);+ p = stpcpy(p, ",\n");+ p = stpcpy(p, " \"tokens_provided\": ");+ p = encode_json(p, toks->size());+ p = stpcpy(p, ",\n");+ p = stpcpy(p, " \"tokens_used\": ");+ p = encode_json(p, count);+ p = stpcpy(p, ",\n");+ }+p = stpcpy(p, " \"embedding\": [");for (size_t i = 0; i < embeddings->size(); ++i) {if (i) {@@ -231,6 +305,8 @@ Client::embedding()p = encode_json(p, (*embeddings)[i]);}p = stpcpy(p, "]\n");+ if (in_openai_mode)+ p = stpcpy(p, " }]\n");p = stpcpy(p, "}\n");ctl::string_view content(obuf.p, p - obuf.p);diff --git a/llamafile/server/main.cpp b/llamafile/server/main.cppindex e404ef54..b3482c91 100644--- a/llamafile/server/main.cpp+++ b/llamafile/server/main.cpp@@ -58,12 +58,6 @@ main(int argc, char* argv[])LoadZipArgs(&argc, &argv);llamafile_get_flags(argc, argv);- // bounce users wanting gpu support (not ready yet)- if (FLAG_n_gpu_layers) {- fprintf(stderr, "error: llamafiler doesn't support gpu yet\n");- return 1;- }-// initialize subsystemstime_init();tokenbucket_init();