Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Support configurable URL prefix in server (#597)
Vlad Laskyeee4b46268e6
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cppindex 41eab79b..cbdf18aa 100644--- a/llama.cpp/common.cpp+++ b/llama.cpp/common.cpp@@ -1078,6 +1078,11 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_paparams.public_path = argv[i];return true;}+ if (arg == "--url-prefix") {+ CHECK_ARG+ params.url_prefix = argv[i];+ return true;+ }if (arg == "--api-key") {CHECK_ARGparams.api_keys.push_back(argv[i]);@@ -1605,6 +1610,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & paramoptions.push_back({ "server", " --host HOST", "ip address to listen (default: %s)", params.hostname.c_str() });options.push_back({ "server", " --port PORT", "port to listen (default: %d)", params.port });options.push_back({ "server", " --path PATH", "path to serve static files from (default: %s)", params.public_path.c_str() });+ options.push_back({ "server", " --url-prefix PREFIX", "Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile (default: %s)", params.url_prefix.c_str() });options.push_back({ "server", " --embedding(s)", "enable embedding endpoint (default: %s)", params.embedding ? "enabled" : "disabled" });options.push_back({ "server", " --api-key KEY", "API key to use for authentication (default: none)" });options.push_back({ "server", " --api-key-file FNAME", "path to file containing API keys (default: none)" });diff --git a/llama.cpp/common.h b/llama.cpp/common.hindex 44fd7a80..10012492 100644--- a/llama.cpp/common.h+++ b/llama.cpp/common.h@@ -214,6 +214,7 @@ struct gpt_params {std::string hostname = "127.0.0.1";std::string public_path = "";+ std::string url_prefix = "";std::string chat_template = "";std::string system_prompt = "";bool enable_chat_template = true;diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1index b396f331..8df738c4 100644--- a/llama.cpp/main/main.1+++ b/llama.cpp/main/main.1@@ -759,6 +759,11 @@ Path from which to serve static files..PpDefault:.Pa /zip/llama.cpp/server/public+.It Fl Fl url-prefix Ar PREFIX+Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile+.Pp+Default:+.Pa /.It Fl Fl nobrowserDo not attempt to open a web browser tab at startup..It Fl gan Ar N , Fl Fl grp-attn-n Ar Ndiff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.ascindex 6acae6fe..99a81de2 100644--- a/llama.cpp/main/main.1.asc+++ b/llama.cpp/main/main.1.asc@@ -735,6 +735,11 @@Default: [4m/zip/llama.cpp/server/public[0m+ [1m--url-prefix [4m[22mPREFIX[0m+ Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile.++ Default: [4m/[0m+[1m--nobrowser[0mDo not attempt to open a web browser tab at startup.diff --git a/llama.cpp/server/public/completion.js b/llama.cpp/server/public/completion.jsindex ab38a7b4..e5b404e2 100644--- a/llama.cpp/server/public/completion.js+++ b/llama.cpp/server/public/completion.js@@ -21,14 +21,14 @@ let generation_settings = null;//export async function* llama(prompt, params = {}, config = {}) {let controller = config.controller;-+ const url_prefix = config.url_prefix || "";if (!controller) {controller = new AbortController();}const completionParams = { ...paramDefaults, ...params, prompt };- const response = await fetch("/completion", {+ const response = await fetch(`${url_prefix}/completion`, {method: 'POST',body: JSON.stringify(completionParams),headers: {@@ -193,9 +193,10 @@ export const llamaComplete = async (params, controller, callback) => {}// Get the model info from the server. This is useful for getting the context window and so on.-export const llamaModelInfo = async () => {+export const llamaModelInfo = async (config = {}) => {if (!generation_settings) {- const props = await fetch("/props").then(r => r.json());+ const url_prefix = config.url_prefix || "";+ const props = await fetch(`${url_prefix}/props`).then(r => r.json());generation_settings = props.default_generation_settings;}return generation_settings;diff --git a/llama.cpp/server/public/index.html b/llama.cpp/server/public/index.htmlindex dba3669d..324c6412 100644--- a/llama.cpp/server/public/index.html+++ b/llama.cpp/server/public/index.html@@ -213,10 +213,10 @@<script type="module">import {html, h, signal, effect, computed, render, useSignal, useEffect, useRef, Component- } from '/index.js';+ } from './index.js';- import { llama } from '/completion.js';- import { SchemaConverter } from '/json-schema-to-grammar.mjs';+ import { llama } from './completion.js';+ import { SchemaConverter } from './json-schema-to-grammar.mjs';let selected_image = false;var slot_id = -1;@@ -419,7 +419,7 @@throw new Error("already running");}controller.value = new AbortController();- for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) {+ for await (const chunk of llama(prompt, llamaParams, { controller: controller.value, url_prefix: document.baseURI.replace(/\/+$/, '') })) {const data = chunk.data;if (data.stop) {diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cppindex da758c81..66de5c2e 100644--- a/llama.cpp/server/server.cpp+++ b/llama.cpp/server/server.cpp@@ -47,6 +47,7 @@ struct server_paramsbool nobrowser = false;bool slots_endpoint = true;bool metrics_endpoint = false;+ std::string url_prefix = "";};bool server_verbose = false;@@ -2171,6 +2172,7 @@ static void server_print_usage(const char *argv0, const gpt_params ¶ms,printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());+ printf(" --url-prefix PREFIX Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile (default: '/')\n");printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);@@ -2257,6 +2259,31 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,}sparams.public_path = argv[i];}+ else if (arg == "--url-prefix")+ {+ if (++i >= argc)+ {+ invalid_param = true;+ break;+ }+ sparams.url_prefix = argv[i];+ // 1. Consolidate consecutive slashes+ while (sparams.url_prefix.find("//") != std::string::npos) {+ sparams.url_prefix.replace(sparams.url_prefix.find("//"), 2, "/");+ }+ // 2 Ensure single slash at start+ if (sparams.url_prefix.empty() || sparams.url_prefix[0] != '/') {+ sparams.url_prefix = "/" + sparams.url_prefix;+ }+ // 3. Remove trailing slash if present+ if (!sparams.url_prefix.empty() && sparams.url_prefix.back() == '/') {+ sparams.url_prefix.pop_back();+ }+ // 4. If only a single slash remains, convert to empty string+ if (sparams.url_prefix == "/") {+ sparams.url_prefix = "";+ }+ }else if (arg == "--api-key"){if (++i >= argc)@@ -2883,7 +2910,7 @@ int server_cli(int argc, char **argv)res.set_header("Access-Control-Allow-Headers", "*");});- svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) {+ svr.Get(sparams.url_prefix + "/health", [&](const httplib::Request& req, httplib::Response& res) {server_state current_state = state.load();switch(current_state) {case SERVER_STATE_READY: {@@ -2933,7 +2960,7 @@ int server_cli(int argc, char **argv)});if (sparams.slots_endpoint) {- svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) {+ svr.Get(sparams.url_prefix + "/slots", [&](const httplib::Request&, httplib::Response& res) {// request slots data using task queuetask_server task;task.id = llama.queue_tasks.get_new_id();@@ -2953,7 +2980,7 @@ int server_cli(int argc, char **argv)}if (sparams.metrics_endpoint) {- svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) {+ svr.Get(sparams.url_prefix + "/metrics", [&](const httplib::Request&, httplib::Response& res) {// request slots data using task queuetask_server task;task.id = llama.queue_tasks.get_new_id();@@ -3099,14 +3126,14 @@ int server_cli(int argc, char **argv)}// Set the base directory for serving static files- svr.set_base_dir(sparams.public_path);+ svr.set_base_dir(sparams.public_path, sparams.url_prefix);// to make it ctrl+clickable:const char *connect_host;if (sparams.hostname != "0.0.0.0") {connect_host = sparams.hostname.c_str();- LOG_TEE("\nllama server listening at http://%s:%d\n\n",- sparams.hostname.c_str(), sparams.port);+ LOG_TEE("\nllama server listening at http://%s:%d%s\n\n",+ sparams.hostname.c_str(), sparams.port, sparams.url_prefix.c_str());} else {struct ifaddrs *ifaddrs;connect_host = "127.0.0.1";@@ -3115,23 +3142,25 @@ int server_cli(int argc, char **argv)for (struct ifaddrs *ifa = ifaddrs; ifa; ifa = ifa->ifa_next) {char buf[128];if (!(ifa->ifa_flags & IFF_UP)) continue;- LOG_TEE("llama server listening at http://%s%s%s:%d\n",+ LOG_TEE("llama server listening at http://%s%s%s:%d%s\n",ifa->ifa_addr->sa_family == AF_INET6 ? "[" : "",sockaddr2str(ifa->ifa_addr, buf, sizeof(buf)),ifa->ifa_addr->sa_family == AF_INET6 ? "]" : "",- sparams.port);+ sparams.port,+ sparams.url_prefix.c_str());}LOG_TEE("\n");} else {perror("getifaddrs");- LOG_TEE("\nllama server listening at http://%s:%d\n\n",- sparams.hostname.c_str(), sparams.port);+ LOG_TEE("\nllama server listening at http://%s:%d%s\n\n",+ sparams.hostname.c_str(), sparams.port, sparams.url_prefix.c_str());}}std::unordered_map<std::string, std::string> log_data;log_data["hostname"] = sparams.hostname;log_data["port"] = std::to_string(sparams.port);+ log_data["url_prefix"] = sparams.url_prefix;if (sparams.api_keys.size() == 1) {log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4);@@ -3141,8 +3170,18 @@ int server_cli(int argc, char **argv)// launch browser tabif (!sparams.nobrowser && !g_server_background_mode) {- char url[128];- snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);+ const size_t max_url_size = 2048; // commonly accepted maximum URL length+ char url[max_url_size];+ int written = snprintf(url, sizeof(url), "http://%s:%d%s/",+ connect_host, sparams.port, sparams.url_prefix.c_str());+ if (written < 0 || written >= (int)sizeof(url)) {+ LOG_ERROR("URL truncation occurred", {+ {"connect_host", connect_host},+ {"port", sparams.port},+ {"url_prefix", sparams.url_prefix},+ });+ return 1;+ }llamafile_launch_browser(url);}if (g_server_on_listening) {@@ -3250,7 +3289,7 @@ int server_cli(int argc, char **argv)return false;};- svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res)+ svr.Get(sparams.url_prefix + "/props", [&llama](const httplib::Request & req, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));json data = {@@ -3262,7 +3301,7 @@ int server_cli(int argc, char **argv)res.set_content(data.dump(), "application/json; charset=utf-8");});- svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)+ svr.Post(sparams.url_prefix + "/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));if (!validate_api_key(req, res)) {@@ -3339,7 +3378,7 @@ int server_cli(int argc, char **argv)}});- svr.Get("/v1/models", [¶ms](const httplib::Request& req, httplib::Response& res)+ svr.Get(sparams.url_prefix + "/v1/models", [¶ms](const httplib::Request& req, httplib::Response& res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));std::time_t t = std::time(0);@@ -3361,7 +3400,7 @@ int server_cli(int argc, char **argv)// TODO: add mount point without "/v1" prefix -- how?- svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res)+ svr.Post(sparams.url_prefix + "/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));if (!validate_api_key(req, res)) {@@ -3441,7 +3480,7 @@ int server_cli(int argc, char **argv)}});- svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)+ svr.Post(sparams.url_prefix + "/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));if (!validate_api_key(req, res)) {@@ -3511,7 +3550,7 @@ int server_cli(int argc, char **argv)svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res){ return res.set_content("", "application/json; charset=utf-8"); });- svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res)+ svr.Post(sparams.url_prefix + "/tokenize", [&llama](const httplib::Request &req, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));const json body = json::parse(req.body);@@ -3524,7 +3563,7 @@ int server_cli(int argc, char **argv)return res.set_content(data.dump(), "application/json; charset=utf-8");});- svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)+ svr.Post(sparams.url_prefix + "/detokenize", [&llama](const httplib::Request &req, httplib::Response &res){res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));const json body = json::parse(req.body);@@ -3539,7 +3578,7 @@ int server_cli(int argc, char **argv)return res.set_content(data.dump(), "application/json; charset=utf-8");});- svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res)+ svr.Post(sparams.url_prefix + "/embedding", [&llama](const httplib::Request &req, httplib::Response &res){// TODO(jart): something llama.cpp did upstream causes@@ -3592,7 +3631,7 @@ int server_cli(int argc, char **argv)return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");});- svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res)+ svr.Post(sparams.url_prefix + "/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res){// TODO(jart): something llama.cpp did upstream causes