Skip to content

Distribute and run LLMs with a single file.

Support configurable URL prefix in server (#597)

Vlad Laskyeee4b46268e6
diff --git a/llama.cpp/common.cpp b/llama.cpp/common.cpp
index 41eab79b..cbdf18aa 100644
--- a/llama.cpp/common.cpp
+++ b/llama.cpp/common.cpp
@@ -1078,6 +1078,11 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa
params.public_path = argv[i];
return true;
}
+ if (arg == "--url-prefix") {
+ CHECK_ARG
+ params.url_prefix = argv[i];
+ return true;
+ }
if (arg == "--api-key") {
CHECK_ARG
params.api_keys.push_back(argv[i]);
@@ -1605,6 +1610,7 @@ void gpt_params_print_usage(int /*argc*/, char ** argv, const gpt_params & param
options.push_back({ "server", " --host HOST", "ip address to listen (default: %s)", params.hostname.c_str() });
options.push_back({ "server", " --port PORT", "port to listen (default: %d)", params.port });
options.push_back({ "server", " --path PATH", "path to serve static files from (default: %s)", params.public_path.c_str() });
+ options.push_back({ "server", " --url-prefix PREFIX", "Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile (default: %s)", params.url_prefix.c_str() });
options.push_back({ "server", " --embedding(s)", "enable embedding endpoint (default: %s)", params.embedding ? "enabled" : "disabled" });
options.push_back({ "server", " --api-key KEY", "API key to use for authentication (default: none)" });
options.push_back({ "server", " --api-key-file FNAME", "path to file containing API keys (default: none)" });
diff --git a/llama.cpp/common.h b/llama.cpp/common.h
index 44fd7a80..10012492 100644
--- a/llama.cpp/common.h
+++ b/llama.cpp/common.h
@@ -214,6 +214,7 @@ struct gpt_params {
std::string hostname = "127.0.0.1";
std::string public_path = "";
+ std::string url_prefix = "";
std::string chat_template = "";
std::string system_prompt = "";
bool enable_chat_template = true;
diff --git a/llama.cpp/main/main.1 b/llama.cpp/main/main.1
index b396f331..8df738c4 100644
--- a/llama.cpp/main/main.1
+++ b/llama.cpp/main/main.1
@@ -759,6 +759,11 @@ Path from which to serve static files.
.Pp
Default:
.Pa /zip/llama.cpp/server/public
+.It Fl Fl url-prefix Ar PREFIX
+Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile
+.Pp
+Default:
+.Pa /
.It Fl Fl nobrowser
Do not attempt to open a web browser tab at startup.
.It Fl gan Ar N , Fl Fl grp-attn-n Ar N
diff --git a/llama.cpp/main/main.1.asc b/llama.cpp/main/main.1.asc
index 6acae6fe..99a81de2 100644
--- a/llama.cpp/main/main.1.asc
+++ b/llama.cpp/main/main.1.asc
@@ -735,6 +735,11 @@
Default: /zip/llama.cpp/server/public
+ --url-prefix PREFIX
+ Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile.
+
+ Default: /
+
--nobrowser
Do not attempt to open a web browser tab at startup.
diff --git a/llama.cpp/server/public/completion.js b/llama.cpp/server/public/completion.js
index ab38a7b4..e5b404e2 100644
--- a/llama.cpp/server/public/completion.js
+++ b/llama.cpp/server/public/completion.js
@@ -21,14 +21,14 @@ let generation_settings = null;
//
export async function* llama(prompt, params = {}, config = {}) {
let controller = config.controller;
-
+ const url_prefix = config.url_prefix || "";
if (!controller) {
controller = new AbortController();
}
const completionParams = { ...paramDefaults, ...params, prompt };
- const response = await fetch("/completion", {
+ const response = await fetch(`${url_prefix}/completion`, {
method: 'POST',
body: JSON.stringify(completionParams),
headers: {
@@ -193,9 +193,10 @@ export const llamaComplete = async (params, controller, callback) => {
}
// Get the model info from the server. This is useful for getting the context window and so on.
-export const llamaModelInfo = async () => {
+export const llamaModelInfo = async (config = {}) => {
if (!generation_settings) {
- const props = await fetch("/props").then(r => r.json());
+ const url_prefix = config.url_prefix || "";
+ const props = await fetch(`${url_prefix}/props`).then(r => r.json());
generation_settings = props.default_generation_settings;
}
return generation_settings;
diff --git a/llama.cpp/server/public/index.html b/llama.cpp/server/public/index.html
index dba3669d..324c6412 100644
--- a/llama.cpp/server/public/index.html
+++ b/llama.cpp/server/public/index.html
@@ -213,10 +213,10 @@
<script type="module">
import {
html, h, signal, effect, computed, render, useSignal, useEffect, useRef, Component
- } from '/index.js';
+ } from './index.js';
- import { llama } from '/completion.js';
- import { SchemaConverter } from '/json-schema-to-grammar.mjs';
+ import { llama } from './completion.js';
+ import { SchemaConverter } from './json-schema-to-grammar.mjs';
let selected_image = false;
var slot_id = -1;
@@ -419,7 +419,7 @@
throw new Error("already running");
}
controller.value = new AbortController();
- for await (const chunk of llama(prompt, llamaParams, { controller: controller.value })) {
+ for await (const chunk of llama(prompt, llamaParams, { controller: controller.value, url_prefix: document.baseURI.replace(/\/+$/, '') })) {
const data = chunk.data;
if (data.stop) {
diff --git a/llama.cpp/server/server.cpp b/llama.cpp/server/server.cpp
index da758c81..66de5c2e 100644
--- a/llama.cpp/server/server.cpp
+++ b/llama.cpp/server/server.cpp
@@ -47,6 +47,7 @@ struct server_params
bool nobrowser = false;
bool slots_endpoint = true;
bool metrics_endpoint = false;
+ std::string url_prefix = "";
};
bool server_verbose = false;
@@ -2171,6 +2172,7 @@ static void server_print_usage(const char *argv0, const gpt_params &params,
printf(" --host ip address to listen (default (default: %s)\n", sparams.hostname.c_str());
printf(" --port PORT port to listen (default (default: %d)\n", sparams.port);
printf(" --path PUBLIC_PATH path from which to serve static files (default %s)\n", sparams.public_path.c_str());
+ printf(" --url-prefix PREFIX Specify a URL prefix (subdirectory) under which the API will be served, e.g. /llamafile (default: '/')\n");
printf(" --api-key API_KEY optional api key to enhance server security. If set, requests must include this key for access.\n");
printf(" --api-key-file FNAME path to file containing api keys delimited by new lines. If set, requests must include one of the keys for access.\n");
printf(" -to N, --timeout N server read/write timeout in seconds (default: %d)\n", sparams.read_timeout);
@@ -2257,6 +2259,31 @@ static void server_params_parse(int argc, char **argv, server_params &sparams,
}
sparams.public_path = argv[i];
}
+ else if (arg == "--url-prefix")
+ {
+ if (++i >= argc)
+ {
+ invalid_param = true;
+ break;
+ }
+ sparams.url_prefix = argv[i];
+ // 1. Consolidate consecutive slashes
+ while (sparams.url_prefix.find("//") != std::string::npos) {
+ sparams.url_prefix.replace(sparams.url_prefix.find("//"), 2, "/");
+ }
+ // 2 Ensure single slash at start
+ if (sparams.url_prefix.empty() || sparams.url_prefix[0] != '/') {
+ sparams.url_prefix = "/" + sparams.url_prefix;
+ }
+ // 3. Remove trailing slash if present
+ if (!sparams.url_prefix.empty() && sparams.url_prefix.back() == '/') {
+ sparams.url_prefix.pop_back();
+ }
+ // 4. If only a single slash remains, convert to empty string
+ if (sparams.url_prefix == "/") {
+ sparams.url_prefix = "";
+ }
+ }
else if (arg == "--api-key")
{
if (++i >= argc)
@@ -2883,7 +2910,7 @@ int server_cli(int argc, char **argv)
res.set_header("Access-Control-Allow-Headers", "*");
});
- svr.Get("/health", [&](const httplib::Request& req, httplib::Response& res) {
+ svr.Get(sparams.url_prefix + "/health", [&](const httplib::Request& req, httplib::Response& res) {
server_state current_state = state.load();
switch(current_state) {
case SERVER_STATE_READY: {
@@ -2933,7 +2960,7 @@ int server_cli(int argc, char **argv)
});
if (sparams.slots_endpoint) {
- svr.Get("/slots", [&](const httplib::Request&, httplib::Response& res) {
+ svr.Get(sparams.url_prefix + "/slots", [&](const httplib::Request&, httplib::Response& res) {
// request slots data using task queue
task_server task;
task.id = llama.queue_tasks.get_new_id();
@@ -2953,7 +2980,7 @@ int server_cli(int argc, char **argv)
}
if (sparams.metrics_endpoint) {
- svr.Get("/metrics", [&](const httplib::Request&, httplib::Response& res) {
+ svr.Get(sparams.url_prefix + "/metrics", [&](const httplib::Request&, httplib::Response& res) {
// request slots data using task queue
task_server task;
task.id = llama.queue_tasks.get_new_id();
@@ -3099,14 +3126,14 @@ int server_cli(int argc, char **argv)
}
// Set the base directory for serving static files
- svr.set_base_dir(sparams.public_path);
+ svr.set_base_dir(sparams.public_path, sparams.url_prefix);
// to make it ctrl+clickable:
const char *connect_host;
if (sparams.hostname != "0.0.0.0") {
connect_host = sparams.hostname.c_str();
- LOG_TEE("\nllama server listening at http://%s:%d\n\n",
- sparams.hostname.c_str(), sparams.port);
+ LOG_TEE("\nllama server listening at http://%s:%d%s\n\n",
+ sparams.hostname.c_str(), sparams.port, sparams.url_prefix.c_str());
} else {
struct ifaddrs *ifaddrs;
connect_host = "127.0.0.1";
@@ -3115,23 +3142,25 @@ int server_cli(int argc, char **argv)
for (struct ifaddrs *ifa = ifaddrs; ifa; ifa = ifa->ifa_next) {
char buf[128];
if (!(ifa->ifa_flags & IFF_UP)) continue;
- LOG_TEE("llama server listening at http://%s%s%s:%d\n",
+ LOG_TEE("llama server listening at http://%s%s%s:%d%s\n",
ifa->ifa_addr->sa_family == AF_INET6 ? "[" : "",
sockaddr2str(ifa->ifa_addr, buf, sizeof(buf)),
ifa->ifa_addr->sa_family == AF_INET6 ? "]" : "",
- sparams.port);
+ sparams.port,
+ sparams.url_prefix.c_str());
}
LOG_TEE("\n");
} else {
perror("getifaddrs");
- LOG_TEE("\nllama server listening at http://%s:%d\n\n",
- sparams.hostname.c_str(), sparams.port);
+ LOG_TEE("\nllama server listening at http://%s:%d%s\n\n",
+ sparams.hostname.c_str(), sparams.port, sparams.url_prefix.c_str());
}
}
std::unordered_map<std::string, std::string> log_data;
log_data["hostname"] = sparams.hostname;
log_data["port"] = std::to_string(sparams.port);
+ log_data["url_prefix"] = sparams.url_prefix;
if (sparams.api_keys.size() == 1) {
log_data["api_key"] = "api_key: ****" + sparams.api_keys[0].substr(sparams.api_keys[0].length() - 4);
@@ -3141,8 +3170,18 @@ int server_cli(int argc, char **argv)
// launch browser tab
if (!sparams.nobrowser && !g_server_background_mode) {
- char url[128];
- snprintf(url, sizeof(url), "http://%s:%d/", connect_host, sparams.port);
+ const size_t max_url_size = 2048; // commonly accepted maximum URL length
+ char url[max_url_size];
+ int written = snprintf(url, sizeof(url), "http://%s:%d%s/",
+ connect_host, sparams.port, sparams.url_prefix.c_str());
+ if (written < 0 || written >= (int)sizeof(url)) {
+ LOG_ERROR("URL truncation occurred", {
+ {"connect_host", connect_host},
+ {"port", sparams.port},
+ {"url_prefix", sparams.url_prefix},
+ });
+ return 1;
+ }
llamafile_launch_browser(url);
}
if (g_server_on_listening) {
@@ -3250,7 +3289,7 @@ int server_cli(int argc, char **argv)
return false;
};
- svr.Get("/props", [&llama](const httplib::Request & req, httplib::Response &res)
+ svr.Get(sparams.url_prefix + "/props", [&llama](const httplib::Request & req, httplib::Response &res)
{
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
json data = {
@@ -3262,7 +3301,7 @@ int server_cli(int argc, char **argv)
res.set_content(data.dump(), "application/json; charset=utf-8");
});
- svr.Post("/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)
+ svr.Post(sparams.url_prefix + "/completion", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)
{
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
if (!validate_api_key(req, res)) {
@@ -3339,7 +3378,7 @@ int server_cli(int argc, char **argv)
}
});
- svr.Get("/v1/models", [&params](const httplib::Request& req, httplib::Response& res)
+ svr.Get(sparams.url_prefix + "/v1/models", [&params](const httplib::Request& req, httplib::Response& res)
{
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
std::time_t t = std::time(0);
@@ -3361,7 +3400,7 @@ int server_cli(int argc, char **argv)
// TODO: add mount point without "/v1" prefix -- how?
- svr.Post("/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res)
+ svr.Post(sparams.url_prefix + "/v1/chat/completions", [&llama, &validate_api_key, &sparams](const httplib::Request &req, httplib::Response &res)
{
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
if (!validate_api_key(req, res)) {
@@ -3441,7 +3480,7 @@ int server_cli(int argc, char **argv)
}
});
- svr.Post("/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)
+ svr.Post(sparams.url_prefix + "/infill", [&llama, &validate_api_key](const httplib::Request &req, httplib::Response &res)
{
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
if (!validate_api_key(req, res)) {
@@ -3511,7 +3550,7 @@ int server_cli(int argc, char **argv)
svr.Options(R"(/.*)", [](const httplib::Request &, httplib::Response &res)
{ return res.set_content("", "application/json; charset=utf-8"); });
- svr.Post("/tokenize", [&llama](const httplib::Request &req, httplib::Response &res)
+ svr.Post(sparams.url_prefix + "/tokenize", [&llama](const httplib::Request &req, httplib::Response &res)
{
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
const json body = json::parse(req.body);
@@ -3524,7 +3563,7 @@ int server_cli(int argc, char **argv)
return res.set_content(data.dump(), "application/json; charset=utf-8");
});
- svr.Post("/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)
+ svr.Post(sparams.url_prefix + "/detokenize", [&llama](const httplib::Request &req, httplib::Response &res)
{
res.set_header("Access-Control-Allow-Origin", req.get_header_value("Origin"));
const json body = json::parse(req.body);
@@ -3539,7 +3578,7 @@ int server_cli(int argc, char **argv)
return res.set_content(data.dump(), "application/json; charset=utf-8");
});
- svr.Post("/embedding", [&llama](const httplib::Request &req, httplib::Response &res)
+ svr.Post(sparams.url_prefix + "/embedding", [&llama](const httplib::Request &req, httplib::Response &res)
{
// TODO(jart): something llama.cpp did upstream causes
@@ -3592,7 +3631,7 @@ int server_cli(int argc, char **argv)
return res.set_content(result.result_json.dump(), "application/json; charset=utf-8");
});
- svr.Post("/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res)
+ svr.Post(sparams.url_prefix + "/v1/embeddings", [&llama](const httplib::Request &req, httplib::Response &res)
{
// TODO(jart): something llama.cpp did upstream causes