Distribute and run LLMs with a single file.
C++ cross-platformggufllama-cpplocal-ailocal-inferencelocal-llmopen-source-aisingle-file-executablespeech-to-text
Add stream_options parameter for OpenAI compatible endpoints
corebonts97167eb63395
diff --git a/llamafile/server/doc/v1_chat_completions.md b/llamafile/server/doc/v1_chat_completions.mdindex 4eda4aa7..14d8df93 100644--- a/llamafile/server/doc/v1_chat_completions.md+++ b/llamafile/server/doc/v1_chat_completions.md@@ -78,6 +78,19 @@ This endpoint supports the following features:will be named delta instead. It's assumed the client will reconstructthe full conversation.+- `stream_options`: `object|null`++ Options for streaming the API response. This parameter is only+ applicable when `stream: true` is also specified. Default is `null`.++ - `include_usage`: `boolean|null`++ Whether to include usage statistics in the streaming response. Default is `false`.++ If set to `true`, a `usage` field with the usage information will be+ included in an additional empty chunk. Note that all other chunks will+ also contain this field, but with `null` value.+- `max_tokens`: `integer|null`Specifies an upper bound for the number of tokens that can bediff --git a/llamafile/server/v1_chat_completions.cpp b/llamafile/server/v1_chat_completions.cppindex 659815ee..b362341f 100644--- a/llamafile/server/v1_chat_completions.cpp+++ b/llamafile/server/v1_chat_completions.cpp@@ -46,6 +46,7 @@ namespace server {struct V1ChatCompletionParams{bool stream = false;+ bool stream_include_usage = false;long max_tokens = -1;long seed = _rand64();double top_p = 1;@@ -276,6 +277,26 @@ Client::get_v1_chat_completions_params(V1ChatCompletionParams* params)if (!stream.isBool())return send_error(400, "stream field must be boolean");params->stream = stream.getBool();++ // stream_options: object|null+ //+ // Options for the streaming response.+ Json& stream_options = json["stream_options"];+ if (!stream_options.isNull()) {+ if (!stream_options.isObject())+ return send_error(400, "stream_options field must be object");++ // include_usage: bool|null+ //+ // Include usage also for streaming responses. The actual usage will be reported before+ // the [DONE] message, but all chunks contain an empty usage field.+ Json& include_usage = stream_options["include_usage"];+ if (!include_usage.isNull()) {+ if (!include_usage.isBool())+ return send_error(400, "include_usage field must be boolean");+ params->stream_include_usage = include_usage.getBool();+ }+ }}// max_tokens: integer|null@@ -570,6 +591,8 @@ Client::v1_chat_completions()return false;choice["delta"]["role"] = "assistant";choice["delta"]["content"] = "";+ if (params->stream_include_usage)+ response->json["usage"] = nullptr;}// prefill time@@ -661,6 +684,12 @@ Client::v1_chat_completions()if (params->stream) {choice["delta"]["content"] = "";response->json["created"] = timespec_real().tv_sec;+ if (params->stream_include_usage) {+ Json& usage = response->json["usage"];+ usage["prompt_tokens"] = prompt_tokens;+ usage["completion_tokens"] = completion_tokens;+ usage["total_tokens"] = completion_tokens + prompt_tokens;+ }response->content = make_event(response->json);choice.getObject().erase("delta");if (!send_response_chunk(response->content))diff --git a/llamafile/server/v1_completions.cpp b/llamafile/server/v1_completions.cppindex 5e110420..b07fb8c8 100644--- a/llamafile/server/v1_completions.cpp+++ b/llamafile/server/v1_completions.cpp@@ -46,6 +46,7 @@ struct V1CompletionParams{bool echo = false;bool stream = false;+ bool stream_include_usage = false;long max_tokens = -1;long seed = _rand64();double top_p = 1;@@ -248,6 +249,26 @@ Client::get_v1_completions_params(V1CompletionParams* params)if (!stream.isBool())return send_error(400, "stream field must be boolean");params->stream = stream.getBool();++ // stream_options: object|null+ //+ // Options for the streaming response.+ Json& stream_options = json["stream_options"];+ if (!stream_options.isNull()) {+ if (!stream_options.isObject())+ return send_error(400, "stream_options field must be object");++ // include_usage: bool|null+ //+ // Include usage also for streaming responses. The actual usage will be reported before+ // the [DONE] message, but all chunks contain an empty usage field.+ Json& include_usage = stream_options["include_usage"];+ if (!include_usage.isNull()) {+ if (!include_usage.isBool())+ return send_error(400, "include_usage field must be boolean");+ params->stream_include_usage = include_usage.getBool();+ }+ }}// max_tokens: integer|null@@ -441,6 +462,8 @@ Client::v1_completions()choice["delta"]["role"] = "assistant";choice["delta"]["content"] = "";response->json["created"] = timespec_real().tv_sec;+ if (params->stream_include_usage)+ response->json["usage"] = nullptr;response->content = make_event(response->json);choice.getObject().erase("delta");if (!send_response_chunk(response->content))@@ -494,6 +517,12 @@ Client::v1_completions()if (params->stream) {choice["text"] = "";response->json["created"] = timespec_real().tv_sec;+ if (params->stream_include_usage) {+ Json& usage = response->json["usage"];+ usage["prompt_tokens"] = prompt_tokens;+ usage["completion_tokens"] = completion_tokens;+ usage["total_tokens"] = completion_tokens + prompt_tokens;+ }response->content = make_event(response->json);if (!send_response_chunk(response->content))return false;