Skip to content

Distribute and run LLMs with a single file.

Add stream_options parameter for OpenAI compatible endpoints

corebonts97167eb63395
diff --git a/llamafile/server/doc/v1_chat_completions.md b/llamafile/server/doc/v1_chat_completions.md
index 4eda4aa7..14d8df93 100644
--- a/llamafile/server/doc/v1_chat_completions.md
+++ b/llamafile/server/doc/v1_chat_completions.md
@@ -78,6 +78,19 @@ This endpoint supports the following features:
will be named delta instead. It's assumed the client will reconstruct
the full conversation.
+- `stream_options`: `object|null`
+
+ Options for streaming the API response. This parameter is only
+ applicable when `stream: true` is also specified. Default is `null`.
+
+ - `include_usage`: `boolean|null`
+
+ Whether to include usage statistics in the streaming response. Default is `false`.
+
+ If set to `true`, a `usage` field with the usage information will be
+ included in an additional empty chunk. Note that all other chunks will
+ also contain this field, but with `null` value.
+
- `max_tokens`: `integer|null`
Specifies an upper bound for the number of tokens that can be
diff --git a/llamafile/server/v1_chat_completions.cpp b/llamafile/server/v1_chat_completions.cpp
index 659815ee..b362341f 100644
--- a/llamafile/server/v1_chat_completions.cpp
+++ b/llamafile/server/v1_chat_completions.cpp
@@ -46,6 +46,7 @@ namespace server {
struct V1ChatCompletionParams
{
bool stream = false;
+ bool stream_include_usage = false;
long max_tokens = -1;
long seed = _rand64();
double top_p = 1;
@@ -276,6 +277,26 @@ Client::get_v1_chat_completions_params(V1ChatCompletionParams* params)
if (!stream.isBool())
return send_error(400, "stream field must be boolean");
params->stream = stream.getBool();
+
+ // stream_options: object|null
+ //
+ // Options for the streaming response.
+ Json& stream_options = json["stream_options"];
+ if (!stream_options.isNull()) {
+ if (!stream_options.isObject())
+ return send_error(400, "stream_options field must be object");
+
+ // include_usage: bool|null
+ //
+ // Include usage also for streaming responses. The actual usage will be reported before
+ // the [DONE] message, but all chunks contain an empty usage field.
+ Json& include_usage = stream_options["include_usage"];
+ if (!include_usage.isNull()) {
+ if (!include_usage.isBool())
+ return send_error(400, "include_usage field must be boolean");
+ params->stream_include_usage = include_usage.getBool();
+ }
+ }
}
// max_tokens: integer|null
@@ -570,6 +591,8 @@ Client::v1_chat_completions()
return false;
choice["delta"]["role"] = "assistant";
choice["delta"]["content"] = "";
+ if (params->stream_include_usage)
+ response->json["usage"] = nullptr;
}
// prefill time
@@ -661,6 +684,12 @@ Client::v1_chat_completions()
if (params->stream) {
choice["delta"]["content"] = "";
response->json["created"] = timespec_real().tv_sec;
+ if (params->stream_include_usage) {
+ Json& usage = response->json["usage"];
+ usage["prompt_tokens"] = prompt_tokens;
+ usage["completion_tokens"] = completion_tokens;
+ usage["total_tokens"] = completion_tokens + prompt_tokens;
+ }
response->content = make_event(response->json);
choice.getObject().erase("delta");
if (!send_response_chunk(response->content))
diff --git a/llamafile/server/v1_completions.cpp b/llamafile/server/v1_completions.cpp
index 5e110420..b07fb8c8 100644
--- a/llamafile/server/v1_completions.cpp
+++ b/llamafile/server/v1_completions.cpp
@@ -46,6 +46,7 @@ struct V1CompletionParams
{
bool echo = false;
bool stream = false;
+ bool stream_include_usage = false;
long max_tokens = -1;
long seed = _rand64();
double top_p = 1;
@@ -248,6 +249,26 @@ Client::get_v1_completions_params(V1CompletionParams* params)
if (!stream.isBool())
return send_error(400, "stream field must be boolean");
params->stream = stream.getBool();
+
+ // stream_options: object|null
+ //
+ // Options for the streaming response.
+ Json& stream_options = json["stream_options"];
+ if (!stream_options.isNull()) {
+ if (!stream_options.isObject())
+ return send_error(400, "stream_options field must be object");
+
+ // include_usage: bool|null
+ //
+ // Include usage also for streaming responses. The actual usage will be reported before
+ // the [DONE] message, but all chunks contain an empty usage field.
+ Json& include_usage = stream_options["include_usage"];
+ if (!include_usage.isNull()) {
+ if (!include_usage.isBool())
+ return send_error(400, "include_usage field must be boolean");
+ params->stream_include_usage = include_usage.getBool();
+ }
+ }
}
// max_tokens: integer|null
@@ -441,6 +462,8 @@ Client::v1_completions()
choice["delta"]["role"] = "assistant";
choice["delta"]["content"] = "";
response->json["created"] = timespec_real().tv_sec;
+ if (params->stream_include_usage)
+ response->json["usage"] = nullptr;
response->content = make_event(response->json);
choice.getObject().erase("delta");
if (!send_response_chunk(response->content))
@@ -494,6 +517,12 @@ Client::v1_completions()
if (params->stream) {
choice["text"] = "";
response->json["created"] = timespec_real().tv_sec;
+ if (params->stream_include_usage) {
+ Json& usage = response->json["usage"];
+ usage["prompt_tokens"] = prompt_tokens;
+ usage["completion_tokens"] = completion_tokens;
+ usage["total_tokens"] = completion_tokens + prompt_tokens;
+ }
response->content = make_event(response->json);
if (!send_response_chunk(response->content))
return false;