Skip to content

LLM inference in C/C++

C++ ggml

server: add "verbose" field to schema (#24864)

Xuan-Son Nguyenbfa3219177c8
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cpp
index d5d747a654..ed4bda2412 100644
--- a/tools/server/server-schema.cpp
+++ b/tools/server/server-schema.cpp
@@ -14,6 +14,9 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &
fields.emplace_back(f);
};
+ add((new field_bool("verbose", params.verbose))
+ ->set_desc("Include __verbose field in the response with additional debug information"));
+
add((new field_bool("timings_per_token", params.timings_per_token))
->set_desc("Include prompt processing and text generation speed information in each response"));
diff --git a/tools/server/tests/unit/test_chat_completion.py b/tools/server/tests/unit/test_chat_completion.py
index b00aac649d..0258b539ed 100644
--- a/tools/server/tests/unit/test_chat_completion.py
+++ b/tools/server/tests/unit/test_chat_completion.py
@@ -603,3 +603,23 @@ def test_chat_completions_token_count():
})
assert res.status_code == 200
assert res.body["input_tokens"] > 5
+
+
+def test_verbose_debug():
+ global server
+ server.start()
+ for verbose in [True, False]:
+ res = server.make_request("POST", "/chat/completions", data={
+ "max_tokens": 2,
+ "messages": [
+ {"role": "system", "content": "Book"},
+ {"role": "user", "content": "What is the best book"},
+ ],
+ "verbose": verbose,
+ })
+ assert res.status_code == 200
+ if verbose:
+ assert "__verbose" in res.body
+ assert "Book" in res.body["__verbose"]["prompt"]
+ else:
+ assert "__verbose" not in res.body