LLM inference in C/C++
C++ ggml
server: add "verbose" field to schema (#24864)
Xuan-Son Nguyenbfa3219177c8
diff --git a/tools/server/server-schema.cpp b/tools/server/server-schema.cppindex d5d747a654..ed4bda2412 100644--- a/tools/server/server-schema.cpp+++ b/tools/server/server-schema.cpp@@ -14,6 +14,9 @@ std::vector<std::unique_ptr<field>> make_llama_cmpl_schema(const common_params &fields.emplace_back(f);};+ add((new field_bool("verbose", params.verbose))+ ->set_desc("Include __verbose field in the response with additional debug information"));+add((new field_bool("timings_per_token", params.timings_per_token))->set_desc("Include prompt processing and text generation speed information in each response"));diff --git a/tools/server/tests/unit/test_chat_completion.py b/tools/server/tests/unit/test_chat_completion.pyindex b00aac649d..0258b539ed 100644--- a/tools/server/tests/unit/test_chat_completion.py+++ b/tools/server/tests/unit/test_chat_completion.py@@ -603,3 +603,23 @@ def test_chat_completions_token_count():})assert res.status_code == 200assert res.body["input_tokens"] > 5+++def test_verbose_debug():+ global server+ server.start()+ for verbose in [True, False]:+ res = server.make_request("POST", "/chat/completions", data={+ "max_tokens": 2,+ "messages": [+ {"role": "system", "content": "Book"},+ {"role": "user", "content": "What is the best book"},+ ],+ "verbose": verbose,+ })+ assert res.status_code == 200+ if verbose:+ assert "__verbose" in res.body+ assert "Book" in res.body["__verbose"]["prompt"]+ else:+ assert "__verbose" not in res.body