LLM inference in C/C++
C++ ggml
server : return HTTP 400 on invalid grammar (#24144) (#24154)
Anuj Attri10786217e9d4
diff --git a/common/sampling.cpp b/common/sampling.cppindex c537f33503..75a299e23e 100644--- a/common/sampling.cpp+++ b/common/sampling.cpp@@ -259,6 +259,9 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st}}}+ if (!grmr && !grammar_str.empty()) {+ throw std::runtime_error("failed to parse grammar");+ }// Compute prefill tokens from the generation promptstd::vector<llama_token> prefill_tokens;diff --git a/tools/server/tests/unit/test_chat_completion.py b/tools/server/tests/unit/test_chat_completion.pyindex fe55dc5ab1..b00aac649d 100644--- a/tools/server/tests/unit/test_chat_completion.py+++ b/tools/server/tests/unit/test_chat_completion.py@@ -307,6 +307,20 @@ def test_completion_with_grammar(jinja: bool, grammar: str, n_predicted: int, reassert match_regex(re_content, choice["message"]["content"]), choice["message"]["content"]+def test_completion_with_invalid_grammar():+ global server+ server.start()+ res = server.make_request("POST", "/chat/completions", data={+ "max_tokens": 8,+ "messages": [+ {"role": "user", "content": "Does not matter what I say, does it?"},+ ],+ "grammar": "root ::= this is (not valid GBNF",+ })+ assert res.status_code == 400, res.body+ assert "error" in res.body++@pytest.mark.parametrize("messages", [None,"string",