Skip to content

LLM inference in C/C++

C++ ggml

json-schema : accept escaped hyphen in regex patterns (#29127)

chiheb ben cheikheb1e1f495f86
diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp
index e0426098c0..3349ae4fe0 100644
--- a/common/json-schema-to-grammar.cpp
+++ b/common/json-schema-to-grammar.cpp
@@ -321,7 +321,8 @@ static size_t gbnf_escape_length(const std::string & pattern, size_t pos) {
case 'x': n_hex = 2; break;
case 'u': n_hex = 4; break;
case 'U': n_hex = 8; break;
- case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']':
+ // keep in sync with parse_char() in src/llama-grammar.cpp
+ case 't': case 'r': case 'n': case '\\': case '"': case '[': case ']': case '-':
return 2;
default:
return 0;
diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp
index 4c4206c6e6..d5873c19d0 100755
--- a/tests/test-json-schema-to-grammar.cpp
+++ b/tests/test-json-schema-to-grammar.cpp
@@ -1476,6 +1476,32 @@ static void test_all(const std::string & title, std::function<void(const TestCas
)"""
});
+ test({
+ SUCCESS,
+ "regexp with escaped hyphen in a character class",
+ R"""({
+ "type": "string",
+ "pattern": "^[a-z\\-]+$"
+ })""",
+ R"""(
+ root ::= "\"" ([a-z\-]+) "\""
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )"""
+ });
+
+ test({
+ SUCCESS,
+ "regexp with escaped hyphen outside a character class",
+ R"""({
+ "type": "string",
+ "pattern": "^a\\-b$"
+ })""",
+ R"""(
+ root ::= "\"" ("a\-b") "\""
+ space ::= | " " | "\n"{1,2} [ \t]{0,20}
+ )"""
+ });
+
// a regexp that is invalid under any flavor is still an error
test({
FAILURE,
@@ -1494,7 +1520,7 @@ static void test_all(const std::string & title, std::function<void(const TestCas
R"""({
"type": "object",
"properties": {
- "a": { "type": "string", "pattern": "^[a-z\\-]+$" }
+ "a": { "type": "string", "pattern": "^(?=a)a$" }
},
"required": ["a"],
"additionalProperties": false