LLM inference in C/C++
C++ ggml
common/peg : suppress incomplete escape sequences (#26780)
Aldehir Rojas48d22e295e2b
diff --git a/common/peg-parser.cpp b/common/peg-parser.cppindex ef290ed7c0..4a4be7cf78 100644--- a/common/peg-parser.cpp+++ b/common/peg-parser.cpp@@ -570,23 +570,34 @@ struct parser_executor {}static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos, const char delimiter) {+ auto save = pos;+++pos; // consume '\'if (pos >= ctx.input.size()) {if (!ctx.is_lenient()) {return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);}+ pos = save; // suppress unmatched '\'return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);}char c = ctx.input[pos];+if (c == delimiter || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') {++pos;return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos);- } else if (c == 'u') {- return handle_unicode_escape(ctx, start, pos);- } else {- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);}++ if (c == 'u') {+ auto result = handle_unicode_escape(ctx, start, pos);+ if (result.need_more_input()) {+ pos = save; // suppress incomplete sequence+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos);+ }+ return result;+ }++ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start);}static common_peg_parse_result handle_unicode_escape(common_peg_parse_context & ctx, size_t start, size_t & pos) {diff --git a/tests/peg-parser/test-json-parser.cpp b/tests/peg-parser/test-json-parser.cppindex 5dd00115ce..ec7c2e668f 100644--- a/tests/peg-parser/test-json-parser.cpp+++ b/tests/peg-parser/test-json-parser.cpp@@ -77,6 +77,30 @@ void test_json_parser(testing &t) {t.assert_equal("result_is_need_more_input", true, result.need_more_input());});+ // Test need_more_input() parsing - incomplete escape sequence in a string value+ t.test("need_more_input() parsing - incomplete escape sequence", [](testing &t) {+ auto json = build_peg_parser([](common_peg_parser_builder & p) { return p.json(); });++ std::vector<std::string> inputs {+ R"({"text": "hello\)", // dangling backslash+ R"({"text": "hello\u)", // incomplete unicode escape sequence+ R"({"text": "hello\u00)",+ };++ for (const auto & input : inputs) {+ t.test(input, [&](testing &t) {+ common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);++ auto result = json.parse(ctx);++ t.assert_equal("result_is_need_more_input", true, result.need_more_input());++ // the incomplete escape sequence is not part of the partial value+ t.assert_equal("result_end", input.find('\\'), result.end);+ });+ }+ });+t.test("object member", [](testing &t) {auto parser = build_peg_parser([](common_peg_parser_builder & p) {return p.json_member("name", "\"" + p.chars("[a-z]") + "\"");