LLM inference in C/C++
C++ ggml
common/peg : handle invalid utf-8 sequences in the AST (#29161)
Aldehir Rojas3d82ef62d47f
diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cppindex ffa43a3188..f827974952 100644--- a/common/chat-peg-parser.cpp+++ b/common/chat-peg-parser.cpp@@ -318,13 +318,13 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) {bool is_content = node.tag == common_chat_peg_builder::CONTENT;if (is_reasoning) { // GPT OSS can have more than 1 reasoning block, so concatenate here- result.reasoning_content += std::string(node.text);+ result.reasoning_content += node.sanitized_text();}if (is_content) {// Concatenate content from multiple content nodes (e.g., when reasoning markers// are preserved before content markers in reasoning_format=NONE mode)- result.content += std::string(node.text);+ result.content += node.sanitized_text();}// Handle tool-related tags (supporting both JSON and tagged formats)@@ -1058,12 +1058,12 @@ void common_chat_peg_gemma4_mapper::visit(const common_peg_ast_arena & arena, coconst auto & node = arena.get(id);if (node.tag == "reasoning") {- result.reasoning_content += std::string(node.text);+ result.reasoning_content += node.sanitized_text();return;}if (node.tag == "content") {- result.content += std::string(node.text);+ result.content += node.sanitized_text();return;}@@ -1206,12 +1206,12 @@ void common_chat_peg_minimax_m3_mapper::visit(const common_peg_ast_arena & arenaconst auto & node = arena.get(id);if (node.tag == common_chat_peg_builder::REASONING) {- result.reasoning_content += std::string(node.text);+ result.reasoning_content += node.sanitized_text();return;}if (node.tag == common_chat_peg_builder::CONTENT) {- result.content += std::string(node.text);+ result.content += node.sanitized_text();return;}diff --git a/common/peg-parser.cpp b/common/peg-parser.cppindex 10735389ea..75a908a281 100644--- a/common/peg-parser.cpp+++ b/common/peg-parser.cpp@@ -166,6 +166,25 @@ common_peg_ast_id common_peg_ast_arena::find_by_rule(const common_peg_ast_node &return COMMON_PEG_INVALID_AST_ID;}+std::string common_peg_ast_node::sanitized_text() const {+ if (invalid_utf8.empty()) {+ return std::string(text);+ }++ std::string out;+ out.reserve(text.size() + 2 * invalid_utf8.size());++ size_t seg_start = start;+ for (const auto & invalid : invalid_utf8) {+ out.append(text.data() + (seg_start - start), invalid.pos - seg_start);+ out.append("\xEF\xBF\xBD");+ seg_start = invalid.pos + invalid.len;+ }+ out.append(text.data() + (seg_start - start), end - seg_start);++ return out;+}+void common_peg_ast_arena::visit(common_peg_ast_id id, const common_peg_ast_visitor & visitor) const {if (id == COMMON_PEG_INVALID_AST_ID) {return;@@ -282,6 +301,7 @@ struct parser_executor {auto pos = start_pos;std::vector<common_peg_ast_id> nodes;+ std::vector<common_peg_invalid_utf8> invalid_utf8;for (size_t i = 0; i < p.children.size(); i++) {const auto & child_id = p.children[i];@@ -306,13 +326,14 @@ struct parser_executor {if (!result.nodes.empty()) {nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());}+ invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());if (result.need_more_input()) {ctx.parse_depth--;if (ctx.is_debug()) {fprintf(stderr, "%sSEQ -> NEED_MORE\n", debug_indent().c_str());}- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes));+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8));}pos = result.end;@@ -322,7 +343,7 @@ struct parser_executor {if (ctx.is_debug()) {fprintf(stderr, "%sSEQ -> SUCCESS at %zu->%zu\n", debug_indent().c_str(), start_pos, pos);}- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes));+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8));}common_peg_parse_result operator()(const common_peg_choice_parser & p) {@@ -370,6 +391,7 @@ struct parser_executor {auto pos = start_pos;int match_count = 0;std::vector<common_peg_ast_id> nodes;+ std::vector<common_peg_invalid_utf8> invalid_utf8;// Try to match up to max_count times (or unlimited if max_count is -1)while (p.max_count == -1 || match_count < p.max_count) {@@ -400,6 +422,7 @@ struct parser_executor {if (!result.nodes.empty()) {nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());}+ invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());pos = result.end;match_count++;@@ -410,13 +433,14 @@ struct parser_executor {if (!result.nodes.empty()) {nodes.insert(nodes.end(), result.nodes.begin(), result.nodes.end());}+ invalid_utf8.insert(invalid_utf8.end(), result.invalid_utf8.begin(), result.invalid_utf8.end());ctx.parse_depth--;if (ctx.is_debug()) {fprintf(stderr, "%sREPEAT -> NEED_MORE (count=%d, nodes=%zu)\n", debug_indent().c_str(),match_count, nodes.size());}- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes));+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, result.end, std::move(nodes), std::move(invalid_utf8));}// Child failed - stop trying@@ -434,7 +458,7 @@ struct parser_executor {fprintf(stderr, "%sREPEAT -> NEED_MORE (not enough matches: %d < %d)\n", debug_indent().c_str(),match_count, p.min_count);}- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes));+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, pos, std::move(nodes), std::move(invalid_utf8));}if (ctx.is_debug()) {fprintf(stderr, "%sREPEAT -> FAIL (not enough matches: %d < %d)\n", debug_indent().c_str(), match_count,@@ -448,7 +472,7 @@ struct parser_executor {fprintf(stderr, "%sREPEAT -> SUCCESS (count=%d, nodes=%zu)\n", debug_indent().c_str(), match_count,nodes.size());}- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes));+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, std::move(nodes), std::move(invalid_utf8));}common_peg_parse_result operator()(const common_peg_and_parser & p) {@@ -664,23 +688,23 @@ struct parser_executor {// Scan input and check for delimiterssize_t pos = start_pos;size_t last_valid_pos = start_pos;+ std::vector<common_peg_invalid_utf8> invalid_utf8;while (pos < ctx.input.size()) {auto utf8_result = common_parse_utf8_codepoint(ctx.input, pos);- if (utf8_result.status == utf8_parse_result::INCOMPLETE) {- // Incomplete UTF-8 sequence- if (!ctx.is_lenient()) {- // Input is complete but UTF-8 is incomplete = malformed- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);- }- // Return what we have so far (before incomplete sequence)- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos);+ if (utf8_result.status == utf8_parse_result::INCOMPLETE && ctx.is_lenient()) {+ // The rest of the sequence may still arrive, return what we have so far+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8));}- if (utf8_result.status == utf8_parse_result::INVALID) {- // Malformed UTF-8- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start_pos);+ if (utf8_result.status != utf8_parse_result::SUCCESS) {+ // Malformed UTF-8, or a sequence truncated by the end of a complete input.+ // A delimiter cannot start inside bytes that fail to decode, so consume them and move on+ invalid_utf8.push_back({pos, utf8_result.bytes_consumed});+ pos += utf8_result.bytes_consumed;+ last_valid_pos = pos;+ continue;}// Check if a delimiter starts at this position@@ -688,12 +712,12 @@ struct parser_executor {if (match == common_trie::COMPLETE_MATCH) {// Found a complete delimiter, return everything before it- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));}if (match == common_trie::PARTIAL_MATCH) {// Found a partial match extending to end of input, return everything before it- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos);+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, pos, {}, std::move(invalid_utf8));}pos += utf8_result.bytes_consumed;@@ -702,9 +726,9 @@ struct parser_executor {if (last_valid_pos == ctx.input.size() && ctx.is_lenient()) {// Reached the end of a partial stream, there might still be more input that we need to consume.- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos);+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start_pos, last_valid_pos, {}, std::move(invalid_utf8));}- return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos);+ return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start_pos, last_valid_pos, {}, std::move(invalid_utf8));}common_peg_parse_result operator()(const common_peg_schema_parser & p) {@@ -728,10 +752,11 @@ struct parser_executor {result.end,text,std::move(result.nodes),- result.need_more_input()+ result.need_more_input(),+ result.invalid_utf8);- return common_peg_parse_result(result.type, result.start, result.end, { node_id });+ return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8));}return result;@@ -757,10 +782,11 @@ struct parser_executor {result.end,text,std::move(result.nodes),- result.need_more_input()+ result.need_more_input(),+ result.invalid_utf8);- return common_peg_parse_result(result.type, result.start, result.end, { node_id });+ return common_peg_parse_result(result.type, result.start, result.end, { node_id }, std::move(result.invalid_utf8));}return result;diff --git a/common/peg-parser.h b/common/peg-parser.hindex fb5d82b30f..8883259457 100644--- a/common/peg-parser.h+++ b/common/peg-parser.h@@ -72,6 +72,12 @@ enum common_peg_parse_result_type {const char * common_peg_parse_result_type_name(common_peg_parse_result_type type);+// A run of input bytes that does not decode as UTF-8+struct common_peg_invalid_utf8 {+ size_t pos;+ size_t len;+};+struct common_peg_ast_node {common_peg_ast_id id;std::string rule;@@ -82,6 +88,12 @@ struct common_peg_ast_node {std::vector<common_peg_ast_id> children;bool is_partial = false;++ // Invalid UTF-8 inside the node, in ascending order+ std::vector<common_peg_invalid_utf8> invalid_utf8;++ // Returns the text with every invalid run replaced by U+FFFD+ std::string sanitized_text() const;};struct common_peg_parse_result;@@ -98,10 +110,11 @@ class common_peg_ast_arena {size_t end,std::string_view text,std::vector<common_peg_ast_id> children,- bool is_partial = false+ bool is_partial = false,+ std::vector<common_peg_invalid_utf8> invalid_utf8 = {}) {common_peg_ast_id id = nodes_.size();- nodes_.push_back({id, rule, tag, start, end, text, std::move(children), is_partial});+ nodes_.push_back({id, rule, tag, start, end, text, std::move(children), is_partial, std::move(invalid_utf8)});return id;}@@ -127,6 +140,9 @@ struct common_peg_parse_result {std::vector<common_peg_ast_id> nodes;+ // Invalid UTF-8 consumed by this result, carried up to the enclosing AST nodes+ std::vector<common_peg_invalid_utf8> invalid_utf8;+common_peg_parse_result() = default;common_peg_parse_result(common_peg_parse_result_type type, size_t start)@@ -135,8 +151,8 @@ struct common_peg_parse_result {common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end): type(type), start(start), end(end) {}- common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end, std::vector<common_peg_ast_id> nodes)- : type(type), start(start), end(end), nodes(std::move(nodes)) {}+ common_peg_parse_result(common_peg_parse_result_type type, size_t start, size_t end, std::vector<common_peg_ast_id> nodes, std::vector<common_peg_invalid_utf8> invalid_utf8 = {})+ : type(type), start(start), end(end), nodes(std::move(nodes)), invalid_utf8(std::move(invalid_utf8)) {}bool fail() const { return type == COMMON_PEG_PARSE_RESULT_FAIL; }bool need_more_input() const { return type == COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT; }@@ -430,6 +446,7 @@ class common_peg_parser_builder {common_peg_parser space() { return add(common_peg_space_parser{}); }// Matches all characters until a delimiter is found (delimiter not consumed).+ // Invalid UTF-8 is consumed and recorded on the AST nodes.// S -> (!delim .)*common_peg_parser until(const std::string & delimiter) { return add(common_peg_until_parser{{delimiter}}); }diff --git a/common/unicode.cpp b/common/unicode.cppindex f71fe56783..4ebe94a9ec 100644--- a/common/unicode.cpp+++ b/common/unicode.cpp@@ -26,16 +26,16 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off// Invalid: continuation byte as first byteif (!(input[offset] & 0x40)) {- return utf8_parse_result(utf8_parse_result::INVALID);+ return utf8_parse_result(utf8_parse_result::INVALID, 0, 1);}// 2-byte sequenceif (!(input[offset] & 0x20)) {if (offset + 1 >= input.size()) {- return utf8_parse_result(utf8_parse_result::INCOMPLETE);+ return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, 1);}if ((input[offset + 1] & 0xc0) != 0x80) {- return utf8_parse_result(utf8_parse_result::INVALID);+ return utf8_parse_result(utf8_parse_result::INVALID, 0, 1);}auto result = ((input[offset] & 0x1f) << 6) | (input[offset + 1] & 0x3f);return utf8_parse_result(utf8_parse_result::SUCCESS, result, 2);@@ -43,11 +43,14 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off// 3-byte sequenceif (!(input[offset] & 0x10)) {- if (offset + 2 >= input.size()) {- return utf8_parse_result(utf8_parse_result::INCOMPLETE);- }- if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80) {- return utf8_parse_result(utf8_parse_result::INVALID);+ // Check one byte at a time so a bad byte is reported before a short input+ for (size_t i = 1; i < 3; i++) {+ if (offset + i >= input.size()) {+ return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, i);+ }+ if ((input[offset + i] & 0xc0) != 0x80) {+ return utf8_parse_result(utf8_parse_result::INVALID, 0, i);+ }}auto result = ((input[offset] & 0x0f) << 12) | ((input[offset + 1] & 0x3f) << 6) | (input[offset + 2] & 0x3f);return utf8_parse_result(utf8_parse_result::SUCCESS, result, 3);@@ -55,18 +58,20 @@ utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t off// 4-byte sequenceif (!(input[offset] & 0x08)) {- if (offset + 3 >= input.size()) {- return utf8_parse_result(utf8_parse_result::INCOMPLETE);- }- if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80 || (input[offset + 3] & 0xc0) != 0x80) {- return utf8_parse_result(utf8_parse_result::INVALID);+ for (size_t i = 1; i < 4; i++) {+ if (offset + i >= input.size()) {+ return utf8_parse_result(utf8_parse_result::INCOMPLETE, 0, i);+ }+ if ((input[offset + i] & 0xc0) != 0x80) {+ return utf8_parse_result(utf8_parse_result::INVALID, 0, i);+ }}auto result = ((input[offset] & 0x07) << 18) | ((input[offset + 1] & 0x3f) << 12) | ((input[offset + 2] & 0x3f) << 6) | (input[offset + 3] & 0x3f);return utf8_parse_result(utf8_parse_result::SUCCESS, result, 4);}// Invalid first byte- return utf8_parse_result(utf8_parse_result::INVALID);+ return utf8_parse_result(utf8_parse_result::INVALID, 0, 1);}bool common_utf8_is_complete(const std::string & s) {diff --git a/common/unicode.h b/common/unicode.hindex 9b32fa19d6..380e77e4c7 100644--- a/common/unicode.h+++ b/common/unicode.h@@ -9,7 +9,7 @@struct utf8_parse_result {uint32_t codepoint; // Decoded codepoint (only valid if status == SUCCESS)- size_t bytes_consumed; // How many bytes this codepoint uses (1-4)+ size_t bytes_consumed; // How many bytes this codepoint uses (1-4), or the length of the valid prefix if status != SUCCESSenum status { SUCCESS, INCOMPLETE, INVALID } status;utf8_parse_result(enum status s, uint32_t cp = 0, size_t bytes = 0)diff --git a/tests/peg-parser/test-unicode.cpp b/tests/peg-parser/test-unicode.cppindex 24663d7017..2eaafa1742 100644--- a/tests/peg-parser/test-unicode.cpp+++ b/tests/peg-parser/test-unicode.cpp@@ -273,19 +273,35 @@ void test_unicode(testing &t) {});t.test("malformed UTF-8", [](testing &t) {- std::vector<test_case> test_cases {+ struct passthrough_case {+ std::string input;+ std::string expected_text;+ std::string expected_sanitized;+ };++ std::vector<passthrough_case> test_cases {// Invalid UTF-8 bytes- {std::string("Hello\xFF\xFE"), "", COMMON_PEG_PARSE_RESULT_FAIL},+ {std::string("Hello\xFF\xFE</tag>"), std::string("Hello\xFF\xFE"), "Hello\xEF\xBF\xBD\xEF\xBF\xBD"},// Continuation byte without lead byte- {std::string("Hello\x80World"), "", COMMON_PEG_PARSE_RESULT_FAIL},+ {std::string("Hello\x80World</tag>"), std::string("Hello\x80World"), "Hello\xEF\xBF\xBDWorld"},- // Invalid continuation byte- {std::string("\xC3\x28"), "", COMMON_PEG_PARSE_RESULT_FAIL},+ // Invalid continuation byte, the lead byte is dropped and '(' survives+ {std::string("\xC3\x28</tag>"), std::string("\xC3\x28"), "\xEF\xBF\xBD("},++ // Two good bytes of a 3-byte sequence then a bad third byte, the prefix is replaced once and the third byte is kept+ {std::string("\xE4\xB8" "A</tag>"), std::string("\xE4\xB8" "A"), "\xEF\xBF\xBD" "A"},+ {std::string("\xE4\xB8</tag>"), std::string("\xE4\xB8"), "\xEF\xBF\xBD"},++ // Truncated sequence in a complete input, the leftover prefix is replaced once+ {std::string("Hello\xE4\xB8"), std::string("Hello\xE4\xB8"), "Hello\xEF\xBF\xBD"},++ // Valid multi-byte content around the bad byte is left alone+ {std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD</tag>"), std::string("\xE4\xBD\xA0\xFF\xE5\xA5\xBD"), "\xE4\xBD\xA0\xEF\xBF\xBD\xE5\xA5\xBD"},};auto parser = build_peg_parser([](common_peg_parser_builder& p) {- return p.until("</tag>");+ return p.tag("body", p.until("</tag>")) + p.optional(p.literal("</tag>"));});for (size_t i = 0; i < test_cases.size(); i++) {@@ -296,10 +312,28 @@ void test_unicode(testing &t) {common_peg_parse_context ctx(tc.input);auto result = parser.parse(ctx);- assert_result_equal(t, tc.expected_result, result.type);+ assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type);+ const auto & node = ctx.ast.get(result.nodes[0]);+ t.assert_equal("raw text", tc.expected_text, std::string(node.text));+ t.assert_equal("sanitized text", tc.expected_sanitized, node.sanitized_text());});}});++ t.test("malformed UTF-8 rescanned by backtracking", [](testing &t) {+ // The failed alternative and the lookahead scan the same bad byte, it must only be recorded once+ auto parser = build_peg_parser([](common_peg_parser_builder& p) {+ return (p.until("<a>") + p.literal("<a>")) | (p.peek(p.until("<b>")) + p.until("<b>") + p.literal("<b>"));+ });++ std::string input("x\xFFy<b>");+ common_peg_parse_context ctx(input);+ auto result = parser.parse(ctx);++ assert_result_equal(t, COMMON_PEG_PARSE_RESULT_SUCCESS, result.type);+ t.assert_equal("invalid count", 1u, result.invalid_utf8.size());+ t.assert_equal("invalid offset", 1u, result.invalid_utf8[0].pos);+ });});t.test("json_string parser", [](testing &t) {diff --git a/tests/test-chat-peg-parser.cpp b/tests/test-chat-peg-parser.cppindex 9d15796f7a..36e11a30bb 100644--- a/tests/test-chat-peg-parser.cpp+++ b/tests/test-chat-peg-parser.cpp@@ -23,6 +23,7 @@ static void test_command7_parser_compare(testing & t);static void test_prefix_tool_names(testing & t);static void test_tagged_peg_parser(testing & t);static void test_permute(testing & t);+static void test_invalid_utf8(testing & t);int main(int argc, char * argv[]) {testing t(std::cout);@@ -42,6 +43,7 @@ int main(int argc, char * argv[]) {t.test("prefix tool names", test_prefix_tool_names);t.test("tagged peg parser", test_tagged_peg_parser);t.test("permute", test_permute);+ t.test("invalid utf8", test_invalid_utf8);return t.summary();}@@ -1069,3 +1071,36 @@ static void test_permute(testing & t) {)""", gbnf_of(parser));});}++static void test_invalid_utf8(testing & t) {+ auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) {+ return "<think>" + p.reasoning(p.until("</think>")) + "</think>" + p.content(p.rest()) + p.end();+ });++ t.test("replaced in reasoning and content", [&](testing & t) {+ std::string input("<think>plan\xFF\xFE</think>caf\xC3\xA9 \x80 done");+ common_peg_parse_context ctx(input);+ auto result = parser.parse(ctx);+ t.assert_true("success", result.success());++ common_chat_msg msg;+ auto mapper = common_chat_peg_mapper(msg);+ mapper.from_ast(ctx.ast, result);++ t.assert_equal("reasoning", "plan\xEF\xBF\xBD\xEF\xBF\xBD", msg.reasoning_content);+ t.assert_equal("content", "caf\xC3\xA9 \xEF\xBF\xBD done", msg.content);+ });++ t.test("partial input keeps trailing incomplete sequence out", [&](testing & t) {+ std::string input("<think>x</think>a\x80" "b\xE4\xB8");+ common_peg_parse_context ctx(input, COMMON_PEG_PARSE_FLAG_LENIENT);+ auto result = parser.parse(ctx);+ t.assert_true("not fail", !result.fail());++ common_chat_msg msg;+ auto mapper = common_chat_peg_mapper(msg);+ mapper.from_ast(ctx.ast, result);++ t.assert_equal("content", "a\xEF\xBF\xBD" "b", msg.content);+ });+}