RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: harden native analyzer conversions and JSON output (#22070)
buua43658c78b9aa4b2
diff --git a/internal/binding/cpp/CMakeLists.txt b/internal/binding/cpp/CMakeLists.txtindex 2215b65905..26e8a78337 100644--- a/internal/binding/cpp/CMakeLists.txt+++ b/internal/binding/cpp/CMakeLists.txt@@ -139,6 +139,7 @@ add_library(rag_tokenizer_c_api STATICthinc_ner.hthinc_parser.cppthinc_parser.h+ thinc_json.hrag_analyzer.cpprag_analyzer.hdart_trie.h@@ -178,6 +179,17 @@ set_target_properties(rag_analyzer_c_test PROPERTIES)if(BUILD_TESTING)+ add_executable(thinc_json_test+ thinc_json_test.cpp+ )+ target_link_libraries(thinc_json_test rag_tokenizer_c_api stdc++ m ${PCRE2_LIB})+ target_include_directories(thinc_json_test PUBLIC "${CMAKE_SOURCE_DIR}")+ set_target_properties(thinc_json_test PROPERTIES+ CXX_STANDARD 20+ CXX_STANDARD_REQUIRED ON+ )+ add_test(NAME thinc_json COMMAND thinc_json_test)+add_executable(rag_analyzer_opencc_copy_testrag_analyzer_opencc_copy_test.cpp)diff --git a/internal/binding/cpp/Makefile b/internal/binding/cpp/Makefileindex 87e6ee5d5a..cfde129de2 100644--- a/internal/binding/cpp/Makefile+++ b/internal/binding/cpp/Makefile@@ -39,7 +39,7 @@ c_api_asan: $(ASAN_BUILD_DIR)test_string_utils:mkdir -p $(BUILD_DIR)- $(CXX) $(CPPFLAGS) $(SIMDE_CPPFLAGS) $(CXXFLAGS) -std=c++20 -fsanitize=address -fno-omit-frame-pointer -g -I. string_utils_test.cpp -o $(BUILD_DIR)/string_utils_test+ $(CXX) $(CPPFLAGS) $(SIMDE_CPPFLAGS) $(CXXFLAGS) -D_GLIBCXX_ASSERTIONS -std=c++20 -fsanitize=address -fno-omit-frame-pointer -g -I. string_utils_test.cpp -o $(BUILD_DIR)/string_utils_testASAN_OPTIONS=detect_leaks=0 ./$(BUILD_DIR)/string_utils_test# Build debug version of C API library with memory trackingdiff --git a/internal/binding/cpp/opencc/openccxx.cpp b/internal/binding/cpp/opencc/openccxx.cppindex 54b27e0d26..c0937fc1d2 100644--- a/internal/binding/cpp/opencc/openccxx.cpp+++ b/internal/binding/cpp/opencc/openccxx.cpp@@ -1,19 +1,18 @@#include "openccxx.h"#include "opencc.h"-#include "utils.h"-#include <iostream>+#include <stdexcept>#include <string>OpenCC::OpenCC(const std::string &home_dir) : od((opencc_t)-1) {- config_file = mstrcpy(OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP);- open(config_file, home_dir.c_str());+ if (open(OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP, home_dir.c_str()) != 0) {+ throw std::runtime_error("failed to load OpenCC config " OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP " from " + home_dir);+ }}OpenCC::~OpenCC() {if (od != (opencc_t)-1)opencc_close(od);- free(config_file);}int OpenCC::open(const char *config_file, const char *home_dir) {diff --git a/internal/binding/cpp/opencc/openccxx.h b/internal/binding/cpp/opencc/openccxx.hindex 844bbacdb5..0c2c88bbe2 100644--- a/internal/binding/cpp/opencc/openccxx.h+++ b/internal/binding/cpp/opencc/openccxx.h@@ -15,6 +15,5 @@ public:long convert(const std::wstring &in, std::wstring &out, long length = -1);private:- char *config_file;opencc_t od;};diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cppindex b8b6f1e5bb..3906a17954 100644--- a/internal/binding/cpp/rag_analyzer.cpp+++ b/internal/binding/cpp/rag_analyzer.cpp@@ -29,6 +29,7 @@#include <cmath>#include <fstream>#include <algorithm>+#include <stdexcept>#include <unordered_set>// import :term;// import :stemmer;@@ -808,16 +809,11 @@ int32_t RAGAnalyzer::Load() {wordnet_lemma_ = new WordNetLemmatizer(lemma_path.string());fs::path opencc_path(root / OPENCC_PATH);- printf("Validate opencc file: %s\n", opencc_path.string().c_str());- if (!fs::exists(opencc_path)) {- printf("opencc_path not exists: %s\n", opencc_path.string().c_str());- return -6;- }try {- printf("Load opencc file: %s\n", opencc_path.string().c_str());+ printf("Load OpenCC config from: %s\n", opencc_path.string().c_str());opencc_ = std::make_unique<::OpenCC>(opencc_path.string());} catch (const std::exception &e) {- printf("Fail to open opencc: %s\n", opencc_path.string().c_str());+ printf("Fail to load OpenCC config from %s: %s\n", opencc_path.string().c_str(), e.what());return -7;}@@ -853,53 +849,6 @@ void RAGAnalyzer::BuildPositionMapping(const std::string &original, const std::s}}-std::string RAGAnalyzer::StrQ2B(const std::string &input) {- std::string output;- size_t i = 0;-- while (i < input.size()) {- unsigned char c = input[i];-- uint32_t codepoint = 0;- if (c < 0x80) {- codepoint = c;- i += 1;- } else if ((c & 0xE0) == 0xC0) {- codepoint = (c & 0x1F) << 6;- codepoint |= (input[i + 1] & 0x3F);- i += 2;- } else if ((c & 0xF0) == 0xE0) {- codepoint = (c & 0x0F) << 12;- codepoint |= (input[i + 1] & 0x3F) << 6;- codepoint |= (input[i + 2] & 0x3F);- i += 3;- } else {- output += c;- i += 1;- continue;- }-- if (codepoint >= 0xFF01 && codepoint <= 0xFF5E) {- output += static_cast<char>(codepoint - 0xFEE0);- } else if (codepoint == 0x3000) {- output += ' ';- } else {- if (codepoint < 0x80) {- output += static_cast<char>(codepoint);- } else if (codepoint < 0x800) {- output += static_cast<char>(0xC0 | (codepoint >> 6));- output += static_cast<char>(0x80 | (codepoint & 0x3F));- } else if (codepoint < 0x10000) {- output += static_cast<char>(0xE0 | (codepoint >> 12));- output += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));- output += static_cast<char>(0x80 | (codepoint & 0x3F));- }- }- }-- return output;-}-// ASCII base letter for each codepoint in Latin-1 Supplement (U+00C0–U+00FF)// and Latin Extended-A (U+0100–U+017F), indexed by codepoint - 0xC0. Only// letters whose NFD decomposition is exactly one ASCII letter plus combining@@ -1869,9 +1818,11 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {// Python-style simple tokenization: re.sub(r"\\W+", " ", line)std::string processed_line = PCRE2GlobalReplace(*input, R"#(\W+)#", " ");- std::string str1 = StrQ2B(processed_line);+ std::string str1 = FullWidthToHalfWidth(processed_line);std::string strline;- opencc_->convert(str1, strline);+ if (opencc_->convert(str1, strline) < 0) {+ throw std::runtime_error("OpenCC conversion failed during tokenization");+ }std::vector<std::string> res;@@ -1938,7 +1889,7 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> RAGAnalyzer::TokenizeWithPosition(const std::string &line) {// Fold diacritics to ASCII before any other processing when the// language asks for it (Slovak, Czech). Folding shrinks 2-byte UTF-8- // letters to 1-byte ASCII, so byte offsets shift; like the StrQ2B and+ // letters to 1-byte ASCII, so byte offsets shift; like full-width folding and// OpenCC stages below, a position mapping recovers offsets on the// original input.std::string folded_line;@@ -1954,15 +1905,17 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>// Get processed line and position mapping from PCRE2GlobalReplaceauto [processed_line, pcre2_pos_mapping] = PCRE2GlobalReplaceWithPosition(*input, R"#(\W+)#", " ");- std::string str1 = StrQ2B(processed_line);+ std::string str1 = FullWidthToHalfWidth(processed_line);std::string strline;- opencc_->convert(str1, strline);+ if (opencc_->convert(str1, strline) < 0) {+ throw std::runtime_error("OpenCC conversion failed during tokenization with positions");+ }std::vector<std::string> tokens;std::vector<std::pair<unsigned, unsigned>> positions;- // Build character position mapping from StrQ2B conversion- std::vector<unsigned> strq2b_pos_mapping;- BuildPositionMapping(processed_line, str1, strq2b_pos_mapping);+ // Build character position mapping from full-width folding+ std::vector<unsigned> fullwidth_pos_mapping;+ BuildPositionMapping(processed_line, str1, fullwidth_pos_mapping);// Build character position mapping from OpenCC conversionstd::vector<unsigned> opencc_pos_mapping;@@ -1975,8 +1928,8 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>for (size_t i = 0; i < strline.size(); ++i) {if (i < opencc_pos_mapping.size()) {unsigned str1_pos = opencc_pos_mapping[i];- if (str1_pos < strq2b_pos_mapping.size()) {- unsigned processed_pos = strq2b_pos_mapping[str1_pos];+ if (str1_pos < fullwidth_pos_mapping.size()) {+ unsigned processed_pos = fullwidth_pos_mapping[str1_pos];if (processed_pos < pcre2_pos_mapping.size()) {final_pos_mapping[i] = pcre2_pos_mapping[processed_pos].first;} else {diff --git a/internal/binding/cpp/rag_analyzer.h b/internal/binding/cpp/rag_analyzer.hindex 2145d15843..3b9b889d28 100644--- a/internal/binding/cpp/rag_analyzer.h+++ b/internal/binding/cpp/rag_analyzer.h@@ -100,8 +100,6 @@ protected:private:static constexpr float DENOMINATOR = 1000000;- static std::string StrQ2B(const std::string& input);-// Fold Latin-1 Supplement / Latin Extended-A letters whose NFD// decomposition is one ASCII letter plus combining marks to that base// letter, preserving case (Š→S, ď→d). Letters without such adiff --git a/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cppindex 81897f801d..1d986d187b 100644--- a/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp+++ b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp@@ -19,6 +19,20 @@ int main() {const fs::path opencc_path = resource_path / "opencc";fs::create_directories(opencc_path);+ const fs::path invalid_opencc_path = resource_path / "invalid_opencc";+ fs::create_directories(invalid_opencc_path);+ bool rejected_missing_config = false;+ try {+ OpenCC missing_config(invalid_opencc_path.string());+ } catch (const std::exception&) {+ rejected_missing_config = true;+ }+ if (!rejected_missing_config) {+ std::cerr << "OpenCC accepted a missing configuration in an existing directory\n";+ fs::remove_all(resource_path);+ return 1;+ }+{std::ofstream config(opencc_path / "zht2zhs.ini");config << "title = test\n"diff --git a/internal/binding/cpp/string_utils.h b/internal/binding/cpp/string_utils.hindex e02654a7ea..626b916eb8 100644--- a/internal/binding/cpp/string_utils.h+++ b/internal/binding/cpp/string_utils.h@@ -26,6 +26,68 @@#include <cstring>#include <string>+inline std::string FullWidthToHalfWidth(const std::string& input)+{+ std::string output;+ size_t i = 0;++ while (i < input.size())+ {+ const unsigned char first = static_cast<unsigned char>(input[i]);+ if (first < 0x80)+ {+ output += static_cast<char>(first);+ ++i;+ continue;+ }++ size_t length = 0;+ uint32_t codepoint = 0;+ if ((first & 0xE0) == 0xC0 && first >= 0xC2 && i + 1 < input.size())+ {+ const unsigned char second = static_cast<unsigned char>(input[i + 1]);+ if ((second & 0xC0) == 0x80)+ {+ codepoint = ((first & 0x1F) << 6) | (second & 0x3F);+ length = 2;+ }+ }+ else if ((first & 0xF0) == 0xE0 && i + 2 < input.size())+ {+ const unsigned char second = static_cast<unsigned char>(input[i + 1]);+ const unsigned char third = static_cast<unsigned char>(input[i + 2]);+ if ((second & 0xC0) == 0x80 && (third & 0xC0) == 0x80 &&+ !(first == 0xE0 && second < 0xA0) && !(first == 0xED && second >= 0xA0))+ {+ codepoint = ((first & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F);+ length = 3;+ }+ }++ if (length == 0)+ {+ output += input[i++];+ continue;+ }++ if (codepoint >= 0xFF01 && codepoint <= 0xFF5E)+ {+ output += static_cast<char>(codepoint - 0xFEE0);+ }+ else if (codepoint == 0x3000)+ {+ output += ' ';+ }+ else+ {+ output.append(input, i, length);+ }+ i += length;+ }++ return output;+}+[[nodiscard]] constexpr uint8_t ToUpper(uint8_t ch) noexcept { return ch >= 'a' && ch <= 'z' ? ch - 32 : ch; }[[nodiscard]] constexpr uint8_t ToLower(uint8_t ch) noexcept { return ch >= 'A' && ch <= 'Z' ? ch + 32 : ch; }diff --git a/internal/binding/cpp/string_utils_test.cpp b/internal/binding/cpp/string_utils_test.cppindex b81b690620..f853d18bc7 100644--- a/internal/binding/cpp/string_utils_test.cpp+++ b/internal/binding/cpp/string_utils_test.cpp@@ -5,6 +5,12 @@#include "string_utils.h"int main() {+ const std::string truncated_utf8 = "\xE3";+ assert(FullWidthToHalfWidth(truncated_utf8) == truncated_utf8);+ const std::string invalid_continuation = "\xE3\x80";+ assert(FullWidthToHalfWidth(invalid_continuation) == invalid_continuation);+ assert(FullWidthToHalfWidth("\xEF\xBC\xA1\xE3\x80\x80") == "A ");+constexpr size_t output_limit = 4096 * 3;std::string input(output_limit * 2, 'A');std::vector<char> guarded_output(output_limit + 2, 'x');diff --git a/internal/binding/cpp/thinc_json.h b/internal/binding/cpp/thinc_json.hnew file mode 100644index 0000000000..480b9a2d2c--- /dev/null+++ b/internal/binding/cpp/thinc_json.h@@ -0,0 +1,42 @@+#pragma once++#include <string>+#include <string_view>++namespace thinc {++inline std::string JsonString(std::string_view value)+{+ static constexpr char hex[] = "0123456789abcdef";+ std::string result;+ result.reserve(value.size() + 2);+ result.push_back('"');+ for (unsigned char c : value)+ {+ switch (c)+ {+ case '"': result += "\\\""; break;+ case '\\': result += "\\\\"; break;+ case '\b': result += "\\b"; break;+ case '\f': result += "\\f"; break;+ case '\n': result += "\\n"; break;+ case '\r': result += "\\r"; break;+ case '\t': result += "\\t"; break;+ default:+ if (c < 0x20)+ {+ result += "\\u00";+ result.push_back(hex[c >> 4]);+ result.push_back(hex[c & 0x0F]);+ }+ else+ {+ result.push_back(static_cast<char>(c));+ }+ }+ }+ result.push_back('"');+ return result;+}++} // namespace thincdiff --git a/internal/binding/cpp/thinc_json_test.cpp b/internal/binding/cpp/thinc_json_test.cppnew file mode 100644index 0000000000..b8de2f7e9d--- /dev/null+++ b/internal/binding/cpp/thinc_json_test.cpp@@ -0,0 +1,29 @@+#include "thinc_ner.h"+#include "thinc_json.h"++#include <iostream>+#include <string>++int main() {+ const std::string escaped = thinc::JsonString("quote\" slash\\ line\n tab\t control\x01");+ const std::string expected_escaped = R"json("quote\" slash\\ line\n tab\t control\u0001")json";+ if (escaped != expected_escaped) {+ std::cerr << "JSON string escaping failed\nexpected: " << expected_escaped << "\nactual: " << escaped << '\n';+ return 1;+ }++ char* result = ThincNER_Tokenize("\"say\" \\ now", "en");+ if (!result) {+ std::cerr << "ThincNER_Tokenize returned null\n";+ return 1;+ }++ const std::string expected = R"json(["\"","say","\"","\\","now"])json";+ const std::string actual(result);+ ThincNER_FreeString(result);+ if (actual != expected) {+ std::cerr << "invalid Thinc tokenizer JSON\nexpected: " << expected << "\nactual: " << actual << '\n';+ return 1;+ }+ return 0;+}diff --git a/internal/binding/cpp/thinc_ner.cpp b/internal/binding/cpp/thinc_ner.cppindex 8e111549c6..728d2c7eec 100644--- a/internal/binding/cpp/thinc_ner.cpp+++ b/internal/binding/cpp/thinc_ner.cpp@@ -1,6 +1,7 @@#pragma STDC FP_CONTRACT OFF#include "thinc_ner.h"+#include "thinc_json.h"#include <algorithm>#include <cmath>@@ -608,7 +609,14 @@ char* ThincNER_Predict(ThincNERHandle h, const char* tj) {// ---- Step 8: BILUO decode ----auto ents=decode_biluo(tok,tl);- std::string r="["; for(size_t i=0;i<ents.size();i++){if(i)r+=",";r+="{\"text\":\""+ents[i].text+"\",\"label\":\""+ents[i].label+"\",\"start\":"+std::to_string(ents[i].start)+",\"end\":"+std::to_string(ents[i].end)+",\"confidence\":"+std::to_string(ents[i].conf)+"}";} r+="]";+ std::string r="[";+ for(size_t i=0;i<ents.size();i++){+ if(i)r+=",";+ r+="{\"text\":"+thinc::JsonString(ents[i].text)+",\"label\":"+thinc::JsonString(ents[i].label)++ ",\"start\":"+std::to_string(ents[i].start)+",\"end\":"+std::to_string(ents[i].end)++ ",\"confidence\":"+std::to_string(ents[i].conf)+"}";+ }+ r+="]";return strdup(r.c_str());}@@ -620,6 +628,11 @@ char* ThincNER_Tokenize(const char* t, const char* l) {// de, fr, es, pt: European Latin -> en tokenizer; zh, ja: CJK -> zh tokenizerbool is_cjk = (lang == "zh" || lang == "ja");auto tok = is_cjk ? tokenize_zh(t) : tokenize_en(t);- std::string r="["; for(size_t i=0;i<tok.size();i++){if(i)r+=",";r+="\""+tok[i]+"\"";} r+="]";+ std::string r="[";+ for(size_t i=0;i<tok.size();i++){+ if(i)r+=",";+ r+=thinc::JsonString(tok[i]);+ }+ r+="]";return strdup(r.c_str());}diff --git a/internal/binding/cpp/thinc_parser.cpp b/internal/binding/cpp/thinc_parser.cppindex 044af97e15..122de9c44d 100644--- a/internal/binding/cpp/thinc_parser.cpp+++ b/internal/binding/cpp/thinc_parser.cpp@@ -1,4 +1,5 @@#include "thinc_parser.h"+#include "thinc_json.h"#include <algorithm>#include <cmath>@@ -453,8 +454,8 @@ char* ThincParser_Predict(ThincParserHandle h, const char* tokens_json) {std::string r="[";for(int i=0;i<n;i++){if(i)r+=",";- r+="{\"text\":\""+tokens[i]+"\",\"head\":"+std::to_string(heads[i])+- ",\"dep\":\""+labels[i]+"\",\"index\":"+std::to_string(i)+"}";+ r+="{\"text\":"+thinc::JsonString(tokens[i])+",\"head\":"+std::to_string(heads[i])++ ",\"dep\":"+thinc::JsonString(labels[i])+",\"index\":"+std::to_string(i)+"}";}r+="]";return strdup(r.c_str());@@ -536,7 +537,8 @@ char* ThincTagger_Predict(ThincTaggerHandle h, const char* tokens_json) {for(int i=0;i<n;i++){if(i)r+=",";std::string tag = best_tags[i] < (int)s->tags.size() ? s->tags[best_tags[i]] : "";- r+="{\"text\":\""+tokens[i]+"\",\"tag\":\""+pos_only(tag)+"\",\"index\":"+std::to_string(i)+"}";+ r+="{\"text\":"+thinc::JsonString(tokens[i])+",\"tag\":"+thinc::JsonString(pos_only(tag))++ ",\"index\":"+std::to_string(i)+"}";}r+="]";return strdup(r.c_str());