Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: harden native analyzer conversions and JSON output (#22070)

buua43658c78b9aa4b2
diff --git a/internal/binding/cpp/CMakeLists.txt b/internal/binding/cpp/CMakeLists.txt
index 2215b65905..26e8a78337 100644
--- a/internal/binding/cpp/CMakeLists.txt
+++ b/internal/binding/cpp/CMakeLists.txt
@@ -139,6 +139,7 @@ add_library(rag_tokenizer_c_api STATIC
thinc_ner.h
thinc_parser.cpp
thinc_parser.h
+ thinc_json.h
rag_analyzer.cpp
rag_analyzer.h
dart_trie.h
@@ -178,6 +179,17 @@ set_target_properties(rag_analyzer_c_test PROPERTIES
)
if(BUILD_TESTING)
+ add_executable(thinc_json_test
+ thinc_json_test.cpp
+ )
+ target_link_libraries(thinc_json_test rag_tokenizer_c_api stdc++ m ${PCRE2_LIB})
+ target_include_directories(thinc_json_test PUBLIC "${CMAKE_SOURCE_DIR}")
+ set_target_properties(thinc_json_test PROPERTIES
+ CXX_STANDARD 20
+ CXX_STANDARD_REQUIRED ON
+ )
+ add_test(NAME thinc_json COMMAND thinc_json_test)
+
add_executable(rag_analyzer_opencc_copy_test
rag_analyzer_opencc_copy_test.cpp
)
diff --git a/internal/binding/cpp/Makefile b/internal/binding/cpp/Makefile
index 87e6ee5d5a..cfde129de2 100644
--- a/internal/binding/cpp/Makefile
+++ b/internal/binding/cpp/Makefile
@@ -39,7 +39,7 @@ c_api_asan: $(ASAN_BUILD_DIR)
test_string_utils:
mkdir -p $(BUILD_DIR)
- $(CXX) $(CPPFLAGS) $(SIMDE_CPPFLAGS) $(CXXFLAGS) -std=c++20 -fsanitize=address -fno-omit-frame-pointer -g -I. string_utils_test.cpp -o $(BUILD_DIR)/string_utils_test
+ $(CXX) $(CPPFLAGS) $(SIMDE_CPPFLAGS) $(CXXFLAGS) -D_GLIBCXX_ASSERTIONS -std=c++20 -fsanitize=address -fno-omit-frame-pointer -g -I. string_utils_test.cpp -o $(BUILD_DIR)/string_utils_test
ASAN_OPTIONS=detect_leaks=0 ./$(BUILD_DIR)/string_utils_test
# Build debug version of C API library with memory tracking
diff --git a/internal/binding/cpp/opencc/openccxx.cpp b/internal/binding/cpp/opencc/openccxx.cpp
index 54b27e0d26..c0937fc1d2 100644
--- a/internal/binding/cpp/opencc/openccxx.cpp
+++ b/internal/binding/cpp/opencc/openccxx.cpp
@@ -1,19 +1,18 @@
#include "openccxx.h"
#include "opencc.h"
-#include "utils.h"
-#include <iostream>
+#include <stdexcept>
#include <string>
OpenCC::OpenCC(const std::string &home_dir) : od((opencc_t)-1) {
- config_file = mstrcpy(OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP);
- open(config_file, home_dir.c_str());
+ if (open(OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP, home_dir.c_str()) != 0) {
+ throw std::runtime_error("failed to load OpenCC config " OPENCC_DEFAULT_CONFIG_TRAD_TO_SIMP " from " + home_dir);
+ }
}
OpenCC::~OpenCC() {
if (od != (opencc_t)-1)
opencc_close(od);
- free(config_file);
}
int OpenCC::open(const char *config_file, const char *home_dir) {
diff --git a/internal/binding/cpp/opencc/openccxx.h b/internal/binding/cpp/opencc/openccxx.h
index 844bbacdb5..0c2c88bbe2 100644
--- a/internal/binding/cpp/opencc/openccxx.h
+++ b/internal/binding/cpp/opencc/openccxx.h
@@ -15,6 +15,5 @@ public:
long convert(const std::wstring &in, std::wstring &out, long length = -1);
private:
- char *config_file;
opencc_t od;
};
diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cpp
index b8b6f1e5bb..3906a17954 100644
--- a/internal/binding/cpp/rag_analyzer.cpp
+++ b/internal/binding/cpp/rag_analyzer.cpp
@@ -29,6 +29,7 @@
#include <cmath>
#include <fstream>
#include <algorithm>
+#include <stdexcept>
#include <unordered_set>
// import :term;
// import :stemmer;
@@ -808,16 +809,11 @@ int32_t RAGAnalyzer::Load() {
wordnet_lemma_ = new WordNetLemmatizer(lemma_path.string());
fs::path opencc_path(root / OPENCC_PATH);
- printf("Validate opencc file: %s\n", opencc_path.string().c_str());
- if (!fs::exists(opencc_path)) {
- printf("opencc_path not exists: %s\n", opencc_path.string().c_str());
- return -6;
- }
try {
- printf("Load opencc file: %s\n", opencc_path.string().c_str());
+ printf("Load OpenCC config from: %s\n", opencc_path.string().c_str());
opencc_ = std::make_unique<::OpenCC>(opencc_path.string());
} catch (const std::exception &e) {
- printf("Fail to open opencc: %s\n", opencc_path.string().c_str());
+ printf("Fail to load OpenCC config from %s: %s\n", opencc_path.string().c_str(), e.what());
return -7;
}
@@ -853,53 +849,6 @@ void RAGAnalyzer::BuildPositionMapping(const std::string &original, const std::s
}
}
-std::string RAGAnalyzer::StrQ2B(const std::string &input) {
- std::string output;
- size_t i = 0;
-
- while (i < input.size()) {
- unsigned char c = input[i];
-
- uint32_t codepoint = 0;
- if (c < 0x80) {
- codepoint = c;
- i += 1;
- } else if ((c & 0xE0) == 0xC0) {
- codepoint = (c & 0x1F) << 6;
- codepoint |= (input[i + 1] & 0x3F);
- i += 2;
- } else if ((c & 0xF0) == 0xE0) {
- codepoint = (c & 0x0F) << 12;
- codepoint |= (input[i + 1] & 0x3F) << 6;
- codepoint |= (input[i + 2] & 0x3F);
- i += 3;
- } else {
- output += c;
- i += 1;
- continue;
- }
-
- if (codepoint >= 0xFF01 && codepoint <= 0xFF5E) {
- output += static_cast<char>(codepoint - 0xFEE0);
- } else if (codepoint == 0x3000) {
- output += ' ';
- } else {
- if (codepoint < 0x80) {
- output += static_cast<char>(codepoint);
- } else if (codepoint < 0x800) {
- output += static_cast<char>(0xC0 | (codepoint >> 6));
- output += static_cast<char>(0x80 | (codepoint & 0x3F));
- } else if (codepoint < 0x10000) {
- output += static_cast<char>(0xE0 | (codepoint >> 12));
- output += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3F));
- output += static_cast<char>(0x80 | (codepoint & 0x3F));
- }
- }
- }
-
- return output;
-}
-
// ASCII base letter for each codepoint in Latin-1 Supplement (U+00C0–U+00FF)
// and Latin Extended-A (U+0100–U+017F), indexed by codepoint - 0xC0. Only
// letters whose NFD decomposition is exactly one ASCII letter plus combining
@@ -1869,9 +1818,11 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {
// Python-style simple tokenization: re.sub(r"\\W+", " ", line)
std::string processed_line = PCRE2GlobalReplace(*input, R"#(\W+)#", " ");
- std::string str1 = StrQ2B(processed_line);
+ std::string str1 = FullWidthToHalfWidth(processed_line);
std::string strline;
- opencc_->convert(str1, strline);
+ if (opencc_->convert(str1, strline) < 0) {
+ throw std::runtime_error("OpenCC conversion failed during tokenization");
+ }
std::vector<std::string> res;
@@ -1938,7 +1889,7 @@ std::string RAGAnalyzer::Tokenize(const std::string &line) {
std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>> RAGAnalyzer::TokenizeWithPosition(const std::string &line) {
// Fold diacritics to ASCII before any other processing when the
// language asks for it (Slovak, Czech). Folding shrinks 2-byte UTF-8
- // letters to 1-byte ASCII, so byte offsets shift; like the StrQ2B and
+ // letters to 1-byte ASCII, so byte offsets shift; like full-width folding and
// OpenCC stages below, a position mapping recovers offsets on the
// original input.
std::string folded_line;
@@ -1954,15 +1905,17 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
// Get processed line and position mapping from PCRE2GlobalReplace
auto [processed_line, pcre2_pos_mapping] = PCRE2GlobalReplaceWithPosition(*input, R"#(\W+)#", " ");
- std::string str1 = StrQ2B(processed_line);
+ std::string str1 = FullWidthToHalfWidth(processed_line);
std::string strline;
- opencc_->convert(str1, strline);
+ if (opencc_->convert(str1, strline) < 0) {
+ throw std::runtime_error("OpenCC conversion failed during tokenization with positions");
+ }
std::vector<std::string> tokens;
std::vector<std::pair<unsigned, unsigned>> positions;
- // Build character position mapping from StrQ2B conversion
- std::vector<unsigned> strq2b_pos_mapping;
- BuildPositionMapping(processed_line, str1, strq2b_pos_mapping);
+ // Build character position mapping from full-width folding
+ std::vector<unsigned> fullwidth_pos_mapping;
+ BuildPositionMapping(processed_line, str1, fullwidth_pos_mapping);
// Build character position mapping from OpenCC conversion
std::vector<unsigned> opencc_pos_mapping;
@@ -1975,8 +1928,8 @@ std::pair<std::vector<std::string>, std::vector<std::pair<unsigned, unsigned>>>
for (size_t i = 0; i < strline.size(); ++i) {
if (i < opencc_pos_mapping.size()) {
unsigned str1_pos = opencc_pos_mapping[i];
- if (str1_pos < strq2b_pos_mapping.size()) {
- unsigned processed_pos = strq2b_pos_mapping[str1_pos];
+ if (str1_pos < fullwidth_pos_mapping.size()) {
+ unsigned processed_pos = fullwidth_pos_mapping[str1_pos];
if (processed_pos < pcre2_pos_mapping.size()) {
final_pos_mapping[i] = pcre2_pos_mapping[processed_pos].first;
} else {
diff --git a/internal/binding/cpp/rag_analyzer.h b/internal/binding/cpp/rag_analyzer.h
index 2145d15843..3b9b889d28 100644
--- a/internal/binding/cpp/rag_analyzer.h
+++ b/internal/binding/cpp/rag_analyzer.h
@@ -100,8 +100,6 @@ protected:
private:
static constexpr float DENOMINATOR = 1000000;
- static std::string StrQ2B(const std::string& input);
-
// Fold Latin-1 Supplement / Latin Extended-A letters whose NFD
// decomposition is one ASCII letter plus combining marks to that base
// letter, preserving case (Š→S, ď→d). Letters without such a
diff --git a/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp
index 81897f801d..1d986d187b 100644
--- a/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp
+++ b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp
@@ -19,6 +19,20 @@ int main() {
const fs::path opencc_path = resource_path / "opencc";
fs::create_directories(opencc_path);
+ const fs::path invalid_opencc_path = resource_path / "invalid_opencc";
+ fs::create_directories(invalid_opencc_path);
+ bool rejected_missing_config = false;
+ try {
+ OpenCC missing_config(invalid_opencc_path.string());
+ } catch (const std::exception&) {
+ rejected_missing_config = true;
+ }
+ if (!rejected_missing_config) {
+ std::cerr << "OpenCC accepted a missing configuration in an existing directory\n";
+ fs::remove_all(resource_path);
+ return 1;
+ }
+
{
std::ofstream config(opencc_path / "zht2zhs.ini");
config << "title = test\n"
diff --git a/internal/binding/cpp/string_utils.h b/internal/binding/cpp/string_utils.h
index e02654a7ea..626b916eb8 100644
--- a/internal/binding/cpp/string_utils.h
+++ b/internal/binding/cpp/string_utils.h
@@ -26,6 +26,68 @@
#include <cstring>
#include <string>
+inline std::string FullWidthToHalfWidth(const std::string& input)
+{
+ std::string output;
+ size_t i = 0;
+
+ while (i < input.size())
+ {
+ const unsigned char first = static_cast<unsigned char>(input[i]);
+ if (first < 0x80)
+ {
+ output += static_cast<char>(first);
+ ++i;
+ continue;
+ }
+
+ size_t length = 0;
+ uint32_t codepoint = 0;
+ if ((first & 0xE0) == 0xC0 && first >= 0xC2 && i + 1 < input.size())
+ {
+ const unsigned char second = static_cast<unsigned char>(input[i + 1]);
+ if ((second & 0xC0) == 0x80)
+ {
+ codepoint = ((first & 0x1F) << 6) | (second & 0x3F);
+ length = 2;
+ }
+ }
+ else if ((first & 0xF0) == 0xE0 && i + 2 < input.size())
+ {
+ const unsigned char second = static_cast<unsigned char>(input[i + 1]);
+ const unsigned char third = static_cast<unsigned char>(input[i + 2]);
+ if ((second & 0xC0) == 0x80 && (third & 0xC0) == 0x80 &&
+ !(first == 0xE0 && second < 0xA0) && !(first == 0xED && second >= 0xA0))
+ {
+ codepoint = ((first & 0x0F) << 12) | ((second & 0x3F) << 6) | (third & 0x3F);
+ length = 3;
+ }
+ }
+
+ if (length == 0)
+ {
+ output += input[i++];
+ continue;
+ }
+
+ if (codepoint >= 0xFF01 && codepoint <= 0xFF5E)
+ {
+ output += static_cast<char>(codepoint - 0xFEE0);
+ }
+ else if (codepoint == 0x3000)
+ {
+ output += ' ';
+ }
+ else
+ {
+ output.append(input, i, length);
+ }
+ i += length;
+ }
+
+ return output;
+}
+
[[nodiscard]] constexpr uint8_t ToUpper(uint8_t ch) noexcept { return ch >= 'a' && ch <= 'z' ? ch - 32 : ch; }
[[nodiscard]] constexpr uint8_t ToLower(uint8_t ch) noexcept { return ch >= 'A' && ch <= 'Z' ? ch + 32 : ch; }
diff --git a/internal/binding/cpp/string_utils_test.cpp b/internal/binding/cpp/string_utils_test.cpp
index b81b690620..f853d18bc7 100644
--- a/internal/binding/cpp/string_utils_test.cpp
+++ b/internal/binding/cpp/string_utils_test.cpp
@@ -5,6 +5,12 @@
#include "string_utils.h"
int main() {
+ const std::string truncated_utf8 = "\xE3";
+ assert(FullWidthToHalfWidth(truncated_utf8) == truncated_utf8);
+ const std::string invalid_continuation = "\xE3\x80";
+ assert(FullWidthToHalfWidth(invalid_continuation) == invalid_continuation);
+ assert(FullWidthToHalfWidth("\xEF\xBC\xA1\xE3\x80\x80") == "A ");
+
constexpr size_t output_limit = 4096 * 3;
std::string input(output_limit * 2, 'A');
std::vector<char> guarded_output(output_limit + 2, 'x');
diff --git a/internal/binding/cpp/thinc_json.h b/internal/binding/cpp/thinc_json.h
new file mode 100644
index 0000000000..480b9a2d2c
--- /dev/null
+++ b/internal/binding/cpp/thinc_json.h
@@ -0,0 +1,42 @@
+#pragma once
+
+#include <string>
+#include <string_view>
+
+namespace thinc {
+
+inline std::string JsonString(std::string_view value)
+{
+ static constexpr char hex[] = "0123456789abcdef";
+ std::string result;
+ result.reserve(value.size() + 2);
+ result.push_back('"');
+ for (unsigned char c : value)
+ {
+ switch (c)
+ {
+ case '"': result += "\\\""; break;
+ case '\\': result += "\\\\"; break;
+ case '\b': result += "\\b"; break;
+ case '\f': result += "\\f"; break;
+ case '\n': result += "\\n"; break;
+ case '\r': result += "\\r"; break;
+ case '\t': result += "\\t"; break;
+ default:
+ if (c < 0x20)
+ {
+ result += "\\u00";
+ result.push_back(hex[c >> 4]);
+ result.push_back(hex[c & 0x0F]);
+ }
+ else
+ {
+ result.push_back(static_cast<char>(c));
+ }
+ }
+ }
+ result.push_back('"');
+ return result;
+}
+
+} // namespace thinc
diff --git a/internal/binding/cpp/thinc_json_test.cpp b/internal/binding/cpp/thinc_json_test.cpp
new file mode 100644
index 0000000000..b8de2f7e9d
--- /dev/null
+++ b/internal/binding/cpp/thinc_json_test.cpp
@@ -0,0 +1,29 @@
+#include "thinc_ner.h"
+#include "thinc_json.h"
+
+#include <iostream>
+#include <string>
+
+int main() {
+ const std::string escaped = thinc::JsonString("quote\" slash\\ line\n tab\t control\x01");
+ const std::string expected_escaped = R"json("quote\" slash\\ line\n tab\t control\u0001")json";
+ if (escaped != expected_escaped) {
+ std::cerr << "JSON string escaping failed\nexpected: " << expected_escaped << "\nactual: " << escaped << '\n';
+ return 1;
+ }
+
+ char* result = ThincNER_Tokenize("\"say\" \\ now", "en");
+ if (!result) {
+ std::cerr << "ThincNER_Tokenize returned null\n";
+ return 1;
+ }
+
+ const std::string expected = R"json(["\"","say","\"","\\","now"])json";
+ const std::string actual(result);
+ ThincNER_FreeString(result);
+ if (actual != expected) {
+ std::cerr << "invalid Thinc tokenizer JSON\nexpected: " << expected << "\nactual: " << actual << '\n';
+ return 1;
+ }
+ return 0;
+}
diff --git a/internal/binding/cpp/thinc_ner.cpp b/internal/binding/cpp/thinc_ner.cpp
index 8e111549c6..728d2c7eec 100644
--- a/internal/binding/cpp/thinc_ner.cpp
+++ b/internal/binding/cpp/thinc_ner.cpp
@@ -1,6 +1,7 @@
#pragma STDC FP_CONTRACT OFF
#include "thinc_ner.h"
+#include "thinc_json.h"
#include <algorithm>
#include <cmath>
@@ -608,7 +609,14 @@ char* ThincNER_Predict(ThincNERHandle h, const char* tj) {
// ---- Step 8: BILUO decode ----
auto ents=decode_biluo(tok,tl);
- std::string r="["; for(size_t i=0;i<ents.size();i++){if(i)r+=",";r+="{\"text\":\""+ents[i].text+"\",\"label\":\""+ents[i].label+"\",\"start\":"+std::to_string(ents[i].start)+",\"end\":"+std::to_string(ents[i].end)+",\"confidence\":"+std::to_string(ents[i].conf)+"}";} r+="]";
+ std::string r="[";
+ for(size_t i=0;i<ents.size();i++){
+ if(i)r+=",";
+ r+="{\"text\":"+thinc::JsonString(ents[i].text)+",\"label\":"+thinc::JsonString(ents[i].label)+
+ ",\"start\":"+std::to_string(ents[i].start)+",\"end\":"+std::to_string(ents[i].end)+
+ ",\"confidence\":"+std::to_string(ents[i].conf)+"}";
+ }
+ r+="]";
return strdup(r.c_str());
}
@@ -620,6 +628,11 @@ char* ThincNER_Tokenize(const char* t, const char* l) {
// de, fr, es, pt: European Latin -> en tokenizer; zh, ja: CJK -> zh tokenizer
bool is_cjk = (lang == "zh" || lang == "ja");
auto tok = is_cjk ? tokenize_zh(t) : tokenize_en(t);
- std::string r="["; for(size_t i=0;i<tok.size();i++){if(i)r+=",";r+="\""+tok[i]+"\"";} r+="]";
+ std::string r="[";
+ for(size_t i=0;i<tok.size();i++){
+ if(i)r+=",";
+ r+=thinc::JsonString(tok[i]);
+ }
+ r+="]";
return strdup(r.c_str());
}
diff --git a/internal/binding/cpp/thinc_parser.cpp b/internal/binding/cpp/thinc_parser.cpp
index 044af97e15..122de9c44d 100644
--- a/internal/binding/cpp/thinc_parser.cpp
+++ b/internal/binding/cpp/thinc_parser.cpp
@@ -1,4 +1,5 @@
#include "thinc_parser.h"
+#include "thinc_json.h"
#include <algorithm>
#include <cmath>
@@ -453,8 +454,8 @@ char* ThincParser_Predict(ThincParserHandle h, const char* tokens_json) {
std::string r="[";
for(int i=0;i<n;i++){
if(i)r+=",";
- r+="{\"text\":\""+tokens[i]+"\",\"head\":"+std::to_string(heads[i])+
- ",\"dep\":\""+labels[i]+"\",\"index\":"+std::to_string(i)+"}";
+ r+="{\"text\":"+thinc::JsonString(tokens[i])+",\"head\":"+std::to_string(heads[i])+
+ ",\"dep\":"+thinc::JsonString(labels[i])+",\"index\":"+std::to_string(i)+"}";
}
r+="]";
return strdup(r.c_str());
@@ -536,7 +537,8 @@ char* ThincTagger_Predict(ThincTaggerHandle h, const char* tokens_json) {
for(int i=0;i<n;i++){
if(i)r+=",";
std::string tag = best_tags[i] < (int)s->tags.size() ? s->tags[best_tags[i]] : "";
- r+="{\"text\":\""+tokens[i]+"\",\"tag\":\""+pos_only(tag)+"\",\"index\":"+std::to_string(i)+"}";
+ r+="{\"text\":"+thinc::JsonString(tokens[i])+",\"tag\":"+thinc::JsonString(pos_only(tag))+
+ ",\"index\":"+std::to_string(i)+"}";
}
r+="]";
return strdup(r.c_str());