Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: isolate analyzer and benchmark worker state (#21583)

buua4361ed2737ffab9
diff --git a/internal/binding/cpp/CMakeLists.txt b/internal/binding/cpp/CMakeLists.txt
index 12d206aed2..2215b65905 100644
--- a/internal/binding/cpp/CMakeLists.txt
+++ b/internal/binding/cpp/CMakeLists.txt
@@ -178,6 +178,17 @@ set_target_properties(rag_analyzer_c_test PROPERTIES
)
if(BUILD_TESTING)
+ add_executable(rag_analyzer_opencc_copy_test
+ rag_analyzer_opencc_copy_test.cpp
+ )
+ target_link_libraries(rag_analyzer_opencc_copy_test rag_tokenizer_c_api stdc++ m ${PCRE2_LIB})
+ target_include_directories(rag_analyzer_opencc_copy_test PUBLIC "${CMAKE_SOURCE_DIR}")
+ set_target_properties(rag_analyzer_opencc_copy_test PROPERTIES
+ CXX_STANDARD 20
+ CXX_STANDARD_REQUIRED ON
+ )
+ add_test(NAME rag_analyzer_opencc_copy COMMAND rag_analyzer_opencc_copy_test)
+
add_executable(wordnet_lemmatizer_missing_exceptions_test
wordnet_lemmatizer_test.cpp
wordnet_lemmatizer.cpp
diff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cpp
index 530774d6de..b8b6f1e5bb 100644
--- a/internal/binding/cpp/rag_analyzer.cpp
+++ b/internal/binding/cpp/rag_analyzer.cpp
@@ -682,8 +682,11 @@ RAGAnalyzer::RAGAnalyzer(const std::string &path)
}
RAGAnalyzer::RAGAnalyzer(const RAGAnalyzer &other)
- : own_dict_(false), trie_(other.trie_), pos_table_(other.pos_table_), wordnet_lemma_(other.wordnet_lemma_), stemmer_(std::make_unique<Stemmer>()),
- opencc_(other.opencc_), lowercase_string_buffer_(term_string_buffer_limit_), fine_grained_(other.fine_grained_) {
+ : dict_path_(other.dict_path_), own_dict_(false), trie_(other.trie_), pos_table_(other.pos_table_), wordnet_lemma_(other.wordnet_lemma_),
+ stemmer_(std::make_unique<Stemmer>()),
+ // OpenCC keeps mutable conversion buffers, so each analyzer needs its own converter.
+ opencc_(other.opencc_ ? std::make_unique<::OpenCC>((fs::path(other.dict_path_) / OPENCC_PATH).string()) : nullptr),
+ lowercase_string_buffer_(term_string_buffer_limit_), fine_grained_(other.fine_grained_) {
InitStemmer(STEM_LANG_ENGLISH);
}
@@ -692,7 +695,6 @@ RAGAnalyzer::~RAGAnalyzer() {
delete trie_;
delete pos_table_;
delete wordnet_lemma_;
- delete opencc_;
}
}
@@ -813,7 +815,7 @@ int32_t RAGAnalyzer::Load() {
}
try {
printf("Load opencc file: %s\n", opencc_path.string().c_str());
- opencc_ = new ::OpenCC(opencc_path.string());
+ opencc_ = std::make_unique<::OpenCC>(opencc_path.string());
} catch (const std::exception &e) {
printf("Fail to open opencc: %s\n", opencc_path.string().c_str());
return -7;
diff --git a/internal/binding/cpp/rag_analyzer.h b/internal/binding/cpp/rag_analyzer.h
index c2b7c31885..2145d15843 100644
--- a/internal/binding/cpp/rag_analyzer.h
+++ b/internal/binding/cpp/rag_analyzer.h
@@ -163,7 +163,7 @@ public:
std::unique_ptr<Stemmer> stemmer_;
- OpenCC* opencc_{nullptr};
+ std::unique_ptr<OpenCC> opencc_;
mutable std::vector<char> lowercase_string_buffer_;
diff --git a/internal/binding/cpp/rag_analyzer_c_api.h b/internal/binding/cpp/rag_analyzer_c_api.h
index 510a11bb3d..4d77d891bd 100644
--- a/internal/binding/cpp/rag_analyzer_c_api.h
+++ b/internal/binding/cpp/rag_analyzer_c_api.h
@@ -98,7 +98,7 @@ char* RAGAnalyzer_GetTermTag(RAGAnalyzerHandle handle, const char* term);
// Copy an existing RAGAnalyzer instance to create a new independent instance
// This is useful for creating per-request analyzer instances in multi-threaded environments
-// The new instance shares the loaded dictionaries with the original but has independent internal state
+// The new instance shares the tokenizer dictionaries and owns its OpenCC converter state.
// Returns: handle to the new analyzer instance, or NULL on failure
RAGAnalyzerHandle RAGAnalyzer_Copy(RAGAnalyzerHandle handle);
diff --git a/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp
new file mode 100644
index 0000000000..81897f801d
--- /dev/null
+++ b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp
@@ -0,0 +1,95 @@
+#include "rag_analyzer.h"
+
+#include <atomic>
+#include <chrono>
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+int main() {
+ namespace fs = std::filesystem;
+
+ const fs::path resource_path = fs::temp_directory_path() /
+ ("rag_analyzer_opencc_copy_test_" +
+ std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));
+ const fs::path opencc_path = resource_path / "opencc";
+ fs::create_directories(opencc_path);
+
+ {
+ std::ofstream config(opencc_path / "zht2zhs.ini");
+ config << "title = test\n"
+ << "dict0 = TEXT test.txt\n";
+ if (!config) {
+ std::cerr << "failed to write OpenCC config\n";
+ fs::remove_all(resource_path);
+ return 1;
+ }
+ }
+ {
+ std::ofstream dictionary(opencc_path / "test.txt");
+ dictionary << "繁體\t繁体\n";
+ if (!dictionary) {
+ std::cerr << "failed to write OpenCC dictionary\n";
+ fs::remove_all(resource_path);
+ return 1;
+ }
+ }
+
+ RAGAnalyzer base(resource_path.string());
+ base.opencc_ = std::make_unique<::OpenCC>(opencc_path.string());
+
+ constexpr int worker_count = 8;
+ std::vector<std::unique_ptr<RAGAnalyzer>> analyzers;
+ analyzers.reserve(worker_count);
+ for (int i = 0; i < worker_count; ++i) {
+ auto copy = std::make_unique<RAGAnalyzer>(base);
+ if (!copy->opencc_ || copy->opencc_.get() == base.opencc_.get()) {
+ std::cerr << "RAGAnalyzer copy shares its OpenCC converter\n";
+ fs::remove_all(resource_path);
+ return 1;
+ }
+ for (const auto& existing : analyzers) {
+ if (copy->opencc_.get() == existing->opencc_.get()) {
+ std::cerr << "RAGAnalyzer copies share an OpenCC converter\n";
+ fs::remove_all(resource_path);
+ return 1;
+ }
+ }
+ analyzers.push_back(std::move(copy));
+ }
+
+ std::string input;
+ std::string expected;
+ for (int i = 0; i < 128; ++i) {
+ input += "繁體";
+ expected += "繁体";
+ }
+
+ std::atomic<int> failures{0};
+ std::vector<std::thread> workers;
+ workers.reserve(worker_count);
+ for (int i = 0; i < worker_count; ++i) {
+ workers.emplace_back([&, i] {
+ for (int iteration = 0; iteration < 100; ++iteration) {
+ std::string converted;
+ if (analyzers[i]->opencc_->convert(input, converted) < 0 || converted != expected) {
+ ++failures;
+ }
+ }
+ });
+ }
+ for (auto& worker : workers) {
+ worker.join();
+ }
+
+ fs::remove_all(resource_path);
+ if (failures != 0) {
+ std::cerr << "concurrent OpenCC conversions failed " << failures << " times\n";
+ return 1;
+ }
+ return 0;
+}
diff --git a/internal/binding/rag_analyzer.go b/internal/binding/rag_analyzer.go
index 7a83c2b180..32269180f1 100644
--- a/internal/binding/rag_analyzer.go
+++ b/internal/binding/rag_analyzer.go
@@ -264,9 +264,9 @@ func (a *Analyzer) GetTermTag(term string) string {
return C.GoString(cResult)
}
-// Copy creates a new independent analyzer instance from the current one
-// The new instance shares the loaded dictionaries but has independent internal state
-// This is useful for creating per-request analyzer instances in concurrent environments
+// Copy creates a new independent analyzer instance from the current one.
+// The instance shares the tokenizer dictionaries and owns an independent OpenCC converter.
+// This is useful for creating per-request analyzer instances in concurrent environments.
func (a *Analyzer) Copy() *Analyzer {
if a.handle == nil {
return nil
diff --git a/internal/cli/benchmark.go b/internal/cli/benchmark.go
index 253d31285f..d31f63791f 100644
--- a/internal/cli/benchmark.go
+++ b/internal/cli/benchmark.go
@@ -58,9 +58,17 @@ func (c *CLI) RunBenchmark(commandCount int, cmd *Command) (ResponseIf, error) {
nestedCmd.Params["iterations"] = iterations
if concurrency == 1 {
- return c.runBenchmarkSingle(iterations, nestedCmd)
+ response, err := c.runBenchmarkSingle(iterations, nestedCmd)
+ if err != nil {
+ return nil, err
+ }
+ return response, nil
+ }
+ response, err := c.runBenchmarkConcurrent(concurrency, iterations, nestedCmd)
+ if err != nil {
+ return nil, err
}
- return c.runBenchmarkConcurrent(concurrency, iterations, nestedCmd)
+ return response, nil
}
// runBenchmarkSingle runs benchmark with single concurrency (sequential execution)
@@ -137,7 +145,11 @@ func (c *CLI) runBenchmarkSingle(iterations int, nestedCmd *Command) (*Benchmark
// runBenchmarkConcurrent runs benchmark with multiple concurrent workers
func (c *CLI) runBenchmarkConcurrent(concurrency, iterations int, nestedCmd *Command) (*BenchmarkResponse, error) {
- results := make([]map[string]interface{}, concurrency)
+ type workerResult struct {
+ response *BenchmarkResponse
+ err error
+ }
+ results := make([]workerResult, concurrency)
var wg sync.WaitGroup
// For search_on_datasets, convert dataset names to IDs first
@@ -164,12 +176,12 @@ func (c *CLI) runBenchmarkConcurrent(concurrency, iterations int, nestedCmd *Com
go func(idx int) {
defer wg.Done()
- benchmarkResponse, _ := c.runBenchmarkSingle(iterations, nestedCmd)
-
- results[idx] = map[string]interface{}{
- "duration": 0.0,
- "response_list": benchmarkResponse,
+ workerCmd := *nestedCmd
+ workerCmd.Params = make(map[string]interface{}, len(nestedCmd.Params))
+ for key, value := range nestedCmd.Params {
+ workerCmd.Params[key] = value
}
+ results[idx].response, results[idx].err = c.runBenchmarkSingle(iterations, &workerCmd)
}(i)
}
@@ -180,13 +192,15 @@ func (c *CLI) runBenchmarkConcurrent(concurrency, iterations int, nestedCmd *Com
successCount := 0
failureCount := 0
- for _, result := range results {
- if result == nil {
- continue
+ for i, result := range results {
+ if result.err != nil {
+ return nil, fmt.Errorf("benchmark worker %d: %w", i+1, result.err)
+ }
+ if result.response == nil {
+ return nil, fmt.Errorf("benchmark worker %d returned no response", i+1)
}
- res, _ := result["response_list"].(*BenchmarkResponse)
- successCount += res.SuccessCount
- failureCount += res.FailureCount
+ successCount += result.response.SuccessCount
+ failureCount += result.response.FailureCount
}
//totalCommands := iterations * concurrency
diff --git a/internal/cli/benchmark_test.go b/internal/cli/benchmark_test.go
new file mode 100644
index 0000000000..c6e8039edc
--- /dev/null
+++ b/internal/cli/benchmark_test.go
@@ -0,0 +1,39 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package cli
+
+import "testing"
+
+func TestRunBenchmarkConcurrentPropagatesWorkerError(t *testing.T) {
+ cli := &CLI{Config: &CommandLineConfig{CLIMode: APIMode}}
+ command := &Command{Type: "benchmark", Params: map[string]interface{}{
+ "concurrency": 2,
+ "iterations": 1,
+ "command": &Command{
+ Type: "not_a_command",
+ Params: map[string]interface{}{},
+ },
+ }}
+
+ response, err := cli.RunBenchmark(1, command)
+ if err == nil {
+ t.Fatal("RunBenchmark() error = nil, want worker command error")
+ }
+ if response != nil {
+ t.Fatalf("RunBenchmark() response = %#v, want nil on worker error", response)
+ }
+}
diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.go
index 6cf3b11749..0cd7e44200 100644
--- a/internal/tokenizer/tokenizer.go
+++ b/internal/tokenizer/tokenizer.go
@@ -442,10 +442,10 @@ func (p *analyzerPool) Close() {
// the lock; stopCh is already closed, so it returns promptly.
p.wg.Wait()
- // A copied analyzer references the base analyzer's loaded dictionaries
+ // A copied analyzer references the base analyzer's loaded tokenizer dictionaries
// (RAGAnalyzer_Copy shares trie_/pos_table_/wordnet_lemma_ and does not own
- // them), so the base must outlive every checked-out instance. Wait for the
- // borrowers to return before destroying it.
+ // them), but owns its OpenCC converter. Keep the base alive until every checked-out
+ // instance has been returned.
p.borrowers.Wait()
// Phase 2: close the channel and release the analyzers. The read lock keeps