RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: isolate analyzer and benchmark worker state (#21583)
buua4361ed2737ffab9
diff --git a/internal/binding/cpp/CMakeLists.txt b/internal/binding/cpp/CMakeLists.txtindex 12d206aed2..2215b65905 100644--- a/internal/binding/cpp/CMakeLists.txt+++ b/internal/binding/cpp/CMakeLists.txt@@ -178,6 +178,17 @@ set_target_properties(rag_analyzer_c_test PROPERTIES)if(BUILD_TESTING)+ add_executable(rag_analyzer_opencc_copy_test+ rag_analyzer_opencc_copy_test.cpp+ )+ target_link_libraries(rag_analyzer_opencc_copy_test rag_tokenizer_c_api stdc++ m ${PCRE2_LIB})+ target_include_directories(rag_analyzer_opencc_copy_test PUBLIC "${CMAKE_SOURCE_DIR}")+ set_target_properties(rag_analyzer_opencc_copy_test PROPERTIES+ CXX_STANDARD 20+ CXX_STANDARD_REQUIRED ON+ )+ add_test(NAME rag_analyzer_opencc_copy COMMAND rag_analyzer_opencc_copy_test)+add_executable(wordnet_lemmatizer_missing_exceptions_testwordnet_lemmatizer_test.cppwordnet_lemmatizer.cppdiff --git a/internal/binding/cpp/rag_analyzer.cpp b/internal/binding/cpp/rag_analyzer.cppindex 530774d6de..b8b6f1e5bb 100644--- a/internal/binding/cpp/rag_analyzer.cpp+++ b/internal/binding/cpp/rag_analyzer.cpp@@ -682,8 +682,11 @@ RAGAnalyzer::RAGAnalyzer(const std::string &path)}RAGAnalyzer::RAGAnalyzer(const RAGAnalyzer &other)- : own_dict_(false), trie_(other.trie_), pos_table_(other.pos_table_), wordnet_lemma_(other.wordnet_lemma_), stemmer_(std::make_unique<Stemmer>()),- opencc_(other.opencc_), lowercase_string_buffer_(term_string_buffer_limit_), fine_grained_(other.fine_grained_) {+ : dict_path_(other.dict_path_), own_dict_(false), trie_(other.trie_), pos_table_(other.pos_table_), wordnet_lemma_(other.wordnet_lemma_),+ stemmer_(std::make_unique<Stemmer>()),+ // OpenCC keeps mutable conversion buffers, so each analyzer needs its own converter.+ opencc_(other.opencc_ ? std::make_unique<::OpenCC>((fs::path(other.dict_path_) / OPENCC_PATH).string()) : nullptr),+ lowercase_string_buffer_(term_string_buffer_limit_), fine_grained_(other.fine_grained_) {InitStemmer(STEM_LANG_ENGLISH);}@@ -692,7 +695,6 @@ RAGAnalyzer::~RAGAnalyzer() {delete trie_;delete pos_table_;delete wordnet_lemma_;- delete opencc_;}}@@ -813,7 +815,7 @@ int32_t RAGAnalyzer::Load() {}try {printf("Load opencc file: %s\n", opencc_path.string().c_str());- opencc_ = new ::OpenCC(opencc_path.string());+ opencc_ = std::make_unique<::OpenCC>(opencc_path.string());} catch (const std::exception &e) {printf("Fail to open opencc: %s\n", opencc_path.string().c_str());return -7;diff --git a/internal/binding/cpp/rag_analyzer.h b/internal/binding/cpp/rag_analyzer.hindex c2b7c31885..2145d15843 100644--- a/internal/binding/cpp/rag_analyzer.h+++ b/internal/binding/cpp/rag_analyzer.h@@ -163,7 +163,7 @@ public:std::unique_ptr<Stemmer> stemmer_;- OpenCC* opencc_{nullptr};+ std::unique_ptr<OpenCC> opencc_;mutable std::vector<char> lowercase_string_buffer_;diff --git a/internal/binding/cpp/rag_analyzer_c_api.h b/internal/binding/cpp/rag_analyzer_c_api.hindex 510a11bb3d..4d77d891bd 100644--- a/internal/binding/cpp/rag_analyzer_c_api.h+++ b/internal/binding/cpp/rag_analyzer_c_api.h@@ -98,7 +98,7 @@ char* RAGAnalyzer_GetTermTag(RAGAnalyzerHandle handle, const char* term);// Copy an existing RAGAnalyzer instance to create a new independent instance// This is useful for creating per-request analyzer instances in multi-threaded environments-// The new instance shares the loaded dictionaries with the original but has independent internal state+// The new instance shares the tokenizer dictionaries and owns its OpenCC converter state.// Returns: handle to the new analyzer instance, or NULL on failureRAGAnalyzerHandle RAGAnalyzer_Copy(RAGAnalyzerHandle handle);diff --git a/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cppnew file mode 100644index 0000000000..81897f801d--- /dev/null+++ b/internal/binding/cpp/rag_analyzer_opencc_copy_test.cpp@@ -0,0 +1,95 @@+#include "rag_analyzer.h"++#include <atomic>+#include <chrono>+#include <filesystem>+#include <fstream>+#include <iostream>+#include <memory>+#include <string>+#include <thread>+#include <vector>++int main() {+ namespace fs = std::filesystem;++ const fs::path resource_path = fs::temp_directory_path() /+ ("rag_analyzer_opencc_copy_test_" ++ std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()));+ const fs::path opencc_path = resource_path / "opencc";+ fs::create_directories(opencc_path);++ {+ std::ofstream config(opencc_path / "zht2zhs.ini");+ config << "title = test\n"+ << "dict0 = TEXT test.txt\n";+ if (!config) {+ std::cerr << "failed to write OpenCC config\n";+ fs::remove_all(resource_path);+ return 1;+ }+ }+ {+ std::ofstream dictionary(opencc_path / "test.txt");+ dictionary << "繁體\t繁体\n";+ if (!dictionary) {+ std::cerr << "failed to write OpenCC dictionary\n";+ fs::remove_all(resource_path);+ return 1;+ }+ }++ RAGAnalyzer base(resource_path.string());+ base.opencc_ = std::make_unique<::OpenCC>(opencc_path.string());++ constexpr int worker_count = 8;+ std::vector<std::unique_ptr<RAGAnalyzer>> analyzers;+ analyzers.reserve(worker_count);+ for (int i = 0; i < worker_count; ++i) {+ auto copy = std::make_unique<RAGAnalyzer>(base);+ if (!copy->opencc_ || copy->opencc_.get() == base.opencc_.get()) {+ std::cerr << "RAGAnalyzer copy shares its OpenCC converter\n";+ fs::remove_all(resource_path);+ return 1;+ }+ for (const auto& existing : analyzers) {+ if (copy->opencc_.get() == existing->opencc_.get()) {+ std::cerr << "RAGAnalyzer copies share an OpenCC converter\n";+ fs::remove_all(resource_path);+ return 1;+ }+ }+ analyzers.push_back(std::move(copy));+ }++ std::string input;+ std::string expected;+ for (int i = 0; i < 128; ++i) {+ input += "繁體";+ expected += "繁体";+ }++ std::atomic<int> failures{0};+ std::vector<std::thread> workers;+ workers.reserve(worker_count);+ for (int i = 0; i < worker_count; ++i) {+ workers.emplace_back([&, i] {+ for (int iteration = 0; iteration < 100; ++iteration) {+ std::string converted;+ if (analyzers[i]->opencc_->convert(input, converted) < 0 || converted != expected) {+ ++failures;+ }+ }+ });+ }+ for (auto& worker : workers) {+ worker.join();+ }++ fs::remove_all(resource_path);+ if (failures != 0) {+ std::cerr << "concurrent OpenCC conversions failed " << failures << " times\n";+ return 1;+ }+ return 0;+}diff --git a/internal/binding/rag_analyzer.go b/internal/binding/rag_analyzer.goindex 7a83c2b180..32269180f1 100644--- a/internal/binding/rag_analyzer.go+++ b/internal/binding/rag_analyzer.go@@ -264,9 +264,9 @@ func (a *Analyzer) GetTermTag(term string) string {return C.GoString(cResult)}-// Copy creates a new independent analyzer instance from the current one-// The new instance shares the loaded dictionaries but has independent internal state-// This is useful for creating per-request analyzer instances in concurrent environments+// Copy creates a new independent analyzer instance from the current one.+// The instance shares the tokenizer dictionaries and owns an independent OpenCC converter.+// This is useful for creating per-request analyzer instances in concurrent environments.func (a *Analyzer) Copy() *Analyzer {if a.handle == nil {return nildiff --git a/internal/cli/benchmark.go b/internal/cli/benchmark.goindex 253d31285f..d31f63791f 100644--- a/internal/cli/benchmark.go+++ b/internal/cli/benchmark.go@@ -58,9 +58,17 @@ func (c *CLI) RunBenchmark(commandCount int, cmd *Command) (ResponseIf, error) {nestedCmd.Params["iterations"] = iterationsif concurrency == 1 {- return c.runBenchmarkSingle(iterations, nestedCmd)+ response, err := c.runBenchmarkSingle(iterations, nestedCmd)+ if err != nil {+ return nil, err+ }+ return response, nil+ }+ response, err := c.runBenchmarkConcurrent(concurrency, iterations, nestedCmd)+ if err != nil {+ return nil, err}- return c.runBenchmarkConcurrent(concurrency, iterations, nestedCmd)+ return response, nil}// runBenchmarkSingle runs benchmark with single concurrency (sequential execution)@@ -137,7 +145,11 @@ func (c *CLI) runBenchmarkSingle(iterations int, nestedCmd *Command) (*Benchmark// runBenchmarkConcurrent runs benchmark with multiple concurrent workersfunc (c *CLI) runBenchmarkConcurrent(concurrency, iterations int, nestedCmd *Command) (*BenchmarkResponse, error) {- results := make([]map[string]interface{}, concurrency)+ type workerResult struct {+ response *BenchmarkResponse+ err error+ }+ results := make([]workerResult, concurrency)var wg sync.WaitGroup// For search_on_datasets, convert dataset names to IDs first@@ -164,12 +176,12 @@ func (c *CLI) runBenchmarkConcurrent(concurrency, iterations int, nestedCmd *Comgo func(idx int) {defer wg.Done()- benchmarkResponse, _ := c.runBenchmarkSingle(iterations, nestedCmd)-- results[idx] = map[string]interface{}{- "duration": 0.0,- "response_list": benchmarkResponse,+ workerCmd := *nestedCmd+ workerCmd.Params = make(map[string]interface{}, len(nestedCmd.Params))+ for key, value := range nestedCmd.Params {+ workerCmd.Params[key] = value}+ results[idx].response, results[idx].err = c.runBenchmarkSingle(iterations, &workerCmd)}(i)}@@ -180,13 +192,15 @@ func (c *CLI) runBenchmarkConcurrent(concurrency, iterations int, nestedCmd *ComsuccessCount := 0failureCount := 0- for _, result := range results {- if result == nil {- continue+ for i, result := range results {+ if result.err != nil {+ return nil, fmt.Errorf("benchmark worker %d: %w", i+1, result.err)+ }+ if result.response == nil {+ return nil, fmt.Errorf("benchmark worker %d returned no response", i+1)}- res, _ := result["response_list"].(*BenchmarkResponse)- successCount += res.SuccessCount- failureCount += res.FailureCount+ successCount += result.response.SuccessCount+ failureCount += result.response.FailureCount}//totalCommands := iterations * concurrencydiff --git a/internal/cli/benchmark_test.go b/internal/cli/benchmark_test.gonew file mode 100644index 0000000000..c6e8039edc--- /dev/null+++ b/internal/cli/benchmark_test.go@@ -0,0 +1,39 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package cli++import "testing"++func TestRunBenchmarkConcurrentPropagatesWorkerError(t *testing.T) {+ cli := &CLI{Config: &CommandLineConfig{CLIMode: APIMode}}+ command := &Command{Type: "benchmark", Params: map[string]interface{}{+ "concurrency": 2,+ "iterations": 1,+ "command": &Command{+ Type: "not_a_command",+ Params: map[string]interface{}{},+ },+ }}++ response, err := cli.RunBenchmark(1, command)+ if err == nil {+ t.Fatal("RunBenchmark() error = nil, want worker command error")+ }+ if response != nil {+ t.Fatalf("RunBenchmark() response = %#v, want nil on worker error", response)+ }+}diff --git a/internal/tokenizer/tokenizer.go b/internal/tokenizer/tokenizer.goindex 6cf3b11749..0cd7e44200 100644--- a/internal/tokenizer/tokenizer.go+++ b/internal/tokenizer/tokenizer.go@@ -442,10 +442,10 @@ func (p *analyzerPool) Close() {// the lock; stopCh is already closed, so it returns promptly.p.wg.Wait()- // A copied analyzer references the base analyzer's loaded dictionaries+ // A copied analyzer references the base analyzer's loaded tokenizer dictionaries// (RAGAnalyzer_Copy shares trie_/pos_table_/wordnet_lemma_ and does not own- // them), so the base must outlive every checked-out instance. Wait for the- // borrowers to return before destroying it.+ // them), but owns its OpenCC converter. Keep the base alive until every checked-out+ // instance has been returned.p.borrowers.Wait()// Phase 2: close the channel and release the analyzers. The read lock keeps