Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Port PR19661 to go (keyword delimiter) (#20069)

qinling02100e21c029e287
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.go
index d3bb211073..0bd78e723d 100644
--- a/internal/service/chat_pipeline.go
+++ b/internal/service/chat_pipeline.go
@@ -646,7 +646,7 @@ func (s *ChatPipelineService) AsyncChat(
if useKW, _ := chat.PromptConfig["keyword"].(bool); useKW && chatModel != nil && len(questions) > 0 {
if kw, err := KeywordExtraction(ctx, chatModel, questions[len(questions)-1], 3); err == nil && kw != "" {
original := questions[len(questions)-1]
- questions[len(questions)-1] = questions[len(questions)-1] + "," + kw
+ questions[len(questions)-1] = AppendKeywords(original, kw)
common.Debug("keyword extraction applied",
zap.String("original_question", original),
zap.String("augmented_question", questions[len(questions)-1]))
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 61a71873ef..e75a315e9a 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -337,8 +337,8 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.Retrieval
extractedKeywords, err := service.KeywordExtraction(ctx, chatModel, modifiedQuestion, 3)
if err != nil {
common.Warn("Failed to extract keywords from question", zap.Error(err))
- } else if extractedKeywords != "" {
- modifiedQuestion = modifiedQuestion + " " + extractedKeywords
+ } else {
+ modifiedQuestion = service.AppendKeywords(modifiedQuestion, extractedKeywords)
}
}
diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.go
index 630dee101e..5d7e6fcfd5 100644
--- a/internal/service/dataset/search.go
+++ b/internal/service/dataset/search.go
@@ -264,8 +264,8 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.Search
extractedKeywords, err := service.KeywordExtraction(ctx, chatModel, modifiedQuestion, 3)
if err != nil {
common.Warn("Failed to extract keywords from question", zap.Error(err))
- } else if extractedKeywords != "" {
- modifiedQuestion = modifiedQuestion + extractedKeywords
+ } else {
+ modifiedQuestion = service.AppendKeywords(modifiedQuestion, extractedKeywords)
}
}
}
diff --git a/internal/service/generator.go b/internal/service/generator.go
index 9f4241941c..6665493b30 100644
--- a/internal/service/generator.go
+++ b/internal/service/generator.go
@@ -32,6 +32,22 @@ import (
"go.uber.org/zap"
)
+// KeywordDelimiter separates the original question from the keywords appended
+// by keyword extraction. Without it the question's last token merges with the
+// first keyword during tokenization, which silently degrades lexical matching.
+// Every question-augmentation callsite uses this one delimiter.
+const KeywordDelimiter = ","
+
+// AppendKeywords joins question with the keywords extracted from it, separated
+// by KeywordDelimiter. An empty keywords string (failed or empty extraction)
+// leaves question untouched, so a dangling delimiter is never appended.
+func AppendKeywords(question, keywords string) string {
+ if keywords == "" {
+ return question
+ }
+ return question + KeywordDelimiter + keywords
+}
+
// KeywordExtraction extracts keywords from content using LLM.
//
// Uses ChatModel to call the LLM with a keyword extraction prompt.
diff --git a/internal/service/generator_test.go b/internal/service/generator_test.go
new file mode 100644
index 0000000000..661eac7215
--- /dev/null
+++ b/internal/service/generator_test.go
@@ -0,0 +1,52 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import "testing"
+
+// TestAppendKeywordsSeparatesQuestionFromKeywords pins the delimiter used when
+// keyword extraction output is appended to the retrieval question. The
+// delimiter matters because without it the question's last token merges with
+// the first keyword during tokenization.
+func TestAppendKeywordsSeparatesQuestionFromKeywords(t *testing.T) {
+ if got := AppendKeywords("什么是RAG", "RAG,检索,增强"); got != "什么是RAG,RAG,检索,增强" {
+ t.Fatalf("AppendKeywords = %q", got)
+ }
+ if got := AppendKeywords("how do I reset my password", "password,reset"); got != "how do I reset my password,password,reset" {
+ t.Fatalf("AppendKeywords = %q", got)
+ }
+}
+
+// TestAppendKeywordsEmptyKeepsQuestion guards against a dangling delimiter when
+// keyword extraction yields nothing (LLM error or empty reply).
+func TestAppendKeywordsEmptyKeepsQuestion(t *testing.T) {
+ if got := AppendKeywords("what is rag", ""); got != "what is rag" {
+ t.Fatalf("AppendKeywords with empty keywords = %q, want the question unchanged", got)
+ }
+ if got := AppendKeywords("", "rag"); got != ",rag" {
+ t.Fatalf("AppendKeywords with empty question = %q", got)
+ }
+}
+
+// TestKeywordDelimiterIsComma keeps every Go callsite on the same delimiter;
+// the value is asserted here so a drift shows up as a single failing test
+// rather than three divergent concatenations.
+func TestKeywordDelimiterIsComma(t *testing.T) {
+ if KeywordDelimiter != "," {
+ t.Fatalf("KeywordDelimiter = %q, want %q", KeywordDelimiter, ",")
+ }
+}