RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Port PR19661 to go (keyword delimiter) (#20069)
qinling02100e21c029e287
diff --git a/internal/service/chat_pipeline.go b/internal/service/chat_pipeline.goindex d3bb211073..0bd78e723d 100644--- a/internal/service/chat_pipeline.go+++ b/internal/service/chat_pipeline.go@@ -646,7 +646,7 @@ func (s *ChatPipelineService) AsyncChat(if useKW, _ := chat.PromptConfig["keyword"].(bool); useKW && chatModel != nil && len(questions) > 0 {if kw, err := KeywordExtraction(ctx, chatModel, questions[len(questions)-1], 3); err == nil && kw != "" {original := questions[len(questions)-1]- questions[len(questions)-1] = questions[len(questions)-1] + "," + kw+ questions[len(questions)-1] = AppendKeywords(original, kw)common.Debug("keyword extraction applied",zap.String("original_question", original),zap.String("augmented_question", questions[len(questions)-1]))diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex 61a71873ef..e75a315e9a 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -337,8 +337,8 @@ func (s *ChunkService) RetrievalTest(ctx context.Context, req *service.RetrievalextractedKeywords, err := service.KeywordExtraction(ctx, chatModel, modifiedQuestion, 3)if err != nil {common.Warn("Failed to extract keywords from question", zap.Error(err))- } else if extractedKeywords != "" {- modifiedQuestion = modifiedQuestion + " " + extractedKeywords+ } else {+ modifiedQuestion = service.AppendKeywords(modifiedQuestion, extractedKeywords)}}diff --git a/internal/service/dataset/search.go b/internal/service/dataset/search.goindex 630dee101e..5d7e6fcfd5 100644--- a/internal/service/dataset/search.go+++ b/internal/service/dataset/search.go@@ -264,8 +264,8 @@ func (d *DatasetService) SearchDatasets(ctx context.Context, req *service.SearchextractedKeywords, err := service.KeywordExtraction(ctx, chatModel, modifiedQuestion, 3)if err != nil {common.Warn("Failed to extract keywords from question", zap.Error(err))- } else if extractedKeywords != "" {- modifiedQuestion = modifiedQuestion + extractedKeywords+ } else {+ modifiedQuestion = service.AppendKeywords(modifiedQuestion, extractedKeywords)}}}diff --git a/internal/service/generator.go b/internal/service/generator.goindex 9f4241941c..6665493b30 100644--- a/internal/service/generator.go+++ b/internal/service/generator.go@@ -32,6 +32,22 @@ import ("go.uber.org/zap")+// KeywordDelimiter separates the original question from the keywords appended+// by keyword extraction. Without it the question's last token merges with the+// first keyword during tokenization, which silently degrades lexical matching.+// Every question-augmentation callsite uses this one delimiter.+const KeywordDelimiter = ","++// AppendKeywords joins question with the keywords extracted from it, separated+// by KeywordDelimiter. An empty keywords string (failed or empty extraction)+// leaves question untouched, so a dangling delimiter is never appended.+func AppendKeywords(question, keywords string) string {+ if keywords == "" {+ return question+ }+ return question + KeywordDelimiter + keywords+}+// KeywordExtraction extracts keywords from content using LLM.//// Uses ChatModel to call the LLM with a keyword extraction prompt.diff --git a/internal/service/generator_test.go b/internal/service/generator_test.gonew file mode 100644index 0000000000..661eac7215--- /dev/null+++ b/internal/service/generator_test.go@@ -0,0 +1,52 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import "testing"++// TestAppendKeywordsSeparatesQuestionFromKeywords pins the delimiter used when+// keyword extraction output is appended to the retrieval question. The+// delimiter matters because without it the question's last token merges with+// the first keyword during tokenization.+func TestAppendKeywordsSeparatesQuestionFromKeywords(t *testing.T) {+ if got := AppendKeywords("什么是RAG", "RAG,检索,增强"); got != "什么是RAG,RAG,检索,增强" {+ t.Fatalf("AppendKeywords = %q", got)+ }+ if got := AppendKeywords("how do I reset my password", "password,reset"); got != "how do I reset my password,password,reset" {+ t.Fatalf("AppendKeywords = %q", got)+ }+}++// TestAppendKeywordsEmptyKeepsQuestion guards against a dangling delimiter when+// keyword extraction yields nothing (LLM error or empty reply).+func TestAppendKeywordsEmptyKeepsQuestion(t *testing.T) {+ if got := AppendKeywords("what is rag", ""); got != "what is rag" {+ t.Fatalf("AppendKeywords with empty keywords = %q, want the question unchanged", got)+ }+ if got := AppendKeywords("", "rag"); got != ",rag" {+ t.Fatalf("AppendKeywords with empty question = %q", got)+ }+}++// TestKeywordDelimiterIsComma keeps every Go callsite on the same delimiter;+// the value is asserted here so a drift shows up as a single failing test+// rather than three divergent concatenations.+func TestKeywordDelimiterIsComma(t *testing.T) {+ if KeywordDelimiter != "," {+ t.Fatalf("KeywordDelimiter = %q, want %q", KeywordDelimiter, ",")+ }+}