Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feature: go highlight follow python, follow #19488 (#19792)

Wang Qi1f229c8e37a7
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index bb50aeee86..f2e85c35cb 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -31,6 +31,8 @@ import (
"sort"
"strconv"
"strings"
+ "unicode"
+ "unicode/utf8"
"ragflow/internal/common"
"ragflow/internal/engine/types"
@@ -48,14 +50,6 @@ var jsonIterator = sonic.Config{
var memoryMessageVectorFieldRE = regexp.MustCompile(`^q_\d+_vec$`)
-var (
- elasticsearchHighlightEmTagRE = regexp.MustCompile(`<em>[^<>]+</em>`)
- elasticsearchHighlightNewlineRE = regexp.MustCompile(`[\r\n]`)
- elasticsearchHighlightDelimiterRE = regexp.MustCompile(`[.?!;\n]`)
- elasticsearchLetterRE = regexp.MustCompile(`\pL`)
- elasticsearchEnglishLetterRE = regexp.MustCompile(`[A-Za-z]`)
-)
-
// CreateChunkStore creates an index
func (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {
if baseName == "" {
@@ -2358,13 +2352,7 @@ func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string {
// GetHighlight returns highlighted text for matching keywords
func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {
result := make(map[string]string)
- if len(chunks) == 0 || len(keywords) == 0 {
- return result
- }
-
- normalizedKeywords := normalizeElasticsearchHighlightKeywords(keywords)
- englishPatterns := compileElasticsearchHighlightPatterns(normalizedKeywords)
- nonEnglishPattern := compileElasticsearchNonEnglishHighlightPattern(normalizedKeywords)
+ pattern := compileElasticsearchHighlightPattern(keywords)
for _, chunk := range chunks {
docID, ok := elasticsearchChunkID(chunk)
@@ -2372,47 +2360,16 @@ func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string
continue
}
- if highlightText := firstElasticsearchHighlight(chunk); highlightText != "" {
- result[docID] = highlightText
- continue
- }
-
txt, ok := chunk[fieldName].(string)
- if fieldName == "content_with_weight" && (!ok || txt == "") {
- txt, ok = chunk["content"].(string)
- }
- if !ok || txt == "" {
- continue
- }
-
- if elasticsearchHighlightEmTagRE.MatchString(txt) {
- result[docID] = txt
+ if !ok {
continue
}
-
- txt = elasticsearchHighlightNewlineRE.ReplaceAllString(txt, " ")
- segments := elasticsearchHighlightDelimiterRE.Split(txt, -1)
-
- var highlightedSegments []string
- for _, segment := range segments {
- segmentToCheck := segment
- if isMostlyEnglishElasticsearchSegment(segment) {
- for _, pattern := range englishPatterns {
- segmentToCheck = pattern.ReplaceAllString(segmentToCheck, "$1<em>$2</em>$3")
- }
- } else if nonEnglishPattern != nil {
- segmentToCheck = nonEnglishPattern.ReplaceAllStringFunc(segmentToCheck, func(match string) string {
- return "<em>" + match + "</em>"
- })
- }
- if segmentToCheck != segment {
- highlightedSegments = append(highlightedSegments, strings.TrimSpace(segmentToCheck))
- }
- }
-
- if len(highlightedSegments) > 0 {
- result[docID] = strings.Join(highlightedSegments, "... ")
+ if pattern != nil {
+ txt = pattern.ReplaceAllStringFunc(txt, func(match string) string {
+ return "<em>" + match + "</em>"
+ })
}
+ result[docID] = txt
}
return result
}
@@ -2427,68 +2384,42 @@ func elasticsearchChunkID(chunk map[string]interface{}) (string, bool) {
return "", false
}
-func firstElasticsearchHighlight(chunk map[string]interface{}) string {
- highlight, ok := chunk["highlight"].(map[string]interface{})
- if !ok || len(highlight) == 0 {
- return ""
- }
-
- for _, vals := range highlight {
- if arr, ok := vals.([]interface{}); ok && len(arr) > 0 {
- if str, ok := arr[0].(string); ok {
- return str
- }
- }
- }
- return ""
-}
-
func countElasticsearchAggregationTag(counts map[string]int, tag string) {
if tag = strings.TrimSpace(tag); tag != "" {
counts[tag]++
}
}
-func isMostlyEnglishElasticsearchSegment(segment string) bool {
- totalCount := len(elasticsearchLetterRE.FindAllString(segment, -1))
- return totalCount > 0 && float64(len(elasticsearchEnglishLetterRE.FindAllString(segment, -1)))/float64(totalCount) > 0.5
-}
-
-func compileElasticsearchHighlightPatterns(keywords []string) []*regexp.Regexp {
- patterns := make([]*regexp.Regexp, 0, len(keywords))
+func compileElasticsearchHighlightPattern(keywords []string) *regexp.Regexp {
+ nonEmpty := make([]string, 0, len(keywords))
for _, kw := range keywords {
- patterns = append(patterns, regexp.MustCompile(`(?i)(^|[ .?/'\"\(\)!,:;-])(`+regexp.QuoteMeta(kw)+`)([ .?/'\"\(\)!,:;-]|$)`))
+ if kw != "" {
+ nonEmpty = append(nonEmpty, kw)
+ }
}
- return patterns
-}
-
-func compileElasticsearchNonEnglishHighlightPattern(keywords []string) *regexp.Regexp {
- if len(keywords) == 0 {
+ if len(nonEmpty) == 0 {
return nil
}
- parts := make([]string, 0, len(keywords))
- for _, kw := range keywords {
- parts = append(parts, regexp.QuoteMeta(kw))
+ slices.SortStableFunc(nonEmpty, func(a, b string) int {
+ return cmp.Compare(utf8.RuneCountInString(b), utf8.RuneCountInString(a))
+ })
+ parts := make([]string, len(nonEmpty))
+ for i, keyword := range nonEmpty {
+ parts[i] = regexp.QuoteMeta(keyword)
+ if isLatinKeyword(keyword) {
+ parts[i] += `\p{Latin}*`
+ }
}
- return regexp.MustCompile(strings.Join(parts, "|"))
+ return regexp.MustCompile("(?i)" + strings.Join(parts, "|"))
}
-func normalizeElasticsearchHighlightKeywords(keywords []string) []string {
- seen := make(map[string]struct{}, len(keywords))
- normalized := make([]string, 0, len(keywords))
- for _, kw := range keywords {
- if kw == "" {
- continue
- }
- if _, ok := seen[kw]; !ok {
- seen[kw] = struct{}{}
- normalized = append(normalized, kw)
+func isLatinKeyword(keyword string) bool {
+ for _, r := range keyword {
+ if !unicode.In(r, unicode.Latin) {
+ return false
}
}
- slices.SortStableFunc(normalized, func(a, b string) int {
- return cmp.Compare(len(b), len(a))
- })
- return normalized
+ return keyword != ""
}
// DropChunkStore deletes a chunk index
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index 807d7fb3a4..d33af456f5 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -551,62 +551,46 @@ func TestElasticsearchGetChunkIDsPreservesOrderWithFallback(t *testing.T) {
}
}
-func TestElasticsearchGetHighlightFallbackAndBoundaries(t *testing.T) {
+func TestElasticsearchGetHighlightUsesRegex(t *testing.T) {
engine := &Engine{}
chunks := []map[string]interface{}{
{
- "_id": "fallback-id",
- "content": "Alpha beta.\nbetamax soup. BETA again!",
+ "_id": "full-text",
+ "content_with_weight": "Alpha beta.\nbetamax soup. BETA again!",
},
+ {"id": "no-match", "content_with_weight": "Keep the complete text."},
+ {"id": "missing"},
}
got := engine.GetHighlight(chunks, []string{"beta"}, "content_with_weight")
assertEqual(t, got, map[string]string{
- "fallback-id": "Alpha <em>beta</em>... <em>BETA</em> again",
- })
- if gotText := got["fallback-id"]; strings.Contains(gotText, "<em>beta</em>max") {
- t.Fatalf("highlight matched inside a larger token: %q", gotText)
- }
-
- gotLaterFallback := engine.GetHighlight([]map[string]interface{}{
- {"_id": "first"},
- {"_id": "second", "content": "Gamma beta."},
- }, []string{"beta"}, "content_with_weight")
- assertEqual(t, gotLaterFallback, map[string]string{
- "second": "Gamma <em>beta</em>",
- })
-
- gotMixedFallback := engine.GetHighlight([]map[string]interface{}{
- {"id": "weighted", "content_with_weight": "Weighted beta."},
- {"id": "plain", "content": "Plain beta."},
- }, []string{"beta"}, "content_with_weight")
- assertEqual(t, gotMixedFallback, map[string]string{
- "plain": "Plain <em>beta</em>",
- "weighted": "Weighted <em>beta</em>",
+ "full-text": "Alpha <em>beta</em>.\n<em>betamax</em> soup. <em>BETA</em> again!",
+ "no-match": "Keep the complete text.",
})
+}
- gotEmptyFallback := engine.GetHighlight([]map[string]interface{}{
- {"id": "empty-weighted", "content_with_weight": "", "content": "Empty fallback beta."},
- }, []string{"beta"}, "content_with_weight")
- assertEqual(t, gotEmptyFallback, map[string]string{
- "empty-weighted": "Empty fallback <em>beta</em>",
- })
+func TestElasticsearchGetHighlightExpandsLatinStems(t *testing.T) {
+ engine := &Engine{}
+ chunks := []map[string]interface{}{
+ {
+ "_id": "latin",
+ "content_with_weight": "Required coordinated isolation. Coordinación.",
+ },
+ }
- gotInvalidFallback := engine.GetHighlight([]map[string]interface{}{
- {"id": "invalid-weighted", "content_with_weight": nil, "content": "Invalid fallback beta."},
- }, []string{"beta"}, "content_with_weight")
- assertEqual(t, gotInvalidFallback, map[string]string{
- "invalid-weighted": "Invalid fallback <em>beta</em>",
+ got := engine.GetHighlight(chunks, []string{"requir", "coordin", "isol"}, "content_with_weight")
+ assertEqual(t, got, map[string]string{
+ "latin": "<em>Required</em> <em>coordinated</em> <em>isolation</em>. <em>Coordinación</em>.",
})
}
-func TestElasticsearchGetHighlightPreservesExistingAndNonEnglish(t *testing.T) {
+func TestElasticsearchGetHighlightPrefersLongKeywords(t *testing.T) {
engine := &Engine{}
gotExisting := engine.GetHighlight([]map[string]interface{}{
{"id": "existing", "content_with_weight": "already <em>marked</em> text"},
}, []string{"marked"}, "content_with_weight")
- assertEqual(t, gotExisting, map[string]string{"existing": "already <em>marked</em> text"})
+ assertEqual(t, gotExisting, map[string]string{"existing": "already <em><em>marked</em></em> text"})
gotNonEnglish := engine.GetHighlight([]map[string]interface{}{
{"id": "cn", "content_with_weight": "这是世界。你好世界"},
@@ -625,8 +609,8 @@ func TestElasticsearchGetHighlightPreservesExistingAndNonEnglish(t *testing.T) {
if got := engine.GetHighlight([]map[string]interface{}{{"id": "x"}}, []string{"x"}, "content_with_weight"); got == nil || len(got) != 0 {
t.Fatalf("missing field highlight=%#v, want empty non-nil map", got)
}
- if got := engine.GetHighlight([]map[string]interface{}{{"id": "x", "content": "x"}}, nil, "content"); got == nil || len(got) != 0 {
- t.Fatalf("empty keyword highlight=%#v, want empty non-nil map", got)
+ if got := engine.GetHighlight([]map[string]interface{}{{"id": "x", "content": "x"}}, nil, "content"); !reflect.DeepEqual(got, map[string]string{"x": "x"}) {
+ t.Fatalf("empty keyword highlight=%#v, want unchanged text", got)
}
}
diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.go
index 4d56e17012..18f66e3618 100644
--- a/internal/engine/infinity/chunk.go
+++ b/internal/engine/infinity/chunk.go
@@ -31,6 +31,8 @@ import (
"strconv"
"strings"
"sync"
+ "unicode"
+ "unicode/utf8"
infinity "github.com/infiniflow/infinity-go-sdk"
"go.uber.org/zap"
@@ -1896,19 +1898,75 @@ func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string {
return ids
}
-// GetHighlight generates highlighted text snippets for search results.
+// GetHighlight returns highlighted text for search results.
// Matches keywords in text and wraps them with <em> tags.
func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {
result := make(map[string]string)
- if len(chunks) == 0 || len(keywords) == 0 {
- return result
+ if fieldName == "content_with_weight" && !hasInfinityHighlightField(chunks, fieldName) {
+ fieldName = "content"
}
+ pattern := compileInfinityHighlightPattern(keywords)
- // For Infinity, scores are already returned in search results (_score column)
- // So GetScores just extracts scores from chunks, mimicking Python's approach
+ for _, chunk := range chunks {
+ id, ok := chunk["id"].(string)
+ if !ok || id == "" {
+ continue
+ }
+ txt, ok := chunk[fieldName].(string)
+ if !ok {
+ continue
+ }
+ if pattern != nil {
+ txt = pattern.ReplaceAllStringFunc(txt, func(match string) string {
+ return "<em>" + match + "</em>"
+ })
+ }
+ result[id] = txt
+ }
return result
}
+func hasInfinityHighlightField(chunks []map[string]interface{}, fieldName string) bool {
+ for _, chunk := range chunks {
+ if _, ok := chunk[fieldName]; ok {
+ return true
+ }
+ }
+ return false
+}
+
+func compileInfinityHighlightPattern(keywords []string) *regexp.Regexp {
+ nonEmpty := make([]string, 0, len(keywords))
+ for _, keyword := range keywords {
+ if keyword != "" {
+ nonEmpty = append(nonEmpty, keyword)
+ }
+ }
+ if len(nonEmpty) == 0 {
+ return nil
+ }
+ sort.SliceStable(nonEmpty, func(i, j int) bool {
+ return utf8.RuneCountInString(nonEmpty[i]) > utf8.RuneCountInString(nonEmpty[j])
+ })
+ parts := make([]string, len(nonEmpty))
+ for i, keyword := range nonEmpty {
+ parts[i] = regexp.QuoteMeta(keyword)
+ if isLatinKeyword(keyword) {
+ parts[i] += `\p{Latin}*`
+ }
+ }
+ return regexp.MustCompile("(?i)" + strings.Join(parts, "|"))
+}
+
+func isLatinKeyword(keyword string) bool {
+ for _, r := range keyword {
+ if !unicode.In(r, unicode.Latin) {
+ return false
+ }
+ }
+ return keyword != ""
+}
+
// KNNScores for Infinity - since Infinity normalizes scores during fusion,
// we just need to return a result structure that GetScores can parse.
// This matches Python's approach where Infinity doesn't use the two-pass KNN.
diff --git a/internal/engine/infinity/chunk_highlight_test.go b/internal/engine/infinity/chunk_highlight_test.go
new file mode 100644
index 0000000000..d458aff191
--- /dev/null
+++ b/internal/engine/infinity/chunk_highlight_test.go
@@ -0,0 +1,74 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package infinity
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestGetHighlightUsesRegex(t *testing.T) {
+ engine := &Engine{}
+ chunks := []map[string]interface{}{
+ {"id": "full-text", "content_with_weight": "Alpha beta.\nbetamax soup. BETA again!"},
+ {"id": "no-match", "content_with_weight": "Keep the complete text."},
+ }
+
+ got := engine.GetHighlight(chunks, []string{"beta"}, "content_with_weight")
+ want := map[string]string{
+ "full-text": "Alpha <em>beta</em>.\n<em>betamax</em> soup. <em>BETA</em> again!",
+ "no-match": "Keep the complete text.",
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("GetHighlight() = %#v, want %#v", got, want)
+ }
+}
+
+func TestGetHighlightExpandsLatinStems(t *testing.T) {
+ engine := &Engine{}
+ chunks := []map[string]interface{}{
+ {
+ "id": "latin",
+ "content_with_weight": "Required coordinated isolation. Coordinación.",
+ },
+ }
+
+ got := engine.GetHighlight(chunks, []string{"requir", "coordin", "isol"}, "content_with_weight")
+ want := map[string]string{
+ "latin": "<em>Required</em> <em>coordinated</em> <em>isolation</em>. <em>Coordinación</em>.",
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("GetHighlight() = %#v, want %#v", got, want)
+ }
+}
+
+func TestGetHighlightPrefersLongKeywordsAndFallsBackToContent(t *testing.T) {
+ engine := &Engine{}
+ chunks := []map[string]interface{}{
+ {"id": "overlap", "content": "世界和世"},
+ }
+
+ got := engine.GetHighlight(chunks, []string{"世", "世界", ""}, "content_with_weight")
+ want := map[string]string{"overlap": "<em>世界</em>和<em>世</em>"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("GetHighlight() = %#v, want %#v", got, want)
+ }
+
+ got = engine.GetHighlight(chunks, nil, "content_with_weight")
+ want = map[string]string{"overlap": "世界和世"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("GetHighlight() without keywords = %#v, want %#v", got, want)
+ }
+}
diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.go
index 147fba4ebd..25ccbdac0e 100644
--- a/internal/service/nlp/retrieval.go
+++ b/internal/service/nlp/retrieval.go
@@ -144,6 +144,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
RankFeature: *req.RankFeature,
EmbeddingModel: req.EmbeddingModel,
VectorSimilarityWeight: req.VectorSimilarityWeight,
+ Highlight: req.Highlight,
AllowDenseFallback: req.AllowDenseFallback,
Filter: req.Filter,
}
@@ -332,15 +333,9 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)
resultChunk["vector"] = zeroVector
}
- highlightEnabled := false
- if req.Highlight != nil && *req.Highlight {
- highlightEnabled = true
- }
- if highlightEnabled && searchResult.Highlight != nil {
+ if searchResult.Highlight != nil {
if highlightText, ok := searchResult.Highlight[chunkID]; ok {
- resultChunk["highlight"] = RemoveRedundantSpaces(highlightText)
- } else if contentWithWeight, ok := chunk["content_with_weight"].(string); ok {
- resultChunk["highlight"] = RemoveRedundantSpaces(contentWithWeight)
+ resultChunk["highlight"] = highlightText
}
}
filteredChunks = append(filteredChunks, resultChunk)
@@ -800,8 +795,8 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchReque
aggregation := s.docEngine.GetAggregation(searchResult.Chunks, "docnm_kwd")
// Build Highlight using GetHighlight
- var highlight map[string]string
- if len(keywordsList) > 0 {
+ highlight := make(map[string]string)
+ if *req.Highlight {
highlight = s.docEngine.GetHighlight(searchResult.Chunks, keywordsList, "content_with_weight")
}
diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.go
index 3a3c3d4192..6e3542d6a7 100644
--- a/internal/service/nlp/retrieval_test.go
+++ b/internal/service/nlp/retrieval_test.go
@@ -68,12 +68,60 @@ func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {
t.Fatalf("must_not filter = %#v", filters["must_not"])
}
}
+ if engine.highlightCalls != 0 {
+ t.Fatalf("GetHighlight calls = %d with highlighting disabled, want 0", engine.highlightCalls)
+ }
+}
+
+func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {
+ oldQueryBuilder := globalQueryBuilder
+ globalQueryBuilder = NewQueryBuilder()
+ defer func() { globalQueryBuilder = oldQueryBuilder }()
+
+ rows := []map[string]interface{}{
+ {"id": "highlighted", "content_ltks": "alpha", "content_with_weight": "alpha élève", "_score": 0.9},
+ {"id": "unmatched", "content_ltks": "alpha", "content_with_weight": "plain content", "_score": 0.8},
+ }
+ docEngine := &retrievalCountEngine{
+ rows: rows,
+ highlights: map[string]string{"highlighted": "<em>alpha</em> élève"},
+ }
+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})
+ top := 10
+ threshold := 0.5
+ vectorWeight := 1.0
+ highlight := true
+
+ result, err := service.Retrieval(t.Context(), &RetrievalRequest{
+ Question: "alpha",
+ TenantIDs: []string{"tenant-1"},
+ Page: 1,
+ PageSize: 2,
+ KNNTopK: &top,
+ SimilarityThreshold: &threshold,
+ VectorSimilarityWeight: &vectorWeight,
+ Highlight: &highlight,
+ })
+ if err != nil {
+ t.Fatalf("Retrieval failed: %v", err)
+ }
+ if docEngine.highlightCalls != 1 {
+ t.Fatalf("GetHighlight calls = %d, want 1", docEngine.highlightCalls)
+ }
+ if got := result.Chunks[0]["highlight"]; got != "<em>alpha</em> élève" {
+ t.Fatalf("highlight = %q, want spacing preserved", got)
+ }
+ if _, ok := result.Chunks[1]["highlight"]; ok {
+ t.Fatalf("unmatched chunk received fallback highlight: %#v", result.Chunks[1])
+ }
}
type retrievalCountEngine struct {
- rows []map[string]interface{}
- searchLimits []int
- searchFilters []map[string]interface{}
+ rows []map[string]interface{}
+ searchLimits []int
+ searchFilters []map[string]interface{}
+ highlights map[string]string
+ highlightCalls int
}
func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
@@ -184,7 +232,8 @@ func (e *retrievalCountEngine) GetAggregation([]map[string]interface{}, string)
return nil
}
func (e *retrievalCountEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {
- return nil
+ e.highlightCalls++
+ return e.highlights
}
func (e *retrievalCountEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {
return nil, nil