RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feature: go highlight follow python, follow #19488 (#19792)
Wang Qi1f229c8e37a7
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.goindex bb50aeee86..f2e85c35cb 100644--- a/internal/engine/elasticsearch/chunk.go+++ b/internal/engine/elasticsearch/chunk.go@@ -31,6 +31,8 @@ import ("sort""strconv""strings"+ "unicode"+ "unicode/utf8""ragflow/internal/common""ragflow/internal/engine/types"@@ -48,14 +50,6 @@ var jsonIterator = sonic.Config{var memoryMessageVectorFieldRE = regexp.MustCompile(`^q_\d+_vec$`)-var (- elasticsearchHighlightEmTagRE = regexp.MustCompile(`<em>[^<>]+</em>`)- elasticsearchHighlightNewlineRE = regexp.MustCompile(`[\r\n]`)- elasticsearchHighlightDelimiterRE = regexp.MustCompile(`[.?!;\n]`)- elasticsearchLetterRE = regexp.MustCompile(`\pL`)- elasticsearchEnglishLetterRE = regexp.MustCompile(`[A-Za-z]`)-)-// CreateChunkStore creates an indexfunc (e *Engine) CreateChunkStore(ctx context.Context, baseName, datasetID string, vectorSize int, parserID string) error {if baseName == "" {@@ -2358,13 +2352,7 @@ func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string {// GetHighlight returns highlighted text for matching keywordsfunc (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {result := make(map[string]string)- if len(chunks) == 0 || len(keywords) == 0 {- return result- }-- normalizedKeywords := normalizeElasticsearchHighlightKeywords(keywords)- englishPatterns := compileElasticsearchHighlightPatterns(normalizedKeywords)- nonEnglishPattern := compileElasticsearchNonEnglishHighlightPattern(normalizedKeywords)+ pattern := compileElasticsearchHighlightPattern(keywords)for _, chunk := range chunks {docID, ok := elasticsearchChunkID(chunk)@@ -2372,47 +2360,16 @@ func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []stringcontinue}- if highlightText := firstElasticsearchHighlight(chunk); highlightText != "" {- result[docID] = highlightText- continue- }-txt, ok := chunk[fieldName].(string)- if fieldName == "content_with_weight" && (!ok || txt == "") {- txt, ok = chunk["content"].(string)- }- if !ok || txt == "" {- continue- }-- if elasticsearchHighlightEmTagRE.MatchString(txt) {- result[docID] = txt+ if !ok {continue}-- txt = elasticsearchHighlightNewlineRE.ReplaceAllString(txt, " ")- segments := elasticsearchHighlightDelimiterRE.Split(txt, -1)-- var highlightedSegments []string- for _, segment := range segments {- segmentToCheck := segment- if isMostlyEnglishElasticsearchSegment(segment) {- for _, pattern := range englishPatterns {- segmentToCheck = pattern.ReplaceAllString(segmentToCheck, "$1<em>$2</em>$3")- }- } else if nonEnglishPattern != nil {- segmentToCheck = nonEnglishPattern.ReplaceAllStringFunc(segmentToCheck, func(match string) string {- return "<em>" + match + "</em>"- })- }- if segmentToCheck != segment {- highlightedSegments = append(highlightedSegments, strings.TrimSpace(segmentToCheck))- }- }-- if len(highlightedSegments) > 0 {- result[docID] = strings.Join(highlightedSegments, "... ")+ if pattern != nil {+ txt = pattern.ReplaceAllStringFunc(txt, func(match string) string {+ return "<em>" + match + "</em>"+ })}+ result[docID] = txt}return result}@@ -2427,68 +2384,42 @@ func elasticsearchChunkID(chunk map[string]interface{}) (string, bool) {return "", false}-func firstElasticsearchHighlight(chunk map[string]interface{}) string {- highlight, ok := chunk["highlight"].(map[string]interface{})- if !ok || len(highlight) == 0 {- return ""- }-- for _, vals := range highlight {- if arr, ok := vals.([]interface{}); ok && len(arr) > 0 {- if str, ok := arr[0].(string); ok {- return str- }- }- }- return ""-}-func countElasticsearchAggregationTag(counts map[string]int, tag string) {if tag = strings.TrimSpace(tag); tag != "" {counts[tag]++}}-func isMostlyEnglishElasticsearchSegment(segment string) bool {- totalCount := len(elasticsearchLetterRE.FindAllString(segment, -1))- return totalCount > 0 && float64(len(elasticsearchEnglishLetterRE.FindAllString(segment, -1)))/float64(totalCount) > 0.5-}--func compileElasticsearchHighlightPatterns(keywords []string) []*regexp.Regexp {- patterns := make([]*regexp.Regexp, 0, len(keywords))+func compileElasticsearchHighlightPattern(keywords []string) *regexp.Regexp {+ nonEmpty := make([]string, 0, len(keywords))for _, kw := range keywords {- patterns = append(patterns, regexp.MustCompile(`(?i)(^|[ .?/'\"\(\)!,:;-])(`+regexp.QuoteMeta(kw)+`)([ .?/'\"\(\)!,:;-]|$)`))+ if kw != "" {+ nonEmpty = append(nonEmpty, kw)+ }}- return patterns-}--func compileElasticsearchNonEnglishHighlightPattern(keywords []string) *regexp.Regexp {- if len(keywords) == 0 {+ if len(nonEmpty) == 0 {return nil}- parts := make([]string, 0, len(keywords))- for _, kw := range keywords {- parts = append(parts, regexp.QuoteMeta(kw))+ slices.SortStableFunc(nonEmpty, func(a, b string) int {+ return cmp.Compare(utf8.RuneCountInString(b), utf8.RuneCountInString(a))+ })+ parts := make([]string, len(nonEmpty))+ for i, keyword := range nonEmpty {+ parts[i] = regexp.QuoteMeta(keyword)+ if isLatinKeyword(keyword) {+ parts[i] += `\p{Latin}*`+ }}- return regexp.MustCompile(strings.Join(parts, "|"))+ return regexp.MustCompile("(?i)" + strings.Join(parts, "|"))}-func normalizeElasticsearchHighlightKeywords(keywords []string) []string {- seen := make(map[string]struct{}, len(keywords))- normalized := make([]string, 0, len(keywords))- for _, kw := range keywords {- if kw == "" {- continue- }- if _, ok := seen[kw]; !ok {- seen[kw] = struct{}{}- normalized = append(normalized, kw)+func isLatinKeyword(keyword string) bool {+ for _, r := range keyword {+ if !unicode.In(r, unicode.Latin) {+ return false}}- slices.SortStableFunc(normalized, func(a, b string) int {- return cmp.Compare(len(b), len(a))- })- return normalized+ return keyword != ""}// DropChunkStore deletes a chunk indexdiff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.goindex 807d7fb3a4..d33af456f5 100644--- a/internal/engine/elasticsearch/chunk_helpers_test.go+++ b/internal/engine/elasticsearch/chunk_helpers_test.go@@ -551,62 +551,46 @@ func TestElasticsearchGetChunkIDsPreservesOrderWithFallback(t *testing.T) {}}-func TestElasticsearchGetHighlightFallbackAndBoundaries(t *testing.T) {+func TestElasticsearchGetHighlightUsesRegex(t *testing.T) {engine := &Engine{}chunks := []map[string]interface{}{{- "_id": "fallback-id",- "content": "Alpha beta.\nbetamax soup. BETA again!",+ "_id": "full-text",+ "content_with_weight": "Alpha beta.\nbetamax soup. BETA again!",},+ {"id": "no-match", "content_with_weight": "Keep the complete text."},+ {"id": "missing"},}got := engine.GetHighlight(chunks, []string{"beta"}, "content_with_weight")assertEqual(t, got, map[string]string{- "fallback-id": "Alpha <em>beta</em>... <em>BETA</em> again",- })- if gotText := got["fallback-id"]; strings.Contains(gotText, "<em>beta</em>max") {- t.Fatalf("highlight matched inside a larger token: %q", gotText)- }-- gotLaterFallback := engine.GetHighlight([]map[string]interface{}{- {"_id": "first"},- {"_id": "second", "content": "Gamma beta."},- }, []string{"beta"}, "content_with_weight")- assertEqual(t, gotLaterFallback, map[string]string{- "second": "Gamma <em>beta</em>",- })-- gotMixedFallback := engine.GetHighlight([]map[string]interface{}{- {"id": "weighted", "content_with_weight": "Weighted beta."},- {"id": "plain", "content": "Plain beta."},- }, []string{"beta"}, "content_with_weight")- assertEqual(t, gotMixedFallback, map[string]string{- "plain": "Plain <em>beta</em>",- "weighted": "Weighted <em>beta</em>",+ "full-text": "Alpha <em>beta</em>.\n<em>betamax</em> soup. <em>BETA</em> again!",+ "no-match": "Keep the complete text.",})+}- gotEmptyFallback := engine.GetHighlight([]map[string]interface{}{- {"id": "empty-weighted", "content_with_weight": "", "content": "Empty fallback beta."},- }, []string{"beta"}, "content_with_weight")- assertEqual(t, gotEmptyFallback, map[string]string{- "empty-weighted": "Empty fallback <em>beta</em>",- })+func TestElasticsearchGetHighlightExpandsLatinStems(t *testing.T) {+ engine := &Engine{}+ chunks := []map[string]interface{}{+ {+ "_id": "latin",+ "content_with_weight": "Required coordinated isolation. Coordinación.",+ },+ }- gotInvalidFallback := engine.GetHighlight([]map[string]interface{}{- {"id": "invalid-weighted", "content_with_weight": nil, "content": "Invalid fallback beta."},- }, []string{"beta"}, "content_with_weight")- assertEqual(t, gotInvalidFallback, map[string]string{- "invalid-weighted": "Invalid fallback <em>beta</em>",+ got := engine.GetHighlight(chunks, []string{"requir", "coordin", "isol"}, "content_with_weight")+ assertEqual(t, got, map[string]string{+ "latin": "<em>Required</em> <em>coordinated</em> <em>isolation</em>. <em>Coordinación</em>.",})}-func TestElasticsearchGetHighlightPreservesExistingAndNonEnglish(t *testing.T) {+func TestElasticsearchGetHighlightPrefersLongKeywords(t *testing.T) {engine := &Engine{}gotExisting := engine.GetHighlight([]map[string]interface{}{{"id": "existing", "content_with_weight": "already <em>marked</em> text"},}, []string{"marked"}, "content_with_weight")- assertEqual(t, gotExisting, map[string]string{"existing": "already <em>marked</em> text"})+ assertEqual(t, gotExisting, map[string]string{"existing": "already <em><em>marked</em></em> text"})gotNonEnglish := engine.GetHighlight([]map[string]interface{}{{"id": "cn", "content_with_weight": "这是世界。你好世界"},@@ -625,8 +609,8 @@ func TestElasticsearchGetHighlightPreservesExistingAndNonEnglish(t *testing.T) {if got := engine.GetHighlight([]map[string]interface{}{{"id": "x"}}, []string{"x"}, "content_with_weight"); got == nil || len(got) != 0 {t.Fatalf("missing field highlight=%#v, want empty non-nil map", got)}- if got := engine.GetHighlight([]map[string]interface{}{{"id": "x", "content": "x"}}, nil, "content"); got == nil || len(got) != 0 {- t.Fatalf("empty keyword highlight=%#v, want empty non-nil map", got)+ if got := engine.GetHighlight([]map[string]interface{}{{"id": "x", "content": "x"}}, nil, "content"); !reflect.DeepEqual(got, map[string]string{"x": "x"}) {+ t.Fatalf("empty keyword highlight=%#v, want unchanged text", got)}}diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex 4d56e17012..18f66e3618 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -31,6 +31,8 @@ import ("strconv""strings""sync"+ "unicode"+ "unicode/utf8"infinity "github.com/infiniflow/infinity-go-sdk""go.uber.org/zap"@@ -1896,19 +1898,75 @@ func (e *Engine) GetChunkIDs(chunks []map[string]interface{}) []string {return ids}-// GetHighlight generates highlighted text snippets for search results.+// GetHighlight returns highlighted text for search results.// Matches keywords in text and wraps them with <em> tags.func (e *Engine) GetHighlight(chunks []map[string]interface{}, keywords []string, fieldName string) map[string]string {result := make(map[string]string)- if len(chunks) == 0 || len(keywords) == 0 {- return result+ if fieldName == "content_with_weight" && !hasInfinityHighlightField(chunks, fieldName) {+ fieldName = "content"}+ pattern := compileInfinityHighlightPattern(keywords)- // For Infinity, scores are already returned in search results (_score column)- // So GetScores just extracts scores from chunks, mimicking Python's approach+ for _, chunk := range chunks {+ id, ok := chunk["id"].(string)+ if !ok || id == "" {+ continue+ }+ txt, ok := chunk[fieldName].(string)+ if !ok {+ continue+ }+ if pattern != nil {+ txt = pattern.ReplaceAllStringFunc(txt, func(match string) string {+ return "<em>" + match + "</em>"+ })+ }+ result[id] = txt+ }return result}+func hasInfinityHighlightField(chunks []map[string]interface{}, fieldName string) bool {+ for _, chunk := range chunks {+ if _, ok := chunk[fieldName]; ok {+ return true+ }+ }+ return false+}++func compileInfinityHighlightPattern(keywords []string) *regexp.Regexp {+ nonEmpty := make([]string, 0, len(keywords))+ for _, keyword := range keywords {+ if keyword != "" {+ nonEmpty = append(nonEmpty, keyword)+ }+ }+ if len(nonEmpty) == 0 {+ return nil+ }+ sort.SliceStable(nonEmpty, func(i, j int) bool {+ return utf8.RuneCountInString(nonEmpty[i]) > utf8.RuneCountInString(nonEmpty[j])+ })+ parts := make([]string, len(nonEmpty))+ for i, keyword := range nonEmpty {+ parts[i] = regexp.QuoteMeta(keyword)+ if isLatinKeyword(keyword) {+ parts[i] += `\p{Latin}*`+ }+ }+ return regexp.MustCompile("(?i)" + strings.Join(parts, "|"))+}++func isLatinKeyword(keyword string) bool {+ for _, r := range keyword {+ if !unicode.In(r, unicode.Latin) {+ return false+ }+ }+ return keyword != ""+}+// KNNScores for Infinity - since Infinity normalizes scores during fusion,// we just need to return a result structure that GetScores can parse.// This matches Python's approach where Infinity doesn't use the two-pass KNN.diff --git a/internal/engine/infinity/chunk_highlight_test.go b/internal/engine/infinity/chunk_highlight_test.gonew file mode 100644index 0000000000..d458aff191--- /dev/null+++ b/internal/engine/infinity/chunk_highlight_test.go@@ -0,0 +1,74 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package infinity++import (+ "reflect"+ "testing"+)++func TestGetHighlightUsesRegex(t *testing.T) {+ engine := &Engine{}+ chunks := []map[string]interface{}{+ {"id": "full-text", "content_with_weight": "Alpha beta.\nbetamax soup. BETA again!"},+ {"id": "no-match", "content_with_weight": "Keep the complete text."},+ }++ got := engine.GetHighlight(chunks, []string{"beta"}, "content_with_weight")+ want := map[string]string{+ "full-text": "Alpha <em>beta</em>.\n<em>betamax</em> soup. <em>BETA</em> again!",+ "no-match": "Keep the complete text.",+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("GetHighlight() = %#v, want %#v", got, want)+ }+}++func TestGetHighlightExpandsLatinStems(t *testing.T) {+ engine := &Engine{}+ chunks := []map[string]interface{}{+ {+ "id": "latin",+ "content_with_weight": "Required coordinated isolation. Coordinación.",+ },+ }++ got := engine.GetHighlight(chunks, []string{"requir", "coordin", "isol"}, "content_with_weight")+ want := map[string]string{+ "latin": "<em>Required</em> <em>coordinated</em> <em>isolation</em>. <em>Coordinación</em>.",+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("GetHighlight() = %#v, want %#v", got, want)+ }+}++func TestGetHighlightPrefersLongKeywordsAndFallsBackToContent(t *testing.T) {+ engine := &Engine{}+ chunks := []map[string]interface{}{+ {"id": "overlap", "content": "世界和世"},+ }++ got := engine.GetHighlight(chunks, []string{"世", "世界", ""}, "content_with_weight")+ want := map[string]string{"overlap": "<em>世界</em>和<em>世</em>"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("GetHighlight() = %#v, want %#v", got, want)+ }++ got = engine.GetHighlight(chunks, nil, "content_with_weight")+ want = map[string]string{"overlap": "世界和世"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("GetHighlight() without keywords = %#v, want %#v", got, want)+ }+}diff --git a/internal/service/nlp/retrieval.go b/internal/service/nlp/retrieval.goindex 147fba4ebd..25ccbdac0e 100644--- a/internal/service/nlp/retrieval.go+++ b/internal/service/nlp/retrieval.go@@ -144,6 +144,7 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)RankFeature: *req.RankFeature,EmbeddingModel: req.EmbeddingModel,VectorSimilarityWeight: req.VectorSimilarityWeight,+ Highlight: req.Highlight,AllowDenseFallback: req.AllowDenseFallback,Filter: req.Filter,}@@ -332,15 +333,9 @@ func (s *RetrievalService) Retrieval(ctx context.Context, req *RetrievalRequest)resultChunk["vector"] = zeroVector}- highlightEnabled := false- if req.Highlight != nil && *req.Highlight {- highlightEnabled = true- }- if highlightEnabled && searchResult.Highlight != nil {+ if searchResult.Highlight != nil {if highlightText, ok := searchResult.Highlight[chunkID]; ok {- resultChunk["highlight"] = RemoveRedundantSpaces(highlightText)- } else if contentWithWeight, ok := chunk["content_with_weight"].(string); ok {- resultChunk["highlight"] = RemoveRedundantSpaces(contentWithWeight)+ resultChunk["highlight"] = highlightText}}filteredChunks = append(filteredChunks, resultChunk)@@ -800,8 +795,8 @@ func (s *RetrievalService) Search(ctx context.Context, req *RetrievalSearchRequeaggregation := s.docEngine.GetAggregation(searchResult.Chunks, "docnm_kwd")// Build Highlight using GetHighlight- var highlight map[string]string- if len(keywordsList) > 0 {+ highlight := make(map[string]string)+ if *req.Highlight {highlight = s.docEngine.GetHighlight(searchResult.Chunks, keywordsList, "content_with_weight")}diff --git a/internal/service/nlp/retrieval_test.go b/internal/service/nlp/retrieval_test.goindex 3a3c3d4192..6e3542d6a7 100644--- a/internal/service/nlp/retrieval_test.go+++ b/internal/service/nlp/retrieval_test.go@@ -68,12 +68,60 @@ func TestRetrievalUsesRerankCandidatesCountAsCandidateSet(t *testing.T) {t.Fatalf("must_not filter = %#v", filters["must_not"])}}+ if engine.highlightCalls != 0 {+ t.Fatalf("GetHighlight calls = %d with highlighting disabled, want 0", engine.highlightCalls)+ }+}++func TestRetrievalReturnsRegexHighlightWithoutFallbackOrCleanup(t *testing.T) {+ oldQueryBuilder := globalQueryBuilder+ globalQueryBuilder = NewQueryBuilder()+ defer func() { globalQueryBuilder = oldQueryBuilder }()++ rows := []map[string]interface{}{+ {"id": "highlighted", "content_ltks": "alpha", "content_with_weight": "alpha élève", "_score": 0.9},+ {"id": "unmatched", "content_ltks": "alpha", "content_with_weight": "plain content", "_score": 0.8},+ }+ docEngine := &retrievalCountEngine{+ rows: rows,+ highlights: map[string]string{"highlighted": "<em>alpha</em> élève"},+ }+ service := NewRetrievalService(docEngine, &dao.DocumentDAO{})+ top := 10+ threshold := 0.5+ vectorWeight := 1.0+ highlight := true++ result, err := service.Retrieval(t.Context(), &RetrievalRequest{+ Question: "alpha",+ TenantIDs: []string{"tenant-1"},+ Page: 1,+ PageSize: 2,+ KNNTopK: &top,+ SimilarityThreshold: &threshold,+ VectorSimilarityWeight: &vectorWeight,+ Highlight: &highlight,+ })+ if err != nil {+ t.Fatalf("Retrieval failed: %v", err)+ }+ if docEngine.highlightCalls != 1 {+ t.Fatalf("GetHighlight calls = %d, want 1", docEngine.highlightCalls)+ }+ if got := result.Chunks[0]["highlight"]; got != "<em>alpha</em> élève" {+ t.Fatalf("highlight = %q, want spacing preserved", got)+ }+ if _, ok := result.Chunks[1]["highlight"]; ok {+ t.Fatalf("unmatched chunk received fallback highlight: %#v", result.Chunks[1])+ }}type retrievalCountEngine struct {- rows []map[string]interface{}- searchLimits []int- searchFilters []map[string]interface{}+ rows []map[string]interface{}+ searchLimits []int+ searchFilters []map[string]interface{}+ highlights map[string]string+ highlightCalls int}func (e *retrievalCountEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {@@ -184,7 +232,8 @@ func (e *retrievalCountEngine) GetAggregation([]map[string]interface{}, string)return nil}func (e *retrievalCountEngine) GetHighlight([]map[string]interface{}, []string, string) map[string]string {- return nil+ e.highlightCalls+++ return e.highlights}func (e *retrievalCountEngine) RunSQL(context.Context, string, string, []string, string) ([]map[string]interface{}, error) {return nil, nil