Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(agent,ingestion): carry the grep/bm25 contracts and index declared titles (#20542)

Zhichang Yu2400ca8eb514
diff --git a/internal/agent/runtime/retrieval.go b/internal/agent/runtime/retrieval.go
index ec9722fde9..098224c571 100644
--- a/internal/agent/runtime/retrieval.go
+++ b/internal/agent/runtime/retrieval.go
@@ -195,6 +195,11 @@ var ErrGrepServiceMissing = errors.New(
"grep service not registered — call runtime.SetGrepService(...) at boot",
)
+// ErrBm25ServiceMissing is returned when no Bm25Service has been registered.
+var ErrBm25ServiceMissing = errors.New(
+ "bm25 service not registered — call runtime.SetBm25Service(...) at boot",
+)
+
// ErrRegexpNotSupported is returned when the underlying doc engine does not
// implement regex matching on chunk content (e.g. Infinity).
var ErrRegexpNotSupported = errors.New(
@@ -209,11 +214,6 @@ var ErrRegexpNotSupported = errors.New(
// surfacing an opaque backend error.
var ErrRegexpPushdown = errors.New("grep_chunks: regexp pushdown failed")
-// ErrBm25ServiceMissing is returned when no Bm25Service has been registered.
-var ErrBm25ServiceMissing = errors.New(
- "bm25 service not registered — call runtime.SetBm25Service(...) at boot",
-)
-
var (
retrievalServiceMu sync.RWMutex
retrievalServiceImpl RetrievalService = stubRetrievalService{}
diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.go
index b80bd0393f..3c12b1a4c4 100644
--- a/internal/agent/tool/retrieval_nlp.go
+++ b/internal/agent/tool/retrieval_nlp.go
@@ -386,6 +386,17 @@ func nlpRequestFromRetrieval(
"must_not": map[string]interface{}{"exists": "compile_kwd"},
}
}
+ // Restrict to ordinary document text chunks with the SAME filter grep_chunks
+ // applies: available_int=1 (must_not available_int<1 semantics so chunks
+ // whose available_int is absent still pass) and must_not exists compile_kwd
+ // to exclude knowledge-compiled products. Setting available_int explicitly
+ // here matches grep exactly (both go through buildBoolQueryFromCondition).
+ if req.OnlyOriginalText {
+ nlpReq.Filter = map[string]interface{}{
+ "available_int": 1,
+ "must_not": map[string]interface{}{"exists": "compile_kwd"},
+ }
+ }
return nlpReq
}
diff --git a/internal/agent/tool/retrieval_service.go b/internal/agent/tool/retrieval_service.go
index 527d9652e7..f39c63f96d 100644
--- a/internal/agent/tool/retrieval_service.go
+++ b/internal/agent/tool/retrieval_service.go
@@ -15,9 +15,9 @@
//
// Retrieval contracts live in internal/agent/runtime (the engine-agnostic
-// package the canvas agent depends on). This file re-exports them under the
-// historical tool.XXX names so the canvas tool package keeps its public API
-// stable without owning a second copy.
+// package both the canvas agent and the smart-reasoning agent depend on). This
+// file re-exports them under the historical tool.XXX names so the canvas tool
+// package keeps its public API stable without owning a second copy.
package tool
import (
@@ -28,9 +28,11 @@ import (
type (
RetrievalChunk = runtime.RetrievalChunk
RetrievalRequest = runtime.RetrievalRequest
+ GrepRequest = runtime.GrepRequest
RetrievalService = runtime.RetrievalService
MemoryRetrievalService = runtime.MemoryRetrievalService
KGRetrievalService = runtime.KGRetrievalService
+ GrepService = runtime.GrepService
)
// ErrRetrievalServiceMissing is declared in retrieval.go so callers and the
@@ -39,6 +41,8 @@ var (
ErrRetrievalServiceMissing = runtime.ErrRetrievalServiceMissing
ErrMemoryRetrievalServiceMissing = runtime.ErrMemoryRetrievalServiceMissing
ErrKGRetrievalServiceMissing = runtime.ErrKGRetrievalServiceMissing
+ ErrGrepServiceMissing = runtime.ErrGrepServiceMissing
+ ErrRegexpNotSupported = runtime.ErrRegexpNotSupported
)
func SetRetrievalService(svc RetrievalService) { runtime.SetRetrievalService(svc) }
@@ -50,6 +54,15 @@ func GetMemoryRetrievalService() MemoryRetrievalService { return runtime.GetM
func SetKGRetrievalService(svc KGRetrievalService) { runtime.SetKGRetrievalService(svc) }
func GetKGRetrievalService() KGRetrievalService { return runtime.GetKGRetrievalService() }
+func SetGrepService(svc GrepService) { runtime.SetGrepService(svc) }
+func GetGrepService() GrepService { return runtime.GetGrepService() }
+
+// Bm25Service mirrors runtime.Bm25Service for injection outside this package.
+type Bm25Service = runtime.Bm25Service
+
+func SetBm25Service(svc Bm25Service) { runtime.SetBm25Service(svc) }
+func GetBm25Service() Bm25Service { return runtime.GetBm25Service() }
+
// SetSimpleRetrievalService installs deterministic synthetic retrieval for
// tests and local demos.
func SetSimpleRetrievalService() { runtime.SetSimpleRetrievalService() }
diff --git a/internal/agentic_rag/helper.go b/internal/agentic_rag/helper.go
index 0a56cfd455..ce6f696914 100644
--- a/internal/agentic_rag/helper.go
+++ b/internal/agentic_rag/helper.go
@@ -450,20 +450,36 @@ func regexMatchSpan(re *regexp.Regexp, content string) (int, int, bool) {
// termMatchSpan is regexMatchSpan's literal-term counterpart: over every
// case-insensitive occurrence of every term it takes the earliest start and the
// latest end, so multi-keyword queries centre the window on the whole hit set.
+//
+// It scans one lowercased copy of the content instead of compiling an (?i)
+// regex per term — measured ~30x faster on the locate path, where this runs once
+// per located chunk — and lowercases the term so the scan stays case-insensitive
+// regardless of how the caller cased it. The offsets therefore come from the
+// lowercased copy: for the few characters whose lowercase form has a different
+// byte length (U+0130, U+1E9E) they can land a byte or two off in the original,
+// which shifts a snippet window but never hides the match.
func termMatchSpan(terms []string, content string) (int, int, bool) {
+ lower := strings.ToLower(content)
first, last := -1, -1
for _, t := range terms {
if t == "" {
continue
}
- re := regexp.MustCompile("(?i)" + regexp.QuoteMeta(t))
- for _, loc := range re.FindAllStringIndex(content, -1) {
- if first < 0 || loc[0] < first {
- first = loc[0]
+ t = strings.ToLower(t)
+ from := 0
+ for {
+ i := strings.Index(lower[from:], t)
+ if i < 0 {
+ break
+ }
+ lo, hi := from+i, from+i+len(t)
+ if first < 0 || lo < first {
+ first = lo
}
- if loc[1] > last {
- last = loc[1]
+ if hi > last {
+ last = hi
}
+ from = hi
}
}
if first < 0 {
diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.go
index e3580e6502..4560559009 100644
--- a/internal/engine/nats/nats.go
+++ b/internal/engine/nats/nats.go
@@ -126,7 +126,7 @@ func (n *NatsEngine) Init() error {
}
common.Info("NATS stream updated with current config (discard=new, larger limits)")
} else {
- common.Info("NATS stream create successfully")
+ common.Info(fmt.Sprintf("NATS stream create successfully at %s", natsURL))
}
common.Info(fmt.Sprintf("NATS stream RAGFLOW_TASKS ready at %s", natsURL))
diff --git a/internal/engine/oceanbase/search.go b/internal/engine/oceanbase/search.go
index a1f0aa8376..9e0943fc81 100644
--- a/internal/engine/oceanbase/search.go
+++ b/internal/engine/oceanbase/search.go
@@ -359,7 +359,7 @@ func (e *Engine) fullTextFields(kind string, text *types.MatchTextExpr) ([]strin
if e.flags.searchOriginalContent {
specifications = []string{"docnm_kwd^10", "content_with_weight", "important_tks^20", "question_tks^20"}
} else {
- specifications = []string{"title_tks^10", "title_sm_tks^5", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}
+ specifications = []string{"title_tks^2", "title_sm_tks^2", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}
}
}
fields := make([]string, 0, len(specifications))
@@ -794,7 +794,7 @@ func tokenizedDBMSFields(kind string) []string {
case "skill":
return []string{"name_tks^10", "tags_tks^5", "description_tks^3", "content_tks"}
default:
- return []string{"title_tks^10", "title_sm_tks^5", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}
+ return []string{"title_tks^2", "title_sm_tks^2", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}
}
}
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index da393ae60f..5694a57e01 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -368,6 +368,9 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map
zap.Int("input_chunks", len(chunks)),
)
titleStem := titleExtRE.ReplaceAllString(name, "")
+ if toks := declaredTitleTokens(chunks); len(toks) > 0 {
+ titleStem = strings.TrimSpace(titleStem + " " + strings.Join(toks, " "))
+ }
// chunk_order_int is the position of the chunk in the (post-filter) reading
// sequence. It is set unconditionally on every surviving chunk so that all
@@ -754,6 +757,151 @@ func cloneTokenizerChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {
return out
}
+// declaredTitleTokens returns the tokens of the title a document declares about itself, when its
+// header block carries one ("title: Pragyan Ojha", "name: Lesley Manyathela", "fullname: ...").
+//
+// Why it exists: a corpus whose files are named by id ("93372.md") otherwise leaves title_tks
+// holding nothing but the id, while the retriever weights title_tks on a par with content_ltks -
+// so the declared title is a ranking signal that costs nothing to index. Label words, the word
+// "wikipedia", English stopwords and pure numbers are dropped: they are noise that would match
+// unrelated queries once the field is weighted.
+//
+// Only the first non-empty chunk is inspected, because the header block (when present at all)
+// belongs to the head of the document.
+//
+// The extraction is script-agnostic. A word that is not pure ASCII — Chinese, Japanese, Korean,
+// Cyrillic, Arabic, Greek, Hebrew, Thai, or Latin carrying diacritics — is kept whole and handed
+// to the analyzer downstream, which is what segments it. Splitting such a word here, on ASCII
+// boundaries, is the older behaviour and it dissolved the whole title into nothing.
+// Tokens that ARE pure ASCII keep the older treatment: englishStopwords are dropped (the
+// tokenizer keeps function words such as "the", "for" and "and" as-is — verified via the ES
+// _analyze API on title_tks and via tokenizer.Tokenize — so they have to be filtered here), and
+// so are the label words and pure numbers. The stopword list stays English-only on purpose: a
+// multilingual one would have to be invented, and getting a language wrong silently deletes real
+// content words from that language's titles.
+// This replaced an earlier length heuristic (keep only tokens >= 5 chars) which also removed real
+// content words such as "icc", "cup", "john", "star", "bros".
+var englishStopwords = map[string]bool{
+ "a": true, "an": true, "and": true, "are": true, "as": true, "at": true, "be": true, "been": true,
+ "but": true, "by": true, "d": true, "for": true, "from": true, "had": true, "has": true, "have": true,
+ "he": true, "her": true, "him": true, "his": true, "i": true, "if": true, "in": true, "into": true,
+ "is": true, "it": true, "its": true, "ll": true, "m": true, "me": true, "my": true, "no": true,
+ "not": true, "of": true, "on": true, "or": true, "our": true, "re": true, "s": true, "she": true,
+ "so": true, "t": true, "that": true, "the": true, "their": true, "them": true, "then": true,
+ "there": true, "these": true, "they": true, "this": true, "to": true, "too": true, "up": true,
+ "us": true, "ve": true, "was": true, "we": true, "were": true, "what": true, "when": true,
+ "where": true, "which": true, "who": true, "will": true, "with": true, "y": true,
+ "you": true, "your": true,
+}
+
+// asciiOnly reports whether s is made of ASCII bytes alone. A pure-ASCII string is the only thing
+// ASCII decoding produces, so a byte scan decides it.
+func asciiOnly(s string) bool {
+ for i := 0; i < len(s); i++ {
+ if s[i] >= utf8.RuneSelf {
+ return false
+ }
+ }
+ return true
+}
+
+// allDigits reports whether s is non-empty and made only of decimal digits, in any script.
+func allDigits(s string) bool {
+ if s == "" {
+ return false
+ }
+ for _, r := range s {
+ if !unicode.IsDigit(r) {
+ return false
+ }
+ }
+ return true
+}
+
+// trimWordEdges drops leading and trailing runes that are neither letters nor digits, so the
+// punctuation a title tends to carry — 《三国演义》, «Война и мир», (1917), a trailing comma — does
+// not reach the token stream.
+func trimWordEdges(s string) string {
+ return strings.TrimFunc(s, func(r rune) bool {
+ return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+ })
+}
+
+func declaredTitleTokens(chunks []schema.ChunkDoc) []string {
+ const (
+ headerLines = 20
+ maxTokens = 20
+ )
+ var values []string
+ for i := range chunks {
+ head := chunks[i].Text
+ if head == "" {
+ continue
+ }
+ lines := strings.Split(head, "\n")
+ if len(lines) > headerLines {
+ lines = lines[:headerLines]
+ }
+ for _, line := range lines {
+ l := strings.ToLower(strings.TrimSpace(line))
+ for _, key := range []string{"title:", "name:", "fullname:"} {
+ if strings.HasPrefix(l, key) {
+ values = append(values, l[len(key):])
+ }
+ }
+ }
+ break
+ }
+ if len(values) == 0 {
+ return nil
+ }
+ skip := map[string]bool{"title": true, "name": true, "fullname": true, "wikipedia": true}
+ seen := map[string]bool{}
+ var out []string
+ // keep records one accepted token and reports whether the cap is now reached.
+ keep := func(token string) bool {
+ if seen[token] {
+ return false
+ }
+ seen[token] = true
+ out = append(out, token)
+ return len(out) >= maxTokens
+ }
+ for _, v := range values {
+ // Split on whitespace, NOT on ASCII boundaries. A word in another script is a word, and
+ // the analyzer downstream segments it (RAGFlow's tokenizer handles CJK); the
+ // ASCII-boundary splitter dissolved such a word into fragments of length zero.
+ for _, word := range strings.Fields(v) {
+ word = trimWordEdges(word)
+ if word == "" {
+ continue
+ }
+ if !asciiOnly(word) {
+ // Pure numbers stay out of the title channel whichever script writes them: the
+ // corpus file stem is numeric, so a numeric title token matches every document.
+ if !allDigits(word) && keep(word) {
+ return out
+ }
+ continue
+ }
+ for _, w := range strings.FieldsFunc(word, func(r rune) bool {
+ return !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9')
+ }) {
+ // Pure numbers stay out (years, ids, and the corpus file stem are all numeric
+ // noise for the title channel); everything else is kept unless it is an English
+ // stopword, a label or a duplicate.
+ if englishStopwords[w] || skip[w] || allDigits(w) {
+ continue
+ }
+ if keep(w) {
+ return out
+ }
+ }
+ }
+ }
+ return out
+}
+
// tokenizeChunks annotates each chunk with title_tks, content_ltks,
// and (when applicable) question_tks / important_tks / summary fields.
// Mirrors python tokenizer.py:130-185 and rag/nlp/__init__.py tokenize() /
diff --git a/internal/ingestion/component/tokenizer_title_test.go b/internal/ingestion/component/tokenizer_title_test.go
new file mode 100644
index 0000000000..cb69dd9cac
--- /dev/null
+++ b/internal/ingestion/component/tokenizer_title_test.go
@@ -0,0 +1,124 @@
+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package component
+
+import (
+ "reflect"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// A corpus named by id ("93372.md") leaves title_tks with the id alone; when the document
+// declares a title in its header block that title must reach the field, filtered — in whatever
+// script the document is written.
+func TestDeclaredTitleTokens(t *testing.T) {
+ header := "Document 44250\nSource URL: https://www.sportskeeda.com/player/pp-ojha\n" +
+ "title: Pragyan Ojha\ndate: 2025-01-01\nPersonal Information\n| Full Name | Pragyan Prayash Ojha |\n"
+ cases := []struct {
+ name string
+ chunks []schema.ChunkDoc
+ want []string
+ }{
+ {
+ // Only the declared-title lines feed the field ("Source URL", "date", "| Full Name |"
+ // are not title lines) and label words are dropped. Short content words are KEPT:
+ // the >=5 length rule was replaced by English stopword filtering (2026-09-22), because
+ // the length rule also removed real words such as "icc", "cup", "john", "star".
+ name: "declared title is tokenised, labels dropped, short words kept",
+ chunks: []schema.ChunkDoc{{Text: header}},
+ want: []string{"pragyan", "ojha"},
+ },
+ {
+ name: "title/name/fullname are all read, duplicates and wikipedia dropped",
+ chunks: []schema.ChunkDoc{{Text: "title: Lesley Manyathela - Wikipedia\nname: Lesley Manyathela\nfullname: Lesley Phuti Manyathela\n"}},
+ want: []string{"lesley", "manyathela", "phuti"},
+ },
+ {
+ name: "no header block means no title",
+ chunks: []schema.ChunkDoc{{Text: "AW: I began teaching at Zama Dance School in January 2008.\n"}},
+ want: nil,
+ },
+ {
+ // Pure numbers are still dropped (the corpus file stem is numeric, so a
+ // numeric title token would match every document), but the short words that
+ // the old >=5 rule removed are now kept.
+ name: "numeric tokens dropped, short words kept",
+ chunks: []schema.ChunkDoc{{Text: "title: 1917 Ok Go\n"}},
+ want: []string{"ok", "go"},
+ },
+ {
+ // A declared title is kept whatever script writes it. The ASCII-boundary
+ // splitter this replaced produced nothing at all for these titles, which
+ // left the title channel empty for every non-Latin document.
+ name: "Chinese title is kept whole",
+ chunks: []schema.ChunkDoc{{Text: "title: 三国演义\n"}},
+ want: []string{"三国演义"},
+ },
+ {
+ name: "Chinese title keeps its book-title marks out and the date line stays untouched",
+ chunks: []schema.ChunkDoc{{Text: "title: 《三国演义》\ndate: 2025-01-01\n"}},
+ want: []string{"三国演义"},
+ },
+ {
+ name: "a localised wikipedia suffix is a separate word, not a label",
+ chunks: []schema.ChunkDoc{{Text: "title: 《三国演义》 - 维基百科\n"}},
+ want: []string{"三国演义", "维基百科"},
+ },
+ {
+ name: "Japanese and Korean titles are kept whole",
+ chunks: []schema.ChunkDoc{{Text: "name: 吾輩は猫である\nfullname: 태백산맥\n"}},
+ want: []string{"吾輩は猫である", "태백산맥"},
+ },
+ {
+ // No multilingual stopword list exists, so a non-English function word
+ // survives: "и" is here on purpose, to pin that residual.
+ name: "Cyrillic title is split on whitespace and quoted form is trimmed",
+ chunks: []schema.ChunkDoc{{Text: "title: «Война и мир»\n"}},
+ want: []string{"война", "и", "мир"},
+ },
+ {
+ // Diacritics used to be ASCII-boundary separators: "José" came out as
+ // "jos"+"é" (and "Müller" as "m"+"ller", with "m" then dropped as a
+ // stopword). The whole accented word is what belongs in the index.
+ name: "accented Latin words survive intact",
+ chunks: []schema.ChunkDoc{{Text: "fullname: José Müller\n"}},
+ want: []string{"josé", "müller"},
+ },
+ {
+ name: "a mixed-script title keeps both sides and still drops English stopwords",
+ chunks: []schema.ChunkDoc{{Text: "title: 三国演义 Romance of the Three Kingdoms\n"}},
+ want: []string{"三国演义", "romance", "three", "kingdoms"},
+ },
+ {
+ name: "pure numbers are dropped in any script",
+ chunks: []schema.ChunkDoc{{Text: "title: ٣٤٥\n"}},
+ want: nil,
+ },
+ {
+ name: "empty chunks are skipped safely",
+ chunks: []schema.ChunkDoc{{}, {Text: ""}},
+ want: nil,
+ },
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ got := declaredTitleTokens(c.chunks)
+ if !reflect.DeepEqual(got, c.want) {
+ t.Errorf("declaredTitleTokens() = %v, want %v", got, c.want)
+ }
+ })
+ }
+}