RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(agent,ingestion): carry the grep/bm25 contracts and index declared titles (#20542)
Zhichang Yu2400ca8eb514
diff --git a/internal/agent/runtime/retrieval.go b/internal/agent/runtime/retrieval.goindex ec9722fde9..098224c571 100644--- a/internal/agent/runtime/retrieval.go+++ b/internal/agent/runtime/retrieval.go@@ -195,6 +195,11 @@ var ErrGrepServiceMissing = errors.New("grep service not registered — call runtime.SetGrepService(...) at boot",)+// ErrBm25ServiceMissing is returned when no Bm25Service has been registered.+var ErrBm25ServiceMissing = errors.New(+ "bm25 service not registered — call runtime.SetBm25Service(...) at boot",+)+// ErrRegexpNotSupported is returned when the underlying doc engine does not// implement regex matching on chunk content (e.g. Infinity).var ErrRegexpNotSupported = errors.New(@@ -209,11 +214,6 @@ var ErrRegexpNotSupported = errors.New(// surfacing an opaque backend error.var ErrRegexpPushdown = errors.New("grep_chunks: regexp pushdown failed")-// ErrBm25ServiceMissing is returned when no Bm25Service has been registered.-var ErrBm25ServiceMissing = errors.New(- "bm25 service not registered — call runtime.SetBm25Service(...) at boot",-)-var (retrievalServiceMu sync.RWMutexretrievalServiceImpl RetrievalService = stubRetrievalService{}diff --git a/internal/agent/tool/retrieval_nlp.go b/internal/agent/tool/retrieval_nlp.goindex b80bd0393f..3c12b1a4c4 100644--- a/internal/agent/tool/retrieval_nlp.go+++ b/internal/agent/tool/retrieval_nlp.go@@ -386,6 +386,17 @@ func nlpRequestFromRetrieval("must_not": map[string]interface{}{"exists": "compile_kwd"},}}+ // Restrict to ordinary document text chunks with the SAME filter grep_chunks+ // applies: available_int=1 (must_not available_int<1 semantics so chunks+ // whose available_int is absent still pass) and must_not exists compile_kwd+ // to exclude knowledge-compiled products. Setting available_int explicitly+ // here matches grep exactly (both go through buildBoolQueryFromCondition).+ if req.OnlyOriginalText {+ nlpReq.Filter = map[string]interface{}{+ "available_int": 1,+ "must_not": map[string]interface{}{"exists": "compile_kwd"},+ }+ }return nlpReq}diff --git a/internal/agent/tool/retrieval_service.go b/internal/agent/tool/retrieval_service.goindex 527d9652e7..f39c63f96d 100644--- a/internal/agent/tool/retrieval_service.go+++ b/internal/agent/tool/retrieval_service.go@@ -15,9 +15,9 @@//// Retrieval contracts live in internal/agent/runtime (the engine-agnostic-// package the canvas agent depends on). This file re-exports them under the-// historical tool.XXX names so the canvas tool package keeps its public API-// stable without owning a second copy.+// package both the canvas agent and the smart-reasoning agent depend on). This+// file re-exports them under the historical tool.XXX names so the canvas tool+// package keeps its public API stable without owning a second copy.package toolimport (@@ -28,9 +28,11 @@ import (type (RetrievalChunk = runtime.RetrievalChunkRetrievalRequest = runtime.RetrievalRequest+ GrepRequest = runtime.GrepRequestRetrievalService = runtime.RetrievalServiceMemoryRetrievalService = runtime.MemoryRetrievalServiceKGRetrievalService = runtime.KGRetrievalService+ GrepService = runtime.GrepService)// ErrRetrievalServiceMissing is declared in retrieval.go so callers and the@@ -39,6 +41,8 @@ var (ErrRetrievalServiceMissing = runtime.ErrRetrievalServiceMissingErrMemoryRetrievalServiceMissing = runtime.ErrMemoryRetrievalServiceMissingErrKGRetrievalServiceMissing = runtime.ErrKGRetrievalServiceMissing+ ErrGrepServiceMissing = runtime.ErrGrepServiceMissing+ ErrRegexpNotSupported = runtime.ErrRegexpNotSupported)func SetRetrievalService(svc RetrievalService) { runtime.SetRetrievalService(svc) }@@ -50,6 +54,15 @@ func GetMemoryRetrievalService() MemoryRetrievalService { return runtime.GetMfunc SetKGRetrievalService(svc KGRetrievalService) { runtime.SetKGRetrievalService(svc) }func GetKGRetrievalService() KGRetrievalService { return runtime.GetKGRetrievalService() }+func SetGrepService(svc GrepService) { runtime.SetGrepService(svc) }+func GetGrepService() GrepService { return runtime.GetGrepService() }++// Bm25Service mirrors runtime.Bm25Service for injection outside this package.+type Bm25Service = runtime.Bm25Service++func SetBm25Service(svc Bm25Service) { runtime.SetBm25Service(svc) }+func GetBm25Service() Bm25Service { return runtime.GetBm25Service() }+// SetSimpleRetrievalService installs deterministic synthetic retrieval for// tests and local demos.func SetSimpleRetrievalService() { runtime.SetSimpleRetrievalService() }diff --git a/internal/agentic_rag/helper.go b/internal/agentic_rag/helper.goindex 0a56cfd455..ce6f696914 100644--- a/internal/agentic_rag/helper.go+++ b/internal/agentic_rag/helper.go@@ -450,20 +450,36 @@ func regexMatchSpan(re *regexp.Regexp, content string) (int, int, bool) {// termMatchSpan is regexMatchSpan's literal-term counterpart: over every// case-insensitive occurrence of every term it takes the earliest start and the// latest end, so multi-keyword queries centre the window on the whole hit set.+//+// It scans one lowercased copy of the content instead of compiling an (?i)+// regex per term — measured ~30x faster on the locate path, where this runs once+// per located chunk — and lowercases the term so the scan stays case-insensitive+// regardless of how the caller cased it. The offsets therefore come from the+// lowercased copy: for the few characters whose lowercase form has a different+// byte length (U+0130, U+1E9E) they can land a byte or two off in the original,+// which shifts a snippet window but never hides the match.func termMatchSpan(terms []string, content string) (int, int, bool) {+ lower := strings.ToLower(content)first, last := -1, -1for _, t := range terms {if t == "" {continue}- re := regexp.MustCompile("(?i)" + regexp.QuoteMeta(t))- for _, loc := range re.FindAllStringIndex(content, -1) {- if first < 0 || loc[0] < first {- first = loc[0]+ t = strings.ToLower(t)+ from := 0+ for {+ i := strings.Index(lower[from:], t)+ if i < 0 {+ break+ }+ lo, hi := from+i, from+i+len(t)+ if first < 0 || lo < first {+ first = lo}- if loc[1] > last {- last = loc[1]+ if hi > last {+ last = hi}+ from = hi}}if first < 0 {diff --git a/internal/engine/nats/nats.go b/internal/engine/nats/nats.goindex e3580e6502..4560559009 100644--- a/internal/engine/nats/nats.go+++ b/internal/engine/nats/nats.go@@ -126,7 +126,7 @@ func (n *NatsEngine) Init() error {}common.Info("NATS stream updated with current config (discard=new, larger limits)")} else {- common.Info("NATS stream create successfully")+ common.Info(fmt.Sprintf("NATS stream create successfully at %s", natsURL))}common.Info(fmt.Sprintf("NATS stream RAGFLOW_TASKS ready at %s", natsURL))diff --git a/internal/engine/oceanbase/search.go b/internal/engine/oceanbase/search.goindex a1f0aa8376..9e0943fc81 100644--- a/internal/engine/oceanbase/search.go+++ b/internal/engine/oceanbase/search.go@@ -359,7 +359,7 @@ func (e *Engine) fullTextFields(kind string, text *types.MatchTextExpr) ([]strinif e.flags.searchOriginalContent {specifications = []string{"docnm_kwd^10", "content_with_weight", "important_tks^20", "question_tks^20"}} else {- specifications = []string{"title_tks^10", "title_sm_tks^5", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}+ specifications = []string{"title_tks^2", "title_sm_tks^2", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}}}fields := make([]string, 0, len(specifications))@@ -794,7 +794,7 @@ func tokenizedDBMSFields(kind string) []string {case "skill":return []string{"name_tks^10", "tags_tks^5", "description_tks^3", "content_tks"}default:- return []string{"title_tks^10", "title_sm_tks^5", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}+ return []string{"title_tks^2", "title_sm_tks^2", "important_tks^20", "question_tks^20", "content_ltks^2", "content_sm_ltks"}}}diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex da393ae60f..5694a57e01 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -368,6 +368,9 @@ func (c *TokenizerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs mapzap.Int("input_chunks", len(chunks)),)titleStem := titleExtRE.ReplaceAllString(name, "")+ if toks := declaredTitleTokens(chunks); len(toks) > 0 {+ titleStem = strings.TrimSpace(titleStem + " " + strings.Join(toks, " "))+ }// chunk_order_int is the position of the chunk in the (post-filter) reading// sequence. It is set unconditionally on every surviving chunk so that all@@ -754,6 +757,151 @@ func cloneTokenizerChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {return out}+// declaredTitleTokens returns the tokens of the title a document declares about itself, when its+// header block carries one ("title: Pragyan Ojha", "name: Lesley Manyathela", "fullname: ...").+//+// Why it exists: a corpus whose files are named by id ("93372.md") otherwise leaves title_tks+// holding nothing but the id, while the retriever weights title_tks on a par with content_ltks -+// so the declared title is a ranking signal that costs nothing to index. Label words, the word+// "wikipedia", English stopwords and pure numbers are dropped: they are noise that would match+// unrelated queries once the field is weighted.+//+// Only the first non-empty chunk is inspected, because the header block (when present at all)+// belongs to the head of the document.+//+// The extraction is script-agnostic. A word that is not pure ASCII — Chinese, Japanese, Korean,+// Cyrillic, Arabic, Greek, Hebrew, Thai, or Latin carrying diacritics — is kept whole and handed+// to the analyzer downstream, which is what segments it. Splitting such a word here, on ASCII+// boundaries, is the older behaviour and it dissolved the whole title into nothing.+// Tokens that ARE pure ASCII keep the older treatment: englishStopwords are dropped (the+// tokenizer keeps function words such as "the", "for" and "and" as-is — verified via the ES+// _analyze API on title_tks and via tokenizer.Tokenize — so they have to be filtered here), and+// so are the label words and pure numbers. The stopword list stays English-only on purpose: a+// multilingual one would have to be invented, and getting a language wrong silently deletes real+// content words from that language's titles.+// This replaced an earlier length heuristic (keep only tokens >= 5 chars) which also removed real+// content words such as "icc", "cup", "john", "star", "bros".+var englishStopwords = map[string]bool{+ "a": true, "an": true, "and": true, "are": true, "as": true, "at": true, "be": true, "been": true,+ "but": true, "by": true, "d": true, "for": true, "from": true, "had": true, "has": true, "have": true,+ "he": true, "her": true, "him": true, "his": true, "i": true, "if": true, "in": true, "into": true,+ "is": true, "it": true, "its": true, "ll": true, "m": true, "me": true, "my": true, "no": true,+ "not": true, "of": true, "on": true, "or": true, "our": true, "re": true, "s": true, "she": true,+ "so": true, "t": true, "that": true, "the": true, "their": true, "them": true, "then": true,+ "there": true, "these": true, "they": true, "this": true, "to": true, "too": true, "up": true,+ "us": true, "ve": true, "was": true, "we": true, "were": true, "what": true, "when": true,+ "where": true, "which": true, "who": true, "will": true, "with": true, "y": true,+ "you": true, "your": true,+}++// asciiOnly reports whether s is made of ASCII bytes alone. A pure-ASCII string is the only thing+// ASCII decoding produces, so a byte scan decides it.+func asciiOnly(s string) bool {+ for i := 0; i < len(s); i++ {+ if s[i] >= utf8.RuneSelf {+ return false+ }+ }+ return true+}++// allDigits reports whether s is non-empty and made only of decimal digits, in any script.+func allDigits(s string) bool {+ if s == "" {+ return false+ }+ for _, r := range s {+ if !unicode.IsDigit(r) {+ return false+ }+ }+ return true+}++// trimWordEdges drops leading and trailing runes that are neither letters nor digits, so the+// punctuation a title tends to carry — 《三国演义》, «Война и мир», (1917), a trailing comma — does+// not reach the token stream.+func trimWordEdges(s string) string {+ return strings.TrimFunc(s, func(r rune) bool {+ return !unicode.IsLetter(r) && !unicode.IsDigit(r)+ })+}++func declaredTitleTokens(chunks []schema.ChunkDoc) []string {+ const (+ headerLines = 20+ maxTokens = 20+ )+ var values []string+ for i := range chunks {+ head := chunks[i].Text+ if head == "" {+ continue+ }+ lines := strings.Split(head, "\n")+ if len(lines) > headerLines {+ lines = lines[:headerLines]+ }+ for _, line := range lines {+ l := strings.ToLower(strings.TrimSpace(line))+ for _, key := range []string{"title:", "name:", "fullname:"} {+ if strings.HasPrefix(l, key) {+ values = append(values, l[len(key):])+ }+ }+ }+ break+ }+ if len(values) == 0 {+ return nil+ }+ skip := map[string]bool{"title": true, "name": true, "fullname": true, "wikipedia": true}+ seen := map[string]bool{}+ var out []string+ // keep records one accepted token and reports whether the cap is now reached.+ keep := func(token string) bool {+ if seen[token] {+ return false+ }+ seen[token] = true+ out = append(out, token)+ return len(out) >= maxTokens+ }+ for _, v := range values {+ // Split on whitespace, NOT on ASCII boundaries. A word in another script is a word, and+ // the analyzer downstream segments it (RAGFlow's tokenizer handles CJK); the+ // ASCII-boundary splitter dissolved such a word into fragments of length zero.+ for _, word := range strings.Fields(v) {+ word = trimWordEdges(word)+ if word == "" {+ continue+ }+ if !asciiOnly(word) {+ // Pure numbers stay out of the title channel whichever script writes them: the+ // corpus file stem is numeric, so a numeric title token matches every document.+ if !allDigits(word) && keep(word) {+ return out+ }+ continue+ }+ for _, w := range strings.FieldsFunc(word, func(r rune) bool {+ return !(r >= 'a' && r <= 'z') && !(r >= '0' && r <= '9')+ }) {+ // Pure numbers stay out (years, ids, and the corpus file stem are all numeric+ // noise for the title channel); everything else is kept unless it is an English+ // stopword, a label or a duplicate.+ if englishStopwords[w] || skip[w] || allDigits(w) {+ continue+ }+ if keep(w) {+ return out+ }+ }+ }+ }+ return out+}+// tokenizeChunks annotates each chunk with title_tks, content_ltks,// and (when applicable) question_tks / important_tks / summary fields.// Mirrors python tokenizer.py:130-185 and rag/nlp/__init__.py tokenize() /diff --git a/internal/ingestion/component/tokenizer_title_test.go b/internal/ingestion/component/tokenizer_title_test.gonew file mode 100644index 0000000000..cb69dd9cac--- /dev/null+++ b/internal/ingestion/component/tokenizer_title_test.go@@ -0,0 +1,124 @@+// Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package component++import (+ "reflect"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// A corpus named by id ("93372.md") leaves title_tks with the id alone; when the document+// declares a title in its header block that title must reach the field, filtered — in whatever+// script the document is written.+func TestDeclaredTitleTokens(t *testing.T) {+ header := "Document 44250\nSource URL: https://www.sportskeeda.com/player/pp-ojha\n" ++ "title: Pragyan Ojha\ndate: 2025-01-01\nPersonal Information\n| Full Name | Pragyan Prayash Ojha |\n"+ cases := []struct {+ name string+ chunks []schema.ChunkDoc+ want []string+ }{+ {+ // Only the declared-title lines feed the field ("Source URL", "date", "| Full Name |"+ // are not title lines) and label words are dropped. Short content words are KEPT:+ // the >=5 length rule was replaced by English stopword filtering (2026-09-22), because+ // the length rule also removed real words such as "icc", "cup", "john", "star".+ name: "declared title is tokenised, labels dropped, short words kept",+ chunks: []schema.ChunkDoc{{Text: header}},+ want: []string{"pragyan", "ojha"},+ },+ {+ name: "title/name/fullname are all read, duplicates and wikipedia dropped",+ chunks: []schema.ChunkDoc{{Text: "title: Lesley Manyathela - Wikipedia\nname: Lesley Manyathela\nfullname: Lesley Phuti Manyathela\n"}},+ want: []string{"lesley", "manyathela", "phuti"},+ },+ {+ name: "no header block means no title",+ chunks: []schema.ChunkDoc{{Text: "AW: I began teaching at Zama Dance School in January 2008.\n"}},+ want: nil,+ },+ {+ // Pure numbers are still dropped (the corpus file stem is numeric, so a+ // numeric title token would match every document), but the short words that+ // the old >=5 rule removed are now kept.+ name: "numeric tokens dropped, short words kept",+ chunks: []schema.ChunkDoc{{Text: "title: 1917 Ok Go\n"}},+ want: []string{"ok", "go"},+ },+ {+ // A declared title is kept whatever script writes it. The ASCII-boundary+ // splitter this replaced produced nothing at all for these titles, which+ // left the title channel empty for every non-Latin document.+ name: "Chinese title is kept whole",+ chunks: []schema.ChunkDoc{{Text: "title: 三国演义\n"}},+ want: []string{"三国演义"},+ },+ {+ name: "Chinese title keeps its book-title marks out and the date line stays untouched",+ chunks: []schema.ChunkDoc{{Text: "title: 《三国演义》\ndate: 2025-01-01\n"}},+ want: []string{"三国演义"},+ },+ {+ name: "a localised wikipedia suffix is a separate word, not a label",+ chunks: []schema.ChunkDoc{{Text: "title: 《三国演义》 - 维基百科\n"}},+ want: []string{"三国演义", "维基百科"},+ },+ {+ name: "Japanese and Korean titles are kept whole",+ chunks: []schema.ChunkDoc{{Text: "name: 吾輩は猫である\nfullname: 태백산맥\n"}},+ want: []string{"吾輩は猫である", "태백산맥"},+ },+ {+ // No multilingual stopword list exists, so a non-English function word+ // survives: "и" is here on purpose, to pin that residual.+ name: "Cyrillic title is split on whitespace and quoted form is trimmed",+ chunks: []schema.ChunkDoc{{Text: "title: «Война и мир»\n"}},+ want: []string{"война", "и", "мир"},+ },+ {+ // Diacritics used to be ASCII-boundary separators: "José" came out as+ // "jos"+"é" (and "Müller" as "m"+"ller", with "m" then dropped as a+ // stopword). The whole accented word is what belongs in the index.+ name: "accented Latin words survive intact",+ chunks: []schema.ChunkDoc{{Text: "fullname: José Müller\n"}},+ want: []string{"josé", "müller"},+ },+ {+ name: "a mixed-script title keeps both sides and still drops English stopwords",+ chunks: []schema.ChunkDoc{{Text: "title: 三国演义 Romance of the Three Kingdoms\n"}},+ want: []string{"三国演义", "romance", "three", "kingdoms"},+ },+ {+ name: "pure numbers are dropped in any script",+ chunks: []schema.ChunkDoc{{Text: "title: ٣٤٥\n"}},+ want: nil,+ },+ {+ name: "empty chunks are skipped safely",+ chunks: []schema.ChunkDoc{{}, {Text: ""}},+ want: nil,+ },+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ got := declaredTitleTokens(c.chunks)+ if !reflect.DeepEqual(got, c.want) {+ t.Errorf("declaredTitleTokens() = %v, want %v", got, c.want)+ }+ })+ }+}