RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(infinity): implement SearchByRegexp for the agent grep tool (#21068)
qinling021054cb269d0d0b
diff --git a/conf/infinity_mapping.json b/conf/infinity_mapping.jsonindex 56eb58c966..9264248dce 100644--- a/conf/infinity_mapping.json+++ b/conf/infinity_mapping.json@@ -14,7 +14,7 @@"important_kwd_empty_count": {"type": "integer", "default": 0},"important_keywords": {"type": "varchar", "default": "", "analyzer": ["rag-coarse", "rag-fine"], "comment": "important_kwd, important_tks"},"questions": {"type": "varchar", "default": "", "analyzer": ["rag-coarse", "rag-fine"], "comment": "question_kwd, question_tks"},- "content": {"type": "varchar", "default": "", "analyzer": ["rag-coarse", "rag-fine"], "comment": "content_with_weight, content_ltks, content_sm_ltks"},+ "content": {"type": "varchar", "default": "", "analyzer": ["rag-coarse", "rag-fine", "sparsegram-3-12-fold"], "comment": "content_with_weight, content_ltks, content_sm_ltks"},"authors": {"type": "varchar", "default": "", "analyzer": ["rag-coarse", "rag-fine"], "comment": "authors_tks, authors_sm_tks"},"page_num_int": {"type": "varchar", "default": ""},"top_int": {"type": "varchar", "default": ""},diff --git a/internal/agent/runtime/retrieval.go b/internal/agent/runtime/retrieval.goindex b7cf23c23d..ebf87355cf 100644--- a/internal/agent/runtime/retrieval.go+++ b/internal/agent/runtime/retrieval.go@@ -201,7 +201,7 @@ var ErrBm25ServiceMissing = errors.New()// ErrRegexpNotSupported is returned when the underlying doc engine does not-// implement regex matching on chunk content (e.g. Infinity).+// implement regex matching on chunk content (no SearchByRegexp method).var ErrRegexpNotSupported = errors.New("grep_chunks: regex matching is not supported by this document engine",)diff --git a/internal/agentic_rag/grep_degrade_test.go b/internal/agentic_rag/grep_degrade_test.goindex 5030401660..55001fe58f 100644--- a/internal/agentic_rag/grep_degrade_test.go+++ b/internal/agentic_rag/grep_degrade_test.go@@ -156,8 +156,8 @@ func TestGrepChunksSplitsExplodingAlternation(t *testing.T) {if err != nil {t.Fatalf("split must recover, got error: %v", err)}- if !strings.Contains(out, toolErrorMarker) || !strings.Contains(out, "ES regexp query was rejected") {- t.Errorf("an exact split must explain the ES rejection, got %.200q", out)+ if !strings.Contains(out, toolErrorMarker) || !strings.Contains(out, "regexp query was rejected by the engine") {+ t.Errorf("an exact split must explain the engine rejection, got %.200q", out)}if !strings.Contains(out, `chunk_id="c1"`) {t.Errorf("branch hits must ship, got %.200q", out)@@ -190,8 +190,8 @@ func TestGrepChunksSanitizesLuceneUnsafePattern(t *testing.T) {if err != nil {t.Fatalf("sanitized retry must recover, got error: %v", err)}- if !strings.Contains(out, toolErrorMarker) || !strings.Contains(out, "ES regexp query was rejected") {- t.Errorf("a sanitized fallback must explain the ES rejection, got %.200q", out)+ if !strings.Contains(out, toolErrorMarker) || !strings.Contains(out, "regexp query was rejected by the engine") {+ t.Errorf("a sanitized fallback must explain the engine rejection, got %.200q", out)}if got := svc.calls[len(svc.calls)-1]; strings.Contains(got, `\b`) {t.Errorf("retry pattern must be sanitized, got %q", got)@@ -225,7 +225,7 @@ func TestGrepChunksDegradesToLexicalPrefilter(t *testing.T) {}if !strings.Contains(out, `<tool_error tool="grep_chunks" severity="warn">`) ||!strings.Contains(out, "APPROXIMATE") ||- !strings.Contains(out, "ES regexp query was rejected") {+ !strings.Contains(out, "regexp query was rejected by the engine") {t.Errorf("expected a warn notice about approximate coverage, got %.300q", out)}if !strings.Contains(out, `chunk_id="c1"`) {diff --git a/internal/agentic_rag/grep_service.go b/internal/agentic_rag/grep_service.goindex c97327f901..6f80336e88 100644--- a/internal/agentic_rag/grep_service.go+++ b/internal/agentic_rag/grep_service.go@@ -35,9 +35,10 @@ import ("go.uber.org/zap")-// Elasticsearch it pushes the regex down to a native `regexp` query; on engines-// without native regex support it falls back to a broad recall + in-memory RE2-// filter. It is stateless and safe to share across goroutines.+// Elasticsearch it pushes the regex down to a native `regexp` query, and on+// Infinity to a native `regex()` filter; on engines without native regex+// support it falls back to a broad recall + in-memory RE2 filter. It is+// stateless and safe to share across goroutines.type GrepAdapter struct {docEngine engine.DocEngine}@@ -48,7 +49,7 @@ func NewGrepAdapter(docEngine engine.DocEngine) *GrepAdapter {}// regexpSearchable is the narrow engine capability for native regex search.-// *elasticsearch.Engine implements it; other engines do not.+// *elasticsearch.Engine and *infinity.Engine implement it; other engines do not.type regexpSearchable interface {SearchByRegexp(ctx context.Context, req *enginetypes.RegexpSearchRequest) (*enginetypes.SearchResult, error)}@@ -56,9 +57,10 @@ type regexpSearchable interface {// ListByDocIDs reads the full original chunks belonging to the given document// ids, skipping graph (relation/entity/location) chunks so callers get the// actual document text. It backs the list_chunks deep-read runtime.-// Because content_with_weight is now a searchable keyword field, it reads the-// docs' chunks directly via the regexp pushdown (match-all "." scoped by-// doc_id), no longer needing a content_ltks-based general recall.+// Because content_with_weight is now a searchable keyword field (content with a+// sparsegram index on Infinity), it reads the docs' chunks directly via the+// regexp pushdown (match-all "." scoped by doc_id), no longer needing a+// content_ltks-based general recall.func (g *GrepAdapter) ListByDocIDs(ctx context.Context, req runtime.GrepRequest) ([]runtime.RetrievalChunk, error) {if g == nil || g.docEngine == nil {return nil, runtime.ErrGrepServiceMissing@@ -78,8 +80,8 @@ func (g *GrepAdapter) ListByDocIDs(ctx context.Context, req runtime.GrepRequest)}// Match-all on the (now keyword) content_with_weight, scoped to the docs,// restricted to ordinary text chunks (available_int=1, no compile_kwd). When- // the caller requests a sort (e.g. reading order), push it down so ES returns- // the deterministic first page.+ // the caller requests a sort (e.g. reading order), push it down so the engine+ // returns the deterministic first page.res, err := se.SearchByRegexp(ectx, &enginetypes.RegexpSearchRequest{TenantID: req.TenantID,KbIDs: req.DatasetIDs,@@ -281,10 +283,11 @@ func (g *GrepAdapter) Grep(ctx context.Context, req runtime.GrepRequest) ([]runtectx, ecancel := engineCallContext(ctx)defer ecancel()- // Path 1: native regex pushdown (Elasticsearch). With content_with_weight- // mapped as a searchable keyword field, ES regexp matching is the sole path;- // there is intentionally no in-memory RE2 fallback (a pushdown failure is a- // real error the caller should surface, not silently "no matches").+ // Path 1: native regex pushdown (Elasticsearch, Infinity). With+ // content_with_weight mapped as a searchable keyword field, native regexp+ // matching is the sole path; there is intentionally no in-memory RE2+ // fallback (a pushdown failure is a real error the caller should surface,+ // not silently "no matches").if se, ok := g.docEngine.(regexpSearchable); ok {res, err := se.SearchByRegexp(ectx, &enginetypes.RegexpSearchRequest{TenantID: req.TenantID,@@ -300,7 +303,7 @@ func (g *GrepAdapter) Grep(ctx context.Context, req runtime.GrepRequest) ([]runtreturn translateGrepChunks(res.Chunks), nil}- // Non-ES engines do not implement regex matching on chunk content; surface+ // Engines without native regex support cannot match on chunk content; surface// an explicit error rather than returning (empty) regex "matches".return nil, runtime.ErrRegexpNotSupported}@@ -399,9 +402,9 @@ func degradeGrep(return nil, "", pushErr}-// pushdownWarning keeps the original ES rejection visible whenever a fallback-// produces results. The model must distinguish a real regexp result from a-// result obtained after the engine rejected the requested pattern.+// pushdownWarning keeps the original pushdown rejection visible whenever a+// fallback produces results. The model must distinguish a real regexp result+// from a result obtained after the engine rejected the requested pattern.func pushdownWarning(pushErr error, detail string) string {if detail == "" {detail = "the pattern was split into smaller regexp branches and the returned matches were verified with the original Go regexp"@@ -413,7 +416,7 @@ func pushdownWarning(pushErr error, detail string) string {if r := []rune(summary); len(r) > 400 {summary = string(r[:400]) + "…"}- return fmt.Sprintf("ES regexp query was rejected: %s. %s", summary, detail)+ return fmt.Sprintf("regexp query was rejected by the engine: %s. %s", summary, detail)}// grepBySplitAlternation pushes each top-level alternation branch as its own@@ -637,7 +640,7 @@ func engineCallContext(ctx context.Context) (context.Context, context.CancelFuncreturn context.WithCancel(context.WithoutCancel(ctx))}-// translateGrepChunks converts ES regexp-search result maps into RetrievalChunk.+// translateGrepChunks converts regexp-search result maps into RetrievalChunk.// regexpPushdownHint turns an engine-side regexp rejection into something the// caller can act on.//@@ -656,8 +659,8 @@ func engineCallContext(ctx context.Context) (context.Context, context.CancelFuncfunc regexpPushdownHint(pattern string) string {if m := boundedRepetitionRe.FindString(pattern); m != "" {return " Retry WITHOUT the bounded repetition `" + m + "`: the engine compiles a pattern " +- "into an automaton with a fixed state budget (it already wraps every pattern in " +- "`.*(...).*`), and repetition over CJK text exhausts it. Join the terms with `.*` " ++ "into a bounded automaton, and repetition over CJK text can exhaust it (Elasticsearch " ++ "wraps every pattern in `.*(...).*`, which costs more states). Join the terms with `.*` " +"(e.g. `关羽.*杀`) or list them as alternatives (`关羽.*杀|关公.*斩`)."}if backreferenceRe.MatchString(pattern) {@@ -669,7 +672,7 @@ func regexpPushdownHint(pattern string) string {"express the constraint as plain terms."}return " Retry with a CHEAPER pattern: literal terms joined by `.*`, at most 2-3 alternation " +- "branches, no bounded repetition — this pattern exceeded the engine's automaton state budget."+ "branches, no bounded repetition — this pattern was too expensive for the engine to compile."}var (diff --git a/internal/engine/infinity/chunk.go b/internal/engine/infinity/chunk.goindex e08c460a0a..a9b82530b2 100644--- a/internal/engine/infinity/chunk.go+++ b/internal/engine/infinity/chunk.go@@ -1417,32 +1417,7 @@ func (e *Engine) Search(ctx context.Context, req *types.SearchRequest) (*types.S}// Convert DataFrame to chunks format (column-oriented to row-oriented)- searchChunks := make([]map[string]interface{}, 0)- for colName, colData := range df.ColumnData {- for i, val := range colData {- // A null cell must not materialize a row of its own. Infinity- // can return a single all-null cell for a projection it- // matched nothing for (observed with a full-text filter plus a- // json-list projection: source_chunk_ids came back as a nil- // slice while the other columns were empty), and fabricating- // one empty row out of it leaks phantom documents to callers —- // GetWikiGraph then read an empty slug_kwd off it and issued- // filter_fulltext('from_kwd', ''), which Infinity rejects.- if isEmptyEngineCell(val) {- continue- }- for len(searchChunks) <= i {- searchChunks = append(searchChunks, make(map[string]interface{}))- }- searchChunks[i][colName] = val- }- }-- // Infinity returns a Json/Array column as ONE cell holding one- // length-prefixed segment per row, so the loop above parks the whole- // column on row 0. Redistribute it before applyFieldMappings renames- // anything.- realignJSONColumnCells(searchChunks, df.ColumnData)+ searchChunks := chunksFromColumnData(df.ColumnData)// Apply field name mapping and row_id handling// Skill index uses different schema@@ -1536,6 +1511,309 @@ func shouldDefaultAvailableFilter(req *types.SearchRequest, isSkillIndex bool) breturn !isSkillIndex && !req.IncludeUnavailable}+const (+ // regexpSearchDefaultLimit caps a regexp search, matching the Elasticsearch+ // engine so grep_chunks returns the same page on either backend.+ regexpSearchDefaultLimit = 30+ // regexpSearchReturnAllLimit is the ceiling for list_chunks' deep read.+ regexpSearchReturnAllLimit = 10000+)++// SearchByRegexp is a regex-match-only search over chunk content for the agent's+// grep_chunks/list_chunks tools, mirroring the Elasticsearch engine. The pattern goes+// down as regex(content, ...), which the content column's sparse gram index narrows by+// the pattern's mandatory literals; Infinity evaluates it with RE2, so the pattern is+// used verbatim (no ".*(...).*" wrapper) and "(?i)" carries case folding.+func (e *Engine) SearchByRegexp(ctx context.Context, req *types.RegexpSearchRequest) (*types.SearchResult, error) {+ if req == nil {+ return nil, fmt.Errorf("regexp search request cannot be nil")+ }+ if strings.TrimSpace(req.Pattern) == "" {+ return nil, fmt.Errorf("regexp pattern cannot be empty")+ }+ tenantID := strings.TrimSpace(req.TenantID)+ if tenantID == "" {+ return nil, fmt.Errorf("tenant id cannot be empty")+ }+ // One index per tenant; a list would fan out to unrelated tables.+ if strings.Contains(tenantID, ",") {+ return nil, fmt.Errorf("regexp search supports exactly one tenant")+ }+ // Chunk tables are per dataset: an unscoped request would read the bare+ // ragflow_<tenant>, which holds no chunks, and report an empty result.+ scoped := false+ for _, kbID := range req.KbIDs {+ if strings.TrimSpace(kbID) != "" {+ scoped = true+ break+ }+ }+ if !scoped {+ return nil, fmt.Errorf("regexp search requires at least one dataset id")+ }++ maxResults := regexpSearchDefaultLimit+ if req.ReturnAll {+ maxResults = regexpSearchReturnAllLimit+ }++ db, release, err := e.client.checkoutDatabase(ctx, "chunk.go")+ if err != nil {+ return nil, fmt.Errorf("failed to get database: %w", err)+ }+ defer release()++ // Chunk tables are per dataset: ragflow_<tenant>_<datasetID>.+ baseName := fmt.Sprintf("ragflow_%s", tenantID)+ tableNames := make([]string, 0, len(req.KbIDs))+ for _, kbID := range req.KbIDs {+ if kbID == "" {+ tableNames = append(tableNames, baseName)+ } else {+ tableNames = append(tableNames, fmt.Sprintf("%s_%s", baseName, kbID))+ }+ }++ outputColumns := []string{+ "id", "doc_id", "kb_id", "content_ltks", "content_with_weight",+ "title_tks", "docnm_kwd", "img_id", "available_int", "important_kwd",+ "position_int", "page_num_int", "top_int", "chunk_order_int",+ "create_timestamp_flt", "knowledge_graph_kwd", "question_kwd", "question_tks",+ "doc_type_kwd", "mom_id", "tag_kwd", "pagerank_fea", "tag_feas",+ }+ if len(req.SelectFields) > 0 {+ outputColumns = append([]string(nil), req.SelectFields...)+ }+ // The merge at the end restores the order Go-side, so the sort keys must be+ // projected even when the caller did not ask for them.+ outputColumns = unionSortColumns(outputColumns, req.Sort)+ // No MATCH is applied, so Infinity accepts no score column to project.+ outputColumns = stripScorePseudoColumns(outputColumns)+ outputColumns = convertSelectFields(outputColumns, false, false)++ // Same filter for every dataset table; kb_id is implied by the table name.+ filterStr := buildRegexpFilter(req.Filter, req.Pattern)++ var allResults []map[string]interface{}+ totalHits := int64(0)+ // A missing table is normal (created on first write); other failures surface+ // only if no table answered.+ var queryErr error++ for _, tableName := range tableNames {+ tbl, err := db.GetTable(tableName)+ if err != nil {+ // A dataset table that does not exist yet is normal (tables are created on+ // first write); any other open failure is real and must reach the caller.+ errMsg := strings.ToLower(err.Error())+ tableMissing := strings.Contains(errMsg, "not found") || strings.Contains(errMsg, "doesn't exist")+ if isConnectionLevelError(err) {+ e.client.dropConnectionFor(db, "SearchByRegexp")+ }+ if !tableMissing && queryErr == nil {+ queryErr = fmt.Errorf("open table %s: %w", tableName, err)+ }+ continue+ }++ table := tbl.Output(outputColumns).Filter(filterStr)++ if req.Sort != nil && len(req.Sort.Fields) > 0 {+ var sortFields [][2]interface{}+ for _, orderField := range req.Sort.Fields {+ sortType := infinity.SortTypeAsc+ if orderField.Type == types.SortDesc {+ sortType = infinity.SortTypeDesc+ }+ sortFields = append(sortFields, [2]interface{}{orderField.Field, sortType})+ }+ table = table.Sort(sortFields)+ }++ table = table.Limit(maxResults)+ table = table.Option(map[string]interface{}{"total_hits_count": true})++ df, err := table.ToDataFrame()+ if err != nil {+ common.Warn("Infinity regexp query failed",+ zap.String("tableName", tableName),+ zap.String("filter", filterStr),+ zap.Int("limit", maxResults),+ zap.Error(err))+ if isConnectionLevelError(err) {+ e.client.dropConnectionFor(db, "SearchByRegexp")+ }+ if queryErr == nil {+ queryErr = fmt.Errorf("query table %s: %w", tableName, err)+ }+ continue+ }++ chunks := chunksFromColumnData(df.ColumnData)+ applyFieldMappings(chunks)++ var tableTotal int64+ if df.ExtraInfo != "" {+ var extraResult map[string]interface{}+ if err := json.Unmarshal([]byte(df.ExtraInfo), &extraResult); err == nil {+ if count, ok := extraResult["total_hits_count"].(float64); ok {+ tableTotal = int64(count)+ }+ }+ }++ common.Debug("Infinity regexp search",+ zap.String("table", tableName),+ zap.String("filter", filterStr),+ zap.Int("rows", len(chunks)),+ zap.Int64("total", tableTotal))++ allResults = append(allResults, chunks...)+ totalHits += tableTotal+ }++ // limit() applies per table, so merge the tables' pages into the requested order+ // and cap once, as the Elasticsearch engine does after its single-index query.+ // The cap must follow the sort, and Total stays the pre-cap count.+ sortChunksByFields(allResults, req.Sort)+ if len(allResults) > maxResults {+ allResults = allResults[:maxResults]+ }++ // All tables failed: surface the error rather than an empty result.+ if len(allResults) == 0 && totalHits == 0 && queryErr != nil {+ return nil, fmt.Errorf("infinity regexp search failed on all tables: %w", queryErr)+ }++ return &types.SearchResult{+ Chunks: allResults,+ Total: totalHits,+ }, nil+}++// buildRegexpFilter renders the regexp search filter: the caller's scope (minus+// kb_id, already implied by the per-dataset table name) AND regex(content, ...)+// with the case-insensitive "(?i)" the grep contract requires.+func buildRegexpFilter(filter map[string]interface{}, pattern string) string {+ filterCopy := map[string]interface{}{}+ for k, v := range filter {+ if k == "kb_id" {+ continue+ }+ filterCopy[k] = v+ }+ parts := make([]string, 0, 2)+ if len(filterCopy) > 0 {+ if condition := equivalentConditionToStr(filterCopy, nil); condition != "" {+ parts = append(parts, condition)+ }+ }+ parts = append(parts, infinity.RegexFilter("content", "(?i)"+pattern))+ return strings.Join(parts, " AND ")+}++// unionSortColumns appends the requested sort keys to a projection, preserving the+// caller's column order: the engine orders the merged rows itself.+func unionSortColumns(columns []string, sort *types.OrderByExpr) []string {+ if sort == nil || len(sort.Fields) == 0 {+ return columns+ }+ seen := make(map[string]struct{}, len(columns)+len(sort.Fields))+ for _, column := range columns {+ seen[column] = struct{}{}+ }+ for _, field := range sort.Fields {+ if field.Field == "" {+ continue+ }+ if _, ok := seen[field.Field]; ok {+ continue+ }+ seen[field.Field] = struct{}{}+ columns = append(columns, field.Field)+ }+ return columns+}++// sortChunksByFields orders merged rows by the requested fields, ascending unless a+// field asks for descending, mirroring the Elasticsearch engine's sortByFields: a+// missing field sorts first, numbers numerically, strings lexicographically.+// page_num_int arrives hex-decoded as an int slice, compared on its leading element.+func sortChunksByFields(chunks []map[string]interface{}, expr *types.OrderByExpr) {+ if expr == nil || len(expr.Fields) == 0 {+ return+ }+ sort.SliceStable(chunks, func(i, j int) bool {+ for _, field := range expr.Fields {+ vi, oki := chunks[i][field.Field]+ vj, okj := chunks[j][field.Field]+ if !oki || !okj {+ if !oki && !okj {+ continue+ }+ // Missing field sorts before a present one.+ return !oki+ }+ less, equal := compareOrderValues(vi, vj)+ if equal {+ continue+ }+ if field.Type == types.SortDesc {+ return !less+ }+ return less+ }+ return false+ })+}++// compareOrderValues compares two raw field values: numbers numerically, strings+// lexicographically, anything else by text. An unset key (the empty column default, as+// a page number that was never set) ranks before a set one in either argument order, so+// the pair cannot order differently by comparison direction; equality is reported+// separately, so only a real tie falls through to the next sort field.+func compareOrderValues(a, b interface{}) (less, equal bool) {+ fa, okA := numericOrderValue(a)+ fb, okB := numericOrderValue(b)+ if okA != okB {+ return !okA, false+ }+ if okA {+ return fa < fb, fa == fb+ }+ sa, okSA := a.(string)+ sb, okSB := b.(string)+ if okSA && okSB {+ return sa < sb, sa == sb+ }+ ta, tb := fmt.Sprintf("%v", a), fmt.Sprintf("%v", b)+ return ta < tb, ta == tb+}++// numericOrderValue reads a scalar order key, unwrapping the int slice+// applyFieldMappings leaves behind for hex-encoded columns.+func numericOrderValue(v interface{}) (float64, bool) {+ switch n := v.(type) {+ case int:+ return float64(n), true+ case int64:+ return float64(n), true+ case float64:+ return n, true+ case float32:+ return float64(n), true+ case []int:+ if len(n) > 0 {+ return float64(n[0]), true+ }+ case []interface{}:+ if len(n) > 0 {+ return numericOrderValue(n[0])+ }+ }+ return 0, false+}+// GetChunk gets a chunk by IDfunc (e *Engine) GetChunk(ctx context.Context, tableName, chunkID string, datasetIDs []string) (interface{}, error) {if e.client == nil || e.client.pool == nil {@@ -1702,6 +1980,28 @@ func splitKeywordList(field, value string) ([]interface{}, bool) {return list, true}+// chunksFromColumnData converts an Infinity column-oriented DataFrame into the+// row-oriented chunk maps the engine returns. A null cell must not become a row+// of its own (Infinity returns one all-null cell for a projection that matched+// nothing, and a phantom row leaks to callers); a Json/Array column arrives+// parked on row 0, so realignJSONColumnCells spreads it before applyFieldMappings.+func chunksFromColumnData(columnData map[string][]interface{}) []map[string]interface{} {+ rows := make([]map[string]interface{}, 0)+ for colName, colData := range columnData {+ for i, val := range colData {+ if isEmptyEngineCell(val) {+ continue+ }+ for len(rows) <= i {+ rows = append(rows, make(map[string]interface{}))+ }+ rows[i][colName] = val+ }+ }+ realignJSONColumnCells(rows, columnData)+ return rows+}+// applyFieldMappings applies field mappings to chunks (side-effect only).// Used by Search() to mutate chunks with derived fields before returning.func applyFieldMappings(chunks []map[string]interface{}) {diff --git a/internal/engine/infinity/chunk_filter_test.go b/internal/engine/infinity/chunk_filter_test.goindex b65e3fd52b..bf3c98742d 100644--- a/internal/engine/infinity/chunk_filter_test.go+++ b/internal/engine/infinity/chunk_filter_test.go@@ -1,6 +1,7 @@package infinityimport (+ "context""fmt""strings""testing"@@ -111,3 +112,66 @@ func TestListFilterConditionKeepsDocumentScope(t *testing.T) {})}}++// buildRegexpFilter must keep the caller's document scope, drop kb_id (the+// per-dataset table name already scopes the query), and append the regex()+// predicate with the case-insensitive inline flag on the content column.+func TestBuildRegexpFilterScopesAndMatchesContent(t *testing.T) {+ got := buildRegexpFilter(map[string]interface{}{+ "available_int": 1,+ "doc_id": []string{"d1", "d2"},+ "must_not": map[string]interface{}{"exists": "compile_kwd"},+ "kb_id": "kb-must-not-appear",+ }, `foo.*bar`)++ for _, want := range []string{+ "regex(content, '(?i)foo.*bar')",+ "available_int=1",+ "doc_id IN ('d1', 'd2')",+ "NOT (compile_kwd!='')",+ } {+ if !strings.Contains(got, want) {+ t.Errorf("buildRegexpFilter() = %q, want it to contain %q", got, want)+ }+ }+ if strings.Contains(got, "kb_id") {+ t.Errorf("buildRegexpFilter() = %q, must not carry kb_id into the per-dataset table filter", got)+ }+}++// A pattern quoted in SQL-ish filter syntax must double its single quotes, or+// the SDK's expression parser would read a truncated literal.+func TestBuildRegexpFilterEscapesSingleQuotes(t *testing.T) {+ got := buildRegexpFilter(nil, `it's a match`)+ if want := `regex(content, '(?i)it''s a match')`; got != want {+ t.Errorf("buildRegexpFilter() = %q, want %q", got, want)+ }+}++// A regexp search from the agent's grep tools carries only a pattern and no+// scope filter; the predicate itself must still be well-formed.+func TestBuildRegexpFilterWithoutScope(t *testing.T) {+ if got, want := buildRegexpFilter(nil, `.*`), `regex(content, '(?i).*')`; got != want {+ t.Errorf("buildRegexpFilter() = %q, want %q", got, want)+ }+}++// An unscoped regexp search must fail rather than scan the bare ragflow_<tenant> table:+// chunk tables are per dataset, so an empty KbIDs would come back as an error-free empty+// result. The check runs before the client is touched, so a zero Engine pins it.+func TestSearchByRegexpRejectsEmptyScope(t *testing.T) {+ eng := &Engine{}+ for _, kbIDs := range [][]string{nil, {}, {""}, {"", " "}} {+ _, err := eng.SearchByRegexp(context.Background(), &types.RegexpSearchRequest{+ TenantID: "tenant1",+ KbIDs: kbIDs,+ Pattern: "foo.*bar",+ })+ if err == nil {+ t.Fatalf("SearchByRegexp(KbIDs=%v) must reject an empty dataset scope", kbIDs)+ }+ if !strings.Contains(err.Error(), "at least one dataset id") {+ t.Errorf("SearchByRegexp(KbIDs=%v) error = %v, want it to name the missing dataset scope", kbIDs, err)+ }+ }+}diff --git a/internal/engine/infinity/chunk_transform_test.go b/internal/engine/infinity/chunk_transform_test.goindex 7495fb1cd1..7a689d62ee 100644--- a/internal/engine/infinity/chunk_transform_test.go+++ b/internal/engine/infinity/chunk_transform_test.go@@ -23,6 +23,7 @@ import ("strings""testing"+ "ragflow/internal/engine/types""ragflow/internal/utility"infinitysdk "github.com/infiniflow/infinity-go-sdk"@@ -953,3 +954,91 @@ func TestDedupeByChunkIDKeepsRowsWithoutID(t *testing.T) {t.Fatalf("rows = %d, want 3 (only the duplicate id collapses): %#v", len(got), got)}}++// chunksFromColumnData must turn the column-oriented DataFrame into rows and+// must not fabricate a row out of an all-null column.+func TestChunksFromColumnDataConvertsAndSkipsNullCells(t *testing.T) {+ rows := chunksFromColumnData(map[string][]interface{}{+ "id": {"a", "b"},+ "content": {"alpha", "beta"},+ "junk": {nil, nil},+ })+ if len(rows) != 2 {+ t.Fatalf("chunksFromColumnData() returned %d rows, want 2", len(rows))+ }+ if rows[0]["id"] != "a" || rows[0]["content"] != "alpha" {+ t.Errorf("row 0 = %v, want id=a content=alpha", rows[0])+ }+ if rows[1]["id"] != "b" || rows[1]["content"] != "beta" {+ t.Errorf("row 1 = %v, want id=b content=beta", rows[1])+ }+ if _, ok := rows[0]["junk"]; ok {+ t.Errorf("row 0 = %v, a null column must not contribute a cell", rows[0])+ }+}++// An entirely null projection (an empty match) must yield no rows rather than+// one phantom empty row.+func TestChunksFromColumnDataAllNullIsEmpty(t *testing.T) {+ rows := chunksFromColumnData(map[string][]interface{}{"content": {nil}})+ if len(rows) != 0 {+ t.Fatalf("chunksFromColumnData() returned %d rows, want 0", len(rows))+ }+}++// TestSortChunksByFieldsRestoresReadingOrder covers the cross-table merge: a table's+// rows arrive in that table's order, so the requested order has to be re-applied across+// the merged set, with page_num_int compared numerically (it arrives as a hex-decoded+// int slice).+func TestSortChunksByFieldsRestoresReadingOrder(t *testing.T) {+ rows := []map[string]interface{}{+ {"id": "b1", "doc_id": "doc-b", "page_num_int": []int{10}, "chunk_order_int": 0},+ {"id": "b2", "doc_id": "doc-b", "page_num_int": []int{3}, "chunk_order_int": 5},+ {"id": "a1", "doc_id": "doc-a", "page_num_int": []int{2}, "chunk_order_int": 9},+ {"id": "a2", "doc_id": "doc-a", "page_num_int": []int{2}, "chunk_order_int": 1},+ }+ sortChunksByFields(rows, &types.OrderByExpr{Fields: []types.OrderByField{+ {Field: "doc_id"},+ {Field: "page_num_int"},+ {Field: "chunk_order_int"},+ }})++ got := make([]string, 0, len(rows))+ for _, row := range rows {+ got = append(got, row["id"].(string))+ }+ // Document first, then the page within it (numerically), then the chunk order+ // within that page.+ want := []string{"a2", "a1", "b2", "b1"}+ if !reflect.DeepEqual(got, want) {+ t.Errorf("merged reading order = %v, want %v", got, want)+ }+}++// TestSortChunksByFieldsRanksUnsetKeyFirst pins the mixed-key case: a chunk with no page+// number arrives as the empty column default, a chunk that has one as a hex-decoded int+// slice. An unset key ranks first, and both argument orders must agree on that.+func TestSortChunksByFieldsRanksUnsetKeyFirst(t *testing.T) {+ order := &types.OrderByExpr{Fields: []types.OrderByField{+ {Field: "doc_id"},+ {Field: "page_num_int"},+ }}+ sorted := func(rows []map[string]interface{}) []string {+ sortChunksByFields(rows, order)+ ids := make([]string, 0, len(rows))+ for _, row := range rows {+ ids = append(ids, row["id"].(string))+ }+ return ids+ }+ withPage := map[string]interface{}{"id": "page-5", "doc_id": "doc-1", "page_num_int": []int{5}}+ unset := map[string]interface{}{"id": "page-unset", "doc_id": "doc-1", "page_num_int": ""}++ want := []string{"page-unset", "page-5"}+ if got := sorted([]map[string]interface{}{withPage, unset}); !reflect.DeepEqual(got, want) {+ t.Errorf("sort(page-first input) = %v, want %v", got, want)+ }+ if got := sorted([]map[string]interface{}{unset, withPage}); !reflect.DeepEqual(got, want) {+ t.Errorf("sort(unset-first input) = %v, want %v", got, want)+ }+}diff --git a/internal/service/agentic_reference.go b/internal/service/agentic_reference.goindex e34f88d055..8494f2e569 100644--- a/internal/service/agentic_reference.go+++ b/internal/service/agentic_reference.go@@ -149,7 +149,7 @@ func (s *ChatPipelineService) buildAgenticReference(ctx context.Context, tenantI// fetchChunksByIDs loads the cited chunks from the tenant's index, keeping the// engine row shape (chunk_id / content_with_weight / docnm_kwd / doc_id /-// kb_id / img_id / positions) so chunksFormat can normalize it. The index is+// kb_id / img_id / position_int) so chunksFormat can normalize it. The index is// the chat's OWNING tenant's - the same scoping the agent's retrieval tools// use.func fetchChunksByIDs(ctx context.Context, tenantID string, datasetIDs, ids []string) []map[string]interface{} {@@ -161,7 +161,7 @@ func fetchChunksByIDs(ctx context.Context, tenantID string, datasetIDs, ids []stIndexNames: []string{fmt.Sprintf("ragflow_%s", tenantID)},KbIDs: datasetIDs,Filter: map[string]interface{}{"id": ids},- SelectFields: []string{"content_with_weight", "docnm_kwd", "doc_id", "kb_id", "img_id", "positions"},+ SelectFields: []string{"content_with_weight", "docnm_kwd", "doc_id", "kb_id", "img_id", "position_int"},Limit: len(ids),}res, err := de.Search(ctx, req)