RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunk): list parent chunks after source chunks on every doc engine (#20990)
qinling0210131974ce67ba
diff --git a/internal/common/constants.go b/internal/common/constants.goindex bddc6a87e2..85d27dc05c 100644--- a/internal/common/constants.go+++ b/internal/common/constants.go@@ -79,3 +79,23 @@ const StatusDialogValid = "1"// DialogStatus is a typed alias for dialog.status to avoid raw string// comparisons in call sites.type DialogStatus string++// ParentChunkOrderBase is the first chunk_order_int of the range reserved for+// parent rows — the hidden rows MaterializeParentChunks creates for retrieval+// promotion. Source chunks live in [0, ParentChunkOrderBase): the tokenizer+// numbers ingested chunks, and AddChunk appends whatever the user adds after the+// highest order already present. Keeping the two ranges apart is what makes a+// parent sort after every source chunk on every engine, including source chunks+// added after the document was parsed:+//+// - Elasticsearch already sorted parents last because the field was simply+// absent on the parent row (a missing sort key counts as greatest);+// - Infinity cannot express "absent": an unwritten integer column falls back to+// its mapping default 0, which tied every parent with the first source chunk,+// so the management chunk list listed parents first and sliced that tie group+// differently on every page (LIMIT/OFFSET).+//+// The value fits a signed 32-bit integer, the narrowest type the column has+// across engines (Infinity integer, OceanBase INTEGER, Elasticsearch long), and+// leaves room for a billion parent rows per document.+const ParentChunkOrderBase = 1 << 30diff --git a/internal/common/number.go b/internal/common/number.goindex b735b47af4..c59e789b77 100644--- a/internal/common/number.go+++ b/internal/common/number.go@@ -143,10 +143,20 @@ func PairwiseSum(xs []float64) float64 {return xs[0]}+// GetInt returns the int behind a value and whether it was a number at all.+// Callers read values that come from config files, JSON decoding, or a document+// engine row — the last two hand numbers back as int, int32, int64, float32 or+// float64 depending on the encoder and the driver, so all of them are accepted.func GetInt(value interface{}) (int, bool) {switch v := value.(type) {case int:return v, true+ case int32:+ return int(v), true+ case int64:+ return int(v), true+ case float32:+ return int(v), truecase float64:return int(v), truedefault:diff --git a/internal/ingestion/task/indexdoc/parent_child.go b/internal/ingestion/task/indexdoc/parent_child.goindex 67d97a40c9..9fd6b61a3a 100644--- a/internal/ingestion/task/indexdoc/parent_child.go+++ b/internal/ingestion/task/indexdoc/parent_child.go@@ -5,15 +5,36 @@ import ("strings""github.com/cespare/xxhash/v2"++ "ragflow/internal/common")// MaterializeParentChunks turns the chunker's transient parent text into// hidden index rows and links every child row to its parent. A parent ID is// scoped to its dataset and document so equal parent text from separate// documents or datasets cannot overwrite each other in a shared index.+//+// Every parent row also receives a chunk_order_int from the range reserved for+// parents (common.ParentChunkOrderBase + k, where k follows the order in which+// the parents are discovered, i.e. the reading order of the first child+// referencing each one), while source chunks stay in [0, ParentChunkOrderBase).+// That keeps every source chunk — the ones just ingested and the ones a user+// adds later through AddChunk — ahead of every parent on every engine:+// Elasticsearch already placed parents last because the field was simply absent+// on the parent row (a missing sort key counts as greatest), but Infinity cannot+// express "absent": an unwritten column falls back to the mapping default 0+// (conf/infinity_mapping.json), which tied every parent with the first source+// chunk, so the management chunk list listed parents first and sliced that tie+// group differently on every page (LIMIT/OFFSET).+//+// Documents whose chunks carry no chunk_order_int at all are left alone: the+// parent rows keep the field absent so the Elasticsearch placement is preserved+// and no ordering is invented for pipelines that never numbered their chunks.func MaterializeParentChunks(datasetID string, chunks []map[string]any) []map[string]any {seen := make(map[string]struct{})parents := make([]map[string]any, 0)+ order := common.ParentChunkOrderBase+ ordered := chunksOrdered(chunks)for _, chunk := range chunks {mom := parentText(chunk)@@ -46,12 +67,29 @@ func MaterializeParentChunks(datasetID string, chunks []map[string]any) []map[stparent[field] = value}}+ if ordered {+ parent["chunk_order_int"] = order+ order+++ }seen[parentID] = struct{}{}parents = append(parents, parent)}return parents}+// chunksOrdered reports whether the document's source chunks are numbered at+// all. The tokenizer numbers every surviving chunk; a pipeline without one+// leaves the field unset, and those documents keep their previous placement+// instead of getting invented order values.+func chunksOrdered(chunks []map[string]any) bool {+ for _, chunk := range chunks {+ if _, ok := common.GetInt(chunk["chunk_order_int"]); ok {+ return true+ }+ }+ return false+}+// parentChunkID returns the deterministic parent row identifier. NUL// separators make the dataset, document, and parent-text boundaries// unambiguous.diff --git a/internal/ingestion/task/indexdoc/parent_child_test.go b/internal/ingestion/task/indexdoc/parent_child_test.goindex df56f7877a..f21322c4ee 100644--- a/internal/ingestion/task/indexdoc/parent_child_test.go+++ b/internal/ingestion/task/indexdoc/parent_child_test.go@@ -1,6 +1,10 @@package indexdoc-import "testing"+import (+ "testing"++ "ragflow/internal/common"+)func TestParentChunkIDScopesDatasetAndDocument(t *testing.T) {if got, again := parentChunkID("kb-a", "doc-a", "same parent text"), parentChunkID("kb-a", "doc-a", "same parent text"); got != again {@@ -29,3 +33,75 @@ func TestMaterializeParentChunksScopesSameParentTextToDocument(t *testing.T) {t.Fatalf("parent docs = %#v, want one parent per document", parents)}}++// TestMaterializeParentChunksOrdersParentsAfterChildren pins the ordering+// contract every chunk_order_int reader relies on (management chunk list,+// agent grep / deep read): a parent sorts after all source chunks of its+// document, its key is unique within that document so LIMIT/OFFSET and+// search_after pagination slice the same group the same way on every engine, and+// it sits in the reserved parent range so chunks added later through AddChunk+// (which keep appending into [0, ParentChunkOrderBase)) still list before it.+func TestMaterializeParentChunksOrdersParentsAfterChildren(t *testing.T) {+ chunks := []map[string]any{+ {"id": "child-1", "doc_id": "doc-1", "mom": "parent one", "chunk_order_int": float64(0)},+ {"id": "child-2", "doc_id": "doc-1", "mom": "parent one", "chunk_order_int": float64(1)},+ {"id": "child-3", "doc_id": "doc-1", "chunk_order_int": float64(2)},+ {"id": "child-4", "doc_id": "doc-1", "mom": "parent two", "chunk_order_int": float64(3)},+ }++ parents := MaterializeParentChunks("kb-1", chunks)++ if len(parents) != 2 {+ t.Fatalf("parent count = %d, want 2", len(parents))+ }+ for i, want := range []int{common.ParentChunkOrderBase, common.ParentChunkOrderBase + 1} {+ if got, ok := common.GetInt(parents[i]["chunk_order_int"]); !ok || got != want {+ t.Errorf("parent %d chunk_order_int = %v, want %d", i, parents[i]["chunk_order_int"], want)+ }+ }+ // Source chunks keep their own values; the parent values are what moves.+ for i, want := range []int{0, 1, 2, 3} {+ if got, ok := common.GetInt(chunks[i]["chunk_order_int"]); !ok || got != want {+ t.Errorf("child %d chunk_order_int = %v, want %d", i, chunks[i]["chunk_order_int"], want)+ }+ }++ seen := map[int]bool{}+ rows := append(append([]map[string]any{}, parents...), chunks...)+ for _, row := range rows {+ order, ok := common.GetInt(row["chunk_order_int"])+ if !ok {+ t.Fatalf("row %#v has no numeric chunk_order_int", row)+ }+ if seen[order] {+ t.Fatalf("chunk_order_int %d is not unique per document: %#v", order, rows)+ }+ seen[order] = true+ }++ // The document's real slice count must not be affected by the extra rows.+ if chunks[0]["mom_id"] != chunks[1]["mom_id"] {+ t.Fatalf("children of the same parent lost their shared mom_id: %#v", chunks)+ }+ if chunks[2]["mom_id"] != nil {+ t.Fatalf("child without a parent must not gain a mom_id: %#v", chunks[2])+ }+}++// TestMaterializeParentChunksLeavesUnnumberedDocumentsAlone keeps the+// Elasticsearch placement for pipelines that never numbered their chunks:+// inventing values there would move parents from "missing (last)" to the front.+func TestMaterializeParentChunksLeavesUnnumberedDocumentsAlone(t *testing.T) {+ chunks := []map[string]any{+ {"id": "child-1", "doc_id": "doc-1", "mom": "parent one"},+ }++ parents := MaterializeParentChunks("kb-1", chunks)++ if len(parents) != 1 {+ t.Fatalf("parent count = %d, want 1", len(parents))+ }+ if _, exists := parents[0]["chunk_order_int"]; exists {+ t.Fatalf("parent chunk_order_int = %v, want it absent", parents[0]["chunk_order_int"])+ }+}diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.goindex 232c90d62d..8d3b641470 100644--- a/internal/service/chunk/chunk.go+++ b/internal/service/chunk/chunk.go@@ -1319,6 +1319,54 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChunreturn deletedCount, nil}+// nextSourceChunkOrder returns the chunk_order_int a newly added source chunk+// should carry: one past the highest order already used by the document's source+// chunks, so manual chunks append after the ingested ones. Parent rows live in+// the reserved range starting at common.ParentChunkOrderBase, so the new chunk+// still lists before every parent of the document — including when the document+// has been parsed (and its parents materialized) long before the chunk was added.+//+// The lookup asks the engine for the document's highest-numbered source chunk.+// doc_type_kwd separates the two ranges without a range condition: every chunker+// output carries it (itemDocType defaults to "text"), it survives the tokenizer,+// and this API writes it below, while parent rows never copy it — so descending+// by chunk_order_int among those rows lands on the last source chunk rather than+// on a parent. A pipeline that emits chunks without doc_type_kwd (custom+// pipelines that never set it) makes the lookup come back empty, and the new+// chunk falls back to order 0, still inside the source range.+func (s *ChunkService) nextSourceChunkOrder(ctx context.Context, indexName, datasetID, documentID string) int {+ // A small page rather than a single row: the lookup only needs the first row+ // that actually carries a source order, so being tolerant of rows without one+ // (products written after the tokenizer, e.g. RAPTOR summaries) keeps this+ // independent of how an engine orders rows missing the sort key in DESC.+ searchResp, err := s.docEngine.Search(ctx, &types.SearchRequest{+ IndexNames: []string{indexName},+ KbIDs: []string{datasetID},+ Limit: 16,+ IncludeUnavailable: true, // a disabled source chunk still holds its order+ SelectFields: []string{"id", "chunk_order_int"},+ OrderBy: (&types.OrderByExpr{}).Desc("chunk_order_int"),+ Filter: map[string]interface{}{+ "doc_id": documentID,+ "exists": "doc_type_kwd",+ "must_not": map[string]interface{}{"exists": "compile_kwd"},+ },+ })+ if err != nil || searchResp == nil {+ // The order only decides where the chunk lists: never fail a write over+ // it. 0 keeps the chunk inside the source range, ahead of the parents.+ return 0+ }+ for _, row := range searchResp.Chunks {+ order, ok := common.GetInt(row["chunk_order_int"])+ if !ok || order < 0 || order >= common.ParentChunkOrderBase {+ continue+ }+ return order + 1+ }+ return 0+}+func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {if s.docEngine == nil {return nil, addChunkError{code: common.CodeServerError, message: "doc engine not initialized"}@@ -1385,9 +1433,17 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequesimportantKeywords = []string{}}+ // Append the manual chunk after the document's existing source chunks. The+ // value stays inside the source range [0, common.ParentChunkOrderBase) so the+ // new chunk keeps listing before the parent rows MaterializeParentChunks+ // created for this document.+ chunkOrder := s.nextSourceChunkOrder(ctx, indexName, req.DatasetID, req.DocumentID)+chunkData := map[string]interface{}{"id": chunkID,"content_with_weight": req.Content,+ "doc_type_kwd": "text", // Python chunk_api.py:1225; the image branch below overrides it+ "chunk_order_int": chunkOrder,"content_ltks": contentLtks,"content_sm_ltks": contentSmLtks,"important_kwd": importantKeywords,diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.goindex cfcf963224..5364e00ac2 100644--- a/internal/service/chunk/chunk_test.go+++ b/internal/service/chunk/chunk_test.go@@ -347,6 +347,13 @@ func TestListSortsChunksByDocumentPosition(t *testing.T) {if !engine.searchReq.IncludeUnavailable {t.Fatal("management chunk list must include disabled parent chunks")}+ // Parent rows must not be filtered out: they are listed after their children+ // on every engine (MaterializeParentChunks numbers them past the source+ // chunks), and excluding them by doc_type_kwd made the list engine-agnostic+ // but no longer showed the parent-child structure users see on ES.+ if _, exists := engine.searchReq.Filter["exists"]; exists {+ t.Fatalf("unexpected existence filter on the chunk list: %#v", engine.searchReq.Filter)+ }if engine.searchReq.OrderBy == nil {t.Fatal("expected OrderBy to be set")}@@ -1066,7 +1073,18 @@ func TestAddChunkSuccess(t *testing.T) {insertChunkTestKB(t, datasetID, userID)insertChunkTestDoc(t, documentID, datasetID)- engine := &addChunkTestEngine{}+ // The document already holds source chunks whose highest order is 9; the+ // lookup page also carries rows the lookup must ignore (a parent-range value+ // and a row without any order), so the added chunk still continues from 9.+ engine := &addChunkTestEngine{+ searchResp: &types.SearchResult{+ Chunks: []map[string]interface{}{+ {"id": "parent-1", "chunk_order_int": float64(common.ParentChunkOrderBase + 3)},+ {"id": "raptor-1"},+ {"id": "chunk-9", "chunk_order_int": float64(9)},+ },+ },+ }var incrementTokenNum, incrementChunkNum int64language := "Slovak"var tokenizeLanguages []string@@ -1164,6 +1182,36 @@ func TestAddChunkSuccess(t *testing.T) {if engine.insertLanguage != language {t.Fatalf("InsertChunks language = %q, want %q", engine.insertLanguage, language)}++ // A manually added text chunk must carry doc_type_kwd (Python chunk_api.py:1225):+ // the management chunk list selects source chunks by that field, so a missing+ // value would hide the chunk from the list.+ if inserted["doc_type_kwd"] != "text" {+ t.Fatalf("doc_type_kwd = %#v, want text", inserted["doc_type_kwd"])+ }+ // The added chunk continues the document's source ordering (highest source+ // order + 1) and stays inside [0, common.ParentChunkOrderBase), so it lists+ // after the ingested chunks and still before the document's parent rows.+ if got := inserted["chunk_order_int"]; got != 10 {+ t.Fatalf("chunk_order_int = %#v, want 10 (highest source order + 1)", got)+ }+ if engine.searchReq == nil {+ t.Fatal("expected the document's highest source order to be looked up")+ }+ if got := engine.searchReq.Filter["exists"]; got != "doc_type_kwd" {+ t.Fatalf("order lookup exists filter = %#v, want doc_type_kwd", got)+ }+ if got := engine.searchReq.Filter["doc_id"]; got != documentID {+ t.Fatalf("order lookup doc_id = %#v, want %q", got, documentID)+ }+ if !engine.searchReq.IncludeUnavailable {+ t.Fatal("order lookup must count disabled source chunks")+ }+ if engine.searchReq.OrderBy == nil || len(engine.searchReq.OrderBy.Fields) != 1 ||+ engine.searchReq.OrderBy.Fields[0].Field != "chunk_order_int" ||+ engine.searchReq.OrderBy.Fields[0].Type != types.SortDesc {+ t.Fatalf("order lookup OrderBy = %#v, want descending chunk_order_int", engine.searchReq.OrderBy)+ }vec, ok := inserted["q_2_vec"].([]float64)if !ok {t.Fatalf("expected q_2_vec []float64, got %T", inserted["q_2_vec"])@@ -2129,6 +2177,8 @@ type addChunkTestEngine struct {insertDataset stringinsertLanguage stringinsertErr error+ searchReq *types.SearchRequest+ searchResp *types.SearchResult}func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {@@ -2139,6 +2189,11 @@ func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[stringreturn nil, e.insertErr}+func (e *addChunkTestEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {+ e.searchReq = req+ return e.searchResp, nil+}+type listChunksSearchEngine struct {parseTestDocEnginesearchReq *types.SearchRequest