Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunk): list parent chunks after source chunks on every doc engine (#20990)

qinling0210131974ce67ba
diff --git a/internal/common/constants.go b/internal/common/constants.go
index bddc6a87e2..85d27dc05c 100644
--- a/internal/common/constants.go
+++ b/internal/common/constants.go
@@ -79,3 +79,23 @@ const StatusDialogValid = "1"
// DialogStatus is a typed alias for dialog.status to avoid raw string
// comparisons in call sites.
type DialogStatus string
+
+// ParentChunkOrderBase is the first chunk_order_int of the range reserved for
+// parent rows — the hidden rows MaterializeParentChunks creates for retrieval
+// promotion. Source chunks live in [0, ParentChunkOrderBase): the tokenizer
+// numbers ingested chunks, and AddChunk appends whatever the user adds after the
+// highest order already present. Keeping the two ranges apart is what makes a
+// parent sort after every source chunk on every engine, including source chunks
+// added after the document was parsed:
+//
+// - Elasticsearch already sorted parents last because the field was simply
+// absent on the parent row (a missing sort key counts as greatest);
+// - Infinity cannot express "absent": an unwritten integer column falls back to
+// its mapping default 0, which tied every parent with the first source chunk,
+// so the management chunk list listed parents first and sliced that tie group
+// differently on every page (LIMIT/OFFSET).
+//
+// The value fits a signed 32-bit integer, the narrowest type the column has
+// across engines (Infinity integer, OceanBase INTEGER, Elasticsearch long), and
+// leaves room for a billion parent rows per document.
+const ParentChunkOrderBase = 1 << 30
diff --git a/internal/common/number.go b/internal/common/number.go
index b735b47af4..c59e789b77 100644
--- a/internal/common/number.go
+++ b/internal/common/number.go
@@ -143,10 +143,20 @@ func PairwiseSum(xs []float64) float64 {
return xs[0]
}
+// GetInt returns the int behind a value and whether it was a number at all.
+// Callers read values that come from config files, JSON decoding, or a document
+// engine row — the last two hand numbers back as int, int32, int64, float32 or
+// float64 depending on the encoder and the driver, so all of them are accepted.
func GetInt(value interface{}) (int, bool) {
switch v := value.(type) {
case int:
return v, true
+ case int32:
+ return int(v), true
+ case int64:
+ return int(v), true
+ case float32:
+ return int(v), true
case float64:
return int(v), true
default:
diff --git a/internal/ingestion/task/indexdoc/parent_child.go b/internal/ingestion/task/indexdoc/parent_child.go
index 67d97a40c9..9fd6b61a3a 100644
--- a/internal/ingestion/task/indexdoc/parent_child.go
+++ b/internal/ingestion/task/indexdoc/parent_child.go
@@ -5,15 +5,36 @@ import (
"strings"
"github.com/cespare/xxhash/v2"
+
+ "ragflow/internal/common"
)
// MaterializeParentChunks turns the chunker's transient parent text into
// hidden index rows and links every child row to its parent. A parent ID is
// scoped to its dataset and document so equal parent text from separate
// documents or datasets cannot overwrite each other in a shared index.
+//
+// Every parent row also receives a chunk_order_int from the range reserved for
+// parents (common.ParentChunkOrderBase + k, where k follows the order in which
+// the parents are discovered, i.e. the reading order of the first child
+// referencing each one), while source chunks stay in [0, ParentChunkOrderBase).
+// That keeps every source chunk — the ones just ingested and the ones a user
+// adds later through AddChunk — ahead of every parent on every engine:
+// Elasticsearch already placed parents last because the field was simply absent
+// on the parent row (a missing sort key counts as greatest), but Infinity cannot
+// express "absent": an unwritten column falls back to the mapping default 0
+// (conf/infinity_mapping.json), which tied every parent with the first source
+// chunk, so the management chunk list listed parents first and sliced that tie
+// group differently on every page (LIMIT/OFFSET).
+//
+// Documents whose chunks carry no chunk_order_int at all are left alone: the
+// parent rows keep the field absent so the Elasticsearch placement is preserved
+// and no ordering is invented for pipelines that never numbered their chunks.
func MaterializeParentChunks(datasetID string, chunks []map[string]any) []map[string]any {
seen := make(map[string]struct{})
parents := make([]map[string]any, 0)
+ order := common.ParentChunkOrderBase
+ ordered := chunksOrdered(chunks)
for _, chunk := range chunks {
mom := parentText(chunk)
@@ -46,12 +67,29 @@ func MaterializeParentChunks(datasetID string, chunks []map[string]any) []map[st
parent[field] = value
}
}
+ if ordered {
+ parent["chunk_order_int"] = order
+ order++
+ }
seen[parentID] = struct{}{}
parents = append(parents, parent)
}
return parents
}
+// chunksOrdered reports whether the document's source chunks are numbered at
+// all. The tokenizer numbers every surviving chunk; a pipeline without one
+// leaves the field unset, and those documents keep their previous placement
+// instead of getting invented order values.
+func chunksOrdered(chunks []map[string]any) bool {
+ for _, chunk := range chunks {
+ if _, ok := common.GetInt(chunk["chunk_order_int"]); ok {
+ return true
+ }
+ }
+ return false
+}
+
// parentChunkID returns the deterministic parent row identifier. NUL
// separators make the dataset, document, and parent-text boundaries
// unambiguous.
diff --git a/internal/ingestion/task/indexdoc/parent_child_test.go b/internal/ingestion/task/indexdoc/parent_child_test.go
index df56f7877a..f21322c4ee 100644
--- a/internal/ingestion/task/indexdoc/parent_child_test.go
+++ b/internal/ingestion/task/indexdoc/parent_child_test.go
@@ -1,6 +1,10 @@
package indexdoc
-import "testing"
+import (
+ "testing"
+
+ "ragflow/internal/common"
+)
func TestParentChunkIDScopesDatasetAndDocument(t *testing.T) {
if got, again := parentChunkID("kb-a", "doc-a", "same parent text"), parentChunkID("kb-a", "doc-a", "same parent text"); got != again {
@@ -29,3 +33,75 @@ func TestMaterializeParentChunksScopesSameParentTextToDocument(t *testing.T) {
t.Fatalf("parent docs = %#v, want one parent per document", parents)
}
}
+
+// TestMaterializeParentChunksOrdersParentsAfterChildren pins the ordering
+// contract every chunk_order_int reader relies on (management chunk list,
+// agent grep / deep read): a parent sorts after all source chunks of its
+// document, its key is unique within that document so LIMIT/OFFSET and
+// search_after pagination slice the same group the same way on every engine, and
+// it sits in the reserved parent range so chunks added later through AddChunk
+// (which keep appending into [0, ParentChunkOrderBase)) still list before it.
+func TestMaterializeParentChunksOrdersParentsAfterChildren(t *testing.T) {
+ chunks := []map[string]any{
+ {"id": "child-1", "doc_id": "doc-1", "mom": "parent one", "chunk_order_int": float64(0)},
+ {"id": "child-2", "doc_id": "doc-1", "mom": "parent one", "chunk_order_int": float64(1)},
+ {"id": "child-3", "doc_id": "doc-1", "chunk_order_int": float64(2)},
+ {"id": "child-4", "doc_id": "doc-1", "mom": "parent two", "chunk_order_int": float64(3)},
+ }
+
+ parents := MaterializeParentChunks("kb-1", chunks)
+
+ if len(parents) != 2 {
+ t.Fatalf("parent count = %d, want 2", len(parents))
+ }
+ for i, want := range []int{common.ParentChunkOrderBase, common.ParentChunkOrderBase + 1} {
+ if got, ok := common.GetInt(parents[i]["chunk_order_int"]); !ok || got != want {
+ t.Errorf("parent %d chunk_order_int = %v, want %d", i, parents[i]["chunk_order_int"], want)
+ }
+ }
+ // Source chunks keep their own values; the parent values are what moves.
+ for i, want := range []int{0, 1, 2, 3} {
+ if got, ok := common.GetInt(chunks[i]["chunk_order_int"]); !ok || got != want {
+ t.Errorf("child %d chunk_order_int = %v, want %d", i, chunks[i]["chunk_order_int"], want)
+ }
+ }
+
+ seen := map[int]bool{}
+ rows := append(append([]map[string]any{}, parents...), chunks...)
+ for _, row := range rows {
+ order, ok := common.GetInt(row["chunk_order_int"])
+ if !ok {
+ t.Fatalf("row %#v has no numeric chunk_order_int", row)
+ }
+ if seen[order] {
+ t.Fatalf("chunk_order_int %d is not unique per document: %#v", order, rows)
+ }
+ seen[order] = true
+ }
+
+ // The document's real slice count must not be affected by the extra rows.
+ if chunks[0]["mom_id"] != chunks[1]["mom_id"] {
+ t.Fatalf("children of the same parent lost their shared mom_id: %#v", chunks)
+ }
+ if chunks[2]["mom_id"] != nil {
+ t.Fatalf("child without a parent must not gain a mom_id: %#v", chunks[2])
+ }
+}
+
+// TestMaterializeParentChunksLeavesUnnumberedDocumentsAlone keeps the
+// Elasticsearch placement for pipelines that never numbered their chunks:
+// inventing values there would move parents from "missing (last)" to the front.
+func TestMaterializeParentChunksLeavesUnnumberedDocumentsAlone(t *testing.T) {
+ chunks := []map[string]any{
+ {"id": "child-1", "doc_id": "doc-1", "mom": "parent one"},
+ }
+
+ parents := MaterializeParentChunks("kb-1", chunks)
+
+ if len(parents) != 1 {
+ t.Fatalf("parent count = %d, want 1", len(parents))
+ }
+ if _, exists := parents[0]["chunk_order_int"]; exists {
+ t.Fatalf("parent chunk_order_int = %v, want it absent", parents[0]["chunk_order_int"])
+ }
+}
diff --git a/internal/service/chunk/chunk.go b/internal/service/chunk/chunk.go
index 232c90d62d..8d3b641470 100644
--- a/internal/service/chunk/chunk.go
+++ b/internal/service/chunk/chunk.go
@@ -1319,6 +1319,54 @@ func (s *ChunkService) RemoveChunks(ctx context.Context, req *service.RemoveChun
return deletedCount, nil
}
+// nextSourceChunkOrder returns the chunk_order_int a newly added source chunk
+// should carry: one past the highest order already used by the document's source
+// chunks, so manual chunks append after the ingested ones. Parent rows live in
+// the reserved range starting at common.ParentChunkOrderBase, so the new chunk
+// still lists before every parent of the document — including when the document
+// has been parsed (and its parents materialized) long before the chunk was added.
+//
+// The lookup asks the engine for the document's highest-numbered source chunk.
+// doc_type_kwd separates the two ranges without a range condition: every chunker
+// output carries it (itemDocType defaults to "text"), it survives the tokenizer,
+// and this API writes it below, while parent rows never copy it — so descending
+// by chunk_order_int among those rows lands on the last source chunk rather than
+// on a parent. A pipeline that emits chunks without doc_type_kwd (custom
+// pipelines that never set it) makes the lookup come back empty, and the new
+// chunk falls back to order 0, still inside the source range.
+func (s *ChunkService) nextSourceChunkOrder(ctx context.Context, indexName, datasetID, documentID string) int {
+ // A small page rather than a single row: the lookup only needs the first row
+ // that actually carries a source order, so being tolerant of rows without one
+ // (products written after the tokenizer, e.g. RAPTOR summaries) keeps this
+ // independent of how an engine orders rows missing the sort key in DESC.
+ searchResp, err := s.docEngine.Search(ctx, &types.SearchRequest{
+ IndexNames: []string{indexName},
+ KbIDs: []string{datasetID},
+ Limit: 16,
+ IncludeUnavailable: true, // a disabled source chunk still holds its order
+ SelectFields: []string{"id", "chunk_order_int"},
+ OrderBy: (&types.OrderByExpr{}).Desc("chunk_order_int"),
+ Filter: map[string]interface{}{
+ "doc_id": documentID,
+ "exists": "doc_type_kwd",
+ "must_not": map[string]interface{}{"exists": "compile_kwd"},
+ },
+ })
+ if err != nil || searchResp == nil {
+ // The order only decides where the chunk lists: never fail a write over
+ // it. 0 keeps the chunk inside the source range, ahead of the parents.
+ return 0
+ }
+ for _, row := range searchResp.Chunks {
+ order, ok := common.GetInt(row["chunk_order_int"])
+ if !ok || order < 0 || order >= common.ParentChunkOrderBase {
+ continue
+ }
+ return order + 1
+ }
+ return 0
+}
+
func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkRequest, userID string) (*service.AddChunkResponse, error) {
if s.docEngine == nil {
return nil, addChunkError{code: common.CodeServerError, message: "doc engine not initialized"}
@@ -1385,9 +1433,17 @@ func (s *ChunkService) AddChunk(ctx context.Context, req *service.AddChunkReques
importantKeywords = []string{}
}
+ // Append the manual chunk after the document's existing source chunks. The
+ // value stays inside the source range [0, common.ParentChunkOrderBase) so the
+ // new chunk keeps listing before the parent rows MaterializeParentChunks
+ // created for this document.
+ chunkOrder := s.nextSourceChunkOrder(ctx, indexName, req.DatasetID, req.DocumentID)
+
chunkData := map[string]interface{}{
"id": chunkID,
"content_with_weight": req.Content,
+ "doc_type_kwd": "text", // Python chunk_api.py:1225; the image branch below overrides it
+ "chunk_order_int": chunkOrder,
"content_ltks": contentLtks,
"content_sm_ltks": contentSmLtks,
"important_kwd": importantKeywords,
diff --git a/internal/service/chunk/chunk_test.go b/internal/service/chunk/chunk_test.go
index cfcf963224..5364e00ac2 100644
--- a/internal/service/chunk/chunk_test.go
+++ b/internal/service/chunk/chunk_test.go
@@ -347,6 +347,13 @@ func TestListSortsChunksByDocumentPosition(t *testing.T) {
if !engine.searchReq.IncludeUnavailable {
t.Fatal("management chunk list must include disabled parent chunks")
}
+ // Parent rows must not be filtered out: they are listed after their children
+ // on every engine (MaterializeParentChunks numbers them past the source
+ // chunks), and excluding them by doc_type_kwd made the list engine-agnostic
+ // but no longer showed the parent-child structure users see on ES.
+ if _, exists := engine.searchReq.Filter["exists"]; exists {
+ t.Fatalf("unexpected existence filter on the chunk list: %#v", engine.searchReq.Filter)
+ }
if engine.searchReq.OrderBy == nil {
t.Fatal("expected OrderBy to be set")
}
@@ -1066,7 +1073,18 @@ func TestAddChunkSuccess(t *testing.T) {
insertChunkTestKB(t, datasetID, userID)
insertChunkTestDoc(t, documentID, datasetID)
- engine := &addChunkTestEngine{}
+ // The document already holds source chunks whose highest order is 9; the
+ // lookup page also carries rows the lookup must ignore (a parent-range value
+ // and a row without any order), so the added chunk still continues from 9.
+ engine := &addChunkTestEngine{
+ searchResp: &types.SearchResult{
+ Chunks: []map[string]interface{}{
+ {"id": "parent-1", "chunk_order_int": float64(common.ParentChunkOrderBase + 3)},
+ {"id": "raptor-1"},
+ {"id": "chunk-9", "chunk_order_int": float64(9)},
+ },
+ },
+ }
var incrementTokenNum, incrementChunkNum int64
language := "Slovak"
var tokenizeLanguages []string
@@ -1164,6 +1182,36 @@ func TestAddChunkSuccess(t *testing.T) {
if engine.insertLanguage != language {
t.Fatalf("InsertChunks language = %q, want %q", engine.insertLanguage, language)
}
+
+ // A manually added text chunk must carry doc_type_kwd (Python chunk_api.py:1225):
+ // the management chunk list selects source chunks by that field, so a missing
+ // value would hide the chunk from the list.
+ if inserted["doc_type_kwd"] != "text" {
+ t.Fatalf("doc_type_kwd = %#v, want text", inserted["doc_type_kwd"])
+ }
+ // The added chunk continues the document's source ordering (highest source
+ // order + 1) and stays inside [0, common.ParentChunkOrderBase), so it lists
+ // after the ingested chunks and still before the document's parent rows.
+ if got := inserted["chunk_order_int"]; got != 10 {
+ t.Fatalf("chunk_order_int = %#v, want 10 (highest source order + 1)", got)
+ }
+ if engine.searchReq == nil {
+ t.Fatal("expected the document's highest source order to be looked up")
+ }
+ if got := engine.searchReq.Filter["exists"]; got != "doc_type_kwd" {
+ t.Fatalf("order lookup exists filter = %#v, want doc_type_kwd", got)
+ }
+ if got := engine.searchReq.Filter["doc_id"]; got != documentID {
+ t.Fatalf("order lookup doc_id = %#v, want %q", got, documentID)
+ }
+ if !engine.searchReq.IncludeUnavailable {
+ t.Fatal("order lookup must count disabled source chunks")
+ }
+ if engine.searchReq.OrderBy == nil || len(engine.searchReq.OrderBy.Fields) != 1 ||
+ engine.searchReq.OrderBy.Fields[0].Field != "chunk_order_int" ||
+ engine.searchReq.OrderBy.Fields[0].Type != types.SortDesc {
+ t.Fatalf("order lookup OrderBy = %#v, want descending chunk_order_int", engine.searchReq.OrderBy)
+ }
vec, ok := inserted["q_2_vec"].([]float64)
if !ok {
t.Fatalf("expected q_2_vec []float64, got %T", inserted["q_2_vec"])
@@ -2129,6 +2177,8 @@ type addChunkTestEngine struct {
insertDataset string
insertLanguage string
insertErr error
+ searchReq *types.SearchRequest
+ searchResp *types.SearchResult
}
func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string]interface{}, baseName string, datasetID string, language string) ([]string, error) {
@@ -2139,6 +2189,11 @@ func (e *addChunkTestEngine) InsertChunks(_ context.Context, chunks []map[string
return nil, e.insertErr
}
+func (e *addChunkTestEngine) Search(_ context.Context, req *types.SearchRequest) (*types.SearchResult, error) {
+ e.searchReq = req
+ return e.searchResp, nil
+}
+
type listChunksSearchEngine struct {
parseTestDocEngine
searchReq *types.SearchRequest