Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

8.0 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

// Slice 2 tests for port-rag-flow-pipeline-to-go.md Phase 3.
// Pin the chunkFromItem pass-through of python-side fields:
//
//   - mom          — parent-section identifier (title / hierarchy
//                    chunkers populate; TokenChunker forwards it).
//   - img_id       — image attachment identifier.
//   - layout       — layout classification.
//   - _pdf_positions — PDF bbox coordinates.
//
// Each test seeds a JSON-style chunk item with the field set,
// invokes TokenChunker through its public path, and asserts the
// emitted chunk carries the field through unchanged.

package chunker

import (
	"context"
	"strings"
	"testing"

	"ragflow/internal/agent/runtime"
	"ragflow/internal/ingestion/component/schema"
)

// invokeAsTokenChunker constructs a TokenChunker component and
// drives a single JSON-payload Invoke. Mirrors the production
// token.go:invokeJSONPayload path.
func invokeAsTokenChunker(t *testing.T, items []map[string]any) map[string]any {
	t.Helper()
	comp, err := NewTokenChunker(map[string]any{})
	if err != nil {
		t.Fatalf("NewTokenChunker: %v", err)
	}
	out, err := comp.Invoke(context.Background(), nil, map[string]any{
		"name":          "test",
		"output_format": "json",
		"json":          items,
	})
	if err != nil {
		t.Fatalf("TokenChunker.Invoke: %v", err)
	}
	return out
}

// TestChunker_PreservesMomField pins the mom pass-through. Title
// and hierarchy chunkers populate mom on each item; the
// TokenChunker pass-through must preserve it on the emitted
// chunk so downstream components can re-attach the parent section.
func TestChunker_PreservesMomField(t *testing.T) {
	items := []map[string]any{
		{"text": "Section body.", "doc_type_kwd": "text", "mom": "sec-1"},
	}
	out := invokeAsTokenChunker(t, items)
	chunks := chunksFromOutput(t, out)
	if len(chunks) == 0 {
		t.Fatal("no chunks emitted")
	}
	if got, want := chunks[0]["mom"], "sec-1"; got != want {
		t.Errorf("chunks[0].mom = %v, want %v", got, want)
	}
}

// TestChunker_PreservesImgIDField pins the img_id pass-through.
// PDF / DOCX parsers attach img_id to image-bearing items; the
// chunker must preserve it so downstream image2id can upload
// the binary to MinIO.
func TestChunker_PreservesImgIDField(t *testing.T) {
	items := []map[string]any{
		{"text": "Caption.", "doc_type_kwd": "image", "img_id": "img-007"},
	}
	out := invokeAsTokenChunker(t, items)
	chunks := chunksFromOutput(t, out)
	if len(chunks) == 0 {
		t.Fatal("no chunks emitted")
	}
	if got, want := chunks[0]["img_id"], "img-007"; got != want {
		t.Errorf("chunks[0].img_id = %v, want %v", got, want)
	}
}

// TestChunker_PreservesLayoutField pins the layout pass-through.
// Layout-aware components (token_chunker._layout) use this field
// to switch processing paths (text vs table vs figure).
func TestChunker_PreservesLayoutField(t *testing.T) {
	items := []map[string]any{
		{"text": "Cell A1 | Cell B1\nCell A2 | Cell B2", "doc_type_kwd": "table", "layout": "table"},
	}
	out := invokeAsTokenChunker(t, items)
	chunks := chunksFromOutput(t, out)
	if len(chunks) == 0 {
		t.Fatal("no chunks emitted")
	}
	if got, want := chunks[0]["layout"], "table"; got != want {
		t.Errorf("chunks[0].layout = %v, want %v", got, want)
	}
}

// TestChunker_PassesPDFPositionsThrough pins the _pdf_positions
// pass-through. DeepDOC's PDF parser attaches bounding boxes under this key;
// downstream layout-aware components use them to rebuild page geometry.
func TestChunker_PassesPDFPositionsThrough(t *testing.T) {
	positions := [][]float64{{0.1, 0.2, 0.3, 0.4}, {0.5, 0.6, 0.7, 0.8}}
	items := []map[string]any{
		{"text": "PDF paragraph.", "doc_type_kwd": "text", "_pdf_positions": positions},
	}
	out := invokeAsTokenChunker(t, items)
	chunks := chunksFromOutput(t, out)
	if len(chunks) == 0 {
		t.Fatal("no chunks emitted")
	}
	got, ok := chunks[0]["_pdf_positions"]
	if !ok {
		t.Fatalf("chunks[0] missing _pdf_positions; keys=%v", keysOf(chunks[0]))
	}
	if len(got.([][]float64)) != 2 {
		t.Errorf("chunks[0]._pdf_positions len = %d, want 2", len(got.([][]float64)))
	}
}

func TestChunker_PreservesVisualParentMetadata(t *testing.T) {
	items := []map[string]any{
		{
			"text":            "OCR text from a table image",
			"doc_type_kwd":    "image",
			"parent_table_id": "docx-table-1",
			"media_order":     2,
		},
	}
	out := invokeAsTokenChunker(t, items)
	chunks := chunksFromOutput(t, out)
	if len(chunks) == 0 {
		t.Fatal("no chunks emitted")
	}
	if got, want := chunks[0]["parent_table_id"], "docx-table-1"; got != want {
		t.Errorf("chunks[0].parent_table_id = %v, want %v", got, want)
	}
	if got, want := chunks[0]["media_order"], float64(2); got != want {
		t.Errorf("chunks[0].media_order = %v, want %v", got, want)
	}
}

// TestChunker_DropsContentWithWeightOnlyItems enforces the pre-index wire
// contract: chunkers consume canonical "text" only.
func TestChunker_DropsContentWithWeightOnlyItems(t *testing.T) {
	items := []map[string]any{
		{"content_with_weight": "fallback text", "doc_type_kwd": "text"},
	}
	out := invokeAsTokenChunker(t, items)
	chunks := chunksFromOutput(t, out)
	if len(chunks) != 0 {
		t.Fatalf("content_with_weight-only items must not produce chunks, got %d", len(chunks))
	}
}

// TestChunker_EmptyInputProducesNoChunks pins the empty-input
// behaviour. An empty items slice should produce an empty chunks
// list (mirrors python `_build_json_chunks` for an empty input).
func TestChunker_EmptyInputProducesNoChunks(t *testing.T) {
	out := invokeAsTokenChunker(t, nil)
	chunks := chunksFromOutput(t, out)
	if len(chunks) != 0 {
		t.Errorf("empty input produced %d chunks, want 0", len(chunks))
	}
}

// chunksFromOutput pulls the chunks list out of a TokenChunker
// output. The component emits output_format=chunks with the list
// under "chunks".
func chunksFromOutput(t *testing.T, out map[string]any) []map[string]any {
	t.Helper()
	raw, ok := out["chunks"]
	if !ok {
		t.Fatalf("output missing 'chunks' key; got keys=%v", keysOf(out))
	}
	maps, ok := schema.ChunkCarrierToMaps(raw)
	if !ok {
		t.Fatalf("chunks: unexpected carrier type %T", raw)
	}
	return maps
}

func keysOf(m map[string]any) []string {
	out := make([]string, 0, len(m))
	for k := range m {
		out = append(out, k)
	}
	return out
}

// TestSlice2_RegisteredContract pins the runtime registry contract
// for TokenChunker. Phase 3 docs reference it as the canonical
// chunker component; this test ensures the registration survived
// the chunkFromItem refactor.
func TestSlice2_RegisteredContract(t *testing.T) {
	factory, category, _, ok := runtime.DefaultRegistry.Lookup("TokenChunker")
	if !ok || factory == nil {
		t.Fatal("TokenChunker not registered in DefaultRegistry")
	}
	if category != runtime.CategoryIngestion {
		t.Errorf("category = %q, want %q", category, runtime.CategoryIngestion)
	}
}

// TestSlice2_NoStringOnlyLayout silently checks that the
// buildChunkMap helper does not panic when the input item has
// layout="string-with-spaces" (a regex error scenario). The
// pass-through uses a free-form map copy, so any string is OK.
func TestSlice2_NoStringOnlyLayout(t *testing.T) {
	items := []map[string]any{
		{"text": "x", "doc_type_kwd": "text", "layout": "figure with caption"},
	}
	out := invokeAsTokenChunker(t, items)
	chunks := chunksFromOutput(t, out)
	if chunks[0]["layout"] != "figure with caption" {
		t.Errorf("layout pass-through broken: got %v", chunks[0]["layout"])
	}
	_ = strings.TrimSpace // avoid unused-import warning when the test list shrinks
}