Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor[Go]: remove chunker dead code & fix stale/false descriptors (L1/L2/L3) (#17960)

Jack57cacaddf9b1
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 7a07252b97..69fbfa5b97 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -178,8 +178,6 @@ func emptyOutputs() map[string]any {
}
}
-func emptyChunkDocs() []schema.ChunkDoc { return []schema.ChunkDoc{} }
-
// chunkOutputs builds the canonical chunker output (output_format="chunks" +
// chunks). The Go runtime passes only this explicit output to the next node,
// so the run-level metadata that downstream components still need (e.g.
diff --git a/internal/ingestion/component/chunker/image.go b/internal/ingestion/component/chunker/image.go
deleted file mode 100644
index e00b628697..0000000000
--- a/internal/ingestion/component/chunker/image.go
+++ /dev/null
@@ -1,109 +0,0 @@
-//
-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-
-package chunker
-
-import (
- "bytes"
- "image"
- "image/draw"
- _ "image/gif"
- _ "image/jpeg"
- "image/png"
-)
-
-// concatImg vertically concatenates two images represented as raw bytes.
-// Mirrors Python's rag/nlp.concat_img:
-//
-// - img1 is img2 (same slice pointer) → returns img1 unchanged.
-// - one side is nil → returns the non-nil side.
-// - both nil → returns nil.
-// - both non-nil → decodes both, creates a new RGB image of
-// max(width1, width2) × (height1 + height2), pastes img1 at the top
-// and img2 below it, then re-encodes as PNG.
-//
-// Unlike Python's version this function works with raw image bytes rather
-// than PIL Image / LazyImage objects. The LazyImage blob-list merge
-// (LazyImage.merge) is not needed here because Go chunker items carry
-// decoded image bytes rather than deferred-load blobs.
-func concatImg(img1, img2 []byte) []byte {
- // Same-reference guard (Python: img1 is img2).
- if len(img1) > 0 && len(img2) > 0 && &img1[0] == &img2[0] {
- return img1
- }
- // Nil / empty guard (Python: img1 and not img2 → img1, etc.).
- if len(img1) == 0 && len(img2) == 0 {
- return nil
- }
- if len(img1) == 0 {
- return img2
- }
- if len(img2) == 0 {
- return img1
- }
-
- // Decode both images.
- dec1, _, err1 := image.Decode(bytes.NewReader(img1))
- dec2, _, err2 := image.Decode(bytes.NewReader(img2))
- if err1 != nil {
- if err2 != nil {
- return nil
- }
- return img2
- }
- if err2 != nil {
- return img1
- }
-
- // Pixel-data equality guard (Python: img1.tobytes() == img2.tobytes()).
- if img1data, img2data := imgBytes(dec1), imgBytes(dec2); img1data != nil && img2data != nil && bytes.Equal(img1data, img2data) {
- return img1
- }
-
- // Compute dimensions.
- bounds1 := dec1.Bounds()
- bounds2 := dec2.Bounds()
- w1, h1 := bounds1.Dx(), bounds1.Dy()
- w2, h2 := bounds2.Dx(), bounds2.Dy()
-
- newW := w1
- if w2 > newW {
- newW = w2
- }
- newH := h1 + h2
-
- // Create new RGBA image and paste img1 at top, img2 below.
- dst := image.NewRGBA(image.Rect(0, 0, newW, newH))
- draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src)
- draw.Draw(dst, image.Rect(0, 0, w1, h1), dec1, bounds1.Min, draw.Over)
- draw.Draw(dst, image.Rect(0, h1, w2, h1+h2), dec2, bounds2.Min, draw.Over)
-
- var buf bytes.Buffer
- if err := png.Encode(&buf, dst); err != nil {
- return nil
- }
- return buf.Bytes()
-}
-
-// imgBytes returns the raw RGBA pixel data of img as a []byte. Returns nil
-// when img is not convertible (should not happen for formats decoded by
-// the Go image library, which always produce RGBA or NRGBA).
-func imgBytes(img image.Image) []byte {
- b := img.Bounds()
- rgba := image.NewRGBA(b)
- draw.Draw(rgba, b, img, b.Min, draw.Src)
- return rgba.Pix
-}
diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go
index 3c28a24223..3c26c1f258 100644
--- a/internal/ingestion/component/chunker/register.go
+++ b/internal/ingestion/component/chunker/register.go
@@ -14,10 +14,10 @@
// limitations under the License.
//
-// Package chunker holds the ingestion chunker components: TokenChunker,
-// TitleChunker, GroupTitleChunker, HierarchyTitleChunker. The four
-// components share the same upstream payload (schema.ChunkerFromUpstream)
-// and the same output shape (schema.ChunkerOutputs).
+// Package chunker holds the ingestion chunker components. The variants share
+// the same upstream payload (schema.ChunkerFromUpstream) and the same output
+// shape (schema.ChunkerOutputs); each registers via MustRegisterChunker from
+// its own file.
//
// The package is intentionally separate from internal/agent/component/
// (the agent canvas) and from internal/ingestion/component/schema/
@@ -35,7 +35,7 @@ import (
)
// MustRegisterChunker registers a single chunker component under
-// CategoryIngestion. The four chunker files each carry exactly one
+// CategoryIngestion. Each chunker file carries exactly one
// init() that calls this with the registered component's name; the
// factory body resolves the typed constructor via newChunkerByName
// (in common.go).
@@ -100,23 +100,26 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
}
// ChunkerInputs is the static, registered input descriptor shared
-// by all four chunker variants.
+// by all chunker variants.
var ChunkerInputs = map[string]string{
"text": "Plain-text input. The chunker slices this into downstream chunks.",
"content": "Alias for \"text\".",
"chunks": "Optional upstream chunk list (structured JSON form).",
- "name": "Source document name. Required by the upstream payload convention.",
+ "name": "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.",
"_created_time": "Optional upstream timestamp (RFC3339Nano, s).",
"_elapsed_time": "Optional upstream elapsed time (s).",
}
// ChunkerOutputs is the static, registered output descriptor shared
-// by all four chunker variants.
+// by all chunker variants.
+//
+// Note: this map is the component's emitted output only. Run-level metadata
+// that downstream components still need — name, tenant_id, kb_id — is NOT
+// re-emitted here; it lives in the workflow-wide CanvasState.Globals bag and
+// is read directly from ctx (see runtime.CanvasState.Globals and
+// globals.GlobalOrInput). Do not add those keys here.
var ChunkerOutputs = map[string]string{
"output_format": "Always \"chunks\" on success.",
"chunks": "list[object]: per-chunk map (text + optional meta keys).",
- "name": "Source document name, carried forward from upstream (pass-through) when present — Tokenizer consumes it for title embedding.",
- "tenant_id": "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.",
- "kb_id": "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.",
"_ERROR": "Set only on validation failure.",
}
diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.go
new file mode 100644
index 0000000000..5b55ddf326
--- /dev/null
+++ b/internal/ingestion/component/chunker/token_delim_contract_test.go
@@ -0,0 +1,102 @@
+package chunker
+
+import (
+ "context"
+ "strings"
+ "testing"
+)
+
+// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick)
+// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting
+// delimiter_mode + a bare delimiter is effectively a no-op — the text is
+// merged by token_size and the bare token survives inside a chunk rather
+// than acting as a split point. Regression guard for the "bare entries are
+// ignored" contract.
+func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": []string{"::"},
+ "chunk_token_size": float64(8),
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ const text = "alpha::beta::gamma::delta"
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.txt",
+ "output_format": "text",
+ "text": text,
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) == 0 {
+ t.Fatalf("no chunks produced")
+ }
+ var joined strings.Builder
+ for _, ck := range chunks {
+ joined.WriteString(ck["text"].(string))
+ }
+ // No content dropped and the bare "::" is preserved (just chunked by
+ // token size, not split on "::").
+ if joined.String() != text {
+ t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks))
+ }
+}
+
+// TestTokenChunker_MultiByteBacktickDelimiter locks T2: a backtick-wrapped
+// multi-byte (CJK) delimiter contributes its INNER content as the split
+// pattern, and the longest delimiter wins over a shorter prefix of it
+// (rune-descending sort). Both ensure the chunker-level delimiter handling
+// is correct for non-ASCII delimiters.
+func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {
+ // Single multi-byte delimiter splits on its inner content.
+ c, err := NewTokenChunker(map[string]any{
+ "delimiters": []string{"`段落`"},
+ "chunk_token_size": float64(1024),
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.txt",
+ "output_format": "text",
+ "text": "第一部分段落第二部分",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ want := []string{"第一部分", "第二部分"}
+ if len(chunks) != len(want) {
+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+ }
+ for i, w := range want {
+ if got := chunks[i]["text"].(string); got != w {
+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+ }
+ }
+
+ // Longest delimiter must win over its shorter prefix: `段落` (2 runes)
+ // beats `段` (1 rune), so "A段落B" splits on "段落", not "段".
+ c2, err := NewTokenChunker(map[string]any{
+ "delimiters": []string{"`段落`", "`段`"},
+ "chunk_token_size": float64(1024),
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out2, err := c2.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.txt",
+ "output_format": "text",
+ "text": "A段落B",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks2, _ := out2["chunks"].([]map[string]any)
+ if len(chunks2) != 2 || chunks2[0]["text"].(string) != "A" || chunks2[1]["text"].(string) != "B" {
+ t.Errorf("longest delimiter not preferred: got %v", chunkTexts(chunks2))
+ }
+}