RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor[Go]: remove chunker dead code & fix stale/false descriptors (L1/L2/L3) (#17960)
Jack57cacaddf9b1
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 7a07252b97..69fbfa5b97 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -178,8 +178,6 @@ func emptyOutputs() map[string]any {}}-func emptyChunkDocs() []schema.ChunkDoc { return []schema.ChunkDoc{} }-// chunkOutputs builds the canonical chunker output (output_format="chunks" +// chunks). The Go runtime passes only this explicit output to the next node,// so the run-level metadata that downstream components still need (e.g.diff --git a/internal/ingestion/component/chunker/image.go b/internal/ingestion/component/chunker/image.godeleted file mode 100644index e00b628697..0000000000--- a/internal/ingestion/component/chunker/image.go+++ /dev/null@@ -1,109 +0,0 @@-//-// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.-//-// Licensed under the Apache License, Version 2.0 (the "License");-// you may not use this file except in compliance with the License.-// You may obtain a copy of the License at-//-// http://www.apache.org/licenses/LICENSE-2.0-//-// Unless required by applicable law or agreed to in writing, software-// distributed under the License is distributed on an "AS IS" BASIS,-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-// See the License for the specific language governing permissions and-// limitations under the License.-//--package chunker--import (- "bytes"- "image"- "image/draw"- _ "image/gif"- _ "image/jpeg"- "image/png"-)--// concatImg vertically concatenates two images represented as raw bytes.-// Mirrors Python's rag/nlp.concat_img:-//-// - img1 is img2 (same slice pointer) → returns img1 unchanged.-// - one side is nil → returns the non-nil side.-// - both nil → returns nil.-// - both non-nil → decodes both, creates a new RGB image of-// max(width1, width2) × (height1 + height2), pastes img1 at the top-// and img2 below it, then re-encodes as PNG.-//-// Unlike Python's version this function works with raw image bytes rather-// than PIL Image / LazyImage objects. The LazyImage blob-list merge-// (LazyImage.merge) is not needed here because Go chunker items carry-// decoded image bytes rather than deferred-load blobs.-func concatImg(img1, img2 []byte) []byte {- // Same-reference guard (Python: img1 is img2).- if len(img1) > 0 && len(img2) > 0 && &img1[0] == &img2[0] {- return img1- }- // Nil / empty guard (Python: img1 and not img2 → img1, etc.).- if len(img1) == 0 && len(img2) == 0 {- return nil- }- if len(img1) == 0 {- return img2- }- if len(img2) == 0 {- return img1- }-- // Decode both images.- dec1, _, err1 := image.Decode(bytes.NewReader(img1))- dec2, _, err2 := image.Decode(bytes.NewReader(img2))- if err1 != nil {- if err2 != nil {- return nil- }- return img2- }- if err2 != nil {- return img1- }-- // Pixel-data equality guard (Python: img1.tobytes() == img2.tobytes()).- if img1data, img2data := imgBytes(dec1), imgBytes(dec2); img1data != nil && img2data != nil && bytes.Equal(img1data, img2data) {- return img1- }-- // Compute dimensions.- bounds1 := dec1.Bounds()- bounds2 := dec2.Bounds()- w1, h1 := bounds1.Dx(), bounds1.Dy()- w2, h2 := bounds2.Dx(), bounds2.Dy()-- newW := w1- if w2 > newW {- newW = w2- }- newH := h1 + h2-- // Create new RGBA image and paste img1 at top, img2 below.- dst := image.NewRGBA(image.Rect(0, 0, newW, newH))- draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src)- draw.Draw(dst, image.Rect(0, 0, w1, h1), dec1, bounds1.Min, draw.Over)- draw.Draw(dst, image.Rect(0, h1, w2, h1+h2), dec2, bounds2.Min, draw.Over)-- var buf bytes.Buffer- if err := png.Encode(&buf, dst); err != nil {- return nil- }- return buf.Bytes()-}--// imgBytes returns the raw RGBA pixel data of img as a []byte. Returns nil-// when img is not convertible (should not happen for formats decoded by-// the Go image library, which always produce RGBA or NRGBA).-func imgBytes(img image.Image) []byte {- b := img.Bounds()- rgba := image.NewRGBA(b)- draw.Draw(rgba, b, img, b.Min, draw.Src)- return rgba.Pix-}diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.goindex 3c28a24223..3c26c1f258 100644--- a/internal/ingestion/component/chunker/register.go+++ b/internal/ingestion/component/chunker/register.go@@ -14,10 +14,10 @@// limitations under the License.//-// Package chunker holds the ingestion chunker components: TokenChunker,-// TitleChunker, GroupTitleChunker, HierarchyTitleChunker. The four-// components share the same upstream payload (schema.ChunkerFromUpstream)-// and the same output shape (schema.ChunkerOutputs).+// Package chunker holds the ingestion chunker components. The variants share+// the same upstream payload (schema.ChunkerFromUpstream) and the same output+// shape (schema.ChunkerOutputs); each registers via MustRegisterChunker from+// its own file.//// The package is intentionally separate from internal/agent/component/// (the agent canvas) and from internal/ingestion/component/schema/@@ -35,7 +35,7 @@ import ()// MustRegisterChunker registers a single chunker component under-// CategoryIngestion. The four chunker files each carry exactly one+// CategoryIngestion. Each chunker file carries exactly one// init() that calls this with the registered component's name; the// factory body resolves the typed constructor via newChunkerByName// (in common.go).@@ -100,23 +100,26 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m}// ChunkerInputs is the static, registered input descriptor shared-// by all four chunker variants.+// by all chunker variants.var ChunkerInputs = map[string]string{"text": "Plain-text input. The chunker slices this into downstream chunks.","content": "Alias for \"text\".","chunks": "Optional upstream chunk list (structured JSON form).",- "name": "Source document name. Required by the upstream payload convention.",+ "name": "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.","_created_time": "Optional upstream timestamp (RFC3339Nano, s).","_elapsed_time": "Optional upstream elapsed time (s).",}// ChunkerOutputs is the static, registered output descriptor shared-// by all four chunker variants.+// by all chunker variants.+//+// Note: this map is the component's emitted output only. Run-level metadata+// that downstream components still need — name, tenant_id, kb_id — is NOT+// re-emitted here; it lives in the workflow-wide CanvasState.Globals bag and+// is read directly from ctx (see runtime.CanvasState.Globals and+// globals.GlobalOrInput). Do not add those keys here.var ChunkerOutputs = map[string]string{"output_format": "Always \"chunks\" on success.","chunks": "list[object]: per-chunk map (text + optional meta keys).",- "name": "Source document name, carried forward from upstream (pass-through) when present — Tokenizer consumes it for title embedding.",- "tenant_id": "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.",- "kb_id": "Carried forward from upstream (pass-through) when present — Tokenizer consumes it to resolve the embedding model.","_ERROR": "Set only on validation failure.",}diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.gonew file mode 100644index 0000000000..5b55ddf326--- /dev/null+++ b/internal/ingestion/component/chunker/token_delim_contract_test.go@@ -0,0 +1,102 @@+package chunker++import (+ "context"+ "strings"+ "testing"+)++// TestTokenChunker_BareDelimiterIgnored locks T1: a bare (non-backtick)+// delimiter entry is IGNORED by CompileDelimiterListPattern, so setting+// delimiter_mode + a bare delimiter is effectively a no-op — the text is+// merged by token_size and the bare token survives inside a chunk rather+// than acting as a split point. Regression guard for the "bare entries are+// ignored" contract.+func TestTokenChunker_BareDelimiterIgnored(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{"::"},+ "chunk_token_size": float64(8),+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ const text = "alpha::beta::gamma::delta"+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.txt",+ "output_format": "text",+ "text": text,+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) == 0 {+ t.Fatalf("no chunks produced")+ }+ var joined strings.Builder+ for _, ck := range chunks {+ joined.WriteString(ck["text"].(string))+ }+ // No content dropped and the bare "::" is preserved (just chunked by+ // token size, not split on "::").+ if joined.String() != text {+ t.Errorf("bare delimiter not ignored: joined=%q want %q (chunks=%v)", joined.String(), text, chunkTexts(chunks))+ }+}++// TestTokenChunker_MultiByteBacktickDelimiter locks T2: a backtick-wrapped+// multi-byte (CJK) delimiter contributes its INNER content as the split+// pattern, and the longest delimiter wins over a shorter prefix of it+// (rune-descending sort). Both ensure the chunker-level delimiter handling+// is correct for non-ASCII delimiters.+func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {+ // Single multi-byte delimiter splits on its inner content.+ c, err := NewTokenChunker(map[string]any{+ "delimiters": []string{"`段落`"},+ "chunk_token_size": float64(1024),+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.txt",+ "output_format": "text",+ "text": "第一部分段落第二部分",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ want := []string{"第一部分", "第二部分"}+ if len(chunks) != len(want) {+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))+ }+ for i, w := range want {+ if got := chunks[i]["text"].(string); got != w {+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)+ }+ }++ // Longest delimiter must win over its shorter prefix: `段落` (2 runes)+ // beats `段` (1 rune), so "A段落B" splits on "段落", not "段".+ c2, err := NewTokenChunker(map[string]any{+ "delimiters": []string{"`段落`", "`段`"},+ "chunk_token_size": float64(1024),+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out2, err := c2.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.txt",+ "output_format": "text",+ "text": "A段落B",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks2, _ := out2["chunks"].([]map[string]any)+ if len(chunks2) != 2 || chunks2[0]["text"].(string) != "A" || chunks2[1]["text"].(string) != "B" {+ t.Errorf("longest delimiter not preferred: got %v", chunkTexts(chunks2))+ }+}