Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(ingestion): migrate general parser and chunker contract (#19611)

jay777213a04af1dcd91
diff --git a/internal/agent/canvas/scheduler.go b/internal/agent/canvas/scheduler.go
index b836357e70..65d4986ef6 100644
--- a/internal/agent/canvas/scheduler.go
+++ b/internal/agent/canvas/scheduler.go
@@ -124,6 +124,8 @@ func isKnownPrimitive(name string) bool {
"stringtransform", "variableaggregator", "variableassigner",
"loop", "parallel": // macros in BuildWorkflow; the pre-pass absorbs them.
return true
+ case "generalchunker":
+ return true
}
return false
}
diff --git a/internal/agent/dsl/normalize.go b/internal/agent/dsl/normalize.go
index 9f6e184790..45db7f2990 100644
--- a/internal/agent/dsl/normalize.go
+++ b/internal/agent/dsl/normalize.go
@@ -601,6 +601,7 @@ var componentNameToNodeTypeMap = map[string]string{
"File": "fileNode",
"Parser": "parserNode",
"Tokenizer": "tokenizerNode",
+ "GeneralChunker": "chunkerNode",
"TokenChunker": "chunkerNode",
"TitleChunker": "chunkerNode",
"OneChunker": "chunkerNode",
diff --git a/internal/agent/dsl/normalize_test.go b/internal/agent/dsl/normalize_test.go
index 424cb595c8..79fb49a245 100644
--- a/internal/agent/dsl/normalize_test.go
+++ b/internal/agent/dsl/normalize_test.go
@@ -48,6 +48,12 @@ func loadFixture(t *testing.T, name string) map[string]any {
return m
}
+func TestComponentNameToNodeTypeRecognizesGeneralChunker(t *testing.T) {
+ if got := componentNameToNodeType("GeneralChunker"); got != "chunkerNode" {
+ t.Fatalf("GeneralChunker node type = %q, want chunkerNode", got)
+ }
+}
+
// TestNormalize_NoopWhenGraphPresent guards against accidentally
// clobbering a payload the front-end just saved. Any input with a
// non-empty `graph.nodes` must round-trip with `graph` untouched.
diff --git a/internal/dao/migration.go b/internal/dao/migration.go
index a99af13081..559d4d2128 100644
--- a/internal/dao/migration.go
+++ b/internal/dao/migration.go
@@ -68,6 +68,10 @@ func RunMigrations(ctx context.Context, db *gorm.DB) error {
return fmt.Errorf("failed to add unique index on user_canvas (user_id, canvas_category, title): %w", err)
}
+ if err := migrateGeneralChunkerParserConfigs(ctx, db); err != nil {
+ return fmt.Errorf("failed to migrate general chunker parser configs: %w", err)
+ }
+
common.Info("All manual migrations completed successfully")
return nil
}
diff --git a/internal/dao/parser_config_migration.go b/internal/dao/parser_config_migration.go
new file mode 100644
index 0000000000..eb06bbd351
--- /dev/null
+++ b/internal/dao/parser_config_migration.go
@@ -0,0 +1,168 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package dao
+
+import (
+ "context"
+ "fmt"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+ parserchunk "ragflow/internal/parser/chunk"
+
+ "go.uber.org/zap"
+ "gorm.io/gorm"
+)
+
+const (
+ legacyGeneralChunkerID = "TokenChunker:SixApplesFall"
+ currentGeneralChunkerID = "GeneralChunker:SixApplesFall"
+)
+
+var generalChunkerParamKeys = map[string]struct{}{
+ "children_delimiters": {},
+ "chunk_token_size": {},
+ "delimiter": {},
+ "delimiters": {},
+ "image_context_size": {},
+ "overlapped_percent": {},
+ "table_context_size": {},
+}
+
+// migrateGeneralChunkerConfig rewrites the exact legacy component key used by
+// the built-in general pipeline. Explicit values already stored under the new
+// key win; unsupported TokenChunker-only fields are dropped because
+// GeneralChunker has no delimiter_mode or outputs parameter.
+func migrateGeneralChunkerConfig(config map[string]interface{}) bool {
+ legacy, ok := config[legacyGeneralChunkerID]
+ if !ok {
+ return false
+ }
+ legacyParams, ok := legacy.(map[string]interface{})
+ if !ok {
+ return false
+ }
+
+ if _, exists := config[currentGeneralChunkerID]; !exists {
+ params := make(map[string]interface{}, len(legacyParams))
+ for key, value := range legacyParams {
+ if key == "delimiter" {
+ if _, exists := legacyParams["delimiters"]; !exists {
+ if delimiters, ok := normalizeLegacyDelimiter(value); ok {
+ params["delimiters"] = delimiters
+ }
+ }
+ continue
+ }
+ if _, supported := generalChunkerParamKeys[key]; supported {
+ params[key] = value
+ }
+ }
+ config[currentGeneralChunkerID] = params
+ }
+ delete(config, legacyGeneralChunkerID)
+ return true
+}
+
+func migrateGeneralChunkerParserConfigs(ctx context.Context, db *gorm.DB) error {
+ if db == nil {
+ return nil
+ }
+
+ var migratedKnowledgebases int
+ if db.WithContext(ctx).Migrator().HasTable("knowledgebase") {
+ count, err := migrateParserConfigRows(ctx, db, &entity.Knowledgebase{})
+ if err != nil {
+ return fmt.Errorf("migrate knowledgebase parser configs: %w", err)
+ }
+ migratedKnowledgebases = count
+ }
+
+ var migratedDocuments int
+ if db.WithContext(ctx).Migrator().HasTable("document") {
+ count, err := migrateParserConfigRows(ctx, db, &entity.Document{})
+ if err != nil {
+ return fmt.Errorf("migrate document parser configs: %w", err)
+ }
+ migratedDocuments = count
+ }
+
+ if migratedKnowledgebases > 0 || migratedDocuments > 0 {
+ common.Info("Migrated general chunker parser configs",
+ zap.Int("knowledgebases", migratedKnowledgebases),
+ zap.Int("documents", migratedDocuments))
+ }
+ return nil
+}
+
+const parserConfigMigrationBatchSize = 256
+
+type parserConfigMigrationRow struct {
+ ID string `gorm:"column:id"`
+ ParserConfig entity.JSONMap `gorm:"column:parser_config"`
+}
+
+func migrateParserConfigRows(ctx context.Context, db *gorm.DB, model any) (int, error) {
+ var migrated int
+ query := db.WithContext(ctx).
+ Model(model).
+ Select("id", "parser_config").
+ Where("parser_id IN ? AND (pipeline_id IS NULL OR pipeline_id = '')", []string{"general", "naive"}).
+ Order("id")
+ var rows []parserConfigMigrationRow
+ result := query.FindInBatches(&rows, parserConfigMigrationBatchSize, func(batchDB *gorm.DB, _ int) error {
+ return batchDB.Transaction(func(tx *gorm.DB) error {
+ for _, row := range rows {
+ config := map[string]interface{}(row.ParserConfig)
+ if !migrateGeneralChunkerConfig(config) {
+ continue
+ }
+ if err := tx.Model(model).Where("id = ?", row.ID).
+ Update("parser_config", entity.JSONMap(config)).Error; err != nil {
+ return fmt.Errorf("update parser config %q: %w", row.ID, err)
+ }
+ migrated++
+ }
+ return nil
+ })
+ })
+ if result.Error != nil {
+ return migrated, result.Error
+ }
+ return migrated, nil
+}
+
+func normalizeLegacyDelimiter(value any) (any, bool) {
+ switch value := value.(type) {
+ case string:
+ delimiters := parserchunk.ParseDelimiterField(value)
+ out := make([]interface{}, len(delimiters))
+ for i, delimiter := range delimiters {
+ out[i] = delimiter
+ }
+ return out, true
+ case []string:
+ out := make([]interface{}, len(value))
+ for i, delimiter := range value {
+ out[i] = delimiter
+ }
+ return out, true
+ case []interface{}:
+ return value, true
+ default:
+ return nil, false
+ }
+}
diff --git a/internal/dao/parser_config_migration_test.go b/internal/dao/parser_config_migration_test.go
new file mode 100644
index 0000000000..da8b629751
--- /dev/null
+++ b/internal/dao/parser_config_migration_test.go
@@ -0,0 +1,234 @@
+package dao
+
+import (
+ "context"
+ "reflect"
+ "testing"
+
+ "ragflow/internal/entity"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+func TestMigrateGeneralChunkerConfigCopiesSupportedParams(t *testing.T) {
+ config := map[string]interface{}{
+ "TokenChunker:SixApplesFall": map[string]interface{}{
+ "chunk_token_size": 768,
+ "delimiters": []interface{}{"\n", ";"},
+ "children_delimiters": []interface{}{"|"},
+ "image_context_size": 4,
+ "overlapped_percent": 0.2,
+ "table_context_size": 3,
+ "delimiter_mode": "delimiter",
+ "outputs": map[string]interface{}{"chunks": map[string]interface{}{}},
+ },
+ }
+
+ changed := migrateGeneralChunkerConfig(config)
+ if !changed {
+ t.Fatal("migrateGeneralChunkerConfig reported no change")
+ }
+ if _, ok := config[legacyGeneralChunkerID]; ok {
+ t.Fatalf("legacy component id remains: %#v", config)
+ }
+
+ want := map[string]interface{}{
+ "chunk_token_size": 768,
+ "delimiters": []interface{}{"\n", ";"},
+ "children_delimiters": []interface{}{"|"},
+ "image_context_size": 4,
+ "overlapped_percent": 0.2,
+ "table_context_size": 3,
+ }
+ got, ok := config[currentGeneralChunkerID].(map[string]interface{})
+ if !ok {
+ t.Fatalf("new component params type = %T", config[currentGeneralChunkerID])
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("new component params = %#v, want %#v", got, want)
+ }
+}
+
+func TestMigrateGeneralChunkerConfigKeepsExplicitNewParams(t *testing.T) {
+ newParams := map[string]interface{}{"chunk_token_size": 256}
+ config := map[string]interface{}{
+ legacyGeneralChunkerID: map[string]interface{}{"chunk_token_size": 768},
+ currentGeneralChunkerID: newParams,
+ }
+
+ if !migrateGeneralChunkerConfig(config) {
+ t.Fatal("migrateGeneralChunkerConfig reported no change")
+ }
+ if got := config[currentGeneralChunkerID]; !reflect.DeepEqual(got, newParams) {
+ t.Fatalf("explicit new params changed: %#v", got)
+ }
+}
+
+func TestMigrateGeneralChunkerConfigNormalizesSingularDelimiter(t *testing.T) {
+ config := map[string]interface{}{
+ legacyGeneralChunkerID: map[string]interface{}{
+ "delimiter": "\n!?;。;!?",
+ },
+ }
+
+ if !migrateGeneralChunkerConfig(config) {
+ t.Fatal("migrateGeneralChunkerConfig reported no change")
+ }
+ params, ok := config[currentGeneralChunkerID].(map[string]interface{})
+ if !ok {
+ t.Fatalf("new component params type = %T", config[currentGeneralChunkerID])
+ }
+ want := []interface{}{"\n", "!", "?", ";", "。", ";", "!", "?"}
+ if !reflect.DeepEqual(params["delimiters"], want) {
+ t.Fatalf("migrated delimiters = %#v, want %#v", params["delimiters"], want)
+ }
+ if _, ok := params["delimiter"]; ok {
+ t.Fatalf("singular delimiter leaked into new params: %#v", params)
+ }
+}
+
+func TestMigrateGeneralChunkerConfigIgnoresOtherChunkers(t *testing.T) {
+ config := map[string]interface{}{
+ "TokenChunker:WarmBreadSmells": map[string]interface{}{"chunk_token_size": 128},
+ }
+
+ if migrateGeneralChunkerConfig(config) {
+ t.Fatal("migration changed a non-general component")
+ }
+ if _, ok := config["TokenChunker:WarmBreadSmells"]; !ok {
+ t.Fatal("non-general component was removed")
+ }
+}
+
+func TestMigrateGeneralChunkerParserConfigsUpdatesBuiltinRows(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.Document{}); err != nil {
+ t.Fatalf("automigrate: %v", err)
+ }
+
+ legacyParams := map[string]interface{}{
+ "chunk_token_size": 256,
+ "delimiter_mode": "delimiter",
+ "outputs": map[string]interface{}{},
+ }
+ kb := &entity.Knowledgebase{
+ ID: "kb-migrate",
+ TenantID: "tenant-migrate",
+ Name: "migration",
+ EmbdID: "embedding",
+ CreatedBy: "user-migrate",
+ ParserID: "general",
+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},
+ }
+ if err := db.Create(kb).Error; err != nil {
+ t.Fatalf("create knowledgebase: %v", err)
+ }
+ doc := &entity.Document{
+ ID: "doc-migrate",
+ KbID: kb.ID,
+ ParserID: "naive",
+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},
+ SourceType: "local",
+ Type: "file",
+ CreatedBy: "user-migrate",
+ Suffix: "txt",
+ }
+ if err := db.Create(doc).Error; err != nil {
+ t.Fatalf("create document: %v", err)
+ }
+ emptyPipelineID := ""
+ emptyPipelineKB := &entity.Knowledgebase{
+ ID: "kb-empty-pipeline",
+ TenantID: "tenant-migrate",
+ Name: "empty-pipeline",
+ EmbdID: "embedding",
+ CreatedBy: "user-migrate",
+ ParserID: "general",
+ PipelineID: &emptyPipelineID,
+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},
+ }
+ if err := db.Create(emptyPipelineKB).Error; err != nil {
+ t.Fatalf("create empty-pipeline knowledgebase: %v", err)
+ }
+ emptyPipelineDoc := &entity.Document{
+ ID: "doc-empty-pipeline",
+ KbID: kb.ID,
+ ParserID: "naive",
+ PipelineID: &emptyPipelineID,
+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},
+ SourceType: "local",
+ Type: "file",
+ CreatedBy: "user-migrate",
+ Suffix: "txt",
+ }
+ if err := db.Create(emptyPipelineDoc).Error; err != nil {
+ t.Fatalf("create empty-pipeline document: %v", err)
+ }
+ customPipelineID := "canvas-migrate"
+ customKB := &entity.Knowledgebase{
+ ID: "kb-custom",
+ TenantID: "tenant-migrate",
+ Name: "custom",
+ EmbdID: "embedding",
+ CreatedBy: "user-migrate",
+ ParserID: "naive",
+ PipelineID: &customPipelineID,
+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},
+ }
+ if err := db.Create(customKB).Error; err != nil {
+ t.Fatalf("create custom knowledgebase: %v", err)
+ }
+
+ if err := migrateGeneralChunkerParserConfigs(context.Background(), db); err != nil {
+ t.Fatalf("migrate parser configs: %v", err)
+ }
+
+ var gotKB entity.Knowledgebase
+ if err := db.First(&gotKB, "id = ?", kb.ID).Error; err != nil {
+ t.Fatalf("reload knowledgebase: %v", err)
+ }
+ var gotDoc entity.Document
+ if err := db.First(&gotDoc, "id = ?", doc.ID).Error; err != nil {
+ t.Fatalf("reload document: %v", err)
+ }
+ var gotEmptyPipelineKB entity.Knowledgebase
+ if err := db.First(&gotEmptyPipelineKB, "id = ?", emptyPipelineKB.ID).Error; err != nil {
+ t.Fatalf("reload empty-pipeline knowledgebase: %v", err)
+ }
+ var gotEmptyPipelineDoc entity.Document
+ if err := db.First(&gotEmptyPipelineDoc, "id = ?", emptyPipelineDoc.ID).Error; err != nil {
+ t.Fatalf("reload empty-pipeline document: %v", err)
+ }
+ for name, config := range map[string]entity.JSONMap{
+ "knowledgebase": gotKB.ParserConfig,
+ "document": gotDoc.ParserConfig,
+ "empty-pipeline knowledgebase": gotEmptyPipelineKB.ParserConfig,
+ "empty-pipeline document": gotEmptyPipelineDoc.ParserConfig,
+ } {
+ if _, ok := config[legacyGeneralChunkerID]; ok {
+ t.Errorf("%s retained legacy component id: %#v", name, config)
+ }
+ params, ok := config[currentGeneralChunkerID].(map[string]interface{})
+ if !ok {
+ t.Errorf("%s new component params type = %T", name, config[currentGeneralChunkerID])
+ continue
+ }
+ if params["chunk_token_size"] != float64(256) && params["chunk_token_size"] != 256 {
+ t.Errorf("%s migrated params = %#v", name, params)
+ }
+ if _, ok := params["delimiter_mode"]; ok {
+ t.Errorf("%s retained delimiter_mode: %#v", name, params)
+ }
+ }
+ var gotCustomKB entity.Knowledgebase
+ if err := db.First(&gotCustomKB, "id = ?", customKB.ID).Error; err != nil {
+ t.Fatalf("reload custom knowledgebase: %v", err)
+ }
+ if _, ok := gotCustomKB.ParserConfig[legacyGeneralChunkerID]; !ok {
+ t.Fatalf("custom pipeline parser config was unexpectedly migrated: %#v", gotCustomKB.ParserConfig)
+ }
+}
diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.go
index c22f0de118..d12d1aa24f 100644
--- a/internal/handler/components_testpkg/components_handler_test.go
+++ b/internal/handler/components_testpkg/components_handler_test.go
@@ -134,7 +134,7 @@ func TestComponentsHandler_NoFilter(t *testing.T) {
// TestComponentsHandler_FilterIngestion verifies the
// ?category=ingestion filter returns the ingestion components
-// (Compiler, Extractor, File, Parser, Tokenizer + 9 chunker variants).
+// (Compiler, Extractor, File, Parser, Tokenizer + 10 chunker variants).
// Names must be sorted ascending (plan §4 task 1 stable output).
func TestComponentsHandler_FilterIngestion(t *testing.T) {
eng := newComponentsTestRig(t)
@@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
+ "compiler", "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "pagechunker", "qachunker", "parser", "tablechunker",
"titlechunker", "tokenchunker", "tokenizer",
}
@@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
+ "compiler", "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "pagechunker", "qachunker", "parser", "tablechunker",
"titlechunker", "tokenchunker", "tokenizer",
}
@@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {
}
_, _, data := decodeEnvelope(t, w.Body.Bytes())
wantNames := []string{
- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
+ "compiler", "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "pagechunker", "qachunker", "parser", "tablechunker",
"titlechunker", "tokenchunker", "tokenizer",
}
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index 1beccc0cb4..cbdfe9fae0 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -59,6 +59,8 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, er
return NewTableChunker(params)
case ComponentNamePageChunker:
return NewPageChunker(params)
+ case ComponentNameGeneralChunker:
+ return NewGeneralChunker(params)
default:
return nil, fmt.Errorf("chunker: unknown component %q", name)
}
@@ -89,7 +91,9 @@ func stringListFromAny(in []any) []string {
// content. invokeTextPayload decides whether an active delimiter yields one
// chunk per segment (custom/backtick, no merge) or splits into paragraphs that
// are merged by token size (bare). Canonical single-string parser_config.delimiter
-// parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField).
+// Legacy single-string parsing is performed by GeneralChunker at its
+// configuration boundary; the shared regex helper only consumes canonical
+// delimiter lists.
func compileDelimPattern(delims []string) *regexp.Regexp {
return chunk.CompileDelimiterPatternList(delims, true)
}
@@ -98,9 +102,10 @@ func compileDelimPattern(delims []string) *regexp.Regexp {
// (token_chunker.py:79-90). The captured delimiter is DISCARDED rather than
// glued to a segment: re.split with a captured group keeps delimiters at odd
// indices, and only the even-index (text) parts are kept. This is the
-// behavior every delimiter path (primary and children, text/markdown/html
-// and JSON) must reproduce so a split chunk reads "first sentence here"
-// without the trailing delimiter.
+// behavior shared TokenChunker paths and General's primary/Markdown splits
+// reproduce so a split chunk reads "first sentence here" without the trailing
+// delimiter. General's legacy-compatible children split is implemented
+// separately because that path keeps the delimiter attached to its parent.
func splitDroppingDelim(text string, pattern *regexp.Regexp) []string {
if pattern == nil {
return []string{text}
diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
new file mode 100644
index 0000000000..ed800d4f83
--- /dev/null
+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
@@ -0,0 +1,182 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
+)
+
+func TestSplitTextParserSentencesKeepsSentenceUnits(t *testing.T) {
+ item := schema.ChunkDoc{Text: "First!Second?第三。Fourth;", DocType: "text", CKType: "text"}
+ parts := splitTextParserSentences(item)
+ if got, want := len(parts), 4; got != want {
+ t.Fatalf("sentence parts = %d, want %d: %#v", got, want, parts)
+ }
+ want := []string{"First!", "Second?", "第三。", "Fourth;"}
+ for i, part := range parts {
+ if part.Text != want[i] {
+ t.Errorf("part[%d] = %q, want %q", i, part.Text, want[i])
+ }
+ }
+}
+
+func TestTokenChunkerTextParserSentenceFallbackEndToEnd(t *testing.T) {
+ const source = "First!Second?第三。Fourth;"
+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "sample.txt", []byte(source))
+ if parsed.Err != nil {
+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
+ }
+ if len(parsed.JSON) != 1 {
+ t.Fatalf("TextParser JSON items = %d, want one parser item", len(parsed.JSON))
+ }
+
+ chunker, err := NewTokenChunker(map[string]any{"chunk_token_size": 3})
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := chunker.Invoke(context.Background(), nil, map[string]any{
+ "name": "sample.txt",
+ "file_type": "txt",
+ "output_format": "json",
+ "json": parsed.JSON,
+ })
+ if err != nil {
+ t.Fatalf("TokenChunker.Invoke: %v", err)
+ }
+ chunks := chunksFromOutput(t, out)
+ want := []string{"First!", "Second?", "第三。", "Fourth;"}
+ if len(chunks) != len(want) {
+ t.Fatalf("chunks = %#v, want one chunk per sentence", chunks)
+ }
+ for i, chunk := range chunks {
+ if got, wantText := chunk["text"], want[i]; got != wantText {
+ t.Errorf("chunk[%d].text = %v, want %q", i, got, wantText)
+ }
+ }
+}
+
+func TestGeneralChunkerTextAppliesOverlapThroughInvoke(t *testing.T) {
+ chunker, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 2,
+ "overlapped_percent": 50,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.txt",
+ "file_type": "txt",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "alpha beta", "doc_type_kwd": "text"},
+ {"text": "gamma delta", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ texts := outputTexts(t, out)
+ if len(texts) != 2 {
+ t.Fatalf("texts = %q, want two overlapped chunks", texts)
+ }
+ if texts[1] == "gamma delta" || !strings.HasSuffix(texts[1], "\ngamma delta") {
+ t.Fatalf("text overlap = %q, want previous tail plus current text", texts[1])
+ }
+}
+
+func TestGeneralChunkerEmptyJSONReturnsNoChunksForEachStrategy(t *testing.T) {
+ for _, fileType := range []string{"txt", "md", "docx", "pdf", "xlsx"} {
+ t.Run(fileType, func(t *testing.T) {
+ chunker, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+ "name": "empty." + fileType,
+ "file_type": fileType,
+ "output_format": "json",
+ "json": []map[string]any{},
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ if chunks := outputChunks(t, out); len(chunks) != 0 {
+ t.Fatalf("chunks = %#v, want empty output", chunks)
+ }
+ })
+ }
+}
+
+func TestGeneralChunkerMarkdownLongHeadingKeepsFollowingTableAtomic(t *testing.T) {
+ chunker, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ longHeading := "# " + strings.Repeat("heading ", 60)
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want separate heading and table chunks", chunks)
+ }
+ if chunks[0]["ck_type"] != "heading" || chunks[0]["text"] != strings.TrimSpace(longHeading) {
+ t.Errorf("heading chunk = %#v", chunks[0])
+ }
+ if chunks[1]["ck_type"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {
+ t.Errorf("table chunk = %#v", chunks[1])
+ }
+}
+
+func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {
+ chunker, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+ "name": "empty-sheets.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "Name", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},
+ {"text": "Amount", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2, "table_id": "sheet-2"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want one header-only chunk per sheet", chunks)
+ }
+ if chunks[0]["text"] != "Name" || chunks[1]["text"] != "Amount" {
+ t.Fatalf("header-only chunks = %#v", chunks)
+ }
+}
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
new file mode 100644
index 0000000000..d945232c75
--- /dev/null
+++ b/internal/ingestion/component/chunker/general.go
@@ -0,0 +1,1191 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "image"
+ "image/draw"
+ _ "image/gif"
+ _ "image/jpeg"
+ "image/png"
+ "log/slog"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+
+ "gorm.io/gorm"
+
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/chunk"
+)
+
+const ComponentNameGeneralChunker = "GeneralChunker"
+
+var generalSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)
+
+type generalChunkerParam struct {
+ ChunkTokenSize int
+ Delimiters []string
+ OverlappedPercent float64
+ ChildrenDelimiters []string
+ TableContextSize int
+ ImageContextSize int
+}
+
+func (generalChunkerParam) Defaults() generalChunkerParam {
+ return generalChunkerParam{
+ ChunkTokenSize: 512,
+ Delimiters: []string{"\n"},
+ ChildrenDelimiters: []string{},
+ }
+}
+
+func (p *generalChunkerParam) Update(conf map[string]any) {
+ if value, ok := schema.NumericFromAny(conf["chunk_token_size"]); ok {
+ p.ChunkTokenSize = int(value)
+ }
+ if value, ok := conf["delimiters"]; ok {
+ if delimiters, recognized := normalizeGeneralDelimiterValue(value); recognized {
+ p.Delimiters = delimiters
+ }
+ } else if value, ok := conf["delimiter"]; ok {
+ if delimiters, recognized := normalizeGeneralDelimiterValue(value); recognized {
+ p.Delimiters = delimiters
+ }
+ }
+ if value, ok := conf["overlapped_percent"]; ok {
+ p.OverlappedPercent = schema.NormalizeOverlappedPercent(value)
+ }
+ if value, ok := conf["children_delimiters"]; ok {
+ if delimiters, recognized := normalizeGeneralDelimiterValue(value); recognized {
+ p.ChildrenDelimiters = delimiters
+ }
+ }
+ if value, ok := schema.NumericFromAny(conf["table_context_size"]); ok {
+ p.TableContextSize = max(0, int(value))
+ }
+ if value, ok := schema.NumericFromAny(conf["image_context_size"]); ok {
+ p.ImageContextSize = max(0, int(value))
+ }
+}
+
+// normalizeGeneralDelimiterValue accepts both the canonical list form and
+// the legacy Python single-string form. A legacy string is split into Unicode
+// rune delimiters, except backtick-wrapped tokens which remain one delimiter.
+func normalizeGeneralDelimiterValue(value any) ([]string, bool) {
+ switch value := value.(type) {
+ case string:
+ return splitGeneralDelimiterString(value), true
+ case []string:
+ return append([]string(nil), value...), true
+ case []any:
+ return stringListFromAny(value), true
+ default:
+ return nil, false
+ }
+}
+
+func splitGeneralDelimiterString(value string) []string {
+ return chunk.ParseDelimiterField(value)
+}
+
+// GeneralChunkerComponent owns the general ingestion chunking policy and
+// selects its format-specific strategy from the Parser's canonical file_type.
+type GeneralChunkerComponent struct {
+ param generalChunkerParam
+}
+
+func NewGeneralChunker(params map[string]any) (runtime.Component, error) {
+ param := generalChunkerParam{}.Defaults()
+ param.Update(params)
+ if err := param.Validate(); err != nil {
+ return nil, err
+ }
+ return &GeneralChunkerComponent{param: param}, nil
+}
+
+func (p generalChunkerParam) Validate() error {
+ if p.ChunkTokenSize < 0 {
+ return fmt.Errorf("chunk_token_size must be non-negative")
+ }
+ return nil
+}
+
+func (c *GeneralChunkerComponent) Inputs() map[string]string { return ChunkerInputs }
+
+func (c *GeneralChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }
+
+func (c *GeneralChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
+ upstream, err := decodeChunkerFromUpstream(inputs)
+ if err != nil {
+ return nil, fmt.Errorf("GeneralChunker: decode input: %w", err)
+ }
+ fileType := strings.ToLower(strings.TrimSpace(upstream.FileType))
+ if fileType == "" {
+ fileType = inferGeneralFileType(upstream)
+ if fileType == "" {
+ return nil, fmt.Errorf("GeneralChunker: file_type is required when source name has no extension")
+ }
+ slog.Warn("GeneralChunker: missing parser file_type; inferred from source name", "name", upstream.Name, "file_type", fileType)
+ }
+
+ strategy := generalStrategyForFileType(fileType)
+ if strategy == generalStrategyText && !isKnownGeneralFileType(fileType) {
+ slog.Debug("GeneralChunker: unknown file_type; using text fallback", "file_type", fileType)
+ }
+
+ switch strategy {
+ case generalStrategyPDF:
+ return c.chunkPDF(ctx, db, upstream)
+ case generalStrategyDOCX:
+ return c.chunkDOCX(ctx, upstream)
+ case generalStrategyMarkdown:
+ return c.chunkMarkdown(ctx, upstream)
+ case generalStrategySpreadsheet:
+ return c.chunkSpreadsheet(ctx, upstream)
+ default:
+ return c.chunkGeneral(ctx, upstream)
+ }
+}
+
+type generalStrategy uint8
+
+const (
+ generalStrategyText generalStrategy = iota
+ generalStrategyPDF
+ generalStrategyDOCX
+ generalStrategyMarkdown
+ generalStrategySpreadsheet
+)
+
+func generalStrategyForFileType(fileType string) generalStrategy {
+ switch strings.ToLower(strings.TrimSpace(fileType)) {
+ case "pdf":
+ return generalStrategyPDF
+ case "docx":
+ return generalStrategyDOCX
+ case "md":
+ return generalStrategyMarkdown
+ case "xls", "xlsx", "csv":
+ return generalStrategySpreadsheet
+ default:
+ return generalStrategyText
+ }
+}
+
+func inferGeneralFileType(upstream schema.ChunkerFromUpstream) string {
+ name := strings.TrimSpace(upstream.Name)
+ if name == "" && upstream.File != nil {
+ name = strings.TrimSpace(upstream.File.Name)
+ }
+ ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(name)), ".")
+ return ext
+}
+
+func isKnownGeneralFileType(fileType string) bool {
+ switch strings.ToLower(strings.TrimSpace(fileType)) {
+ case "pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv",
+ "html", "md", "txt", "epub", "json", "video", "email", "visual",
+ "aural", "folder", "other":
+ return true
+ default:
+ return false
+ }
+}
+
+func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upstream schema.ChunkerFromUpstream) (map[string]any, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, fmt.Errorf("GeneralChunker: %w", err)
+ }
+ units := upstream.JSONResult
+ if upstream.OutputFormat == schema.PayloadFormatChunks {
+ units = upstream.Chunks
+ }
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+
+ primaryPattern := compileDelimPattern(c.param.Delimiters)
+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ units = splitGeneralUnits(units, primaryPattern)
+ units = sortPDFUnits(units)
+ if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {
+ slog.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")
+ }
+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+
+ chunks := make([]schema.ChunkDoc, 0, len(units))
+ body := make([]schema.ChunkDoc, 0, len(units))
+ flushBody := func() {
+ if len(body) == 0 {
+ return
+ }
+ chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")...)
+ body = body[:0]
+ }
+ for _, unit := range units {
+ if itemDocType(unit) == "text" {
+ body = append(body, unit)
+ } else {
+ flushBody()
+ chunks = append(chunks, cloneChunkDoc(unit))
+ }
+ }
+ flushBody()
+
+ engine, err := newPDFEngineFromUpstream(ctx, db, upstream)
+ if err != nil {
+ slog.Warn("GeneralChunker: could not open PDF for on-demand cropping", "err", err)
+ }
+ if engine != nil {
+ defer engine.Close()
+ chunks = cropImageChunks(ctx, engine, chunks)
+ }
+ chunks = finalizeGeneralChunks(chunks, childrenPattern)
+ if err := ctx.Err(); err != nil {
+ return nil, fmt.Errorf("GeneralChunker: %w", err)
+ }
+ if len(chunks) == 0 {
+ return emptyOutputs(), nil
+ }
+ attachPDFOutline(chunks, upstream.File)
+ return chunkOutputs(chunks), nil
+}
+
+// sortPDFUnits orders each contiguous positioned run by physical reading
+// order. Coordinate-free units are barriers: they stay at their parser
+// positions and preserve their relative order instead of being moved to the
+// end of the document.
+func sortPDFUnits(units []schema.ChunkDoc) []schema.ChunkDoc {
+ type positionedUnit struct {
+ unit schema.ChunkDoc
+ row []float64
+ positioned bool
+ }
+ ordered := make([]positionedUnit, 0, len(units))
+ for _, unit := range units {
+ row, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions})
+ ordered = append(ordered, positionedUnit{unit: cloneChunkDoc(unit), row: row, positioned: ok})
+ }
+ runStart := -1
+ flush := func(end int) {
+ if runStart < 0 {
+ return
+ }
+ sort.SliceStable(ordered[runStart:end], func(i, j int) bool {
+ return pdfPosRowLess(ordered[runStart+i].row, ordered[runStart+j].row)
+ })
+ runStart = -1
+ }
+ for i := 0; i <= len(ordered); i++ {
+ positioned := i < len(ordered) && ordered[i].positioned
+ if positioned {
+ if runStart < 0 {
+ runStart = i
+ }
+ continue
+ }
+ flush(i)
+ }
+ result := make([]schema.ChunkDoc, 0, len(ordered))
+ for _, item := range ordered {
+ result = append(result, item.unit)
+ }
+ return result
+}
+
+func hasPDFPositions(units []schema.ChunkDoc) bool {
+ for _, unit := range units {
+ if _, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions}); ok {
+ return true
+ }
+ }
+ return false
+}
+
+func hasUnpositionedPDFMedia(units []schema.ChunkDoc) bool {
+ for _, unit := range units {
+ if itemDocType(unit) == "text" {
+ continue
+ }
+ if _, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions}); !ok {
+ return true
+ }
+ }
+ return false
+}
+
+func attachPDFOutline(chunks []schema.ChunkDoc, file *schema.ChunkerFileMeta) {
+ if len(chunks) == 0 || file == nil || len(file.Extra) == 0 {
+ return
+ }
+ raw, ok := file.Extra["outline"]
+ if !ok || len(raw) == 0 {
+ return
+ }
+ var input []map[string]any
+ if err := json.Unmarshal(raw, &input); err != nil || len(input) == 0 {
+ return
+ }
+ outline := make([]map[string]any, 0, len(input))
+ for _, entry := range input {
+ title, _ := entry["title"].(string)
+ if title == "" {
+ continue
+ }
+ depth := entry["level"]
+ if depth == nil {
+ depth = entry["depth"]
+ }
+ outline = append(outline, map[string]any{"title": title, "depth": depth})
+ }
+ if len(outline) == 0 {
+ return
+ }
+ encoded, err := json.Marshal(outline)
+ if err != nil {
+ return
+ }
+ if chunks[0].Extra == nil {
+ chunks[0].Extra = make(map[string]json.RawMessage)
+ }
+ chunks[0].Extra["__outline__"] = encoded
+}
+
+func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, fmt.Errorf("GeneralChunker: %w", err)
+ }
+ units := upstream.JSONResult
+ if upstream.OutputFormat == schema.PayloadFormatChunks {
+ units = upstream.Chunks
+ }
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+ primaryPattern := compileDelimPattern(c.param.Delimiters)
+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ units = splitGeneralUnits(units, primaryPattern)
+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+ units = mergeDOCXUnits(units, c.param.ChunkTokenSize, hasCustomDelim(c.param.Delimiters), "\n")
+ units = applyGeneralOverlap(units, c.param.OverlappedPercent, "\n")
+ units = finalizeGeneralChunks(units, childrenPattern)
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+ return chunkOutputs(units), nil
+}
+
+// mergeDOCXUnits mirrors Python naive_merge_docx: text units keep the
+// previous text merge target across intervening table/image units. This means
+// a later paragraph can extend an earlier text chunk while the media item
+// remains in its original output position.
+func mergeDOCXUnits(units []schema.ChunkDoc, target int, customDelimiter bool, joinSep string) []schema.ChunkDoc {
+ merged := make([]schema.ChunkDoc, 0, len(units))
+ previousText := -1
+ for _, unit := range units {
+ if itemDocType(unit) != "text" {
+ media := cloneChunkDoc(unit)
+ media.DocType = itemDocType(media)
+ media.CKType = media.DocType
+ merged = append(merged, media)
+ continue
+ }
+
+ text := cloneChunkDoc(unit)
+ text.DocType = "text"
+ text.CKType = "text"
+ text.TKNums = intPtr(generalUnitTokens(text))
+ if previousText < 0 || customDelimiter || intValue(merged[previousText].TKNums) >= target {
+ merged = append(merged, text)
+ previousText = len(merged) - 1
+ continue
+ }
+ mergeGeneralChunk(&merged[previousText], text, joinSep)
+ merged[previousText].DocType = "text"
+ merged[previousText].CKType = "text"
+ }
+ return merged
+}
+
+// attachGeneralMediaContext follows Python's sentence-aware media context
+// extraction. The shared TokenChunker helper intentionally returns the
+// smallest token-fitting rune suffix/prefix; DOCX/PDF general strategies use
+// complete sentence units instead.
+func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int) {
+ for i := range units {
+ if units[i].CKType != "table" && units[i].CKType != "image" {
+ continue
+ }
+ budget := imageTokens
+ if units[i].CKType == "table" {
+ budget = tableTokens
+ }
+ if budget <= 0 {
+ continue
+ }
+ units[i].ContextAbove = collectGeneralMediaContext(units, i, budget, true)
+ units[i].ContextBelow = collectGeneralMediaContext(units, i, budget, false)
+ }
+}
+
+func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, above bool) string {
+ var parts []string
+ remaining := budget
+ step := -1
+ if !above {
+ step = 1
+ }
+ for cursor := index + step; cursor >= 0 && cursor < len(units) && remaining > 0; cursor += step {
+ if generalMediaContextCrossesSpreadsheetBoundary(units[index], units[cursor]) {
+ break
+ }
+ if itemDocType(units[cursor]) != "text" {
+ continue
+ }
+ text := units[cursor].Text
+ tokens := generalUnitTokens(units[cursor])
+ if tokens >= remaining {
+ piece := takeGeneralContextSentence(text, remaining, above)
+ if above {
+ parts = append([]string{piece}, parts...)
+ } else {
+ parts = append(parts, piece)
+ }
+ break
+ }
+ if above {
+ parts = append([]string{text}, parts...)
+ } else {
+ parts = append(parts, text)
+ }
+ remaining -= tokens
+ }
+ return strings.Join(parts, "\n")
+}
+
+func generalMediaContextCrossesSpreadsheetBoundary(media, candidate schema.ChunkDoc) bool {
+ if !hasSpreadsheetIdentity(media) && !hasSpreadsheetIdentity(candidate) {
+ return false
+ }
+ return !sameSpreadsheetTable(media, candidate)
+}
+
+func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {
+ return doc.TableID != "" || doc.Sheet != "" || doc.SheetIndex != nil
+}
+
+func takeGeneralContextSentence(text string, budget int, fromEnd bool) string {
+ sentences := splitGeneralContextSentences(text)
+ if len(sentences) == 0 {
+ return text
+ }
+ // The sentence pieces are contiguous slices of text. Track byte offsets
+ // instead of repeatedly prepending slices and joining the selected text;
+ // each candidate is still tokenized independently because BPE token counts
+ // are not additive across sentence boundaries.
+ if fromEnd {
+ start := len(text)
+ for i := len(sentences) - 1; i >= 0; i-- {
+ start -= len(sentences[i])
+ if tokenizeStr(text[start:]) >= budget {
+ break
+ }
+ }
+ return text[start:]
+ }
+
+ end := 0
+ for _, sentence := range sentences {
+ end += len(sentence)
+ if tokenizeStr(text[:end]) >= budget {
+ break
+ }
+ }
+ return text[:end]
+}
+
+func splitGeneralContextSentences(text string) []string {
+ indices := generalSentencePattern.FindAllStringIndex(text, -1)
+ if len(indices) == 0 {
+ if text == "" {
+ return nil
+ }
+ return []string{text}
+ }
+ parts := make([]string, 0, len(indices)+1)
+ start := 0
+ for _, index := range indices {
+ parts = append(parts, text[start:index[1]])
+ start = index[1]
+ }
+ if start < len(text) {
+ parts = append(parts, text[start:])
+ }
+ return parts
+}
+
+func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, fmt.Errorf("GeneralChunker: %w", err)
+ }
+ units := upstream.JSONResult
+ if upstream.OutputFormat == schema.PayloadFormatChunks {
+ units = upstream.Chunks
+ }
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+ primaryPattern := compileDelimPattern(c.param.Delimiters)
+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ units = splitMarkdownUnits(units, primaryPattern)
+ units = mergeMarkdownUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")
+ units = finalizeGeneralChunks(units, childrenPattern)
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+ return chunkOutputs(units), nil
+}
+
+// splitMarkdownUnits expands only text units while retaining the semantic
+// parser type on every piece. Markdown heading detection depends on ck_type,
+// so the generic splitter's type normalization cannot be used here.
+func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
+ result := make([]schema.ChunkDoc, 0, len(units))
+ for _, unit := range units {
+ unit = cloneChunkDoc(unit)
+ unit.Text = strings.TrimSpace(normalizeGeneralNewlines(itemTextOrFallback(unit)))
+ unit.DocType = itemDocType(unit)
+ if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {
+ unit.TKNums = intPtr(tokenizeStr(unit.Text))
+ result = append(result, unit)
+ continue
+ }
+ for _, part := range splitDroppingDelim(unit.Text, pattern) {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+ piece := cloneChunkDoc(unit)
+ piece.Text = part
+ piece.TKNums = intPtr(tokenizeStr(part))
+ result = append(result, piece)
+ }
+ }
+ return result
+}
+
+// mergeMarkdownUnits mirrors the Markdown branch of Python naive.chunk:
+// ordinary units use a projected token cap, while a short heading is always
+// kept with the following unit. Markdown images are block attachments rather
+// than standalone media chunks, so image-bearing units participate in the
+// text merge and retain their image payload.
+func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
+ overlapPct = max(0, min(100, overlapPct))
+ merged := make([]schema.ChunkDoc, 0, len(units))
+ current := -1
+ currentTokens := 0
+ for _, unit := range units {
+ if itemDocType(unit) == "table" {
+ if current >= 0 && isShortMarkdownHeading(merged[current]) {
+ mergedImage, imagesOK := mergeMarkdownImagesChecked(merged[current].Image, unit.Image)
+ if !imagesOK {
+ merged = append(merged, cloneChunkDoc(unit))
+ current = -1
+ currentTokens = 0
+ continue
+ }
+ table := cloneChunkDoc(unit)
+ heading := merged[current]
+ if heading.Text != "" && table.Text != "" {
+ table.Text = heading.Text + joinSep + table.Text
+ } else {
+ table.Text = heading.Text + table.Text
+ }
+ table.TKNums = intPtr(currentTokens + generalUnitTokens(unit))
+ table.PDFPositions = mergeGeneralPositions(heading.PDFPositions, table.PDFPositions)
+ table.Positions = mergeGeneralPositions(heading.Positions, table.Positions)
+ mergeGeneralMetadata(&table, heading)
+ table.Image = mergedImage
+ table.DocType = "table"
+ table.CKType = "table"
+ merged[current] = table
+ current = -1
+ currentTokens = 0
+ continue
+ }
+ merged = append(merged, cloneChunkDoc(unit))
+ current = -1
+ currentTokens = 0
+ continue
+ }
+
+ unit = cloneChunkDoc(unit)
+ unit.DocType = "text"
+ if unit.CKType == "" {
+ unit.CKType = "text"
+ }
+ unit.TKNums = intPtr(generalUnitTokens(unit))
+ if current < 0 {
+ merged = append(merged, unit)
+ current = len(merged) - 1
+ currentTokens = generalUnitTokens(unit)
+ continue
+ }
+
+ previous := &merged[current]
+ unitTokens := generalUnitTokens(unit)
+ if unit.CKType == "heading" && previous.CKType != "heading" {
+ merged = append(merged, unit)
+ current = len(merged) - 1
+ currentTokens = unitTokens
+ continue
+ }
+ forceMerge := isShortMarkdownHeading(*previous)
+ mergedImage, imagesOK := mergeMarkdownImagesChecked(previous.Image, unit.Image)
+ if !imagesOK {
+ merged = append(merged, unit)
+ current = len(merged) - 1
+ currentTokens = unitTokens
+ continue
+ }
+ projected := currentTokens + unitTokens
+ if !forceMerge && projected > target {
+ overlap, _ := computeOverlapPrefix(previous.Text, overlapPct)
+ if overlap != "" {
+ // The emitted chunk belongs to the current unit. Clone it
+ // first so media and other metadata from the previous chunk
+ // cannot leak through the overlap prefix.
+ next := cloneChunkDoc(unit)
+ next.Text = joinGeneralOverlapText(overlap, unit.Text, joinSep)
+ currentTokens = tokenizeStr(overlap) + unitTokens
+ next.TKNums = intPtr(currentTokens)
+ next.PDFPositions = mergeGeneralPositions(previous.PDFPositions, next.PDFPositions)
+ next.Positions = mergeGeneralPositions(previous.Positions, next.Positions)
+ merged = append(merged, next)
+ } else {
+ merged = append(merged, unit)
+ currentTokens = unitTokens
+ }
+ current = len(merged) - 1
+ continue
+ }
+ mergeMarkdownChunk(previous, unit, joinSep)
+ previous.Image = mergedImage
+ previous.DocType = "text"
+ previous.CKType = "text"
+ currentTokens = projected
+ previous.TKNums = intPtr(currentTokens)
+ }
+ return merged
+}
+
+func isShortMarkdownHeading(unit schema.ChunkDoc) bool {
+ if unit.CKType != "heading" {
+ return false
+ }
+ return tokenizeStr(markdownHeadingContent(unit.Text)) < 50
+}
+
+func markdownHeadingContent(text string) string {
+ text = strings.TrimSpace(text)
+ if len(text) == 0 || text[0] != '#' {
+ return text
+ }
+ level := 0
+ for level < len(text) && text[level] == '#' {
+ level++
+ }
+ if level == 0 || level > 6 || level == len(text) || text[level] != ' ' {
+ return text
+ }
+ return strings.TrimSpace(text[level:])
+}
+
+func mergeMarkdownChunk(dst *schema.ChunkDoc, src schema.ChunkDoc, joinSep string) {
+ mergeGeneralChunk(dst, src, joinSep)
+}
+
+const (
+ maxMarkdownImageDimension = 8192
+ maxMarkdownImagePixels = 32 * 1024 * 1024
+)
+
+func mergeMarkdownImagesChecked(first, second string) (string, bool) {
+ if first == "" {
+ return second, true
+ }
+ if second == "" || first == second {
+ return first, true
+ }
+ firstImage, firstOK := decodeMarkdownImage(first)
+ secondImage, secondOK := decodeMarkdownImage(second)
+ if !firstOK || !secondOK {
+ return first, false
+ }
+ width := firstImage.Bounds().Dx()
+ if secondImage.Bounds().Dx() > width {
+ width = secondImage.Bounds().Dx()
+ }
+ height := firstImage.Bounds().Dy() + secondImage.Bounds().Dy()
+ if !markdownImageWithinLimits(width, height) {
+ return first, false
+ }
+ canvas := image.NewRGBA(image.Rect(0, 0, width, height))
+ draw.Draw(canvas, image.Rect(0, 0, firstImage.Bounds().Dx(), firstImage.Bounds().Dy()), firstImage, image.Point{}, draw.Src)
+ draw.Draw(canvas, image.Rect(0, firstImage.Bounds().Dy(), secondImage.Bounds().Dx(), canvas.Bounds().Dy()), secondImage, image.Point{}, draw.Src)
+ var encoded bytes.Buffer
+ if err := png.Encode(&encoded, canvas); err != nil {
+ return first, false
+ }
+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), true
+}
+
+func markdownImageWithinLimits(width, height int) bool {
+ if width <= 0 || height <= 0 || width > maxMarkdownImageDimension || height > maxMarkdownImageDimension {
+ return false
+ }
+ return width <= maxMarkdownImagePixels/height
+}
+
+func decodeMarkdownImage(value string) (image.Image, bool) {
+ payload := value
+ if marker := strings.Index(value, "base64,"); marker >= 0 {
+ payload = value[marker+len("base64,"):]
+ } else if strings.HasPrefix(value, "data:") {
+ marker := strings.IndexByte(value, ',')
+ if marker < 0 {
+ return nil, false
+ }
+ payload = value[marker+1:]
+ }
+ raw, err := base64.StdEncoding.DecodeString(payload)
+ if err != nil {
+ return nil, false
+ }
+ config, _, err := image.DecodeConfig(bytes.NewReader(raw))
+ if err != nil || !markdownImageWithinLimits(config.Width, config.Height) {
+ return nil, false
+ }
+ decoded, _, err := image.Decode(bytes.NewReader(raw))
+ if err != nil {
+ return nil, false
+ }
+ return decoded, true
+}
+
+func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, fmt.Errorf("GeneralChunker: %w", err)
+ }
+ units := upstream.JSONResult
+ if upstream.OutputFormat == schema.PayloadFormatChunks {
+ units = upstream.Chunks
+ }
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+ chunks := make([]schema.ChunkDoc, 0, len(units))
+ pendingRows := make([]schema.ChunkDoc, 0)
+ var pendingHeader *schema.ChunkDoc
+ flushRows := func() {
+ if len(pendingRows) == 0 {
+ return
+ }
+ chunks = append(chunks, mergeSpreadsheetRows(pendingRows, c.param.ChunkTokenSize)...)
+ pendingRows = pendingRows[:0]
+ }
+ flushHeader := func() {
+ if pendingHeader == nil {
+ return
+ }
+ chunks = append(chunks, cloneChunkDoc(*pendingHeader))
+ pendingHeader = nil
+ }
+ for _, unit := range units {
+ if unit.CKType == "table_header" {
+ flushRows()
+ flushHeader()
+ header := cloneChunkDoc(unit)
+ pendingHeader = &header
+ continue
+ }
+ if unit.CKType == "table_row" {
+ // A header immediately followed by row units is parser metadata for
+ // that table, not an additional data chunk. If no row follows, the
+ // deferred header is emitted as a header-only table below.
+ pendingHeader = nil
+ pendingRows = append(pendingRows, unit)
+ continue
+ }
+ flushRows()
+ flushHeader()
+ chunks = append(chunks, cloneChunkDoc(unit))
+ }
+ flushRows()
+ flushHeader()
+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ chunks = finalizeGeneralChunks(chunks, childrenPattern)
+ if len(chunks) == 0 {
+ return emptyOutputs(), nil
+ }
+ return chunkOutputs(chunks), nil
+}
+
+func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, fmt.Errorf("GeneralChunker: %w", err)
+ }
+ units := upstream.JSONResult
+ if upstream.OutputFormat == schema.PayloadFormatChunks {
+ units = upstream.Chunks
+ }
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+ primaryPattern := compileDelimPattern(c.param.Delimiters)
+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
+ units = splitGeneralUnits(units, primaryPattern)
+ if !hasCustomDelim(c.param.Delimiters) {
+ // Python naive_merge prefixes each delimiter atom with a newline before
+ // counting it. The prefix is a budgeting detail, not emitted content;
+ // without it BPE counts can shift General's soft-cap boundaries.
+ for i := range units {
+ if itemDocType(units[i]) == "text" {
+ units[i].TKNums = intPtr(tokenizeStr("\n" + units[i].Text))
+ }
+ }
+ units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")
+ }
+ units = finalizeGeneralChunks(units, childrenPattern)
+ if len(units) == 0 {
+ return emptyOutputs(), nil
+ }
+ return chunkOutputs(units), nil
+}
+
+// mergeGeneralUnits implements the general OVER_CAP policy. A text unit is
+// appended while the current chunk is at or below the overlap-scaled target;
+// the append itself may cross the target. Oversized units remain indivisible.
+func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
+ overlapPct = max(0, min(100, overlapPct))
+ threshold := float64(target) * (100 - overlapPct) / 100
+ merged := make([]schema.ChunkDoc, 0, len(units))
+ current := -1
+
+ for _, unit := range units {
+ if itemDocType(unit) != "text" {
+ merged = append(merged, cloneChunkDoc(unit))
+ current = -1
+ continue
+ }
+ if strings.TrimSpace(unit.Text) == "" {
+ continue
+ }
+
+ count := generalUnitTokens(unit)
+ unit.DocType = "text"
+ unit.CKType = "text"
+ unit.TKNums = intPtr(count)
+ if count > target {
+ merged = append(merged, cloneChunkDoc(unit))
+ current = -1
+ continue
+ }
+ if current < 0 || float64(intValue(merged[current].TKNums)) > threshold {
+ merged = append(merged, cloneChunkDoc(unit))
+ current = len(merged) - 1
+ continue
+ }
+
+ mergeGeneralChunk(&merged[current], unit, joinSep)
+ }
+
+ return applyGeneralOverlap(merged, overlapPct, joinSep)
+}
+
+func mergeSpreadsheetRows(rows []schema.ChunkDoc, target int) []schema.ChunkDoc {
+ merged := make([]schema.ChunkDoc, 0, len(rows))
+ current := -1
+ for _, row := range rows {
+ if row.CKType != "table_row" || itemDocType(row) != "text" {
+ merged = append(merged, cloneChunkDoc(row))
+ current = -1
+ continue
+ }
+ count := generalUnitTokens(row)
+ if current < 0 || count > target || !sameSpreadsheetTable(merged[current], row) || intValue(merged[current].TKNums)+count > target {
+ row.TKNums = intPtr(count)
+ merged = append(merged, cloneChunkDoc(row))
+ current = len(merged) - 1
+ continue
+ }
+ mergeGeneralChunk(&merged[current], row, "\n")
+ mergeSpreadsheetRowRange(&merged[current], row)
+ }
+ return merged
+}
+
+func sameSpreadsheetTable(first, second schema.ChunkDoc) bool {
+ if first.TableID != "" || second.TableID != "" {
+ return first.TableID == second.TableID
+ }
+ if first.SheetIndex != nil || second.SheetIndex != nil {
+ return first.SheetIndex != nil && second.SheetIndex != nil && *first.SheetIndex == *second.SheetIndex
+ }
+ if firstSheet, firstOK := spreadsheetPositionSheet(first); firstOK {
+ secondSheet, secondOK := spreadsheetPositionSheet(second)
+ return secondOK && firstSheet == secondSheet
+ }
+ if first.Sheet != "" || second.Sheet != "" {
+ return first.Sheet != "" && second.Sheet != "" && first.Sheet == second.Sheet
+ }
+ return false
+}
+
+func spreadsheetPositionSheet(doc schema.ChunkDoc) (float64, bool) {
+ row, ok := firstPositionRow(lineRecord{positions: doc.Positions})
+ if !ok {
+ return 0, false
+ }
+ return row[0], true
+}
+
+func mergeSpreadsheetRowRange(dst *schema.ChunkDoc, src schema.ChunkDoc) {
+ dst.RowStart = minSpreadsheetInt(dst.RowStart, src.RowStart, false)
+ dst.RowEnd = minSpreadsheetInt(dst.RowEnd, src.RowEnd, true)
+ dst.ColStart = minSpreadsheetInt(dst.ColStart, src.ColStart, false)
+ dst.ColEnd = minSpreadsheetInt(dst.ColEnd, src.ColEnd, true)
+}
+
+func minSpreadsheetInt(first, second *int, maximum bool) *int {
+ if first == nil {
+ if second == nil {
+ return nil
+ }
+ value := *second
+ return &value
+ }
+ if second == nil {
+ value := *first
+ return &value
+ }
+ value := *first
+ if (maximum && *second > value) || (!maximum && *second < value) {
+ value = *second
+ }
+ return &value
+}
+
+func generalUnitTokens(unit schema.ChunkDoc) int {
+ if unit.TKNums != nil && *unit.TKNums > 0 {
+ return *unit.TKNums
+ }
+ return tokenizeStr(unit.Text)
+}
+
+func mergeGeneralChunk(dst *schema.ChunkDoc, src schema.ChunkDoc, joinSep string) {
+ if dst.Text != "" && src.Text != "" {
+ dst.Text += joinSep
+ }
+ dst.Text += src.Text
+ dst.TKNums = intPtr(intValue(dst.TKNums) + generalUnitTokens(src))
+ dst.PDFPositions = mergeGeneralPositions(dst.PDFPositions, src.PDFPositions)
+ dst.Positions = mergeGeneralPositions(dst.Positions, src.Positions)
+ mergeGeneralMetadata(dst, src)
+}
+
+func mergeGeneralMetadata(dst *schema.ChunkDoc, src schema.ChunkDoc) {
+ if dst.Image == "" {
+ dst.Image = src.Image
+ }
+ if dst.ImgID == "" {
+ dst.ImgID = src.ImgID
+ }
+ if dst.Layout == "" {
+ dst.Layout = src.Layout
+ }
+ if dst.LayoutType == "" {
+ dst.LayoutType = src.LayoutType
+ }
+ if dst.LayoutNo == "" {
+ dst.LayoutNo = src.LayoutNo
+ }
+ if dst.PageNumber == nil {
+ dst.PageNumber = src.PageNumber
+ }
+ if dst.Extra == nil && len(src.Extra) > 0 {
+ dst.Extra = make(map[string]json.RawMessage, len(src.Extra))
+ }
+ for key, value := range src.Extra {
+ if _, exists := dst.Extra[key]; !exists {
+ dst.Extra[key] = append(json.RawMessage(nil), value...)
+ }
+ }
+}
+
+func mergeGeneralPositions(first, second json.RawMessage) json.RawMessage {
+ if positions := mergePositionMatrix(first, second); len(positions) > 0 {
+ encoded, err := json.Marshal(positions)
+ if err == nil {
+ return encoded
+ }
+ }
+ return extendRawJSONArray(first, second)
+}
+
+func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep string) []schema.ChunkDoc {
+ if overlapPct <= 0 {
+ return chunks
+ }
+ previousText := -1
+ for i := range chunks {
+ if itemDocType(chunks[i]) != "text" {
+ previousText = -1
+ continue
+ }
+ if previousText >= 0 {
+ prefix, _ := computeOverlapPrefix(chunks[previousText].Text, overlapPct)
+ if prefix != "" {
+ current := cloneChunkDoc(chunks[i])
+ current.Text = joinGeneralOverlapText(prefix, current.Text, joinSep)
+ current.TKNums = intPtr(tokenizeStr(current.Text))
+ chunks[i] = current
+ }
+ }
+ previousText = i
+ }
+ return chunks
+}
+
+func joinGeneralOverlapText(prefix, text, separator string) string {
+ if prefix == "" {
+ return text
+ }
+ if text == "" || separator == "" {
+ return prefix + text
+ }
+ if strings.HasSuffix(prefix, separator) || strings.HasPrefix(text, separator) {
+ return prefix + text
+ }
+ return prefix + separator + text
+}
+
+func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
+ result := make([]schema.ChunkDoc, 0, len(units))
+ for _, unit := range units {
+ unit = cloneChunkDoc(unit)
+ unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))
+ unit.DocType = itemDocType(unit)
+ unit.CKType = unit.DocType
+ if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {
+ unit.TKNums = intPtr(tokenizeStr(unit.Text))
+ result = append(result, unit)
+ continue
+ }
+ for _, part := range splitDroppingDelim(unit.Text, pattern) {
+ if strings.TrimSpace(part) == "" {
+ continue
+ }
+ piece := cloneChunkDoc(unit)
+ piece.Text = part
+ piece.TKNums = intPtr(tokenizeStr(part))
+ result = append(result, piece)
+ }
+ }
+ return result
+}
+
+func normalizeGeneralNewlines(text string) string {
+ text = strings.ReplaceAll(text, "\r\n", "\n")
+ return strings.ReplaceAll(text, "\r", "\n")
+}
+
+func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Regexp) []schema.ChunkDoc {
+ visible := make([]schema.ChunkDoc, 0, len(chunks))
+ for _, chunk := range chunks {
+ chunk.Text = removeTag(chunk.Text)
+ if strings.TrimSpace(chunk.Text) == "" && chunk.Image == "" {
+ continue
+ }
+ visible = append(visible, chunk)
+ }
+ visible = splitGeneralChildren(visible, childrenPattern)
+ for i := range visible {
+ visible[i].TKNums = intPtr(tokenizeStr(visible[i].Text))
+ }
+ return visible
+}
+
+// splitGeneralChildren keeps the matched child delimiter attached to the
+// preceding child. This matches the legacy General/naive path; the shared
+// TokenChunker splitByChildren helper intentionally drops delimiters and is
+// therefore not reused here.
+func splitGeneralChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
+ if pattern == nil {
+ return chunks
+ }
+ result := make([]schema.ChunkDoc, 0, len(chunks))
+ for _, chunk := range chunks {
+ if itemDocType(chunk) != "text" {
+ result = append(result, chunk)
+ continue
+ }
+ mom := strings.TrimPrefix(chunk.Text, "\n")
+ for _, part := range splitKeepingGeneralDelimiter(chunk.Text, pattern) {
+ if strings.TrimSpace(part) == "" {
+ continue
+ }
+ piece := cloneChunkDoc(chunk)
+ piece.Text = part
+ piece.Mom = mom
+ result = append(result, piece)
+ }
+ }
+ return result
+}
+
+func splitKeepingGeneralDelimiter(text string, pattern *regexp.Regexp) []string {
+ matches := pattern.FindAllStringIndex(text, -1)
+ if len(matches) == 0 {
+ return []string{text}
+ }
+ result := make([]string, 0, len(matches)+1)
+ cursor := 0
+ for _, match := range matches {
+ if match[0] > cursor {
+ result = append(result, text[cursor:match[1]])
+ }
+ cursor = match[1]
+ }
+ if cursor < len(text) {
+ result = append(result, text[cursor:])
+ }
+ return result
+}
+
+func init() {
+ MustRegisterChunker(ComponentNameGeneralChunker)
+}
diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.go
new file mode 100644
index 0000000000..c8717c32ce
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_merge_test.go
@@ -0,0 +1,505 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "encoding/json"
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func TestMergeGeneralUnitsAllowsOneBoundaryOverflow(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "gamma", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ }
+
+ got := mergeGeneralUnits(units, 3, 0, "\n")
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"alpha\nbeta", "gamma"}) {
+ t.Fatalf("texts = %q, want [alpha\\nbeta gamma]", texts)
+ }
+ if count := intValue(got[0].TKNums); count != 4 {
+ t.Errorf("first chunk tokens = %d, want running sum 4", count)
+ }
+}
+
+func TestMergeSpreadsheetRowsUsesProjectedCapAndSheetBoundary(t *testing.T) {
+ sheetOne := 1
+ sheetTwo := 2
+ rows := []schema.ChunkDoc{
+ {Text: "row-1", DocType: "text", CKType: "table_row", TKNums: intPtr(2), SheetIndex: &sheetOne, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},
+ {Text: "row-2", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetOne, RowStart: intPtr(3), RowEnd: intPtr(3), ColStart: intPtr(1), ColEnd: intPtr(2)},
+ {Text: "row-3", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetTwo, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},
+ }
+
+ got := mergeSpreadsheetRows(rows, 3)
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"row-1\nrow-2", "row-3"}) {
+ t.Fatalf("texts = %q", texts)
+ }
+ if len(got) != 2 || got[0].TKNums == nil || *got[0].TKNums != 3 {
+ t.Fatalf("merged row token count = %#v, want 3", got)
+ }
+ if got[0].RowStart == nil || *got[0].RowStart != 2 || got[0].RowEnd == nil || *got[0].RowEnd != 3 {
+ t.Fatalf("merged row range = start:%v end:%v", got[0].RowStart, got[0].RowEnd)
+ }
+ if got[0].SheetIndex == nil || *got[0].SheetIndex != 1 || got[1].SheetIndex == nil || *got[1].SheetIndex != 2 {
+ t.Fatalf("sheet boundaries not preserved: %#v", got)
+ }
+}
+
+func TestMergeSpreadsheetRowsKeepsOversizedRowAndCapZeroAtomic(t *testing.T) {
+ sheet := 1
+ rows := []schema.ChunkDoc{
+ {Text: "small", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(2), RowEnd: intPtr(2)},
+ {Text: "oversized", DocType: "text", CKType: "table_row", TKNums: intPtr(8), SheetIndex: &sheet, RowStart: intPtr(3), RowEnd: intPtr(3)},
+ {Text: "after", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(4), RowEnd: intPtr(4)},
+ }
+ if got := mergeSpreadsheetRows(rows, 3); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {
+ t.Fatalf("oversized texts = %q", generalChunkTexts(got))
+ }
+ if got := mergeSpreadsheetRows(rows, 0); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {
+ t.Fatalf("zero-cap texts = %q", generalChunkTexts(got))
+ }
+}
+
+func TestGeneralChunkerSpreadsheetConsumesRowIR(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 3})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "orders.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},
+ {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 2, "sheet_index": 1, "table_id": "sheet-1", "row_start": 2, "row_end": 2},
+ {"text": "ID:A-101; Status:pending", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 1, "sheet_index": 1, "table_id": "sheet-1", "row_start": 3, "row_end": 3},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"ID:A-100; Status:paid\nID:A-101; Status:pending"}) {
+ t.Fatalf("spreadsheet chunks = %q", texts)
+ }
+}
+
+func TestMergeGeneralUnitsMergesWhenCurrentEqualsCap(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "at cap", DocType: "text", CKType: "text", TKNums: intPtr(3)},
+ {Text: "overflow", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ }
+
+ got := mergeGeneralUnits(units, 3, 0, "\n")
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"at cap\noverflow"}) {
+ t.Fatalf("texts = %q, want one overflow chunk", texts)
+ }
+}
+
+func TestMergeGeneralUnitsKeepsOversizedAtomWhole(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "oversized atom stays whole", DocType: "text", CKType: "text", TKNums: intPtr(8)},
+ {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ }
+
+ got := mergeGeneralUnits(units, 3, 0, "\n")
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "oversized atom stays whole", "after"}) {
+ t.Fatalf("texts = %q", texts)
+ }
+}
+
+func TestMergeGeneralUnitsAppliesUnconditionalCharacterOverlap(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "abcdefghij@@1\t0\t1\t2\t3##", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "klmnopqrst", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "uvwxyzABCD", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ }
+
+ got := mergeGeneralUnits(units, 2, 50, "\n")
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{
+ "abcdefghij@@1\t0\t1\t2\t3##",
+ "fghij\nklmnopqrst",
+ "mnopqrst\nuvwxyzABCD",
+ }) {
+ t.Fatalf("texts = %q", texts)
+ }
+ if intValue(got[1].TKNums) <= 2 {
+ t.Errorf("overlap was trimmed to cap: token count = %d", intValue(got[1].TKNums))
+ }
+}
+
+func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {
+ position := func(page int) json.RawMessage {
+ return json.RawMessage(fmt.Sprintf("[[%d,0,10,0,5]]", page))
+ }
+ chunks := []schema.ChunkDoc{
+ {Text: "alpha beta", DocType: "text", CKType: "text", PDFPositions: position(1), Positions: position(1)},
+ {Text: "gamma delta", DocType: "text", CKType: "text", PDFPositions: position(2), Positions: position(2)},
+ {Text: "epsilon zeta", DocType: "text", CKType: "text", PDFPositions: position(3), Positions: position(3)},
+ }
+
+ got := applyGeneralOverlap(chunks, 50, "\n")
+ if len(got) != 3 {
+ t.Fatalf("chunks = %d, want 3", len(got))
+ }
+ if string(got[1].PDFPositions) != string(position(2)) || string(got[1].Positions) != string(position(2)) {
+ t.Fatalf("second overlap chunk inherited prior positions: pdf=%s positions=%s", got[1].PDFPositions, got[1].Positions)
+ }
+ if string(got[2].PDFPositions) != string(position(3)) || string(got[2].Positions) != string(position(3)) {
+ t.Fatalf("third overlap chunk accumulated prior positions: pdf=%s positions=%s", got[2].PDFPositions, got[2].Positions)
+ }
+}
+
+func TestMergeSpreadsheetRowsUsesPositionSheetWhenIdentityFieldsAreMissing(t *testing.T) {
+ rows := []schema.ChunkDoc{
+ {Text: "sheet-1-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[1,2,2,1,2]]`)},
+ {Text: "sheet-2-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[2,2,2,1,2]]`)},
+ }
+
+ got := mergeSpreadsheetRows(rows, 10)
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"sheet-1-row", "sheet-2-row"}) {
+ t.Fatalf("position-only sheet boundary was lost: texts = %q", texts)
+ }
+}
+
+func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "alpha beta", DocType: "text", CKType: "text"},
+ {Text: "gamma delta", DocType: "text", CKType: "text"},
+ {Text: "epsilon zeta", DocType: "text", CKType: "text"},
+ {Text: "eta theta", DocType: "text", CKType: "text"},
+ }
+ for i := range units {
+ units[i].TKNums = intPtr(tokenizeStr(units[i].Text))
+ }
+ target := intValue(units[0].TKNums) + intValue(units[1].TKNums)
+
+ got := mergeMarkdownUnits(units, target, 50, "\n")
+ want := []string{
+ "alpha beta\ngamma delta",
+ "gamma delta\nepsilon zeta",
+ "epsilon zeta\neta theta",
+ }
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, want) {
+ t.Fatalf("markdown overlap = %q, want %q", texts, want)
+ }
+}
+
+func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {
+ previousPage := 4
+ currentPage := 5
+ units := []schema.ChunkDoc{
+ {
+ Text: "alpha beta",
+ DocType: "text",
+ CKType: "text",
+ Image: "previous-image",
+ ImgID: "previous-image-id",
+ PageNumber: &previousPage,
+ Positions: json.RawMessage(`[[0,0,10,0,10]]`),
+ Extra: map[string]json.RawMessage{"previous": json.RawMessage(`true`)},
+ },
+ {
+ Text: "gamma delta",
+ DocType: "text",
+ CKType: "text",
+ PageNumber: &currentPage,
+ Positions: json.RawMessage(`[[0,0,10,20,30]]`),
+ Extra: map[string]json.RawMessage{"current": json.RawMessage(`true`)},
+ },
+ }
+ for i := range units {
+ units[i].TKNums = intPtr(tokenizeStr(units[i].Text))
+ }
+
+ target := intValue(units[0].TKNums)
+ got := mergeMarkdownUnits(units, target, 50, "\n")
+ if len(got) != 2 {
+ t.Fatalf("chunks = %#v, want previous and overlapped current chunks", got)
+ }
+ if got[1].Text == "gamma delta" || !strings.HasSuffix(got[1].Text, "\ngamma delta") {
+ t.Fatalf("overlapped text = %q, want a prefix followed by current text", got[1].Text)
+ }
+ if got[1].Image != "" {
+ t.Errorf("overlap chunk inherited previous image %q", got[1].Image)
+ }
+ if got[1].ImgID != "" {
+ t.Errorf("overlap chunk inherited previous image id %q", got[1].ImgID)
+ }
+ if got[1].PageNumber == nil || *got[1].PageNumber != currentPage {
+ t.Errorf("overlap chunk page number = %v, want current page %d", got[1].PageNumber, currentPage)
+ }
+ if string(got[1].Positions) != `[[0,0,10,0,10],[0,0,10,20,30]]` {
+ t.Errorf("overlap chunk positions = %s, want previous and current positions", got[1].Positions)
+ }
+ if _, ok := got[1].Extra["previous"]; ok {
+ t.Errorf("overlap chunk inherited previous extra metadata: %#v", got[1].Extra)
+ }
+ if _, ok := got[1].Extra["current"]; !ok {
+ t.Errorf("overlap chunk lost current extra metadata: %#v", got[1].Extra)
+ }
+}
+
+func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "Background details", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "## Architecture", DocType: "text", CKType: "heading", TKNums: intPtr(1)},
+ {Text: "The architecture is split into parser and chunker.", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ }
+
+ got := mergeMarkdownUnits(units, 3, 0, "\n")
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{
+ "Background details",
+ "## Architecture\nThe architecture is split into parser and chunker.",
+ }) {
+ t.Fatalf("markdown chunks = %q, want body and heading/body chunks", texts)
+ }
+ if got[1].CKType != "text" {
+ t.Fatalf("heading/body chunk CKType = %q, want text after heading absorption", got[1].CKType)
+ }
+}
+
+func TestShortMarkdownHeadingBudgetIgnoresATXMarker(t *testing.T) {
+ content := strings.TrimSpace(strings.Repeat("word ", 49))
+ unit := schema.ChunkDoc{CKType: "heading", Text: "## " + content}
+ if tokenizeStr(content) >= 50 {
+ t.Fatalf("test heading content unexpectedly reached threshold: %d", tokenizeStr(content))
+ }
+ if tokenizeStr(unit.Text) < 50 {
+ t.Fatalf("test ATX heading did not cross raw threshold: %d", tokenizeStr(unit.Text))
+ }
+ if !isShortMarkdownHeading(unit) {
+ t.Fatalf("heading %q was not classified short after removing ATX marker", unit.Text)
+ }
+}
+
+func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T) {
+ previousPage := 4
+ units := []schema.ChunkDoc{
+ {Text: "first", DocType: "text", CKType: "text", TKNums: intPtr(2), Image: "previous-image", ImgID: "previous-image-id", PageNumber: &previousPage, Extra: map[string]json.RawMessage{"source": json.RawMessage(`"previous"`)}},
+ {Text: "second", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ {Text: "third", DocType: "text", CKType: "text", TKNums: intPtr(2)},
+ }
+
+ got := mergeGeneralUnits(units, 4, 50, "\n")
+ if len(got) != 2 {
+ t.Fatalf("chunks = %#v, want two chunks", got)
+ }
+ current := got[1]
+ if current.Image != "" || current.ImgID != "" {
+ t.Fatalf("overlap chunk copied previous media: image=%q img_id=%q", current.Image, current.ImgID)
+ }
+ if current.PageNumber != nil {
+ t.Fatalf("overlap chunk copied previous page number: %v", current.PageNumber)
+ }
+ if _, ok := current.Extra["source"]; ok {
+ t.Fatalf("overlap chunk copied previous extra metadata: %#v", current.Extra)
+ }
+}
+
+func TestTakeGeneralContextSentencePreservesBoundarySelection(t *testing.T) {
+ text := "one!two!three!"
+ if got, want := takeGeneralContextSentence(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {
+ t.Fatalf("suffix context = %q, want %q", got, want)
+ }
+ if got, want := takeGeneralContextSentence(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {
+ t.Fatalf("prefix context = %q, want %q", got, want)
+ }
+}
+
+func TestMergeGeneralUnitsKeepsMediaAtomicAndBreaksTextRun(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "A|B", DocType: "table", CKType: "table", TKNums: intPtr(2)},
+ {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ }
+
+ got := mergeGeneralUnits(units, 10, 0, "\n")
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "A|B", "after"}) {
+ t.Fatalf("texts = %q", texts)
+ }
+ if got[1].DocType != "table" || got[1].CKType != "table" {
+ t.Errorf("table type changed: %#v", got[1])
+ }
+}
+
+func TestMergeGeneralUnitsCombinesPositionsAndMetadata(t *testing.T) {
+ first := schema.ChunkDoc{
+ Text: "alpha",
+ DocType: "text",
+ CKType: "text",
+ TKNums: intPtr(1),
+ Image: "image-a",
+ Positions: json.RawMessage(`[[0,0,10,0,10]]`),
+ PDFPositions: json.RawMessage(`[[0,0,10,0,10]]`),
+ Extra: map[string]json.RawMessage{"source_order": json.RawMessage(`1`)},
+ PageNumber: intPtr(4),
+ }
+ second := schema.ChunkDoc{
+ Text: "beta",
+ DocType: "text",
+ CKType: "text",
+ TKNums: intPtr(1),
+ Positions: json.RawMessage(`[[0,0,10,20,30]]`),
+ PDFPositions: json.RawMessage(`[[0,0,10,20,30]]`),
+ Extra: map[string]json.RawMessage{"heading_level": json.RawMessage(`2`)},
+ }
+
+ got := mergeGeneralUnits([]schema.ChunkDoc{first, second}, 1, 0, "\n")
+ if len(got) != 1 {
+ t.Fatalf("chunks = %d, want 1", len(got))
+ }
+ if got[0].Image != "image-a" {
+ t.Errorf("image = %q, want image-a", got[0].Image)
+ }
+ if got[0].PageNumber == nil || *got[0].PageNumber != 4 {
+ t.Errorf("page number = %v, want 4", got[0].PageNumber)
+ }
+ if string(got[0].Positions) != `[[0,0,10,0,10],[0,0,10,20,30]]` {
+ t.Errorf("positions = %s", got[0].Positions)
+ }
+ if string(got[0].PDFPositions) != `[[0,0,10,0,10],[0,0,10,20,30]]` {
+ t.Errorf("PDF positions = %s", got[0].PDFPositions)
+ }
+ if string(got[0].Extra["source_order"]) != "1" || string(got[0].Extra["heading_level"]) != "2" {
+ t.Errorf("extra metadata = %#v", got[0].Extra)
+ }
+}
+
+func TestGeneralChunkerPreservesImageWithoutText(t *testing.T) {
+ component, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{{
+ "doc_type_kwd": "image",
+ "image": "image-payload",
+ }},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 1 || chunks[0]["image"] != "image-payload" {
+ t.Fatalf("chunks = %#v, want image-only unit", chunks)
+ }
+}
+
+func TestGeneralChunkerCustomDelimiterBypassesMerge(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 1,
+ "delimiters": []string{"`||`"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, generalTextInput("alpha beta||gamma delta"))
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha beta", "gamma delta"}) {
+ t.Fatalf("texts = %q", texts)
+ }
+}
+
+func TestGeneralChunkerBareDelimiterCreatesMergeAtoms(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 0,
+ "delimiters": []string{"|"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, generalTextInput("alpha|beta"))
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha", "beta"}) {
+ t.Fatalf("texts = %q", texts)
+ }
+}
+
+func TestGeneralChunkerSplitsChildrenAfterParentMerge(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 100,
+ "delimiters": []string{},
+ "children_delimiters": []string{"|"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, generalTextInput("alpha|beta"))
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha|", "beta"}) {
+ t.Fatalf("texts = %q", texts)
+ }
+ for i, chunk := range chunks {
+ if chunk["mom"] != "alpha|beta" {
+ t.Errorf("chunk[%d].mom = %q", i, chunk["mom"])
+ }
+ }
+}
+
+func generalChunkTexts(chunks []schema.ChunkDoc) []string {
+ texts := make([]string, len(chunks))
+ for i := range chunks {
+ texts[i] = chunks[i].Text
+ }
+ return texts
+}
+
+func generalTextInput(text string) map[string]any {
+ return map[string]any{
+ "name": "document.txt",
+ "file_type": "txt",
+ "output_format": "json",
+ "json": []map[string]any{{"text": text, "doc_type_kwd": "text"}},
+ }
+}
+
+func outputChunks(t *testing.T, out map[string]any) []map[string]any {
+ t.Helper()
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok {
+ t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])
+ }
+ return chunks
+}
+
+func outputTexts(t *testing.T, out map[string]any) []string {
+ t.Helper()
+ chunks := outputChunks(t, out)
+ texts := make([]string, len(chunks))
+ for i := range chunks {
+ texts[i], _ = chunks[i]["text"].(string)
+ }
+ return texts
+}
diff --git a/internal/ingestion/component/chunker/general_pdf_fallback_cgo_test.go b/internal/ingestion/component/chunker/general_pdf_fallback_cgo_test.go
new file mode 100644
index 0000000000..28d277b7f3
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_pdf_fallback_cgo_test.go
@@ -0,0 +1,62 @@
+//go:build cgo
+
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "errors"
+ "log/slog"
+ "strings"
+ "testing"
+
+ "ragflow/internal/ingestion/component"
+)
+
+func TestGeneralChunkerPDFContinuesWhenStorageResolutionFails(t *testing.T) {
+ previousResolver := component.ResolveDocumentStorageOverride
+ component.ResolveDocumentStorageOverride = func(string) (*component.DocumentStorageRef, error) {
+ return nil, errors.New("storage unavailable")
+ }
+ t.Cleanup(func() { component.ResolveDocumentStorageOverride = previousResolver })
+
+ var logs strings.Builder
+ previousLogger := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})))
+ t.Cleanup(func() { slog.SetDefault(previousLogger) })
+
+ chunker, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.pdf",
+ "file_type": "pdf",
+ "doc_id": "doc-with-missing-storage",
+ "output_format": "json",
+ "json": []map[string]any{{"text": "body", "doc_type_kwd": "text"}},
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+ if !strings.Contains(logs.String(), "could not open PDF") {
+ t.Fatalf("logs = %q, want PDF open warning", logs.String())
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 1 || chunks[0]["text"] != "body" {
+ t.Fatalf("chunks = %#v, want body chunk after crop fallback", chunks)
+ }
+}
diff --git a/internal/ingestion/component/chunker/general_python_parity_test.go b/internal/ingestion/component/chunker/general_python_parity_test.go
new file mode 100644
index 0000000000..4e467f7706
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_python_parity_test.go
@@ -0,0 +1,65 @@
+package chunker
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "reflect"
+ "strings"
+ "testing"
+
+ "ragflow/internal/parser/parser"
+)
+
+type generalTextPythonGolden struct {
+ Input string `json:"input"`
+ Params map[string]any `json:"params"`
+ Chunks []struct {
+ Text string `json:"text"`
+ } `json:"chunks"`
+}
+
+// TestGeneralChunkerTextPythonParity verifies the post-refactor boundary:
+// Python's built-in general path parses and chunks text in one operation,
+// whereas Go sends the full normalized TextParser result to GeneralChunker.
+// The golden is captured from rag.app.naive.chunk and locks the resulting
+// chunk boundaries after removing Python's representation-only leading newline.
+func TestGeneralChunkerTextPythonParity(t *testing.T) {
+ raw, err := os.ReadFile("testdata/general/text_default.python.golden.json")
+ if err != nil {
+ t.Fatalf("read Python golden: %v", err)
+ }
+ var golden generalTextPythonGolden
+ if err := json.Unmarshal(raw, &golden); err != nil {
+ t.Fatalf("decode Python golden: %v", err)
+ }
+
+ parsed := parser.NewTextParser().ParseWithResult(
+ context.Background(), "general.txt", []byte(golden.Input),
+ )
+ if parsed.Err != nil {
+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
+ }
+ component, err := NewGeneralChunker(golden.Params)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "general.txt",
+ "file_type": "txt",
+ "output_format": "json",
+ "json": parsed.JSON,
+ })
+ if err != nil {
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
+ }
+
+ got := outputTexts(t, out)
+ want := make([]string, 0, len(golden.Chunks))
+ for _, chunk := range golden.Chunks {
+ want = append(want, strings.TrimPrefix(chunk.Text, "\n"))
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("GeneralChunker text chunks = %#v, want Python %v", got, want)
+ }
+}
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
new file mode 100644
index 0000000000..ee3938ef49
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -0,0 +1,847 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "image"
+ "image/color"
+ "image/jpeg"
+ "log/slog"
+ "reflect"
+ "strings"
+ "testing"
+
+ "ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/schema"
+)
+
+func TestGeneralChunkerRegistered(t *testing.T) {
+ factory, category, metadata, ok := runtime.DefaultRegistry.Lookup(ComponentNameGeneralChunker)
+ if !ok {
+ t.Fatal("GeneralChunker is not registered")
+ }
+ if category != runtime.CategoryIngestion {
+ t.Errorf("category = %q, want %q", category, runtime.CategoryIngestion)
+ }
+ if factory == nil {
+ t.Fatal("factory is nil")
+ }
+ if _, ok := metadata.Inputs["file_type"]; !ok {
+ t.Error("registered inputs missing file_type")
+ }
+ component, err := factory(ComponentNameGeneralChunker, nil)
+ if err != nil {
+ t.Fatalf("factory: %v", err)
+ }
+ if component == nil {
+ t.Fatal("factory returned nil component")
+ }
+}
+
+func TestGeneralChunkerDefaults(t *testing.T) {
+ component, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ general := component.(*GeneralChunkerComponent)
+ if general.param.ChunkTokenSize != 512 {
+ t.Errorf("ChunkTokenSize = %d, want 512", general.param.ChunkTokenSize)
+ }
+ if general.param.OverlappedPercent != 0 {
+ t.Errorf("OverlappedPercent = %v, want 0", general.param.OverlappedPercent)
+ }
+ if general.param.TableContextSize != 0 || general.param.ImageContextSize != 0 {
+ t.Errorf("context sizes = (%d, %d), want (0, 0)", general.param.TableContextSize, general.param.ImageContextSize)
+ }
+ if len(general.param.Delimiters) != 1 || general.param.Delimiters[0] != "\n" {
+ t.Errorf("Delimiters = %q, want [newline]", general.param.Delimiters)
+ }
+ if len(general.param.ChildrenDelimiters) != 0 {
+ t.Errorf("ChildrenDelimiters = %q, want empty", general.param.ChildrenDelimiters)
+ }
+}
+
+func TestGeneralChunkerParamsNormalizeConfiguration(t *testing.T) {
+ delimiters := []string{"\n", "`---`"}
+ children := []any{". ", "! "}
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 0,
+ "delimiters": delimiters,
+ "overlapped_percent": 0.25,
+ "children_delimiters": children,
+ "table_context_size": -2,
+ "image_context_size": 7,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ general := component.(*GeneralChunkerComponent)
+ if general.param.ChunkTokenSize != 0 {
+ t.Errorf("ChunkTokenSize = %d, want 0", general.param.ChunkTokenSize)
+ }
+ if general.param.OverlappedPercent != 25 {
+ t.Errorf("OverlappedPercent = %v, want 25", general.param.OverlappedPercent)
+ }
+ if got := strings.Join(general.param.Delimiters, "|"); got != "\n|`---`" {
+ t.Errorf("Delimiters = %q", got)
+ }
+ if got := strings.Join(general.param.ChildrenDelimiters, "|"); got != ". |! " {
+ t.Errorf("ChildrenDelimiters = %q", got)
+ }
+ if general.param.TableContextSize != 0 || general.param.ImageContextSize != 7 {
+ t.Errorf("context sizes = (%d, %d), want (0, 7)", general.param.TableContextSize, general.param.ImageContextSize)
+ }
+
+ delimiters[0] = "changed"
+ children[0] = "changed"
+ if general.param.Delimiters[0] != "\n" || general.param.ChildrenDelimiters[0] != ". " {
+ t.Fatal("component params retain caller-owned slices")
+ }
+}
+
+func TestGeneralChunkerRejectsNegativeTokenBudget(t *testing.T) {
+ if _, err := NewGeneralChunker(map[string]any{"chunk_token_size": -1}); err == nil {
+ t.Fatal("NewGeneralChunker accepted a negative chunk_token_size")
+ }
+}
+
+func TestGeneralChunkerNormalizesLegacyDelimiterString(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "delimiter": "\n!?;。;!?",
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ general := component.(*GeneralChunkerComponent)
+ want := []string{"\n", "!", "?", ";", "。", ";", "!", "?"}
+ if !reflect.DeepEqual(general.param.Delimiters, want) {
+ t.Fatalf("Delimiters = %#v, want %#v", general.param.Delimiters, want)
+ }
+}
+
+func TestGeneralStrategyForFileType(t *testing.T) {
+ tests := []struct {
+ fileType string
+ want generalStrategy
+ }{
+ {"pdf", generalStrategyPDF},
+ {"docx", generalStrategyDOCX},
+ {"md", generalStrategyMarkdown},
+ {"xls", generalStrategySpreadsheet},
+ {"xlsx", generalStrategySpreadsheet},
+ {"csv", generalStrategySpreadsheet},
+ {"txt", generalStrategyText},
+ {"other", generalStrategyText},
+ }
+ for _, test := range tests {
+ t.Run(test.fileType, func(t *testing.T) {
+ if got := generalStrategyForFileType(test.fileType); got != test.want {
+ t.Errorf("generalStrategyForFileType(%q) = %v, want %v", test.fileType, got, test.want)
+ }
+ })
+ }
+}
+
+func TestGeneralChunkerLogsUnknownFileTypeFallback(t *testing.T) {
+ var logs bytes.Buffer
+ previousLogger := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
+ t.Cleanup(func() { slog.SetDefault(previousLogger) })
+
+ component, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ _, err = component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.bin",
+ "file_type": "application/x-custom",
+ "output_format": "json",
+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke unknown file type: %v", err)
+ }
+ if !strings.Contains(logs.String(), "unknown file_type") {
+ t.Fatalf("logs = %q, want unknown file_type fallback diagnostic", logs.String())
+ }
+
+ logs.Reset()
+ _, err = component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.txt",
+ "file_type": "txt",
+ "output_format": "json",
+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke canonical text file type: %v", err)
+ }
+ if strings.Contains(logs.String(), "unknown file_type") {
+ t.Fatalf("canonical text fallback emitted unknown-file diagnostic: %q", logs.String())
+ }
+}
+
+func TestGeneralChunkerInfersFileTypeFromSourceName(t *testing.T) {
+ component, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ _, err = component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.txt",
+ "output_format": "json",
+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke inferred file type: %v", err)
+ }
+}
+
+func TestGeneralChunkerRequiresFileTypeWhenSourceHasNoExtension(t *testing.T) {
+ component, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ _, err = component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document",
+ "output_format": "json",
+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},
+ })
+ if err == nil || !strings.Contains(err.Error(), "file_type") {
+ t.Fatalf("Invoke error = %v, want missing file_type error", err)
+ }
+}
+
+func TestGeneralChunkerPreservesSingleParserUnit(t *testing.T) {
+ component, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{{
+ "text": "alpha",
+ "doc_type_kwd": "text",
+ "source_order": 3,
+ "heading_level": 2,
+ }},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 1 {
+ t.Fatalf("chunks = %#v, want one chunk", out["chunks"])
+ }
+ if chunks[0]["text"] != "alpha" || chunks[0]["source_order"] != float64(3) || chunks[0]["heading_level"] != float64(2) {
+ t.Errorf("single parser unit was not preserved: %#v", chunks[0])
+ }
+}
+
+func TestGeneralChunkerMarkdownShortHeadingForcesNextUnit(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %#v, want one forced heading/body chunk", chunks)
+ }
+ if text, _ := chunks[0]["text"].(string); text != "Title\nbody" {
+ t.Errorf("text = %q, want %q", text, "Title\nbody")
+ }
+}
+
+func TestGeneralChunkerMarkdownLongHeadingUsesNormalCap(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ longHeading := strings.Repeat("heading ", 60)
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want separate long heading and body chunks", chunks)
+ }
+ if text, _ := chunks[0]["text"].(string); text != strings.TrimSpace(longHeading) {
+ t.Errorf("first text = %q, want long heading", text)
+ }
+ if text, _ := chunks[1]["text"].(string); text != "body" {
+ t.Errorf("second text = %q, want body", text)
+ }
+}
+
+func TestGeneralChunkerMarkdownImageMergesWithTextAndPreservesPayload(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "figure caption", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %#v, want one heading/image chunk", chunks)
+ }
+ if text, _ := chunks[0]["text"].(string); text != "Title\nfigure caption" {
+ t.Errorf("text = %q, want %q", text, "Title\nfigure caption")
+ }
+ if image, _ := chunks[0]["image"].(string); image != "data:image/png;base64,AAAA" {
+ t.Errorf("image = %q, want original image payload", image)
+ }
+}
+
+func TestMergeMarkdownImagesStacksRasterPayloads(t *testing.T) {
+ const pixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
+ const secondPixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYPj/HwADAgH/5ncLrgAAAABJRU5ErkJggg=="
+ got, ok := mergeMarkdownImagesChecked("data:image/png;base64,"+pixel, "data:image/png;base64,"+secondPixel)
+ if !ok {
+ t.Fatal("mergeMarkdownImagesChecked rejected valid PNG payloads")
+ }
+ decoded, ok := decodeMarkdownImage(got)
+ if !ok {
+ t.Fatalf("merged image is not decodable: %q", got)
+ }
+ if got := decoded.Bounds(); got.Dx() != 1 || got.Dy() != 2 {
+ t.Fatalf("merged image bounds = %v, want 1x2", got)
+ }
+}
+
+func TestDecodeMarkdownImageSupportsJPEGAndGIF(t *testing.T) {
+ var jpegData bytes.Buffer
+ img := image.NewRGBA(image.Rect(0, 0, 1, 1))
+ img.Set(0, 0, color.RGBA{R: 255, A: 255})
+ if err := jpeg.Encode(&jpegData, img, nil); err != nil {
+ t.Fatalf("jpeg.Encode: %v", err)
+ }
+ cases := []struct {
+ name string
+ mime string
+ data []byte
+ }{
+ {
+ name: "jpeg",
+ mime: "image/jpeg",
+ data: jpegData.Bytes(),
+ },
+ {
+ name: "gif",
+ mime: "image/gif",
+ data: mustDecodeBase64(t, "R0lGODdhAgACAIEAAP8AAAAAAAAAAAAAACwAAAAAAgACAAAIBgABCAQQEAA7"),
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ value := "data:" + tc.mime + ";base64," + base64.StdEncoding.EncodeToString(tc.data)
+ if _, ok := decodeMarkdownImage(value); !ok {
+ t.Fatalf("decodeMarkdownImage rejected %s payload", tc.mime)
+ }
+ })
+ }
+}
+
+func mustDecodeBase64(t *testing.T, value string) []byte {
+ t.Helper()
+ decoded, err := base64.StdEncoding.DecodeString(value)
+ if err != nil {
+ t.Fatalf("DecodeString: %v", err)
+ }
+ return decoded
+}
+
+func TestMergeMarkdownImagesAcceptsBareBase64Payloads(t *testing.T) {
+ const pixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="
+ const secondPixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYPj/HwADAgH/5ncLrgAAAABJRU5ErkJggg=="
+ got, ok := mergeMarkdownImagesChecked(pixel, secondPixel)
+ if !ok {
+ t.Fatal("mergeMarkdownImagesChecked rejected bare base64 payloads")
+ }
+ decoded, ok := decodeMarkdownImage(got)
+ if !ok {
+ t.Fatalf("merged bare-base64 image is not decodable: %q", got)
+ }
+ if got := decoded.Bounds(); got.Dx() != 1 || got.Dy() != 2 {
+ t.Fatalf("merged image bounds = %v, want 1x2", got)
+ }
+}
+
+func TestMergeMarkdownImagesRejectsOversizedCanvas(t *testing.T) {
+ if markdownImageWithinLimits(maxMarkdownImageDimension, maxMarkdownImageDimension) {
+ t.Fatal("maximum-dimension canvas should exceed the pixel safety limit")
+ }
+ if !markdownImageWithinLimits(1024, 1024) {
+ t.Fatal("normal image dimensions should remain mergeable")
+ }
+ if got, ok := mergeMarkdownImagesChecked("s3://bucket/first.png", "s3://bucket/second.png"); ok || got != "s3://bucket/first.png" {
+ t.Fatalf("opaque image references were changed: %q", got)
+ }
+}
+
+func TestGeneralChunkerMarkdownDoesNotDropUnmergeableImages(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "first", "doc_type_kwd": "image", "image": "s3://bucket/first.png"},
+ {"text": "second", "doc_type_kwd": "image", "image": "s3://bucket/second.png"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want two chunks so neither image reference is lost", chunks)
+ }
+ if chunks[0]["image"] != "s3://bucket/first.png" || chunks[1]["image"] != "s3://bucket/second.png" {
+ t.Fatalf("image references = %#v, want both source references", chunks)
+ }
+}
+
+func TestGeneralChunkerMarkdownShortHeadingKeepsFollowingTableAtomic(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},
+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want table and body", chunks)
+ }
+ if text, _ := chunks[0]["text"].(string); text != "Title\n<table><tr><td>A</td></tr></table>" {
+ t.Errorf("table text = %q", text)
+ }
+ if got := chunks[0]["doc_type_kwd"]; got != "table" {
+ t.Errorf("table doc_type_kwd = %v, want table", got)
+ }
+ if got := chunks[0]["ck_type"]; got != "table" {
+ t.Errorf("table ck_type = %v, want table", got)
+ }
+}
+
+func TestGeneralChunkerDOCXMediaDoesNotBreakTextMerge(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "", "doc_type_kwd": "image", "image": "figure"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want merged text and image", chunks)
+ }
+ if chunks[0]["text"] != "before\nafter" {
+ t.Errorf("merged text = %q, want before\\nafter", chunks[0]["text"])
+ }
+ if chunks[1]["doc_type_kwd"] != "image" || chunks[1]["image"] != "figure" {
+ t.Errorf("image chunk = %+v", chunks[1])
+ }
+}
+
+func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "table_context_size": 2,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want merged text and table", chunks)
+ }
+ if chunks[1]["doc_type_kwd"] != "table" {
+ t.Fatalf("table chunk = %+v", chunks[1])
+ }
+ if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {
+ t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
+ }
+}
+
+func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "before one", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "before two", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "", DocType: "table", CKType: "table"},
+ {Text: "after one", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "after two", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ }
+
+ if got, want := collectGeneralMediaContext(units, 2, 10, true), "before one\nbefore two"; got != want {
+ t.Fatalf("above context = %q, want %q", got, want)
+ }
+ if got, want := collectGeneralMediaContext(units, 2, 10, false), "after one\nafter two"; got != want {
+ t.Fatalf("below context = %q, want %q", got, want)
+ }
+}
+
+func TestGeneralChunkerDOCXCustomDelimiterDisablesTextMerge(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "delimiters": []string{"`|`"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before|after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"before", "after"}) {
+ t.Fatalf("texts = %q, want custom-delimiter units", texts)
+ }
+}
+
+func TestGeneralChunkerDOCXAppliesTextOverlap(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 2,
+ "overlapped_percent": 50,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "alpha beta", "doc_type_kwd": "text"},
+ {"text": "gamma delta", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ texts := outputTexts(t, out)
+ if len(texts) != 2 {
+ t.Fatalf("texts = %q, want two overlapped chunks", texts)
+ }
+ if texts[1] == "gamma delta" || !strings.HasSuffix(texts[1], "\ngamma delta") {
+ t.Fatalf("DOCX overlap = %q, want previous tail plus current paragraph", texts[1])
+ }
+}
+
+func TestGeneralChunkerChildrenDelimiterKeepsDelimiter(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "children_delimiters": []string{";"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.txt",
+ "file_type": "txt",
+ "output_format": "json",
+ "json": []map[string]any{{"text": "part A;part B", "doc_type_kwd": "text"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"part A;", "part B"}) {
+ t.Fatalf("children delimiter texts = %q, want [part A; part B]", texts)
+ }
+}
+
+func TestGeneralChunkerPDFPreservesPhysicalOrderAroundMedia(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 3 {
+ t.Fatalf("chunks = %#v, want text, media, text in physical order", chunks)
+ }
+ if chunks[0]["doc_type_kwd"] != "text" || chunks[0]["text"] != "before" {
+ t.Errorf("first chunk = %+v", chunks[0])
+ }
+ if chunks[1]["doc_type_kwd"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {
+ t.Errorf("media chunk = %+v", chunks[1])
+ }
+ if chunks[2]["doc_type_kwd"] != "text" || chunks[2]["text"] != "after" {
+ t.Errorf("last chunk = %+v", chunks[2])
+ }
+}
+
+func TestGeneralChunkerPDFUsesPositionOrderForMediaContext(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "table_context_size": 1,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ position := func(top float64) []any {
+ return []any{[]any{1.0, 0.0, 10.0, top, top + 5}}
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "after", "doc_type_kwd": "text", "positions": position(30)},
+ {"text": "<table>A</table>", "doc_type_kwd": "table", "positions": position(20)},
+ {"text": "before", "doc_type_kwd": "text", "positions": position(10)},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 3 {
+ t.Fatalf("chunks = %#v, want text, media, text", chunks)
+ }
+ if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {
+ t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
+ }
+ if chunks[0]["text"] != "before" || chunks[2]["text"] != "after" {
+ t.Errorf("position-ordered text = [%v, %v]", chunks[0]["text"], chunks[2]["text"])
+ }
+}
+
+func TestSortPDFUnitsKeepsUnpositionedMediaInInputOrder(t *testing.T) {
+ position := func(top float64) json.RawMessage {
+ return json.RawMessage(fmt.Sprintf("[[1,0,10,%g,%g]]", top, top+5))
+ }
+ units := []schema.ChunkDoc{
+ {Text: "before", DocType: "text", CKType: "text", Positions: position(10)},
+ {Text: "figure", DocType: "image", CKType: "image"},
+ {Text: "after", DocType: "text", CKType: "text", Positions: position(30)},
+ }
+
+ got := sortPDFUnits(units)
+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "figure", "after"}) {
+ t.Fatalf("PDF units = %q, want unpositioned media to keep its input position", texts)
+ }
+}
+
+func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {
+ component, err := NewGeneralChunker(nil)
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "headers.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{{
+ "text": "Name; Amount",
+ "doc_type_kwd": "table",
+ "ck_type": "table_header",
+ "sheet_index": 1,
+ "table_id": "sheet-1",
+ "positions": []any{[]any{1.0, 1.0, 1.0, 1.0, 2.0}},
+ }},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %#v, want one header-only chunk", chunks)
+ }
+ if chunks[0]["text"] != "Name; Amount" || chunks[0]["ck_type"] != "table_header" {
+ t.Fatalf("header-only chunk = %#v", chunks[0])
+ }
+}
+
+func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "image_context_size": 10,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "figures.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "Revenue", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},
+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},
+ {"text": "Growth", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 3 {
+ t.Fatalf("chunks = %#v, want row, image, row", chunks)
+ }
+ if chunks[1]["ck_type"] != "image" {
+ t.Fatalf("image chunk = %#v", chunks[1])
+ }
+ if chunks[1]["context_above"] != "Revenue" || chunks[1]["context_below"] != "Growth" {
+ t.Fatalf("image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
+ }
+}
+
+func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"image_context_size": 10})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "figures.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "Sheet one", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},
+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},
+ {"text": "Sheet two", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2, "tk_nums": 1},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 3 {
+ t.Fatalf("chunks = %#v, want row, image, row", chunks)
+ }
+ if chunks[1]["context_above"] != "Sheet one" || chunks[1]["context_below"] != nil {
+ t.Fatalf("cross-sheet image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])
+ }
+}
+
+func TestGeneralChunkerPDFAttachesOutlineOnce(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.pdf",
+ "file_type": "pdf",
+ "output_format": "json",
+ "file": map[string]any{
+ "outline": []map[string]any{{"title": "Chapter 1", "level": 0}},
+ },
+ "json": []map[string]any{{"text": "body", "doc_type_kwd": "text"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %#v, want one chunk", chunks)
+ }
+ outline, ok := chunks[0]["__outline__"].([]any)
+ if !ok || len(outline) != 1 {
+ t.Fatalf("outline = %#v, want one entry", chunks[0]["__outline__"])
+ }
+ entry, _ := outline[0].(map[string]any)
+ if entry["title"] != "Chapter 1" || entry["depth"] != float64(0) {
+ t.Errorf("outline entry = %#v", entry)
+ }
+}
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index 46b994a0a3..3cab60a8d8 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -438,37 +438,54 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {
if dt == "" {
dt = "text"
}
- var imgID *string
- if it.ImgID != "" {
- img := it.ImgID
- imgID = &img
- }
- meta := make(map[string]any)
- if it.ContentLtks != "" {
- meta["content_ltks"] = it.ContentLtks
- }
- if it.ContentSmLtks != "" {
- meta["content_sm_ltks"] = it.ContentSmLtks
- }
- if it.TitleTks != "" {
- meta["title_tks"] = it.TitleTks
- }
- if it.TitleSmTks != "" {
- meta["title_sm_tks"] = it.TitleSmTks
+ appendRecord := func(recordText string) {
+ var imgID *string
+ if it.ImgID != "" {
+ img := it.ImgID
+ imgID = &img
+ }
+ meta := make(map[string]any)
+ if it.ContentLtks != "" {
+ meta["content_ltks"] = it.ContentLtks
+ }
+ if it.ContentSmLtks != "" {
+ meta["content_sm_ltks"] = it.ContentSmLtks
+ }
+ if it.TitleTks != "" {
+ meta["title_tks"] = it.TitleTks
+ }
+ if it.TitleSmTks != "" {
+ meta["title_sm_tks"] = it.TitleSmTks
+ }
+ for k, v := range it.Extra {
+ meta[k] = json.RawMessage(v)
+ }
+ out = append(out, lineRecord{
+ text: recordText,
+ docType: dt,
+ imgID: imgID,
+ layout: it.Layout,
+ ckType: it.CKType,
+ pdfPositions: it.PDFPositions,
+ positions: it.Positions,
+ parentMeta: meta,
+ })
}
- for k, v := range it.Extra {
- meta[k] = json.RawMessage(v)
+ // TextParser intentionally emits one coordinate-free text item for the
+ // whole document. Title-family chunkers still need one record per line
+ // so heading regexes can see headings after the first line. Structured
+ // media/table items and positioned text remain atomic.
+ plainText := itemDocType(it) == "text" && (it.CKType == "" || it.CKType == "text")
+ unpositioned := len(it.PDFPositions) == 0 && len(it.Positions) == 0
+ if plainText && unpositioned && strings.Contains(text, "\n") {
+ for _, line := range strings.Split(normalizeGeneralNewlines(text), "\n") {
+ if trim(line) != "" {
+ appendRecord(line)
+ }
+ }
+ continue
}
- out = append(out, lineRecord{
- text: text,
- docType: dt,
- imgID: imgID,
- layout: it.Layout,
- ckType: it.CKType,
- pdfPositions: it.PDFPositions,
- positions: it.Positions,
- parentMeta: meta,
- })
+ appendRecord(text)
}
return out
}
diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.go
index bf97caabc5..a9edcea927 100644
--- a/internal/ingestion/component/chunker/group_test.go
+++ b/internal/ingestion/component/chunker/group_test.go
@@ -22,6 +22,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
)
func TestGroupTitleChunker_Registered(t *testing.T) {
@@ -106,6 +107,71 @@ func TestExtractLineRecords_TextKeyStillWorks(t *testing.T) {
}
}
+func TestExtractLineRecords_ParserStructuredTextSplitsLines(t *testing.T) {
+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "notes.txt", []byte("# Chapter 1\nfirst body\n# Chapter 2\nsecond body"))
+ if parsed.Err != nil {
+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
+ }
+
+ records := extractLineRecords(map[string]any{
+ "name": "notes.txt",
+ "output_format": "json",
+ "json": parsed.JSON,
+ })
+ if len(records) != 4 {
+ t.Fatalf("got %d records, want one record per non-empty line", len(records))
+ }
+ want := []string{"# Chapter 1", "first body", "# Chapter 2", "second body"}
+ for i, record := range records {
+ if record.text != want[i] {
+ t.Errorf("record[%d].text = %q, want %q", i, record.text, want[i])
+ }
+ }
+}
+
+func TestTitleFamilyChunkersRecognizeHeadingsAfterFirstParserLine(t *testing.T) {
+ source := "# Chapter 1\n" + strings.Repeat("first section body ", 40) + "\n# Chapter 2\n" + strings.Repeat("second section body ", 40)
+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "notes.txt", []byte(source))
+ if parsed.Err != nil {
+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
+ }
+ input := map[string]any{
+ "name": "notes.txt",
+ "output_format": "json",
+ "json": parsed.JSON,
+ }
+
+ group, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ manual, err := NewManualChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewManualChunker: %v", err)
+ }
+ for _, tc := range []struct {
+ name string
+ component runtime.Component
+ }{
+ {name: "group", component: group},
+ {name: "manual", component: manual},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ out, err := tc.component.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want one per heading section", len(chunks))
+ }
+ if !strings.Contains(chunks[1]["text"].(string), "# Chapter 2") {
+ t.Fatalf("second chunk lost its heading: %#v", chunks[1])
+ }
+ })
+ }
+}
+
func TestGroupTitleChunker_InvokeEmptyInput(t *testing.T) {
c, err := NewGroupTitleChunker(map[string]any{
"levels": [][]string{{`^# `}},
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index 1d49834b6f..aa819d5127 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -17,10 +17,11 @@
// QAChunker extracts question-answer pairs from parsed content.
//
// Input formats and extraction strategies:
-// - Text (txt, csv) → delimiter-based Q&A (comma or tab)
-// - Markdown (md) → heading-based Q&A
-// - HTML (xls, xlsx) → table-based Q&A (first two columns)
-// - JSON (pdf, docx, xlsx) → text sections via delimiter; table items via extractQATable
+// - Text (txt, csv) → delimiter-based Q&A (comma or tab)
+// - Markdown (md) → heading-based Q&A
+// - HTML table → table-based Q&A (first two columns)
+// - JSON → typed spreadsheet cells first; otherwise text sections
+// or the HTML-table fallback for parsers that emit table markup.
//
// Every Q&A pair becomes a single chunk whose text is
// "Question: {q}\tAnswer: {a}" (ingestion renames text to
@@ -119,7 +120,11 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (m
case schema.PayloadFormatText:
qaPairs = extractQAText(stringPtrVal(upstream.TextResult))
default:
- qaPairs = extractQAJSON(upstream.JSONResult)
+ fileType := upstream.FileType
+ if strings.TrimSpace(fileType) == "" && isCSV(upstream.Name) {
+ fileType = "csv"
+ }
+ qaPairs = extractQAJSON(upstream.JSONResult, fileType)
}
chunks := make([]schema.ChunkDoc, 0, len(qaPairs))
@@ -322,6 +327,9 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {
for _, cells := range rows {
// Python qa.py:365 requires exactly two fields for CSV pairs.
if strictPairs && len(cells) != 2 {
+ if len(pairs) > 0 {
+ pairs[len(pairs)-1].Answer += "\n" + strings.Join(cells, ",")
+ }
continue
}
var texts []string
@@ -530,22 +538,33 @@ func detectDelimiter(lines []string) string {
// JSON / structured QA extraction
// ---------------------------------------------------------------------------
-func extractQAJSON(items []schema.ChunkDoc) []qaPair {
+func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {
var pairs []qaPair
+ strictCSV := strings.EqualFold(fileType, "csv")
for _, item := range items {
- txt, _ := itemText(item)
- if txt == "" {
- continue
- }
- // XLSX (#18800) emits OutputFormat json with HTML tables in item
- // text and doc_type_kwd=table. Route those through extractQATable
- // so spreadsheet QA keeps working; plain text items stay on the
- // delimiter path used by pdf/docx.
var tmp []qaPair
- if itemDocType(item) == "table" {
- tmp = extractQATable(txt, false)
+ if item.CKType == "table_header" || item.CKType == "table_row" {
+ // QA spreadsheets have no header row: the parser's structural
+ // table_header is still the first question/answer record.
+ if strictCSV && len(item.Cells) != 2 {
+ if len(pairs) > 0 {
+ pairs[len(pairs)-1].Answer += "\n" + strings.Join(item.Cells, ",")
+ }
+ continue
+ }
+ tmp = extractQARowCells(item.Cells, strictCSV, item.RowStart)
} else {
- tmp = extractQAText(txt)
+ txt, _ := itemText(item)
+ if txt == "" {
+ continue
+ }
+ // Non-spreadsheet table items may still carry HTML markup. Keep
+ // the HTML fallback for parsers that do not expose typed cells.
+ if itemDocType(item) == "table" {
+ tmp = extractQATable(txt, strictCSV)
+ } else {
+ tmp = extractQAText(txt)
+ }
}
// Preserve the source item's image id and coordinates on each
// extracted pair
@@ -559,6 +578,26 @@ func extractQAJSON(items []schema.ChunkDoc) []qaPair {
return pairs
}
+func extractQARowCells(cells []string, strict bool, rowStart *int) []qaPair {
+ if strict && len(cells) != 2 {
+ return nil
+ }
+ texts := make([]string, 0, len(cells))
+ for _, cell := range cells {
+ if cell = strings.TrimSpace(cell); cell != "" {
+ texts = append(texts, cell)
+ }
+ }
+ if len(texts) < 2 {
+ return nil
+ }
+ rowNum := -1
+ if rowStart != nil && *rowStart > 0 {
+ rowNum = *rowStart - 1
+ }
+ return []qaPair{{Question: texts[0], Answer: texts[1], RowNum: rowNum}}
+}
+
func init() {
MustRegisterChunker(ComponentNameQAChunker)
}
diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
new file mode 100644
index 0000000000..4c5161d91f
--- /dev/null
+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
@@ -0,0 +1,93 @@
+package chunker
+
+import (
+ "context"
+ "strings"
+ "testing"
+
+ "ragflow/internal/parser/parser"
+)
+
+func TestCSVQAFirstRowIsData(t *testing.T) {
+ p := parser.NewCSVParser()
+ res := p.ParseWithResult(context.Background(), "qa.csv", []byte("q1,a1\nq2,a2\n"))
+ if res.Err != nil {
+ t.Fatal(res.Err)
+ }
+
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{
+ "name": "qa.csv",
+ "file_type": "csv",
+ "output_format": res.OutputFormat,
+ "json": res.JSON,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 2 {
+ t.Fatalf("expected both QA rows, got %d chunks: %#v", len(chunks), chunks)
+ }
+ if got, _ := chunks[0]["text"].(string); got != "Question: q1\tAnswer: a1" {
+ t.Fatalf("first row was not emitted as QA data: %q", got)
+ }
+}
+
+func TestCSVTypedRowsFoldMalformedRowIntoPreviousAnswer(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{
+ "name": "qa.csv",
+ "file_type": "csv",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"q1", "a1"}},
+ {"doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"question", "", "extra"}},
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("malformed row created or removed a pair: %#v", chunks)
+ }
+ got, _ := chunks[0]["text"].(string)
+ if !strings.Contains(got, "Answer: a1\nquestion,,extra") {
+ t.Fatalf("malformed row was not folded into the previous answer: %q", got)
+ }
+}
+
+func TestCSVJSONHTMLTableUsesStrictContinuationRules(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{
+ "name": "qa.csv",
+ "file_type": "csv",
+ "output_format": "json",
+ "json": []map[string]any{{
+ "doc_type_kwd": "table",
+ "text": "<table><tr><td>q1</td><td>a1</td></tr>" +
+ "<tr><td>q2</td><td>a2</td><td>trailing</td></tr></table>",
+ }},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("malformed HTML row created a QA pair: %#v", chunks)
+ }
+ got, _ := chunks[0]["text"].(string)
+ if !strings.Contains(got, "Answer: a1\nq2,a2,trailing") {
+ t.Fatalf("malformed HTML row was not folded into the previous answer: %q", got)
+ }
+}
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index a3af4c2cb6..472771afd8 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -171,6 +171,30 @@ func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {
}
}
+func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ inputs := map[string]any{
+ "name": "questions.csv",
+ "output_format": "json",
+ "json": []map[string]any{{
+ "doc_type_kwd": "text",
+ "ck_type": "table_row",
+ "cells": []string{"question", "answer", "unexpected"},
+ }},
+ }
+ out, err := comp.Invoke(t.Context(), nil, inputs)
+ if err != nil {
+ t.Fatalf("Invoke failed: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 0 {
+ t.Fatalf("chunks = %#v, want malformed CSV row rejected", chunks)
+ }
+}
+
// Non-CSV names keep the old "first two non-empty cells" rule on the HTML
// table path. Since #18800 an .xlsx file reaches the chunker as "json", not
// "html", so this covers the shared HTML branch and not the XLSX pipeline.
@@ -394,3 +418,32 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {
}
}
}
+
+func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := comp.Invoke(t.Context(), nil, map[string]any{
+ "name": "orders.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},
+ {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want both QA rows", out["chunks"])
+ }
+ if got, _ := chunks[0]["text"].(string); got != "Question: ID\tAnswer: Status" {
+ t.Fatalf("first-row QA = %q", got)
+ }
+ if got, _ := chunks[1]["text"].(string); got != "Question: A-100\tAnswer: paid" {
+ t.Fatalf("second-row QA = %q", got)
+ }
+}
diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go
index b3b90a12c2..585c9197de 100644
--- a/internal/ingestion/component/chunker/register.go
+++ b/internal/ingestion/component/chunker/register.go
@@ -124,6 +124,7 @@ var ChunkerInputs = map[string]string{
"content": "Alias for \"text\".",
"chunks": "Optional upstream chunk list (structured JSON form).",
"name": "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.",
+ "file_type": "Canonical parser routing extension used by GeneralChunker.",
"_created_time": "Optional upstream timestamp (RFC3339Nano, s).",
"_elapsed_time": "Optional upstream elapsed time (s).",
}
diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.go
index 74fb1bee18..0dc53979fb 100644
--- a/internal/ingestion/component/chunker/table.go
+++ b/internal/ingestion/component/chunker/table.go
@@ -113,10 +113,49 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)
// tableItems returns the per-row records, preferring JSONResult and
// falling back to Chunks. Each record becomes exactly one chunk.
func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {
- if len(items) > 0 {
- return items
+ source := items
+ if len(source) == 0 {
+ source = chunks
}
- return chunks
+ if len(source) == 0 {
+ return nil
+ }
+ dataTables := make(map[string]struct{})
+ for _, item := range source {
+ if item.CKType == "table_row" {
+ dataTables[spreadsheetTableKey(item)] = struct{}{}
+ }
+ }
+ filtered := make([]schema.ChunkDoc, 0, len(source))
+ for _, item := range source {
+ // Spreadsheet parsers expose the header as schema metadata. It is
+ // already represented in each table_row and must not become a data
+ // chunk of its own. A table with no data rows still needs its header
+ // as the only searchable representation.
+ if item.CKType == "table_header" {
+ if _, hasRows := dataTables[spreadsheetTableKey(item)]; hasRows {
+ continue
+ }
+ }
+ filtered = append(filtered, item)
+ }
+ return filtered
+}
+
+func spreadsheetTableKey(item schema.ChunkDoc) string {
+ if item.TableID != "" {
+ return "table:" + item.TableID
+ }
+ if item.SheetIndex != nil {
+ return fmt.Sprintf("sheet-index:%d", *item.SheetIndex)
+ }
+ if item.Sheet != "" {
+ return "sheet:" + item.Sheet
+ }
+ if sheet, ok := spreadsheetPositionSheet(item); ok {
+ return fmt.Sprintf("position-sheet:%g", sheet)
+ }
+ return "unknown"
}
func init() {
diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.go
index 556a47caae..c5cde7d98f 100644
--- a/internal/ingestion/component/chunker/table_test.go
+++ b/internal/ingestion/component/chunker/table_test.go
@@ -79,3 +79,54 @@ func TestTableChunker_EmptyRows(t *testing.T) {
t.Errorf("got %d chunks, want 0", len(chunks))
}
}
+
+func TestTableChunkerSkipsSpreadsheetHeaderRecord(t *testing.T) {
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "orders.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},
+ {"text": "A-100; paid", "doc_type_kwd": "table", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},
+ },
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want only the data row", len(chunks))
+ }
+ if chunks[0]["ck_type"] != "table_row" || chunks[0]["text"] != "A-100; paid" {
+ t.Fatalf("chunks = %#v, want the table row only", chunks)
+ }
+}
+
+func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "empty.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},
+ },
+ })
+ if len(chunks) != 1 {
+ t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))
+ }
+ if chunks[0]["ck_type"] != "table_header" || chunks[0]["text"] != "ID; Name" {
+ t.Fatalf("chunk = %#v, want the header record", chunks[0])
+ }
+}
+
+func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
+ chunks := tableChunksOf(t, map[string]any{
+ "name": "mixed.xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},
+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},
+ {"text": "A-1; paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2},
+ },
+ })
+ if len(chunks) != 2 {
+ t.Fatalf("got %d chunks, want header-only sheet plus data row", len(chunks))
+ }
+ if chunks[0]["ck_type"] != "table_header" || chunks[1]["ck_type"] != "table_row" {
+ t.Fatalf("chunks = %#v, want header then row", chunks)
+ }
+}
diff --git a/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json b/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json
new file mode 100644
index 0000000000..6b2c4aeb87
--- /dev/null
+++ b/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json
@@ -0,0 +1,34 @@
+{
+ "input": "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six;Eta seven!Theta eight?Iota nine.",
+ "params": {
+ "chunk_token_size": 5,
+ "delimiters": [
+ "\n",
+ "!",
+ "?",
+ ";",
+ "。",
+ ";",
+ "!",
+ "?"
+ ],
+ "overlapped_percent": 0
+ },
+ "chunks": [
+ {
+ "text": "\nAlpha one.\nBeta two"
+ },
+ {
+ "text": "\n Gamma three\nDelta four"
+ },
+ {
+ "text": "\n Epsilon five\nZeta six"
+ },
+ {
+ "text": "\nEta seven\nTheta eight"
+ },
+ {
+ "text": "\nIota nine."
+ }
+ ]
+}
diff --git a/internal/ingestion/component/chunker/textcode_parity_test.go b/internal/ingestion/component/chunker/textcode_parity_test.go
index e7ba5a50b6..04daf104d0 100644
--- a/internal/ingestion/component/chunker/textcode_parity_test.go
+++ b/internal/ingestion/component/chunker/textcode_parity_test.go
@@ -37,14 +37,13 @@ type textCodeGolden struct {
Chunks []map[string]any `json:"chunks"`
}
-// TestTextParserTokenChunkerParity covers the production text&code handoff:
-// TextParser.ParseWithResult emits structured JSON items after splitting on
-// the text&code delimiters while keeping matched delimiters in the item text;
-// TokenChunker owns the final token-budget merge. The Python golden is
-// generated by the same TxtParser path used by parser._code; comparison
-// follows the parser alignment contract and ignores representation-only chunk
-// boundaries and whitespace.
-func TestTextParserTokenChunkerParity(t *testing.T) {
+// TestTextParserGeneralChunkerParity covers the production text&code handoff:
+// TextParser emits one normalized JSON unit and GeneralChunker owns delimiter
+// splitting plus the final token-budget merge. The Python golden is generated
+// by the same TxtParser path used by parser._code; comparison follows the
+// content alignment contract and ignores representation-only boundaries and
+// whitespace.
+func TestTextParserGeneralChunkerParity(t *testing.T) {
cases := []struct {
name string
samplePath string
@@ -90,29 +89,28 @@ func TestTextParserTokenChunkerParity(t *testing.T) {
t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
}
- component, err := NewTokenChunker(map[string]any{
+ component, err := NewGeneralChunker(map[string]any{
"chunk_token_size": float64(textCodeTokenSize),
- // Parser already consumed the text&code delimiters. An empty
- // list makes this invocation exercise only the downstream merge.
- "delimiters": []string{},
+ "delimiters": textCodeDelimiter,
})
if err != nil {
- t.Fatalf("NewTokenChunker: %v", err)
+ t.Fatalf("NewGeneralChunker: %v", err)
}
out, err := component.Invoke(context.Background(), nil, map[string]any{
"name": tc.samplePath,
+ "file_type": "txt",
"output_format": "json",
"json": parsed.JSON,
})
if err != nil {
- t.Fatalf("TokenChunker.Invoke: %v", err)
+ t.Fatalf("GeneralChunker.Invoke: %v", err)
}
if msg, ok := out["_ERROR"].(string); ok && msg != "" {
- t.Fatalf("TokenChunker returned _ERROR: %s", msg)
+ t.Fatalf("GeneralChunker returned _ERROR: %s", msg)
}
got, ok := out["chunks"].([]map[string]any)
if !ok {
- t.Fatalf("TokenChunker chunks has type %T", out["chunks"])
+ t.Fatalf("GeneralChunker chunks has type %T", out["chunks"])
}
if len(got) < tc.minChunks {
t.Fatalf("final chunk count = %d, want at least %d", len(got), tc.minChunks)
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 1cdefce641..91269b6af8 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -238,7 +238,7 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs
if engine != nil {
defer engine.Close()
}
- return c.invokeJSONPayload(ctx, items, delimPattern, childrenPattern, engine), nil
+ return c.invokeJSONPayload(ctx, items, delimPattern, childrenPattern, engine, upstream.FileType), nil
}
}
@@ -531,7 +531,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil
// invokeJSONPayload handles structured upstream input. Items fan
// across 4 goroutines; merge is by input index.
-func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []schema.ChunkDoc, delimPattern, childrenPattern *regexp.Regexp, engine deepdoctype.PDFEngine) map[string]any {
+func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []schema.ChunkDoc, delimPattern, childrenPattern *regexp.Regexp, engine deepdoctype.PDFEngine, fileType string) map[string]any {
if len(items) == 0 {
return emptyOutputs()
}
@@ -544,6 +544,12 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
}
lanes := partition(len(items), workers)
perItem := make([][]schema.ChunkDoc, len(items))
+ dataTables := make(map[string]struct{})
+ for _, item := range items {
+ if item.CKType == "table_row" {
+ dataTables[spreadsheetTableKey(item)] = struct{}{}
+ }
+ }
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
@@ -556,7 +562,20 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
perItem[i] = nil
continue
}
- perItem[i] = chunkFromItem(items[i], delimPattern)
+ if items[i].CKType == "table_header" {
+ if _, hasRows := dataTables[spreadsheetTableKey(items[i])]; hasRows {
+ // The typed header is metadata carried by every row. It
+ // must not become an independent TokenChunker chunk when
+ // row IR is consumed by a legacy pipeline.
+ perItem[i] = nil
+ continue
+ }
+ }
+ if isTextParserSentenceFallback(fileType, c.param.Delimiters, items[i]) {
+ perItem[i] = splitTextParserSentences(items[i])
+ } else {
+ perItem[i] = chunkFromItem(items[i], delimPattern)
+ }
}
}(lane.start, lane.end)
}
@@ -610,12 +629,56 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s
return chunkOutputs(out)
}
+// isTextParserSentenceFallback restores the semantic sentence units that the
+// shared TextParser used to emit. Parser boundaries remain owned by the
+// chunker: this only applies the text/code parser's default sentence grammar
+// before TokenChunker merges units by its configured token budget.
+func isTextParserSentenceFallback(fileType string, delimiters []string, item schema.ChunkDoc) bool {
+ if itemDocType(item) != "text" || len(delimiters) != 1 || delimiters[0] != "\n" {
+ return false
+ }
+ switch strings.ToLower(strings.TrimSpace(fileType)) {
+ case "txt", "py", "js", "java", "c", "cpp", "h", "php", "go", "ts", "sh", "cs", "kt", "sql":
+ return true
+ default:
+ return false
+ }
+}
+
+func splitTextParserSentences(item schema.ChunkDoc) []schema.ChunkDoc {
+ text := itemTextOrFallback(item)
+ if !sentenceDelimiter.MatchString(text) {
+ return []schema.ChunkDoc{buildChunkDoc(item, "text", text, "", "")}
+ }
+ parts := splitSentencesLossless(text)
+ out := make([]schema.ChunkDoc, 0, len(parts))
+ for _, part := range parts {
+ part = strings.TrimSpace(part)
+ if part == "" || strings.TrimSpace(sentenceDelimiter.ReplaceAllString(part, "")) == "" {
+ continue
+ }
+ out = append(out, buildChunkDoc(item, "text", part, "", ""))
+ }
+ if len(out) == 0 {
+ return []schema.ChunkDoc{buildChunkDoc(item, "text", text, "", "")}
+ }
+ return out
+}
+
// ---------------------------------------------------------------------------
// JSON-payload internals
// ---------------------------------------------------------------------------
// chunkFromItem mirrors _build_json_chunks for a single item.
func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp) []schema.ChunkDoc {
+ if it.CKType == "table_row" {
+ row := cloneChunkDoc(it)
+ row.DocType = "text"
+ row.CKType = "table_row"
+ row.Text = itemTextOrFallback(it)
+ row.TKNums = intPtr(tokenizeStr(row.Text))
+ return []schema.ChunkDoc{row}
+ }
ckType := itemDocType(it)
txt := itemTextOrFallback(it)
if ckType != "text" {
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index 4fe546146e..ed0053003c 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -26,6 +26,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/ingestion/component/schema"
+ textparser "ragflow/internal/parser/parser"
)
// TestTokenChunker_Registered asserts the registry has a CategoryIngestion
@@ -69,6 +70,69 @@ func TestTokenChunker_InvokeEmptyInput(t *testing.T) {
}
}
+func TestTokenChunkerPreservesSpreadsheetRowBoundaries(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{"chunk_token_size": 512})
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "orders.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1},
+ {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 1},
+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2},
+ {"text": "ID: B-1; Status: open", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 2},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok {
+ t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])
+ }
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want one row chunk per sheet", chunks)
+ }
+ if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "ID: B-1; Status: open" {
+ t.Fatalf("row chunks = %#v", chunks)
+ }
+ for i, chunk := range chunks {
+ if chunk["ck_type"] != "table_row" {
+ t.Errorf("chunk[%d] ck_type = %v, want table_row", i, chunk["ck_type"])
+ }
+ }
+}
+
+func TestTokenChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{"chunk_token_size": 512})
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "orders.xlsx",
+ "file_type": "xlsx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},
+ {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1},
+ {"text": "Name; Owner", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want data row and header-only sheet", chunks)
+ }
+ if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "Name; Owner" {
+ t.Fatalf("chunks = %#v, want row followed by header-only sheet", chunks)
+ }
+}
+
// TestTokenChunker_InvokeDelimMode_BasicChunking drives the
// delimiter-mode path with a backtick delimiter and asserts each
// chunk carries the matched delimiter text within itself (split
@@ -228,6 +292,33 @@ func TestTokenChunker_InvokeJSONPayload(t *testing.T) {
}
}
+func TestTokenChunkerTextParserJSONKeepsSentenceBoundaries(t *testing.T) {
+ parsed := textparser.NewTextParser().ParseWithResult(t.Context(), "doc.txt", []byte("first!second!"))
+ if parsed.Err != nil {
+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
+ }
+
+ c, err := NewTokenChunker(map[string]any{
+ "chunk_token_size": 128,
+ "delimiters": []string{"\n"},
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.txt",
+ "file_type": "txt",
+ "output_format": "json",
+ "json": parsed.JSON,
+ })
+ if err != nil {
+ t.Fatalf("TokenChunker.Invoke: %v", err)
+ }
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"first!\nsecond!"}) {
+ t.Fatalf("text parser JSON chunks = %q, want sentence boundary between parser units", texts)
+ }
+}
+
func TestTokenChunker_InvokeJSONPayload_IndexesHeaderOnlyEmail(t *testing.T) {
c, err := NewTokenChunker(map[string]any{
"delimiter_mode": "delimiter",
diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
index 4700710f2c..2434affcd9 100644
--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
@@ -45,7 +45,7 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {
t.Fatal(res.Err)
}
- inputs := map[string]any{"name": "qa.xlsx", "output_format": res.OutputFormat}
+ inputs := map[string]any{"name": "qa.xlsx", "file_type": "xlsx", "output_format": res.OutputFormat}
switch res.OutputFormat {
case "json":
inputs["json"] = res.JSON
@@ -65,38 +65,38 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {
return chunks
}
-// TestXLSXQARegression is the end-to-end smoke test: every row of the sheet
-// becomes one chunk. The chunker has no header concept, so the first row is
-// a Q&A pair like any other.
-func TestXLSXQARegression(t *testing.T) {
+// TestXLSXQAFirstRowIsData protects the QA spreadsheet contract: workbooks
+// contain question/answer rows without a header, so the parser's structural
+// table_header item is still the first QA pair.
+func TestXLSXQAFirstRowIsData(t *testing.T) {
chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{
- {"question", "answer"},
- {"What is RAGFlow?", "A RAG engine."},
- {"Where are the docs?", "On the website."},
+ {"q1", "a1"},
+ {"q2", "a2"},
}))
- t.Logf("QA chunks=%d", len(chunks))
- if len(chunks) != 3 {
- t.Fatalf("expected 3 chunks, got %d", len(chunks))
+ if len(chunks) != 2 {
+ t.Fatalf("expected both QA rows, got %d chunks: %#v", len(chunks), chunks)
+ }
+ if got := chunkTexts(chunks); !strings.Contains(got[0], "q1") || !strings.Contains(got[0], "a1") {
+ t.Fatalf("first row was not emitted as QA data: %#v", got)
}
}
-// A spreadsheet cell keeps the newline its author typed (Alt+Enter) and the
-// parser renders it into the <tr>/<td> HTML verbatim, so a QA pair whose
-// question or answer spans lines must survive extraction intact. These rows
-// used to disappear from the chunk list without a trace.
+// A spreadsheet cell keeps the newline its author typed (Alt+Enter), so a QA
+// pair whose question or answer spans lines must survive cells-first
+// extraction intact. These rows used to disappear from the chunk list without
+// a trace.
func TestXLSXQAMultilineCells(t *testing.T) {
const multilineQ = "请问全国碳排放权交易市场纳入配额管理的重点排放单\n位名录,是否会公布?"
const multilineA = "需要公布。根据《碳排放权交易管理办法(试行)》。"
const multilineAnswer = "跨行的答案\n第二行\n第三行"
chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{
- {"question", "answer"},
{multilineQ, multilineA},
{"跨行的问句\n第二行", multilineAnswer},
}))
t.Logf("QA chunks=%d", len(chunks))
- if len(chunks) != 3 {
- t.Fatalf("expected 3 chunks, got %d", len(chunks))
+ if len(chunks) != 2 {
+ t.Fatalf("expected 2 data-row chunks, got %d", len(chunks))
}
texts := strings.Join(chunkTexts(chunks), "\n")
for _, want := range []string{multilineQ, multilineA, multilineAnswer} {
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index f35ba2b6b9..051075cd85 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -715,6 +715,9 @@ func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in
// them on ck["important_kwd"]. Keyword extraction pins
// temperature to extractorTemperature (0.2) to mirror generator.py.
func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
+ if strings.TrimSpace(chunkText) == "" {
+ return nil
+ }
if _, exists := ck["important_kwd"]; exists {
return nil
}
@@ -757,6 +760,9 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, i
// runAutoQuestions extracts questions for the current chunk and stores
// them on ck["question_kwd"]. See runAutoKeywords for the temperature pin.
func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
+ if strings.TrimSpace(chunkText) == "" {
+ return nil
+ }
if _, exists := ck["question_kwd"]; exists {
return nil
}
@@ -807,6 +813,9 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,
// runAutoSummary extracts a concise summary for the current chunk using autoSummaryPrompt
// and stores it on ck["summary"].
func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
+ if strings.TrimSpace(chunkText) == "" {
+ return nil
+ }
if _, exists := ck["summary"]; exists {
return nil
}
@@ -845,6 +854,9 @@ func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.D
for i, ck := range in.chunks {
i, ck := i, ck
text := extractorChunkText(ck)
+ if strings.TrimSpace(text) == "" {
+ continue
+ }
fn := func() error {
if err := c.runAutoKeywords(ctx, db, in, ck, text); err != nil {
return fmt.Errorf("chunk %d keywords: %w", i, err)
@@ -870,6 +882,9 @@ func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *go
for i, ck := range in.chunks {
i, ck := i, ck
text := extractorChunkText(ck)
+ if strings.TrimSpace(text) == "" {
+ continue
+ }
fn := c.remainingExtractionJob(ctx, db, in, i, ck, text)
f, err := extractorPool.Submit(ctx, fn)
if err != nil {
@@ -936,6 +951,9 @@ func awaitFutures(ctx context.Context, futs []utility.WorkerPoolFuture[extractor
// runAutoKeywords/runAutoQuestions shape but parses a JSON object and
// merges into the chunk's metadata map.
func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {
+ if strings.TrimSpace(chunkText) == "" {
+ return nil
+ }
if !c.Param.Metadata.Enabled || len(c.Param.Metadata.Metadata) == 0 {
return nil
}
@@ -1006,10 +1024,24 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,
}
// extractorChunkText resolves the body an extraction is run against.
-// Pre-index components must carry canonical string "text" only.
+//
+// Canonical text wins. Media-only chunks may use their explicit surrounding
+// context; chunks with neither body nor context are skipped by the extraction
+// schedulers.
func extractorChunkText(ck map[string]any) string {
- v, _ := ck["text"].(string)
- return v
+ if v, _ := ck["text"].(string); strings.TrimSpace(v) != "" {
+ return v
+ }
+ var contextParts []string
+ for _, key := range []string{"context_above", "context_below"} {
+ if v, _ := ck[key].(string); strings.TrimSpace(v) != "" {
+ contextParts = append(contextParts, strings.TrimSpace(v))
+ }
+ }
+ if len(contextParts) > 0 {
+ return strings.Join(contextParts, " ")
+ }
+ return ""
}
// chunkCacheID returns the chunk's stable per-chunk id, assigned by the chunker
@@ -1063,6 +1095,9 @@ func setMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, s
// - callStructured wraps callRaw with cleanup + explicit JSON parsing and
// returns a map — the metadata path, matching Python gen_metadata.
func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (*extractorChatResponse, error) {
+ if strings.TrimSpace(chunkText) == "" {
+ return nil, fmt.Errorf("extractor: chunk text is empty")
+ }
driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)
if err != nil {
return nil, err
@@ -1548,8 +1583,9 @@ func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs [
}
// buildExtractorMessages assembles system + user messages for one extraction
-// call. The user message strictly carries chunkText (or a fallback single space
-// if empty), ensuring a clean and consistent message contract.
+// call. Normal per-chunk schedulers reject empty chunk text before reaching
+// this low-level builder; the single-space fallback remains for direct
+// callRaw callers that rely on the historical message shape.
// System prompt is omitted if empty or whitespace-only to avoid sending empty
// system turns to LLM providers.
func buildExtractorMessages(systemPrompt, chunkText string) []eschema.Message {
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index 3863345322..9c9ab461c0 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -167,6 +167,43 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {
}
}
+func TestExtractorComponent_SkipsMediaChunksWithoutTextAndUsesContext(t *testing.T) {
+ stub := withStubChatInvoker(t, stubResponse{Content: "context summary"})
+
+ c := &ExtractorComponent{Param: schema.ExtractorParam{
+ LLMID: "gpt-4o-mini",
+ Summary: schema.SummaryExtractConfig{Enabled: true},
+ }}
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "chunks": []map[string]any{
+ {"image": "image-only"},
+ {"image": "context-image", "context_above": "The table shows", "context_below": "annual revenue."},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if calls := stub.Calls(); calls != 1 {
+ t.Fatalf("LLM calls = %d, want one call for the context-bearing media chunk", calls)
+ }
+ chunks, ok := out["chunks"].([]map[string]any)
+ if !ok || len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want two input chunks", out["chunks"])
+ }
+ if _, exists := chunks[0]["summary"]; exists {
+ t.Errorf("image-only chunk received a summary: %#v", chunks[0]["summary"])
+ }
+ if chunks[1]["summary"] != "context summary" {
+ t.Errorf("context-bearing media summary = %#v, want context summary", chunks[1]["summary"])
+ }
+ stub.mu.Lock()
+ defer stub.mu.Unlock()
+ request := stub.lastRequest()
+ if got := request.Messages[len(request.Messages)-1].Content; got != "The table shows annual revenue." {
+ t.Errorf("media extraction prompt = %q, want joined context", got)
+ }
+}
+
// TestExtractorComponent_Invoke_LLMError verifies a mock LLM
// error is surfaced through Invoke with the component-name prefix.
func TestExtractorComponent_Invoke_LLMError(t *testing.T) {
@@ -957,7 +994,7 @@ func TestCleanLLMText(t *testing.T) {
func TestExtractorComponent_callStructured(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `{"a": 1}`})
c := &ExtractorComponent{}
- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")
+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "chunk text")
if err != nil {
t.Fatalf("callStructured: %v", err)
}
@@ -967,7 +1004,7 @@ func TestExtractorComponent_callStructured(t *testing.T) {
// Non-JSON response → (nil, nil), not an error.
withStubChatInvoker(t, stubResponse{Content: "this is not JSON"})
- got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")
+ got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "chunk text")
if err != nil {
t.Fatalf("callStructured on non-JSON: %v", err)
}
@@ -980,7 +1017,7 @@ func TestExtractorComponent_callStructured(t *testing.T) {
func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `preamble<think>reasoning</think>{"a": 1}`})
c := &ExtractorComponent{}
- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")
+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "chunk text")
if err != nil {
t.Fatalf("callStructured: %v", err)
}
@@ -989,6 +1026,14 @@ func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {
}
}
+func TestExtractorComponentRejectsEmptyLLMInput(t *testing.T) {
+ withStubChatInvoker(t, stubResponse{Content: `{"unexpected": true}`})
+ c := &ExtractorComponent{}
+ if _, err := c.callRaw(t.Context(), nil, extractorInputs{llmID: "m"}, "system", " "); err == nil {
+ t.Fatal("callRaw accepted an empty chunk text")
+ }
+}
+
// TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions verifies
// keyword and question extraction on chunks.
func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T) {
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index d264ac9513..a336963a44 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -274,6 +274,7 @@ func defaultSetups() map[string]schema.ParserSetup {
"spreadsheet": {
"parse_method": "deepdoc",
"flatten_media_to_text": false,
+ "html4excel": false,
"output_format": "json",
"suffix": []string{"xls", "xlsx", "csv"},
},
@@ -385,6 +386,7 @@ func (c *ParserComponent) Inputs() map[string]string {
//
// name string — carried over from the upstream file/document
// name (or doc_id when no name is available).
+// file_type string — canonical extension used for parser dispatch.
// output_format string — always "json".
// json []map[string]any — canonical structured parser items.
// lang string — language for tokenization.
@@ -399,6 +401,7 @@ func (c *ParserComponent) Inputs() map[string]string {
func (c *ParserComponent) Outputs() map[string]string {
return map[string]string{
"name": "string: the upstream file/document name (or doc_id when no name is available).",
+ "file_type": "string: canonical extension used for parser dispatch (for example pdf, md, xlsx, or other).",
"output_format": "string: always \"json\".",
"json": "[]map[string]any: canonical structured parser items.",
"lang": "string: the language for tokenization (e.g. English, Dutch, Chinese).",
@@ -415,6 +418,7 @@ func (c *ParserComponent) Outputs() map[string]string {
//
// {
// "name": string (from inputs["name"], file.name, or doc_id),
+// "file_type": string (canonical extension used for parser dispatch),
// "output_format": "json",
// "json": []map[string]any,
// "lang": string (from inputs["lang"]; e.g. English, Dutch),
@@ -513,6 +517,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
lang, _ := getString(inputs, "lang")
out := buildParserOutputs(ctx, dispatched, filename, binary, lang)
+ out["file_type"] = string(fileTypeExt)
// Forward the storage references so a downstream chunker can
// re-acquire the source PDF and crop section images on demand,
// instead of carrying the binary across the component boundary.
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index 24daa44820..5770ec4e06 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -231,7 +231,7 @@ func pythonFamilyName(raw string) string {
return "epub"
case "json", "jsonl", "ldjson":
return "json"
- case "txt", "py", "js", "java", "c", "cpp", "h", "php",
+ case "text&code", "txt", "py", "js", "java", "c", "cpp", "h", "php",
"go", "ts", "sh", "cs", "kt", "sql":
return "text&code"
case "mp4", "avi", "mkv", "mov", "webm", "flv",
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index ef20c6a6c4..d24b1b5521 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -142,8 +142,8 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
if got := items[0]["ck_type"]; got != "heading" {
t.Errorf("markdown item[0].ck_type = %v, want heading", got)
}
- if got := items[0]["text"]; got != "Title" {
- t.Errorf("markdown item[0].text = %v, want Title", got)
+ if got := items[0]["text"]; got != "# Title" {
+ t.Errorf("markdown item[0].text = %v, want # Title", got)
}
if got := items[1]["text"]; got != "Body" {
t.Errorf("markdown item[1].text = %v, want Body", got)
@@ -168,6 +168,29 @@ func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {
requireJSONText(t, out, "Recovered title")
}
+func TestBuildParserOutputsPreservesPDFOutlineMetadata(t *testing.T) {
+ wantOutline := []map[string]any{{"title": "Overview", "level": 0, "page_number": 1}}
+ out := buildParserOutputs(t.Context(), parser.ParseResult{
+ OutputFormat: "json",
+ File: map[string]any{
+ "name": "report.pdf",
+ "outline": wantOutline,
+ },
+ JSON: []map[string]any{{"text": "body", "doc_type_kwd": "text"}},
+ }, "report.pdf", nil, "")
+ file, ok := out["file"].(map[string]any)
+ if !ok {
+ t.Fatalf("file = %T/%v, want parser file metadata", out["file"], out["file"])
+ }
+ gotOutline, ok := file["outline"].([]map[string]any)
+ if !ok {
+ t.Fatalf("file.outline = %T/%v, want []map[string]any", file["outline"], file["outline"])
+ }
+ if !reflect.DeepEqual(gotOutline, wantOutline) {
+ t.Fatalf("file.outline = %#v, want %#v", gotOutline, wantOutline)
+ }
+}
+
func TestParseMarkdownToJSONItemsDoesNotResolveRemoteImages(t *testing.T) {
originalResolver := net.DefaultResolver
dnsLookup := make(chan struct{}, 1)
diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go
index aecbd754e0..4318713dd3 100644
--- a/internal/ingestion/component/parser_test.go
+++ b/internal/ingestion/component/parser_test.go
@@ -90,7 +90,7 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {
if _, ok := out["json"]; !ok {
t.Errorf("Outputs() missing key %q", "json")
}
- for _, key := range []string{"name", "lang", "file", "doc_id", "bucket", "path"} {
+ for _, key := range []string{"name", "file_type", "lang", "file", "doc_id", "bucket", "path"} {
if _, ok := out[key]; !ok {
t.Errorf("Outputs() missing runtime output key %q", key)
}
@@ -100,6 +100,58 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {
}
}
+func TestParserComponentInvokeOutputsResolvedFileType(t *testing.T) {
+ tests := []struct {
+ name string
+ inputs map[string]any
+ want string
+ }{
+ {
+ name: "explicit family hint wins over filename",
+ inputs: map[string]any{
+ "binary": "package main",
+ "file_type": "text&code",
+ "name": "misleading.pdf",
+ },
+ want: "txt",
+ },
+ {
+ name: "top-level name",
+ inputs: map[string]any{"binary": "hello", "name": "notes.txt"},
+ want: "txt",
+ },
+ {
+ name: "file descriptor name",
+ inputs: map[string]any{
+ "binary": "hello",
+ "file": map[string]any{"name": "notes.txt"},
+ },
+ want: "txt",
+ },
+ {
+ name: "unknown input",
+ inputs: map[string]any{"binary": "hello", "name": "notes.unknown"},
+ want: "other",
+ },
+ }
+
+ component := &ParserComponent{setups: defaultSetups()}
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ if routed := string(fileTypeFromInputs(test.inputs)); routed != test.want {
+ t.Fatalf("routing file type = %q, want %q", routed, test.want)
+ }
+ out, err := component.Invoke(t.Context(), nil, test.inputs)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if got := out["file_type"]; got != test.want {
+ t.Errorf("file_type = %v, want %q", got, test.want)
+ }
+ })
+ }
+}
+
func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {
component, err := NewParserComponent(map[string]any{
"pdf": map[string]any{"output_format": "markdown"},
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index eb9aa1f893..f4d3372b35 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -124,6 +124,15 @@ type ChunkDoc struct {
TopInt []int `json:"top_int,omitempty"`
PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`
Positions json.RawMessage `json:"positions,omitempty"`
+ TableID string `json:"table_id,omitempty"`
+ Sheet string `json:"sheet,omitempty"`
+ SheetIndex *int `json:"sheet_index,omitempty"`
+ Headers []string `json:"headers,omitempty"`
+ Cells []string `json:"cells,omitempty"`
+ RowStart *int `json:"row_start,omitempty"`
+ RowEnd *int `json:"row_end,omitempty"`
+ ColStart *int `json:"col_start,omitempty"`
+ ColEnd *int `json:"col_end,omitempty"`
Extra map[string]json.RawMessage `json:"-"`
}
@@ -143,6 +152,7 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {
"context_above", "context_below", "questions", "keywords", "summary",
"chunk_order_int", "title_tks", "title_sm_tks", "content_ltks",
"content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",
+ "table_id", "sheet", "sheet_index", "headers", "cells", "row_start", "row_end", "col_start", "col_end",
} {
delete(raw, key)
}
@@ -236,6 +246,10 @@ func (d ChunkDoc) ToMap() map[string]any {
for k, raw := range d.Extra {
out[k] = decodeExtraValue(raw)
}
+ // Every pre-index chunk has one canonical text field. Keep the key even
+ // for media-only chunks so chunk identity and image upload use the same
+ // wire contract; contextual text remains explicit retrieval metadata.
+ out["text"] = d.Text
delete(out, "content_with_weight")
if len(d.PDFPositions) > 0 {
out["_pdf_positions"] = decodeStructuredValue(d.PDFPositions)
diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.go
index 3cb5987527..3b280e4cb5 100644
--- a/internal/ingestion/component/schema/chunker.go
+++ b/internal/ingestion/component/schema/chunker.go
@@ -23,9 +23,9 @@ import (
"strings"
)
-// ChunkerFromUpstream is the shared upstream payload consumed by all four
-// chunker variants (TokenChunker, TitleChunker, GroupTitleChunker,
-// HierarchyTitleChunker).
+// ChunkerFromUpstream is the shared upstream payload consumed by the chunker
+// variants (GeneralChunker, TokenChunker, TitleChunker, GroupTitleChunker,
+// HierarchyTitleChunker, and QAChunker).
//
// It mirrors the wire shape defined across two equivalent Pydantic
// schemas in the Python codebase:
@@ -41,6 +41,7 @@ import (
// created_time: float | None (alias _created_time)
// elapsed_time: float | None (alias _elapsed_time)
// name: str (required)
+// file_type: str (canonical parser routing extension)
// file: dict | None
// chunks: list[dict] | None
// output_format: Literal["json","markdown","text","html","chunks"] | None
@@ -55,6 +56,9 @@ type ChunkerFromUpstream struct {
// Name is the source document name. Required.
Name string `json:"name"`
+ // FileType is the canonical extension selected by the upstream Parser.
+ FileType string `json:"file_type,omitempty"`
+
// DocID is the originating document ID. When set, the chunker can
// re-resolve the source PDF from storage to crop section images on
// demand, instead of carrying the binary across the component
diff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.go
index da429fa717..e70b09b5a2 100644
--- a/internal/ingestion/component/schema/parser.go
+++ b/internal/ingestion/component/schema/parser.go
@@ -27,6 +27,9 @@ type ParserOutputs struct {
// Name is the resolved source filename.
Name string `json:"name"`
+ // FileType is the canonical extension used for parser dispatch.
+ FileType string `json:"file_type"`
+
// OutputFormat is always "json". Downstream components consume structured JSON items.
OutputFormat string `json:"output_format,omitempty"`
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index 0943c23a73..eb1e745a79 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -18,6 +18,7 @@ package schema
import (
"encoding/json"
+ "reflect"
"strings"
"testing"
)
@@ -123,6 +124,7 @@ func TestFileOutputsJSONRoundTrip(t *testing.T) {
func TestParserOutputsJSONRoundTrip(t *testing.T) {
original := ParserOutputs{
Name: "input.pdf",
+ FileType: "pdf",
OutputFormat: "json",
JSON: []map[string]any{{"text": "hello", "doc_type_kwd": "text"}},
Lang: "English",
@@ -138,6 +140,9 @@ func TestParserOutputsJSONRoundTrip(t *testing.T) {
if !strings.Contains(string(data), `"output_format":"json"`) {
t.Errorf("expected output_format in JSON, got %s", data)
}
+ if !strings.Contains(string(data), `"file_type":"pdf"`) {
+ t.Errorf("expected file_type in JSON, got %s", data)
+ }
var decoded ParserOutputs
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
@@ -145,6 +150,9 @@ func TestParserOutputsJSONRoundTrip(t *testing.T) {
if decoded.OutputFormat != "json" {
t.Errorf("OutputFormat round-trip mismatch: got %q", decoded.OutputFormat)
}
+ if decoded.FileType != "pdf" {
+ t.Errorf("FileType round-trip mismatch: got %q", decoded.FileType)
+ }
if len(decoded.JSON) != 1 {
t.Errorf("JSON round-trip mismatch: got %d", len(decoded.JSON))
}
@@ -198,6 +206,7 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {
md := "# title"
original := ChunkerFromUpstream{
Name: "doc.pdf",
+ FileType: "pdf",
OutputFormat: PayloadFormatChunks,
Chunks: []ChunkDoc{{Text: "alpha"}},
MarkdownResult: &md,
@@ -209,6 +218,9 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {
if !strings.Contains(string(data), `"output_format":"chunks"`) {
t.Errorf("expected output_format in JSON, got %s", data)
}
+ if !strings.Contains(string(data), `"file_type":"pdf"`) {
+ t.Errorf("expected file_type in JSON, got %s", data)
+ }
var decoded ChunkerFromUpstream
if err := json.Unmarshal(data, &decoded); err != nil {
t.Fatalf("unmarshal: %v", err)
@@ -216,11 +228,56 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {
if decoded.Name != "doc.pdf" || decoded.OutputFormat != PayloadFormatChunks {
t.Errorf("round-trip mismatch: %+v", decoded)
}
+ if decoded.FileType != "pdf" {
+ t.Errorf("FileType round-trip mismatch: got %q", decoded.FileType)
+ }
if len(decoded.Chunks) != 1 {
t.Errorf("Chunks round-trip mismatch: got %d", len(decoded.Chunks))
}
}
+func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {
+ sheetIndex, rowStart, rowEnd, colStart, colEnd := 2, 42, 42, 1, 3
+ original := ChunkDoc{
+ Text: "ID:A-100; Status:paid",
+ DocType: "text",
+ CKType: "table_row",
+ TableID: "sheet-2",
+ Sheet: "Orders",
+ SheetIndex: &sheetIndex,
+ Headers: []string{"ID", "Status", "Note"},
+ Cells: []string{"A-100", "paid", ""},
+ RowStart: &rowStart,
+ RowEnd: &rowEnd,
+ ColStart: &colStart,
+ ColEnd: &colEnd,
+ }
+
+ data, err := json.Marshal(original)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ var decoded ChunkDoc
+ if err := json.Unmarshal(data, &decoded); err != nil {
+ t.Fatalf("unmarshal: %v", err)
+ }
+ if decoded.TableID != original.TableID || decoded.Sheet != original.Sheet {
+ t.Fatalf("spreadsheet identity mismatch: got table=%q sheet=%q", decoded.TableID, decoded.Sheet)
+ }
+ if decoded.SheetIndex == nil || *decoded.SheetIndex != sheetIndex {
+ t.Fatalf("sheet index mismatch: got %v", decoded.SheetIndex)
+ }
+ if decoded.RowStart == nil || *decoded.RowStart != rowStart || decoded.RowEnd == nil || *decoded.RowEnd != rowEnd {
+ t.Fatalf("row range mismatch: start=%v end=%v", decoded.RowStart, decoded.RowEnd)
+ }
+ if decoded.ColStart == nil || *decoded.ColStart != colStart || decoded.ColEnd == nil || *decoded.ColEnd != colEnd {
+ t.Fatalf("column range mismatch: start=%v end=%v", decoded.ColStart, decoded.ColEnd)
+ }
+ if !reflect.DeepEqual(decoded.Headers, original.Headers) || !reflect.DeepEqual(decoded.Cells, original.Cells) {
+ t.Fatalf("cells mismatch: headers=%v cells=%v", decoded.Headers, decoded.Cells)
+ }
+}
+
func TestChunkerOutputsJSONRoundTrip(t *testing.T) {
original := ChunkerOutputs{
OutputFormat: PayloadFormatChunks,
diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.go
index 533fb5c75f..e580db68af 100644
--- a/internal/ingestion/component/tokenizer.go
+++ b/internal/ingestion/component/tokenizer.go
@@ -646,10 +646,15 @@ func chunksFromTokenizerUpstream(in schema.TokenizerFromUpstream) []schema.Chunk
default:
raw = cloneChunkDocs(in.JSONResult)
}
- // Keep only chunks that carry canonical pre-index text.
+ // Keep only chunks that have retrievable content: a chunk is dropped only
+ // when canonical text and both media-context fields are empty.
+ // Context-bearing media chunks may intentionally have no display text, but
+ // their surrounding prose is still searchable and must survive to the
+ // tokenizer.
filtered := raw[:0]
for _, ck := range raw {
- if strings.TrimSpace(ck.Text) == "" {
+ if strings.TrimSpace(ck.Text) == "" &&
+ strings.TrimSpace(ck.ContextAbove) == "" && strings.TrimSpace(ck.ContextBelow) == "" {
continue
}
filtered = append(filtered, ck)
@@ -786,7 +791,7 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)
smt = st
}
ck.ContentSmLtks = smt
- } else if t := ck.Text; strings.TrimSpace(t) != "" {
+ } else if t := contextualChunkText(*ck); strings.TrimSpace(t) != "" {
tt, err := tok.Tokenize(t)
if err != nil {
return fmt.Errorf("tokenizer: text tokenize: %w", err)
@@ -816,7 +821,7 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {
for _, f := range fields {
switch f {
case "text":
- b.WriteString(ck.Text)
+ b.WriteString(contextualChunkText(ck))
case "questions":
b.WriteString(ck.Questions)
case "keywords":
@@ -836,6 +841,34 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {
return b.String()
}
+// contextualChunkText is the retrieval representation of a chunk's body.
+// ContextAbove/ContextBelow are generated by GeneralChunker for media blocks
+// and must participate in both full-text tokenization and embeddings, while
+// ChunkDoc.Text remains the display/storage payload (for example, the table
+// HTML or image caption). Context values normally carry their own punctuation
+// or whitespace, matching Python's doc_tokenize_chunks_with_images contract.
+// If a producer trims both sides of a boundary, contextualChunkText inserts a
+// newline so retrieval and embedding do not fuse adjacent English words.
+func contextualChunkText(ck schema.ChunkDoc) string {
+ text := ""
+ for _, part := range []string{ck.ContextAbove, ck.Text, ck.ContextBelow} {
+ if part == "" {
+ continue
+ }
+ if text != "" && !contextBoundaryHasWhitespace(text, part) {
+ text += "\n"
+ }
+ text += part
+ }
+ return text
+}
+
+func contextBoundaryHasWhitespace(left, right string) bool {
+ leftRune, _ := utf8.DecodeLastRuneInString(left)
+ rightRune, _ := utf8.DecodeRuneInString(right)
+ return unicode.IsSpace(leftRune) || unicode.IsSpace(rightRune)
+}
+
// shouldHaveEmbedding reports whether the tokenizer must attach embedding
// vectors: the search method requests embedding AND a KB is present.
//
@@ -871,7 +904,7 @@ func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []
}
func requiresFullTextTokens(ck schema.ChunkDoc) bool {
- return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(ck.Text) != ""
+ return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(contextualChunkText(ck)) != ""
}
func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 1a616939c7..ae05a28a9f 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -240,6 +240,81 @@ func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {
}
}
+func TestTokenizerComponent_FullTextIncludesMediaContext(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ defer tokenizer.SetEngineType("")
+
+ comp, err := NewTokenizerComponent(map[string]any{
+ "search_method": []any{"full_text"},
+ })
+ if err != nil {
+ t.Fatalf("NewTokenizerComponent: %v", err)
+ }
+ out, err := comp.(*TokenizerComponent).Invoke(t.Context(), nil, map[string]any{
+ "name": "report.pdf",
+ "output_format": "chunks",
+ "chunks": []map[string]any{
+ {
+ "text": "table body",
+ "context_above": "above ",
+ "context_below": " below",
+ },
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := out["chunks"].([]map[string]any)
+ if got := chunks[0]["content_ltks"]; got != "above table body below" {
+ t.Fatalf("content_ltks = %q, want contextual text", got)
+ }
+ if got := chunks[0]["content_sm_ltks"]; got != "above table body below" {
+ t.Fatalf("content_sm_ltks = %q, want contextual text", got)
+ }
+}
+
+func TestTokenizerComponent_EmbeddingIncludesMediaContext(t *testing.T) {
+ stub := newStubEmbedder(3)
+ comp, err := NewTokenizerComponentWithResolver(
+ map[string]any{"search_method": []any{"embedding"}},
+ func(context.Context, string, string) (Embedder, string, error) {
+ return stub, "", nil
+ },
+ )
+ if err != nil {
+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)
+ }
+ _, err = comp.(*TokenizerComponent).Invoke(t.Context(), nil, map[string]any{
+ "output_format": "chunks",
+ "chunks": []map[string]any{{
+ "text": "table body",
+ "context_above": "above ",
+ "context_below": " below",
+ }},
+ "kb_id": "kb-1",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ if len(stub.callInputs) != 1 || len(stub.callInputs[0]) != 1 {
+ t.Fatalf("embedder calls = %#v, want one content input", stub.callInputs)
+ }
+ if got := stub.callInputs[0][0]; got != "above table body below" {
+ t.Fatalf("embedding input = %q, want contextual text", got)
+ }
+}
+
+func TestContextualChunkTextSeparatesTightMediaBoundaries(t *testing.T) {
+ ck := schema.ChunkDoc{
+ ContextAbove: "See results in table:",
+ Text: "Revenue grew by 20%.",
+ ContextBelow: "Conclusion follows.",
+ }
+ if got, want := contextualChunkText(ck), "See results in table:\nRevenue grew by 20%.\nConclusion follows."; got != want {
+ t.Fatalf("contextual text = %q, want %q", got, want)
+ }
+}
+
// TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero uses an
// empty chunk list, so tokenizeChunks is a no-op and the C++ pool is not needed.
func TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero(t *testing.T) {
@@ -543,6 +618,36 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {
}
}
+func TestTokenizerComponent_KeepsContextOnlyMediaChunk(t *testing.T) {
+ tokenizer.SetEngineType("infinity")
+ defer tokenizer.SetEngineType("")
+
+ comp, err := NewTokenizerComponent(map[string]any{
+ "search_method": []any{"full_text"},
+ })
+ if err != nil {
+ t.Fatalf("NewTokenizerComponent: %v", err)
+ }
+ out, err := comp.(*TokenizerComponent).Invoke(t.Context(), nil, map[string]any{
+ "name": "report.pdf",
+ "output_format": "chunks",
+ "chunks": []map[string]any{{
+ "image": "data:image/png;base64,placeholder",
+ "context_above": "figure description",
+ }},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("chunks = %#v, want context-bearing media chunk retained", chunks)
+ }
+ if got := chunks[0]["content_ltks"]; got != "figure description" {
+ t.Fatalf("content_ltks = %q, want media context", got)
+ }
+}
+
// TestChunkOrderInt_AllPathsUnconditional is the A6 regression gate: every
// surviving chunk must carry chunk_order_int, equal to its index in the
// (post-filter) reading sequence — on every output_format and every
diff --git a/internal/ingestion/pipeline/general_template_contract_test.go b/internal/ingestion/pipeline/general_template_contract_test.go
new file mode 100644
index 0000000000..c10bb074cf
--- /dev/null
+++ b/internal/ingestion/pipeline/general_template_contract_test.go
@@ -0,0 +1,127 @@
+package pipeline
+
+import (
+ "encoding/json"
+ "testing"
+)
+
+func TestGeneralTemplateUsesGeneralChunker(t *testing.T) {
+ dsl, err := LoadBuiltinDSL("general")
+ if err != nil {
+ t.Fatalf("LoadBuiltinDSL: %v", err)
+ }
+
+ var root map[string]interface{}
+ if err := json.Unmarshal([]byte(dsl), &root); err != nil {
+ t.Fatalf("decode general DSL: %v", err)
+ }
+ components, ok := root["components"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general DSL has no components")
+ }
+
+ const generalID = "GeneralChunker:SixApplesFall"
+ if _, ok := components[generalID]; !ok {
+ t.Fatalf("general DSL has no %s component", generalID)
+ }
+ if _, ok := components["TokenChunker:SixApplesFall"]; ok {
+ t.Fatal("general DSL still contains the legacy TokenChunker component")
+ }
+ component := components[generalID].(map[string]interface{})
+ obj := component["obj"].(map[string]interface{})
+ if got := obj["component_name"]; got != "GeneralChunker" {
+ t.Fatalf("component_name = %v, want GeneralChunker", got)
+ }
+ params := obj["params"].(map[string]interface{})
+ if _, ok := params["delimiter_mode"]; ok {
+ t.Fatal("GeneralChunker params contain unsupported delimiter_mode")
+ }
+ delimiters, ok := params["delimiters"].([]interface{})
+ if !ok {
+ t.Fatalf("delimiters type = %T", params["delimiters"])
+ }
+ seenSemicolon := false
+ for _, delimiter := range delimiters {
+ if delimiter == ";" {
+ seenSemicolon = true
+ }
+ }
+ if !seenSemicolon {
+ t.Fatalf("general delimiters = %#v, want ASCII semicolon", delimiters)
+ }
+
+ parserComponent, ok := components["Parser:HipSignsRhyme"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general DSL has no Parser:HipSignsRhyme component")
+ }
+ parserObj, ok := parserComponent["obj"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general parser has no object definition")
+ }
+ parserParams, ok := parserObj["params"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general parser has no params")
+ }
+ if _, ok := parserParams["outputs"]; ok {
+ t.Fatal("general parser retains obsolete html/markdown/text output declarations")
+ }
+ spreadsheet, ok := parserParams["spreadsheet"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general parser has no spreadsheet setup")
+ }
+ if got := spreadsheet["output_format"]; got != "json" {
+ t.Fatalf("general spreadsheet output_format = %v, want json", got)
+ }
+}
+
+func TestGeneralTemplateParserGraphUsesJSONContract(t *testing.T) {
+ dsl, err := LoadBuiltinDSL("general")
+ if err != nil {
+ t.Fatalf("LoadBuiltinDSL: %v", err)
+ }
+ var root map[string]interface{}
+ if err := json.Unmarshal([]byte(dsl), &root); err != nil {
+ t.Fatalf("decode general DSL: %v", err)
+ }
+ graph, ok := root["graph"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general DSL has no graph")
+ }
+ nodes, ok := graph["nodes"].([]interface{})
+ if !ok {
+ t.Fatal("general graph has no nodes")
+ }
+ for _, rawNode := range nodes {
+ node, ok := rawNode.(map[string]interface{})
+ if !ok || node["id"] != "Parser:HipSignsRhyme" {
+ continue
+ }
+ data, ok := node["data"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general parser graph node has no data")
+ }
+ form, ok := data["form"].(map[string]interface{})
+ if !ok {
+ t.Fatal("general parser graph node has no form")
+ }
+ if _, ok := form["outputs"]; ok {
+ t.Fatal("general parser graph form retains obsolete output declarations")
+ }
+ setups, ok := form["setups"].([]interface{})
+ if !ok {
+ t.Fatal("general parser graph form has no setups")
+ }
+ for _, rawSetup := range setups {
+ setup, ok := rawSetup.(map[string]interface{})
+ if !ok || setup["fileFormat"] != "spreadsheet" {
+ continue
+ }
+ if got := setup["output_format"]; got != "json" {
+ t.Fatalf("general graph spreadsheet output_format = %v, want json", got)
+ }
+ return
+ }
+ t.Fatal("general parser graph form has no spreadsheet setup")
+ }
+ t.Fatal("general graph has no Parser:HipSignsRhyme node")
+}
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 82312e8cce..08d75876c2 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -8,6 +8,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/entity"
"ragflow/internal/ingestion/component/schema"
+ parserchunk "ragflow/internal/parser/chunk"
"go.uber.org/zap"
)
@@ -95,12 +96,17 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
}
validCPNs := make(map[string]map[string]struct{}, len(schemas))
+ componentNames := make(map[string]string, len(schemas))
for _, s := range schemas {
keys := make(map[string]struct{}, len(s.ParamsDefaults))
for k := range s.ParamsDefaults {
keys[k] = struct{}{}
}
+ if s.ComponentName == "GeneralChunker" {
+ keys["delimiters"] = struct{}{}
+ }
validCPNs[s.CpnID] = keys
+ componentNames[s.CpnID] = s.ComponentName
}
result := make(map[string]interface{}, len(rawConfig))
@@ -120,6 +126,9 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
if !ok {
continue
}
+ if componentNames[key] == "GeneralChunker" {
+ params = normalizeGeneralComponentParams(params)
+ }
dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)
if hasDynamic {
params = NormalizeExtractorParams(params)
@@ -147,6 +156,35 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[
return result
}
+func normalizeGeneralComponentParams(params map[string]any) map[string]any {
+ var normalized map[string]any
+ clone := func() map[string]any {
+ if normalized == nil {
+ normalized = make(map[string]any, len(params)+1)
+ for key, value := range params {
+ normalized[key] = value
+ }
+ }
+ return normalized
+ }
+ if _, hasCanonical := params["delimiters"]; !hasCanonical {
+ if value, ok := params["delimiter"].(string); ok {
+ normalizedParams := clone()
+ normalizedParams["delimiters"] = parserchunk.ParseDelimiterField(value)
+ delete(normalizedParams, "delimiter")
+ }
+ } else if value, ok := params["delimiters"].(string); ok {
+ clone()["delimiters"] = parserchunk.ParseDelimiterField(value)
+ }
+ if value, ok := params["children_delimiters"].(string); ok {
+ clone()["children_delimiters"] = parserchunk.ParseDelimiterField(value)
+ }
+ if normalized == nil {
+ return params
+ }
+ return normalized
+}
+
// NormalizeExtractorParams normalizes a raw Extractor parameters map into the canonical modular format.
func NormalizeExtractorParams(raw map[string]any) map[string]any {
if raw == nil {
diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.go
index fbe0aee400..33422be57d 100644
--- a/internal/ingestion/pipeline/pipeline_params_test.go
+++ b/internal/ingestion/pipeline/pipeline_params_test.go
@@ -2,6 +2,7 @@ package pipeline
import (
"encoding/json"
+ "reflect"
"testing"
"ragflow/internal/entity"
@@ -41,6 +42,77 @@ func generalDSL(t *testing.T) []byte {
}
return raw
}
+
+func TestCleanComponentParamsNormalizesGeneralLegacyDelimiter(t *testing.T) {
+ dsl := map[string]any{
+ "components": map[string]any{
+ "GeneralChunker:Legacy": map[string]any{
+ "obj": map[string]any{
+ "component_name": "GeneralChunker",
+ "params": map[string]any{
+ "delimiters": []any{"\n"},
+ },
+ },
+ },
+ },
+ }
+ dslJSON, err := json.Marshal(dsl)
+ if err != nil {
+ t.Fatalf("marshal dsl: %v", err)
+ }
+ result := CleanComponentParams(dslJSON, map[string]any{
+ "GeneralChunker:Legacy": map[string]any{
+ "delimiter": "\n!?;。;!?",
+ },
+ })
+ params, ok := result["GeneralChunker:Legacy"].(map[string]any)
+ if !ok {
+ t.Fatalf("normalized component params = %#v", result)
+ }
+ want := []string{"\n", "!", "?", ";", "。", ";", "!", "?"}
+ if !reflect.DeepEqual(params["delimiters"], want) {
+ t.Fatalf("delimiters = %#v, want %#v", params["delimiters"], want)
+ }
+ if _, ok := params["delimiter"]; ok {
+ t.Fatal("legacy delimiter key survived normalization")
+ }
+}
+
+func TestCleanComponentParamsNormalizesDelimiterWhenDSLDeclaresOnlyLegacyKey(t *testing.T) {
+ dsl := map[string]any{
+ "components": map[string]any{
+ "GeneralChunker:Custom": map[string]any{
+ "obj": map[string]any{
+ "component_name": "GeneralChunker",
+ "params": map[string]any{
+ "delimiter": "\n",
+ },
+ },
+ },
+ },
+ }
+ dslJSON, err := json.Marshal(dsl)
+ if err != nil {
+ t.Fatalf("marshal DSL: %v", err)
+ }
+ result := CleanComponentParams(dslJSON, map[string]any{
+ "GeneralChunker:Custom": map[string]any{
+ "delimiter": "\n!?",
+ },
+ })
+ params, ok := result["GeneralChunker:Custom"].(map[string]any)
+ if !ok {
+ t.Fatalf("normalized component params = %#v", result)
+ }
+ want := []string{"\n", "!", "?"}
+ if !reflect.DeepEqual(params["delimiters"], want) {
+ t.Fatalf("delimiters = %#v, want %#v", params["delimiters"], want)
+ }
+ if _, ok := params["delimiter"]; ok {
+ t.Fatal("legacy delimiter key survived normalization")
+ }
+}
+
func TestCleanComponentParams_DropsLegacyFlatFields(t *testing.T) {
dslJSON := generalDSL(t)
raw := map[string]any{
diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.go
index 4ec571ad01..b6ad120e83 100644
--- a/internal/ingestion/pipeline/real_storage_integration_test.go
+++ b/internal/ingestion/pipeline/real_storage_integration_test.go
@@ -121,12 +121,11 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
- // The TokenChunker merges the two short paragraphs into one chunk under
+ // The GeneralChunker merges the two short paragraphs into one chunk under
// the global token-size budget (chunk_token_size=512, no backtick
// delimiter): this matches Python's _merge_text_chunks_by_token_size and
- // is the intended behaviour. The parser still emits two json items
- // (see the Parser state assertion below); only the downstream chunker
- // collapses them.
+ // is the intended behaviour. TextParser emits one normalized json item;
+ // GeneralChunker owns the delimiter split and collapses the atoms.
wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}
if len(chunks) != len(wantChunkTexts) {
t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))
@@ -169,21 +168,19 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
t.Fatalf("parser output_format = %v, want json", got)
}
jsonItems, ok := parserState["json"].([]map[string]any)
- if !ok || len(jsonItems) != 2 {
- t.Fatalf("parser json = %T/%v, want 2 items", parserState["json"], parserState["json"])
+ if !ok || len(jsonItems) != 1 {
+ t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])
}
- // The parser emits one json item per paragraph (two items), which the
- // downstream TokenChunker later merges into a single chunk.
- wantParserTexts := []string{"Alpha paragraph.", "Beta paragraph."}
+ wantParserTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}
for i, wantText := range wantParserTexts {
if got := jsonItems[i]["text"]; got != wantText {
t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)
}
}
- chunkerState, ok := state["TokenChunker:SixApplesFall"]
+ chunkerState, ok := state["GeneralChunker:SixApplesFall"]
if !ok {
- t.Fatal("missing TokenChunker:SixApplesFall state")
+ t.Fatal("missing GeneralChunker:SixApplesFall state")
}
if got := chunkerState["output_format"]; got != "chunks" {
t.Fatalf("chunker output_format = %v, want chunks", got)
diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
index fb237bd9cb..f745094bf4 100644
--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json
@@ -26,29 +26,11 @@
},
"Parser:HipSignsRhyme": {
"downstream": [
- "TokenChunker:SixApplesFall"
+ "GeneralChunker:SixApplesFall"
],
"obj": {
"component_name": "Parser",
"params": {
- "outputs": {
- "html": {
- "type": "string",
- "value": ""
- },
- "json": {
- "type": "Array<object>",
- "value": []
- },
- "markdown": {
- "type": "string",
- "value": ""
- },
- "text": {
- "type": "string",
- "value": ""
- }
- },
"doc": {
"output_format": "json",
"preprocess": [
@@ -112,7 +94,7 @@
},
"spreadsheet": {
"flatten_media_to_text": false,
- "output_format": "html",
+ "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": [
"main_content"
@@ -152,20 +134,20 @@
"File"
]
},
- "TokenChunker:SixApplesFall": {
+ "GeneralChunker:SixApplesFall": {
"downstream": [
"Extractor:AutoExtractDefault"
],
"obj": {
- "component_name": "TokenChunker",
+ "component_name": "GeneralChunker",
"params": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "delimiter",
"delimiters": [
"\n",
"!",
"?",
+ ";",
"。",
";",
"!",
@@ -206,7 +188,7 @@
},
"Extractor:AutoExtractDefault": {
"upstream": [
- "TokenChunker:SixApplesFall"
+ "GeneralChunker:SixApplesFall"
],
"downstream": [
"Tokenizer:LegalReadersDecide"
@@ -250,15 +232,15 @@
"targetHandle": "end"
},
{
- "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:SixApplesFallend",
+ "id": "xy-edge__Parser:HipSignsRhymestart-GeneralChunker:SixApplesFallend",
"source": "Parser:HipSignsRhyme",
"sourceHandle": "start",
- "target": "TokenChunker:SixApplesFall",
+ "target": "GeneralChunker:SixApplesFall",
"targetHandle": "end"
},
{
- "id": "xy-edge__TokenChunker:SixApplesFallstart-Extractor:AutoExtractDefaultend",
- "source": "TokenChunker:SixApplesFall",
+ "id": "xy-edge__GeneralChunker:SixApplesFallstart-Extractor:AutoExtractDefaultend",
+ "source": "GeneralChunker:SixApplesFall",
"sourceHandle": "start",
"target": "Extractor:AutoExtractDefault",
"targetHandle": "end"
@@ -293,24 +275,6 @@
{
"data": {
"form": {
- "outputs": {
- "html": {
- "type": "string",
- "value": ""
- },
- "json": {
- "type": "Array<object>",
- "value": []
- },
- "markdown": {
- "type": "string",
- "value": ""
- },
- "text": {
- "type": "string",
- "value": ""
- }
- },
"setups": [
{
"fileFormat": "pdf",
@@ -325,7 +289,7 @@
{
"fileFormat": "spreadsheet",
"flatten_media_to_text": false,
- "output_format": "html",
+ "output_format": "json",
"parse_method": "DeepDOC",
"preprocess": [
"main_content"
@@ -396,7 +360,6 @@
"form": {
"children_delimiters": [],
"chunk_token_size": 512,
- "delimiter_mode": "delimiter",
"delimiters": [
{
"value": "\n"
@@ -407,6 +370,9 @@
{
"value": "?"
},
+ {
+ "value": ";"
+ },
{
"value": "。"
},
@@ -429,10 +395,10 @@
},
"overlapped_percent": 0
},
- "label": "TokenChunker",
- "name": "Token Chunker"
+ "label": "GeneralChunker",
+ "name": "General Chunker"
},
- "id": "TokenChunker:SixApplesFall",
+ "id": "GeneralChunker:SixApplesFall",
"measured": {
"height": 74,
"width": 200
diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go
index d05aad94e3..3dcaae9fd9 100644
--- a/internal/ingestion/pipeline/template_integration_test.go
+++ b/internal/ingestion/pipeline/template_integration_test.go
@@ -133,9 +133,9 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
}
state := stateFromRunOutput(t, out)
- chunkerState, ok := state["TokenChunker:SixApplesFall"]
+ chunkerState, ok := state["GeneralChunker:SixApplesFall"]
if !ok {
- t.Fatal("missing TokenChunker:SixApplesFall state")
+ t.Fatal("missing GeneralChunker:SixApplesFall state")
}
if got := chunkerState["output_format"]; got != "chunks" {
t.Fatalf("chunker output_format = %v, want chunks", got)
@@ -191,7 +191,6 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
}
payload := terminalPayloadFromRunOutput(t, out, terminalIDs[0])
- wantTexts := []string{"Alpha paragraph.", "Beta paragraph."}
wantMergedText := "Alpha paragraph.\nBeta paragraph."
assertTokenizerTerminalChunk(t, payload, filename, wantMergedText)
@@ -217,14 +216,11 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
t.Fatalf("parser output_format = %v, want json", got)
}
jsonItems, ok := parserState["json"].([]map[string]any)
- if !ok || len(jsonItems) != 2 {
- t.Fatalf("parser json = %T/%v, want 2 items", parserState["json"], parserState["json"])
+ if !ok || len(jsonItems) != 1 {
+ t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])
}
- for i, wantText := range wantTexts {
- item := jsonItems[i]
- if got := item["text"]; got != wantText {
- t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)
- }
+ if got := jsonItems[0]["text"]; got != "Alpha paragraph.\n\nBeta paragraph." {
+ t.Fatalf("parser json[0].text = %v, want normalized full text", got)
}
chunkerState, ok := state["OneChunker:DryDrinksVisit"]
if !ok {
diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.go
index f4b5d5d561..82cc5529a2 100644
--- a/internal/ingestion/task/pipeline_executor_defaults_test.go
+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go
@@ -41,7 +41,7 @@ var builtinComponentParamsGolden = map[string]string{
"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
"one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",
diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.go
index 3e40b87505..f4762e85fd 100644
--- a/internal/parser/chunk/delim.go
+++ b/internal/parser/chunk/delim.go
@@ -38,6 +38,42 @@ func HasWrappedDelimiter(s string) bool {
return backtickWrappedRE.MatchString(s)
}
+// ParseDelimiterField converts the legacy Python delimiter string into the
+// canonical delimiter list used by the Go pipeline. Characters outside
+// backticks are individual Unicode delimiters; a backtick-wrapped token stays
+// as one list entry so multi-character delimiters remain lossless.
+func ParseDelimiterField(value string) []string {
+ if value == "" {
+ return []string{}
+ }
+ var delimiters []string
+ var wrapped strings.Builder
+ inWrapped := false
+ for _, r := range value {
+ if r == '`' {
+ if inWrapped {
+ wrapped.WriteRune(r)
+ delimiters = append(delimiters, wrapped.String())
+ wrapped.Reset()
+ inWrapped = false
+ } else {
+ inWrapped = true
+ wrapped.WriteRune(r)
+ }
+ continue
+ }
+ if inWrapped {
+ wrapped.WriteRune(r)
+ continue
+ }
+ delimiters = append(delimiters, string(r))
+ }
+ if wrapped.Len() > 0 {
+ delimiters = append(delimiters, wrapped.String())
+ }
+ return delimiters
+}
+
// CompileDelimiterPattern builds an alternation regex from delimiter strings.
//
// Each delimiter is regexp.QuoteMeta'd so whitespace and metacharacters match
@@ -79,8 +115,8 @@ func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {
// bare ones.
//
// Returns nil when no active entry remains. This helper exists for the
-// dataflow list API. The single-string parser_config.delimiter field is not
-// parsed in Go; only the []string list API is consumed.
+// dataflow list API. Call ParseDelimiterField before it when consuming the
+// legacy single-string parser_config field.
func CompileDelimiterPatternList(delims []string, keepBare bool) *regexp.Regexp {
var out []string
seen := make(map[string]struct{})
diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.go
index 5723743957..b6fea0c68f 100644
--- a/internal/parser/chunk/delim_test.go
+++ b/internal/parser/chunk/delim_test.go
@@ -40,6 +40,14 @@ func TestHasWrappedDelimiter(t *testing.T) {
}
}
+func TestParseDelimiterField(t *testing.T) {
+ got := ParseDelimiterField("\n!?;。;!?`---`")
+ want := []string{"\n", "!", "?", ";", "。", ";", "!", "?", "`---`"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("ParseDelimiterField = %#v, want %#v", got, want)
+ }
+}
+
func TestCompileDelimiterPattern(t *testing.T) {
if CompileDelimiterPattern(nil) != nil {
t.Fatal("empty list should yield nil")
diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go
index 7f50d792e4..26383ccaf0 100644
--- a/internal/parser/parser/charset_test.go
+++ b/internal/parser/parser/charset_test.go
@@ -165,13 +165,17 @@ func TestCSVParser_GBK(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatalf("CSVParser returned no JSON items")
}
- tableHTML, _ := res.JSON[0]["text"].(string)
- if strings.ContainsRune(tableHTML, '\ufffd') {
- t.Errorf("CSV HTML output contains replacement runes: %s", tableHTML)
+ var rowText strings.Builder
+ for _, item := range res.JSON {
+ text, _ := item["text"].(string)
+ rowText.WriteString(text)
+ }
+ if strings.ContainsRune(rowText.String(), '\ufffd') {
+ t.Errorf("CSV output contains replacement runes: %s", rowText.String())
}
for _, want := range []string{"笔记本电脑", "电子", "特价销售,送鼠标", "无线耳机"} {
- if !strings.Contains(tableHTML, want) {
- t.Errorf("CSV HTML missing expected substring %q; got: %s", want, tableHTML)
+ if !strings.Contains(rowText.String(), want) {
+ t.Errorf("CSV output missing expected substring %q; got: %s", want, rowText.String())
}
}
if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" && enc != "gb2312" {
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 64a3d206d6..2a53e9ef80 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -14,15 +14,9 @@
// limitations under the License.
//
-// CSVParser renders CSV data as HTML table text in structured JSON table items.
-//
-// Mirrors Python's deepdoc/parser/excel_parser.py:RAGFlowExcelParser.html():
-// - CSV data is rendered as an HTML <table> with <caption> "Data".
-// - The first row is treated as the header (<th>).
-// - Illegal control characters are replaced with spaces.
-// - Large sheets are split into chunks of chunk_rows data rows, each
-// chunk being a self-contained <table> with its own <caption> and
-// repeated header row.
+// CSVParser emits ordered spreadsheet row items in structured JSON.
+// Illegal control characters are replaced with spaces; final chunking is
+// owned by GeneralChunker.
//
// It implements the ParseResultProducer contract so the dispatch seam in
// parser_dispatch.go routes .csv files through the structured path.
@@ -36,14 +30,13 @@ import (
"strings"
)
-const csvDefaultChunkRows = defaultTableChunkRows
const csvSheetName = "Data"
// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.
type CSVParser struct {
ParseMethod string
OutputFormat string
- ChunkRows int
+ HTML4Excel bool
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -52,7 +45,6 @@ type CSVParser struct {
func NewCSVParser() *CSVParser {
return &CSVParser{
- ChunkRows: csvDefaultChunkRows,
TCADPTableResultType: "1",
TCADPMarkdownImageResponseType: "1",
}
@@ -72,19 +64,10 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
- if v, ok := setup["chunk_rows"]; ok {
- switch n := v.(type) {
- case float64:
- p.ChunkRows = int(n)
- case int:
- p.ChunkRows = n
- case int64:
- p.ChunkRows = int(n)
- }
- if p.ChunkRows <= 0 {
- p.ChunkRows = csvDefaultChunkRows
- }
+ if v, ok := setup["html4excel"].(bool); ok {
+ p.HTML4Excel = v
}
+ deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
}
@@ -99,10 +82,8 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {
}
}
-// ParseWithResult implements ParseResultProducer. It reads CSV rows
-// and renders them as HTML <table> chunks with <caption>, header row
-// repeated per chunk, and illegal-character filtering — mirroring
-// Python's RAGFlowExcelParser.html().
+// ParseWithResult implements ParseResultProducer. It reads CSV rows and emits
+// a header item followed by ordered data-row items.
// When TCADP parse_method is configured, the file is dispatched to
// the Tencent Cloud Document Parsing API.
func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
@@ -127,7 +108,10 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
decoded, encName := DecodeToUTF8(data, "text/csv")
text := string(decoded)
if strings.TrimSpace(text) == "" {
- emptyHTML := "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>"
+ var emptyJSON []map[string]any
+ if p.HTML4Excel {
+ emptyJSON = []map[string]any{NewTableJSONItem("<table><caption>Data</caption></table>", csvSheetName, [][]float64{{1, 1, 1, 1, 1}})}
+ }
return ParseResult{
OutputFormat: spreadsheetOutputFormat,
File: map[string]any{
@@ -137,7 +121,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
"format": "csv",
"sheets": 1,
},
- JSON: []map[string]any{NewTableJSONItem(emptyHTML, csvSheetName, [][]float64{{1, 1, 1, 1, 1}})},
+ JSON: emptyJSON,
}
}
@@ -154,21 +138,17 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
// Clean illegal control characters from all cells.
records = cleanIllegalControlChars(records)
- chunkRows := p.ChunkRows
- if chunkRows <= 0 {
- chunkRows = csvDefaultChunkRows
+ dataRows := make([]int, len(records)-1)
+ for i := range dataRows {
+ dataRows[i] = i + 2
}
-
- chunks := recordsToHTMLTableChunkList(records, chunkRows, csvSheetName, 1)
- items := make([]map[string]any, 0, len(chunks))
- for _, ch := range chunks {
- items = append(items, NewTableJSONItem(ch.HTML, csvSheetName, [][]float64{{
- 1,
- float64(ch.RowStart),
- float64(ch.RowEnd),
- float64(ch.ColStart),
- float64(ch.ColEnd),
- }}))
+ var items []map[string]any
+ if p.HTML4Excel {
+ if table := recordsToHTMLTableItem(records, csvSheetName, 1, 1, dataRows); table != nil {
+ items = []map[string]any{table}
+ }
+ } else {
+ items = recordsToSpreadsheetItems(records, csvSheetName, 1, 1, dataRows)
}
return ParseResult{
OutputFormat: spreadsheetOutputFormat,
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index a8504ba47b..ac2f4da99d 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -2,11 +2,10 @@ package parser
import (
"context"
- "strings"
"testing"
)
-func TestCSVParser_BasicHTMLAndJSON(t *testing.T) {
+func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {
csvData := []byte("Name,Age,City\nAlice,30,New York\nBob,25,San Francisco\n")
p := NewCSVParser()
@@ -20,34 +19,44 @@ func TestCSVParser_BasicHTMLAndJSON(t *testing.T) {
t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, "json")
}
- if len(res.JSON) != 1 {
- t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))
+ if len(res.JSON) != 3 {
+ t.Fatalf("len(res.JSON) = %d, want header plus two rows", len(res.JSON))
}
-
- item := res.JSON[0]
- if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {
- t.Fatal("item text should contain rendered <table>")
+ if res.JSON[0]["ck_type"] != "table_header" {
+ t.Fatalf("header ck_type = %v, want table_header", res.JSON[0]["ck_type"])
}
- if item["doc_type_kwd"] != "table" {
- t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
+ if res.JSON[1]["ck_type"] != "table_row" {
+ t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])
}
- if item["ck_type"] != "table" {
- t.Errorf("item ck_type = %v, want 'table'", item["ck_type"])
+ if res.JSON[1]["text"] != "Name:Alice; Age:30; City:New York ——Data" {
+ t.Errorf("row text = %v", res.JSON[1]["text"])
}
- if item["sheet"] != "Data" {
- t.Errorf("item sheet = %v, want 'Data'", item["sheet"])
+ if res.JSON[1]["sheet"] != "Data" || res.JSON[1]["sheet_index"] != 1 {
+ t.Errorf("row sheet metadata = %#v", res.JSON[1])
}
- positions, ok := item["positions"].([][]float64)
+ positions, ok := res.JSON[1]["positions"].([][]float64)
if !ok || len(positions) == 0 {
- t.Fatalf("item positions invalid: %v", item["positions"])
+ t.Fatalf("item positions invalid: %v", res.JSON[1]["positions"])
}
pos := positions[0]
// [sheet, rowStart, rowEnd, colStart, colEnd]
- if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 3 || pos[3] != 1 || pos[4] != 3 {
+ if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 2 || pos[3] != 1 || pos[4] != 3 {
t.Errorf("unexpected positions: %v", pos)
}
}
+func TestCSVParserHTML4ExcelRemainsAtomic(t *testing.T) {
+ p := NewCSVParser()
+ p.ConfigureFromSetup(map[string]any{"html4excel": true})
+ res := p.ParseWithResult(context.Background(), "qa.csv", []byte("Question,Answer\nQ1,A1\n"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+ if len(res.JSON) != 1 || res.JSON[0]["ck_type"] != "table" {
+ t.Fatalf("items = %#v, want one table item", res.JSON)
+ }
+}
+
func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
csvData := []byte("col1,col2\nval1,val2\n")
p := NewCSVParser()
@@ -63,11 +72,11 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
if res.OutputFormat != "json" {
t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
}
- if len(res.JSON) != 1 {
- t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))
+ if len(res.JSON) != 2 {
+ t.Fatalf("len(res.JSON) = %d, want header plus row", len(res.JSON))
}
- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
- t.Fatal("item text should contain rendered <table>")
+ if res.JSON[1]["ck_type"] != "table_row" {
+ t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])
}
}
@@ -85,40 +94,45 @@ func TestCSVParser_EmptyContent(t *testing.T) {
if res.OutputFormat != "json" {
t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
}
- if len(res.JSON) != 1 {
- t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))
- }
- item := res.JSON[0]
- if item["doc_type_kwd"] != "table" {
- t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
+ if len(res.JSON) != 0 {
+ t.Fatalf("len(res.JSON) = %d, want 0", len(res.JSON))
}
}
-func TestCSVParser_Chunking(t *testing.T) {
- // Header + 4 data rows, chunk size = 2
+func TestCSVParser_PreservesEveryDataRow(t *testing.T) {
+ // Header + 4 data rows; GeneralChunker owns any later token merge.
csvData := []byte("Header\nr1\nr2\nr3\nr4\n")
p := NewCSVParser()
- p.ChunkRows = 2
-
res := p.ParseWithResult(context.Background(), "chunked.csv", csvData)
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- // 4 data rows with chunk_rows=2 => 2 chunks
- if len(res.JSON) != 2 {
- t.Fatalf("len(res.JSON) = %d, want 2", len(res.JSON))
+ if len(res.JSON) != 5 {
+ t.Fatalf("len(res.JSON) = %d, want header plus four rows", len(res.JSON))
}
+ for i, item := range res.JSON[1:] {
+ pos := item["positions"].([][]float64)[0]
+ wantRow := float64(i + 2)
+ if pos[1] != wantRow || pos[2] != wantRow {
+ t.Errorf("row %d position = [%v, %v], want [%v, %v]", i, pos[1], pos[2], wantRow, wantRow)
+ }
+ }
+}
- pos1 := res.JSON[0]["positions"].([][]float64)[0]
- pos2 := res.JSON[1]["positions"].([][]float64)[0]
-
- // chunk 1: rows 2..3
- if pos1[1] != 2 || pos1[2] != 3 {
- t.Errorf("chunk 1 row range: [%v, %v], want [2, 3]", pos1[1], pos1[2])
+func TestCSVParserKeepsVariableRowWidths(t *testing.T) {
+ p := NewCSVParser()
+ res := p.ParseWithResult(context.Background(), "mixed.csv", []byte("Question,Answer\nQ1,A1\nQ2,A2,extra\n"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult failed: %v", res.Err)
+ }
+ if len(res.JSON) != 3 {
+ t.Fatalf("items = %d, want header plus two rows", len(res.JSON))
+ }
+ if got := len(res.JSON[1]["cells"].([]string)); got != 2 {
+ t.Fatalf("first data row width = %d, want 2", got)
}
- // chunk 2: rows 4..5
- if pos2[1] != 4 || pos2[2] != 5 {
- t.Errorf("chunk 2 row range: [%v, %v], want [4, 5]", pos2[1], pos2[2])
+ if got := len(res.JSON[2]["cells"].([]string)); got != 3 {
+ t.Fatalf("second data row width = %d, want 3", got)
}
}
diff --git a/internal/parser/parser/delimiter.go b/internal/parser/parser/delimiter.go
index 9ab0e62e1d..e06e6d5104 100644
--- a/internal/parser/parser/delimiter.go
+++ b/internal/parser/parser/delimiter.go
@@ -25,10 +25,3 @@ package parser
// alignment test normalizes delimiters. It is a non-test symbol so both the
// production parsers and the alignment tests share one source of truth.
const DefaultMarkdownDelimiter = "\n!?;。;!?"
-
-// DefaultTextCodeDelimiter is the flow parser's default text&code delimiter
-// set, used by TextParser and when the text&code alignment test normalizes
-// delimiters. It is a non-test symbol so both the production parser and the
-// alignment tests share one source of truth (no duplicate hard-coded copy in
-// text_parser.go, which would otherwise drift silently).
-const DefaultTextCodeDelimiter = "\n!?;。;!?"
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go
index 71286bea47..8ba42ad7d6 100644
--- a/internal/parser/parser/docx_ir.go
+++ b/internal/parser/parser/docx_ir.go
@@ -102,6 +102,7 @@ type docxIRList struct {
type docxIRRun struct {
Type string `json:"type"` // "text", "line_break", "image"
Text string `json:"text"`
+ Data []byte `json:"data"` // raw image bytes for inline image runs
Content []docxIRElement `json:"content"` // nested elements (used in table cells)
}
@@ -262,19 +263,7 @@ func buildDOCXJSONSections(irJSON string) []map[string]any {
for _, el := range sec.Elements {
switch el.Type {
case "paragraph", "heading":
- text := joinDOCXIRRuns(el.contentRuns())
- if strings.TrimSpace(text) == "" {
- continue
- }
- item := map[string]any{
- "text": text,
- "image": nil,
- "doc_type_kwd": "text",
- }
- if el.Type == "heading" {
- item["ck_type"] = "heading"
- }
- sections = append(sections, item)
+ sections = appendDOCXParagraphSections(sections, el, el.Type == "heading")
case "image":
b64 := base64.StdEncoding.EncodeToString(el.Data)
@@ -324,6 +313,58 @@ func buildDOCXJSONSections(irJSON string) []map[string]any {
return sections
}
+// appendDOCXParagraphSections emits text and inline-image runs in their IR
+// order. A paragraph is not necessarily a single text unit: office_oxide can
+// place an image between two text runs, and dropping that run would make the
+// downstream DOCX chunker lose both media order and adjacency context.
+func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool) []map[string]any {
+ var text strings.Builder
+ flushText := func() {
+ value := strings.TrimSpace(text.String())
+ text.Reset()
+ if value == "" {
+ return
+ }
+ item := map[string]any{
+ "text": value,
+ "image": nil,
+ "doc_type_kwd": "text",
+ }
+ if heading {
+ item["ck_type"] = "heading"
+ }
+ sections = append(sections, item)
+ }
+ appendImage := func(data []byte) {
+ if len(data) == 0 {
+ return
+ }
+ sections = append(sections, map[string]any{
+ "text": "",
+ "image": base64.StdEncoding.EncodeToString(data),
+ "doc_type_kwd": "image",
+ })
+ }
+ for _, run := range el.contentRuns() {
+ switch run.Type {
+ case "text":
+ text.WriteString(run.Text)
+ case "line_break":
+ text.WriteByte('\n')
+ case "image":
+ flushText()
+ appendImage(run.Data)
+ for _, nested := range run.Content {
+ if nested.Type == "image" {
+ appendImage(nested.Data)
+ }
+ }
+ }
+ }
+ flushText()
+ return sections
+}
+
// --- figure extraction (used by the cgo parser path) ---
// extractDOCXFiguresFromIR parses the office_oxide IR JSON and
@@ -348,6 +389,10 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
flat = append(flat, flatBlock{image: b64})
continue
}
+ if el.Type == "paragraph" || el.Type == "heading" {
+ flat = append(flat, docxInlineFlatBlocks(el)...)
+ continue
+ }
text := docxElementText(el, "\n")
flat = append(flat, flatBlock{text: text})
}
@@ -385,6 +430,45 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
return figures
}
+// docxInlineFlatBlocks preserves the text/image sequence inside a paragraph
+// for figure context extraction. Top-level element order alone is not enough
+// for DOCX because office_oxide can encode an inline image as a paragraph run.
+func docxInlineFlatBlocks(el docxIRElement) []flatBlock {
+ var blocks []flatBlock
+ var text strings.Builder
+ flushText := func() {
+ if text.Len() == 0 {
+ return
+ }
+ blocks = append(blocks, flatBlock{text: text.String()})
+ text.Reset()
+ }
+ appendImage := func(data []byte) {
+ if len(data) == 0 {
+ return
+ }
+ blocks = append(blocks, flatBlock{image: base64.StdEncoding.EncodeToString(data)})
+ }
+ for _, run := range el.contentRuns() {
+ switch run.Type {
+ case "text":
+ text.WriteString(run.Text)
+ case "line_break":
+ text.WriteByte('\n')
+ case "image":
+ flushText()
+ appendImage(run.Data)
+ for _, nested := range run.Content {
+ if nested.Type == "image" {
+ appendImage(nested.Data)
+ }
+ }
+ }
+ }
+ flushText()
+ return blocks
+}
+
// --- internal types ---
// flatBlock is a flattened IR element used internally to collect
diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go
index 48b7b9196e..1a33eeeb8b 100644
--- a/internal/parser/parser/docx_ir_test.go
+++ b/internal/parser/parser/docx_ir_test.go
@@ -49,6 +49,29 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {
}
}
+func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[
+ {"type":"paragraph","content":[
+ {"type":"text","text":"before"},
+ {"type":"image","data":"aGVsbG8="},
+ {"type":"text","text":"after"}
+ ]}
+ ]}]}`
+ got := buildDOCXJSONSections(irJSON)
+ if len(got) != 3 {
+ t.Fatalf("sections = %+v, want text/image/text sequence", got)
+ }
+ if got[0]["text"] != "before" || got[0]["doc_type_kwd"] != "text" {
+ t.Errorf("first item = %+v, want text before", got[0])
+ }
+ if got[1]["image"] != "aGVsbG8=" || got[1]["doc_type_kwd"] != "image" {
+ t.Errorf("inline image item = %+v", got[1])
+ }
+ if got[2]["text"] != "after" || got[2]["doc_type_kwd"] != "text" {
+ t.Errorf("last item = %+v, want text after", got[2])
+ }
+}
+
// TestJoinDOCXIRRuns pins that only text-type runs are concatenated;
// non-text runs (e.g. nested image runs) are skipped.
func TestJoinDOCXIRRuns(t *testing.T) {
@@ -104,6 +127,23 @@ func TestExtractDOCXFiguresFromIR(t *testing.T) {
}
}
+func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[
+ {"type":"paragraph","content":[
+ {"type":"text","text":"before"},
+ {"type":"image","data":"aGVsbG8="},
+ {"type":"text","text":"after"}
+ ]}
+ ]}]}`
+ figs := extractDOCXFiguresFromIR(irJSON)
+ if len(figs) != 1 {
+ t.Fatalf("expected 1 inline figure, got %d", len(figs))
+ }
+ if figs[0].Image != "aGVsbG8=" || figs[0].ContextAbove != "before" || figs[0].ContextBelow != "after" {
+ t.Fatalf("inline figure = %+v", figs[0])
+ }
+}
+
// TestExtractDOCXFiguresFromIR_NoImage returns nil when the IR has no
// image blocks.
func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index bb3dce85be..4986bb357b 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -543,14 +543,23 @@ func resolveAndValidateHost(host string) (net.IP, error) {
return addrs[0].IP, nil
}
-// headingText returns the inline-text of a heading node by
-// concatenating every Leaf / Text child. Empty headings emit "".
+// headingText returns a canonical ATX heading (marker plus inline text).
+// Keeping the marker in the parser payload preserves the Markdown structure
+// for GeneralChunker and downstream LLM prompts; ck_type alone does not carry
+// the heading level.
func headingText(h *ast.Heading) string {
var buf bytes.Buffer
for _, c := range h.GetChildren() {
buf.WriteString(leafText(c))
}
- return strings.TrimSpace(buf.String())
+ text := strings.TrimSpace(buf.String())
+ if h.Level <= 0 {
+ return text
+ }
+ if text == "" {
+ return strings.Repeat("#", h.Level)
+ }
+ return strings.Repeat("#", h.Level) + " " + text
}
// leafText mirrors gomarkdown's leaf walker: walks every descendant
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index d8c819d797..8616b49e48 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -27,8 +27,8 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
t.Fatal("JSON is empty; want at least one item")
}
// Verify heading
- if got, _ := res.JSON[0]["text"].(string); got != "Hello" {
- t.Fatalf("first item text = %q, want %q", got, "Hello")
+ if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {
+ t.Fatalf("first item text = %q, want %q", got, "# Hello")
}
if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {
t.Fatalf("first item ck_type = %q, want %q", got, "heading")
@@ -321,7 +321,7 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- // Title, Intro, table item, Trailing = 4.
+ // # Title, Intro, table item, Trailing = 4.
if len(res.JSON) != 4 {
t.Fatalf("len(JSON) = %d, want 4 (one per top-level block, table as one item)", len(res.JSON))
}
@@ -332,7 +332,7 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
text, _ := item["text"].(string)
all.WriteString(text)
all.WriteString("\n")
- if text == "Title" {
+ if text == "# Title" {
sawTitle = true
}
switch kd, _ := item["doc_type_kwd"].(string); kd {
@@ -569,7 +569,7 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
switch kd, _ := item["doc_type_kwd"].(string); kd {
case "text":
switch text {
- case "Title":
+ case "# Title":
titleIdx = i
case "Middle.":
middleIdx = i
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 91cb78a25a..c7837acd3e 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -20,7 +20,9 @@ import (
"encoding/base64"
"fmt"
"html"
+ "log/slog"
"regexp"
+ "sort"
"strconv"
"strings"
@@ -38,13 +40,241 @@ var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`
// value parsing.
var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`)
-// defaultTableChunkRows is the row ceiling for one spreadsheet <table>. It is
-// deliberately far beyond any real sheet: a sheet is emitted as ONE
-// self-contained <table> and is never split by row count. Any split would cut
-// inside <td> content or, worse, drop the delimiter characters it cut on —
-// the bug the typed table item exists to prevent. Keep in sync with Python's
-// TABLE_NO_SPLIT_ROWS (rag/flow/parser/parser.py).
-const defaultTableChunkRows = 1 << 30
+func recordsToSpreadsheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) []map[string]any {
+ if len(records) == 0 {
+ return nil
+ }
+ if headerRow <= 0 {
+ headerRow = 1
+ }
+ if len(dataRows) != len(records)-1 {
+ dataRows = make([]int, len(records)-1)
+ for i := range dataRows {
+ dataRows[i] = headerRow + i + 1
+ }
+ }
+ maxCols := 0
+ for _, row := range records {
+ if len(row) > maxCols {
+ maxCols = len(row)
+ }
+ }
+ if maxCols == 0 {
+ return nil
+ }
+ header := append([]string(nil), records[0]...)
+ if len(header) == 0 && maxCols > 0 {
+ header = padSpreadsheetRow(header, maxCols)
+ }
+ headerColEnd := len(header)
+ if headerColEnd == 0 {
+ headerColEnd = maxCols
+ }
+ if headerColEnd == 0 {
+ headerColEnd = 1
+ }
+ tableID := fmt.Sprintf("sheet-%d", sheetIndex)
+ items := make([]map[string]any, 0, len(records))
+ headerText := spreadsheetRowText(header, nil, sheet)
+ items = append(items, map[string]any{
+ "text": headerText,
+ "doc_type_kwd": DocTypeTable,
+ "ck_type": "table_header",
+ "table_id": tableID,
+ "sheet": sheet,
+ "sheet_index": sheetIndex,
+ "cells": header,
+ "row_start": headerRow,
+ "row_end": headerRow,
+ "col_start": 1,
+ "col_end": headerColEnd,
+ "positions": [][]float64{{float64(sheetIndex), float64(headerRow), float64(headerRow), 1, float64(headerColEnd)}},
+ })
+ for i, sourceRow := range records[1:] {
+ cells := append([]string(nil), sourceRow...)
+ colStart, colEnd := nonEmptyColumnRange(cells)
+ if colStart == 0 {
+ continue
+ }
+ rowNumber := dataRows[i]
+ items = append(items, map[string]any{
+ "text": spreadsheetRowText(header, cells, sheet),
+ "doc_type_kwd": DocTypeText,
+ "ck_type": "table_row",
+ "table_id": tableID,
+ "sheet": sheet,
+ "sheet_index": sheetIndex,
+ "headers": header,
+ "cells": cells,
+ "row_start": rowNumber,
+ "row_end": rowNumber,
+ "col_start": colStart,
+ "col_end": colEnd,
+ "positions": [][]float64{{float64(sheetIndex), float64(rowNumber), float64(rowNumber), float64(colStart), float64(colEnd)}},
+ })
+ }
+ return items
+}
+
+func recordsToHTMLTableItem(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) map[string]any {
+ if len(records) == 0 {
+ return nil
+ }
+ if headerRow <= 0 {
+ headerRow = 1
+ }
+ colEnd := 1
+ for _, row := range records {
+ if len(row) > colEnd {
+ colEnd = len(row)
+ }
+ }
+ var builder strings.Builder
+ builder.WriteString("<table><caption>")
+ builder.WriteString(html.EscapeString(sheet))
+ builder.WriteString("</caption>\n<tr>")
+ for _, cell := range records[0] {
+ builder.WriteString("<th>")
+ builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))
+ builder.WriteString("</th>")
+ }
+ builder.WriteString("</tr>\n")
+ for _, row := range records[1:] {
+ builder.WriteString("<tr>")
+ for _, cell := range row {
+ builder.WriteString("<td>")
+ builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))
+ builder.WriteString("</td>")
+ }
+ builder.WriteString("</tr>\n")
+ }
+ builder.WriteString("</table>\n")
+ rowStart, rowEnd := headerRow, headerRow
+ if len(dataRows) > 0 {
+ rowStart, rowEnd = dataRows[0], dataRows[len(dataRows)-1]
+ }
+ return NewTableJSONItem(builder.String(), sheet, [][]float64{{
+ float64(sheetIndex), float64(rowStart), float64(rowEnd), 1, float64(colEnd),
+ }})
+}
+
+func padSpreadsheetRow(row []string, width int) []string {
+ padded := make([]string, width)
+ copy(padded, row)
+ return padded
+}
+
+func nonEmptyColumnRange(row []string) (int, int) {
+ start, end := 0, 0
+ for i, cell := range row {
+ if strings.TrimSpace(cell) == "" {
+ continue
+ }
+ col := i + 1
+ if start == 0 {
+ start = col
+ }
+ end = col
+ }
+ return start, end
+}
+
+func spreadsheetRowText(headers, cells []string, sheet string) string {
+ values := headers
+ if cells != nil {
+ values = cells
+ }
+ parts := make([]string, 0, len(values))
+ for i, value := range values {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ continue
+ }
+ if cells == nil {
+ parts = append(parts, value)
+ continue
+ }
+ header := ""
+ if i < len(headers) {
+ header = strings.TrimSpace(headers[i])
+ }
+ if header != "" {
+ parts = append(parts, header+":"+value)
+ } else {
+ parts = append(parts, value)
+ }
+ }
+ text := strings.Join(parts, "; ")
+ if cells != nil && sheet != "" && !strings.Contains(strings.ToLower(sheet), "sheet") {
+ text += " ——" + sheet
+ }
+ return text
+}
+
+func sortSpreadsheetItems(items []map[string]any) {
+ sort.SliceStable(items, func(i, j int) bool {
+ headerI := items[i]["ck_type"] == "table_header"
+ headerJ := items[j]["ck_type"] == "table_header"
+ if headerI != headerJ {
+ return headerI
+ }
+ ri, ci := spreadsheetItemCoordinate(items[i])
+ rj, cj := spreadsheetItemCoordinate(items[j])
+ if ri != rj {
+ return ri < rj
+ }
+ if ci != cj {
+ return ci < cj
+ }
+ return spreadsheetItemKindRank(items[i]) < spreadsheetItemKindRank(items[j])
+ })
+}
+
+func spreadsheetItemCoordinate(item map[string]any) (int, int) {
+ row, _ := numericItemInt(item["row_start"])
+ col, _ := numericItemInt(item["col_start"])
+ return row, col
+}
+
+func spreadsheetItemKindRank(item map[string]any) int {
+ if item["ck_type"] == "image" {
+ return 1
+ }
+ return 0
+}
+
+func numericItemInt(value any) (int, bool) {
+ switch n := value.(type) {
+ case int:
+ return n, true
+ case int64:
+ return int(n), true
+ case float64:
+ return int(n), true
+ case string:
+ parsed, err := strconv.Atoi(strings.TrimSpace(n))
+ return parsed, err == nil
+ default:
+ return 0, false
+ }
+}
+
+// deprecatedChunkRows reports the removed parser-side chunking option. Row
+// boundaries are now emitted as structured IR and the selected chunker owns
+// token-budget merging; silently accepting chunk_rows would make an existing
+// configuration look effective when it is not.
+func deprecatedChunkRows(setup map[string]any, parserName string) {
+ raw, exists := setup["chunk_rows"]
+ if !exists {
+ return
+ }
+ rows, ok := numericItemInt(raw)
+ if !ok {
+ slog.Warn("spreadsheet parser ignored invalid chunk_rows; configure row merging on the chunker", "parser", parserName, "chunk_rows", raw)
+ return
+ }
+ slog.Warn("spreadsheet parser ignored deprecated chunk_rows; configure row merging on the chunker", "parser", parserName, "chunk_rows", rows)
+}
// extractXLSXImages returns the floating and in-cell images anchored to a
// worksheet as structured parser items. Excelize exposes both kinds through
@@ -81,6 +311,7 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []stri
if picture.Format != nil && picture.Format.AltText != "" {
alt = picture.Format.AltText
}
+ row, col := axisToRC(cell)
items = append(items, map[string]any{
"text": alt,
"doc_type_kwd": "image",
@@ -88,6 +319,10 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []stri
"image": "data:" + mimeType + ";base64," + encoded,
"sheet": sheet,
"cell": cell,
+ "row_start": row,
+ "row_end": row,
+ "col_start": col,
+ "col_end": col,
})
}
}
@@ -144,106 +379,6 @@ func cleanIllegalControlChars(records [][]string) [][]string {
return out
}
-// buildHeaderRow renders a row as an HTML <th> header row.
-func buildHeaderRow(row []string) string {
- var b strings.Builder
- b.WriteString("<tr>")
- for _, cell := range row {
- b.WriteString("<th>")
- b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
- b.WriteString("</th>")
- }
- b.WriteString("</tr>\n")
- return b.String()
-}
-
-// htmlTableChunk is one self-contained <table> chunk with 1-based sheet/row/col
-// coordinates for source location (mirrors Excel position_int semantics).
-type htmlTableChunk struct {
- HTML string
- RowStart int // inclusive, 1-based Excel row of first data row (header-only chunks use headerRowAbs)
- RowEnd int // inclusive, 1-based Excel row of last data row
- ColStart int
- ColEnd int
-}
-
-// recordsToHTMLTableChunkList renders records as structured htmlTableChunk items.
-// headerRowAbs is the 1-based workbook row number of records[0] (normally 1).
-func recordsToHTMLTableChunkList(records [][]string, chunkRows int, caption string, headerRowAbs int) []htmlTableChunk {
- if headerRowAbs <= 0 {
- headerRowAbs = 1
- }
- colEnd := 1
- for _, row := range records {
- if n := len(row); n > colEnd {
- colEnd = n
- }
- }
- if len(records) == 0 {
- return []htmlTableChunk{{
- HTML: "<table><caption>" + html.EscapeString(caption) + "</caption></table>",
- RowStart: headerRowAbs,
- RowEnd: headerRowAbs,
- ColStart: 1,
- ColEnd: colEnd,
- }}
- }
-
- headerHTML := buildHeaderRow(records[0])
- dataRows := records[1:]
- nData := len(dataRows)
-
- if nData == 0 {
- return []htmlTableChunk{{
- HTML: "<table><caption>" + html.EscapeString(caption) + "</caption>\n" + headerHTML + "</table>",
- RowStart: headerRowAbs,
- RowEnd: headerRowAbs,
- ColStart: 1,
- ColEnd: colEnd,
- }}
- }
-
- if chunkRows <= 0 {
- chunkRows = defaultTableChunkRows
- }
-
- nChunks := (nData + chunkRows - 1) / chunkRows
- out := make([]htmlTableChunk, 0, nChunks)
- for ci := 0; ci < nChunks; ci++ {
- start := ci * chunkRows
- end := start + chunkRows
- if end > nData {
- end = nData
- }
-
- var b strings.Builder
- b.WriteString("<table><caption>")
- b.WriteString(html.EscapeString(caption))
- b.WriteString("</caption>\n")
- b.WriteString(headerHTML)
-
- for _, row := range dataRows[start:end] {
- b.WriteString("<tr>")
- for _, cell := range row {
- b.WriteString("<td>")
- b.WriteString(html.EscapeString(strings.TrimSpace(cell)))
- b.WriteString("</td>")
- }
- b.WriteString("</tr>\n")
- }
- b.WriteString("</table>\n")
-
- out = append(out, htmlTableChunk{
- HTML: b.String(),
- RowStart: headerRowAbs + start + 1,
- RowEnd: headerRowAbs + end,
- ColStart: 1,
- ColEnd: colEnd,
- })
- }
- return out
-}
-
// ──────────────────────────────────────────────────────────── axis helpers
// axisToRC parses an "A1"-style cell reference (optionally with a leading "$")
@@ -590,45 +725,13 @@ func isSubtotalRow(row []string) bool {
return false
}
-// decodeChunkRows reads the "chunk_rows" setup knob, returning the default when
-// it is absent or non-positive.
-func decodeChunkRows(setup map[string]any) int {
- if setup == nil {
- return defaultTableChunkRows
- }
- v, ok := setup["chunk_rows"]
- if !ok {
- return defaultTableChunkRows
- }
- switch n := v.(type) {
- case float64:
- rows := int(n)
- if rows <= 0 {
- return defaultTableChunkRows
- }
- return rows
- case int:
- if n <= 0 {
- return defaultTableChunkRows
- }
- return n
- case int64:
- rows := int(n)
- if rows <= 0 {
- return defaultTableChunkRows
- }
- return rows
- }
- return defaultTableChunkRows
-}
-
-func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]htmlTableChunk, []string, error) {
+func readSpreadsheetRecords(f *excelize.File, sheet string) ([][]string, []int, int, []string, error) {
rows, err := f.GetRows(sheet)
if err != nil {
- return nil, nil, fmt.Errorf("read XLSX sheet %q rows: %w", sheet, err)
+ return nil, nil, 0, nil, fmt.Errorf("read XLSX sheet %q rows: %w", sheet, err)
}
if len(rows) == 0 {
- return nil, nil, nil
+ return nil, nil, 0, nil, nil
}
rows = cleanIllegalControlChars(rows)
@@ -669,23 +772,5 @@ func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]ht
absDataRows = append(absDataRows, i+1)
}
- chunks := recordsToHTMLTableChunkList(records, chunkRows, sheet, headerRow)
- if len(chunks) == 0 || len(absDataRows) == 0 {
- return chunks, warnings, nil
- }
- if chunkRows <= 0 {
- chunkRows = defaultTableChunkRows
- }
- for ci := range chunks {
- start := ci * chunkRows
- end := start + chunkRows
- if end > len(absDataRows) {
- end = len(absDataRows)
- }
- if start < end {
- chunks[ci].RowStart = absDataRows[start]
- chunks[ci].RowEnd = absDataRows[end-1]
- }
- }
- return chunks, warnings, nil
+ return records, absDataRows, headerRow, warnings, nil
}
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index 4b6efc1e1c..b66a67ec46 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -1,24 +1,159 @@
package parser
import (
+ "reflect"
"strings"
"testing"
"github.com/xuri/excelize/v2"
)
-func xlsxTableHTML(res ParseResult) string {
- var out strings.Builder
+func TestXLSXParserEmitsSpreadsheetRows(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "ID")
+ mustSetCell(t, f, "Sheet1", "B1", "Status")
+ mustSetCell(t, f, "Sheet1", "A2", "A-100")
+ mustSetCell(t, f, "Sheet1", "B2", "paid")
+ mustSetCell(t, f, "Sheet1", "A3", "A-101")
+ mustSetCell(t, f, "Sheet1", "B3", "pending")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "orders.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) != 3 {
+ t.Fatalf("items = %d, want header plus two rows", len(res.JSON))
+ }
+ header := res.JSON[0]
+ if header["ck_type"] != "table_header" {
+ t.Fatalf("header ck_type = %v, want table_header", header["ck_type"])
+ }
+ if got, want := header["cells"], []string{"ID", "Status"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("header cells = %#v, want %#v", got, want)
+ }
+ row := res.JSON[1]
+ if row["ck_type"] != "table_row" || row["doc_type_kwd"] != "text" {
+ t.Fatalf("row types = ck_type:%v doc_type:%v", row["ck_type"], row["doc_type_kwd"])
+ }
+ if got, want := row["cells"], []string{"A-100", "paid"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("row cells = %#v, want %#v", got, want)
+ }
+ if row["sheet"] != "Sheet1" || row["sheet_index"] != 1 {
+ t.Fatalf("row sheet metadata = %#v", row)
+ }
+ if row["row_start"] != 2 || row["row_end"] != 2 || row["col_start"] != 1 || row["col_end"] != 2 {
+ t.Fatalf("row coordinates = %#v", row)
+ }
+}
+
+func TestSortSpreadsheetItemsKeepsHeaderBeforeRowsWhenHeaderIsLater(t *testing.T) {
+ items := []map[string]any{
+ {"ck_type": "table_row", "row_start": 1, "col_start": 1},
+ {"ck_type": "table_row", "row_start": 2, "col_start": 1},
+ {"ck_type": "table_header", "row_start": 5, "col_start": 1},
+ }
+
+ sortSpreadsheetItems(items)
+ if items[0]["ck_type"] != "table_header" {
+ t.Fatalf("sorted items = %#v, want table_header first", items)
+ }
+}
+
+func TestXLSXParserHTML4ExcelRemainsAtomic(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Question")
+ mustSetCell(t, f, "Sheet1", "B1", "Answer")
+ mustSetCell(t, f, "Sheet1", "A2", "Q1")
+ mustSetCell(t, f, "Sheet1", "B2", "A1")
+ })
+ p, _ := NewXLSXParser("")
+ p.ConfigureFromSetup(map[string]any{"html4excel": true})
+ res := p.ParseWithResult(t.Context(), "qa.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) != 1 {
+ t.Fatalf("items = %d, want one atomic table item", len(res.JSON))
+ }
+ if res.JSON[0]["ck_type"] != "table" || !strings.Contains(res.JSON[0]["text"].(string), "<table>") {
+ t.Fatalf("html4excel item = %#v", res.JSON[0])
+ }
+}
+
+func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {
+ const dataRows = 257
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "Value")
+ for row := 2; row <= dataRows+1; row++ {
+ mustSetCell(t, f, "Sheet1", cellAxis(row, 1), row)
+ }
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "rows.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) != dataRows+1 {
+ t.Fatalf("items = %d, want one header plus %d data rows", len(res.JSON), dataRows)
+ }
+ last := res.JSON[len(res.JSON)-1]
+ if last["ck_type"] != "table_row" || last["row_start"] != dataRows+1 {
+ t.Fatalf("last row = %#v, want source row %d", last, dataRows+1)
+ }
+}
+
+func TestXLSXParserPreservesSheetOrderAndIdentity(t *testing.T) {
+ data := newTestXLSX(t, func(f *excelize.File) {
+ mustSetCell(t, f, "Sheet1", "A1", "First")
+ mustSetCell(t, f, "Sheet1", "A2", "one")
+ _, err := f.NewSheet("Orders")
+ if err != nil {
+ t.Fatalf("new sheet: %v", err)
+ }
+ mustSetCell(t, f, "Orders", "A1", "Second")
+ mustSetCell(t, f, "Orders", "A2", "two")
+ })
+ p, _ := NewXLSXParser("")
+ res := p.ParseWithResult(t.Context(), "multi.xlsx", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if len(res.JSON) != 4 {
+ t.Fatalf("items = %d, want two headers and two rows", len(res.JSON))
+ }
+ seen := map[string]map[string]any{}
for _, item := range res.JSON {
- if item["doc_type_kwd"] != "table" {
- continue
+ if item["ck_type"] == "table_header" {
+ seen[item["sheet"].(string)] = item
}
+ }
+ if first := seen["Sheet1"]; first == nil || first["table_id"] != "sheet-1" || first["sheet_index"] != 1 {
+ t.Fatalf("Sheet1 identity = %#v", first)
+ }
+ if second := seen["Orders"]; second == nil || second["table_id"] != "sheet-2" || second["sheet_index"] != 2 {
+ t.Fatalf("Orders identity = %#v", second)
+ }
+}
+
+func spreadsheetText(res ParseResult) string {
+ var out strings.Builder
+ for _, item := range res.JSON {
text, _ := item["text"].(string)
out.WriteString(text)
}
return out.String()
}
+func spreadsheetHeaderItem(res ParseResult) map[string]any {
+ for _, item := range res.JSON {
+ if item["ck_type"] == "table_header" {
+ return item
+ }
+ }
+ return nil
+}
+
func TestXLSXImageMIMEType(t *testing.T) {
tests := []struct {
extension string
@@ -103,108 +238,8 @@ func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.
}
}
-// TestRecordsToHTMLTableChunkList_Alignment asserts the chunked output uses the
-// shared schema: <caption>, first row as <th>, data as <td>, repeated header
-// per 256-row chunk, and NO <thead>/<tbody> wrapper.
-func TestRecordsToHTMLTableChunkList_Alignment(t *testing.T) {
- records := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}
- chunks := recordsToHTMLTableChunkList(records, 256, "Sheet1", 1)
- if len(chunks) != 1 {
- t.Fatalf("want 1 chunk, got %d", len(chunks))
- }
- out := chunks[0].HTML
-
- if !strings.Contains(out, `<caption>Sheet1</caption>`) {
- t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", out)
- }
- if !strings.Contains(out, "<tr><th>Name</th><th>Age</th></tr>") {
- t.Fatalf("want header row as <th>, got:\n%s", out)
- }
- if !strings.Contains(out, "<tr><td>Alice</td><td>30</td></tr>") {
- t.Fatalf("want data row as <td>, got:\n%s", out)
- }
- if strings.Contains(out, "<thead>") || strings.Contains(out, "<tbody>") {
- t.Fatalf("must not emit <thead>/<tbody> to stay byte-compatible with Python/CSV, got:\n%s", out)
- }
-}
-
-// TestRecordsToHTMLTableChunkList_DefaultCeilingDoesNotSplit pins the default row
-// ceiling: a sheet is emitted as ONE self-contained <table>. Splitting it would
-// cut inside <td> content or drop the delimiter characters it cut on, and the
-// value must stay in sync with Python's TABLE_NO_SPLIT_ROWS.
-func TestRecordsToHTMLTableChunkList_DefaultCeilingDoesNotSplit(t *testing.T) {
- const dataRows = 5000
- records := make([][]string, 0, dataRows+1)
- records = append(records, []string{"C1", "C2"})
- for i := 0; i < dataRows; i++ {
- records = append(records, []string{"x", "y"})
- }
-
- chunks := recordsToHTMLTableChunkList(records, defaultTableChunkRows, "S", 1)
- if len(chunks) != 1 {
- t.Fatalf("want 1 chunk with the default ceiling, got %d", len(chunks))
- }
- // The single chunk spans every data row.
- if chunks[0].RowStart != 2 || chunks[0].RowEnd != dataRows+1 {
- t.Fatalf("want rows 2..%d, got %d..%d", dataRows+1, chunks[0].RowStart, chunks[0].RowEnd)
- }
-}
-
-// TestRecordsToHTMLTableChunkList_Chunking asserts 256-row chunking with a repeated
-// header (ceil(n_data / chunk_rows) chunks).
-func TestRecordsToHTMLTableChunkList_Chunking(t *testing.T) {
- const dataRows = 300
- records := make([][]string, 0, dataRows+1)
- records = append(records, []string{"C1", "C2"})
- for i := 0; i < dataRows; i++ {
- records = append(records, []string{"x", "y"})
- }
- chunks := recordsToHTMLTableChunkList(records, 256, "S", 1)
- // 300 data rows → ceil(300/256) = 2 chunks, each repeating the header.
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
- for i, ch := range chunks {
- if !strings.Contains(ch.HTML, "<tr><th>C1</th><th>C2</th></tr>") {
- t.Fatalf("want header repeated in chunk %d, got %s", i, ch.HTML)
- }
- }
-}
-
-func TestRecordsToHTMLTableChunkList_RowRange(t *testing.T) {
- records := make([][]string, 0, 25)
- records = append(records, []string{"H"})
- for i := 0; i < 24; i++ {
- records = append(records, []string{"x"})
- }
- chunks := recordsToHTMLTableChunkList(records, 12, "S", 1)
- if len(chunks) != 2 {
- t.Fatalf("want 2 chunks, got %d", len(chunks))
- }
- if chunks[0].RowStart != 2 || chunks[0].RowEnd != 13 {
- t.Fatalf("chunk0 rows = %d-%d, want 2-13", chunks[0].RowStart, chunks[0].RowEnd)
- }
- if chunks[1].RowStart != 14 || chunks[1].RowEnd != 25 {
- t.Fatalf("chunk1 rows = %d-%d, want 14-25", chunks[1].RowStart, chunks[1].RowEnd)
- }
-}
-
-func TestRecordsToHTMLTableChunkList_ColEndUsesWidestRow(t *testing.T) {
- records := [][]string{
- {"H"},
- {"a", "b", "c"},
- }
- chunks := recordsToHTMLTableChunkList(records, 12, "S", 1)
- if len(chunks) != 1 {
- t.Fatalf("want 1 chunk, got %d", len(chunks))
- }
- if chunks[0].ColEnd != 3 {
- t.Fatalf("ColEnd = %d, want 3", chunks[0].ColEnd)
- }
-}
-
-// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a <caption> and
-// renders the first row as <th>, and that the header text appears only in <th>.
+// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a typed header
+// row and data row with the header labels preserved separately from values.
func TestXLSXParser_HeaderAndCaption(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "Product")
@@ -219,22 +254,22 @@ func TestXLSXParser_HeaderAndCaption(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- html := xlsxTableHTML(res)
- if !strings.Contains(html, `<caption>Sheet1</caption>`) {
- t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", html)
+ text := spreadsheetText(res)
+ if !strings.Contains(text, "Product; Price") {
+ t.Fatalf("want header text, got:\n%s", text)
}
- if !strings.Contains(html, "<tr><th>Product</th><th>Price</th></tr>") {
- t.Fatalf("want header as <th>, got:\n%s", html)
+ if !strings.Contains(text, "Product:Widget; Price:9.99") {
+ t.Fatalf("want row text, got:\n%s", text)
}
- // "Product" must only appear inside a <th>, never inside a <td>.
- if strings.Contains(html, "<td>Product</td>") {
- t.Fatalf("header value leaked into <td>:\n%s", html)
+ // The header label must not be duplicated as a value in the row text.
+ if strings.Contains(text, "Product:Product") {
+ t.Fatalf("header value leaked into row text:\n%s", text)
}
}
// TestXLSXParser_MergedHeaderInheritance asserts a horizontally merged header
// cell's slave columns inherit the master text, so a wide merged title does not
-// render as a row of blank <th> cells.
+// render as a row of blank header cells.
func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
// Row 1 is the header; A1:C1 merged into one wide label "Sales Report".
@@ -255,10 +290,9 @@ func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- html := xlsxTableHTML(res)
- // The merged master text must have propagated into all three <th> slots.
- if !strings.Contains(html, "<tr><th>Sales Report</th><th>Sales Report</th><th>Sales Report</th></tr>") {
- t.Fatalf("merged master text not inherited into header <th>, got:\n%s", html)
+ text := spreadsheetText(res)
+ if !strings.Contains(text, "Sales Report; Sales Report; Sales Report") {
+ t.Fatalf("merged master text not inherited into header cells, got:\n%s", text)
}
}
@@ -279,11 +313,15 @@ func TestDetectHeaderRow_ListObject(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "<tr><th>Name</th><th>Age</th></tr>") {
- t.Fatalf("want ListObject header row detected, got:\n%s", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "Name; Age") {
+ t.Fatalf("want ListObject header row detected, got:\n%s", spreadsheetText(res))
+ }
+ header := spreadsheetHeaderItem(res)
+ if header == nil {
+ t.Fatal("missing table_header item")
}
- if strings.Contains(xlsxTableHTML(res), "<th>Title</th>") {
- t.Fatalf("title row must not be the header:\n%s", xlsxTableHTML(res))
+ if got := header["text"]; got != "Name; Age" {
+ t.Fatalf("header text = %v, want Name; Age", got)
}
}
@@ -306,16 +344,16 @@ func TestDetectHeaderRow_Lightweight(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "<tr><th>Item</th><th>Count</th></tr>") {
- t.Fatalf("want row-2 header detected, got:\n%s", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "Item; Count") {
+ t.Fatalf("want row-2 header detected, got:\n%s", spreadsheetText(res))
}
- if strings.Contains(xlsxTableHTML(res), "<th>100</th>") {
- t.Fatalf("numeric row 1 must not be the header:\n%s", xlsxTableHTML(res))
+ if strings.Contains(spreadsheetText(res), "100; 200") {
+ t.Fatalf("numeric row 1 must not be the header:\n%s", spreadsheetText(res))
}
}
// TestXLSXParser_CommonCaseNoRegression asserts the dominant case (header on
-// row 1, no merges, no ListObject) renders row 1 as <th> unchanged.
+// row 1, no merges, no ListObject) keeps the header row unchanged.
func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {
data := newTestXLSX(t, func(f *excelize.File) {
mustSetCell(t, f, "Sheet1", "A1", "col_a")
@@ -328,8 +366,8 @@ func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "<tr><th>col_a</th><th>col_b</th></tr>") {
- t.Fatalf("common-case header must render as <th>:\n%s", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "col_a; col_b") {
+ t.Fatalf("common-case header must be preserved:\n%s", spreadsheetText(res))
}
}
@@ -356,11 +394,11 @@ func TestDetectHeaderRow_BoldSubtotalNotHeader(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "<tr><th>2023</th><th>2024</th></tr>") {
- t.Fatalf("numeric row 1 must remain the header:\n%s", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "2023; 2024") {
+ t.Fatalf("numeric row 1 must remain the header:\n%s", spreadsheetText(res))
}
- if strings.Contains(xlsxTableHTML(res), "<th>Total</th>") {
- t.Fatalf("bold subtotal row must not be promoted to header:\n%s", xlsxTableHTML(res))
+ if strings.Contains(spreadsheetText(res), "Total; Summary") {
+ t.Fatalf("bold subtotal row must not be promoted to header:\n%s", spreadsheetText(res))
}
}
@@ -389,11 +427,15 @@ func TestDetectHeaderRow_StyledHeaderPastFarMerge(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "<th>Name</th><th>Desc</th>") {
- t.Fatalf("narrow bold header past far merge must be detected:\n%s", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "Name; Desc") {
+ t.Fatalf("narrow bold header past far merge must be detected:\n%s", spreadsheetText(res))
}
- if strings.Contains(xlsxTableHTML(res), "<th>Sales Report</th>") {
- t.Fatalf("wide merged title must not be the header:\n%s", xlsxTableHTML(res))
+ header := spreadsheetHeaderItem(res)
+ if header == nil {
+ t.Fatal("missing table_header item")
+ }
+ if got := header["text"]; got != "Name; Desc" {
+ t.Fatalf("header text = %v, want Name; Desc", got)
}
}
@@ -419,11 +461,15 @@ func TestDetectHeaderRow_StyledTextHeaderOverNumeric(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "<tr><th>Product</th><th>Units</th></tr>") {
- t.Fatalf("styled text header over numeric data must be detected:\n%s", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "Product; Units") {
+ t.Fatalf("styled text header over numeric data must be detected:\n%s", spreadsheetText(res))
+ }
+ header := spreadsheetHeaderItem(res)
+ if header == nil {
+ t.Fatal("missing table_header item")
}
- if strings.Contains(xlsxTableHTML(res), "<th>Report Title</th>") {
- t.Fatalf("title stub must not be the header:\n%s", xlsxTableHTML(res))
+ if got := header["text"]; got != "Product; Units" {
+ t.Fatalf("header text = %v, want Product; Units", got)
}
}
@@ -484,115 +530,8 @@ func TestMergeExtentCol(t *testing.T) {
}
}
-// TestDecodeChunkRows exercises the chunk_rows setup knob across all the types
-// JSON/yaml decoding can produce, plus the defaulting paths.
-func TestDecodeChunkRows(t *testing.T) {
- if got := decodeChunkRows(nil); got != defaultTableChunkRows {
- t.Fatalf("nil setup: want default %d, got %d", defaultTableChunkRows, got)
- }
- if got := decodeChunkRows(map[string]any{}); got != defaultTableChunkRows {
- t.Fatalf("empty setup: want default, got %d", got)
- }
- if got := decodeChunkRows(map[string]any{"chunk_rows": 100.0}); got != 100 {
- t.Fatalf("float64 100: want 100, got %d", got)
- }
- if got := decodeChunkRows(map[string]any{"chunk_rows": 0.0}); got != defaultTableChunkRows {
- t.Fatalf("float64 0: want default, got %d", got)
- }
- if got := decodeChunkRows(map[string]any{"chunk_rows": -5.0}); got != defaultTableChunkRows {
- t.Fatalf("float64 -5: want default, got %d", got)
- }
- if got := decodeChunkRows(map[string]any{"chunk_rows": 256}); got != 256 {
- t.Fatalf("int 256: want 256, got %d", got)
- }
- if got := decodeChunkRows(map[string]any{"chunk_rows": int64(512)}); got != 512 {
- t.Fatalf("int64 512: want 512, got %d", got)
- }
- if got := decodeChunkRows(map[string]any{"chunk_rows": "256"}); got != defaultTableChunkRows {
- t.Fatalf("string (unsupported type): want default, got %d", got)
- }
-}
-
-// TestRenderSheetTables_FarMergeDoesNotBloatDataRows guards the memory blow-up
-// from review: a far merge on a non-header row must not widen the header or any
-// data row to the merge's full span. The header stays its natural width and no
-// data row carries empty <td> cells.
-func TestRenderSheetTables_FarMergeDoesNotBloatDataRows(t *testing.T) {
- data := newTestXLSX(t, func(f *excelize.File) {
- // Row 1: a wide merged title A1:Z1 (NOT the header).
- mustSetCell(t, f, "Sheet1", "A1", "Report Title")
- mustMergeCell(t, f, "Sheet1", "A1", "Z1")
- // Row 2: the real header, made bold so detection anchors it.
- hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})
- mustSetCell(t, f, "Sheet1", "A2", "Name")
- mustSetCell(t, f, "Sheet1", "B2", "Price")
- mustSetCellStyle(t, f, "Sheet1", "A2", "B2", hdrIdx)
- // Rows 3-4: data.
- mustSetCell(t, f, "Sheet1", "A3", "Alice")
- mustSetCell(t, f, "Sheet1", "B3", "9.99")
- mustSetCell(t, f, "Sheet1", "A4", "Bob")
- mustSetCell(t, f, "Sheet1", "B4", "19.99")
- })
- p, _ := NewXLSXParser("")
- res := p.ParseWithResult(t.Context(), "t.xlsx", data)
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- html := xlsxTableHTML(res)
- if !strings.Contains(html, "<tr><th>Name</th><th>Price</th></tr>") {
- t.Fatalf("want row-2 header detected, got:\n%s", html)
- }
- if strings.Contains(html, "<th>Title</th>") {
- t.Fatalf("wide merged title must not be the header:\n%s", html)
- }
- // The far merge is on row 1, not the header, so nothing is widened to 26
- // columns: the header keeps 2 columns (no empty <th>) and data rows carry
- // no empty <td>.
- if strings.Contains(html, "<th></th>") {
- t.Fatalf("header was bloated by the far merge:\n%s", html)
- }
- if strings.Count(html, "<td></td>") != 0 {
- t.Fatalf("data rows were bloated by the far merge:\n%s", html)
- }
-}
-
-// TestRenderSheetTables_MultiSheet asserts each sheet becomes its own <table>
-// chunk, each carrying its own <caption>.
-func TestRenderSheetTables_MultiSheet(t *testing.T) {
- data := newTestXLSX(t, func(f *excelize.File) {
- mustSetCell(t, f, "Sheet1", "A1", "Name")
- mustSetCell(t, f, "Sheet1", "B1", "Age")
- mustSetCell(t, f, "Sheet1", "A2", "Alice")
- mustSetCell(t, f, "Sheet1", "B2", "30")
- if _, err := f.NewSheet("HR"); err != nil {
- t.Fatalf("NewSheet: %v", err)
- }
- mustSetCell(t, f, "HR", "A1", "Dept")
- mustSetCell(t, f, "HR", "B1", "Head")
- mustSetCell(t, f, "HR", "A2", "Eng")
- mustSetCell(t, f, "HR", "B2", "Bob")
- })
- p, _ := NewXLSXParser("")
- res := p.ParseWithResult(t.Context(), "t.xlsx", data)
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- html := xlsxTableHTML(res)
- if !strings.Contains(html, "<caption>Sheet1</caption>") {
- t.Fatalf("want Sheet1 caption, got:\n%s", html)
- }
- if !strings.Contains(html, "<caption>HR</caption>") {
- t.Fatalf("want HR caption, got:\n%s", html)
- }
- // Two small sheets → two <table> chunks.
- if n := strings.Count(html, "<table>"); n != 2 {
- t.Fatalf("want 2 <table> chunks, got %d:\n%s", n, html)
- }
-}
-
-// TestRenderSheetTables_EmptySheet asserts an empty/unreadable sheet yields an
-// empty string rather than an empty <table> wrapper.
-func TestRenderSheetTables_EmptySheet(t *testing.T) {
+// TestXLSXParser_EmptySheet asserts an empty sheet yields no parser items.
+func TestXLSXParser_EmptySheet(t *testing.T) {
// excelize.NewFile yields a single empty "Sheet1".
data := newTestXLSX(t, func(f *excelize.File) {})
p, _ := NewXLSXParser("")
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index 7fe65d9839..a7646a7b87 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -20,7 +20,7 @@
//
// - HTMLParser — block-level walker that emits the python-compatible
// {text, doc_type_kwd, ck_type} shape.
-// - TextParser — paragraph-splitting for the text&code family
+// - TextParser — normalized one-unit output for the text&code family
// (.txt / .py / .js / .java / .c / .cpp / .h / .php / .go / .ts
// / .sh / .cs / .kt / .sql).
//
@@ -43,31 +43,31 @@ import (
"ragflow/internal/utility"
)
-// TestTextParser_ParseWithResult_ParaSplit pins the paragraph-split
-// rule. A blank-line-separated input yields one item per
-// paragraph; the python TxtParser does the same.
-func TestTextParser_ParseWithResult_ParaSplit(t *testing.T) {
+// TestTextParser_ParseWithResult_PreservesOneTextUnit verifies that parser
+// output does not claim chunk boundaries by splitting on its historical
+// fixed delimiter set. GeneralChunker owns that decision from this point on.
+func TestTextParser_ParseWithResult_PreservesOneTextUnit(t *testing.T) {
p := NewTextParser()
- src := []byte("First paragraph.\n\nSecond paragraph.\n\nThird.")
- ctx := t.Context()
- res := p.ParseWithResult(ctx, "doc.txt", src)
+ res := p.ParseWithResult(t.Context(), "doc.txt", []byte("First sentence.\r\n\r\nSecond sentence!\rThird sentence?"))
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
if res.OutputFormat != "json" {
- t.Errorf("OutputFormat = %q, want json", res.OutputFormat)
+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)
}
if got, want := res.File["name"], "doc.txt"; got != want {
- t.Errorf("File.name = %v, want %v", got, want)
+ t.Fatalf("File.name = %v, want %v", got, want)
}
- if len(res.JSON) != 3 {
- t.Fatalf("JSON len = %d, want 3 (one per paragraph)", len(res.JSON))
+ if len(res.JSON) != 1 {
+ t.Fatalf("JSON len = %d, want 1: %#v", len(res.JSON), res.JSON)
}
- if got, want := res.JSON[0]["text"], "First paragraph."; got != want {
- t.Errorf("JSON[0].text = %v, want %v", got, want)
+ if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {
+ t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)
}
- if got, want := res.JSON[2]["text"], "Third."; got != want {
- t.Errorf("JSON[2].text = %v, want %v", got, want)
+ got, _ := res.JSON[0]["text"].(string)
+ want := "First sentence.\n\nSecond sentence!\nThird sentence?"
+ if got != want {
+ t.Errorf("JSON[0].text = %q, want %q", got, want)
}
}
@@ -297,55 +297,9 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) {
}
}
-// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix:
-// TextParser now splits on the flow parser's default delimiter set
-// ("\n!?;。;!?"), mirroring deepdoc TxtParser.parser_txt, instead of only on
-// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing
-// delimiter attached, so sentence-ending punctuation survives the split.
-func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {
- ctx := t.Context()
- p := NewTextParser()
-
- // Single newlines now split too (previously only "\n\n" did).
- src := []byte("First line.\nSecond line.\nThird line.")
- res := p.ParseWithResult(ctx, "doc.txt", src)
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- if len(res.JSON) != 3 {
- t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON))
- }
-
- // Sentence delimiters split and keep the delimiter attached. The period
- // "." is NOT in the default set, so "Foo. Bar" stays joined until the ";".
- // TrimSpace drops the incidental leading space before each delimiter (the
- // package's established convention, also used by markdown leafText).
- src = []byte("Hello! World? Foo. Bar; Baz。 Qux!")
- res = p.ParseWithResult(ctx, "doc.txt", src)
- want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux!"}
- if len(res.JSON) != len(want) {
- t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
- }
- for i, w := range want {
- if got := res.JSON[i]["text"]; got != w {
- t.Errorf("JSON[%d].text = %v, want %v", i, got, w)
- }
- }
-
- // Chinese sentence delimiters split the same way.
- src = []byte("这是第一句。这是第二句!第三句?结尾。")
- res = p.ParseWithResult(ctx, "doc.txt", src)
- if len(res.JSON) != 4 {
- t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON))
- }
-}
-
// TestTextParser_ParseWithResult_NewlineNormalization pins the
// normalizeTextNewlines contract: CRLF ("\r\n") and lone-CR ("\r") line
-// endings fold to LF before splitting, so every variant of the same logical
-// content yields identical items. This mirrors rag/nlp/delim.
-// normalize_text_newlines, which is what Python splits on, so Windows-line
-// documents parse identically to Unix ones.
+// endings fold to LF while the complete text remains one parser unit.
func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
@@ -394,22 +348,13 @@ func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {
// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is
// content-equivalent to Python's _code on the shared sample, using the shared
-// concatenation-normalization alignment tool (align_test.go). Python applies
-// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per
-// contract #17799), so item counts differ; the
-// comparison normalizes both (delimiters stripped, whitespace collapsed) and
-// joins on whitespace, so only CONTENT equivalence — not byte-exact layout — is
-// checked. The golden files are a NORMALIZED content baseline, not a verbatim
-// Python transcript: a fresh _code run at chunk_token_num=128 may merge into a
-// different item count/structure (e.g. the en sample collapses to one chunk while
-// the golden keeps prose and code as two items) and may collapse inter-sentence
-// newlines, so do not treat them as byte-exact.
+// concatenation-normalization alignment tool (align_test.go). Item boundaries
+// intentionally differ because Parser no longer performs Python's delimiter
+// split; the comparison joins normalized text and checks content only.
//
-// No generator script is committed. The baseline meta records generator, sample,
-// delimiter, keep_delimiters and chunk_token_num (see textcode.python.en/zh.golden.json);
-// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow
-// _code on the sample with keep_delimiters=True and the default delimiter set,
-// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}.
+// No generator script is committed. The baseline metadata records the Python
+// generator and delimiter used for normalization; Parser output is compared
+// for content equivalence, not Python's historical item boundaries.
func TestTextParser_AlignmentGolden(t *testing.T) {
ctx := t.Context()
p := NewTextParser()
@@ -439,54 +384,10 @@ func TestTextParser_AlignmentGolden(t *testing.T) {
goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)
pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)
- if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok {
+ delimiter, _ := gd.Meta["delimiter"].(string)
+ if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(delimiter)); !ok {
t.Fatalf("text&code parser not aligned with Python golden:%s", diff)
}
})
}
}
-
-// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent
-// delimiters, confirming it matches Python's deepdoc TxtParser.parser_txt
-// delimiter-loop exactly (not a divergence). Both ports run the same
-// re.split(r"(%s)" % dels, txt) loop with keep_delimiters=True and merge a
-// run of adjacent delimiters into the preceding segment, so the standalone
-// second delimiter is dropped on both sides: "a!!b" → ["a!", "b"] (verified
-// against deepdoc/parser/txt_parser.py). The alignment test's delimiter-strip
-// normalization also reconciles this, but this test guards splitCapturingDelims
-// directly so a future silent change there is caught independently.
-func TestTextParser_AdjacentDelimiters(t *testing.T) {
- ctx := t.Context()
- p := NewTextParser()
-
- // Two adjacent sentence delimiters: Go (and Python's parser_txt) merge
- // them into the preceding segment and drop the standalone second delimiter.
- src := []byte("a!!b")
- res := p.ParseWithResult(ctx, "doc.txt", src)
- if res.Err != nil {
- t.Fatalf("ParseWithResult: %v", res.Err)
- }
- want := []string{"a!", "b"}
- if len(res.JSON) != len(want) {
- t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
- }
- for i, w := range want {
- if got := res.JSON[i]["text"]; got != w {
- t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w)
- }
- }
-
- // Delimiters separated by text each attach to their own segment (no merge
- // across the gap).
- src = []byte("x?y!z")
- res = p.ParseWithResult(ctx, "doc.txt", src)
- want = []string{"x?", "y!", "z"}
- if len(res.JSON) != len(want) {
- t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
- }
- for i, w := range want {
- if got := res.JSON[i]["text"]; got != w {
- t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w)
- }
- }
-}
diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go
index 4a88d9b856..1c1efd33b0 100644
--- a/internal/parser/parser/text_parser.go
+++ b/internal/parser/parser/text_parser.go
@@ -22,21 +22,13 @@
// side needs a parser for these families so `text&code` resolves to a
// real ParseResultProducer.
//
-// TextParser fills that gap with a real implementation: it splits the
-// input into fine segments on the flow parser's default delimiter set and
-// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block
-// boundaries converge to the Python flow TxtParser (which uses the same
-// delimiter set); the OVER_CAP token merge that Python applies afterwards is
-// intentionally NOT performed here — chunking ownership stays with the
-// downstream Chunker (contract #17799), so
-// item counts differ from Python's merged chunks and are reconciled by the
-// stitch-compare alignment test (align_test.go).
+// TextParser decodes and normalizes text&code input, then emits one parser
+// unit. Chunk boundaries belong to GeneralChunker, not to this parser.
package parser
import (
"context"
- "regexp"
"strings"
)
@@ -49,19 +41,11 @@ func NewTextParser() *TextParser {
return &TextParser{}
}
-// ParseWithResult emits one item per non-empty paragraph. The
-// output format is "json" to mirror the python TxtParser's
-// behaviour (it emits a list of items with text + doc_type_kwd).
-//
-// The items slice is always non-nil so downstream chunkers see a
-// non-empty JSON payload even for an empty input (mirrors the
-// MarkdownParser convention at markdown_parser.go:71-76).
+// ParseWithResult emits the complete normalized text as one parser unit. The
+// output format is "json" to mirror the parser component boundary. No fixed
+// delimiter or token budget is applied here.
func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
decoded, encName := DecodeToUTF8(data, "text/plain")
- items := textParserItems(decoded)
- if items == nil {
- items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
- }
return ParseResult{
OutputFormat: "json",
File: map[string]any{
@@ -69,7 +53,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data
"size": len(data),
"encoding": encName,
},
- JSON: items,
+ JSON: []map[string]any{NewTextJSONItem(normalizeTextNewlines(string(decoded)))},
}
}
@@ -77,36 +61,8 @@ func (p *TextParser) String() string {
return "TextParser"
}
-// defaultTextDelimiterPattern is the regexp alternation of the flow parser's
-// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code
-// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror
-// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing
-// delimiter config entry, so this default
-// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter
-// const lives in delimiter.go so production and the alignment tests use one source
-// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims
-// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt)
-// interleaving.
-var (
- defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter)
- textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern)
- textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")
-)
-
-// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes
-// (longest-first is a no-op here: every delimiter in the default set is a
-// single rune, so insertion order is preserved like python's stable sort).
-func buildDelimiterPattern(delims string) string {
- parts := make([]string, 0, len(delims))
- for _, r := range delims {
- parts = append(parts, regexp.QuoteMeta(string(r)))
- }
- return strings.Join(parts, "|")
-}
-
-// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring
-// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split
-// identically to Unix ones.
+// normalizeTextNewlines folds CRLF and standalone CR to LF while preserving
+// the rest of the document content for GeneralChunker.
func normalizeTextNewlines(s string) string {
if s == "" {
return s
@@ -114,76 +70,3 @@ func normalizeTextNewlines(s string) string {
s = strings.ReplaceAll(s, "\r\n", "\n")
return strings.ReplaceAll(s, "\r", "\n")
}
-
-// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it
-// splits on the regexp and includes each matched delimiter as its own element
-// (with empty strings between adjacent delimiters) so callers can keep or drop
-// them. Go's regexp.Split discards captured groups, hence the manual walk.
-func splitCapturingDelims(s string, re *regexp.Regexp) []string {
- locs := re.FindAllStringIndex(s, -1)
- if len(locs) == 0 {
- return []string{s}
- }
- out := make([]string, 0, 2*len(locs)+1)
- prev := 0
- for _, loc := range locs {
- out = append(out, s[prev:loc[0]])
- out = append(out, s[loc[0]:loc[1]])
- prev = loc[1]
- }
- out = append(out, s[prev:])
- return out
-}
-
-// textParserItems splits data into fine segments on the flow parser's default
-// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to
-// (but not including) the OVER_CAP token merge. The token merge is intentionally
-// NOT performed here — chunking ownership stays with the downstream Chunker per
-// contract #17799 — so item counts differ from the
-// python flow's merged chunks and are reconciled by the stitch-compare alignment
-// test (align_test.go).
-//
-// Unlike the python signature default keep_delimiters=False, the flow _code
-// path calls TxtParser with keep_delimiters=True, so each segment keeps its
-// trailing delimiter attached (sentence-ending punctuation preserved for code
-// and prose).
-//
-// No per-item byte cap is applied: the parser is a pure delimiter splitter, just
-// like python's parser_txt (which also does no size slicing). Sizing belongs to
-// the chunker and the embedding truncation step, so a continuous run longer than
-// the embedding token budget (e.g. a minified / no-newline file) is kept as one
-// item here and collapsed to one chunk downstream — matching python's behaviour
-// rather than diverging from it.
-func textParserItems(data []byte) []map[string]any {
- txt := normalizeTextNewlines(string(data))
- secs := splitCapturingDelims(txt, textDelimiterSplitRe)
-
- var paras []string
- for i, sec := range secs {
- if textDelimiterExactRe.MatchString(sec) {
- continue
- }
- if sec == "" {
- continue
- }
- // keep_delimiters=True: append the delimiter to the segment it
- // follows, mirroring python's parser_txt.
- if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) {
- sec += secs[i+1]
- }
- paras = append(paras, sec)
- }
-
- var items []map[string]any
- for _, para := range paras {
- text := strings.TrimSpace(para)
- if text == "" {
- continue
- }
- items = append(items, map[string]any{
- "text": text,
- "doc_type_kwd": "text",
- })
- }
- return items
-}
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index 3dc510333b..e807ea8b63 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -25,7 +25,7 @@ type XLSParser struct {
libType string
ParseMethod string
OutputFormat string
- ChunkRows int
+ HTML4Excel bool
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -38,7 +38,6 @@ func NewXLSParser(libType string) (*XLSParser, error) {
}
return &XLSParser{
libType: libType,
- ChunkRows: defaultTableChunkRows,
TCADPTableResultType: "1",
TCADPMarkdownImageResponseType: "1",
}, nil
@@ -58,7 +57,10 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
- p.ChunkRows = decodeChunkRows(setup)
+ if v, ok := setup["html4excel"].(bool); ok {
+ p.HTML4Excel = v
+ }
+ deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
}
@@ -91,12 +93,7 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
}
- chunkRows := p.ChunkRows
- if chunkRows <= 0 {
- chunkRows = defaultTableChunkRows
- }
-
- items, warnings, sheetsCount, err := parseXLSXBytes(data, chunkRows)
+ items, warnings, sheetsCount, err := parseXLSXBytes(data, p.HTML4Excel)
if err != nil {
return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}
}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index 6aef5e68cd..c9ac44e4db 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -3,7 +3,6 @@ package parser
import (
"bytes"
"context"
- "strings"
"testing"
"github.com/xuri/excelize/v2"
@@ -27,7 +26,7 @@ func createTestExcelBytes(t *testing.T) []byte {
return buf.Bytes()
}
-func TestXLSParser_HTMLAndJSONOutput(t *testing.T) {
+func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
data := createTestExcelBytes(t)
p, err := NewXLSParser("excelize")
if err != nil {
@@ -47,14 +46,14 @@ func TestXLSParser_HTMLAndJSONOutput(t *testing.T) {
t.Fatalf("res.JSON should have structured table items")
}
item := res.JSON[0]
- if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {
- t.Errorf("table item text should contain rendered <table>, got %v", item["text"])
+ if item["ck_type"] != "table_header" {
+ t.Errorf("header ck_type = %v, want table_header", item["ck_type"])
}
- if item["doc_type_kwd"] != "table" {
- t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
+ if len(res.JSON) < 2 || res.JSON[1]["text"] != "Header1:Val1; Header2:Val2" {
+ t.Errorf("row item = %#v", res.JSON)
}
- if item["ck_type"] != "table" {
- t.Errorf("ck_type = %v, want 'table'", item["ck_type"])
+ if item["doc_type_kwd"] != "table" {
+ t.Errorf("header doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])
}
// Configure output format to "json"
@@ -88,11 +87,11 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
if res.OutputFormat != "json" {
t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)
}
- if len(res.JSON) == 0 {
- t.Fatalf("res.JSON should have items")
+ if len(res.JSON) < 2 {
+ t.Fatalf("res.JSON = %#v, want header and row", res.JSON)
}
- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
- t.Errorf("item text should contain HTML table, got %v", res.JSON[0]["text"])
+ if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {
+ t.Errorf("items should be header and row, got %#v", res.JSON)
}
// Even if configured to legacy "html", format is unified to "json"
diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go
index bf3fe936d7..568225603d 100644
--- a/internal/parser/parser/xls_tcadp_test.go
+++ b/internal/parser/parser/xls_tcadp_test.go
@@ -1,6 +1,8 @@
package parser
import (
+ "bytes"
+ "log/slog"
"net/http"
"net/http/httptest"
"strings"
@@ -32,6 +34,7 @@ func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {
p.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
"output_format": "json",
+ "chunk_rows": 50,
"tcadp_apiserver": server.URL,
"tcadp_api_key": "tcadp-secret",
"table_result_type": "1",
@@ -220,11 +223,11 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {
if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if len(res.JSON) == 0 {
- t.Fatal("JSON items is empty; want structured table items")
+ if len(res.JSON) < 2 {
+ t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))
}
- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
- t.Fatalf("JSON item text = %v; want rendered table", res.JSON[0]["text"])
+ if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {
+ t.Fatalf("JSON items = %#v; want header and row", res.JSON)
}
}
@@ -236,6 +239,7 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {
p.ConfigureFromSetup(map[string]any{
"parse_method": "TCADP parser",
"output_format": "json",
+ "chunk_rows": 50,
"tcadp_apiserver": "https://tcadp.example.com",
"tcadp_api_key": "secret",
"table_result_type": "2",
@@ -257,3 +261,16 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {
t.Fatalf("TCADPMarkdownImageResponseType = %q, want %q", got, want)
}
}
+
+func TestSpreadsheetParserWarnsForDeprecatedChunkRows(t *testing.T) {
+ var logs bytes.Buffer
+ previousLogger := slog.Default()
+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})))
+ t.Cleanup(func() { slog.SetDefault(previousLogger) })
+
+ p := NewCSVParser()
+ p.ConfigureFromSetup(map[string]any{"chunk_rows": 32})
+ if !strings.Contains(logs.String(), "deprecated chunk_rows") {
+ t.Fatalf("logs = %q, want deprecated chunk_rows warning", logs.String())
+ }
+}
diff --git a/internal/parser/parser/xlsx_normalize_test.go b/internal/parser/parser/xlsx_normalize_test.go
index 381909d42b..3f3fb8e0fc 100644
--- a/internal/parser/parser/xlsx_normalize_test.go
+++ b/internal/parser/parser/xlsx_normalize_test.go
@@ -29,10 +29,10 @@ func TestXLSXParser_NormalizesInvalidSheetName(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- html := xlsxTableHTML(res)
+ text := spreadsheetText(res)
for _, want := range []string{"Name", "Amount", "Alice", "100"} {
- if !strings.Contains(html, want) {
- t.Fatalf("parsed table = %q, want %q", html, want)
+ if !strings.Contains(text, want) {
+ t.Fatalf("parsed rows = %q, want %q", text, want)
}
}
if !strings.Contains(strings.Join(res.Warnings, "\n"), "Visible:Data") {
@@ -76,8 +76,8 @@ func TestXLSXParser_NormalizesEmojiSheetNameWithinUTF16Limit(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "Alice") {
- t.Fatalf("parsed table = %q, want Alice", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "Alice") {
+ t.Fatalf("parsed rows = %q, want Alice", spreadsheetText(res))
}
}
@@ -138,8 +138,8 @@ func TestXLSXParser_WarnsWhenMergedCellsCannotBeRead(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "Alice") {
- t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "Alice") {
+ t.Fatalf("parsed rows = %q, want sheet data", spreadsheetText(res))
}
if !strings.Contains(strings.Join(res.Warnings, "\n"), "merged cells") {
t.Fatalf("warnings = %v, want merged-cells warning", res.Warnings)
@@ -161,8 +161,8 @@ func TestXLSXParser_WarnsWhenTableMetadataCannotBeRead(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if !strings.Contains(xlsxTableHTML(res), "Alice") {
- t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))
+ if !strings.Contains(spreadsheetText(res), "Alice") {
+ t.Fatalf("parsed rows = %q, want sheet data", spreadsheetText(res))
}
if !strings.Contains(strings.Join(res.Warnings, "\n"), "table metadata") {
t.Fatalf("warnings = %v, want table-metadata warning", res.Warnings)
diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.go
index e91c5c97eb..c5e9ae57ba 100644
--- a/internal/parser/parser/xlsx_parse_method_test.go
+++ b/internal/parser/parser/xlsx_parse_method_test.go
@@ -78,7 +78,7 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
if len(res.JSON) != 1 {
- t.Fatalf("JSON item count = %d, want 1", len(res.JSON))
+ t.Fatalf("JSON item count = %d, want header-only result", len(res.JSON))
}
text, _ := res.JSON[0]["text"].(string)
if !strings.Contains(text, tc.cellValue) {
@@ -120,6 +120,9 @@ func TestXLSXParser_ExtractsFloatingImages(t *testing.T) {
if image["image"] != "data:image/png;base64,"+pngBase64 {
t.Fatalf("image data = %v, want data URL", image["image"])
}
+ if image["sheet_index"] != 1 || image["row_start"] != 3 || image["row_end"] != 3 || image["col_start"] != 3 || image["col_end"] != 3 {
+ t.Fatalf("image coordinates = %#v, want Sheet1!C3", image)
+ }
}
func TestXLSXParser_ImageWithoutAltTextUsesAnchorCell(t *testing.T) {
@@ -157,7 +160,7 @@ func mustDecodeBase64(t *testing.T, encoded string) []byte {
}
// TestCSVParser_DeepDocParseMethod asserts the CSV parser accepts the
-// default "deepdoc" parse_method and renders the default HTML table.
+// default "deepdoc" parse_method and emits row IR.
func TestCSVParser_DeepDocParseMethod(t *testing.T) {
p := NewCSVParser()
p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"})
@@ -170,11 +173,11 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {
if got, want := res.OutputFormat, "json"; got != want {
t.Fatalf("OutputFormat = %q, want %q", got, want)
}
- if len(res.JSON) == 0 {
- t.Fatal("JSON items is empty; want structured table items")
+ if len(res.JSON) < 2 {
+ t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))
}
- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {
- t.Fatalf("JSON item text = %v, want a rendered <table>", res.JSON[0]["text"])
+ if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {
+ t.Fatalf("JSON items = %#v, want header and row", res.JSON)
}
}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index eea92c925d..aff077bd90 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -29,7 +29,7 @@ type XLSXParser struct {
libType string
ParseMethod string
OutputFormat string
- ChunkRows int
+ HTML4Excel bool
TCADPAPIServer string
TCADPAPIKey string
TCADPTableResultType string
@@ -42,7 +42,6 @@ func NewXLSXParser(libType string) (*XLSXParser, error) {
}
return &XLSXParser{
libType: libType,
- ChunkRows: defaultTableChunkRows,
TCADPTableResultType: "1",
TCADPMarkdownImageResponseType: "1",
}, nil
@@ -62,7 +61,10 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {
if v, ok := setup["output_format"].(string); ok && v != "" {
p.OutputFormat = v
}
- p.ChunkRows = decodeChunkRows(setup)
+ if v, ok := setup["html4excel"].(bool); ok {
+ p.HTML4Excel = v
+ }
+ deprecatedChunkRows(setup, p.String())
if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {
p.TCADPAPIServer = v
}
@@ -113,12 +115,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
// for spreadsheet processing.
}
- chunkRows := p.ChunkRows
- if chunkRows <= 0 {
- chunkRows = defaultTableChunkRows
- }
-
- items, warnings, sheets, err := parseXLSXBytes(data, chunkRows)
+ items, warnings, sheets, err := parseXLSXBytes(data, p.HTML4Excel)
if err == nil {
return xlsxParseResult(filename, items, warnings, sheets)
}
@@ -130,7 +127,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
if !changed {
return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}
}
- items, warnings, sheets, retryErr := parseXLSXBytes(normalized, chunkRows)
+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized, p.HTML4Excel)
if retryErr != nil {
return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}
}
@@ -138,7 +135,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
return xlsxParseResult(filename, items, warnings, sheets)
}
-func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int, error) {
+func parseXLSXBytes(data []byte, html4excel bool) ([]map[string]any, []string, int, error) {
f, err := excelize.OpenReader(bytes.NewReader(data))
if err != nil {
return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)
@@ -149,22 +146,30 @@ func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int
items := make([]map[string]any, 0)
warnings := make([]string, 0)
for sheetIdx, sheet := range sheets {
- tables, sheetWarnings, err := renderSheetTableChunks(f, sheet, chunkRows)
+ records, dataRows, headerRow, sheetWarnings, err := readSpreadsheetRecords(f, sheet)
if err != nil {
return nil, warnings, len(sheets), err
}
warnings = append(warnings, sheetWarnings...)
- for _, table := range tables {
- items = append(items, NewTableJSONItem(table.HTML, sheet, [][]float64{{
- float64(sheetIdx + 1),
- float64(table.RowStart),
- float64(table.RowEnd),
- float64(table.ColStart),
- float64(table.ColEnd),
- }}))
+ var sheetItems []map[string]any
+ if html4excel {
+ if table := recordsToHTMLTableItem(records, sheet, sheetIdx+1, headerRow, dataRows); table != nil {
+ sheetItems = []map[string]any{table}
+ }
+ } else {
+ sheetItems = recordsToSpreadsheetItems(records, sheet, sheetIdx+1, headerRow, dataRows)
}
images, imageWarnings := extractXLSXImages(f, sheet)
- items = append(items, images...)
+ for _, image := range images {
+ row, _ := numericItemInt(image["row_start"])
+ col, _ := numericItemInt(image["col_start"])
+ image["sheet_index"] = sheetIdx + 1
+ image["table_id"] = fmt.Sprintf("sheet-%d", sheetIdx+1)
+ image["positions"] = [][]float64{{float64(sheetIdx + 1), float64(row), float64(row), float64(col), float64(col)}}
+ }
+ sheetItems = append(sheetItems, images...)
+ sortSpreadsheetItems(sheetItems)
+ items = append(items, sheetItems...)
warnings = append(warnings, imageWarnings...)
}
return items, warnings, len(sheets), nil
diff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.go
index b45653b6e5..d20ddc9bc5 100644
--- a/internal/service/components_service_test.go
+++ b/internal/service/components_service_test.go
@@ -28,7 +28,7 @@ import (
_ "ragflow/internal/agent/component" // registers agent components
"ragflow/internal/agent/runtime"
_ "ragflow/internal/ingestion/component" // registers ingestion main components
- _ "ragflow/internal/ingestion/component/chunker" // registers 4 chunker variants
+ _ "ragflow/internal/ingestion/component/chunker" // registers chunker variants
)
func TestComponentsService_List_NoFilter(t *testing.T) {
@@ -62,7 +62,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) {
t.Fatalf("List(Ingestion) returned error: %v", err)
}
wantNames := []string{
- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
+ "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",
}
assertComponentNameSet(t, "ingestion", namesOf(got), wantNames)
@@ -85,7 +85,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) {
t.Fatalf("List(Ingestion,Shared) returned error: %v", err)
}
wantNames := []string{
- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
+ "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",
}
assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames)
@@ -104,7 +104,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) {
t.Fatalf("List(Ingestion x3) returned error: %v", err)
}
wantNames := []string{
- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
+ "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",
"onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",
}
if len(got) != len(wantNames) {
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 5e0d8b9c02..012b4d4e46 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -5,6 +5,7 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
+ "html"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/parser/parser"
@@ -232,6 +233,9 @@ func parseResultText(res parser.ParseResult) (string, error) {
return res.HTML, nil
case "json":
if len(res.JSON) > 0 {
+ if rendered, ok := renderSpreadsheetJSON(res.JSON); ok {
+ return rendered, nil
+ }
parts := make([]string, 0, len(res.JSON))
for _, item := range res.JSON {
if text, ok := item["text"].(string); ok {
@@ -260,6 +264,164 @@ func parseResultText(res parser.ParseResult) (string, error) {
}
}
+func renderSpreadsheetJSON(items []map[string]any) (string, bool) {
+ parts := make([]string, 0, len(items))
+ rendered := false
+ for i := 0; i < len(items); {
+ if !isSpreadsheetRowItem(items[i]) {
+ if text, ok := items[i]["text"].(string); ok {
+ parts = append(parts, text)
+ } else {
+ raw, err := json.Marshal(items[i])
+ if err != nil {
+ return "", false
+ }
+ parts = append(parts, string(raw))
+ }
+ i++
+ continue
+ }
+
+ start := i
+ i = spreadsheetTableEnd(items, start)
+ parts = append(parts, renderSpreadsheetTable(items[start:i]))
+ rendered = true
+ }
+ if !rendered {
+ return "", false
+ }
+ return strings.Join(parts, "\n"), true
+}
+
+func isSpreadsheetRowItem(item map[string]any) bool {
+ kind, _ := item["ck_type"].(string)
+ return kind == "table_header" || kind == "table_row"
+}
+
+func spreadsheetTableKey(item map[string]any) string {
+ if tableID, _ := item["table_id"].(string); strings.TrimSpace(tableID) != "" {
+ return "table:" + tableID
+ }
+ if sheetIndex, ok := item["sheet_index"].(float64); ok {
+ return fmt.Sprintf("sheet-index:%g", sheetIndex)
+ }
+ if sheetIndex, ok := item["sheet_index"].(int); ok {
+ return fmt.Sprintf("sheet-index:%d", sheetIndex)
+ }
+ if sheet, _ := item["sheet"].(string); strings.TrimSpace(sheet) != "" {
+ return "sheet:" + sheet
+ }
+ return ""
+}
+
+func spreadsheetTableEnd(items []map[string]any, start int) int {
+ key := spreadsheetTableKey(items[start])
+ if key != "" {
+ i := start
+ for i < len(items) && isSpreadsheetRowItem(items[i]) && spreadsheetTableKey(items[i]) == key {
+ i++
+ }
+ return i
+ }
+
+ // Without a stable table identity, do not merge unrelated row sequences.
+ // A header can still own the following anonymous rows until the next
+ // anonymous header; anonymous row-only inputs are kept as separate tables
+ // because their table boundary cannot be inferred safely.
+ if kind, _ := items[start]["ck_type"].(string); kind == "table_header" {
+ i := start + 1
+ for i < len(items) && isSpreadsheetRowItem(items[i]) {
+ if nextKind, _ := items[i]["ck_type"].(string); nextKind == "table_header" {
+ break
+ }
+ if spreadsheetTableKey(items[i]) != "" {
+ break
+ }
+ i++
+ }
+ return i
+ }
+ return start + 1
+}
+
+func renderSpreadsheetTable(items []map[string]any) string {
+ if len(items) == 0 {
+ return ""
+ }
+ var header []string
+ if kind, _ := items[0]["ck_type"].(string); kind == "table_header" {
+ header = spreadsheetStringSlice(items[0]["cells"])
+ if len(header) == 0 {
+ header = spreadsheetStringSlice(items[0]["text"])
+ }
+ } else {
+ // Row IR carries the logical column names separately from its cells.
+ // Use those names as a preview header without consuming the first row.
+ header = spreadsheetStringSlice(items[0]["headers"])
+ }
+
+ sheet, _ := items[0]["sheet"].(string)
+ var builder strings.Builder
+ builder.WriteString("<table>")
+ if strings.TrimSpace(sheet) != "" {
+ builder.WriteString("<caption>")
+ builder.WriteString(html.EscapeString(strings.TrimSpace(sheet)))
+ builder.WriteString("</caption>")
+ }
+ if len(header) > 0 {
+ builder.WriteString("<tr>")
+ for _, cell := range header {
+ builder.WriteString("<th>")
+ builder.WriteString(html.EscapeString(cell))
+ builder.WriteString("</th>")
+ }
+ builder.WriteString("</tr>")
+ }
+ for _, item := range items {
+ kind, _ := item["ck_type"].(string)
+ if kind == "table_header" {
+ continue
+ }
+ cells := spreadsheetStringSlice(item["cells"])
+ if len(cells) == 0 {
+ cells = spreadsheetStringSlice(item["text"])
+ }
+ builder.WriteString("<tr>")
+ for _, cell := range cells {
+ builder.WriteString("<td>")
+ builder.WriteString(html.EscapeString(cell))
+ builder.WriteString("</td>")
+ }
+ builder.WriteString("</tr>")
+ }
+ builder.WriteString("</table>")
+ return builder.String()
+}
+
+func spreadsheetStringSlice(value any) []string {
+ switch values := value.(type) {
+ case []string:
+ out := make([]string, len(values))
+ for i, value := range values {
+ out[i] = strings.TrimSpace(value)
+ }
+ return out
+ case []any:
+ out := make([]string, 0, len(values))
+ for _, value := range values {
+ out = append(out, strings.TrimSpace(fmt.Sprint(value)))
+ }
+ return out
+ case string:
+ if strings.TrimSpace(values) == "" {
+ return nil
+ }
+ return []string{strings.TrimSpace(values)}
+ default:
+ return nil
+ }
+}
+
// parseFileContent tries to parse a file's contents using the appropriate parser.
// Falls back to returning raw text if no parser is available.
func parseFileContent(ctx context.Context, filename string, data []byte) string {
diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.go
index 795de53997..5a70e6edf3 100644
--- a/internal/service/file/file_content_test.go
+++ b/internal/service/file/file_content_test.go
@@ -75,6 +75,91 @@ func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T)
}
}
+func TestParseResultText_SpreadsheetRowsRenderReadableHTMLTable(t *testing.T) {
+ result, err := parseResultText(parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {
+ "text": "a; b",
+ "doc_type_kwd": "table",
+ "ck_type": "table_header",
+ "table_id": "sheet-1",
+ "sheet": "Data",
+ "cells": []string{"a", "b"},
+ },
+ {
+ "text": "a:1; b:2 ——Data",
+ "doc_type_kwd": "text",
+ "ck_type": "table_row",
+ "table_id": "sheet-1",
+ "sheet": "Data",
+ "headers": []string{"a", "b"},
+ "cells": []string{"1", "2"},
+ },
+ },
+ })
+ if err != nil {
+ t.Fatalf("parseResultText: %v", err)
+ }
+ for _, want := range []string{"<table", "<caption>Data</caption>", "<th>a</th>", "<td>1</td>"} {
+ if !strings.Contains(result, want) {
+ t.Errorf("parseResultText = %q, want %q", result, want)
+ }
+ }
+}
+
+func TestParseResultText_SpreadsheetRowsWithoutHeaderDoNotDuplicateFirstRow(t *testing.T) {
+ result, err := parseResultText(parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {
+ "ck_type": "table_row",
+ "headers": []string{"name", "value"},
+ "cells": []string{"alpha", "1"},
+ },
+ },
+ })
+ if err != nil {
+ t.Fatalf("parseResultText: %v", err)
+ }
+ if !strings.Contains(result, "<th>name</th>") {
+ t.Fatalf("row headers were not rendered: %q", result)
+ }
+ if got := strings.Count(result, "<td>alpha</td>"); got != 1 {
+ t.Fatalf("first row rendered %d times, want once: %q", got, result)
+ }
+}
+
+func TestParseResultText_SpreadsheetRowsWithoutIdentityStaySeparate(t *testing.T) {
+ result, err := parseResultText(parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {
+ "ck_type": "table_header",
+ "cells": []string{"first"},
+ },
+ {
+ "ck_type": "table_row",
+ "cells": []string{"one"},
+ },
+ {
+ "ck_type": "table_header",
+ "cells": []string{"second"},
+ },
+ {
+ "ck_type": "table_row",
+ "cells": []string{"two"},
+ },
+ },
+ })
+ if err != nil {
+ t.Fatalf("parseResultText: %v", err)
+ }
+ if got := strings.Count(result, "<table>"); got != 2 {
+ t.Fatalf("unknown spreadsheet tables were merged: got %d tables in %q", got, result)
+ }
+}
+
func TestParseResultText_EmailJSONKeepsSearchableHeaders(t *testing.T) {
raw := []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: Parser contract\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEmail body")
p := parser.NewEmailParser()
diff --git a/web/src/components/operator-icon.tsx b/web/src/components/operator-icon.tsx
index ff52e29325..6052a11954 100644
--- a/web/src/components/operator-icon.tsx
+++ b/web/src/components/operator-icon.tsx
@@ -93,6 +93,7 @@ export const LucideIconMap = {
[Operator.File]: LucideFile,
[Operator.Parser]: LucideFilePlay,
[Operator.Tokenizer]: LucideListPlus,
+ [Operator.GeneralChunker]: LucideBlocks,
[Operator.TokenChunker]: LucideBlocks,
[Operator.TitleChunker]: LucideHeading,
[Operator.Extractor]: LucideFileStack,
diff --git a/web/src/components/pipeline-operator-tabs/parser-config-validation.ts b/web/src/components/pipeline-operator-tabs/parser-config-validation.ts
index 59ca040bae..6e7a8044b7 100644
--- a/web/src/components/pipeline-operator-tabs/parser-config-validation.ts
+++ b/web/src/components/pipeline-operator-tabs/parser-config-validation.ts
@@ -17,6 +17,7 @@
import { Operator } from '@/constants/agent';
import {
initialCompilationValues,
+ initialGeneralChunkerValues,
initialGoExtractorValues,
initialParserValues,
initialTitleChunkerValues,
@@ -42,6 +43,8 @@ function getOperatorFormSchema(operatorType: Operator, t: TFunction) {
return ParserFormSchema;
case Operator.TokenChunker:
return TokenChunkerFormSchema;
+ case Operator.GeneralChunker:
+ return TokenChunkerFormSchema;
case Operator.TitleChunker:
return TitleChunkerFormSchema;
case Operator.Extractor:
@@ -61,6 +64,8 @@ function getOperatorInitialValues(operatorType: Operator) {
return initialParserValues;
case Operator.TokenChunker:
return initialTokenChunkerValues;
+ case Operator.GeneralChunker:
+ return initialGeneralChunkerValues;
case Operator.TitleChunker:
return initialTitleChunkerValues;
case Operator.Extractor:
diff --git a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx
index a9b32b5766..a1aa1b7252 100644
--- a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx
+++ b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx
@@ -72,6 +72,16 @@ const PipelineOperatorForm = ({
externalErrors={externalErrors}
/>
);
+ case Operator.GeneralChunker:
+ return (
+ <TokenChunkerForm
+ node={node}
+ onValuesChange={handleValuesChange}
+ hideOutputs
+ externalErrors={externalErrors}
+ isGeneralChunker
+ />
+ );
case Operator.TitleChunker:
return (
<TitleChunkerForm
diff --git a/web/src/constants/agent.tsx b/web/src/constants/agent.tsx
index fd81020f4b..c3fb973a83 100644
--- a/web/src/constants/agent.tsx
+++ b/web/src/constants/agent.tsx
@@ -130,6 +130,7 @@ export enum Operator {
File = 'File', // pipeline
Parser = 'Parser',
Tokenizer = 'Tokenizer',
+ GeneralChunker = 'GeneralChunker',
TokenChunker = 'TokenChunker',
TitleChunker = 'TitleChunker',
Extractor = 'Extractor',
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index d10501d1e8..daba5365fd 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -670,6 +670,12 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
imageTableContextWindow: 'Image & table context window',
imageTableContextWindowTip:
'Captures N tokens of text above and below the image & table to provide richer background context.',
+ tableContextWindow: 'Table context window',
+ tableContextWindowTip:
+ 'Captures N tokens of text above and below a table to provide richer background context.',
+ imageContextWindow: 'Image context window',
+ imageContextWindowTip:
+ 'Captures N tokens of text above and below an image to provide richer background context.',
autoMetadata: 'Auto metadata',
mineruOptions: 'MinerU options',
mineruParseMethod: 'Parse method',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index e28f62edc9..ee80787056 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -609,6 +609,12 @@ export default {
imageTableContextWindow: '图像与表格上下文窗口',
imageTableContextWindowTip:
'抓取图像与表格上下方的 N 个 token,为该 Chunk 提供更丰富的背景上下文。',
+ tableContextWindow: '表格上下文窗口',
+ tableContextWindowTip:
+ '抓取表格上下方的 N 个 token,为该 Chunk 提供更丰富的背景上下文。',
+ imageContextWindow: '图像上下文窗口',
+ imageContextWindowTip:
+ '抓取图像上下方的 N 个 token,为该 Chunk 提供更丰富的背景上下文。',
autoMetadata: '自动元数据',
mineruOptions: 'MinerU 选项',
mineruParseMethod: '解析方法',
diff --git a/web/src/pages/agent/constant/index.tsx b/web/src/pages/agent/constant/index.tsx
index 180e47f7ce..57a08589d1 100644
--- a/web/src/pages/agent/constant/index.tsx
+++ b/web/src/pages/agent/constant/index.tsx
@@ -825,6 +825,7 @@ export const RestrictedUpstreamMap = {
[Operator.VariableAssigner]: [Operator.Begin],
[Operator.VariableAggregator]: [Operator.Begin],
[Operator.Parser]: [Operator.Begin], // pipeline
+ [Operator.GeneralChunker]: [Operator.Begin],
[Operator.TokenChunker]: [Operator.Begin],
[Operator.TitleChunker]: [Operator.Begin],
[Operator.Tokenizer]: [Operator.Begin],
@@ -881,6 +882,7 @@ export const NodeMap = {
[Operator.File]: 'fileNode',
[Operator.Parser]: 'parserNode',
[Operator.Tokenizer]: 'tokenizerNode',
+ [Operator.GeneralChunker]: 'chunkerNode',
[Operator.TokenChunker]: 'chunkerNode',
[Operator.TitleChunker]: 'chunkerNode',
[Operator.Extractor]: 'contextNode',
@@ -928,6 +930,7 @@ export const NoDebugOperatorsList = [
Operator.File,
Operator.Parser,
Operator.Tokenizer,
+ Operator.GeneralChunker,
Operator.TokenChunker,
Operator.TitleChunker,
Operator.Extractor,
@@ -940,6 +943,7 @@ export const NoCopyOperatorsList = [
Operator.File,
Operator.Parser,
Operator.Tokenizer,
+ Operator.GeneralChunker,
Operator.TokenChunker,
Operator.TitleChunker,
Operator.Extractor,
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index e0dd0dd2f6..d6ee82567d 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -285,6 +285,28 @@ export const initialTokenChunkerValues = {
children_delimiters: [],
};
+export const initialGeneralChunkerValues = {
+ outputs: {
+ chunks: { type: 'Array<Object>', value: [] },
+ },
+ chunk_token_size: 512,
+ overlapped_percent: 0,
+ delimiters: [
+ { value: '\n' },
+ { value: '!' },
+ { value: '?' },
+ { value: ';' },
+ { value: '。' },
+ { value: ';' },
+ { value: '!' },
+ { value: '?' },
+ ],
+ table_context_size: 0,
+ image_context_size: 0,
+ enable_children: false,
+ children_delimiters: [],
+};
+
export enum Hierarchy {
H1 = '1',
H2 = '2',
@@ -473,6 +495,7 @@ export const FileTypeSuffixMap = {
export const SingleOperators = [
Operator.Tokenizer,
+ Operator.GeneralChunker,
Operator.TokenChunker,
Operator.TitleChunker,
Operator.Parser,
diff --git a/web/src/pages/agent/form-sheet/form-config-map.tsx b/web/src/pages/agent/form-sheet/form-config-map.tsx
index eba7cd7496..d4cb0611cb 100644
--- a/web/src/pages/agent/form-sheet/form-config-map.tsx
+++ b/web/src/pages/agent/form-sheet/form-config-map.tsx
@@ -48,6 +48,7 @@ import VariableAssignerForm from '../form/variable-assigner-form';
import WenCaiForm from '../form/wencai-form';
import WikipediaForm from '../form/wikipedia-form';
import YahooFinanceForm from '../form/yahoo-finance-form';
+import type { INextOperatorForm } from '../interface';
export const FormConfigMap = {
[Operator.Begin]: {
@@ -183,6 +184,11 @@ export const FormConfigMap = {
[Operator.Tokenizer]: {
component: TokenizerForm,
},
+ [Operator.GeneralChunker]: {
+ component: (props: INextOperatorForm) => (
+ <TokenChunkerForm {...props} isGeneralChunker />
+ ),
+ },
[Operator.TokenChunker]: {
component: TokenChunkerForm,
},
diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsx
index b749f9efec..e8dd32ba03 100644
--- a/web/src/pages/agent/form/token-chunker-form/index.tsx
+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx
@@ -13,7 +13,10 @@ import { memo } from 'react';
import { useFieldArray, useForm } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
-import { initialTokenChunkerValues } from '../../constant/pipeline';
+import {
+ initialGeneralChunkerValues,
+ initialTokenChunkerValues,
+} from '../../constant/pipeline';
import { useFormChangeCallback } from '../../hooks/use-form-change-callback';
import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
@@ -26,7 +29,9 @@ const outputList = buildOutputList(initialTokenChunkerValues.outputs);
export const FormSchema = z.object({
chunk_token_size: z.number(),
- image_table_context_window: z.number(),
+ image_table_context_window: z.number().optional(),
+ table_context_size: z.number().optional(),
+ image_context_size: z.number().optional(),
delimiters: z.array(
z.object({
value: z.string().optional(),
@@ -44,13 +49,21 @@ export const FormSchema = z.object({
export type TokenChunkerFormSchemaType = z.infer<typeof FormSchema>;
+type TokenChunkerFormProps = INextOperatorForm & {
+ isGeneralChunker?: boolean;
+};
+
const TokenChunkerForm = ({
node,
onValuesChange,
hideOutputs,
externalErrors,
-}: INextOperatorForm) => {
- const defaultValues = useFormValues(initialTokenChunkerValues, node);
+ isGeneralChunker = false,
+}: TokenChunkerFormProps) => {
+ const initialValues = isGeneralChunker
+ ? initialGeneralChunkerValues
+ : initialTokenChunkerValues;
+ const defaultValues = useFormValues(initialValues, node);
const { t } = useTranslation();
// Normalize legacy values: 'token_size' (removed tab) and empty fall back
@@ -92,17 +105,19 @@ const TokenChunkerForm = ({
return (
<Form {...form}>
<FormWrapper>
- <RenderField
- field={{
- name: 'delimiter_mode',
- type: FormFieldType.Segmented,
- label: '',
- options: [
- { label: t('flow.delimiters'), value: 'delimiter' },
- { label: t('flow.one'), value: 'one' },
- ],
- }}
- />
+ {!isGeneralChunker && (
+ <RenderField
+ field={{
+ name: 'delimiter_mode',
+ type: FormFieldType.Segmented,
+ label: '',
+ options: [
+ { label: t('flow.delimiters'), value: 'delimiter' },
+ { label: t('flow.one'), value: 'one' },
+ ],
+ }}
+ />
+ )}
{delimiterMode === 'delimiter' && (
<>
@@ -120,14 +135,35 @@ const TokenChunkerForm = ({
integer
label={t('flow.overlappedPercent')}
/>
- <SliderInputFormField
- name="image_table_context_window"
- max={256}
- min={0}
- integer
- label={t('knowledgeConfiguration.imageTableContextWindow')}
- tooltip={t('knowledgeConfiguration.imageTableContextWindowTip')}
- />
+ {isGeneralChunker ? (
+ <>
+ <SliderInputFormField
+ name="table_context_size"
+ max={256}
+ min={0}
+ integer
+ label={t('knowledgeConfiguration.tableContextWindow')}
+ tooltip={t('knowledgeConfiguration.tableContextWindowTip')}
+ />
+ <SliderInputFormField
+ name="image_context_size"
+ max={256}
+ min={0}
+ integer
+ label={t('knowledgeConfiguration.imageContextWindow')}
+ tooltip={t('knowledgeConfiguration.imageContextWindowTip')}
+ />
+ </>
+ ) : (
+ <SliderInputFormField
+ name="image_table_context_window"
+ max={256}
+ min={0}
+ integer
+ label={t('knowledgeConfiguration.imageTableContextWindow')}
+ tooltip={t('knowledgeConfiguration.imageTableContextWindowTip')}
+ />
+ )}
<section>
<span className="mb-2 inline-block">{t('flow.delimiters')}</span>
<div className="space-y-4">
diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.ts
index e79a2043e4..77e682c8ca 100644
--- a/web/src/pages/agent/hooks/use-add-node.ts
+++ b/web/src/pages/agent/hooks/use-add-node.ts
@@ -51,6 +51,7 @@ import {
initialTavilyValues,
initialTitleChunkerValues,
initialTokenChunkerValues,
+ initialGeneralChunkerValues,
initialTokenizerValues,
initialUserFillUpValues,
initialVariableAggregatorValues,
@@ -185,6 +186,7 @@ export const useInitializeOperatorParams = () => {
[Operator.Parser]: buildInitialParserValues(defaultModelDictionary),
[Operator.Tokenizer]: initialTokenizerValues,
[Operator.TokenChunker]: initialTokenChunkerValues,
+ [Operator.GeneralChunker]: initialGeneralChunkerValues,
[Operator.TitleChunker]: initialTitleChunkerValues,
[Operator.Extractor]: {
...getInitialExtractorValues(),
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index 52ac5d6f0f..0005405f0e 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -367,6 +367,16 @@ export function transformTokenChunkerParams(
};
}
+export function transformGeneralChunkerParams(
+ params: TokenChunkerFormSchemaType,
+) {
+ const result = transformTokenChunkerParams(params);
+ result.table_context_size = Number(params.table_context_size || 0);
+ result.image_context_size = Number(params.image_context_size || 0);
+ delete result.delimiter_mode;
+ return result;
+}
+
export function transformTitleChunkerParams(
params: TitleChunkerFormSchemaType,
) {
@@ -640,6 +650,10 @@ export const buildDslComponentsByGraph = (
params = transformTokenChunkerParams(params);
break;
+ case Operator.GeneralChunker:
+ params = transformGeneralChunkerParams(params);
+ break;
+
case Operator.TitleChunker:
params = transformTitleChunkerParams(params);
break;
diff --git a/web/src/utils/__tests__/pipeline-operator.test.ts b/web/src/utils/__tests__/pipeline-operator.test.ts
index 8718bc4aac..a84f51b1d3 100644
--- a/web/src/utils/__tests__/pipeline-operator.test.ts
+++ b/web/src/utils/__tests__/pipeline-operator.test.ts
@@ -1,4 +1,9 @@
-import { buildOperatorNode } from '@/utils/pipeline-operator';
+import {
+ buildOperatorNode,
+ getOperatorType,
+ transformApiConfigToForm,
+ transformFormConfigToApi,
+} from '@/utils/pipeline-operator';
let mockIsGoBackend = true;
jest.mock('@/utils/backend-runtime', () => ({
@@ -10,6 +15,42 @@ const extractorNode = {
data: { form: {} },
} as any;
+describe('GeneralChunker operator bridge', () => {
+ it('recognizes and transforms the built-in general chunker config', () => {
+ expect(getOperatorType('GeneralChunker:SixApplesFall')).toBe(
+ 'GeneralChunker',
+ );
+
+ const form = transformApiConfigToForm('GeneralChunker', {
+ chunk_token_size: 512,
+ delimiters: ['\n', ';'],
+ overlapped_percent: 0.1,
+ table_context_size: 2,
+ image_context_size: 3,
+ });
+ expect(form.delimiters).toEqual([{ value: '\n' }, { value: ';' }]);
+ expect(form.table_context_size).toBe(2);
+ expect(form.image_context_size).toBe(3);
+ expect(form).not.toHaveProperty('image_table_context_window');
+ expect(form).not.toHaveProperty('delimiter_mode');
+
+ const api = transformFormConfigToApi('GeneralChunker', {
+ delimiter_mode: 'delimiter',
+ chunk_token_size: 512,
+ delimiters: [{ value: '\n' }, { value: ';' }],
+ children_delimiters: [],
+ enable_children: false,
+ overlapped_percent: 10,
+ table_context_size: 2,
+ image_context_size: 3,
+ });
+ expect(api.delimiters).toEqual(['\n', ';']);
+ expect(api.table_context_size).toBe(2);
+ expect(api.image_context_size).toBe(3);
+ expect(api).not.toHaveProperty('delimiter_mode');
+ });
+});
+
describe('buildOperatorNode dataset-level metadata precedence', () => {
beforeEach(() => {
mockIsGoBackend = true;
diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.ts
index 6b22136a77..25fb249fe2 100644
--- a/web/src/utils/pipeline-operator.ts
+++ b/web/src/utils/pipeline-operator.ts
@@ -19,6 +19,7 @@ import { DSL, RAGFlowNodeType } from '@/interfaces/database/agent';
import {
getInitialExtractorValues,
initialGoExtractorValues,
+ initialGeneralChunkerValues,
initialParserValues,
initialTitleChunkerValues,
initialTokenChunkerValues,
@@ -26,6 +27,7 @@ import {
} from '@/pages/agent/constant/pipeline';
import {
transformExtractorParams,
+ transformGeneralChunkerParams,
transformParserParams,
transformTitleChunkerParams,
transformTokenChunkerParams,
@@ -252,6 +254,17 @@ function transformTokenChunkerConfigToForm(
return result;
}
+function transformGeneralChunkerConfigToForm(
+ config: Record<string, any> | undefined,
+): Record<string, any> {
+ const result = transformTokenChunkerConfigToForm(config);
+ result.table_context_size = Number(config?.table_context_size ?? 0);
+ result.image_context_size = Number(config?.image_context_size ?? 0);
+ delete result.image_table_context_window;
+ delete result.delimiter_mode;
+ return result;
+}
+
/**
* Converts TitleChunker config from API/DSL format to form format.
* DSL: { method: "hierarchy", hierarchy: "3", levels: [...], include_heading_content, root_chunk_as_heading }
@@ -321,6 +334,8 @@ export function transformApiConfigToForm(
return transformTokenizerConfigToForm(config);
case Operator.TokenChunker:
return transformTokenChunkerConfigToForm(config);
+ case Operator.GeneralChunker:
+ return transformGeneralChunkerConfigToForm(config);
case Operator.TitleChunker:
return transformTitleChunkerConfigToForm(config);
default:
@@ -384,6 +399,8 @@ export function transformFormConfigToApi(
return config; // passthrough for Tokenizer
case Operator.TokenChunker:
return transformTokenChunkerParams(config as any);
+ case Operator.GeneralChunker:
+ return transformGeneralChunkerParams(config as any);
case Operator.TitleChunker:
return transformTitleChunkerParams(config as any);
default:
@@ -417,6 +434,11 @@ export function normalizeOperatorForm(
...cloneDeep(initialTokenChunkerValues),
...rawForm,
};
+ case Operator.GeneralChunker:
+ return {
+ ...cloneDeep(initialGeneralChunkerValues),
+ ...rawForm,
+ };
case Operator.Extractor:
return {
...cloneDeep(getInitialExtractorValues()),