RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(ingestion): migrate general parser and chunker contract (#19611)
jay777213a04af1dcd91
diff --git a/internal/agent/canvas/scheduler.go b/internal/agent/canvas/scheduler.goindex b836357e70..65d4986ef6 100644--- a/internal/agent/canvas/scheduler.go+++ b/internal/agent/canvas/scheduler.go@@ -124,6 +124,8 @@ func isKnownPrimitive(name string) bool {"stringtransform", "variableaggregator", "variableassigner","loop", "parallel": // macros in BuildWorkflow; the pre-pass absorbs them.return true+ case "generalchunker":+ return true}return false}diff --git a/internal/agent/dsl/normalize.go b/internal/agent/dsl/normalize.goindex 9f6e184790..45db7f2990 100644--- a/internal/agent/dsl/normalize.go+++ b/internal/agent/dsl/normalize.go@@ -601,6 +601,7 @@ var componentNameToNodeTypeMap = map[string]string{"File": "fileNode","Parser": "parserNode","Tokenizer": "tokenizerNode",+ "GeneralChunker": "chunkerNode","TokenChunker": "chunkerNode","TitleChunker": "chunkerNode","OneChunker": "chunkerNode",diff --git a/internal/agent/dsl/normalize_test.go b/internal/agent/dsl/normalize_test.goindex 424cb595c8..79fb49a245 100644--- a/internal/agent/dsl/normalize_test.go+++ b/internal/agent/dsl/normalize_test.go@@ -48,6 +48,12 @@ func loadFixture(t *testing.T, name string) map[string]any {return m}+func TestComponentNameToNodeTypeRecognizesGeneralChunker(t *testing.T) {+ if got := componentNameToNodeType("GeneralChunker"); got != "chunkerNode" {+ t.Fatalf("GeneralChunker node type = %q, want chunkerNode", got)+ }+}+// TestNormalize_NoopWhenGraphPresent guards against accidentally// clobbering a payload the front-end just saved. Any input with a// non-empty `graph.nodes` must round-trip with `graph` untouched.diff --git a/internal/dao/migration.go b/internal/dao/migration.goindex a99af13081..559d4d2128 100644--- a/internal/dao/migration.go+++ b/internal/dao/migration.go@@ -68,6 +68,10 @@ func RunMigrations(ctx context.Context, db *gorm.DB) error {return fmt.Errorf("failed to add unique index on user_canvas (user_id, canvas_category, title): %w", err)}+ if err := migrateGeneralChunkerParserConfigs(ctx, db); err != nil {+ return fmt.Errorf("failed to migrate general chunker parser configs: %w", err)+ }+common.Info("All manual migrations completed successfully")return nil}diff --git a/internal/dao/parser_config_migration.go b/internal/dao/parser_config_migration.gonew file mode 100644index 0000000000..eb06bbd351--- /dev/null+++ b/internal/dao/parser_config_migration.go@@ -0,0 +1,168 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package dao++import (+ "context"+ "fmt"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ parserchunk "ragflow/internal/parser/chunk"++ "go.uber.org/zap"+ "gorm.io/gorm"+)++const (+ legacyGeneralChunkerID = "TokenChunker:SixApplesFall"+ currentGeneralChunkerID = "GeneralChunker:SixApplesFall"+)++var generalChunkerParamKeys = map[string]struct{}{+ "children_delimiters": {},+ "chunk_token_size": {},+ "delimiter": {},+ "delimiters": {},+ "image_context_size": {},+ "overlapped_percent": {},+ "table_context_size": {},+}++// migrateGeneralChunkerConfig rewrites the exact legacy component key used by+// the built-in general pipeline. Explicit values already stored under the new+// key win; unsupported TokenChunker-only fields are dropped because+// GeneralChunker has no delimiter_mode or outputs parameter.+func migrateGeneralChunkerConfig(config map[string]interface{}) bool {+ legacy, ok := config[legacyGeneralChunkerID]+ if !ok {+ return false+ }+ legacyParams, ok := legacy.(map[string]interface{})+ if !ok {+ return false+ }++ if _, exists := config[currentGeneralChunkerID]; !exists {+ params := make(map[string]interface{}, len(legacyParams))+ for key, value := range legacyParams {+ if key == "delimiter" {+ if _, exists := legacyParams["delimiters"]; !exists {+ if delimiters, ok := normalizeLegacyDelimiter(value); ok {+ params["delimiters"] = delimiters+ }+ }+ continue+ }+ if _, supported := generalChunkerParamKeys[key]; supported {+ params[key] = value+ }+ }+ config[currentGeneralChunkerID] = params+ }+ delete(config, legacyGeneralChunkerID)+ return true+}++func migrateGeneralChunkerParserConfigs(ctx context.Context, db *gorm.DB) error {+ if db == nil {+ return nil+ }++ var migratedKnowledgebases int+ if db.WithContext(ctx).Migrator().HasTable("knowledgebase") {+ count, err := migrateParserConfigRows(ctx, db, &entity.Knowledgebase{})+ if err != nil {+ return fmt.Errorf("migrate knowledgebase parser configs: %w", err)+ }+ migratedKnowledgebases = count+ }++ var migratedDocuments int+ if db.WithContext(ctx).Migrator().HasTable("document") {+ count, err := migrateParserConfigRows(ctx, db, &entity.Document{})+ if err != nil {+ return fmt.Errorf("migrate document parser configs: %w", err)+ }+ migratedDocuments = count+ }++ if migratedKnowledgebases > 0 || migratedDocuments > 0 {+ common.Info("Migrated general chunker parser configs",+ zap.Int("knowledgebases", migratedKnowledgebases),+ zap.Int("documents", migratedDocuments))+ }+ return nil+}++const parserConfigMigrationBatchSize = 256++type parserConfigMigrationRow struct {+ ID string `gorm:"column:id"`+ ParserConfig entity.JSONMap `gorm:"column:parser_config"`+}++func migrateParserConfigRows(ctx context.Context, db *gorm.DB, model any) (int, error) {+ var migrated int+ query := db.WithContext(ctx).+ Model(model).+ Select("id", "parser_config").+ Where("parser_id IN ? AND (pipeline_id IS NULL OR pipeline_id = '')", []string{"general", "naive"}).+ Order("id")+ var rows []parserConfigMigrationRow+ result := query.FindInBatches(&rows, parserConfigMigrationBatchSize, func(batchDB *gorm.DB, _ int) error {+ return batchDB.Transaction(func(tx *gorm.DB) error {+ for _, row := range rows {+ config := map[string]interface{}(row.ParserConfig)+ if !migrateGeneralChunkerConfig(config) {+ continue+ }+ if err := tx.Model(model).Where("id = ?", row.ID).+ Update("parser_config", entity.JSONMap(config)).Error; err != nil {+ return fmt.Errorf("update parser config %q: %w", row.ID, err)+ }+ migrated+++ }+ return nil+ })+ })+ if result.Error != nil {+ return migrated, result.Error+ }+ return migrated, nil+}++func normalizeLegacyDelimiter(value any) (any, bool) {+ switch value := value.(type) {+ case string:+ delimiters := parserchunk.ParseDelimiterField(value)+ out := make([]interface{}, len(delimiters))+ for i, delimiter := range delimiters {+ out[i] = delimiter+ }+ return out, true+ case []string:+ out := make([]interface{}, len(value))+ for i, delimiter := range value {+ out[i] = delimiter+ }+ return out, true+ case []interface{}:+ return value, true+ default:+ return nil, false+ }+}diff --git a/internal/dao/parser_config_migration_test.go b/internal/dao/parser_config_migration_test.gonew file mode 100644index 0000000000..da8b629751--- /dev/null+++ b/internal/dao/parser_config_migration_test.go@@ -0,0 +1,234 @@+package dao++import (+ "context"+ "reflect"+ "testing"++ "ragflow/internal/entity"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"+)++func TestMigrateGeneralChunkerConfigCopiesSupportedParams(t *testing.T) {+ config := map[string]interface{}{+ "TokenChunker:SixApplesFall": map[string]interface{}{+ "chunk_token_size": 768,+ "delimiters": []interface{}{"\n", ";"},+ "children_delimiters": []interface{}{"|"},+ "image_context_size": 4,+ "overlapped_percent": 0.2,+ "table_context_size": 3,+ "delimiter_mode": "delimiter",+ "outputs": map[string]interface{}{"chunks": map[string]interface{}{}},+ },+ }++ changed := migrateGeneralChunkerConfig(config)+ if !changed {+ t.Fatal("migrateGeneralChunkerConfig reported no change")+ }+ if _, ok := config[legacyGeneralChunkerID]; ok {+ t.Fatalf("legacy component id remains: %#v", config)+ }++ want := map[string]interface{}{+ "chunk_token_size": 768,+ "delimiters": []interface{}{"\n", ";"},+ "children_delimiters": []interface{}{"|"},+ "image_context_size": 4,+ "overlapped_percent": 0.2,+ "table_context_size": 3,+ }+ got, ok := config[currentGeneralChunkerID].(map[string]interface{})+ if !ok {+ t.Fatalf("new component params type = %T", config[currentGeneralChunkerID])+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("new component params = %#v, want %#v", got, want)+ }+}++func TestMigrateGeneralChunkerConfigKeepsExplicitNewParams(t *testing.T) {+ newParams := map[string]interface{}{"chunk_token_size": 256}+ config := map[string]interface{}{+ legacyGeneralChunkerID: map[string]interface{}{"chunk_token_size": 768},+ currentGeneralChunkerID: newParams,+ }++ if !migrateGeneralChunkerConfig(config) {+ t.Fatal("migrateGeneralChunkerConfig reported no change")+ }+ if got := config[currentGeneralChunkerID]; !reflect.DeepEqual(got, newParams) {+ t.Fatalf("explicit new params changed: %#v", got)+ }+}++func TestMigrateGeneralChunkerConfigNormalizesSingularDelimiter(t *testing.T) {+ config := map[string]interface{}{+ legacyGeneralChunkerID: map[string]interface{}{+ "delimiter": "\n!?;。;!?",+ },+ }++ if !migrateGeneralChunkerConfig(config) {+ t.Fatal("migrateGeneralChunkerConfig reported no change")+ }+ params, ok := config[currentGeneralChunkerID].(map[string]interface{})+ if !ok {+ t.Fatalf("new component params type = %T", config[currentGeneralChunkerID])+ }+ want := []interface{}{"\n", "!", "?", ";", "。", ";", "!", "?"}+ if !reflect.DeepEqual(params["delimiters"], want) {+ t.Fatalf("migrated delimiters = %#v, want %#v", params["delimiters"], want)+ }+ if _, ok := params["delimiter"]; ok {+ t.Fatalf("singular delimiter leaked into new params: %#v", params)+ }+}++func TestMigrateGeneralChunkerConfigIgnoresOtherChunkers(t *testing.T) {+ config := map[string]interface{}{+ "TokenChunker:WarmBreadSmells": map[string]interface{}{"chunk_token_size": 128},+ }++ if migrateGeneralChunkerConfig(config) {+ t.Fatal("migration changed a non-general component")+ }+ if _, ok := config["TokenChunker:WarmBreadSmells"]; !ok {+ t.Fatal("non-general component was removed")+ }+}++func TestMigrateGeneralChunkerParserConfigsUpdatesBuiltinRows(t *testing.T) {+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err := db.AutoMigrate(&entity.Knowledgebase{}, &entity.Document{}); err != nil {+ t.Fatalf("automigrate: %v", err)+ }++ legacyParams := map[string]interface{}{+ "chunk_token_size": 256,+ "delimiter_mode": "delimiter",+ "outputs": map[string]interface{}{},+ }+ kb := &entity.Knowledgebase{+ ID: "kb-migrate",+ TenantID: "tenant-migrate",+ Name: "migration",+ EmbdID: "embedding",+ CreatedBy: "user-migrate",+ ParserID: "general",+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},+ }+ if err := db.Create(kb).Error; err != nil {+ t.Fatalf("create knowledgebase: %v", err)+ }+ doc := &entity.Document{+ ID: "doc-migrate",+ KbID: kb.ID,+ ParserID: "naive",+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},+ SourceType: "local",+ Type: "file",+ CreatedBy: "user-migrate",+ Suffix: "txt",+ }+ if err := db.Create(doc).Error; err != nil {+ t.Fatalf("create document: %v", err)+ }+ emptyPipelineID := ""+ emptyPipelineKB := &entity.Knowledgebase{+ ID: "kb-empty-pipeline",+ TenantID: "tenant-migrate",+ Name: "empty-pipeline",+ EmbdID: "embedding",+ CreatedBy: "user-migrate",+ ParserID: "general",+ PipelineID: &emptyPipelineID,+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},+ }+ if err := db.Create(emptyPipelineKB).Error; err != nil {+ t.Fatalf("create empty-pipeline knowledgebase: %v", err)+ }+ emptyPipelineDoc := &entity.Document{+ ID: "doc-empty-pipeline",+ KbID: kb.ID,+ ParserID: "naive",+ PipelineID: &emptyPipelineID,+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},+ SourceType: "local",+ Type: "file",+ CreatedBy: "user-migrate",+ Suffix: "txt",+ }+ if err := db.Create(emptyPipelineDoc).Error; err != nil {+ t.Fatalf("create empty-pipeline document: %v", err)+ }+ customPipelineID := "canvas-migrate"+ customKB := &entity.Knowledgebase{+ ID: "kb-custom",+ TenantID: "tenant-migrate",+ Name: "custom",+ EmbdID: "embedding",+ CreatedBy: "user-migrate",+ ParserID: "naive",+ PipelineID: &customPipelineID,+ ParserConfig: entity.JSONMap{legacyGeneralChunkerID: legacyParams},+ }+ if err := db.Create(customKB).Error; err != nil {+ t.Fatalf("create custom knowledgebase: %v", err)+ }++ if err := migrateGeneralChunkerParserConfigs(context.Background(), db); err != nil {+ t.Fatalf("migrate parser configs: %v", err)+ }++ var gotKB entity.Knowledgebase+ if err := db.First(&gotKB, "id = ?", kb.ID).Error; err != nil {+ t.Fatalf("reload knowledgebase: %v", err)+ }+ var gotDoc entity.Document+ if err := db.First(&gotDoc, "id = ?", doc.ID).Error; err != nil {+ t.Fatalf("reload document: %v", err)+ }+ var gotEmptyPipelineKB entity.Knowledgebase+ if err := db.First(&gotEmptyPipelineKB, "id = ?", emptyPipelineKB.ID).Error; err != nil {+ t.Fatalf("reload empty-pipeline knowledgebase: %v", err)+ }+ var gotEmptyPipelineDoc entity.Document+ if err := db.First(&gotEmptyPipelineDoc, "id = ?", emptyPipelineDoc.ID).Error; err != nil {+ t.Fatalf("reload empty-pipeline document: %v", err)+ }+ for name, config := range map[string]entity.JSONMap{+ "knowledgebase": gotKB.ParserConfig,+ "document": gotDoc.ParserConfig,+ "empty-pipeline knowledgebase": gotEmptyPipelineKB.ParserConfig,+ "empty-pipeline document": gotEmptyPipelineDoc.ParserConfig,+ } {+ if _, ok := config[legacyGeneralChunkerID]; ok {+ t.Errorf("%s retained legacy component id: %#v", name, config)+ }+ params, ok := config[currentGeneralChunkerID].(map[string]interface{})+ if !ok {+ t.Errorf("%s new component params type = %T", name, config[currentGeneralChunkerID])+ continue+ }+ if params["chunk_token_size"] != float64(256) && params["chunk_token_size"] != 256 {+ t.Errorf("%s migrated params = %#v", name, params)+ }+ if _, ok := params["delimiter_mode"]; ok {+ t.Errorf("%s retained delimiter_mode: %#v", name, params)+ }+ }+ var gotCustomKB entity.Knowledgebase+ if err := db.First(&gotCustomKB, "id = ?", customKB.ID).Error; err != nil {+ t.Fatalf("reload custom knowledgebase: %v", err)+ }+ if _, ok := gotCustomKB.ParserConfig[legacyGeneralChunkerID]; !ok {+ t.Fatalf("custom pipeline parser config was unexpectedly migrated: %#v", gotCustomKB.ParserConfig)+ }+}diff --git a/internal/handler/components_testpkg/components_handler_test.go b/internal/handler/components_testpkg/components_handler_test.goindex c22f0de118..d12d1aa24f 100644--- a/internal/handler/components_testpkg/components_handler_test.go+++ b/internal/handler/components_testpkg/components_handler_test.go@@ -134,7 +134,7 @@ func TestComponentsHandler_NoFilter(t *testing.T) {// TestComponentsHandler_FilterIngestion verifies the// ?category=ingestion filter returns the ingestion components-// (Compiler, Extractor, File, Parser, Tokenizer + 9 chunker variants).+// (Compiler, Extractor, File, Parser, Tokenizer + 10 chunker variants).// Names must be sorted ascending (plan §4 task 1 stable output).func TestComponentsHandler_FilterIngestion(t *testing.T) {eng := newComponentsTestRig(t)@@ -146,7 +146,7 @@ func TestComponentsHandler_FilterIngestion(t *testing.T) {_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",+ "compiler", "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "pagechunker", "qachunker", "parser", "tablechunker","titlechunker", "tokenchunker", "tokenizer",}@@ -172,7 +172,7 @@ func TestComponentsHandler_FilterMultiple(t *testing.T) {_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",+ "compiler", "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "pagechunker", "qachunker", "parser", "tablechunker","titlechunker", "tokenchunker", "tokenizer",}@@ -273,7 +273,7 @@ func TestComponentsHandler_CaseInsensitive(t *testing.T) {}_, _, data := decodeEnvelope(t, w.Body.Bytes())wantNames := []string{- "compiler", "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",+ "compiler", "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "pagechunker", "qachunker", "parser", "tablechunker","titlechunker", "tokenchunker", "tokenizer",}diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 1beccc0cb4..cbdfe9fae0 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -59,6 +59,8 @@ func newChunkerByName(name string, params map[string]any) (runtime.Component, erreturn NewTableChunker(params)case ComponentNamePageChunker:return NewPageChunker(params)+ case ComponentNameGeneralChunker:+ return NewGeneralChunker(params)default:return nil, fmt.Errorf("chunker: unknown component %q", name)}@@ -89,7 +91,9 @@ func stringListFromAny(in []any) []string {// content. invokeTextPayload decides whether an active delimiter yields one// chunk per segment (custom/backtick, no merge) or splits into paragraphs that// are merged by token size (bare). Canonical single-string parser_config.delimiter-// parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField).+// Legacy single-string parsing is performed by GeneralChunker at its+// configuration boundary; the shared regex helper only consumes canonical+// delimiter lists.func compileDelimPattern(delims []string) *regexp.Regexp {return chunk.CompileDelimiterPatternList(delims, true)}@@ -98,9 +102,10 @@ func compileDelimPattern(delims []string) *regexp.Regexp {// (token_chunker.py:79-90). The captured delimiter is DISCARDED rather than// glued to a segment: re.split with a captured group keeps delimiters at odd// indices, and only the even-index (text) parts are kept. This is the-// behavior every delimiter path (primary and children, text/markdown/html-// and JSON) must reproduce so a split chunk reads "first sentence here"-// without the trailing delimiter.+// behavior shared TokenChunker paths and General's primary/Markdown splits+// reproduce so a split chunk reads "first sentence here" without the trailing+// delimiter. General's legacy-compatible children split is implemented+// separately because that path keeps the delimiter attached to its parent.func splitDroppingDelim(text string, pattern *regexp.Regexp) []string {if pattern == nil {return []string{text}diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.gonew file mode 100644index 0000000000..ed800d4f83--- /dev/null+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go@@ -0,0 +1,182 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser"+)++func TestSplitTextParserSentencesKeepsSentenceUnits(t *testing.T) {+ item := schema.ChunkDoc{Text: "First!Second?第三。Fourth;", DocType: "text", CKType: "text"}+ parts := splitTextParserSentences(item)+ if got, want := len(parts), 4; got != want {+ t.Fatalf("sentence parts = %d, want %d: %#v", got, want, parts)+ }+ want := []string{"First!", "Second?", "第三。", "Fourth;"}+ for i, part := range parts {+ if part.Text != want[i] {+ t.Errorf("part[%d] = %q, want %q", i, part.Text, want[i])+ }+ }+}++func TestTokenChunkerTextParserSentenceFallbackEndToEnd(t *testing.T) {+ const source = "First!Second?第三。Fourth;"+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "sample.txt", []byte(source))+ if parsed.Err != nil {+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)+ }+ if len(parsed.JSON) != 1 {+ t.Fatalf("TextParser JSON items = %d, want one parser item", len(parsed.JSON))+ }++ chunker, err := NewTokenChunker(map[string]any{"chunk_token_size": 3})+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := chunker.Invoke(context.Background(), nil, map[string]any{+ "name": "sample.txt",+ "file_type": "txt",+ "output_format": "json",+ "json": parsed.JSON,+ })+ if err != nil {+ t.Fatalf("TokenChunker.Invoke: %v", err)+ }+ chunks := chunksFromOutput(t, out)+ want := []string{"First!", "Second?", "第三。", "Fourth;"}+ if len(chunks) != len(want) {+ t.Fatalf("chunks = %#v, want one chunk per sentence", chunks)+ }+ for i, chunk := range chunks {+ if got, wantText := chunk["text"], want[i]; got != wantText {+ t.Errorf("chunk[%d].text = %v, want %q", i, got, wantText)+ }+ }+}++func TestGeneralChunkerTextAppliesOverlapThroughInvoke(t *testing.T) {+ chunker, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 2,+ "overlapped_percent": 50,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{+ "name": "document.txt",+ "file_type": "txt",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "alpha beta", "doc_type_kwd": "text"},+ {"text": "gamma delta", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ texts := outputTexts(t, out)+ if len(texts) != 2 {+ t.Fatalf("texts = %q, want two overlapped chunks", texts)+ }+ if texts[1] == "gamma delta" || !strings.HasSuffix(texts[1], "\ngamma delta") {+ t.Fatalf("text overlap = %q, want previous tail plus current text", texts[1])+ }+}++func TestGeneralChunkerEmptyJSONReturnsNoChunksForEachStrategy(t *testing.T) {+ for _, fileType := range []string{"txt", "md", "docx", "pdf", "xlsx"} {+ t.Run(fileType, func(t *testing.T) {+ chunker, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{+ "name": "empty." + fileType,+ "file_type": fileType,+ "output_format": "json",+ "json": []map[string]any{},+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ if chunks := outputChunks(t, out); len(chunks) != 0 {+ t.Fatalf("chunks = %#v, want empty output", chunks)+ }+ })+ }+}++func TestGeneralChunkerMarkdownLongHeadingKeepsFollowingTableAtomic(t *testing.T) {+ chunker, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ longHeading := "# " + strings.Repeat("heading ", 60)+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},+ },+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want separate heading and table chunks", chunks)+ }+ if chunks[0]["ck_type"] != "heading" || chunks[0]["text"] != strings.TrimSpace(longHeading) {+ t.Errorf("heading chunk = %#v", chunks[0])+ }+ if chunks[1]["ck_type"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {+ t.Errorf("table chunk = %#v", chunks[1])+ }+}++func TestGeneralChunkerKeepsHeaderOnlySheetsSeparate(t *testing.T) {+ chunker, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{+ "name": "empty-sheets.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "Name", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},+ {"text": "Amount", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2, "table_id": "sheet-2"},+ },+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want one header-only chunk per sheet", chunks)+ }+ if chunks[0]["text"] != "Name" || chunks[1]["text"] != "Amount" {+ t.Fatalf("header-only chunks = %#v", chunks)+ }+}diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.gonew file mode 100644index 0000000000..d945232c75--- /dev/null+++ b/internal/ingestion/component/chunker/general.go@@ -0,0 +1,1191 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "bytes"+ "context"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "image"+ "image/draw"+ _ "image/gif"+ _ "image/jpeg"+ "image/png"+ "log/slog"+ "path/filepath"+ "regexp"+ "sort"+ "strings"++ "gorm.io/gorm"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/chunk"+)++const ComponentNameGeneralChunker = "GeneralChunker"++var generalSentencePattern = regexp.MustCompile(`([。!??;!\n]|\. )`)++type generalChunkerParam struct {+ ChunkTokenSize int+ Delimiters []string+ OverlappedPercent float64+ ChildrenDelimiters []string+ TableContextSize int+ ImageContextSize int+}++func (generalChunkerParam) Defaults() generalChunkerParam {+ return generalChunkerParam{+ ChunkTokenSize: 512,+ Delimiters: []string{"\n"},+ ChildrenDelimiters: []string{},+ }+}++func (p *generalChunkerParam) Update(conf map[string]any) {+ if value, ok := schema.NumericFromAny(conf["chunk_token_size"]); ok {+ p.ChunkTokenSize = int(value)+ }+ if value, ok := conf["delimiters"]; ok {+ if delimiters, recognized := normalizeGeneralDelimiterValue(value); recognized {+ p.Delimiters = delimiters+ }+ } else if value, ok := conf["delimiter"]; ok {+ if delimiters, recognized := normalizeGeneralDelimiterValue(value); recognized {+ p.Delimiters = delimiters+ }+ }+ if value, ok := conf["overlapped_percent"]; ok {+ p.OverlappedPercent = schema.NormalizeOverlappedPercent(value)+ }+ if value, ok := conf["children_delimiters"]; ok {+ if delimiters, recognized := normalizeGeneralDelimiterValue(value); recognized {+ p.ChildrenDelimiters = delimiters+ }+ }+ if value, ok := schema.NumericFromAny(conf["table_context_size"]); ok {+ p.TableContextSize = max(0, int(value))+ }+ if value, ok := schema.NumericFromAny(conf["image_context_size"]); ok {+ p.ImageContextSize = max(0, int(value))+ }+}++// normalizeGeneralDelimiterValue accepts both the canonical list form and+// the legacy Python single-string form. A legacy string is split into Unicode+// rune delimiters, except backtick-wrapped tokens which remain one delimiter.+func normalizeGeneralDelimiterValue(value any) ([]string, bool) {+ switch value := value.(type) {+ case string:+ return splitGeneralDelimiterString(value), true+ case []string:+ return append([]string(nil), value...), true+ case []any:+ return stringListFromAny(value), true+ default:+ return nil, false+ }+}++func splitGeneralDelimiterString(value string) []string {+ return chunk.ParseDelimiterField(value)+}++// GeneralChunkerComponent owns the general ingestion chunking policy and+// selects its format-specific strategy from the Parser's canonical file_type.+type GeneralChunkerComponent struct {+ param generalChunkerParam+}++func NewGeneralChunker(params map[string]any) (runtime.Component, error) {+ param := generalChunkerParam{}.Defaults()+ param.Update(params)+ if err := param.Validate(); err != nil {+ return nil, err+ }+ return &GeneralChunkerComponent{param: param}, nil+}++func (p generalChunkerParam) Validate() error {+ if p.ChunkTokenSize < 0 {+ return fmt.Errorf("chunk_token_size must be non-negative")+ }+ return nil+}++func (c *GeneralChunkerComponent) Inputs() map[string]string { return ChunkerInputs }++func (c *GeneralChunkerComponent) Outputs() map[string]string { return ChunkerOutputs }++func (c *GeneralChunkerComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {+ upstream, err := decodeChunkerFromUpstream(inputs)+ if err != nil {+ return nil, fmt.Errorf("GeneralChunker: decode input: %w", err)+ }+ fileType := strings.ToLower(strings.TrimSpace(upstream.FileType))+ if fileType == "" {+ fileType = inferGeneralFileType(upstream)+ if fileType == "" {+ return nil, fmt.Errorf("GeneralChunker: file_type is required when source name has no extension")+ }+ slog.Warn("GeneralChunker: missing parser file_type; inferred from source name", "name", upstream.Name, "file_type", fileType)+ }++ strategy := generalStrategyForFileType(fileType)+ if strategy == generalStrategyText && !isKnownGeneralFileType(fileType) {+ slog.Debug("GeneralChunker: unknown file_type; using text fallback", "file_type", fileType)+ }++ switch strategy {+ case generalStrategyPDF:+ return c.chunkPDF(ctx, db, upstream)+ case generalStrategyDOCX:+ return c.chunkDOCX(ctx, upstream)+ case generalStrategyMarkdown:+ return c.chunkMarkdown(ctx, upstream)+ case generalStrategySpreadsheet:+ return c.chunkSpreadsheet(ctx, upstream)+ default:+ return c.chunkGeneral(ctx, upstream)+ }+}++type generalStrategy uint8++const (+ generalStrategyText generalStrategy = iota+ generalStrategyPDF+ generalStrategyDOCX+ generalStrategyMarkdown+ generalStrategySpreadsheet+)++func generalStrategyForFileType(fileType string) generalStrategy {+ switch strings.ToLower(strings.TrimSpace(fileType)) {+ case "pdf":+ return generalStrategyPDF+ case "docx":+ return generalStrategyDOCX+ case "md":+ return generalStrategyMarkdown+ case "xls", "xlsx", "csv":+ return generalStrategySpreadsheet+ default:+ return generalStrategyText+ }+}++func inferGeneralFileType(upstream schema.ChunkerFromUpstream) string {+ name := strings.TrimSpace(upstream.Name)+ if name == "" && upstream.File != nil {+ name = strings.TrimSpace(upstream.File.Name)+ }+ ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(name)), ".")+ return ext+}++func isKnownGeneralFileType(fileType string) bool {+ switch strings.ToLower(strings.TrimSpace(fileType)) {+ case "pdf", "doc", "docx", "ppt", "pptx", "xls", "xlsx", "csv",+ "html", "md", "txt", "epub", "json", "video", "email", "visual",+ "aural", "folder", "other":+ return true+ default:+ return false+ }+}++func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upstream schema.ChunkerFromUpstream) (map[string]any, error) {+ if err := ctx.Err(); err != nil {+ return nil, fmt.Errorf("GeneralChunker: %w", err)+ }+ units := upstream.JSONResult+ if upstream.OutputFormat == schema.PayloadFormatChunks {+ units = upstream.Chunks+ }+ if len(units) == 0 {+ return emptyOutputs(), nil+ }++ primaryPattern := compileDelimPattern(c.param.Delimiters)+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ units = splitGeneralUnits(units, primaryPattern)+ units = sortPDFUnits(units)+ if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {+ slog.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")+ }+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)++ chunks := make([]schema.ChunkDoc, 0, len(units))+ body := make([]schema.ChunkDoc, 0, len(units))+ flushBody := func() {+ if len(body) == 0 {+ return+ }+ chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")...)+ body = body[:0]+ }+ for _, unit := range units {+ if itemDocType(unit) == "text" {+ body = append(body, unit)+ } else {+ flushBody()+ chunks = append(chunks, cloneChunkDoc(unit))+ }+ }+ flushBody()++ engine, err := newPDFEngineFromUpstream(ctx, db, upstream)+ if err != nil {+ slog.Warn("GeneralChunker: could not open PDF for on-demand cropping", "err", err)+ }+ if engine != nil {+ defer engine.Close()+ chunks = cropImageChunks(ctx, engine, chunks)+ }+ chunks = finalizeGeneralChunks(chunks, childrenPattern)+ if err := ctx.Err(); err != nil {+ return nil, fmt.Errorf("GeneralChunker: %w", err)+ }+ if len(chunks) == 0 {+ return emptyOutputs(), nil+ }+ attachPDFOutline(chunks, upstream.File)+ return chunkOutputs(chunks), nil+}++// sortPDFUnits orders each contiguous positioned run by physical reading+// order. Coordinate-free units are barriers: they stay at their parser+// positions and preserve their relative order instead of being moved to the+// end of the document.+func sortPDFUnits(units []schema.ChunkDoc) []schema.ChunkDoc {+ type positionedUnit struct {+ unit schema.ChunkDoc+ row []float64+ positioned bool+ }+ ordered := make([]positionedUnit, 0, len(units))+ for _, unit := range units {+ row, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions})+ ordered = append(ordered, positionedUnit{unit: cloneChunkDoc(unit), row: row, positioned: ok})+ }+ runStart := -1+ flush := func(end int) {+ if runStart < 0 {+ return+ }+ sort.SliceStable(ordered[runStart:end], func(i, j int) bool {+ return pdfPosRowLess(ordered[runStart+i].row, ordered[runStart+j].row)+ })+ runStart = -1+ }+ for i := 0; i <= len(ordered); i++ {+ positioned := i < len(ordered) && ordered[i].positioned+ if positioned {+ if runStart < 0 {+ runStart = i+ }+ continue+ }+ flush(i)+ }+ result := make([]schema.ChunkDoc, 0, len(ordered))+ for _, item := range ordered {+ result = append(result, item.unit)+ }+ return result+}++func hasPDFPositions(units []schema.ChunkDoc) bool {+ for _, unit := range units {+ if _, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions}); ok {+ return true+ }+ }+ return false+}++func hasUnpositionedPDFMedia(units []schema.ChunkDoc) bool {+ for _, unit := range units {+ if itemDocType(unit) == "text" {+ continue+ }+ if _, ok := firstPositionRow(lineRecord{pdfPositions: unit.PDFPositions, positions: unit.Positions}); !ok {+ return true+ }+ }+ return false+}++func attachPDFOutline(chunks []schema.ChunkDoc, file *schema.ChunkerFileMeta) {+ if len(chunks) == 0 || file == nil || len(file.Extra) == 0 {+ return+ }+ raw, ok := file.Extra["outline"]+ if !ok || len(raw) == 0 {+ return+ }+ var input []map[string]any+ if err := json.Unmarshal(raw, &input); err != nil || len(input) == 0 {+ return+ }+ outline := make([]map[string]any, 0, len(input))+ for _, entry := range input {+ title, _ := entry["title"].(string)+ if title == "" {+ continue+ }+ depth := entry["level"]+ if depth == nil {+ depth = entry["depth"]+ }+ outline = append(outline, map[string]any{"title": title, "depth": depth})+ }+ if len(outline) == 0 {+ return+ }+ encoded, err := json.Marshal(outline)+ if err != nil {+ return+ }+ if chunks[0].Extra == nil {+ chunks[0].Extra = make(map[string]json.RawMessage)+ }+ chunks[0].Extra["__outline__"] = encoded+}++func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {+ if err := ctx.Err(); err != nil {+ return nil, fmt.Errorf("GeneralChunker: %w", err)+ }+ units := upstream.JSONResult+ if upstream.OutputFormat == schema.PayloadFormatChunks {+ units = upstream.Chunks+ }+ if len(units) == 0 {+ return emptyOutputs(), nil+ }+ primaryPattern := compileDelimPattern(c.param.Delimiters)+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ units = splitGeneralUnits(units, primaryPattern)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ units = mergeDOCXUnits(units, c.param.ChunkTokenSize, hasCustomDelim(c.param.Delimiters), "\n")+ units = applyGeneralOverlap(units, c.param.OverlappedPercent, "\n")+ units = finalizeGeneralChunks(units, childrenPattern)+ if len(units) == 0 {+ return emptyOutputs(), nil+ }+ return chunkOutputs(units), nil+}++// mergeDOCXUnits mirrors Python naive_merge_docx: text units keep the+// previous text merge target across intervening table/image units. This means+// a later paragraph can extend an earlier text chunk while the media item+// remains in its original output position.+func mergeDOCXUnits(units []schema.ChunkDoc, target int, customDelimiter bool, joinSep string) []schema.ChunkDoc {+ merged := make([]schema.ChunkDoc, 0, len(units))+ previousText := -1+ for _, unit := range units {+ if itemDocType(unit) != "text" {+ media := cloneChunkDoc(unit)+ media.DocType = itemDocType(media)+ media.CKType = media.DocType+ merged = append(merged, media)+ continue+ }++ text := cloneChunkDoc(unit)+ text.DocType = "text"+ text.CKType = "text"+ text.TKNums = intPtr(generalUnitTokens(text))+ if previousText < 0 || customDelimiter || intValue(merged[previousText].TKNums) >= target {+ merged = append(merged, text)+ previousText = len(merged) - 1+ continue+ }+ mergeGeneralChunk(&merged[previousText], text, joinSep)+ merged[previousText].DocType = "text"+ merged[previousText].CKType = "text"+ }+ return merged+}++// attachGeneralMediaContext follows Python's sentence-aware media context+// extraction. The shared TokenChunker helper intentionally returns the+// smallest token-fitting rune suffix/prefix; DOCX/PDF general strategies use+// complete sentence units instead.+func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int) {+ for i := range units {+ if units[i].CKType != "table" && units[i].CKType != "image" {+ continue+ }+ budget := imageTokens+ if units[i].CKType == "table" {+ budget = tableTokens+ }+ if budget <= 0 {+ continue+ }+ units[i].ContextAbove = collectGeneralMediaContext(units, i, budget, true)+ units[i].ContextBelow = collectGeneralMediaContext(units, i, budget, false)+ }+}++func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, above bool) string {+ var parts []string+ remaining := budget+ step := -1+ if !above {+ step = 1+ }+ for cursor := index + step; cursor >= 0 && cursor < len(units) && remaining > 0; cursor += step {+ if generalMediaContextCrossesSpreadsheetBoundary(units[index], units[cursor]) {+ break+ }+ if itemDocType(units[cursor]) != "text" {+ continue+ }+ text := units[cursor].Text+ tokens := generalUnitTokens(units[cursor])+ if tokens >= remaining {+ piece := takeGeneralContextSentence(text, remaining, above)+ if above {+ parts = append([]string{piece}, parts...)+ } else {+ parts = append(parts, piece)+ }+ break+ }+ if above {+ parts = append([]string{text}, parts...)+ } else {+ parts = append(parts, text)+ }+ remaining -= tokens+ }+ return strings.Join(parts, "\n")+}++func generalMediaContextCrossesSpreadsheetBoundary(media, candidate schema.ChunkDoc) bool {+ if !hasSpreadsheetIdentity(media) && !hasSpreadsheetIdentity(candidate) {+ return false+ }+ return !sameSpreadsheetTable(media, candidate)+}++func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {+ return doc.TableID != "" || doc.Sheet != "" || doc.SheetIndex != nil+}++func takeGeneralContextSentence(text string, budget int, fromEnd bool) string {+ sentences := splitGeneralContextSentences(text)+ if len(sentences) == 0 {+ return text+ }+ // The sentence pieces are contiguous slices of text. Track byte offsets+ // instead of repeatedly prepending slices and joining the selected text;+ // each candidate is still tokenized independently because BPE token counts+ // are not additive across sentence boundaries.+ if fromEnd {+ start := len(text)+ for i := len(sentences) - 1; i >= 0; i-- {+ start -= len(sentences[i])+ if tokenizeStr(text[start:]) >= budget {+ break+ }+ }+ return text[start:]+ }++ end := 0+ for _, sentence := range sentences {+ end += len(sentence)+ if tokenizeStr(text[:end]) >= budget {+ break+ }+ }+ return text[:end]+}++func splitGeneralContextSentences(text string) []string {+ indices := generalSentencePattern.FindAllStringIndex(text, -1)+ if len(indices) == 0 {+ if text == "" {+ return nil+ }+ return []string{text}+ }+ parts := make([]string, 0, len(indices)+1)+ start := 0+ for _, index := range indices {+ parts = append(parts, text[start:index[1]])+ start = index[1]+ }+ if start < len(text) {+ parts = append(parts, text[start:])+ }+ return parts+}++func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {+ if err := ctx.Err(); err != nil {+ return nil, fmt.Errorf("GeneralChunker: %w", err)+ }+ units := upstream.JSONResult+ if upstream.OutputFormat == schema.PayloadFormatChunks {+ units = upstream.Chunks+ }+ if len(units) == 0 {+ return emptyOutputs(), nil+ }+ primaryPattern := compileDelimPattern(c.param.Delimiters)+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ units = splitMarkdownUnits(units, primaryPattern)+ units = mergeMarkdownUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")+ units = finalizeGeneralChunks(units, childrenPattern)+ if len(units) == 0 {+ return emptyOutputs(), nil+ }+ return chunkOutputs(units), nil+}++// splitMarkdownUnits expands only text units while retaining the semantic+// parser type on every piece. Markdown heading detection depends on ck_type,+// so the generic splitter's type normalization cannot be used here.+func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {+ result := make([]schema.ChunkDoc, 0, len(units))+ for _, unit := range units {+ unit = cloneChunkDoc(unit)+ unit.Text = strings.TrimSpace(normalizeGeneralNewlines(itemTextOrFallback(unit)))+ unit.DocType = itemDocType(unit)+ if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {+ unit.TKNums = intPtr(tokenizeStr(unit.Text))+ result = append(result, unit)+ continue+ }+ for _, part := range splitDroppingDelim(unit.Text, pattern) {+ part = strings.TrimSpace(part)+ if part == "" {+ continue+ }+ piece := cloneChunkDoc(unit)+ piece.Text = part+ piece.TKNums = intPtr(tokenizeStr(part))+ result = append(result, piece)+ }+ }+ return result+}++// mergeMarkdownUnits mirrors the Markdown branch of Python naive.chunk:+// ordinary units use a projected token cap, while a short heading is always+// kept with the following unit. Markdown images are block attachments rather+// than standalone media chunks, so image-bearing units participate in the+// text merge and retain their image payload.+func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {+ overlapPct = max(0, min(100, overlapPct))+ merged := make([]schema.ChunkDoc, 0, len(units))+ current := -1+ currentTokens := 0+ for _, unit := range units {+ if itemDocType(unit) == "table" {+ if current >= 0 && isShortMarkdownHeading(merged[current]) {+ mergedImage, imagesOK := mergeMarkdownImagesChecked(merged[current].Image, unit.Image)+ if !imagesOK {+ merged = append(merged, cloneChunkDoc(unit))+ current = -1+ currentTokens = 0+ continue+ }+ table := cloneChunkDoc(unit)+ heading := merged[current]+ if heading.Text != "" && table.Text != "" {+ table.Text = heading.Text + joinSep + table.Text+ } else {+ table.Text = heading.Text + table.Text+ }+ table.TKNums = intPtr(currentTokens + generalUnitTokens(unit))+ table.PDFPositions = mergeGeneralPositions(heading.PDFPositions, table.PDFPositions)+ table.Positions = mergeGeneralPositions(heading.Positions, table.Positions)+ mergeGeneralMetadata(&table, heading)+ table.Image = mergedImage+ table.DocType = "table"+ table.CKType = "table"+ merged[current] = table+ current = -1+ currentTokens = 0+ continue+ }+ merged = append(merged, cloneChunkDoc(unit))+ current = -1+ currentTokens = 0+ continue+ }++ unit = cloneChunkDoc(unit)+ unit.DocType = "text"+ if unit.CKType == "" {+ unit.CKType = "text"+ }+ unit.TKNums = intPtr(generalUnitTokens(unit))+ if current < 0 {+ merged = append(merged, unit)+ current = len(merged) - 1+ currentTokens = generalUnitTokens(unit)+ continue+ }++ previous := &merged[current]+ unitTokens := generalUnitTokens(unit)+ if unit.CKType == "heading" && previous.CKType != "heading" {+ merged = append(merged, unit)+ current = len(merged) - 1+ currentTokens = unitTokens+ continue+ }+ forceMerge := isShortMarkdownHeading(*previous)+ mergedImage, imagesOK := mergeMarkdownImagesChecked(previous.Image, unit.Image)+ if !imagesOK {+ merged = append(merged, unit)+ current = len(merged) - 1+ currentTokens = unitTokens+ continue+ }+ projected := currentTokens + unitTokens+ if !forceMerge && projected > target {+ overlap, _ := computeOverlapPrefix(previous.Text, overlapPct)+ if overlap != "" {+ // The emitted chunk belongs to the current unit. Clone it+ // first so media and other metadata from the previous chunk+ // cannot leak through the overlap prefix.+ next := cloneChunkDoc(unit)+ next.Text = joinGeneralOverlapText(overlap, unit.Text, joinSep)+ currentTokens = tokenizeStr(overlap) + unitTokens+ next.TKNums = intPtr(currentTokens)+ next.PDFPositions = mergeGeneralPositions(previous.PDFPositions, next.PDFPositions)+ next.Positions = mergeGeneralPositions(previous.Positions, next.Positions)+ merged = append(merged, next)+ } else {+ merged = append(merged, unit)+ currentTokens = unitTokens+ }+ current = len(merged) - 1+ continue+ }+ mergeMarkdownChunk(previous, unit, joinSep)+ previous.Image = mergedImage+ previous.DocType = "text"+ previous.CKType = "text"+ currentTokens = projected+ previous.TKNums = intPtr(currentTokens)+ }+ return merged+}++func isShortMarkdownHeading(unit schema.ChunkDoc) bool {+ if unit.CKType != "heading" {+ return false+ }+ return tokenizeStr(markdownHeadingContent(unit.Text)) < 50+}++func markdownHeadingContent(text string) string {+ text = strings.TrimSpace(text)+ if len(text) == 0 || text[0] != '#' {+ return text+ }+ level := 0+ for level < len(text) && text[level] == '#' {+ level+++ }+ if level == 0 || level > 6 || level == len(text) || text[level] != ' ' {+ return text+ }+ return strings.TrimSpace(text[level:])+}++func mergeMarkdownChunk(dst *schema.ChunkDoc, src schema.ChunkDoc, joinSep string) {+ mergeGeneralChunk(dst, src, joinSep)+}++const (+ maxMarkdownImageDimension = 8192+ maxMarkdownImagePixels = 32 * 1024 * 1024+)++func mergeMarkdownImagesChecked(first, second string) (string, bool) {+ if first == "" {+ return second, true+ }+ if second == "" || first == second {+ return first, true+ }+ firstImage, firstOK := decodeMarkdownImage(first)+ secondImage, secondOK := decodeMarkdownImage(second)+ if !firstOK || !secondOK {+ return first, false+ }+ width := firstImage.Bounds().Dx()+ if secondImage.Bounds().Dx() > width {+ width = secondImage.Bounds().Dx()+ }+ height := firstImage.Bounds().Dy() + secondImage.Bounds().Dy()+ if !markdownImageWithinLimits(width, height) {+ return first, false+ }+ canvas := image.NewRGBA(image.Rect(0, 0, width, height))+ draw.Draw(canvas, image.Rect(0, 0, firstImage.Bounds().Dx(), firstImage.Bounds().Dy()), firstImage, image.Point{}, draw.Src)+ draw.Draw(canvas, image.Rect(0, firstImage.Bounds().Dy(), secondImage.Bounds().Dx(), canvas.Bounds().Dy()), secondImage, image.Point{}, draw.Src)+ var encoded bytes.Buffer+ if err := png.Encode(&encoded, canvas); err != nil {+ return first, false+ }+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), true+}++func markdownImageWithinLimits(width, height int) bool {+ if width <= 0 || height <= 0 || width > maxMarkdownImageDimension || height > maxMarkdownImageDimension {+ return false+ }+ return width <= maxMarkdownImagePixels/height+}++func decodeMarkdownImage(value string) (image.Image, bool) {+ payload := value+ if marker := strings.Index(value, "base64,"); marker >= 0 {+ payload = value[marker+len("base64,"):]+ } else if strings.HasPrefix(value, "data:") {+ marker := strings.IndexByte(value, ',')+ if marker < 0 {+ return nil, false+ }+ payload = value[marker+1:]+ }+ raw, err := base64.StdEncoding.DecodeString(payload)+ if err != nil {+ return nil, false+ }+ config, _, err := image.DecodeConfig(bytes.NewReader(raw))+ if err != nil || !markdownImageWithinLimits(config.Width, config.Height) {+ return nil, false+ }+ decoded, _, err := image.Decode(bytes.NewReader(raw))+ if err != nil {+ return nil, false+ }+ return decoded, true+}++func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {+ if err := ctx.Err(); err != nil {+ return nil, fmt.Errorf("GeneralChunker: %w", err)+ }+ units := upstream.JSONResult+ if upstream.OutputFormat == schema.PayloadFormatChunks {+ units = upstream.Chunks+ }+ if len(units) == 0 {+ return emptyOutputs(), nil+ }+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ chunks := make([]schema.ChunkDoc, 0, len(units))+ pendingRows := make([]schema.ChunkDoc, 0)+ var pendingHeader *schema.ChunkDoc+ flushRows := func() {+ if len(pendingRows) == 0 {+ return+ }+ chunks = append(chunks, mergeSpreadsheetRows(pendingRows, c.param.ChunkTokenSize)...)+ pendingRows = pendingRows[:0]+ }+ flushHeader := func() {+ if pendingHeader == nil {+ return+ }+ chunks = append(chunks, cloneChunkDoc(*pendingHeader))+ pendingHeader = nil+ }+ for _, unit := range units {+ if unit.CKType == "table_header" {+ flushRows()+ flushHeader()+ header := cloneChunkDoc(unit)+ pendingHeader = &header+ continue+ }+ if unit.CKType == "table_row" {+ // A header immediately followed by row units is parser metadata for+ // that table, not an additional data chunk. If no row follows, the+ // deferred header is emitted as a header-only table below.+ pendingHeader = nil+ pendingRows = append(pendingRows, unit)+ continue+ }+ flushRows()+ flushHeader()+ chunks = append(chunks, cloneChunkDoc(unit))+ }+ flushRows()+ flushHeader()+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ chunks = finalizeGeneralChunks(chunks, childrenPattern)+ if len(chunks) == 0 {+ return emptyOutputs(), nil+ }+ return chunkOutputs(chunks), nil+}++func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {+ if err := ctx.Err(); err != nil {+ return nil, fmt.Errorf("GeneralChunker: %w", err)+ }+ units := upstream.JSONResult+ if upstream.OutputFormat == schema.PayloadFormatChunks {+ units = upstream.Chunks+ }+ if len(units) == 0 {+ return emptyOutputs(), nil+ }+ primaryPattern := compileDelimPattern(c.param.Delimiters)+ childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)+ units = splitGeneralUnits(units, primaryPattern)+ if !hasCustomDelim(c.param.Delimiters) {+ // Python naive_merge prefixes each delimiter atom with a newline before+ // counting it. The prefix is a budgeting detail, not emitted content;+ // without it BPE counts can shift General's soft-cap boundaries.+ for i := range units {+ if itemDocType(units[i]) == "text" {+ units[i].TKNums = intPtr(tokenizeStr("\n" + units[i].Text))+ }+ }+ units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")+ }+ units = finalizeGeneralChunks(units, childrenPattern)+ if len(units) == 0 {+ return emptyOutputs(), nil+ }+ return chunkOutputs(units), nil+}++// mergeGeneralUnits implements the general OVER_CAP policy. A text unit is+// appended while the current chunk is at or below the overlap-scaled target;+// the append itself may cross the target. Oversized units remain indivisible.+func mergeGeneralUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {+ overlapPct = max(0, min(100, overlapPct))+ threshold := float64(target) * (100 - overlapPct) / 100+ merged := make([]schema.ChunkDoc, 0, len(units))+ current := -1++ for _, unit := range units {+ if itemDocType(unit) != "text" {+ merged = append(merged, cloneChunkDoc(unit))+ current = -1+ continue+ }+ if strings.TrimSpace(unit.Text) == "" {+ continue+ }++ count := generalUnitTokens(unit)+ unit.DocType = "text"+ unit.CKType = "text"+ unit.TKNums = intPtr(count)+ if count > target {+ merged = append(merged, cloneChunkDoc(unit))+ current = -1+ continue+ }+ if current < 0 || float64(intValue(merged[current].TKNums)) > threshold {+ merged = append(merged, cloneChunkDoc(unit))+ current = len(merged) - 1+ continue+ }++ mergeGeneralChunk(&merged[current], unit, joinSep)+ }++ return applyGeneralOverlap(merged, overlapPct, joinSep)+}++func mergeSpreadsheetRows(rows []schema.ChunkDoc, target int) []schema.ChunkDoc {+ merged := make([]schema.ChunkDoc, 0, len(rows))+ current := -1+ for _, row := range rows {+ if row.CKType != "table_row" || itemDocType(row) != "text" {+ merged = append(merged, cloneChunkDoc(row))+ current = -1+ continue+ }+ count := generalUnitTokens(row)+ if current < 0 || count > target || !sameSpreadsheetTable(merged[current], row) || intValue(merged[current].TKNums)+count > target {+ row.TKNums = intPtr(count)+ merged = append(merged, cloneChunkDoc(row))+ current = len(merged) - 1+ continue+ }+ mergeGeneralChunk(&merged[current], row, "\n")+ mergeSpreadsheetRowRange(&merged[current], row)+ }+ return merged+}++func sameSpreadsheetTable(first, second schema.ChunkDoc) bool {+ if first.TableID != "" || second.TableID != "" {+ return first.TableID == second.TableID+ }+ if first.SheetIndex != nil || second.SheetIndex != nil {+ return first.SheetIndex != nil && second.SheetIndex != nil && *first.SheetIndex == *second.SheetIndex+ }+ if firstSheet, firstOK := spreadsheetPositionSheet(first); firstOK {+ secondSheet, secondOK := spreadsheetPositionSheet(second)+ return secondOK && firstSheet == secondSheet+ }+ if first.Sheet != "" || second.Sheet != "" {+ return first.Sheet != "" && second.Sheet != "" && first.Sheet == second.Sheet+ }+ return false+}++func spreadsheetPositionSheet(doc schema.ChunkDoc) (float64, bool) {+ row, ok := firstPositionRow(lineRecord{positions: doc.Positions})+ if !ok {+ return 0, false+ }+ return row[0], true+}++func mergeSpreadsheetRowRange(dst *schema.ChunkDoc, src schema.ChunkDoc) {+ dst.RowStart = minSpreadsheetInt(dst.RowStart, src.RowStart, false)+ dst.RowEnd = minSpreadsheetInt(dst.RowEnd, src.RowEnd, true)+ dst.ColStart = minSpreadsheetInt(dst.ColStart, src.ColStart, false)+ dst.ColEnd = minSpreadsheetInt(dst.ColEnd, src.ColEnd, true)+}++func minSpreadsheetInt(first, second *int, maximum bool) *int {+ if first == nil {+ if second == nil {+ return nil+ }+ value := *second+ return &value+ }+ if second == nil {+ value := *first+ return &value+ }+ value := *first+ if (maximum && *second > value) || (!maximum && *second < value) {+ value = *second+ }+ return &value+}++func generalUnitTokens(unit schema.ChunkDoc) int {+ if unit.TKNums != nil && *unit.TKNums > 0 {+ return *unit.TKNums+ }+ return tokenizeStr(unit.Text)+}++func mergeGeneralChunk(dst *schema.ChunkDoc, src schema.ChunkDoc, joinSep string) {+ if dst.Text != "" && src.Text != "" {+ dst.Text += joinSep+ }+ dst.Text += src.Text+ dst.TKNums = intPtr(intValue(dst.TKNums) + generalUnitTokens(src))+ dst.PDFPositions = mergeGeneralPositions(dst.PDFPositions, src.PDFPositions)+ dst.Positions = mergeGeneralPositions(dst.Positions, src.Positions)+ mergeGeneralMetadata(dst, src)+}++func mergeGeneralMetadata(dst *schema.ChunkDoc, src schema.ChunkDoc) {+ if dst.Image == "" {+ dst.Image = src.Image+ }+ if dst.ImgID == "" {+ dst.ImgID = src.ImgID+ }+ if dst.Layout == "" {+ dst.Layout = src.Layout+ }+ if dst.LayoutType == "" {+ dst.LayoutType = src.LayoutType+ }+ if dst.LayoutNo == "" {+ dst.LayoutNo = src.LayoutNo+ }+ if dst.PageNumber == nil {+ dst.PageNumber = src.PageNumber+ }+ if dst.Extra == nil && len(src.Extra) > 0 {+ dst.Extra = make(map[string]json.RawMessage, len(src.Extra))+ }+ for key, value := range src.Extra {+ if _, exists := dst.Extra[key]; !exists {+ dst.Extra[key] = append(json.RawMessage(nil), value...)+ }+ }+}++func mergeGeneralPositions(first, second json.RawMessage) json.RawMessage {+ if positions := mergePositionMatrix(first, second); len(positions) > 0 {+ encoded, err := json.Marshal(positions)+ if err == nil {+ return encoded+ }+ }+ return extendRawJSONArray(first, second)+}++func applyGeneralOverlap(chunks []schema.ChunkDoc, overlapPct float64, joinSep string) []schema.ChunkDoc {+ if overlapPct <= 0 {+ return chunks+ }+ previousText := -1+ for i := range chunks {+ if itemDocType(chunks[i]) != "text" {+ previousText = -1+ continue+ }+ if previousText >= 0 {+ prefix, _ := computeOverlapPrefix(chunks[previousText].Text, overlapPct)+ if prefix != "" {+ current := cloneChunkDoc(chunks[i])+ current.Text = joinGeneralOverlapText(prefix, current.Text, joinSep)+ current.TKNums = intPtr(tokenizeStr(current.Text))+ chunks[i] = current+ }+ }+ previousText = i+ }+ return chunks+}++func joinGeneralOverlapText(prefix, text, separator string) string {+ if prefix == "" {+ return text+ }+ if text == "" || separator == "" {+ return prefix + text+ }+ if strings.HasSuffix(prefix, separator) || strings.HasPrefix(text, separator) {+ return prefix + text+ }+ return prefix + separator + text+}++func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {+ result := make([]schema.ChunkDoc, 0, len(units))+ for _, unit := range units {+ unit = cloneChunkDoc(unit)+ unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))+ unit.DocType = itemDocType(unit)+ unit.CKType = unit.DocType+ if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {+ unit.TKNums = intPtr(tokenizeStr(unit.Text))+ result = append(result, unit)+ continue+ }+ for _, part := range splitDroppingDelim(unit.Text, pattern) {+ if strings.TrimSpace(part) == "" {+ continue+ }+ piece := cloneChunkDoc(unit)+ piece.Text = part+ piece.TKNums = intPtr(tokenizeStr(part))+ result = append(result, piece)+ }+ }+ return result+}++func normalizeGeneralNewlines(text string) string {+ text = strings.ReplaceAll(text, "\r\n", "\n")+ return strings.ReplaceAll(text, "\r", "\n")+}++func finalizeGeneralChunks(chunks []schema.ChunkDoc, childrenPattern *regexp.Regexp) []schema.ChunkDoc {+ visible := make([]schema.ChunkDoc, 0, len(chunks))+ for _, chunk := range chunks {+ chunk.Text = removeTag(chunk.Text)+ if strings.TrimSpace(chunk.Text) == "" && chunk.Image == "" {+ continue+ }+ visible = append(visible, chunk)+ }+ visible = splitGeneralChildren(visible, childrenPattern)+ for i := range visible {+ visible[i].TKNums = intPtr(tokenizeStr(visible[i].Text))+ }+ return visible+}++// splitGeneralChildren keeps the matched child delimiter attached to the+// preceding child. This matches the legacy General/naive path; the shared+// TokenChunker splitByChildren helper intentionally drops delimiters and is+// therefore not reused here.+func splitGeneralChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {+ if pattern == nil {+ return chunks+ }+ result := make([]schema.ChunkDoc, 0, len(chunks))+ for _, chunk := range chunks {+ if itemDocType(chunk) != "text" {+ result = append(result, chunk)+ continue+ }+ mom := strings.TrimPrefix(chunk.Text, "\n")+ for _, part := range splitKeepingGeneralDelimiter(chunk.Text, pattern) {+ if strings.TrimSpace(part) == "" {+ continue+ }+ piece := cloneChunkDoc(chunk)+ piece.Text = part+ piece.Mom = mom+ result = append(result, piece)+ }+ }+ return result+}++func splitKeepingGeneralDelimiter(text string, pattern *regexp.Regexp) []string {+ matches := pattern.FindAllStringIndex(text, -1)+ if len(matches) == 0 {+ return []string{text}+ }+ result := make([]string, 0, len(matches)+1)+ cursor := 0+ for _, match := range matches {+ if match[0] > cursor {+ result = append(result, text[cursor:match[1]])+ }+ cursor = match[1]+ }+ if cursor < len(text) {+ result = append(result, text[cursor:])+ }+ return result+}++func init() {+ MustRegisterChunker(ComponentNameGeneralChunker)+}diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.gonew file mode 100644index 0000000000..c8717c32ce--- /dev/null+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -0,0 +1,505 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "encoding/json"+ "fmt"+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++func TestMergeGeneralUnitsAllowsOneBoundaryOverflow(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "alpha", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "beta", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "gamma", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ }++ got := mergeGeneralUnits(units, 3, 0, "\n")+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"alpha\nbeta", "gamma"}) {+ t.Fatalf("texts = %q, want [alpha\\nbeta gamma]", texts)+ }+ if count := intValue(got[0].TKNums); count != 4 {+ t.Errorf("first chunk tokens = %d, want running sum 4", count)+ }+}++func TestMergeSpreadsheetRowsUsesProjectedCapAndSheetBoundary(t *testing.T) {+ sheetOne := 1+ sheetTwo := 2+ rows := []schema.ChunkDoc{+ {Text: "row-1", DocType: "text", CKType: "table_row", TKNums: intPtr(2), SheetIndex: &sheetOne, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},+ {Text: "row-2", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetOne, RowStart: intPtr(3), RowEnd: intPtr(3), ColStart: intPtr(1), ColEnd: intPtr(2)},+ {Text: "row-3", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheetTwo, RowStart: intPtr(2), RowEnd: intPtr(2), ColStart: intPtr(1), ColEnd: intPtr(2)},+ }++ got := mergeSpreadsheetRows(rows, 3)+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"row-1\nrow-2", "row-3"}) {+ t.Fatalf("texts = %q", texts)+ }+ if len(got) != 2 || got[0].TKNums == nil || *got[0].TKNums != 3 {+ t.Fatalf("merged row token count = %#v, want 3", got)+ }+ if got[0].RowStart == nil || *got[0].RowStart != 2 || got[0].RowEnd == nil || *got[0].RowEnd != 3 {+ t.Fatalf("merged row range = start:%v end:%v", got[0].RowStart, got[0].RowEnd)+ }+ if got[0].SheetIndex == nil || *got[0].SheetIndex != 1 || got[1].SheetIndex == nil || *got[1].SheetIndex != 2 {+ t.Fatalf("sheet boundaries not preserved: %#v", got)+ }+}++func TestMergeSpreadsheetRowsKeepsOversizedRowAndCapZeroAtomic(t *testing.T) {+ sheet := 1+ rows := []schema.ChunkDoc{+ {Text: "small", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(2), RowEnd: intPtr(2)},+ {Text: "oversized", DocType: "text", CKType: "table_row", TKNums: intPtr(8), SheetIndex: &sheet, RowStart: intPtr(3), RowEnd: intPtr(3)},+ {Text: "after", DocType: "text", CKType: "table_row", TKNums: intPtr(1), SheetIndex: &sheet, RowStart: intPtr(4), RowEnd: intPtr(4)},+ }+ if got := mergeSpreadsheetRows(rows, 3); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {+ t.Fatalf("oversized texts = %q", generalChunkTexts(got))+ }+ if got := mergeSpreadsheetRows(rows, 0); !reflect.DeepEqual(generalChunkTexts(got), []string{"small", "oversized", "after"}) {+ t.Fatalf("zero-cap texts = %q", generalChunkTexts(got))+ }+}++func TestGeneralChunkerSpreadsheetConsumesRowIR(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 3})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "orders.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1, "table_id": "sheet-1"},+ {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 2, "sheet_index": 1, "table_id": "sheet-1", "row_start": 2, "row_end": 2},+ {"text": "ID:A-101; Status:pending", "doc_type_kwd": "text", "ck_type": "table_row", "tk_nums": 1, "sheet_index": 1, "table_id": "sheet-1", "row_start": 3, "row_end": 3},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"ID:A-100; Status:paid\nID:A-101; Status:pending"}) {+ t.Fatalf("spreadsheet chunks = %q", texts)+ }+}++func TestMergeGeneralUnitsMergesWhenCurrentEqualsCap(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "at cap", DocType: "text", CKType: "text", TKNums: intPtr(3)},+ {Text: "overflow", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ }++ got := mergeGeneralUnits(units, 3, 0, "\n")+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"at cap\noverflow"}) {+ t.Fatalf("texts = %q, want one overflow chunk", texts)+ }+}++func TestMergeGeneralUnitsKeepsOversizedAtomWhole(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "oversized atom stays whole", DocType: "text", CKType: "text", TKNums: intPtr(8)},+ {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ }++ got := mergeGeneralUnits(units, 3, 0, "\n")+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "oversized atom stays whole", "after"}) {+ t.Fatalf("texts = %q", texts)+ }+}++func TestMergeGeneralUnitsAppliesUnconditionalCharacterOverlap(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "abcdefghij@@1\t0\t1\t2\t3##", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "klmnopqrst", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "uvwxyzABCD", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ }++ got := mergeGeneralUnits(units, 2, 50, "\n")+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{+ "abcdefghij@@1\t0\t1\t2\t3##",+ "fghij\nklmnopqrst",+ "mnopqrst\nuvwxyzABCD",+ }) {+ t.Fatalf("texts = %q", texts)+ }+ if intValue(got[1].TKNums) <= 2 {+ t.Errorf("overlap was trimmed to cap: token count = %d", intValue(got[1].TKNums))+ }+}++func TestApplyGeneralOverlapDoesNotAccumulatePositions(t *testing.T) {+ position := func(page int) json.RawMessage {+ return json.RawMessage(fmt.Sprintf("[[%d,0,10,0,5]]", page))+ }+ chunks := []schema.ChunkDoc{+ {Text: "alpha beta", DocType: "text", CKType: "text", PDFPositions: position(1), Positions: position(1)},+ {Text: "gamma delta", DocType: "text", CKType: "text", PDFPositions: position(2), Positions: position(2)},+ {Text: "epsilon zeta", DocType: "text", CKType: "text", PDFPositions: position(3), Positions: position(3)},+ }++ got := applyGeneralOverlap(chunks, 50, "\n")+ if len(got) != 3 {+ t.Fatalf("chunks = %d, want 3", len(got))+ }+ if string(got[1].PDFPositions) != string(position(2)) || string(got[1].Positions) != string(position(2)) {+ t.Fatalf("second overlap chunk inherited prior positions: pdf=%s positions=%s", got[1].PDFPositions, got[1].Positions)+ }+ if string(got[2].PDFPositions) != string(position(3)) || string(got[2].Positions) != string(position(3)) {+ t.Fatalf("third overlap chunk accumulated prior positions: pdf=%s positions=%s", got[2].PDFPositions, got[2].Positions)+ }+}++func TestMergeSpreadsheetRowsUsesPositionSheetWhenIdentityFieldsAreMissing(t *testing.T) {+ rows := []schema.ChunkDoc{+ {Text: "sheet-1-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[1,2,2,1,2]]`)},+ {Text: "sheet-2-row", DocType: "text", CKType: "table_row", TKNums: intPtr(1), Positions: json.RawMessage(`[[2,2,2,1,2]]`)},+ }++ got := mergeSpreadsheetRows(rows, 10)+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"sheet-1-row", "sheet-2-row"}) {+ t.Fatalf("position-only sheet boundary was lost: texts = %q", texts)+ }+}++func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "alpha beta", DocType: "text", CKType: "text"},+ {Text: "gamma delta", DocType: "text", CKType: "text"},+ {Text: "epsilon zeta", DocType: "text", CKType: "text"},+ {Text: "eta theta", DocType: "text", CKType: "text"},+ }+ for i := range units {+ units[i].TKNums = intPtr(tokenizeStr(units[i].Text))+ }+ target := intValue(units[0].TKNums) + intValue(units[1].TKNums)++ got := mergeMarkdownUnits(units, target, 50, "\n")+ want := []string{+ "alpha beta\ngamma delta",+ "gamma delta\nepsilon zeta",+ "epsilon zeta\neta theta",+ }+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, want) {+ t.Fatalf("markdown overlap = %q, want %q", texts, want)+ }+}++func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {+ previousPage := 4+ currentPage := 5+ units := []schema.ChunkDoc{+ {+ Text: "alpha beta",+ DocType: "text",+ CKType: "text",+ Image: "previous-image",+ ImgID: "previous-image-id",+ PageNumber: &previousPage,+ Positions: json.RawMessage(`[[0,0,10,0,10]]`),+ Extra: map[string]json.RawMessage{"previous": json.RawMessage(`true`)},+ },+ {+ Text: "gamma delta",+ DocType: "text",+ CKType: "text",+ PageNumber: ¤tPage,+ Positions: json.RawMessage(`[[0,0,10,20,30]]`),+ Extra: map[string]json.RawMessage{"current": json.RawMessage(`true`)},+ },+ }+ for i := range units {+ units[i].TKNums = intPtr(tokenizeStr(units[i].Text))+ }++ target := intValue(units[0].TKNums)+ got := mergeMarkdownUnits(units, target, 50, "\n")+ if len(got) != 2 {+ t.Fatalf("chunks = %#v, want previous and overlapped current chunks", got)+ }+ if got[1].Text == "gamma delta" || !strings.HasSuffix(got[1].Text, "\ngamma delta") {+ t.Fatalf("overlapped text = %q, want a prefix followed by current text", got[1].Text)+ }+ if got[1].Image != "" {+ t.Errorf("overlap chunk inherited previous image %q", got[1].Image)+ }+ if got[1].ImgID != "" {+ t.Errorf("overlap chunk inherited previous image id %q", got[1].ImgID)+ }+ if got[1].PageNumber == nil || *got[1].PageNumber != currentPage {+ t.Errorf("overlap chunk page number = %v, want current page %d", got[1].PageNumber, currentPage)+ }+ if string(got[1].Positions) != `[[0,0,10,0,10],[0,0,10,20,30]]` {+ t.Errorf("overlap chunk positions = %s, want previous and current positions", got[1].Positions)+ }+ if _, ok := got[1].Extra["previous"]; ok {+ t.Errorf("overlap chunk inherited previous extra metadata: %#v", got[1].Extra)+ }+ if _, ok := got[1].Extra["current"]; !ok {+ t.Errorf("overlap chunk lost current extra metadata: %#v", got[1].Extra)+ }+}++func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "Background details", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "## Architecture", DocType: "text", CKType: "heading", TKNums: intPtr(1)},+ {Text: "The architecture is split into parser and chunker.", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ }++ got := mergeMarkdownUnits(units, 3, 0, "\n")+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{+ "Background details",+ "## Architecture\nThe architecture is split into parser and chunker.",+ }) {+ t.Fatalf("markdown chunks = %q, want body and heading/body chunks", texts)+ }+ if got[1].CKType != "text" {+ t.Fatalf("heading/body chunk CKType = %q, want text after heading absorption", got[1].CKType)+ }+}++func TestShortMarkdownHeadingBudgetIgnoresATXMarker(t *testing.T) {+ content := strings.TrimSpace(strings.Repeat("word ", 49))+ unit := schema.ChunkDoc{CKType: "heading", Text: "## " + content}+ if tokenizeStr(content) >= 50 {+ t.Fatalf("test heading content unexpectedly reached threshold: %d", tokenizeStr(content))+ }+ if tokenizeStr(unit.Text) < 50 {+ t.Fatalf("test ATX heading did not cross raw threshold: %d", tokenizeStr(unit.Text))+ }+ if !isShortMarkdownHeading(unit) {+ t.Fatalf("heading %q was not classified short after removing ATX marker", unit.Text)+ }+}++func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T) {+ previousPage := 4+ units := []schema.ChunkDoc{+ {Text: "first", DocType: "text", CKType: "text", TKNums: intPtr(2), Image: "previous-image", ImgID: "previous-image-id", PageNumber: &previousPage, Extra: map[string]json.RawMessage{"source": json.RawMessage(`"previous"`)}},+ {Text: "second", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ {Text: "third", DocType: "text", CKType: "text", TKNums: intPtr(2)},+ }++ got := mergeGeneralUnits(units, 4, 50, "\n")+ if len(got) != 2 {+ t.Fatalf("chunks = %#v, want two chunks", got)+ }+ current := got[1]+ if current.Image != "" || current.ImgID != "" {+ t.Fatalf("overlap chunk copied previous media: image=%q img_id=%q", current.Image, current.ImgID)+ }+ if current.PageNumber != nil {+ t.Fatalf("overlap chunk copied previous page number: %v", current.PageNumber)+ }+ if _, ok := current.Extra["source"]; ok {+ t.Fatalf("overlap chunk copied previous extra metadata: %#v", current.Extra)+ }+}++func TestTakeGeneralContextSentencePreservesBoundarySelection(t *testing.T) {+ text := "one!two!three!"+ if got, want := takeGeneralContextSentence(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {+ t.Fatalf("suffix context = %q, want %q", got, want)+ }+ if got, want := takeGeneralContextSentence(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {+ t.Fatalf("prefix context = %q, want %q", got, want)+ }+}++func TestMergeGeneralUnitsKeepsMediaAtomicAndBreaksTextRun(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "A|B", DocType: "table", CKType: "table", TKNums: intPtr(2)},+ {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ }++ got := mergeGeneralUnits(units, 10, 0, "\n")+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "A|B", "after"}) {+ t.Fatalf("texts = %q", texts)+ }+ if got[1].DocType != "table" || got[1].CKType != "table" {+ t.Errorf("table type changed: %#v", got[1])+ }+}++func TestMergeGeneralUnitsCombinesPositionsAndMetadata(t *testing.T) {+ first := schema.ChunkDoc{+ Text: "alpha",+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(1),+ Image: "image-a",+ Positions: json.RawMessage(`[[0,0,10,0,10]]`),+ PDFPositions: json.RawMessage(`[[0,0,10,0,10]]`),+ Extra: map[string]json.RawMessage{"source_order": json.RawMessage(`1`)},+ PageNumber: intPtr(4),+ }+ second := schema.ChunkDoc{+ Text: "beta",+ DocType: "text",+ CKType: "text",+ TKNums: intPtr(1),+ Positions: json.RawMessage(`[[0,0,10,20,30]]`),+ PDFPositions: json.RawMessage(`[[0,0,10,20,30]]`),+ Extra: map[string]json.RawMessage{"heading_level": json.RawMessage(`2`)},+ }++ got := mergeGeneralUnits([]schema.ChunkDoc{first, second}, 1, 0, "\n")+ if len(got) != 1 {+ t.Fatalf("chunks = %d, want 1", len(got))+ }+ if got[0].Image != "image-a" {+ t.Errorf("image = %q, want image-a", got[0].Image)+ }+ if got[0].PageNumber == nil || *got[0].PageNumber != 4 {+ t.Errorf("page number = %v, want 4", got[0].PageNumber)+ }+ if string(got[0].Positions) != `[[0,0,10,0,10],[0,0,10,20,30]]` {+ t.Errorf("positions = %s", got[0].Positions)+ }+ if string(got[0].PDFPositions) != `[[0,0,10,0,10],[0,0,10,20,30]]` {+ t.Errorf("PDF positions = %s", got[0].PDFPositions)+ }+ if string(got[0].Extra["source_order"]) != "1" || string(got[0].Extra["heading_level"]) != "2" {+ t.Errorf("extra metadata = %#v", got[0].Extra)+ }+}++func TestGeneralChunkerPreservesImageWithoutText(t *testing.T) {+ component, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{{+ "doc_type_kwd": "image",+ "image": "image-payload",+ }},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 || chunks[0]["image"] != "image-payload" {+ t.Fatalf("chunks = %#v, want image-only unit", chunks)+ }+}++func TestGeneralChunkerCustomDelimiterBypassesMerge(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 1,+ "delimiters": []string{"`||`"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, generalTextInput("alpha beta||gamma delta"))+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha beta", "gamma delta"}) {+ t.Fatalf("texts = %q", texts)+ }+}++func TestGeneralChunkerBareDelimiterCreatesMergeAtoms(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 0,+ "delimiters": []string{"|"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, generalTextInput("alpha|beta"))+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha", "beta"}) {+ t.Fatalf("texts = %q", texts)+ }+}++func TestGeneralChunkerSplitsChildrenAfterParentMerge(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 100,+ "delimiters": []string{},+ "children_delimiters": []string{"|"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, generalTextInput("alpha|beta"))+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha|", "beta"}) {+ t.Fatalf("texts = %q", texts)+ }+ for i, chunk := range chunks {+ if chunk["mom"] != "alpha|beta" {+ t.Errorf("chunk[%d].mom = %q", i, chunk["mom"])+ }+ }+}++func generalChunkTexts(chunks []schema.ChunkDoc) []string {+ texts := make([]string, len(chunks))+ for i := range chunks {+ texts[i] = chunks[i].Text+ }+ return texts+}++func generalTextInput(text string) map[string]any {+ return map[string]any{+ "name": "document.txt",+ "file_type": "txt",+ "output_format": "json",+ "json": []map[string]any{{"text": text, "doc_type_kwd": "text"}},+ }+}++func outputChunks(t *testing.T, out map[string]any) []map[string]any {+ t.Helper()+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok {+ t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])+ }+ return chunks+}++func outputTexts(t *testing.T, out map[string]any) []string {+ t.Helper()+ chunks := outputChunks(t, out)+ texts := make([]string, len(chunks))+ for i := range chunks {+ texts[i], _ = chunks[i]["text"].(string)+ }+ return texts+}diff --git a/internal/ingestion/component/chunker/general_pdf_fallback_cgo_test.go b/internal/ingestion/component/chunker/general_pdf_fallback_cgo_test.gonew file mode 100644index 0000000000..28d277b7f3--- /dev/null+++ b/internal/ingestion/component/chunker/general_pdf_fallback_cgo_test.go@@ -0,0 +1,62 @@+//go:build cgo++//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "errors"+ "log/slog"+ "strings"+ "testing"++ "ragflow/internal/ingestion/component"+)++func TestGeneralChunkerPDFContinuesWhenStorageResolutionFails(t *testing.T) {+ previousResolver := component.ResolveDocumentStorageOverride+ component.ResolveDocumentStorageOverride = func(string) (*component.DocumentStorageRef, error) {+ return nil, errors.New("storage unavailable")+ }+ t.Cleanup(func() { component.ResolveDocumentStorageOverride = previousResolver })++ var logs strings.Builder+ previousLogger := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})))+ t.Cleanup(func() { slog.SetDefault(previousLogger) })++ chunker, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := chunker.Invoke(t.Context(), nil, map[string]any{+ "name": "document.pdf",+ "file_type": "pdf",+ "doc_id": "doc-with-missing-storage",+ "output_format": "json",+ "json": []map[string]any{{"text": "body", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }+ if !strings.Contains(logs.String(), "could not open PDF") {+ t.Fatalf("logs = %q, want PDF open warning", logs.String())+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 || chunks[0]["text"] != "body" {+ t.Fatalf("chunks = %#v, want body chunk after crop fallback", chunks)+ }+}diff --git a/internal/ingestion/component/chunker/general_python_parity_test.go b/internal/ingestion/component/chunker/general_python_parity_test.gonew file mode 100644index 0000000000..4e467f7706--- /dev/null+++ b/internal/ingestion/component/chunker/general_python_parity_test.go@@ -0,0 +1,65 @@+package chunker++import (+ "context"+ "encoding/json"+ "os"+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/parser/parser"+)++type generalTextPythonGolden struct {+ Input string `json:"input"`+ Params map[string]any `json:"params"`+ Chunks []struct {+ Text string `json:"text"`+ } `json:"chunks"`+}++// TestGeneralChunkerTextPythonParity verifies the post-refactor boundary:+// Python's built-in general path parses and chunks text in one operation,+// whereas Go sends the full normalized TextParser result to GeneralChunker.+// The golden is captured from rag.app.naive.chunk and locks the resulting+// chunk boundaries after removing Python's representation-only leading newline.+func TestGeneralChunkerTextPythonParity(t *testing.T) {+ raw, err := os.ReadFile("testdata/general/text_default.python.golden.json")+ if err != nil {+ t.Fatalf("read Python golden: %v", err)+ }+ var golden generalTextPythonGolden+ if err := json.Unmarshal(raw, &golden); err != nil {+ t.Fatalf("decode Python golden: %v", err)+ }++ parsed := parser.NewTextParser().ParseWithResult(+ context.Background(), "general.txt", []byte(golden.Input),+ )+ if parsed.Err != nil {+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)+ }+ component, err := NewGeneralChunker(golden.Params)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "general.txt",+ "file_type": "txt",+ "output_format": "json",+ "json": parsed.JSON,+ })+ if err != nil {+ t.Fatalf("GeneralChunker.Invoke: %v", err)+ }++ got := outputTexts(t, out)+ want := make([]string, 0, len(golden.Chunks))+ for _, chunk := range golden.Chunks {+ want = append(want, strings.TrimPrefix(chunk.Text, "\n"))+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("GeneralChunker text chunks = %#v, want Python %v", got, want)+ }+}diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.gonew file mode 100644index 0000000000..ee3938ef49--- /dev/null+++ b/internal/ingestion/component/chunker/general_test.go@@ -0,0 +1,847 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "bytes"+ "encoding/base64"+ "encoding/json"+ "fmt"+ "image"+ "image/color"+ "image/jpeg"+ "log/slog"+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema"+)++func TestGeneralChunkerRegistered(t *testing.T) {+ factory, category, metadata, ok := runtime.DefaultRegistry.Lookup(ComponentNameGeneralChunker)+ if !ok {+ t.Fatal("GeneralChunker is not registered")+ }+ if category != runtime.CategoryIngestion {+ t.Errorf("category = %q, want %q", category, runtime.CategoryIngestion)+ }+ if factory == nil {+ t.Fatal("factory is nil")+ }+ if _, ok := metadata.Inputs["file_type"]; !ok {+ t.Error("registered inputs missing file_type")+ }+ component, err := factory(ComponentNameGeneralChunker, nil)+ if err != nil {+ t.Fatalf("factory: %v", err)+ }+ if component == nil {+ t.Fatal("factory returned nil component")+ }+}++func TestGeneralChunkerDefaults(t *testing.T) {+ component, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ general := component.(*GeneralChunkerComponent)+ if general.param.ChunkTokenSize != 512 {+ t.Errorf("ChunkTokenSize = %d, want 512", general.param.ChunkTokenSize)+ }+ if general.param.OverlappedPercent != 0 {+ t.Errorf("OverlappedPercent = %v, want 0", general.param.OverlappedPercent)+ }+ if general.param.TableContextSize != 0 || general.param.ImageContextSize != 0 {+ t.Errorf("context sizes = (%d, %d), want (0, 0)", general.param.TableContextSize, general.param.ImageContextSize)+ }+ if len(general.param.Delimiters) != 1 || general.param.Delimiters[0] != "\n" {+ t.Errorf("Delimiters = %q, want [newline]", general.param.Delimiters)+ }+ if len(general.param.ChildrenDelimiters) != 0 {+ t.Errorf("ChildrenDelimiters = %q, want empty", general.param.ChildrenDelimiters)+ }+}++func TestGeneralChunkerParamsNormalizeConfiguration(t *testing.T) {+ delimiters := []string{"\n", "`---`"}+ children := []any{". ", "! "}+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 0,+ "delimiters": delimiters,+ "overlapped_percent": 0.25,+ "children_delimiters": children,+ "table_context_size": -2,+ "image_context_size": 7,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ general := component.(*GeneralChunkerComponent)+ if general.param.ChunkTokenSize != 0 {+ t.Errorf("ChunkTokenSize = %d, want 0", general.param.ChunkTokenSize)+ }+ if general.param.OverlappedPercent != 25 {+ t.Errorf("OverlappedPercent = %v, want 25", general.param.OverlappedPercent)+ }+ if got := strings.Join(general.param.Delimiters, "|"); got != "\n|`---`" {+ t.Errorf("Delimiters = %q", got)+ }+ if got := strings.Join(general.param.ChildrenDelimiters, "|"); got != ". |! " {+ t.Errorf("ChildrenDelimiters = %q", got)+ }+ if general.param.TableContextSize != 0 || general.param.ImageContextSize != 7 {+ t.Errorf("context sizes = (%d, %d), want (0, 7)", general.param.TableContextSize, general.param.ImageContextSize)+ }++ delimiters[0] = "changed"+ children[0] = "changed"+ if general.param.Delimiters[0] != "\n" || general.param.ChildrenDelimiters[0] != ". " {+ t.Fatal("component params retain caller-owned slices")+ }+}++func TestGeneralChunkerRejectsNegativeTokenBudget(t *testing.T) {+ if _, err := NewGeneralChunker(map[string]any{"chunk_token_size": -1}); err == nil {+ t.Fatal("NewGeneralChunker accepted a negative chunk_token_size")+ }+}++func TestGeneralChunkerNormalizesLegacyDelimiterString(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "delimiter": "\n!?;。;!?",+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ general := component.(*GeneralChunkerComponent)+ want := []string{"\n", "!", "?", ";", "。", ";", "!", "?"}+ if !reflect.DeepEqual(general.param.Delimiters, want) {+ t.Fatalf("Delimiters = %#v, want %#v", general.param.Delimiters, want)+ }+}++func TestGeneralStrategyForFileType(t *testing.T) {+ tests := []struct {+ fileType string+ want generalStrategy+ }{+ {"pdf", generalStrategyPDF},+ {"docx", generalStrategyDOCX},+ {"md", generalStrategyMarkdown},+ {"xls", generalStrategySpreadsheet},+ {"xlsx", generalStrategySpreadsheet},+ {"csv", generalStrategySpreadsheet},+ {"txt", generalStrategyText},+ {"other", generalStrategyText},+ }+ for _, test := range tests {+ t.Run(test.fileType, func(t *testing.T) {+ if got := generalStrategyForFileType(test.fileType); got != test.want {+ t.Errorf("generalStrategyForFileType(%q) = %v, want %v", test.fileType, got, test.want)+ }+ })+ }+}++func TestGeneralChunkerLogsUnknownFileTypeFallback(t *testing.T) {+ var logs bytes.Buffer+ previousLogger := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))+ t.Cleanup(func() { slog.SetDefault(previousLogger) })++ component, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ _, err = component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.bin",+ "file_type": "application/x-custom",+ "output_format": "json",+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatalf("Invoke unknown file type: %v", err)+ }+ if !strings.Contains(logs.String(), "unknown file_type") {+ t.Fatalf("logs = %q, want unknown file_type fallback diagnostic", logs.String())+ }++ logs.Reset()+ _, err = component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.txt",+ "file_type": "txt",+ "output_format": "json",+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatalf("Invoke canonical text file type: %v", err)+ }+ if strings.Contains(logs.String(), "unknown file_type") {+ t.Fatalf("canonical text fallback emitted unknown-file diagnostic: %q", logs.String())+ }+}++func TestGeneralChunkerInfersFileTypeFromSourceName(t *testing.T) {+ component, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ _, err = component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.txt",+ "output_format": "json",+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatalf("Invoke inferred file type: %v", err)+ }+}++func TestGeneralChunkerRequiresFileTypeWhenSourceHasNoExtension(t *testing.T) {+ component, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ _, err = component.Invoke(t.Context(), nil, map[string]any{+ "name": "document",+ "output_format": "json",+ "json": []map[string]any{{"text": "alpha", "doc_type_kwd": "text"}},+ })+ if err == nil || !strings.Contains(err.Error(), "file_type") {+ t.Fatalf("Invoke error = %v, want missing file_type error", err)+ }+}++func TestGeneralChunkerPreservesSingleParserUnit(t *testing.T) {+ component, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{{+ "text": "alpha",+ "doc_type_kwd": "text",+ "source_order": 3,+ "heading_level": 2,+ }},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 1 {+ t.Fatalf("chunks = %#v, want one chunk", out["chunks"])+ }+ if chunks[0]["text"] != "alpha" || chunks[0]["source_order"] != float64(3) || chunks[0]["heading_level"] != float64(2) {+ t.Errorf("single parser unit was not preserved: %#v", chunks[0])+ }+}++func TestGeneralChunkerMarkdownShortHeadingForcesNextUnit(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %#v, want one forced heading/body chunk", chunks)+ }+ if text, _ := chunks[0]["text"].(string); text != "Title\nbody" {+ t.Errorf("text = %q, want %q", text, "Title\nbody")+ }+}++func TestGeneralChunkerMarkdownLongHeadingUsesNormalCap(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ longHeading := strings.Repeat("heading ", 60)+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": longHeading, "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want separate long heading and body chunks", chunks)+ }+ if text, _ := chunks[0]["text"].(string); text != strings.TrimSpace(longHeading) {+ t.Errorf("first text = %q, want long heading", text)+ }+ if text, _ := chunks[1]["text"].(string); text != "body" {+ t.Errorf("second text = %q, want body", text)+ }+}++func TestGeneralChunkerMarkdownImageMergesWithTextAndPreservesPayload(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "figure caption", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %#v, want one heading/image chunk", chunks)+ }+ if text, _ := chunks[0]["text"].(string); text != "Title\nfigure caption" {+ t.Errorf("text = %q, want %q", text, "Title\nfigure caption")+ }+ if image, _ := chunks[0]["image"].(string); image != "data:image/png;base64,AAAA" {+ t.Errorf("image = %q, want original image payload", image)+ }+}++func TestMergeMarkdownImagesStacksRasterPayloads(t *testing.T) {+ const pixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="+ const secondPixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYPj/HwADAgH/5ncLrgAAAABJRU5ErkJggg=="+ got, ok := mergeMarkdownImagesChecked("data:image/png;base64,"+pixel, "data:image/png;base64,"+secondPixel)+ if !ok {+ t.Fatal("mergeMarkdownImagesChecked rejected valid PNG payloads")+ }+ decoded, ok := decodeMarkdownImage(got)+ if !ok {+ t.Fatalf("merged image is not decodable: %q", got)+ }+ if got := decoded.Bounds(); got.Dx() != 1 || got.Dy() != 2 {+ t.Fatalf("merged image bounds = %v, want 1x2", got)+ }+}++func TestDecodeMarkdownImageSupportsJPEGAndGIF(t *testing.T) {+ var jpegData bytes.Buffer+ img := image.NewRGBA(image.Rect(0, 0, 1, 1))+ img.Set(0, 0, color.RGBA{R: 255, A: 255})+ if err := jpeg.Encode(&jpegData, img, nil); err != nil {+ t.Fatalf("jpeg.Encode: %v", err)+ }+ cases := []struct {+ name string+ mime string+ data []byte+ }{+ {+ name: "jpeg",+ mime: "image/jpeg",+ data: jpegData.Bytes(),+ },+ {+ name: "gif",+ mime: "image/gif",+ data: mustDecodeBase64(t, "R0lGODdhAgACAIEAAP8AAAAAAAAAAAAAACwAAAAAAgACAAAIBgABCAQQEAA7"),+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ value := "data:" + tc.mime + ";base64," + base64.StdEncoding.EncodeToString(tc.data)+ if _, ok := decodeMarkdownImage(value); !ok {+ t.Fatalf("decodeMarkdownImage rejected %s payload", tc.mime)+ }+ })+ }+}++func mustDecodeBase64(t *testing.T, value string) []byte {+ t.Helper()+ decoded, err := base64.StdEncoding.DecodeString(value)+ if err != nil {+ t.Fatalf("DecodeString: %v", err)+ }+ return decoded+}++func TestMergeMarkdownImagesAcceptsBareBase64Payloads(t *testing.T) {+ const pixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGP4z8DwHwAFAAH/iZk9HQAAAABJRU5ErkJggg=="+ const secondPixel = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYPj/HwADAgH/5ncLrgAAAABJRU5ErkJggg=="+ got, ok := mergeMarkdownImagesChecked(pixel, secondPixel)+ if !ok {+ t.Fatal("mergeMarkdownImagesChecked rejected bare base64 payloads")+ }+ decoded, ok := decodeMarkdownImage(got)+ if !ok {+ t.Fatalf("merged bare-base64 image is not decodable: %q", got)+ }+ if got := decoded.Bounds(); got.Dx() != 1 || got.Dy() != 2 {+ t.Fatalf("merged image bounds = %v, want 1x2", got)+ }+}++func TestMergeMarkdownImagesRejectsOversizedCanvas(t *testing.T) {+ if markdownImageWithinLimits(maxMarkdownImageDimension, maxMarkdownImageDimension) {+ t.Fatal("maximum-dimension canvas should exceed the pixel safety limit")+ }+ if !markdownImageWithinLimits(1024, 1024) {+ t.Fatal("normal image dimensions should remain mergeable")+ }+ if got, ok := mergeMarkdownImagesChecked("s3://bucket/first.png", "s3://bucket/second.png"); ok || got != "s3://bucket/first.png" {+ t.Fatalf("opaque image references were changed: %q", got)+ }+}++func TestGeneralChunkerMarkdownDoesNotDropUnmergeableImages(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "first", "doc_type_kwd": "image", "image": "s3://bucket/first.png"},+ {"text": "second", "doc_type_kwd": "image", "image": "s3://bucket/second.png"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want two chunks so neither image reference is lost", chunks)+ }+ if chunks[0]["image"] != "s3://bucket/first.png" || chunks[1]["image"] != "s3://bucket/second.png" {+ t.Fatalf("image references = %#v, want both source references", chunks)+ }+}++func TestGeneralChunkerMarkdownShortHeadingKeepsFollowingTableAtomic(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "Title", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table", "ck_type": "table"},+ {"text": "body", "doc_type_kwd": "text", "ck_type": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want table and body", chunks)+ }+ if text, _ := chunks[0]["text"].(string); text != "Title\n<table><tr><td>A</td></tr></table>" {+ t.Errorf("table text = %q", text)+ }+ if got := chunks[0]["doc_type_kwd"]; got != "table" {+ t.Errorf("table doc_type_kwd = %v, want table", got)+ }+ if got := chunks[0]["ck_type"]; got != "table" {+ t.Errorf("table ck_type = %v, want table", got)+ }+}++func TestGeneralChunkerDOCXMediaDoesNotBreakTextMerge(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "", "doc_type_kwd": "image", "image": "figure"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want merged text and image", chunks)+ }+ if chunks[0]["text"] != "before\nafter" {+ t.Errorf("merged text = %q, want before\\nafter", chunks[0]["text"])+ }+ if chunks[1]["doc_type_kwd"] != "image" || chunks[1]["image"] != "figure" {+ t.Errorf("image chunk = %+v", chunks[1])+ }+}++func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "table_context_size": 2,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want merged text and table", chunks)+ }+ if chunks[1]["doc_type_kwd"] != "table" {+ t.Fatalf("table chunk = %+v", chunks[1])+ }+ if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {+ t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])+ }+}++func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "before one", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "before two", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "", DocType: "table", CKType: "table"},+ {Text: "after one", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "after two", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ }++ if got, want := collectGeneralMediaContext(units, 2, 10, true), "before one\nbefore two"; got != want {+ t.Fatalf("above context = %q, want %q", got, want)+ }+ if got, want := collectGeneralMediaContext(units, 2, 10, false), "after one\nafter two"; got != want {+ t.Fatalf("below context = %q, want %q", got, want)+ }+}++func TestGeneralChunkerDOCXCustomDelimiterDisablesTextMerge(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "delimiters": []string{"`|`"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before|after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"before", "after"}) {+ t.Fatalf("texts = %q, want custom-delimiter units", texts)+ }+}++func TestGeneralChunkerDOCXAppliesTextOverlap(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 2,+ "overlapped_percent": 50,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "alpha beta", "doc_type_kwd": "text"},+ {"text": "gamma delta", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ texts := outputTexts(t, out)+ if len(texts) != 2 {+ t.Fatalf("texts = %q, want two overlapped chunks", texts)+ }+ if texts[1] == "gamma delta" || !strings.HasSuffix(texts[1], "\ngamma delta") {+ t.Fatalf("DOCX overlap = %q, want previous tail plus current paragraph", texts[1])+ }+}++func TestGeneralChunkerChildrenDelimiterKeepsDelimiter(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "children_delimiters": []string{";"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.txt",+ "file_type": "txt",+ "output_format": "json",+ "json": []map[string]any{{"text": "part A;part B", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"part A;", "part B"}) {+ t.Fatalf("children delimiter texts = %q, want [part A; part B]", texts)+ }+}++func TestGeneralChunkerPDFPreservesPhysicalOrderAroundMedia(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks = %#v, want text, media, text in physical order", chunks)+ }+ if chunks[0]["doc_type_kwd"] != "text" || chunks[0]["text"] != "before" {+ t.Errorf("first chunk = %+v", chunks[0])+ }+ if chunks[1]["doc_type_kwd"] != "table" || chunks[1]["text"] != "<table><tr><td>A</td></tr></table>" {+ t.Errorf("media chunk = %+v", chunks[1])+ }+ if chunks[2]["doc_type_kwd"] != "text" || chunks[2]["text"] != "after" {+ t.Errorf("last chunk = %+v", chunks[2])+ }+}++func TestGeneralChunkerPDFUsesPositionOrderForMediaContext(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "table_context_size": 1,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ position := func(top float64) []any {+ return []any{[]any{1.0, 0.0, 10.0, top, top + 5}}+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "after", "doc_type_kwd": "text", "positions": position(30)},+ {"text": "<table>A</table>", "doc_type_kwd": "table", "positions": position(20)},+ {"text": "before", "doc_type_kwd": "text", "positions": position(10)},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks = %#v, want text, media, text", chunks)+ }+ if chunks[1]["context_above"] != "before" || chunks[1]["context_below"] != "after" {+ t.Errorf("table context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])+ }+ if chunks[0]["text"] != "before" || chunks[2]["text"] != "after" {+ t.Errorf("position-ordered text = [%v, %v]", chunks[0]["text"], chunks[2]["text"])+ }+}++func TestSortPDFUnitsKeepsUnpositionedMediaInInputOrder(t *testing.T) {+ position := func(top float64) json.RawMessage {+ return json.RawMessage(fmt.Sprintf("[[1,0,10,%g,%g]]", top, top+5))+ }+ units := []schema.ChunkDoc{+ {Text: "before", DocType: "text", CKType: "text", Positions: position(10)},+ {Text: "figure", DocType: "image", CKType: "image"},+ {Text: "after", DocType: "text", CKType: "text", Positions: position(30)},+ }++ got := sortPDFUnits(units)+ if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"before", "figure", "after"}) {+ t.Fatalf("PDF units = %q, want unpositioned media to keep its input position", texts)+ }+}++func TestGeneralChunkerSpreadsheetHeaderOnlyPreservesHeaderChunk(t *testing.T) {+ component, err := NewGeneralChunker(nil)+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "headers.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{{+ "text": "Name; Amount",+ "doc_type_kwd": "table",+ "ck_type": "table_header",+ "sheet_index": 1,+ "table_id": "sheet-1",+ "positions": []any{[]any{1.0, 1.0, 1.0, 1.0, 2.0}},+ }},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %#v, want one header-only chunk", chunks)+ }+ if chunks[0]["text"] != "Name; Amount" || chunks[0]["ck_type"] != "table_header" {+ t.Fatalf("header-only chunk = %#v", chunks[0])+ }+}++func TestGeneralChunkerSpreadsheetAttachesImageContext(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "image_context_size": 10,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "figures.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "Revenue", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},+ {"text": "Growth", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks = %#v, want row, image, row", chunks)+ }+ if chunks[1]["ck_type"] != "image" {+ t.Fatalf("image chunk = %#v", chunks[1])+ }+ if chunks[1]["context_above"] != "Revenue" || chunks[1]["context_below"] != "Growth" {+ t.Fatalf("image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])+ }+}++func TestGeneralChunkerSpreadsheetImageContextStopsAtSheetBoundary(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"image_context_size": 10})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "figures.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "Sheet one", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1, "tk_nums": 1},+ {"text": "B2", "doc_type_kwd": "image", "ck_type": "image", "image": "figure", "table_id": "sheet-1", "sheet_index": 1},+ {"text": "Sheet two", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2, "tk_nums": 1},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks = %#v, want row, image, row", chunks)+ }+ if chunks[1]["context_above"] != "Sheet one" || chunks[1]["context_below"] != nil {+ t.Fatalf("cross-sheet image context = above:%q below:%q", chunks[1]["context_above"], chunks[1]["context_below"])+ }+}++func TestGeneralChunkerPDFAttachesOutlineOnce(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.pdf",+ "file_type": "pdf",+ "output_format": "json",+ "file": map[string]any{+ "outline": []map[string]any{{"title": "Chapter 1", "level": 0}},+ },+ "json": []map[string]any{{"text": "body", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %#v, want one chunk", chunks)+ }+ outline, ok := chunks[0]["__outline__"].([]any)+ if !ok || len(outline) != 1 {+ t.Fatalf("outline = %#v, want one entry", chunks[0]["__outline__"])+ }+ entry, _ := outline[0].(map[string]any)+ if entry["title"] != "Chapter 1" || entry["depth"] != float64(0) {+ t.Errorf("outline entry = %#v", entry)+ }+}diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.goindex 46b994a0a3..3cab60a8d8 100644--- a/internal/ingestion/component/chunker/group.go+++ b/internal/ingestion/component/chunker/group.go@@ -438,37 +438,54 @@ func recordsFromStructured(items []schema.ChunkDoc) []lineRecord {if dt == "" {dt = "text"}- var imgID *string- if it.ImgID != "" {- img := it.ImgID- imgID = &img- }- meta := make(map[string]any)- if it.ContentLtks != "" {- meta["content_ltks"] = it.ContentLtks- }- if it.ContentSmLtks != "" {- meta["content_sm_ltks"] = it.ContentSmLtks- }- if it.TitleTks != "" {- meta["title_tks"] = it.TitleTks- }- if it.TitleSmTks != "" {- meta["title_sm_tks"] = it.TitleSmTks+ appendRecord := func(recordText string) {+ var imgID *string+ if it.ImgID != "" {+ img := it.ImgID+ imgID = &img+ }+ meta := make(map[string]any)+ if it.ContentLtks != "" {+ meta["content_ltks"] = it.ContentLtks+ }+ if it.ContentSmLtks != "" {+ meta["content_sm_ltks"] = it.ContentSmLtks+ }+ if it.TitleTks != "" {+ meta["title_tks"] = it.TitleTks+ }+ if it.TitleSmTks != "" {+ meta["title_sm_tks"] = it.TitleSmTks+ }+ for k, v := range it.Extra {+ meta[k] = json.RawMessage(v)+ }+ out = append(out, lineRecord{+ text: recordText,+ docType: dt,+ imgID: imgID,+ layout: it.Layout,+ ckType: it.CKType,+ pdfPositions: it.PDFPositions,+ positions: it.Positions,+ parentMeta: meta,+ })}- for k, v := range it.Extra {- meta[k] = json.RawMessage(v)+ // TextParser intentionally emits one coordinate-free text item for the+ // whole document. Title-family chunkers still need one record per line+ // so heading regexes can see headings after the first line. Structured+ // media/table items and positioned text remain atomic.+ plainText := itemDocType(it) == "text" && (it.CKType == "" || it.CKType == "text")+ unpositioned := len(it.PDFPositions) == 0 && len(it.Positions) == 0+ if plainText && unpositioned && strings.Contains(text, "\n") {+ for _, line := range strings.Split(normalizeGeneralNewlines(text), "\n") {+ if trim(line) != "" {+ appendRecord(line)+ }+ }+ continue}- out = append(out, lineRecord{- text: text,- docType: dt,- imgID: imgID,- layout: it.Layout,- ckType: it.CKType,- pdfPositions: it.PDFPositions,- positions: it.Positions,- parentMeta: meta,- })+ appendRecord(text)}return out}diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.goindex bf97caabc5..a9edcea927 100644--- a/internal/ingestion/component/chunker/group_test.go+++ b/internal/ingestion/component/chunker/group_test.go@@ -22,6 +22,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser")func TestGroupTitleChunker_Registered(t *testing.T) {@@ -106,6 +107,71 @@ func TestExtractLineRecords_TextKeyStillWorks(t *testing.T) {}}+func TestExtractLineRecords_ParserStructuredTextSplitsLines(t *testing.T) {+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "notes.txt", []byte("# Chapter 1\nfirst body\n# Chapter 2\nsecond body"))+ if parsed.Err != nil {+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)+ }++ records := extractLineRecords(map[string]any{+ "name": "notes.txt",+ "output_format": "json",+ "json": parsed.JSON,+ })+ if len(records) != 4 {+ t.Fatalf("got %d records, want one record per non-empty line", len(records))+ }+ want := []string{"# Chapter 1", "first body", "# Chapter 2", "second body"}+ for i, record := range records {+ if record.text != want[i] {+ t.Errorf("record[%d].text = %q, want %q", i, record.text, want[i])+ }+ }+}++func TestTitleFamilyChunkersRecognizeHeadingsAfterFirstParserLine(t *testing.T) {+ source := "# Chapter 1\n" + strings.Repeat("first section body ", 40) + "\n# Chapter 2\n" + strings.Repeat("second section body ", 40)+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "notes.txt", []byte(source))+ if parsed.Err != nil {+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)+ }+ input := map[string]any{+ "name": "notes.txt",+ "output_format": "json",+ "json": parsed.JSON,+ }++ group, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ manual, err := NewManualChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewManualChunker: %v", err)+ }+ for _, tc := range []struct {+ name string+ component runtime.Component+ }{+ {name: "group", component: group},+ {name: "manual", component: manual},+ } {+ t.Run(tc.name, func(t *testing.T) {+ out, err := tc.component.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 2 {+ t.Fatalf("got %d chunks, want one per heading section", len(chunks))+ }+ if !strings.Contains(chunks[1]["text"].(string), "# Chapter 2") {+ t.Fatalf("second chunk lost its heading: %#v", chunks[1])+ }+ })+ }+}+func TestGroupTitleChunker_InvokeEmptyInput(t *testing.T) {c, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}},diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex 1d49834b6f..aa819d5127 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -17,10 +17,11 @@// QAChunker extracts question-answer pairs from parsed content.//// Input formats and extraction strategies:-// - Text (txt, csv) → delimiter-based Q&A (comma or tab)-// - Markdown (md) → heading-based Q&A-// - HTML (xls, xlsx) → table-based Q&A (first two columns)-// - JSON (pdf, docx, xlsx) → text sections via delimiter; table items via extractQATable+// - Text (txt, csv) → delimiter-based Q&A (comma or tab)+// - Markdown (md) → heading-based Q&A+// - HTML table → table-based Q&A (first two columns)+// - JSON → typed spreadsheet cells first; otherwise text sections+// or the HTML-table fallback for parsers that emit table markup.//// Every Q&A pair becomes a single chunk whose text is// "Question: {q}\tAnswer: {a}" (ingestion renames text to@@ -119,7 +120,11 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (mcase schema.PayloadFormatText:qaPairs = extractQAText(stringPtrVal(upstream.TextResult))default:- qaPairs = extractQAJSON(upstream.JSONResult)+ fileType := upstream.FileType+ if strings.TrimSpace(fileType) == "" && isCSV(upstream.Name) {+ fileType = "csv"+ }+ qaPairs = extractQAJSON(upstream.JSONResult, fileType)}chunks := make([]schema.ChunkDoc, 0, len(qaPairs))@@ -322,6 +327,9 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {for _, cells := range rows {// Python qa.py:365 requires exactly two fields for CSV pairs.if strictPairs && len(cells) != 2 {+ if len(pairs) > 0 {+ pairs[len(pairs)-1].Answer += "\n" + strings.Join(cells, ",")+ }continue}var texts []string@@ -530,22 +538,33 @@ func detectDelimiter(lines []string) string {// JSON / structured QA extraction// ----------------------------------------------------------------------------func extractQAJSON(items []schema.ChunkDoc) []qaPair {+func extractQAJSON(items []schema.ChunkDoc, fileType string) []qaPair {var pairs []qaPair+ strictCSV := strings.EqualFold(fileType, "csv")for _, item := range items {- txt, _ := itemText(item)- if txt == "" {- continue- }- // XLSX (#18800) emits OutputFormat json with HTML tables in item- // text and doc_type_kwd=table. Route those through extractQATable- // so spreadsheet QA keeps working; plain text items stay on the- // delimiter path used by pdf/docx.var tmp []qaPair- if itemDocType(item) == "table" {- tmp = extractQATable(txt, false)+ if item.CKType == "table_header" || item.CKType == "table_row" {+ // QA spreadsheets have no header row: the parser's structural+ // table_header is still the first question/answer record.+ if strictCSV && len(item.Cells) != 2 {+ if len(pairs) > 0 {+ pairs[len(pairs)-1].Answer += "\n" + strings.Join(item.Cells, ",")+ }+ continue+ }+ tmp = extractQARowCells(item.Cells, strictCSV, item.RowStart)} else {- tmp = extractQAText(txt)+ txt, _ := itemText(item)+ if txt == "" {+ continue+ }+ // Non-spreadsheet table items may still carry HTML markup. Keep+ // the HTML fallback for parsers that do not expose typed cells.+ if itemDocType(item) == "table" {+ tmp = extractQATable(txt, strictCSV)+ } else {+ tmp = extractQAText(txt)+ }}// Preserve the source item's image id and coordinates on each// extracted pair@@ -559,6 +578,26 @@ func extractQAJSON(items []schema.ChunkDoc) []qaPair {return pairs}+func extractQARowCells(cells []string, strict bool, rowStart *int) []qaPair {+ if strict && len(cells) != 2 {+ return nil+ }+ texts := make([]string, 0, len(cells))+ for _, cell := range cells {+ if cell = strings.TrimSpace(cell); cell != "" {+ texts = append(texts, cell)+ }+ }+ if len(texts) < 2 {+ return nil+ }+ rowNum := -1+ if rowStart != nil && *rowStart > 0 {+ rowNum = *rowStart - 1+ }+ return []qaPair{{Question: texts[0], Answer: texts[1], RowNum: rowNum}}+}+func init() {MustRegisterChunker(ComponentNameQAChunker)}diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.gonew file mode 100644index 0000000000..4c5161d91f--- /dev/null+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go@@ -0,0 +1,93 @@+package chunker++import (+ "context"+ "strings"+ "testing"++ "ragflow/internal/parser/parser"+)++func TestCSVQAFirstRowIsData(t *testing.T) {+ p := parser.NewCSVParser()+ res := p.ParseWithResult(context.Background(), "qa.csv", []byte("q1,a1\nq2,a2\n"))+ if res.Err != nil {+ t.Fatal(res.Err)+ }++ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ out, err := comp.Invoke(t.Context(), nil, map[string]any{+ "name": "qa.csv",+ "file_type": "csv",+ "output_format": res.OutputFormat,+ "json": res.JSON,+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 2 {+ t.Fatalf("expected both QA rows, got %d chunks: %#v", len(chunks), chunks)+ }+ if got, _ := chunks[0]["text"].(string); got != "Question: q1\tAnswer: a1" {+ t.Fatalf("first row was not emitted as QA data: %q", got)+ }+}++func TestCSVTypedRowsFoldMalformedRowIntoPreviousAnswer(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ out, err := comp.Invoke(t.Context(), nil, map[string]any{+ "name": "qa.csv",+ "file_type": "csv",+ "output_format": "json",+ "json": []map[string]any{+ {"doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"q1", "a1"}},+ {"doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"question", "", "extra"}},+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("malformed row created or removed a pair: %#v", chunks)+ }+ got, _ := chunks[0]["text"].(string)+ if !strings.Contains(got, "Answer: a1\nquestion,,extra") {+ t.Fatalf("malformed row was not folded into the previous answer: %q", got)+ }+}++func TestCSVJSONHTMLTableUsesStrictContinuationRules(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ out, err := comp.Invoke(t.Context(), nil, map[string]any{+ "name": "qa.csv",+ "file_type": "csv",+ "output_format": "json",+ "json": []map[string]any{{+ "doc_type_kwd": "table",+ "text": "<table><tr><td>q1</td><td>a1</td></tr>" ++ "<tr><td>q2</td><td>a2</td><td>trailing</td></tr></table>",+ }},+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("malformed HTML row created a QA pair: %#v", chunks)+ }+ got, _ := chunks[0]["text"].(string)+ if !strings.Contains(got, "Answer: a1\nq2,a2,trailing") {+ t.Fatalf("malformed HTML row was not folded into the previous answer: %q", got)+ }+}diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex a3af4c2cb6..472771afd8 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -171,6 +171,30 @@ func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {}}+func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ inputs := map[string]any{+ "name": "questions.csv",+ "output_format": "json",+ "json": []map[string]any{{+ "doc_type_kwd": "text",+ "ck_type": "table_row",+ "cells": []string{"question", "answer", "unexpected"},+ }},+ }+ out, err := comp.Invoke(t.Context(), nil, inputs)+ if err != nil {+ t.Fatalf("Invoke failed: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 0 {+ t.Fatalf("chunks = %#v, want malformed CSV row rejected", chunks)+ }+}+// Non-CSV names keep the old "first two non-empty cells" rule on the HTML// table path. Since #18800 an .xlsx file reaches the chunker as "json", not// "html", so this covers the shared HTML branch and not the XLSX pipeline.@@ -394,3 +418,32 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {}}}++func TestQAChunkerSpreadsheetRowIRTreatsFirstRowAsQAData(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ out, err := comp.Invoke(t.Context(), nil, map[string]any{+ "name": "orders.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},+ {"text": "ID:A-100; Status:paid", "doc_type_kwd": "text", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want both QA rows", out["chunks"])+ }+ if got, _ := chunks[0]["text"].(string); got != "Question: ID\tAnswer: Status" {+ t.Fatalf("first-row QA = %q", got)+ }+ if got, _ := chunks[1]["text"].(string); got != "Question: A-100\tAnswer: paid" {+ t.Fatalf("second-row QA = %q", got)+ }+}diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.goindex b3b90a12c2..585c9197de 100644--- a/internal/ingestion/component/chunker/register.go+++ b/internal/ingestion/component/chunker/register.go@@ -124,6 +124,7 @@ var ChunkerInputs = map[string]string{"content": "Alias for \"text\".","chunks": "Optional upstream chunk list (structured JSON form).","name": "Source document name. Not required on the payload: when absent it is read from the workflow-wide globals bag (CanvasState.Globals) via globals.GlobalOrInput.",+ "file_type": "Canonical parser routing extension used by GeneralChunker.","_created_time": "Optional upstream timestamp (RFC3339Nano, s).","_elapsed_time": "Optional upstream elapsed time (s).",}diff --git a/internal/ingestion/component/chunker/table.go b/internal/ingestion/component/chunker/table.goindex 74fb1bee18..0dc53979fb 100644--- a/internal/ingestion/component/chunker/table.go+++ b/internal/ingestion/component/chunker/table.go@@ -113,10 +113,49 @@ func (c *TableChunkerComponent) invoke(_ context.Context, inputs map[string]any)// tableItems returns the per-row records, preferring JSONResult and// falling back to Chunks. Each record becomes exactly one chunk.func tableItems(items, chunks []schema.ChunkDoc) []schema.ChunkDoc {- if len(items) > 0 {- return items+ source := items+ if len(source) == 0 {+ source = chunks}- return chunks+ if len(source) == 0 {+ return nil+ }+ dataTables := make(map[string]struct{})+ for _, item := range source {+ if item.CKType == "table_row" {+ dataTables[spreadsheetTableKey(item)] = struct{}{}+ }+ }+ filtered := make([]schema.ChunkDoc, 0, len(source))+ for _, item := range source {+ // Spreadsheet parsers expose the header as schema metadata. It is+ // already represented in each table_row and must not become a data+ // chunk of its own. A table with no data rows still needs its header+ // as the only searchable representation.+ if item.CKType == "table_header" {+ if _, hasRows := dataTables[spreadsheetTableKey(item)]; hasRows {+ continue+ }+ }+ filtered = append(filtered, item)+ }+ return filtered+}++func spreadsheetTableKey(item schema.ChunkDoc) string {+ if item.TableID != "" {+ return "table:" + item.TableID+ }+ if item.SheetIndex != nil {+ return fmt.Sprintf("sheet-index:%d", *item.SheetIndex)+ }+ if item.Sheet != "" {+ return "sheet:" + item.Sheet+ }+ if sheet, ok := spreadsheetPositionSheet(item); ok {+ return fmt.Sprintf("position-sheet:%g", sheet)+ }+ return "unknown"}func init() {diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.goindex 556a47caae..c5cde7d98f 100644--- a/internal/ingestion/component/chunker/table_test.go+++ b/internal/ingestion/component/chunker/table_test.go@@ -79,3 +79,54 @@ func TestTableChunker_EmptyRows(t *testing.T) {t.Errorf("got %d chunks, want 0", len(chunks))}}++func TestTableChunkerSkipsSpreadsheetHeaderRecord(t *testing.T) {+ chunks := tableChunksOf(t, map[string]any{+ "name": "orders.xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "cells": []string{"ID", "Status"}},+ {"text": "A-100; paid", "doc_type_kwd": "table", "ck_type": "table_row", "cells": []string{"A-100", "paid"}},+ },+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want only the data row", len(chunks))+ }+ if chunks[0]["ck_type"] != "table_row" || chunks[0]["text"] != "A-100; paid" {+ t.Fatalf("chunks = %#v, want the table row only", chunks)+ }+}++func TestTableChunkerPreservesHeaderOnlyTable(t *testing.T) {+ chunks := tableChunksOf(t, map[string]any{+ "name": "empty.xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},+ },+ })+ if len(chunks) != 1 {+ t.Fatalf("got %d chunks, want the header-only table preserved", len(chunks))+ }+ if chunks[0]["ck_type"] != "table_header" || chunks[0]["text"] != "ID; Name" {+ t.Fatalf("chunk = %#v, want the header record", chunks[0])+ }+}++func TestTableChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {+ chunks := tableChunksOf(t, map[string]any{+ "name": "mixed.xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "ID; Name", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},+ {"text": "A-1; paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-2", "sheet_index": 2},+ },+ })+ if len(chunks) != 2 {+ t.Fatalf("got %d chunks, want header-only sheet plus data row", len(chunks))+ }+ if chunks[0]["ck_type"] != "table_header" || chunks[1]["ck_type"] != "table_row" {+ t.Fatalf("chunks = %#v, want header then row", chunks)+ }+}diff --git a/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json b/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.jsonnew file mode 100644index 0000000000..6b2c4aeb87--- /dev/null+++ b/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json@@ -0,0 +1,34 @@+{+ "input": "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six;Eta seven!Theta eight?Iota nine.",+ "params": {+ "chunk_token_size": 5,+ "delimiters": [+ "\n",+ "!",+ "?",+ ";",+ "。",+ ";",+ "!",+ "?"+ ],+ "overlapped_percent": 0+ },+ "chunks": [+ {+ "text": "\nAlpha one.\nBeta two"+ },+ {+ "text": "\n Gamma three\nDelta four"+ },+ {+ "text": "\n Epsilon five\nZeta six"+ },+ {+ "text": "\nEta seven\nTheta eight"+ },+ {+ "text": "\nIota nine."+ }+ ]+}diff --git a/internal/ingestion/component/chunker/textcode_parity_test.go b/internal/ingestion/component/chunker/textcode_parity_test.goindex e7ba5a50b6..04daf104d0 100644--- a/internal/ingestion/component/chunker/textcode_parity_test.go+++ b/internal/ingestion/component/chunker/textcode_parity_test.go@@ -37,14 +37,13 @@ type textCodeGolden struct {Chunks []map[string]any `json:"chunks"`}-// TestTextParserTokenChunkerParity covers the production text&code handoff:-// TextParser.ParseWithResult emits structured JSON items after splitting on-// the text&code delimiters while keeping matched delimiters in the item text;-// TokenChunker owns the final token-budget merge. The Python golden is-// generated by the same TxtParser path used by parser._code; comparison-// follows the parser alignment contract and ignores representation-only chunk-// boundaries and whitespace.-func TestTextParserTokenChunkerParity(t *testing.T) {+// TestTextParserGeneralChunkerParity covers the production text&code handoff:+// TextParser emits one normalized JSON unit and GeneralChunker owns delimiter+// splitting plus the final token-budget merge. The Python golden is generated+// by the same TxtParser path used by parser._code; comparison follows the+// content alignment contract and ignores representation-only boundaries and+// whitespace.+func TestTextParserGeneralChunkerParity(t *testing.T) {cases := []struct {name stringsamplePath string@@ -90,29 +89,28 @@ func TestTextParserTokenChunkerParity(t *testing.T) {t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)}- component, err := NewTokenChunker(map[string]any{+ component, err := NewGeneralChunker(map[string]any{"chunk_token_size": float64(textCodeTokenSize),- // Parser already consumed the text&code delimiters. An empty- // list makes this invocation exercise only the downstream merge.- "delimiters": []string{},+ "delimiters": textCodeDelimiter,})if err != nil {- t.Fatalf("NewTokenChunker: %v", err)+ t.Fatalf("NewGeneralChunker: %v", err)}out, err := component.Invoke(context.Background(), nil, map[string]any{"name": tc.samplePath,+ "file_type": "txt","output_format": "json","json": parsed.JSON,})if err != nil {- t.Fatalf("TokenChunker.Invoke: %v", err)+ t.Fatalf("GeneralChunker.Invoke: %v", err)}if msg, ok := out["_ERROR"].(string); ok && msg != "" {- t.Fatalf("TokenChunker returned _ERROR: %s", msg)+ t.Fatalf("GeneralChunker returned _ERROR: %s", msg)}got, ok := out["chunks"].([]map[string]any)if !ok {- t.Fatalf("TokenChunker chunks has type %T", out["chunks"])+ t.Fatalf("GeneralChunker chunks has type %T", out["chunks"])}if len(got) < tc.minChunks {t.Fatalf("final chunk count = %d, want at least %d", len(got), tc.minChunks)diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 1cdefce641..91269b6af8 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -238,7 +238,7 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputsif engine != nil {defer engine.Close()}- return c.invokeJSONPayload(ctx, items, delimPattern, childrenPattern, engine), nil+ return c.invokeJSONPayload(ctx, items, delimPattern, childrenPattern, engine, upstream.FileType), nil}}@@ -531,7 +531,7 @@ func (c *TokenChunkerComponent) mergeByTokenSize(text string, delimPattern, chil// invokeJSONPayload handles structured upstream input. Items fan// across 4 goroutines; merge is by input index.-func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []schema.ChunkDoc, delimPattern, childrenPattern *regexp.Regexp, engine deepdoctype.PDFEngine) map[string]any {+func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []schema.ChunkDoc, delimPattern, childrenPattern *regexp.Regexp, engine deepdoctype.PDFEngine, fileType string) map[string]any {if len(items) == 0 {return emptyOutputs()}@@ -544,6 +544,12 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []s}lanes := partition(len(items), workers)perItem := make([][]schema.ChunkDoc, len(items))+ dataTables := make(map[string]struct{})+ for _, item := range items {+ if item.CKType == "table_row" {+ dataTables[spreadsheetTableKey(item)] = struct{}{}+ }+ }var wg sync.WaitGroupfor w := 0; w < workers; w++ {@@ -556,7 +562,20 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []sperItem[i] = nilcontinue}- perItem[i] = chunkFromItem(items[i], delimPattern)+ if items[i].CKType == "table_header" {+ if _, hasRows := dataTables[spreadsheetTableKey(items[i])]; hasRows {+ // The typed header is metadata carried by every row. It+ // must not become an independent TokenChunker chunk when+ // row IR is consumed by a legacy pipeline.+ perItem[i] = nil+ continue+ }+ }+ if isTextParserSentenceFallback(fileType, c.param.Delimiters, items[i]) {+ perItem[i] = splitTextParserSentences(items[i])+ } else {+ perItem[i] = chunkFromItem(items[i], delimPattern)+ }}}(lane.start, lane.end)}@@ -610,12 +629,56 @@ func (c *TokenChunkerComponent) invokeJSONPayload(ctx context.Context, items []sreturn chunkOutputs(out)}+// isTextParserSentenceFallback restores the semantic sentence units that the+// shared TextParser used to emit. Parser boundaries remain owned by the+// chunker: this only applies the text/code parser's default sentence grammar+// before TokenChunker merges units by its configured token budget.+func isTextParserSentenceFallback(fileType string, delimiters []string, item schema.ChunkDoc) bool {+ if itemDocType(item) != "text" || len(delimiters) != 1 || delimiters[0] != "\n" {+ return false+ }+ switch strings.ToLower(strings.TrimSpace(fileType)) {+ case "txt", "py", "js", "java", "c", "cpp", "h", "php", "go", "ts", "sh", "cs", "kt", "sql":+ return true+ default:+ return false+ }+}++func splitTextParserSentences(item schema.ChunkDoc) []schema.ChunkDoc {+ text := itemTextOrFallback(item)+ if !sentenceDelimiter.MatchString(text) {+ return []schema.ChunkDoc{buildChunkDoc(item, "text", text, "", "")}+ }+ parts := splitSentencesLossless(text)+ out := make([]schema.ChunkDoc, 0, len(parts))+ for _, part := range parts {+ part = strings.TrimSpace(part)+ if part == "" || strings.TrimSpace(sentenceDelimiter.ReplaceAllString(part, "")) == "" {+ continue+ }+ out = append(out, buildChunkDoc(item, "text", part, "", ""))+ }+ if len(out) == 0 {+ return []schema.ChunkDoc{buildChunkDoc(item, "text", text, "", "")}+ }+ return out+}+// ---------------------------------------------------------------------------// JSON-payload internals// ---------------------------------------------------------------------------// chunkFromItem mirrors _build_json_chunks for a single item.func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp) []schema.ChunkDoc {+ if it.CKType == "table_row" {+ row := cloneChunkDoc(it)+ row.DocType = "text"+ row.CKType = "table_row"+ row.Text = itemTextOrFallback(it)+ row.TKNums = intPtr(tokenizeStr(row.Text))+ return []schema.ChunkDoc{row}+ }ckType := itemDocType(it)txt := itemTextOrFallback(it)if ckType != "text" {diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex 4fe546146e..ed0053003c 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -26,6 +26,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/schema"+ textparser "ragflow/internal/parser/parser")// TestTokenChunker_Registered asserts the registry has a CategoryIngestion@@ -69,6 +70,69 @@ func TestTokenChunker_InvokeEmptyInput(t *testing.T) {}}+func TestTokenChunkerPreservesSpreadsheetRowBoundaries(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "orders.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 1},+ {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 1},+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "sheet_index": 2},+ {"text": "ID: B-1; Status: open", "doc_type_kwd": "text", "ck_type": "table_row", "sheet_index": 2},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok {+ t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])+ }+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want one row chunk per sheet", chunks)+ }+ if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "ID: B-1; Status: open" {+ t.Fatalf("row chunks = %#v", chunks)+ }+ for i, chunk := range chunks {+ if chunk["ck_type"] != "table_row" {+ t.Errorf("chunk[%d] ck_type = %v, want table_row", i, chunk["ck_type"])+ }+ }+}++func TestTokenChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "orders.xlsx",+ "file_type": "xlsx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "ID; Status", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-1", "sheet_index": 1},+ {"text": "ID: A-1; Status: paid", "doc_type_kwd": "text", "ck_type": "table_row", "table_id": "sheet-1", "sheet_index": 1},+ {"text": "Name; Owner", "doc_type_kwd": "table", "ck_type": "table_header", "table_id": "sheet-2", "sheet_index": 2},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want data row and header-only sheet", chunks)+ }+ if chunks[0]["text"] != "ID: A-1; Status: paid" || chunks[1]["text"] != "Name; Owner" {+ t.Fatalf("chunks = %#v, want row followed by header-only sheet", chunks)+ }+}+// TestTokenChunker_InvokeDelimMode_BasicChunking drives the// delimiter-mode path with a backtick delimiter and asserts each// chunk carries the matched delimiter text within itself (split@@ -228,6 +292,33 @@ func TestTokenChunker_InvokeJSONPayload(t *testing.T) {}}+func TestTokenChunkerTextParserJSONKeepsSentenceBoundaries(t *testing.T) {+ parsed := textparser.NewTextParser().ParseWithResult(t.Context(), "doc.txt", []byte("first!second!"))+ if parsed.Err != nil {+ t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)+ }++ c, err := NewTokenChunker(map[string]any{+ "chunk_token_size": 128,+ "delimiters": []string{"\n"},+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.txt",+ "file_type": "txt",+ "output_format": "json",+ "json": parsed.JSON,+ })+ if err != nil {+ t.Fatalf("TokenChunker.Invoke: %v", err)+ }+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"first!\nsecond!"}) {+ t.Fatalf("text parser JSON chunks = %q, want sentence boundary between parser units", texts)+ }+}+func TestTokenChunker_InvokeJSONPayload_IndexesHeaderOnlyEmail(t *testing.T) {c, err := NewTokenChunker(map[string]any{"delimiter_mode": "delimiter",diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.goindex 4700710f2c..2434affcd9 100644--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go@@ -45,7 +45,7 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {t.Fatal(res.Err)}- inputs := map[string]any{"name": "qa.xlsx", "output_format": res.OutputFormat}+ inputs := map[string]any{"name": "qa.xlsx", "file_type": "xlsx", "output_format": res.OutputFormat}switch res.OutputFormat {case "json":inputs["json"] = res.JSON@@ -65,38 +65,38 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {return chunks}-// TestXLSXQARegression is the end-to-end smoke test: every row of the sheet-// becomes one chunk. The chunker has no header concept, so the first row is-// a Q&A pair like any other.-func TestXLSXQARegression(t *testing.T) {+// TestXLSXQAFirstRowIsData protects the QA spreadsheet contract: workbooks+// contain question/answer rows without a header, so the parser's structural+// table_header item is still the first QA pair.+func TestXLSXQAFirstRowIsData(t *testing.T) {chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{- {"question", "answer"},- {"What is RAGFlow?", "A RAG engine."},- {"Where are the docs?", "On the website."},+ {"q1", "a1"},+ {"q2", "a2"},}))- t.Logf("QA chunks=%d", len(chunks))- if len(chunks) != 3 {- t.Fatalf("expected 3 chunks, got %d", len(chunks))+ if len(chunks) != 2 {+ t.Fatalf("expected both QA rows, got %d chunks: %#v", len(chunks), chunks)+ }+ if got := chunkTexts(chunks); !strings.Contains(got[0], "q1") || !strings.Contains(got[0], "a1") {+ t.Fatalf("first row was not emitted as QA data: %#v", got)}}-// A spreadsheet cell keeps the newline its author typed (Alt+Enter) and the-// parser renders it into the <tr>/<td> HTML verbatim, so a QA pair whose-// question or answer spans lines must survive extraction intact. These rows-// used to disappear from the chunk list without a trace.+// A spreadsheet cell keeps the newline its author typed (Alt+Enter), so a QA+// pair whose question or answer spans lines must survive cells-first+// extraction intact. These rows used to disappear from the chunk list without+// a trace.func TestXLSXQAMultilineCells(t *testing.T) {const multilineQ = "请问全国碳排放权交易市场纳入配额管理的重点排放单\n位名录,是否会公布?"const multilineA = "需要公布。根据《碳排放权交易管理办法(试行)》。"const multilineAnswer = "跨行的答案\n第二行\n第三行"chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{- {"question", "answer"},{multilineQ, multilineA},{"跨行的问句\n第二行", multilineAnswer},}))t.Logf("QA chunks=%d", len(chunks))- if len(chunks) != 3 {- t.Fatalf("expected 3 chunks, got %d", len(chunks))+ if len(chunks) != 2 {+ t.Fatalf("expected 2 data-row chunks, got %d", len(chunks))}texts := strings.Join(chunkTexts(chunks), "\n")for _, want := range []string{multilineQ, multilineA, multilineAnswer} {diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex f35ba2b6b9..051075cd85 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -715,6 +715,9 @@ func (c *ExtractorComponent) callTextCached(ctx context.Context, db *gorm.DB, in// them on ck["important_kwd"]. Keyword extraction pins// temperature to extractorTemperature (0.2) to mirror generator.py.func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {+ if strings.TrimSpace(chunkText) == "" {+ return nil+ }if _, exists := ck["important_kwd"]; exists {return nil}@@ -757,6 +760,9 @@ func (c *ExtractorComponent) runAutoKeywords(ctx context.Context, db *gorm.DB, i// runAutoQuestions extracts questions for the current chunk and stores// them on ck["question_kwd"]. See runAutoKeywords for the temperature pin.func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {+ if strings.TrimSpace(chunkText) == "" {+ return nil+ }if _, exists := ck["question_kwd"]; exists {return nil}@@ -807,6 +813,9 @@ func (c *ExtractorComponent) runAutoQuestions(ctx context.Context, db *gorm.DB,// runAutoSummary extracts a concise summary for the current chunk using autoSummaryPrompt// and stores it on ck["summary"].func (c *ExtractorComponent) runAutoSummary(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {+ if strings.TrimSpace(chunkText) == "" {+ return nil+ }if _, exists := ck["summary"]; exists {return nil}@@ -845,6 +854,9 @@ func (c *ExtractorComponent) runAutoKeywordsPool(ctx context.Context, db *gorm.Dfor i, ck := range in.chunks {i, ck := i, cktext := extractorChunkText(ck)+ if strings.TrimSpace(text) == "" {+ continue+ }fn := func() error {if err := c.runAutoKeywords(ctx, db, in, ck, text); err != nil {return fmt.Errorf("chunk %d keywords: %w", i, err)@@ -870,6 +882,9 @@ func (c *ExtractorComponent) runRemainingExtractions(ctx context.Context, db *gofor i, ck := range in.chunks {i, ck := i, cktext := extractorChunkText(ck)+ if strings.TrimSpace(text) == "" {+ continue+ }fn := c.remainingExtractionJob(ctx, db, in, i, ck, text)f, err := extractorPool.Submit(ctx, fn)if err != nil {@@ -936,6 +951,9 @@ func awaitFutures(ctx context.Context, futs []utility.WorkerPoolFuture[extractor// runAutoKeywords/runAutoQuestions shape but parses a JSON object and// merges into the chunk's metadata map.func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB, in extractorInputs, ck map[string]any, chunkText string) error {+ if strings.TrimSpace(chunkText) == "" {+ return nil+ }if !c.Param.Metadata.Enabled || len(c.Param.Metadata.Metadata) == 0 {return nil}@@ -1006,10 +1024,24 @@ func (c *ExtractorComponent) runEnableMetadata(ctx context.Context, db *gorm.DB,}// extractorChunkText resolves the body an extraction is run against.-// Pre-index components must carry canonical string "text" only.+//+// Canonical text wins. Media-only chunks may use their explicit surrounding+// context; chunks with neither body nor context are skipped by the extraction+// schedulers.func extractorChunkText(ck map[string]any) string {- v, _ := ck["text"].(string)- return v+ if v, _ := ck["text"].(string); strings.TrimSpace(v) != "" {+ return v+ }+ var contextParts []string+ for _, key := range []string{"context_above", "context_below"} {+ if v, _ := ck[key].(string); strings.TrimSpace(v) != "" {+ contextParts = append(contextParts, strings.TrimSpace(v))+ }+ }+ if len(contextParts) > 0 {+ return strings.Join(contextParts, " ")+ }+ return ""}// chunkCacheID returns the chunk's stable per-chunk id, assigned by the chunker@@ -1063,6 +1095,9 @@ func setMetadataLLMCache(ctx context.Context, store chunkcache.Store, modelID, s// - callStructured wraps callRaw with cleanup + explicit JSON parsing and// returns a map — the metadata path, matching Python gen_metadata.func (c *ExtractorComponent) callRaw(ctx context.Context, db *gorm.DB, in extractorInputs, systemPrompt, chunkText string) (*extractorChatResponse, error) {+ if strings.TrimSpace(chunkText) == "" {+ return nil, fmt.Errorf("extractor: chunk text is empty")+ }driver, modelName, apiKey, baseURL, err := resolveExtractorChatTarget(ctx, db, in.llmID)if err != nil {return nil, err@@ -1548,8 +1583,9 @@ func fitExtractorMessages(ctx context.Context, db *gorm.DB, llmID string, msgs [}// buildExtractorMessages assembles system + user messages for one extraction-// call. The user message strictly carries chunkText (or a fallback single space-// if empty), ensuring a clean and consistent message contract.+// call. Normal per-chunk schedulers reject empty chunk text before reaching+// this low-level builder; the single-space fallback remains for direct+// callRaw callers that rely on the historical message shape.// System prompt is omitted if empty or whitespace-only to avoid sending empty// system turns to LLM providers.func buildExtractorMessages(systemPrompt, chunkText string) []eschema.Message {diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex 3863345322..9c9ab461c0 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -167,6 +167,43 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {}}+func TestExtractorComponent_SkipsMediaChunksWithoutTextAndUsesContext(t *testing.T) {+ stub := withStubChatInvoker(t, stubResponse{Content: "context summary"})++ c := &ExtractorComponent{Param: schema.ExtractorParam{+ LLMID: "gpt-4o-mini",+ Summary: schema.SummaryExtractConfig{Enabled: true},+ }}+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "chunks": []map[string]any{+ {"image": "image-only"},+ {"image": "context-image", "context_above": "The table shows", "context_below": "annual revenue."},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if calls := stub.Calls(); calls != 1 {+ t.Fatalf("LLM calls = %d, want one call for the context-bearing media chunk", calls)+ }+ chunks, ok := out["chunks"].([]map[string]any)+ if !ok || len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want two input chunks", out["chunks"])+ }+ if _, exists := chunks[0]["summary"]; exists {+ t.Errorf("image-only chunk received a summary: %#v", chunks[0]["summary"])+ }+ if chunks[1]["summary"] != "context summary" {+ t.Errorf("context-bearing media summary = %#v, want context summary", chunks[1]["summary"])+ }+ stub.mu.Lock()+ defer stub.mu.Unlock()+ request := stub.lastRequest()+ if got := request.Messages[len(request.Messages)-1].Content; got != "The table shows annual revenue." {+ t.Errorf("media extraction prompt = %q, want joined context", got)+ }+}+// TestExtractorComponent_Invoke_LLMError verifies a mock LLM// error is surfaced through Invoke with the component-name prefix.func TestExtractorComponent_Invoke_LLMError(t *testing.T) {@@ -957,7 +994,7 @@ func TestCleanLLMText(t *testing.T) {func TestExtractorComponent_callStructured(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: `{"a": 1}`})c := &ExtractorComponent{}- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "chunk text")if err != nil {t.Fatalf("callStructured: %v", err)}@@ -967,7 +1004,7 @@ func TestExtractorComponent_callStructured(t *testing.T) {// Non-JSON response → (nil, nil), not an error.withStubChatInvoker(t, stubResponse{Content: "this is not JSON"})- got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")+ got, err = c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "chunk text")if err != nil {t.Fatalf("callStructured on non-JSON: %v", err)}@@ -980,7 +1017,7 @@ func TestExtractorComponent_callStructured(t *testing.T) {func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {withStubChatInvoker(t, stubResponse{Content: `preamble<think>reasoning</think>{"a": 1}`})c := &ExtractorComponent{}- got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "")+ got, err := c.callStructured(t.Context(), nil, extractorInputs{llmID: "m"}, "system", "chunk text")if err != nil {t.Fatalf("callStructured: %v", err)}@@ -989,6 +1026,14 @@ func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {}}+func TestExtractorComponentRejectsEmptyLLMInput(t *testing.T) {+ withStubChatInvoker(t, stubResponse{Content: `{"unexpected": true}`})+ c := &ExtractorComponent{}+ if _, err := c.callRaw(t.Context(), nil, extractorInputs{llmID: "m"}, "system", " "); err == nil {+ t.Fatal("callRaw accepted an empty chunk text")+ }+}+// TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions verifies// keyword and question extraction on chunks.func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T) {diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex d264ac9513..a336963a44 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -274,6 +274,7 @@ func defaultSetups() map[string]schema.ParserSetup {"spreadsheet": {"parse_method": "deepdoc","flatten_media_to_text": false,+ "html4excel": false,"output_format": "json","suffix": []string{"xls", "xlsx", "csv"},},@@ -385,6 +386,7 @@ func (c *ParserComponent) Inputs() map[string]string {//// name string — carried over from the upstream file/document// name (or doc_id when no name is available).+// file_type string — canonical extension used for parser dispatch.// output_format string — always "json".// json []map[string]any — canonical structured parser items.// lang string — language for tokenization.@@ -399,6 +401,7 @@ func (c *ParserComponent) Inputs() map[string]string {func (c *ParserComponent) Outputs() map[string]string {return map[string]string{"name": "string: the upstream file/document name (or doc_id when no name is available).",+ "file_type": "string: canonical extension used for parser dispatch (for example pdf, md, xlsx, or other).","output_format": "string: always \"json\".","json": "[]map[string]any: canonical structured parser items.","lang": "string: the language for tokenization (e.g. English, Dutch, Chinese).",@@ -415,6 +418,7 @@ func (c *ParserComponent) Outputs() map[string]string {//// {// "name": string (from inputs["name"], file.name, or doc_id),+// "file_type": string (canonical extension used for parser dispatch),// "output_format": "json",// "json": []map[string]any,// "lang": string (from inputs["lang"]; e.g. English, Dutch),@@ -513,6 +517,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st}lang, _ := getString(inputs, "lang")out := buildParserOutputs(ctx, dispatched, filename, binary, lang)+ out["file_type"] = string(fileTypeExt)// Forward the storage references so a downstream chunker can// re-acquire the source PDF and crop section images on demand,// instead of carrying the binary across the component boundary.diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex 24daa44820..5770ec4e06 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -231,7 +231,7 @@ func pythonFamilyName(raw string) string {return "epub"case "json", "jsonl", "ldjson":return "json"- case "txt", "py", "js", "java", "c", "cpp", "h", "php",+ case "text&code", "txt", "py", "js", "java", "c", "cpp", "h", "php","go", "ts", "sh", "cs", "kt", "sql":return "text&code"case "mp4", "avi", "mkv", "mov", "webm", "flv",diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex ef20c6a6c4..d24b1b5521 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -142,8 +142,8 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {if got := items[0]["ck_type"]; got != "heading" {t.Errorf("markdown item[0].ck_type = %v, want heading", got)}- if got := items[0]["text"]; got != "Title" {- t.Errorf("markdown item[0].text = %v, want Title", got)+ if got := items[0]["text"]; got != "# Title" {+ t.Errorf("markdown item[0].text = %v, want # Title", got)}if got := items[1]["text"]; got != "Body" {t.Errorf("markdown item[1].text = %v, want Body", got)@@ -168,6 +168,29 @@ func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {requireJSONText(t, out, "Recovered title")}+func TestBuildParserOutputsPreservesPDFOutlineMetadata(t *testing.T) {+ wantOutline := []map[string]any{{"title": "Overview", "level": 0, "page_number": 1}}+ out := buildParserOutputs(t.Context(), parser.ParseResult{+ OutputFormat: "json",+ File: map[string]any{+ "name": "report.pdf",+ "outline": wantOutline,+ },+ JSON: []map[string]any{{"text": "body", "doc_type_kwd": "text"}},+ }, "report.pdf", nil, "")+ file, ok := out["file"].(map[string]any)+ if !ok {+ t.Fatalf("file = %T/%v, want parser file metadata", out["file"], out["file"])+ }+ gotOutline, ok := file["outline"].([]map[string]any)+ if !ok {+ t.Fatalf("file.outline = %T/%v, want []map[string]any", file["outline"], file["outline"])+ }+ if !reflect.DeepEqual(gotOutline, wantOutline) {+ t.Fatalf("file.outline = %#v, want %#v", gotOutline, wantOutline)+ }+}+func TestParseMarkdownToJSONItemsDoesNotResolveRemoteImages(t *testing.T) {originalResolver := net.DefaultResolverdnsLookup := make(chan struct{}, 1)diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.goindex aecbd754e0..4318713dd3 100644--- a/internal/ingestion/component/parser_test.go+++ b/internal/ingestion/component/parser_test.go@@ -90,7 +90,7 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {if _, ok := out["json"]; !ok {t.Errorf("Outputs() missing key %q", "json")}- for _, key := range []string{"name", "lang", "file", "doc_id", "bucket", "path"} {+ for _, key := range []string{"name", "file_type", "lang", "file", "doc_id", "bucket", "path"} {if _, ok := out[key]; !ok {t.Errorf("Outputs() missing runtime output key %q", key)}@@ -100,6 +100,58 @@ func TestParserComponent_InputsOutputs_NonEmpty(t *testing.T) {}}+func TestParserComponentInvokeOutputsResolvedFileType(t *testing.T) {+ tests := []struct {+ name string+ inputs map[string]any+ want string+ }{+ {+ name: "explicit family hint wins over filename",+ inputs: map[string]any{+ "binary": "package main",+ "file_type": "text&code",+ "name": "misleading.pdf",+ },+ want: "txt",+ },+ {+ name: "top-level name",+ inputs: map[string]any{"binary": "hello", "name": "notes.txt"},+ want: "txt",+ },+ {+ name: "file descriptor name",+ inputs: map[string]any{+ "binary": "hello",+ "file": map[string]any{"name": "notes.txt"},+ },+ want: "txt",+ },+ {+ name: "unknown input",+ inputs: map[string]any{"binary": "hello", "name": "notes.unknown"},+ want: "other",+ },+ }++ component := &ParserComponent{setups: defaultSetups()}+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ if routed := string(fileTypeFromInputs(test.inputs)); routed != test.want {+ t.Fatalf("routing file type = %q, want %q", routed, test.want)+ }+ out, err := component.Invoke(t.Context(), nil, test.inputs)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if got := out["file_type"]; got != test.want {+ t.Errorf("file_type = %v, want %q", got, test.want)+ }+ })+ }+}+func TestNewParserComponentNormalizesOutputFormatToJSON(t *testing.T) {component, err := NewParserComponent(map[string]any{"pdf": map[string]any{"output_format": "markdown"},diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex eb9aa1f893..f4d3372b35 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -124,6 +124,15 @@ type ChunkDoc struct {TopInt []int `json:"top_int,omitempty"`PDFPositions json.RawMessage `json:"_pdf_positions,omitempty"`Positions json.RawMessage `json:"positions,omitempty"`+ TableID string `json:"table_id,omitempty"`+ Sheet string `json:"sheet,omitempty"`+ SheetIndex *int `json:"sheet_index,omitempty"`+ Headers []string `json:"headers,omitempty"`+ Cells []string `json:"cells,omitempty"`+ RowStart *int `json:"row_start,omitempty"`+ RowEnd *int `json:"row_end,omitempty"`+ ColStart *int `json:"col_start,omitempty"`+ ColEnd *int `json:"col_end,omitempty"`Extra map[string]json.RawMessage `json:"-"`}@@ -143,6 +152,7 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {"context_above", "context_below", "questions", "keywords", "summary","chunk_order_int", "title_tks", "title_sm_tks", "content_ltks","content_sm_ltks", "tag_kwd", "page_number", "top_int", "_pdf_positions", "positions",+ "table_id", "sheet", "sheet_index", "headers", "cells", "row_start", "row_end", "col_start", "col_end",} {delete(raw, key)}@@ -236,6 +246,10 @@ func (d ChunkDoc) ToMap() map[string]any {for k, raw := range d.Extra {out[k] = decodeExtraValue(raw)}+ // Every pre-index chunk has one canonical text field. Keep the key even+ // for media-only chunks so chunk identity and image upload use the same+ // wire contract; contextual text remains explicit retrieval metadata.+ out["text"] = d.Textdelete(out, "content_with_weight")if len(d.PDFPositions) > 0 {out["_pdf_positions"] = decodeStructuredValue(d.PDFPositions)diff --git a/internal/ingestion/component/schema/chunker.go b/internal/ingestion/component/schema/chunker.goindex 3cb5987527..3b280e4cb5 100644--- a/internal/ingestion/component/schema/chunker.go+++ b/internal/ingestion/component/schema/chunker.go@@ -23,9 +23,9 @@ import ("strings")-// ChunkerFromUpstream is the shared upstream payload consumed by all four-// chunker variants (TokenChunker, TitleChunker, GroupTitleChunker,-// HierarchyTitleChunker).+// ChunkerFromUpstream is the shared upstream payload consumed by the chunker+// variants (GeneralChunker, TokenChunker, TitleChunker, GroupTitleChunker,+// HierarchyTitleChunker, and QAChunker).//// It mirrors the wire shape defined across two equivalent Pydantic// schemas in the Python codebase:@@ -41,6 +41,7 @@ import (// created_time: float | None (alias _created_time)// elapsed_time: float | None (alias _elapsed_time)// name: str (required)+// file_type: str (canonical parser routing extension)// file: dict | None// chunks: list[dict] | None// output_format: Literal["json","markdown","text","html","chunks"] | None@@ -55,6 +56,9 @@ type ChunkerFromUpstream struct {// Name is the source document name. Required.Name string `json:"name"`+ // FileType is the canonical extension selected by the upstream Parser.+ FileType string `json:"file_type,omitempty"`+// DocID is the originating document ID. When set, the chunker can// re-resolve the source PDF from storage to crop section images on// demand, instead of carrying the binary across the componentdiff --git a/internal/ingestion/component/schema/parser.go b/internal/ingestion/component/schema/parser.goindex da429fa717..e70b09b5a2 100644--- a/internal/ingestion/component/schema/parser.go+++ b/internal/ingestion/component/schema/parser.go@@ -27,6 +27,9 @@ type ParserOutputs struct {// Name is the resolved source filename.Name string `json:"name"`+ // FileType is the canonical extension used for parser dispatch.+ FileType string `json:"file_type"`+// OutputFormat is always "json". Downstream components consume structured JSON items.OutputFormat string `json:"output_format,omitempty"`diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex 0943c23a73..eb1e745a79 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -18,6 +18,7 @@ package schemaimport ("encoding/json"+ "reflect""strings""testing")@@ -123,6 +124,7 @@ func TestFileOutputsJSONRoundTrip(t *testing.T) {func TestParserOutputsJSONRoundTrip(t *testing.T) {original := ParserOutputs{Name: "input.pdf",+ FileType: "pdf",OutputFormat: "json",JSON: []map[string]any{{"text": "hello", "doc_type_kwd": "text"}},Lang: "English",@@ -138,6 +140,9 @@ func TestParserOutputsJSONRoundTrip(t *testing.T) {if !strings.Contains(string(data), `"output_format":"json"`) {t.Errorf("expected output_format in JSON, got %s", data)}+ if !strings.Contains(string(data), `"file_type":"pdf"`) {+ t.Errorf("expected file_type in JSON, got %s", data)+ }var decoded ParserOutputsif err := json.Unmarshal(data, &decoded); err != nil {t.Fatalf("unmarshal: %v", err)@@ -145,6 +150,9 @@ func TestParserOutputsJSONRoundTrip(t *testing.T) {if decoded.OutputFormat != "json" {t.Errorf("OutputFormat round-trip mismatch: got %q", decoded.OutputFormat)}+ if decoded.FileType != "pdf" {+ t.Errorf("FileType round-trip mismatch: got %q", decoded.FileType)+ }if len(decoded.JSON) != 1 {t.Errorf("JSON round-trip mismatch: got %d", len(decoded.JSON))}@@ -198,6 +206,7 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {md := "# title"original := ChunkerFromUpstream{Name: "doc.pdf",+ FileType: "pdf",OutputFormat: PayloadFormatChunks,Chunks: []ChunkDoc{{Text: "alpha"}},MarkdownResult: &md,@@ -209,6 +218,9 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {if !strings.Contains(string(data), `"output_format":"chunks"`) {t.Errorf("expected output_format in JSON, got %s", data)}+ if !strings.Contains(string(data), `"file_type":"pdf"`) {+ t.Errorf("expected file_type in JSON, got %s", data)+ }var decoded ChunkerFromUpstreamif err := json.Unmarshal(data, &decoded); err != nil {t.Fatalf("unmarshal: %v", err)@@ -216,11 +228,56 @@ func TestChunkerFromUpstreamJSONRoundTrip(t *testing.T) {if decoded.Name != "doc.pdf" || decoded.OutputFormat != PayloadFormatChunks {t.Errorf("round-trip mismatch: %+v", decoded)}+ if decoded.FileType != "pdf" {+ t.Errorf("FileType round-trip mismatch: got %q", decoded.FileType)+ }if len(decoded.Chunks) != 1 {t.Errorf("Chunks round-trip mismatch: got %d", len(decoded.Chunks))}}+func TestChunkDocSpreadsheetFieldsRoundTrip(t *testing.T) {+ sheetIndex, rowStart, rowEnd, colStart, colEnd := 2, 42, 42, 1, 3+ original := ChunkDoc{+ Text: "ID:A-100; Status:paid",+ DocType: "text",+ CKType: "table_row",+ TableID: "sheet-2",+ Sheet: "Orders",+ SheetIndex: &sheetIndex,+ Headers: []string{"ID", "Status", "Note"},+ Cells: []string{"A-100", "paid", ""},+ RowStart: &rowStart,+ RowEnd: &rowEnd,+ ColStart: &colStart,+ ColEnd: &colEnd,+ }++ data, err := json.Marshal(original)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ var decoded ChunkDoc+ if err := json.Unmarshal(data, &decoded); err != nil {+ t.Fatalf("unmarshal: %v", err)+ }+ if decoded.TableID != original.TableID || decoded.Sheet != original.Sheet {+ t.Fatalf("spreadsheet identity mismatch: got table=%q sheet=%q", decoded.TableID, decoded.Sheet)+ }+ if decoded.SheetIndex == nil || *decoded.SheetIndex != sheetIndex {+ t.Fatalf("sheet index mismatch: got %v", decoded.SheetIndex)+ }+ if decoded.RowStart == nil || *decoded.RowStart != rowStart || decoded.RowEnd == nil || *decoded.RowEnd != rowEnd {+ t.Fatalf("row range mismatch: start=%v end=%v", decoded.RowStart, decoded.RowEnd)+ }+ if decoded.ColStart == nil || *decoded.ColStart != colStart || decoded.ColEnd == nil || *decoded.ColEnd != colEnd {+ t.Fatalf("column range mismatch: start=%v end=%v", decoded.ColStart, decoded.ColEnd)+ }+ if !reflect.DeepEqual(decoded.Headers, original.Headers) || !reflect.DeepEqual(decoded.Cells, original.Cells) {+ t.Fatalf("cells mismatch: headers=%v cells=%v", decoded.Headers, decoded.Cells)+ }+}+func TestChunkerOutputsJSONRoundTrip(t *testing.T) {original := ChunkerOutputs{OutputFormat: PayloadFormatChunks,diff --git a/internal/ingestion/component/tokenizer.go b/internal/ingestion/component/tokenizer.goindex 533fb5c75f..e580db68af 100644--- a/internal/ingestion/component/tokenizer.go+++ b/internal/ingestion/component/tokenizer.go@@ -646,10 +646,15 @@ func chunksFromTokenizerUpstream(in schema.TokenizerFromUpstream) []schema.Chunkdefault:raw = cloneChunkDocs(in.JSONResult)}- // Keep only chunks that carry canonical pre-index text.+ // Keep only chunks that have retrievable content: a chunk is dropped only+ // when canonical text and both media-context fields are empty.+ // Context-bearing media chunks may intentionally have no display text, but+ // their surrounding prose is still searchable and must survive to the+ // tokenizer.filtered := raw[:0]for _, ck := range raw {- if strings.TrimSpace(ck.Text) == "" {+ if strings.TrimSpace(ck.Text) == "" &&+ strings.TrimSpace(ck.ContextAbove) == "" && strings.TrimSpace(ck.ContextBelow) == "" {continue}filtered = append(filtered, ck)@@ -786,7 +791,7 @@ func tokenizeChunks(chunks []schema.ChunkDoc, titleStem string, language string)smt = st}ck.ContentSmLtks = smt- } else if t := ck.Text; strings.TrimSpace(t) != "" {+ } else if t := contextualChunkText(*ck); strings.TrimSpace(t) != "" {tt, err := tok.Tokenize(t)if err != nil {return fmt.Errorf("tokenizer: text tokenize: %w", err)@@ -816,7 +821,7 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {for _, f := range fields {switch f {case "text":- b.WriteString(ck.Text)+ b.WriteString(contextualChunkText(ck))case "questions":b.WriteString(ck.Questions)case "keywords":@@ -836,6 +841,34 @@ func concatFields(ck schema.ChunkDoc, fields []string) string {return b.String()}+// contextualChunkText is the retrieval representation of a chunk's body.+// ContextAbove/ContextBelow are generated by GeneralChunker for media blocks+// and must participate in both full-text tokenization and embeddings, while+// ChunkDoc.Text remains the display/storage payload (for example, the table+// HTML or image caption). Context values normally carry their own punctuation+// or whitespace, matching Python's doc_tokenize_chunks_with_images contract.+// If a producer trims both sides of a boundary, contextualChunkText inserts a+// newline so retrieval and embedding do not fuse adjacent English words.+func contextualChunkText(ck schema.ChunkDoc) string {+ text := ""+ for _, part := range []string{ck.ContextAbove, ck.Text, ck.ContextBelow} {+ if part == "" {+ continue+ }+ if text != "" && !contextBoundaryHasWhitespace(text, part) {+ text += "\n"+ }+ text += part+ }+ return text+}++func contextBoundaryHasWhitespace(left, right string) bool {+ leftRune, _ := utf8.DecodeLastRuneInString(left)+ rightRune, _ := utf8.DecodeRuneInString(right)+ return unicode.IsSpace(leftRune) || unicode.IsSpace(rightRune)+}+// shouldHaveEmbedding reports whether the tokenizer must attach embedding// vectors: the search method requests embedding AND a KB is present.//@@ -871,7 +904,7 @@ func validateTokenizerOutputs(chunks []schema.ChunkDoc, searchMethods, fields []}func requiresFullTextTokens(ck schema.ChunkDoc) bool {- return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(ck.Text) != ""+ return strings.TrimSpace(ck.Summary) != "" || strings.TrimSpace(contextualChunkText(ck)) != ""}func requiresEmbeddingVector(ck schema.ChunkDoc, fields []string) bool {diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 1a616939c7..ae05a28a9f 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -240,6 +240,81 @@ func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {}}+func TestTokenizerComponent_FullTextIncludesMediaContext(t *testing.T) {+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")++ comp, err := NewTokenizerComponent(map[string]any{+ "search_method": []any{"full_text"},+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponent: %v", err)+ }+ out, err := comp.(*TokenizerComponent).Invoke(t.Context(), nil, map[string]any{+ "name": "report.pdf",+ "output_format": "chunks",+ "chunks": []map[string]any{+ {+ "text": "table body",+ "context_above": "above ",+ "context_below": " below",+ },+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := out["chunks"].([]map[string]any)+ if got := chunks[0]["content_ltks"]; got != "above table body below" {+ t.Fatalf("content_ltks = %q, want contextual text", got)+ }+ if got := chunks[0]["content_sm_ltks"]; got != "above table body below" {+ t.Fatalf("content_sm_ltks = %q, want contextual text", got)+ }+}++func TestTokenizerComponent_EmbeddingIncludesMediaContext(t *testing.T) {+ stub := newStubEmbedder(3)+ comp, err := NewTokenizerComponentWithResolver(+ map[string]any{"search_method": []any{"embedding"}},+ func(context.Context, string, string) (Embedder, string, error) {+ return stub, "", nil+ },+ )+ if err != nil {+ t.Fatalf("NewTokenizerComponentWithResolver: %v", err)+ }+ _, err = comp.(*TokenizerComponent).Invoke(t.Context(), nil, map[string]any{+ "output_format": "chunks",+ "chunks": []map[string]any{{+ "text": "table body",+ "context_above": "above ",+ "context_below": " below",+ }},+ "kb_id": "kb-1",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ if len(stub.callInputs) != 1 || len(stub.callInputs[0]) != 1 {+ t.Fatalf("embedder calls = %#v, want one content input", stub.callInputs)+ }+ if got := stub.callInputs[0][0]; got != "above table body below" {+ t.Fatalf("embedding input = %q, want contextual text", got)+ }+}++func TestContextualChunkTextSeparatesTightMediaBoundaries(t *testing.T) {+ ck := schema.ChunkDoc{+ ContextAbove: "See results in table:",+ Text: "Revenue grew by 20%.",+ ContextBelow: "Conclusion follows.",+ }+ if got, want := contextualChunkText(ck), "See results in table:\nRevenue grew by 20%.\nConclusion follows."; got != want {+ t.Fatalf("contextual text = %q, want %q", got, want)+ }+}+// TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero uses an// empty chunk list, so tokenizeChunks is a no-op and the C++ pool is not needed.func TestTokenizerComponent_Embedding_ZeroChunksStillEmitsConsumptionZero(t *testing.T) {@@ -543,6 +618,36 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {}}+func TestTokenizerComponent_KeepsContextOnlyMediaChunk(t *testing.T) {+ tokenizer.SetEngineType("infinity")+ defer tokenizer.SetEngineType("")++ comp, err := NewTokenizerComponent(map[string]any{+ "search_method": []any{"full_text"},+ })+ if err != nil {+ t.Fatalf("NewTokenizerComponent: %v", err)+ }+ out, err := comp.(*TokenizerComponent).Invoke(t.Context(), nil, map[string]any{+ "name": "report.pdf",+ "output_format": "chunks",+ "chunks": []map[string]any{{+ "image": "data:image/png;base64,placeholder",+ "context_above": "figure description",+ }},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %#v, want context-bearing media chunk retained", chunks)+ }+ if got := chunks[0]["content_ltks"]; got != "figure description" {+ t.Fatalf("content_ltks = %q, want media context", got)+ }+}+// TestChunkOrderInt_AllPathsUnconditional is the A6 regression gate: every// surviving chunk must carry chunk_order_int, equal to its index in the// (post-filter) reading sequence — on every output_format and everydiff --git a/internal/ingestion/pipeline/general_template_contract_test.go b/internal/ingestion/pipeline/general_template_contract_test.gonew file mode 100644index 0000000000..c10bb074cf--- /dev/null+++ b/internal/ingestion/pipeline/general_template_contract_test.go@@ -0,0 +1,127 @@+package pipeline++import (+ "encoding/json"+ "testing"+)++func TestGeneralTemplateUsesGeneralChunker(t *testing.T) {+ dsl, err := LoadBuiltinDSL("general")+ if err != nil {+ t.Fatalf("LoadBuiltinDSL: %v", err)+ }++ var root map[string]interface{}+ if err := json.Unmarshal([]byte(dsl), &root); err != nil {+ t.Fatalf("decode general DSL: %v", err)+ }+ components, ok := root["components"].(map[string]interface{})+ if !ok {+ t.Fatal("general DSL has no components")+ }++ const generalID = "GeneralChunker:SixApplesFall"+ if _, ok := components[generalID]; !ok {+ t.Fatalf("general DSL has no %s component", generalID)+ }+ if _, ok := components["TokenChunker:SixApplesFall"]; ok {+ t.Fatal("general DSL still contains the legacy TokenChunker component")+ }+ component := components[generalID].(map[string]interface{})+ obj := component["obj"].(map[string]interface{})+ if got := obj["component_name"]; got != "GeneralChunker" {+ t.Fatalf("component_name = %v, want GeneralChunker", got)+ }+ params := obj["params"].(map[string]interface{})+ if _, ok := params["delimiter_mode"]; ok {+ t.Fatal("GeneralChunker params contain unsupported delimiter_mode")+ }+ delimiters, ok := params["delimiters"].([]interface{})+ if !ok {+ t.Fatalf("delimiters type = %T", params["delimiters"])+ }+ seenSemicolon := false+ for _, delimiter := range delimiters {+ if delimiter == ";" {+ seenSemicolon = true+ }+ }+ if !seenSemicolon {+ t.Fatalf("general delimiters = %#v, want ASCII semicolon", delimiters)+ }++ parserComponent, ok := components["Parser:HipSignsRhyme"].(map[string]interface{})+ if !ok {+ t.Fatal("general DSL has no Parser:HipSignsRhyme component")+ }+ parserObj, ok := parserComponent["obj"].(map[string]interface{})+ if !ok {+ t.Fatal("general parser has no object definition")+ }+ parserParams, ok := parserObj["params"].(map[string]interface{})+ if !ok {+ t.Fatal("general parser has no params")+ }+ if _, ok := parserParams["outputs"]; ok {+ t.Fatal("general parser retains obsolete html/markdown/text output declarations")+ }+ spreadsheet, ok := parserParams["spreadsheet"].(map[string]interface{})+ if !ok {+ t.Fatal("general parser has no spreadsheet setup")+ }+ if got := spreadsheet["output_format"]; got != "json" {+ t.Fatalf("general spreadsheet output_format = %v, want json", got)+ }+}++func TestGeneralTemplateParserGraphUsesJSONContract(t *testing.T) {+ dsl, err := LoadBuiltinDSL("general")+ if err != nil {+ t.Fatalf("LoadBuiltinDSL: %v", err)+ }+ var root map[string]interface{}+ if err := json.Unmarshal([]byte(dsl), &root); err != nil {+ t.Fatalf("decode general DSL: %v", err)+ }+ graph, ok := root["graph"].(map[string]interface{})+ if !ok {+ t.Fatal("general DSL has no graph")+ }+ nodes, ok := graph["nodes"].([]interface{})+ if !ok {+ t.Fatal("general graph has no nodes")+ }+ for _, rawNode := range nodes {+ node, ok := rawNode.(map[string]interface{})+ if !ok || node["id"] != "Parser:HipSignsRhyme" {+ continue+ }+ data, ok := node["data"].(map[string]interface{})+ if !ok {+ t.Fatal("general parser graph node has no data")+ }+ form, ok := data["form"].(map[string]interface{})+ if !ok {+ t.Fatal("general parser graph node has no form")+ }+ if _, ok := form["outputs"]; ok {+ t.Fatal("general parser graph form retains obsolete output declarations")+ }+ setups, ok := form["setups"].([]interface{})+ if !ok {+ t.Fatal("general parser graph form has no setups")+ }+ for _, rawSetup := range setups {+ setup, ok := rawSetup.(map[string]interface{})+ if !ok || setup["fileFormat"] != "spreadsheet" {+ continue+ }+ if got := setup["output_format"]; got != "json" {+ t.Fatalf("general graph spreadsheet output_format = %v, want json", got)+ }+ return+ }+ t.Fatal("general parser graph form has no spreadsheet setup")+ }+ t.Fatal("general graph has no Parser:HipSignsRhyme node")+}diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 82312e8cce..08d75876c2 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -8,6 +8,7 @@ import ("ragflow/internal/common""ragflow/internal/entity""ragflow/internal/ingestion/component/schema"+ parserchunk "ragflow/internal/parser/chunk""go.uber.org/zap")@@ -95,12 +96,17 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[}validCPNs := make(map[string]map[string]struct{}, len(schemas))+ componentNames := make(map[string]string, len(schemas))for _, s := range schemas {keys := make(map[string]struct{}, len(s.ParamsDefaults))for k := range s.ParamsDefaults {keys[k] = struct{}{}}+ if s.ComponentName == "GeneralChunker" {+ keys["delimiters"] = struct{}{}+ }validCPNs[s.CpnID] = keys+ componentNames[s.CpnID] = s.ComponentName}result := make(map[string]interface{}, len(rawConfig))@@ -120,6 +126,9 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[if !ok {continue}+ if componentNames[key] == "GeneralChunker" {+ params = normalizeGeneralComponentParams(params)+ }dynamicWhitelist, hasDynamic := getComponentParamWhitelist(key)if hasDynamic {params = NormalizeExtractorParams(params)@@ -147,6 +156,35 @@ func CleanComponentParams(dslJSON []byte, rawConfig map[string]interface{}) map[return result}+func normalizeGeneralComponentParams(params map[string]any) map[string]any {+ var normalized map[string]any+ clone := func() map[string]any {+ if normalized == nil {+ normalized = make(map[string]any, len(params)+1)+ for key, value := range params {+ normalized[key] = value+ }+ }+ return normalized+ }+ if _, hasCanonical := params["delimiters"]; !hasCanonical {+ if value, ok := params["delimiter"].(string); ok {+ normalizedParams := clone()+ normalizedParams["delimiters"] = parserchunk.ParseDelimiterField(value)+ delete(normalizedParams, "delimiter")+ }+ } else if value, ok := params["delimiters"].(string); ok {+ clone()["delimiters"] = parserchunk.ParseDelimiterField(value)+ }+ if value, ok := params["children_delimiters"].(string); ok {+ clone()["children_delimiters"] = parserchunk.ParseDelimiterField(value)+ }+ if normalized == nil {+ return params+ }+ return normalized+}+// NormalizeExtractorParams normalizes a raw Extractor parameters map into the canonical modular format.func NormalizeExtractorParams(raw map[string]any) map[string]any {if raw == nil {diff --git a/internal/ingestion/pipeline/pipeline_params_test.go b/internal/ingestion/pipeline/pipeline_params_test.goindex fbe0aee400..33422be57d 100644--- a/internal/ingestion/pipeline/pipeline_params_test.go+++ b/internal/ingestion/pipeline/pipeline_params_test.go@@ -2,6 +2,7 @@ package pipelineimport ("encoding/json"+ "reflect""testing""ragflow/internal/entity"@@ -41,6 +42,77 @@ func generalDSL(t *testing.T) []byte {}return raw}++func TestCleanComponentParamsNormalizesGeneralLegacyDelimiter(t *testing.T) {+ dsl := map[string]any{+ "components": map[string]any{+ "GeneralChunker:Legacy": map[string]any{+ "obj": map[string]any{+ "component_name": "GeneralChunker",+ "params": map[string]any{+ "delimiters": []any{"\n"},+ },+ },+ },+ },+ }+ dslJSON, err := json.Marshal(dsl)+ if err != nil {+ t.Fatalf("marshal dsl: %v", err)+ }+ result := CleanComponentParams(dslJSON, map[string]any{+ "GeneralChunker:Legacy": map[string]any{+ "delimiter": "\n!?;。;!?",+ },+ })+ params, ok := result["GeneralChunker:Legacy"].(map[string]any)+ if !ok {+ t.Fatalf("normalized component params = %#v", result)+ }+ want := []string{"\n", "!", "?", ";", "。", ";", "!", "?"}+ if !reflect.DeepEqual(params["delimiters"], want) {+ t.Fatalf("delimiters = %#v, want %#v", params["delimiters"], want)+ }+ if _, ok := params["delimiter"]; ok {+ t.Fatal("legacy delimiter key survived normalization")+ }+}++func TestCleanComponentParamsNormalizesDelimiterWhenDSLDeclaresOnlyLegacyKey(t *testing.T) {+ dsl := map[string]any{+ "components": map[string]any{+ "GeneralChunker:Custom": map[string]any{+ "obj": map[string]any{+ "component_name": "GeneralChunker",+ "params": map[string]any{+ "delimiter": "\n",+ },+ },+ },+ },+ }+ dslJSON, err := json.Marshal(dsl)+ if err != nil {+ t.Fatalf("marshal DSL: %v", err)+ }+ result := CleanComponentParams(dslJSON, map[string]any{+ "GeneralChunker:Custom": map[string]any{+ "delimiter": "\n!?",+ },+ })+ params, ok := result["GeneralChunker:Custom"].(map[string]any)+ if !ok {+ t.Fatalf("normalized component params = %#v", result)+ }+ want := []string{"\n", "!", "?"}+ if !reflect.DeepEqual(params["delimiters"], want) {+ t.Fatalf("delimiters = %#v, want %#v", params["delimiters"], want)+ }+ if _, ok := params["delimiter"]; ok {+ t.Fatal("legacy delimiter key survived normalization")+ }+}+func TestCleanComponentParams_DropsLegacyFlatFields(t *testing.T) {dslJSON := generalDSL(t)raw := map[string]any{diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.goindex 4ec571ad01..b6ad120e83 100644--- a/internal/ingestion/pipeline/real_storage_integration_test.go+++ b/internal/ingestion/pipeline/real_storage_integration_test.go@@ -121,12 +121,11 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}- // The TokenChunker merges the two short paragraphs into one chunk under+ // The GeneralChunker merges the two short paragraphs into one chunk under// the global token-size budget (chunk_token_size=512, no backtick// delimiter): this matches Python's _merge_text_chunks_by_token_size and- // is the intended behaviour. The parser still emits two json items- // (see the Parser state assertion below); only the downstream chunker- // collapses them.+ // is the intended behaviour. TextParser emits one normalized json item;+ // GeneralChunker owns the delimiter split and collapses the atoms.wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}if len(chunks) != len(wantChunkTexts) {t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))@@ -169,21 +168,19 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {t.Fatalf("parser output_format = %v, want json", got)}jsonItems, ok := parserState["json"].([]map[string]any)- if !ok || len(jsonItems) != 2 {- t.Fatalf("parser json = %T/%v, want 2 items", parserState["json"], parserState["json"])+ if !ok || len(jsonItems) != 1 {+ t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])}- // The parser emits one json item per paragraph (two items), which the- // downstream TokenChunker later merges into a single chunk.- wantParserTexts := []string{"Alpha paragraph.", "Beta paragraph."}+ wantParserTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}for i, wantText := range wantParserTexts {if got := jsonItems[i]["text"]; got != wantText {t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)}}- chunkerState, ok := state["TokenChunker:SixApplesFall"]+ chunkerState, ok := state["GeneralChunker:SixApplesFall"]if !ok {- t.Fatal("missing TokenChunker:SixApplesFall state")+ t.Fatal("missing GeneralChunker:SixApplesFall state")}if got := chunkerState["output_format"]; got != "chunks" {t.Fatalf("chunker output_format = %v, want chunks", got)diff --git a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json b/internal/ingestion/pipeline/template/ingestion_pipeline_general.jsonindex fb237bd9cb..f745094bf4 100644--- a/internal/ingestion/pipeline/template/ingestion_pipeline_general.json+++ b/internal/ingestion/pipeline/template/ingestion_pipeline_general.json@@ -26,29 +26,11 @@},"Parser:HipSignsRhyme": {"downstream": [- "TokenChunker:SixApplesFall"+ "GeneralChunker:SixApplesFall"],"obj": {"component_name": "Parser","params": {- "outputs": {- "html": {- "type": "string",- "value": ""- },- "json": {- "type": "Array<object>",- "value": []- },- "markdown": {- "type": "string",- "value": ""- },- "text": {- "type": "string",- "value": ""- }- },"doc": {"output_format": "json","preprocess": [@@ -112,7 +94,7 @@},"spreadsheet": {"flatten_media_to_text": false,- "output_format": "html",+ "output_format": "json","parse_method": "DeepDOC","preprocess": ["main_content"@@ -152,20 +134,20 @@"File"]},- "TokenChunker:SixApplesFall": {+ "GeneralChunker:SixApplesFall": {"downstream": ["Extractor:AutoExtractDefault"],"obj": {- "component_name": "TokenChunker",+ "component_name": "GeneralChunker","params": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "delimiter","delimiters": ["\n","!","?",+ ";","。",";","!",@@ -206,7 +188,7 @@},"Extractor:AutoExtractDefault": {"upstream": [- "TokenChunker:SixApplesFall"+ "GeneralChunker:SixApplesFall"],"downstream": ["Tokenizer:LegalReadersDecide"@@ -250,15 +232,15 @@"targetHandle": "end"},{- "id": "xy-edge__Parser:HipSignsRhymestart-TokenChunker:SixApplesFallend",+ "id": "xy-edge__Parser:HipSignsRhymestart-GeneralChunker:SixApplesFallend","source": "Parser:HipSignsRhyme","sourceHandle": "start",- "target": "TokenChunker:SixApplesFall",+ "target": "GeneralChunker:SixApplesFall","targetHandle": "end"},{- "id": "xy-edge__TokenChunker:SixApplesFallstart-Extractor:AutoExtractDefaultend",- "source": "TokenChunker:SixApplesFall",+ "id": "xy-edge__GeneralChunker:SixApplesFallstart-Extractor:AutoExtractDefaultend",+ "source": "GeneralChunker:SixApplesFall","sourceHandle": "start","target": "Extractor:AutoExtractDefault","targetHandle": "end"@@ -293,24 +275,6 @@{"data": {"form": {- "outputs": {- "html": {- "type": "string",- "value": ""- },- "json": {- "type": "Array<object>",- "value": []- },- "markdown": {- "type": "string",- "value": ""- },- "text": {- "type": "string",- "value": ""- }- },"setups": [{"fileFormat": "pdf",@@ -325,7 +289,7 @@{"fileFormat": "spreadsheet","flatten_media_to_text": false,- "output_format": "html",+ "output_format": "json","parse_method": "DeepDOC","preprocess": ["main_content"@@ -396,7 +360,6 @@"form": {"children_delimiters": [],"chunk_token_size": 512,- "delimiter_mode": "delimiter","delimiters": [{"value": "\n"@@ -407,6 +370,9 @@{"value": "?"},+ {+ "value": ";"+ },{"value": "。"},@@ -429,10 +395,10 @@},"overlapped_percent": 0},- "label": "TokenChunker",- "name": "Token Chunker"+ "label": "GeneralChunker",+ "name": "General Chunker"},- "id": "TokenChunker:SixApplesFall",+ "id": "GeneralChunker:SixApplesFall","measured": {"height": 74,"width": 200diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex d05aad94e3..3dcaae9fd9 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -133,9 +133,9 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {}state := stateFromRunOutput(t, out)- chunkerState, ok := state["TokenChunker:SixApplesFall"]+ chunkerState, ok := state["GeneralChunker:SixApplesFall"]if !ok {- t.Fatal("missing TokenChunker:SixApplesFall state")+ t.Fatal("missing GeneralChunker:SixApplesFall state")}if got := chunkerState["output_format"]; got != "chunks" {t.Fatalf("chunker output_format = %v, want chunks", got)@@ -191,7 +191,6 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {}payload := terminalPayloadFromRunOutput(t, out, terminalIDs[0])- wantTexts := []string{"Alpha paragraph.", "Beta paragraph."}wantMergedText := "Alpha paragraph.\nBeta paragraph."assertTokenizerTerminalChunk(t, payload, filename, wantMergedText)@@ -217,14 +216,11 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {t.Fatalf("parser output_format = %v, want json", got)}jsonItems, ok := parserState["json"].([]map[string]any)- if !ok || len(jsonItems) != 2 {- t.Fatalf("parser json = %T/%v, want 2 items", parserState["json"], parserState["json"])+ if !ok || len(jsonItems) != 1 {+ t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])}- for i, wantText := range wantTexts {- item := jsonItems[i]- if got := item["text"]; got != wantText {- t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)- }+ if got := jsonItems[0]["text"]; got != "Alpha paragraph.\n\nBeta paragraph." {+ t.Fatalf("parser json[0].text = %v, want normalized full text", got)}chunkerState, ok := state["OneChunker:DryDrinksVisit"]if !ok {diff --git a/internal/ingestion/task/pipeline_executor_defaults_test.go b/internal/ingestion/task/pipeline_executor_defaults_test.goindex f4b5d5d561..82cc5529a2 100644--- a/internal/ingestion/task/pipeline_executor_defaults_test.go+++ b/internal/ingestion/task/pipeline_executor_defaults_test.go@@ -41,7 +41,7 @@ var builtinComponentParamsGolden = map[string]string{"audio": "{\"File\": {}, \"Parser:SongsFillAir\": {\"audio\": {\"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"aac\", \"aiff\", \"ape\", \"au\", \"da\", \"flac\", \"midi\", \"mp3\", \"ogg\", \"oggvorbis\", \"realaudio\", \"vqf\", \"wav\", \"wave\", \"wma\"]}}, \"TokenChunker:BlueSkiesLaugh\": {}, \"Tokenizer:KindEyesWatch\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","book": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:GrumpyGarlicsBake\": {\"hierarchy\": 5, \"include_heading_content\": true, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:HotDonutsRing\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","email": "{\"File\": {}, \"Parser:BirdsFlutterHigh\": {\"email\": {\"fields\": [\"from\", \"to\", \"cc\", \"bcc\", \"date\", \"subject\", \"body\", \"attachments\"], \"output_format\": \"text\", \"preprocess\": [\"main_content\"], \"suffix\": [\"eml\"]}}, \"TokenChunker:WarmBreadSmells\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:NiceWordsSpoken\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",- "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"TokenChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiter_mode\": \"delimiter\", \"delimiters\": [\"\\n\", \"!\", \"?\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",+ "general": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"pages\": [[1, 100000]], \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\", \"csv\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\", \"py\", \"js\", \"java\", \"c\", \"cpp\", \"h\", \"php\", \"go\", \"ts\", \"sh\", \"cs\", \"kt\", \"sql\"]}}, \"GeneralChunker:SixApplesFall\": {\"children_delimiters\": [], \"chunk_token_size\": 512, \"delimiters\": [\"\\n\", \"!\", \"?\", \";\", \"。\", \";\", \"!\", \"?\"], \"image_context_size\": 0, \"overlapped_percent\": 0, \"table_context_size\": 0}, \"Tokenizer:LegalReadersDecide\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","laws": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"remove_toc\": true, \"suffix\": [\"pdf\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"TitleChunker:SpicyKeysKick\": {\"hierarchy\": 2, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"hierarchy\"}, \"Tokenizer:PublicJobsTake\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","manual": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}}, \"ManualChunker:NineInsectsFind\": {\"hierarchy\": 0, \"include_heading_content\": false, \"levels\": [[\"^#[^#]\", \"^##[^#]\", \"^###[^#]\", \"^####[^#]\"], [\"第[零一二三四五六七八九十百0-9]+(分?编|部分)\", \"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"第[零一二三四五六七八九十百0-9]+条\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\"], [\"第[0-9]+章\", \"第[0-9]+节\", \"[0-9]{1,2}[\\\\. 、]\", \"[0-9]{1,2}\\\\.[0-9]{1,2}($|[^a-zA-Z/%~.-])\", \"[0-9]{1,2}\\\\.[0-9]{1,2}\\\\.[0-9]{1,2}\"], [\"第[零一二三四五六七八九十百0-9]+章\", \"第[零一二三四五六七八九十百0-9]+节\", \"[零一二三四五六七八九十百]+[ 、]\", \"[\\\\((][零一二三四五六七八九十百]+[\\\\))]\", \"[\\\\((][0-9]{,2}[\\\\))]\"], [\"PART (ONE|TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT|NINE|TEN)\", \"Chapter (I+V?|VI*|XI|IX|X)\", \"Section [0-9]+\", \"Article [0-9]+\"]], \"method\": \"group\"}, \"Tokenizer:FunnyBalloonsGrin\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}","one": "{\"File\": {}, \"Parser:HipSignsRhyme\": {\"doc\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"doc\"]}, \"docx\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"docx\"], \"vlm\": {}}, \"html\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"htm\", \"html\"]}, \"markdown\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"md\", \"markdown\", \"mdx\"], \"vlm\": {}}, \"pdf\": {\"flatten_media_to_text\": false, \"output_format\": \"json\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"pdf\"], \"vlm\": {}}, \"spreadsheet\": {\"flatten_media_to_text\": false, \"output_format\": \"html\", \"parse_method\": \"DeepDOC\", \"preprocess\": [\"main_content\"], \"suffix\": [\"xls\", \"xlsx\"], \"vlm\": {}}, \"text&code\": {\"output_format\": \"json\", \"preprocess\": [\"main_content\"], \"suffix\": [\"txt\"]}}, \"OneChunker:DryDrinksVisit\": {}, \"Tokenizer:FrankWeeksListen\": {\"fields\": \"text\", \"filename_embd_weight\": 0.1, \"search_method\": [\"embedding\", \"full_text\"]}, \"Extractor:AutoExtractDefault\": {\"llm_id\": \"\", \"keywords\": {\"top_n\": 0}, \"questions\": {\"top_n\": 0}, \"tags\": {\"top_n\": 0, \"tag_file_id\": \"\"}, \"summary\": {\"enabled\": false}, \"metadata\": {\"enabled\": false, \"metadata\": [], \"built_in_metadata\": []}}}",diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.goindex 3e40b87505..f4762e85fd 100644--- a/internal/parser/chunk/delim.go+++ b/internal/parser/chunk/delim.go@@ -38,6 +38,42 @@ func HasWrappedDelimiter(s string) bool {return backtickWrappedRE.MatchString(s)}+// ParseDelimiterField converts the legacy Python delimiter string into the+// canonical delimiter list used by the Go pipeline. Characters outside+// backticks are individual Unicode delimiters; a backtick-wrapped token stays+// as one list entry so multi-character delimiters remain lossless.+func ParseDelimiterField(value string) []string {+ if value == "" {+ return []string{}+ }+ var delimiters []string+ var wrapped strings.Builder+ inWrapped := false+ for _, r := range value {+ if r == '`' {+ if inWrapped {+ wrapped.WriteRune(r)+ delimiters = append(delimiters, wrapped.String())+ wrapped.Reset()+ inWrapped = false+ } else {+ inWrapped = true+ wrapped.WriteRune(r)+ }+ continue+ }+ if inWrapped {+ wrapped.WriteRune(r)+ continue+ }+ delimiters = append(delimiters, string(r))+ }+ if wrapped.Len() > 0 {+ delimiters = append(delimiters, wrapped.String())+ }+ return delimiters+}+// CompileDelimiterPattern builds an alternation regex from delimiter strings.//// Each delimiter is regexp.QuoteMeta'd so whitespace and metacharacters match@@ -79,8 +115,8 @@ func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {// bare ones.//// Returns nil when no active entry remains. This helper exists for the-// dataflow list API. The single-string parser_config.delimiter field is not-// parsed in Go; only the []string list API is consumed.+// dataflow list API. Call ParseDelimiterField before it when consuming the+// legacy single-string parser_config field.func CompileDelimiterPatternList(delims []string, keepBare bool) *regexp.Regexp {var out []stringseen := make(map[string]struct{})diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.goindex 5723743957..b6fea0c68f 100644--- a/internal/parser/chunk/delim_test.go+++ b/internal/parser/chunk/delim_test.go@@ -40,6 +40,14 @@ func TestHasWrappedDelimiter(t *testing.T) {}}+func TestParseDelimiterField(t *testing.T) {+ got := ParseDelimiterField("\n!?;。;!?`---`")+ want := []string{"\n", "!", "?", ";", "。", ";", "!", "?", "`---`"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("ParseDelimiterField = %#v, want %#v", got, want)+ }+}+func TestCompileDelimiterPattern(t *testing.T) {if CompileDelimiterPattern(nil) != nil {t.Fatal("empty list should yield nil")diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.goindex 7f50d792e4..26383ccaf0 100644--- a/internal/parser/parser/charset_test.go+++ b/internal/parser/parser/charset_test.go@@ -165,13 +165,17 @@ func TestCSVParser_GBK(t *testing.T) {if len(res.JSON) == 0 {t.Fatalf("CSVParser returned no JSON items")}- tableHTML, _ := res.JSON[0]["text"].(string)- if strings.ContainsRune(tableHTML, '\ufffd') {- t.Errorf("CSV HTML output contains replacement runes: %s", tableHTML)+ var rowText strings.Builder+ for _, item := range res.JSON {+ text, _ := item["text"].(string)+ rowText.WriteString(text)+ }+ if strings.ContainsRune(rowText.String(), '\ufffd') {+ t.Errorf("CSV output contains replacement runes: %s", rowText.String())}for _, want := range []string{"笔记本电脑", "电子", "特价销售,送鼠标", "无线耳机"} {- if !strings.Contains(tableHTML, want) {- t.Errorf("CSV HTML missing expected substring %q; got: %s", want, tableHTML)+ if !strings.Contains(rowText.String(), want) {+ t.Errorf("CSV output missing expected substring %q; got: %s", want, rowText.String())}}if enc, _ := res.File["encoding"].(string); enc != "gb18030" && enc != "gbk" && enc != "gb2312" {diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 64a3d206d6..2a53e9ef80 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -14,15 +14,9 @@// limitations under the License.//-// CSVParser renders CSV data as HTML table text in structured JSON table items.-//-// Mirrors Python's deepdoc/parser/excel_parser.py:RAGFlowExcelParser.html():-// - CSV data is rendered as an HTML <table> with <caption> "Data".-// - The first row is treated as the header (<th>).-// - Illegal control characters are replaced with spaces.-// - Large sheets are split into chunks of chunk_rows data rows, each-// chunk being a self-contained <table> with its own <caption> and-// repeated header row.+// CSVParser emits ordered spreadsheet row items in structured JSON.+// Illegal control characters are replaced with spaces; final chunking is+// owned by GeneralChunker.//// It implements the ParseResultProducer contract so the dispatch seam in// parser_dispatch.go routes .csv files through the structured path.@@ -36,14 +30,13 @@ import ("strings")-const csvDefaultChunkRows = defaultTableChunkRowsconst csvSheetName = "Data"// CSVParser reads RFC-4180 CSV data and emits structured table JSON items.type CSVParser struct {ParseMethod stringOutputFormat string- ChunkRows int+ HTML4Excel boolTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType string@@ -52,7 +45,6 @@ type CSVParser struct {func NewCSVParser() *CSVParser {return &CSVParser{- ChunkRows: csvDefaultChunkRows,TCADPTableResultType: "1",TCADPMarkdownImageResponseType: "1",}@@ -72,19 +64,10 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["output_format"].(string); ok && v != "" {p.OutputFormat = v}- if v, ok := setup["chunk_rows"]; ok {- switch n := v.(type) {- case float64:- p.ChunkRows = int(n)- case int:- p.ChunkRows = n- case int64:- p.ChunkRows = int(n)- }- if p.ChunkRows <= 0 {- p.ChunkRows = csvDefaultChunkRows- }+ if v, ok := setup["html4excel"].(bool); ok {+ p.HTML4Excel = v}+ deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {p.TCADPAPIServer = v}@@ -99,10 +82,8 @@ func (p *CSVParser) ConfigureFromSetup(setup map[string]any) {}}-// ParseWithResult implements ParseResultProducer. It reads CSV rows-// and renders them as HTML <table> chunks with <caption>, header row-// repeated per chunk, and illegal-character filtering — mirroring-// Python's RAGFlowExcelParser.html().+// ParseWithResult implements ParseResultProducer. It reads CSV rows and emits+// a header item followed by ordered data-row items.// When TCADP parse_method is configured, the file is dispatched to// the Tencent Cloud Document Parsing API.func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {@@ -127,7 +108,10 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [decoded, encName := DecodeToUTF8(data, "text/csv")text := string(decoded)if strings.TrimSpace(text) == "" {- emptyHTML := "<table><caption>" + csvSheetName + "</caption><tr><td></td></tr></table>"+ var emptyJSON []map[string]any+ if p.HTML4Excel {+ emptyJSON = []map[string]any{NewTableJSONItem("<table><caption>Data</caption></table>", csvSheetName, [][]float64{{1, 1, 1, 1, 1}})}+ }return ParseResult{OutputFormat: spreadsheetOutputFormat,File: map[string]any{@@ -137,7 +121,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data ["format": "csv","sheets": 1,},- JSON: []map[string]any{NewTableJSONItem(emptyHTML, csvSheetName, [][]float64{{1, 1, 1, 1, 1}})},+ JSON: emptyJSON,}}@@ -154,21 +138,17 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [// Clean illegal control characters from all cells.records = cleanIllegalControlChars(records)- chunkRows := p.ChunkRows- if chunkRows <= 0 {- chunkRows = csvDefaultChunkRows+ dataRows := make([]int, len(records)-1)+ for i := range dataRows {+ dataRows[i] = i + 2}-- chunks := recordsToHTMLTableChunkList(records, chunkRows, csvSheetName, 1)- items := make([]map[string]any, 0, len(chunks))- for _, ch := range chunks {- items = append(items, NewTableJSONItem(ch.HTML, csvSheetName, [][]float64{{- 1,- float64(ch.RowStart),- float64(ch.RowEnd),- float64(ch.ColStart),- float64(ch.ColEnd),- }}))+ var items []map[string]any+ if p.HTML4Excel {+ if table := recordsToHTMLTableItem(records, csvSheetName, 1, 1, dataRows); table != nil {+ items = []map[string]any{table}+ }+ } else {+ items = recordsToSpreadsheetItems(records, csvSheetName, 1, 1, dataRows)}return ParseResult{OutputFormat: spreadsheetOutputFormat,diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex a8504ba47b..ac2f4da99d 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -2,11 +2,10 @@ package parserimport ("context"- "strings""testing")-func TestCSVParser_BasicHTMLAndJSON(t *testing.T) {+func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {csvData := []byte("Name,Age,City\nAlice,30,New York\nBob,25,San Francisco\n")p := NewCSVParser()@@ -20,34 +19,44 @@ func TestCSVParser_BasicHTMLAndJSON(t *testing.T) {t.Fatalf("res.OutputFormat = %q, want %q", res.OutputFormat, "json")}- if len(res.JSON) != 1 {- t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))+ if len(res.JSON) != 3 {+ t.Fatalf("len(res.JSON) = %d, want header plus two rows", len(res.JSON))}-- item := res.JSON[0]- if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {- t.Fatal("item text should contain rendered <table>")+ if res.JSON[0]["ck_type"] != "table_header" {+ t.Fatalf("header ck_type = %v, want table_header", res.JSON[0]["ck_type"])}- if item["doc_type_kwd"] != "table" {- t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])+ if res.JSON[1]["ck_type"] != "table_row" {+ t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])}- if item["ck_type"] != "table" {- t.Errorf("item ck_type = %v, want 'table'", item["ck_type"])+ if res.JSON[1]["text"] != "Name:Alice; Age:30; City:New York ——Data" {+ t.Errorf("row text = %v", res.JSON[1]["text"])}- if item["sheet"] != "Data" {- t.Errorf("item sheet = %v, want 'Data'", item["sheet"])+ if res.JSON[1]["sheet"] != "Data" || res.JSON[1]["sheet_index"] != 1 {+ t.Errorf("row sheet metadata = %#v", res.JSON[1])}- positions, ok := item["positions"].([][]float64)+ positions, ok := res.JSON[1]["positions"].([][]float64)if !ok || len(positions) == 0 {- t.Fatalf("item positions invalid: %v", item["positions"])+ t.Fatalf("item positions invalid: %v", res.JSON[1]["positions"])}pos := positions[0]// [sheet, rowStart, rowEnd, colStart, colEnd]- if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 3 || pos[3] != 1 || pos[4] != 3 {+ if len(pos) != 5 || pos[0] != 1 || pos[1] != 2 || pos[2] != 2 || pos[3] != 1 || pos[4] != 3 {t.Errorf("unexpected positions: %v", pos)}}+func TestCSVParserHTML4ExcelRemainsAtomic(t *testing.T) {+ p := NewCSVParser()+ p.ConfigureFromSetup(map[string]any{"html4excel": true})+ res := p.ParseWithResult(context.Background(), "qa.csv", []byte("Question,Answer\nQ1,A1\n"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }+ if len(res.JSON) != 1 || res.JSON[0]["ck_type"] != "table" {+ t.Fatalf("items = %#v, want one table item", res.JSON)+ }+}+func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {csvData := []byte("col1,col2\nval1,val2\n")p := NewCSVParser()@@ -63,11 +72,11 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {if res.OutputFormat != "json" {t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)}- if len(res.JSON) != 1 {- t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))+ if len(res.JSON) != 2 {+ t.Fatalf("len(res.JSON) = %d, want header plus row", len(res.JSON))}- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {- t.Fatal("item text should contain rendered <table>")+ if res.JSON[1]["ck_type"] != "table_row" {+ t.Fatalf("row ck_type = %v, want table_row", res.JSON[1]["ck_type"])}}@@ -85,40 +94,45 @@ func TestCSVParser_EmptyContent(t *testing.T) {if res.OutputFormat != "json" {t.Fatalf("res.OutputFormat = %q, want 'json'", res.OutputFormat)}- if len(res.JSON) != 1 {- t.Fatalf("len(res.JSON) = %d, want 1", len(res.JSON))- }- item := res.JSON[0]- if item["doc_type_kwd"] != "table" {- t.Errorf("item doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])+ if len(res.JSON) != 0 {+ t.Fatalf("len(res.JSON) = %d, want 0", len(res.JSON))}}-func TestCSVParser_Chunking(t *testing.T) {- // Header + 4 data rows, chunk size = 2+func TestCSVParser_PreservesEveryDataRow(t *testing.T) {+ // Header + 4 data rows; GeneralChunker owns any later token merge.csvData := []byte("Header\nr1\nr2\nr3\nr4\n")p := NewCSVParser()- p.ChunkRows = 2-res := p.ParseWithResult(context.Background(), "chunked.csv", csvData)if res.Err != nil {t.Fatalf("ParseWithResult failed: %v", res.Err)}- // 4 data rows with chunk_rows=2 => 2 chunks- if len(res.JSON) != 2 {- t.Fatalf("len(res.JSON) = %d, want 2", len(res.JSON))+ if len(res.JSON) != 5 {+ t.Fatalf("len(res.JSON) = %d, want header plus four rows", len(res.JSON))}+ for i, item := range res.JSON[1:] {+ pos := item["positions"].([][]float64)[0]+ wantRow := float64(i + 2)+ if pos[1] != wantRow || pos[2] != wantRow {+ t.Errorf("row %d position = [%v, %v], want [%v, %v]", i, pos[1], pos[2], wantRow, wantRow)+ }+ }+}- pos1 := res.JSON[0]["positions"].([][]float64)[0]- pos2 := res.JSON[1]["positions"].([][]float64)[0]-- // chunk 1: rows 2..3- if pos1[1] != 2 || pos1[2] != 3 {- t.Errorf("chunk 1 row range: [%v, %v], want [2, 3]", pos1[1], pos1[2])+func TestCSVParserKeepsVariableRowWidths(t *testing.T) {+ p := NewCSVParser()+ res := p.ParseWithResult(context.Background(), "mixed.csv", []byte("Question,Answer\nQ1,A1\nQ2,A2,extra\n"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult failed: %v", res.Err)+ }+ if len(res.JSON) != 3 {+ t.Fatalf("items = %d, want header plus two rows", len(res.JSON))+ }+ if got := len(res.JSON[1]["cells"].([]string)); got != 2 {+ t.Fatalf("first data row width = %d, want 2", got)}- // chunk 2: rows 4..5- if pos2[1] != 4 || pos2[2] != 5 {- t.Errorf("chunk 2 row range: [%v, %v], want [4, 5]", pos2[1], pos2[2])+ if got := len(res.JSON[2]["cells"].([]string)); got != 3 {+ t.Fatalf("second data row width = %d, want 3", got)}}diff --git a/internal/parser/parser/delimiter.go b/internal/parser/parser/delimiter.goindex 9ab0e62e1d..e06e6d5104 100644--- a/internal/parser/parser/delimiter.go+++ b/internal/parser/parser/delimiter.go@@ -25,10 +25,3 @@ package parser// alignment test normalizes delimiters. It is a non-test symbol so both the// production parsers and the alignment tests share one source of truth.const DefaultMarkdownDelimiter = "\n!?;。;!?"--// DefaultTextCodeDelimiter is the flow parser's default text&code delimiter-// set, used by TextParser and when the text&code alignment test normalizes-// delimiters. It is a non-test symbol so both the production parser and the-// alignment tests share one source of truth (no duplicate hard-coded copy in-// text_parser.go, which would otherwise drift silently).-const DefaultTextCodeDelimiter = "\n!?;。;!?"diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex 71286bea47..8ba42ad7d6 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -102,6 +102,7 @@ type docxIRList struct {type docxIRRun struct {Type string `json:"type"` // "text", "line_break", "image"Text string `json:"text"`+ Data []byte `json:"data"` // raw image bytes for inline image runsContent []docxIRElement `json:"content"` // nested elements (used in table cells)}@@ -262,19 +263,7 @@ func buildDOCXJSONSections(irJSON string) []map[string]any {for _, el := range sec.Elements {switch el.Type {case "paragraph", "heading":- text := joinDOCXIRRuns(el.contentRuns())- if strings.TrimSpace(text) == "" {- continue- }- item := map[string]any{- "text": text,- "image": nil,- "doc_type_kwd": "text",- }- if el.Type == "heading" {- item["ck_type"] = "heading"- }- sections = append(sections, item)+ sections = appendDOCXParagraphSections(sections, el, el.Type == "heading")case "image":b64 := base64.StdEncoding.EncodeToString(el.Data)@@ -324,6 +313,58 @@ func buildDOCXJSONSections(irJSON string) []map[string]any {return sections}+// appendDOCXParagraphSections emits text and inline-image runs in their IR+// order. A paragraph is not necessarily a single text unit: office_oxide can+// place an image between two text runs, and dropping that run would make the+// downstream DOCX chunker lose both media order and adjacency context.+func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool) []map[string]any {+ var text strings.Builder+ flushText := func() {+ value := strings.TrimSpace(text.String())+ text.Reset()+ if value == "" {+ return+ }+ item := map[string]any{+ "text": value,+ "image": nil,+ "doc_type_kwd": "text",+ }+ if heading {+ item["ck_type"] = "heading"+ }+ sections = append(sections, item)+ }+ appendImage := func(data []byte) {+ if len(data) == 0 {+ return+ }+ sections = append(sections, map[string]any{+ "text": "",+ "image": base64.StdEncoding.EncodeToString(data),+ "doc_type_kwd": "image",+ })+ }+ for _, run := range el.contentRuns() {+ switch run.Type {+ case "text":+ text.WriteString(run.Text)+ case "line_break":+ text.WriteByte('\n')+ case "image":+ flushText()+ appendImage(run.Data)+ for _, nested := range run.Content {+ if nested.Type == "image" {+ appendImage(nested.Data)+ }+ }+ }+ }+ flushText()+ return sections+}+// --- figure extraction (used by the cgo parser path) ---// extractDOCXFiguresFromIR parses the office_oxide IR JSON and@@ -348,6 +389,10 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {flat = append(flat, flatBlock{image: b64})continue}+ if el.Type == "paragraph" || el.Type == "heading" {+ flat = append(flat, docxInlineFlatBlocks(el)...)+ continue+ }text := docxElementText(el, "\n")flat = append(flat, flatBlock{text: text})}@@ -385,6 +430,45 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {return figures}+// docxInlineFlatBlocks preserves the text/image sequence inside a paragraph+// for figure context extraction. Top-level element order alone is not enough+// for DOCX because office_oxide can encode an inline image as a paragraph run.+func docxInlineFlatBlocks(el docxIRElement) []flatBlock {+ var blocks []flatBlock+ var text strings.Builder+ flushText := func() {+ if text.Len() == 0 {+ return+ }+ blocks = append(blocks, flatBlock{text: text.String()})+ text.Reset()+ }+ appendImage := func(data []byte) {+ if len(data) == 0 {+ return+ }+ blocks = append(blocks, flatBlock{image: base64.StdEncoding.EncodeToString(data)})+ }+ for _, run := range el.contentRuns() {+ switch run.Type {+ case "text":+ text.WriteString(run.Text)+ case "line_break":+ text.WriteByte('\n')+ case "image":+ flushText()+ appendImage(run.Data)+ for _, nested := range run.Content {+ if nested.Type == "image" {+ appendImage(nested.Data)+ }+ }+ }+ }+ flushText()+ return blocks+}+// --- internal types ---// flatBlock is a flattened IR element used internally to collectdiff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 48b7b9196e..1a33eeeb8b 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -49,6 +49,29 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {}}+func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[+ {"type":"text","text":"before"},+ {"type":"image","data":"aGVsbG8="},+ {"type":"text","text":"after"}+ ]}+ ]}]}`+ got := buildDOCXJSONSections(irJSON)+ if len(got) != 3 {+ t.Fatalf("sections = %+v, want text/image/text sequence", got)+ }+ if got[0]["text"] != "before" || got[0]["doc_type_kwd"] != "text" {+ t.Errorf("first item = %+v, want text before", got[0])+ }+ if got[1]["image"] != "aGVsbG8=" || got[1]["doc_type_kwd"] != "image" {+ t.Errorf("inline image item = %+v", got[1])+ }+ if got[2]["text"] != "after" || got[2]["doc_type_kwd"] != "text" {+ t.Errorf("last item = %+v, want text after", got[2])+ }+}+// TestJoinDOCXIRRuns pins that only text-type runs are concatenated;// non-text runs (e.g. nested image runs) are skipped.func TestJoinDOCXIRRuns(t *testing.T) {@@ -104,6 +127,23 @@ func TestExtractDOCXFiguresFromIR(t *testing.T) {}}+func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[+ {"type":"text","text":"before"},+ {"type":"image","data":"aGVsbG8="},+ {"type":"text","text":"after"}+ ]}+ ]}]}`+ figs := extractDOCXFiguresFromIR(irJSON)+ if len(figs) != 1 {+ t.Fatalf("expected 1 inline figure, got %d", len(figs))+ }+ if figs[0].Image != "aGVsbG8=" || figs[0].ContextAbove != "before" || figs[0].ContextBelow != "after" {+ t.Fatalf("inline figure = %+v", figs[0])+ }+}+// TestExtractDOCXFiguresFromIR_NoImage returns nil when the IR has no// image blocks.func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex bb3dce85be..4986bb357b 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -543,14 +543,23 @@ func resolveAndValidateHost(host string) (net.IP, error) {return addrs[0].IP, nil}-// headingText returns the inline-text of a heading node by-// concatenating every Leaf / Text child. Empty headings emit "".+// headingText returns a canonical ATX heading (marker plus inline text).+// Keeping the marker in the parser payload preserves the Markdown structure+// for GeneralChunker and downstream LLM prompts; ck_type alone does not carry+// the heading level.func headingText(h *ast.Heading) string {var buf bytes.Bufferfor _, c := range h.GetChildren() {buf.WriteString(leafText(c))}- return strings.TrimSpace(buf.String())+ text := strings.TrimSpace(buf.String())+ if h.Level <= 0 {+ return text+ }+ if text == "" {+ return strings.Repeat("#", h.Level)+ }+ return strings.Repeat("#", h.Level) + " " + text}// leafText mirrors gomarkdown's leaf walker: walks every descendantdiff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex d8c819d797..8616b49e48 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -27,8 +27,8 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {t.Fatal("JSON is empty; want at least one item")}// Verify heading- if got, _ := res.JSON[0]["text"].(string); got != "Hello" {- t.Fatalf("first item text = %q, want %q", got, "Hello")+ if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {+ t.Fatalf("first item text = %q, want %q", got, "# Hello")}if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {t.Fatalf("first item ck_type = %q, want %q", got, "heading")@@ -321,7 +321,7 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- // Title, Intro, table item, Trailing = 4.+ // # Title, Intro, table item, Trailing = 4.if len(res.JSON) != 4 {t.Fatalf("len(JSON) = %d, want 4 (one per top-level block, table as one item)", len(res.JSON))}@@ -332,7 +332,7 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {text, _ := item["text"].(string)all.WriteString(text)all.WriteString("\n")- if text == "Title" {+ if text == "# Title" {sawTitle = true}switch kd, _ := item["doc_type_kwd"].(string); kd {@@ -569,7 +569,7 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {switch kd, _ := item["doc_type_kwd"].(string); kd {case "text":switch text {- case "Title":+ case "# Title":titleIdx = icase "Middle.":middleIdx = idiff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 91cb78a25a..c7837acd3e 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -20,7 +20,9 @@ import ("encoding/base64""fmt""html"+ "log/slog""regexp"+ "sort""strconv""strings"@@ -38,13 +40,241 @@ var tableIllegalCharsRe = regexp.MustCompile(`[\x00-\x08]|\x0B|\x0C|[\x0E-\x1F]`// value parsing.var numericCellRe = regexp.MustCompile(`^[\$\+\-]?[\d,]+(\.\d+)?%?$`)-// defaultTableChunkRows is the row ceiling for one spreadsheet <table>. It is-// deliberately far beyond any real sheet: a sheet is emitted as ONE-// self-contained <table> and is never split by row count. Any split would cut-// inside <td> content or, worse, drop the delimiter characters it cut on —-// the bug the typed table item exists to prevent. Keep in sync with Python's-// TABLE_NO_SPLIT_ROWS (rag/flow/parser/parser.py).-const defaultTableChunkRows = 1 << 30+func recordsToSpreadsheetItems(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) []map[string]any {+ if len(records) == 0 {+ return nil+ }+ if headerRow <= 0 {+ headerRow = 1+ }+ if len(dataRows) != len(records)-1 {+ dataRows = make([]int, len(records)-1)+ for i := range dataRows {+ dataRows[i] = headerRow + i + 1+ }+ }+ maxCols := 0+ for _, row := range records {+ if len(row) > maxCols {+ maxCols = len(row)+ }+ }+ if maxCols == 0 {+ return nil+ }+ header := append([]string(nil), records[0]...)+ if len(header) == 0 && maxCols > 0 {+ header = padSpreadsheetRow(header, maxCols)+ }+ headerColEnd := len(header)+ if headerColEnd == 0 {+ headerColEnd = maxCols+ }+ if headerColEnd == 0 {+ headerColEnd = 1+ }+ tableID := fmt.Sprintf("sheet-%d", sheetIndex)+ items := make([]map[string]any, 0, len(records))+ headerText := spreadsheetRowText(header, nil, sheet)+ items = append(items, map[string]any{+ "text": headerText,+ "doc_type_kwd": DocTypeTable,+ "ck_type": "table_header",+ "table_id": tableID,+ "sheet": sheet,+ "sheet_index": sheetIndex,+ "cells": header,+ "row_start": headerRow,+ "row_end": headerRow,+ "col_start": 1,+ "col_end": headerColEnd,+ "positions": [][]float64{{float64(sheetIndex), float64(headerRow), float64(headerRow), 1, float64(headerColEnd)}},+ })+ for i, sourceRow := range records[1:] {+ cells := append([]string(nil), sourceRow...)+ colStart, colEnd := nonEmptyColumnRange(cells)+ if colStart == 0 {+ continue+ }+ rowNumber := dataRows[i]+ items = append(items, map[string]any{+ "text": spreadsheetRowText(header, cells, sheet),+ "doc_type_kwd": DocTypeText,+ "ck_type": "table_row",+ "table_id": tableID,+ "sheet": sheet,+ "sheet_index": sheetIndex,+ "headers": header,+ "cells": cells,+ "row_start": rowNumber,+ "row_end": rowNumber,+ "col_start": colStart,+ "col_end": colEnd,+ "positions": [][]float64{{float64(sheetIndex), float64(rowNumber), float64(rowNumber), float64(colStart), float64(colEnd)}},+ })+ }+ return items+}++func recordsToHTMLTableItem(records [][]string, sheet string, sheetIndex, headerRow int, dataRows []int) map[string]any {+ if len(records) == 0 {+ return nil+ }+ if headerRow <= 0 {+ headerRow = 1+ }+ colEnd := 1+ for _, row := range records {+ if len(row) > colEnd {+ colEnd = len(row)+ }+ }+ var builder strings.Builder+ builder.WriteString("<table><caption>")+ builder.WriteString(html.EscapeString(sheet))+ builder.WriteString("</caption>\n<tr>")+ for _, cell := range records[0] {+ builder.WriteString("<th>")+ builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))+ builder.WriteString("</th>")+ }+ builder.WriteString("</tr>\n")+ for _, row := range records[1:] {+ builder.WriteString("<tr>")+ for _, cell := range row {+ builder.WriteString("<td>")+ builder.WriteString(html.EscapeString(strings.TrimSpace(cell)))+ builder.WriteString("</td>")+ }+ builder.WriteString("</tr>\n")+ }+ builder.WriteString("</table>\n")+ rowStart, rowEnd := headerRow, headerRow+ if len(dataRows) > 0 {+ rowStart, rowEnd = dataRows[0], dataRows[len(dataRows)-1]+ }+ return NewTableJSONItem(builder.String(), sheet, [][]float64{{+ float64(sheetIndex), float64(rowStart), float64(rowEnd), 1, float64(colEnd),+ }})+}++func padSpreadsheetRow(row []string, width int) []string {+ padded := make([]string, width)+ copy(padded, row)+ return padded+}++func nonEmptyColumnRange(row []string) (int, int) {+ start, end := 0, 0+ for i, cell := range row {+ if strings.TrimSpace(cell) == "" {+ continue+ }+ col := i + 1+ if start == 0 {+ start = col+ }+ end = col+ }+ return start, end+}++func spreadsheetRowText(headers, cells []string, sheet string) string {+ values := headers+ if cells != nil {+ values = cells+ }+ parts := make([]string, 0, len(values))+ for i, value := range values {+ value = strings.TrimSpace(value)+ if value == "" {+ continue+ }+ if cells == nil {+ parts = append(parts, value)+ continue+ }+ header := ""+ if i < len(headers) {+ header = strings.TrimSpace(headers[i])+ }+ if header != "" {+ parts = append(parts, header+":"+value)+ } else {+ parts = append(parts, value)+ }+ }+ text := strings.Join(parts, "; ")+ if cells != nil && sheet != "" && !strings.Contains(strings.ToLower(sheet), "sheet") {+ text += " ——" + sheet+ }+ return text+}++func sortSpreadsheetItems(items []map[string]any) {+ sort.SliceStable(items, func(i, j int) bool {+ headerI := items[i]["ck_type"] == "table_header"+ headerJ := items[j]["ck_type"] == "table_header"+ if headerI != headerJ {+ return headerI+ }+ ri, ci := spreadsheetItemCoordinate(items[i])+ rj, cj := spreadsheetItemCoordinate(items[j])+ if ri != rj {+ return ri < rj+ }+ if ci != cj {+ return ci < cj+ }+ return spreadsheetItemKindRank(items[i]) < spreadsheetItemKindRank(items[j])+ })+}++func spreadsheetItemCoordinate(item map[string]any) (int, int) {+ row, _ := numericItemInt(item["row_start"])+ col, _ := numericItemInt(item["col_start"])+ return row, col+}++func spreadsheetItemKindRank(item map[string]any) int {+ if item["ck_type"] == "image" {+ return 1+ }+ return 0+}++func numericItemInt(value any) (int, bool) {+ switch n := value.(type) {+ case int:+ return n, true+ case int64:+ return int(n), true+ case float64:+ return int(n), true+ case string:+ parsed, err := strconv.Atoi(strings.TrimSpace(n))+ return parsed, err == nil+ default:+ return 0, false+ }+}++// deprecatedChunkRows reports the removed parser-side chunking option. Row+// boundaries are now emitted as structured IR and the selected chunker owns+// token-budget merging; silently accepting chunk_rows would make an existing+// configuration look effective when it is not.+func deprecatedChunkRows(setup map[string]any, parserName string) {+ raw, exists := setup["chunk_rows"]+ if !exists {+ return+ }+ rows, ok := numericItemInt(raw)+ if !ok {+ slog.Warn("spreadsheet parser ignored invalid chunk_rows; configure row merging on the chunker", "parser", parserName, "chunk_rows", raw)+ return+ }+ slog.Warn("spreadsheet parser ignored deprecated chunk_rows; configure row merging on the chunker", "parser", parserName, "chunk_rows", rows)+}// extractXLSXImages returns the floating and in-cell images anchored to a// worksheet as structured parser items. Excelize exposes both kinds through@@ -81,6 +311,7 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []striif picture.Format != nil && picture.Format.AltText != "" {alt = picture.Format.AltText}+ row, col := axisToRC(cell)items = append(items, map[string]any{"text": alt,"doc_type_kwd": "image",@@ -88,6 +319,10 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []stri"image": "data:" + mimeType + ";base64," + encoded,"sheet": sheet,"cell": cell,+ "row_start": row,+ "row_end": row,+ "col_start": col,+ "col_end": col,})}}@@ -144,106 +379,6 @@ func cleanIllegalControlChars(records [][]string) [][]string {return out}-// buildHeaderRow renders a row as an HTML <th> header row.-func buildHeaderRow(row []string) string {- var b strings.Builder- b.WriteString("<tr>")- for _, cell := range row {- b.WriteString("<th>")- b.WriteString(html.EscapeString(strings.TrimSpace(cell)))- b.WriteString("</th>")- }- b.WriteString("</tr>\n")- return b.String()-}--// htmlTableChunk is one self-contained <table> chunk with 1-based sheet/row/col-// coordinates for source location (mirrors Excel position_int semantics).-type htmlTableChunk struct {- HTML string- RowStart int // inclusive, 1-based Excel row of first data row (header-only chunks use headerRowAbs)- RowEnd int // inclusive, 1-based Excel row of last data row- ColStart int- ColEnd int-}--// recordsToHTMLTableChunkList renders records as structured htmlTableChunk items.-// headerRowAbs is the 1-based workbook row number of records[0] (normally 1).-func recordsToHTMLTableChunkList(records [][]string, chunkRows int, caption string, headerRowAbs int) []htmlTableChunk {- if headerRowAbs <= 0 {- headerRowAbs = 1- }- colEnd := 1- for _, row := range records {- if n := len(row); n > colEnd {- colEnd = n- }- }- if len(records) == 0 {- return []htmlTableChunk{{- HTML: "<table><caption>" + html.EscapeString(caption) + "</caption></table>",- RowStart: headerRowAbs,- RowEnd: headerRowAbs,- ColStart: 1,- ColEnd: colEnd,- }}- }-- headerHTML := buildHeaderRow(records[0])- dataRows := records[1:]- nData := len(dataRows)-- if nData == 0 {- return []htmlTableChunk{{- HTML: "<table><caption>" + html.EscapeString(caption) + "</caption>\n" + headerHTML + "</table>",- RowStart: headerRowAbs,- RowEnd: headerRowAbs,- ColStart: 1,- ColEnd: colEnd,- }}- }-- if chunkRows <= 0 {- chunkRows = defaultTableChunkRows- }-- nChunks := (nData + chunkRows - 1) / chunkRows- out := make([]htmlTableChunk, 0, nChunks)- for ci := 0; ci < nChunks; ci++ {- start := ci * chunkRows- end := start + chunkRows- if end > nData {- end = nData- }-- var b strings.Builder- b.WriteString("<table><caption>")- b.WriteString(html.EscapeString(caption))- b.WriteString("</caption>\n")- b.WriteString(headerHTML)-- for _, row := range dataRows[start:end] {- b.WriteString("<tr>")- for _, cell := range row {- b.WriteString("<td>")- b.WriteString(html.EscapeString(strings.TrimSpace(cell)))- b.WriteString("</td>")- }- b.WriteString("</tr>\n")- }- b.WriteString("</table>\n")-- out = append(out, htmlTableChunk{- HTML: b.String(),- RowStart: headerRowAbs + start + 1,- RowEnd: headerRowAbs + end,- ColStart: 1,- ColEnd: colEnd,- })- }- return out-}-// ──────────────────────────────────────────────────────────── axis helpers// axisToRC parses an "A1"-style cell reference (optionally with a leading "$")@@ -590,45 +725,13 @@ func isSubtotalRow(row []string) bool {return false}-// decodeChunkRows reads the "chunk_rows" setup knob, returning the default when-// it is absent or non-positive.-func decodeChunkRows(setup map[string]any) int {- if setup == nil {- return defaultTableChunkRows- }- v, ok := setup["chunk_rows"]- if !ok {- return defaultTableChunkRows- }- switch n := v.(type) {- case float64:- rows := int(n)- if rows <= 0 {- return defaultTableChunkRows- }- return rows- case int:- if n <= 0 {- return defaultTableChunkRows- }- return n- case int64:- rows := int(n)- if rows <= 0 {- return defaultTableChunkRows- }- return rows- }- return defaultTableChunkRows-}--func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]htmlTableChunk, []string, error) {+func readSpreadsheetRecords(f *excelize.File, sheet string) ([][]string, []int, int, []string, error) {rows, err := f.GetRows(sheet)if err != nil {- return nil, nil, fmt.Errorf("read XLSX sheet %q rows: %w", sheet, err)+ return nil, nil, 0, nil, fmt.Errorf("read XLSX sheet %q rows: %w", sheet, err)}if len(rows) == 0 {- return nil, nil, nil+ return nil, nil, 0, nil, nil}rows = cleanIllegalControlChars(rows)@@ -669,23 +772,5 @@ func renderSheetTableChunks(f *excelize.File, sheet string, chunkRows int) ([]htabsDataRows = append(absDataRows, i+1)}- chunks := recordsToHTMLTableChunkList(records, chunkRows, sheet, headerRow)- if len(chunks) == 0 || len(absDataRows) == 0 {- return chunks, warnings, nil- }- if chunkRows <= 0 {- chunkRows = defaultTableChunkRows- }- for ci := range chunks {- start := ci * chunkRows- end := start + chunkRows- if end > len(absDataRows) {- end = len(absDataRows)- }- if start < end {- chunks[ci].RowStart = absDataRows[start]- chunks[ci].RowEnd = absDataRows[end-1]- }- }- return chunks, warnings, nil+ return records, absDataRows, headerRow, warnings, nil}diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex 4b6efc1e1c..b66a67ec46 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -1,24 +1,159 @@package parserimport (+ "reflect""strings""testing""github.com/xuri/excelize/v2")-func xlsxTableHTML(res ParseResult) string {- var out strings.Builder+func TestXLSXParserEmitsSpreadsheetRows(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "ID")+ mustSetCell(t, f, "Sheet1", "B1", "Status")+ mustSetCell(t, f, "Sheet1", "A2", "A-100")+ mustSetCell(t, f, "Sheet1", "B2", "paid")+ mustSetCell(t, f, "Sheet1", "A3", "A-101")+ mustSetCell(t, f, "Sheet1", "B3", "pending")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "orders.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 3 {+ t.Fatalf("items = %d, want header plus two rows", len(res.JSON))+ }+ header := res.JSON[0]+ if header["ck_type"] != "table_header" {+ t.Fatalf("header ck_type = %v, want table_header", header["ck_type"])+ }+ if got, want := header["cells"], []string{"ID", "Status"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("header cells = %#v, want %#v", got, want)+ }+ row := res.JSON[1]+ if row["ck_type"] != "table_row" || row["doc_type_kwd"] != "text" {+ t.Fatalf("row types = ck_type:%v doc_type:%v", row["ck_type"], row["doc_type_kwd"])+ }+ if got, want := row["cells"], []string{"A-100", "paid"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("row cells = %#v, want %#v", got, want)+ }+ if row["sheet"] != "Sheet1" || row["sheet_index"] != 1 {+ t.Fatalf("row sheet metadata = %#v", row)+ }+ if row["row_start"] != 2 || row["row_end"] != 2 || row["col_start"] != 1 || row["col_end"] != 2 {+ t.Fatalf("row coordinates = %#v", row)+ }+}++func TestSortSpreadsheetItemsKeepsHeaderBeforeRowsWhenHeaderIsLater(t *testing.T) {+ items := []map[string]any{+ {"ck_type": "table_row", "row_start": 1, "col_start": 1},+ {"ck_type": "table_row", "row_start": 2, "col_start": 1},+ {"ck_type": "table_header", "row_start": 5, "col_start": 1},+ }++ sortSpreadsheetItems(items)+ if items[0]["ck_type"] != "table_header" {+ t.Fatalf("sorted items = %#v, want table_header first", items)+ }+}++func TestXLSXParserHTML4ExcelRemainsAtomic(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Question")+ mustSetCell(t, f, "Sheet1", "B1", "Answer")+ mustSetCell(t, f, "Sheet1", "A2", "Q1")+ mustSetCell(t, f, "Sheet1", "B2", "A1")+ })+ p, _ := NewXLSXParser("")+ p.ConfigureFromSetup(map[string]any{"html4excel": true})+ res := p.ParseWithResult(t.Context(), "qa.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("items = %d, want one atomic table item", len(res.JSON))+ }+ if res.JSON[0]["ck_type"] != "table" || !strings.Contains(res.JSON[0]["text"].(string), "<table>") {+ t.Fatalf("html4excel item = %#v", res.JSON[0])+ }+}++func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {+ const dataRows = 257+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "Value")+ for row := 2; row <= dataRows+1; row++ {+ mustSetCell(t, f, "Sheet1", cellAxis(row, 1), row)+ }+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "rows.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != dataRows+1 {+ t.Fatalf("items = %d, want one header plus %d data rows", len(res.JSON), dataRows)+ }+ last := res.JSON[len(res.JSON)-1]+ if last["ck_type"] != "table_row" || last["row_start"] != dataRows+1 {+ t.Fatalf("last row = %#v, want source row %d", last, dataRows+1)+ }+}++func TestXLSXParserPreservesSheetOrderAndIdentity(t *testing.T) {+ data := newTestXLSX(t, func(f *excelize.File) {+ mustSetCell(t, f, "Sheet1", "A1", "First")+ mustSetCell(t, f, "Sheet1", "A2", "one")+ _, err := f.NewSheet("Orders")+ if err != nil {+ t.Fatalf("new sheet: %v", err)+ }+ mustSetCell(t, f, "Orders", "A1", "Second")+ mustSetCell(t, f, "Orders", "A2", "two")+ })+ p, _ := NewXLSXParser("")+ res := p.ParseWithResult(t.Context(), "multi.xlsx", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 4 {+ t.Fatalf("items = %d, want two headers and two rows", len(res.JSON))+ }+ seen := map[string]map[string]any{}for _, item := range res.JSON {- if item["doc_type_kwd"] != "table" {- continue+ if item["ck_type"] == "table_header" {+ seen[item["sheet"].(string)] = item}+ }+ if first := seen["Sheet1"]; first == nil || first["table_id"] != "sheet-1" || first["sheet_index"] != 1 {+ t.Fatalf("Sheet1 identity = %#v", first)+ }+ if second := seen["Orders"]; second == nil || second["table_id"] != "sheet-2" || second["sheet_index"] != 2 {+ t.Fatalf("Orders identity = %#v", second)+ }+}++func spreadsheetText(res ParseResult) string {+ var out strings.Builder+ for _, item := range res.JSON {text, _ := item["text"].(string)out.WriteString(text)}return out.String()}+func spreadsheetHeaderItem(res ParseResult) map[string]any {+ for _, item := range res.JSON {+ if item["ck_type"] == "table_header" {+ return item+ }+ }+ return nil+}+func TestXLSXImageMIMEType(t *testing.T) {tests := []struct {extension string@@ -103,108 +238,8 @@ func mustAddTable(t *testing.T, f *excelize.File, sheet string, table *excelize.}}-// TestRecordsToHTMLTableChunkList_Alignment asserts the chunked output uses the-// shared schema: <caption>, first row as <th>, data as <td>, repeated header-// per 256-row chunk, and NO <thead>/<tbody> wrapper.-func TestRecordsToHTMLTableChunkList_Alignment(t *testing.T) {- records := [][]string{{"Name", "Age"}, {"Alice", "30"}, {"Bob", "25"}}- chunks := recordsToHTMLTableChunkList(records, 256, "Sheet1", 1)- if len(chunks) != 1 {- t.Fatalf("want 1 chunk, got %d", len(chunks))- }- out := chunks[0].HTML-- if !strings.Contains(out, `<caption>Sheet1</caption>`) {- t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", out)- }- if !strings.Contains(out, "<tr><th>Name</th><th>Age</th></tr>") {- t.Fatalf("want header row as <th>, got:\n%s", out)- }- if !strings.Contains(out, "<tr><td>Alice</td><td>30</td></tr>") {- t.Fatalf("want data row as <td>, got:\n%s", out)- }- if strings.Contains(out, "<thead>") || strings.Contains(out, "<tbody>") {- t.Fatalf("must not emit <thead>/<tbody> to stay byte-compatible with Python/CSV, got:\n%s", out)- }-}--// TestRecordsToHTMLTableChunkList_DefaultCeilingDoesNotSplit pins the default row-// ceiling: a sheet is emitted as ONE self-contained <table>. Splitting it would-// cut inside <td> content or drop the delimiter characters it cut on, and the-// value must stay in sync with Python's TABLE_NO_SPLIT_ROWS.-func TestRecordsToHTMLTableChunkList_DefaultCeilingDoesNotSplit(t *testing.T) {- const dataRows = 5000- records := make([][]string, 0, dataRows+1)- records = append(records, []string{"C1", "C2"})- for i := 0; i < dataRows; i++ {- records = append(records, []string{"x", "y"})- }-- chunks := recordsToHTMLTableChunkList(records, defaultTableChunkRows, "S", 1)- if len(chunks) != 1 {- t.Fatalf("want 1 chunk with the default ceiling, got %d", len(chunks))- }- // The single chunk spans every data row.- if chunks[0].RowStart != 2 || chunks[0].RowEnd != dataRows+1 {- t.Fatalf("want rows 2..%d, got %d..%d", dataRows+1, chunks[0].RowStart, chunks[0].RowEnd)- }-}--// TestRecordsToHTMLTableChunkList_Chunking asserts 256-row chunking with a repeated-// header (ceil(n_data / chunk_rows) chunks).-func TestRecordsToHTMLTableChunkList_Chunking(t *testing.T) {- const dataRows = 300- records := make([][]string, 0, dataRows+1)- records = append(records, []string{"C1", "C2"})- for i := 0; i < dataRows; i++ {- records = append(records, []string{"x", "y"})- }- chunks := recordsToHTMLTableChunkList(records, 256, "S", 1)- // 300 data rows → ceil(300/256) = 2 chunks, each repeating the header.- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }- for i, ch := range chunks {- if !strings.Contains(ch.HTML, "<tr><th>C1</th><th>C2</th></tr>") {- t.Fatalf("want header repeated in chunk %d, got %s", i, ch.HTML)- }- }-}--func TestRecordsToHTMLTableChunkList_RowRange(t *testing.T) {- records := make([][]string, 0, 25)- records = append(records, []string{"H"})- for i := 0; i < 24; i++ {- records = append(records, []string{"x"})- }- chunks := recordsToHTMLTableChunkList(records, 12, "S", 1)- if len(chunks) != 2 {- t.Fatalf("want 2 chunks, got %d", len(chunks))- }- if chunks[0].RowStart != 2 || chunks[0].RowEnd != 13 {- t.Fatalf("chunk0 rows = %d-%d, want 2-13", chunks[0].RowStart, chunks[0].RowEnd)- }- if chunks[1].RowStart != 14 || chunks[1].RowEnd != 25 {- t.Fatalf("chunk1 rows = %d-%d, want 14-25", chunks[1].RowStart, chunks[1].RowEnd)- }-}--func TestRecordsToHTMLTableChunkList_ColEndUsesWidestRow(t *testing.T) {- records := [][]string{- {"H"},- {"a", "b", "c"},- }- chunks := recordsToHTMLTableChunkList(records, 12, "S", 1)- if len(chunks) != 1 {- t.Fatalf("want 1 chunk, got %d", len(chunks))- }- if chunks[0].ColEnd != 3 {- t.Fatalf("ColEnd = %d, want 3", chunks[0].ColEnd)- }-}--// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a <caption> and-// renders the first row as <th>, and that the header text appears only in <th>.+// TestXLSXParser_HeaderAndCaption asserts the XLSX parser emits a typed header+// row and data row with the header labels preserved separately from values.func TestXLSXParser_HeaderAndCaption(t *testing.T) {data := newTestXLSX(t, func(f *excelize.File) {mustSetCell(t, f, "Sheet1", "A1", "Product")@@ -219,22 +254,22 @@ func TestXLSXParser_HeaderAndCaption(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- html := xlsxTableHTML(res)- if !strings.Contains(html, `<caption>Sheet1</caption>`) {- t.Fatalf("want <caption>Sheet1</caption>, got:\n%s", html)+ text := spreadsheetText(res)+ if !strings.Contains(text, "Product; Price") {+ t.Fatalf("want header text, got:\n%s", text)}- if !strings.Contains(html, "<tr><th>Product</th><th>Price</th></tr>") {- t.Fatalf("want header as <th>, got:\n%s", html)+ if !strings.Contains(text, "Product:Widget; Price:9.99") {+ t.Fatalf("want row text, got:\n%s", text)}- // "Product" must only appear inside a <th>, never inside a <td>.- if strings.Contains(html, "<td>Product</td>") {- t.Fatalf("header value leaked into <td>:\n%s", html)+ // The header label must not be duplicated as a value in the row text.+ if strings.Contains(text, "Product:Product") {+ t.Fatalf("header value leaked into row text:\n%s", text)}}// TestXLSXParser_MergedHeaderInheritance asserts a horizontally merged header// cell's slave columns inherit the master text, so a wide merged title does not-// render as a row of blank <th> cells.+// render as a row of blank header cells.func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {data := newTestXLSX(t, func(f *excelize.File) {// Row 1 is the header; A1:C1 merged into one wide label "Sales Report".@@ -255,10 +290,9 @@ func TestXLSXParser_MergedHeaderInheritance(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- html := xlsxTableHTML(res)- // The merged master text must have propagated into all three <th> slots.- if !strings.Contains(html, "<tr><th>Sales Report</th><th>Sales Report</th><th>Sales Report</th></tr>") {- t.Fatalf("merged master text not inherited into header <th>, got:\n%s", html)+ text := spreadsheetText(res)+ if !strings.Contains(text, "Sales Report; Sales Report; Sales Report") {+ t.Fatalf("merged master text not inherited into header cells, got:\n%s", text)}}@@ -279,11 +313,15 @@ func TestDetectHeaderRow_ListObject(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "<tr><th>Name</th><th>Age</th></tr>") {- t.Fatalf("want ListObject header row detected, got:\n%s", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "Name; Age") {+ t.Fatalf("want ListObject header row detected, got:\n%s", spreadsheetText(res))+ }+ header := spreadsheetHeaderItem(res)+ if header == nil {+ t.Fatal("missing table_header item")}- if strings.Contains(xlsxTableHTML(res), "<th>Title</th>") {- t.Fatalf("title row must not be the header:\n%s", xlsxTableHTML(res))+ if got := header["text"]; got != "Name; Age" {+ t.Fatalf("header text = %v, want Name; Age", got)}}@@ -306,16 +344,16 @@ func TestDetectHeaderRow_Lightweight(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "<tr><th>Item</th><th>Count</th></tr>") {- t.Fatalf("want row-2 header detected, got:\n%s", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "Item; Count") {+ t.Fatalf("want row-2 header detected, got:\n%s", spreadsheetText(res))}- if strings.Contains(xlsxTableHTML(res), "<th>100</th>") {- t.Fatalf("numeric row 1 must not be the header:\n%s", xlsxTableHTML(res))+ if strings.Contains(spreadsheetText(res), "100; 200") {+ t.Fatalf("numeric row 1 must not be the header:\n%s", spreadsheetText(res))}}// TestXLSXParser_CommonCaseNoRegression asserts the dominant case (header on-// row 1, no merges, no ListObject) renders row 1 as <th> unchanged.+// row 1, no merges, no ListObject) keeps the header row unchanged.func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {data := newTestXLSX(t, func(f *excelize.File) {mustSetCell(t, f, "Sheet1", "A1", "col_a")@@ -328,8 +366,8 @@ func TestXLSXParser_CommonCaseNoRegression(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "<tr><th>col_a</th><th>col_b</th></tr>") {- t.Fatalf("common-case header must render as <th>:\n%s", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "col_a; col_b") {+ t.Fatalf("common-case header must be preserved:\n%s", spreadsheetText(res))}}@@ -356,11 +394,11 @@ func TestDetectHeaderRow_BoldSubtotalNotHeader(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "<tr><th>2023</th><th>2024</th></tr>") {- t.Fatalf("numeric row 1 must remain the header:\n%s", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "2023; 2024") {+ t.Fatalf("numeric row 1 must remain the header:\n%s", spreadsheetText(res))}- if strings.Contains(xlsxTableHTML(res), "<th>Total</th>") {- t.Fatalf("bold subtotal row must not be promoted to header:\n%s", xlsxTableHTML(res))+ if strings.Contains(spreadsheetText(res), "Total; Summary") {+ t.Fatalf("bold subtotal row must not be promoted to header:\n%s", spreadsheetText(res))}}@@ -389,11 +427,15 @@ func TestDetectHeaderRow_StyledHeaderPastFarMerge(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "<th>Name</th><th>Desc</th>") {- t.Fatalf("narrow bold header past far merge must be detected:\n%s", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "Name; Desc") {+ t.Fatalf("narrow bold header past far merge must be detected:\n%s", spreadsheetText(res))}- if strings.Contains(xlsxTableHTML(res), "<th>Sales Report</th>") {- t.Fatalf("wide merged title must not be the header:\n%s", xlsxTableHTML(res))+ header := spreadsheetHeaderItem(res)+ if header == nil {+ t.Fatal("missing table_header item")+ }+ if got := header["text"]; got != "Name; Desc" {+ t.Fatalf("header text = %v, want Name; Desc", got)}}@@ -419,11 +461,15 @@ func TestDetectHeaderRow_StyledTextHeaderOverNumeric(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "<tr><th>Product</th><th>Units</th></tr>") {- t.Fatalf("styled text header over numeric data must be detected:\n%s", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "Product; Units") {+ t.Fatalf("styled text header over numeric data must be detected:\n%s", spreadsheetText(res))+ }+ header := spreadsheetHeaderItem(res)+ if header == nil {+ t.Fatal("missing table_header item")}- if strings.Contains(xlsxTableHTML(res), "<th>Report Title</th>") {- t.Fatalf("title stub must not be the header:\n%s", xlsxTableHTML(res))+ if got := header["text"]; got != "Product; Units" {+ t.Fatalf("header text = %v, want Product; Units", got)}}@@ -484,115 +530,8 @@ func TestMergeExtentCol(t *testing.T) {}}-// TestDecodeChunkRows exercises the chunk_rows setup knob across all the types-// JSON/yaml decoding can produce, plus the defaulting paths.-func TestDecodeChunkRows(t *testing.T) {- if got := decodeChunkRows(nil); got != defaultTableChunkRows {- t.Fatalf("nil setup: want default %d, got %d", defaultTableChunkRows, got)- }- if got := decodeChunkRows(map[string]any{}); got != defaultTableChunkRows {- t.Fatalf("empty setup: want default, got %d", got)- }- if got := decodeChunkRows(map[string]any{"chunk_rows": 100.0}); got != 100 {- t.Fatalf("float64 100: want 100, got %d", got)- }- if got := decodeChunkRows(map[string]any{"chunk_rows": 0.0}); got != defaultTableChunkRows {- t.Fatalf("float64 0: want default, got %d", got)- }- if got := decodeChunkRows(map[string]any{"chunk_rows": -5.0}); got != defaultTableChunkRows {- t.Fatalf("float64 -5: want default, got %d", got)- }- if got := decodeChunkRows(map[string]any{"chunk_rows": 256}); got != 256 {- t.Fatalf("int 256: want 256, got %d", got)- }- if got := decodeChunkRows(map[string]any{"chunk_rows": int64(512)}); got != 512 {- t.Fatalf("int64 512: want 512, got %d", got)- }- if got := decodeChunkRows(map[string]any{"chunk_rows": "256"}); got != defaultTableChunkRows {- t.Fatalf("string (unsupported type): want default, got %d", got)- }-}--// TestRenderSheetTables_FarMergeDoesNotBloatDataRows guards the memory blow-up-// from review: a far merge on a non-header row must not widen the header or any-// data row to the merge's full span. The header stays its natural width and no-// data row carries empty <td> cells.-func TestRenderSheetTables_FarMergeDoesNotBloatDataRows(t *testing.T) {- data := newTestXLSX(t, func(f *excelize.File) {- // Row 1: a wide merged title A1:Z1 (NOT the header).- mustSetCell(t, f, "Sheet1", "A1", "Report Title")- mustMergeCell(t, f, "Sheet1", "A1", "Z1")- // Row 2: the real header, made bold so detection anchors it.- hdrIdx := mustNewStyle(t, f, &excelize.Style{Font: &excelize.Font{Bold: true}})- mustSetCell(t, f, "Sheet1", "A2", "Name")- mustSetCell(t, f, "Sheet1", "B2", "Price")- mustSetCellStyle(t, f, "Sheet1", "A2", "B2", hdrIdx)- // Rows 3-4: data.- mustSetCell(t, f, "Sheet1", "A3", "Alice")- mustSetCell(t, f, "Sheet1", "B3", "9.99")- mustSetCell(t, f, "Sheet1", "A4", "Bob")- mustSetCell(t, f, "Sheet1", "B4", "19.99")- })- p, _ := NewXLSXParser("")- res := p.ParseWithResult(t.Context(), "t.xlsx", data)- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- html := xlsxTableHTML(res)- if !strings.Contains(html, "<tr><th>Name</th><th>Price</th></tr>") {- t.Fatalf("want row-2 header detected, got:\n%s", html)- }- if strings.Contains(html, "<th>Title</th>") {- t.Fatalf("wide merged title must not be the header:\n%s", html)- }- // The far merge is on row 1, not the header, so nothing is widened to 26- // columns: the header keeps 2 columns (no empty <th>) and data rows carry- // no empty <td>.- if strings.Contains(html, "<th></th>") {- t.Fatalf("header was bloated by the far merge:\n%s", html)- }- if strings.Count(html, "<td></td>") != 0 {- t.Fatalf("data rows were bloated by the far merge:\n%s", html)- }-}--// TestRenderSheetTables_MultiSheet asserts each sheet becomes its own <table>-// chunk, each carrying its own <caption>.-func TestRenderSheetTables_MultiSheet(t *testing.T) {- data := newTestXLSX(t, func(f *excelize.File) {- mustSetCell(t, f, "Sheet1", "A1", "Name")- mustSetCell(t, f, "Sheet1", "B1", "Age")- mustSetCell(t, f, "Sheet1", "A2", "Alice")- mustSetCell(t, f, "Sheet1", "B2", "30")- if _, err := f.NewSheet("HR"); err != nil {- t.Fatalf("NewSheet: %v", err)- }- mustSetCell(t, f, "HR", "A1", "Dept")- mustSetCell(t, f, "HR", "B1", "Head")- mustSetCell(t, f, "HR", "A2", "Eng")- mustSetCell(t, f, "HR", "B2", "Bob")- })- p, _ := NewXLSXParser("")- res := p.ParseWithResult(t.Context(), "t.xlsx", data)- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- html := xlsxTableHTML(res)- if !strings.Contains(html, "<caption>Sheet1</caption>") {- t.Fatalf("want Sheet1 caption, got:\n%s", html)- }- if !strings.Contains(html, "<caption>HR</caption>") {- t.Fatalf("want HR caption, got:\n%s", html)- }- // Two small sheets → two <table> chunks.- if n := strings.Count(html, "<table>"); n != 2 {- t.Fatalf("want 2 <table> chunks, got %d:\n%s", n, html)- }-}--// TestRenderSheetTables_EmptySheet asserts an empty/unreadable sheet yields an-// empty string rather than an empty <table> wrapper.-func TestRenderSheetTables_EmptySheet(t *testing.T) {+// TestXLSXParser_EmptySheet asserts an empty sheet yields no parser items.+func TestXLSXParser_EmptySheet(t *testing.T) {// excelize.NewFile yields a single empty "Sheet1".data := newTestXLSX(t, func(f *excelize.File) {})p, _ := NewXLSXParser("")diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.goindex 7fe65d9839..a7646a7b87 100644--- a/internal/parser/parser/parse_with_result_test.go+++ b/internal/parser/parser/parse_with_result_test.go@@ -20,7 +20,7 @@//// - HTMLParser — block-level walker that emits the python-compatible// {text, doc_type_kwd, ck_type} shape.-// - TextParser — paragraph-splitting for the text&code family+// - TextParser — normalized one-unit output for the text&code family// (.txt / .py / .js / .java / .c / .cpp / .h / .php / .go / .ts// / .sh / .cs / .kt / .sql).//@@ -43,31 +43,31 @@ import ("ragflow/internal/utility")-// TestTextParser_ParseWithResult_ParaSplit pins the paragraph-split-// rule. A blank-line-separated input yields one item per-// paragraph; the python TxtParser does the same.-func TestTextParser_ParseWithResult_ParaSplit(t *testing.T) {+// TestTextParser_ParseWithResult_PreservesOneTextUnit verifies that parser+// output does not claim chunk boundaries by splitting on its historical+// fixed delimiter set. GeneralChunker owns that decision from this point on.+func TestTextParser_ParseWithResult_PreservesOneTextUnit(t *testing.T) {p := NewTextParser()- src := []byte("First paragraph.\n\nSecond paragraph.\n\nThird.")- ctx := t.Context()- res := p.ParseWithResult(ctx, "doc.txt", src)+ res := p.ParseWithResult(t.Context(), "doc.txt", []byte("First sentence.\r\n\r\nSecond sentence!\rThird sentence?"))if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}if res.OutputFormat != "json" {- t.Errorf("OutputFormat = %q, want json", res.OutputFormat)+ t.Fatalf("OutputFormat = %q, want json", res.OutputFormat)}if got, want := res.File["name"], "doc.txt"; got != want {- t.Errorf("File.name = %v, want %v", got, want)+ t.Fatalf("File.name = %v, want %v", got, want)}- if len(res.JSON) != 3 {- t.Fatalf("JSON len = %d, want 3 (one per paragraph)", len(res.JSON))+ if len(res.JSON) != 1 {+ t.Fatalf("JSON len = %d, want 1: %#v", len(res.JSON), res.JSON)}- if got, want := res.JSON[0]["text"], "First paragraph."; got != want {- t.Errorf("JSON[0].text = %v, want %v", got, want)+ if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {+ t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)}- if got, want := res.JSON[2]["text"], "Third."; got != want {- t.Errorf("JSON[2].text = %v, want %v", got, want)+ got, _ := res.JSON[0]["text"].(string)+ want := "First sentence.\n\nSecond sentence!\nThird sentence?"+ if got != want {+ t.Errorf("JSON[0].text = %q, want %q", got, want)}}@@ -297,55 +297,9 @@ func TestGetParser_RoutesTextAndCode(t *testing.T) {}}-// TestTextParser_ParseWithResult_DefaultDelimiter pins the alignment fix:-// TextParser now splits on the flow parser's default delimiter set-// ("\n!?;。;!?"), mirroring deepdoc TxtParser.parser_txt, instead of only on-// blank lines. keep_delimiters=True (the flow _code path) keeps each trailing-// delimiter attached, so sentence-ending punctuation survives the split.-func TestTextParser_ParseWithResult_DefaultDelimiter(t *testing.T) {- ctx := t.Context()- p := NewTextParser()-- // Single newlines now split too (previously only "\n\n" did).- src := []byte("First line.\nSecond line.\nThird line.")- res := p.ParseWithResult(ctx, "doc.txt", src)- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- if len(res.JSON) != 3 {- t.Fatalf("JSON len = %d, want 3 (single-newline split)", len(res.JSON))- }-- // Sentence delimiters split and keep the delimiter attached. The period- // "." is NOT in the default set, so "Foo. Bar" stays joined until the ";".- // TrimSpace drops the incidental leading space before each delimiter (the- // package's established convention, also used by markdown leafText).- src = []byte("Hello! World? Foo. Bar; Baz。 Qux!")- res = p.ParseWithResult(ctx, "doc.txt", src)- want := []string{"Hello!", "World?", "Foo. Bar;", "Baz。", "Qux!"}- if len(res.JSON) != len(want) {- t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)- }- for i, w := range want {- if got := res.JSON[i]["text"]; got != w {- t.Errorf("JSON[%d].text = %v, want %v", i, got, w)- }- }-- // Chinese sentence delimiters split the same way.- src = []byte("这是第一句。这是第二句!第三句?结尾。")- res = p.ParseWithResult(ctx, "doc.txt", src)- if len(res.JSON) != 4 {- t.Fatalf("JSON len = %d, want 4 (CJK delimiter split)", len(res.JSON))- }-}-// TestTextParser_ParseWithResult_NewlineNormalization pins the// normalizeTextNewlines contract: CRLF ("\r\n") and lone-CR ("\r") line-// endings fold to LF before splitting, so every variant of the same logical-// content yields identical items. This mirrors rag/nlp/delim.-// normalize_text_newlines, which is what Python splits on, so Windows-line-// documents parse identically to Unix ones.+// endings fold to LF while the complete text remains one parser unit.func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {ctx := t.Context()p := NewTextParser()@@ -394,22 +348,13 @@ func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {// TestTextParser_AlignmentGolden verifies Go's ParseWithResult output is// content-equivalent to Python's _code on the shared sample, using the shared-// concatenation-normalization alignment tool (align_test.go). Python applies-// the OVER_CAP token merge (chunking ownership retained by the Go Chunker per-// contract #17799), so item counts differ; the-// comparison normalizes both (delimiters stripped, whitespace collapsed) and-// joins on whitespace, so only CONTENT equivalence — not byte-exact layout — is-// checked. The golden files are a NORMALIZED content baseline, not a verbatim-// Python transcript: a fresh _code run at chunk_token_num=128 may merge into a-// different item count/structure (e.g. the en sample collapses to one chunk while-// the golden keeps prose and code as two items) and may collapse inter-sentence-// newlines, so do not treat them as byte-exact.+// concatenation-normalization alignment tool (align_test.go). Item boundaries+// intentionally differ because Parser no longer performs Python's delimiter+// split; the comparison joins normalized text and checks content only.//-// No generator script is committed. The baseline meta records generator, sample,-// delimiter, keep_delimiters and chunk_token_num (see textcode.python.en/zh.golden.json);-// sample, delimiter, keep_delimiters, chunk_token_num): call the python flow-// _code on the sample with keep_delimiters=True and the default delimiter set,-// then project each merged section to {"text": section[0], "doc_type_kwd": "text"}.+// No generator script is committed. The baseline metadata records the Python+// generator and delimiter used for normalization; Parser output is compared+// for content equivalence, not Python's historical item boundaries.func TestTextParser_AlignmentGolden(t *testing.T) {ctx := t.Context()p := NewTextParser()@@ -439,54 +384,10 @@ func TestTextParser_AlignmentGolden(t *testing.T) {goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)- if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(DefaultTextCodeDelimiter)); !ok {+ delimiter, _ := gd.Meta["delimiter"].(string)+ if ok, diff := CompareAlignment(goText, pyText, TextCodeAlignOptions(delimiter)); !ok {t.Fatalf("text&code parser not aligned with Python golden:%s", diff)}})}}--// TestTextParser_AdjacentDelimiters pins Go's behavior on adjacent-// delimiters, confirming it matches Python's deepdoc TxtParser.parser_txt-// delimiter-loop exactly (not a divergence). Both ports run the same-// re.split(r"(%s)" % dels, txt) loop with keep_delimiters=True and merge a-// run of adjacent delimiters into the preceding segment, so the standalone-// second delimiter is dropped on both sides: "a!!b" → ["a!", "b"] (verified-// against deepdoc/parser/txt_parser.py). The alignment test's delimiter-strip-// normalization also reconciles this, but this test guards splitCapturingDelims-// directly so a future silent change there is caught independently.-func TestTextParser_AdjacentDelimiters(t *testing.T) {- ctx := t.Context()- p := NewTextParser()-- // Two adjacent sentence delimiters: Go (and Python's parser_txt) merge- // them into the preceding segment and drop the standalone second delimiter.- src := []byte("a!!b")- res := p.ParseWithResult(ctx, "doc.txt", src)- if res.Err != nil {- t.Fatalf("ParseWithResult: %v", res.Err)- }- want := []string{"a!", "b"}- if len(res.JSON) != len(want) {- t.Fatalf("adjacent delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)- }- for i, w := range want {- if got := res.JSON[i]["text"]; got != w {- t.Errorf("adjacent delimiters: JSON[%d].text = %v, want %v", i, got, w)- }- }-- // Delimiters separated by text each attach to their own segment (no merge- // across the gap).- src = []byte("x?y!z")- res = p.ParseWithResult(ctx, "doc.txt", src)- want = []string{"x?", "y!", "z"}- if len(res.JSON) != len(want) {- t.Fatalf("mixed delimiters: JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)- }- for i, w := range want {- if got := res.JSON[i]["text"]; got != w {- t.Errorf("mixed delimiters: JSON[%d].text = %v, want %v", i, got, w)- }- }-}diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.goindex 4a88d9b856..1c1efd33b0 100644--- a/internal/parser/parser/text_parser.go+++ b/internal/parser/parser/text_parser.go@@ -22,21 +22,13 @@// side needs a parser for these families so `text&code` resolves to a// real ParseResultProducer.//-// TextParser fills that gap with a real implementation: it splits the-// input into fine segments on the flow parser's default delimiter set and-// emits the python-compatible `{text, doc_type_kwd:"text"}` shape. Block-// boundaries converge to the Python flow TxtParser (which uses the same-// delimiter set); the OVER_CAP token merge that Python applies afterwards is-// intentionally NOT performed here — chunking ownership stays with the-// downstream Chunker (contract #17799), so-// item counts differ from Python's merged chunks and are reconciled by the-// stitch-compare alignment test (align_test.go).+// TextParser decodes and normalizes text&code input, then emits one parser+// unit. Chunk boundaries belong to GeneralChunker, not to this parser.package parserimport ("context"- "regexp""strings")@@ -49,19 +41,11 @@ func NewTextParser() *TextParser {return &TextParser{}}-// ParseWithResult emits one item per non-empty paragraph. The-// output format is "json" to mirror the python TxtParser's-// behaviour (it emits a list of items with text + doc_type_kwd).-//-// The items slice is always non-nil so downstream chunkers see a-// non-empty JSON payload even for an empty input (mirrors the-// MarkdownParser convention at markdown_parser.go:71-76).+// ParseWithResult emits the complete normalized text as one parser unit. The+// output format is "json" to mirror the parser component boundary. No fixed+// delimiter or token budget is applied here.func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {decoded, encName := DecodeToUTF8(data, "text/plain")- items := textParserItems(decoded)- if items == nil {- items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}- }return ParseResult{OutputFormat: "json",File: map[string]any{@@ -69,7 +53,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data"size": len(data),"encoding": encName,},- JSON: items,+ JSON: []map[string]any{NewTextJSONItem(normalizeTextNewlines(string(decoded)))},}}@@ -77,36 +61,8 @@ func (p *TextParser) String() string {return "TextParser"}-// defaultTextDelimiterPattern is the regexp alternation of the flow parser's-// default delimiter set DefaultTextCodeDelimiter (rag/flow/parser/parser.py:_code-// → deepdoc TxtParser default "\n!?;。;!?"), each rune re.escape'd to mirror-// rag/nlp/delim.compile_delimiter_pattern. The Parser component has no user-facing-// delimiter config entry, so this default-// is exactly what the python flow always splits on. The shared DefaultTextCodeDelimiter-// const lives in delimiter.go so production and the alignment tests use one source-// of truth. Go's regexp.Split drops captured delimiters, so splitCapturingDelims-// walks the match indexes manually to reproduce python's re.split(r"(%s)" % pattern, txt)-// interleaving.-var (- defaultTextDelimiterPattern = buildDelimiterPattern(DefaultTextCodeDelimiter)- textDelimiterSplitRe = regexp.MustCompile(defaultTextDelimiterPattern)- textDelimiterExactRe = regexp.MustCompile("^(?:" + defaultTextDelimiterPattern + ")$")-)--// buildDelimiterPattern builds an alternation of re.escape'd delimiter runes-// (longest-first is a no-op here: every delimiter in the default set is a-// single rune, so insertion order is preserved like python's stable sort).-func buildDelimiterPattern(delims string) string {- parts := make([]string, 0, len(delims))- for _, r := range delims {- parts = append(parts, regexp.QuoteMeta(string(r)))- }- return strings.Join(parts, "|")-}--// normalizeTextNewlines folds CRLF and standalone CR to LF, mirroring-// rag/nlp/delim.normalize_text_newlines so Windows-line-ending documents split-// identically to Unix ones.+// normalizeTextNewlines folds CRLF and standalone CR to LF while preserving+// the rest of the document content for GeneralChunker.func normalizeTextNewlines(s string) string {if s == "" {return s@@ -114,76 +70,3 @@ func normalizeTextNewlines(s string) string {s = strings.ReplaceAll(s, "\r\n", "\n")return strings.ReplaceAll(s, "\r", "\n")}--// splitCapturingDelims reproduces python re.split(r"(%s)" % pattern, s): it-// splits on the regexp and includes each matched delimiter as its own element-// (with empty strings between adjacent delimiters) so callers can keep or drop-// them. Go's regexp.Split discards captured groups, hence the manual walk.-func splitCapturingDelims(s string, re *regexp.Regexp) []string {- locs := re.FindAllStringIndex(s, -1)- if len(locs) == 0 {- return []string{s}- }- out := make([]string, 0, 2*len(locs)+1)- prev := 0- for _, loc := range locs {- out = append(out, s[prev:loc[0]])- out = append(out, s[loc[0]:loc[1]])- prev = loc[1]- }- out = append(out, s[prev:])- return out-}--// textParserItems splits data into fine segments on the flow parser's default-// delimiter set, mirroring deepdoc.parser.txt_parser.TxtParser.parser_txt up to-// (but not including) the OVER_CAP token merge. The token merge is intentionally-// NOT performed here — chunking ownership stays with the downstream Chunker per-// contract #17799 — so item counts differ from the-// python flow's merged chunks and are reconciled by the stitch-compare alignment-// test (align_test.go).-//-// Unlike the python signature default keep_delimiters=False, the flow _code-// path calls TxtParser with keep_delimiters=True, so each segment keeps its-// trailing delimiter attached (sentence-ending punctuation preserved for code-// and prose).-//-// No per-item byte cap is applied: the parser is a pure delimiter splitter, just-// like python's parser_txt (which also does no size slicing). Sizing belongs to-// the chunker and the embedding truncation step, so a continuous run longer than-// the embedding token budget (e.g. a minified / no-newline file) is kept as one-// item here and collapsed to one chunk downstream — matching python's behaviour-// rather than diverging from it.-func textParserItems(data []byte) []map[string]any {- txt := normalizeTextNewlines(string(data))- secs := splitCapturingDelims(txt, textDelimiterSplitRe)-- var paras []string- for i, sec := range secs {- if textDelimiterExactRe.MatchString(sec) {- continue- }- if sec == "" {- continue- }- // keep_delimiters=True: append the delimiter to the segment it- // follows, mirroring python's parser_txt.- if i+1 < len(secs) && textDelimiterExactRe.MatchString(secs[i+1]) {- sec += secs[i+1]- }- paras = append(paras, sec)- }-- var items []map[string]any- for _, para := range paras {- text := strings.TrimSpace(para)- if text == "" {- continue- }- items = append(items, map[string]any{- "text": text,- "doc_type_kwd": "text",- })- }- return items-}diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex 3dc510333b..e807ea8b63 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -25,7 +25,7 @@ type XLSParser struct {libType stringParseMethod stringOutputFormat string- ChunkRows int+ HTML4Excel boolTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType string@@ -38,7 +38,6 @@ func NewXLSParser(libType string) (*XLSParser, error) {}return &XLSParser{libType: libType,- ChunkRows: defaultTableChunkRows,TCADPTableResultType: "1",TCADPMarkdownImageResponseType: "1",}, nil@@ -58,7 +57,10 @@ func (p *XLSParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["output_format"].(string); ok && v != "" {p.OutputFormat = v}- p.ChunkRows = decodeChunkRows(setup)+ if v, ok := setup["html4excel"].(bool); ok {+ p.HTML4Excel = v+ }+ deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {p.TCADPAPIServer = v}@@ -91,12 +93,7 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [}}- chunkRows := p.ChunkRows- if chunkRows <= 0 {- chunkRows = defaultTableChunkRows- }-- items, warnings, sheetsCount, err := parseXLSXBytes(data, chunkRows)+ items, warnings, sheetsCount, err := parseXLSXBytes(data, p.HTML4Excel)if err != nil {return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex 6aef5e68cd..c9ac44e4db 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -3,7 +3,6 @@ package parserimport ("bytes""context"- "strings""testing""github.com/xuri/excelize/v2"@@ -27,7 +26,7 @@ func createTestExcelBytes(t *testing.T) []byte {return buf.Bytes()}-func TestXLSParser_HTMLAndJSONOutput(t *testing.T) {+func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {data := createTestExcelBytes(t)p, err := NewXLSParser("excelize")if err != nil {@@ -47,14 +46,14 @@ func TestXLSParser_HTMLAndJSONOutput(t *testing.T) {t.Fatalf("res.JSON should have structured table items")}item := res.JSON[0]- if text, ok := item["text"].(string); !ok || !strings.Contains(text, "<table>") {- t.Errorf("table item text should contain rendered <table>, got %v", item["text"])+ if item["ck_type"] != "table_header" {+ t.Errorf("header ck_type = %v, want table_header", item["ck_type"])}- if item["doc_type_kwd"] != "table" {- t.Errorf("doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])+ if len(res.JSON) < 2 || res.JSON[1]["text"] != "Header1:Val1; Header2:Val2" {+ t.Errorf("row item = %#v", res.JSON)}- if item["ck_type"] != "table" {- t.Errorf("ck_type = %v, want 'table'", item["ck_type"])+ if item["doc_type_kwd"] != "table" {+ t.Errorf("header doc_type_kwd = %v, want 'table'", item["doc_type_kwd"])}// Configure output format to "json"@@ -88,11 +87,11 @@ func TestXLSXParser_JSONOutput(t *testing.T) {if res.OutputFormat != "json" {t.Errorf("res.OutputFormat = %q, want 'json'", res.OutputFormat)}- if len(res.JSON) == 0 {- t.Fatalf("res.JSON should have items")+ if len(res.JSON) < 2 {+ t.Fatalf("res.JSON = %#v, want header and row", res.JSON)}- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {- t.Errorf("item text should contain HTML table, got %v", res.JSON[0]["text"])+ if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {+ t.Errorf("items should be header and row, got %#v", res.JSON)}// Even if configured to legacy "html", format is unified to "json"diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.goindex bf3fe936d7..568225603d 100644--- a/internal/parser/parser/xls_tcadp_test.go+++ b/internal/parser/parser/xls_tcadp_test.go@@ -1,6 +1,8 @@package parserimport (+ "bytes"+ "log/slog""net/http""net/http/httptest""strings"@@ -32,6 +34,7 @@ func TestXLSXParser_ParseWithResult_TCADPJSONIntegration(t *testing.T) {p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser","output_format": "json",+ "chunk_rows": 50,"tcadp_apiserver": server.URL,"tcadp_api_key": "tcadp-secret","table_result_type": "1",@@ -220,11 +223,11 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if len(res.JSON) == 0 {- t.Fatal("JSON items is empty; want structured table items")+ if len(res.JSON) < 2 {+ t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))}- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {- t.Fatalf("JSON item text = %v; want rendered table", res.JSON[0]["text"])+ if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {+ t.Fatalf("JSON items = %#v; want header and row", res.JSON)}}@@ -236,6 +239,7 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {p.ConfigureFromSetup(map[string]any{"parse_method": "TCADP parser","output_format": "json",+ "chunk_rows": 50,"tcadp_apiserver": "https://tcadp.example.com","tcadp_api_key": "secret","table_result_type": "2",@@ -257,3 +261,16 @@ func TestXLSXParser_ConfigureFromSetup_TCADP(t *testing.T) {t.Fatalf("TCADPMarkdownImageResponseType = %q, want %q", got, want)}}++func TestSpreadsheetParserWarnsForDeprecatedChunkRows(t *testing.T) {+ var logs bytes.Buffer+ previousLogger := slog.Default()+ slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})))+ t.Cleanup(func() { slog.SetDefault(previousLogger) })++ p := NewCSVParser()+ p.ConfigureFromSetup(map[string]any{"chunk_rows": 32})+ if !strings.Contains(logs.String(), "deprecated chunk_rows") {+ t.Fatalf("logs = %q, want deprecated chunk_rows warning", logs.String())+ }+}diff --git a/internal/parser/parser/xlsx_normalize_test.go b/internal/parser/parser/xlsx_normalize_test.goindex 381909d42b..3f3fb8e0fc 100644--- a/internal/parser/parser/xlsx_normalize_test.go+++ b/internal/parser/parser/xlsx_normalize_test.go@@ -29,10 +29,10 @@ func TestXLSXParser_NormalizesInvalidSheetName(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}- html := xlsxTableHTML(res)+ text := spreadsheetText(res)for _, want := range []string{"Name", "Amount", "Alice", "100"} {- if !strings.Contains(html, want) {- t.Fatalf("parsed table = %q, want %q", html, want)+ if !strings.Contains(text, want) {+ t.Fatalf("parsed rows = %q, want %q", text, want)}}if !strings.Contains(strings.Join(res.Warnings, "\n"), "Visible:Data") {@@ -76,8 +76,8 @@ func TestXLSXParser_NormalizesEmojiSheetNameWithinUTF16Limit(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "Alice") {- t.Fatalf("parsed table = %q, want Alice", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "Alice") {+ t.Fatalf("parsed rows = %q, want Alice", spreadsheetText(res))}}@@ -138,8 +138,8 @@ func TestXLSXParser_WarnsWhenMergedCellsCannotBeRead(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "Alice") {- t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "Alice") {+ t.Fatalf("parsed rows = %q, want sheet data", spreadsheetText(res))}if !strings.Contains(strings.Join(res.Warnings, "\n"), "merged cells") {t.Fatalf("warnings = %v, want merged-cells warning", res.Warnings)@@ -161,8 +161,8 @@ func TestXLSXParser_WarnsWhenTableMetadataCannotBeRead(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if !strings.Contains(xlsxTableHTML(res), "Alice") {- t.Fatalf("parsed table = %q, want sheet data", xlsxTableHTML(res))+ if !strings.Contains(spreadsheetText(res), "Alice") {+ t.Fatalf("parsed rows = %q, want sheet data", spreadsheetText(res))}if !strings.Contains(strings.Join(res.Warnings, "\n"), "table metadata") {t.Fatalf("warnings = %v, want table-metadata warning", res.Warnings)diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.goindex e91c5c97eb..c5e9ae57ba 100644--- a/internal/parser/parser/xlsx_parse_method_test.go+++ b/internal/parser/parser/xlsx_parse_method_test.go@@ -78,7 +78,7 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {t.Fatalf("OutputFormat = %q, want %q", got, want)}if len(res.JSON) != 1 {- t.Fatalf("JSON item count = %d, want 1", len(res.JSON))+ t.Fatalf("JSON item count = %d, want header-only result", len(res.JSON))}text, _ := res.JSON[0]["text"].(string)if !strings.Contains(text, tc.cellValue) {@@ -120,6 +120,9 @@ func TestXLSXParser_ExtractsFloatingImages(t *testing.T) {if image["image"] != "data:image/png;base64,"+pngBase64 {t.Fatalf("image data = %v, want data URL", image["image"])}+ if image["sheet_index"] != 1 || image["row_start"] != 3 || image["row_end"] != 3 || image["col_start"] != 3 || image["col_end"] != 3 {+ t.Fatalf("image coordinates = %#v, want Sheet1!C3", image)+ }}func TestXLSXParser_ImageWithoutAltTextUsesAnchorCell(t *testing.T) {@@ -157,7 +160,7 @@ func mustDecodeBase64(t *testing.T, encoded string) []byte {}// TestCSVParser_DeepDocParseMethod asserts the CSV parser accepts the-// default "deepdoc" parse_method and renders the default HTML table.+// default "deepdoc" parse_method and emits row IR.func TestCSVParser_DeepDocParseMethod(t *testing.T) {p := NewCSVParser()p.ConfigureFromSetup(map[string]any{"parse_method": "deepdoc"})@@ -170,11 +173,11 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {if got, want := res.OutputFormat, "json"; got != want {t.Fatalf("OutputFormat = %q, want %q", got, want)}- if len(res.JSON) == 0 {- t.Fatal("JSON items is empty; want structured table items")+ if len(res.JSON) < 2 {+ t.Fatalf("JSON items = %d, want at least header and one data row", len(res.JSON))}- if text, ok := res.JSON[0]["text"].(string); !ok || !strings.Contains(text, "<table>") {- t.Fatalf("JSON item text = %v, want a rendered <table>", res.JSON[0]["text"])+ if res.JSON[0]["ck_type"] != "table_header" || res.JSON[1]["ck_type"] != "table_row" {+ t.Fatalf("JSON items = %#v, want header and row", res.JSON)}}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex eea92c925d..aff077bd90 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -29,7 +29,7 @@ type XLSXParser struct {libType stringParseMethod stringOutputFormat string- ChunkRows int+ HTML4Excel boolTCADPAPIServer stringTCADPAPIKey stringTCADPTableResultType string@@ -42,7 +42,6 @@ func NewXLSXParser(libType string) (*XLSXParser, error) {}return &XLSXParser{libType: libType,- ChunkRows: defaultTableChunkRows,TCADPTableResultType: "1",TCADPMarkdownImageResponseType: "1",}, nil@@ -62,7 +61,10 @@ func (p *XLSXParser) ConfigureFromSetup(setup map[string]any) {if v, ok := setup["output_format"].(string); ok && v != "" {p.OutputFormat = v}- p.ChunkRows = decodeChunkRows(setup)+ if v, ok := setup["html4excel"].(bool); ok {+ p.HTML4Excel = v+ }+ deprecatedChunkRows(setup, p.String())if v, ok := setup["tcadp_apiserver"].(string); ok && v != "" {p.TCADPAPIServer = v}@@ -113,12 +115,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data// for spreadsheet processing.}- chunkRows := p.ChunkRows- if chunkRows <= 0 {- chunkRows = defaultTableChunkRows- }-- items, warnings, sheets, err := parseXLSXBytes(data, chunkRows)+ items, warnings, sheets, err := parseXLSXBytes(data, p.HTML4Excel)if err == nil {return xlsxParseResult(filename, items, warnings, sheets)}@@ -130,7 +127,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, dataif !changed {return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}}- items, warnings, sheets, retryErr := parseXLSXBytes(normalized, chunkRows)+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized, p.HTML4Excel)if retryErr != nil {return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}}@@ -138,7 +135,7 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, datareturn xlsxParseResult(filename, items, warnings, sheets)}-func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, int, error) {+func parseXLSXBytes(data []byte, html4excel bool) ([]map[string]any, []string, int, error) {f, err := excelize.OpenReader(bytes.NewReader(data))if err != nil {return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)@@ -149,22 +146,30 @@ func parseXLSXBytes(data []byte, chunkRows int) ([]map[string]any, []string, intitems := make([]map[string]any, 0)warnings := make([]string, 0)for sheetIdx, sheet := range sheets {- tables, sheetWarnings, err := renderSheetTableChunks(f, sheet, chunkRows)+ records, dataRows, headerRow, sheetWarnings, err := readSpreadsheetRecords(f, sheet)if err != nil {return nil, warnings, len(sheets), err}warnings = append(warnings, sheetWarnings...)- for _, table := range tables {- items = append(items, NewTableJSONItem(table.HTML, sheet, [][]float64{{- float64(sheetIdx + 1),- float64(table.RowStart),- float64(table.RowEnd),- float64(table.ColStart),- float64(table.ColEnd),- }}))+ var sheetItems []map[string]any+ if html4excel {+ if table := recordsToHTMLTableItem(records, sheet, sheetIdx+1, headerRow, dataRows); table != nil {+ sheetItems = []map[string]any{table}+ }+ } else {+ sheetItems = recordsToSpreadsheetItems(records, sheet, sheetIdx+1, headerRow, dataRows)}images, imageWarnings := extractXLSXImages(f, sheet)- items = append(items, images...)+ for _, image := range images {+ row, _ := numericItemInt(image["row_start"])+ col, _ := numericItemInt(image["col_start"])+ image["sheet_index"] = sheetIdx + 1+ image["table_id"] = fmt.Sprintf("sheet-%d", sheetIdx+1)+ image["positions"] = [][]float64{{float64(sheetIdx + 1), float64(row), float64(row), float64(col), float64(col)}}+ }+ sheetItems = append(sheetItems, images...)+ sortSpreadsheetItems(sheetItems)+ items = append(items, sheetItems...)warnings = append(warnings, imageWarnings...)}return items, warnings, len(sheets), nildiff --git a/internal/service/components_service_test.go b/internal/service/components_service_test.goindex b45653b6e5..d20ddc9bc5 100644--- a/internal/service/components_service_test.go+++ b/internal/service/components_service_test.go@@ -28,7 +28,7 @@ import (_ "ragflow/internal/agent/component" // registers agent components"ragflow/internal/agent/runtime"_ "ragflow/internal/ingestion/component" // registers ingestion main components- _ "ragflow/internal/ingestion/component/chunker" // registers 4 chunker variants+ _ "ragflow/internal/ingestion/component/chunker" // registers chunker variants)func TestComponentsService_List_NoFilter(t *testing.T) {@@ -62,7 +62,7 @@ func TestComponentsService_List_FilterIngestion(t *testing.T) {t.Fatalf("List(Ingestion) returned error: %v", err)}wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",+ "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",}assertComponentNameSet(t, "ingestion", namesOf(got), wantNames)@@ -85,7 +85,7 @@ func TestComponentsService_List_FilterIngestionAndShared(t *testing.T) {t.Fatalf("List(Ingestion,Shared) returned error: %v", err)}wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",+ "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",}assertComponentNameSet(t, "ingestion+shared", namesOf(got), wantNames)@@ -104,7 +104,7 @@ func TestComponentsService_List_FilterDuplicates(t *testing.T) {t.Fatalf("List(Ingestion x3) returned error: %v", err)}wantNames := []string{- "extractor", "file", "grouptitlechunker", "hierarchytitlechunker", "manualchunker",+ "extractor", "file", "generalchunker", "grouptitlechunker", "hierarchytitlechunker", "manualchunker","onechunker", "pagechunker", "qachunker", "parser", "tablechunker", "titlechunker", "tokenchunker", "tokenizer",}if len(got) != len(wantNames) {diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 5e0d8b9c02..012b4d4e46 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -5,6 +5,7 @@ import ("encoding/base64""encoding/json""fmt"+ "html""ragflow/internal/dao""ragflow/internal/entity""ragflow/internal/parser/parser"@@ -232,6 +233,9 @@ func parseResultText(res parser.ParseResult) (string, error) {return res.HTML, nilcase "json":if len(res.JSON) > 0 {+ if rendered, ok := renderSpreadsheetJSON(res.JSON); ok {+ return rendered, nil+ }parts := make([]string, 0, len(res.JSON))for _, item := range res.JSON {if text, ok := item["text"].(string); ok {@@ -260,6 +264,164 @@ func parseResultText(res parser.ParseResult) (string, error) {}}+func renderSpreadsheetJSON(items []map[string]any) (string, bool) {+ parts := make([]string, 0, len(items))+ rendered := false+ for i := 0; i < len(items); {+ if !isSpreadsheetRowItem(items[i]) {+ if text, ok := items[i]["text"].(string); ok {+ parts = append(parts, text)+ } else {+ raw, err := json.Marshal(items[i])+ if err != nil {+ return "", false+ }+ parts = append(parts, string(raw))+ }+ i+++ continue+ }++ start := i+ i = spreadsheetTableEnd(items, start)+ parts = append(parts, renderSpreadsheetTable(items[start:i]))+ rendered = true+ }+ if !rendered {+ return "", false+ }+ return strings.Join(parts, "\n"), true+}++func isSpreadsheetRowItem(item map[string]any) bool {+ kind, _ := item["ck_type"].(string)+ return kind == "table_header" || kind == "table_row"+}++func spreadsheetTableKey(item map[string]any) string {+ if tableID, _ := item["table_id"].(string); strings.TrimSpace(tableID) != "" {+ return "table:" + tableID+ }+ if sheetIndex, ok := item["sheet_index"].(float64); ok {+ return fmt.Sprintf("sheet-index:%g", sheetIndex)+ }+ if sheetIndex, ok := item["sheet_index"].(int); ok {+ return fmt.Sprintf("sheet-index:%d", sheetIndex)+ }+ if sheet, _ := item["sheet"].(string); strings.TrimSpace(sheet) != "" {+ return "sheet:" + sheet+ }+ return ""+}++func spreadsheetTableEnd(items []map[string]any, start int) int {+ key := spreadsheetTableKey(items[start])+ if key != "" {+ i := start+ for i < len(items) && isSpreadsheetRowItem(items[i]) && spreadsheetTableKey(items[i]) == key {+ i+++ }+ return i+ }++ // Without a stable table identity, do not merge unrelated row sequences.+ // A header can still own the following anonymous rows until the next+ // anonymous header; anonymous row-only inputs are kept as separate tables+ // because their table boundary cannot be inferred safely.+ if kind, _ := items[start]["ck_type"].(string); kind == "table_header" {+ i := start + 1+ for i < len(items) && isSpreadsheetRowItem(items[i]) {+ if nextKind, _ := items[i]["ck_type"].(string); nextKind == "table_header" {+ break+ }+ if spreadsheetTableKey(items[i]) != "" {+ break+ }+ i+++ }+ return i+ }+ return start + 1+}++func renderSpreadsheetTable(items []map[string]any) string {+ if len(items) == 0 {+ return ""+ }+ var header []string+ if kind, _ := items[0]["ck_type"].(string); kind == "table_header" {+ header = spreadsheetStringSlice(items[0]["cells"])+ if len(header) == 0 {+ header = spreadsheetStringSlice(items[0]["text"])+ }+ } else {+ // Row IR carries the logical column names separately from its cells.+ // Use those names as a preview header without consuming the first row.+ header = spreadsheetStringSlice(items[0]["headers"])+ }++ sheet, _ := items[0]["sheet"].(string)+ var builder strings.Builder+ builder.WriteString("<table>")+ if strings.TrimSpace(sheet) != "" {+ builder.WriteString("<caption>")+ builder.WriteString(html.EscapeString(strings.TrimSpace(sheet)))+ builder.WriteString("</caption>")+ }+ if len(header) > 0 {+ builder.WriteString("<tr>")+ for _, cell := range header {+ builder.WriteString("<th>")+ builder.WriteString(html.EscapeString(cell))+ builder.WriteString("</th>")+ }+ builder.WriteString("</tr>")+ }+ for _, item := range items {+ kind, _ := item["ck_type"].(string)+ if kind == "table_header" {+ continue+ }+ cells := spreadsheetStringSlice(item["cells"])+ if len(cells) == 0 {+ cells = spreadsheetStringSlice(item["text"])+ }+ builder.WriteString("<tr>")+ for _, cell := range cells {+ builder.WriteString("<td>")+ builder.WriteString(html.EscapeString(cell))+ builder.WriteString("</td>")+ }+ builder.WriteString("</tr>")+ }+ builder.WriteString("</table>")+ return builder.String()+}++func spreadsheetStringSlice(value any) []string {+ switch values := value.(type) {+ case []string:+ out := make([]string, len(values))+ for i, value := range values {+ out[i] = strings.TrimSpace(value)+ }+ return out+ case []any:+ out := make([]string, 0, len(values))+ for _, value := range values {+ out = append(out, strings.TrimSpace(fmt.Sprint(value)))+ }+ return out+ case string:+ if strings.TrimSpace(values) == "" {+ return nil+ }+ return []string{strings.TrimSpace(values)}+ default:+ return nil+ }+}+// parseFileContent tries to parse a file's contents using the appropriate parser.// Falls back to returning raw text if no parser is available.func parseFileContent(ctx context.Context, filename string, data []byte) string {diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.goindex 795de53997..5a70e6edf3 100644--- a/internal/service/file/file_content_test.go+++ b/internal/service/file/file_content_test.go@@ -75,6 +75,91 @@ func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T)}}+func TestParseResultText_SpreadsheetRowsRenderReadableHTMLTable(t *testing.T) {+ result, err := parseResultText(parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {+ "text": "a; b",+ "doc_type_kwd": "table",+ "ck_type": "table_header",+ "table_id": "sheet-1",+ "sheet": "Data",+ "cells": []string{"a", "b"},+ },+ {+ "text": "a:1; b:2 ——Data",+ "doc_type_kwd": "text",+ "ck_type": "table_row",+ "table_id": "sheet-1",+ "sheet": "Data",+ "headers": []string{"a", "b"},+ "cells": []string{"1", "2"},+ },+ },+ })+ if err != nil {+ t.Fatalf("parseResultText: %v", err)+ }+ for _, want := range []string{"<table", "<caption>Data</caption>", "<th>a</th>", "<td>1</td>"} {+ if !strings.Contains(result, want) {+ t.Errorf("parseResultText = %q, want %q", result, want)+ }+ }+}++func TestParseResultText_SpreadsheetRowsWithoutHeaderDoNotDuplicateFirstRow(t *testing.T) {+ result, err := parseResultText(parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {+ "ck_type": "table_row",+ "headers": []string{"name", "value"},+ "cells": []string{"alpha", "1"},+ },+ },+ })+ if err != nil {+ t.Fatalf("parseResultText: %v", err)+ }+ if !strings.Contains(result, "<th>name</th>") {+ t.Fatalf("row headers were not rendered: %q", result)+ }+ if got := strings.Count(result, "<td>alpha</td>"); got != 1 {+ t.Fatalf("first row rendered %d times, want once: %q", got, result)+ }+}++func TestParseResultText_SpreadsheetRowsWithoutIdentityStaySeparate(t *testing.T) {+ result, err := parseResultText(parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {+ "ck_type": "table_header",+ "cells": []string{"first"},+ },+ {+ "ck_type": "table_row",+ "cells": []string{"one"},+ },+ {+ "ck_type": "table_header",+ "cells": []string{"second"},+ },+ {+ "ck_type": "table_row",+ "cells": []string{"two"},+ },+ },+ })+ if err != nil {+ t.Fatalf("parseResultText: %v", err)+ }+ if got := strings.Count(result, "<table>"); got != 2 {+ t.Fatalf("unknown spreadsheet tables were merged: got %d tables in %q", got, result)+ }+}+func TestParseResultText_EmailJSONKeepsSearchableHeaders(t *testing.T) {raw := []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: Parser contract\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEmail body")p := parser.NewEmailParser()diff --git a/web/src/components/operator-icon.tsx b/web/src/components/operator-icon.tsxindex ff52e29325..6052a11954 100644--- a/web/src/components/operator-icon.tsx+++ b/web/src/components/operator-icon.tsx@@ -93,6 +93,7 @@ export const LucideIconMap = {[Operator.File]: LucideFile,[Operator.Parser]: LucideFilePlay,[Operator.Tokenizer]: LucideListPlus,+ [Operator.GeneralChunker]: LucideBlocks,[Operator.TokenChunker]: LucideBlocks,[Operator.TitleChunker]: LucideHeading,[Operator.Extractor]: LucideFileStack,diff --git a/web/src/components/pipeline-operator-tabs/parser-config-validation.ts b/web/src/components/pipeline-operator-tabs/parser-config-validation.tsindex 59ca040bae..6e7a8044b7 100644--- a/web/src/components/pipeline-operator-tabs/parser-config-validation.ts+++ b/web/src/components/pipeline-operator-tabs/parser-config-validation.ts@@ -17,6 +17,7 @@import { Operator } from '@/constants/agent';import {initialCompilationValues,+ initialGeneralChunkerValues,initialGoExtractorValues,initialParserValues,initialTitleChunkerValues,@@ -42,6 +43,8 @@ function getOperatorFormSchema(operatorType: Operator, t: TFunction) {return ParserFormSchema;case Operator.TokenChunker:return TokenChunkerFormSchema;+ case Operator.GeneralChunker:+ return TokenChunkerFormSchema;case Operator.TitleChunker:return TitleChunkerFormSchema;case Operator.Extractor:@@ -61,6 +64,8 @@ function getOperatorInitialValues(operatorType: Operator) {return initialParserValues;case Operator.TokenChunker:return initialTokenChunkerValues;+ case Operator.GeneralChunker:+ return initialGeneralChunkerValues;case Operator.TitleChunker:return initialTitleChunkerValues;case Operator.Extractor:diff --git a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsxindex a9b32b5766..a1aa1b7252 100644--- a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx+++ b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx@@ -72,6 +72,16 @@ const PipelineOperatorForm = ({externalErrors={externalErrors}/>);+ case Operator.GeneralChunker:+ return (+ <TokenChunkerForm+ node={node}+ onValuesChange={handleValuesChange}+ hideOutputs+ externalErrors={externalErrors}+ isGeneralChunker+ />+ );case Operator.TitleChunker:return (<TitleChunkerFormdiff --git a/web/src/constants/agent.tsx b/web/src/constants/agent.tsxindex fd81020f4b..c3fb973a83 100644--- a/web/src/constants/agent.tsx+++ b/web/src/constants/agent.tsx@@ -130,6 +130,7 @@ export enum Operator {File = 'File', // pipelineParser = 'Parser',Tokenizer = 'Tokenizer',+ GeneralChunker = 'GeneralChunker',TokenChunker = 'TokenChunker',TitleChunker = 'TitleChunker',Extractor = 'Extractor',diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex d10501d1e8..daba5365fd 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -670,6 +670,12 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default limimageTableContextWindow: 'Image & table context window',imageTableContextWindowTip:'Captures N tokens of text above and below the image & table to provide richer background context.',+ tableContextWindow: 'Table context window',+ tableContextWindowTip:+ 'Captures N tokens of text above and below a table to provide richer background context.',+ imageContextWindow: 'Image context window',+ imageContextWindowTip:+ 'Captures N tokens of text above and below an image to provide richer background context.',autoMetadata: 'Auto metadata',mineruOptions: 'MinerU options',mineruParseMethod: 'Parse method',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex e28f62edc9..ee80787056 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -609,6 +609,12 @@ export default {imageTableContextWindow: '图像与表格上下文窗口',imageTableContextWindowTip:'抓取图像与表格上下方的 N 个 token,为该 Chunk 提供更丰富的背景上下文。',+ tableContextWindow: '表格上下文窗口',+ tableContextWindowTip:+ '抓取表格上下方的 N 个 token,为该 Chunk 提供更丰富的背景上下文。',+ imageContextWindow: '图像上下文窗口',+ imageContextWindowTip:+ '抓取图像上下方的 N 个 token,为该 Chunk 提供更丰富的背景上下文。',autoMetadata: '自动元数据',mineruOptions: 'MinerU 选项',mineruParseMethod: '解析方法',diff --git a/web/src/pages/agent/constant/index.tsx b/web/src/pages/agent/constant/index.tsxindex 180e47f7ce..57a08589d1 100644--- a/web/src/pages/agent/constant/index.tsx+++ b/web/src/pages/agent/constant/index.tsx@@ -825,6 +825,7 @@ export const RestrictedUpstreamMap = {[Operator.VariableAssigner]: [Operator.Begin],[Operator.VariableAggregator]: [Operator.Begin],[Operator.Parser]: [Operator.Begin], // pipeline+ [Operator.GeneralChunker]: [Operator.Begin],[Operator.TokenChunker]: [Operator.Begin],[Operator.TitleChunker]: [Operator.Begin],[Operator.Tokenizer]: [Operator.Begin],@@ -881,6 +882,7 @@ export const NodeMap = {[Operator.File]: 'fileNode',[Operator.Parser]: 'parserNode',[Operator.Tokenizer]: 'tokenizerNode',+ [Operator.GeneralChunker]: 'chunkerNode',[Operator.TokenChunker]: 'chunkerNode',[Operator.TitleChunker]: 'chunkerNode',[Operator.Extractor]: 'contextNode',@@ -928,6 +930,7 @@ export const NoDebugOperatorsList = [Operator.File,Operator.Parser,Operator.Tokenizer,+ Operator.GeneralChunker,Operator.TokenChunker,Operator.TitleChunker,Operator.Extractor,@@ -940,6 +943,7 @@ export const NoCopyOperatorsList = [Operator.File,Operator.Parser,Operator.Tokenizer,+ Operator.GeneralChunker,Operator.TokenChunker,Operator.TitleChunker,Operator.Extractor,diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex e0dd0dd2f6..d6ee82567d 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -285,6 +285,28 @@ export const initialTokenChunkerValues = {children_delimiters: [],};+export const initialGeneralChunkerValues = {+ outputs: {+ chunks: { type: 'Array<Object>', value: [] },+ },+ chunk_token_size: 512,+ overlapped_percent: 0,+ delimiters: [+ { value: '\n' },+ { value: '!' },+ { value: '?' },+ { value: ';' },+ { value: '。' },+ { value: ';' },+ { value: '!' },+ { value: '?' },+ ],+ table_context_size: 0,+ image_context_size: 0,+ enable_children: false,+ children_delimiters: [],+};+export enum Hierarchy {H1 = '1',H2 = '2',@@ -473,6 +495,7 @@ export const FileTypeSuffixMap = {export const SingleOperators = [Operator.Tokenizer,+ Operator.GeneralChunker,Operator.TokenChunker,Operator.TitleChunker,Operator.Parser,diff --git a/web/src/pages/agent/form-sheet/form-config-map.tsx b/web/src/pages/agent/form-sheet/form-config-map.tsxindex eba7cd7496..d4cb0611cb 100644--- a/web/src/pages/agent/form-sheet/form-config-map.tsx+++ b/web/src/pages/agent/form-sheet/form-config-map.tsx@@ -48,6 +48,7 @@ import VariableAssignerForm from '../form/variable-assigner-form';import WenCaiForm from '../form/wencai-form';import WikipediaForm from '../form/wikipedia-form';import YahooFinanceForm from '../form/yahoo-finance-form';+import type { INextOperatorForm } from '../interface';export const FormConfigMap = {[Operator.Begin]: {@@ -183,6 +184,11 @@ export const FormConfigMap = {[Operator.Tokenizer]: {component: TokenizerForm,},+ [Operator.GeneralChunker]: {+ component: (props: INextOperatorForm) => (+ <TokenChunkerForm {...props} isGeneralChunker />+ ),+ },[Operator.TokenChunker]: {component: TokenChunkerForm,},diff --git a/web/src/pages/agent/form/token-chunker-form/index.tsx b/web/src/pages/agent/form/token-chunker-form/index.tsxindex b749f9efec..e8dd32ba03 100644--- a/web/src/pages/agent/form/token-chunker-form/index.tsx+++ b/web/src/pages/agent/form/token-chunker-form/index.tsx@@ -13,7 +13,10 @@ import { memo } from 'react';import { useFieldArray, useForm } from 'react-hook-form';import { useTranslation } from 'react-i18next';import { z } from 'zod';-import { initialTokenChunkerValues } from '../../constant/pipeline';+import {+ initialGeneralChunkerValues,+ initialTokenChunkerValues,+} from '../../constant/pipeline';import { useFormChangeCallback } from '../../hooks/use-form-change-callback';import { useFormValues } from '../../hooks/use-form-values';import { useWatchFormChange } from '../../hooks/use-watch-form-change';@@ -26,7 +29,9 @@ const outputList = buildOutputList(initialTokenChunkerValues.outputs);export const FormSchema = z.object({chunk_token_size: z.number(),- image_table_context_window: z.number(),+ image_table_context_window: z.number().optional(),+ table_context_size: z.number().optional(),+ image_context_size: z.number().optional(),delimiters: z.array(z.object({value: z.string().optional(),@@ -44,13 +49,21 @@ export const FormSchema = z.object({export type TokenChunkerFormSchemaType = z.infer<typeof FormSchema>;+type TokenChunkerFormProps = INextOperatorForm & {+ isGeneralChunker?: boolean;+};+const TokenChunkerForm = ({node,onValuesChange,hideOutputs,externalErrors,-}: INextOperatorForm) => {- const defaultValues = useFormValues(initialTokenChunkerValues, node);+ isGeneralChunker = false,+}: TokenChunkerFormProps) => {+ const initialValues = isGeneralChunker+ ? initialGeneralChunkerValues+ : initialTokenChunkerValues;+ const defaultValues = useFormValues(initialValues, node);const { t } = useTranslation();// Normalize legacy values: 'token_size' (removed tab) and empty fall back@@ -92,17 +105,19 @@ const TokenChunkerForm = ({return (<Form {...form}><FormWrapper>- <RenderField- field={{- name: 'delimiter_mode',- type: FormFieldType.Segmented,- label: '',- options: [- { label: t('flow.delimiters'), value: 'delimiter' },- { label: t('flow.one'), value: 'one' },- ],- }}- />+ {!isGeneralChunker && (+ <RenderField+ field={{+ name: 'delimiter_mode',+ type: FormFieldType.Segmented,+ label: '',+ options: [+ { label: t('flow.delimiters'), value: 'delimiter' },+ { label: t('flow.one'), value: 'one' },+ ],+ }}+ />+ )}{delimiterMode === 'delimiter' && (<>@@ -120,14 +135,35 @@ const TokenChunkerForm = ({integerlabel={t('flow.overlappedPercent')}/>- <SliderInputFormField- name="image_table_context_window"- max={256}- min={0}- integer- label={t('knowledgeConfiguration.imageTableContextWindow')}- tooltip={t('knowledgeConfiguration.imageTableContextWindowTip')}- />+ {isGeneralChunker ? (+ <>+ <SliderInputFormField+ name="table_context_size"+ max={256}+ min={0}+ integer+ label={t('knowledgeConfiguration.tableContextWindow')}+ tooltip={t('knowledgeConfiguration.tableContextWindowTip')}+ />+ <SliderInputFormField+ name="image_context_size"+ max={256}+ min={0}+ integer+ label={t('knowledgeConfiguration.imageContextWindow')}+ tooltip={t('knowledgeConfiguration.imageContextWindowTip')}+ />+ </>+ ) : (+ <SliderInputFormField+ name="image_table_context_window"+ max={256}+ min={0}+ integer+ label={t('knowledgeConfiguration.imageTableContextWindow')}+ tooltip={t('knowledgeConfiguration.imageTableContextWindowTip')}+ />+ )}<section><span className="mb-2 inline-block">{t('flow.delimiters')}</span><div className="space-y-4">diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.tsindex e79a2043e4..77e682c8ca 100644--- a/web/src/pages/agent/hooks/use-add-node.ts+++ b/web/src/pages/agent/hooks/use-add-node.ts@@ -51,6 +51,7 @@ import {initialTavilyValues,initialTitleChunkerValues,initialTokenChunkerValues,+ initialGeneralChunkerValues,initialTokenizerValues,initialUserFillUpValues,initialVariableAggregatorValues,@@ -185,6 +186,7 @@ export const useInitializeOperatorParams = () => {[Operator.Parser]: buildInitialParserValues(defaultModelDictionary),[Operator.Tokenizer]: initialTokenizerValues,[Operator.TokenChunker]: initialTokenChunkerValues,+ [Operator.GeneralChunker]: initialGeneralChunkerValues,[Operator.TitleChunker]: initialTitleChunkerValues,[Operator.Extractor]: {...getInitialExtractorValues(),diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex 52ac5d6f0f..0005405f0e 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -367,6 +367,16 @@ export function transformTokenChunkerParams(};}+export function transformGeneralChunkerParams(+ params: TokenChunkerFormSchemaType,+) {+ const result = transformTokenChunkerParams(params);+ result.table_context_size = Number(params.table_context_size || 0);+ result.image_context_size = Number(params.image_context_size || 0);+ delete result.delimiter_mode;+ return result;+}+export function transformTitleChunkerParams(params: TitleChunkerFormSchemaType,) {@@ -640,6 +650,10 @@ export const buildDslComponentsByGraph = (params = transformTokenChunkerParams(params);break;+ case Operator.GeneralChunker:+ params = transformGeneralChunkerParams(params);+ break;+case Operator.TitleChunker:params = transformTitleChunkerParams(params);break;diff --git a/web/src/utils/__tests__/pipeline-operator.test.ts b/web/src/utils/__tests__/pipeline-operator.test.tsindex 8718bc4aac..a84f51b1d3 100644--- a/web/src/utils/__tests__/pipeline-operator.test.ts+++ b/web/src/utils/__tests__/pipeline-operator.test.ts@@ -1,4 +1,9 @@-import { buildOperatorNode } from '@/utils/pipeline-operator';+import {+ buildOperatorNode,+ getOperatorType,+ transformApiConfigToForm,+ transformFormConfigToApi,+} from '@/utils/pipeline-operator';let mockIsGoBackend = true;jest.mock('@/utils/backend-runtime', () => ({@@ -10,6 +15,42 @@ const extractorNode = {data: { form: {} },} as any;+describe('GeneralChunker operator bridge', () => {+ it('recognizes and transforms the built-in general chunker config', () => {+ expect(getOperatorType('GeneralChunker:SixApplesFall')).toBe(+ 'GeneralChunker',+ );++ const form = transformApiConfigToForm('GeneralChunker', {+ chunk_token_size: 512,+ delimiters: ['\n', ';'],+ overlapped_percent: 0.1,+ table_context_size: 2,+ image_context_size: 3,+ });+ expect(form.delimiters).toEqual([{ value: '\n' }, { value: ';' }]);+ expect(form.table_context_size).toBe(2);+ expect(form.image_context_size).toBe(3);+ expect(form).not.toHaveProperty('image_table_context_window');+ expect(form).not.toHaveProperty('delimiter_mode');++ const api = transformFormConfigToApi('GeneralChunker', {+ delimiter_mode: 'delimiter',+ chunk_token_size: 512,+ delimiters: [{ value: '\n' }, { value: ';' }],+ children_delimiters: [],+ enable_children: false,+ overlapped_percent: 10,+ table_context_size: 2,+ image_context_size: 3,+ });+ expect(api.delimiters).toEqual(['\n', ';']);+ expect(api.table_context_size).toBe(2);+ expect(api.image_context_size).toBe(3);+ expect(api).not.toHaveProperty('delimiter_mode');+ });+});+describe('buildOperatorNode dataset-level metadata precedence', () => {beforeEach(() => {mockIsGoBackend = true;diff --git a/web/src/utils/pipeline-operator.ts b/web/src/utils/pipeline-operator.tsindex 6b22136a77..25fb249fe2 100644--- a/web/src/utils/pipeline-operator.ts+++ b/web/src/utils/pipeline-operator.ts@@ -19,6 +19,7 @@ import { DSL, RAGFlowNodeType } from '@/interfaces/database/agent';import {getInitialExtractorValues,initialGoExtractorValues,+ initialGeneralChunkerValues,initialParserValues,initialTitleChunkerValues,initialTokenChunkerValues,@@ -26,6 +27,7 @@ import {} from '@/pages/agent/constant/pipeline';import {transformExtractorParams,+ transformGeneralChunkerParams,transformParserParams,transformTitleChunkerParams,transformTokenChunkerParams,@@ -252,6 +254,17 @@ function transformTokenChunkerConfigToForm(return result;}+function transformGeneralChunkerConfigToForm(+ config: Record<string, any> | undefined,+): Record<string, any> {+ const result = transformTokenChunkerConfigToForm(config);+ result.table_context_size = Number(config?.table_context_size ?? 0);+ result.image_context_size = Number(config?.image_context_size ?? 0);+ delete result.image_table_context_window;+ delete result.delimiter_mode;+ return result;+}+/*** Converts TitleChunker config from API/DSL format to form format.* DSL: { method: "hierarchy", hierarchy: "3", levels: [...], include_heading_content, root_chunk_as_heading }@@ -321,6 +334,8 @@ export function transformApiConfigToForm(return transformTokenizerConfigToForm(config);case Operator.TokenChunker:return transformTokenChunkerConfigToForm(config);+ case Operator.GeneralChunker:+ return transformGeneralChunkerConfigToForm(config);case Operator.TitleChunker:return transformTitleChunkerConfigToForm(config);default:@@ -384,6 +399,8 @@ export function transformFormConfigToApi(return config; // passthrough for Tokenizercase Operator.TokenChunker:return transformTokenChunkerParams(config as any);+ case Operator.GeneralChunker:+ return transformGeneralChunkerParams(config as any);case Operator.TitleChunker:return transformTitleChunkerParams(config as any);default:@@ -417,6 +434,11 @@ export function normalizeOperatorForm(...cloneDeep(initialTokenChunkerValues),...rawForm,};+ case Operator.GeneralChunker:+ return {+ ...cloneDeep(initialGeneralChunkerValues),+ ...rawForm,+ };case Operator.Extractor:return {...cloneDeep(getInitialExtractorValues()),