Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix(dataset):auto parse unsupported upload (#21018)

ciel26ec6912865f
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.go
index 9d5d55a517..529235ac9c 100644
--- a/internal/ingestion/pipeline/pipeline_params.go
+++ b/internal/ingestion/pipeline/pipeline_params.go
@@ -139,11 +139,17 @@ func isCompilerComponent(cpnID string) bool {
strings.HasPrefix(lowerID, "compiler_")
}
+// IsParserComponentID reports whether a component ID names a Parser component.
+// A parser_config map is keyed by component ID, where the component name is not
+// available, so this is the config-side half of isParserComponent.
+func IsParserComponentID(cpnID string) bool {
+ return strings.HasPrefix(strings.ToLower(cpnID), "parser:")
+}
+
// isParserComponent returns true if the component name or component ID
// indicates a Parser component.
func isParserComponent(cpnID, componentName string) bool {
- return strings.EqualFold(componentName, "parser") ||
- strings.HasPrefix(strings.ToLower(cpnID), "parser:")
+ return strings.EqualFold(componentName, "parser") || IsParserComponentID(cpnID)
}
// NormalizeParserConfigSetups lifts legacy nested Parser "setups" entries in
diff --git a/internal/service/document/document_parser_selection.go b/internal/service/document/document_parser_selection.go
index fd10f48750..00ffe02fee 100644
--- a/internal/service/document/document_parser_selection.go
+++ b/internal/service/document/document_parser_selection.go
@@ -7,6 +7,8 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ componentpkg "ragflow/internal/ingestion/component"
+ "ragflow/internal/ingestion/component/schema"
pipelinepkg "ragflow/internal/ingestion/pipeline"
"ragflow/internal/service"
"ragflow/internal/utility"
@@ -14,12 +16,18 @@ import (
"go.uber.org/zap"
)
-// defaultDocumentParserID returns the parser that owns a document of the given
-// file type, mirroring Python's FileService.get_parser
+// generalParserID is the built-in catch-all ingestion pipeline. A file whose
+// family the dataset's selection does not declare and which no dedicated
+// pipeline owns falls back to it.
+const generalParserID = "general"
+
+// dedicatedDocumentParserID returns the pipeline that owns a document of the
+// given file type, or "" when no pipeline owns it and the dataset's selection
+// governs the file. Mirrors Python's FileService.get_parser
// (api/db/services/file_service.py:734-743): images and video are parsed by the
// picture pipeline, audio by audio, presentations by presentation and mail by
-// email. Every other file type keeps the dataset's parser.
-func defaultDocumentParserID(filetype utility.FileType, filename, fallback string) string {
+// email.
+func dedicatedDocumentParserID(filetype utility.FileType, filename string) string {
switch filetype {
case utility.FileTypeVISUAL:
return "picture"
@@ -40,37 +48,144 @@ func defaultDocumentParserID(filetype utility.FileType, filename, fallback strin
strings.HasSuffix(name, ".eml"):
return "email"
}
- return fallback
+ return ""
+}
+
+// parserParamsDeclaresFamily reports whether a Parser component's params
+// declare the given file family ("pdf", "text&code", "slides", ...). The
+// declaration is the one the web parser-gap validation reads
+// (transformParserConfigSetups), so both sides judge files by one contract.
+func parserParamsDeclaresFamily(params map[string]any, family string) bool {
+ _, declared := schema.FlattenLegacyParserSetups(params)[family]
+ return declared
+}
+
+// configDeclaredFamily reports whether a parser_config declares the family.
+// found is false when the config has no readable Parser entry (a legacy flat
+// config, or a malformed one): the config cannot answer then.
+func configDeclaredFamily(config entity.JSONMap, family string) (declared, found bool) {
+ for cpnID, raw := range config {
+ if !pipelinepkg.IsParserComponentID(cpnID) {
+ continue
+ }
+ params, ok := raw.(map[string]any)
+ if !ok {
+ return false, false
+ }
+ declared = declared || parserParamsDeclaresFamily(params, family)
+ found = true
+ }
+ return declared, found
+}
+
+// templateDeclaredFamily reports whether a builtin template declares the family.
+// found is false when the template cannot be loaded or carries no Parser
+// component. The registry resolves legacy aliases ("naive" -> "general"), so a
+// dataset row storing a historical parser_id is judged by the template it
+// actually runs.
+func templateDeclaredFamily(ctx context.Context, parserID, family string) (declared, found bool) {
+ dslJSON, err := service.LoadPipelineDSL(ctx, false, parserID, nil)
+ if err != nil {
+ return false, false
+ }
+ defaults, err := pipelinepkg.ComponentParamsDefaults(dslJSON)
+ if err != nil {
+ return false, false
+ }
+ for cpnID, params := range defaults {
+ if !pipelinepkg.IsParserComponentID(cpnID) {
+ continue
+ }
+ return parserParamsDeclaresFamily(params, family), true
+ }
+ return false, false
}
-// resolveDocumentParser returns the parser_id and parser_config a newly created
-// document should carry.
+// selectionGovernsFamily reports whether the dataset's selection declares a
+// file family.
//
-// A file type owned by a dedicated pipeline overrides the dataset's parser and
-// the document is seeded with that pipeline's component params, so the row
-// matches the parser it actually runs with: document parser_config drives both
-// the parser-gap validation and the runtime override params. Without the config
-// swap the document would keep another template's component ids and be reported
-// as unsupported by its own file type.
+// The saved config carries the declaration the web gap validation reads, so it
+// decides whenever it has a Parser entry. A legacy flat config has none, and a
+// builtin selection is then judged by the template its parser_id actually runs;
+// a canvas has no template to consult. Callers only ask about a family the
+// pipeline vocabulary can name; an unreadable declaration answers true, keeping
+// the choice that cannot be judged.
+func selectionGovernsFamily(ctx context.Context, sel *service.ParserSelection, config entity.JSONMap, family string) bool {
+ if declared, found := configDeclaredFamily(config, family); found {
+ return declared
+ }
+ if sel != nil && sel.IsBuiltIn() {
+ if declared, found := templateDeclaredFamily(ctx, sel.ParserID, family); found {
+ return declared
+ }
+ }
+ return true
+}
+
+// parserSelectionColumns maps a resolved parser selection onto the document
+// row's (parser_id, pipeline_id) columns. A canvas selection leaves parser_id
+// empty: parser_id is not applicable in canvas pipeline mode.
+func parserSelectionColumns(sel *service.ParserSelection) (string, *string) {
+ if sel != nil && sel.IsPipeline() {
+ pipelineID := sel.PipelineID
+ return "", &pipelineID
+ }
+ parserID := ""
+ if sel != nil {
+ parserID = sel.ParserID
+ }
+ return parserID, nil
+}
+
+// resolveDocumentParser returns the parser selection and parser_config a newly
+// created document should carry.
//
-// Canvas pipeline mode keeps the dataset parser untouched (parser_id is blanked
-// by the caller). On a DSL failure the dataset parser and config are kept: a
-// registry lookup must not fail an upload, and a half-switched row (new parser,
-// old config) is exactly the inconsistent state this function exists to avoid.
-func resolveDocumentParser(ctx context.Context, kb *entity.Knowledgebase, filename string, filetype utility.FileType, base entity.JSONMap) (string, entity.JSONMap) {
- if kb.PipelineID != nil {
- return kb.ParserID, base
- }
- parserID := defaultDocumentParserID(filetype, filename, kb.ParserID)
- if parserID == kb.ParserID || parserID == "" {
- return kb.ParserID, base
+// The dataset's selection governs every file family its config declares; a file
+// the selection does not declare is handed to the pipeline that owns its family
+// (picture / audio / presentation / email), and failing that to the catch-all
+// general pipeline, with the config reseeded from that template. A family the
+// pipeline vocabulary cannot name cannot be judged against a declaration: the
+// owning pipeline still takes the file, and with no owner (rtf, wps, ...) the
+// dataset's choice is honored. The row then matches the pipeline that actually
+// runs it: document parser_config drives both the parser-gap validation and the
+// runtime override params, and a row left on a parser that cannot run its file
+// is reported as unsupported by its own type. An unchanged parser keeps its
+// config, and on a DSL failure the selection and config are kept: a registry
+// lookup must not fail an upload, and a half-switched row (new parser, old
+// config) is the inconsistent state this function exists to avoid.
+func resolveDocumentParser(ctx context.Context, kb *entity.Knowledgebase, filename string, filetype utility.FileType, base entity.JSONMap) (*service.ParserSelection, entity.JSONMap) {
+ selection := service.CurrentSelection(kb.ParserID, kb.PipelineID)
+ family := componentpkg.ParserFileFamily(utility.GetFileExtension(filename))
+ if family != "" && selectionGovernsFamily(ctx, selection, kb.ParserConfig, family) {
+ return selection, base
}
- dslJSON, err := service.LoadPipelineDSL(ctx, false, parserID, nil)
+ target := dedicatedDocumentParserID(filetype, filename)
+ if target == "" {
+ if family == "" {
+ return selection, base
+ }
+ target = generalParserID
+ }
+ if selection.IsBuiltIn() && selection.ParserID == target {
+ return selection, base
+ }
+ if rebuilt, ok := rebuildParserConfig(ctx, kb, target, filename, base); ok {
+ return &service.ParserSelection{Mode: service.ParseModeBuiltIn, ParserID: target}, rebuilt
+ }
+ return selection, base
+}
+
+// rebuildParserConfig reseeds a document's parser_config from the target
+// built-in pipeline's DSL. It reports false when the DSL cannot be loaded, so
+// the caller keeps the dataset's selection instead of writing a half-switched
+// row.
+func rebuildParserConfig(ctx context.Context, kb *entity.Knowledgebase, target, filename string, base entity.JSONMap) (entity.JSONMap, bool) {
+ dslJSON, err := service.LoadPipelineDSL(ctx, false, target, nil)
if err != nil {
- common.Warn("document: file-type parser DSL unavailable, keeping dataset parser",
- zap.String("parser_id", parserID), zap.String("filename", filename), zap.Error(err))
- return kb.ParserID, base
+ common.Warn("document: target pipeline DSL unavailable, keeping dataset selection",
+ zap.String("parser_id", target), zap.String("filename", filename), zap.Error(err))
+ return nil, false
}
raw := map[string]interface{}(base)
cleaned := pipelinepkg.BuildParserConfig(dslJSON, raw)
@@ -84,5 +199,5 @@ func resolveDocumentParser(ctx context.Context, kb *entity.Knowledgebase, filena
if tenant, tenantErr := dao.NewTenantDAO().GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {
cleaned = service.ApplyComponentScopedParserConfig(cleaned, tenant.LLMID)
}
- return parserID, cleaned
+ return cleaned, true
}
diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.go
index 9ac7f58e99..baba45f2f3 100644
--- a/internal/service/document/document_parser_selection_test.go
+++ b/internal/service/document/document_parser_selection_test.go
@@ -2,44 +2,399 @@ package document
import (
"mime/multipart"
+ "reflect"
"strings"
"testing"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "ragflow/internal/service"
"ragflow/internal/storage"
+ syncerconnector "ragflow/internal/syncer/connector"
"ragflow/internal/utility"
)
-func TestDefaultDocumentParserID(t *testing.T) {
+func TestDedicatedDocumentParserID(t *testing.T) {
cases := []struct {
name string
filetype utility.FileType
filename string
- fallback string
want string
}{
- {"image routes to picture", utility.FileTypeVISUAL, "photo.jpg", "naive", "picture"},
- {"video routes to picture", utility.FileTypeVISUAL, "clip.mp4", "naive", "picture"},
- {"audio routes to audio", utility.FileTypeAURAL, "talk.mp3", "naive", "audio"},
- {"pptx routes to presentation", utility.FileTypeDOC, "deck.pptx", "naive", "presentation"},
- {"legacy ppt routes to presentation", utility.FileTypeDOC, "deck.ppt", "naive", "presentation"},
- {"pages routes to presentation", utility.FileTypeDOC, "deck.pages", "naive", "presentation"},
- {"presentation extension is case-insensitive", utility.FileTypeDOC, "DECK.PPTX", "naive", "presentation"},
- {"eml routes to email", utility.FileTypeDOC, "mail.eml", "naive", "email"},
- {"msg routes to email", utility.FileTypeDOC, "mail.msg", "naive", "email"},
- {"pdf keeps the dataset parser", utility.FileTypePDF, "report.pdf", "naive", "naive"},
- {"text keeps the dataset parser", utility.FileTypeDOC, "notes.txt", "manual", "manual"},
- {"no substring false positive", utility.FileTypeDOC, "deck.pptx.md", "naive", "naive"},
- {"dedicated dataset parser is kept", utility.FileTypeVISUAL, "photo.jpg", "picture", "picture"},
+ {"image routes to picture", utility.FileTypeVISUAL, "photo.jpg", "picture"},
+ {"video routes to picture", utility.FileTypeVISUAL, "clip.mp4", "picture"},
+ {"audio routes to audio", utility.FileTypeAURAL, "talk.mp3", "audio"},
+ {"pptx routes to presentation", utility.FileTypeDOC, "deck.pptx", "presentation"},
+ {"legacy ppt routes to presentation", utility.FileTypeDOC, "deck.ppt", "presentation"},
+ {"pages routes to presentation", utility.FileTypeDOC, "deck.pages", "presentation"},
+ {"presentation extension is case-insensitive", utility.FileTypeDOC, "DECK.PPTX", "presentation"},
+ {"eml routes to email", utility.FileTypeDOC, "mail.eml", "email"},
+ {"msg routes to email", utility.FileTypeDOC, "mail.msg", "email"},
+ {"pdf has no dedicated pipeline", utility.FileTypePDF, "report.pdf", ""},
+ {"text has no dedicated pipeline", utility.FileTypeDOC, "notes.txt", ""},
+ {"no substring false positive", utility.FileTypeDOC, "deck.pptx.md", ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- got := defaultDocumentParserID(tc.filetype, tc.filename, tc.fallback)
- if got != tc.want {
- t.Fatalf("defaultDocumentParserID(%q, %q, %q) = %q, want %q",
- tc.filetype, tc.filename, tc.fallback, got, tc.want)
+ if got := dedicatedDocumentParserID(tc.filetype, tc.filename); got != tc.want {
+ t.Fatalf("dedicatedDocumentParserID(%q, %q) = %q, want %q",
+ tc.filetype, tc.filename, got, tc.want)
+ }
+ })
+ }
+}
+
+func TestConfigDeclaredFamily(t *testing.T) {
+ flat := entity.JSONMap{
+ "Parser:HipSignsRhyme": map[string]any{
+ "pdf": map[string]any{"suffix": []any{"pdf"}},
+ "text&code": map[string]any{},
+ "vlm": map[string]any{},
+ "enable_vision_enhancement": false,
+ },
+ }
+ nested := entity.JSONMap{
+ "Parser:HipSignsRhyme": map[string]any{
+ "setups": map[string]any{
+ "pdf": map[string]any{},
+ "slides": map[string]any{},
+ },
+ },
+ }
+ // A config never carries two Parser components today; the union keeps the
+ // answer independent of map iteration order if one ever does.
+ multiple := entity.JSONMap{
+ "Parser:One": map[string]any{"pdf": map[string]any{}},
+ "Parser:Two": map[string]any{"slides": map[string]any{}},
+ }
+
+ cases := []struct {
+ name string
+ config entity.JSONMap
+ family string
+ wantDeclared bool
+ wantFound bool
+ }{
+ {"flat declares the family", flat, "pdf", true, true},
+ {"flat misses the family", flat, "slides", false, true},
+ {"nested legacy setups are lifted", nested, "slides", true, true},
+ {"nested legacy setups miss the family", nested, "text&code", false, true},
+ {"any parser entry declaring the family wins", multiple, "slides", true, true},
+ {"an entry missing the family still answers", multiple, "audio", false, true},
+ {"no parser entry cannot answer", entity.JSONMap{"File": map[string]any{}}, "pdf", false, false},
+ {"nil config cannot answer", nil, "pdf", false, false},
+ {"non-map parser entry cannot answer", entity.JSONMap{"Parser:X": "oops"}, "pdf", false, false},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ declared, found := configDeclaredFamily(tc.config, tc.family)
+ if declared != tc.wantDeclared || found != tc.wantFound {
+ t.Fatalf("configDeclaredFamily(%#v, %q) = (%v, %v), want (%v, %v)",
+ tc.config, tc.family, declared, found, tc.wantDeclared, tc.wantFound)
+ }
+ })
+ }
+}
+
+func TestSelectionGovernsFamilyMissesAnUndeclaredFamily(t *testing.T) {
+ sel := service.CurrentSelection("presentation", nil)
+ if selectionGovernsFamily(t.Context(), sel, presentationParserConfig(), "text&code") {
+ t.Fatal("selectionGovernsFamily reported an undeclared family as declared")
+ }
+}
+
+func presentationParserConfig() entity.JSONMap {
+ return entity.JSONMap{
+ "Parser:HipSignsRhyme": map[string]any{
+ "pdf": map[string]any{"suffix": []any{"pdf"}},
+ "slides": map[string]any{"suffix": []any{"pptx", "ppt"}},
+ },
+ }
+}
+
+func generalParserConfig() entity.JSONMap {
+ return entity.JSONMap{
+ "Parser:HipSignsRhyme": map[string]any{
+ "pdf": map[string]any{"suffix": []any{"pdf"}},
+ "text&code": map[string]any{},
+ "docx": map[string]any{},
+ },
+ }
+}
+
+func pictureParserConfig() entity.JSONMap {
+ return entity.JSONMap{
+ "Parser:ViewsCaptureLight": map[string]any{
+ "image": map[string]any{"suffix": []any{"jpg"}},
+ "video": map[string]any{"suffix": []any{"mp4"}},
+ },
+ }
+}
+
+func tableParserConfig() entity.JSONMap {
+ return entity.JSONMap{
+ "Parser:TableRoundStone": map[string]any{
+ "spreadsheet": map[string]any{"suffix": []any{"xlsx"}},
+ "text&code": map[string]any{},
+ },
+ }
+}
+
+// TestResolveDocumentParser_Priority pins the resolution order: the dataset's
+// selection governs the families its config declares, an undeclared family goes
+// to the builtin pipeline that owns it, and failing that to general. Undetermined
+// support (a canvas without a Parser node) keeps the selection, and so does an
+// extension the pipeline vocabulary cannot name unless an owning pipeline takes
+// it.
+func TestResolveDocumentParser_Priority(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := dao.DB.Create(&entity.Tenant{ID: "tenant-1", LLMID: "llm-default"}).Error; err != nil {
+ t.Fatalf("insert tenant: %v", err)
+ }
+
+ canvasID := "canvas-1"
+ legacyConfig := entity.JSONMap{"llm_id": "ministral", "chunk_token_num": 512}
+ canvasWithoutParser := entity.JSONMap{"File": map[string]any{}}
+
+ cases := []struct {
+ name string
+ kbParser string
+ kbPipelineID *string
+ kbConfig entity.JSONMap
+ filename string
+ filetype utility.FileType
+ wantParserID string
+ wantPipeline string
+ wantFamilyIn string
+ wantFamilyOut string
+ wantBaseConfigKept bool
+ }{
+ {
+ name: "general keeps a text family it declares",
+ kbParser: "general",
+ kbConfig: generalParserConfig(),
+ filename: "laws.txt",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "general",
+ wantFamilyIn: "text&code",
+ wantFamilyOut: "slides",
+ },
+ {
+ name: "presentation keeps a pdf it declares",
+ kbParser: "presentation",
+ kbConfig: presentationParserConfig(),
+ filename: "report.pdf",
+ filetype: utility.FileTypePDF,
+ wantParserID: "presentation",
+ wantFamilyIn: "slides",
+ wantFamilyOut: "text&code",
+ },
+ {
+ name: "presentation falls back to general for text",
+ kbParser: "presentation",
+ kbConfig: presentationParserConfig(),
+ filename: "laws.txt",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "general",
+ wantFamilyIn: "text&code",
+ wantFamilyOut: "slides",
+ },
+ {
+ name: "table falls back to general for pdf",
+ kbParser: "table",
+ kbConfig: tableParserConfig(),
+ filename: "report.pdf",
+ filetype: utility.FileTypePDF,
+ wantParserID: "general",
+ wantFamilyIn: "docx",
+ wantFamilyOut: "",
+ },
+ {
+ name: "picture owns an image the selection does not declare",
+ kbParser: "presentation",
+ kbConfig: presentationParserConfig(),
+ filename: "photo.jpg",
+ filetype: utility.FileTypeVISUAL,
+ wantParserID: "picture",
+ wantFamilyIn: "image",
+ wantFamilyOut: "slides",
+ },
+ {
+ name: "audio owns audio the selection does not declare",
+ kbParser: "picture",
+ kbConfig: pictureParserConfig(),
+ filename: "talk.mp3",
+ filetype: utility.FileTypeAURAL,
+ wantParserID: "audio",
+ wantFamilyIn: "audio",
+ wantFamilyOut: "image",
+ },
+ {
+ name: "legacy flat config is judged by its template",
+ kbParser: "naive",
+ kbConfig: legacyConfig,
+ filename: "laws.txt",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "naive",
+ wantFamilyIn: "",
+ wantFamilyOut: "",
+ },
+ {
+ name: "legacy flat config still routes an owned family",
+ kbParser: "naive",
+ kbConfig: legacyConfig,
+ filename: "deck.pptx",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "presentation",
+ wantFamilyIn: "slides",
+ wantFamilyOut: "",
+ },
+ {
+ name: "canvas keeps a family it declares",
+ kbParser: "presentation",
+ kbPipelineID: &canvasID,
+ kbConfig: presentationParserConfig(),
+ filename: "report.pdf",
+ filetype: utility.FileTypePDF,
+ wantParserID: "",
+ wantPipeline: canvasID,
+ wantFamilyIn: "slides",
+ wantFamilyOut: "",
+ },
+ {
+ name: "canvas falls back to general for an undeclared family",
+ kbParser: "presentation",
+ kbPipelineID: &canvasID,
+ kbConfig: presentationParserConfig(),
+ filename: "laws.txt",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "general",
+ wantPipeline: "",
+ wantFamilyIn: "text&code",
+ wantFamilyOut: "slides",
+ },
+ {
+ name: "canvas without a parser node keeps the canvas",
+ kbParser: "presentation",
+ kbPipelineID: &canvasID,
+ kbConfig: canvasWithoutParser,
+ filename: "laws.txt",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "",
+ wantPipeline: canvasID,
+ },
+ {
+ name: "canvas without a parser node owns nothing either",
+ kbParser: "presentation",
+ kbPipelineID: &canvasID,
+ kbConfig: canvasWithoutParser,
+ filename: "photo.jpg",
+ filetype: utility.FileTypeVISUAL,
+ wantParserID: "",
+ wantPipeline: canvasID,
+ },
+ {
+ name: "canvas selects into general for an unowned family",
+ kbParser: "presentation",
+ kbPipelineID: &canvasID,
+ kbConfig: pictureParserConfig(),
+ filename: "report.pdf",
+ filetype: utility.FileTypePDF,
+ wantParserID: "general",
+ wantFamilyIn: "text&code",
+ },
+ {
+ name: "an unmappable extension still reaches its owner pipeline",
+ kbParser: "general",
+ kbConfig: generalParserConfig(),
+ filename: "deck.pages",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "presentation",
+ wantFamilyIn: "slides",
+ },
+ {
+ name: "an unmappable visual extension still reaches picture",
+ kbParser: "general",
+ kbConfig: generalParserConfig(),
+ filename: "clip.rm",
+ filetype: utility.FileTypeVISUAL,
+ wantParserID: "picture",
+ wantFamilyIn: "video",
+ },
+ {
+ name: "an unmappable audio extension still reaches audio",
+ kbParser: "general",
+ kbConfig: generalParserConfig(),
+ filename: "song.vorbis",
+ filetype: utility.FileTypeAURAL,
+ wantParserID: "audio",
+ wantFamilyIn: "audio",
+ },
+ {
+ name: "an unmappable extension with no owner keeps the selection",
+ kbParser: "general",
+ kbConfig: generalParserConfig(),
+ filename: "notes.rtf",
+ filetype: utility.FileTypeDOC,
+ wantParserID: "general",
+ wantBaseConfigKept: true,
+ },
+ {
+ name: "an owner already selected keeps its edited config",
+ kbParser: "picture",
+ kbConfig: pictureParserConfig(),
+ filename: "clip.psd",
+ filetype: utility.FileTypeVISUAL,
+ wantParserID: "picture",
+ wantBaseConfigKept: true,
+ },
+ {
+ name: "a canvas loses an unmappable extension to its owner",
+ kbParser: "presentation",
+ kbPipelineID: &canvasID,
+ kbConfig: presentationParserConfig(),
+ filename: "clip.rm",
+ filetype: utility.FileTypeVISUAL,
+ wantParserID: "picture",
+ wantFamilyIn: "video",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ kb := &entity.Knowledgebase{
+ ID: "kb-" + tc.name,
+ TenantID: "tenant-1",
+ Name: "kb",
+ CreatedBy: "user-1",
+ ParserID: tc.kbParser,
+ PipelineID: tc.kbPipelineID,
+ ParserConfig: tc.kbConfig,
+ }
+ base := cloneParserConfigForDocument(tc.kbConfig)
+
+ selection, gotConfig := resolveDocumentParser(t.Context(), kb, tc.filename, tc.filetype, base)
+
+ gotParser, gotPipelinePtr := parserSelectionColumns(selection)
+ gotPipeline := ""
+ if gotPipelinePtr != nil {
+ gotPipeline = *gotPipelinePtr
+ }
+ if gotParser != tc.wantParserID || gotPipeline != tc.wantPipeline {
+ t.Fatalf("selection = (%q, %q), want (%q, %q)",
+ gotParser, gotPipeline, tc.wantParserID, tc.wantPipeline)
+ }
+ if tc.wantBaseConfigKept && !reflect.DeepEqual(gotConfig, base) {
+ t.Fatalf("parser_config = %#v, want the base config kept", gotConfig)
+ }
+ if tc.wantFamilyIn != "" && !configHasParserFamily(gotConfig, tc.wantFamilyIn) {
+ t.Fatalf("parser_config misses family %q: %#v", tc.wantFamilyIn, gotConfig)
+ }
+ if tc.wantFamilyOut != "" && configHasParserFamily(gotConfig, tc.wantFamilyOut) {
+ t.Fatalf("parser_config unexpectedly keeps family %q: %#v", tc.wantFamilyOut, gotConfig)
}
})
}
@@ -141,6 +496,162 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {
}
}
+// TestUploadLocalDocuments_FallsBackWhenSelectionLacksFamily covers the upload
+// path end to end: a text file uploaded to a dataset whose selection declares
+// only pdf and slides is reseeded with the catch-all general pipeline, and a
+// canvas dataset keeps a family its canvas declares while falling back for the
+// rest.
+func TestUploadLocalDocuments_FallsBackWhenSelectionLacksFamily(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ mockStorage := newFakeUploadStorage()
+ factory := storage.GetStorageFactory()
+ origStorage := factory.GetStorage()
+ factory.SetStorage(mockStorage)
+ t.Cleanup(func() { factory.SetStorage(origStorage) })
+
+ if err := dao.DB.Create(&entity.Tenant{
+ ID: "tenant-1", LLMID: "llm-default", EmbdID: "embd-default", ASRID: "asr-default",
+ }).Error; err != nil {
+ t.Fatalf("insert tenant: %v", err)
+ }
+
+ ctx := t.Context()
+ svc := testDocumentService(t)
+
+ presentationKB := &entity.Knowledgebase{
+ ID: "kb-presentation",
+ TenantID: "tenant-1",
+ Name: "kb-presentation",
+ CreatedBy: "user-1",
+ ParserID: "presentation",
+ ParserConfig: presentationParserConfig(),
+ }
+ if err := dao.DB.Create(presentationKB).Error; err != nil {
+ t.Fatalf("insert presentation kb: %v", err)
+ }
+ got, errs := svc.UploadLocalDocuments(ctx, presentationKB, "user-1", []*multipart.FileHeader{
+ makeTestFileHeader(t, "file", "laws.txt", []byte("plain text")),
+ makeTestFileHeader(t, "file", "deck.pptx", []byte("pptx-bytes")),
+ }, "", nil)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected errs: %v", errs)
+ }
+ if parserID, _ := got[0]["parser_id"].(string); parserID != "general" {
+ t.Fatalf("txt parser_id = %q, want general", parserID)
+ }
+ parserFamilySetup(t, got[0], "text&code")
+ if parserID, _ := got[1]["parser_id"].(string); parserID != "presentation" {
+ t.Fatalf("pptx parser_id = %q, want presentation", parserID)
+ }
+ parserFamilySetup(t, got[1], "slides")
+
+ canvasID := "canvas-1"
+ canvasKB := &entity.Knowledgebase{
+ ID: "kb-canvas",
+ TenantID: "tenant-1",
+ Name: "kb-canvas",
+ CreatedBy: "user-1",
+ ParserID: "presentation",
+ PipelineID: &canvasID,
+ ParserConfig: presentationParserConfig(),
+ }
+ if err := dao.DB.Create(canvasKB).Error; err != nil {
+ t.Fatalf("insert canvas kb: %v", err)
+ }
+ got, errs = svc.UploadLocalDocuments(ctx, canvasKB, "user-1", []*multipart.FileHeader{
+ makeTestFileHeader(t, "file", "report.pdf", []byte("%PDF-1.4")),
+ makeTestFileHeader(t, "file", "laws.txt", []byte("plain text")),
+ }, "", nil)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected errs: %v", errs)
+ }
+ // The canvas declares pdf: the document stays on the canvas with parser_id
+ // blanked and the canvas carried on pipeline_id.
+ if parserID, _ := got[0]["parser_id"].(string); parserID != "" {
+ t.Fatalf("canvas pdf parser_id = %q, want blank", parserID)
+ }
+ if pipelineID, _ := got[0]["pipeline_id"].(string); pipelineID != canvasID {
+ t.Fatalf("canvas pdf pipeline_id = %q, want %q", pipelineID, canvasID)
+ }
+ parserFamilySetup(t, got[0], "pdf")
+ // The canvas does not declare text&code: the document leaves the canvas for
+ // general.
+ if parserID, _ := got[1]["parser_id"].(string); parserID != "general" {
+ t.Fatalf("canvas txt parser_id = %q, want general", parserID)
+ }
+ if pipelineID, ok := got[1]["pipeline_id"].(string); ok && pipelineID != "" {
+ t.Fatalf("canvas txt pipeline_id = %q, want none", pipelineID)
+ }
+ parserFamilySetup(t, got[1], "text&code")
+}
+
+// TestInsertSyncDocument_FallsBackWhenSelectionLacksFamily covers the connector
+// path: it shares resolveDocumentParser with the upload path, so a synced text
+// file lands on general too.
+func TestInsertSyncDocument_FallsBackWhenSelectionLacksFamily(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ mockStorage := newFakeUploadStorage()
+ factory := storage.GetStorageFactory()
+ origStorage := factory.GetStorage()
+ factory.SetStorage(mockStorage)
+ t.Cleanup(func() { factory.SetStorage(origStorage) })
+
+ if err := dao.DB.Create(&entity.Tenant{
+ ID: "tenant-1", LLMID: "llm-default", EmbdID: "embd-default", ASRID: "asr-default",
+ }).Error; err != nil {
+ t.Fatalf("insert tenant: %v", err)
+ }
+
+ kb := &entity.Knowledgebase{
+ ID: "kb-sync",
+ TenantID: "tenant-1",
+ Name: "kb-sync",
+ CreatedBy: "user-1",
+ ParserID: "presentation",
+ ParserConfig: presentationParserConfig(),
+ }
+ if err := dao.DB.Create(kb).Error; err != nil {
+ t.Fatalf("insert kb: %v", err)
+ }
+
+ input := service.DocumentUpsertInput{
+ TaskContext: dao.SyncTaskContext{Knowledgebase: *kb},
+ SourceType: "connector",
+ DocumentID: "doc-sync-1",
+ SourceDocument: syncerconnector.SourceDocument{Blob: []byte("plain text")},
+ }
+ svc := testDocumentService(t)
+ res, err := svc.insertSyncDocument(t.Context(), input, "laws.txt", "", "txt", "", "tenant-1")
+ if err != nil {
+ t.Fatalf("insertSyncDocument: %v", err)
+ }
+
+ doc, err := svc.documentDAO.GetByID(t.Context(), dao.DB, res.DocID)
+ if err != nil {
+ t.Fatalf("load synced document: %v", err)
+ }
+ if doc.ParserID != "general" {
+ t.Fatalf("synced txt parser_id = %q, want general", doc.ParserID)
+ }
+ if doc.PipelineID != nil {
+ t.Fatalf("synced txt pipeline_id = %v, want nil", *doc.PipelineID)
+ }
+ if !configHasParserFamily(doc.ParserConfig, "text&code") {
+ t.Fatalf("synced txt parser_config misses text&code: %#v", doc.ParserConfig)
+ }
+}
+
+// configHasParserFamily reports whether the config's Parser entry declares the
+// given family, read through the production declaration reader.
+func configHasParserFamily(config entity.JSONMap, family string) bool {
+ declared, _ := configDeclaredFamily(config, family)
+ return declared
+}
+
// parserFamilySetup returns the family setup of the document's Parser node, the
// shape the web parser-gap validation keys on.
func parserFamilySetup(t *testing.T, doc map[string]interface{}, family string) map[string]interface{} {
diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.go
index 3ff7fdd654..fd89e32ba1 100644
--- a/internal/service/document/document_upload.go
+++ b/internal/service/document/document_upload.go
@@ -10,6 +10,7 @@ import (
"ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "ragflow/internal/service"
"ragflow/internal/storage"
"ragflow/internal/utility"
"strings"
@@ -88,8 +89,8 @@ func (s *DocumentService) UploadLocalDocuments(ctx context.Context, kb *entity.K
continue
}
- parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, merged)
- doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), parserID, parserConfig, "local", int64(len(blob)), blob)
+ selection, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, merged)
+ doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), selection, parserConfig, "local", int64(len(blob)), blob)
if err = s.InsertDocument(doc); err != nil {
// Roll back the orphaned blob so a failed insert doesn't leak storage.
rmErr := removeObjectBestEffort(ctx, storageImpl, kb.ID, location)
@@ -133,7 +134,7 @@ func (s *DocumentService) UploadEmptyDocument(ctx context.Context, kb *entity.Kn
// Virtual documents have no file type to route on, so they keep the
// dataset parser (mirrors Python's _upload_empty_document).
- doc := s.newDatasetDocument(kb, tenantID, name, "", "virtual", kb.ParserID, kb.ParserConfig, "local", 0, nil)
+ doc := s.newDatasetDocument(kb, tenantID, name, "", "virtual", service.CurrentSelection(kb.ParserID, kb.PipelineID), kb.ParserConfig, "local", 0, nil)
if err = s.InsertDocument(doc); err != nil {
return nil, common.CodeServerError, err
}
@@ -271,8 +272,8 @@ func (s *DocumentService) UploadWebDocument(ctx context.Context, kb *entity.Know
return nil, common.CodeServerError, err
}
- parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, kb.ParserConfig)
- doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), parserID, parserConfig, "web", int64(len(blob)), blob)
+ selection, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, kb.ParserConfig)
+ doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), selection, parserConfig, "web", int64(len(blob)), blob)
if err = s.InsertDocument(doc); err != nil {
rmErr := removeObjectBestEffort(ctx, storageImpl, kb.ID, location)
if rmErr != nil {
@@ -308,9 +309,9 @@ func normalizeWebDocumentName(name, contentType string, blob []byte) string {
}
// newDatasetDocument builds a Document row for an upload, deriving suffix and
-// content hash. parserID/parserConfig are resolved by the caller (see
+// content hash. selection/parserConfig are resolved by the caller (see
// resolveDocumentParser); blob may be nil for the empty/virtual document.
-func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, filename, location, filetype, parserID string, parserConfig entity.JSONMap, src string, size int64, blob []byte) *entity.Document {
+func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, filename, location, filetype string, selection *service.ParserSelection, parserConfig entity.JSONMap, src string, size int64, blob []byte) *entity.Document {
docID := common.GenerateToken()
parserConfig = cloneParserConfigForDocument(parserConfig)
status := "1"
@@ -318,15 +319,13 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID,
if i := strings.LastIndex(filename, "."); i >= 0 {
suffix = filename[i+1:]
}
- if kb.PipelineID != nil {
- parserID = "" // canvas pipeline mode — parser_id not applicable
- }
+ parserID, pipelineID := parserSelectionColumns(selection)
loc := location
doc := &entity.Document{
ID: docID,
KbID: kb.ID,
ParserID: parserID,
- PipelineID: kb.PipelineID,
+ PipelineID: pipelineID,
ParserConfig: parserConfig,
CreatedBy: tenantID,
Type: filetype,
diff --git a/internal/service/document/file2document.go b/internal/service/document/file2document.go
index 90a2425f80..8133c0f532 100644
--- a/internal/service/document/file2document.go
+++ b/internal/service/document/file2document.go
@@ -241,12 +241,14 @@ func (s *File2DocumentService) convertFiles(ctx context.Context, fileIDs, kbIDs
continue
}
- parserID, parserConfig := resolveDocumentParser(ctx, kb, docName, utility.FileType(file.Type), cloneParserConfigForDocument(kb.ParserConfig))
+ selection, parserConfig := resolveDocumentParser(ctx, kb, docName, utility.FileType(file.Type), cloneParserConfigForDocument(kb.ParserConfig))
+ parserID, pipelineID := parserSelectionColumns(selection)
suffix := strings.TrimPrefix(filepath.Ext(docName), ".")
doc := &entity.Document{
ID: common.GenerateUUID(),
KbID: kb.ID,
ParserID: parserID,
+ PipelineID: pipelineID,
ParserConfig: parserConfig,
CreatedBy: userID,
Type: file.Type,
@@ -257,10 +259,6 @@ func (s *File2DocumentService) convertFiles(ctx context.Context, fileIDs, kbIDs
if file.Location != nil {
doc.Location = file.Location
}
- if kb.PipelineID != nil {
- doc.PipelineID = kb.PipelineID
- doc.ParserID = "" // canvas pipeline mode — parser_id not applicable
- }
// InsertDocument creates the row and increments KB doc_num in one
// transaction, so a failed insert never leaves a stale counter.
diff --git a/internal/service/document/sync_document.go b/internal/service/document/sync_document.go
index 5395688c09..12604745fd 100644
--- a/internal/service/document/sync_document.go
+++ b/internal/service/document/sync_document.go
@@ -134,8 +134,8 @@ func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.
}
// create the 'doc'
- parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, utility.FileType(filetype), kb.ParserConfig)
- doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, parserID, parserConfig, input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)
+ selection, parserConfig := resolveDocumentParser(ctx, kb, filename, utility.FileType(filetype), kb.ParserConfig)
+ doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, selection, parserConfig, input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)
doc.ID = input.DocumentID
// A fingerprint is acknowledged only after all dependent work succeeds.
doc.ContentHash = nil
diff --git a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
index c7bb118b24..5d975f09a9 100644
--- a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
+++ b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
@@ -35,10 +35,7 @@ export function useParserGapValidation() {
if (!config) {
return [];
}
- return attachAutoSwitchTargets(
- findFilesParserGaps(names, config),
- knowledgeBase?.pipeline_id,
- );
+ return attachAutoSwitchTargets(findFilesParserGaps(names, config));
},
[loading, knowledgeBase],
);
diff --git a/web/src/pages/dataset/dataset/use-upload-document.test.ts b/web/src/pages/dataset/dataset/use-upload-document.test.ts
new file mode 100644
index 0000000000..8a39c85fae
--- /dev/null
+++ b/web/src/pages/dataset/dataset/use-upload-document.test.ts
@@ -0,0 +1,219 @@
+import { act, renderHook, waitFor } from '@testing-library/react';
+
+// NOTE: this suite uses jest.mock, so it is transformed by esbuild-jest's
+// babel pipeline, which does not strip TypeScript. Imported bindings must not
+// appear in type annotations, so shapes are declared locally and values are
+// cast at the call site.
+type Doc = {
+ id: string;
+ name: string;
+ parser_config?: Record<string, unknown>;
+ [key: string]: unknown;
+};
+
+const mockUpload = jest.fn();
+const mockRun = jest.fn();
+const mockWarning = jest.fn();
+
+// Per-test gap fixtures: pre-upload gaps by file name, and post-upload
+// document gaps keyed by document name. The real gap detection is covered by
+// utils.test.ts; here both validators are stubbed so the wiring under test is
+// which documents the hook auto-parses.
+let mockUploadGaps: Array<Record<string, unknown>> = [];
+let mockDocumentGaps: Record<string, Array<Record<string, unknown>>> = {};
+
+jest.mock('@/hooks/use-document-request', () => ({
+ useUploadDocument: () => ({ uploadDocument: mockUpload, loading: false }),
+ useRunDocument: () => ({ runDocumentByIds: mockRun, loading: false }),
+}));
+
+jest.mock('./use-parser-gap-validation', () => ({
+ useParserGapValidation: () => ({
+ findParseGaps: (names: string[]) =>
+ mockUploadGaps.filter((gap) => names.includes(gap.name as string)),
+ findDocumentParseGaps: (docs: Doc[]) =>
+ docs.flatMap((doc) => mockDocumentGaps[doc.name] ?? []),
+ }),
+}));
+
+jest.mock('@/components/ui/modal/modal', () => ({
+ Modal: {
+ warning: (options: { onOk?: () => void }) => {
+ mockWarning(options);
+ options.onOk?.();
+ },
+ },
+}));
+
+jest.mock('react-i18next', () => ({
+ useTranslation: () => ({ t: (key: string) => key }),
+}));
+
+import { useHandleUploadDocument } from './use-upload-document';
+
+function makeDoc(overrides: Partial<Doc> = {}): Doc {
+ return { id: 'doc-1', name: 'photo.jpg', parser_config: {}, ...overrides };
+}
+
+function uploadOkResponse(docs: Doc[]) {
+ return { code: 0, data: docs };
+}
+
+beforeEach(() => {
+ jest.clearAllMocks();
+ mockUploadGaps = [];
+ mockDocumentGaps = {};
+ mockUpload.mockResolvedValue(uploadOkResponse([makeDoc()]));
+});
+
+// The regression case for the auto-parse bug: an unsupported file type (here
+// an image) is reseeded by the backend with its dedicated pipeline, and the
+// uploaded response carries the converted parser_config. The pre-upload gap
+// prediction must not exclude the document from auto-parse anymore —
+// runnability follows the document's actual config.
+it('auto-parses a document the backend switched to a dedicated parser', async () => {
+ mockUploadGaps = [
+ { reason: 'unsupportedType', name: 'photo.jpg', fileType: 'Image' },
+ ];
+ mockUpload.mockResolvedValue(
+ uploadOkResponse([
+ makeDoc({
+ id: 'doc-img',
+ parser_config: {
+ 'Parser:parse': { setups: [{ fileFormat: 'Image' }] },
+ },
+ }),
+ ]),
+ );
+
+ const { result } = renderHook(() => useHandleUploadDocument());
+ await act(async () => {
+ await result.current.onDocumentUploadOk({
+ fileList: [new File(['x'], 'photo.jpg')],
+ parseOnCreation: true,
+ });
+ });
+
+ expect(mockWarning).toHaveBeenCalledTimes(1);
+ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));
+ expect(mockRun).toHaveBeenCalledWith({
+ documentIds: ['doc-img'],
+ run: 1,
+ });
+});
+
+it('skips auto-parse when the uploaded config still has a gap', async () => {
+ mockUploadGaps = [
+ { reason: 'missingModel', name: 'song.mp3', fileType: 'Audio' },
+ ];
+ mockUpload.mockResolvedValue(
+ uploadOkResponse([makeDoc({ id: 'doc-audio', name: 'song.mp3' })]),
+ );
+ mockDocumentGaps = {
+ 'song.mp3': [
+ { reason: 'missingModel', name: 'song.mp3', fileType: 'Audio' },
+ ],
+ };
+
+ const { result } = renderHook(() => useHandleUploadDocument());
+ await act(async () => {
+ await result.current.onDocumentUploadOk({
+ fileList: [new File(['x'], 'song.mp3')],
+ parseOnCreation: true,
+ });
+ });
+
+ await waitFor(() => expect(mockUpload).toHaveBeenCalledTimes(1));
+ // Give the floating proceedUpload promise a chance to settle before
+ // asserting it did not schedule a run.
+ await act(async () => {
+ await Promise.resolve();
+ });
+ expect(mockRun).not.toHaveBeenCalled();
+});
+
+it('auto-parses only the runnable documents of a mixed batch', async () => {
+ mockUpload.mockResolvedValue(
+ uploadOkResponse([
+ makeDoc({ id: 'doc-pdf', name: 'a.pdf' }),
+ makeDoc({ id: 'doc-audio', name: 'song.mp3' }),
+ ]),
+ );
+ mockDocumentGaps = {
+ 'song.mp3': [
+ { reason: 'missingModel', name: 'song.mp3', fileType: 'Audio' },
+ ],
+ };
+
+ const { result } = renderHook(() => useHandleUploadDocument());
+ await act(async () => {
+ await result.current.onDocumentUploadOk({
+ fileList: [new File(['x'], 'a.pdf'), new File(['x'], 'song.mp3')],
+ parseOnCreation: true,
+ });
+ });
+
+ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));
+ expect(mockRun).toHaveBeenCalledWith({
+ documentIds: ['doc-pdf'],
+ run: 1,
+ });
+});
+
+it('does not run when auto-parse is unchecked', async () => {
+ const { result } = renderHook(() => useHandleUploadDocument());
+ await act(async () => {
+ await result.current.onDocumentUploadOk({
+ fileList: [new File(['x'], 'a.pdf')],
+ parseOnCreation: false,
+ });
+ });
+
+ await waitFor(() => expect(mockUpload).toHaveBeenCalledTimes(1));
+ await act(async () => {
+ await Promise.resolve();
+ });
+ expect(mockRun).not.toHaveBeenCalled();
+});
+
+it('does not run when the upload fails', async () => {
+ mockUpload.mockResolvedValue({ code: 123, message: 'boom' });
+
+ const { result } = renderHook(() => useHandleUploadDocument());
+ await act(async () => {
+ await result.current.onDocumentUploadOk({
+ fileList: [new File(['x'], 'a.pdf')],
+ parseOnCreation: true,
+ });
+ });
+
+ await waitFor(() => expect(mockUpload).toHaveBeenCalledTimes(1));
+ await act(async () => {
+ await Promise.resolve();
+ });
+ expect(mockRun).not.toHaveBeenCalled();
+});
+
+it('auto-parses every document on the no-gap path without a warning modal', async () => {
+ mockUpload.mockResolvedValue(
+ uploadOkResponse([
+ makeDoc({ id: 'doc-a', name: 'a.pdf' }),
+ makeDoc({ id: 'doc-b', name: 'b.docx' }),
+ ]),
+ );
+
+ const { result } = renderHook(() => useHandleUploadDocument());
+ await act(async () => {
+ await result.current.onDocumentUploadOk({
+ fileList: [new File(['x'], 'a.pdf'), new File(['x'], 'b.docx')],
+ parseOnCreation: true,
+ });
+ });
+
+ expect(mockWarning).not.toHaveBeenCalled();
+ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));
+ expect(mockRun).toHaveBeenCalledWith({
+ documentIds: ['doc-a', 'doc-b'],
+ run: 1,
+ });
+});
diff --git a/web/src/pages/dataset/dataset/use-upload-document.ts b/web/src/pages/dataset/dataset/use-upload-document.ts
index 004baf9a8b..0d6f6663e5 100644
--- a/web/src/pages/dataset/dataset/use-upload-document.ts
+++ b/web/src/pages/dataset/dataset/use-upload-document.ts
@@ -5,14 +5,13 @@ import {
useRunDocument,
useUploadDocument,
} from '@/hooks/use-document-request';
-import { FileType } from '@/constants/file';
import { IDocumentInfo } from '@/interfaces/database/document';
-import { getExtension, getUnSupportedFilesCount } from '@/utils/document-util';
+import { getUnSupportedFilesCount } from '@/utils/document-util';
import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { buildUploadGapModalContent } from './parser-gap-content';
import { useParserGapValidation } from './use-parser-gap-validation';
-import { getFileTypeByExtension, hasUnsupportedTypeGap } from './utils';
+import { hasUnsupportedTypeGap } from './utils';
export const useHandleUploadDocument = () => {
const { t } = useTranslation();
@@ -23,18 +22,15 @@ export const useHandleUploadDocument = () => {
} = useSetModalState();
const { uploadDocument, loading } = useUploadDocument();
const { runDocumentByIds } = useRunDocument();
- const { findParseGaps } = useParserGapValidation();
+ const { findParseGaps, findDocumentParseGaps } = useParserGapValidation();
const proceedUpload = useCallback(
- async (
- {
- fileList,
- parseOnCreation,
- tableColumnMode,
- tableColumnRoles,
- }: UploadFormSchemaType,
- failingFileTypes: Set<FileType>,
- ) => {
+ async ({
+ fileList,
+ parseOnCreation,
+ tableColumnMode,
+ tableColumnRoles,
+ }: UploadFormSchemaType) => {
// Build parser_config if column roles are configured
let parserConfig: Record<string, any> | undefined;
if (
@@ -58,19 +54,18 @@ export const useHandleUploadDocument = () => {
return;
}
- // Trigger parsing for both full and partial success when parseOnCreation
- // is enabled; files whose required model is missing are uploaded but not
- // auto-parsed.
+ // Auto-parse is decided from each uploaded document's own parser_config —
+ // the config the backend resolved for it (an undeclared family is
+ // reseeded with the pipeline that owns it, or with general). This is the
+ // same validation the parse button applies, so files whose required model
+ // is missing are uploaded but not auto-parsed.
if (
(isSuccess || isPartialSuccess) &&
parseOnCreation &&
ret.data?.length > 0
) {
const runnableDocuments = (ret.data as IDocumentInfo[]).filter(
- (doc) => {
- const fileType = getFileTypeByExtension(getExtension(doc.name));
- return !fileType || !failingFileTypes.has(fileType);
- },
+ (doc) => findDocumentParseGaps([doc]).length === 0,
);
if (runnableDocuments.length > 0) {
runDocumentByIds({
@@ -94,7 +89,12 @@ export const useHandleUploadDocument = () => {
return ret?.code;
},
- [uploadDocument, runDocumentByIds, hideDocumentUploadModal],
+ [
+ uploadDocument,
+ runDocumentByIds,
+ findDocumentParseGaps,
+ hideDocumentUploadModal,
+ ],
);
const onDocumentUploadOk = useCallback(
@@ -109,7 +109,6 @@ export const useHandleUploadDocument = () => {
);
const gaps = findParseGaps(names);
if (gaps.length > 0) {
- const failingFileTypes = new Set(gaps.map((gap) => gap.fileType));
Modal.warning({
title: t(
hasUnsupportedTypeGap(gaps)
@@ -126,13 +125,13 @@ export const useHandleUploadDocument = () => {
cancelText: t('common.cancel'),
closable: false,
onOk: () => {
- proceedUpload(values, failingFileTypes);
+ proceedUpload(values);
},
});
return;
}
- return proceedUpload(values, new Set());
+ return proceedUpload(values);
},
[findParseGaps, proceedUpload, t],
);
diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.ts
index c08d05a15f..4aa50509f5 100644
--- a/web/src/pages/dataset/dataset/utils.test.ts
+++ b/web/src/pages/dataset/dataset/utils.test.ts
@@ -1,4 +1,5 @@
import { FileType } from '@/constants/file';
+import { getExtension } from '@/utils/document-util';
import {
IngestionTaskStatus,
ParserModelKind,
@@ -24,6 +25,11 @@ import {
type UploadFileParserGap,
} from './utils';
+// Mirrors how findFilesParserGaps feeds the switch decision: the family comes
+// from the extension map, the suffix sets from the backend's accepted owners.
+const switchTarget = (name: string) =>
+ builtinParserSwitchTarget(name, getFileTypeByExtension(getExtension(name)));
+
describe('isDocumentStopping', () => {
it('is true only for the STOPPING status', () => {
expect(
@@ -270,80 +276,64 @@ describe('hasUnsupportedTypeGap', () => {
});
describe('builtinParserSwitchTarget', () => {
- it('never switches for a dataset on a custom pipeline', () => {
- expect(builtinParserSwitchTarget('photo.jpg', 'canvas-1')).toBeNull();
- expect(builtinParserSwitchTarget('song.mp3', 'canvas-1')).toBeNull();
- expect(builtinParserSwitchTarget('mail.msg', 'canvas-1')).toBeNull();
- });
-
it('maps owned file types to their built-in pipelines', () => {
// The visual bucket includes the video extensions, mirroring the backend.
- expect(builtinParserSwitchTarget('photo.JPG')).toBe('picture');
- expect(builtinParserSwitchTarget('movie.mp4')).toBe('picture');
- expect(builtinParserSwitchTarget('clip.mov')).toBe('picture');
- expect(builtinParserSwitchTarget('song.mp3')).toBe('audio');
- expect(builtinParserSwitchTarget('song.opus')).toBe('audio');
- expect(builtinParserSwitchTarget('deck.ppt')).toBe('presentation');
- expect(builtinParserSwitchTarget('deck.pptx')).toBe('presentation');
- expect(builtinParserSwitchTarget('deck.pages')).toBe('presentation');
- expect(builtinParserSwitchTarget('mail.msg')).toBe('email');
- expect(builtinParserSwitchTarget('mail.eml')).toBe('email');
- });
-
- it('keeps the dataset parser for every other type', () => {
- expect(builtinParserSwitchTarget('laws.txt')).toBeNull();
- expect(builtinParserSwitchTarget('laws.pdf')).toBeNull();
- expect(builtinParserSwitchTarget('book.docx')).toBeNull();
- expect(builtinParserSwitchTarget('sheet.xlsx')).toBeNull();
+ expect(switchTarget('photo.JPG')).toBe('picture');
+ expect(switchTarget('movie.mp4')).toBe('picture');
+ expect(switchTarget('clip.mov')).toBe('picture');
+ expect(switchTarget('song.mp3')).toBe('audio');
+ expect(switchTarget('song.opus')).toBe('audio');
+ expect(switchTarget('deck.ppt')).toBe('presentation');
+ expect(switchTarget('deck.pptx')).toBe('presentation');
+ expect(switchTarget('deck.pages')).toBe('presentation');
+ expect(switchTarget('mail.msg')).toBe('email');
+ expect(switchTarget('mail.eml')).toBe('email');
+ });
+
+ it('falls back to the general pipeline for the families it declares', () => {
+ // The backend reseeds these with the catch-all general pipeline when the
+ // dataset selection does not declare the family (the upload-gap case).
+ expect(switchTarget('laws.txt')).toBe('general');
+ expect(switchTarget('laws.pdf')).toBe('general');
+ expect(switchTarget('book.docx')).toBe('general');
+ expect(switchTarget('sheet.xlsx')).toBe('general');
+ expect(switchTarget('notes.md')).toBe('general');
+ expect(switchTarget('page.html')).toBe('general');
+ });
+
+ it('keeps the dataset selection for types no pipeline owns', () => {
// Frontend-known audio suffixes the backend rejects at upload keep the
- // dataset parser too — the switch map mirrors the backend exactly.
- expect(builtinParserSwitchTarget('song.wma')).toBeNull();
+ // dataset selection too — the switch map mirrors the backend exactly.
+ expect(switchTarget('song.wma')).toBeNull();
});
});
describe('attachAutoSwitchTargets', () => {
it('annotates unsupported gaps with the target and other gaps with null', () => {
- const gaps = attachAutoSwitchTargets(
- [
- {
- name: 'photo.jpg',
- reason: ParserGapReason.UnsupportedType,
- fileType: FileType.Image,
- },
- {
- name: 'song.mp3',
- reason: ParserGapReason.MissingModel,
- fileType: FileType.Audio,
- modelKind: ParserModelKind.Asr,
- },
- {
- name: 'notes.txt',
- reason: ParserGapReason.UnsupportedType,
- fileType: FileType.Code,
- },
- ],
- null,
- );
+ const gaps = attachAutoSwitchTargets([
+ {
+ name: 'photo.jpg',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Image,
+ },
+ {
+ name: 'song.mp3',
+ reason: ParserGapReason.MissingModel,
+ fileType: FileType.Audio,
+ modelKind: ParserModelKind.Asr,
+ },
+ {
+ name: 'notes.txt',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Code,
+ },
+ ]);
expect(gaps.map((x) => x.autoSwitchTarget)).toEqual([
'picture',
null,
- null,
+ 'general',
]);
});
-
- it('annotates nothing switchable on a custom pipeline', () => {
- const gaps = attachAutoSwitchTargets(
- [
- {
- name: 'photo.jpg',
- reason: ParserGapReason.UnsupportedType,
- fileType: FileType.Image,
- },
- ],
- 'canvas-1',
- );
- expect(gaps[0].autoSwitchTarget).toBeNull();
- });
});
describe('pickBySwitchFate', () => {
diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.ts
index 25b6d84bd6..4cae6cc71a 100644
--- a/web/src/pages/dataset/dataset/utils.ts
+++ b/web/src/pages/dataset/dataset/utils.ts
@@ -168,20 +168,23 @@ export function pickByGapKind<T>(
}
// --- Upload-time built-in parser switch ------------------------------------
-// Mirrors the backend's document parser selection
-// (internal/service/document/document_parser_selection.go): for a dataset
-// without a custom pipeline, upload reseeds a document with the built-in
-// pipeline owning its file type — visual (images and video) -> picture,
-// aural -> audio, .ppt/.pptx/.pages -> presentation, .msg/.eml -> email —
-// and keeps the dataset parser for everything else. A dataset on a custom
-// pipeline never switches. The upload modal annotates each unsupported file
-// with this fate so it never promises a switch that will not happen.
+// Mirrors the backend's document parser resolution
+// (internal/service/document/document_parser_selection.go): a file whose
+// family the dataset's selection declares runs on that selection; an undeclared
+// family is reseeded with the built-in pipeline owning it — visual (images and
+// video) -> picture, aural -> audio, .ppt/.pptx/.pages -> presentation,
+// .msg/.eml -> email — and failing that with the catch-all general pipeline.
+// A dataset selection whose support cannot be determined (a canvas without a
+// Parser node) produces no gap and never switches. The upload modal annotates
+// each unsupported file with this fate so it never promises a switch that will
+// not happen.
export type BuiltinSwitchTarget =
| 'picture'
| 'audio'
| 'presentation'
- | 'email';
+ | 'email'
+ | 'general';
// Display labels match the dataset table's parse column, which renders the
// parser id capitalized.
@@ -190,8 +193,22 @@ export const BuiltinSwitchTargetLabels: Record<BuiltinSwitchTarget, string> = {
audio: 'Audio',
presentation: 'Presentation',
email: 'Email',
+ general: 'General',
};
+// FileType members the catch-all general pipeline declares. A file no dedicated
+// pipeline owns and whose family the dataset selection does not declare runs
+// under general after upload.
+const GeneralFamilyFileTypes = new Set<FileType>([
+ FileType.PDF,
+ FileType.Spreadsheet,
+ FileType.TextMarkdown,
+ FileType.Code,
+ FileType.Html,
+ FileType.Doc,
+ FileType.Docx,
+]);
+
// Extension sets mirror utility.FilenameType (internal/utility/file.go); the
// visual bucket intentionally includes the video extensions.
const VisualSwitchSuffixes = new Set([
@@ -251,17 +268,17 @@ const AuralSwitchSuffixes = new Set([
]);
/**
- * The built-in pipeline the backend switches an uploaded file to, or null
- * when the file keeps the dataset parser. `pipelineId` is the dataset's
- * custom pipeline id: once set, the backend never switches.
+ * The built-in pipeline the backend switches an uploaded file to, or null when
+ * the file keeps the dataset selection. Callers only consult this for files the
+ * selection does not declare (an unsupported-type gap), so the remaining
+ * general-family types resolve to the catch-all general pipeline. `fileType` is
+ * the uploaded file's family; a type the backend rejects at upload (or an
+ * unmapped one) keeps the dataset selection.
*/
export function builtinParserSwitchTarget(
name: string,
- pipelineId?: string | null,
+ fileType: FileType | undefined,
): BuiltinSwitchTarget | null {
- if (pipelineId) {
- return null;
- }
const suffix = getExtension(name);
if (VisualSwitchSuffixes.has(suffix)) {
return 'picture';
@@ -275,7 +292,7 @@ export function builtinParserSwitchTarget(
if (suffix === 'msg' || suffix === 'eml') {
return 'email';
}
- return null;
+ return fileType && GeneralFamilyFileTypes.has(fileType) ? 'general' : null;
}
/** An upload-time gap annotated with the file's parser-switch fate. */
@@ -286,13 +303,12 @@ export type UploadFileParserGap = FileParserGap & {
export function attachAutoSwitchTargets(
gaps: FileParserGap[],
- pipelineId?: string | null,
): UploadFileParserGap[] {
return gaps.map((gap) => ({
...gap,
autoSwitchTarget:
gap.reason === ParserGapReason.UnsupportedType
- ? builtinParserSwitchTarget(gap.name, pipelineId)
+ ? builtinParserSwitchTarget(gap.name, gap.fileType)
: null,
}));
}