RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix(dataset):auto parse unsupported upload (#21018)
ciel26ec6912865f
diff --git a/internal/ingestion/pipeline/pipeline_params.go b/internal/ingestion/pipeline/pipeline_params.goindex 9d5d55a517..529235ac9c 100644--- a/internal/ingestion/pipeline/pipeline_params.go+++ b/internal/ingestion/pipeline/pipeline_params.go@@ -139,11 +139,17 @@ func isCompilerComponent(cpnID string) bool {strings.HasPrefix(lowerID, "compiler_")}+// IsParserComponentID reports whether a component ID names a Parser component.+// A parser_config map is keyed by component ID, where the component name is not+// available, so this is the config-side half of isParserComponent.+func IsParserComponentID(cpnID string) bool {+ return strings.HasPrefix(strings.ToLower(cpnID), "parser:")+}+// isParserComponent returns true if the component name or component ID// indicates a Parser component.func isParserComponent(cpnID, componentName string) bool {- return strings.EqualFold(componentName, "parser") ||- strings.HasPrefix(strings.ToLower(cpnID), "parser:")+ return strings.EqualFold(componentName, "parser") || IsParserComponentID(cpnID)}// NormalizeParserConfigSetups lifts legacy nested Parser "setups" entries indiff --git a/internal/service/document/document_parser_selection.go b/internal/service/document/document_parser_selection.goindex fd10f48750..00ffe02fee 100644--- a/internal/service/document/document_parser_selection.go+++ b/internal/service/document/document_parser_selection.go@@ -7,6 +7,8 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"+ componentpkg "ragflow/internal/ingestion/component"+ "ragflow/internal/ingestion/component/schema"pipelinepkg "ragflow/internal/ingestion/pipeline""ragflow/internal/service""ragflow/internal/utility"@@ -14,12 +16,18 @@ import ("go.uber.org/zap")-// defaultDocumentParserID returns the parser that owns a document of the given-// file type, mirroring Python's FileService.get_parser+// generalParserID is the built-in catch-all ingestion pipeline. A file whose+// family the dataset's selection does not declare and which no dedicated+// pipeline owns falls back to it.+const generalParserID = "general"++// dedicatedDocumentParserID returns the pipeline that owns a document of the+// given file type, or "" when no pipeline owns it and the dataset's selection+// governs the file. Mirrors Python's FileService.get_parser// (api/db/services/file_service.py:734-743): images and video are parsed by the// picture pipeline, audio by audio, presentations by presentation and mail by-// email. Every other file type keeps the dataset's parser.-func defaultDocumentParserID(filetype utility.FileType, filename, fallback string) string {+// email.+func dedicatedDocumentParserID(filetype utility.FileType, filename string) string {switch filetype {case utility.FileTypeVISUAL:return "picture"@@ -40,37 +48,144 @@ func defaultDocumentParserID(filetype utility.FileType, filename, fallback strinstrings.HasSuffix(name, ".eml"):return "email"}- return fallback+ return ""+}++// parserParamsDeclaresFamily reports whether a Parser component's params+// declare the given file family ("pdf", "text&code", "slides", ...). The+// declaration is the one the web parser-gap validation reads+// (transformParserConfigSetups), so both sides judge files by one contract.+func parserParamsDeclaresFamily(params map[string]any, family string) bool {+ _, declared := schema.FlattenLegacyParserSetups(params)[family]+ return declared+}++// configDeclaredFamily reports whether a parser_config declares the family.+// found is false when the config has no readable Parser entry (a legacy flat+// config, or a malformed one): the config cannot answer then.+func configDeclaredFamily(config entity.JSONMap, family string) (declared, found bool) {+ for cpnID, raw := range config {+ if !pipelinepkg.IsParserComponentID(cpnID) {+ continue+ }+ params, ok := raw.(map[string]any)+ if !ok {+ return false, false+ }+ declared = declared || parserParamsDeclaresFamily(params, family)+ found = true+ }+ return declared, found+}++// templateDeclaredFamily reports whether a builtin template declares the family.+// found is false when the template cannot be loaded or carries no Parser+// component. The registry resolves legacy aliases ("naive" -> "general"), so a+// dataset row storing a historical parser_id is judged by the template it+// actually runs.+func templateDeclaredFamily(ctx context.Context, parserID, family string) (declared, found bool) {+ dslJSON, err := service.LoadPipelineDSL(ctx, false, parserID, nil)+ if err != nil {+ return false, false+ }+ defaults, err := pipelinepkg.ComponentParamsDefaults(dslJSON)+ if err != nil {+ return false, false+ }+ for cpnID, params := range defaults {+ if !pipelinepkg.IsParserComponentID(cpnID) {+ continue+ }+ return parserParamsDeclaresFamily(params, family), true+ }+ return false, false}-// resolveDocumentParser returns the parser_id and parser_config a newly created-// document should carry.+// selectionGovernsFamily reports whether the dataset's selection declares a+// file family.//-// A file type owned by a dedicated pipeline overrides the dataset's parser and-// the document is seeded with that pipeline's component params, so the row-// matches the parser it actually runs with: document parser_config drives both-// the parser-gap validation and the runtime override params. Without the config-// swap the document would keep another template's component ids and be reported-// as unsupported by its own file type.+// The saved config carries the declaration the web gap validation reads, so it+// decides whenever it has a Parser entry. A legacy flat config has none, and a+// builtin selection is then judged by the template its parser_id actually runs;+// a canvas has no template to consult. Callers only ask about a family the+// pipeline vocabulary can name; an unreadable declaration answers true, keeping+// the choice that cannot be judged.+func selectionGovernsFamily(ctx context.Context, sel *service.ParserSelection, config entity.JSONMap, family string) bool {+ if declared, found := configDeclaredFamily(config, family); found {+ return declared+ }+ if sel != nil && sel.IsBuiltIn() {+ if declared, found := templateDeclaredFamily(ctx, sel.ParserID, family); found {+ return declared+ }+ }+ return true+}++// parserSelectionColumns maps a resolved parser selection onto the document+// row's (parser_id, pipeline_id) columns. A canvas selection leaves parser_id+// empty: parser_id is not applicable in canvas pipeline mode.+func parserSelectionColumns(sel *service.ParserSelection) (string, *string) {+ if sel != nil && sel.IsPipeline() {+ pipelineID := sel.PipelineID+ return "", &pipelineID+ }+ parserID := ""+ if sel != nil {+ parserID = sel.ParserID+ }+ return parserID, nil+}++// resolveDocumentParser returns the parser selection and parser_config a newly+// created document should carry.//-// Canvas pipeline mode keeps the dataset parser untouched (parser_id is blanked-// by the caller). On a DSL failure the dataset parser and config are kept: a-// registry lookup must not fail an upload, and a half-switched row (new parser,-// old config) is exactly the inconsistent state this function exists to avoid.-func resolveDocumentParser(ctx context.Context, kb *entity.Knowledgebase, filename string, filetype utility.FileType, base entity.JSONMap) (string, entity.JSONMap) {- if kb.PipelineID != nil {- return kb.ParserID, base- }- parserID := defaultDocumentParserID(filetype, filename, kb.ParserID)- if parserID == kb.ParserID || parserID == "" {- return kb.ParserID, base+// The dataset's selection governs every file family its config declares; a file+// the selection does not declare is handed to the pipeline that owns its family+// (picture / audio / presentation / email), and failing that to the catch-all+// general pipeline, with the config reseeded from that template. A family the+// pipeline vocabulary cannot name cannot be judged against a declaration: the+// owning pipeline still takes the file, and with no owner (rtf, wps, ...) the+// dataset's choice is honored. The row then matches the pipeline that actually+// runs it: document parser_config drives both the parser-gap validation and the+// runtime override params, and a row left on a parser that cannot run its file+// is reported as unsupported by its own type. An unchanged parser keeps its+// config, and on a DSL failure the selection and config are kept: a registry+// lookup must not fail an upload, and a half-switched row (new parser, old+// config) is the inconsistent state this function exists to avoid.+func resolveDocumentParser(ctx context.Context, kb *entity.Knowledgebase, filename string, filetype utility.FileType, base entity.JSONMap) (*service.ParserSelection, entity.JSONMap) {+ selection := service.CurrentSelection(kb.ParserID, kb.PipelineID)+ family := componentpkg.ParserFileFamily(utility.GetFileExtension(filename))+ if family != "" && selectionGovernsFamily(ctx, selection, kb.ParserConfig, family) {+ return selection, base}- dslJSON, err := service.LoadPipelineDSL(ctx, false, parserID, nil)+ target := dedicatedDocumentParserID(filetype, filename)+ if target == "" {+ if family == "" {+ return selection, base+ }+ target = generalParserID+ }+ if selection.IsBuiltIn() && selection.ParserID == target {+ return selection, base+ }+ if rebuilt, ok := rebuildParserConfig(ctx, kb, target, filename, base); ok {+ return &service.ParserSelection{Mode: service.ParseModeBuiltIn, ParserID: target}, rebuilt+ }+ return selection, base+}++// rebuildParserConfig reseeds a document's parser_config from the target+// built-in pipeline's DSL. It reports false when the DSL cannot be loaded, so+// the caller keeps the dataset's selection instead of writing a half-switched+// row.+func rebuildParserConfig(ctx context.Context, kb *entity.Knowledgebase, target, filename string, base entity.JSONMap) (entity.JSONMap, bool) {+ dslJSON, err := service.LoadPipelineDSL(ctx, false, target, nil)if err != nil {- common.Warn("document: file-type parser DSL unavailable, keeping dataset parser",- zap.String("parser_id", parserID), zap.String("filename", filename), zap.Error(err))- return kb.ParserID, base+ common.Warn("document: target pipeline DSL unavailable, keeping dataset selection",+ zap.String("parser_id", target), zap.String("filename", filename), zap.Error(err))+ return nil, false}raw := map[string]interface{}(base)cleaned := pipelinepkg.BuildParserConfig(dslJSON, raw)@@ -84,5 +199,5 @@ func resolveDocumentParser(ctx context.Context, kb *entity.Knowledgebase, filenaif tenant, tenantErr := dao.NewTenantDAO().GetByID(ctx, dao.DB, kb.TenantID); tenantErr == nil && tenant != nil {cleaned = service.ApplyComponentScopedParserConfig(cleaned, tenant.LLMID)}- return parserID, cleaned+ return cleaned, true}diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.goindex 9ac7f58e99..baba45f2f3 100644--- a/internal/service/document/document_parser_selection_test.go+++ b/internal/service/document/document_parser_selection_test.go@@ -2,44 +2,399 @@ package documentimport ("mime/multipart"+ "reflect""strings""testing""ragflow/internal/dao""ragflow/internal/entity"+ "ragflow/internal/service""ragflow/internal/storage"+ syncerconnector "ragflow/internal/syncer/connector""ragflow/internal/utility")-func TestDefaultDocumentParserID(t *testing.T) {+func TestDedicatedDocumentParserID(t *testing.T) {cases := []struct {name stringfiletype utility.FileTypefilename string- fallback stringwant string}{- {"image routes to picture", utility.FileTypeVISUAL, "photo.jpg", "naive", "picture"},- {"video routes to picture", utility.FileTypeVISUAL, "clip.mp4", "naive", "picture"},- {"audio routes to audio", utility.FileTypeAURAL, "talk.mp3", "naive", "audio"},- {"pptx routes to presentation", utility.FileTypeDOC, "deck.pptx", "naive", "presentation"},- {"legacy ppt routes to presentation", utility.FileTypeDOC, "deck.ppt", "naive", "presentation"},- {"pages routes to presentation", utility.FileTypeDOC, "deck.pages", "naive", "presentation"},- {"presentation extension is case-insensitive", utility.FileTypeDOC, "DECK.PPTX", "naive", "presentation"},- {"eml routes to email", utility.FileTypeDOC, "mail.eml", "naive", "email"},- {"msg routes to email", utility.FileTypeDOC, "mail.msg", "naive", "email"},- {"pdf keeps the dataset parser", utility.FileTypePDF, "report.pdf", "naive", "naive"},- {"text keeps the dataset parser", utility.FileTypeDOC, "notes.txt", "manual", "manual"},- {"no substring false positive", utility.FileTypeDOC, "deck.pptx.md", "naive", "naive"},- {"dedicated dataset parser is kept", utility.FileTypeVISUAL, "photo.jpg", "picture", "picture"},+ {"image routes to picture", utility.FileTypeVISUAL, "photo.jpg", "picture"},+ {"video routes to picture", utility.FileTypeVISUAL, "clip.mp4", "picture"},+ {"audio routes to audio", utility.FileTypeAURAL, "talk.mp3", "audio"},+ {"pptx routes to presentation", utility.FileTypeDOC, "deck.pptx", "presentation"},+ {"legacy ppt routes to presentation", utility.FileTypeDOC, "deck.ppt", "presentation"},+ {"pages routes to presentation", utility.FileTypeDOC, "deck.pages", "presentation"},+ {"presentation extension is case-insensitive", utility.FileTypeDOC, "DECK.PPTX", "presentation"},+ {"eml routes to email", utility.FileTypeDOC, "mail.eml", "email"},+ {"msg routes to email", utility.FileTypeDOC, "mail.msg", "email"},+ {"pdf has no dedicated pipeline", utility.FileTypePDF, "report.pdf", ""},+ {"text has no dedicated pipeline", utility.FileTypeDOC, "notes.txt", ""},+ {"no substring false positive", utility.FileTypeDOC, "deck.pptx.md", ""},}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- got := defaultDocumentParserID(tc.filetype, tc.filename, tc.fallback)- if got != tc.want {- t.Fatalf("defaultDocumentParserID(%q, %q, %q) = %q, want %q",- tc.filetype, tc.filename, tc.fallback, got, tc.want)+ if got := dedicatedDocumentParserID(tc.filetype, tc.filename); got != tc.want {+ t.Fatalf("dedicatedDocumentParserID(%q, %q) = %q, want %q",+ tc.filetype, tc.filename, got, tc.want)+ }+ })+ }+}++func TestConfigDeclaredFamily(t *testing.T) {+ flat := entity.JSONMap{+ "Parser:HipSignsRhyme": map[string]any{+ "pdf": map[string]any{"suffix": []any{"pdf"}},+ "text&code": map[string]any{},+ "vlm": map[string]any{},+ "enable_vision_enhancement": false,+ },+ }+ nested := entity.JSONMap{+ "Parser:HipSignsRhyme": map[string]any{+ "setups": map[string]any{+ "pdf": map[string]any{},+ "slides": map[string]any{},+ },+ },+ }+ // A config never carries two Parser components today; the union keeps the+ // answer independent of map iteration order if one ever does.+ multiple := entity.JSONMap{+ "Parser:One": map[string]any{"pdf": map[string]any{}},+ "Parser:Two": map[string]any{"slides": map[string]any{}},+ }++ cases := []struct {+ name string+ config entity.JSONMap+ family string+ wantDeclared bool+ wantFound bool+ }{+ {"flat declares the family", flat, "pdf", true, true},+ {"flat misses the family", flat, "slides", false, true},+ {"nested legacy setups are lifted", nested, "slides", true, true},+ {"nested legacy setups miss the family", nested, "text&code", false, true},+ {"any parser entry declaring the family wins", multiple, "slides", true, true},+ {"an entry missing the family still answers", multiple, "audio", false, true},+ {"no parser entry cannot answer", entity.JSONMap{"File": map[string]any{}}, "pdf", false, false},+ {"nil config cannot answer", nil, "pdf", false, false},+ {"non-map parser entry cannot answer", entity.JSONMap{"Parser:X": "oops"}, "pdf", false, false},+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ declared, found := configDeclaredFamily(tc.config, tc.family)+ if declared != tc.wantDeclared || found != tc.wantFound {+ t.Fatalf("configDeclaredFamily(%#v, %q) = (%v, %v), want (%v, %v)",+ tc.config, tc.family, declared, found, tc.wantDeclared, tc.wantFound)+ }+ })+ }+}++func TestSelectionGovernsFamilyMissesAnUndeclaredFamily(t *testing.T) {+ sel := service.CurrentSelection("presentation", nil)+ if selectionGovernsFamily(t.Context(), sel, presentationParserConfig(), "text&code") {+ t.Fatal("selectionGovernsFamily reported an undeclared family as declared")+ }+}++func presentationParserConfig() entity.JSONMap {+ return entity.JSONMap{+ "Parser:HipSignsRhyme": map[string]any{+ "pdf": map[string]any{"suffix": []any{"pdf"}},+ "slides": map[string]any{"suffix": []any{"pptx", "ppt"}},+ },+ }+}++func generalParserConfig() entity.JSONMap {+ return entity.JSONMap{+ "Parser:HipSignsRhyme": map[string]any{+ "pdf": map[string]any{"suffix": []any{"pdf"}},+ "text&code": map[string]any{},+ "docx": map[string]any{},+ },+ }+}++func pictureParserConfig() entity.JSONMap {+ return entity.JSONMap{+ "Parser:ViewsCaptureLight": map[string]any{+ "image": map[string]any{"suffix": []any{"jpg"}},+ "video": map[string]any{"suffix": []any{"mp4"}},+ },+ }+}++func tableParserConfig() entity.JSONMap {+ return entity.JSONMap{+ "Parser:TableRoundStone": map[string]any{+ "spreadsheet": map[string]any{"suffix": []any{"xlsx"}},+ "text&code": map[string]any{},+ },+ }+}++// TestResolveDocumentParser_Priority pins the resolution order: the dataset's+// selection governs the families its config declares, an undeclared family goes+// to the builtin pipeline that owns it, and failing that to general. Undetermined+// support (a canvas without a Parser node) keeps the selection, and so does an+// extension the pipeline vocabulary cannot name unless an owning pipeline takes+// it.+func TestResolveDocumentParser_Priority(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ if err := dao.DB.Create(&entity.Tenant{ID: "tenant-1", LLMID: "llm-default"}).Error; err != nil {+ t.Fatalf("insert tenant: %v", err)+ }++ canvasID := "canvas-1"+ legacyConfig := entity.JSONMap{"llm_id": "ministral", "chunk_token_num": 512}+ canvasWithoutParser := entity.JSONMap{"File": map[string]any{}}++ cases := []struct {+ name string+ kbParser string+ kbPipelineID *string+ kbConfig entity.JSONMap+ filename string+ filetype utility.FileType+ wantParserID string+ wantPipeline string+ wantFamilyIn string+ wantFamilyOut string+ wantBaseConfigKept bool+ }{+ {+ name: "general keeps a text family it declares",+ kbParser: "general",+ kbConfig: generalParserConfig(),+ filename: "laws.txt",+ filetype: utility.FileTypeDOC,+ wantParserID: "general",+ wantFamilyIn: "text&code",+ wantFamilyOut: "slides",+ },+ {+ name: "presentation keeps a pdf it declares",+ kbParser: "presentation",+ kbConfig: presentationParserConfig(),+ filename: "report.pdf",+ filetype: utility.FileTypePDF,+ wantParserID: "presentation",+ wantFamilyIn: "slides",+ wantFamilyOut: "text&code",+ },+ {+ name: "presentation falls back to general for text",+ kbParser: "presentation",+ kbConfig: presentationParserConfig(),+ filename: "laws.txt",+ filetype: utility.FileTypeDOC,+ wantParserID: "general",+ wantFamilyIn: "text&code",+ wantFamilyOut: "slides",+ },+ {+ name: "table falls back to general for pdf",+ kbParser: "table",+ kbConfig: tableParserConfig(),+ filename: "report.pdf",+ filetype: utility.FileTypePDF,+ wantParserID: "general",+ wantFamilyIn: "docx",+ wantFamilyOut: "",+ },+ {+ name: "picture owns an image the selection does not declare",+ kbParser: "presentation",+ kbConfig: presentationParserConfig(),+ filename: "photo.jpg",+ filetype: utility.FileTypeVISUAL,+ wantParserID: "picture",+ wantFamilyIn: "image",+ wantFamilyOut: "slides",+ },+ {+ name: "audio owns audio the selection does not declare",+ kbParser: "picture",+ kbConfig: pictureParserConfig(),+ filename: "talk.mp3",+ filetype: utility.FileTypeAURAL,+ wantParserID: "audio",+ wantFamilyIn: "audio",+ wantFamilyOut: "image",+ },+ {+ name: "legacy flat config is judged by its template",+ kbParser: "naive",+ kbConfig: legacyConfig,+ filename: "laws.txt",+ filetype: utility.FileTypeDOC,+ wantParserID: "naive",+ wantFamilyIn: "",+ wantFamilyOut: "",+ },+ {+ name: "legacy flat config still routes an owned family",+ kbParser: "naive",+ kbConfig: legacyConfig,+ filename: "deck.pptx",+ filetype: utility.FileTypeDOC,+ wantParserID: "presentation",+ wantFamilyIn: "slides",+ wantFamilyOut: "",+ },+ {+ name: "canvas keeps a family it declares",+ kbParser: "presentation",+ kbPipelineID: &canvasID,+ kbConfig: presentationParserConfig(),+ filename: "report.pdf",+ filetype: utility.FileTypePDF,+ wantParserID: "",+ wantPipeline: canvasID,+ wantFamilyIn: "slides",+ wantFamilyOut: "",+ },+ {+ name: "canvas falls back to general for an undeclared family",+ kbParser: "presentation",+ kbPipelineID: &canvasID,+ kbConfig: presentationParserConfig(),+ filename: "laws.txt",+ filetype: utility.FileTypeDOC,+ wantParserID: "general",+ wantPipeline: "",+ wantFamilyIn: "text&code",+ wantFamilyOut: "slides",+ },+ {+ name: "canvas without a parser node keeps the canvas",+ kbParser: "presentation",+ kbPipelineID: &canvasID,+ kbConfig: canvasWithoutParser,+ filename: "laws.txt",+ filetype: utility.FileTypeDOC,+ wantParserID: "",+ wantPipeline: canvasID,+ },+ {+ name: "canvas without a parser node owns nothing either",+ kbParser: "presentation",+ kbPipelineID: &canvasID,+ kbConfig: canvasWithoutParser,+ filename: "photo.jpg",+ filetype: utility.FileTypeVISUAL,+ wantParserID: "",+ wantPipeline: canvasID,+ },+ {+ name: "canvas selects into general for an unowned family",+ kbParser: "presentation",+ kbPipelineID: &canvasID,+ kbConfig: pictureParserConfig(),+ filename: "report.pdf",+ filetype: utility.FileTypePDF,+ wantParserID: "general",+ wantFamilyIn: "text&code",+ },+ {+ name: "an unmappable extension still reaches its owner pipeline",+ kbParser: "general",+ kbConfig: generalParserConfig(),+ filename: "deck.pages",+ filetype: utility.FileTypeDOC,+ wantParserID: "presentation",+ wantFamilyIn: "slides",+ },+ {+ name: "an unmappable visual extension still reaches picture",+ kbParser: "general",+ kbConfig: generalParserConfig(),+ filename: "clip.rm",+ filetype: utility.FileTypeVISUAL,+ wantParserID: "picture",+ wantFamilyIn: "video",+ },+ {+ name: "an unmappable audio extension still reaches audio",+ kbParser: "general",+ kbConfig: generalParserConfig(),+ filename: "song.vorbis",+ filetype: utility.FileTypeAURAL,+ wantParserID: "audio",+ wantFamilyIn: "audio",+ },+ {+ name: "an unmappable extension with no owner keeps the selection",+ kbParser: "general",+ kbConfig: generalParserConfig(),+ filename: "notes.rtf",+ filetype: utility.FileTypeDOC,+ wantParserID: "general",+ wantBaseConfigKept: true,+ },+ {+ name: "an owner already selected keeps its edited config",+ kbParser: "picture",+ kbConfig: pictureParserConfig(),+ filename: "clip.psd",+ filetype: utility.FileTypeVISUAL,+ wantParserID: "picture",+ wantBaseConfigKept: true,+ },+ {+ name: "a canvas loses an unmappable extension to its owner",+ kbParser: "presentation",+ kbPipelineID: &canvasID,+ kbConfig: presentationParserConfig(),+ filename: "clip.rm",+ filetype: utility.FileTypeVISUAL,+ wantParserID: "picture",+ wantFamilyIn: "video",+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ kb := &entity.Knowledgebase{+ ID: "kb-" + tc.name,+ TenantID: "tenant-1",+ Name: "kb",+ CreatedBy: "user-1",+ ParserID: tc.kbParser,+ PipelineID: tc.kbPipelineID,+ ParserConfig: tc.kbConfig,+ }+ base := cloneParserConfigForDocument(tc.kbConfig)++ selection, gotConfig := resolveDocumentParser(t.Context(), kb, tc.filename, tc.filetype, base)++ gotParser, gotPipelinePtr := parserSelectionColumns(selection)+ gotPipeline := ""+ if gotPipelinePtr != nil {+ gotPipeline = *gotPipelinePtr+ }+ if gotParser != tc.wantParserID || gotPipeline != tc.wantPipeline {+ t.Fatalf("selection = (%q, %q), want (%q, %q)",+ gotParser, gotPipeline, tc.wantParserID, tc.wantPipeline)+ }+ if tc.wantBaseConfigKept && !reflect.DeepEqual(gotConfig, base) {+ t.Fatalf("parser_config = %#v, want the base config kept", gotConfig)+ }+ if tc.wantFamilyIn != "" && !configHasParserFamily(gotConfig, tc.wantFamilyIn) {+ t.Fatalf("parser_config misses family %q: %#v", tc.wantFamilyIn, gotConfig)+ }+ if tc.wantFamilyOut != "" && configHasParserFamily(gotConfig, tc.wantFamilyOut) {+ t.Fatalf("parser_config unexpectedly keeps family %q: %#v", tc.wantFamilyOut, gotConfig)}})}@@ -141,6 +496,162 @@ func TestUploadLocalDocuments_RoutesFileTypesToDedicatedParsers(t *testing.T) {}}+// TestUploadLocalDocuments_FallsBackWhenSelectionLacksFamily covers the upload+// path end to end: a text file uploaded to a dataset whose selection declares+// only pdf and slides is reseeded with the catch-all general pipeline, and a+// canvas dataset keeps a family its canvas declares while falling back for the+// rest.+func TestUploadLocalDocuments_FallsBackWhenSelectionLacksFamily(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ mockStorage := newFakeUploadStorage()+ factory := storage.GetStorageFactory()+ origStorage := factory.GetStorage()+ factory.SetStorage(mockStorage)+ t.Cleanup(func() { factory.SetStorage(origStorage) })++ if err := dao.DB.Create(&entity.Tenant{+ ID: "tenant-1", LLMID: "llm-default", EmbdID: "embd-default", ASRID: "asr-default",+ }).Error; err != nil {+ t.Fatalf("insert tenant: %v", err)+ }++ ctx := t.Context()+ svc := testDocumentService(t)++ presentationKB := &entity.Knowledgebase{+ ID: "kb-presentation",+ TenantID: "tenant-1",+ Name: "kb-presentation",+ CreatedBy: "user-1",+ ParserID: "presentation",+ ParserConfig: presentationParserConfig(),+ }+ if err := dao.DB.Create(presentationKB).Error; err != nil {+ t.Fatalf("insert presentation kb: %v", err)+ }+ got, errs := svc.UploadLocalDocuments(ctx, presentationKB, "user-1", []*multipart.FileHeader{+ makeTestFileHeader(t, "file", "laws.txt", []byte("plain text")),+ makeTestFileHeader(t, "file", "deck.pptx", []byte("pptx-bytes")),+ }, "", nil)+ if len(errs) != 0 {+ t.Fatalf("unexpected errs: %v", errs)+ }+ if parserID, _ := got[0]["parser_id"].(string); parserID != "general" {+ t.Fatalf("txt parser_id = %q, want general", parserID)+ }+ parserFamilySetup(t, got[0], "text&code")+ if parserID, _ := got[1]["parser_id"].(string); parserID != "presentation" {+ t.Fatalf("pptx parser_id = %q, want presentation", parserID)+ }+ parserFamilySetup(t, got[1], "slides")++ canvasID := "canvas-1"+ canvasKB := &entity.Knowledgebase{+ ID: "kb-canvas",+ TenantID: "tenant-1",+ Name: "kb-canvas",+ CreatedBy: "user-1",+ ParserID: "presentation",+ PipelineID: &canvasID,+ ParserConfig: presentationParserConfig(),+ }+ if err := dao.DB.Create(canvasKB).Error; err != nil {+ t.Fatalf("insert canvas kb: %v", err)+ }+ got, errs = svc.UploadLocalDocuments(ctx, canvasKB, "user-1", []*multipart.FileHeader{+ makeTestFileHeader(t, "file", "report.pdf", []byte("%PDF-1.4")),+ makeTestFileHeader(t, "file", "laws.txt", []byte("plain text")),+ }, "", nil)+ if len(errs) != 0 {+ t.Fatalf("unexpected errs: %v", errs)+ }+ // The canvas declares pdf: the document stays on the canvas with parser_id+ // blanked and the canvas carried on pipeline_id.+ if parserID, _ := got[0]["parser_id"].(string); parserID != "" {+ t.Fatalf("canvas pdf parser_id = %q, want blank", parserID)+ }+ if pipelineID, _ := got[0]["pipeline_id"].(string); pipelineID != canvasID {+ t.Fatalf("canvas pdf pipeline_id = %q, want %q", pipelineID, canvasID)+ }+ parserFamilySetup(t, got[0], "pdf")+ // The canvas does not declare text&code: the document leaves the canvas for+ // general.+ if parserID, _ := got[1]["parser_id"].(string); parserID != "general" {+ t.Fatalf("canvas txt parser_id = %q, want general", parserID)+ }+ if pipelineID, ok := got[1]["pipeline_id"].(string); ok && pipelineID != "" {+ t.Fatalf("canvas txt pipeline_id = %q, want none", pipelineID)+ }+ parserFamilySetup(t, got[1], "text&code")+}++// TestInsertSyncDocument_FallsBackWhenSelectionLacksFamily covers the connector+// path: it shares resolveDocumentParser with the upload path, so a synced text+// file lands on general too.+func TestInsertSyncDocument_FallsBackWhenSelectionLacksFamily(t *testing.T) {+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)++ mockStorage := newFakeUploadStorage()+ factory := storage.GetStorageFactory()+ origStorage := factory.GetStorage()+ factory.SetStorage(mockStorage)+ t.Cleanup(func() { factory.SetStorage(origStorage) })++ if err := dao.DB.Create(&entity.Tenant{+ ID: "tenant-1", LLMID: "llm-default", EmbdID: "embd-default", ASRID: "asr-default",+ }).Error; err != nil {+ t.Fatalf("insert tenant: %v", err)+ }++ kb := &entity.Knowledgebase{+ ID: "kb-sync",+ TenantID: "tenant-1",+ Name: "kb-sync",+ CreatedBy: "user-1",+ ParserID: "presentation",+ ParserConfig: presentationParserConfig(),+ }+ if err := dao.DB.Create(kb).Error; err != nil {+ t.Fatalf("insert kb: %v", err)+ }++ input := service.DocumentUpsertInput{+ TaskContext: dao.SyncTaskContext{Knowledgebase: *kb},+ SourceType: "connector",+ DocumentID: "doc-sync-1",+ SourceDocument: syncerconnector.SourceDocument{Blob: []byte("plain text")},+ }+ svc := testDocumentService(t)+ res, err := svc.insertSyncDocument(t.Context(), input, "laws.txt", "", "txt", "", "tenant-1")+ if err != nil {+ t.Fatalf("insertSyncDocument: %v", err)+ }++ doc, err := svc.documentDAO.GetByID(t.Context(), dao.DB, res.DocID)+ if err != nil {+ t.Fatalf("load synced document: %v", err)+ }+ if doc.ParserID != "general" {+ t.Fatalf("synced txt parser_id = %q, want general", doc.ParserID)+ }+ if doc.PipelineID != nil {+ t.Fatalf("synced txt pipeline_id = %v, want nil", *doc.PipelineID)+ }+ if !configHasParserFamily(doc.ParserConfig, "text&code") {+ t.Fatalf("synced txt parser_config misses text&code: %#v", doc.ParserConfig)+ }+}++// configHasParserFamily reports whether the config's Parser entry declares the+// given family, read through the production declaration reader.+func configHasParserFamily(config entity.JSONMap, family string) bool {+ declared, _ := configDeclaredFamily(config, family)+ return declared+}+// parserFamilySetup returns the family setup of the document's Parser node, the// shape the web parser-gap validation keys on.func parserFamilySetup(t *testing.T, doc map[string]interface{}, family string) map[string]interface{} {diff --git a/internal/service/document/document_upload.go b/internal/service/document/document_upload.goindex 3ff7fdd654..fd89e32ba1 100644--- a/internal/service/document/document_upload.go+++ b/internal/service/document/document_upload.go@@ -10,6 +10,7 @@ import ("ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity"+ "ragflow/internal/service""ragflow/internal/storage""ragflow/internal/utility""strings"@@ -88,8 +89,8 @@ func (s *DocumentService) UploadLocalDocuments(ctx context.Context, kb *entity.Kcontinue}- parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, merged)- doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), parserID, parserConfig, "local", int64(len(blob)), blob)+ selection, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, merged)+ doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), selection, parserConfig, "local", int64(len(blob)), blob)if err = s.InsertDocument(doc); err != nil {// Roll back the orphaned blob so a failed insert doesn't leak storage.rmErr := removeObjectBestEffort(ctx, storageImpl, kb.ID, location)@@ -133,7 +134,7 @@ func (s *DocumentService) UploadEmptyDocument(ctx context.Context, kb *entity.Kn// Virtual documents have no file type to route on, so they keep the// dataset parser (mirrors Python's _upload_empty_document).- doc := s.newDatasetDocument(kb, tenantID, name, "", "virtual", kb.ParserID, kb.ParserConfig, "local", 0, nil)+ doc := s.newDatasetDocument(kb, tenantID, name, "", "virtual", service.CurrentSelection(kb.ParserID, kb.PipelineID), kb.ParserConfig, "local", 0, nil)if err = s.InsertDocument(doc); err != nil {return nil, common.CodeServerError, err}@@ -271,8 +272,8 @@ func (s *DocumentService) UploadWebDocument(ctx context.Context, kb *entity.Knowreturn nil, common.CodeServerError, err}- parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, kb.ParserConfig)- doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), parserID, parserConfig, "web", int64(len(blob)), blob)+ selection, parserConfig := resolveDocumentParser(ctx, kb, filename, filetype, kb.ParserConfig)+ doc := s.newDatasetDocument(kb, tenantID, filename, location, string(filetype), selection, parserConfig, "web", int64(len(blob)), blob)if err = s.InsertDocument(doc); err != nil {rmErr := removeObjectBestEffort(ctx, storageImpl, kb.ID, location)if rmErr != nil {@@ -308,9 +309,9 @@ func normalizeWebDocumentName(name, contentType string, blob []byte) string {}// newDatasetDocument builds a Document row for an upload, deriving suffix and-// content hash. parserID/parserConfig are resolved by the caller (see+// content hash. selection/parserConfig are resolved by the caller (see// resolveDocumentParser); blob may be nil for the empty/virtual document.-func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, filename, location, filetype, parserID string, parserConfig entity.JSONMap, src string, size int64, blob []byte) *entity.Document {+func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID, filename, location, filetype string, selection *service.ParserSelection, parserConfig entity.JSONMap, src string, size int64, blob []byte) *entity.Document {docID := common.GenerateToken()parserConfig = cloneParserConfigForDocument(parserConfig)status := "1"@@ -318,15 +319,13 @@ func (s *DocumentService) newDatasetDocument(kb *entity.Knowledgebase, tenantID,if i := strings.LastIndex(filename, "."); i >= 0 {suffix = filename[i+1:]}- if kb.PipelineID != nil {- parserID = "" // canvas pipeline mode — parser_id not applicable- }+ parserID, pipelineID := parserSelectionColumns(selection)loc := locationdoc := &entity.Document{ID: docID,KbID: kb.ID,ParserID: parserID,- PipelineID: kb.PipelineID,+ PipelineID: pipelineID,ParserConfig: parserConfig,CreatedBy: tenantID,Type: filetype,diff --git a/internal/service/document/file2document.go b/internal/service/document/file2document.goindex 90a2425f80..8133c0f532 100644--- a/internal/service/document/file2document.go+++ b/internal/service/document/file2document.go@@ -241,12 +241,14 @@ func (s *File2DocumentService) convertFiles(ctx context.Context, fileIDs, kbIDscontinue}- parserID, parserConfig := resolveDocumentParser(ctx, kb, docName, utility.FileType(file.Type), cloneParserConfigForDocument(kb.ParserConfig))+ selection, parserConfig := resolveDocumentParser(ctx, kb, docName, utility.FileType(file.Type), cloneParserConfigForDocument(kb.ParserConfig))+ parserID, pipelineID := parserSelectionColumns(selection)suffix := strings.TrimPrefix(filepath.Ext(docName), ".")doc := &entity.Document{ID: common.GenerateUUID(),KbID: kb.ID,ParserID: parserID,+ PipelineID: pipelineID,ParserConfig: parserConfig,CreatedBy: userID,Type: file.Type,@@ -257,10 +259,6 @@ func (s *File2DocumentService) convertFiles(ctx context.Context, fileIDs, kbIDsif file.Location != nil {doc.Location = file.Location}- if kb.PipelineID != nil {- doc.PipelineID = kb.PipelineID- doc.ParserID = "" // canvas pipeline mode — parser_id not applicable- }// InsertDocument creates the row and increments KB doc_num in one// transaction, so a failed insert never leaves a stale counter.diff --git a/internal/service/document/sync_document.go b/internal/service/document/sync_document.goindex 5395688c09..12604745fd 100644--- a/internal/service/document/sync_document.go+++ b/internal/service/document/sync_document.go@@ -134,8 +134,8 @@ func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.}// create the 'doc'- parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, utility.FileType(filetype), kb.ParserConfig)- doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, parserID, parserConfig, input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)+ selection, parserConfig := resolveDocumentParser(ctx, kb, filename, utility.FileType(filetype), kb.ParserConfig)+ doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, selection, parserConfig, input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)doc.ID = input.DocumentID// A fingerprint is acknowledged only after all dependent work succeeds.doc.ContentHash = nildiff --git a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts b/web/src/pages/dataset/dataset/use-parser-gap-validation.tsindex c7bb118b24..5d975f09a9 100644--- a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts+++ b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts@@ -35,10 +35,7 @@ export function useParserGapValidation() {if (!config) {return [];}- return attachAutoSwitchTargets(- findFilesParserGaps(names, config),- knowledgeBase?.pipeline_id,- );+ return attachAutoSwitchTargets(findFilesParserGaps(names, config));},[loading, knowledgeBase],);diff --git a/web/src/pages/dataset/dataset/use-upload-document.test.ts b/web/src/pages/dataset/dataset/use-upload-document.test.tsnew file mode 100644index 0000000000..8a39c85fae--- /dev/null+++ b/web/src/pages/dataset/dataset/use-upload-document.test.ts@@ -0,0 +1,219 @@+import { act, renderHook, waitFor } from '@testing-library/react';++// NOTE: this suite uses jest.mock, so it is transformed by esbuild-jest's+// babel pipeline, which does not strip TypeScript. Imported bindings must not+// appear in type annotations, so shapes are declared locally and values are+// cast at the call site.+type Doc = {+ id: string;+ name: string;+ parser_config?: Record<string, unknown>;+ [key: string]: unknown;+};++const mockUpload = jest.fn();+const mockRun = jest.fn();+const mockWarning = jest.fn();++// Per-test gap fixtures: pre-upload gaps by file name, and post-upload+// document gaps keyed by document name. The real gap detection is covered by+// utils.test.ts; here both validators are stubbed so the wiring under test is+// which documents the hook auto-parses.+let mockUploadGaps: Array<Record<string, unknown>> = [];+let mockDocumentGaps: Record<string, Array<Record<string, unknown>>> = {};++jest.mock('@/hooks/use-document-request', () => ({+ useUploadDocument: () => ({ uploadDocument: mockUpload, loading: false }),+ useRunDocument: () => ({ runDocumentByIds: mockRun, loading: false }),+}));++jest.mock('./use-parser-gap-validation', () => ({+ useParserGapValidation: () => ({+ findParseGaps: (names: string[]) =>+ mockUploadGaps.filter((gap) => names.includes(gap.name as string)),+ findDocumentParseGaps: (docs: Doc[]) =>+ docs.flatMap((doc) => mockDocumentGaps[doc.name] ?? []),+ }),+}));++jest.mock('@/components/ui/modal/modal', () => ({+ Modal: {+ warning: (options: { onOk?: () => void }) => {+ mockWarning(options);+ options.onOk?.();+ },+ },+}));++jest.mock('react-i18next', () => ({+ useTranslation: () => ({ t: (key: string) => key }),+}));++import { useHandleUploadDocument } from './use-upload-document';++function makeDoc(overrides: Partial<Doc> = {}): Doc {+ return { id: 'doc-1', name: 'photo.jpg', parser_config: {}, ...overrides };+}++function uploadOkResponse(docs: Doc[]) {+ return { code: 0, data: docs };+}++beforeEach(() => {+ jest.clearAllMocks();+ mockUploadGaps = [];+ mockDocumentGaps = {};+ mockUpload.mockResolvedValue(uploadOkResponse([makeDoc()]));+});++// The regression case for the auto-parse bug: an unsupported file type (here+// an image) is reseeded by the backend with its dedicated pipeline, and the+// uploaded response carries the converted parser_config. The pre-upload gap+// prediction must not exclude the document from auto-parse anymore —+// runnability follows the document's actual config.+it('auto-parses a document the backend switched to a dedicated parser', async () => {+ mockUploadGaps = [+ { reason: 'unsupportedType', name: 'photo.jpg', fileType: 'Image' },+ ];+ mockUpload.mockResolvedValue(+ uploadOkResponse([+ makeDoc({+ id: 'doc-img',+ parser_config: {+ 'Parser:parse': { setups: [{ fileFormat: 'Image' }] },+ },+ }),+ ]),+ );++ const { result } = renderHook(() => useHandleUploadDocument());+ await act(async () => {+ await result.current.onDocumentUploadOk({+ fileList: [new File(['x'], 'photo.jpg')],+ parseOnCreation: true,+ });+ });++ expect(mockWarning).toHaveBeenCalledTimes(1);+ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));+ expect(mockRun).toHaveBeenCalledWith({+ documentIds: ['doc-img'],+ run: 1,+ });+});++it('skips auto-parse when the uploaded config still has a gap', async () => {+ mockUploadGaps = [+ { reason: 'missingModel', name: 'song.mp3', fileType: 'Audio' },+ ];+ mockUpload.mockResolvedValue(+ uploadOkResponse([makeDoc({ id: 'doc-audio', name: 'song.mp3' })]),+ );+ mockDocumentGaps = {+ 'song.mp3': [+ { reason: 'missingModel', name: 'song.mp3', fileType: 'Audio' },+ ],+ };++ const { result } = renderHook(() => useHandleUploadDocument());+ await act(async () => {+ await result.current.onDocumentUploadOk({+ fileList: [new File(['x'], 'song.mp3')],+ parseOnCreation: true,+ });+ });++ await waitFor(() => expect(mockUpload).toHaveBeenCalledTimes(1));+ // Give the floating proceedUpload promise a chance to settle before+ // asserting it did not schedule a run.+ await act(async () => {+ await Promise.resolve();+ });+ expect(mockRun).not.toHaveBeenCalled();+});++it('auto-parses only the runnable documents of a mixed batch', async () => {+ mockUpload.mockResolvedValue(+ uploadOkResponse([+ makeDoc({ id: 'doc-pdf', name: 'a.pdf' }),+ makeDoc({ id: 'doc-audio', name: 'song.mp3' }),+ ]),+ );+ mockDocumentGaps = {+ 'song.mp3': [+ { reason: 'missingModel', name: 'song.mp3', fileType: 'Audio' },+ ],+ };++ const { result } = renderHook(() => useHandleUploadDocument());+ await act(async () => {+ await result.current.onDocumentUploadOk({+ fileList: [new File(['x'], 'a.pdf'), new File(['x'], 'song.mp3')],+ parseOnCreation: true,+ });+ });++ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));+ expect(mockRun).toHaveBeenCalledWith({+ documentIds: ['doc-pdf'],+ run: 1,+ });+});++it('does not run when auto-parse is unchecked', async () => {+ const { result } = renderHook(() => useHandleUploadDocument());+ await act(async () => {+ await result.current.onDocumentUploadOk({+ fileList: [new File(['x'], 'a.pdf')],+ parseOnCreation: false,+ });+ });++ await waitFor(() => expect(mockUpload).toHaveBeenCalledTimes(1));+ await act(async () => {+ await Promise.resolve();+ });+ expect(mockRun).not.toHaveBeenCalled();+});++it('does not run when the upload fails', async () => {+ mockUpload.mockResolvedValue({ code: 123, message: 'boom' });++ const { result } = renderHook(() => useHandleUploadDocument());+ await act(async () => {+ await result.current.onDocumentUploadOk({+ fileList: [new File(['x'], 'a.pdf')],+ parseOnCreation: true,+ });+ });++ await waitFor(() => expect(mockUpload).toHaveBeenCalledTimes(1));+ await act(async () => {+ await Promise.resolve();+ });+ expect(mockRun).not.toHaveBeenCalled();+});++it('auto-parses every document on the no-gap path without a warning modal', async () => {+ mockUpload.mockResolvedValue(+ uploadOkResponse([+ makeDoc({ id: 'doc-a', name: 'a.pdf' }),+ makeDoc({ id: 'doc-b', name: 'b.docx' }),+ ]),+ );++ const { result } = renderHook(() => useHandleUploadDocument());+ await act(async () => {+ await result.current.onDocumentUploadOk({+ fileList: [new File(['x'], 'a.pdf'), new File(['x'], 'b.docx')],+ parseOnCreation: true,+ });+ });++ expect(mockWarning).not.toHaveBeenCalled();+ await waitFor(() => expect(mockRun).toHaveBeenCalledTimes(1));+ expect(mockRun).toHaveBeenCalledWith({+ documentIds: ['doc-a', 'doc-b'],+ run: 1,+ });+});diff --git a/web/src/pages/dataset/dataset/use-upload-document.ts b/web/src/pages/dataset/dataset/use-upload-document.tsindex 004baf9a8b..0d6f6663e5 100644--- a/web/src/pages/dataset/dataset/use-upload-document.ts+++ b/web/src/pages/dataset/dataset/use-upload-document.ts@@ -5,14 +5,13 @@ import {useRunDocument,useUploadDocument,} from '@/hooks/use-document-request';-import { FileType } from '@/constants/file';import { IDocumentInfo } from '@/interfaces/database/document';-import { getExtension, getUnSupportedFilesCount } from '@/utils/document-util';+import { getUnSupportedFilesCount } from '@/utils/document-util';import { useCallback } from 'react';import { useTranslation } from 'react-i18next';import { buildUploadGapModalContent } from './parser-gap-content';import { useParserGapValidation } from './use-parser-gap-validation';-import { getFileTypeByExtension, hasUnsupportedTypeGap } from './utils';+import { hasUnsupportedTypeGap } from './utils';export const useHandleUploadDocument = () => {const { t } = useTranslation();@@ -23,18 +22,15 @@ export const useHandleUploadDocument = () => {} = useSetModalState();const { uploadDocument, loading } = useUploadDocument();const { runDocumentByIds } = useRunDocument();- const { findParseGaps } = useParserGapValidation();+ const { findParseGaps, findDocumentParseGaps } = useParserGapValidation();const proceedUpload = useCallback(- async (- {- fileList,- parseOnCreation,- tableColumnMode,- tableColumnRoles,- }: UploadFormSchemaType,- failingFileTypes: Set<FileType>,- ) => {+ async ({+ fileList,+ parseOnCreation,+ tableColumnMode,+ tableColumnRoles,+ }: UploadFormSchemaType) => {// Build parser_config if column roles are configuredlet parserConfig: Record<string, any> | undefined;if (@@ -58,19 +54,18 @@ export const useHandleUploadDocument = () => {return;}- // Trigger parsing for both full and partial success when parseOnCreation- // is enabled; files whose required model is missing are uploaded but not- // auto-parsed.+ // Auto-parse is decided from each uploaded document's own parser_config —+ // the config the backend resolved for it (an undeclared family is+ // reseeded with the pipeline that owns it, or with general). This is the+ // same validation the parse button applies, so files whose required model+ // is missing are uploaded but not auto-parsed.if ((isSuccess || isPartialSuccess) &&parseOnCreation &&ret.data?.length > 0) {const runnableDocuments = (ret.data as IDocumentInfo[]).filter(- (doc) => {- const fileType = getFileTypeByExtension(getExtension(doc.name));- return !fileType || !failingFileTypes.has(fileType);- },+ (doc) => findDocumentParseGaps([doc]).length === 0,);if (runnableDocuments.length > 0) {runDocumentByIds({@@ -94,7 +89,12 @@ export const useHandleUploadDocument = () => {return ret?.code;},- [uploadDocument, runDocumentByIds, hideDocumentUploadModal],+ [+ uploadDocument,+ runDocumentByIds,+ findDocumentParseGaps,+ hideDocumentUploadModal,+ ],);const onDocumentUploadOk = useCallback(@@ -109,7 +109,6 @@ export const useHandleUploadDocument = () => {);const gaps = findParseGaps(names);if (gaps.length > 0) {- const failingFileTypes = new Set(gaps.map((gap) => gap.fileType));Modal.warning({title: t(hasUnsupportedTypeGap(gaps)@@ -126,13 +125,13 @@ export const useHandleUploadDocument = () => {cancelText: t('common.cancel'),closable: false,onOk: () => {- proceedUpload(values, failingFileTypes);+ proceedUpload(values);},});return;}- return proceedUpload(values, new Set());+ return proceedUpload(values);},[findParseGaps, proceedUpload, t],);diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.tsindex c08d05a15f..4aa50509f5 100644--- a/web/src/pages/dataset/dataset/utils.test.ts+++ b/web/src/pages/dataset/dataset/utils.test.ts@@ -1,4 +1,5 @@import { FileType } from '@/constants/file';+import { getExtension } from '@/utils/document-util';import {IngestionTaskStatus,ParserModelKind,@@ -24,6 +25,11 @@ import {type UploadFileParserGap,} from './utils';+// Mirrors how findFilesParserGaps feeds the switch decision: the family comes+// from the extension map, the suffix sets from the backend's accepted owners.+const switchTarget = (name: string) =>+ builtinParserSwitchTarget(name, getFileTypeByExtension(getExtension(name)));+describe('isDocumentStopping', () => {it('is true only for the STOPPING status', () => {expect(@@ -270,80 +276,64 @@ describe('hasUnsupportedTypeGap', () => {});describe('builtinParserSwitchTarget', () => {- it('never switches for a dataset on a custom pipeline', () => {- expect(builtinParserSwitchTarget('photo.jpg', 'canvas-1')).toBeNull();- expect(builtinParserSwitchTarget('song.mp3', 'canvas-1')).toBeNull();- expect(builtinParserSwitchTarget('mail.msg', 'canvas-1')).toBeNull();- });-it('maps owned file types to their built-in pipelines', () => {// The visual bucket includes the video extensions, mirroring the backend.- expect(builtinParserSwitchTarget('photo.JPG')).toBe('picture');- expect(builtinParserSwitchTarget('movie.mp4')).toBe('picture');- expect(builtinParserSwitchTarget('clip.mov')).toBe('picture');- expect(builtinParserSwitchTarget('song.mp3')).toBe('audio');- expect(builtinParserSwitchTarget('song.opus')).toBe('audio');- expect(builtinParserSwitchTarget('deck.ppt')).toBe('presentation');- expect(builtinParserSwitchTarget('deck.pptx')).toBe('presentation');- expect(builtinParserSwitchTarget('deck.pages')).toBe('presentation');- expect(builtinParserSwitchTarget('mail.msg')).toBe('email');- expect(builtinParserSwitchTarget('mail.eml')).toBe('email');- });-- it('keeps the dataset parser for every other type', () => {- expect(builtinParserSwitchTarget('laws.txt')).toBeNull();- expect(builtinParserSwitchTarget('laws.pdf')).toBeNull();- expect(builtinParserSwitchTarget('book.docx')).toBeNull();- expect(builtinParserSwitchTarget('sheet.xlsx')).toBeNull();+ expect(switchTarget('photo.JPG')).toBe('picture');+ expect(switchTarget('movie.mp4')).toBe('picture');+ expect(switchTarget('clip.mov')).toBe('picture');+ expect(switchTarget('song.mp3')).toBe('audio');+ expect(switchTarget('song.opus')).toBe('audio');+ expect(switchTarget('deck.ppt')).toBe('presentation');+ expect(switchTarget('deck.pptx')).toBe('presentation');+ expect(switchTarget('deck.pages')).toBe('presentation');+ expect(switchTarget('mail.msg')).toBe('email');+ expect(switchTarget('mail.eml')).toBe('email');+ });++ it('falls back to the general pipeline for the families it declares', () => {+ // The backend reseeds these with the catch-all general pipeline when the+ // dataset selection does not declare the family (the upload-gap case).+ expect(switchTarget('laws.txt')).toBe('general');+ expect(switchTarget('laws.pdf')).toBe('general');+ expect(switchTarget('book.docx')).toBe('general');+ expect(switchTarget('sheet.xlsx')).toBe('general');+ expect(switchTarget('notes.md')).toBe('general');+ expect(switchTarget('page.html')).toBe('general');+ });++ it('keeps the dataset selection for types no pipeline owns', () => {// Frontend-known audio suffixes the backend rejects at upload keep the- // dataset parser too — the switch map mirrors the backend exactly.- expect(builtinParserSwitchTarget('song.wma')).toBeNull();+ // dataset selection too — the switch map mirrors the backend exactly.+ expect(switchTarget('song.wma')).toBeNull();});});describe('attachAutoSwitchTargets', () => {it('annotates unsupported gaps with the target and other gaps with null', () => {- const gaps = attachAutoSwitchTargets(- [- {- name: 'photo.jpg',- reason: ParserGapReason.UnsupportedType,- fileType: FileType.Image,- },- {- name: 'song.mp3',- reason: ParserGapReason.MissingModel,- fileType: FileType.Audio,- modelKind: ParserModelKind.Asr,- },- {- name: 'notes.txt',- reason: ParserGapReason.UnsupportedType,- fileType: FileType.Code,- },- ],- null,- );+ const gaps = attachAutoSwitchTargets([+ {+ name: 'photo.jpg',+ reason: ParserGapReason.UnsupportedType,+ fileType: FileType.Image,+ },+ {+ name: 'song.mp3',+ reason: ParserGapReason.MissingModel,+ fileType: FileType.Audio,+ modelKind: ParserModelKind.Asr,+ },+ {+ name: 'notes.txt',+ reason: ParserGapReason.UnsupportedType,+ fileType: FileType.Code,+ },+ ]);expect(gaps.map((x) => x.autoSwitchTarget)).toEqual(['picture',null,- null,+ 'general',]);});-- it('annotates nothing switchable on a custom pipeline', () => {- const gaps = attachAutoSwitchTargets(- [- {- name: 'photo.jpg',- reason: ParserGapReason.UnsupportedType,- fileType: FileType.Image,- },- ],- 'canvas-1',- );- expect(gaps[0].autoSwitchTarget).toBeNull();- });});describe('pickBySwitchFate', () => {diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.tsindex 25b6d84bd6..4cae6cc71a 100644--- a/web/src/pages/dataset/dataset/utils.ts+++ b/web/src/pages/dataset/dataset/utils.ts@@ -168,20 +168,23 @@ export function pickByGapKind<T>(}// --- Upload-time built-in parser switch -------------------------------------// Mirrors the backend's document parser selection-// (internal/service/document/document_parser_selection.go): for a dataset-// without a custom pipeline, upload reseeds a document with the built-in-// pipeline owning its file type — visual (images and video) -> picture,-// aural -> audio, .ppt/.pptx/.pages -> presentation, .msg/.eml -> email —-// and keeps the dataset parser for everything else. A dataset on a custom-// pipeline never switches. The upload modal annotates each unsupported file-// with this fate so it never promises a switch that will not happen.+// Mirrors the backend's document parser resolution+// (internal/service/document/document_parser_selection.go): a file whose+// family the dataset's selection declares runs on that selection; an undeclared+// family is reseeded with the built-in pipeline owning it — visual (images and+// video) -> picture, aural -> audio, .ppt/.pptx/.pages -> presentation,+// .msg/.eml -> email — and failing that with the catch-all general pipeline.+// A dataset selection whose support cannot be determined (a canvas without a+// Parser node) produces no gap and never switches. The upload modal annotates+// each unsupported file with this fate so it never promises a switch that will+// not happen.export type BuiltinSwitchTarget =| 'picture'| 'audio'| 'presentation'- | 'email';+ | 'email'+ | 'general';// Display labels match the dataset table's parse column, which renders the// parser id capitalized.@@ -190,8 +193,22 @@ export const BuiltinSwitchTargetLabels: Record<BuiltinSwitchTarget, string> = {audio: 'Audio',presentation: 'Presentation',email: 'Email',+ general: 'General',};+// FileType members the catch-all general pipeline declares. A file no dedicated+// pipeline owns and whose family the dataset selection does not declare runs+// under general after upload.+const GeneralFamilyFileTypes = new Set<FileType>([+ FileType.PDF,+ FileType.Spreadsheet,+ FileType.TextMarkdown,+ FileType.Code,+ FileType.Html,+ FileType.Doc,+ FileType.Docx,+]);+// Extension sets mirror utility.FilenameType (internal/utility/file.go); the// visual bucket intentionally includes the video extensions.const VisualSwitchSuffixes = new Set([@@ -251,17 +268,17 @@ const AuralSwitchSuffixes = new Set([]);/**- * The built-in pipeline the backend switches an uploaded file to, or null- * when the file keeps the dataset parser. `pipelineId` is the dataset's- * custom pipeline id: once set, the backend never switches.+ * The built-in pipeline the backend switches an uploaded file to, or null when+ * the file keeps the dataset selection. Callers only consult this for files the+ * selection does not declare (an unsupported-type gap), so the remaining+ * general-family types resolve to the catch-all general pipeline. `fileType` is+ * the uploaded file's family; a type the backend rejects at upload (or an+ * unmapped one) keeps the dataset selection.*/export function builtinParserSwitchTarget(name: string,- pipelineId?: string | null,+ fileType: FileType | undefined,): BuiltinSwitchTarget | null {- if (pipelineId) {- return null;- }const suffix = getExtension(name);if (VisualSwitchSuffixes.has(suffix)) {return 'picture';@@ -275,7 +292,7 @@ export function builtinParserSwitchTarget(if (suffix === 'msg' || suffix === 'eml') {return 'email';}- return null;+ return fileType && GeneralFamilyFileTypes.has(fileType) ? 'general' : null;}/** An upload-time gap annotated with the file's parser-switch fate. */@@ -286,13 +303,12 @@ export type UploadFileParserGap = FileParserGap & {export function attachAutoSwitchTargets(gaps: FileParserGap[],- pipelineId?: string | null,): UploadFileParserGap[] {return gaps.map((gap) => ({...gap,autoSwitchTarget:gap.reason === ParserGapReason.UnsupportedType- ? builtinParserSwitchTarget(gap.name, pipelineId)+ ? builtinParserSwitchTarget(gap.name, gap.fileType): null,}));}