RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: Dataset file upload: For unsupported file types, prompt the user to configure the corresponding file parser. (#19710)
balibabu4d24efec2827
diff --git a/web/src/constants/file.ts b/web/src/constants/file.tsindex 2814093051..cf32404d4c 100644--- a/web/src/constants/file.ts+++ b/web/src/constants/file.ts@@ -14,6 +14,67 @@* limitations under the License.*/+export enum FileType {+ PDF = 'pdf',+ Spreadsheet = 'spreadsheet',+ Image = 'image',+ Email = 'email',+ TextMarkdown = 'markdown',+ Code = 'text&code',+ Html = 'html',+ Doc = 'doc',+ Docx = 'docx',+ PowerPoint = 'slides',+ Video = 'video',+ Audio = 'audio',+}++export const FileTypeSuffixMap = {+ [FileType.PDF]: ['pdf'],+ [FileType.Spreadsheet]: ['xls', 'xlsx', 'csv'],+ [FileType.Image]: ['jpg', 'jpeg', 'png', 'gif'],+ [FileType.Email]: ['eml', 'msg'],+ [FileType.TextMarkdown]: ['md', 'markdown', 'mdx'],+ [FileType.Code]: [+ 'txt',+ 'py',+ 'js',+ 'java',+ 'c',+ 'cpp',+ 'h',+ 'php',+ 'go',+ 'ts',+ 'sh',+ 'cs',+ 'kt',+ 'sql',+ ],+ [FileType.Html]: ['htm', 'html'],+ [FileType.Doc]: ['doc'],+ [FileType.Docx]: ['docx'],+ [FileType.PowerPoint]: ['pptx', 'ppt'],+ [FileType.Video]: ['mp4', 'avi', 'mkv'],+ [FileType.Audio]: [+ 'da',+ 'wave',+ 'wav',+ 'mp3',+ 'aac',+ 'flac',+ 'ogg',+ 'aiff',+ 'au',+ 'midi',+ 'wma',+ 'realaudio',+ 'vqf',+ 'oggvorbis',+ 'ape',+ ],+};+export const FileIconMap = {doc: 'doc',docx: 'doc',diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex df922c0058..5930e0cf54 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -397,18 +397,21 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim},knowledgeDetails: {continueUpload: 'Continue upload',- goToConfiguration: 'Go to configuration',+ reselectParser: 'Reselect parse method',uploadMissingModelsTitle: 'Some files lack a required model',fileModelMissing:'{{name}} ({{fileType}}) requires a configured {{model}} model',missingModelAsr: 'audio (ASR)',missingModelVision: 'vision',- configureInDatasetSettingHint:- 'You can configure it in the dataset Configuration page (Parser operator), or continue anyway.',+ uploadUnsupportedTypesTitle: 'Some file types cannot be parsed',+ fileTypeUnsupported:+ '{{name}} ({{fileType}}): not supported by the current parser configuration',+ reselectParserAfterUploadHint:+ 'Continue uploading, then reselect a parse method for these files in the file list.',+ reselectParserToParseHint:+ 'Reselect a parse method for the affected files, then parse again.',parseBlockedTitle: 'Cannot parse',parseBlockedPartialTitle: 'Some files cannot be parsed',- parseBlockedHint:- 'Configure the model in the dataset Configuration page (Parser operator), then try again.',parseValidFiles: 'Parse valid files',parseValidFilesNote: '{{count}} valid files will be parsed.',metadata: {diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex 49fcb9d01c..c13d3b0423 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -356,17 +356,19 @@ export default {},knowledgeDetails: {continueUpload: '继续上传',- goToConfiguration: '前往配置',+ reselectParser: '重新选择解析方法',uploadMissingModelsTitle: '部分文件缺少解析所需的模型',fileModelMissing: '{{name}}({{fileType}})需要配置{{model}}模型',missingModelAsr: '音频(ASR)',missingModelVision: '视觉',- configureInDatasetSettingHint:- '可在数据集「配置」页的 Parser 算子中设置对应模型,或继续操作。',+ uploadUnsupportedTypesTitle: '部分文件类型不支持解析',+ fileTypeUnsupported:+ '{{name}}({{fileType}}):当前解析配置不支持该文件类型',+ reselectParserAfterUploadHint:+ '可继续上传,上传后在文件列表中重新选择这些文件的解析方法。',+ reselectParserToParseHint: '请重新选择对应文件的解析方法后再解析。',parseBlockedTitle: '无法解析',parseBlockedPartialTitle: '部分文件无法解析',- parseBlockedHint:- '请先在数据集「配置」页的 Parser 算子中设置对应模型,再重新解析。',parseValidFiles: '解析有效文件',parseValidFilesNote: '将解析其余 {{count}} 个有效文件。',metadata: {diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex d6ee82567d..131a7cdc38 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -1,24 +1,10 @@import { ParseDocumentType } from '@/components/layout-recognize-form-field';import { initialLlmBaseValues, Operator } from '@/constants/agent';+import { FileType } from '@/constants/file';import { ModelTypeToField } from '@/constants/llm';import { pickByBackend } from '@/utils/backend-variant';import { cloneDeep } from 'lodash';-export enum FileType {- PDF = 'pdf',- Spreadsheet = 'spreadsheet',- Image = 'image',- Email = 'email',- TextMarkdown = 'markdown',- Code = 'text&code',- Html = 'html',- Doc = 'doc',- Docx = 'docx',- PowerPoint = 'slides',- Video = 'video',- Audio = 'audio',-}-export enum PdfOutputFormat {Json = 'json',Markdown = 'markdown',@@ -447,52 +433,6 @@ export const initialCompilationValues = {export const NoDebugOperatorsList = [Operator.File];-export const FileTypeSuffixMap = {- [FileType.PDF]: ['pdf'],- [FileType.Spreadsheet]: ['xls', 'xlsx', 'csv'],- [FileType.Image]: ['jpg', 'jpeg', 'png', 'gif'],- [FileType.Email]: ['eml', 'msg'],- [FileType.TextMarkdown]: ['md', 'markdown', 'mdx'],- [FileType.Code]: [- 'txt',- 'py',- 'js',- 'java',- 'c',- 'cpp',- 'h',- 'php',- 'go',- 'ts',- 'sh',- 'cs',- 'kt',- 'sql',- ],- [FileType.Html]: ['htm', 'html'],- [FileType.Doc]: ['doc'],- [FileType.Docx]: ['docx'],- [FileType.PowerPoint]: ['pptx', 'ppt'],- [FileType.Video]: ['mp4', 'avi', 'mkv'],- [FileType.Audio]: [- 'da',- 'wave',- 'wav',- 'mp3',- 'aac',- 'flac',- 'ogg',- 'aiff',- 'au',- 'midi',- 'wma',- 'realaudio',- 'vqf',- 'oggvorbis',- 'ape',- ],-};-export const SingleOperators = [Operator.Tokenizer,Operator.GeneralChunker,diff --git a/web/src/pages/agent/form/parser-form/common-form-fields.tsx b/web/src/pages/agent/form/parser-form/common-form-fields.tsxindex 2d0a1fd145..23f343a65d 100644--- a/web/src/pages/agent/form/parser-form/common-form-fields.tsx+++ b/web/src/pages/agent/form/parser-form/common-form-fields.tsx@@ -6,11 +6,11 @@ import {} from '@/components/originui/select-with-search';import { RAGFlowFormItem } from '@/components/ragflow-form';import { Switch } from '@/components/ui/switch';+import { FileType } from '@/constants/file';import { upperCase, upperFirst } from 'lodash';import { useTranslation } from 'react-i18next';import { useOwnerTenantId } from '../../context';import {- FileType,OutputFormatMap,SpreadsheetOutputFormat,} from '../../constant/pipeline';diff --git a/web/src/pages/agent/form/parser-form/index.tsx b/web/src/pages/agent/form/parser-form/index.tsxindex 95f395d47f..835a3b3f30 100644--- a/web/src/pages/agent/form/parser-form/index.tsx+++ b/web/src/pages/agent/form/parser-form/index.tsx@@ -9,6 +9,7 @@ import {DropdownMenuTrigger,} from '@/components/ui/dropdown-menu';import { Form } from '@/components/ui/form';+import { FileType } from '@/constants/file';import { useFetchDefaultModelDictionary } from '@/hooks/use-llm-request';import { zodResolver } from '@hookform/resolvers/zod';import { LucideTrash2 } from 'lucide-react';@@ -16,7 +17,7 @@ import { memo, useCallback, useMemo } from 'react';import { useFieldArray, useForm, useFormContext } from 'react-hook-form';import { useTranslation } from 'react-i18next';import { z } from 'zod';-import { FileType, initialParserValues } from '../../constant/pipeline';+import { initialParserValues } from '../../constant/pipeline';import { useFormChangeCallback } from '../../hooks/use-form-change-callback';import { useFormValues } from '../../hooks/use-form-values';import { useWatchFormChange } from '../../hooks/use-watch-form-change';diff --git a/web/src/pages/agent/form/parser-form/schema.test.ts b/web/src/pages/agent/form/parser-form/schema.test.tsindex 6e5001283e..71add82be3 100644--- a/web/src/pages/agent/form/parser-form/schema.test.ts+++ b/web/src/pages/agent/form/parser-form/schema.test.ts@@ -1,4 +1,5 @@-import { FileType, ParserFields } from '../../constant/pipeline';+import { FileType } from '@/constants/file';+import { ParserFields } from '../../constant/pipeline';import { FormSchema } from './schema';describe('parser FormSchema', () => {diff --git a/web/src/pages/agent/form/parser-form/schema.ts b/web/src/pages/agent/form/parser-form/schema.tsindex 8dc4163514..0af3f8dae0 100644--- a/web/src/pages/agent/form/parser-form/schema.ts+++ b/web/src/pages/agent/form/parser-form/schema.ts@@ -1,6 +1,6 @@+import { FileType } from '@/constants/file';import i18n from '@/locales/config';import { z } from 'zod';-import { FileType } from '../../constant/pipeline';export const SetupSchema = z.object({diff --git a/web/src/pages/agent/form/parser-form/utils.test.ts b/web/src/pages/agent/form/parser-form/utils.test.tsindex 984036b62e..764ec77573 100644--- a/web/src/pages/agent/form/parser-form/utils.test.ts+++ b/web/src/pages/agent/form/parser-form/utils.test.ts@@ -1,5 +1,6 @@+import { FileType } from '@/constants/file';import { ModelTypeToField } from '@/constants/llm';-import { FileType, initialParserValues } from '../../constant/pipeline';+import { initialParserValues } from '../../constant/pipeline';import { buildInitialParserSetup, buildInitialParserValues } from './utils';describe('parser-form utils', () => {diff --git a/web/src/pages/agent/form/parser-form/utils.ts b/web/src/pages/agent/form/parser-form/utils.tsindex b1aa6bb4d3..05d35248d3 100644--- a/web/src/pages/agent/form/parser-form/utils.ts+++ b/web/src/pages/agent/form/parser-form/utils.ts@@ -1,6 +1,6 @@+import { FileType } from '@/constants/file';import { cloneDeep } from 'lodash';import {- FileType,FileTypeDefaultModelFieldMap,initialParserValues,} from '../../constant/pipeline';diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.tsindex 77ce72caef..18b8b1b4e8 100644--- a/web/src/pages/agent/utils.ts+++ b/web/src/pages/agent/utils.ts@@ -1,3 +1,4 @@+import { FileType, FileTypeSuffixMap } from '@/constants/file';import {DSL,DSLComponents,@@ -28,8 +29,6 @@ import isObject from 'lodash/isObject';import {AgentDialogueMode,CategorizeAnchorPointPositions,- FileType,- FileTypeSuffixMap,InputMode,NoCopyOperatorsList,NoDebugOperatorsList,diff --git a/web/src/pages/dataset/dataset/constant.ts b/web/src/pages/dataset/dataset/constant.tsindex 96bebd79b3..5516e9b8d5 100644--- a/web/src/pages/dataset/dataset/constant.ts+++ b/web/src/pages/dataset/dataset/constant.ts@@ -31,3 +31,21 @@ export const RunningStatusMap = {};export * from '@/constants/knowledge';++/** Why a file cannot be parsed under the dataset's current Parser operator. */+export enum ParserGapReason {+ // The operator setups declare no entry for the file's type family at all.+ UnsupportedType = 'unsupportedType',+ // The family is declared, but its required model (audio/video) is not set.+ MissingModel = 'missingModel',+}++/**+ * Model capability a Parser operator setup needs for audio/video files.+ * Values follow the parser-configuration vocabulary (the ModelTypeToField+ * keys), not the LLM registry types (LlmModelType).+ */+export enum ParserModelKind {+ Asr = 'asr',+ Vision = 'vision',+}diff --git a/web/src/pages/dataset/dataset/parser-gap-content.tsx b/web/src/pages/dataset/dataset/parser-gap-content.tsxnew file mode 100644index 0000000000..705ab92ccc--- /dev/null+++ b/web/src/pages/dataset/dataset/parser-gap-content.tsx@@ -0,0 +1,41 @@+import type { TFunction } from 'i18next';+import type { ReactNode } from 'react';+import { ParserGapReason, ParserModelKind } from './constant';+import type { FileParserGap } from './utils';++/**+ * Builds the body of the parser-gap modals (upload warning, parse-click+ * error). Strings are resolved with the caller's `t` because the static Modal+ * API renders content in a separate React root without the app providers.+ */+export function buildParserGapModalContent(+ t: TFunction,+ gaps: FileParserGap[],+ hintKey: string,+): ReactNode {+ return (+ <div className="space-y-2">+ <ul className="list-disc pl-4 space-y-1">+ {gaps.map((gap, index) => (+ <li key={`${gap.name}-${index}`}>+ {gap.reason === ParserGapReason.MissingModel+ ? t('knowledgeDetails.fileModelMissing', {+ name: gap.name,+ fileType: t(`flow.fileFormatOptions.${gap.fileType}`),+ model: t(+ gap.modelKind === ParserModelKind.Asr+ ? 'knowledgeDetails.missingModelAsr'+ : 'knowledgeDetails.missingModelVision',+ ),+ })+ : t('knowledgeDetails.fileTypeUnsupported', {+ name: gap.name,+ fileType: t(`flow.fileFormatOptions.${gap.fileType}`),+ })}+ </li>+ ))}+ </ul>+ <p className="text-text-secondary">{t(hintKey)}</p>+ </div>+ );+}diff --git a/web/src/pages/dataset/dataset/parser-model-gap-content.tsx b/web/src/pages/dataset/dataset/parser-model-gap-content.tsxdeleted file mode 100644index 317032b1b1..0000000000--- a/web/src/pages/dataset/dataset/parser-model-gap-content.tsx+++ /dev/null@@ -1,35 +0,0 @@-import type { TFunction } from 'i18next';-import type { ReactNode } from 'react';-import type { FileModelGap } from './utils';--/**- * Builds the body of the missing-model modals (upload warning, parse-click- * error). Strings are resolved with the caller's `t` because the static Modal- * API renders content in a separate React root without the app providers.- */-export function buildMissingModelModalContent(- t: TFunction,- gaps: FileModelGap[],- hintKey: string,-): ReactNode {- return (- <div className="space-y-2">- <ul className="list-disc pl-4 space-y-1">- {gaps.map((gap, index) => (- <li key={`${gap.name}-${index}`}>- {t('knowledgeDetails.fileModelMissing', {- name: gap.name,- fileType: t(`flow.fileFormatOptions.${gap.fileType}`),- model: t(- gap.modelKind === 'asr'- ? 'knowledgeDetails.missingModelAsr'- : 'knowledgeDetails.missingModelVision',- ),- })}- </li>- ))}- </ul>- <p className="text-text-secondary">{t(hintKey)}</p>- </div>- );-}diff --git a/web/src/pages/dataset/dataset/parsing-status-cell.tsx b/web/src/pages/dataset/dataset/parsing-status-cell.tsxindex 3c3eb15b80..7ba4d90efe 100644--- a/web/src/pages/dataset/dataset/parsing-status-cell.tsx+++ b/web/src/pages/dataset/dataset/parsing-status-cell.tsx@@ -116,6 +116,7 @@ export function ParseDropdownButton({export function ParsingStatusCell({record,showLog,+ showChangeParserModal,}: {record: IDocumentInfo;showLog: (record: IDocumentInfo) => void;@@ -133,7 +134,7 @@ export function ParsingStatusCell({visible: reparseDialogVisible,showModal: showReparseDialogModal,hideModal: hideReparseDialogModal,- } = useHandleRunDocumentByIds(id);+ } = useHandleRunDocumentByIds(id, showChangeParserModal);const isGo = useIsGoBackend();const isRunning = isDocumentProcessing(record);const isQueued = effectiveRun === RunningStatus.QUEUED;diff --git a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsxindex 69c324802d..c5b8d49b9d 100644--- a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx+++ b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx@@ -24,8 +24,8 @@ import { useParams } from 'react-router';import { toast } from 'sonner';import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';import { DocumentType } from './constant';-import { buildMissingModelModalContent } from './parser-model-gap-content';-import { useParserModelValidation } from './use-parser-model-validation';+import { buildParserGapModalContent } from './parser-gap-content';+import { useParserGapValidation } from './use-parser-gap-validation';import { isDocumentProcessing } from './utils';export function useBulkOperateDataset({@@ -46,8 +46,7 @@ export function useBulkOperateDataset({const { setDocumentStatus } = useSetDocumentStatus();const { removeDocument } = useRemoveDocument();const { visible, showModal, hideModal } = useSetModalState();- const { findFilesMissingModels, goToDatasetConfiguration } =- useParserModelValidation();+ const { findDocumentParseGaps } = useParserGapValidation();const { knowledgeBase } = useKnowledgeBaseContext();const chunkNum = useMemo(() => {@@ -73,15 +72,14 @@ export function useBulkOperateDataset({return;}- // Starting a parse requires the models of each file type to be- // configured; cancelling is always allowed.+ // Starting a parse requires each file type to be supported by the+ // Parser operator its document actually runs with; cancelling is+ // always allowed.if (run === 1) {const selectedDocuments = documents.filter((x) =>nonVirtualKeys.includes(x.id),);- const gaps = findFilesMissingModels(- selectedDocuments.map((x) => x.name),- );+ const gaps = findDocumentParseGaps(selectedDocuments);if (gaps.length > 0) {hideModal();const failingNames = new Set(gaps.map((gap) => gap.name));@@ -92,15 +90,14 @@ export function useBulkOperateDataset({if (validIds.length === 0) {Modal.error({title: t('knowledgeDetails.parseBlockedTitle'),- content: buildMissingModelModalContent(+ content: buildParserGapModalContent(t,gaps,- 'knowledgeDetails.parseBlockedHint',+ 'knowledgeDetails.reselectParserToParseHint',),- okText: t('knowledgeDetails.goToConfiguration'),- cancelText: t('common.cancel'),+ showCancel: false,+ okText: t('common.cancel'),closable: false,- onOk: goToDatasetConfiguration,});return;}@@ -109,10 +106,10 @@ export function useBulkOperateDataset({title: t('knowledgeDetails.parseBlockedPartialTitle'),content: (<div className="space-y-2">- {buildMissingModelModalContent(+ {buildParserGapModalContent(t,gaps,- 'knowledgeDetails.parseBlockedHint',+ 'knowledgeDetails.reselectParserToParseHint',)}<p>{t('knowledgeDetails.parseValidFilesNote', {@@ -145,8 +142,7 @@ export function useBulkOperateDataset({selectedRowKeys,hideModal,t,- findFilesMissingModels,- goToDatasetConfiguration,+ findDocumentParseGaps,],);diff --git a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts b/web/src/pages/dataset/dataset/use-parser-gap-validation.tsnew file mode 100644index 0000000000..57263f56fa--- /dev/null+++ b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts@@ -0,0 +1,60 @@+import type { IDocumentInfo } from '@/interfaces/database/document';+import { useIsGoBackend } from '@/utils/backend-variant';+import { useCallback } from 'react';+import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';+import {+ FileParserGap,+ findDocumentsParserGaps,+ findFilesParserGaps,+ getSavedParserSetups,+} from './utils';++/**+ * Validates that each file can be parsed under the Parser operator config it+ * actually runs with: the file type must be declared in the operator setups,+ * and audio/video additionally need their model configured.+ * Only the Go backend has parser operator settings — on the Python backend+ * every file passes.+ */+export function useParserGapValidation() {+ const isGoBackend = useIsGoBackend();+ const { knowledgeBase, loading } = useKnowledgeBaseContext();++ // Skip while the dataset is loading: its parser_config is not readable+ // yet and every file would look unsupported. A missing Parser entry means+ // the config predates operator-scoped settings, so type support cannot be+ // determined either.+ const findParseGaps = useCallback(+ (names: string[]): FileParserGap[] => {+ if (!isGoBackend || loading) {+ return [];+ }+ const setups = getSavedParserSetups(knowledgeBase);+ if (!setups) {+ return [];+ }+ return findFilesParserGaps(names, setups);+ },+ [isGoBackend, loading, knowledgeBase],+ );++ // Parse-time validation: a document may override the dataset parser, and+ // its row carries the config it actually runs with — validate against that,+ // falling back to the dataset-level config for rows without one.+ const findDocumentParseGaps = useCallback(+ (+ documents: Pick<IDocumentInfo, 'name' | 'parser_config'>[],+ ): FileParserGap[] => {+ if (!isGoBackend || loading) {+ return [];+ }+ return findDocumentsParserGaps(+ documents,+ getSavedParserSetups(knowledgeBase),+ );+ },+ [isGoBackend, loading, knowledgeBase],+ );++ return { findParseGaps, findDocumentParseGaps };+}diff --git a/web/src/pages/dataset/dataset/use-parser-model-validation.ts b/web/src/pages/dataset/dataset/use-parser-model-validation.tsdeleted file mode 100644index d06613ea21..0000000000--- a/web/src/pages/dataset/dataset/use-parser-model-validation.ts+++ /dev/null@@ -1,46 +0,0 @@-import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';-import { useIsGoBackend } from '@/utils/backend-variant';-import { useCallback } from 'react';-import { useParams } from 'react-router';-import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';-import {- FileModelGap,- findFilesMissingParserModels,- getEffectiveParserSetups,-} from './utils';--/**- * Validates that the models required to parse a file type (audio/video/image)- * are configured on the dataset's Parser operator.- * Only the Go backend has parser operator settings — on the Python backend- * every file passes.- */-export function useParserModelValidation() {- const isGoBackend = useIsGoBackend();- const { knowledgeBase, loading } = useKnowledgeBaseContext();- const { id } = useParams();- const { navigateToDatasetSetting } = useNavigatePage();-- const findFilesMissingModels = useCallback(- (names: string[]): FileModelGap[] => {- // Skip while the dataset is loading: its parser_config is not readable- // yet and every media file would look unconfigured.- if (!isGoBackend || loading) {- return [];- }- return findFilesMissingParserModels(- names,- getEffectiveParserSetups(knowledgeBase),- );- },- [isGoBackend, loading, knowledgeBase],- );-- const goToDatasetConfiguration = useCallback(() => {- if (id) {- navigateToDatasetSetting(id);- }- }, [id, navigateToDatasetSetting]);-- return { findFilesMissingModels, goToDatasetConfiguration };-}diff --git a/web/src/pages/dataset/dataset/use-run-document.ts b/web/src/pages/dataset/dataset/use-run-document.tsindex cd48016eb8..290ab157a9 100644--- a/web/src/pages/dataset/dataset/use-run-document.ts+++ b/web/src/pages/dataset/dataset/use-run-document.ts@@ -4,17 +4,20 @@ import { useRunDocument } from '@/hooks/use-document-request';import { IDocumentInfo } from '@/interfaces/database/document';import { useState } from 'react';import { useTranslation } from 'react-i18next';-import { buildMissingModelModalContent } from './parser-model-gap-content';-import { useParserModelValidation } from './use-parser-model-validation';+import { buildParserGapModalContent } from './parser-gap-content';+import { UseChangeDocumentParserShowType } from './use-change-document-parser';+import { useParserGapValidation } from './use-parser-gap-validation';-export const useHandleRunDocumentByIds = (id: string) => {+export const useHandleRunDocumentByIds = (+ id: string,+ showChangeParserModal: UseChangeDocumentParserShowType['showChangeParserModal'],+) => {const { t } = useTranslation();const { runDocumentByIds, loading } = useRunDocument();const [currentId, setCurrentId] = useState<string>('');const isLoading = loading && currentId !== '' && currentId === id;const { visible, showModal, hideModal } = useSetModalState();- const { findFilesMissingModels, goToDatasetConfiguration } =- useParserModelValidation();+ const { findDocumentParseGaps } = useParserGapValidation();const handleRunDocumentByIds = async (record: IDocumentInfo,@@ -24,23 +27,26 @@ export const useHandleRunDocumentByIds = (id: string) => {if (isLoading) {return;}- // Starting a parse requires the models of the file type to be configured;- // cancelling is always allowed.+ // Starting a parse requires the file type to be supported by the+ // Parser operator the document actually runs with; cancelling is+ // always allowed.if (!isRunning) {- const gaps = findFilesMissingModels([record.name]);+ const gaps = findDocumentParseGaps([record]);if (gaps.length > 0) {hideModal();Modal.error({title: t('knowledgeDetails.parseBlockedTitle'),- content: buildMissingModelModalContent(+ content: buildParserGapModalContent(t,gaps,- 'knowledgeDetails.parseBlockedHint',+ 'knowledgeDetails.reselectParserToParseHint',),- okText: t('knowledgeDetails.goToConfiguration'),+ okText: t('knowledgeDetails.reselectParser'),cancelText: t('common.cancel'),closable: false,- onOk: goToDatasetConfiguration,+ onOk: () => {+ showChangeParserModal(record);+ },});return;}diff --git a/web/src/pages/dataset/dataset/use-upload-document.ts b/web/src/pages/dataset/dataset/use-upload-document.tsindex 81b7ef3c92..86a8d0568b 100644--- a/web/src/pages/dataset/dataset/use-upload-document.ts+++ b/web/src/pages/dataset/dataset/use-upload-document.ts@@ -5,14 +5,14 @@ import {useRunDocument,useUploadDocument,} from '@/hooks/use-document-request';+import { FileType } from '@/constants/file';import { IDocumentInfo } from '@/interfaces/database/document';-import { FileType } from '@/pages/agent/constant/pipeline';import { getExtension, getUnSupportedFilesCount } from '@/utils/document-util';import { useCallback } from 'react';import { useTranslation } from 'react-i18next';-import { buildMissingModelModalContent } from './parser-model-gap-content';-import { useParserModelValidation } from './use-parser-model-validation';-import { getFileTypeByExtension } from './utils';+import { buildParserGapModalContent } from './parser-gap-content';+import { useParserGapValidation } from './use-parser-gap-validation';+import { getFileTypeByExtension, hasUnsupportedTypeGap } from './utils';export const useHandleUploadDocument = () => {const { t } = useTranslation();@@ -23,8 +23,7 @@ export const useHandleUploadDocument = () => {} = useSetModalState();const { uploadDocument, loading } = useUploadDocument();const { runDocumentByIds } = useRunDocument();- const { findFilesMissingModels, goToDatasetConfiguration } =- useParserModelValidation();+ const { findParseGaps } = useParserGapValidation();const proceedUpload = useCallback(async (@@ -108,39 +107,33 @@ export const useHandleUploadDocument = () => {const names = fileList.map((file) =>file instanceof File ? file.name : file.file.name,);- const gaps = findFilesMissingModels(names);+ const gaps = findParseGaps(names);if (gaps.length > 0) {const failingFileTypes = new Set(gaps.map((gap) => gap.fileType));Modal.warning({- title: t('knowledgeDetails.uploadMissingModelsTitle'),- content: buildMissingModelModalContent(+ title: t(+ hasUnsupportedTypeGap(gaps)+ ? 'knowledgeDetails.uploadUnsupportedTypesTitle'+ : 'knowledgeDetails.uploadMissingModelsTitle',+ ),+ content: buildParserGapModalContent(t,gaps,- 'knowledgeDetails.configureInDatasetSettingHint',+ 'knowledgeDetails.reselectParserAfterUploadHint',),okText: t('knowledgeDetails.continueUpload'),- cancelText: t('knowledgeDetails.goToConfiguration'),+ cancelText: t('common.cancel'),closable: false,onOk: () => {proceedUpload(values, failingFileTypes);},- onCancel: () => {- hideDocumentUploadModal();- goToDatasetConfiguration();- },});return;}return proceedUpload(values, new Set());},- [- findFilesMissingModels,- goToDatasetConfiguration,- hideDocumentUploadModal,- proceedUpload,- t,- ],+ [findParseGaps, proceedUpload, t],);return {diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.tsindex 83832380ac..bc903f8e1a 100644--- a/web/src/pages/dataset/dataset/utils.test.ts+++ b/web/src/pages/dataset/dataset/utils.test.ts@@ -1,11 +1,18 @@-import { FileType, initialParserValues } from '@/pages/agent/constant/pipeline';-import { IngestionTaskStatus, RunningStatus } from './constant';+import { FileType } from '@/constants/file';import {- findFilesMissingParserModels,- findMissingParserModel,+ IngestionTaskStatus,+ ParserModelKind,+ ParserGapReason,+ RunningStatus,+} from './constant';+import {+ findDocumentsParserGaps,+ findFilesParserGaps,+ findParserGap,getDocumentRunningStatus,- getEffectiveParserSetups,getFileTypeByExtension,+ getSavedParserSetups,+ hasUnsupportedTypeGap,ingestionStatusToRunningStatus,isDocumentProcessing,isDocumentStopping,@@ -175,95 +182,141 @@ describe('getFileTypeByExtension', () => {});});-// Minimal structural stand-in for IDataset in these tests. It is declared-// locally (instead of importing the interface) because the esbuild-jest-// babel hoisting pipeline cannot elide imported bindings used only in type-// positions.-type DatasetWithParserConfig = { parser_config: any };--const DefaultSetups = initialParserValues.setups as Record<string, any>[];+// Saved setups shaped like a pipeline that declares pdf/image plus audio+// (model missing) and video (model configured).+const SavedSetups = [+ { fileFormat: FileType.PDF, parse_method: 'DeepDOC' },+ { fileFormat: FileType.Image, parse_method: 'ocr' },+ { fileFormat: FileType.Audio, vlm: { llm_id: '' } },+ { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },+] as Record<string, any>[];-describe('findMissingParserModel', () => {- it('passes audio when the setup carries a model', () => {- const setups = [- { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } },- ];- expect(findMissingParserModel(FileType.Audio, setups)).toBeNull();+describe('findParserGap', () => {+ it('flags a file type missing from the saved setups as unsupported', () => {+ expect(findParserGap(FileType.Docx, SavedSetups)).toEqual({+ reason: ParserGapReason.UnsupportedType,+ fileType: FileType.Docx,+ });});- it('flags audio when the setup has no model, ignoring the tenant default', () => {- expect(findMissingParserModel(FileType.Audio, DefaultSetups)).toEqual({+ it('flags audio/video without a declared family as unsupported, not missing-model', () => {+ const setups = [{ fileFormat: FileType.PDF }];+ expect(findParserGap(FileType.Audio, setups)).toEqual({+ reason: ParserGapReason.UnsupportedType,fileType: FileType.Audio,- modelKind: 'asr',+ });+ expect(findParserGap(FileType.Video, setups)).toEqual({+ reason: ParserGapReason.UnsupportedType,+ fileType: FileType.Video,});});- it('passes video when the setup carries a model', () => {- const setups = [- { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },- ];- expect(findMissingParserModel(FileType.Video, setups)).toBeNull();+ it('flags declared audio without a model as a missing asr model', () => {+ expect(findParserGap(FileType.Audio, SavedSetups)).toEqual({+ reason: ParserGapReason.MissingModel,+ fileType: FileType.Audio,+ modelKind: ParserModelKind.Asr,+ });});- it('flags video when the setup has no model', () => {- expect(findMissingParserModel(FileType.Video, DefaultSetups)).toEqual({+ it('flags declared video without a model as a missing vision model', () => {+ const setups = [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }];+ expect(findParserGap(FileType.Video, setups)).toEqual({+ reason: ParserGapReason.MissingModel,fileType: FileType.Video,- modelKind: 'vision',+ modelKind: ParserModelKind.Vision,});});- it('never flags image — ocr-only parsing needs no model', () => {- // The image parser always runs OCR; the vision model (picked as- // parse_method) is an optional supplement, so neither ocr nor a model- // id constitutes a gap.- expect(findMissingParserModel(FileType.Image, DefaultSetups)).toBeNull();- expect(- findMissingParserModel(FileType.Image, [- { fileFormat: FileType.Image, parse_method: 'gpt-4o@OpenAI' },- ]),- ).toBeNull();+ it('passes video when the setup carries a model', () => {+ expect(findParserGap(FileType.Video, SavedSetups)).toBeNull();});- it('ignores file types without model requirements', () => {- expect(findMissingParserModel(FileType.PDF, DefaultSetups)).toBeNull();- expect(findMissingParserModel(undefined, DefaultSetups)).toBeNull();+ it('passes image with an ocr-only setup — no model required', () => {+ expect(findParserGap(FileType.Image, SavedSetups)).toBeNull();+ });++ it('passes supported non-media types and unknown file types', () => {+ expect(findParserGap(FileType.PDF, SavedSetups)).toBeNull();+ expect(findParserGap(undefined, SavedSetups)).toBeNull();});});-describe('findFilesMissingParserModels', () => {- it('returns a gap per file that lacks its required model', () => {- const gaps = findFilesMissingParserModels(- ['song.mp3', 'notes.txt', 'photo.jpg'],- DefaultSetups,+describe('findFilesParserGaps', () => {+ it('returns a gap per failing file, mixing both reasons', () => {+ const gaps = findFilesParserGaps(+ ['song.mp3', 'notes.docx', 'photo.jpg', 'movie.mp4'],+ SavedSetups,);expect(gaps).toEqual([- { name: 'song.mp3', fileType: FileType.Audio, modelKind: 'asr' },+ {+ name: 'song.mp3',+ reason: ParserGapReason.MissingModel,+ fileType: FileType.Audio,+ modelKind: ParserModelKind.Asr,+ },+ {+ name: 'notes.docx',+ reason: ParserGapReason.UnsupportedType,+ fileType: FileType.Docx,+ },]);});});-describe('getEffectiveParserSetups', () => {- it('falls back to the default setups without a saved parser config', () => {- const setups = getEffectiveParserSetups(null);- expect(setups.map((x) => x.fileFormat)).toEqual(- initialParserValues.setups.map((x) => x.fileFormat),- );+describe('hasUnsupportedTypeGap', () => {+ it('detects unsupported-type gaps among mixed gaps', () => {+ expect(+ hasUnsupportedTypeGap([+ {+ reason: ParserGapReason.MissingModel,+ fileType: FileType.Audio,+ modelKind: ParserModelKind.Asr,+ },+ { reason: ParserGapReason.UnsupportedType, fileType: FileType.Image },+ ]),+ ).toBe(true);+ expect(+ hasUnsupportedTypeGap([+ {+ reason: ParserGapReason.MissingModel,+ fileType: FileType.Audio,+ modelKind: ParserModelKind.Asr,+ },+ ]),+ ).toBe(false);+ expect(hasUnsupportedTypeGap([])).toBe(false);+ });+});++describe('getSavedParserSetups', () => {+ it('returns null without a saved parser config', () => {+ expect(getSavedParserSetups(null)).toBeNull();+ expect(+ getSavedParserSetups({+ parser_config: {},+ }),+ ).toBeNull();+ });++ it('returns null for legacy flat-shape configs without a Parser entry', () => {+ const knowledgeBase = {+ parser_config: { chunk_token_num: 128, delimiter: '\n' },+ };+ expect(getSavedParserSetups(knowledgeBase)).toBeNull();});- it('applies saved overrides keyed by file format', () => {+ it('returns only the saved Parser setups, without merging defaults', () => {const knowledgeBase = {parser_config: {'Parser:1': {- audio: { vlm: { llm_id: 'whisper@OpenAI' } },+ pdf: { parse_method: 'DeepDOC' },+ image: { parse_method: 'ocr' },},},- } as unknown as DatasetWithParserConfig;- const setups = getEffectiveParserSetups(knowledgeBase);- const audio = setups.find((x) => x.fileFormat === FileType.Audio);- expect(audio?.vlm?.llm_id).toBe('whisper@OpenAI');- // Untouched types keep their defaults- const image = setups.find((x) => x.fileFormat === FileType.Image);- expect(image?.parse_method).toBe('ocr');+ };+ const setups = getSavedParserSetups(knowledgeBase);+ expect(setups?.map((x) => x.fileFormat).sort()).toEqual(['image', 'pdf']);});it('ignores non-parser operator entries', () => {@@ -271,8 +324,53 @@ describe('getEffectiveParserSetups', () => {parser_config: {'TokenChunker:1': { chunk_token_size: 128 },},- } as unknown as DatasetWithParserConfig;- const setups = getEffectiveParserSetups(knowledgeBase);- expect(setups).toHaveLength(initialParserValues.setups.length);+ };+ expect(getSavedParserSetups(knowledgeBase)).toBeNull();+ });+});++describe('findDocumentsParserGaps', () => {+ // A document overridden to a general-like parser: its row parser_config+ // declares pdf even though the dataset-level fallback does not.+ const OverriddenDoc = {+ name: 'civil-code.pdf',+ parser_config: {+ 'Parser:1': { pdf: { parse_method: 'DeepDOC' } },+ },+ };++ it('validates against the document row config when present', () => {+ expect(findDocumentsParserGaps([OverriddenDoc], null)).toEqual([]);+ });++ it('falls back to the dataset-level setups for rows without a Parser entry', () => {+ const legacyRow = { name: 'song.mp3', parser_config: {} };+ expect(findDocumentsParserGaps([legacyRow], SavedSetups)).toEqual([+ {+ name: 'song.mp3',+ reason: ParserGapReason.MissingModel,+ fileType: FileType.Audio,+ modelKind: ParserModelKind.Asr,+ },+ ]);+ // No fallback either: nothing can be determined, so nothing is flagged.+ expect(findDocumentsParserGaps([legacyRow], null)).toEqual([]);+ });++ it('evaluates each document against its own config in a mixed batch', () => {+ const inheritedRow = { name: 'photo.jpg', parser_config: undefined };+ const gaps = findDocumentsParserGaps(+ [OverriddenDoc, inheritedRow],+ // Dataset-level config declares only audio: the overridden pdf passes+ // while the inherited jpg is flagged.+ [{ fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } }],+ );+ expect(gaps).toEqual([+ {+ name: 'photo.jpg',+ reason: ParserGapReason.UnsupportedType,+ fileType: FileType.Image,+ },+ ]);});});diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.tsindex 89334692af..86b0aac712 100644--- a/web/src/pages/dataset/dataset/utils.ts+++ b/web/src/pages/dataset/dataset/utils.ts@@ -1,19 +1,18 @@import { Operator } from '@/constants/agent';-import type { IDataset } from '@/interfaces/database/dataset';+import { FileType, FileTypeSuffixMap } from '@/constants/file';import type { IDocumentInfo } from '@/interfaces/database/document';-import {- FileType,- FileTypeSuffixMap,- initialParserValues,-} from '@/pages/agent/constant/pipeline';import { pickByBackend } from '@/utils/backend-variant';import { getExtension } from '@/utils/document-util';import {getOperatorType,transformParserConfigSetups,} from '@/utils/pipeline-operator';-import { cloneDeep } from 'lodash';-import { IngestionTaskStatus, RunningStatus } from './constant';+import {+ IngestionTaskStatus,+ ParserGapReason,+ ParserModelKind,+ RunningStatus,+} from './constant';/** Ingestion statuses that represent an active or canceling parse task on Go. */const activeIngestionStatuses = new Set<IngestionTaskStatus>([@@ -158,22 +157,34 @@ export const isDocumentProcessing = (python: isPythonDocumentProcessing(document),});-// --- Parser model prerequisite checks --------------------------------------// Audio/video files can only be parsed when the matching model is configured-// on the dataset's Parser operator. The tenant default is deliberately not-// consulted: parsing reads the operator setup, so a global default does not-// make the file parsable. Image files are exempt: the image parser always-// runs OCR and only supplements it with the vision model (picked as the-// image parse_method, with the tenant default as fallback), so a static-// method like ocr is a complete configuration.+// --- Parser prerequisite checks -------------------------------------------+// A file cannot be parsed under the dataset's current Parser operator when+// the operator does not declare the file's type family at all+// (unsupportedType), or — for audio/video — when the declared setup carries+// no model (missingModel). The tenant default is deliberately not consulted:+// parsing reads the operator setup, so a global default does not make the+// file parsable. Image files only need their family declared: the image+// parser always runs OCR and merely supplements it with the vision model+// (picked as the image parse_method, with the tenant default as fallback).// These helpers power the upload warning and the parse-click validation.export type ParserModelGap = {+ reason: ParserGapReason.MissingModel;fileType: FileType;- modelKind: 'asr' | 'vision';+ modelKind: ParserModelKind.Asr | ParserModelKind.Vision;};-export type FileModelGap = ParserModelGap & { name: string };+export type UnsupportedTypeGap = {+ reason: ParserGapReason.UnsupportedType;+ fileType: FileType;+};++export type ParserGap = ParserModelGap | UnsupportedTypeGap;++export type FileParserGap = ParserGap & { name: string };++export const hasUnsupportedTypeGap = (gaps: ParserGap[]) =>+ gaps.some((gap) => gap.reason === ParserGapReason.UnsupportedType);type ParserSetup = Record<string, any> & { fileFormat?: string };@@ -193,65 +204,89 @@ export function getFileTypeByExtension(}/**- * Effective parser setups for a dataset: the saved Parser operator entry- * (stored as a flattened map keyed by file format) merged over the default- * setups, so untouched file types validate against their defaults.+ * Parser setups exactly as saved, keyed by file format — the keys are the+ * type families the pipeline/built-in method declares. Accepts either a+ * dataset or a document row: a document's parser_config snapshots the parser+ * it actually runs with (dataset config at upload, the override after a+ * parser change). Returns null when parser_config has no Parser operator+ * entry (legacy flat-shape configs, pipelines without a Parser stage): type+ * support cannot be determined then and validation is skipped.*/-export function getEffectiveParserSetups(- knowledgeBase: Pick<IDataset, 'parser_config'> | null | undefined,-): ParserSetup[] {- const parserConfig = knowledgeBase?.parser_config as- | Record<string, any>- | undefined;+export function getSavedParserSetups(+ source: { parser_config?: unknown } | null | undefined,+): ParserSetup[] | null {+ const parserConfig = source?.parser_config as Record<string, any> | undefined;const parserEntry = Object.entries(parserConfig ?? {}).find(([operatorId]) => getOperatorType(operatorId) === Operator.Parser,)?.[1];- const savedSetups = transformParserConfigSetups(parserEntry);- const savedByFileType = new Map(- savedSetups.map((setup) => [setup.fileFormat, setup]),- );-- const defaultSetups = cloneDeep(initialParserValues.setups) as ParserSetup[];- const merged = defaultSetups.map(- (setup) => savedByFileType.get(setup.fileFormat) ?? setup,- );- for (const setup of savedSetups) {- if (!defaultSetups.some((x) => x.fileFormat === setup.fileFormat)) {- merged.push(setup);- }+ if (!parserEntry) {+ return null;}- return merged;+ return transformParserConfigSetups(parserEntry);}-export function findMissingParserModel(+export function findParserGap(fileType: FileType | undefined,setups: ParserSetup[],-): ParserModelGap | null {+): ParserGap | null {+ if (!fileType) {+ return null;+ }+const setup = setups.find((x) => x.fileFormat === fileType);+ if (!setup) {+ return { reason: ParserGapReason.UnsupportedType, fileType };+ }switch (fileType) {case FileType.Audio: {- const configured = setup?.vlm?.llm_id;- return configured ? null : { fileType, modelKind: 'asr' };+ return setup.vlm?.llm_id+ ? null+ : {+ reason: ParserGapReason.MissingModel,+ fileType,+ modelKind: ParserModelKind.Asr,+ };}case FileType.Video: {- const configured = setup?.vlm?.llm_id;- return configured ? null : { fileType, modelKind: 'vision' };+ return setup.vlm?.llm_id+ ? null+ : {+ reason: ParserGapReason.MissingModel,+ fileType,+ modelKind: ParserModelKind.Vision,+ };}default:return null;}}-export function findFilesMissingParserModels(+export function findFilesParserGaps(names: string[],setups: ParserSetup[],-): FileModelGap[] {+): FileParserGap[] {return names.flatMap((name) => {const fileType = getFileTypeByExtension(getExtension(name));- const gap = findMissingParserModel(fileType, setups);+ const gap = findParserGap(fileType, setups);return gap ? [{ ...gap, name }] : [];});}++/**+ * Per-document parse validation: each document row carries the parser config+ * it actually runs with (dataset snapshot at upload, the override after a+ * parser change), so it is validated against its own setups. Rows without a+ * Parser entry fall back to `fallbackSetups` (the dataset-level config).+ */+export function findDocumentsParserGaps(+ documents: Array<{ name: string; parser_config?: unknown }>,+ fallbackSetups: ParserSetup[] | null,+): FileParserGap[] {+ return documents.flatMap((doc) => {+ const setups = getSavedParserSetups(doc) ?? fallbackSetups;+ return setups ? findFilesParserGaps([doc.name], setups) : [];+ });+}