Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: Dataset file upload: For unsupported file types, prompt the user to configure the corresponding file parser. (#19710)

balibabu4d24efec2827
diff --git a/web/src/constants/file.ts b/web/src/constants/file.ts
index 2814093051..cf32404d4c 100644
--- a/web/src/constants/file.ts
+++ b/web/src/constants/file.ts
@@ -14,6 +14,67 @@
* limitations under the License.
*/
+export enum FileType {
+ PDF = 'pdf',
+ Spreadsheet = 'spreadsheet',
+ Image = 'image',
+ Email = 'email',
+ TextMarkdown = 'markdown',
+ Code = 'text&code',
+ Html = 'html',
+ Doc = 'doc',
+ Docx = 'docx',
+ PowerPoint = 'slides',
+ Video = 'video',
+ Audio = 'audio',
+}
+
+export const FileTypeSuffixMap = {
+ [FileType.PDF]: ['pdf'],
+ [FileType.Spreadsheet]: ['xls', 'xlsx', 'csv'],
+ [FileType.Image]: ['jpg', 'jpeg', 'png', 'gif'],
+ [FileType.Email]: ['eml', 'msg'],
+ [FileType.TextMarkdown]: ['md', 'markdown', 'mdx'],
+ [FileType.Code]: [
+ 'txt',
+ 'py',
+ 'js',
+ 'java',
+ 'c',
+ 'cpp',
+ 'h',
+ 'php',
+ 'go',
+ 'ts',
+ 'sh',
+ 'cs',
+ 'kt',
+ 'sql',
+ ],
+ [FileType.Html]: ['htm', 'html'],
+ [FileType.Doc]: ['doc'],
+ [FileType.Docx]: ['docx'],
+ [FileType.PowerPoint]: ['pptx', 'ppt'],
+ [FileType.Video]: ['mp4', 'avi', 'mkv'],
+ [FileType.Audio]: [
+ 'da',
+ 'wave',
+ 'wav',
+ 'mp3',
+ 'aac',
+ 'flac',
+ 'ogg',
+ 'aiff',
+ 'au',
+ 'midi',
+ 'wma',
+ 'realaudio',
+ 'vqf',
+ 'oggvorbis',
+ 'ape',
+ ],
+};
+
export const FileIconMap = {
doc: 'doc',
docx: 'doc',
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index df922c0058..5930e0cf54 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -397,18 +397,21 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
},
knowledgeDetails: {
continueUpload: 'Continue upload',
- goToConfiguration: 'Go to configuration',
+ reselectParser: 'Reselect parse method',
uploadMissingModelsTitle: 'Some files lack a required model',
fileModelMissing:
'{{name}} ({{fileType}}) requires a configured {{model}} model',
missingModelAsr: 'audio (ASR)',
missingModelVision: 'vision',
- configureInDatasetSettingHint:
- 'You can configure it in the dataset Configuration page (Parser operator), or continue anyway.',
+ uploadUnsupportedTypesTitle: 'Some file types cannot be parsed',
+ fileTypeUnsupported:
+ '{{name}} ({{fileType}}): not supported by the current parser configuration',
+ reselectParserAfterUploadHint:
+ 'Continue uploading, then reselect a parse method for these files in the file list.',
+ reselectParserToParseHint:
+ 'Reselect a parse method for the affected files, then parse again.',
parseBlockedTitle: 'Cannot parse',
parseBlockedPartialTitle: 'Some files cannot be parsed',
- parseBlockedHint:
- 'Configure the model in the dataset Configuration page (Parser operator), then try again.',
parseValidFiles: 'Parse valid files',
parseValidFilesNote: '{{count}} valid files will be parsed.',
metadata: {
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 49fcb9d01c..c13d3b0423 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -356,17 +356,19 @@ export default {
},
knowledgeDetails: {
continueUpload: '继续上传',
- goToConfiguration: '前往配置',
+ reselectParser: '重新选择解析方法',
uploadMissingModelsTitle: '部分文件缺少解析所需的模型',
fileModelMissing: '{{name}}({{fileType}})需要配置{{model}}模型',
missingModelAsr: '音频(ASR)',
missingModelVision: '视觉',
- configureInDatasetSettingHint:
- '可在数据集「配置」页的 Parser 算子中设置对应模型,或继续操作。',
+ uploadUnsupportedTypesTitle: '部分文件类型不支持解析',
+ fileTypeUnsupported:
+ '{{name}}({{fileType}}):当前解析配置不支持该文件类型',
+ reselectParserAfterUploadHint:
+ '可继续上传,上传后在文件列表中重新选择这些文件的解析方法。',
+ reselectParserToParseHint: '请重新选择对应文件的解析方法后再解析。',
parseBlockedTitle: '无法解析',
parseBlockedPartialTitle: '部分文件无法解析',
- parseBlockedHint:
- '请先在数据集「配置」页的 Parser 算子中设置对应模型,再重新解析。',
parseValidFiles: '解析有效文件',
parseValidFilesNote: '将解析其余 {{count}} 个有效文件。',
metadata: {
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index d6ee82567d..131a7cdc38 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -1,24 +1,10 @@
import { ParseDocumentType } from '@/components/layout-recognize-form-field';
import { initialLlmBaseValues, Operator } from '@/constants/agent';
+import { FileType } from '@/constants/file';
import { ModelTypeToField } from '@/constants/llm';
import { pickByBackend } from '@/utils/backend-variant';
import { cloneDeep } from 'lodash';
-export enum FileType {
- PDF = 'pdf',
- Spreadsheet = 'spreadsheet',
- Image = 'image',
- Email = 'email',
- TextMarkdown = 'markdown',
- Code = 'text&code',
- Html = 'html',
- Doc = 'doc',
- Docx = 'docx',
- PowerPoint = 'slides',
- Video = 'video',
- Audio = 'audio',
-}
-
export enum PdfOutputFormat {
Json = 'json',
Markdown = 'markdown',
@@ -447,52 +433,6 @@ export const initialCompilationValues = {
export const NoDebugOperatorsList = [Operator.File];
-export const FileTypeSuffixMap = {
- [FileType.PDF]: ['pdf'],
- [FileType.Spreadsheet]: ['xls', 'xlsx', 'csv'],
- [FileType.Image]: ['jpg', 'jpeg', 'png', 'gif'],
- [FileType.Email]: ['eml', 'msg'],
- [FileType.TextMarkdown]: ['md', 'markdown', 'mdx'],
- [FileType.Code]: [
- 'txt',
- 'py',
- 'js',
- 'java',
- 'c',
- 'cpp',
- 'h',
- 'php',
- 'go',
- 'ts',
- 'sh',
- 'cs',
- 'kt',
- 'sql',
- ],
- [FileType.Html]: ['htm', 'html'],
- [FileType.Doc]: ['doc'],
- [FileType.Docx]: ['docx'],
- [FileType.PowerPoint]: ['pptx', 'ppt'],
- [FileType.Video]: ['mp4', 'avi', 'mkv'],
- [FileType.Audio]: [
- 'da',
- 'wave',
- 'wav',
- 'mp3',
- 'aac',
- 'flac',
- 'ogg',
- 'aiff',
- 'au',
- 'midi',
- 'wma',
- 'realaudio',
- 'vqf',
- 'oggvorbis',
- 'ape',
- ],
-};
-
export const SingleOperators = [
Operator.Tokenizer,
Operator.GeneralChunker,
diff --git a/web/src/pages/agent/form/parser-form/common-form-fields.tsx b/web/src/pages/agent/form/parser-form/common-form-fields.tsx
index 2d0a1fd145..23f343a65d 100644
--- a/web/src/pages/agent/form/parser-form/common-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/common-form-fields.tsx
@@ -6,11 +6,11 @@ import {
} from '@/components/originui/select-with-search';
import { RAGFlowFormItem } from '@/components/ragflow-form';
import { Switch } from '@/components/ui/switch';
+import { FileType } from '@/constants/file';
import { upperCase, upperFirst } from 'lodash';
import { useTranslation } from 'react-i18next';
import { useOwnerTenantId } from '../../context';
import {
- FileType,
OutputFormatMap,
SpreadsheetOutputFormat,
} from '../../constant/pipeline';
diff --git a/web/src/pages/agent/form/parser-form/index.tsx b/web/src/pages/agent/form/parser-form/index.tsx
index 95f395d47f..835a3b3f30 100644
--- a/web/src/pages/agent/form/parser-form/index.tsx
+++ b/web/src/pages/agent/form/parser-form/index.tsx
@@ -9,6 +9,7 @@ import {
DropdownMenuTrigger,
} from '@/components/ui/dropdown-menu';
import { Form } from '@/components/ui/form';
+import { FileType } from '@/constants/file';
import { useFetchDefaultModelDictionary } from '@/hooks/use-llm-request';
import { zodResolver } from '@hookform/resolvers/zod';
import { LucideTrash2 } from 'lucide-react';
@@ -16,7 +17,7 @@ import { memo, useCallback, useMemo } from 'react';
import { useFieldArray, useForm, useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
-import { FileType, initialParserValues } from '../../constant/pipeline';
+import { initialParserValues } from '../../constant/pipeline';
import { useFormChangeCallback } from '../../hooks/use-form-change-callback';
import { useFormValues } from '../../hooks/use-form-values';
import { useWatchFormChange } from '../../hooks/use-watch-form-change';
diff --git a/web/src/pages/agent/form/parser-form/schema.test.ts b/web/src/pages/agent/form/parser-form/schema.test.ts
index 6e5001283e..71add82be3 100644
--- a/web/src/pages/agent/form/parser-form/schema.test.ts
+++ b/web/src/pages/agent/form/parser-form/schema.test.ts
@@ -1,4 +1,5 @@
-import { FileType, ParserFields } from '../../constant/pipeline';
+import { FileType } from '@/constants/file';
+import { ParserFields } from '../../constant/pipeline';
import { FormSchema } from './schema';
describe('parser FormSchema', () => {
diff --git a/web/src/pages/agent/form/parser-form/schema.ts b/web/src/pages/agent/form/parser-form/schema.ts
index 8dc4163514..0af3f8dae0 100644
--- a/web/src/pages/agent/form/parser-form/schema.ts
+++ b/web/src/pages/agent/form/parser-form/schema.ts
@@ -1,6 +1,6 @@
+import { FileType } from '@/constants/file';
import i18n from '@/locales/config';
import { z } from 'zod';
-import { FileType } from '../../constant/pipeline';
export const SetupSchema = z
.object({
diff --git a/web/src/pages/agent/form/parser-form/utils.test.ts b/web/src/pages/agent/form/parser-form/utils.test.ts
index 984036b62e..764ec77573 100644
--- a/web/src/pages/agent/form/parser-form/utils.test.ts
+++ b/web/src/pages/agent/form/parser-form/utils.test.ts
@@ -1,5 +1,6 @@
+import { FileType } from '@/constants/file';
import { ModelTypeToField } from '@/constants/llm';
-import { FileType, initialParserValues } from '../../constant/pipeline';
+import { initialParserValues } from '../../constant/pipeline';
import { buildInitialParserSetup, buildInitialParserValues } from './utils';
describe('parser-form utils', () => {
diff --git a/web/src/pages/agent/form/parser-form/utils.ts b/web/src/pages/agent/form/parser-form/utils.ts
index b1aa6bb4d3..05d35248d3 100644
--- a/web/src/pages/agent/form/parser-form/utils.ts
+++ b/web/src/pages/agent/form/parser-form/utils.ts
@@ -1,6 +1,6 @@
+import { FileType } from '@/constants/file';
import { cloneDeep } from 'lodash';
import {
- FileType,
FileTypeDefaultModelFieldMap,
initialParserValues,
} from '../../constant/pipeline';
diff --git a/web/src/pages/agent/utils.ts b/web/src/pages/agent/utils.ts
index 77ce72caef..18b8b1b4e8 100644
--- a/web/src/pages/agent/utils.ts
+++ b/web/src/pages/agent/utils.ts
@@ -1,3 +1,4 @@
+import { FileType, FileTypeSuffixMap } from '@/constants/file';
import {
DSL,
DSLComponents,
@@ -28,8 +29,6 @@ import isObject from 'lodash/isObject';
import {
AgentDialogueMode,
CategorizeAnchorPointPositions,
- FileType,
- FileTypeSuffixMap,
InputMode,
NoCopyOperatorsList,
NoDebugOperatorsList,
diff --git a/web/src/pages/dataset/dataset/constant.ts b/web/src/pages/dataset/dataset/constant.ts
index 96bebd79b3..5516e9b8d5 100644
--- a/web/src/pages/dataset/dataset/constant.ts
+++ b/web/src/pages/dataset/dataset/constant.ts
@@ -31,3 +31,21 @@ export const RunningStatusMap = {
};
export * from '@/constants/knowledge';
+
+/** Why a file cannot be parsed under the dataset's current Parser operator. */
+export enum ParserGapReason {
+ // The operator setups declare no entry for the file's type family at all.
+ UnsupportedType = 'unsupportedType',
+ // The family is declared, but its required model (audio/video) is not set.
+ MissingModel = 'missingModel',
+}
+
+/**
+ * Model capability a Parser operator setup needs for audio/video files.
+ * Values follow the parser-configuration vocabulary (the ModelTypeToField
+ * keys), not the LLM registry types (LlmModelType).
+ */
+export enum ParserModelKind {
+ Asr = 'asr',
+ Vision = 'vision',
+}
diff --git a/web/src/pages/dataset/dataset/parser-gap-content.tsx b/web/src/pages/dataset/dataset/parser-gap-content.tsx
new file mode 100644
index 0000000000..705ab92ccc
--- /dev/null
+++ b/web/src/pages/dataset/dataset/parser-gap-content.tsx
@@ -0,0 +1,41 @@
+import type { TFunction } from 'i18next';
+import type { ReactNode } from 'react';
+import { ParserGapReason, ParserModelKind } from './constant';
+import type { FileParserGap } from './utils';
+
+/**
+ * Builds the body of the parser-gap modals (upload warning, parse-click
+ * error). Strings are resolved with the caller's `t` because the static Modal
+ * API renders content in a separate React root without the app providers.
+ */
+export function buildParserGapModalContent(
+ t: TFunction,
+ gaps: FileParserGap[],
+ hintKey: string,
+): ReactNode {
+ return (
+ <div className="space-y-2">
+ <ul className="list-disc pl-4 space-y-1">
+ {gaps.map((gap, index) => (
+ <li key={`${gap.name}-${index}`}>
+ {gap.reason === ParserGapReason.MissingModel
+ ? t('knowledgeDetails.fileModelMissing', {
+ name: gap.name,
+ fileType: t(`flow.fileFormatOptions.${gap.fileType}`),
+ model: t(
+ gap.modelKind === ParserModelKind.Asr
+ ? 'knowledgeDetails.missingModelAsr'
+ : 'knowledgeDetails.missingModelVision',
+ ),
+ })
+ : t('knowledgeDetails.fileTypeUnsupported', {
+ name: gap.name,
+ fileType: t(`flow.fileFormatOptions.${gap.fileType}`),
+ })}
+ </li>
+ ))}
+ </ul>
+ <p className="text-text-secondary">{t(hintKey)}</p>
+ </div>
+ );
+}
diff --git a/web/src/pages/dataset/dataset/parser-model-gap-content.tsx b/web/src/pages/dataset/dataset/parser-model-gap-content.tsx
deleted file mode 100644
index 317032b1b1..0000000000
--- a/web/src/pages/dataset/dataset/parser-model-gap-content.tsx
+++ /dev/null
@@ -1,35 +0,0 @@
-import type { TFunction } from 'i18next';
-import type { ReactNode } from 'react';
-import type { FileModelGap } from './utils';
-
-/**
- * Builds the body of the missing-model modals (upload warning, parse-click
- * error). Strings are resolved with the caller's `t` because the static Modal
- * API renders content in a separate React root without the app providers.
- */
-export function buildMissingModelModalContent(
- t: TFunction,
- gaps: FileModelGap[],
- hintKey: string,
-): ReactNode {
- return (
- <div className="space-y-2">
- <ul className="list-disc pl-4 space-y-1">
- {gaps.map((gap, index) => (
- <li key={`${gap.name}-${index}`}>
- {t('knowledgeDetails.fileModelMissing', {
- name: gap.name,
- fileType: t(`flow.fileFormatOptions.${gap.fileType}`),
- model: t(
- gap.modelKind === 'asr'
- ? 'knowledgeDetails.missingModelAsr'
- : 'knowledgeDetails.missingModelVision',
- ),
- })}
- </li>
- ))}
- </ul>
- <p className="text-text-secondary">{t(hintKey)}</p>
- </div>
- );
-}
diff --git a/web/src/pages/dataset/dataset/parsing-status-cell.tsx b/web/src/pages/dataset/dataset/parsing-status-cell.tsx
index 3c3eb15b80..7ba4d90efe 100644
--- a/web/src/pages/dataset/dataset/parsing-status-cell.tsx
+++ b/web/src/pages/dataset/dataset/parsing-status-cell.tsx
@@ -116,6 +116,7 @@ export function ParseDropdownButton({
export function ParsingStatusCell({
record,
showLog,
+ showChangeParserModal,
}: {
record: IDocumentInfo;
showLog: (record: IDocumentInfo) => void;
@@ -133,7 +134,7 @@ export function ParsingStatusCell({
visible: reparseDialogVisible,
showModal: showReparseDialogModal,
hideModal: hideReparseDialogModal,
- } = useHandleRunDocumentByIds(id);
+ } = useHandleRunDocumentByIds(id, showChangeParserModal);
const isGo = useIsGoBackend();
const isRunning = isDocumentProcessing(record);
const isQueued = effectiveRun === RunningStatus.QUEUED;
diff --git a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
index 69c324802d..c5b8d49b9d 100644
--- a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
+++ b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
@@ -24,8 +24,8 @@ import { useParams } from 'react-router';
import { toast } from 'sonner';
import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';
import { DocumentType } from './constant';
-import { buildMissingModelModalContent } from './parser-model-gap-content';
-import { useParserModelValidation } from './use-parser-model-validation';
+import { buildParserGapModalContent } from './parser-gap-content';
+import { useParserGapValidation } from './use-parser-gap-validation';
import { isDocumentProcessing } from './utils';
export function useBulkOperateDataset({
@@ -46,8 +46,7 @@ export function useBulkOperateDataset({
const { setDocumentStatus } = useSetDocumentStatus();
const { removeDocument } = useRemoveDocument();
const { visible, showModal, hideModal } = useSetModalState();
- const { findFilesMissingModels, goToDatasetConfiguration } =
- useParserModelValidation();
+ const { findDocumentParseGaps } = useParserGapValidation();
const { knowledgeBase } = useKnowledgeBaseContext();
const chunkNum = useMemo(() => {
@@ -73,15 +72,14 @@ export function useBulkOperateDataset({
return;
}
- // Starting a parse requires the models of each file type to be
- // configured; cancelling is always allowed.
+ // Starting a parse requires each file type to be supported by the
+ // Parser operator its document actually runs with; cancelling is
+ // always allowed.
if (run === 1) {
const selectedDocuments = documents.filter((x) =>
nonVirtualKeys.includes(x.id),
);
- const gaps = findFilesMissingModels(
- selectedDocuments.map((x) => x.name),
- );
+ const gaps = findDocumentParseGaps(selectedDocuments);
if (gaps.length > 0) {
hideModal();
const failingNames = new Set(gaps.map((gap) => gap.name));
@@ -92,15 +90,14 @@ export function useBulkOperateDataset({
if (validIds.length === 0) {
Modal.error({
title: t('knowledgeDetails.parseBlockedTitle'),
- content: buildMissingModelModalContent(
+ content: buildParserGapModalContent(
t,
gaps,
- 'knowledgeDetails.parseBlockedHint',
+ 'knowledgeDetails.reselectParserToParseHint',
),
- okText: t('knowledgeDetails.goToConfiguration'),
- cancelText: t('common.cancel'),
+ showCancel: false,
+ okText: t('common.cancel'),
closable: false,
- onOk: goToDatasetConfiguration,
});
return;
}
@@ -109,10 +106,10 @@ export function useBulkOperateDataset({
title: t('knowledgeDetails.parseBlockedPartialTitle'),
content: (
<div className="space-y-2">
- {buildMissingModelModalContent(
+ {buildParserGapModalContent(
t,
gaps,
- 'knowledgeDetails.parseBlockedHint',
+ 'knowledgeDetails.reselectParserToParseHint',
)}
<p>
{t('knowledgeDetails.parseValidFilesNote', {
@@ -145,8 +142,7 @@ export function useBulkOperateDataset({
selectedRowKeys,
hideModal,
t,
- findFilesMissingModels,
- goToDatasetConfiguration,
+ findDocumentParseGaps,
],
);
diff --git a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
new file mode 100644
index 0000000000..57263f56fa
--- /dev/null
+++ b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
@@ -0,0 +1,60 @@
+import type { IDocumentInfo } from '@/interfaces/database/document';
+import { useIsGoBackend } from '@/utils/backend-variant';
+import { useCallback } from 'react';
+import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';
+import {
+ FileParserGap,
+ findDocumentsParserGaps,
+ findFilesParserGaps,
+ getSavedParserSetups,
+} from './utils';
+
+/**
+ * Validates that each file can be parsed under the Parser operator config it
+ * actually runs with: the file type must be declared in the operator setups,
+ * and audio/video additionally need their model configured.
+ * Only the Go backend has parser operator settings — on the Python backend
+ * every file passes.
+ */
+export function useParserGapValidation() {
+ const isGoBackend = useIsGoBackend();
+ const { knowledgeBase, loading } = useKnowledgeBaseContext();
+
+ // Skip while the dataset is loading: its parser_config is not readable
+ // yet and every file would look unsupported. A missing Parser entry means
+ // the config predates operator-scoped settings, so type support cannot be
+ // determined either.
+ const findParseGaps = useCallback(
+ (names: string[]): FileParserGap[] => {
+ if (!isGoBackend || loading) {
+ return [];
+ }
+ const setups = getSavedParserSetups(knowledgeBase);
+ if (!setups) {
+ return [];
+ }
+ return findFilesParserGaps(names, setups);
+ },
+ [isGoBackend, loading, knowledgeBase],
+ );
+
+ // Parse-time validation: a document may override the dataset parser, and
+ // its row carries the config it actually runs with — validate against that,
+ // falling back to the dataset-level config for rows without one.
+ const findDocumentParseGaps = useCallback(
+ (
+ documents: Pick<IDocumentInfo, 'name' | 'parser_config'>[],
+ ): FileParserGap[] => {
+ if (!isGoBackend || loading) {
+ return [];
+ }
+ return findDocumentsParserGaps(
+ documents,
+ getSavedParserSetups(knowledgeBase),
+ );
+ },
+ [isGoBackend, loading, knowledgeBase],
+ );
+
+ return { findParseGaps, findDocumentParseGaps };
+}
diff --git a/web/src/pages/dataset/dataset/use-parser-model-validation.ts b/web/src/pages/dataset/dataset/use-parser-model-validation.ts
deleted file mode 100644
index d06613ea21..0000000000
--- a/web/src/pages/dataset/dataset/use-parser-model-validation.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
-import { useIsGoBackend } from '@/utils/backend-variant';
-import { useCallback } from 'react';
-import { useParams } from 'react-router';
-import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';
-import {
- FileModelGap,
- findFilesMissingParserModels,
- getEffectiveParserSetups,
-} from './utils';
-
-/**
- * Validates that the models required to parse a file type (audio/video/image)
- * are configured on the dataset's Parser operator.
- * Only the Go backend has parser operator settings — on the Python backend
- * every file passes.
- */
-export function useParserModelValidation() {
- const isGoBackend = useIsGoBackend();
- const { knowledgeBase, loading } = useKnowledgeBaseContext();
- const { id } = useParams();
- const { navigateToDatasetSetting } = useNavigatePage();
-
- const findFilesMissingModels = useCallback(
- (names: string[]): FileModelGap[] => {
- // Skip while the dataset is loading: its parser_config is not readable
- // yet and every media file would look unconfigured.
- if (!isGoBackend || loading) {
- return [];
- }
- return findFilesMissingParserModels(
- names,
- getEffectiveParserSetups(knowledgeBase),
- );
- },
- [isGoBackend, loading, knowledgeBase],
- );
-
- const goToDatasetConfiguration = useCallback(() => {
- if (id) {
- navigateToDatasetSetting(id);
- }
- }, [id, navigateToDatasetSetting]);
-
- return { findFilesMissingModels, goToDatasetConfiguration };
-}
diff --git a/web/src/pages/dataset/dataset/use-run-document.ts b/web/src/pages/dataset/dataset/use-run-document.ts
index cd48016eb8..290ab157a9 100644
--- a/web/src/pages/dataset/dataset/use-run-document.ts
+++ b/web/src/pages/dataset/dataset/use-run-document.ts
@@ -4,17 +4,20 @@ import { useRunDocument } from '@/hooks/use-document-request';
import { IDocumentInfo } from '@/interfaces/database/document';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
-import { buildMissingModelModalContent } from './parser-model-gap-content';
-import { useParserModelValidation } from './use-parser-model-validation';
+import { buildParserGapModalContent } from './parser-gap-content';
+import { UseChangeDocumentParserShowType } from './use-change-document-parser';
+import { useParserGapValidation } from './use-parser-gap-validation';
-export const useHandleRunDocumentByIds = (id: string) => {
+export const useHandleRunDocumentByIds = (
+ id: string,
+ showChangeParserModal: UseChangeDocumentParserShowType['showChangeParserModal'],
+) => {
const { t } = useTranslation();
const { runDocumentByIds, loading } = useRunDocument();
const [currentId, setCurrentId] = useState<string>('');
const isLoading = loading && currentId !== '' && currentId === id;
const { visible, showModal, hideModal } = useSetModalState();
- const { findFilesMissingModels, goToDatasetConfiguration } =
- useParserModelValidation();
+ const { findDocumentParseGaps } = useParserGapValidation();
const handleRunDocumentByIds = async (
record: IDocumentInfo,
@@ -24,23 +27,26 @@ export const useHandleRunDocumentByIds = (id: string) => {
if (isLoading) {
return;
}
- // Starting a parse requires the models of the file type to be configured;
- // cancelling is always allowed.
+ // Starting a parse requires the file type to be supported by the
+ // Parser operator the document actually runs with; cancelling is
+ // always allowed.
if (!isRunning) {
- const gaps = findFilesMissingModels([record.name]);
+ const gaps = findDocumentParseGaps([record]);
if (gaps.length > 0) {
hideModal();
Modal.error({
title: t('knowledgeDetails.parseBlockedTitle'),
- content: buildMissingModelModalContent(
+ content: buildParserGapModalContent(
t,
gaps,
- 'knowledgeDetails.parseBlockedHint',
+ 'knowledgeDetails.reselectParserToParseHint',
),
- okText: t('knowledgeDetails.goToConfiguration'),
+ okText: t('knowledgeDetails.reselectParser'),
cancelText: t('common.cancel'),
closable: false,
- onOk: goToDatasetConfiguration,
+ onOk: () => {
+ showChangeParserModal(record);
+ },
});
return;
}
diff --git a/web/src/pages/dataset/dataset/use-upload-document.ts b/web/src/pages/dataset/dataset/use-upload-document.ts
index 81b7ef3c92..86a8d0568b 100644
--- a/web/src/pages/dataset/dataset/use-upload-document.ts
+++ b/web/src/pages/dataset/dataset/use-upload-document.ts
@@ -5,14 +5,14 @@ import {
useRunDocument,
useUploadDocument,
} from '@/hooks/use-document-request';
+import { FileType } from '@/constants/file';
import { IDocumentInfo } from '@/interfaces/database/document';
-import { FileType } from '@/pages/agent/constant/pipeline';
import { getExtension, getUnSupportedFilesCount } from '@/utils/document-util';
import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
-import { buildMissingModelModalContent } from './parser-model-gap-content';
-import { useParserModelValidation } from './use-parser-model-validation';
-import { getFileTypeByExtension } from './utils';
+import { buildParserGapModalContent } from './parser-gap-content';
+import { useParserGapValidation } from './use-parser-gap-validation';
+import { getFileTypeByExtension, hasUnsupportedTypeGap } from './utils';
export const useHandleUploadDocument = () => {
const { t } = useTranslation();
@@ -23,8 +23,7 @@ export const useHandleUploadDocument = () => {
} = useSetModalState();
const { uploadDocument, loading } = useUploadDocument();
const { runDocumentByIds } = useRunDocument();
- const { findFilesMissingModels, goToDatasetConfiguration } =
- useParserModelValidation();
+ const { findParseGaps } = useParserGapValidation();
const proceedUpload = useCallback(
async (
@@ -108,39 +107,33 @@ export const useHandleUploadDocument = () => {
const names = fileList.map((file) =>
file instanceof File ? file.name : file.file.name,
);
- const gaps = findFilesMissingModels(names);
+ const gaps = findParseGaps(names);
if (gaps.length > 0) {
const failingFileTypes = new Set(gaps.map((gap) => gap.fileType));
Modal.warning({
- title: t('knowledgeDetails.uploadMissingModelsTitle'),
- content: buildMissingModelModalContent(
+ title: t(
+ hasUnsupportedTypeGap(gaps)
+ ? 'knowledgeDetails.uploadUnsupportedTypesTitle'
+ : 'knowledgeDetails.uploadMissingModelsTitle',
+ ),
+ content: buildParserGapModalContent(
t,
gaps,
- 'knowledgeDetails.configureInDatasetSettingHint',
+ 'knowledgeDetails.reselectParserAfterUploadHint',
),
okText: t('knowledgeDetails.continueUpload'),
- cancelText: t('knowledgeDetails.goToConfiguration'),
+ cancelText: t('common.cancel'),
closable: false,
onOk: () => {
proceedUpload(values, failingFileTypes);
},
- onCancel: () => {
- hideDocumentUploadModal();
- goToDatasetConfiguration();
- },
});
return;
}
return proceedUpload(values, new Set());
},
- [
- findFilesMissingModels,
- goToDatasetConfiguration,
- hideDocumentUploadModal,
- proceedUpload,
- t,
- ],
+ [findParseGaps, proceedUpload, t],
);
return {
diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.ts
index 83832380ac..bc903f8e1a 100644
--- a/web/src/pages/dataset/dataset/utils.test.ts
+++ b/web/src/pages/dataset/dataset/utils.test.ts
@@ -1,11 +1,18 @@
-import { FileType, initialParserValues } from '@/pages/agent/constant/pipeline';
-import { IngestionTaskStatus, RunningStatus } from './constant';
+import { FileType } from '@/constants/file';
import {
- findFilesMissingParserModels,
- findMissingParserModel,
+ IngestionTaskStatus,
+ ParserModelKind,
+ ParserGapReason,
+ RunningStatus,
+} from './constant';
+import {
+ findDocumentsParserGaps,
+ findFilesParserGaps,
+ findParserGap,
getDocumentRunningStatus,
- getEffectiveParserSetups,
getFileTypeByExtension,
+ getSavedParserSetups,
+ hasUnsupportedTypeGap,
ingestionStatusToRunningStatus,
isDocumentProcessing,
isDocumentStopping,
@@ -175,95 +182,141 @@ describe('getFileTypeByExtension', () => {
});
});
-// Minimal structural stand-in for IDataset in these tests. It is declared
-// locally (instead of importing the interface) because the esbuild-jest
-// babel hoisting pipeline cannot elide imported bindings used only in type
-// positions.
-type DatasetWithParserConfig = { parser_config: any };
-
-const DefaultSetups = initialParserValues.setups as Record<string, any>[];
+// Saved setups shaped like a pipeline that declares pdf/image plus audio
+// (model missing) and video (model configured).
+const SavedSetups = [
+ { fileFormat: FileType.PDF, parse_method: 'DeepDOC' },
+ { fileFormat: FileType.Image, parse_method: 'ocr' },
+ { fileFormat: FileType.Audio, vlm: { llm_id: '' } },
+ { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
+] as Record<string, any>[];
-describe('findMissingParserModel', () => {
- it('passes audio when the setup carries a model', () => {
- const setups = [
- { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } },
- ];
- expect(findMissingParserModel(FileType.Audio, setups)).toBeNull();
+describe('findParserGap', () => {
+ it('flags a file type missing from the saved setups as unsupported', () => {
+ expect(findParserGap(FileType.Docx, SavedSetups)).toEqual({
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Docx,
+ });
});
- it('flags audio when the setup has no model, ignoring the tenant default', () => {
- expect(findMissingParserModel(FileType.Audio, DefaultSetups)).toEqual({
+ it('flags audio/video without a declared family as unsupported, not missing-model', () => {
+ const setups = [{ fileFormat: FileType.PDF }];
+ expect(findParserGap(FileType.Audio, setups)).toEqual({
+ reason: ParserGapReason.UnsupportedType,
fileType: FileType.Audio,
- modelKind: 'asr',
+ });
+ expect(findParserGap(FileType.Video, setups)).toEqual({
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Video,
});
});
- it('passes video when the setup carries a model', () => {
- const setups = [
- { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
- ];
- expect(findMissingParserModel(FileType.Video, setups)).toBeNull();
+ it('flags declared audio without a model as a missing asr model', () => {
+ expect(findParserGap(FileType.Audio, SavedSetups)).toEqual({
+ reason: ParserGapReason.MissingModel,
+ fileType: FileType.Audio,
+ modelKind: ParserModelKind.Asr,
+ });
});
- it('flags video when the setup has no model', () => {
- expect(findMissingParserModel(FileType.Video, DefaultSetups)).toEqual({
+ it('flags declared video without a model as a missing vision model', () => {
+ const setups = [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }];
+ expect(findParserGap(FileType.Video, setups)).toEqual({
+ reason: ParserGapReason.MissingModel,
fileType: FileType.Video,
- modelKind: 'vision',
+ modelKind: ParserModelKind.Vision,
});
});
- it('never flags image — ocr-only parsing needs no model', () => {
- // The image parser always runs OCR; the vision model (picked as
- // parse_method) is an optional supplement, so neither ocr nor a model
- // id constitutes a gap.
- expect(findMissingParserModel(FileType.Image, DefaultSetups)).toBeNull();
- expect(
- findMissingParserModel(FileType.Image, [
- { fileFormat: FileType.Image, parse_method: 'gpt-4o@OpenAI' },
- ]),
- ).toBeNull();
+ it('passes video when the setup carries a model', () => {
+ expect(findParserGap(FileType.Video, SavedSetups)).toBeNull();
});
- it('ignores file types without model requirements', () => {
- expect(findMissingParserModel(FileType.PDF, DefaultSetups)).toBeNull();
- expect(findMissingParserModel(undefined, DefaultSetups)).toBeNull();
+ it('passes image with an ocr-only setup — no model required', () => {
+ expect(findParserGap(FileType.Image, SavedSetups)).toBeNull();
+ });
+
+ it('passes supported non-media types and unknown file types', () => {
+ expect(findParserGap(FileType.PDF, SavedSetups)).toBeNull();
+ expect(findParserGap(undefined, SavedSetups)).toBeNull();
});
});
-describe('findFilesMissingParserModels', () => {
- it('returns a gap per file that lacks its required model', () => {
- const gaps = findFilesMissingParserModels(
- ['song.mp3', 'notes.txt', 'photo.jpg'],
- DefaultSetups,
+describe('findFilesParserGaps', () => {
+ it('returns a gap per failing file, mixing both reasons', () => {
+ const gaps = findFilesParserGaps(
+ ['song.mp3', 'notes.docx', 'photo.jpg', 'movie.mp4'],
+ SavedSetups,
);
expect(gaps).toEqual([
- { name: 'song.mp3', fileType: FileType.Audio, modelKind: 'asr' },
+ {
+ name: 'song.mp3',
+ reason: ParserGapReason.MissingModel,
+ fileType: FileType.Audio,
+ modelKind: ParserModelKind.Asr,
+ },
+ {
+ name: 'notes.docx',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Docx,
+ },
]);
});
});
-describe('getEffectiveParserSetups', () => {
- it('falls back to the default setups without a saved parser config', () => {
- const setups = getEffectiveParserSetups(null);
- expect(setups.map((x) => x.fileFormat)).toEqual(
- initialParserValues.setups.map((x) => x.fileFormat),
- );
+describe('hasUnsupportedTypeGap', () => {
+ it('detects unsupported-type gaps among mixed gaps', () => {
+ expect(
+ hasUnsupportedTypeGap([
+ {
+ reason: ParserGapReason.MissingModel,
+ fileType: FileType.Audio,
+ modelKind: ParserModelKind.Asr,
+ },
+ { reason: ParserGapReason.UnsupportedType, fileType: FileType.Image },
+ ]),
+ ).toBe(true);
+ expect(
+ hasUnsupportedTypeGap([
+ {
+ reason: ParserGapReason.MissingModel,
+ fileType: FileType.Audio,
+ modelKind: ParserModelKind.Asr,
+ },
+ ]),
+ ).toBe(false);
+ expect(hasUnsupportedTypeGap([])).toBe(false);
+ });
+});
+
+describe('getSavedParserSetups', () => {
+ it('returns null without a saved parser config', () => {
+ expect(getSavedParserSetups(null)).toBeNull();
+ expect(
+ getSavedParserSetups({
+ parser_config: {},
+ }),
+ ).toBeNull();
+ });
+
+ it('returns null for legacy flat-shape configs without a Parser entry', () => {
+ const knowledgeBase = {
+ parser_config: { chunk_token_num: 128, delimiter: '\n' },
+ };
+ expect(getSavedParserSetups(knowledgeBase)).toBeNull();
});
- it('applies saved overrides keyed by file format', () => {
+ it('returns only the saved Parser setups, without merging defaults', () => {
const knowledgeBase = {
parser_config: {
'Parser:1': {
- audio: { vlm: { llm_id: 'whisper@OpenAI' } },
+ pdf: { parse_method: 'DeepDOC' },
+ image: { parse_method: 'ocr' },
},
},
- } as unknown as DatasetWithParserConfig;
- const setups = getEffectiveParserSetups(knowledgeBase);
- const audio = setups.find((x) => x.fileFormat === FileType.Audio);
- expect(audio?.vlm?.llm_id).toBe('whisper@OpenAI');
- // Untouched types keep their defaults
- const image = setups.find((x) => x.fileFormat === FileType.Image);
- expect(image?.parse_method).toBe('ocr');
+ };
+ const setups = getSavedParserSetups(knowledgeBase);
+ expect(setups?.map((x) => x.fileFormat).sort()).toEqual(['image', 'pdf']);
});
it('ignores non-parser operator entries', () => {
@@ -271,8 +324,53 @@ describe('getEffectiveParserSetups', () => {
parser_config: {
'TokenChunker:1': { chunk_token_size: 128 },
},
- } as unknown as DatasetWithParserConfig;
- const setups = getEffectiveParserSetups(knowledgeBase);
- expect(setups).toHaveLength(initialParserValues.setups.length);
+ };
+ expect(getSavedParserSetups(knowledgeBase)).toBeNull();
+ });
+});
+
+describe('findDocumentsParserGaps', () => {
+ // A document overridden to a general-like parser: its row parser_config
+ // declares pdf even though the dataset-level fallback does not.
+ const OverriddenDoc = {
+ name: 'civil-code.pdf',
+ parser_config: {
+ 'Parser:1': { pdf: { parse_method: 'DeepDOC' } },
+ },
+ };
+
+ it('validates against the document row config when present', () => {
+ expect(findDocumentsParserGaps([OverriddenDoc], null)).toEqual([]);
+ });
+
+ it('falls back to the dataset-level setups for rows without a Parser entry', () => {
+ const legacyRow = { name: 'song.mp3', parser_config: {} };
+ expect(findDocumentsParserGaps([legacyRow], SavedSetups)).toEqual([
+ {
+ name: 'song.mp3',
+ reason: ParserGapReason.MissingModel,
+ fileType: FileType.Audio,
+ modelKind: ParserModelKind.Asr,
+ },
+ ]);
+ // No fallback either: nothing can be determined, so nothing is flagged.
+ expect(findDocumentsParserGaps([legacyRow], null)).toEqual([]);
+ });
+
+ it('evaluates each document against its own config in a mixed batch', () => {
+ const inheritedRow = { name: 'photo.jpg', parser_config: undefined };
+ const gaps = findDocumentsParserGaps(
+ [OverriddenDoc, inheritedRow],
+ // Dataset-level config declares only audio: the overridden pdf passes
+ // while the inherited jpg is flagged.
+ [{ fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } }],
+ );
+ expect(gaps).toEqual([
+ {
+ name: 'photo.jpg',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Image,
+ },
+ ]);
});
});
diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.ts
index 89334692af..86b0aac712 100644
--- a/web/src/pages/dataset/dataset/utils.ts
+++ b/web/src/pages/dataset/dataset/utils.ts
@@ -1,19 +1,18 @@
import { Operator } from '@/constants/agent';
-import type { IDataset } from '@/interfaces/database/dataset';
+import { FileType, FileTypeSuffixMap } from '@/constants/file';
import type { IDocumentInfo } from '@/interfaces/database/document';
-import {
- FileType,
- FileTypeSuffixMap,
- initialParserValues,
-} from '@/pages/agent/constant/pipeline';
import { pickByBackend } from '@/utils/backend-variant';
import { getExtension } from '@/utils/document-util';
import {
getOperatorType,
transformParserConfigSetups,
} from '@/utils/pipeline-operator';
-import { cloneDeep } from 'lodash';
-import { IngestionTaskStatus, RunningStatus } from './constant';
+import {
+ IngestionTaskStatus,
+ ParserGapReason,
+ ParserModelKind,
+ RunningStatus,
+} from './constant';
/** Ingestion statuses that represent an active or canceling parse task on Go. */
const activeIngestionStatuses = new Set<IngestionTaskStatus>([
@@ -158,22 +157,34 @@ export const isDocumentProcessing = (
python: isPythonDocumentProcessing(document),
});
-// --- Parser model prerequisite checks -------------------------------------
-// Audio/video files can only be parsed when the matching model is configured
-// on the dataset's Parser operator. The tenant default is deliberately not
-// consulted: parsing reads the operator setup, so a global default does not
-// make the file parsable. Image files are exempt: the image parser always
-// runs OCR and only supplements it with the vision model (picked as the
-// image parse_method, with the tenant default as fallback), so a static
-// method like ocr is a complete configuration.
+// --- Parser prerequisite checks -------------------------------------------
+// A file cannot be parsed under the dataset's current Parser operator when
+// the operator does not declare the file's type family at all
+// (unsupportedType), or — for audio/video — when the declared setup carries
+// no model (missingModel). The tenant default is deliberately not consulted:
+// parsing reads the operator setup, so a global default does not make the
+// file parsable. Image files only need their family declared: the image
+// parser always runs OCR and merely supplements it with the vision model
+// (picked as the image parse_method, with the tenant default as fallback).
// These helpers power the upload warning and the parse-click validation.
export type ParserModelGap = {
+ reason: ParserGapReason.MissingModel;
fileType: FileType;
- modelKind: 'asr' | 'vision';
+ modelKind: ParserModelKind.Asr | ParserModelKind.Vision;
};
-export type FileModelGap = ParserModelGap & { name: string };
+export type UnsupportedTypeGap = {
+ reason: ParserGapReason.UnsupportedType;
+ fileType: FileType;
+};
+
+export type ParserGap = ParserModelGap | UnsupportedTypeGap;
+
+export type FileParserGap = ParserGap & { name: string };
+
+export const hasUnsupportedTypeGap = (gaps: ParserGap[]) =>
+ gaps.some((gap) => gap.reason === ParserGapReason.UnsupportedType);
type ParserSetup = Record<string, any> & { fileFormat?: string };
@@ -193,65 +204,89 @@ export function getFileTypeByExtension(
}
/**
- * Effective parser setups for a dataset: the saved Parser operator entry
- * (stored as a flattened map keyed by file format) merged over the default
- * setups, so untouched file types validate against their defaults.
+ * Parser setups exactly as saved, keyed by file format — the keys are the
+ * type families the pipeline/built-in method declares. Accepts either a
+ * dataset or a document row: a document's parser_config snapshots the parser
+ * it actually runs with (dataset config at upload, the override after a
+ * parser change). Returns null when parser_config has no Parser operator
+ * entry (legacy flat-shape configs, pipelines without a Parser stage): type
+ * support cannot be determined then and validation is skipped.
*/
-export function getEffectiveParserSetups(
- knowledgeBase: Pick<IDataset, 'parser_config'> | null | undefined,
-): ParserSetup[] {
- const parserConfig = knowledgeBase?.parser_config as
- | Record<string, any>
- | undefined;
+export function getSavedParserSetups(
+ source: { parser_config?: unknown } | null | undefined,
+): ParserSetup[] | null {
+ const parserConfig = source?.parser_config as Record<string, any> | undefined;
const parserEntry = Object.entries(parserConfig ?? {}).find(
([operatorId]) => getOperatorType(operatorId) === Operator.Parser,
)?.[1];
- const savedSetups = transformParserConfigSetups(parserEntry);
- const savedByFileType = new Map(
- savedSetups.map((setup) => [setup.fileFormat, setup]),
- );
-
- const defaultSetups = cloneDeep(initialParserValues.setups) as ParserSetup[];
- const merged = defaultSetups.map(
- (setup) => savedByFileType.get(setup.fileFormat) ?? setup,
- );
- for (const setup of savedSetups) {
- if (!defaultSetups.some((x) => x.fileFormat === setup.fileFormat)) {
- merged.push(setup);
- }
+ if (!parserEntry) {
+ return null;
}
- return merged;
+ return transformParserConfigSetups(parserEntry);
}
-export function findMissingParserModel(
+export function findParserGap(
fileType: FileType | undefined,
setups: ParserSetup[],
-): ParserModelGap | null {
+): ParserGap | null {
+ if (!fileType) {
+ return null;
+ }
+
const setup = setups.find((x) => x.fileFormat === fileType);
+ if (!setup) {
+ return { reason: ParserGapReason.UnsupportedType, fileType };
+ }
switch (fileType) {
case FileType.Audio: {
- const configured = setup?.vlm?.llm_id;
- return configured ? null : { fileType, modelKind: 'asr' };
+ return setup.vlm?.llm_id
+ ? null
+ : {
+ reason: ParserGapReason.MissingModel,
+ fileType,
+ modelKind: ParserModelKind.Asr,
+ };
}
case FileType.Video: {
- const configured = setup?.vlm?.llm_id;
- return configured ? null : { fileType, modelKind: 'vision' };
+ return setup.vlm?.llm_id
+ ? null
+ : {
+ reason: ParserGapReason.MissingModel,
+ fileType,
+ modelKind: ParserModelKind.Vision,
+ };
}
default:
return null;
}
}
-export function findFilesMissingParserModels(
+export function findFilesParserGaps(
names: string[],
setups: ParserSetup[],
-): FileModelGap[] {
+): FileParserGap[] {
return names.flatMap((name) => {
const fileType = getFileTypeByExtension(getExtension(name));
- const gap = findMissingParserModel(fileType, setups);
+ const gap = findParserGap(fileType, setups);
return gap ? [{ ...gap, name }] : [];
});
}
+
+/**
+ * Per-document parse validation: each document row carries the parser config
+ * it actually runs with (dataset snapshot at upload, the override after a
+ * parser change), so it is validated against its own setups. Rows without a
+ * Parser entry fall back to `fallbackSetups` (the dataset-level config).
+ */
+export function findDocumentsParserGaps(
+ documents: Array<{ name: string; parser_config?: unknown }>,
+ fallbackSetups: ParserSetup[] | null,
+): FileParserGap[] {
+ return documents.flatMap((doc) => {
+ const setups = getSavedParserSetups(doc) ?? fallbackSetups;
+ return setups ? findFilesParserGaps([doc.name], setups) : [];
+ });
+}