Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feat: Make the files of the parser operators in the pipeline deletable. (#19384)

balibabue3bb9e1cd930
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py
index 402669ef76..367f73f040 100644
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -306,15 +306,6 @@ class ParserParam(ProcessParamBase):
html_output_format = html_config.get("output_format", "")
self.check_valid_value(html_output_format, "HTML output format abnormal.", self.allowed_output_format["html"])
- audio_config = self.setups.get("audio", "")
- if audio_config:
- audio_vlm = audio_config.get("vlm") or {}
- self.check_empty(audio_vlm.get("llm_id"), "Audio VLM")
-
- video_config = self.setups.get("video", "")
- if video_config:
- video_vlm = video_config.get("vlm") or {}
- self.check_empty(video_vlm.get("llm_id"), "Video VLM")
email_config = self.setups.get("email", "")
if email_config:
email_output_format = email_config.get("output_format", "")
@@ -1223,14 +1214,17 @@ class Parser(ProcessBase):
self.callback(random.randint(1, 5) / 100.0, "Start to work on an audio.")
conf = self._param.setups["audio"]
- vlm = conf.get("vlm")
+ vlm = conf.get("vlm") or {}
self.set_output("output_format", conf["output_format"])
_, ext = os.path.splitext(name)
with tempfile.NamedTemporaryFile(suffix=ext) as tmpf:
tmpf.write(blob)
tmpf.flush()
tmp_path = os.path.abspath(tmpf.name)
- seq2txt_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.ASR, vlm["llm_id"])
+ if vlm.get("llm_id"):
+ seq2txt_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.ASR, vlm["llm_id"])
+ else:
+ seq2txt_model_config = get_tenant_default_model_by_type(self._canvas.get_tenant_id(), LLMType.ASR)
seq2txt_mdl = LLMBundle(self._canvas.get_tenant_id(), seq2txt_model_config)
txt = seq2txt_mdl.transcription(tmp_path)
@@ -1241,9 +1235,12 @@ class Parser(ProcessBase):
self.callback(random.randint(1, 5) / 100.0, "Start to work on an video.")
conf = self._param.setups["video"]
- vlm = conf.get("vlm")
+ vlm = conf.get("vlm") or {}
self.set_output("output_format", conf["output_format"])
- cv_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.VISION, vlm["llm_id"])
+ if vlm.get("llm_id"):
+ cv_model_config = resolve_model_config(self._canvas.get_tenant_id(), LLMType.VISION, vlm["llm_id"])
+ else:
+ cv_model_config = get_tenant_default_model_by_type(self._canvas.get_tenant_id(), LLMType.VISION)
cv_mdl = LLMBundle(self._canvas.get_tenant_id(), cv_model_config)
video_prompt = str(conf.get("prompt", "") or "")
txt = asyncio.run(cv_mdl.async_chat(system="", history=[], gen_conf={}, video_bytes=blob, filename=name, video_prompt=video_prompt))
diff --git a/web/src/components/document-pipeline-dialog/index.tsx b/web/src/components/document-pipeline-dialog/index.tsx
index b4975b518b..f67f72ab8e 100644
--- a/web/src/components/document-pipeline-dialog/index.tsx
+++ b/web/src/components/document-pipeline-dialog/index.tsx
@@ -128,6 +128,7 @@ export function DocumentPipelineDialog({
| Record<string, FieldErrors | undefined>
| undefined
}
+ fixedFileFormats
/>
)}
</form>
diff --git a/web/src/components/layout-recognize-form-field.tsx b/web/src/components/layout-recognize-form-field.tsx
index e0ff99021d..a426932184 100644
--- a/web/src/components/layout-recognize-form-field.tsx
+++ b/web/src/components/layout-recognize-form-field.tsx
@@ -91,7 +91,7 @@ export function LayoutRecognizeFormField({
const modelTree = buildModelTree(
allAddedModels,
- ['image2text', 'ocr'],
+ ['image2text', 'ocr', 'vision'],
(node) => (
<div className="flex justify-between items-center gap-2 w-full">
<span className="flex items-center gap-1.5 truncate">
diff --git a/web/src/components/list-filter-bar/use-handle-filter-submit.ts b/web/src/components/list-filter-bar/use-handle-filter-submit.ts
index 6fced36903..2744a31ccc 100644
--- a/web/src/components/list-filter-bar/use-handle-filter-submit.ts
+++ b/web/src/components/list-filter-bar/use-handle-filter-submit.ts
@@ -88,7 +88,11 @@ export function useHandleFilterSubmit() {
if (!preValue) return preValue;
const newValue: FilterValue = mergeFilterValue(preValue, validFields);
- return isEqual(newValue, preValue) ? preValue : newValue;
+ // Keep the previous reference when nothing was actually pruned, so a
+ // catalog refresh (e.g. the document poll invalidating the filter
+ // counts every 5s) doesn't trigger a throwaway state update and the
+ // re-render churn that follows.
+ return isEqual(preValue, newValue) ? preValue : newValue;
});
}, []);
diff --git a/web/src/components/pipeline-operator-tabs/index.tsx b/web/src/components/pipeline-operator-tabs/index.tsx
index c1d3c26d4b..5b633c9dd9 100644
--- a/web/src/components/pipeline-operator-tabs/index.tsx
+++ b/web/src/components/pipeline-operator-tabs/index.tsx
@@ -35,6 +35,9 @@ type PipelineOperatorTabsProps = {
// Validation errors from the outer form's parser_config, keyed by
// operatorId; each entry is mirrored onto the operator form's fields.
operatorFormErrors?: Record<string, FieldErrors | undefined>;
+ // Dataset-side embeddings show a fixed set of parser file types; only the
+ // canvas parser allows add/remove.
+ fixedFileFormats?: boolean;
};
const PipelineOperatorTabs = ({
@@ -44,6 +47,7 @@ const PipelineOperatorTabs = ({
onOperatorValuesChange,
operatorValues,
operatorFormErrors,
+ fixedFileFormats,
}: PipelineOperatorTabsProps) => {
const getOperatorId = useCallback((node: RAGFlowNodeType) => {
return (
@@ -105,6 +109,7 @@ const PipelineOperatorTabs = ({
node={node}
onValuesChange={handleValuesChange(node)}
externalErrors={operatorFormErrors?.[getOperatorId(node)]}
+ fixedFileFormats={fixedFileFormats}
/>
</TabsContent>
);
diff --git a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx
index 4c76a44ca7..a9b32b5766 100644
--- a/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx
+++ b/web/src/components/pipeline-operator-tabs/pipeline-operator-form.tsx
@@ -30,12 +30,16 @@ type PipelineOperatorFormProps = {
node: RAGFlowNodeType;
onValuesChange?: (values: any) => void;
externalErrors?: FieldErrors;
+ // Dataset-side embeddings show a fixed set of parser file types; only the
+ // canvas parser allows add/remove.
+ fixedFileFormats?: boolean;
};
const PipelineOperatorForm = ({
node,
onValuesChange,
externalErrors,
+ fixedFileFormats,
}: PipelineOperatorFormProps) => {
const operatorType = getOperatorType(
(node.data as Record<string, any>)?.operatorId || node.data?.label || '',
@@ -56,6 +60,7 @@ const PipelineOperatorForm = ({
onValuesChange={handleValuesChange}
hideOutputs
externalErrors={externalErrors}
+ fixedFileFormats={fixedFileFormats}
/>
);
case Operator.TokenChunker:
diff --git a/web/src/hooks/logic-hooks/navigate-hooks.ts b/web/src/hooks/logic-hooks/navigate-hooks.ts
index 3d0748f9a0..f53d814993 100644
--- a/web/src/hooks/logic-hooks/navigate-hooks.ts
+++ b/web/src/hooks/logic-hooks/navigate-hooks.ts
@@ -73,6 +73,13 @@ export const useNavigatePage = () => {
[navigate],
);
+ const navigateToDatasetSetting = useCallback(
+ (id: string) => {
+ navigate(`${Routes.DatasetBase}${Routes.DataSetSetting}/${id}`);
+ },
+ [navigate],
+ );
+
const navigateToCompilation = useCallback(
(id: string) => () => {
navigate(`${Routes.DatasetBase}${Routes.Compilation}/${id}`);
@@ -251,6 +258,7 @@ export const useNavigatePage = () => {
navigateToOldProfile,
navigateToDataflowResult,
navigateToDataFile,
+ navigateToDatasetSetting,
navigateToCompilation,
navigateToDataSourceDetail,
navigateToMemory,
diff --git a/web/src/hooks/use-llm-request.tsx b/web/src/hooks/use-llm-request.tsx
index fcf5d3c154..43e3a4a936 100644
--- a/web/src/hooks/use-llm-request.tsx
+++ b/web/src/hooks/use-llm-request.tsx
@@ -644,8 +644,15 @@ export const useUpdateProviderInstance = () => {
export const useFetchDefaultModels = () => {
const { data, isFetching: loading } = useQuery<IDefaultModel[]>({
queryKey: LlmKeys.defaultModels(),
+ // Tenant default models change rarely, and every mutation that can change
+ // them invalidates this key explicitly, so hold the result for a while
+ // instead of refetching per observer — there is one per dataset row.
+ staleTime: 5 * 60 * 1000,
initialData: [],
- gcTime: 0,
+ // `initialData` alone would be timestamped as of now and so counted fresh
+ // for the whole stale window, meaning the seeded empty list would never be
+ // replaced by a real fetch. Backdate it so the first mount still fetches.
+ initialDataUpdatedAt: 0,
queryFn: async () => {
const { data } = await llmService.listDefaultModels({}, true);
return data?.data?.models ?? [];
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index fe0cff53ed..c54adb2907 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -396,6 +396,21 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
dataFlowRequired: 'Data flow is required',
},
knowledgeDetails: {
+ continueUpload: 'Continue upload',
+ goToConfiguration: 'Go to configuration',
+ uploadMissingModelsTitle: 'Some files lack a required model',
+ fileModelMissing:
+ '{{name}} ({{fileType}}) requires a configured {{model}} model',
+ missingModelAsr: 'audio (ASR)',
+ missingModelVision: 'vision',
+ configureInDatasetSettingHint:
+ 'You can configure it in the dataset Configuration page (Parser operator), or continue anyway.',
+ parseBlockedTitle: 'Cannot parse',
+ parseBlockedPartialTitle: 'Some files cannot be parsed',
+ parseBlockedHint:
+ 'Configure the model in the dataset Configuration page (Parser operator), then try again.',
+ parseValidFiles: 'Parse valid files',
+ parseValidFilesNote: '{{count}} valid files will be parsed.',
metadata: {
fields: 'fields',
selectFiles: 'Selected {{count}} files',
@@ -3442,6 +3457,8 @@ This process aggregates variables from multiple branches into a single variable
audio: 'Audio',
video: 'Video',
},
+ addFileType: 'Add file type',
+ atLeastOneFileType: 'At least one file type is required',
fields: 'Field',
rule: 'Rule',
addRule: 'Add rule',
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 6e824f2ae6..d2883f2a82 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -355,6 +355,20 @@ export default {
dataFlowRequired: '数据流必填',
},
knowledgeDetails: {
+ continueUpload: '继续上传',
+ goToConfiguration: '前往配置',
+ uploadMissingModelsTitle: '部分文件缺少解析所需的模型',
+ fileModelMissing: '{{name}}({{fileType}})需要配置{{model}}模型',
+ missingModelAsr: '音频(ASR)',
+ missingModelVision: '视觉',
+ configureInDatasetSettingHint:
+ '可在数据集「配置」页的 Parser 算子中设置对应模型,或继续操作。',
+ parseBlockedTitle: '无法解析',
+ parseBlockedPartialTitle: '部分文件无法解析',
+ parseBlockedHint:
+ '请先在数据集「配置」页的 Parser 算子中设置对应模型,再重新解析。',
+ parseValidFiles: '解析有效文件',
+ parseValidFilesNote: '将解析其余 {{count}} 个有效文件。',
metadata: {
fields: '字段',
selectFiles: '已选择 {{count}} 个文件',
@@ -2997,6 +3011,8 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 R
audio: '音频',
video: '视频',
},
+ addFileType: '添加文件类型',
+ atLeastOneFileType: '至少保留一个文件类型',
fields: '字段',
rule: '规则',
addRule: '增加规则',
diff --git a/web/src/pages/agent/form/parser-form/image-form-fields.tsx b/web/src/pages/agent/form/parser-form/image-form-fields.tsx
index 3d073f21c9..7a669fb906 100644
--- a/web/src/pages/agent/form/parser-form/image-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/image-form-fields.tsx
@@ -2,18 +2,17 @@ import { RAGFlowFormItem } from '@/components/ragflow-form';
import { Textarea } from '@/components/ui/textarea';
import { buildOptions } from '@/utils/form';
import { isEmpty } from 'lodash';
-import { useEffect, useMemo } from 'react';
-import { useFormContext, useWatch } from 'react-hook-form';
+import { useMemo } from 'react';
+import { useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
-import { FileType, ImageParseMethod } from '../../constant/pipeline';
+import { ImageParseMethod } from '../../constant/pipeline';
import { LanguageFormField, ParserMethodFormField } from './common-form-fields';
import { CommonProps } from './interface';
import { useSetInitialLanguage } from './use-set-initial-language';
-import { buildFieldNameWithPrefix, isForeignParseMethod } from './utils';
+import { buildFieldNameWithPrefix } from './utils';
export function ImageFormFields({ prefix }: CommonProps) {
const { t } = useTranslation();
- const form = useFormContext();
const options = buildOptions(
ImageParseMethod,
t,
@@ -29,19 +28,6 @@ export function ImageFormFields({ prefix }: CommonProps) {
return !isEmpty(parseMethod) && parseMethod !== ImageParseMethod.OCR;
}, [parseMethod]);
- useEffect(() => {
- const current = form.getValues(parseMethodName);
- // On a file-type switch the field remounts and react-hook-form re-seeds it
- // from the node's saved form data, so it can hold another file type's
- // static parse method (e.g. DeepDOC) — reset it to OCR in that case too.
- if (isEmpty(current) || isForeignParseMethod(FileType.Image, current)) {
- form.setValue(parseMethodName, ImageParseMethod.OCR, {
- shouldValidate: true,
- shouldDirty: true,
- });
- }
- }, [form, parseMethodName]);
-
useSetInitialLanguage({ prefix, languageShown });
return (
diff --git a/web/src/pages/agent/form/parser-form/index.tsx b/web/src/pages/agent/form/parser-form/index.tsx
index b8ae01004c..95f395d47f 100644
--- a/web/src/pages/agent/form/parser-form/index.tsx
+++ b/web/src/pages/agent/form/parser-form/index.tsx
@@ -1,11 +1,18 @@
import { Collapse } from '@/components/collapse';
import { useSyncExternalFormErrors } from '@/components/pipeline-operator-tabs/use-sync-external-form-errors';
+import { BlockButton, Button } from '@/components/ui/button';
import { Card } from '@/components/ui/card';
+import {
+ DropdownMenu,
+ DropdownMenuContent,
+ DropdownMenuItem,
+ DropdownMenuTrigger,
+} from '@/components/ui/dropdown-menu';
import { Form } from '@/components/ui/form';
import { useFetchDefaultModelDictionary } from '@/hooks/use-llm-request';
-import i18n from '@/locales/config';
import { zodResolver } from '@hookform/resolvers/zod';
-import { memo, useMemo } from 'react';
+import { LucideTrash2 } from 'lucide-react';
+import { memo, useCallback, useMemo } from 'react';
import { useFieldArray, useForm, useFormContext } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { z } from 'zod';
@@ -21,15 +28,19 @@ import { EmailFormFields } from './email-form-fields';
import { ImageFormFields } from './image-form-fields';
import { PdfFormFields } from './pdf-form-fields';
import { PptFormFields } from './ppt-form-fields';
+import { FormSchema, ParserFormSchemaType } from './schema';
import { SpreadsheetFormFields } from './spreadsheet-form-fields';
import {
HtmlFormFields,
TextMarkdownFormFields,
} from './text-html-form-fields';
-import { withDefaultParserModels } from './utils';
+import { buildInitialParserSetup } from './utils';
import { AudioFormFields, VideoFormFields } from './video-form-fields';
import { WordFormFields } from './word-form-fields';
+export { FormSchema } from './schema';
+export type { ParserFormSchemaType } from './schema';
+
const outputList = buildOutputList(initialParserValues.outputs);
const FileFormatWidgetMap = {
@@ -49,76 +60,11 @@ const FileFormatWidgetMap = {
type ParserItemProps = {
name: string;
index: number;
+ canRemove: boolean;
+ onRemove: (index: number) => void;
};
-const SetupSchema = z
- .object({
- fileFormat: z.string().nullish(),
- // preprocess: z.array(z.string()).optional(),
- output_format: z.string().optional(),
- parse_method: z.string().optional(),
- lang: z.string().optional(),
- fields: z.array(z.string()).optional(),
- vlm: z.object({ llm_id: z.string().optional() }).optional(),
- flatten_media_to_text: z.boolean().optional(),
- system_prompt: z.string().optional(),
- table_result_type: z.string().optional(),
- markdown_image_response_type: z.string().optional(),
- enable_multi_column: z.boolean().optional(),
- remove_toc: z.boolean().optional(),
- remove_header_footer: z.boolean().optional(),
- pages: z
- .array(
- z
- .object({
- // Keep these checks on the fields themselves: an object-level
- // `superRefine` is skipped whenever the base shape fails to
- // parse, so one missing sibling would silently swallow the
- // other field's error.
- from: z.coerce
- .number()
- .int(i18n.t('knowledgeDetails.pageRangeFromInvalid'))
- .min(1, i18n.t('knowledgeDetails.pageRangeFromInvalid')),
- to: z.coerce
- .number()
- .int(i18n.t('knowledgeDetails.pageRangeToInvalid'))
- .min(1, i18n.t('knowledgeDetails.pageRangeToInvalid')),
- })
- .refine(({ from, to }) => to >= from, {
- path: ['to'],
- message: i18n.t('knowledgeDetails.pageRangeToInvalid'),
- }),
- )
- .optional(),
- })
- .superRefine((values, ctx) => {
- if (values.fileFormat === FileType.Email && !values.fields?.length) {
- ctx.addIssue({
- code: z.ZodIssueCode.custom,
- path: ['fields'],
- message: 'Fields is required',
- });
- }
- if (
- (values.fileFormat === FileType.Video ||
- values.fileFormat === FileType.Audio) &&
- !values.vlm?.llm_id
- ) {
- ctx.addIssue({
- code: z.ZodIssueCode.custom,
- path: ['vlm', 'llm_id'],
- message: 'Model is required',
- });
- }
- });
-
-export const FormSchema = z.object({
- setups: z.array(SetupSchema),
-});
-
-export type ParserFormSchemaType = z.infer<typeof FormSchema>;
-
-function ParserItem({ name, index }: ParserItemProps) {
+function ParserItem({ name, index, canRemove, onRemove }: ParserItemProps) {
const { t } = useTranslation();
const form = useFormContext<ParserFormSchemaType>();
@@ -130,12 +76,31 @@ function ParserItem({ name, index }: ParserItemProps) {
? FileFormatWidgetMap[fileFormat as keyof typeof FileFormatWidgetMap]
: () => <></>;
+ const handleRemove = useCallback(() => {
+ onRemove(index);
+ }, [onRemove, index]);
+
return (
<Card as="section" className="bg-bg-card px-5 py-2.5 border-none">
- {/* The file format is fixed per parser item; keep it registered so it is
+ {/* The file format has no visible input; keep it registered so it is
still submitted with the form. */}
<input type="hidden" {...form.register(`setups.${index}.fileFormat`)} />
- <Collapse title={t(`flow.fileFormatOptions.${fileFormat}`)} defaultOpen>
+ <Collapse
+ title={t(`flow.fileFormatOptions.${fileFormat}`)}
+ defaultOpen
+ rightContent={
+ canRemove ? (
+ <Button
+ type="button"
+ variant="ghost"
+ size="icon-xs"
+ onClick={handleRemove}
+ >
+ <LucideTrash2 className="size-4" />
+ </Button>
+ ) : undefined
+ }
+ >
<div className="space-y-5">
<Widget prefix={prefix} fileType={fileFormat as FileType}></Widget>
</div>
@@ -150,18 +115,43 @@ function ParserItem({ name, index }: ParserItemProps) {
);
}
+type ParserFormProps = INextOperatorForm & {
+ // Dataset-side embeddings (settings page, document pipeline dialog) show a
+ // fixed set of file types; only the canvas parser allows add/remove.
+ fixedFileFormats?: boolean;
+};
+
+type AddFileTypeMenuItemProps = {
+ fileType: FileType;
+ onSelect: (fileType: FileType) => void;
+};
+
+function AddFileTypeMenuItem({ fileType, onSelect }: AddFileTypeMenuItemProps) {
+ const { t } = useTranslation();
+ const handleSelect = useCallback(() => {
+ onSelect(fileType);
+ }, [onSelect, fileType]);
+
+ return (
+ <DropdownMenuItem onSelect={handleSelect}>
+ {t(`flow.fileFormatOptions.${fileType}`)}
+ </DropdownMenuItem>
+ );
+}
+
const ParserForm = ({
node,
onValuesChange,
hideOutputs,
externalErrors,
-}: INextOperatorForm) => {
+ fixedFileFormats,
+}: ParserFormProps) => {
+ const { t } = useTranslation();
const defaultModelDictionary = useFetchDefaultModelDictionary();
- const formValues = useFormValues(initialParserValues, node);
- const defaultValues = useMemo(
- () => withDefaultParserModels(formValues, defaultModelDictionary),
- [formValues, defaultModelDictionary],
- );
+ // Show the saved values as-is: an empty llm_id means the user cleared the
+ // model, and the backend falls back to the tenant default at parse time.
+ // Prefilling it here would make a cleared model reappear and be written back.
+ const defaultValues = useFormValues(initialParserValues, node);
const form = useForm<z.infer<typeof FormSchema>>({
defaultValues,
@@ -172,7 +162,7 @@ const ParserForm = ({
useSyncExternalFormErrors(form, externalErrors);
const name = 'setups';
- const { fields } = useFieldArray({
+ const { fields, append, remove } = useFieldArray({
name,
control: form.control,
});
@@ -180,14 +170,58 @@ const ParserForm = ({
useWatchFormChange(node?.id, form);
useFormChangeCallback(form, onValuesChange);
+ const remainingFileTypes = useMemo(() => {
+ const presentFileTypes = new Set(fields.map((x) => x.fileFormat));
+ return initialParserValues.setups
+ .map((x) => x.fileFormat)
+ .filter(
+ (fileType): fileType is FileType =>
+ !!fileType && !presentFileTypes.has(fileType),
+ );
+ }, [fields]);
+
+ const handleAddFileType = useCallback(
+ (fileType: FileType) => {
+ const setup = buildInitialParserSetup(fileType, defaultModelDictionary);
+ if (setup) {
+ append(setup);
+ }
+ },
+ [append, defaultModelDictionary],
+ );
+
+ const canRemove = !fixedFileFormats && fields.length > 1;
+
return (
<Form {...form}>
<form className="space-y-5 px-5">
{fields.map((field, index) => {
return (
- <ParserItem key={field.id} name={name} index={index}></ParserItem>
+ <ParserItem
+ key={field.id}
+ name={name}
+ index={index}
+ canRemove={canRemove}
+ onRemove={remove}
+ ></ParserItem>
);
})}
+ {!fixedFileFormats && remainingFileTypes.length > 0 && (
+ <DropdownMenu>
+ <DropdownMenuTrigger asChild>
+ <BlockButton type="button">{t('flow.addFileType')}</BlockButton>
+ </DropdownMenuTrigger>
+ <DropdownMenuContent>
+ {remainingFileTypes.map((fileType) => (
+ <AddFileTypeMenuItem
+ key={fileType}
+ fileType={fileType}
+ onSelect={handleAddFileType}
+ />
+ ))}
+ </DropdownMenuContent>
+ </DropdownMenu>
+ )}
</form>
{!hideOutputs && (
<div className="p-5">
diff --git a/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx b/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx
index 01bca52d70..a3fbb2a022 100644
--- a/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/pdf-form-fields.tsx
@@ -14,7 +14,6 @@ import { useEffect, useMemo } from 'react';
import { useFormContext, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { useOwnerTenantId } from '../../context';
-import { FileType } from '../../constant/pipeline';
import {
FlattenMediaToTextFormField,
LanguageFormField,
@@ -26,7 +25,7 @@ import {
import { CommonProps } from './interface';
import { DynamicPageRange } from './dynamic-page-range';
import { useSetInitialLanguage } from './use-set-initial-language';
-import { buildFieldNameWithPrefix, isForeignParseMethod } from './utils';
+import { buildFieldNameWithPrefix } from './utils';
const tableResultTypeOptions: SelectWithSearchFlagOptionType[] = [
{ label: 'Markdown', value: '0' },
@@ -69,19 +68,6 @@ export function PdfFormFields({ prefix }: CommonProps) {
useSetInitialLanguage({ prefix, languageShown });
- useEffect(() => {
- const current = form.getValues(parseMethodName);
- // On a file-type switch the field remounts and react-hook-form re-seeds it
- // from the node's saved form data, so it can hold another file type's
- // static parse method (e.g. ocr) — reset it to DeepDOC in that case too.
- if (isEmpty(current) || isForeignParseMethod(FileType.PDF, current)) {
- form.setValue(parseMethodName, ParseDocumentType.DeepDOC, {
- shouldValidate: true,
- shouldDirty: true,
- });
- }
- }, [form, parseMethodName]);
-
// Set default values for TCADP options when TCADP is selected
useEffect(() => {
if (tcadpOptionsShown) {
diff --git a/web/src/pages/agent/form/parser-form/ppt-form-fields.tsx b/web/src/pages/agent/form/parser-form/ppt-form-fields.tsx
index 22c35e8bbf..2b8153ed2f 100644
--- a/web/src/pages/agent/form/parser-form/ppt-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/ppt-form-fields.tsx
@@ -8,10 +8,9 @@ import { isEmpty } from 'lodash';
import { useEffect, useMemo } from 'react';
import { useFormContext, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
-import { FileType } from '../../constant/pipeline';
import { ParserMethodFormField } from './common-form-fields';
import { CommonProps } from './interface';
-import { buildFieldNameWithPrefix, isForeignParseMethod } from './utils';
+import { buildFieldNameWithPrefix } from './utils';
const tableResultTypeOptions: SelectWithSearchFlagOptionType[] = [
{ label: 'Markdown', value: '0' },
@@ -48,22 +47,6 @@ export function PptFormFields({ prefix }: CommonProps) {
);
}, [parseMethod]);
- useEffect(() => {
- const current = form.getValues(parseMethodName);
- // On a file-type switch the field remounts and react-hook-form re-seeds it
- // from the node's saved form data, so it can hold another file type's
- // static parse method (e.g. ocr) — reset it to DeepDOC in that case too.
- if (
- isEmpty(current) ||
- isForeignParseMethod(FileType.PowerPoint, current)
- ) {
- form.setValue(parseMethodName, ParseDocumentType.DeepDOC, {
- shouldValidate: true,
- shouldDirty: true,
- });
- }
- }, [form, parseMethodName]);
-
// Set default values for TCADP options when TCADP is selected
useEffect(() => {
if (tcadpOptionsShown) {
diff --git a/web/src/pages/agent/form/parser-form/schema.test.ts b/web/src/pages/agent/form/parser-form/schema.test.ts
new file mode 100644
index 0000000000..6e5001283e
--- /dev/null
+++ b/web/src/pages/agent/form/parser-form/schema.test.ts
@@ -0,0 +1,37 @@
+import { FileType, ParserFields } from '../../constant/pipeline';
+import { FormSchema } from './schema';
+
+describe('parser FormSchema', () => {
+ it('accepts video and audio setups without a model', () => {
+ const result = FormSchema.safeParse({
+ setups: [
+ { fileFormat: FileType.Video, output_format: 'text' },
+ { fileFormat: FileType.Audio, output_format: 'text' },
+ ],
+ });
+ expect(result.success).toBe(true);
+ });
+
+ it('requires at least one file type', () => {
+ const result = FormSchema.safeParse({ setups: [] });
+ expect(result.success).toBe(false);
+ });
+
+ it('still requires fields for email', () => {
+ const result = FormSchema.safeParse({
+ setups: [{ fileFormat: FileType.Email, output_format: 'text' }],
+ });
+ expect(result.success).toBe(false);
+
+ const withFields = FormSchema.safeParse({
+ setups: [
+ {
+ fileFormat: FileType.Email,
+ output_format: 'text',
+ fields: Object.values(ParserFields),
+ },
+ ],
+ });
+ expect(withFields.success).toBe(true);
+ });
+});
diff --git a/web/src/pages/agent/form/parser-form/schema.ts b/web/src/pages/agent/form/parser-form/schema.ts
new file mode 100644
index 0000000000..8dc4163514
--- /dev/null
+++ b/web/src/pages/agent/form/parser-form/schema.ts
@@ -0,0 +1,59 @@
+import i18n from '@/locales/config';
+import { z } from 'zod';
+import { FileType } from '../../constant/pipeline';
+
+export const SetupSchema = z
+ .object({
+ fileFormat: z.string().nullish(),
+ // preprocess: z.array(z.string()).optional(),
+ output_format: z.string().optional(),
+ parse_method: z.string().optional(),
+ lang: z.string().optional(),
+ fields: z.array(z.string()).optional(),
+ vlm: z.object({ llm_id: z.string().optional() }).optional(),
+ flatten_media_to_text: z.boolean().optional(),
+ system_prompt: z.string().optional(),
+ table_result_type: z.string().optional(),
+ markdown_image_response_type: z.string().optional(),
+ enable_multi_column: z.boolean().optional(),
+ remove_toc: z.boolean().optional(),
+ remove_header_footer: z.boolean().optional(),
+ pages: z
+ .array(
+ z
+ .object({
+ // Keep these checks on the fields themselves: an object-level
+ // `superRefine` is skipped whenever the base shape fails to
+ // parse, so one missing sibling would silently swallow the
+ // other field's error.
+ from: z.coerce
+ .number()
+ .int(i18n.t('knowledgeDetails.pageRangeFromInvalid'))
+ .min(1, i18n.t('knowledgeDetails.pageRangeFromInvalid')),
+ to: z.coerce
+ .number()
+ .int(i18n.t('knowledgeDetails.pageRangeToInvalid'))
+ .min(1, i18n.t('knowledgeDetails.pageRangeToInvalid')),
+ })
+ .refine(({ from, to }) => to >= from, {
+ path: ['to'],
+ message: i18n.t('knowledgeDetails.pageRangeToInvalid'),
+ }),
+ )
+ .optional(),
+ })
+ .superRefine((values, ctx) => {
+ if (values.fileFormat === FileType.Email && !values.fields?.length) {
+ ctx.addIssue({
+ code: z.ZodIssueCode.custom,
+ path: ['fields'],
+ message: 'Fields is required',
+ });
+ }
+ });
+
+export const FormSchema = z.object({
+ setups: z.array(SetupSchema).min(1, i18n.t('flow.atLeastOneFileType')),
+});
+
+export type ParserFormSchemaType = z.infer<typeof FormSchema>;
diff --git a/web/src/pages/agent/form/parser-form/spreadsheet-form-fields.tsx b/web/src/pages/agent/form/parser-form/spreadsheet-form-fields.tsx
index 0e28ba2490..934d5526e5 100644
--- a/web/src/pages/agent/form/parser-form/spreadsheet-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/spreadsheet-form-fields.tsx
@@ -13,13 +13,12 @@ import { useEffect, useMemo } from 'react';
import { useFormContext, useWatch } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import { useOwnerTenantId } from '../../context';
-import { FileType } from '../../constant/pipeline';
import {
FlattenMediaToTextFormField,
ParserMethodFormField,
} from './common-form-fields';
import { CommonProps } from './interface';
-import { buildFieldNameWithPrefix, isForeignParseMethod } from './utils';
+import { buildFieldNameWithPrefix } from './utils';
const tableResultTypeOptions: SelectWithSearchFlagOptionType[] = [
{ label: 'Markdown', value: '0' },
@@ -60,22 +59,6 @@ export function SpreadsheetFormFields({ prefix }: CommonProps) {
);
}, [parseMethod]);
- useEffect(() => {
- const current = form.getValues(parseMethodName);
- // On a file-type switch the field remounts and react-hook-form re-seeds it
- // from the node's saved form data, so it can hold another file type's
- // static parse method (e.g. ocr) — reset it to DeepDOC in that case too.
- if (
- isEmpty(current) ||
- isForeignParseMethod(FileType.Spreadsheet, current)
- ) {
- form.setValue(parseMethodName, ParseDocumentType.DeepDOC, {
- shouldValidate: true,
- shouldDirty: true,
- });
- }
- }, [form, parseMethodName]);
-
// Set default values for TCADP options when TCADP is selected
useEffect(() => {
if (tcadpOptionsShown) {
diff --git a/web/src/pages/agent/form/parser-form/utils.test.ts b/web/src/pages/agent/form/parser-form/utils.test.ts
index 8ee78f28ae..a2df564e93 100644
--- a/web/src/pages/agent/form/parser-form/utils.test.ts
+++ b/web/src/pages/agent/form/parser-form/utils.test.ts
@@ -1,61 +1,105 @@
import { ParseDocumentType } from '@/components/layout-recognize-form-field';
-import { FileType, ImageParseMethod } from '../../constant/pipeline';
-import { getInitialParseMethod, isForeignParseMethod } from './utils';
+import { ModelTypeToField } from '@/constants/llm';
+import {
+ FileType,
+ ImageParseMethod,
+ initialParserValues,
+} from '../../constant/pipeline';
+import {
+ buildInitialParserSetup,
+ buildInitialParserValues,
+ isStaticParseMethod,
+} from './utils';
describe('parser-form utils', () => {
- describe('getInitialParseMethod', () => {
- it('returns ocr for image', () => {
- expect(getInitialParseMethod(FileType.Image)).toBe(ImageParseMethod.OCR);
+ describe('isStaticParseMethod', () => {
+ it('recognizes the static parse methods', () => {
+ expect(isStaticParseMethod(ImageParseMethod.OCR)).toBe(true);
+ expect(isStaticParseMethod(ParseDocumentType.DeepDOC)).toBe(true);
+ expect(isStaticParseMethod(ParseDocumentType.PlainText)).toBe(true);
+ expect(isStaticParseMethod(ParseDocumentType.Docling)).toBe(true);
+ expect(isStaticParseMethod(ParseDocumentType.OpenDataLoader)).toBe(true);
+ expect(isStaticParseMethod(ParseDocumentType.TCADPParser)).toBe(true);
});
- it('returns DeepDOC for pdf/spreadsheet/powerpoint', () => {
- expect(getInitialParseMethod(FileType.PDF)).toBe(
- ParseDocumentType.DeepDOC,
- );
- expect(getInitialParseMethod(FileType.Spreadsheet)).toBe(
- ParseDocumentType.DeepDOC,
- );
- expect(getInitialParseMethod(FileType.PowerPoint)).toBe(
- ParseDocumentType.DeepDOC,
+ it('treats LLM model ids and empty values as non-static', () => {
+ expect(isStaticParseMethod('gpt-4o@OpenAI')).toBe(false);
+ expect(isStaticParseMethod('')).toBe(false);
+ expect(isStaticParseMethod(undefined)).toBe(false);
+ expect(isStaticParseMethod(null)).toBe(false);
+ });
+ });
+
+ describe('buildInitialParserSetup', () => {
+ it('returns a deep copy of the default setup', () => {
+ const setup = buildInitialParserSetup(FileType.PDF, {});
+ expect(setup?.fileFormat).toBe(FileType.PDF);
+ // Mutating the copy must not touch the shared defaults
+ (setup as any).pages[0].from = 5;
+ expect(buildInitialParserSetup(FileType.PDF, {})?.pages?.[0].from).toBe(
+ 1,
);
});
- it('returns empty string for file types without a parse method', () => {
- expect(getInitialParseMethod(FileType.Email)).toBe('');
+ it('prefills the tenant default model for video and audio', () => {
+ const dictionary = {
+ [ModelTypeToField.vision]: 'gpt-4o@OpenAI',
+ [ModelTypeToField.asr]: 'whisper@OpenAI',
+ };
+ expect(buildInitialParserSetup(FileType.Video, dictionary)?.vlm).toEqual({
+ llm_id: 'gpt-4o@OpenAI',
+ });
+ expect(buildInitialParserSetup(FileType.Audio, dictionary)?.vlm).toEqual({
+ llm_id: 'whisper@OpenAI',
+ });
});
- });
- describe('isForeignParseMethod', () => {
- it('treats a static method from another file type as foreign', () => {
- // The bug: switching pdf -> image left DeepDOC in parse_method
- expect(
- isForeignParseMethod(FileType.Image, ParseDocumentType.DeepDOC),
- ).toBe(true);
- expect(isForeignParseMethod(FileType.PDF, ImageParseMethod.OCR)).toBe(
- true,
+ it('leaves other file types untouched even with defaults present', () => {
+ const dictionary = {
+ [ModelTypeToField.vision]: 'gpt-4o@OpenAI',
+ [ModelTypeToField.asr]: 'whisper@OpenAI',
+ };
+ expect(buildInitialParserSetup(FileType.PDF, dictionary)?.vlm).toBe(
+ undefined,
);
});
- it('accepts the file type’s own initial method', () => {
- expect(isForeignParseMethod(FileType.Image, ImageParseMethod.OCR)).toBe(
- false,
- );
- expect(
- isForeignParseMethod(FileType.PDF, ParseDocumentType.DeepDOC),
- ).toBe(false);
+ it('keeps the empty model id when no tenant default exists', () => {
+ expect(buildInitialParserSetup(FileType.Audio, {})?.vlm).toEqual({
+ llm_id: '',
+ });
+ });
+
+ it('returns undefined for a file type without a default setup', () => {
+ expect(buildInitialParserSetup('nope' as FileType, {})).toBeUndefined();
});
+ });
+
+ describe('buildInitialParserValues', () => {
+ it('prefills tenant default models across every default file type', () => {
+ const values = buildInitialParserValues({
+ [ModelTypeToField.vision]: 'gpt-4o@OpenAI',
+ [ModelTypeToField.asr]: 'whisper@OpenAI',
+ });
- it('never treats an LLM model id as foreign', () => {
- expect(isForeignParseMethod(FileType.PDF, 'gpt-4o@OpenAI')).toBe(false);
- expect(isForeignParseMethod(FileType.Image, 'deepseek-v3@Moonshot')).toBe(
- false,
+ const byFileType = new Map(
+ values.setups.map((setup: any) => [setup.fileFormat, setup]),
);
+ expect(byFileType.get(FileType.Video)?.vlm).toEqual({
+ llm_id: 'gpt-4o@OpenAI',
+ });
+ expect(byFileType.get(FileType.Audio)?.vlm).toEqual({
+ llm_id: 'whisper@OpenAI',
+ });
+ expect(byFileType.get(FileType.PDF)?.vlm).toBe(undefined);
});
- it('ignores empty and non-string values', () => {
- expect(isForeignParseMethod(FileType.Image, '')).toBe(false);
- expect(isForeignParseMethod(FileType.Image, undefined)).toBe(false);
- expect(isForeignParseMethod(FileType.Image, null)).toBe(false);
+ it('keeps every default file type and does not mutate the defaults', () => {
+ const values = buildInitialParserValues({});
+ expect(values.setups).toHaveLength(initialParserValues.setups.length);
+ expect(initialParserValues.setups).toEqual(
+ buildInitialParserValues({}).setups,
+ );
});
});
});
diff --git a/web/src/pages/agent/form/parser-form/utils.ts b/web/src/pages/agent/form/parser-form/utils.ts
index d840f757dd..06b4311739 100644
--- a/web/src/pages/agent/form/parser-form/utils.ts
+++ b/web/src/pages/agent/form/parser-form/utils.ts
@@ -1,4 +1,5 @@
import { ParseDocumentType } from '@/components/layout-recognize-form-field';
+import { cloneDeep } from 'lodash';
import {
FileType,
FileTypeDefaultModelFieldMap,
@@ -10,39 +11,9 @@ export function buildFieldNameWithPrefix(name: string, prefix: string) {
return `${prefix}.${name}`;
}
-export function withDefaultParserModels(
- formValues: Record<string, any>,
- defaultModelDictionary: Record<string, string>,
-) {
- const setups = formValues?.setups;
- if (!Array.isArray(setups)) {
- return formValues;
- }
-
- return {
- ...formValues,
- setups: setups.map((setup) => {
- const field = FileTypeDefaultModelFieldMap[setup?.fileFormat as FileType];
- const modelId = field ? defaultModelDictionary[field] : '';
- if (!modelId || setup?.vlm?.llm_id) {
- return setup;
- }
- return { ...setup, vlm: { ...setup?.vlm, llm_id: modelId } };
- }),
- };
-}
-
-export function getInitialParseMethod(fileType: FileType): string {
- const setup = initialParserValues.setups.find(
- (x) => x.fileFormat === fileType,
- );
- return setup?.parse_method ?? '';
-}
-
-// Static parse-method values across all file types. Forms saved while the
-// file type was still switchable can hold another file type's static value on
-// parse_method. LLM model ids from the model tree are never in this set, so a
-// user-picked model is never treated as foreign.
+// Static parse-method values across all file types. LLM model ids from the
+// model tree are never in this set, so `isStaticParseMethod` can tell a
+// built-in method apart from a user-picked model.
// Note: ParseDocumentType is a const enum — list members explicitly instead of
// Object.values, which is not allowed on const enums (TS2475).
const KnownStaticParseMethods = new Set<string>([
@@ -54,13 +25,46 @@ const KnownStaticParseMethods = new Set<string>([
ImageParseMethod.OCR,
]);
-export function isForeignParseMethod(
+export function isStaticParseMethod(value: unknown): value is string {
+ return typeof value === 'string' && KnownStaticParseMethods.has(value);
+}
+
+// Builds the default setup for a file type being added to the parser form,
+// prefilling the tenant default model for types that have one (video/audio).
+export function buildInitialParserSetup(
fileType: FileType,
- value: unknown,
-): value is string {
- return (
- typeof value === 'string' &&
- KnownStaticParseMethods.has(value) &&
- value !== getInitialParseMethod(fileType)
+ defaultModelDictionary: Record<string, string>,
+) {
+ const setup = initialParserValues.setups.find(
+ (x) => x.fileFormat === fileType,
);
+ if (!setup) {
+ return undefined;
+ }
+
+ const nextSetup = cloneDeep(setup) as Record<string, any>;
+ const field = FileTypeDefaultModelFieldMap[fileType];
+ const modelId = field ? defaultModelDictionary[field] : '';
+ if (modelId) {
+ nextSetup.vlm = { ...nextSetup.vlm, llm_id: modelId };
+ }
+ return nextSetup;
+}
+
+// Form values for a freshly added Parser node: every default file type with its
+// tenant default model prefilled. Only applies to node creation — an existing
+// form is shown as saved, so a cleared model stays cleared.
+export function buildInitialParserValues(
+ defaultModelDictionary: Record<string, string>,
+) {
+ return {
+ ...initialParserValues,
+ setups: initialParserValues.setups.map(
+ (setup) =>
+ buildInitialParserSetup(
+ setup.fileFormat as FileType,
+ defaultModelDictionary,
+ ) ?? setup,
+ ),
+ };
}
diff --git a/web/src/pages/agent/form/parser-form/video-form-fields.tsx b/web/src/pages/agent/form/parser-form/video-form-fields.tsx
index bd45e11729..ee536c45be 100644
--- a/web/src/pages/agent/form/parser-form/video-form-fields.tsx
+++ b/web/src/pages/agent/form/parser-form/video-form-fields.tsx
@@ -20,7 +20,6 @@ export function AudioFormFields({ prefix }: OutputFormatFormFieldProps) {
modelTypes={ModelTypeMap.asr_id}
allowClear
ownerTenantId={ownerTenantId}
- required
/>
</>
);
@@ -39,7 +38,6 @@ export function VideoFormFields({ prefix }: OutputFormatFormFieldProps) {
modelTypes={ModelTypeMap.img2txt_id}
allowClear
ownerTenantId={ownerTenantId}
- required
/>
</>
);
diff --git a/web/src/pages/agent/hooks/use-add-node.ts b/web/src/pages/agent/hooks/use-add-node.ts
index d76f1408a8..e79a2043e4 100644
--- a/web/src/pages/agent/hooks/use-add-node.ts
+++ b/web/src/pages/agent/hooks/use-add-node.ts
@@ -38,7 +38,6 @@ import {
initialLoopValues,
initialMessageValues,
initialNoteValues,
- initialParserValues,
initialPubMedValues,
initialBGPTValues,
initialQueritContentsValues,
@@ -61,7 +60,7 @@ import {
initialWikipediaValues,
initialYahooFinanceValues,
} from '../constant';
-import { withDefaultParserModels } from '../form/parser-form/utils';
+import { buildInitialParserValues } from '../form/parser-form/utils';
import useGraphStore from '../store';
import {
generateNodeNamesWithIncreasingIndex,
@@ -183,10 +182,7 @@ export const useInitializeOperatorParams = () => {
[Operator.TavilyExtract]: initialTavilyExtractValues,
[Operator.Placeholder]: {},
[Operator.File]: {},
- [Operator.Parser]: withDefaultParserModels(
- initialParserValues,
- defaultModelDictionary,
- ),
+ [Operator.Parser]: buildInitialParserValues(defaultModelDictionary),
[Operator.Tokenizer]: initialTokenizerValues,
[Operator.TokenChunker]: initialTokenChunkerValues,
[Operator.TitleChunker]: initialTitleChunkerValues,
diff --git a/web/src/pages/dataset/dataset/index.tsx b/web/src/pages/dataset/dataset/index.tsx
index ad85962e39..05ec190fc8 100644
--- a/web/src/pages/dataset/dataset/index.tsx
+++ b/web/src/pages/dataset/dataset/index.tsx
@@ -260,10 +260,6 @@ export default function Dataset() {
)}
{reparseDialogVisible && (
<ReparseDialog
- hidden={
- chunkNum === 0 && !knowledgeBase?.parser_config?.enable_metadata
- }
- // hidden={false}
enable_metadata={knowledgeBase?.parser_config?.enable_metadata}
handleOperationIconClick={handleOperationIconClick}
chunk_num={chunkNum}
diff --git a/web/src/pages/dataset/dataset/parser-model-gap-content.tsx b/web/src/pages/dataset/dataset/parser-model-gap-content.tsx
new file mode 100644
index 0000000000..317032b1b1
--- /dev/null
+++ b/web/src/pages/dataset/dataset/parser-model-gap-content.tsx
@@ -0,0 +1,35 @@
+import type { TFunction } from 'i18next';
+import type { ReactNode } from 'react';
+import type { FileModelGap } from './utils';
+
+/**
+ * Builds the body of the missing-model modals (upload warning, parse-click
+ * error). Strings are resolved with the caller's `t` because the static Modal
+ * API renders content in a separate React root without the app providers.
+ */
+export function buildMissingModelModalContent(
+ t: TFunction,
+ gaps: FileModelGap[],
+ hintKey: string,
+): ReactNode {
+ return (
+ <div className="space-y-2">
+ <ul className="list-disc pl-4 space-y-1">
+ {gaps.map((gap, index) => (
+ <li key={`${gap.name}-${index}`}>
+ {t('knowledgeDetails.fileModelMissing', {
+ name: gap.name,
+ fileType: t(`flow.fileFormatOptions.${gap.fileType}`),
+ model: t(
+ gap.modelKind === 'asr'
+ ? 'knowledgeDetails.missingModelAsr'
+ : 'knowledgeDetails.missingModelVision',
+ ),
+ })}
+ </li>
+ ))}
+ </ul>
+ <p className="text-text-secondary">{t(hintKey)}</p>
+ </div>
+ );
+}
diff --git a/web/src/pages/dataset/dataset/parsing-status-cell.test.tsx b/web/src/pages/dataset/dataset/parsing-status-cell.test.tsx
index c9285811cb..f3ab0182b6 100644
--- a/web/src/pages/dataset/dataset/parsing-status-cell.test.tsx
+++ b/web/src/pages/dataset/dataset/parsing-status-cell.test.tsx
@@ -1,5 +1,5 @@
-import { render, screen } from '@testing-library/react';
-import React from 'react';
+import { fireEvent, render, screen } from '@testing-library/react';
+import React, { StrictMode } from 'react';
jest.mock('react-i18next', () => ({
useTranslation: () => ({ t: (key: string) => key }),
@@ -9,11 +9,14 @@ jest.mock('./reparse-dialog', () => ({
ReparseDialog: () => null,
}));
+const mockRunDocumentByIds = jest.fn();
+const mockShowReparseDialog = jest.fn();
+
jest.mock('./use-run-document', () => ({
useHandleRunDocumentByIds: () => ({
- handleRunDocumentByIds: jest.fn(),
+ handleRunDocumentByIds: mockRunDocumentByIds,
visible: false,
- showModal: jest.fn(),
+ showModal: mockShowReparseDialog,
hideModal: jest.fn(),
}),
}));
@@ -21,43 +24,57 @@ jest.mock('./use-run-document', () => ({
import { IngestionTaskStatus, RunningStatus } from './constant';
import { ParsingStatusCell } from './parsing-status-cell';
+const baseRecord = {
+ id: 'doc-1',
+ dataset_id: 'dataset-1',
+ name: 'doc.pdf',
+ type: 'document',
+ run: RunningStatus.UNSTART,
+ ingestion_status: IngestionTaskStatus.RUNNING,
+ progress: 0,
+ chunk_count: 0,
+ parser_config: {},
+ create_date: '',
+ create_time: 0,
+ created_by: 'user-1',
+ nickname: '',
+ location: '',
+ pipeline_id: '',
+ pipeline_name: '',
+ process_duration: 0,
+ progress_msg: '',
+ size: 0,
+ source_type: 'local',
+ status: '1',
+ suffix: 'pdf',
+ thumbnail: '',
+ token_num: 0,
+ update_date: '',
+ update_time: 0,
+ chunk_method: 'naive',
+};
+
+function renderCell(overrides: Record<string, any>) {
+ return render(
+ React.createElement(ParsingStatusCell, {
+ record: { ...baseRecord, ...overrides } as any,
+ showLog: jest.fn(),
+ showChangeParserModal: jest.fn(),
+ }),
+ { wrapper: StrictMode },
+ );
+}
+
describe('ParsingStatusCell', () => {
+ beforeEach(() => {
+ mockRunDocumentByIds.mockClear();
+ mockShowReparseDialog.mockClear();
+ });
+
it('shows a cancel icon while ingestion runs before the document run state updates', () => {
- const { container } = render(
- React.createElement(ParsingStatusCell, {
- record: {
- id: 'doc-1',
- dataset_id: 'dataset-1',
- name: 'doc.pdf',
- type: 'document',
- run: RunningStatus.UNSTART,
- ingestion_status: IngestionTaskStatus.RUNNING,
- progress: 0,
- chunk_count: 0,
- parser_config: {},
- create_date: '',
- create_time: 0,
- created_by: 'user-1',
- nickname: '',
- location: '',
- pipeline_id: '',
- pipeline_name: '',
- process_duration: 0,
- progress_msg: '',
- size: 0,
- source_type: 'local',
- status: '1',
- suffix: 'pdf',
- thumbnail: '',
- token_num: 0,
- update_date: '',
- update_time: 0,
- chunk_method: 'naive',
- },
- showLog: jest.fn(),
- showChangeParserModal: jest.fn(),
- }),
- );
+ const { container } = renderCell({
+ ingestion_status: IngestionTaskStatus.RUNNING,
+ });
expect(container.querySelector('svg.lucide-circle-x')).toBeInTheDocument();
expect(
@@ -66,41 +83,10 @@ describe('ParsingStatusCell', () => {
});
it('does not expose cancelling as a document status', () => {
- const { container } = render(
- React.createElement(ParsingStatusCell, {
- record: {
- id: 'doc-1',
- dataset_id: 'dataset-1',
- name: 'doc.pdf',
- type: 'document',
- run: RunningStatus.RUNNING,
- ingestion_status: IngestionTaskStatus.STOPPING,
- progress: 0,
- chunk_count: 0,
- parser_config: {},
- create_date: '',
- create_time: 0,
- created_by: 'user-1',
- nickname: '',
- location: '',
- pipeline_id: '',
- pipeline_name: '',
- process_duration: 0,
- progress_msg: '',
- size: 0,
- source_type: 'local',
- status: '1',
- suffix: 'pdf',
- thumbnail: '',
- token_num: 0,
- update_date: '',
- update_time: 0,
- chunk_method: 'naive',
- },
- showLog: jest.fn(),
- showChangeParserModal: jest.fn(),
- }),
- );
+ const { container } = renderCell({
+ run: RunningStatus.RUNNING,
+ ingestion_status: IngestionTaskStatus.STOPPING,
+ });
expect(
screen.queryByText('knowledgeDetails.runningStatusStopping'),
@@ -108,4 +94,38 @@ describe('ParsingStatusCell', () => {
expect(container.querySelector('svg.lucide-circle-x')).toBeInTheDocument();
expect(container.querySelector('button[disabled]')).toBeInTheDocument();
});
+
+ it('runs a chunkless document once without opening the confirmation', () => {
+ renderCell({ ingestion_status: IngestionTaskStatus.COMPLETED });
+
+ fireEvent.click(screen.getByTestId('document-parse-toggle'));
+
+ expect(mockRunDocumentByIds).toHaveBeenCalledTimes(1);
+ expect(mockShowReparseDialog).not.toHaveBeenCalled();
+ });
+
+ it('opens the confirmation when existing chunks would be dropped', () => {
+ renderCell({
+ run: RunningStatus.DONE,
+ ingestion_status: IngestionTaskStatus.COMPLETED,
+ chunk_count: 3,
+ });
+
+ fireEvent.click(screen.getByTestId('document-parse-toggle'));
+
+ expect(mockShowReparseDialog).toHaveBeenCalledTimes(1);
+ expect(mockRunDocumentByIds).not.toHaveBeenCalled();
+ });
+
+ it('opens the confirmation for a chunkless document when auto-metadata applies', () => {
+ renderCell({
+ ingestion_status: IngestionTaskStatus.COMPLETED,
+ parser_config: { enable_metadata: true },
+ });
+
+ fireEvent.click(screen.getByTestId('document-parse-toggle'));
+
+ expect(mockShowReparseDialog).toHaveBeenCalledTimes(1);
+ expect(mockRunDocumentByIds).not.toHaveBeenCalled();
+ });
});
diff --git a/web/src/pages/dataset/dataset/parsing-status-cell.tsx b/web/src/pages/dataset/dataset/parsing-status-cell.tsx
index d3acbf0eda..e1efa92314 100644
--- a/web/src/pages/dataset/dataset/parsing-status-cell.tsx
+++ b/web/src/pages/dataset/dataset/parsing-status-cell.tsx
@@ -130,7 +130,22 @@ export function ParsingStatusCell({
delete: boolean;
apply_kb: boolean;
}) => {
- handleRunDocumentByIds(record.id, isRunning, option);
+ handleRunDocumentByIds(record, isRunning, option);
+ };
+
+ // The confirmation only offers real choices when there are existing chunks to
+ // drop or auto-metadata to re-apply. Otherwise, and always when cancelling a
+ // run, the action fires straight away.
+ const needsParseConfirm =
+ !isRunning &&
+ (!isZeroChunk || Boolean(record?.parser_config?.enable_metadata));
+
+ const handleParseClick = () => {
+ if (needsParseConfirm) {
+ showReparseDialogModal();
+ return;
+ }
+ handleOperationIconClick();
};
const showParse = useMemo(() => {
@@ -181,7 +196,8 @@ export function ParsingStatusCell({
variant="ghost"
size="icon-xs"
disabled={isStopping}
- onClick={() => showReparseDialogModal()}
+ onClick={handleParseClick}
+ data-testid="document-parse-toggle"
>
<CircleX
color="rgba(var(--state-error))"
@@ -194,9 +210,8 @@ export function ParsingStatusCell({
<Button
variant="ghost"
size="icon-xs"
- onClick={() => {
- showReparseDialogModal();
- }}
+ onClick={handleParseClick}
+ data-testid="document-parse-toggle"
>
{operationIcon}
</Button>
@@ -208,10 +223,6 @@ export function ParsingStatusCell({
)}
{reparseDialogVisible && (
<ReparseDialog
- hidden={
- (isZeroChunk && !record?.parser_config?.enable_metadata) ||
- isRunning
- }
enable_metadata={record?.parser_config?.enable_metadata}
handleOperationIconClick={handleOperationIconClick}
chunk_num={chunk_count}
diff --git a/web/src/pages/dataset/dataset/reparse-dialog.tsx b/web/src/pages/dataset/dataset/reparse-dialog.tsx
index 1485485e8f..ddbd869a19 100644
--- a/web/src/pages/dataset/dataset/reparse-dialog.tsx
+++ b/web/src/pages/dataset/dataset/reparse-dialog.tsx
@@ -16,7 +16,6 @@ export const ReparseDialog = memo(
handleOperationIconClick,
chunk_num,
enable_metadata = false,
- hidden = false,
visible = true,
hideModal,
}: DialogProps & {
@@ -28,24 +27,11 @@ export const ReparseDialog = memo(
enable_metadata?: boolean;
visible: boolean;
hideModal: () => void;
- hidden?: boolean;
}) => {
const [defaultValues, setDefaultValues] = useState<any>(null);
const [fields, setFields] = useState<FormFieldConfig[]>([]);
const { t } = useTranslation();
- const handleOperationIconClickRef = useRef(handleOperationIconClick);
- const hiddenRef = useRef(hidden);
- useEffect(() => {
- handleOperationIconClickRef.current = handleOperationIconClick;
- hiddenRef.current = hidden;
- });
-
- useEffect(() => {
- if (hiddenRef.current) {
- handleOperationIconClickRef.current();
- }
- }, []);
useEffect(() => {
setDefaultValues({
delete: chunk_num > 0,
@@ -139,7 +125,6 @@ export const ReparseDialog = memo(
title={t(`knowledgeDetails.parseFile`)}
onOk={() => handleSave()}
onCancel={() => handleCancel()}
- hidden={hidden}
open={visible}
okButtonText={t('common.confirm')}
content={{
diff --git a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
index d83c5bb918..69c324802d 100644
--- a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
+++ b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
@@ -1,3 +1,4 @@
+import { Modal } from '@/components/ui/modal/modal';
import { useSetModalState } from '@/hooks/common-hooks';
import {
UseRowSelectionType,
@@ -21,7 +22,10 @@ import { useCallback, useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useParams } from 'react-router';
import { toast } from 'sonner';
+import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';
import { DocumentType } from './constant';
+import { buildMissingModelModalContent } from './parser-model-gap-content';
+import { useParserModelValidation } from './use-parser-model-validation';
import { isDocumentProcessing } from './utils';
export function useBulkOperateDataset({
@@ -42,6 +46,9 @@ export function useBulkOperateDataset({
const { setDocumentStatus } = useSetDocumentStatus();
const { removeDocument } = useRemoveDocument();
const { visible, showModal, hideModal } = useSetModalState();
+ const { findFilesMissingModels, goToDatasetConfiguration } =
+ useParserModelValidation();
+ const { knowledgeBase } = useKnowledgeBaseContext();
const chunkNum = useMemo(() => {
if (!documents.length) {
@@ -65,6 +72,66 @@ export function useBulkOperateDataset({
toast.error(t('Please select a non-empty file list'));
return;
}
+
+ // Starting a parse requires the models of each file type to be
+ // configured; cancelling is always allowed.
+ if (run === 1) {
+ const selectedDocuments = documents.filter((x) =>
+ nonVirtualKeys.includes(x.id),
+ );
+ const gaps = findFilesMissingModels(
+ selectedDocuments.map((x) => x.name),
+ );
+ if (gaps.length > 0) {
+ hideModal();
+ const failingNames = new Set(gaps.map((gap) => gap.name));
+ const validIds = selectedDocuments
+ .filter((x) => !failingNames.has(x.name))
+ .map((x) => x.id);
+
+ if (validIds.length === 0) {
+ Modal.error({
+ title: t('knowledgeDetails.parseBlockedTitle'),
+ content: buildMissingModelModalContent(
+ t,
+ gaps,
+ 'knowledgeDetails.parseBlockedHint',
+ ),
+ okText: t('knowledgeDetails.goToConfiguration'),
+ cancelText: t('common.cancel'),
+ closable: false,
+ onOk: goToDatasetConfiguration,
+ });
+ return;
+ }
+
+ Modal.warning({
+ title: t('knowledgeDetails.parseBlockedPartialTitle'),
+ content: (
+ <div className="space-y-2">
+ {buildMissingModelModalContent(
+ t,
+ gaps,
+ 'knowledgeDetails.parseBlockedHint',
+ )}
+ <p>
+ {t('knowledgeDetails.parseValidFilesNote', {
+ count: validIds.length,
+ })}
+ </p>
+ </div>
+ ),
+ okText: t('knowledgeDetails.parseValidFiles'),
+ cancelText: t('common.cancel'),
+ closable: false,
+ onOk: async () => {
+ await runDocumentByIds({ documentIds: validIds, run, option });
+ },
+ });
+ return;
+ }
+ }
+
await runDocumentByIds({
documentIds: nonVirtualKeys,
run,
@@ -72,7 +139,15 @@ export function useBulkOperateDataset({
});
hideModal();
},
- [documents, runDocumentByIds, selectedRowKeys, hideModal, t],
+ [
+ documents,
+ runDocumentByIds,
+ selectedRowKeys,
+ hideModal,
+ t,
+ findFilesMissingModels,
+ goToDatasetConfiguration,
+ ],
);
const handleRunClick = useCallback(
@@ -82,6 +157,19 @@ export function useBulkOperateDataset({
[runDocument],
);
+ // The confirmation only offers real choices when the selection has existing
+ // chunks to drop or auto-metadata to re-apply; otherwise run straight away.
+ const needsRunConfirm =
+ chunkNum > 0 || Boolean(knowledgeBase?.parser_config?.enable_metadata);
+
+ const handleRunMenuClick = useCallback(() => {
+ if (needsRunConfirm) {
+ showModal();
+ return;
+ }
+ handleRunClick();
+ }, [needsRunConfirm, showModal, handleRunClick]);
+
const handleCancelClick = useCallback(() => {
runDocument(2);
}, [runDocument]);
@@ -136,7 +224,7 @@ export function useBulkOperateDataset({
id: 'run',
label: t('knowledgeDetails.run'),
icon: <LucidePlayCircle />,
- onClick: () => showModal(),
+ onClick: handleRunMenuClick,
},
{
id: 'cancel',
diff --git a/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx b/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx
index bc2755e354..f7e2e37971 100644
--- a/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx
+++ b/web/src/pages/dataset/dataset/use-dataset-table-columns.tsx
@@ -15,6 +15,7 @@ import { cn } from '@/lib/utils';
import { formatDate } from '@/utils/date';
import { ColumnDef } from '@tanstack/table-core';
import { ArrowUpDown } from 'lucide-react';
+import { useMemo } from 'react';
import { useTranslation } from 'react-i18next';
import { useParams } from 'react-router';
import { MetadataType } from '../components/metedata/constant';
@@ -46,208 +47,223 @@ export function useDatasetTableColumns({
const { setDocumentStatus } = useSetDocumentStatus();
const { id: datasetId } = useParams();
- const columns: ColumnDef<IDocumentInfo>[] = [
- {
- id: 'select',
- header: ({ table }) => (
- <Checkbox
- checked={
- table.getIsAllPageRowsSelected() ||
- (table.getIsSomePageRowsSelected() && 'indeterminate')
- }
- onCheckedChange={(value) => table.toggleAllPageRowsSelected(!!value)}
- aria-label="Select all"
- />
- ),
- cell: ({ row }) => (
- <Checkbox
- checked={row.getIsSelected()}
- onCheckedChange={(value) => row.toggleSelected(!!value)}
- aria-label="Select row"
- />
- ),
- enableSorting: false,
- enableHiding: false,
- },
- {
- accessorKey: 'name',
- header: ({ column }) => {
- return (
- <div className="flex items-center gap-1">
- {t('name')}
-
- <Button
- variant="ghost"
- size="icon-xs"
- onClick={() =>
- column.toggleSorting(column.getIsSorted() === 'asc')
- }
- >
- <ArrowUpDown />
- </Button>
- </div>
- );
+ const columns: ColumnDef<IDocumentInfo>[] = useMemo(
+ () => [
+ {
+ id: 'select',
+ header: ({ table }) => (
+ <Checkbox
+ checked={
+ table.getIsAllPageRowsSelected() ||
+ (table.getIsSomePageRowsSelected() && 'indeterminate')
+ }
+ onCheckedChange={(value) =>
+ table.toggleAllPageRowsSelected(!!value)
+ }
+ aria-label="Select all"
+ />
+ ),
+ cell: ({ row }) => (
+ <Checkbox
+ checked={row.getIsSelected()}
+ onCheckedChange={(value) => row.toggleSelected(!!value)}
+ aria-label="Select row"
+ />
+ ),
+ enableSorting: false,
+ enableHiding: false,
},
- meta: { cellClassName: 'max-w-[20vw]' },
- cell: ({ row }) => {
- const name: string = row.getValue('name');
+ {
+ accessorKey: 'name',
+ header: ({ column }) => {
+ return (
+ <div className="flex items-center gap-1">
+ {t('name')}
- return (
- <Tooltip>
- <TooltipTrigger asChild>
- <div
- className="flex items-center gap-2 cursor-pointer"
- onClick={navigateToChunkParsedResult(
- row.original.id,
- row.original.dataset_id,
- )}
+ <Button
+ variant="ghost"
+ size="icon-xs"
+ onClick={() =>
+ column.toggleSorting(column.getIsSorted() === 'asc')
+ }
>
- <FileIcon name={name}></FileIcon>
- <span className={cn('truncate')}>{name}</span>
- </div>
- </TooltipTrigger>
- <TooltipContent>
- <p>{name}</p>
- </TooltipContent>
- </Tooltip>
- );
+ <ArrowUpDown />
+ </Button>
+ </div>
+ );
+ },
+ meta: { cellClassName: 'max-w-[20vw]' },
+ cell: ({ row }) => {
+ const name: string = row.getValue('name');
+
+ return (
+ <Tooltip>
+ <TooltipTrigger asChild>
+ <div
+ className="flex items-center gap-2 cursor-pointer"
+ onClick={navigateToChunkParsedResult(
+ row.original.id,
+ row.original.dataset_id,
+ )}
+ >
+ <FileIcon name={name}></FileIcon>
+ <span className={cn('truncate')}>{name}</span>
+ </div>
+ </TooltipTrigger>
+ <TooltipContent>
+ <p>{name}</p>
+ </TooltipContent>
+ </Tooltip>
+ );
+ },
},
- },
- {
- accessorKey: 'create_time',
- header: ({ column }) => {
- return (
- <div className="flex items-center gap-1">
- {t('uploadDate')}
+ {
+ accessorKey: 'create_time',
+ header: ({ column }) => {
+ return (
+ <div className="flex items-center gap-1">
+ {t('uploadDate')}
- <Button
- variant="ghost"
- size="icon-xs"
- onClick={() =>
- column.toggleSorting(column.getIsSorted() === 'asc')
- }
- >
- <ArrowUpDown />
- </Button>
- </div>
- );
+ <Button
+ variant="ghost"
+ size="icon-xs"
+ onClick={() =>
+ column.toggleSorting(column.getIsSorted() === 'asc')
+ }
+ >
+ <ArrowUpDown />
+ </Button>
+ </div>
+ );
+ },
+ cell: ({ row }) => (
+ <time
+ className="lowercase"
+ dateTime={new Date(row.getValue('create_time')).toISOString()}
+ >
+ {formatDate(row.getValue('create_time'))}
+ </time>
+ ),
},
- cell: ({ row }) => (
- <time
- className="lowercase"
- dateTime={new Date(row.getValue('create_time')).toISOString()}
- >
- {formatDate(row.getValue('create_time'))}
- </time>
- ),
- },
- {
- accessorKey: 'status',
- header: t('enabled'),
- cell: ({ row }) => {
- const id = row.original.id;
- return (
- <Switch
- checked={row.getValue('status') === '1'}
- onCheckedChange={(e) => {
- setDocumentStatus({
- status: e,
- documentId: id,
- datasetId: datasetId!,
- });
- }}
- />
- );
+ {
+ accessorKey: 'status',
+ header: t('enabled'),
+ cell: ({ row }) => {
+ const id = row.original.id;
+ return (
+ <Switch
+ checked={row.getValue('status') === '1'}
+ onCheckedChange={(e) => {
+ setDocumentStatus({
+ status: e,
+ documentId: id,
+ datasetId: datasetId!,
+ });
+ }}
+ />
+ );
+ },
+ },
+ {
+ accessorKey: 'chunk_count',
+ header: t('chunkNumber'),
+ cell: ({ row }) => (
+ <div className="capitalize">{row.getValue('chunk_count')}</div>
+ ),
},
- },
- {
- accessorKey: 'chunk_count',
- header: t('chunkNumber'),
- cell: ({ row }) => (
- <div className="capitalize">{row.getValue('chunk_count')}</div>
- ),
- },
- {
- accessorKey: 'meta_fields',
- header: t('metadata.metadata'),
- cell: ({ row }) => {
- const length = Object.keys(row.getValue('meta_fields') || {}).length;
- return (
- <Button
- variant="static"
- size="auto"
- onClick={() => {
- showManageMetadataModal({
- isEditField: false,
- isCanAdd: true,
- isAddValue: true,
- type: MetadataType.UpdateSingle,
- record: row.original,
- title: (
- <div className="flex flex-col gap-2 w-full">
- <div className="text-base font-normal">
- {t('metadata.editMetadata')}
+ {
+ accessorKey: 'meta_fields',
+ header: t('metadata.metadata'),
+ cell: ({ row }) => {
+ const length = Object.keys(row.getValue('meta_fields') || {}).length;
+ return (
+ <Button
+ variant="static"
+ size="auto"
+ onClick={() => {
+ showManageMetadataModal({
+ isEditField: false,
+ isCanAdd: true,
+ isAddValue: true,
+ type: MetadataType.UpdateSingle,
+ record: row.original,
+ title: (
+ <div className="flex flex-col gap-2 w-full">
+ <div className="text-base font-normal">
+ {t('metadata.editMetadata')}
+ </div>
</div>
- </div>
- ),
- secondTitle: (
- <div className="w-full flex gap-1 text-sm text-text-secondary">
- <FileIcon name={row.original.name}></FileIcon>
- <div className="truncate">{row.original.name}</div>
- </div>
- ),
- isDeleteSingleValue: true,
- documentIds: [row.original.id],
- });
- }}
- >
- {length + ' fields'}
- </Button>
- );
+ ),
+ secondTitle: (
+ <div className="w-full flex gap-1 text-sm text-text-secondary">
+ <FileIcon name={row.original.name}></FileIcon>
+ <div className="truncate">{row.original.name}</div>
+ </div>
+ ),
+ isDeleteSingleValue: true,
+ documentIds: [row.original.id],
+ });
+ }}
+ >
+ {length + ' fields'}
+ </Button>
+ );
+ },
},
- },
- {
- accessorKey: 'run',
- header: t('Parse'),
- cell: ({ row }) => {
- return (
- <ParseDropdownButton
- record={row.original}
- showChangeParserModal={showChangeParserModal}
- />
- );
+ {
+ accessorKey: 'run',
+ header: t('Parse'),
+ cell: ({ row }) => {
+ return (
+ <ParseDropdownButton
+ record={row.original}
+ showChangeParserModal={showChangeParserModal}
+ />
+ );
+ },
},
- },
- {
- id: 'run-status',
- header: '',
- cell: ({ row }) => {
- return (
- <ParsingStatusCell
- record={row.original}
- showChangeParserModal={showChangeParserModal}
- showLog={showLog}
- />
- );
+ {
+ id: 'run-status',
+ header: '',
+ cell: ({ row }) => {
+ return (
+ <ParsingStatusCell
+ record={row.original}
+ showChangeParserModal={showChangeParserModal}
+ showLog={showLog}
+ />
+ );
+ },
},
- },
- {
- id: 'actions',
- header: t('action'),
- enableHiding: false,
- cell: ({ row }) => {
- const record = row.original;
+ {
+ id: 'actions',
+ header: t('action'),
+ enableHiding: false,
+ cell: ({ row }) => {
+ const record = row.original;
- return (
- <DatasetActionCell
- record={record}
- showRenameModal={showRenameModal}
- setRowSelection={setRowSelection}
- />
- );
+ return (
+ <DatasetActionCell
+ record={record}
+ showRenameModal={showRenameModal}
+ setRowSelection={setRowSelection}
+ />
+ );
+ },
},
- },
- ];
+ ],
+ [
+ t,
+ navigateToChunkParsedResult,
+ setDocumentStatus,
+ datasetId,
+ showChangeParserModal,
+ showRenameModal,
+ showManageMetadataModal,
+ showLog,
+ setRowSelection,
+ ],
+ );
return columns;
}
diff --git a/web/src/pages/dataset/dataset/use-parser-model-validation.ts b/web/src/pages/dataset/dataset/use-parser-model-validation.ts
new file mode 100644
index 0000000000..d06613ea21
--- /dev/null
+++ b/web/src/pages/dataset/dataset/use-parser-model-validation.ts
@@ -0,0 +1,46 @@
+import { useNavigatePage } from '@/hooks/logic-hooks/navigate-hooks';
+import { useIsGoBackend } from '@/utils/backend-variant';
+import { useCallback } from 'react';
+import { useParams } from 'react-router';
+import { useKnowledgeBaseContext } from '../contexts/knowledge-base-context';
+import {
+ FileModelGap,
+ findFilesMissingParserModels,
+ getEffectiveParserSetups,
+} from './utils';
+
+/**
+ * Validates that the models required to parse a file type (audio/video/image)
+ * are configured on the dataset's Parser operator.
+ * Only the Go backend has parser operator settings — on the Python backend
+ * every file passes.
+ */
+export function useParserModelValidation() {
+ const isGoBackend = useIsGoBackend();
+ const { knowledgeBase, loading } = useKnowledgeBaseContext();
+ const { id } = useParams();
+ const { navigateToDatasetSetting } = useNavigatePage();
+
+ const findFilesMissingModels = useCallback(
+ (names: string[]): FileModelGap[] => {
+ // Skip while the dataset is loading: its parser_config is not readable
+ // yet and every media file would look unconfigured.
+ if (!isGoBackend || loading) {
+ return [];
+ }
+ return findFilesMissingParserModels(
+ names,
+ getEffectiveParserSetups(knowledgeBase),
+ );
+ },
+ [isGoBackend, loading, knowledgeBase],
+ );
+
+ const goToDatasetConfiguration = useCallback(() => {
+ if (id) {
+ navigateToDatasetSetting(id);
+ }
+ }, [id, navigateToDatasetSetting]);
+
+ return { findFilesMissingModels, goToDatasetConfiguration };
+}
diff --git a/web/src/pages/dataset/dataset/use-run-document.ts b/web/src/pages/dataset/dataset/use-run-document.ts
index 8494f78565..cd48016eb8 100644
--- a/web/src/pages/dataset/dataset/use-run-document.ts
+++ b/web/src/pages/dataset/dataset/use-run-document.ts
@@ -1,24 +1,54 @@
+import { Modal } from '@/components/ui/modal/modal';
import { useSetModalState } from '@/hooks/common-hooks';
import { useRunDocument } from '@/hooks/use-document-request';
+import { IDocumentInfo } from '@/interfaces/database/document';
import { useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { buildMissingModelModalContent } from './parser-model-gap-content';
+import { useParserModelValidation } from './use-parser-model-validation';
export const useHandleRunDocumentByIds = (id: string) => {
+ const { t } = useTranslation();
const { runDocumentByIds, loading } = useRunDocument();
const [currentId, setCurrentId] = useState<string>('');
const isLoading = loading && currentId !== '' && currentId === id;
const { visible, showModal, hideModal } = useSetModalState();
+ const { findFilesMissingModels, goToDatasetConfiguration } =
+ useParserModelValidation();
+
const handleRunDocumentByIds = async (
- documentId: string,
+ record: IDocumentInfo,
isRunning: boolean,
option?: { delete: boolean; apply_kb: boolean },
) => {
if (isLoading) {
return;
}
- setCurrentId(documentId);
+ // Starting a parse requires the models of the file type to be configured;
+ // cancelling is always allowed.
+ if (!isRunning) {
+ const gaps = findFilesMissingModels([record.name]);
+ if (gaps.length > 0) {
+ hideModal();
+ Modal.error({
+ title: t('knowledgeDetails.parseBlockedTitle'),
+ content: buildMissingModelModalContent(
+ t,
+ gaps,
+ 'knowledgeDetails.parseBlockedHint',
+ ),
+ okText: t('knowledgeDetails.goToConfiguration'),
+ cancelText: t('common.cancel'),
+ closable: false,
+ onOk: goToDatasetConfiguration,
+ });
+ return;
+ }
+ }
+ setCurrentId(record.id);
try {
await runDocumentByIds({
- documentIds: [documentId],
+ documentIds: [record.id],
run: isRunning ? 2 : 1,
option,
});
diff --git a/web/src/pages/dataset/dataset/use-upload-document.ts b/web/src/pages/dataset/dataset/use-upload-document.ts
index 5b41cd7900..81b7ef3c92 100644
--- a/web/src/pages/dataset/dataset/use-upload-document.ts
+++ b/web/src/pages/dataset/dataset/use-upload-document.ts
@@ -1,13 +1,21 @@
import { UploadFormSchemaType } from '@/components/file-upload-dialog';
+import { Modal } from '@/components/ui/modal/modal';
import { useSetModalState } from '@/hooks/common-hooks';
import {
useRunDocument,
useUploadDocument,
} from '@/hooks/use-document-request';
-import { getUnSupportedFilesCount } from '@/utils/document-util';
+import { IDocumentInfo } from '@/interfaces/database/document';
+import { FileType } from '@/pages/agent/constant/pipeline';
+import { getExtension, getUnSupportedFilesCount } from '@/utils/document-util';
import { useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
+import { buildMissingModelModalContent } from './parser-model-gap-content';
+import { useParserModelValidation } from './use-parser-model-validation';
+import { getFileTypeByExtension } from './utils';
export const useHandleUploadDocument = () => {
+ const { t } = useTranslation();
const {
visible: documentUploadVisible,
hideModal: hideDocumentUploadModal,
@@ -15,68 +23,126 @@ export const useHandleUploadDocument = () => {
} = useSetModalState();
const { uploadDocument, loading } = useUploadDocument();
const { runDocumentByIds } = useRunDocument();
+ const { findFilesMissingModels, goToDatasetConfiguration } =
+ useParserModelValidation();
- const onDocumentUploadOk = useCallback(
- async ({
- fileList,
- parseOnCreation,
- tableColumnMode,
- tableColumnRoles,
- }: UploadFormSchemaType) => {
- if (fileList.length > 0) {
- // Build parser_config if column roles are configured
- let parserConfig: Record<string, any> | undefined;
- if (
- tableColumnMode === 'manual' &&
- tableColumnRoles &&
- Object.keys(tableColumnRoles).length > 0
- ) {
- parserConfig = {
- table_column_mode: 'manual',
- table_column_roles: tableColumnRoles,
- };
- }
+ const proceedUpload = useCallback(
+ async (
+ {
+ fileList,
+ parseOnCreation,
+ tableColumnMode,
+ tableColumnRoles,
+ }: UploadFormSchemaType,
+ failingFileTypes: Set<FileType>,
+ ) => {
+ // Build parser_config if column roles are configured
+ let parserConfig: Record<string, any> | undefined;
+ if (
+ tableColumnMode === 'manual' &&
+ tableColumnRoles &&
+ Object.keys(tableColumnRoles).length > 0
+ ) {
+ parserConfig = {
+ table_column_mode: 'manual',
+ table_column_roles: tableColumnRoles,
+ };
+ }
- const ret = await uploadDocument(fileList as File[], parserConfig);
+ const ret = await uploadDocument(fileList as File[], parserConfig);
- // Check for success (code === 0) or partial success (code === 500 with some files)
- const isSuccess = ret?.code === 0;
- const isPartialSuccess = ret?.code === 500 && ret?.message;
+ // Check for success (code === 0) or partial success (code === 500 with some files)
+ const isSuccess = ret?.code === 0;
+ const isPartialSuccess = ret?.code === 500 && ret?.message;
- if (!isSuccess && !isPartialSuccess) {
- return;
- }
+ if (!isSuccess && !isPartialSuccess) {
+ return;
+ }
- // Trigger parsing for both full and partial success when parseOnCreation is enabled
- if (
- (isSuccess || isPartialSuccess) &&
- parseOnCreation &&
- ret.data?.length > 0
- ) {
+ // Trigger parsing for both full and partial success when parseOnCreation
+ // is enabled; files whose required model is missing are uploaded but not
+ // auto-parsed.
+ if (
+ (isSuccess || isPartialSuccess) &&
+ parseOnCreation &&
+ ret.data?.length > 0
+ ) {
+ const runnableDocuments = (ret.data as IDocumentInfo[]).filter(
+ (doc) => {
+ const fileType = getFileTypeByExtension(getExtension(doc.name));
+ return !fileType || !failingFileTypes.has(fileType);
+ },
+ );
+ if (runnableDocuments.length > 0) {
runDocumentByIds({
- documentIds: ret.data.map((x: any) => x.id),
+ documentIds: runnableDocuments.map((x) => x.id),
run: 1,
});
}
+ }
- if (isSuccess) {
- hideDocumentUploadModal();
- return 0;
- }
-
- // For partial success (code 500), check if any files were uploaded
- const count = getUnSupportedFilesCount(ret?.message);
- if (count !== fileList.length) {
- hideDocumentUploadModal();
- return 0;
- }
+ if (isSuccess) {
+ hideDocumentUploadModal();
+ return 0;
+ }
- return ret?.code;
+ // For partial success (code 500), check if any files were uploaded
+ const count = getUnSupportedFilesCount(ret?.message);
+ if (count !== fileList.length) {
+ hideDocumentUploadModal();
+ return 0;
}
+
+ return ret?.code;
},
[uploadDocument, runDocumentByIds, hideDocumentUploadModal],
);
+ const onDocumentUploadOk = useCallback(
+ async (values: UploadFormSchemaType) => {
+ const { fileList } = values;
+ if (fileList.length === 0) {
+ return;
+ }
+
+ const names = fileList.map((file) =>
+ file instanceof File ? file.name : file.file.name,
+ );
+ const gaps = findFilesMissingModels(names);
+ if (gaps.length > 0) {
+ const failingFileTypes = new Set(gaps.map((gap) => gap.fileType));
+ Modal.warning({
+ title: t('knowledgeDetails.uploadMissingModelsTitle'),
+ content: buildMissingModelModalContent(
+ t,
+ gaps,
+ 'knowledgeDetails.configureInDatasetSettingHint',
+ ),
+ okText: t('knowledgeDetails.continueUpload'),
+ cancelText: t('knowledgeDetails.goToConfiguration'),
+ closable: false,
+ onOk: () => {
+ proceedUpload(values, failingFileTypes);
+ },
+ onCancel: () => {
+ hideDocumentUploadModal();
+ goToDatasetConfiguration();
+ },
+ });
+ return;
+ }
+
+ return proceedUpload(values, new Set());
+ },
+ [
+ findFilesMissingModels,
+ goToDatasetConfiguration,
+ hideDocumentUploadModal,
+ proceedUpload,
+ t,
+ ],
+ );
+
return {
documentUploadLoading: loading,
onDocumentUploadOk,
diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.ts
index b1910a1321..5ba5866c1c 100644
--- a/web/src/pages/dataset/dataset/utils.test.ts
+++ b/web/src/pages/dataset/dataset/utils.test.ts
@@ -1,5 +1,13 @@
+import { IDataset } from '@/interfaces/database/dataset';
+import { FileType, initialParserValues } from '@/pages/agent/constant/pipeline';
import { IngestionTaskStatus, RunningStatus } from './constant';
-import { isDocumentProcessing } from './utils';
+import {
+ findFilesMissingParserModels,
+ findMissingParserModel,
+ getEffectiveParserSetups,
+ getFileTypeByExtension,
+ isDocumentProcessing,
+} from './utils';
describe('isDocumentProcessing', () => {
it('treats a scheduled ingestion task as processing before parsing starts', () => {
@@ -27,3 +35,119 @@ describe('isDocumentProcessing', () => {
},
);
});
+
+describe('getFileTypeByExtension', () => {
+ it('maps media extensions to their file types', () => {
+ expect(getFileTypeByExtension('mp3')).toBe(FileType.Audio);
+ expect(getFileTypeByExtension('wav')).toBe(FileType.Audio);
+ expect(getFileTypeByExtension('mp4')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('jpeg')).toBe(FileType.Image);
+ expect(getFileTypeByExtension('PNG')).toBe(FileType.Image);
+ expect(getFileTypeByExtension('pdf')).toBe(FileType.PDF);
+ });
+
+ it('returns undefined for unknown extensions', () => {
+ expect(getFileTypeByExtension('xyz')).toBeUndefined();
+ expect(getFileTypeByExtension('')).toBeUndefined();
+ });
+});
+
+const DefaultSetups = initialParserValues.setups as Record<string, any>[];
+
+describe('findMissingParserModel', () => {
+ it('passes audio when the setup carries a model', () => {
+ const setups = [
+ { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } },
+ ];
+ expect(findMissingParserModel(FileType.Audio, setups)).toBeNull();
+ });
+
+ it('flags audio when the setup has no model, ignoring the tenant default', () => {
+ expect(findMissingParserModel(FileType.Audio, DefaultSetups)).toEqual({
+ fileType: FileType.Audio,
+ modelKind: 'asr',
+ });
+ });
+
+ it('passes video when the setup carries a model', () => {
+ const setups = [
+ { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
+ ];
+ expect(findMissingParserModel(FileType.Video, setups)).toBeNull();
+ });
+
+ it('flags video when the setup has no model', () => {
+ expect(findMissingParserModel(FileType.Video, DefaultSetups)).toEqual({
+ fileType: FileType.Video,
+ modelKind: 'vision',
+ });
+ });
+
+ it('flags image when parse_method is a static method like ocr', () => {
+ // The default image setup uses ocr — no vision model configured
+ expect(findMissingParserModel(FileType.Image, DefaultSetups)).toEqual({
+ fileType: FileType.Image,
+ modelKind: 'vision',
+ });
+ });
+
+ it('passes image when parse_method is a vision model', () => {
+ const setups = [
+ { fileFormat: FileType.Image, parse_method: 'gpt-4o@OpenAI' },
+ ];
+ expect(findMissingParserModel(FileType.Image, setups)).toBeNull();
+ });
+
+ it('ignores file types without model requirements', () => {
+ expect(findMissingParserModel(FileType.PDF, DefaultSetups)).toBeNull();
+ expect(findMissingParserModel(undefined, DefaultSetups)).toBeNull();
+ });
+});
+
+describe('findFilesMissingParserModels', () => {
+ it('returns a gap per file that lacks its required model', () => {
+ const gaps = findFilesMissingParserModels(
+ ['song.mp3', 'notes.txt', 'photo.jpg'],
+ DefaultSetups,
+ );
+ expect(gaps).toEqual([
+ { name: 'song.mp3', fileType: FileType.Audio, modelKind: 'asr' },
+ { name: 'photo.jpg', fileType: FileType.Image, modelKind: 'vision' },
+ ]);
+ });
+});
+
+describe('getEffectiveParserSetups', () => {
+ it('falls back to the default setups without a saved parser config', () => {
+ const setups = getEffectiveParserSetups(null);
+ expect(setups.map((x) => x.fileFormat)).toEqual(
+ initialParserValues.setups.map((x) => x.fileFormat),
+ );
+ });
+
+ it('applies saved overrides keyed by file format', () => {
+ const knowledgeBase = {
+ parser_config: {
+ 'Parser:1': {
+ audio: { vlm: { llm_id: 'whisper@OpenAI' } },
+ },
+ },
+ } as unknown as IDataset;
+ const setups = getEffectiveParserSetups(knowledgeBase);
+ const audio = setups.find((x) => x.fileFormat === FileType.Audio);
+ expect(audio?.vlm?.llm_id).toBe('whisper@OpenAI');
+ // Untouched types keep their defaults
+ const image = setups.find((x) => x.fileFormat === FileType.Image);
+ expect(image?.parse_method).toBe('ocr');
+ });
+
+ it('ignores non-parser operator entries', () => {
+ const knowledgeBase = {
+ parser_config: {
+ 'TokenChunker:1': { chunk_token_size: 128 },
+ },
+ } as unknown as IDataset;
+ const setups = getEffectiveParserSetups(knowledgeBase);
+ expect(setups).toHaveLength(initialParserValues.setups.length);
+ });
+});
diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.ts
index e899d88ab9..5779f83abc 100644
--- a/web/src/pages/dataset/dataset/utils.ts
+++ b/web/src/pages/dataset/dataset/utils.ts
@@ -1,5 +1,19 @@
-import { IngestionTaskStatus, RunningStatus } from './constant';
+import { Operator } from '@/constants/agent';
+import type { IDataset } from '@/interfaces/database/dataset';
import type { IDocumentInfo } from '@/interfaces/database/document';
+import {
+ FileType,
+ FileTypeSuffixMap,
+ initialParserValues,
+} from '@/pages/agent/constant/pipeline';
+import { isStaticParseMethod } from '@/pages/agent/form/parser-form/utils';
+import { getExtension } from '@/utils/document-util';
+import {
+ getOperatorType,
+ transformParserConfigSetups,
+} from '@/utils/pipeline-operator';
+import { cloneDeep } from 'lodash';
+import { IngestionTaskStatus, RunningStatus } from './constant';
export const isParserRunning = (text: RunningStatus) => {
const isRunning = text === RunningStatus.RUNNING;
@@ -35,3 +49,105 @@ export const isDocumentProcessing = (
document.ingestion_status === IngestionTaskStatus.STOPPING
);
};
+
+// --- Parser model prerequisite checks -------------------------------------
+// Audio/video/image files can only be parsed when the matching model is
+// configured on the dataset's Parser operator. The tenant default is
+// deliberately not consulted: parsing reads the operator setup, so a global
+// default does not make the file parsable.
+// These helpers power the upload warning and the parse-click validation.
+
+export type ParserModelGap = {
+ fileType: FileType;
+ modelKind: 'asr' | 'vision';
+};
+
+export type FileModelGap = ParserModelGap & { name: string };
+
+type ParserSetup = Record<string, any> & { fileFormat?: string };
+
+const ExtensionToFileTypeMap: Record<string, FileType> = Object.entries(
+ FileTypeSuffixMap,
+).reduce<Record<string, FileType>>((acc, [fileType, suffixes]) => {
+ for (const suffix of suffixes) {
+ acc[suffix] = fileType as FileType;
+ }
+ return acc;
+}, {});
+
+export function getFileTypeByExtension(
+ extension: string,
+): FileType | undefined {
+ return ExtensionToFileTypeMap[extension.toLowerCase()];
+}
+
+/**
+ * Effective parser setups for a dataset: the saved Parser operator entry
+ * (stored as a flattened map keyed by file format) merged over the default
+ * setups, so untouched file types validate against their defaults.
+ */
+export function getEffectiveParserSetups(
+ knowledgeBase: Pick<IDataset, 'parser_config'> | null | undefined,
+): ParserSetup[] {
+ const parserConfig = knowledgeBase?.parser_config as
+ | Record<string, any>
+ | undefined;
+
+ const parserEntry = Object.entries(parserConfig ?? {}).find(
+ ([operatorId]) => getOperatorType(operatorId) === Operator.Parser,
+ )?.[1];
+
+ const savedSetups = transformParserConfigSetups(parserEntry);
+ const savedByFileType = new Map(
+ savedSetups.map((setup) => [setup.fileFormat, setup]),
+ );
+
+ const defaultSetups = cloneDeep(initialParserValues.setups) as ParserSetup[];
+ const merged = defaultSetups.map(
+ (setup) => savedByFileType.get(setup.fileFormat) ?? setup,
+ );
+ for (const setup of savedSetups) {
+ if (!defaultSetups.some((x) => x.fileFormat === setup.fileFormat)) {
+ merged.push(setup);
+ }
+ }
+ return merged;
+}
+
+export function findMissingParserModel(
+ fileType: FileType | undefined,
+ setups: ParserSetup[],
+): ParserModelGap | null {
+ const setup = setups.find((x) => x.fileFormat === fileType);
+
+ switch (fileType) {
+ case FileType.Audio: {
+ const configured = setup?.vlm?.llm_id;
+ return configured ? null : { fileType, modelKind: 'asr' };
+ }
+ case FileType.Video: {
+ const configured = setup?.vlm?.llm_id;
+ return configured ? null : { fileType, modelKind: 'vision' };
+ }
+ case FileType.Image: {
+ // The image parser's vision model is picked as parse_method; a static
+ // method (e.g. ocr) means no vision model is configured.
+ const configured =
+ !!setup?.parse_method && !isStaticParseMethod(setup.parse_method);
+ return configured ? null : { fileType, modelKind: 'vision' };
+ }
+ default:
+ return null;
+ }
+}
+
+export function findFilesMissingParserModels(
+ names: string[],
+ setups: ParserSetup[],
+): FileModelGap[] {
+ return names.flatMap((name) => {
+ const fileType = getFileTypeByExtension(getExtension(name));
+ const gap = findMissingParserModel(fileType, setups);
+ return gap ? [{ ...gap, name }] : [];
+ });
+}
diff --git a/web/src/pages/dataset/setting/go/index.tsx b/web/src/pages/dataset/setting/go/index.tsx
index 1ff0eb5c82..6154beea98 100644
--- a/web/src/pages/dataset/setting/go/index.tsx
+++ b/web/src/pages/dataset/setting/go/index.tsx
@@ -243,6 +243,7 @@ export default function DatasetSetting() {
| Record<string, FieldErrors | undefined>
| undefined
}
+ fixedFileFormats
/>
)}