Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix: Even though a vision model has been configured, uploading a video still triggers a notification stating that no vision model is configured. (#20816)

balibabue4695ca5594b
diff --git a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
index 6cb0c30cb4..ed27c98e4b 100644
--- a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
+++ b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts
@@ -5,7 +5,7 @@ import {
FileParserGap,
findDocumentsParserGaps,
findFilesParserGaps,
- getSavedParserSetups,
+ getSavedParserConfig,
} from './utils';
/**
@@ -25,11 +25,11 @@ export function useParserGapValidation() {
if (loading) {
return [];
}
- const setups = getSavedParserSetups(knowledgeBase);
- if (!setups) {
+ const config = getSavedParserConfig(knowledgeBase);
+ if (!config) {
return [];
}
- return findFilesParserGaps(names, setups);
+ return findFilesParserGaps(names, config);
},
[loading, knowledgeBase],
);
@@ -46,7 +46,7 @@ export function useParserGapValidation() {
}
return findDocumentsParserGaps(
documents,
- getSavedParserSetups(knowledgeBase),
+ getSavedParserConfig(knowledgeBase),
);
},
[loading, knowledgeBase],
diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.ts
index 651ebd4443..6ec3d7270b 100644
--- a/web/src/pages/dataset/dataset/utils.test.ts
+++ b/web/src/pages/dataset/dataset/utils.test.ts
@@ -12,11 +12,12 @@ import {
getDocumentRunningStatus,
getDocumentProgressMessage,
getFileTypeByExtension,
- getSavedParserSetups,
+ getSavedParserConfig,
hasUnsupportedTypeGap,
ingestionStatusToRunningStatus,
isDocumentProcessing,
isDocumentStopping,
+ type SavedParserConfig,
} from './utils';
describe('isDocumentStopping', () => {
@@ -124,63 +125,96 @@ describe('getFileTypeByExtension', () => {
});
});
-// Saved setups shaped like a pipeline that declares pdf/image plus audio
-// (model missing) and video (model configured).
-const SavedSetups = [
- { fileFormat: FileType.PDF, parse_method: 'DeepDOC' },
- { fileFormat: FileType.Image, parse_method: 'ocr' },
- { fileFormat: FileType.Audio, vlm: { llm_id: '' } },
- { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
-] as Record<string, any>[];
+// Saved config shaped like a pipeline that declares pdf/image plus audio
+// (ASR model missing) and video, with the Parser-level vision enhancement
+// enabled and its model configured — the current contract, where media
+// families other than audio carry no model of their own.
+const SavedConfig: SavedParserConfig = {
+ setups: [
+ { fileFormat: FileType.PDF, parse_method: 'DeepDOC' },
+ { fileFormat: FileType.Image, ocr_enabled: true },
+ { fileFormat: FileType.Audio, vlm: { llm_id: '' } },
+ { fileFormat: FileType.Video },
+ ],
+ visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },
+};
describe('findParserGap', () => {
it('flags a file type missing from the saved setups as unsupported', () => {
- expect(findParserGap(FileType.Docx, SavedSetups)).toEqual({
+ expect(findParserGap(FileType.Docx, SavedConfig)).toEqual({
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Docx,
});
});
it('flags audio/video without a declared family as unsupported, not missing-model', () => {
- const setups = [{ fileFormat: FileType.PDF }];
- expect(findParserGap(FileType.Audio, setups)).toEqual({
+ const config: SavedParserConfig = {
+ setups: [{ fileFormat: FileType.PDF }],
+ };
+ expect(findParserGap(FileType.Audio, config)).toEqual({
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Audio,
});
- expect(findParserGap(FileType.Video, setups)).toEqual({
+ expect(findParserGap(FileType.Video, config)).toEqual({
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
});
});
it('flags declared audio without a model as a missing asr model', () => {
- expect(findParserGap(FileType.Audio, SavedSetups)).toEqual({
+ expect(findParserGap(FileType.Audio, SavedConfig)).toEqual({
reason: ParserGapReason.MissingModel,
fileType: FileType.Audio,
modelKind: ParserModelKind.Asr,
});
});
- it('flags declared video without a model as a missing vision model', () => {
- const setups = [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }];
- expect(findParserGap(FileType.Video, setups)).toEqual({
+ it('passes video when the Parser-level vision enhancement has a model', () => {
+ expect(findParserGap(FileType.Video, SavedConfig)).toBeNull();
+ });
+
+ it('flags declared video when the vision enhancement is off or has no model', () => {
+ const disabled: SavedParserConfig = {
+ setups: [{ fileFormat: FileType.Video }],
+ visionEnhancement: { enabled: false, modelId: 'glm-4v@ZHIPU-AI' },
+ };
+ const noModel: SavedParserConfig = {
+ setups: [{ fileFormat: FileType.Video }],
+ visionEnhancement: { enabled: true, modelId: '' },
+ };
+ const expected = {
reason: ParserGapReason.MissingModel,
fileType: FileType.Video,
modelKind: ParserModelKind.Vision,
- });
+ };
+ expect(findParserGap(FileType.Video, disabled)).toEqual(expected);
+ expect(findParserGap(FileType.Video, noModel)).toEqual(expected);
});
- it('passes video when the setup carries a model', () => {
- expect(findParserGap(FileType.Video, SavedSetups)).toBeNull();
+ it('reads the video model from the family setup for configs saved before the move', () => {
+ const withModel: SavedParserConfig = {
+ setups: [
+ { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
+ ],
+ };
+ const withoutModel: SavedParserConfig = {
+ setups: [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }],
+ };
+ expect(findParserGap(FileType.Video, withModel)).toBeNull();
+ expect(findParserGap(FileType.Video, withoutModel)).toEqual({
+ reason: ParserGapReason.MissingModel,
+ fileType: FileType.Video,
+ modelKind: ParserModelKind.Vision,
+ });
});
it('passes image with an ocr-only setup — no model required', () => {
- expect(findParserGap(FileType.Image, SavedSetups)).toBeNull();
+ expect(findParserGap(FileType.Image, SavedConfig)).toBeNull();
});
it('passes supported non-media types and unknown file types', () => {
- expect(findParserGap(FileType.PDF, SavedSetups)).toBeNull();
- expect(findParserGap(undefined, SavedSetups)).toBeNull();
+ expect(findParserGap(FileType.PDF, SavedConfig)).toBeNull();
+ expect(findParserGap(undefined, SavedConfig)).toBeNull();
});
});
@@ -188,7 +222,7 @@ describe('findFilesParserGaps', () => {
it('returns a gap per failing file, mixing both reasons', () => {
const gaps = findFilesParserGaps(
['song.mp3', 'notes.docx', 'photo.jpg', 'movie.mp4'],
- SavedSetups,
+ SavedConfig,
);
expect(gaps).toEqual([
{
@@ -231,11 +265,11 @@ describe('hasUnsupportedTypeGap', () => {
});
});
-describe('getSavedParserSetups', () => {
+describe('getSavedParserConfig', () => {
it('returns null without a saved parser config', () => {
- expect(getSavedParserSetups(null)).toBeNull();
+ expect(getSavedParserConfig(null)).toBeNull();
expect(
- getSavedParserSetups({
+ getSavedParserConfig({
parser_config: {},
}),
).toBeNull();
@@ -245,7 +279,7 @@ describe('getSavedParserSetups', () => {
const knowledgeBase = {
parser_config: { chunk_token_num: 128, delimiter: '\n' },
};
- expect(getSavedParserSetups(knowledgeBase)).toBeNull();
+ expect(getSavedParserConfig(knowledgeBase)).toBeNull();
});
it('returns only the saved Parser setups, without merging defaults', () => {
@@ -257,8 +291,42 @@ describe('getSavedParserSetups', () => {
},
},
};
- const setups = getSavedParserSetups(knowledgeBase);
- expect(setups?.map((x) => x.fileFormat).sort()).toEqual(['image', 'pdf']);
+ const config = getSavedParserConfig(knowledgeBase);
+ expect(config?.setups.map((x) => x.fileFormat).sort()).toEqual([
+ 'image',
+ 'pdf',
+ ]);
+ });
+
+ it('reads the vision enhancement globals from the Parser entry top level', () => {
+ const knowledgeBase = {
+ parser_config: {
+ 'Parser:1': {
+ enable_vision_enhancement: true,
+ vlm: { llm_id: 'glm-4v@ZHIPU-AI', system_prompt: 'describe' },
+ video: {},
+ },
+ },
+ };
+ expect(getSavedParserConfig(knowledgeBase)).toEqual({
+ setups: [{ fileFormat: FileType.Video }],
+ visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },
+ });
+ });
+
+ it('leaves the vision enhancement undefined for configs saved before the move', () => {
+ const knowledgeBase = {
+ parser_config: {
+ 'Parser:1': {
+ video: { vlm: { llm_id: 'gpt-4o@OpenAI' } },
+ },
+ },
+ };
+ const config = getSavedParserConfig(knowledgeBase);
+ expect(config?.visionEnhancement).toBeUndefined();
+ expect(config?.setups).toEqual([
+ { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
+ ]);
});
it('ignores non-parser operator entries', () => {
@@ -267,7 +335,7 @@ describe('getSavedParserSetups', () => {
'TokenChunker:1': { chunk_token_size: 128 },
},
};
- expect(getSavedParserSetups(knowledgeBase)).toBeNull();
+ expect(getSavedParserConfig(knowledgeBase)).toBeNull();
});
});
@@ -285,9 +353,9 @@ describe('findDocumentsParserGaps', () => {
expect(findDocumentsParserGaps([OverriddenDoc], null)).toEqual([]);
});
- it('falls back to the dataset-level setups for rows without a Parser entry', () => {
+ it('falls back to the dataset-level config for rows without a Parser entry', () => {
const legacyRow = { name: 'song.mp3', parser_config: {} };
- expect(findDocumentsParserGaps([legacyRow], SavedSetups)).toEqual([
+ expect(findDocumentsParserGaps([legacyRow], SavedConfig)).toEqual([
{
name: 'song.mp3',
reason: ParserGapReason.MissingModel,
@@ -305,7 +373,11 @@ describe('findDocumentsParserGaps', () => {
[OverriddenDoc, inheritedRow],
// Dataset-level config declares only audio: the overridden pdf passes
// while the inherited jpg is flagged.
- [{ fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } }],
+ {
+ setups: [
+ { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } },
+ ],
+ },
);
expect(gaps).toEqual([
{
@@ -315,4 +387,18 @@ describe('findDocumentsParserGaps', () => {
},
]);
});
+
+ it('reads the document row vision globals for video', () => {
+ const videoDoc = {
+ name: 'movie.mp4',
+ parser_config: {
+ 'Parser:1': {
+ enable_vision_enhancement: true,
+ vlm: { llm_id: 'glm-4v@ZHIPU-AI' },
+ video: {},
+ },
+ },
+ };
+ expect(findDocumentsParserGaps([videoDoc], null)).toEqual([]);
+ });
});
diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.ts
index 6040256eaf..bbb6c68604 100644
--- a/web/src/pages/dataset/dataset/utils.ts
+++ b/web/src/pages/dataset/dataset/utils.ts
@@ -128,13 +128,14 @@ export const isDocumentProcessing = (
// --- Parser prerequisite checks -------------------------------------------
// A file cannot be parsed under the dataset's current Parser operator when
// the operator does not declare the file's type family at all
-// (unsupportedType), or — for audio/video — when the declared setup carries
-// no model (missingModel). The tenant default is deliberately not consulted:
-// parsing reads the operator setup, so a global default does not make the
-// file parsable. Image files only need their family declared: the image
+// (unsupportedType), or when the family is declared but the model it needs
+// is not set (missingModel): audio reads its per-setup ASR model, while
+// video reads the Parser-level vision enhancement (enable_vision_enhancement
+// + vlm.llm_id) — its model no longer lives on the family setup. The tenant
+// default is deliberately not consulted: a global default does not make the
+// file parsable here. Image files only need their family declared: the image
// OCR switch runs local extraction independently, and the vision model is a
-// global supplement picked from the top-level vlm.llm_id (falling back to the
-// tenant default).
+// global supplement for it.
// These helpers power the upload warning and the parse-click validation.
export type ParserModelGap = {
@@ -168,6 +169,22 @@ export function pickByGapKind<T>(
type ParserSetup = Record<string, any> & { fileFormat?: string };
+/**
+ * The Parser-level vision enhancement globals (enable_vision_enhancement +
+ * vlm.llm_id): one switch and model serve every media family. Undefined marks
+ * a config saved before the model moved off the per-family setups — those
+ * still carry the video model on the video setup itself.
+ */
+export type ParserVisionEnhancement = {
+ enabled: boolean;
+ modelId: string;
+};
+
+export type SavedParserConfig = {
+ setups: ParserSetup[];
+ visionEnhancement?: ParserVisionEnhancement;
+};
+
const ExtensionToFileTypeMap: Record<string, FileType> = Object.entries(
FileTypeSuffixMap,
).reduce<Record<string, FileType>>((acc, [fileType, suffixes]) => {
@@ -183,18 +200,37 @@ export function getFileTypeByExtension(
return ExtensionToFileTypeMap[extension.toLowerCase()];
}
+// Reads the Parser-level vision globals from a saved Parser entry. A boolean
+// enable_vision_enhancement marks the current contract; configs saved before
+// the model moved to the top level do not have it, and their video setup
+// still holds the model — so the absence is meaningful, not a default-off.
+function readVisionEnhancement(
+ parserEntry: unknown,
+): ParserVisionEnhancement | undefined {
+ if (typeof parserEntry !== 'object' || parserEntry === null) {
+ return undefined;
+ }
+ const params = parserEntry as Record<string, any>;
+ const enabled = params.enable_vision_enhancement;
+ if (typeof enabled !== 'boolean') {
+ return undefined;
+ }
+ const modelId = params.vlm?.llm_id;
+ return { enabled, modelId: typeof modelId === 'string' ? modelId : '' };
+}
+
/**
- * Parser setups exactly as saved, keyed by file format — the keys are the
- * type families the pipeline/built-in method declares. Accepts either a
- * dataset or a document row: a document's parser_config snapshots the parser
- * it actually runs with (dataset config at upload, the override after a
- * parser change). Returns null when parser_config has no Parser operator
- * entry (legacy flat-shape configs, pipelines without a Parser stage): type
- * support cannot be determined then and validation is skipped.
+ * Parser config exactly as saved: the per-family setups plus the Parser-level
+ * vision enhancement globals. Accepts either a dataset or a document row: a
+ * document's parser_config snapshots the parser it actually runs with
+ * (dataset config at upload, the override after a parser change). Returns
+ * null when parser_config has no Parser operator entry (legacy flat-shape
+ * configs, pipelines without a Parser stage): type support cannot be
+ * determined then and validation is skipped.
*/
-export function getSavedParserSetups(
+export function getSavedParserConfig(
source: { parser_config?: unknown } | null | undefined,
-): ParserSetup[] | null {
+): SavedParserConfig | null {
const parserConfig = source?.parser_config as Record<string, any> | undefined;
const parserEntry = Object.entries(parserConfig ?? {}).find(
@@ -204,24 +240,29 @@ export function getSavedParserSetups(
if (!parserEntry) {
return null;
}
- return transformParserConfigSetups(parserEntry);
+ return {
+ setups: transformParserConfigSetups(parserEntry),
+ visionEnhancement: readVisionEnhancement(parserEntry),
+ };
}
export function findParserGap(
fileType: FileType | undefined,
- setups: ParserSetup[],
+ config: SavedParserConfig,
): ParserGap | null {
if (!fileType) {
return null;
}
- const setup = setups.find((x) => x.fileFormat === fileType);
+ const setup = config.setups.find((x) => x.fileFormat === fileType);
if (!setup) {
return { reason: ParserGapReason.UnsupportedType, fileType };
}
switch (fileType) {
case FileType.Audio: {
+ // Audio keeps a per-setup model: it is an ASR model, not the shared
+ // vision one.
return setup.vlm?.llm_id
? null
: {
@@ -231,7 +272,10 @@ export function findParserGap(
};
}
case FileType.Video: {
- return setup.vlm?.llm_id
+ const hasVisionModel = config.visionEnhancement
+ ? config.visionEnhancement.enabled && !!config.visionEnhancement.modelId
+ : !!setup.vlm?.llm_id;
+ return hasVisionModel
? null
: {
reason: ParserGapReason.MissingModel,
@@ -246,11 +290,11 @@ export function findParserGap(
export function findFilesParserGaps(
names: string[],
- setups: ParserSetup[],
+ config: SavedParserConfig,
): FileParserGap[] {
return names.flatMap((name) => {
const fileType = getFileTypeByExtension(getExtension(name));
- const gap = findParserGap(fileType, setups);
+ const gap = findParserGap(fileType, config);
return gap ? [{ ...gap, name }] : [];
});
}
@@ -258,15 +302,15 @@ export function findFilesParserGaps(
/**
* Per-document parse validation: each document row carries the parser config
* it actually runs with (dataset snapshot at upload, the override after a
- * parser change), so it is validated against its own setups. Rows without a
- * Parser entry fall back to `fallbackSetups` (the dataset-level config).
+ * parser change), so it is validated against its own config. Rows without a
+ * Parser entry fall back to `fallbackConfig` (the dataset-level config).
*/
export function findDocumentsParserGaps(
documents: Array<{ name: string; parser_config?: unknown }>,
- fallbackSetups: ParserSetup[] | null,
+ fallbackConfig: SavedParserConfig | null,
): FileParserGap[] {
return documents.flatMap((doc) => {
- const setups = getSavedParserSetups(doc) ?? fallbackSetups;
- return setups ? findFilesParserGaps([doc.name], setups) : [];
+ const config = getSavedParserConfig(doc) ?? fallbackConfig;
+ return config ? findFilesParserGaps([doc.name], config) : [];
});
}