RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix: Even though a vision model has been configured, uploading a video still triggers a notification stating that no vision model is configured. (#20816)
balibabue4695ca5594b
diff --git a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts b/web/src/pages/dataset/dataset/use-parser-gap-validation.tsindex 6cb0c30cb4..ed27c98e4b 100644--- a/web/src/pages/dataset/dataset/use-parser-gap-validation.ts+++ b/web/src/pages/dataset/dataset/use-parser-gap-validation.ts@@ -5,7 +5,7 @@ import {FileParserGap,findDocumentsParserGaps,findFilesParserGaps,- getSavedParserSetups,+ getSavedParserConfig,} from './utils';/**@@ -25,11 +25,11 @@ export function useParserGapValidation() {if (loading) {return [];}- const setups = getSavedParserSetups(knowledgeBase);- if (!setups) {+ const config = getSavedParserConfig(knowledgeBase);+ if (!config) {return [];}- return findFilesParserGaps(names, setups);+ return findFilesParserGaps(names, config);},[loading, knowledgeBase],);@@ -46,7 +46,7 @@ export function useParserGapValidation() {}return findDocumentsParserGaps(documents,- getSavedParserSetups(knowledgeBase),+ getSavedParserConfig(knowledgeBase),);},[loading, knowledgeBase],diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.tsindex 651ebd4443..6ec3d7270b 100644--- a/web/src/pages/dataset/dataset/utils.test.ts+++ b/web/src/pages/dataset/dataset/utils.test.ts@@ -12,11 +12,12 @@ import {getDocumentRunningStatus,getDocumentProgressMessage,getFileTypeByExtension,- getSavedParserSetups,+ getSavedParserConfig,hasUnsupportedTypeGap,ingestionStatusToRunningStatus,isDocumentProcessing,isDocumentStopping,+ type SavedParserConfig,} from './utils';describe('isDocumentStopping', () => {@@ -124,63 +125,96 @@ describe('getFileTypeByExtension', () => {});});-// Saved setups shaped like a pipeline that declares pdf/image plus audio-// (model missing) and video (model configured).-const SavedSetups = [- { fileFormat: FileType.PDF, parse_method: 'DeepDOC' },- { fileFormat: FileType.Image, parse_method: 'ocr' },- { fileFormat: FileType.Audio, vlm: { llm_id: '' } },- { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },-] as Record<string, any>[];+// Saved config shaped like a pipeline that declares pdf/image plus audio+// (ASR model missing) and video, with the Parser-level vision enhancement+// enabled and its model configured — the current contract, where media+// families other than audio carry no model of their own.+const SavedConfig: SavedParserConfig = {+ setups: [+ { fileFormat: FileType.PDF, parse_method: 'DeepDOC' },+ { fileFormat: FileType.Image, ocr_enabled: true },+ { fileFormat: FileType.Audio, vlm: { llm_id: '' } },+ { fileFormat: FileType.Video },+ ],+ visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },+};describe('findParserGap', () => {it('flags a file type missing from the saved setups as unsupported', () => {- expect(findParserGap(FileType.Docx, SavedSetups)).toEqual({+ expect(findParserGap(FileType.Docx, SavedConfig)).toEqual({reason: ParserGapReason.UnsupportedType,fileType: FileType.Docx,});});it('flags audio/video without a declared family as unsupported, not missing-model', () => {- const setups = [{ fileFormat: FileType.PDF }];- expect(findParserGap(FileType.Audio, setups)).toEqual({+ const config: SavedParserConfig = {+ setups: [{ fileFormat: FileType.PDF }],+ };+ expect(findParserGap(FileType.Audio, config)).toEqual({reason: ParserGapReason.UnsupportedType,fileType: FileType.Audio,});- expect(findParserGap(FileType.Video, setups)).toEqual({+ expect(findParserGap(FileType.Video, config)).toEqual({reason: ParserGapReason.UnsupportedType,fileType: FileType.Video,});});it('flags declared audio without a model as a missing asr model', () => {- expect(findParserGap(FileType.Audio, SavedSetups)).toEqual({+ expect(findParserGap(FileType.Audio, SavedConfig)).toEqual({reason: ParserGapReason.MissingModel,fileType: FileType.Audio,modelKind: ParserModelKind.Asr,});});- it('flags declared video without a model as a missing vision model', () => {- const setups = [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }];- expect(findParserGap(FileType.Video, setups)).toEqual({+ it('passes video when the Parser-level vision enhancement has a model', () => {+ expect(findParserGap(FileType.Video, SavedConfig)).toBeNull();+ });++ it('flags declared video when the vision enhancement is off or has no model', () => {+ const disabled: SavedParserConfig = {+ setups: [{ fileFormat: FileType.Video }],+ visionEnhancement: { enabled: false, modelId: 'glm-4v@ZHIPU-AI' },+ };+ const noModel: SavedParserConfig = {+ setups: [{ fileFormat: FileType.Video }],+ visionEnhancement: { enabled: true, modelId: '' },+ };+ const expected = {reason: ParserGapReason.MissingModel,fileType: FileType.Video,modelKind: ParserModelKind.Vision,- });+ };+ expect(findParserGap(FileType.Video, disabled)).toEqual(expected);+ expect(findParserGap(FileType.Video, noModel)).toEqual(expected);});- it('passes video when the setup carries a model', () => {- expect(findParserGap(FileType.Video, SavedSetups)).toBeNull();+ it('reads the video model from the family setup for configs saved before the move', () => {+ const withModel: SavedParserConfig = {+ setups: [+ { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },+ ],+ };+ const withoutModel: SavedParserConfig = {+ setups: [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }],+ };+ expect(findParserGap(FileType.Video, withModel)).toBeNull();+ expect(findParserGap(FileType.Video, withoutModel)).toEqual({+ reason: ParserGapReason.MissingModel,+ fileType: FileType.Video,+ modelKind: ParserModelKind.Vision,+ });});it('passes image with an ocr-only setup — no model required', () => {- expect(findParserGap(FileType.Image, SavedSetups)).toBeNull();+ expect(findParserGap(FileType.Image, SavedConfig)).toBeNull();});it('passes supported non-media types and unknown file types', () => {- expect(findParserGap(FileType.PDF, SavedSetups)).toBeNull();- expect(findParserGap(undefined, SavedSetups)).toBeNull();+ expect(findParserGap(FileType.PDF, SavedConfig)).toBeNull();+ expect(findParserGap(undefined, SavedConfig)).toBeNull();});});@@ -188,7 +222,7 @@ describe('findFilesParserGaps', () => {it('returns a gap per failing file, mixing both reasons', () => {const gaps = findFilesParserGaps(['song.mp3', 'notes.docx', 'photo.jpg', 'movie.mp4'],- SavedSetups,+ SavedConfig,);expect(gaps).toEqual([{@@ -231,11 +265,11 @@ describe('hasUnsupportedTypeGap', () => {});});-describe('getSavedParserSetups', () => {+describe('getSavedParserConfig', () => {it('returns null without a saved parser config', () => {- expect(getSavedParserSetups(null)).toBeNull();+ expect(getSavedParserConfig(null)).toBeNull();expect(- getSavedParserSetups({+ getSavedParserConfig({parser_config: {},}),).toBeNull();@@ -245,7 +279,7 @@ describe('getSavedParserSetups', () => {const knowledgeBase = {parser_config: { chunk_token_num: 128, delimiter: '\n' },};- expect(getSavedParserSetups(knowledgeBase)).toBeNull();+ expect(getSavedParserConfig(knowledgeBase)).toBeNull();});it('returns only the saved Parser setups, without merging defaults', () => {@@ -257,8 +291,42 @@ describe('getSavedParserSetups', () => {},},};- const setups = getSavedParserSetups(knowledgeBase);- expect(setups?.map((x) => x.fileFormat).sort()).toEqual(['image', 'pdf']);+ const config = getSavedParserConfig(knowledgeBase);+ expect(config?.setups.map((x) => x.fileFormat).sort()).toEqual([+ 'image',+ 'pdf',+ ]);+ });++ it('reads the vision enhancement globals from the Parser entry top level', () => {+ const knowledgeBase = {+ parser_config: {+ 'Parser:1': {+ enable_vision_enhancement: true,+ vlm: { llm_id: 'glm-4v@ZHIPU-AI', system_prompt: 'describe' },+ video: {},+ },+ },+ };+ expect(getSavedParserConfig(knowledgeBase)).toEqual({+ setups: [{ fileFormat: FileType.Video }],+ visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },+ });+ });++ it('leaves the vision enhancement undefined for configs saved before the move', () => {+ const knowledgeBase = {+ parser_config: {+ 'Parser:1': {+ video: { vlm: { llm_id: 'gpt-4o@OpenAI' } },+ },+ },+ };+ const config = getSavedParserConfig(knowledgeBase);+ expect(config?.visionEnhancement).toBeUndefined();+ expect(config?.setups).toEqual([+ { fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },+ ]);});it('ignores non-parser operator entries', () => {@@ -267,7 +335,7 @@ describe('getSavedParserSetups', () => {'TokenChunker:1': { chunk_token_size: 128 },},};- expect(getSavedParserSetups(knowledgeBase)).toBeNull();+ expect(getSavedParserConfig(knowledgeBase)).toBeNull();});});@@ -285,9 +353,9 @@ describe('findDocumentsParserGaps', () => {expect(findDocumentsParserGaps([OverriddenDoc], null)).toEqual([]);});- it('falls back to the dataset-level setups for rows without a Parser entry', () => {+ it('falls back to the dataset-level config for rows without a Parser entry', () => {const legacyRow = { name: 'song.mp3', parser_config: {} };- expect(findDocumentsParserGaps([legacyRow], SavedSetups)).toEqual([+ expect(findDocumentsParserGaps([legacyRow], SavedConfig)).toEqual([{name: 'song.mp3',reason: ParserGapReason.MissingModel,@@ -305,7 +373,11 @@ describe('findDocumentsParserGaps', () => {[OverriddenDoc, inheritedRow],// Dataset-level config declares only audio: the overridden pdf passes// while the inherited jpg is flagged.- [{ fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } }],+ {+ setups: [+ { fileFormat: FileType.Audio, vlm: { llm_id: 'whisper@OpenAI' } },+ ],+ },);expect(gaps).toEqual([{@@ -315,4 +387,18 @@ describe('findDocumentsParserGaps', () => {},]);});++ it('reads the document row vision globals for video', () => {+ const videoDoc = {+ name: 'movie.mp4',+ parser_config: {+ 'Parser:1': {+ enable_vision_enhancement: true,+ vlm: { llm_id: 'glm-4v@ZHIPU-AI' },+ video: {},+ },+ },+ };+ expect(findDocumentsParserGaps([videoDoc], null)).toEqual([]);+ });});diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.tsindex 6040256eaf..bbb6c68604 100644--- a/web/src/pages/dataset/dataset/utils.ts+++ b/web/src/pages/dataset/dataset/utils.ts@@ -128,13 +128,14 @@ export const isDocumentProcessing = (// --- Parser prerequisite checks -------------------------------------------// A file cannot be parsed under the dataset's current Parser operator when// the operator does not declare the file's type family at all-// (unsupportedType), or — for audio/video — when the declared setup carries-// no model (missingModel). The tenant default is deliberately not consulted:-// parsing reads the operator setup, so a global default does not make the-// file parsable. Image files only need their family declared: the image+// (unsupportedType), or when the family is declared but the model it needs+// is not set (missingModel): audio reads its per-setup ASR model, while+// video reads the Parser-level vision enhancement (enable_vision_enhancement+// + vlm.llm_id) — its model no longer lives on the family setup. The tenant+// default is deliberately not consulted: a global default does not make the+// file parsable here. Image files only need their family declared: the image// OCR switch runs local extraction independently, and the vision model is a-// global supplement picked from the top-level vlm.llm_id (falling back to the-// tenant default).+// global supplement for it.// These helpers power the upload warning and the parse-click validation.export type ParserModelGap = {@@ -168,6 +169,22 @@ export function pickByGapKind<T>(type ParserSetup = Record<string, any> & { fileFormat?: string };+/**+ * The Parser-level vision enhancement globals (enable_vision_enhancement ++ * vlm.llm_id): one switch and model serve every media family. Undefined marks+ * a config saved before the model moved off the per-family setups — those+ * still carry the video model on the video setup itself.+ */+export type ParserVisionEnhancement = {+ enabled: boolean;+ modelId: string;+};++export type SavedParserConfig = {+ setups: ParserSetup[];+ visionEnhancement?: ParserVisionEnhancement;+};+const ExtensionToFileTypeMap: Record<string, FileType> = Object.entries(FileTypeSuffixMap,).reduce<Record<string, FileType>>((acc, [fileType, suffixes]) => {@@ -183,18 +200,37 @@ export function getFileTypeByExtension(return ExtensionToFileTypeMap[extension.toLowerCase()];}+// Reads the Parser-level vision globals from a saved Parser entry. A boolean+// enable_vision_enhancement marks the current contract; configs saved before+// the model moved to the top level do not have it, and their video setup+// still holds the model — so the absence is meaningful, not a default-off.+function readVisionEnhancement(+ parserEntry: unknown,+): ParserVisionEnhancement | undefined {+ if (typeof parserEntry !== 'object' || parserEntry === null) {+ return undefined;+ }+ const params = parserEntry as Record<string, any>;+ const enabled = params.enable_vision_enhancement;+ if (typeof enabled !== 'boolean') {+ return undefined;+ }+ const modelId = params.vlm?.llm_id;+ return { enabled, modelId: typeof modelId === 'string' ? modelId : '' };+}+/**- * Parser setups exactly as saved, keyed by file format — the keys are the- * type families the pipeline/built-in method declares. Accepts either a- * dataset or a document row: a document's parser_config snapshots the parser- * it actually runs with (dataset config at upload, the override after a- * parser change). Returns null when parser_config has no Parser operator- * entry (legacy flat-shape configs, pipelines without a Parser stage): type- * support cannot be determined then and validation is skipped.+ * Parser config exactly as saved: the per-family setups plus the Parser-level+ * vision enhancement globals. Accepts either a dataset or a document row: a+ * document's parser_config snapshots the parser it actually runs with+ * (dataset config at upload, the override after a parser change). Returns+ * null when parser_config has no Parser operator entry (legacy flat-shape+ * configs, pipelines without a Parser stage): type support cannot be+ * determined then and validation is skipped.*/-export function getSavedParserSetups(+export function getSavedParserConfig(source: { parser_config?: unknown } | null | undefined,-): ParserSetup[] | null {+): SavedParserConfig | null {const parserConfig = source?.parser_config as Record<string, any> | undefined;const parserEntry = Object.entries(parserConfig ?? {}).find(@@ -204,24 +240,29 @@ export function getSavedParserSetups(if (!parserEntry) {return null;}- return transformParserConfigSetups(parserEntry);+ return {+ setups: transformParserConfigSetups(parserEntry),+ visionEnhancement: readVisionEnhancement(parserEntry),+ };}export function findParserGap(fileType: FileType | undefined,- setups: ParserSetup[],+ config: SavedParserConfig,): ParserGap | null {if (!fileType) {return null;}- const setup = setups.find((x) => x.fileFormat === fileType);+ const setup = config.setups.find((x) => x.fileFormat === fileType);if (!setup) {return { reason: ParserGapReason.UnsupportedType, fileType };}switch (fileType) {case FileType.Audio: {+ // Audio keeps a per-setup model: it is an ASR model, not the shared+ // vision one.return setup.vlm?.llm_id? null: {@@ -231,7 +272,10 @@ export function findParserGap(};}case FileType.Video: {- return setup.vlm?.llm_id+ const hasVisionModel = config.visionEnhancement+ ? config.visionEnhancement.enabled && !!config.visionEnhancement.modelId+ : !!setup.vlm?.llm_id;+ return hasVisionModel? null: {reason: ParserGapReason.MissingModel,@@ -246,11 +290,11 @@ export function findParserGap(export function findFilesParserGaps(names: string[],- setups: ParserSetup[],+ config: SavedParserConfig,): FileParserGap[] {return names.flatMap((name) => {const fileType = getFileTypeByExtension(getExtension(name));- const gap = findParserGap(fileType, setups);+ const gap = findParserGap(fileType, config);return gap ? [{ ...gap, name }] : [];});}@@ -258,15 +302,15 @@ export function findFilesParserGaps(/*** Per-document parse validation: each document row carries the parser config* it actually runs with (dataset snapshot at upload, the override after a- * parser change), so it is validated against its own setups. Rows without a- * Parser entry fall back to `fallbackSetups` (the dataset-level config).+ * parser change), so it is validated against its own config. Rows without a+ * Parser entry fall back to `fallbackConfig` (the dataset-level config).*/export function findDocumentsParserGaps(documents: Array<{ name: string; parser_config?: unknown }>,- fallbackSetups: ParserSetup[] | null,+ fallbackConfig: SavedParserConfig | null,): FileParserGap[] {return documents.flatMap((doc) => {- const setups = getSavedParserSetups(doc) ?? fallbackSetups;- return setups ? findFilesParserGaps([doc.name], setups) : [];+ const config = getSavedParserConfig(doc) ?? fallbackConfig;+ return config ? findFilesParserGaps([doc.name], config) : [];});}