Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): block video parse while keeping visual upload routing (#21245)

jay77721b84cadea97c8
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index 8bfa54ca07..e0d50afd8a 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -1101,3 +1101,35 @@ func picturePNG(t *testing.T) []byte {
}
return data.Bytes()
}
+
+// TestMaybeDispatchVideoRejectsExplicitly pins the runtime backstop behind the
+// frontend parse block: a video file whose setups declare the video family
+// gets an explicit "not yet supported" error (not a silent fallback), while
+// configs without a video family fall through to the generic dispatch path.
+func TestMaybeDispatchVideoRejectsExplicitly(t *testing.T) {
+ ctx := t.Context()
+ inputs := map[string]any{"tenant_id": "t1"}
+
+ // Main path: standard component defaults declare the video family, so the
+ // branch handles the file and fails loudly.
+ _, handled, err := maybeDispatchVideo(ctx, nil, utility.FileTypeVIDEO, "clip.mov", []byte("mov-bytes"), inputs, defaultSetups())
+ if !handled {
+ t.Fatal("maybeDispatchVideo did not handle a video file with a video setup")
+ }
+ if err == nil || !strings.Contains(err.Error(), "not yet supported") {
+ t.Fatalf("video error = %v, want explicit not-yet-supported", err)
+ }
+
+ // Without a video family the branch stays out of the way; the generic
+ // dispatch path (GetParser "unsupported file type") answers instead.
+ _, handled, err = maybeDispatchVideo(ctx, nil, utility.FileTypeVIDEO, "clip.mov", []byte("mov-bytes"), inputs, map[string]schema.ParserSetup{})
+ if handled || err != nil {
+ t.Fatalf("handled = %v, err = %v, want pass-through without a video setup", handled, err)
+ }
+
+ // Non-video files never enter the video branch.
+ _, handled, err = maybeDispatchVideo(ctx, nil, utility.FileTypeVISUAL, "photo.jpg", []byte("jpeg-bytes"), inputs, defaultSetups())
+ if handled || err != nil {
+ t.Fatalf("handled = %v, err = %v, want pass-through for images", handled, err)
+ }
+}
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index fafa593fdb..c3e8fbe49f 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -251,6 +251,9 @@ func familyToExt(family string) utility.FileType {
// Python family identifier used by ParserComponent setups.
// Returns "" when the hint is unknown.
func pythonFamilyName(raw string) string {
+ if utility.IsVideoExtension(raw) || raw == "video" {
+ return "video"
+ }
switch raw {
case "pdf":
return "pdf"
@@ -275,9 +278,6 @@ func pythonFamilyName(raw string) string {
case "text&code", "txt", "py", "js", "java", "c", "cpp", "h", "php",
"go", "ts", "sh", "cs", "kt", "sql":
return "text&code"
- case "mp4", "avi", "mkv", "mov", "webm", "flv",
- "mpeg", "mpg", "wmv", "3gp", "3gpp", "video":
- return "video"
case "eml", "msg", "email":
return "email"
case "da", "wave", "wav", "mp3", "aac", "flac", "ogg",
diff --git a/internal/service/document/document_parser_selection_test.go b/internal/service/document/document_parser_selection_test.go
index 4721f284d8..2f07e5af55 100644
--- a/internal/service/document/document_parser_selection_test.go
+++ b/internal/service/document/document_parser_selection_test.go
@@ -638,6 +638,63 @@ func TestUploadLocalDocuments_FallsBackWhenSelectionLacksFamily(t *testing.T) {
parserFamilySetup(t, got[1], "text&code")
}
+// TestUploadLocalDocuments_SwitchesVideoToPicture pins the upload policy:
+// video uploads are accepted (visual bucket) and switched to the picture
+// pipeline; parsing stays blocked later because no parser implements video
+// analysis.
+func TestUploadLocalDocuments_SwitchesVideoToPicture(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+
+ mockStorage := newFakeUploadStorage()
+ factory := storage.GetStorageFactory()
+ origStorage := factory.GetStorage()
+ factory.SetStorage(mockStorage)
+ t.Cleanup(func() { factory.SetStorage(origStorage) })
+
+ if err := dao.DB.Create(&entity.Tenant{
+ ID: "tenant-1", LLMID: "llm-default", EmbdID: "embd-default", ASRID: "asr-default",
+ }).Error; err != nil {
+ t.Fatalf("insert tenant: %v", err)
+ }
+
+ kb := &entity.Knowledgebase{
+ ID: "kb-video-block",
+ TenantID: "tenant-1",
+ Name: "kb-video-block",
+ CreatedBy: "user-1",
+ ParserID: "naive",
+ ParserConfig: presentationParserConfig(),
+ }
+ if err := dao.DB.Create(kb).Error; err != nil {
+ t.Fatalf("insert kb: %v", err)
+ }
+
+ ctx := t.Context()
+ svc := testDocumentService(t)
+ got, errs := svc.UploadLocalDocuments(ctx, kb, "user-1", []*multipart.FileHeader{
+ makeTestFileHeader(t, "file", "clip.mov", []byte("mov-bytes")),
+ makeTestFileHeader(t, "file", "movie.mp4", []byte("mp4-bytes")),
+ makeTestFileHeader(t, "file", "legacy.rmvb", []byte("rmvb-bytes")),
+ makeTestFileHeader(t, "file", "notes.txt", []byte("plain text")),
+ }, "", nil)
+ if len(errs) != 0 {
+ t.Fatalf("unexpected errs: %v", errs)
+ }
+ if len(got) != 4 {
+ t.Fatalf("uploaded docs = %d, want 4", len(got))
+ }
+ for _, doc := range got[:3] {
+ if parserID, _ := doc["parser_id"].(string); parserID != "picture" {
+ t.Fatalf("%v parser_id = %q, want picture", doc["name"], parserID)
+ }
+ if docType, _ := doc["type"].(string); docType != string(utility.FileTypeVISUAL) {
+ t.Fatalf("%v type = %q, want %q", doc["name"], docType, utility.FileTypeVISUAL)
+ }
+ parserFamilySetup(t, doc, "video")
+ }
+}
+
// TestInsertSyncDocument_FallsBackWhenSelectionLacksFamily covers the connector
// path: it shares resolveDocumentParser with the upload path, so a synced text
// file lands on general too.
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 4a702252cd..7db25ec4fc 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -139,6 +139,13 @@ func (s *FileService) GetFileContents(ctx context.Context, uid string, fileDicts
}
ft := utility.FilenameType(name)
+ if utility.IsVideoFilename(name) {
+ // Uploads accept video, but no parser implements video analysis.
+ // Report that instead of inlining the container as a bogus image
+ // data URI (the old visual classification) or dumping raw bytes.
+ texts = append(texts, fmt.Sprintf("\n -----------------\nFile: %s\nContent as following: \n(video content is not supported)\n", name))
+ continue
+ }
if ft == utility.FileTypeVISUAL {
mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))
if mediaType == "" {
@@ -183,6 +190,14 @@ func (s *FileService) ParseAgentUploads(ctx context.Context, userID string, file
return nil, fmt.Errorf("file %q: upload is empty", name)
}
+ // Reject video filenames before the MIME-based image branch: a
+ // mislabeled video (e.g. clip.mp4 declaring image/png) would otherwise
+ // be inlined as a bogus image data URI without ever reaching the
+ // parser-level video check.
+ if utility.IsVideoFilename(name) {
+ return nil, fmt.Errorf("file %q: video content is not supported", name)
+ }
+
mediaType := strings.ToLower(strings.TrimSpace(strings.Split(mimeType, ";")[0]))
if strings.HasPrefix(mediaType, "image/") {
contents = append(contents, "data:"+mediaType+";base64,"+base64.StdEncoding.EncodeToString(data))
diff --git a/internal/service/file/file_test.go b/internal/service/file/file_test.go
index 1ce08d36f7..e59ebdadcc 100644
--- a/internal/service/file/file_test.go
+++ b/internal/service/file/file_test.go
@@ -233,6 +233,40 @@ func TestFileService_GetFileContents_ImageAttachmentAsDataURI(t *testing.T) {
}
}
+// Video attachments are accepted but unparsable: GetFileContents reports a
+// fixed notice as the file's text (so the model sees why there is no content)
+// instead of inlining the container as a bogus image or failing the batch.
+// A mixed batch keeps the valid attachments alongside the notice.
+func TestFileService_GetFileContents_VideoAttachmentReportsNotice(t *testing.T) {
+ memory := storage.NewMemoryStorage()
+ ctx := t.Context()
+ if err := memory.Put(ctx, "user-1-downloads", "vid-1", []byte("mp4-bytes")); err != nil {
+ t.Fatalf("put video: %v", err)
+ }
+ if err := memory.Put(ctx, "user-1-downloads", "img-1", []byte("jpeg-bytes")); err != nil {
+ t.Fatalf("put image: %v", err)
+ }
+ factory := storage.GetStorageFactory()
+ originalStorage := factory.GetStorage()
+ factory.SetStorage(memory)
+ t.Cleanup(func() { factory.SetStorage(originalStorage) })
+
+ svc := testFileService()
+ texts, images, err := svc.GetFileContents(ctx, "user-1", []map[string]interface{}{
+ {"id": "vid-1", "name": "clip.mp4", "mime_type": "video/mp4", "created_by": "user-1"},
+ {"id": "img-1", "name": "photo.jpeg", "mime_type": "image/jpeg", "created_by": "user-1"},
+ })
+ if err != nil {
+ t.Fatalf("GetFileContents failed: %v", err)
+ }
+ if len(texts) != 1 || !strings.Contains(texts[0], "clip.mp4") || !strings.Contains(texts[0], "video content is not supported") {
+ t.Fatalf("video notice = %v, want one text entry naming the file as unsupported", texts)
+ }
+ if len(images) != 1 {
+ t.Fatalf("expected the valid image to survive alongside the notice, got %v", images)
+ }
+}
+
func TestFileService_ParseAgentUploads_TextAndImageInRequestOrder(t *testing.T) {
ctx := t.Context()
memory := storage.NewMemoryStorage()
@@ -265,6 +299,25 @@ func TestFileService_ParseAgentUploads_TextAndImageInRequestOrder(t *testing.T)
}
}
+func TestFileService_ParseAgentUploads_RejectsVideoWithImageMime(t *testing.T) {
+ ctx := t.Context()
+ memory := storage.NewMemoryStorage()
+ if err := memory.Put(ctx, "user-1-downloads", "video-id", []byte("mp4-bytes")); err != nil {
+ t.Fatalf("put video: %v", err)
+ }
+ factory := storage.GetStorageFactory()
+ originalStorage := factory.GetStorage()
+ factory.SetStorage(memory)
+ t.Cleanup(func() { factory.SetStorage(originalStorage) })
+
+ _, err := testFileService().ParseAgentUploads(ctx, "user-1", []map[string]interface{}{
+ {"id": "video-id", "name": "clip.mp4", "mime_type": "image/png", "created_by": "user-1"},
+ }, "")
+ if err == nil || !strings.Contains(err.Error(), "video content is not supported") {
+ t.Fatalf("error = %v, want video rejection even with an image MIME type", err)
+ }
+}
+
func TestFileService_ParseAgentUploads_RejectsForeignOwner(t *testing.T) {
ctx := t.Context()
memory := storage.NewMemoryStorage()
diff --git a/internal/utility/file.go b/internal/utility/file.go
index b83a380272..40fe309b3f 100644
--- a/internal/utility/file.go
+++ b/internal/utility/file.go
@@ -52,8 +52,33 @@ const (
var (
filenameLenLimit = 255
+
+ // videoExtensions is the single source of truth for video containers.
+ // GetFileType (runtime dispatch) and the ingestion parser-family mapping
+ // read this set. FilenameType keeps the legacy upload bucket (video is
+ // admitted as visual and routed to the picture pipeline), so every video
+ // container accepted below must also be present here.
+ videoExtensions = map[string]struct{}{
+ "mp4": {}, "avi": {}, "mkv": {}, "mov": {}, "webm": {}, "flv": {},
+ "mpeg": {}, "mpg": {}, "wmv": {}, "3gp": {}, "3gpp": {},
+ "rm": {}, "rmvb": {}, "asf": {}, "dat": {}, "asx": {}, "wvx": {},
+ "mpe": {}, "mpa": {},
+ }
)
+// IsVideoExtension reports whether ext (with or without a leading dot,
+// case-insensitive) is a video container.
+func IsVideoExtension(ext string) bool {
+ normalized := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(ext), "."))
+ _, ok := videoExtensions[normalized]
+ return ok
+}
+
+// IsVideoFilename reports whether filename has a video extension.
+func IsVideoFilename(filename string) bool {
+ return IsVideoExtension(filepath.Ext(filename))
+}
+
func init() {
}
@@ -81,6 +106,10 @@ func GetFileType(filename string) FileType {
suffix = strings.ToLower(ext)
}
+ if IsVideoExtension(suffix) {
+ return FileTypeVIDEO
+ }
+
switch suffix {
case "pdf":
return FileTypePDF
@@ -114,9 +143,6 @@ func GetFileType(filename string) FileType {
case "da", "wave", "wav", "mp3", "aac", "flac", "ogg",
"aiff", "au", "midi", "wma", "ape", "alac", "wv", "opus":
return FileTypeAURAL
- case "mp4", "avi", "mkv", "mov", "webm", "flv", "mpeg",
- "mpg", "wmv", "3gp", "3gpp":
- return FileTypeVIDEO
case "png", "jpg", "jpeg", "gif", "bmp":
return FileTypeVISUAL
case "tiff", "tif", "webp", "svg", "ico":
@@ -137,6 +163,15 @@ func FilenameType(filename string) FileType {
return FileTypeOTHER
}
+ // Video is admitted at upload in the legacy visual bucket and routed to the
+ // picture pipeline; parsing is blocked later (no parser implements video
+ // analysis). Keep the admission here so every video container in the shared
+ // set passes the upload gate, including webm/flv/3gp/3gpp which the old
+ // bucket missed.
+ if IsVideoFilename(normalized) {
+ return FileTypeVISUAL
+ }
+
if matched, _ := regexp.MatchString(`.*\.pdf$`, normalized); matched {
return FileTypePDF
}
@@ -165,8 +200,6 @@ func FilenameType(filename string) FileType {
visualExtensions := []string{
"jpg", "jpeg", "png", "tif", "gif", "bmp", "pcx", "tga", "exif", "fpx", "svg", "psd", "cdr",
"pcd", "dxf", "ufo", "eps", "ai", "raw", "wmf", "webp", "avif", "apng", "icon", "ico",
- "mpg", "mpeg", "avi", "rm", "rmvb", "mov", "wmv", "asf", "dat", "asx", "wvx", "mpe",
- "mpa", "mp4", "mkv",
}
for _, ext := range visualExtensions {
if strings.HasSuffix(normalized, "."+ext) {
diff --git a/internal/utility/filename_type_test.go b/internal/utility/filename_type_test.go
index 13ef38f44c..494328d09b 100644
--- a/internal/utility/filename_type_test.go
+++ b/internal/utility/filename_type_test.go
@@ -37,3 +37,58 @@ func TestFilenameTypeBMP(t *testing.T) {
}
}
}
+
+// TestFilenameTypeVideo covers the upload bucket: video is admitted as visual
+// (legacy categorization) and routed to the picture pipeline; parsing is
+// blocked later. Every extension in the shared video set must classify as
+// visual here, while GetFileType reports them as video for runtime dispatch.
+func TestFilenameTypeVideo(t *testing.T) {
+ cases := []struct {
+ name string
+ want FileType
+ }{
+ {"clip.mp4", FileTypeVISUAL},
+ {"clip.mov", FileTypeVISUAL},
+ {"CLIP.MOV", FileTypeVISUAL},
+ {"clip.mkv", FileTypeVISUAL},
+ {"clip.webm", FileTypeVISUAL},
+ {"clip.flv", FileTypeVISUAL},
+ {"clip.wmv", FileTypeVISUAL},
+ {"clip.mpeg", FileTypeVISUAL},
+ {"clip.mpg", FileTypeVISUAL},
+ {"clip.3gp", FileTypeVISUAL},
+ {"clip.3gpp", FileTypeVISUAL},
+ {"clip.rm", FileTypeVISUAL},
+ {"clip.rmvb", FileTypeVISUAL},
+ {"clip.asf", FileTypeVISUAL},
+ {"clip.dat", FileTypeVISUAL},
+ {"clip.asx", FileTypeVISUAL},
+ {"clip.wvx", FileTypeVISUAL},
+ {"clip.mpe", FileTypeVISUAL},
+ {"clip.mpa", FileTypeVISUAL},
+ {"photo.jpg", FileTypeVISUAL},
+ }
+ for _, tc := range cases {
+ if got := FilenameType(tc.name); got != tc.want {
+ t.Fatalf("FilenameType(%q) = %q, want %q", tc.name, got, tc.want)
+ }
+ }
+}
+
+// TestIsVideoExtension pins the shared predicate that upload admission, runtime
+// dispatch and the parser-family mapping read.
+func TestIsVideoExtension(t *testing.T) {
+ for _, ext := range []string{"mp4", ".MOV", "rmvb", "3gpp"} {
+ if !IsVideoExtension(ext) {
+ t.Fatalf("IsVideoExtension(%q) = false, want true", ext)
+ }
+ }
+ for _, ext := range []string{"", "jpg", "pdf", "mp4x"} {
+ if IsVideoExtension(ext) {
+ t.Fatalf("IsVideoExtension(%q) = true, want false", ext)
+ }
+ }
+ if !IsVideoFilename("path/to/CLIP.MOV") {
+ t.Fatal("IsVideoFilename(\"path/to/CLIP.MOV\") = false, want true")
+ }
+}
diff --git a/web/src/constants/file.ts b/web/src/constants/file.ts
index cf32404d4c..44d9b018f0 100644
--- a/web/src/constants/file.ts
+++ b/web/src/constants/file.ts
@@ -55,7 +55,29 @@ export const FileTypeSuffixMap = {
[FileType.Doc]: ['doc'],
[FileType.Docx]: ['docx'],
[FileType.PowerPoint]: ['pptx', 'ppt'],
- [FileType.Video]: ['mp4', 'avi', 'mkv'],
+ [FileType.Video]: [
+ // Mirrors utility.videoExtensions (internal/utility/file.go): upload
+ // admission, runtime dispatch and the parser-family mapping read one set.
+ 'mp4',
+ 'avi',
+ 'mkv',
+ 'mov',
+ 'webm',
+ 'flv',
+ 'mpeg',
+ 'mpg',
+ 'wmv',
+ '3gp',
+ '3gpp',
+ 'rm',
+ 'rmvb',
+ 'asf',
+ 'dat',
+ 'asx',
+ 'wvx',
+ 'mpe',
+ 'mpa',
+ ],
[FileType.Audio]: [
'da',
'wave',
@@ -93,7 +115,22 @@ export const FileIconMap = {
mp4: 'mp4',
avi: 'avi',
mkv: 'mkv',
+ mov: 'mp4',
+ webm: 'mp4',
+ flv: 'mp4',
+ wmv: 'mp4',
+ mpeg: 'mp4',
+ mpg: 'mp4',
+ '3gp': 'mp4',
+ '3gpp': 'mp4',
+ rm: 'mp4',
rmvb: 'rmvb',
+ asf: 'mp4',
+ dat: 'mp4',
+ asx: 'mp4',
+ wvx: 'mp4',
+ mpe: 'mp4',
+ mpa: 'mp4',
wav: 'wav',
html: 'html',
json: 'json',
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index 8e154f1a1a..bb6c7a5568 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -416,6 +416,12 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
'{{name}} ({{fileType}}): not supported by the current parser configuration; it will automatically switch to the built-in {{parser}} configuration',
fileTypeUnsupportedManualSwitch:
'{{name}} ({{fileType}}): not supported by the current parser configuration; switch its parsing method manually after upload',
+ fileTypeVideoUnsupported:
+ '{{name}} ({{fileType}}): video files cannot be parsed yet',
+ videoUnsupportedAfterUploadHint:
+ 'You can continue uploading, but video files cannot be parsed yet and will not be processed.',
+ videoUnsupportedParseHint:
+ 'Video files cannot be parsed yet. Delete this file or convert it to a supported format.',
reselectParserAfterUploadHint:
'Continue uploading, then system will automatically switch these files to a supported built-in parsing configuration.',
manualSwitchAfterUploadHint:
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 8071789be5..e0e1c4d08e 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -372,6 +372,12 @@ export default {
'{{name}}({{fileType}}):当前解析配置不支持该文件类型,将自动切换为内置 {{parser}} 配置',
fileTypeUnsupportedManualSwitch:
'{{name}}({{fileType}}):当前解析配置不支持该文件类型,上传后可手动切换解析配置',
+ fileTypeVideoUnsupported:
+ '{{name}}({{fileType}}):视频文件暂不支持解析',
+ videoUnsupportedAfterUploadHint:
+ '可继续上传,但视频文件暂不支持解析,上传后不会被处理。',
+ videoUnsupportedParseHint:
+ '视频文件暂不支持解析,请删除该文件或转换为受支持的格式。',
reselectParserAfterUploadHint:
'可继续上传,系统将自动切换至支持这些文件类型的内置解析配置。',
manualSwitchAfterUploadHint:
diff --git a/web/src/pages/dataset/dataset/parser-gap-content.tsx b/web/src/pages/dataset/dataset/parser-gap-content.tsx
index 745500687c..65a80296cb 100644
--- a/web/src/pages/dataset/dataset/parser-gap-content.tsx
+++ b/web/src/pages/dataset/dataset/parser-gap-content.tsx
@@ -1,5 +1,6 @@
import type { TFunction } from 'i18next';
import type { ReactNode } from 'react';
+import { FileType } from '@/constants/file';
import { ParserGapReason, ParserModelKind } from './constant';
import {
BuiltinSwitchTargetLabels,
@@ -30,7 +31,21 @@ type UploadGapHintKeys = {
mixedSwitch: string;
};
+// Video uploads are accepted but parsing is unsupported (no parser implements
+// video analysis). The generic "reselect the parser" guidance is wrong for
+// them: no selectable parser can process the file, so they get their own copy.
+const isVideoGap = (gap: FileParserGap) => gap.fileType === FileType.Video;
+
+const onlyVideoGaps = (gaps: FileParserGap[]) =>
+ gaps.length > 0 && gaps.every(isVideoGap);
+
function renderGapLine(t: TFunction, gap: FileParserGap): ReactNode {
+ if (gap.fileType === FileType.Video) {
+ return t('knowledgeDetails.fileTypeVideoUnsupported', {
+ name: gap.name,
+ fileType: t(`flow.fileFormatOptions.${gap.fileType}`),
+ });
+ }
if (gap.reason === ParserGapReason.MissingModel) {
return t('knowledgeDetails.fileModelMissing', {
name: gap.name,
@@ -52,6 +67,9 @@ function renderUploadGapLine(
t: TFunction,
gap: UploadFileParserGap,
): ReactNode {
+ if (gap.fileType === FileType.Video) {
+ return renderGapLine(t, gap);
+ }
if (gap.reason === ParserGapReason.MissingModel) {
return renderGapLine(t, gap);
}
@@ -91,7 +109,14 @@ export function buildParserGapModalContent(
gaps: FileParserGap[],
hintKeys: GapHintKeys,
): ReactNode {
- return buildContent(t, gaps, pickByGapKind(gaps, hintKeys), renderGapLine);
+ return buildContent(
+ t,
+ gaps,
+ onlyVideoGaps(gaps)
+ ? 'knowledgeDetails.videoUnsupportedParseHint'
+ : pickByGapKind(gaps, hintKeys),
+ renderGapLine,
+ );
}
export function buildUploadGapModalContent(
@@ -102,7 +127,9 @@ export function buildUploadGapModalContent(
return buildContent(
t,
gaps,
- pickBySwitchFate(gaps, hintKeys),
+ onlyVideoGaps(gaps)
+ ? 'knowledgeDetails.videoUnsupportedAfterUploadHint'
+ : pickBySwitchFate(gaps, hintKeys),
renderUploadGapLine,
);
}
diff --git a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
index 6c93a0387a..40527042a6 100644
--- a/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
+++ b/web/src/pages/dataset/dataset/use-bulk-operate-dataset.tsx
@@ -1,4 +1,5 @@
import { Modal } from '@/components/ui/modal/modal';
+import { FileType } from '@/constants/file';
import {
UseRowSelectionType,
useSelectedIds,
@@ -21,7 +22,7 @@ import { useCallback } from 'react';
import { useTranslation } from 'react-i18next';
import { useParams } from 'react-router';
import { toast } from 'sonner';
-import { DocumentType } from './constant';
+import { DocumentType, ParserGapReason } from './constant';
import { buildParserGapModalContent } from './parser-gap-content';
import { useParserGapValidation } from './use-parser-gap-validation';
import { isDocumentProcessing } from './utils';
@@ -72,14 +73,26 @@ export function useBulkOperateDataset({
.map((x) => x.id);
if (validIds.length === 0) {
+ // Pure-video selections have no parser that can process them, so
+ // offering anything that implies a fix would mislead. Acknowledge
+ // instead, mirroring the single-document run path. The reason
+ // check keeps this narrow: a future gap kind on a parseable file
+ // must keep the actionable dialog.
+ const videoOnly =
+ gaps.length > 0 &&
+ gaps.every(
+ (gap) =>
+ gap.fileType === FileType.Video &&
+ gap.reason === ParserGapReason.UnsupportedType,
+ );
Modal.error({
title: t('knowledgeDetails.parseBlockedTitle'),
content: buildParserGapModalContent(t, gaps, {
missingModel: 'knowledgeDetails.addModelToParseHint',
unsupportedType: 'knowledgeDetails.reselectParserToParseHint',
}),
- showCancel: false,
- okText: t('common.cancel'),
+ showCancel: !videoOnly,
+ okText: t(videoOnly ? 'common.ok' : 'common.cancel'),
closable: false,
});
return;
diff --git a/web/src/pages/dataset/dataset/use-run-document.ts b/web/src/pages/dataset/dataset/use-run-document.ts
index 2c51f8963e..b24f4576f7 100644
--- a/web/src/pages/dataset/dataset/use-run-document.ts
+++ b/web/src/pages/dataset/dataset/use-run-document.ts
@@ -1,10 +1,12 @@
import { Modal } from '@/components/ui/modal/modal';
+import { FileType } from '@/constants/file';
import { useSetModalState } from '@/hooks/common-hooks';
import { useRunDocument } from '@/hooks/use-document-request';
import { IDocumentInfo } from '@/interfaces/database/document';
import { useState } from 'react';
import { useTranslation } from 'react-i18next';
import { buildParserGapModalContent } from './parser-gap-content';
+import { ParserGapReason } from './constant';
import { UseChangeDocumentParserShowType } from './use-change-document-parser';
import { useParserGapValidation } from './use-parser-gap-validation';
import { pickByGapKind } from './utils';
@@ -35,6 +37,18 @@ export const useHandleRunDocumentByIds = (
const gaps = findDocumentParseGaps([record]);
if (gaps.length > 0) {
hideModal();
+ // Video has no parser that can process it, so offering "reselect parse
+ // method" would send the user into a dialog that cannot fix anything.
+ // Acknowledge instead; the change-parser route stays for the other
+ // gap kinds. The reason check keeps this narrow: a future gap kind on
+ // a parseable file must keep the actionable dialog.
+ const videoOnly =
+ gaps.length > 0 &&
+ gaps.every(
+ (gap) =>
+ gap.fileType === FileType.Video &&
+ gap.reason === ParserGapReason.UnsupportedType,
+ );
Modal.error({
title: t('knowledgeDetails.parseBlockedTitle'),
content: buildParserGapModalContent(t, gaps, {
@@ -42,16 +56,17 @@ export const useHandleRunDocumentByIds = (
unsupportedType: 'knowledgeDetails.reselectParserToParseHint',
}),
okText: t(
- pickByGapKind(gaps, {
- missingModel: 'knowledgeDetails.goAddModel',
- unsupportedType: 'knowledgeDetails.reselectParser',
- }),
+ videoOnly
+ ? 'common.ok'
+ : pickByGapKind(gaps, {
+ missingModel: 'knowledgeDetails.goAddModel',
+ unsupportedType: 'knowledgeDetails.reselectParser',
+ }),
),
cancelText: t('common.cancel'),
+ showCancel: !videoOnly,
closable: false,
- onOk: () => {
- showChangeParserModal(record);
- },
+ onOk: videoOnly ? undefined : () => showChangeParserModal(record),
});
return;
}
diff --git a/web/src/pages/dataset/dataset/utils.test.ts b/web/src/pages/dataset/dataset/utils.test.ts
index 26f6b2ddec..04cd589bc3 100644
--- a/web/src/pages/dataset/dataset/utils.test.ts
+++ b/web/src/pages/dataset/dataset/utils.test.ts
@@ -124,11 +124,37 @@ describe('getFileTypeByExtension', () => {
expect(getFileTypeByExtension('mp3')).toBe(FileType.Audio);
expect(getFileTypeByExtension('wav')).toBe(FileType.Audio);
expect(getFileTypeByExtension('mp4')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('mov')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('webm')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('flv')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('wmv')).toBe(FileType.Video);
+ // Legacy containers share the same backend video set, so the upload warning
+ // and parse block must recognize them too.
+ expect(getFileTypeByExtension('rm')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('rmvb')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('asf')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('dat')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('asx')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('wvx')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('mpe')).toBe(FileType.Video);
+ expect(getFileTypeByExtension('mpa')).toBe(FileType.Video);
expect(getFileTypeByExtension('jpeg')).toBe(FileType.Image);
expect(getFileTypeByExtension('PNG')).toBe(FileType.Image);
expect(getFileTypeByExtension('pdf')).toBe(FileType.PDF);
});
+ it('ignores surrounding whitespace like the backend does', () => {
+ // getExtension trims (mirroring utility.normalizeFilename); without it a
+ // name like 'clip.mov ' maps to nothing and skips both the upload warning
+ // and the parse block while the backend still admits it as visual.
+ expect(getFileTypeByExtension(getExtension('clip.mov '))).toBe(
+ FileType.Video,
+ );
+ expect(switchTarget('clip.mov ')).toBe('picture');
+ // The trim is shared: other consumers resolve trailing-space names too.
+ expect(getExtension('report.pdf ')).toBe('pdf');
+ });
+
it('returns undefined for unknown extensions', () => {
expect(getFileTypeByExtension('xyz')).toBeUndefined();
expect(getFileTypeByExtension('')).toBeUndefined();
@@ -136,9 +162,8 @@ describe('getFileTypeByExtension', () => {
});
// Saved config shaped like a pipeline that declares pdf/image plus audio
-// (ASR model missing) and video, with the Parser-level vision enhancement
-// enabled and its model configured — the current contract, where media
-// families other than audio carry no model of their own.
+// (ASR model missing) and video. Video needs no model: parsing it is
+// unsupported regardless of the configured vision enhancement.
const SavedConfig: SavedParserConfig = {
setups: [
{ fileFormat: FileType.PDF, parse_method: 'DeepDOC' },
@@ -147,7 +172,6 @@ const SavedConfig: SavedParserConfig = {
{ fileFormat: FileType.Audio, vlm: { llm_id: '' } },
{ fileFormat: FileType.Video },
],
- visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },
};
describe('findParserGap', () => {
@@ -180,29 +204,16 @@ describe('findParserGap', () => {
});
});
- it('passes video when the Parser-level vision enhancement has a model', () => {
- expect(findParserGap(FileType.Video, SavedConfig)).toBeNull();
- });
-
- it('flags declared video when the vision enhancement is off or has no model', () => {
- const disabled: SavedParserConfig = {
- setups: [{ fileFormat: FileType.Video }],
- visionEnhancement: { enabled: false, modelId: 'glm-4v@ZHIPU-AI' },
- };
- const noModel: SavedParserConfig = {
- setups: [{ fileFormat: FileType.Video }],
- visionEnhancement: { enabled: true, modelId: '' },
- };
- const expected = {
- reason: ParserGapReason.MissingModel,
+ it('flags video as unsupported even when a vision model is configured', () => {
+ // Video parsing is not implemented yet: a declared family with a vision
+ // model still stays unsupported so upload warns and parse-click blocks.
+ expect(findParserGap(FileType.Video, SavedConfig)).toEqual({
+ reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
- modelKind: ParserModelKind.Vision,
- };
- expect(findParserGap(FileType.Video, disabled)).toEqual(expected);
- expect(findParserGap(FileType.Video, noModel)).toEqual(expected);
+ });
});
- it('reads the video model from the family setup for configs saved before the move', () => {
+ it('flags declared video as unsupported regardless of its setup', () => {
const withModel: SavedParserConfig = {
setups: [
{ fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
@@ -211,12 +222,12 @@ describe('findParserGap', () => {
const withoutModel: SavedParserConfig = {
setups: [{ fileFormat: FileType.Video, vlm: { llm_id: '' } }],
};
- expect(findParserGap(FileType.Video, withModel)).toBeNull();
- expect(findParserGap(FileType.Video, withoutModel)).toEqual({
- reason: ParserGapReason.MissingModel,
+ const expected = {
+ reason: ParserGapReason.UnsupportedType,
fileType: FileType.Video,
- modelKind: ParserModelKind.Vision,
- });
+ };
+ expect(findParserGap(FileType.Video, withModel)).toEqual(expected);
+ expect(findParserGap(FileType.Video, withoutModel)).toEqual(expected);
});
it('passes image/email with a general-like setup — no switch needed', () => {
@@ -259,6 +270,11 @@ describe('findFilesParserGaps', () => {
reason: ParserGapReason.UnsupportedType,
fileType: FileType.Docx,
},
+ {
+ name: 'movie.mp4',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Video,
+ },
{
name: 'deck.pptx',
reason: ParserGapReason.UnsupportedType,
@@ -295,10 +311,16 @@ describe('hasUnsupportedTypeGap', () => {
describe('builtinParserSwitchTarget', () => {
it('maps owned file types to their built-in pipelines', () => {
- // The visual bucket includes the video extensions, mirroring the backend.
+ // Images switch to picture; video uploads are accepted into the visual
+ // bucket and switched to picture as well, but parsing stays blocked
+ // because no parser implements video analysis.
expect(switchTarget('photo.JPG')).toBe('picture');
expect(switchTarget('movie.mp4')).toBe('picture');
expect(switchTarget('clip.mov')).toBe('picture');
+ expect(switchTarget('clip.webm')).toBe('picture');
+ expect(switchTarget('clip.rmvb')).toBe('picture');
+ expect(switchTarget('clip.asf')).toBe('picture');
+ expect(switchTarget('clip.mpa')).toBe('picture');
expect(switchTarget('song.mp3')).toBe('audio');
expect(switchTarget('song.opus')).toBe('audio');
expect(switchTarget('deck.ppt')).toBe('presentation');
@@ -408,6 +430,22 @@ describe('pickBySwitchFate', () => {
pickBySwitchFate([unsupportedGap(null), missingModelGap], Options),
).toBe('manual');
});
+
+ it('does not count video as switchable even though it targets picture', () => {
+ // Video uploads switch to picture but stay unparsable, so a batch of a
+ // video plus a non-switchable file must read as manual, not mixed; a
+ // video plus an auto-switchable file reads as mixed, not auto.
+ const videoGap: UploadFileParserGap = {
+ name: 'clip.mov',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Video,
+ autoSwitchTarget: 'picture',
+ };
+ expect(pickBySwitchFate([videoGap], Options)).toBe('manual');
+ expect(
+ pickBySwitchFate([videoGap, unsupportedGap('general')], Options),
+ ).toBe('mixed');
+ });
});
describe('getSavedParserConfig', () => {
@@ -443,7 +481,7 @@ describe('getSavedParserConfig', () => {
]);
});
- it('reads the vision enhancement globals from the Parser entry top level', () => {
+ it('reads only the saved Parser setups from the Parser entry', () => {
const knowledgeBase = {
parser_config: {
'Parser:1': {
@@ -455,11 +493,10 @@ describe('getSavedParserConfig', () => {
};
expect(getSavedParserConfig(knowledgeBase)).toEqual({
setups: [{ fileFormat: FileType.Video }],
- visionEnhancement: { enabled: true, modelId: 'glm-4v@ZHIPU-AI' },
});
});
- it('leaves the vision enhancement undefined for configs saved before the move', () => {
+ it('keeps family-local settings for configs saved before the vision move', () => {
const knowledgeBase = {
parser_config: {
'Parser:1': {
@@ -468,7 +505,6 @@ describe('getSavedParserConfig', () => {
},
};
const config = getSavedParserConfig(knowledgeBase);
- expect(config?.visionEnhancement).toBeUndefined();
expect(config?.setups).toEqual([
{ fileFormat: FileType.Video, vlm: { llm_id: 'gpt-4o@OpenAI' } },
]);
@@ -508,8 +544,18 @@ describe('findDocumentsParserGaps', () => {
modelKind: ParserModelKind.Asr,
},
]);
- // No fallback either: nothing can be determined, so nothing is flagged.
+ // No fallback either: nothing can be determined for other types, so
+ // nothing is flagged — except video, which is never parsable.
expect(findDocumentsParserGaps([legacyRow], null)).toEqual([]);
+ expect(
+ findDocumentsParserGaps([{ name: 'clip.mov', parser_config: {} }], null),
+ ).toEqual([
+ {
+ name: 'clip.mov',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Video,
+ },
+ ]);
});
it('evaluates each document against its own config in a mixed batch', () => {
@@ -544,6 +590,13 @@ describe('findDocumentsParserGaps', () => {
},
},
};
- expect(findDocumentsParserGaps([videoDoc], null)).toEqual([]);
+ // Video stays unsupported even with a vision model configured.
+ expect(findDocumentsParserGaps([videoDoc], null)).toEqual([
+ {
+ name: 'movie.mp4',
+ reason: ParserGapReason.UnsupportedType,
+ fileType: FileType.Video,
+ },
+ ]);
});
});
diff --git a/web/src/pages/dataset/dataset/utils.ts b/web/src/pages/dataset/dataset/utils.ts
index 4cae6cc71a..2778eb7d65 100644
--- a/web/src/pages/dataset/dataset/utils.ts
+++ b/web/src/pages/dataset/dataset/utils.ts
@@ -129,13 +129,13 @@ export const isDocumentProcessing = (
// A file cannot be parsed under the dataset's current Parser operator when
// the operator does not declare the file's type family at all
// (unsupportedType), or when the family is declared but the model it needs
-// is not set (missingModel): audio reads its per-setup ASR model, while
-// video reads the Parser-level vision enhancement (enable_vision_enhancement
-// + vlm.llm_id) — its model no longer lives on the family setup. The tenant
-// default is deliberately not consulted: a global default does not make the
-// file parsable here. Image files only need their family declared: the image
-// OCR switch runs local extraction independently, and the vision model is a
-// global supplement for it.
+// is not set (missingModel): audio reads its per-setup ASR model. Video is
+// temporarily blocked as unsupported until the video-analysis capability
+// lands (maybeDispatchVideo still returns "not yet supported"), so it never
+// reports missingModel even when its family is declared.
+// Image files only need their family declared: the image OCR switch runs
+// local extraction independently, and the vision model is a global supplement
+// for it.
// These helpers power the upload warning and the parse-click validation.
export type ParserModelGap = {
@@ -209,8 +209,11 @@ const GeneralFamilyFileTypes = new Set<FileType>([
FileType.Docx,
]);
-// Extension sets mirror utility.FilenameType (internal/utility/file.go); the
-// visual bucket intentionally includes the video extensions.
+// Extension sets mirror the backend upload buckets (utility.FilenameType):
+// the visual bucket admits image containers plus every video container in the
+// shared backend set, so video keeps the picture switch the backend performs.
+// Parsability is judged separately: pickBySwitchFate excludes video from the
+// switchable count because no parser implements video analysis.
const VisualSwitchSuffixes = new Set([
'jpg',
'jpeg',
@@ -237,21 +240,26 @@ const VisualSwitchSuffixes = new Set([
'apng',
'icon',
'ico',
- 'mpg',
- 'mpeg',
+ // Video containers (mirrors utility.videoExtensions).
+ 'mp4',
'avi',
- 'rm',
- 'rmvb',
+ 'mkv',
'mov',
+ 'webm',
+ 'flv',
+ 'mpeg',
+ 'mpg',
'wmv',
+ '3gp',
+ '3gpp',
+ 'rm',
+ 'rmvb',
'asf',
'dat',
'asx',
'wvx',
'mpe',
'mpa',
- 'mp4',
- 'mkv',
]);
const AuralSwitchSuffixes = new Set([
@@ -279,6 +287,12 @@ export function builtinParserSwitchTarget(
name: string,
fileType: FileType | undefined,
): BuiltinSwitchTarget | null {
+ // Video uploads are accepted (visual bucket) but unsupported for parsing (no
+ // parser implements video analysis). The picture switch below mirrors what
+ // the backend performs; the row is still unparsable, so the user is warned
+ // at upload and blocked at parse-click. pickBySwitchFate separately excludes
+ // video from the switchable count so mixed batches never promise an
+ // automatic switch that cannot make the file parsable.
const suffix = getExtension(name);
if (VisualSwitchSuffixes.has(suffix)) {
return 'picture';
@@ -327,7 +341,12 @@ export function pickBySwitchFate<T>(
if (unsupported.length === 0) {
return options.missingModel;
}
- const switchable = unsupported.filter((gap) => gap.autoSwitchTarget !== null);
+ // Video uploads switch to picture but stay unparsable, so they are not
+ // counted as switchable: counting them would promise an automatic switch
+ // that cannot make the file parsable.
+ const switchable = unsupported.filter(
+ (gap) => gap.autoSwitchTarget !== null && gap.fileType !== FileType.Video,
+ );
if (switchable.length === 0) {
return options.manualSwitch;
}
@@ -338,20 +357,8 @@ export function pickBySwitchFate<T>(
type ParserSetup = Record<string, any> & { fileFormat?: string };
-/**
- * The Parser-level vision enhancement globals (enable_vision_enhancement +
- * vlm.llm_id): one switch and model serve every media family. Undefined marks
- * a config saved before the model moved off the per-family setups — those
- * still carry the video model on the video setup itself.
- */
-export type ParserVisionEnhancement = {
- enabled: boolean;
- modelId: string;
-};
-
export type SavedParserConfig = {
setups: ParserSetup[];
- visionEnhancement?: ParserVisionEnhancement;
};
const ExtensionToFileTypeMap: Record<string, FileType> = Object.entries(
@@ -369,32 +376,12 @@ export function getFileTypeByExtension(
return ExtensionToFileTypeMap[extension.toLowerCase()];
}
-// Reads the Parser-level vision globals from a saved Parser entry. A boolean
-// enable_vision_enhancement marks the current contract; configs saved before
-// the model moved to the top level do not have it, and their video setup
-// still holds the model — so the absence is meaningful, not a default-off.
-function readVisionEnhancement(
- parserEntry: unknown,
-): ParserVisionEnhancement | undefined {
- if (typeof parserEntry !== 'object' || parserEntry === null) {
- return undefined;
- }
- const params = parserEntry as Record<string, any>;
- const enabled = params.enable_vision_enhancement;
- if (typeof enabled !== 'boolean') {
- return undefined;
- }
- const modelId = params.vlm?.llm_id;
- return { enabled, modelId: typeof modelId === 'string' ? modelId : '' };
-}
-
/**
- * Parser config exactly as saved: the per-family setups plus the Parser-level
- * vision enhancement globals. Accepts either a dataset or a document row: a
- * document's parser_config snapshots the parser it actually runs with
- * (dataset config at upload, the override after a parser change). Returns
- * null when parser_config has no Parser operator entry (legacy flat-shape
- * configs, pipelines without a Parser stage): type support cannot be
+ * Parser config exactly as saved: the per-family setups. Accepts either a
+ * dataset or a document row: a document's parser_config snapshots the parser it
+ * actually runs with (dataset config at upload, the override after a parser
+ * change). Returns null when parser_config has no Parser operator entry (legacy
+ * flat-shape configs, pipelines without a Parser stage): type support cannot be
* determined then and validation is skipped.
*/
export function getSavedParserConfig(
@@ -411,7 +398,6 @@ export function getSavedParserConfig(
}
return {
setups: transformParserConfigSetups(parserEntry),
- visionEnhancement: readVisionEnhancement(parserEntry),
};
}
@@ -441,16 +427,12 @@ export function findParserGap(
};
}
case FileType.Video: {
- const hasVisionModel = config.visionEnhancement
- ? config.visionEnhancement.enabled && !!config.visionEnhancement.modelId
- : !!setup.vlm?.llm_id;
- return hasVisionModel
- ? null
- : {
- reason: ParserGapReason.MissingModel,
- fileType,
- modelKind: ParserModelKind.Vision,
- };
+ // Video parsing is not implemented yet (the backend dispatch returns
+ // "video parsing is not yet supported"), so a declared family or a
+ // configured vision model does not make the file parsable. Report it
+ // as unsupported so upload warns and parse-click blocks instead of
+ // prompting for a model that cannot unblock it.
+ return { reason: ParserGapReason.UnsupportedType, fileType };
}
default:
return null;
@@ -480,6 +462,20 @@ export function findDocumentsParserGaps(
): FileParserGap[] {
return documents.flatMap((doc) => {
const config = getSavedParserConfig(doc) ?? fallbackConfig;
- return config ? findFilesParserGaps([doc.name], config) : [];
+ if (!config) {
+ // Video is never parsable regardless of configuration, so it stays
+ // blocked even when type support cannot be determined for other files.
+ const fileType = getFileTypeByExtension(getExtension(doc.name));
+ return fileType === FileType.Video
+ ? [
+ {
+ reason: ParserGapReason.UnsupportedType,
+ fileType,
+ name: doc.name,
+ },
+ ]
+ : [];
+ }
+ return findFilesParserGaps([doc.name], config);
});
}
diff --git a/web/src/utils/document-util.ts b/web/src/utils/document-util.ts
index fb294a012b..f7d3b4e722 100644
--- a/web/src/utils/document-util.ts
+++ b/web/src/utils/document-util.ts
@@ -60,7 +60,10 @@ export const buildChunkHighlights = (
export const isFileUploadDone = (file: UploadFile) => file.status === 'done';
export const getExtension = (name: string) =>
- name?.slice(name.lastIndexOf('.') + 1).toLowerCase() ?? '';
+ name
+ ?.slice(name.lastIndexOf('.') + 1)
+ .trim()
+ .toLowerCase() ?? '';
export const isPdf = (name: string) => {
return getExtension(name) === 'pdf';