Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(import): accept formats advertised by parser engines (#4602)

Jarvis26a37cb2d0f6
diff --git a/internal/application/service/knowledge_util_filetype_test.go b/internal/application/service/knowledge_util_filetype_test.go
index 1ef96c779..7a0084427 100644
--- a/internal/application/service/knowledge_util_filetype_test.go
+++ b/internal/application/service/knowledge_util_filetype_test.go
@@ -2,6 +2,7 @@ package service
import (
"slices"
+ "strings"
"testing"
secutils "github.com/Tencent/WeKnora/internal/utils"
@@ -65,21 +66,27 @@ func TestIsSupportedImportExtension(t *testing.T) {
// The set is pinned so moving it to internal/utils cannot silently change it.
func TestImportExtensionSetIsSharedAcrossPaths(t *testing.T) {
want := []string{
- "csv", "doc", "docx", "epub", "flac", "gif", "htm", "html", "jpeg", "jpg",
- "json", "m4a", "markdown", "md", "mhtml", "mp3", "ogg", "pdf", "png", "ppt",
- "pptx", "txt", "wav", "xls", "xlsx", "xmind",
+ "bmp", "csv", "doc", "docm", "docx", "epub", "flac", "gif", "htm", "html", "jpeg", "jpg",
+ "json", "m4a", "markdown", "md", "mhtml", "mp3", "odp", "ods", "odt", "ogg", "pdf", "png", "ppt",
+ "pptm", "pptx", "rtf", "tiff", "txt", "wav", "webp", "xls", "xlsm", "xlsx", "xmind",
}
exts := secutils.SupportedImportExtensions()
if !slices.Equal(exts, want) {
t.Fatalf("SupportedImportExtensions() = %v, want %v", exts, want)
}
for _, ext := range exts {
- if !isValidFileType("file." + ext) {
- t.Errorf("isValidFileType rejects supported extension %q", ext)
- }
- if err := validateImportFileType(ext); err != nil {
- t.Errorf("validateImportFileType(%q) = %v, want nil", ext, err)
- }
+ t.Run(ext, func(t *testing.T) {
+ for _, variant := range []string{ext, strings.ToUpper(ext)} {
+ if !isValidFileType("file." + variant) {
+ t.Errorf("isValidFileType rejects supported extension %q", variant)
+ }
+ for _, input := range []string{variant, " ." + variant + " "} {
+ if err := validateImportFileType(input); err != nil {
+ t.Errorf("validateImportFileType(%q) = %v, want nil", input, err)
+ }
+ }
+ }
+ })
}
}
diff --git a/internal/infrastructure/docparser/engine_registry_test.go b/internal/infrastructure/docparser/engine_registry_test.go
index 4bd82424d..1b983ba49 100644
--- a/internal/infrastructure/docparser/engine_registry_test.go
+++ b/internal/infrastructure/docparser/engine_registry_test.go
@@ -5,8 +5,24 @@ import (
"github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc"
"github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/utils"
)
+// Locally advertised parser formats must pass the shared import gate used by
+// direct uploads, URL imports, workers and datasource connectors (#4598).
+// Check metadata even when a converter is not linked into this test build.
+func TestLocalParserFormatsPassImportValidation(t *testing.T) {
+ for _, engine := range ListAllEngines(true, nil, nil) {
+ t.Run(engine.Name, func(t *testing.T) {
+ for _, ext := range engine.FileTypes {
+ if !utils.IsSupportedImportExtension(ext) {
+ t.Errorf("engine advertises %q but the shared import gate rejects it", ext)
+ }
+ }
+ })
+ }
+}
+
func TestListAllEnginesBuiltinIncludesDocumentFormats(t *testing.T) {
engines := ListAllEngines(true, nil, nil)
for _, engine := range engines {
diff --git a/internal/utils/import_filetypes.go b/internal/utils/import_filetypes.go
index c196d66ac..a7ebbd4a2 100644
--- a/internal/utils/import_filetypes.go
+++ b/internal/utils/import_filetypes.go
@@ -9,13 +9,18 @@ import (
// accepted by every knowledge import path: direct upload, file-URL download,
// the worker's post-download re-check and data source connectors. Keeping one
// set avoids the drift that let direct upload accept xlsx while URL import
-// rejected it (#2447).
+// rejected it (#2447). Locally registered parser formats are checked against
+// this set by docparser's contract test to prevent advertised-but-rejected
+// formats (#4598).
var supportedImportFileExtensions = map[string]struct{}{
"pdf": {}, "txt": {}, "docx": {}, "doc": {}, "epub": {},
+ "docm": {}, "odt": {}, "rtf": {},
"html": {}, "htm": {}, "mhtml": {}, "md": {}, "markdown": {},
"xmind": {},
"png": {}, "jpg": {}, "jpeg": {}, "gif": {},
+ "bmp": {}, "tiff": {}, "webp": {},
"csv": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {}, "json": {},
+ "xlsm": {}, "ods": {}, "pptm": {}, "odp": {},
"mp3": {}, "wav": {}, "m4a": {}, "flac": {}, "ogg": {},
}
diff --git a/internal/utils/import_filetypes_test.go b/internal/utils/import_filetypes_test.go
index e87654dfe..17df6f466 100644
--- a/internal/utils/import_filetypes_test.go
+++ b/internal/utils/import_filetypes_test.go
@@ -31,6 +31,16 @@ func TestIsSupportedImportExtension(t *testing.T) {
{"pdf", true},
{" .XLSX ", true},
{"XMIND", true},
+ {"xlsm", true},
+ {"docm", true},
+ {"pptm", true},
+ {"odt", true},
+ {"ods", true},
+ {"odp", true},
+ {"rtf", true},
+ {"bmp", true},
+ {"tiff", true},
+ {"webp", true},
{"exe", false},
{"mp4", false},
{"unknown", false},