Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(import): accept formats advertised by parser engines (#4602)
Jarvis26a37cb2d0f6
diff --git a/internal/application/service/knowledge_util_filetype_test.go b/internal/application/service/knowledge_util_filetype_test.goindex 1ef96c779..7a0084427 100644--- a/internal/application/service/knowledge_util_filetype_test.go+++ b/internal/application/service/knowledge_util_filetype_test.go@@ -2,6 +2,7 @@ package serviceimport ("slices"+ "strings""testing"secutils "github.com/Tencent/WeKnora/internal/utils"@@ -65,21 +66,27 @@ func TestIsSupportedImportExtension(t *testing.T) {// The set is pinned so moving it to internal/utils cannot silently change it.func TestImportExtensionSetIsSharedAcrossPaths(t *testing.T) {want := []string{- "csv", "doc", "docx", "epub", "flac", "gif", "htm", "html", "jpeg", "jpg",- "json", "m4a", "markdown", "md", "mhtml", "mp3", "ogg", "pdf", "png", "ppt",- "pptx", "txt", "wav", "xls", "xlsx", "xmind",+ "bmp", "csv", "doc", "docm", "docx", "epub", "flac", "gif", "htm", "html", "jpeg", "jpg",+ "json", "m4a", "markdown", "md", "mhtml", "mp3", "odp", "ods", "odt", "ogg", "pdf", "png", "ppt",+ "pptm", "pptx", "rtf", "tiff", "txt", "wav", "webp", "xls", "xlsm", "xlsx", "xmind",}exts := secutils.SupportedImportExtensions()if !slices.Equal(exts, want) {t.Fatalf("SupportedImportExtensions() = %v, want %v", exts, want)}for _, ext := range exts {- if !isValidFileType("file." + ext) {- t.Errorf("isValidFileType rejects supported extension %q", ext)- }- if err := validateImportFileType(ext); err != nil {- t.Errorf("validateImportFileType(%q) = %v, want nil", ext, err)- }+ t.Run(ext, func(t *testing.T) {+ for _, variant := range []string{ext, strings.ToUpper(ext)} {+ if !isValidFileType("file." + variant) {+ t.Errorf("isValidFileType rejects supported extension %q", variant)+ }+ for _, input := range []string{variant, " ." + variant + " "} {+ if err := validateImportFileType(input); err != nil {+ t.Errorf("validateImportFileType(%q) = %v, want nil", input, err)+ }+ }+ }+ })}}diff --git a/internal/infrastructure/docparser/engine_registry_test.go b/internal/infrastructure/docparser/engine_registry_test.goindex 4bd82424d..1b983ba49 100644--- a/internal/infrastructure/docparser/engine_registry_test.go+++ b/internal/infrastructure/docparser/engine_registry_test.go@@ -5,8 +5,24 @@ import ("github.com/Tencent/WeKnora/internal/infrastructure/docparser/anydoc""github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/utils")+// Locally advertised parser formats must pass the shared import gate used by+// direct uploads, URL imports, workers and datasource connectors (#4598).+// Check metadata even when a converter is not linked into this test build.+func TestLocalParserFormatsPassImportValidation(t *testing.T) {+ for _, engine := range ListAllEngines(true, nil, nil) {+ t.Run(engine.Name, func(t *testing.T) {+ for _, ext := range engine.FileTypes {+ if !utils.IsSupportedImportExtension(ext) {+ t.Errorf("engine advertises %q but the shared import gate rejects it", ext)+ }+ }+ })+ }+}+func TestListAllEnginesBuiltinIncludesDocumentFormats(t *testing.T) {engines := ListAllEngines(true, nil, nil)for _, engine := range engines {diff --git a/internal/utils/import_filetypes.go b/internal/utils/import_filetypes.goindex c196d66ac..a7ebbd4a2 100644--- a/internal/utils/import_filetypes.go+++ b/internal/utils/import_filetypes.go@@ -9,13 +9,18 @@ import (// accepted by every knowledge import path: direct upload, file-URL download,// the worker's post-download re-check and data source connectors. Keeping one// set avoids the drift that let direct upload accept xlsx while URL import-// rejected it (#2447).+// rejected it (#2447). Locally registered parser formats are checked against+// this set by docparser's contract test to prevent advertised-but-rejected+// formats (#4598).var supportedImportFileExtensions = map[string]struct{}{"pdf": {}, "txt": {}, "docx": {}, "doc": {}, "epub": {},+ "docm": {}, "odt": {}, "rtf": {},"html": {}, "htm": {}, "mhtml": {}, "md": {}, "markdown": {},"xmind": {},"png": {}, "jpg": {}, "jpeg": {}, "gif": {},+ "bmp": {}, "tiff": {}, "webp": {},"csv": {}, "xlsx": {}, "xls": {}, "pptx": {}, "ppt": {}, "json": {},+ "xlsm": {}, "ods": {}, "pptm": {}, "odp": {},"mp3": {}, "wav": {}, "m4a": {}, "flac": {}, "ogg": {},}diff --git a/internal/utils/import_filetypes_test.go b/internal/utils/import_filetypes_test.goindex e87654dfe..17df6f466 100644--- a/internal/utils/import_filetypes_test.go+++ b/internal/utils/import_filetypes_test.go@@ -31,6 +31,16 @@ func TestIsSupportedImportExtension(t *testing.T) {{"pdf", true},{" .XLSX ", true},{"XMIND", true},+ {"xlsm", true},+ {"docm", true},+ {"pptm", true},+ {"odt", true},+ {"ods", true},+ {"odp", true},+ {"rtf", true},+ {"bmp", true},+ {"tiff", true},+ {"webp", true},{"exe", false},{"mp4", false},{"unknown", false},