Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(parser): keep PDFs on builtin instead of defaulting to anydoc
Lvcy_cdcc82e75cb5ca
diff --git a/internal/infrastructure/docparser/engine_registry_test.go b/internal/infrastructure/docparser/engine_registry_test.goindex 26ba84e75..4bd82424d 100644--- a/internal/infrastructure/docparser/engine_registry_test.go+++ b/internal/infrastructure/docparser/engine_registry_test.go@@ -33,7 +33,7 @@ func TestListAllEnginesBuiltinIncludesDocumentFormats(t *testing.T) {}func TestDefaultParserEnginePrefersAnydocWhenLinked(t *testing.T) {- cases := []string{"pptx", "ppt", "pdf", "docx"}+ cases := []string{"pptx", "ppt", "docx"}if anydoc.Available() {for _, ft := range cases {if got := types.DefaultParserEngine(ft); got != AnydocEngineName {@@ -43,15 +43,33 @@ func TestDefaultParserEnginePrefersAnydocWhenLinked(t *testing.T) {if got := types.DefaultParserEngine("csv"); got != "" {t.Errorf("DefaultParserEngine(csv) = %q, want empty so the Go simple reader stays default", got)}+ // anydoc claims pdf but cannot model one, so linking the binding must+ // not move PDFs off builtin. This branch is where that is observable.+ for _, ft := range []string{"pdf", ".PDF"} {+ if got := types.DefaultParserEngine(ft); got != "" {+ t.Errorf("DefaultParserEngine(%q) = %q, want empty so PDFs stay on builtin", ft, got)+ }+ }return}if got := types.DefaultParserEngine("pptx"); got != "markitdown" {t.Fatalf("DefaultParserEngine(pptx) = %q, want markitdown when anydoc is unavailable", got)}- if got := types.DefaultParserEngine("pdf"); got != "" {- t.Fatalf("DefaultParserEngine(pdf) = %q, want empty when anydoc is unavailable", got)- }if got := types.DefaultParserEngine("docx"); got != "" {t.Fatalf("DefaultParserEngine(docx) = %q, want empty when anydoc is unavailable", got)}}++// The PDF exclusion only matters because anydoc still claims the type. If a+// future anydoc drops pdf, the exclusion becomes dead code and should go with+// it — this guard runs in every build, tagged or not.+func TestPreferAnydocSkipsPDF(t *testing.T) {+ if !anydoc.Supports("pdf", "") {+ t.Fatal("anydoc no longer claims pdf; drop the explicit exclusion in preferAnydocWhenAvailable")+ }+ for _, ft := range []string{"pdf", ".PDF", " Pdf "} {+ if got := preferAnydocWhenAvailable(ft); got != "" {+ t.Errorf("preferAnydocWhenAvailable(%q) = %q, want empty so PDFs stay on builtin", ft, got)+ }+ }+}diff --git a/internal/infrastructure/docparser/engines.go b/internal/infrastructure/docparser/engines.goindex 0dafa648d..7883094f4 100644--- a/internal/infrastructure/docparser/engines.go+++ b/internal/infrastructure/docparser/engines.go@@ -44,11 +44,20 @@ func init() {// preferAnydocWhenAvailable is the type-level default override: when the// anydoc binding is linked and converts this file type, use it instead of// builtin or the markitdown fallback. Simple formats stay on the Go reader.+//+// PDF is excluded. anydoc has no document model for it, so Convert force-+// disables asset extraction, and AnydocReader only falls back to builtin+// when the whole file yields no text at all. A born-digital PDF that yields+// a few characters is therefore reported as parsed while its figures,+// tables and layout are gone. The builtin parser classifies pages+// individually and routes scanned ones through OCR/VLM, so it is the better+// default for the one type anydoc cannot model.func preferAnydocWhenAvailable(fileType string) string {- if IsSimpleFormat(fileType) {+ ft := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(fileType), "."))+ if IsSimpleFormat(ft) || ft == "pdf" {return ""}- if anydoc.Available() && anydoc.Supports(fileType, "") {+ if anydoc.Available() && anydoc.Supports(ft, "") {return AnydocEngineName}return ""