Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docparser): escape standalone image filenames in Markdown (#4475)
Jarvisfd7a2586fd58
diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.goindex 0a676f0cb..46939fc60 100644--- a/internal/infrastructure/docparser/builtin_converter.go+++ b/internal/infrastructure/docparser/builtin_converter.go@@ -7,6 +7,7 @@ import ("fmt""io""net/http"+ "net/url""path/filepath""strings"@@ -87,17 +88,15 @@ func imageToResult(fileName string, data []byte) *types.ReadResult {if fileName == "" {fileName = "image.png"}- refPath := "images/" + fileName- // Encode spaces so the markdown URL is valid and matches the regex in ResolveAndStore.- safeRef := strings.ReplaceAll(refPath, " ", "%20")+ markdown, refPath := originalImageMarkdown(fileName)mime := http.DetectContentType(data)return &types.ReadResult{- MarkdownContent: fmt.Sprintf("", fileName, safeRef),+ MarkdownContent: markdown,ImageRefs: []types.ImageRef{{Filename: fileName,- OriginalRef: safeRef,+ OriginalRef: refPath,MimeType: mime,ImageData: data,IsOriginal: true,@@ -106,6 +105,18 @@ func imageToResult(fileName string, data []byte) *types.ReadResult {}}+// originalImageMarkdown keeps a filename from changing Markdown syntax. The+// destination is also the exact lookup key used by ImageResolver.+func originalImageMarkdown(fileName string) (markdown, refPath string) {+ refPath = "images/" + url.PathEscape(fileName)+ alt := markdownImageAltReplacer.Replace(fileName)+ return fmt.Sprintf("", alt, refPath), refPath+}++var markdownImageAltReplacer = strings.NewReplacer(+ `\`, `\\`, "[", `\[`, "]", `\]`, "\r\n", " ", "\r", " ", "\n", " ",+)+// IsImageFormat returns true if the file type is a recognized image format.func IsImageFormat(fileType string) bool {return imageFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]@@ -154,13 +165,12 @@ func ensureOriginalImageRef(req *types.ReadRequest, mdContent string, imageRefsif fileName == "" {fileName = "image." + ft}- refPath := "images/" + fileName+ imgLine, refPath := originalImageMarkdown(fileName)- if strings.Contains(mdContent, refPath) {+ if strings.Contains(mdContent, "images/"+fileName) || strings.Contains(mdContent, refPath) {return mdContent, imageRefs}- imgLine := fmt.Sprintf("", fileName, refPath)if strings.TrimSpace(mdContent) == "" {mdContent = imgLine} else {diff --git a/internal/infrastructure/docparser/builtin_image_filename_test.go b/internal/infrastructure/docparser/builtin_image_filename_test.gonew file mode 100644index 000000000..ba43b7460--- /dev/null+++ b/internal/infrastructure/docparser/builtin_image_filename_test.go@@ -0,0 +1,59 @@+package docparser++import (+ "context"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++func TestStandaloneImageFilenamesResolveAndStore(t *testing.T) {+ filenames := []string{+ "diagram.png", "diagram).png", "diagram(.png", "diagram[1].png",+ "diagram].png", "diagram](extra).png", `diagram\.png`, "diagram .png",+ "100%20complete.png", "diagram#1?.png", "第 1 张图片(终稿).png",+ "line\nbreak.png", "line\r\nbreak.png", `quote".png`,+ }+ for _, filename := range filenames {+ for _, fallback := range []bool{false, true} {+ name := "simple/" + filename+ if fallback {+ name = "fallback/" + filename+ }+ t.Run(name, func(t *testing.T) {+ data := createTestPNG(64, 64)+ req := &types.ReadRequest{FileName: filename, FileType: "png", FileContent: data}+ result, err := (&SimpleFormatReader{}).Read(context.Background(), req)+ require.NoError(t, err)+ if fallback {+ result.MarkdownContent, result.ImageRefs = ensureOriginalImageRef(req, "OCR text", nil)+ }+ require.Len(t, result.ImageRefs, 1)+ require.Equal(t, filename, result.ImageRefs[0].Filename)+ svc := &captureSaveBytes{}+ markdown, images, err := NewImageResolver().ResolveAndStore(context.Background(), result, svc, 1)+ require.NoError(t, err)+ require.Len(t, svc.saved, 1, "image must reach storage: %s", result.MarkdownContent)+ require.Equal(t, data, svc.saved[0])+ require.Len(t, images, 1)+ require.Contains(t, markdown, "]("+svc.urls[0]+")")+ require.NotContains(t, markdown, result.ImageRefs[0].OriginalRef)+ if fallback {+ require.Contains(t, markdown, "OCR text")+ }++ // Running the fallback on its own generated reference must not add a second image.+ again, refs := ensureOriginalImageRef(req, result.MarkdownContent, result.ImageRefs)+ require.Equal(t, result.MarkdownContent, again)+ require.Len(t, refs, 1)+ })+ }+ }+}++func TestStandaloneImageMarkdownEscapesSyntax(t *testing.T) {+ result := imageToResult(`a[b]\c).png`, []byte("image"))+ require.Equal(t, `![a\[b\]\\c).png](images/a%5Bb%5D%5Cc%29.png)`, result.MarkdownContent)+ require.Equal(t, "images/a%5Bb%5D%5Cc%29.png", result.ImageRefs[0].OriginalRef)+}