Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docparser): escape standalone image filenames in Markdown (#4475)

Jarvisfd7a2586fd58
diff --git a/internal/infrastructure/docparser/builtin_converter.go b/internal/infrastructure/docparser/builtin_converter.go
index 0a676f0cb..46939fc60 100644
--- a/internal/infrastructure/docparser/builtin_converter.go
+++ b/internal/infrastructure/docparser/builtin_converter.go
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"net/http"
+ "net/url"
"path/filepath"
"strings"
@@ -87,17 +88,15 @@ func imageToResult(fileName string, data []byte) *types.ReadResult {
if fileName == "" {
fileName = "image.png"
}
- refPath := "images/" + fileName
- // Encode spaces so the markdown URL is valid and matches the regex in ResolveAndStore.
- safeRef := strings.ReplaceAll(refPath, " ", "%20")
+ markdown, refPath := originalImageMarkdown(fileName)
mime := http.DetectContentType(data)
return &types.ReadResult{
- MarkdownContent: fmt.Sprintf("![%s](%s)", fileName, safeRef),
+ MarkdownContent: markdown,
ImageRefs: []types.ImageRef{
{
Filename: fileName,
- OriginalRef: safeRef,
+ OriginalRef: refPath,
MimeType: mime,
ImageData: data,
IsOriginal: true,
@@ -106,6 +105,18 @@ func imageToResult(fileName string, data []byte) *types.ReadResult {
}
}
+// originalImageMarkdown keeps a filename from changing Markdown syntax. The
+// destination is also the exact lookup key used by ImageResolver.
+func originalImageMarkdown(fileName string) (markdown, refPath string) {
+ refPath = "images/" + url.PathEscape(fileName)
+ alt := markdownImageAltReplacer.Replace(fileName)
+ return fmt.Sprintf("![%s](%s)", alt, refPath), refPath
+}
+
+var markdownImageAltReplacer = strings.NewReplacer(
+ `\`, `\\`, "[", `\[`, "]", `\]`, "\r\n", " ", "\r", " ", "\n", " ",
+)
+
// IsImageFormat returns true if the file type is a recognized image format.
func IsImageFormat(fileType string) bool {
return imageFormats[strings.ToLower(strings.TrimPrefix(fileType, "."))]
@@ -154,13 +165,12 @@ func ensureOriginalImageRef(req *types.ReadRequest, mdContent string, imageRefs
if fileName == "" {
fileName = "image." + ft
}
- refPath := "images/" + fileName
+ imgLine, refPath := originalImageMarkdown(fileName)
- if strings.Contains(mdContent, refPath) {
+ if strings.Contains(mdContent, "images/"+fileName) || strings.Contains(mdContent, refPath) {
return mdContent, imageRefs
}
- imgLine := fmt.Sprintf("![%s](%s)", fileName, refPath)
if strings.TrimSpace(mdContent) == "" {
mdContent = imgLine
} else {
diff --git a/internal/infrastructure/docparser/builtin_image_filename_test.go b/internal/infrastructure/docparser/builtin_image_filename_test.go
new file mode 100644
index 000000000..ba43b7460
--- /dev/null
+++ b/internal/infrastructure/docparser/builtin_image_filename_test.go
@@ -0,0 +1,59 @@
+package docparser
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+func TestStandaloneImageFilenamesResolveAndStore(t *testing.T) {
+ filenames := []string{
+ "diagram.png", "diagram).png", "diagram(.png", "diagram[1].png",
+ "diagram].png", "diagram](extra).png", `diagram\.png`, "diagram .png",
+ "100%20complete.png", "diagram#1?.png", "第 1 张图片(终稿).png",
+ "line\nbreak.png", "line\r\nbreak.png", `quote".png`,
+ }
+ for _, filename := range filenames {
+ for _, fallback := range []bool{false, true} {
+ name := "simple/" + filename
+ if fallback {
+ name = "fallback/" + filename
+ }
+ t.Run(name, func(t *testing.T) {
+ data := createTestPNG(64, 64)
+ req := &types.ReadRequest{FileName: filename, FileType: "png", FileContent: data}
+ result, err := (&SimpleFormatReader{}).Read(context.Background(), req)
+ require.NoError(t, err)
+ if fallback {
+ result.MarkdownContent, result.ImageRefs = ensureOriginalImageRef(req, "OCR text", nil)
+ }
+ require.Len(t, result.ImageRefs, 1)
+ require.Equal(t, filename, result.ImageRefs[0].Filename)
+ svc := &captureSaveBytes{}
+ markdown, images, err := NewImageResolver().ResolveAndStore(context.Background(), result, svc, 1)
+ require.NoError(t, err)
+ require.Len(t, svc.saved, 1, "image must reach storage: %s", result.MarkdownContent)
+ require.Equal(t, data, svc.saved[0])
+ require.Len(t, images, 1)
+ require.Contains(t, markdown, "]("+svc.urls[0]+")")
+ require.NotContains(t, markdown, result.ImageRefs[0].OriginalRef)
+ if fallback {
+ require.Contains(t, markdown, "OCR text")
+ }
+
+ // Running the fallback on its own generated reference must not add a second image.
+ again, refs := ensureOriginalImageRef(req, result.MarkdownContent, result.ImageRefs)
+ require.Equal(t, result.MarkdownContent, again)
+ require.Len(t, refs, 1)
+ })
+ }
+ }
+}
+
+func TestStandaloneImageMarkdownEscapesSyntax(t *testing.T) {
+ result := imageToResult(`a[b]\c).png`, []byte("image"))
+ require.Equal(t, `![a\[b\]\\c).png](images/a%5Bb%5D%5Cc%29.png)`, result.MarkdownContent)
+ require.Equal(t, "images/a%5Bb%5D%5Cc%29.png", result.ImageRefs[0].OriginalRef)
+}