Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(wiki): restore rewritten wkimg placeholders instead of persisting broken links (#4552)
u7-u72bceacb6378f
diff --git a/internal/application/service/wiki_ingest.go b/internal/application/service/wiki_ingest.goindex 307ea529d..7b271124f 100644--- a/internal/application/service/wiki_ingest.go+++ b/internal/application/service/wiki_ingest.go@@ -3403,7 +3403,14 @@ var (// e.g. <image url="...">. Capture both double- and single-quoted forms.imageURLAttrRE = regexp.MustCompile(`(?i)<image\b[^>]*\surl\s*=\s*(?:"([^"]*)"|'([^']*)')`)- imagePlaceholderTokenRE = regexp.MustCompile(`wkimg:[A-Za-z0-9_-]+`)+ imagePlaceholderTokenRE = regexp.MustCompile(`(?i)wkimg\s*[::/_\-]?\s*[0-9]{1,6}`)++ // looseImagePlaceholderTokenRE matches a whole token that the synthesis+ // model may have rewritten with a different separator (wkimg/0010,+ // wkimg_0010, wkimg-0010, "wkimg 0010", full-width colon). The numeric+ // suffix still identifies the image, so these are normalised back to the+ // canonical wkimg:0010 form before lookup.+ looseImagePlaceholderTokenRE = regexp.MustCompile(`(?i)^wkimg\s*[::/_\-]?\s*([0-9]{1,6})$`))func maskTemplateDataImageURLs(data map[string]string) (map[string]string, map[string]string) {@@ -3509,7 +3516,18 @@ func unmaskImageURLs(out string, urlMap map[string]string) string {}return match[:idx+1] + realURL + ")"}- if strings.HasPrefix(url, "wkimg:") {+ if canonical, ok := canonicalImagePlaceholder(url); ok {+ realURL, known := urlMap[canonical]+ if !known {+ return ""+ }+ idx := strings.LastIndex(match, "(")+ if idx < 0 {+ return match+ }+ return match[:idx+1] + realURL + ")"+ }+ if strings.HasPrefix(strings.ToLower(url), "wkimg") {return ""}return match@@ -3518,6 +3536,18 @@ func unmaskImageURLs(out string, urlMap map[string]string) string {return replaceImagePlaceholderTokensOutsideMarkdown(out, urlMap)}+// canonicalImagePlaceholder normalises the separator variants a model may+// introduce and reports whether token is a whole image placeholder. Only the+// separator is normalised: the digits are preserved verbatim so an invented+// index such as wkimg:9999 does not accidentally resolve to a real image.+func canonicalImagePlaceholder(token string) (string, bool) {+ m := looseImagePlaceholderTokenRE.FindStringSubmatch(strings.TrimSpace(token))+ if m == nil {+ return "", false+ }+ return "wkimg:" + m[1], true+}+func replaceImagePlaceholderTokensOutsideMarkdown(s string, urlMap map[string]string) string {matches := mdImageURLRE.FindAllStringIndex(s, -1)if len(matches) == 0 {@@ -3541,7 +3571,11 @@ func replaceImagePlaceholderTokensOutsideMarkdown(s string, urlMap map[string]stfunc replaceImagePlaceholderTokens(s string, urlMap map[string]string) string {return imagePlaceholderTokenRE.ReplaceAllStringFunc(s, func(token string) string {- if realURL, ok := urlMap[token]; ok {+ canonical, ok := canonicalImagePlaceholder(token)+ if !ok {+ return ""+ }+ if realURL, ok := urlMap[canonical]; ok {return realURL}return ""diff --git a/internal/application/service/wiki_ingest_test.go b/internal/application/service/wiki_ingest_test.goindex e0815f53f..4e2b40704 100644--- a/internal/application/service/wiki_ingest_test.go+++ b/internal/application/service/wiki_ingest_test.go@@ -324,6 +324,43 @@ func TestUnmaskImageURLsDropsUnknownPlaceholders(t *testing.T) {}}+// The synthesis model sometimes rewrites wkimg:0010 with another separator+// (a slash in #4294). The digits still identify the image, so the token must+// be restored — or dropped when the index is unknown — instead of reaching+// storage as a broken image link.+func TestUnmaskImageURLsNormalisesPlaceholderSeparators(t *testing.T) {+ const page1 = "minio://kb/exports/page_1.jpg"+ const page10 = "minio://kb/exports/page_10.jpg"+ urlMap := map[string]string{+ "wkimg:0001": page1,+ "wkimg:0010": page10,+ }++ cases := []struct {+ name string+ input string+ want string+ }{+ {"slash", "", ""},+ {"underscore", "", ""},+ {"dash", "", ""},+ {"space", "", ""},+ {"full width colon", "", ""},+ {"uppercase", "", ""},+ {"outside markdown", "see wkimg/0010 here", "see " + page10 + " here"},+ {"unknown index", "", ""},+ {"non placeholder url", "", ""},+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ if got := unmaskImageURLs(tc.input, urlMap); got != tc.want {+ t.Fatalf("unmaskImageURLs(%q) = %q, want %q", tc.input, got, tc.want)+ }+ })+ }+}+func TestGenerateWithTemplateMasksImageURLsBeforeLLM(t *testing.T) {const realURL = "minio://kb/10000/exports/4135-aaaa-bbbb-cccc/page_1.jpg"model := &templateCaptureChatModel{