Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(chat): keep OCR/caption text of scanned-page image hits in merge (#3064)

LuckVdb037e2626071
diff --git a/internal/application/service/chat_pipeline/merge.go b/internal/application/service/chat_pipeline/merge.go
index d33893a27..b16e79935 100644
--- a/internal/application/service/chat_pipeline/merge.go
+++ b/internal/application/service/chat_pipeline/merge.go
@@ -349,6 +349,13 @@ func (p *PluginMerge) resolveParentChunks(
continue
}
hitImageInfo := r.ImageInfo
+ // The hit chunk itself carries the recognized text (Content is
+ // the OCR text / caption stored by the multimodal pipeline).
+ // Save it before the parent expansion overwrites the field:
+ // force-scanned PDFs cut the parent markdown from image
+ // placeholders only, so dropping the child body used to empty
+ // the context sent to the model (#3052).
+ childRecognizedContent := r.Content
contentSource := textParent
if textParent.ParentChunkID != "" {
if grandparent, found := parentMap[textParent.ParentChunkID]; found &&
@@ -366,6 +373,14 @@ func (p *PluginMerge) resolveParentChunks(
textContent := searchutil.PruneMarkdownImagesByImageInfo(textParent.Content, r.ImageInfo)
parentContent := searchutil.PruneMarkdownImagesByImageInfo(contentSource.Content, r.ImageInfo)
r.Content = searchutil.JoinChunkContent(parentContent, textContent, "\n\n")
+ // Re-attach the recognized text after the parent/grandparent
+ // markdown. JoinChunkContent collapses duplicates, so when the
+ // surrounding text already covers the OCR/caption body nothing
+ // is added.
+ r.Content = searchutil.JoinChunkContent(r.Content, childRecognizedContent, "\n\n")
+ r.ImageInfo = searchutil.ClearImageInfoTextMatchingBody(
+ r.ImageInfo, childRecognizedContent, r.ChunkType,
+ )
r.ContentRewritten = true
pipelineInfo(ctx, "Merge", "image_parent_resolve", map[string]interface{}{
"child_id": r.ID,
diff --git a/internal/application/service/chat_pipeline/merge_parent_resolve_test.go b/internal/application/service/chat_pipeline/merge_parent_resolve_test.go
index 5ae6688d7..a6d447a1e 100644
--- a/internal/application/service/chat_pipeline/merge_parent_resolve_test.go
+++ b/internal/application/service/chat_pipeline/merge_parent_resolve_test.go
@@ -159,3 +159,205 @@ func TestParentChildImageHit_WindowSliceAndFilter(t *testing.T) {
t.Fatalf("infos: %+v", infos)
}
}
+
+// TestResolveImageOCRHit_KeepsRecognizedTextWhenParentIsPlaceholderOnly:
+// force-scanned PDFs produce parent/text chunks cut from image placeholders
+// alone, while the whole recognized page text lives in the image_ocr child.
+// The parent expansion must not discard that recognized text (#3052).
+func TestResolveImageOCRHit_KeepsRecognizedTextWhenParentIsPlaceholderOnly(t *testing.T) {
+ ocrText := "客户编码 A01,客户名称 示例公司,联系人 张三,电话 13800000000。"
+ imageInfo, err := json.Marshal([]types.ImageInfo{{URL: "images/scan_page_1.jpg", OCRText: ocrText}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ repo := &expandChunkRepo{
+ chunks: map[string]*types.Chunk{
+ "text": {
+ ID: "text", ParentChunkID: "parent", ChunkType: types.ChunkTypeText, ChunkIndex: 3,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ "parent": {
+ ID: "parent", ChunkType: types.ChunkTypeParentText,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ },
+ }
+ plugin := &PluginMerge{chunkRepo: repo}
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
+ result := &types.SearchResult{
+ ID: "ocr-1", KnowledgeID: "doc", ChunkType: string(types.ChunkTypeImageOCR),
+ ParentChunkID: "text", Content: ocrText, ImageInfo: string(imageInfo),
+ }
+
+ got := plugin.resolveParentChunks(ctx, &types.ChatManage{}, []*types.SearchResult{result})
+ if len(got) != 1 {
+ t.Fatalf("result count = %d, want 1", len(got))
+ }
+ if !strings.Contains(got[0].Content, "示例公司") || !strings.Contains(got[0].Content, "13800000000") {
+ t.Fatalf("recognized OCR text was dropped by parent expansion: %q", got[0].Content)
+ }
+}
+
+// TestResolveImageOCRHit_DoesNotDuplicateRecognizedText: when the surrounding
+// text already contains the recognized body (normal multimodal mode), the
+// re-attachment must not duplicate it.
+func TestResolveImageOCRHit_DoesNotDuplicateRecognizedText(t *testing.T) {
+ ocrText := "matched image body text"
+ imageInfo, err := json.Marshal([]types.ImageInfo{{URL: "u1", OCRText: ocrText}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ repo := &expandChunkRepo{
+ chunks: map[string]*types.Chunk{
+ "text": {
+ ID: "text", ParentChunkID: "parent", ChunkType: types.ChunkTypeText, ChunkIndex: 2,
+ Content: "intro\n\nmatched image body text\n\noutro\n\n![matched](u1)",
+ },
+ "parent": {
+ ID: "parent", ChunkType: types.ChunkTypeParentText,
+ Content: "long grandparent context\n\nintro\n\n" +
+ "matched image body text\n\noutro\n\n![matched](u1)\n\nmore context",
+ },
+ },
+ }
+ plugin := &PluginMerge{chunkRepo: repo}
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
+ result := &types.SearchResult{
+ ID: "ocr-1", KnowledgeID: "doc", ChunkType: string(types.ChunkTypeImageOCR),
+ ParentChunkID: "text", Content: ocrText, ImageInfo: string(imageInfo),
+ }
+
+ got := plugin.resolveParentChunks(ctx, &types.ChatManage{}, []*types.SearchResult{result})
+ if len(got) != 1 {
+ t.Fatalf("result count = %d, want 1", len(got))
+ }
+ if got[0].Content == "" {
+ t.Fatal("merged content is empty")
+ }
+ if strings.Count(got[0].Content, ocrText) != 1 {
+ t.Fatalf("recognized text duplicated in merged content: %q", got[0].Content)
+ }
+}
+
+func TestResolveImageOCRHit_ChatEnrichmentDoesNotDuplicateOCR(t *testing.T) {
+ ocrText := "客户编码 A01,客户名称 示例公司,联系人 张三,电话 13800000000。"
+ caption := "扫描件第一页客户信息表"
+ imageInfo, err := json.Marshal([]types.ImageInfo{{
+ URL: "images/scan_page_1.jpg", OCRText: ocrText, Caption: caption,
+ }})
+ if err != nil {
+ t.Fatal(err)
+ }
+ repo := &expandChunkRepo{
+ chunks: map[string]*types.Chunk{
+ "text": {
+ ID: "text", ParentChunkID: "parent", ChunkType: types.ChunkTypeText, ChunkIndex: 3,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ "parent": {
+ ID: "parent", ChunkType: types.ChunkTypeParentText,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ },
+ children: map[string][]*types.Chunk{
+ "text": {{
+ ID: "ocr-1", ParentChunkID: "text", ChunkType: types.ChunkTypeImageOCR,
+ ImageInfo: string(imageInfo), IsEnabled: true,
+ }},
+ },
+ }
+ plugin := &PluginMerge{chunkRepo: repo}
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
+ result := &types.SearchResult{
+ ID: "ocr-1", KnowledgeID: "doc", ChunkType: string(types.ChunkTypeImageOCR),
+ ParentChunkID: "text", Content: ocrText, ImageInfo: string(imageInfo),
+ }
+
+ got := plugin.resolveParentChunks(ctx, &types.ChatManage{}, []*types.SearchResult{result})
+ if len(got) != 1 {
+ t.Fatalf("result count = %d, want 1", len(got))
+ }
+ passage := getEnrichedPassageForChat(ctx, got[0])
+ if strings.Count(passage, ocrText) != 1 {
+ t.Fatalf("OCR duplicated after chat enrichment: %q", passage)
+ }
+ if !strings.Contains(passage, caption) {
+ t.Fatalf("caption was dropped from chat enrichment: %q", passage)
+ }
+}
+
+func TestResolveImageCaptionHit_KeepsCaptionWhenParentIsPlaceholderOnly(t *testing.T) {
+ caption := "扫描合同首页,含甲乙双方签章位置"
+ imageInfo, err := json.Marshal([]types.ImageInfo{{URL: "images/scan_page_1.jpg", Caption: caption}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ repo := &expandChunkRepo{
+ chunks: map[string]*types.Chunk{
+ "text": {
+ ID: "text", ParentChunkID: "parent", ChunkType: types.ChunkTypeText, ChunkIndex: 1,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ "parent": {
+ ID: "parent", ChunkType: types.ChunkTypeParentText,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ },
+ }
+ plugin := &PluginMerge{chunkRepo: repo}
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
+ result := &types.SearchResult{
+ ID: "cap-1", KnowledgeID: "doc", ChunkType: string(types.ChunkTypeImageCaption),
+ ParentChunkID: "text", Content: caption, ImageInfo: string(imageInfo),
+ }
+
+ got := plugin.resolveParentChunks(ctx, &types.ChatManage{}, []*types.SearchResult{result})
+ if len(got) != 1 {
+ t.Fatalf("result count = %d, want 1", len(got))
+ }
+ if !strings.Contains(got[0].Content, "甲乙双方") {
+ t.Fatalf("caption was dropped by parent expansion: %q", got[0].Content)
+ }
+ passage := getEnrichedPassageForChat(ctx, got[0])
+ if strings.Count(passage, caption) != 1 {
+ t.Fatalf("caption duplicated after chat enrichment: %q", passage)
+ }
+}
+
+func TestResolveImageOCRHit_KeepsTextWhenPlaceholderIsPruned(t *testing.T) {
+ ocrText := "仅存在于 OCR 子块中的扫描页正文,用于核对账号 6222。"
+ imageInfo, err := json.Marshal([]types.ImageInfo{{URL: "resource://unmatched-scan", OCRText: ocrText}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ repo := &expandChunkRepo{
+ chunks: map[string]*types.Chunk{
+ "text": {
+ ID: "text", ParentChunkID: "parent", ChunkType: types.ChunkTypeText, ChunkIndex: 4,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ "parent": {
+ ID: "parent", ChunkType: types.ChunkTypeParentText,
+ Content: "![scan_page_1.jpg](images/scan_page_1.jpg)",
+ },
+ },
+ }
+ plugin := &PluginMerge{chunkRepo: repo}
+ ctx := context.WithValue(context.Background(), types.TenantIDContextKey, uint64(1))
+ result := &types.SearchResult{
+ ID: "ocr-1", KnowledgeID: "doc", ChunkType: string(types.ChunkTypeImageOCR),
+ ParentChunkID: "text", Content: ocrText, ImageInfo: string(imageInfo),
+ }
+
+ got := plugin.resolveParentChunks(ctx, &types.ChatManage{}, []*types.SearchResult{result})
+ if len(got) != 1 {
+ t.Fatalf("result count = %d, want 1", len(got))
+ }
+ if !strings.Contains(got[0].Content, "6222") {
+ t.Fatalf("OCR text was dropped after placeholder prune: %q", got[0].Content)
+ }
+ passage := getEnrichedPassageForChat(ctx, got[0])
+ if strings.Count(passage, ocrText) != 1 {
+ t.Fatalf("OCR count after chat enrichment = %d: %q", strings.Count(passage, ocrText), passage)
+ }
+}
diff --git a/internal/searchutil/imageinfo.go b/internal/searchutil/imageinfo.go
index 487563aea..3163b44e7 100644
--- a/internal/searchutil/imageinfo.go
+++ b/internal/searchutil/imageinfo.go
@@ -231,6 +231,39 @@ func MergeImageInfoJSON(perChunk map[string]string) string {
return string(data)
}
+// ClearImageInfoTextMatchingBody removes the OCR or caption field that exactly
+// matches recognized from image_info. Merge re-attaches that body onto Content
+// for image_ocr / image_caption hits; chat enrichment would otherwise inject
+// the same text again from ImageInfo.
+func ClearImageInfoTextMatchingBody(imageInfoJSON, recognized, chunkType string) string {
+ if imageInfoJSON == "" || recognized == "" {
+ return imageInfoJSON
+ }
+ var infos []types.ImageInfo
+ if err := json.Unmarshal([]byte(imageInfoJSON), &infos); err != nil || len(infos) == 0 {
+ return imageInfoJSON
+ }
+ changed := false
+ for i := range infos {
+ switch chunkType {
+ case string(types.ChunkTypeImageOCR):
+ if infos[i].OCRText == recognized {
+ infos[i].OCRText = ""
+ changed = true
+ }
+ case string(types.ChunkTypeImageCaption):
+ if infos[i].Caption == recognized {
+ infos[i].Caption = ""
+ changed = true
+ }
+ }
+ }
+ if !changed {
+ return imageInfoJSON
+ }
+ return marshalImageInfos(infos)
+}
+
// EnrichContentWithImageInfo embeds image info as XML tags into text content.
// Inline Markdown image links get wrapped in <image> with <image_caption> / <image_ocr>;
// images not found in content are appended as <image> blocks.
diff --git a/internal/searchutil/imageinfo_match_test.go b/internal/searchutil/imageinfo_match_test.go
index 80060ecf4..fc3ac1c9e 100644
--- a/internal/searchutil/imageinfo_match_test.go
+++ b/internal/searchutil/imageinfo_match_test.go
@@ -172,3 +172,50 @@ func TestImageURLsInContent(t *testing.T) {
t.Fatalf("urls: %#v", urls)
}
}
+
+func TestClearImageInfoTextMatchingBody_ClearsOnlyHitField(t *testing.T) {
+ raw, err := json.Marshal([]types.ImageInfo{
+ {URL: "u1", OCRText: "page one ocr body", Caption: "page one caption"},
+ {URL: "u2", OCRText: "page two ocr body", Caption: "page two caption"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ got := ClearImageInfoTextMatchingBody(string(raw), "page one ocr body", string(types.ChunkTypeImageOCR))
+ var infos []types.ImageInfo
+ if err := json.Unmarshal([]byte(got), &infos); err != nil {
+ t.Fatal(err)
+ }
+ if len(infos) != 2 {
+ t.Fatalf("entries: %d", len(infos))
+ }
+ if infos[0].URL != "u1" || infos[0].OCRText != "" || infos[0].Caption != "page one caption" {
+ t.Fatalf("hit entry: %+v", infos[0])
+ }
+ if infos[1].URL != "u2" || infos[1].OCRText != "page two ocr body" {
+ t.Fatalf("sibling entry stripped: %+v", infos[1])
+ }
+
+ gotCaption := ClearImageInfoTextMatchingBody(string(raw), "page two caption", string(types.ChunkTypeImageCaption))
+ if err := json.Unmarshal([]byte(gotCaption), &infos); err != nil {
+ t.Fatal(err)
+ }
+ if infos[1].Caption != "" || infos[1].OCRText != "page two ocr body" || infos[0].Caption != "page one caption" {
+ t.Fatalf("caption clear: %+v", infos)
+ }
+}
+
+func TestClearImageInfoTextMatchingBody_UnchangedWhenNoMatch(t *testing.T) {
+ raw, err := json.Marshal([]types.ImageInfo{{URL: "u1", OCRText: "kept ocr"}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := ClearImageInfoTextMatchingBody(string(raw), "other body", string(types.ChunkTypeImageOCR))
+ if got != string(raw) {
+ t.Fatalf("expected original JSON, got %q", got)
+ }
+ if got := ClearImageInfoTextMatchingBody("", "body", string(types.ChunkTypeImageOCR)); got != "" {
+ t.Fatalf("empty JSON: %q", got)
+ }
+}