Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(image): stop appending KB custom instructions to the OCR prompt OCR提示词不再错误的附加图片解析用户自定义提示词 (#3291)

tayadwx7ea5bf434950
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go
index aedd671c7..631425f03 100644
--- a/internal/application/service/image_multimodal.go
+++ b/internal/application/service/image_multimodal.go
@@ -59,6 +59,20 @@ func buildVLMCaptionPrompt(ctx context.Context, cfg types.VLMConfig) string {
return types.AppendCustomPromptInstructions(prompt, cfg.CustomInstructions, "image_description")
}
+// buildVLMOCRPrompt returns the system-owned OCR prompt for the given image
+// source type. cfg is part of the signature and deliberately unused: knowledge
+// base custom instructions must never reach the OCR prompt — free-form
+// business rules compete with the "No text content" output contract and turn
+// decorative images into poisoned, vectorized image_ocr chunks. Custom OCR
+// guidance, if ever needed, deserves a dedicated OCR instruction field rather
+// than reusing CustomInstructions.
+func buildVLMOCRPrompt(sourceType string, _ types.VLMConfig) string {
+ if sourceType == "scanned_pdf" {
+ return vlmOCRScannedPDFPrompt
+ }
+ return vlmOCRPrompt
+}
+
// ImageMultimodalService handles image:multimodal asynq tasks.
// It reads images from storage (via FileService for provider:// URLs),
// performs OCR and VLM caption, and creates child chunks.
@@ -258,15 +272,13 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e
}
if payload.EnableOCR {
- prompt := vlmOCRPrompt
+ prompt := buildVLMOCRPrompt(payload.ImageSourceType, vlmCfg)
if payload.ImageSourceType == "scanned_pdf" {
- prompt = vlmOCRScannedPDFPrompt
logger.Infof(ctx, "[ImageMultimodal] Using scanned PDF prompt for OCR: %s", payload.ImageURL)
imgOut["ocr_prompt"] = "scanned_pdf"
} else {
imgOut["ocr_prompt"] = "default"
}
- prompt = types.AppendCustomPromptInstructions(prompt, vlmCfg.CustomInstructions, "image_ocr")
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
if ocrErr != nil {
diff --git a/internal/application/service/image_multimodal_prompt_test.go b/internal/application/service/image_multimodal_prompt_test.go
index 4eaa1eaff..578af6623 100644
--- a/internal/application/service/image_multimodal_prompt_test.go
+++ b/internal/application/service/image_multimodal_prompt_test.go
@@ -27,3 +27,37 @@ func TestBuildVLMCaptionPrompt(t *testing.T) {
}
})
}
+
+// TestVLMOCRPromptExcludesCustomInstructions pins the OCR-side half of the
+// prompt contract: knowledge base custom instructions may shape *how images
+// are described* (caption prompt) but must never reach the OCR prompt, where
+// free-form business rules compete with the "No text content" output contract
+// and produce poisoned image_ocr child chunks.
+func TestVLMOCRPromptExcludesCustomInstructions(t *testing.T) {
+ const marker = "UNIQUE_BUSINESS_MARKER_8f3a"
+ cfg := types.VLMConfig{CustomInstructions: "For images without text output " + marker}
+
+ for _, sourceType := range []string{"", "scanned_pdf"} {
+ got := buildVLMOCRPrompt(sourceType, cfg)
+ if strings.Contains(got, marker) {
+ t.Fatalf("OCR prompt for source %q leaked custom instructions: %s", sourceType, got)
+ }
+ if !strings.Contains(got, "No text content") {
+ t.Fatalf("OCR prompt for source %q lost the No text content contract: %s", sourceType, got)
+ }
+ }
+
+ if got := buildVLMOCRPrompt("scanned_pdf", cfg); got != vlmOCRScannedPDFPrompt {
+ t.Fatalf("scanned_pdf source must use the scanned-PDF prompt verbatim")
+ }
+ if got := buildVLMOCRPrompt("regular", cfg); got != vlmOCRPrompt {
+ t.Fatalf("regular source must use the default OCR prompt verbatim")
+ }
+
+ // The caption path keeps its custom instructions — only the OCR append was
+ // removed.
+ caption := buildVLMCaptionPrompt(context.Background(), cfg)
+ if !strings.Contains(caption, marker) {
+ t.Fatalf("caption prompt should still carry custom instructions: %s", caption)
+ }
+}