Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(image): stop appending KB custom instructions to the OCR prompt OCR提示词不再错误的附加图片解析用户自定义提示词 (#3291)
tayadwx7ea5bf434950
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.goindex aedd671c7..631425f03 100644--- a/internal/application/service/image_multimodal.go+++ b/internal/application/service/image_multimodal.go@@ -59,6 +59,20 @@ func buildVLMCaptionPrompt(ctx context.Context, cfg types.VLMConfig) string {return types.AppendCustomPromptInstructions(prompt, cfg.CustomInstructions, "image_description")}+// buildVLMOCRPrompt returns the system-owned OCR prompt for the given image+// source type. cfg is part of the signature and deliberately unused: knowledge+// base custom instructions must never reach the OCR prompt — free-form+// business rules compete with the "No text content" output contract and turn+// decorative images into poisoned, vectorized image_ocr chunks. Custom OCR+// guidance, if ever needed, deserves a dedicated OCR instruction field rather+// than reusing CustomInstructions.+func buildVLMOCRPrompt(sourceType string, _ types.VLMConfig) string {+ if sourceType == "scanned_pdf" {+ return vlmOCRScannedPDFPrompt+ }+ return vlmOCRPrompt+}+// ImageMultimodalService handles image:multimodal asynq tasks.// It reads images from storage (via FileService for provider:// URLs),// performs OCR and VLM caption, and creates child chunks.@@ -258,15 +272,13 @@ func (s *ImageMultimodalService) Handle(ctx context.Context, task *asynq.Task) e}if payload.EnableOCR {- prompt := vlmOCRPrompt+ prompt := buildVLMOCRPrompt(payload.ImageSourceType, vlmCfg)if payload.ImageSourceType == "scanned_pdf" {- prompt = vlmOCRScannedPDFPromptlogger.Infof(ctx, "[ImageMultimodal] Using scanned PDF prompt for OCR: %s", payload.ImageURL)imgOut["ocr_prompt"] = "scanned_pdf"} else {imgOut["ocr_prompt"] = "default"}- prompt = types.AppendCustomPromptInstructions(prompt, vlmCfg.CustomInstructions, "image_ocr")ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)if ocrErr != nil {diff --git a/internal/application/service/image_multimodal_prompt_test.go b/internal/application/service/image_multimodal_prompt_test.goindex 4eaa1eaff..578af6623 100644--- a/internal/application/service/image_multimodal_prompt_test.go+++ b/internal/application/service/image_multimodal_prompt_test.go@@ -27,3 +27,37 @@ func TestBuildVLMCaptionPrompt(t *testing.T) {}})}++// TestVLMOCRPromptExcludesCustomInstructions pins the OCR-side half of the+// prompt contract: knowledge base custom instructions may shape *how images+// are described* (caption prompt) but must never reach the OCR prompt, where+// free-form business rules compete with the "No text content" output contract+// and produce poisoned image_ocr child chunks.+func TestVLMOCRPromptExcludesCustomInstructions(t *testing.T) {+ const marker = "UNIQUE_BUSINESS_MARKER_8f3a"+ cfg := types.VLMConfig{CustomInstructions: "For images without text output " + marker}++ for _, sourceType := range []string{"", "scanned_pdf"} {+ got := buildVLMOCRPrompt(sourceType, cfg)+ if strings.Contains(got, marker) {+ t.Fatalf("OCR prompt for source %q leaked custom instructions: %s", sourceType, got)+ }+ if !strings.Contains(got, "No text content") {+ t.Fatalf("OCR prompt for source %q lost the No text content contract: %s", sourceType, got)+ }+ }++ if got := buildVLMOCRPrompt("scanned_pdf", cfg); got != vlmOCRScannedPDFPrompt {+ t.Fatalf("scanned_pdf source must use the scanned-PDF prompt verbatim")+ }+ if got := buildVLMOCRPrompt("regular", cfg); got != vlmOCRPrompt {+ t.Fatalf("regular source must use the default OCR prompt verbatim")+ }++ // The caption path keeps its custom instructions — only the OCR append was+ // removed.+ caption := buildVLMCaptionPrompt(context.Background(), cfg)+ if !strings.Contains(caption, marker) {+ t.Fatalf("caption prompt should still carry custom instructions: %s", caption)+ }+}