Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(ocr): reject invalid output and record extraction failures (#4132)

lyingbugd27ecade656a
diff --git a/internal/application/service/image_multimodal.go b/internal/application/service/image_multimodal.go
index 1de915704..fb7949b53 100644
--- a/internal/application/service/image_multimodal.go
+++ b/internal/application/service/image_multimodal.go
@@ -28,17 +28,17 @@ import (
)
const (
- vlmOCRPrompt = "<system_prompt>\n" +
- "You are an OCR assistant. Your task is to extract all body text content from this document image and output in pure Markdown format.\n" +
- "</system_prompt>\n\n" +
- "<instructions>\n" +
- "1. Ignore headers and footers.\n" +
- "2. Use Markdown table syntax for tables.\n" +
- "3. Use LaTeX format for formulas (wrapped with $ or $$).\n" +
- "4. Organize content in the original reading order.\n" +
- "5. Output ONLY the extracted text content. Do NOT include any HTML tags, reasoning, or unrelated comments.\n" +
- "6. If there is absolutely no recognizable text content in the image, reply ONLY with: No text content.\n" +
- "</instructions>"
+ vlmOCRPrompt = "You are an OCR assistant. Transcribe only text that is visibly present in the image, " +
+ "preserving its original language.\n" +
+ "The image may be a screenshot, photograph, diagram, or document. For scattered labels, " +
+ "buttons, scores, and other UI text, output plain text in reading order, one item per line. " +
+ "Do not force this content into a table.\n" +
+ "Use Markdown tables only when the image contains a real table with readable cells. Never " +
+ "output empty table cells, invented layout, repeated filler, or descriptions of the image.\n" +
+ "Preserve readable numbers and symbols. Do not guess unreadable text. Stop after all visible " +
+ "text has been transcribed; repeat text only when it visibly appears more than once.\n" +
+ "Output only the extracted text, without explanations, HTML tags, or code fences. If no text " +
+ "is recognizable, reply only: No text content."
vlmOCRScannedPDFPrompt = "<system_prompt>\n" +
"You are an OCR and document layout extraction assistant. The input image is a page from a scanned PDF document.\n" +
"Your task is to carefully extract all text and layout structure from the image, and output the result in pure Markdown format.\n" +
@@ -348,7 +348,21 @@ func (s *ImageMultimodalService) processImage(
plan := selectImageActionRounds(payload)
for i := 0; i < len(plan); i++ {
round := plan[i]
+ var ocrSpan *Span
+ if round.Contains(types.ActionOCR) && imgSpan != nil {
+ ocrSpan = tracker.BeginSubSpan(ctx, imgSpan, imgSpan.Name+".ocr", types.SpanKindGeneration, nil)
+ }
s.executeActionRound(ctx, payload, vlmModel, imgBytes, vlmCfg, &imageInfo, out, round)
+ if ocrSpan != nil {
+ if message, failed := out["ocr_error"].(string); failed {
+ code, _ := out["ocr_error_code"].(string)
+ tracker.FailSpan(ctx, ocrSpan, code, message, nil)
+ } else {
+ tracker.EndSpan(ctx, ocrSpan, types.JSONMap{
+ "status": out["ocr_status"], "chars": out["ocr_chars"],
+ })
+ }
+ }
// Result processor: only an observation round feeds the OCR decision.
if round.Contains(types.ActionObservation) {
ocrWanted := payload.EnableOCR && DecideOCR(imageInfo.Attrs, payload.ImageActions)
@@ -404,10 +418,19 @@ func (s *ImageMultimodalService) processImage(
})
}
out["chunks_created"] = len(newChunks)
+ if out["ocr_status"] == "failed" {
+ out["outcome"] = "partial_failure"
+ if len(newChunks) == 0 {
+ out["outcome"] = "failed"
+ }
+ }
if len(newChunks) == 0 {
// Deferred finalize will count this image on success.
out["skipped"] = "no_extracted_content"
+ if out["ocr_status"] == "failed" {
+ out["skipped"] = "ocr_failed"
+ }
return nil
}
@@ -499,6 +522,20 @@ func (s *ImageMultimodalService) runImageOCR(
out types.JSONMap,
vlmCfg types.VLMConfig,
) {
+ ctx, span := langfuse.GetManager().StartSpan(ctx, langfuse.SpanOptions{
+ Name: "image.ocr", Metadata: map[string]interface{}{"model_id": vlmModel.GetModelID()},
+ })
+ var failure error
+ defer func() {
+ span.Finish(map[string]interface{}{
+ "status": out["ocr_status"], "error_code": out["ocr_error_code"],
+ "rejection_reason": out["ocr_rejection_reason"], "chars": out["ocr_chars"],
+ "raw_chars": out["ocr_raw_chars"],
+ }, nil, failure)
+ }()
+ // A failed extraction must never leave text from a previous attempt behind.
+ imageInfo.OCRText = ""
+ out["ocr_chars"] = 0
// The OCR prompt is system-owned: knowledge base custom instructions must
// never reach it, or free-form business rules compete with the "No text
// content" contract and poison image_ocr chunks. buildVLMOCRPrompt picks
@@ -513,19 +550,38 @@ func (s *ImageMultimodalService) runImageOCR(
ocrText, ocrErr := vlmModel.Predict(ctx, [][]byte{imgBytes}, prompt)
if ocrErr != nil {
+ failure = ocrErr
logger.Warnf(ctx, "[ImageMultimodal] OCR failed for %s: %v", payload.ImageURL, ocrErr)
+ out["ocr_status"] = "failed"
+ out["ocr_error_code"] = "OCR_REQUEST_FAILED"
+ if errors.Is(ocrErr, vlm.ErrTruncatedCompletion) {
+ out["ocr_error_code"] = "OCR_TRUNCATED"
+ }
out["ocr_error"] = ocrErr.Error()
return
}
- ocrText = sanitizeOCRText(ocrText)
+ out["ocr_raw_chars"] = len([]rune(ocrText))
+ ocrText, ocrErr = validateOCRText(ocrText)
+ if ocrErr != nil {
+ failure = ocrErr
+ out["ocr_status"] = "failed"
+ out["ocr_error_code"] = "OCR_INVALID_OUTPUT"
+ out["ocr_error"] = ocrErr.Error()
+ var invalid *ocrValidationError
+ if errors.As(ocrErr, &invalid) {
+ out["ocr_rejection_reason"] = invalid.reason
+ }
+ logger.Warnf(ctx, "[ImageMultimodal] OCR rejected for %s: %v", payload.ImageURL, ocrErr)
+ return
+ }
if ocrText != "" {
imageInfo.OCRText = ocrText
+ out["ocr_status"] = "succeeded"
out["ocr_chars"] = len([]rune(ocrText))
out["ocr_preview"] = previewText(ocrText, 200)
} else {
- logger.Warnf(ctx, "[ImageMultimodal] OCR returned empty/invalid content for %s, discarded", payload.ImageURL)
- out["ocr_chars"] = 0
- out["ocr_skipped"] = "empty_or_invalid"
+ out["ocr_status"] = "no_text"
+ out["ocr_skipped"] = "no_text"
}
}
diff --git a/internal/application/service/image_ocr_live_test.go b/internal/application/service/image_ocr_live_test.go
new file mode 100644
index 000000000..a8ab34da5
--- /dev/null
+++ b/internal/application/service/image_ocr_live_test.go
@@ -0,0 +1,61 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/Tencent/WeKnora/internal/models/vlm"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+// Opt-in verification against the configured VLM, with production processing
+// and an in-memory chunk repository. No knowledge-base records are modified.
+// Fixture: {"model": <Model>, "image_paths": [<local path>, ...]}.
+// Keep credentials and user images outside version control; private endpoints
+// require the usual explicit SSRF_WHITELIST environment setting.
+func TestLiveImageOCRValidation(t *testing.T) {
+ path := os.Getenv("WEKNORA_OCR_TEST_FIXTURE")
+ if path == "" {
+ t.Skip("set WEKNORA_OCR_TEST_FIXTURE to run live OCR verification")
+ }
+ raw, err := os.ReadFile(path)
+ require.NoError(t, err)
+ var fixture struct {
+ Model types.Model `json:"model"`
+ ImagePaths []string `json:"image_paths"`
+ }
+ require.NoError(t, json.Unmarshal(raw, &fixture))
+ require.NotEmpty(t, fixture.ImagePaths)
+ model, err := vlm.NewRemoteAPIVLM(vlm.ConfigFromModel(&fixture.Model, "", ""))
+ require.NoError(t, err)
+ for _, path := range fixture.ImagePaths {
+ t.Run(filepath.Base(path), func(t *testing.T) {
+ data, err := os.ReadFile(path)
+ require.NoError(t, err)
+ repo := &attrsChunkRepo{}
+ svc := newAttrsTestService(&attrsFileService{body: data}, repo)
+ tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
+ out := types.JSONMap{}
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
+ defer cancel()
+ start := time.Now()
+ err = svc.processImage(ctx, &types.ImageMultimodalPayload{
+ ImageURL: "local://fixture", EnableOCR: true, Attempt: 1,
+ }, model, types.VLMConfig{}, tracker, out)
+ require.NoError(t, err)
+ require.Empty(t, tracker.failed)
+ require.Equal(t, "succeeded", out["ocr_status"], "OCR outcome: %v", out)
+ require.Len(t, repo.created, 1)
+ chunk := repo.created[0]
+ require.Equal(t, types.ChunkTypeImageOCR, chunk.ChunkType)
+ require.NotEmpty(t, chunk.Content)
+ t.Logf("duration=%s chars=%d status=%s preview=%q", time.Since(start), len([]rune(chunk.Content)),
+ out["ocr_status"], previewText(chunk.Content, 180))
+ })
+ }
+}
diff --git a/internal/application/service/image_ocr_validation_test.go b/internal/application/service/image_ocr_validation_test.go
new file mode 100644
index 000000000..be688fb83
--- /dev/null
+++ b/internal/application/service/image_ocr_validation_test.go
@@ -0,0 +1,99 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/vlm"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+type ocrTestTracker struct {
+ noopSpanTracker
+ failed map[string]string
+ outputs map[string]types.JSONMap
+}
+
+func (t *ocrTestTracker) LookupStage(context.Context, string, int, string) *Span {
+ return &Span{Name: "multimodal"}
+}
+
+func (t *ocrTestTracker) BeginSubSpan(_ context.Context, _ *Span, name, _ string, _ types.JSONMap) *Span {
+ return &Span{Name: name}
+}
+
+func (t *ocrTestTracker) FailSpan(_ context.Context, s *Span, code, _ string, _ error) {
+ t.failed[s.Name] = code
+}
+
+func (t *ocrTestTracker) EndSpan(_ context.Context, s *Span, out types.JSONMap) {
+ t.outputs[s.Name] = out
+}
+
+func TestImageOCRRejectsInvalidContentBeforePersistence(t *testing.T) {
+ for _, tc := range []struct {
+ name, raw, status, code string
+ err error
+ }{
+ {name: "recorded empty cells", raw: strings.Repeat("| ", 2500), status: "failed", code: "OCR_INVALID_OUTPUT"},
+ {name: "repeated words", raw: strings.Repeat("传球按钮 ", 200), status: "failed", code: "OCR_INVALID_OUTPUT"},
+ {name: "empty model output", status: "failed", code: "OCR_INVALID_OUTPUT"},
+ {
+ name: "truncated output", status: "failed", code: "OCR_TRUNCATED",
+ err: fmt.Errorf("request: %w", vlm.ErrTruncatedCompletion),
+ },
+ {name: "request failure", status: "failed", code: "OCR_REQUEST_FAILED", err: errors.New("model unavailable")},
+ {name: "no text", raw: "No text content.", status: "no_text"},
+ } {
+ for _, caption := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/caption=%v", tc.name, caption), func(t *testing.T) {
+ repo := &attrsChunkRepo{}
+ svc := newAttrsTestService(&attrsFileService{body: testPNG(t)}, repo)
+ model := &attrsFakeVLM{reply: func(prompt string, _ int) (string, error) {
+ if prompt == vlmOCRPrompt {
+ return tc.raw, tc.err
+ }
+ return "A game screenshot.", nil
+ }}
+ tracker := &ocrTestTracker{failed: map[string]string{}, outputs: map[string]types.JSONMap{}}
+ out := types.JSONMap{}
+ err := svc.processImage(context.Background(), &types.ImageMultimodalPayload{
+ ImageURL: "local://test.png", EnableOCR: true, EnableCaption: caption, Attempt: 1,
+ }, model, types.VLMConfig{}, tracker, out)
+ require.NoError(t, err, "bad OCR must not retry the whole image and duplicate a valid caption")
+ require.Equal(t, tc.status, out["ocr_status"])
+ require.Equal(t, 0, out["ocr_chars"])
+ require.NotContains(t, out, "ocr_preview")
+ require.Equal(t, tc.code, tracker.failed["multimodal.image[0].ocr"])
+ if tc.code != "" {
+ require.Equal(t, tc.code, out["ocr_error_code"])
+ require.NotEmpty(t, out["ocr_error"])
+ require.NotContains(t, tracker.outputs, "multimodal.image[0].ocr",
+ "failed OCR must not be marked done")
+ if caption {
+ require.Equal(t, "partial_failure", out["outcome"])
+ } else {
+ require.Equal(t, "failed", out["outcome"])
+ }
+ } else {
+ require.NotContains(t, out, "ocr_error")
+ require.Equal(t, "no_text", tracker.outputs["multimodal.image[0].ocr"]["status"])
+ }
+ if caption {
+ require.Len(t, repo.created, 1)
+ } else {
+ require.Empty(t, repo.created)
+ }
+ for _, chunk := range repo.created {
+ require.Equal(t, types.ChunkTypeImageCaption, chunk.ChunkType)
+ require.NotContains(t, chunk.ImageInfo, "| |")
+ require.NotContains(t, chunk.ImageInfo, "传球按钮")
+ }
+ })
+ }
+ }
+}
diff --git a/internal/application/service/ocr_sanitizer.go b/internal/application/service/ocr_sanitizer.go
index fba14b033..ab0ad72d3 100644
--- a/internal/application/service/ocr_sanitizer.go
+++ b/internal/application/service/ocr_sanitizer.go
@@ -1,8 +1,10 @@
package service
import (
+ "fmt"
"regexp"
"strings"
+ "unicode"
htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
@@ -28,6 +30,65 @@ var (
// sanitizeOCRText cleans up VLM OCR output by stripping HTML wrappers,
// converting HTML to markdown, and filtering out useless responses.
func sanitizeOCRText(raw string) string {
+ text, _ := validateOCRText(raw)
+ return text
+}
+
+type ocrValidationError struct{ reason string }
+
+func (e *ocrValidationError) Error() string { return fmt.Sprintf("invalid OCR output: %s", e.reason) }
+
+// validateOCRText distinguishes an explicit no-text answer from unusable model
+// output. Reject the entire response, rather than indexing a plausible prefix
+// followed by hallucinated repetition.
+func validateOCRText(raw string) (string, error) {
+ if isKnownEmptyReply(stripMarkdownCodeBlock(strings.TrimSpace(raw))) {
+ return "", nil
+ }
+ text := normalizeOCRText(raw)
+ if text == "" {
+ return "", &ocrValidationError{reason: "empty_content"}
+ }
+ readable := false
+ for _, r := range text {
+ if unicode.IsLetter(r) || unicode.IsNumber(r) ||
+ (unicode.IsSymbol(r) && !strings.ContainsRune("|~`", r)) {
+ readable = true
+ break
+ }
+ }
+ if !readable {
+ return "", &ocrValidationError{reason: "no_readable_content"}
+ }
+ if hasOCRRepetition(text) {
+ return "", &ocrValidationError{reason: "repetitive_content"}
+ }
+ return text, nil
+}
+
+// Require a long, dominant periodic run to avoid rejecting ordinary repeated
+// labels, table separators, or short numeric values. Normalize whitespace so
+// line wrapping does not hide a decoding loop. Work is bounded by 64*n.
+func hasOCRRepetition(text string) bool {
+ runes := []rune(strings.Join(strings.Fields(text), " "))
+ for period := 1; period <= 64 && period <= len(runes)/32; period++ {
+ matched := 0
+ for i := period; i < len(runes); i++ {
+ if runes[i] == runes[i-period] {
+ matched++
+ } else {
+ matched = 0
+ }
+ span := matched + period
+ if span >= 512 && span >= period*32 && span*5 >= len(runes)*4 {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func normalizeOCRText(raw string) string {
text := strings.TrimSpace(raw)
if text == "" {
return ""
diff --git a/internal/application/service/ocr_sanitizer_test.go b/internal/application/service/ocr_sanitizer_test.go
index 1281bc28a..9c858e066 100644
--- a/internal/application/service/ocr_sanitizer_test.go
+++ b/internal/application/service/ocr_sanitizer_test.go
@@ -1,6 +1,7 @@
package service
import (
+ "fmt"
"regexp"
"strings"
"testing"
@@ -255,3 +256,49 @@ func TestIsKnownEmptyReply(t *testing.T) {
})
}
}
+
+func TestValidateOCRTextRejectsDegenerateOutput(t *testing.T) {
+ for _, tt := range []struct {
+ name, input, reason string
+ }{
+ {"recorded empty cells", strings.Repeat("| ", 2500), "no_readable_content"},
+ {"empty response", " \n\t", "empty_content"},
+ {"markdown skeleton", "| --- | --- |\n| | |", "no_readable_content"},
+ {"repeated phrase", strings.Repeat("传球按钮 ", 200), "repetitive_content"},
+ {"repeated suffix after real text", "传球\n" + strings.Repeat("| ", 2500), "repetitive_content"},
+ {"repeated wrapped lines", strings.Repeat("开始\n锁定目标\n", 100), "repetitive_content"},
+ } {
+ t.Run(tt.name, func(t *testing.T) {
+ text, err := validateOCRText(tt.input)
+ invalid, ok := err.(*ocrValidationError)
+ if text != "" || !ok || invalid.reason != tt.reason {
+ t.Fatalf("text=%q error=%v; want rejection %s", text, err, tt.reason)
+ }
+ if got := sanitizeOCRText(tt.input); got != "" {
+ t.Fatal("shared sanitizer must also discard rejected OCR")
+ }
+ })
+ }
+}
+
+func TestValidateOCRTextPreservesRealContent(t *testing.T) {
+ var table strings.Builder
+ table.WriteString("| 项目 | 数值 |\n| --- | --- |\n")
+ for i := 0; i < 100; i++ {
+ fmt.Fprintf(&table, "| 项目%d | %d |\n", i, i)
+ }
+ for _, input := range []string{
+ "A", "3", "传球", "A\nB\nL1\nL2\n传球\n传球", "0:0\n58.29\n取消施法", "±", "$x^2 + y^2 = z^2$",
+ table.String(), strings.Repeat("同一个按钮\n", 5),
+ } {
+ got, err := validateOCRText(input)
+ if err != nil || got != strings.TrimSpace(input) {
+ t.Fatalf("valid OCR rejected or changed: input=%q got=%q err=%v", input, got, err)
+ }
+ }
+ for _, input := range []string{"No text content.", "无文字内容", "```\nNo text content.\n```"} {
+ if got, err := validateOCRText(input); err != nil || got != "" {
+ t.Fatalf("explicit no-text answer should succeed without text: %q %v", got, err)
+ }
+ }
+}
diff --git a/internal/application/service/temporary_document.go b/internal/application/service/temporary_document.go
index af6def7ec..1065cdf5a 100644
--- a/internal/application/service/temporary_document.go
+++ b/internal/application/service/temporary_document.go
@@ -602,7 +602,10 @@ func (s *temporaryDocumentService) understandImagesWithVLM(
logger.Warnf(ctx, "temporary document VLM OCR failed on image %d: %v", idx, ocrErr)
return
}
- ocrResults[idx] = sanitizeOCRText(ocrText)
+ ocrResults[idx], ocrErr = validateOCRText(ocrText)
+ if ocrErr != nil {
+ logger.Warnf(ctx, "temporary document VLM OCR rejected on image %d: %v", idx, ocrErr)
+ }
}(idx, img)
}
wg.Wait()
diff --git a/internal/models/vlm/remote_api.go b/internal/models/vlm/remote_api.go
index 1d1de4efb..7d1bbe970 100644
--- a/internal/models/vlm/remote_api.go
+++ b/internal/models/vlm/remote_api.go
@@ -119,15 +119,10 @@ func (v *RemoteAPIVLM) Predict(ctx context.Context, imgBytesList [][]byte, promp
return "", fmt.Errorf("VLM request: %w", err)
}
content := resp.Content
- if strings.TrimSpace(content) == "" && resp.FinishReason == "length" {
- // Reasoning models spend the completion budget on reasoning before any
- // visible output, so an exhausted budget yields an empty message rather
- // than an API error. Returning "" here would be recorded as
- // "no_extracted_content" and look identical to an image with no text.
- return "", fmt.Errorf(
- "VLM returned no content: completion truncated at %d tokens (finish_reason=length)",
- defaultMaxToks,
- )
+ if resp.FinishReason == "length" {
+ // Non-empty partial output is still incomplete and must not be indexed
+ // as successful OCR. Returning an error also marks the model trace failed.
+ return "", fmt.Errorf("%w at %d tokens (finish_reason=length)", ErrTruncatedCompletion, defaultMaxToks)
}
logger.Infof(ctx, "[VLM] response received, len=%d", len(content))
return content, nil
diff --git a/internal/models/vlm/remote_api_reasoning_test.go b/internal/models/vlm/remote_api_reasoning_test.go
index b59d16d92..6c1ebbcd5 100644
--- a/internal/models/vlm/remote_api_reasoning_test.go
+++ b/internal/models/vlm/remote_api_reasoning_test.go
@@ -2,6 +2,8 @@ package vlm
import (
"encoding/json"
+ "errors"
+ "fmt"
"math"
"net/http"
"net/http/httptest"
@@ -129,40 +131,30 @@ func TestRemoteAPIVLMKeepsMaxTokensForNonReasoningModel(t *testing.T) {
}
}
-// TestRemoteAPIVLMReportsTruncatedCompletion covers the other way a reasoning
-// model yields nothing: the completion budget also covers reasoning tokens, so
-// an exhausted budget returns an empty message with finish_reason=length
-// instead of an API error.
+// A completion stopped by the budget is not successful OCR, even when the
+// response has a non-empty prefix or a long run of repeated empty table cells.
func TestRemoteAPIVLMReportsTruncatedCompletion(t *testing.T) {
withVLMSSRFWhitelist(t, "127.0.0.1")
-
- server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
- w.Header().Set("Content-Type", "application/json")
- _, _ = w.Write([]byte(`{
- "id": "chatcmpl-test",
- "object": "chat.completion",
- "choices": [
- {"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "length"}
- ]
- }`))
- }))
- defer server.Close()
-
- v, err := NewRemoteAPIVLM(&Config{
- BaseURL: server.URL,
- ModelName: "gpt-5-nano",
- APIKey: "sk-test",
- Provider: "openai",
- })
- if err != nil {
- t.Fatalf("NewRemoteAPIVLM: %v", err)
- }
-
- _, err = v.Predict(t.Context(), [][]byte{testPNG}, "extract the text")
- if err == nil {
- t.Fatal("Predict returned nil error for a truncated completion")
- }
- if !strings.Contains(err.Error(), "truncated") {
- t.Errorf("error = %q, want it to mention truncation", err.Error())
+ for _, content := range []string{"", "partial extracted text", strings.Repeat("| ", 2500)} {
+ t.Run(fmt.Sprintf("chars=%d", len(content)), func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "choices": []any{map[string]any{
+ "index": 0, "message": map[string]any{"role": "assistant", "content": content},
+ "finish_reason": "length",
+ }},
+ })
+ }))
+ defer server.Close()
+ v, err := NewRemoteAPIVLM(&Config{BaseURL: server.URL, ModelName: "generic-vision", Provider: "generic"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, err := v.Predict(t.Context(), [][]byte{testPNG}, "extract text")
+ if got != "" || !errors.Is(err, ErrTruncatedCompletion) {
+ t.Fatalf("content=%q error=%v", got, err)
+ }
+ })
}
}
diff --git a/internal/models/vlm/vlm.go b/internal/models/vlm/vlm.go
index 42827aa48..9bec4aa1b 100644
--- a/internal/models/vlm/vlm.go
+++ b/internal/models/vlm/vlm.go
@@ -2,6 +2,7 @@ package vlm
import (
"context"
+ "errors"
"fmt"
"strings"
@@ -12,6 +13,9 @@ import (
"github.com/Tencent/WeKnora/internal/types"
)
+// ErrTruncatedCompletion means the model exhausted its output budget.
+var ErrTruncatedCompletion = errors.New("VLM completion truncated")
+
// VLM defines the interface for Vision Language Model operations.
type VLM interface {
// Predict sends one or more images with a text prompt to the VLM and returns the generated text.
diff --git a/website-docs/03-features/02-knowledge-base.md b/website-docs/03-features/02-knowledge-base.md
index 058e8168b..9af89e399 100644
--- a/website-docs/03-features/02-knowledge-base.md
+++ b/website-docs/03-features/02-knowledge-base.md
@@ -210,6 +210,12 @@ OCR 决策是代码纯函数 `DecideOCR`(`internal/application/service/image_a
管线与观察结果记录在每张图的处理轨迹子 span 上,便于核对与排查:input `pipeline` = `observation_driven`(本开关开启)或 `caption_ocr`(关闭,历史行为);output `attr_policy`(本轮 OCR 决策)、`image_attrs`(观察到的属性)、`ocr_skipped`(`attr_policy`=被策略跳过 / `disabled`=OCR 总开关关闭)、`observation_failed`(未形成有效观察)、`chunks_created`。
+OCR 会在入库前拒绝空响应、只有分隔符的内容,以及占据输出主体的长串重复内容;正常短标签、数字和表格仍可保留。OpenAI 兼容 VLM 返回 `finish_reason=length` 时,非空的截断输出也按失败处理。普通图片提示词按可见文字转录,截图中的按钮和标签逐行输出,仅真实表格使用 Markdown 表格;扫描 PDF 使用专用文档布局提示词。
+
+每次执行 OCR 都有独立的 `multimodal.image[i].ocr` 处理子 span;Langfuse 中对应 `image.ocr`,包含模型调用和输出校验。失败时记录 `ocr_status=failed`、`ocr_error_code`(`OCR_INVALID_OUTPUT` / `OCR_TRUNCATED` / `OCR_REQUEST_FAILED`)和错误原因,不创建或索引该次 OCR 分块。正常的图片描述仍可保留,整张图的 output 用 `outcome=partial_failure` 表明部分失败,不因无效 OCR 重试整个图片任务。明确回答没有文字时记为 `ocr_status=no_text`,与异常区分;正常提取记为 `succeeded`。已有异常分块不会被自动清除,需要重新解析相应文档。
+
+可通过 `WEKNORA_OCR_TEST_FIXTURE` 指定本地 JSON 文件(`{"model": <模型配置>, "image_paths": [<本地图片路径>]}`),运行 `go test ./internal/application/service -run '^TestLiveImageOCRValidation$' -v -count=1` 验证实际模型。私有端点需配置 `SSRF_WHITELIST`。该测试使用内存分块仓库,不写业务数据库;带凭证的配置和原图应放在版本控制之外。
+
单次上传 / 重新解析可在请求体的 `process_config`(`KnowledgeProcessOverrides`)里按文档覆盖 `image_attrs_enabled` 与 `image_actions`;未传的项沿用知识库设置。`image_actions` 按 action key 合并(`on` 整体替换)。接口字段见[知识库 API](../04-api/02-api-knowledge.md),管线细节见[文档解析](03-document-parsing.md)。
**图片向量(`image_vector_enabled`)**:默认关闭,升级后已有知识库也保持关闭(库里存的配置没有这个键,读出来就是 false),需要在知识库设置「图像处理配置 → 图片向量检索」里(开启多模态后出现)或通过 API 显式开启。只有**开关打开且向量模型能处理图片**时才生效,两者缺一则入库不生成图片向量、检索也不召回图片向量。之所以不随模型能力自动开启:能处理图片的向量模型常常只是被用来编码文本,自动开启会给这些知识库凭空增加每张图片一次的向量调用,并改变检索行为——开启后文档向量检索的候选池放大 1.5 倍,图片命中按 `min(VectorThreshold, 0.1)` 的单独阈值过滤。