RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Dispatch tests pin the routing contract:
//
// - FileTypeOTHER + missing setups → text-page mode.
// - FileTypeMarkdown → JSON payload family on the matching output
// key.
// - Historical output_format values, including PDF Markdown, are accepted
// and normalized to the single JSON payload.
package component
import (
"archive/zip"
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"ragflow/internal/common"
"reflect"
"strings"
"testing"
"ragflow/internal/deepdoc/parser/pdf"
deepdoctype "ragflow/internal/deepdoc/parser/pdf/type"
doctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
"ragflow/internal/utility"
"github.com/glebarez/sqlite"
"go.uber.org/zap"
"go.uber.org/zap/zaptest/observer"
"gorm.io/gorm"
)
// useMockDocAnalyzer installs a test-only MockDocAnalyzer as the in-process
// DeepDoc backend via the public factory seam. MockDocAnalyzer is test
// infrastructure and must never sit in the production fallback path; it is
// injected here so the production parse path can be exercised without a real
// DeepDoc service or ONNX Runtime models. The factory is reset to nil on
// cleanup (it is nil in this test binary, which registers no real backend).
func useMockDocAnalyzer(t *testing.T) {
t.Helper()
t.Cleanup(func() { doctype.SetNativeDocAnalyzerFactory(nil) })
doctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
return &pdf.MockDocAnalyzer{Healthy: true}, true
})
}
type captureSetupConfigurer struct {
setup map[string]any
}
func (c *captureSetupConfigurer) ConfigureFromSetup(setup map[string]any) {
c.setup = setup
}
func requireJSONText(t *testing.T, out map[string]any, want string) {
t.Helper()
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
items := parser.ItemsToMaps(jsonDocs)
if !ok || len(items) == 0 {
t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
for _, item := range items {
if text, _ := item["text"].(string); strings.Contains(text, want) {
return
}
}
t.Fatalf("json payload does not contain %q: %#v", want, items)
}
// TestNormalizeParserJSON_EmptyPayloadYieldsEmpty pins that a result with no
// populated payload family normalizes to an empty JSON slice rather than
// injecting a bogus item. HTML is not a supported backend output format
// (PR #19467 unified the component to JSON-only; no Go parser produces an
// HTML payload), so there is no HTML normalization branch to fall through
// to — this guards that regression.
func TestNormalizeParserJSON_EmptyPayloadYieldsEmpty(t *testing.T) {
r := parser.ParseResult{}
items := normalizeParserJSON(t.Context(), "doc.txt", r)
if len(items) != 0 {
t.Fatalf("normalizeParserJSON(empty) = %d items, want 0 (no payload → empty)", len(items))
}
}
func TestBuildParserOutputsPreservesPDFOutlineMetadata(t *testing.T) {
wantOutline := []map[string]any{{"title": "Overview", "level": 0, "page_number": 1}}
out := buildParserOutputs(t.Context(), parser.ParseResult{
File: map[string]any{
"name": "report.pdf",
"outline": wantOutline,
},
JSON: parser.MapsToItems([]map[string]any{{"text": "body", "doc_type_kwd": "text"}}),
}, "report.pdf", nil, "")
file, ok := out["file"].(map[string]any)
if !ok {
t.Fatalf("file = %T/%v, want parser file metadata", out["file"], out["file"])
}
gotOutline, ok := file["outline"].([]map[string]any)
if !ok {
t.Fatalf("file.outline = %T/%v, want []map[string]any", file["outline"], file["outline"])
}
if !reflect.DeepEqual(gotOutline, wantOutline) {
t.Fatalf("file.outline = %#v, want %#v", gotOutline, wantOutline)
}
}
func TestLogParserOutputReportsNormalizationSource(t *testing.T) {
core, logs := observer.New(zap.DebugLevel)
originalLogger := common.Logger
common.Logger = zap.New(core)
t.Cleanup(func() { common.Logger = originalLogger })
logParserOutput(parser.ParseResult{
JSON: []parser.DocItem{parser.NewTextJSONItem("Title"), parser.NewTextJSONItem("Body")},
}, []parser.DocItem{parser.NewTextJSONItem("Title"), parser.NewTextJSONItem("Body")})
entries := logs.FilterMessage("parser stage output").All()
if len(entries) != 1 {
t.Fatalf("parser output log count = %d, want 1", len(entries))
}
fields := entries[0].ContextMap()
if _, ok := fields["output_format"]; ok {
t.Errorf("output_format must be omitted because it is always json: %v", fields)
}
if got := fields["normalized_from"]; got != "json" {
t.Errorf("normalized_from = %v, want json", got)
}
if got := fields["json_items"]; got != int64(2) {
t.Errorf("json_items = %v, want 2", got)
}
}
func TestDispatch_JSONOutput(t *testing.T) {
setups := defaultSetups()
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("# Title\n\nbody\n"),
"doc_id": "doc.md",
"file_type": "md",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got, want := out["output_format"], "json"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json payload missing or wrong type: %T", out["json"])
}
if len(jsonItems) == 0 {
t.Errorf("json payload empty; want at least 1 item")
}
// File metadata is carried through dispatch.
if fm, ok := out["file"].(map[string]any); !ok || fm["name"] != "doc.md" {
t.Errorf("file metadata missing or wrong: %+v", out["file"])
}
}
func TestDispatchNormalizesConfiguredOutputFormatToJSON(t *testing.T) {
setups := defaultSetups()
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("From: [email protected]\r\nTo: [email protected]\r\nSubject: Hello\r\n\r\nBody\r\n"),
"name": "mail.eml",
"file_type": "eml",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
items := parser.ItemsToMaps(jsonDocs)
if !ok || len(items) == 0 {
t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
}
// TestDispatch_TextPageMode_NoFileType pins the no-dispatch
// path. When the upstream inputs supply neither file_type nor
// file.name, the component degrades to text-page mode and
// emits structured JSON items. This is the documented behavior for
// canvas-bound invocations that wire the binary directly without
// a family hint.
func TestDispatch_TextPageMode_NoFileType(t *testing.T) {
setups := defaultSetups()
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("plain content\n"),
"doc_id": "unknown",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got, want := out["output_format"], "json"; got != want {
t.Errorf("output_format = %v, want %v (raw-text mode)", got, want)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) == 0 {
t.Fatalf("json items missing or empty: %T", out["json"])
}
if _, ok := out["text"]; ok {
t.Fatalf("output contains obsolete text payload: %#v", out["text"])
}
}
// TestDispatch_SupportedFamilyFailure_HardErrors pins the agreed
// migration rule: once a supported family is identified, parser
// resolution/execution failures must surface as errors instead of
// silently degrading to text-page mode.
func TestDispatch_SupportedFamilyFailure_HardErrors(t *testing.T) {
setups := defaultSetups()
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("PDF payload as bytes (not a real PDF — stub test)\n"),
"file_type": "pdf",
})
if err == nil {
t.Fatal("Invoke: want error for supported family parse failure, got nil")
}
if !strings.Contains(err.Error(), "pdf") {
t.Errorf("error %q must mention pdf", err.Error())
}
}
// TestFileTypeFromInputs_ResolutionOrder pins the precedence
// rules documented on parser_dispatch.go:fileTypeFromInputs:
//
// 1. inputs["file_type"] (explicit family hint)
// 2. inputs["name"] (the resolved source filename)
// 3. inputs["file"].name (filename in the file descriptor)
// 4. FileTypeOTHER (text-page mode)
func TestFileTypeFromInputs_ResolutionOrder(t *testing.T) {
cases := []struct {
name string
in map[string]any
want string
}{
{"explicit pdf", map[string]any{"file_type": "pdf"}, "pdf"},
{"explicit xls (binary)", map[string]any{"file_type": "xls"}, "xls"},
{"explicit xlsx (OOXML)", map[string]any{"file_type": "xlsx"}, "xlsx"},
{"explicit ppt (binary)", map[string]any{"file_type": "ppt"}, "ppt"},
{"explicit pptx (OOXML)", map[string]any{"file_type": "pptx"}, "pptx"},
{"explicit slides (family name)", map[string]any{"file_type": "slides"}, "pptx"},
{"explicit spreadsheet (family name)", map[string]any{"file_type": "spreadsheet"}, "xlsx"},
{"explicit markdown (family form)", map[string]any{"file_type": "markdown"}, "md"},
{"name wins over conflicting file.name", map[string]any{"name": "report.txt", "file": map[string]any{"name": "report.pdf"}}, "txt"},
{"file.name docx", map[string]any{"file": map[string]any{"name": "report.docx"}}, "docx"},
{"name fallback md", map[string]any{"name": "notes.md"}, "md"},
{"unrelated inputs", map[string]any{"binary": []byte("x"), "doc_id": "abc"}, "other"},
{"unknown family hint", map[string]any{"file_type": "image/xyz"}, "other"},
{"nil inputs", nil, "other"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := fileTypeFromInputs(tc.in)
if string(got) != tc.want {
t.Errorf("fileTypeFromInputs(%v) = %q, want %q", tc.in, got, tc.want)
}
})
}
}
func TestConfigureParserFromSetups_UsesPythonFamilySetup(t *testing.T) {
setups := defaultSetups()
got := &captureSetupConfigurer{}
configureParserFromSetups(got, utility.FileTypePDF, setups)
want := map[string]any(setups["pdf"])
if !reflect.DeepEqual(got.setup, want) {
t.Fatalf("ConfigureFromSetup got %+v, want %+v", got.setup, want)
}
}
func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {
useMockDocAnalyzer(t)
path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s): %v", path, err)
}
setups := defaultSetups()
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": data,
"file_type": "pdf",
"name": "Doc1.pdf",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
items := parser.ItemsToMaps(jsonDocs)
if !ok || len(items) == 0 {
t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
}
// TestDispatch_PDFPlainText_UsesConfiguredBackend pins the plain-text
// dispatch for every spelling the dataset configuration UI can persist.
// The UI option is labelled "Naive" and stores "Plain Text"; treating that
// spelling as a custom VLM model name made the run fail with
// `provider name missing in model name: Plain Text`.
func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {
path := filepath.Join("..", "..", "..", "test", "benchmark", "test_docs", "Doc1.pdf")
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile(%s): %v", path, err)
}
for _, method := range []string{"plain_text", "plaintext", "Plain Text"} {
t.Run(method, func(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = method
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": data,
"file_type": "pdf",
"name": "Doc1.pdf",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) == 0 {
t.Fatalf("json payload missing or empty: %T", out["json"])
}
if got, _ := jsonItems[0]["text"].(string); strings.TrimSpace(got) == "" {
t.Fatalf("json first item text = %q, want non-empty", got)
}
})
}
}
func TestDispatch_PDFUnsupportedParseMethod_HardErrors(t *testing.T) {
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "bad.pdf",
})
if err == nil {
t.Fatal("Invoke: want error for unsupported PDF parse_method, got nil")
}
if !strings.Contains(err.Error(), "parse_method") || !strings.Contains(err.Error(), "tenant_id") {
t.Fatalf("error = %q, want parse_method + tenant_id context", err.Error())
}
}
func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) {
origPromptLoader := pdfVisionPromptLoader
origRenderer := pdfVisionPageRenderer
origResolver := pdfVisionModelResolver
origInvoker := pdfVisionChatInvoker
t.Cleanup(func() {
pdfVisionPromptLoader = origPromptLoader
pdfVisionPageRenderer = origRenderer
pdfVisionModelResolver = origResolver
pdfVisionChatInvoker = origInvoker
})
var prompts []string
pdfVisionPromptLoader = func(name string) (string, error) {
if name != "vision_llm_describe_prompt" {
return "", fmt.Errorf("unexpected prompt %q", name)
}
return "Describe page {{ page }}.", nil
}
pdfVisionPageRenderer = func(_ []byte, _ [][]int) ([]pdfVisionPage, error) {
return []pdfVisionPage{
{PageNumber: 1, WidthPts: 100, HeightPts: 200, ImageURL: "data:image/png;base64,aaa"},
{PageNumber: 2, WidthPts: 120, HeightPts: 240, ImageURL: "data:image/png;base64,bbb"},
}, nil
}
pdfVisionModelResolver = func(ctx context.Context, db *gorm.DB, tenantID string, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
if tenantID != "tenant-1" || modelID != "CustomVLM" {
return nil, "", nil, fmt.Errorf("resolver got tenant/model %q/%q", tenantID, modelID)
}
return nil, "resolved-vlm", nil, nil
}
pdfVisionChatInvoker = func(ctx context.Context, _ models.ModelDriver, modelName string, messages []models.Message, _ *models.APIConfig) (*models.ChatResponse, error) {
if modelName != "resolved-vlm" {
return nil, fmt.Errorf("modelName = %q, want resolved-vlm", modelName)
}
if len(messages) != 1 {
return nil, fmt.Errorf("messages len = %d, want 1", len(messages))
}
content, ok := messages[0].Content.([]interface{})
if !ok || len(content) != 2 {
return nil, fmt.Errorf("content = %#v, want multimodal prompt+image", messages[0].Content)
}
block, ok := content[0].(map[string]any)
if !ok {
return nil, fmt.Errorf("content[0] = %T, want map[string]any", content[0])
}
prompt, _ := block["text"].(string)
prompts = append(prompts, prompt)
answer := "Transcribed " + prompt
return &models.ChatResponse{Answer: &answer}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "vision.pdf",
"tenant_id": "tenant-1",
})
if err == nil {
jsonDocs, ok := out["json"].([]parser.DocItem)
jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) != 2 {
t.Fatalf("json payload = %#v, want 2 items", out["json"])
}
if got, want := jsonItems[0]["page_number"], 1; got != want {
t.Fatalf("json[0].page_number = %v, want %v", got, want)
}
if positions, ok := jsonItems[0]["_pdf_positions"].([][]any); !ok || len(positions) != 1 {
t.Fatalf("json[0]._pdf_positions = %#v, want one normalized page box", jsonItems[0]["_pdf_positions"])
}
if file, ok := out["file"].(map[string]any); !ok || file["parse_method"] != "CustomVLM" || file["page_count"] != 2 {
t.Fatalf("file metadata = %#v, want parse_method/page_count", out["file"])
}
if len(prompts) != 2 || !strings.Contains(prompts[0], "page 1") || !strings.Contains(prompts[1], "page 2") {
t.Fatalf("prompts = %#v, want rendered page-specific prompts", prompts)
}
return
}
t.Fatalf("Invoke: %v", err)
}
func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) {
origPromptLoader := pdfVisionPromptLoader
origRenderer := pdfVisionPageRenderer
origResolver := pdfVisionModelResolver
origInvoker := pdfVisionChatInvoker
t.Cleanup(func() {
pdfVisionPromptLoader = origPromptLoader
pdfVisionPageRenderer = origRenderer
pdfVisionModelResolver = origResolver
pdfVisionChatInvoker = origInvoker
})
pdfVisionPromptLoader = func(string) (string, error) { return "Describe page {{ page }}.", nil }
pdfVisionPageRenderer = func(_ []byte, _ [][]int) ([]pdfVisionPage, error) {
return []pdfVisionPage{
{PageNumber: 1, WidthPts: 100, HeightPts: 200, ImageURL: "data:image/png;base64,aaa"},
{PageNumber: 2, WidthPts: 120, HeightPts: 240, ImageURL: "data:image/png;base64,bbb"},
}, nil
}
pdfVisionModelResolver = func(ctx context.Context, db *gorm.DB, tenantID string, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
return nil, "resolved-vlm", nil, nil
}
call := 0
pdfVisionChatInvoker = func(ctx context.Context, _ models.ModelDriver, _ string, _ []models.Message, _ *models.APIConfig) (*models.ChatResponse, error) {
call++
answer := ""
if call == 1 {
answer = "First page"
}
return &models.ChatResponse{Answer: &answer}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "vision.pdf",
"tenant_id": "tenant-1",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) != 2 {
t.Fatalf("json payload = %#v, want 2 items", out["json"])
}
if got := jsonItems[1]["text"]; got != "" {
t.Fatalf("json[1].text = %#v, want empty string placeholder", got)
}
}
// TestDispatch_PDFVisionJSON_HonorsPageRanges pins that the VLM path renders
// and transcribes only the configured page ranges. The same setup key carries
// the canvas-debug page cap.
func TestDispatch_PDFVisionJSON_HonorsPageRanges(t *testing.T) {
origPromptLoader := pdfVisionPromptLoader
origRenderer := pdfVisionPageRenderer
origResolver := pdfVisionModelResolver
origInvoker := pdfVisionChatInvoker
t.Cleanup(func() {
pdfVisionPromptLoader = origPromptLoader
pdfVisionPageRenderer = origRenderer
pdfVisionModelResolver = origResolver
pdfVisionChatInvoker = origInvoker
})
pdfVisionPromptLoader = func(string) (string, error) { return "Describe page {{ page }}.", nil }
pdfVisionPageRenderer = func(_ []byte, ranges [][]int) ([]pdfVisionPage, error) {
if want := [][]int{{2, 3}}; !reflect.DeepEqual(ranges, want) {
return nil, fmt.Errorf("renderer ranges = %v, want %v", ranges, want)
}
return []pdfVisionPage{
{PageNumber: 2, WidthPts: 100, HeightPts: 200, ImageURL: "data:image/png;base64,bbb"},
{PageNumber: 3, WidthPts: 100, HeightPts: 200, ImageURL: "data:image/png;base64,ccc"},
}, nil
}
pdfVisionModelResolver = func(ctx context.Context, db *gorm.DB, tenantID string, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
return nil, "resolved-vlm", nil, nil
}
pdfVisionChatInvoker = func(ctx context.Context, _ models.ModelDriver, _ string, _ []models.Message, _ *models.APIConfig) (*models.ChatResponse, error) {
answer := "page text"
return &models.ChatResponse{Answer: &answer}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
setups["pdf"]["pages"] = []any{[]any{float64(2), float64(3)}}
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "vision.pdf",
"tenant_id": "tenant-1",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json payload = %T, want []map[string]any", out["json"])
}
var pageNumbers []any
for _, item := range jsonItems {
pageNumbers = append(pageNumbers, item["page_number"])
}
if want := []any{2, 3}; !reflect.DeepEqual(pageNumbers, want) {
t.Fatalf("page numbers = %v, want %v", pageNumbers, want)
}
}
// TestDispatch_PDFVisionJSON_RejectsInvalidPages pins that an invalid pages
// value fails the VLM parse before any page is rendered. A fallback to every
// page would send the whole PDF to the vision model.
func TestDispatch_PDFVisionJSON_RejectsInvalidPages(t *testing.T) {
origRenderer := pdfVisionPageRenderer
t.Cleanup(func() { pdfVisionPageRenderer = origRenderer })
pdfVisionPageRenderer = func(_ []byte, ranges [][]int) ([]pdfVisionPage, error) {
t.Errorf("renderer called with ranges %v, want no render for invalid pages", ranges)
return nil, fmt.Errorf("renderer must not run")
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "CustomVLM"
setups["pdf"]["pages"] = "abc"
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "vision.pdf",
"tenant_id": "tenant-1",
})
if err == nil || !strings.Contains(err.Error(), "pdf vision pages") {
t.Fatalf("Invoke error = %v, want pdf vision pages error", err)
}
}
func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == "/file_parse" {
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody\n"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
return
}
http.NotFound(w, r)
}))
defer server.Close()
// Mock the MinerU provider resolver to return a driver pointing at the test server.
origResolver := resolveTenantOCRModelByProvider
defer func() { resolveTenantOCRModelByProvider = origResolver }()
baseURL := server.URL
apiKey := ""
resolveTenantOCRModelByProvider = func(ctx context.Context, db *gorm.DB, tenantID string, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
if got, want := providerName, "MinerU"; got != want {
t.Fatalf("providerName = %q, want %q", got, want)
}
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
requireJSONText(t, out, "Title")
}
func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == "/file_parse" {
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody paragraph\n"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
return
}
http.NotFound(w, r)
}))
defer server.Close()
origResolver := resolveTenantOCRModelByProvider
defer func() { resolveTenantOCRModelByProvider = origResolver }()
baseURL := server.URL
apiKey := ""
resolveTenantOCRModelByProvider = func(ctx context.Context, db *gorm.DB, tenantID string, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
if got, want := providerName, "MinerU"; got != want {
t.Fatalf("providerName = %q, want %q", got, want)
}
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got, want := out["output_format"], "json"; got != want {
t.Fatalf("output_format = %v, want %v", got, want)
}
jsonDocs, ok := out["json"].([]parser.DocItem)
items := parser.ItemsToMaps(jsonDocs)
if !ok || len(items) == 0 {
t.Fatalf("json payload = %#v, want non-empty structured items", out["json"])
}
}
func TestDispatch_PDFMinerUMarkdown_SendsServerURLFromProviderConfig(t *testing.T) {
withSSRFBypass(t)
var gotBackend, gotServerURL, gotAuth string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost && r.URL.Path == "/file_parse" {
gotAuth = r.Header.Get("Authorization")
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("ParseMultipartForm: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
gotBackend = r.FormValue("backend")
gotServerURL = r.FormValue("server_url")
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"http-client"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
return
}
http.NotFound(w, r)
}))
defer server.Close()
origResolver := resolveMinerUModelForDispatch
defer func() { resolveMinerUModelForDispatch = origResolver }()
baseURL := server.URL
apiKey := `{"mineru_backend":"vlm-http-client","mineru_server_url":"http://vllm-host:30000"}`
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got, want := gotBackend, "vlm-http-client"; got != want {
t.Fatalf("backend = %q, want %q", got, want)
}
if got, want := gotServerURL, "http://vllm-host:30000"; got != want {
t.Fatalf("server_url = %q, want %q", got, want)
}
if gotAuth != "" {
t.Fatalf("Authorization = %q, want empty for provider JSON api_key", gotAuth)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
requireJSONText(t, out, "http-client")
}
func TestDispatch_PDFMinerUMarkdown_RequiresServerURLForHTTPClientBackend(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.NotFoundHandler())
defer server.Close()
origResolver := resolveMinerUModelForDispatch
defer func() { resolveMinerUModelForDispatch = origResolver }()
baseURL := server.URL
apiKey := `{"mineru_backend":"hybrid-http-client"}`
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err == nil {
t.Fatal("Invoke: want error when mineru_server_url is missing, got nil")
}
if !strings.Contains(err.Error(), "mineru_server_url") {
t.Fatalf("error = %q, want mineru_server_url context", err.Error())
}
}
func TestDispatch_PDFMinerUV1_WiresParseAndExtract(t *testing.T) {
withSSRFBypass(t)
invoke := func(t *testing.T, serverURL string) map[string]any {
t.Helper()
origResolver := resolveMinerUModelForDispatch
t.Cleanup(func() { resolveMinerUModelForDispatch = origResolver })
baseURL := serverURL
apiKey := ""
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
setups["pdf"]["mineru_backend"] = "standard"
return invokeMinerUDispatch(t, nil, setups)
}
t.Run("zip content_list uses mineruExtractJSONItems", func(t *testing.T) {
var zipBuf bytes.Buffer
zw := zip.NewWriter(&zipBuf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"V1 body"},{"type":"table","table_body":"<table></table>","table_caption":["Table caption"]}]`))
_ = zw.Close()
zipBytes := zipBuf.Bytes()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
w.WriteHeader(http.StatusOK)
case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
_, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
_, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"}}}]}`))
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBytes)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
out := invoke(t, server.URL)
requireJSONText(t, out, "V1 body")
requireJSONText(t, out, "Table caption")
})
t.Run("markdown fallback when zip has no content_list", func(t *testing.T) {
var zipBuf bytes.Buffer
zw := zip.NewWriter(&zipBuf)
f, _ := zw.Create("middle.json")
_, _ = f.Write([]byte(`{}`))
_ = zw.Close()
zipBytes := zipBuf.Bytes()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
w.WriteHeader(http.StatusOK)
case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
_, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
_, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBytes)
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
_, _ = w.Write([]byte("# Zip had no content_list\n"))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
out := invoke(t, server.URL)
requireJSONText(t, out, "Zip had no content_list")
})
t.Run("markdown fallback when zip sections are blank", func(t *testing.T) {
var zipBuf bytes.Buffer
zw := zip.NewWriter(&zipBuf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":""}]`))
_ = zw.Close()
zipBytes := zipBuf.Bytes()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
w.WriteHeader(http.StatusOK)
case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
_, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
_, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBytes)
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
_, _ = w.Write([]byte("# Blank zip sections\n"))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
out := invoke(t, server.URL)
requireJSONText(t, out, "Blank zip sections")
})
t.Run("markdown fallback when zip download fails", func(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
w.WriteHeader(http.StatusOK)
case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
_, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
_, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"},"markdown":{"file_id":"out-md"}}}]}`))
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
http.Error(w, "zip gone", http.StatusInternalServerError)
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-md/content":
_, _ = w.Write([]byte("# Markdown fallback\n"))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
out := invoke(t, server.URL)
requireJSONText(t, out, "Markdown fallback")
})
t.Run("empty content_list without markdown is an error", func(t *testing.T) {
var zipBuf bytes.Buffer
zw := zip.NewWriter(&zipBuf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[]`))
_ = zw.Close()
zipBytes := zipBuf.Bytes()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.Method == http.MethodGet && r.URL.Path == "/v1/health":
w.WriteHeader(http.StatusOK)
case r.Method == http.MethodPost && r.URL.Path == "/v1/uploads":
_, _ = w.Write([]byte(`{"id":"up-1","status":"completed","file":{"id":"file-1"}}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/parse/jobs":
_, _ = w.Write([]byte(`{"job_id":"job-1","status":"completed","files":[{"name":"sample.pdf","status":"completed","output_files":{"zip":{"file_id":"out-zip"}}}]}`))
case r.Method == http.MethodGet && r.URL.Path == "/v1/files/out-zip/content":
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(zipBytes)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
origResolver := resolveMinerUModelForDispatch
t.Cleanup(func() { resolveMinerUModelForDispatch = origResolver })
baseURL := server.URL
apiKey := ""
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
setups["pdf"]["mineru_backend"] = "standard"
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err == nil {
t.Fatal("Invoke: want error when V1 zip yields no sections")
}
if !strings.Contains(err.Error(), "no usable content_list") && !strings.Contains(err.Error(), "no sections") {
t.Fatalf("error = %q, want empty content_list extract error", err.Error())
}
})
}
func TestDispatch_PDFMonkeyOCRv2Markdown_UsesNativeParseEndpoint(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/parse" {
http.NotFound(w, r)
return
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Fatal(err)
}
if r.FormValue("start_page_id") != "0" || r.FormValue("end_page_id") != "99999" {
t.Fatalf("page range = %q:%q", r.FormValue("start_page_id"), r.FormValue("end_page_id"))
}
file, _, err := r.FormFile("files")
if err != nil {
t.Fatal(err)
}
_ = file.Close()
var output bytes.Buffer
archive := zip.NewWriter(&output)
entry, _ := archive.Create("sample/jsons/sample.json")
_, _ = entry.Write([]byte(`[{"label":"Title","content":"MonkeyOCRv2 title"}]`))
_ = archive.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(output.Bytes())
}))
defer server.Close()
original := resolveTenantOCRModelByProvider
t.Cleanup(func() { resolveTenantOCRModelByProvider = original })
resolveTenantOCRModelByProvider = func(_ context.Context, _ *gorm.DB, tenantID, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
if tenantID != "test-tenant" || providerName != "MonkeyOCRv2" {
t.Fatalf("tenant=%q provider=%q", tenantID, providerName)
}
return &monkeyOCRv2FakeDriver{}, "MonkeyOCRv2-Parsing", &models.APIConfig{BaseURL: &server.URL}, 0, nil
}
component, err := NewParserComponent(map[string]any{
"pdf": map[string]any{"parse_method": "monkeyocrv2"},
})
if err != nil {
t.Fatalf("NewParserComponent: %v", err)
}
out, err := component.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "MonkeyOCRv2 title")
}
func TestDispatch_PDFMonkeyOCRMarkdown_UsesFileParseEndpoint(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
http.NotFound(w, r)
return
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if got, want := r.FormValue("parse_method"), "auto"; got != want {
t.Errorf("parse_method form value = %q, want %q", got, want)
}
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"# MonkeyOCR title\n\nBody\n"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
}))
defer server.Close()
origResolver := resolveMonkeyOCRModelForDispatch
t.Cleanup(func() { resolveMonkeyOCRModelForDispatch = origResolver })
baseURL := server.URL
apiKey := ""
factoryDriver, err := models.NewModelFactory().CreateModelDriver("MonkeyOCR", map[string]string{"default": server.URL}, models.URLSuffix{DocumentParse: "file_parse"})
if err != nil {
t.Fatalf("CreateModelDriver: %v", err)
}
if factoryDriver.Name() != "monkeyocr" {
t.Fatalf("factory driver Name()=%q, want monkeyocr", factoryDriver.Name())
}
resolveMonkeyOCRModelForDispatch = func(_ context.Context, _ *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
if tenantID != "test-tenant" {
t.Fatalf("tenant=%q", tenantID)
}
if modelID != "" {
t.Fatalf("modelID = %q, want empty for the named selector", modelID)
}
return factoryDriver, "MonkeyOCR-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
component, err := NewParserComponent(map[string]any{
"pdf": map[string]any{"parse_method": "monkeyocr", "output_format": "markdown"},
})
if err != nil {
t.Fatalf("NewParserComponent: %v", err)
}
out, err := component.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "MonkeyOCR title")
}
func TestDispatch_PDFMonkeyOCR_CompositeSelectorInParseMethod(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
http.NotFound(w, r)
return
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if got, want := r.FormValue("parse_method"), "auto"; got != want {
t.Errorf("parse_method form value = %q, want %q", got, want)
}
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"# MonkeyOCR composite\n"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
}))
defer server.Close()
selector := "MonkeyOCR-model@default@MonkeyOCR"
origResolver := resolveMonkeyOCRModelForDispatch
t.Cleanup(func() { resolveMonkeyOCRModelForDispatch = origResolver })
baseURL := server.URL
apiKey := ""
factoryDriver, err := models.NewModelFactory().CreateModelDriver("MonkeyOCR", map[string]string{"default": server.URL}, models.URLSuffix{DocumentParse: "file_parse"})
if err != nil {
t.Fatalf("CreateModelDriver: %v", err)
}
if factoryDriver.Name() != "monkeyocr" {
t.Fatalf("factory driver Name()=%q, want monkeyocr", factoryDriver.Name())
}
resolveMonkeyOCRModelForDispatch = func(_ context.Context, _ *gorm.DB, tenantID, modelID string) (models.ModelDriver, string, *models.APIConfig, error) {
if got, want := modelID, selector; got != want {
t.Fatalf("modelID = %q, want %q", got, want)
}
return factoryDriver, "MonkeyOCR-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
component, err := NewParserComponent(map[string]any{
"pdf": map[string]any{"parse_method": selector, "output_format": "markdown"},
})
if err != nil {
t.Fatalf("NewParserComponent: %v", err)
}
out, err := component.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "MonkeyOCR composite")
}
// mineruTestDriver is a minimal ModelDriver mock whose Name() returns "mineru".
type mineruTestDriver struct{}
func (d *mineruTestDriver) NewInstance(baseURL map[string]string) models.ModelDriver { return d }
func (d *mineruTestDriver) Name() string { return "mineru" }
func (d *mineruTestDriver) ChatWithMessages(ctx context.Context, modelName string, messages []models.Message, apiConfig *models.APIConfig, chatModelConfig *models.ChatConfig, usage *common.ModelUsage) (*models.ChatResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) ChatStreamlyWithSender(ctx context.Context, modelName string, messages []models.Message, apiConfig *models.APIConfig, modelConfig *models.ChatConfig, usage *common.ModelUsage, sender func(*string, *string) error) error {
return fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) Embed(ctx context.Context, modelName *string, request models.EmbedRequest, apiConfig *models.APIConfig, embeddingConfig *models.EmbeddingConfig, usage *common.ModelUsage) ([]models.EmbeddingData, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) Rerank(ctx context.Context, modelName *string, request models.RerankRequest, apiConfig *models.APIConfig, rerankConfig *models.RerankConfig, usage *common.ModelUsage) (*models.RerankResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) TranscribeAudio(ctx context.Context, modelName *string, file *string, apiConfig *models.APIConfig, asrConfig *models.ASRConfig, usage *common.ModelUsage) (*models.ASRResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) TranscribeAudioWithSender(ctx context.Context, modelName *string, file *string, apiConfig *models.APIConfig, asrConfig *models.ASRConfig, usage *common.ModelUsage, sender func(*string, *string) error) error {
return fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) AudioSpeech(ctx context.Context, modelName *string, audioContent *string, apiConfig *models.APIConfig, ttsConfig *models.TTSConfig, usage *common.ModelUsage) (*models.TTSResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) AudioSpeechWithSender(ctx context.Context, modelName *string, audioContent *string, apiConfig *models.APIConfig, ttsConfig *models.TTSConfig, usage *common.ModelUsage, sender func(*string, *string) error) error {
return fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) OCRFile(ctx context.Context, modelName *string, content []byte, url *string, apiConfig *models.APIConfig, ocrConfig *models.OCRConfig, usage *common.ModelUsage) (*models.OCRFileResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) ParseFile(ctx context.Context, modelName *string, content []byte, url *string, apiConfig *models.APIConfig, parseFileConfig *models.ParseFileConfig, usage *common.ModelUsage) (*models.ParseFileResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) ListModels(ctx context.Context, apiConfig *models.APIConfig) ([]models.ListModelResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) Balance(ctx context.Context, apiConfig *models.APIConfig) (map[string]interface{}, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) CheckConnection(ctx context.Context, apiConfig *models.APIConfig) error {
return fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) ListTasks(ctx context.Context, apiConfig *models.APIConfig) ([]models.ListTaskStatus, error) {
return nil, fmt.Errorf("not implemented")
}
func (d *mineruTestDriver) ShowTask(ctx context.Context, taskID string, apiConfig *models.APIConfig) (*models.TaskResponse, error) {
return nil, fmt.Errorf("not implemented")
}
func (u *mineruTestDriver) Decide(ctx context.Context, modelName *string, request *models.DecideRequest, apiConfig *models.APIConfig, modelUsage *common.ModelUsage) (*models.DecideResponse, error) {
return nil, nil
}
func TestDispatch_PDFPaddleOCRMarkdown_UsesTenantModel(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/layout-parsing" {
http.NotFound(w, r)
return
}
if got, want := r.Header.Get("Authorization"), "Bearer paddle-secret"; got != want {
t.Errorf("Authorization = %q, want %q", got, want)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"errorCode":0,"result":{"layoutParsingResults":[{"markdown":{"text":"# Paddle Title\n\nPaddle body.\n"}}]}}`))
}))
defer server.Close()
// Mock resolveTenantOCRModelByProvider to return a PaddleOCR driver
// pointing at the test server.
origResolver := resolveTenantOCRModelByProvider
defer func() { resolveTenantOCRModelByProvider = origResolver }()
baseURL := server.URL
apiKey := "paddle-secret"
resolveTenantOCRModelByProvider = func(ctx context.Context, db *gorm.DB, tenantID string, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
if got, want := providerName, "PaddleOCR"; got != want {
t.Fatalf("providerName = %q, want %q", got, want)
}
return models.NewPaddleOCRLocalModel(nil, models.URLSuffix{OCR: "layout-parsing"}), "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "PaddleOCR"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
requireJSONText(t, out, "Paddle Title")
}
// TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload pins the cloud
// PaddleOCR configuration contract: the tenant api_key is a JSON payload
// (paddleocr_api_url / paddleocr_access_token / paddleocr_algorithm) and the
// instance base_url field stays empty, mirroring Python's PaddleOCROcrModel.
// The driver, not the dispatch layer, unwraps that payload into the wire
// bearer token and server endpoint.
func TestDispatch_PDFPaddleOCRMarkdown_UsesAPIKeyPayload(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
switch {
case r.Method == http.MethodPost && r.URL.Path == "/api/v2/ocr/jobs":
if got, want := r.Header.Get("Authorization"), "Bearer tok-123"; got != want {
t.Errorf("Authorization = %q, want %q (driver must unwrap api_key payload)", got, want)
}
_, _ = w.Write([]byte(`{"data":{"jobId":"job-1"}}`))
case r.Method == http.MethodGet && r.URL.Path == "/api/v2/ocr/jobs/job-1":
if got, want := r.Header.Get("Authorization"), "Bearer tok-123"; got != want {
t.Errorf("poll Authorization = %q, want %q", got, want)
}
_, _ = w.Write([]byte(`{"data":{"state":"done","resultUrl":{"jsonUrl":"http://` + r.Host + `/result"}}}`))
case r.Method == http.MethodGet && r.URL.Path == "/result":
_, _ = w.Write([]byte(`[{"logId":"l1","errorCode":0,"result":{"layoutParsingResults":[{"markdown":{"text":"# Unwrapped Title\n\nUnwrapped body.\n"}}]}}]`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
origResolver := resolveTenantOCRModelByProvider
defer func() { resolveTenantOCRModelByProvider = origResolver }()
apiKey := fmt.Sprintf(
`{"paddleocr_api_url":%q,"paddleocr_access_token":"tok-123","paddleocr_algorithm":"PaddleOCR-VL"}`,
server.URL+"/api")
emptyBaseURL := ""
resolveTenantOCRModelByProvider = func(ctx context.Context, db *gorm.DB, tenantID string, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
if got, want := providerName, "PaddleOCR"; got != want {
t.Fatalf("providerName = %q, want %q", got, want)
}
return models.NewPaddleOCRModel(nil, models.URLSuffix{OCR: "v2/ocr/jobs"}), "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &emptyBaseURL}, 0, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "PaddleOCR"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "Unwrapped Title")
}
func TestDispatch_PDFPaddleOCR_NoTenantModel_HardErrors(t *testing.T) {
withSSRFBypass(t)
origResolver := resolveTenantOCRModelByProvider
defer func() { resolveTenantOCRModelByProvider = origResolver }()
resolveTenantOCRModelByProvider = func(ctx context.Context, db *gorm.DB, tenantID string, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
return nil, "", nil, 0, fmt.Errorf("no active PaddleOCR OCR model")
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "paddleocr"
c := &ParserComponent{setups: setups}
_, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err == nil || !strings.Contains(err.Error(), "parser: PaddleOCR model") {
t.Fatalf("Invoke error = %v, want PaddleOCR model error", err)
}
}
// TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel pins the routing of
// a bare tenant model UUID in layout_recognizer — the value the web UI writes
// when a user picks an OCR model for PDF parsing — to the PaddleOCR dispatch
// path. The raw UUID carries no "@provider" hint in the string, so it must be
// resolved first (mirroring Python's get_composite_model_name_by_id before
// normalize_layout_recognizer). Previously the UUID fell through to the
// image2text VLM path and failed with "cannot be used as image2text model"
// for OCR-typed models such as the cloud "PaddleOCR" provider's.
func TestDispatch_PDFPaddleOCR_BareModelUUID_UsesExactModel(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/layout-parsing" {
http.NotFound(w, r)
return
}
if got, want := r.Header.Get("Authorization"), "Bearer paddle-secret"; got != want {
t.Errorf("Authorization = %q, want %q", got, want)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"errorCode":0,"result":{"layoutParsingResults":[{"markdown":{"text":"# Cloud Paddle Title\n\nCloud body.\n"}}]}}`))
}))
defer server.Close()
origIsLayout := isPaddleOCRLayoutModelID
origResolve := resolvePaddleOCRModelForDispatch
defer func() {
isPaddleOCRLayoutModelID = origIsLayout
resolvePaddleOCRModelForDispatch = origResolve
}()
modelID := "d13ffec6c1e34b1abc30e540b692d83d"
isPaddleOCRLayoutModelID = func(ctx context.Context, db *gorm.DB, tenantID, layout string) bool {
if got, want := layout, modelID; got != want {
t.Fatalf("layout = %q, want %q", got, want)
}
return true
}
baseURL := server.URL
apiKey := "paddle-secret"
resolvePaddleOCRModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
if got, want := mid, modelID; got != want {
t.Fatalf("modelID = %q, want %q", got, want)
}
return models.NewPaddleOCRLocalModel(nil, models.URLSuffix{OCR: "layout-parsing"}), "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["layout_recognizer"] = modelID
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
requireJSONText(t, out, "Cloud Paddle Title")
}
// TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod pins the routing of a
// bare tenant model UUID carried in parse_method (with layout_recognizer
// empty) to the PaddleOCR dispatch path. Previously only layout_recognizer
// was probed for a UUID, so a UUID in parse_method fell through to the
// image2text VLM path and failed with "cannot be used as image2text model".
func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/layout-parsing" {
http.NotFound(w, r)
return
}
if got, want := r.Header.Get("Authorization"), "Bearer paddle-secret"; got != want {
t.Errorf("Authorization = %q, want %q", got, want)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"errorCode":0,"result":{"layoutParsingResults":[{"markdown":{"text":"# Cloud Paddle Title\n\nCloud body.\n"}}]}}`))
}))
defer server.Close()
origIsLayout := isPaddleOCRLayoutModelID
origResolve := resolvePaddleOCRModelForDispatch
defer func() {
isPaddleOCRLayoutModelID = origIsLayout
resolvePaddleOCRModelForDispatch = origResolve
}()
modelID := "d13ffec6c1e34b1abc30e540b692d83d"
isPaddleOCRLayoutModelID = func(ctx context.Context, db *gorm.DB, tenantID, selector string) bool {
if got, want := selector, modelID; got != want {
t.Fatalf("selector = %q, want %q", got, want)
}
return true
}
baseURL := server.URL
apiKey := "paddle-secret"
resolvePaddleOCRModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
if got, want := mid, modelID; got != want {
t.Fatalf("modelID = %q, want %q", got, want)
}
return models.NewPaddleOCRLocalModel(nil, models.URLSuffix{OCR: "layout-parsing"}), "PaddleOCR-VL", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = modelID
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
requireJSONText(t, out, "Cloud Paddle Title")
}
// TestDispatch_PDFMinerU_BareModelUUID_InParseMethod pins the routing of a
// bare tenant model UUID in parse_method — the value stored when a user picks
// an OCR model for PDF parsing — to the MinerU dispatch path using that exact
// model. The UUID carries no "@provider" hint, so it must be resolved before
// the dispatch path is chosen instead of falling through to the image2text
// VLM path.
func TestDispatch_PDFMinerU_BareModelUUID_InParseMethod(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
http.NotFound(w, r)
return
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if got, want := r.Header.Get("Authorization"), "Bearer mineru-secret"; got != want {
t.Errorf("Authorization = %q, want %q", got, want)
}
if got, want := r.FormValue("parse_method"), "auto"; got != want {
t.Errorf("parse_method form value = %q, want %q", got, want)
}
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"# Cloud MinerU Title\n\nCloud body.\n"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
}))
defer server.Close()
modelID := "061436bbd8474d54be5bba3efbeac109"
origProbe := isMinerULayoutModelID
origResolve := resolveMinerUModelForDispatch
defer func() {
isMinerULayoutModelID = origProbe
resolveMinerUModelForDispatch = origResolve
}()
isMinerULayoutModelID = func(ctx context.Context, db *gorm.DB, tenantID, selector string) bool {
if got, want := selector, modelID; got != want {
t.Fatalf("selector = %q, want %q", got, want)
}
return true
}
baseURL := server.URL
apiKey := "mineru-secret"
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
if got, want := mid, modelID; got != want {
t.Fatalf("modelID = %q, want %q", got, want)
}
return &mineruTestDriver{}, "vlm", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = modelID
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "scansmpl.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "Cloud MinerU Title")
}
// TestDispatch_PDFMinerU_LanguageChain pins the lang_list chain
// (mineru_lang → setup lang → "English"), mirroring Python's
// mineru_parser.py:1181. The pdf setup's default lang ("Chinese") keeps the
// unconfigured form value on Python's ch code.
func TestDispatch_PDFMinerU_LanguageChain(t *testing.T) {
tests := []struct {
name string
overrides map[string]any
wantLang string
}{
{name: "default setup lang maps to ch", overrides: nil, wantLang: "ch"},
{name: "mineru_lang wins over setup lang", overrides: map[string]any{"mineru_lang": "English"}, wantLang: "en"},
{name: "setup lang used when mineru_lang unset", overrides: map[string]any{"lang": "Japanese"}, wantLang: "japan"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
withSSRFBypass(t)
var gotLang string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
http.NotFound(w, r)
return
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
gotLang = r.FormValue("lang_list")
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"# Title\n\nBody.\n"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
}))
defer server.Close()
origResolve := resolveMinerUModelForDispatch
defer func() { resolveMinerUModelForDispatch = origResolve }()
baseURL := server.URL
apiKey := "mineru-secret"
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
return &mineruTestDriver{}, "vlm", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
for k, v := range tc.overrides {
setups["pdf"][k] = v
}
c := &ParserComponent{setups: setups}
if _, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "scansmpl.pdf",
"tenant_id": "test-tenant",
}); err != nil {
t.Fatalf("Invoke: %v", err)
}
if gotLang != tc.wantLang {
t.Errorf("lang_list = %q, want %q", gotLang, tc.wantLang)
}
})
}
}
// TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer pins the same
// bare-UUID routing when the model id lands in layout_recognizer while
// parse_method stays a named method: the layout selector drives the probe and
// the named parse method never reaches the MinerU API form.
func TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
http.NotFound(w, r)
return
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if got, want := r.FormValue("parse_method"), "auto"; got != want {
t.Errorf("parse_method form value = %q, want %q", got, want)
}
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write([]byte(`[{"type":"text","text":"# Cloud MinerU Title\n\nCloud body.\n"}]`))
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
}))
defer server.Close()
modelID := "061436bbd8474d54be5bba3efbeac109"
origProbe := isMinerULayoutModelID
origResolve := resolveMinerUModelForDispatch
defer func() {
isMinerULayoutModelID = origProbe
resolveMinerUModelForDispatch = origResolve
}()
isMinerULayoutModelID = func(ctx context.Context, db *gorm.DB, tenantID, selector string) bool {
if got, want := selector, modelID; got != want {
t.Fatalf("selector = %q, want %q", got, want)
}
return true
}
baseURL := server.URL
apiKey := "mineru-secret"
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
if got, want := mid, modelID; got != want {
t.Fatalf("modelID = %q, want %q", got, want)
}
return &mineruTestDriver{}, "vlm", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "deepdoc"
setups["pdf"]["layout_recognizer"] = modelID
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "scansmpl.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "Cloud MinerU Title")
}
func writeMinerUZipResponse(w http.ResponseWriter, markdown string) {
payload, _ := json.Marshal([]map[string]string{{"type": "text", "text": markdown}})
buf := new(bytes.Buffer)
zw := zip.NewWriter(buf)
f, _ := zw.Create("content_list.json")
_, _ = f.Write(payload)
_ = zw.Close()
w.Header().Set("Content-Type", "application/zip")
_, _ = w.Write(buf.Bytes())
}
// setupMinerUVisionDispatchDB seeds an in-memory provider -> instance -> OCR
// model chain for the MinerU provider so the production dispatch probe and
// resolver run against tenant data instead of stubs.
func setupMinerUVisionDispatchDB(t *testing.T, baseURL string) (*gorm.DB, string) {
t.Helper()
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(&entity.UserTenant{}, &entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{}); err != nil {
t.Fatalf("migrate model tables: %v", err)
}
modelID := "061436bbd8474d54be5bba3efbeac109"
rows := []any{
&entity.TenantModelProvider{ID: "provider-mineru-1", UserID: "test-tenant", TenantID: "test-tenant", ProviderName: "MinerU"},
&entity.TenantModelInstance{ID: "instance-mineru-1", ProviderID: "provider-mineru-1", InstanceName: "default", APIKey: "mineru-secret", Status: "active", Extra: `{"base_url":"` + baseURL + `"}`},
&entity.TenantModel{ID: modelID, ProviderID: "provider-mineru-1", InstanceID: "instance-mineru-1", ModelName: "MinerU-model", ModelType: int(entity.ModelTypeOCR), Status: "active"},
}
for _, row := range rows {
if err := db.Create(row).Error; err != nil {
t.Fatalf("seed %T: %v", row, err)
}
}
return db, modelID
}
// newMinerUVisionTestServer serves the MinerU /file_parse endpoint, asserting
// the tenant instance credentials and API parse method posted by the
// dispatch.
func newMinerUVisionTestServer(t *testing.T, wantParseMethod string) *httptest.Server {
t.Helper()
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
http.NotFound(w, r)
return
}
if err := r.ParseMultipartForm(1 << 20); err != nil {
t.Errorf("parse form: %v", err)
return
}
if got, want := r.Header.Get("Authorization"), "Bearer mineru-secret"; got != want {
t.Errorf("Authorization = %q, want %q", got, want)
}
if wantParseMethod != "" {
if got := r.FormValue("parse_method"); got != wantParseMethod {
t.Errorf("parse_method form value = %q, want %q", got, wantParseMethod)
}
}
writeMinerUZipResponse(w, "# Real Resolver Title\n\nReal body.\n")
}))
}
func invokeMinerUDispatch(t *testing.T, db *gorm.DB, setups map[string]schema.ParserSetup) map[string]any {
t.Helper()
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), db, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
"tenant_id": "test-tenant",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
return out
}
// TestDispatch_PDFMinerU_HonorsMinerUParseMethodOption pins the dedicated
// mineru_parse_method parser option flowing to the MinerU /file_parse form:
// parse_method stays the dispatch selector while the API method comes from
// mineru_parse_method, mirroring Python's mineru_parser.py.
func TestDispatch_PDFMinerU_HonorsMinerUParseMethodOption(t *testing.T) {
withSSRFBypass(t)
server := newMinerUVisionTestServer(t, "ocr")
defer server.Close()
origResolve := resolveMinerUModelForDispatch
defer func() { resolveMinerUModelForDispatch = origResolve }()
baseURL := server.URL
apiKey := "mineru-secret"
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
if mid != "" {
t.Fatalf("modelID = %q, want empty for the named selector", mid)
}
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
setups["pdf"]["mineru_parse_method"] = "ocr"
out := invokeMinerUDispatch(t, nil, setups)
requireJSONText(t, out, "Real Resolver Title")
}
// TestDispatch_PDFMinerU_CompositeSelectorInParseMethod pins composite
// model@instance@provider selectors naming MinerU in parse_method: the
// dispatch matches the provider suffix and forwards the full selector to
// the model resolver.
func TestDispatch_PDFMinerU_CompositeSelectorInParseMethod(t *testing.T) {
withSSRFBypass(t)
server := newMinerUVisionTestServer(t, "auto")
defer server.Close()
selector := "MinerU-model@default@MinerU"
origResolve := resolveMinerUModelForDispatch
defer func() { resolveMinerUModelForDispatch = origResolve }()
baseURL := server.URL
apiKey := "mineru-secret"
resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
if got, want := mid, selector; got != want {
t.Fatalf("modelID = %q, want %q", got, want)
}
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
}
setups := defaultSetups()
setups["pdf"]["parse_method"] = selector
out := invokeMinerUDispatch(t, nil, setups)
requireJSONText(t, out, "Real Resolver Title")
}
// TestDispatch_PDFMinerU_NamedSelector_ResolvesMinerUProviderModel runs the
// named "mineru" dispatch with the production probe and resolver against a
// seeded tenant MinerU provider, pinning provider -> instance -> model
// resolution and the instance credentials reaching the MinerU API.
func TestDispatch_PDFMinerU_NamedSelector_ResolvesMinerUProviderModel(t *testing.T) {
withSSRFBypass(t)
server := newMinerUVisionTestServer(t, "auto")
defer server.Close()
db, _ := setupMinerUVisionDispatchDB(t, server.URL)
setups := defaultSetups()
setups["pdf"]["parse_method"] = "mineru"
out := invokeMinerUDispatch(t, db, setups)
requireJSONText(t, out, "Real Resolver Title")
}
// TestDispatch_PDFMinerU_BareModelUUID_ResolvesThroughRealProbeAndResolver
// runs the bare model UUID dispatch with the production probe and resolver:
// the UUID resolves to the seeded MinerU OCR model and the exact instance
// credentials reach the MinerU API.
func TestDispatch_PDFMinerU_BareModelUUID_ResolvesThroughRealProbeAndResolver(t *testing.T) {
withSSRFBypass(t)
server := newMinerUVisionTestServer(t, "auto")
defer server.Close()
db, modelID := setupMinerUVisionDispatchDB(t, server.URL)
setups := defaultSetups()
setups["pdf"]["parse_method"] = modelID
out := invokeMinerUDispatch(t, db, setups)
requireJSONText(t, out, "Real Resolver Title")
}
// TestDispatch_PDFMinerU_CompositeSelector_ResolvesThroughRealResolver runs
// the composite model@instance@provider dispatch (layout_recognizer from the
// web model picker, parse_method untouched) with the production resolver
// against the seeded tenant model chain.
func TestDispatch_PDFMinerU_CompositeSelector_ResolvesThroughRealResolver(t *testing.T) {
withSSRFBypass(t)
server := newMinerUVisionTestServer(t, "auto")
defer server.Close()
db, _ := setupMinerUVisionDispatchDB(t, server.URL)
setups := defaultSetups()
setups["pdf"]["parse_method"] = "deepdoc"
setups["pdf"]["layout_recognizer"] = "MinerU-model@default@MinerU"
out := invokeMinerUDispatch(t, db, setups)
requireJSONText(t, out, "Real Resolver Title")
}
// TestResolveTenantOCRModelByProvider_MinerUProviderAlias pins the sibling
// provider spelling tolerance: a tenant that configured only the remote
// "MinerU.Net" provider still resolves through the canonical "MinerU" name.
func TestResolveTenantOCRModelByProvider_MinerUProviderAlias(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
if err := db.AutoMigrate(&entity.UserTenant{}, &entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{}); err != nil {
t.Fatalf("migrate model tables: %v", err)
}
rows := []any{
&entity.TenantModelProvider{ID: "provider-minerunet-1", UserID: "test-tenant", TenantID: "test-tenant", ProviderName: "MinerU.Net"},
&entity.TenantModelInstance{ID: "instance-minerunet-1", ProviderID: "provider-minerunet-1", InstanceName: "default", APIKey: "mineru-secret", Status: "active", Extra: "{}"},
&entity.TenantModel{ID: "model-minerunet-1", ProviderID: "provider-minerunet-1", InstanceID: "instance-minerunet-1", ModelName: "MinerU-model", ModelType: int(entity.ModelTypeOCR), Status: "active"},
}
for _, row := range rows {
if err := db.Create(row).Error; err != nil {
t.Fatalf("seed %T: %v", row, err)
}
}
driver, modelName, _, _, err := resolveTenantOCRModelByProvider(t.Context(), db, "test-tenant", "MinerU")
if err != nil {
t.Fatalf("resolveTenantOCRModelByProvider: %v", err)
}
if got, want := modelName, "MinerU-model"; got != want {
t.Fatalf("modelName = %q, want %q", got, want)
}
if driver == nil || !isMinerUDriver(driver) {
t.Fatalf("driver = %v, want a MinerU driver", driver)
}
}
func TestIsPaddleOCRDriver(t *testing.T) {
for _, tc := range []struct {
name string
d models.ModelDriver
want bool
}{
{"local", &models.PaddleOCRLocalModel{}, true},
{"remote", &models.PaddleOCRModel{}, true},
{"dummy", &models.DummyModel{}, false},
} {
if got := isPaddleOCRDriver(tc.d); got != tc.want {
t.Errorf("isPaddleOCRDriver(%s) = %v, want %v", tc.name, got, tc.want)
}
}
}
func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {
withSSRFBypass(t)
var requestCount int
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requestCount++
if got, want := r.Header.Get("Authorization"), "Bearer doc-secret"; got != want {
t.Errorf("Authorization = %q, want %q", got, want)
return
}
switch {
// The parser tries standard conversions first (they carry page
// provenance) and falls back to chunked endpoints only when the
// server rejects the standard shape, so the third request is the
// chunked /v1 call and answers with a chunk list.
case r.Method == http.MethodPost && r.URL.Path == "/v1/convert/source" && requestCount == 1:
w.WriteHeader(http.StatusUnprocessableEntity)
_, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1alpha/convert/source" && requestCount == 2:
w.WriteHeader(http.StatusUnprocessableEntity)
_, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/convert/source" && requestCount == 3:
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`[{"text":"# Docling Title\n\nDocling body.\n"}]`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "Docling"
setups["pdf"]["docling_server_url"] = server.URL
setups["pdf"]["docling_api_key"] = "doc-secret"
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
requireJSONText(t, out, "Docling Title")
if got, want := requestCount, 3; got != want {
t.Fatalf("requestCount = %d, want %d", got, want)
}
}
func TestDispatch_PDFOpenDataLoaderLegacyMarkdownEmitsJSON(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"json_doc":null,"md_text":"# ODL Title\n\nODL body.\n"}`))
}))
defer server.Close()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "OpenDataLoader"
setups["pdf"]["opendataloader_apiserver"] = server.URL
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "ODL Title")
}
func TestDispatch_PDFSoMarkLegacyMarkdownEmitsJSON(t *testing.T) {
withSSRFBypass(t)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/parse/async":
_, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-4"}}`))
case "/parse/async_check":
_, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"blocks":[{"type":"title","content":"SoMark Title","title_level":1},{"type":"text","content":"Body"}]}]}}}}}`))
default:
http.NotFound(w, r)
}
}))
defer server.Close()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "SoMark"
setups["pdf"]["somark_base_url"] = server.URL
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "SoMark Title")
}
func TestDispatch_PDFTCADPLegacyMarkdownEmitsJSON(t *testing.T) {
withSSRFBypass(t)
zipPayload := tcadpZipFixtureForComponent(t)
var server *httptest.Server
server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/reconstruct_document":
_, _ = w.Write([]byte(`{"DocumentRecognizeResultUrl":"` + server.URL + `/download.zip"}`))
case "/download.zip":
_, _ = w.Write(zipPayload)
default:
http.NotFound(w, r)
}
}))
defer server.Close()
setups := defaultSetups()
setups["pdf"]["parse_method"] = "TCADP parser"
setups["pdf"]["tcadp_apiserver"] = server.URL
c := &ParserComponent{setups: setups}
out, err := c.Invoke(t.Context(), nil, map[string]any{
"binary": []byte("%PDF-1.4"),
"file_type": "pdf",
"name": "sample.pdf",
})
if err != nil {
t.Fatalf("Invoke: %v", err)
}
requireJSONText(t, out, "Hello TCADP")
}
func tcadpZipFixtureForComponent(t *testing.T) []byte {
t.Helper()
var buf bytes.Buffer
writer := zip.NewWriter(&buf)
f1, err := writer.Create("result.md")
if err != nil {
t.Fatalf("Create md: %v", err)
}
_, _ = f1.Write([]byte("Hello TCADP"))
if err := writer.Close(); err != nil {
t.Fatalf("Close zip: %v", err)
}
return buf.Bytes()
}
func TestParserFileFamilyPresentationExtensions(t *testing.T) {
for _, ext := range []string{"ppt", "pptx", "pages", "dps", "dpt"} {
if got := ParserFileFamily(ext); got != "slides" {
t.Errorf("ParserFileFamily(%q) = %q, want slides", ext, got)
}
}
}
// TestPythonFamilyName_FileTypeConstants pins the contract that every
// utility.FileType semantic constant resolves to the setups key used by
// defaultSetups. Before the fix, FileTypeVISUAL mapped
// to "picture" (mismatching the "image" setups key) and FileTypeAURAL matched
// no case at all (returning ""), so output_format validation and
// configureParserFromSetups were silently skipped for image/audio files.
func TestPythonFamilyName_FileTypeConstants(t *testing.T) {
cases := []struct {
ft utility.FileType
want string
}{
{utility.FileTypePDF, "pdf"},
{utility.FileTypeDOC, "doc"},
{utility.FileTypeDOCX, "docx"},
{utility.FileTypePPT, "slides"},
{utility.FileTypePPTX, "slides"},
{utility.FileTypeXLS, "spreadsheet"},
{utility.FileTypeXLSX, "spreadsheet"},
{utility.FileTypeCSV, "spreadsheet"},
{utility.FileTypeHTML, "html"},
{utility.FileTypeMarkdown, "markdown"},
{utility.FileTypeTXT, "text&code"},
{utility.FileTypeEPUB, "epub"},
{utility.FileTypeJSON, "json"},
{utility.FileTypeVISUAL, "image"},
{utility.FileTypeAURAL, "audio"},
{utility.FileTypeVIDEO, "video"},
{utility.FileTypeEMAIL, "email"},
}
for _, c := range cases {
got := pythonFamilyName(string(c.ft))
if got != c.want {
t.Errorf("pythonFamilyName(%q) = %q, want %q", c.ft, got, c.want)
}
// Every mapped family must have a matching setup so its parser receives
// the canonical output format and family-specific configuration.
if _, ok := defaultSetups()[got]; !ok {
t.Errorf("pythonFamilyName(%q) → %q has no defaultSetups entry", c.ft, got)
}
}
}
// TestConfigureParserFromSetups_Visual pins that image files pick up their
// setup (parse_method / lang / vlm) via the family mapping. Before the fix,
// configureParserFromSetups silently skipped configuration because
// resolveParserFamily returned "picture", which did not exist in
// defaultSetups.
func TestConfigureParserFromSetups_Visual(t *testing.T) {
setups := defaultSetups()
t.Run("visual resolves to image setup", func(t *testing.T) {
got := &captureSetupConfigurer{}
configureParserFromSetups(got, utility.FileTypeVISUAL, setups)
want := map[string]any(setups["image"])
if got.setup == nil {
t.Fatal("ConfigureFromSetup not called for FileTypeVISUAL")
}
if v, _ := got.setup["parse_method"].(string); v != want["parse_method"] {
t.Errorf("FileTypeVISUAL parse_method = %v, want %v", v, want["parse_method"])
}
})
}