Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): route bare MinerU OCR model UUID to MinerU PDF dispatch (#19639)

euvre522fe21ae011
diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.go
index 6560f9762f..0ebe28b4a7 100644
--- a/internal/ingestion/component/dispatch_model.go
+++ b/internal/ingestion/component/dispatch_model.go
@@ -111,11 +111,10 @@ func defaultResolveTenantOCRModelByProvider(ctx context.Context, db *gorm.DB, te
providerDAO := dao.NewTenantModelProviderDAO()
provider, err := providerDAO.GetByTenantIDAndProviderName(ctx, db, tenantID, providerName)
if err != nil {
- // Some OCR providers are registered under sibling names: the cloud
- // "PaddleOCR" provider and the local "PaddleOCR.local" provider expose
- // the same OCR capability. Tolerate the alternate spelling before
- // giving up so a tenant configured with either name resolves.
- for _, alias := range paddleOCRProviderAliases() {
+ // Some OCR capabilities are registered under sibling provider names
+ // (cloud vs. local). Tolerate the alternate spelling before giving up
+ // so a tenant configured with either name resolves.
+ for _, alias := range ocrProviderAliases(providerName) {
if alias == providerName {
continue
}
@@ -149,12 +148,21 @@ func defaultResolveTenantOCRModelByProvider(ctx context.Context, db *gorm.DB, te
return nil, "", nil, 0, fmt.Errorf("tenant %s has no active %s OCR model", tenantID, providerName)
}
-// paddleOCRProviderAliases returns the registered provider names that expose
-// PaddleOCR OCR models: the cloud "PaddleOCR" provider and the local
-// "PaddleOCR.local" provider. Both carry OCR-typed models and route through
-// the PaddleOCR PDF dispatch regardless of which spelling a tenant configured.
-func paddleOCRProviderAliases() []string {
- return []string{"PaddleOCR", "PaddleOCR.local"}
+// ocrProviderAliases returns the registered provider names that expose the
+// same OCR capability as providerName: the cloud "PaddleOCR" provider and
+// the local "PaddleOCR.local" provider are interchangeable, and so are the
+// local "MinerU" provider and the remote "MinerU.Net" provider. The PDF
+// dispatch resolves a tenant's OCR model regardless of which spelling was
+// configured.
+func ocrProviderAliases(providerName string) []string {
+ switch providerName {
+ case "PaddleOCR", "PaddleOCR.local":
+ return []string{"PaddleOCR", "PaddleOCR.local"}
+ case "MinerU", "MinerU.Net":
+ return []string{"MinerU", "MinerU.Net"}
+ default:
+ return []string{providerName}
+ }
}
func tenantModelIDByType(tenant *entity.Tenant, modelType entity.ModelType) string {
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 897e80220b..54158964e9 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -28,6 +28,7 @@ import (
"archive/zip"
"bytes"
"context"
+ "encoding/json"
"fmt"
"net"
"net/http"
@@ -44,9 +45,11 @@ import (
doctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
"ragflow/internal/entity/models"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
"ragflow/internal/utility"
+ "github.com/glebarez/sqlite"
"go.uber.org/zap"
"go.uber.org/zap/zaptest/observer"
"gorm.io/gorm"
@@ -676,12 +679,15 @@ func TestDispatch_PDFMinerUMarkdown_UsesConfiguredBackend(t *testing.T) {
}))
defer server.Close()
- // Mock resolveTenantModelByType to return a MinerU driver pointing at the test server.
- origResolver := resolveTenantModelByType
- defer func() { resolveTenantModelByType = origResolver }()
+ // Mock the MinerU provider resolver to return a driver pointing at the test server.
+ origResolver := resolveTenantOCRModelByProvider
+ defer func() { resolveTenantOCRModelByProvider = origResolver }()
baseURL := server.URL
apiKey := ""
- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ resolveTenantOCRModelByProvider = func(ctx context.Context, db *gorm.DB, tenantID string, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ if got, want := providerName, "MinerU"; got != want {
+ t.Fatalf("providerName = %q, want %q", got, want)
+ }
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
}
@@ -722,11 +728,14 @@ func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {
}))
defer server.Close()
- origResolver := resolveTenantModelByType
- defer func() { resolveTenantModelByType = origResolver }()
+ origResolver := resolveTenantOCRModelByProvider
+ defer func() { resolveTenantOCRModelByProvider = origResolver }()
baseURL := server.URL
apiKey := ""
- resolveTenantModelByType = func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ resolveTenantOCRModelByProvider = func(ctx context.Context, db *gorm.DB, tenantID string, providerName string) (models.ModelDriver, string, *models.APIConfig, int, error) {
+ if got, want := providerName, "MinerU"; got != want {
+ t.Fatalf("providerName = %q, want %q", got, want)
+ }
return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, 0, nil
}
@@ -1123,6 +1132,371 @@ func TestDispatch_PDFPaddleOCR_BareModelUUID_InParseMethod(t *testing.T) {
requireJSONText(t, out, "Cloud Paddle Title")
}
+// TestDispatch_PDFMinerU_BareModelUUID_InParseMethod pins the routing of a
+// bare tenant model UUID in parse_method — the value stored when a user picks
+// an OCR model for PDF parsing — to the MinerU dispatch path using that exact
+// model. The UUID carries no "@provider" hint, so it must be resolved before
+// the dispatch path is chosen instead of falling through to the image2text
+// VLM path.
+func TestDispatch_PDFMinerU_BareModelUUID_InParseMethod(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
+ http.NotFound(w, r)
+ return
+ }
+ if err := r.ParseMultipartForm(1 << 20); err != nil {
+ t.Errorf("parse form: %v", err)
+ return
+ }
+ if got, want := r.Header.Get("Authorization"), "Bearer mineru-secret"; got != want {
+ t.Errorf("Authorization = %q, want %q", got, want)
+ }
+ if got, want := r.FormValue("parse_method"), "auto"; got != want {
+ t.Errorf("parse_method form value = %q, want %q", got, want)
+ }
+ buf := new(bytes.Buffer)
+ zw := zip.NewWriter(buf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[{"type":"text","text":"# Cloud MinerU Title\n\nCloud body.\n"}]`))
+ _ = zw.Close()
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(buf.Bytes())
+ }))
+ defer server.Close()
+
+ modelID := "061436bbd8474d54be5bba3efbeac109"
+ origProbe := isMinerULayoutModelID
+ origResolve := resolveMinerUModelForDispatch
+ defer func() {
+ isMinerULayoutModelID = origProbe
+ resolveMinerUModelForDispatch = origResolve
+ }()
+ isMinerULayoutModelID = func(ctx context.Context, db *gorm.DB, tenantID, selector string) bool {
+ if got, want := selector, modelID; got != want {
+ t.Fatalf("selector = %q, want %q", got, want)
+ }
+ return true
+ }
+ baseURL := server.URL
+ apiKey := "mineru-secret"
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
+ if got, want := mid, modelID; got != want {
+ t.Fatalf("modelID = %q, want %q", got, want)
+ }
+ return &mineruTestDriver{}, "vlm", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = modelID
+ setups["pdf"]["output_format"] = "markdown"
+ c := &ParserComponent{setups: setups}
+
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "scansmpl.pdf",
+ "tenant_id": "test-tenant",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ requireJSONText(t, out, "Cloud MinerU Title")
+}
+
+// TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer pins the same
+// bare-UUID routing when the model id lands in layout_recognizer while
+// parse_method stays a named method: the layout selector drives the probe and
+// the named parse method never reaches the MinerU API form.
+func TestDispatch_PDFMinerU_BareModelUUID_InLayoutRecognizer(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
+ http.NotFound(w, r)
+ return
+ }
+ if err := r.ParseMultipartForm(1 << 20); err != nil {
+ t.Errorf("parse form: %v", err)
+ return
+ }
+ if got, want := r.FormValue("parse_method"), "auto"; got != want {
+ t.Errorf("parse_method form value = %q, want %q", got, want)
+ }
+ buf := new(bytes.Buffer)
+ zw := zip.NewWriter(buf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write([]byte(`[{"type":"text","text":"# Cloud MinerU Title\n\nCloud body.\n"}]`))
+ _ = zw.Close()
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(buf.Bytes())
+ }))
+ defer server.Close()
+
+ modelID := "061436bbd8474d54be5bba3efbeac109"
+ origProbe := isMinerULayoutModelID
+ origResolve := resolveMinerUModelForDispatch
+ defer func() {
+ isMinerULayoutModelID = origProbe
+ resolveMinerUModelForDispatch = origResolve
+ }()
+ isMinerULayoutModelID = func(ctx context.Context, db *gorm.DB, tenantID, selector string) bool {
+ if got, want := selector, modelID; got != want {
+ t.Fatalf("selector = %q, want %q", got, want)
+ }
+ return true
+ }
+ baseURL := server.URL
+ apiKey := "mineru-secret"
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
+ if got, want := mid, modelID; got != want {
+ t.Fatalf("modelID = %q, want %q", got, want)
+ }
+ return &mineruTestDriver{}, "vlm", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "deepdoc"
+ setups["pdf"]["layout_recognizer"] = modelID
+ setups["pdf"]["output_format"] = "markdown"
+ c := &ParserComponent{setups: setups}
+
+ out, err := c.Invoke(t.Context(), nil, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "scansmpl.pdf",
+ "tenant_id": "test-tenant",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ requireJSONText(t, out, "Cloud MinerU Title")
+}
+
+func writeMinerUZipResponse(w http.ResponseWriter, markdown string) {
+ payload, _ := json.Marshal([]map[string]string{{"type": "text", "text": markdown}})
+ buf := new(bytes.Buffer)
+ zw := zip.NewWriter(buf)
+ f, _ := zw.Create("content_list.json")
+ _, _ = f.Write(payload)
+ _ = zw.Close()
+ w.Header().Set("Content-Type", "application/zip")
+ _, _ = w.Write(buf.Bytes())
+}
+
+// setupMinerUVisionDispatchDB seeds an in-memory provider -> instance -> OCR
+// model chain for the MinerU provider so the production dispatch probe and
+// resolver run against tenant data instead of stubs.
+func setupMinerUVisionDispatchDB(t *testing.T, baseURL string) (*gorm.DB, string) {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{}); err != nil {
+ t.Fatalf("migrate model tables: %v", err)
+ }
+ modelID := "061436bbd8474d54be5bba3efbeac109"
+ rows := []any{
+ &entity.TenantModelProvider{ID: "provider-mineru-1", TenantID: "test-tenant", ProviderName: "MinerU"},
+ &entity.TenantModelInstance{ID: "instance-mineru-1", ProviderID: "provider-mineru-1", InstanceName: "default", APIKey: "mineru-secret", Status: "active", Extra: `{"base_url":"` + baseURL + `"}`},
+ &entity.TenantModel{ID: modelID, ProviderID: "provider-mineru-1", InstanceID: "instance-mineru-1", ModelName: "MinerU-model", ModelType: int(entity.ModelTypeOCR), Status: "active"},
+ }
+ for _, row := range rows {
+ if err := db.Create(row).Error; err != nil {
+ t.Fatalf("seed %T: %v", row, err)
+ }
+ }
+ return db, modelID
+}
+
+// newMinerUVisionTestServer serves the MinerU /file_parse endpoint, asserting
+// the tenant instance credentials and API parse method posted by the
+// dispatch.
+func newMinerUVisionTestServer(t *testing.T, wantParseMethod string) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost || r.URL.Path != "/file_parse" {
+ http.NotFound(w, r)
+ return
+ }
+ if err := r.ParseMultipartForm(1 << 20); err != nil {
+ t.Errorf("parse form: %v", err)
+ return
+ }
+ if got, want := r.Header.Get("Authorization"), "Bearer mineru-secret"; got != want {
+ t.Errorf("Authorization = %q, want %q", got, want)
+ }
+ if wantParseMethod != "" {
+ if got := r.FormValue("parse_method"); got != wantParseMethod {
+ t.Errorf("parse_method form value = %q, want %q", got, wantParseMethod)
+ }
+ }
+ writeMinerUZipResponse(w, "# Real Resolver Title\n\nReal body.\n")
+ }))
+}
+
+func invokeMinerUDispatch(t *testing.T, db *gorm.DB, setups map[string]schema.ParserSetup) map[string]any {
+ t.Helper()
+ c := &ParserComponent{setups: setups}
+ out, err := c.Invoke(t.Context(), db, map[string]any{
+ "binary": []byte("%PDF-1.4"),
+ "file_type": "pdf",
+ "name": "sample.pdf",
+ "tenant_id": "test-tenant",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ return out
+}
+
+// TestDispatch_PDFMinerU_HonorsMinerUParseMethodOption pins the dedicated
+// mineru_parse_method parser option flowing to the MinerU /file_parse form:
+// parse_method stays the dispatch selector while the API method comes from
+// mineru_parse_method, mirroring Python's mineru_parser.py.
+func TestDispatch_PDFMinerU_HonorsMinerUParseMethodOption(t *testing.T) {
+ withSSRFBypass(t)
+ server := newMinerUVisionTestServer(t, "ocr")
+ defer server.Close()
+
+ origResolve := resolveMinerUModelForDispatch
+ defer func() { resolveMinerUModelForDispatch = origResolve }()
+ baseURL := server.URL
+ apiKey := "mineru-secret"
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
+ if mid != "" {
+ t.Fatalf("modelID = %q, want empty for the named selector", mid)
+ }
+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ setups["pdf"]["mineru_parse_method"] = "ocr"
+ setups["pdf"]["output_format"] = "markdown"
+
+ out := invokeMinerUDispatch(t, nil, setups)
+ requireJSONText(t, out, "Real Resolver Title")
+}
+
+// TestDispatch_PDFMinerU_CompositeSelectorInParseMethod pins composite
+// model@instance@provider selectors naming MinerU in parse_method: the
+// dispatch matches the provider suffix and forwards the full selector to
+// the model resolver.
+func TestDispatch_PDFMinerU_CompositeSelectorInParseMethod(t *testing.T) {
+ withSSRFBypass(t)
+ server := newMinerUVisionTestServer(t, "auto")
+ defer server.Close()
+
+ selector := "MinerU-model@default@MinerU"
+ origResolve := resolveMinerUModelForDispatch
+ defer func() { resolveMinerUModelForDispatch = origResolve }()
+ baseURL := server.URL
+ apiKey := "mineru-secret"
+ resolveMinerUModelForDispatch = func(ctx context.Context, db *gorm.DB, tenantID, mid string) (models.ModelDriver, string, *models.APIConfig, error) {
+ if got, want := mid, selector; got != want {
+ t.Fatalf("modelID = %q, want %q", got, want)
+ }
+ return &mineruTestDriver{}, "mineru-model", &models.APIConfig{ApiKey: &apiKey, BaseURL: &baseURL}, nil
+ }
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = selector
+ setups["pdf"]["output_format"] = "markdown"
+
+ out := invokeMinerUDispatch(t, nil, setups)
+ requireJSONText(t, out, "Real Resolver Title")
+}
+
+// TestDispatch_PDFMinerU_NamedSelector_ResolvesMinerUProviderModel runs the
+// named "mineru" dispatch with the production probe and resolver against a
+// seeded tenant MinerU provider, pinning provider -> instance -> model
+// resolution and the instance credentials reaching the MinerU API.
+func TestDispatch_PDFMinerU_NamedSelector_ResolvesMinerUProviderModel(t *testing.T) {
+ withSSRFBypass(t)
+ server := newMinerUVisionTestServer(t, "auto")
+ defer server.Close()
+ db, _ := setupMinerUVisionDispatchDB(t, server.URL)
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "mineru"
+ setups["pdf"]["output_format"] = "markdown"
+
+ out := invokeMinerUDispatch(t, db, setups)
+ requireJSONText(t, out, "Real Resolver Title")
+}
+
+// TestDispatch_PDFMinerU_BareModelUUID_ResolvesThroughRealProbeAndResolver
+// runs the bare model UUID dispatch with the production probe and resolver:
+// the UUID resolves to the seeded MinerU OCR model and the exact instance
+// credentials reach the MinerU API.
+func TestDispatch_PDFMinerU_BareModelUUID_ResolvesThroughRealProbeAndResolver(t *testing.T) {
+ withSSRFBypass(t)
+ server := newMinerUVisionTestServer(t, "auto")
+ defer server.Close()
+ db, modelID := setupMinerUVisionDispatchDB(t, server.URL)
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = modelID
+ setups["pdf"]["output_format"] = "markdown"
+
+ out := invokeMinerUDispatch(t, db, setups)
+ requireJSONText(t, out, "Real Resolver Title")
+}
+
+// TestDispatch_PDFMinerU_CompositeSelector_ResolvesThroughRealResolver runs
+// the composite model@instance@provider dispatch (layout_recognizer from the
+// web model picker, parse_method untouched) with the production resolver
+// against the seeded tenant model chain.
+func TestDispatch_PDFMinerU_CompositeSelector_ResolvesThroughRealResolver(t *testing.T) {
+ withSSRFBypass(t)
+ server := newMinerUVisionTestServer(t, "auto")
+ defer server.Close()
+ db, _ := setupMinerUVisionDispatchDB(t, server.URL)
+
+ setups := defaultSetups()
+ setups["pdf"]["parse_method"] = "deepdoc"
+ setups["pdf"]["layout_recognizer"] = "MinerU-model@default@MinerU"
+ setups["pdf"]["output_format"] = "markdown"
+
+ out := invokeMinerUDispatch(t, db, setups)
+ requireJSONText(t, out, "Real Resolver Title")
+}
+
+// TestResolveTenantOCRModelByProvider_MinerUProviderAlias pins the sibling
+// provider spelling tolerance: a tenant that configured only the remote
+// "MinerU.Net" provider still resolves through the canonical "MinerU" name.
+func TestResolveTenantOCRModelByProvider_MinerUProviderAlias(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err := db.AutoMigrate(&entity.TenantModelProvider{}, &entity.TenantModelInstance{}, &entity.TenantModel{}); err != nil {
+ t.Fatalf("migrate model tables: %v", err)
+ }
+ rows := []any{
+ &entity.TenantModelProvider{ID: "provider-minerunet-1", TenantID: "test-tenant", ProviderName: "MinerU.Net"},
+ &entity.TenantModelInstance{ID: "instance-minerunet-1", ProviderID: "provider-minerunet-1", InstanceName: "default", APIKey: "mineru-secret", Status: "active", Extra: "{}"},
+ &entity.TenantModel{ID: "model-minerunet-1", ProviderID: "provider-minerunet-1", InstanceID: "instance-minerunet-1", ModelName: "MinerU-model", ModelType: int(entity.ModelTypeOCR), Status: "active"},
+ }
+ for _, row := range rows {
+ if err := db.Create(row).Error; err != nil {
+ t.Fatalf("seed %T: %v", row, err)
+ }
+ }
+
+ driver, modelName, _, _, err := resolveTenantOCRModelByProvider(t.Context(), db, "test-tenant", "MinerU")
+ if err != nil {
+ t.Fatalf("resolveTenantOCRModelByProvider: %v", err)
+ }
+ if got, want := modelName, "MinerU-model"; got != want {
+ t.Fatalf("modelName = %q, want %q", got, want)
+ }
+ if driver == nil || !isMinerUDriver(driver) {
+ t.Fatalf("driver = %v, want a MinerU driver", driver)
+ }
+}
+
func TestIsPaddleOCRDriver(t *testing.T) {
for _, tc := range []struct {
name string
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index 7276db99bf..872860c9e3 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -105,10 +105,25 @@ func maybeDispatchPDFVision(
return res, true, err
}
- // MinerU dispatch: parse_method "mineru" or layout_recognizer "@MinerU"
- if strings.EqualFold(strings.TrimSpace(method), "mineru") ||
+ // MinerU dispatch: parse_method "mineru", a layout_recognizer naming
+ // MinerU, a composite model@instance@provider selector naming MinerU in
+ // either field, or a bare tenant model UUID (in either field) that
+ // resolves to a MinerU OCR model.
+ methodLower := strings.ToLower(strings.TrimSpace(method))
+ isMinerUMatch := methodLower == "mineru" ||
strings.HasPrefix(layoutLower, "mineru") ||
- strings.Contains(layoutLower, "@mineru") {
+ strings.Contains(layoutLower, "@mineru") ||
+ strings.Contains(methodLower, "@mineru")
+ minerUSelector := layout
+ if strings.TrimSpace(minerUSelector) == "" {
+ minerUSelector = method
+ }
+ isMinerUByUUID := false
+ if !isMinerUMatch && strings.TrimSpace(minerUSelector) != "" &&
+ !isNamedPDFParseMethod(minerUSelector) {
+ isMinerUByUUID = isMinerULayoutModelID(ctx, db, tenantID, minerUSelector)
+ }
+ if isMinerUMatch || isMinerUByUUID {
common.Info("pdf vision dispatch: MinerU branch matched",
zap.String("parse_method", method),
zap.String("layout_recognizer", layout),
@@ -117,7 +132,11 @@ func maybeDispatchPDFVision(
return parser.ParseResult{}, true,
fmt.Errorf("parser: MinerU requires tenant_id")
}
- res, err := dispatchMinerUPDF(ctx, db, filename, binary, tenantID, setup)
+ dispatchModelID := ""
+ if isMinerUByUUID || strings.Contains(minerUSelector, "@") {
+ dispatchModelID = minerUSelector
+ }
+ res, err := dispatchMinerUPDF(ctx, db, filename, binary, tenantID, setup, dispatchModelID)
if err != nil {
return parser.ParseResult{}, true, err
}
@@ -473,7 +492,7 @@ func getAnyString(object map[string]any, keys ...string) string {
return ""
}
-// dispatchMinerUPDF submits a PDF to the tenant's MinerU OCR model
+// dispatchMinerUPDF submits a PDF to the selected MinerU OCR model
// via the streaming /file_parse endpoint and returns parsed sections.
// Mirrors Python's mineru_parser.py:parse_PDF which POSTs with
// stream=True and reads the zip response body directly (no polling).
@@ -484,8 +503,9 @@ func dispatchMinerUPDF(
binary []byte,
tenantID string,
setup schema.ParserSetup,
+ modelID string,
) (parser.ParseResult, error) {
- driver, _, apiConfig, _, err := resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeOCR)
+ driver, _, apiConfig, err := resolveMinerUModelForDispatch(ctx, db, tenantID, modelID)
if err != nil {
return parser.ParseResult{}, fmt.Errorf("parser: MinerU model: %w", err)
}
@@ -503,8 +523,7 @@ func dispatchMinerUPDF(
}
apiURL := strings.TrimRight(baseURL, "/") + "/file_parse"
- // Parse method: "raw", "auto", "ocr", "txt" matching Python's MinerUParseMethod.
- parseMethod := getStringOr(setup, "parse_method", "auto")
+ parseMethod := mineruAPIParseMethod(getStringOr(setup, "mineru_parse_method", ""))
lang := getStringOr(setup, "mineru_lang", "English")
mineruLang := mineruLangCode(lang)
backend := getStringOr(setup, "mineru_backend", "pipeline")
@@ -530,6 +549,29 @@ func dispatchMinerUPDF(
return buildMarkdownOCRDispatchResult(md), nil
}
+// resolveMinerUModelForDispatch resolves the OCR model used by the MinerU PDF
+// dispatch. modelID is the raw selector value: a bare tenant model UUID (no
+// "@") selects that exact model, a composite model@instance@provider name
+// resolves through the provider-instance chain, and an empty value falls back
+// to the tenant's first MinerU OCR model — mirroring Python's by_mineru,
+// which falls back to get_first_provider_model_name(tenant_id, "MinerU",
+// LLMType.OCR) for the named "mineru" selector.
+var resolveMinerUModelForDispatch = defaultResolveMinerUModelForDispatch
+
+func defaultResolveMinerUModelForDispatch(ctx context.Context, db *gorm.DB, tenantID, modelID string) (modelModule.ModelDriver, string, *modelModule.APIConfig, error) {
+ modelID = strings.TrimSpace(modelID)
+ if modelID != "" && !strings.Contains(modelID, "@") {
+ driver, modelName, apiConfig, _, err := resolveModelConfigByID(ctx, db, tenantID, entity.ModelTypeOCR, modelID)
+ return driver, modelName, apiConfig, err
+ }
+ if modelID != "" {
+ driver, modelName, apiConfig, _, err := resolveModelConfig(ctx, db, tenantID, entity.ModelTypeOCR, modelID)
+ return driver, modelName, apiConfig, err
+ }
+ driver, modelName, apiConfig, _, err := resolveTenantOCRModelByProvider(ctx, db, tenantID, "MinerU")
+ return driver, modelName, apiConfig, err
+}
+
// resolvePaddleOCRModelForDispatch resolves the OCR model used by the
// PaddleOCR PDF dispatch. modelID is the raw layout_recognizer value: a bare
// tenant model UUID (no "@") selects that exact model regardless of provider
@@ -627,6 +669,21 @@ func mineruLangCode(lang string) string {
}
}
+// mineruAPIParseMethod constrains the /file_parse form field to the values
+// the MinerU API accepts (auto, txt, ocr), mirroring Python's
+// MinerUParseMethod. The value comes from the parser setup's dedicated
+// mineru_parse_method option; the setup's parse_method is a dispatch
+// selector ("mineru", a composite selector, or a tenant model UUID), never
+// an API method, so anything unrecognized falls back to the API default.
+func mineruAPIParseMethod(raw string) string {
+ method := strings.ToLower(strings.TrimSpace(raw))
+ switch method {
+ case "auto", "txt", "ocr":
+ return method
+ }
+ return "auto"
+}
+
// mineruStreamParse POSTs the PDF binary to the MinerU /file_parse
// endpoint with streaming and returns the zip response body.
// Mirrors Python's mineru_parser.py._run_mineru_api with stream=True.
@@ -961,6 +1018,29 @@ func isMinerUDriver(driver modelModule.ModelDriver) bool {
return false
}
+// isMinerULayoutModelID reports whether selector — a bare tenant model UUID
+// with no "model@instance@provider" composite hint — resolves to an active
+// OCR model driven by a MinerU provider (remote mineru.net or local mineru).
+// The raw UUID carries no provider spelling, so it must be resolved before
+// the dispatch path is chosen instead of falling through to the image2text
+// VLM path.
+var isMinerULayoutModelID = defaultIsMinerULayoutModelID
+
+func defaultIsMinerULayoutModelID(ctx context.Context, db *gorm.DB, tenantID, selector string) bool {
+ selector = strings.TrimSpace(selector)
+ if db == nil {
+ return false
+ }
+ if selector == "" || strings.Contains(selector, "@") {
+ return false
+ }
+ driver, _, _, err := defaultResolveMinerUModelForDispatch(ctx, db, tenantID, selector)
+ if err != nil {
+ return false
+ }
+ return isMinerUDriver(driver)
+}
+
// isPaddleOCRDriver reports whether the model driver is a PaddleOCR variant
// (cloud "paddleocr" or local "paddleocr.local").
func isPaddleOCRDriver(driver modelModule.ModelDriver) bool {