Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): extract shared parseWithTCADP core from pptx/xls/pdf wrappers (#17671)

Taranum Wasu28ce77fe5f1e
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 8505fb90ba..11297a8661 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -110,8 +110,8 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
method := normalizeXLSXParseMethod(p.ParseMethod)
switch method {
case "tcadp":
- return parseSpreadsheetWithTCADP(
- filename, data, "CSV",
+ return parseWithTCADP(
+ ctx, filename, data, "CSV",
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
p.OutputFormat,
diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.go
index a9050b1ffe..5cca9141a8 100644
--- a/internal/parser/parser/pdf_parser_tcadp.go
+++ b/internal/parser/parser/pdf_parser_tcadp.go
@@ -4,87 +4,20 @@ import (
"archive/zip"
"bytes"
"context"
- "encoding/base64"
"encoding/json"
"fmt"
"io"
- "net/http"
- "ragflow/internal/common"
"strings"
-
- models "ragflow/internal/entity/models"
)
+// parsePDFWithTCADP sends PDF binary data to the TCADP cloud reconstruction
+// service. Thin wrapper over the shared parseWithTCADP core — env-fallbacks
+// and request construction live there.
func parsePDFWithTCADP(filename string, data []byte, parser *PDFParser) ParseResult {
- if len(data) == 0 {
- return emptyPDFResult(filename)
- }
- baseURL := strings.TrimSpace(parser.TCADPAPIServer)
- if baseURL == "" {
- baseURL = strings.TrimSpace(common.GetEnv(common.EnvTCADPAPIServerURL))
- }
- if baseURL == "" {
- return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")}
- }
- apiKey := strings.TrimSpace(parser.TCADPAPIKey)
- if apiKey == "" {
- apiKey = strings.TrimSpace(common.GetEnv(common.EnvTCADPAPIKey))
- }
- requestBody := map[string]any{
- "file_type": "PDF",
- "file_base64": base64.StdEncoding.EncodeToString(data),
- "file_start_page_number": 1,
- "file_end_page_number": 1000,
- "config": map[string]any{
- "TableResultType": parser.TCADPTableResultType,
- "MarkdownImageResponseType": parser.TCADPMarkdownImageResponseType,
- },
- }
- resp, err := models.PostJSONRequest(context.Background(), models.NewDriverHTTPClient(false), strings.TrimRight(baseURL, "/")+"/reconstruct_document", bearer(apiKey), requestBody)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP submit: %w", err)}
- }
- defer resp.Body.Close()
- raw, err := io.ReadAll(resp.Body)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP read submit: %w", err)}
- }
- if resp.StatusCode >= 300 {
- return ParseResult{Err: fmt.Errorf("parser: TCADP HTTP %d: %s", resp.StatusCode, string(raw))}
- }
- var payload struct {
- DocumentRecognizeResultURL string `json:"DocumentRecognizeResultUrl"`
- }
- if err := json.Unmarshal(raw, &payload); err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP decode submit: %w", err)}
- }
- if payload.DocumentRecognizeResultURL == "" {
- return ParseResult{Err: fmt.Errorf("parser: TCADP returned no DocumentRecognizeResultUrl")}
- }
- downloadReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, payload.DocumentRecognizeResultURL, nil)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP download request: %w", err)}
- }
- if auth := bearer(apiKey); auth != "" {
- downloadReq.Header.Set("Authorization", auth)
- }
- downloadResp, err := models.NewDriverHTTPClient(false).Do(downloadReq)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP download: %w", err)}
- }
- defer downloadResp.Body.Close()
- zipBytes, err := io.ReadAll(downloadResp.Body)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP read zip: %w", err)}
- }
- if downloadResp.StatusCode >= 300 {
- return ParseResult{Err: fmt.Errorf("parser: TCADP download HTTP %d: %s", downloadResp.StatusCode, string(zipBytes))}
- }
- items, pageCount, err := tcadpItemsFromZip(zipBytes)
- if err != nil {
- return ParseResult{Err: err}
- }
- return pdfItemsToResult(filename, items, parser.OutputFormat, pageCount)
+ return parseWithTCADP(context.Background(), filename, data, "PDF",
+ parser.TCADPAPIServer, parser.TCADPAPIKey,
+ parser.TCADPTableResultType, parser.TCADPMarkdownImageResponseType,
+ parser.OutputFormat)
}
func tcadpItemsFromZip(zipBytes []byte) ([]map[string]any, int, error) {
diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.go
index a8d50ab005..9665f814b8 100644
--- a/internal/parser/parser/pptx_parser.go
+++ b/internal/parser/parser/pptx_parser.go
@@ -95,7 +95,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
method := strings.ToLower(strings.TrimSpace(p.ParseMethod))
switch method {
case "tcadp":
- return parsePresentationWithTCADP(ctx,
+ return parseWithTCADP(ctx,
filename, data, strings.ToUpper(p.format),
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
diff --git a/internal/parser/parser/pptx_tcadp.go b/internal/parser/parser/tcadp_common.go
similarity index 65%
rename from internal/parser/parser/pptx_tcadp.go
rename to internal/parser/parser/tcadp_common.go
index ad97d0a4c8..9b2dad9bab 100644
--- a/internal/parser/parser/pptx_tcadp.go
+++ b/internal/parser/parser/tcadp_common.go
@@ -13,23 +13,48 @@ import (
models "ragflow/internal/entity/models"
)
-// parsePresentationWithTCADP sends binary presentation (PPTX/PPT) data
-// to the TCADP cloud reconstruction service and returns the structured
-// parse result. Mirrors the spreadsheet-family parseSpreadsheetWithTCADP
-// in xls_tcadp.go
-func parsePresentationWithTCADP(ctx context.Context, filename string, data []byte, fileType string,
- tcadpAPIServer, tcadpAPIKey, tableResultType, markdownImageResponseType string,
+// parseWithTCADP submits binary data to the TCADP cloud reconstruction service
+// and returns the structured parse result.
+//
+// Shared by all three TCADP-using parser families (PDF, spreadsheet,
+// presentation). Each family wraps this with its own thin function that
+// supplies its parameters (env-fallbacks, fileType, table/format config)
+// so the TCADP API contract lives in one place.
+
+// tcadpAPIBaseURL resolves the TCADP base URL using the order:
+// 1. explicit caller argument (already trimmed)
+// 2. legacy TCADP_APISERVER_URL env var (kept for backward compat with
+// the old parsePDFWithTCADP helper that honoured it)
+// 3. canonical TCADP_APISERVER env var (used by the other two families
+// and the error message)
+//
+// Extracted as a helper so unit tests can exercise the env-fallback
+// ordering without spinning up the full parse flow.
+func tcadpAPIBaseURL(explicit string) string {
+ if v := strings.TrimSpace(explicit); v != "" {
+ return v
+ }
+ if v := strings.TrimSpace(os.Getenv("TCADP_APISERVER_URL")); v != "" {
+ return v
+ }
+ return strings.TrimSpace(os.Getenv("TCADP_APISERVER"))
+}
+
+func parseWithTCADP(
+ ctx context.Context,
+ filename string,
+ data []byte,
+ fileType string,
+ tcadpAPIServer, tcadpAPIKey string,
+ tableResultType, markdownImageResponseType string,
outputFormat string,
) ParseResult {
if len(data) == 0 {
return emptyPDFResult(filename)
}
- baseURL := strings.TrimSpace(tcadpAPIServer)
- if baseURL == "" {
- baseURL = strings.TrimSpace(os.Getenv("TCADP_APISERVER"))
- }
+ baseURL := tcadpAPIBaseURL(tcadpAPIServer)
if baseURL == "" {
- return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")}
+ return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver, TCADP_APISERVER_URL, or TCADP_APISERVER")}
}
apiKey := strings.TrimSpace(tcadpAPIKey)
if apiKey == "" {
diff --git a/internal/parser/parser/tcadp_common_test.go b/internal/parser/parser/tcadp_common_test.go
new file mode 100644
index 0000000000..af83d9aa05
--- /dev/null
+++ b/internal/parser/parser/tcadp_common_test.go
@@ -0,0 +1,55 @@
+package parser
+
+import (
+ "os"
+ "testing"
+)
+
+// TestParseWithTCADPReadsLegacyTCADPAPISERVERURLEnvVar guards against
+// the silent breaking change where the shared parseWithTCADP core
+// dropped the legacy TCADP_APISERVER_URL env var that the old
+// parsePDFWithTCADP wrapper honoured. Existing PDF deployments set
+// this env var; we must keep reading it for backward compat.
+func TestParseWithTCADPReadsLegacyTCADPAPISERVERURLEnvVar(t *testing.T) {
+ t.Setenv("TCADP_APISERVER_URL", "https://legacy.tcadp.example.com")
+ t.Setenv("TCADP_APISERVER", "")
+
+ if got := tcadpAPIBaseURL(""); got != "https://legacy.tcadp.example.com" {
+ t.Fatalf("expected tcadpAPIBaseURL to honor TCADP_APISERVER_URL fallback, got %q", got)
+ }
+}
+
+// TestParseWithTCADPPrefersExplicitArgumentOverEnvVar pins the order:
+// an explicit tcadpAPIServer argument wins over both env vars.
+func TestParseWithTCADPPrefersExplicitArgumentOverEnvVar(t *testing.T) {
+ t.Setenv("TCADP_APISERVER", "https://env.tcadp.example.com")
+ t.Setenv("TCADP_APISERVER_URL", "https://legacy.tcadp.example.com")
+
+ if got := tcadpAPIBaseURL("https://explicit.tcadp.example.com"); got != "https://explicit.tcadp.example.com" {
+ t.Fatalf("explicit arg must win over env vars, got %q", got)
+ }
+}
+
+// TestParseWithTCADPFallsBackToCanonicalTCADPAPISERVEREnvVar verifies
+// the canonical env var works when the legacy one is unset.
+func TestParseWithTCADPFallsBackToCanonicalTCADPAPISERVEREnvVar(t *testing.T) {
+ t.Setenv("TCADP_APISERVER", "https://canonical.tcadp.example.com")
+ t.Setenv("TCADP_APISERVER_URL", "")
+
+ if got := tcadpAPIBaseURL(""); got != "https://canonical.tcadp.example.com" {
+ t.Fatalf("expected tcadpAPIBaseURL to honor TCADP_APISERVER fallback, got %q", got)
+ }
+}
+
+// TestParseWithTCADPReturnsEmptyWhenNoEnvVarOrArgSet guards against a
+// regression where the fallback ordering ever leaks the literal string
+// "" into the parse flow (which would surface as a malformed-URL error
+// later instead of the helpful "configure tcadp_apiserver" error).
+func TestParseWithTCADPReturnsEmptyWhenNoEnvVarOrArgSet(t *testing.T) {
+ os.Unsetenv("TCADP_APISERVER_URL")
+ os.Unsetenv("TCADP_APISERVER")
+
+ if got := tcadpAPIBaseURL(""); got != "" {
+ t.Fatalf("expected empty result when no env var and no arg, got %q", got)
+ }
+}
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index 02a34f9bfa..84f9610291 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -81,8 +81,8 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
method := normalizeXLSXParseMethod(p.ParseMethod)
switch method {
case "tcadp":
- return parseSpreadsheetWithTCADP(
- filename, data, "XLS",
+ return parseWithTCADP(
+ ctx, filename, data, "XLS",
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
p.OutputFormat,
diff --git a/internal/parser/parser/xls_tcadp.go b/internal/parser/parser/xls_tcadp.go
deleted file mode 100644
index 9468450161..0000000000
--- a/internal/parser/parser/xls_tcadp.go
+++ /dev/null
@@ -1,86 +0,0 @@
-package parser
-
-import (
- "context"
- "encoding/base64"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "os"
- "strings"
-
- models "ragflow/internal/entity/models"
-)
-
-func parseSpreadsheetWithTCADP(filename string, data []byte, fileType string, tcadpAPIServer, tcadpAPIKey, tableResultType, markdownImageResponseType string, outputFormat string) ParseResult {
- if len(data) == 0 {
- return emptyPDFResult(filename)
- }
- baseURL := strings.TrimSpace(tcadpAPIServer)
- if baseURL == "" {
- baseURL = strings.TrimSpace(os.Getenv("TCADP_APISERVER"))
- }
- if baseURL == "" {
- return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")}
- }
- apiKey := strings.TrimSpace(tcadpAPIKey)
- if apiKey == "" {
- apiKey = strings.TrimSpace(os.Getenv("TCADP_API_KEY"))
- }
- requestBody := map[string]any{
- "file_type": fileType,
- "file_base64": base64.StdEncoding.EncodeToString(data),
- "file_start_page_number": 1,
- "file_end_page_number": 1000,
- "config": map[string]any{
- "TableResultType": tableResultType,
- "MarkdownImageResponseType": markdownImageResponseType,
- },
- }
- resp, err := models.PostJSONRequest(context.Background(), models.NewDriverHTTPClient(false), strings.TrimRight(baseURL, "/")+"/reconstruct_document", bearer(apiKey), requestBody)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP submit: %w", err)}
- }
- defer resp.Body.Close()
- raw, err := io.ReadAll(resp.Body)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP read submit: %w", err)}
- }
- if resp.StatusCode >= 300 {
- return ParseResult{Err: fmt.Errorf("parser: TCADP HTTP %d: %s", resp.StatusCode, string(raw))}
- }
- var payload struct {
- DocumentRecognizeResultURL string `json:"DocumentRecognizeResultUrl"`
- }
- if err := json.Unmarshal(raw, &payload); err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP decode submit: %w", err)}
- }
- if payload.DocumentRecognizeResultURL == "" {
- return ParseResult{Err: fmt.Errorf("parser: TCADP returned no DocumentRecognizeResultUrl")}
- }
- downloadReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, payload.DocumentRecognizeResultURL, nil)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP download request: %w", err)}
- }
- if auth := bearer(apiKey); auth != "" {
- downloadReq.Header.Set("Authorization", auth)
- }
- downloadResp, err := models.NewDriverHTTPClient(false).Do(downloadReq)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP download: %w", err)}
- }
- defer downloadResp.Body.Close()
- zipBytes, err := io.ReadAll(downloadResp.Body)
- if err != nil {
- return ParseResult{Err: fmt.Errorf("parser: TCADP read zip: %w", err)}
- }
- if downloadResp.StatusCode >= 300 {
- return ParseResult{Err: fmt.Errorf("parser: TCADP download HTTP %d: %s", downloadResp.StatusCode, string(zipBytes))}
- }
- items, pageCount, err := tcadpItemsFromZip(zipBytes)
- if err != nil {
- return ParseResult{Err: err}
- }
- return pdfItemsToResult(filename, items, outputFormat, pageCount)
-}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index 02f698941c..b896a1ed37 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -98,8 +98,8 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
method := normalizeXLSXParseMethod(p.ParseMethod)
switch method {
case "tcadp":
- return parseSpreadsheetWithTCADP(
- filename, data, "XLSX",
+ return parseWithTCADP(
+ ctx, filename, data, "XLSX",
p.TCADPAPIServer, p.TCADPAPIKey,
p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,
p.OutputFormat,