RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): extract shared parseWithTCADP core from pptx/xls/pdf wrappers (#17671)
Taranum Wasu28ce77fe5f1e
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 8505fb90ba..11297a8661 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -110,8 +110,8 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [method := normalizeXLSXParseMethod(p.ParseMethod)switch method {case "tcadp":- return parseSpreadsheetWithTCADP(- filename, data, "CSV",+ return parseWithTCADP(+ ctx, filename, data, "CSV",p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,p.OutputFormat,diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.goindex a9050b1ffe..5cca9141a8 100644--- a/internal/parser/parser/pdf_parser_tcadp.go+++ b/internal/parser/parser/pdf_parser_tcadp.go@@ -4,87 +4,20 @@ import ("archive/zip""bytes""context"- "encoding/base64""encoding/json""fmt""io"- "net/http"- "ragflow/internal/common""strings"-- models "ragflow/internal/entity/models")+// parsePDFWithTCADP sends PDF binary data to the TCADP cloud reconstruction+// service. Thin wrapper over the shared parseWithTCADP core — env-fallbacks+// and request construction live there.func parsePDFWithTCADP(filename string, data []byte, parser *PDFParser) ParseResult {- if len(data) == 0 {- return emptyPDFResult(filename)- }- baseURL := strings.TrimSpace(parser.TCADPAPIServer)- if baseURL == "" {- baseURL = strings.TrimSpace(common.GetEnv(common.EnvTCADPAPIServerURL))- }- if baseURL == "" {- return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")}- }- apiKey := strings.TrimSpace(parser.TCADPAPIKey)- if apiKey == "" {- apiKey = strings.TrimSpace(common.GetEnv(common.EnvTCADPAPIKey))- }- requestBody := map[string]any{- "file_type": "PDF",- "file_base64": base64.StdEncoding.EncodeToString(data),- "file_start_page_number": 1,- "file_end_page_number": 1000,- "config": map[string]any{- "TableResultType": parser.TCADPTableResultType,- "MarkdownImageResponseType": parser.TCADPMarkdownImageResponseType,- },- }- resp, err := models.PostJSONRequest(context.Background(), models.NewDriverHTTPClient(false), strings.TrimRight(baseURL, "/")+"/reconstruct_document", bearer(apiKey), requestBody)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP submit: %w", err)}- }- defer resp.Body.Close()- raw, err := io.ReadAll(resp.Body)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP read submit: %w", err)}- }- if resp.StatusCode >= 300 {- return ParseResult{Err: fmt.Errorf("parser: TCADP HTTP %d: %s", resp.StatusCode, string(raw))}- }- var payload struct {- DocumentRecognizeResultURL string `json:"DocumentRecognizeResultUrl"`- }- if err := json.Unmarshal(raw, &payload); err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP decode submit: %w", err)}- }- if payload.DocumentRecognizeResultURL == "" {- return ParseResult{Err: fmt.Errorf("parser: TCADP returned no DocumentRecognizeResultUrl")}- }- downloadReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, payload.DocumentRecognizeResultURL, nil)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP download request: %w", err)}- }- if auth := bearer(apiKey); auth != "" {- downloadReq.Header.Set("Authorization", auth)- }- downloadResp, err := models.NewDriverHTTPClient(false).Do(downloadReq)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP download: %w", err)}- }- defer downloadResp.Body.Close()- zipBytes, err := io.ReadAll(downloadResp.Body)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP read zip: %w", err)}- }- if downloadResp.StatusCode >= 300 {- return ParseResult{Err: fmt.Errorf("parser: TCADP download HTTP %d: %s", downloadResp.StatusCode, string(zipBytes))}- }- items, pageCount, err := tcadpItemsFromZip(zipBytes)- if err != nil {- return ParseResult{Err: err}- }- return pdfItemsToResult(filename, items, parser.OutputFormat, pageCount)+ return parseWithTCADP(context.Background(), filename, data, "PDF",+ parser.TCADPAPIServer, parser.TCADPAPIKey,+ parser.TCADPTableResultType, parser.TCADPMarkdownImageResponseType,+ parser.OutputFormat)}func tcadpItemsFromZip(zipBytes []byte) ([]map[string]any, int, error) {diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex a8d50ab005..9665f814b8 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -95,7 +95,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datamethod := strings.ToLower(strings.TrimSpace(p.ParseMethod))switch method {case "tcadp":- return parsePresentationWithTCADP(ctx,+ return parseWithTCADP(ctx,filename, data, strings.ToUpper(p.format),p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,diff --git a/internal/parser/parser/pptx_tcadp.go b/internal/parser/parser/tcadp_common.gosimilarity index 65%rename from internal/parser/parser/pptx_tcadp.gorename to internal/parser/parser/tcadp_common.goindex ad97d0a4c8..9b2dad9bab 100644--- a/internal/parser/parser/pptx_tcadp.go+++ b/internal/parser/parser/tcadp_common.go@@ -13,23 +13,48 @@ import (models "ragflow/internal/entity/models")-// parsePresentationWithTCADP sends binary presentation (PPTX/PPT) data-// to the TCADP cloud reconstruction service and returns the structured-// parse result. Mirrors the spreadsheet-family parseSpreadsheetWithTCADP-// in xls_tcadp.go-func parsePresentationWithTCADP(ctx context.Context, filename string, data []byte, fileType string,- tcadpAPIServer, tcadpAPIKey, tableResultType, markdownImageResponseType string,+// parseWithTCADP submits binary data to the TCADP cloud reconstruction service+// and returns the structured parse result.+//+// Shared by all three TCADP-using parser families (PDF, spreadsheet,+// presentation). Each family wraps this with its own thin function that+// supplies its parameters (env-fallbacks, fileType, table/format config)+// so the TCADP API contract lives in one place.++// tcadpAPIBaseURL resolves the TCADP base URL using the order:+// 1. explicit caller argument (already trimmed)+// 2. legacy TCADP_APISERVER_URL env var (kept for backward compat with+// the old parsePDFWithTCADP helper that honoured it)+// 3. canonical TCADP_APISERVER env var (used by the other two families+// and the error message)+//+// Extracted as a helper so unit tests can exercise the env-fallback+// ordering without spinning up the full parse flow.+func tcadpAPIBaseURL(explicit string) string {+ if v := strings.TrimSpace(explicit); v != "" {+ return v+ }+ if v := strings.TrimSpace(os.Getenv("TCADP_APISERVER_URL")); v != "" {+ return v+ }+ return strings.TrimSpace(os.Getenv("TCADP_APISERVER"))+}++func parseWithTCADP(+ ctx context.Context,+ filename string,+ data []byte,+ fileType string,+ tcadpAPIServer, tcadpAPIKey string,+ tableResultType, markdownImageResponseType string,outputFormat string,) ParseResult {if len(data) == 0 {return emptyPDFResult(filename)}- baseURL := strings.TrimSpace(tcadpAPIServer)- if baseURL == "" {- baseURL = strings.TrimSpace(os.Getenv("TCADP_APISERVER"))- }+ baseURL := tcadpAPIBaseURL(tcadpAPIServer)if baseURL == "" {- return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")}+ return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver, TCADP_APISERVER_URL, or TCADP_APISERVER")}}apiKey := strings.TrimSpace(tcadpAPIKey)if apiKey == "" {diff --git a/internal/parser/parser/tcadp_common_test.go b/internal/parser/parser/tcadp_common_test.gonew file mode 100644index 0000000000..af83d9aa05--- /dev/null+++ b/internal/parser/parser/tcadp_common_test.go@@ -0,0 +1,55 @@+package parser++import (+ "os"+ "testing"+)++// TestParseWithTCADPReadsLegacyTCADPAPISERVERURLEnvVar guards against+// the silent breaking change where the shared parseWithTCADP core+// dropped the legacy TCADP_APISERVER_URL env var that the old+// parsePDFWithTCADP wrapper honoured. Existing PDF deployments set+// this env var; we must keep reading it for backward compat.+func TestParseWithTCADPReadsLegacyTCADPAPISERVERURLEnvVar(t *testing.T) {+ t.Setenv("TCADP_APISERVER_URL", "https://legacy.tcadp.example.com")+ t.Setenv("TCADP_APISERVER", "")++ if got := tcadpAPIBaseURL(""); got != "https://legacy.tcadp.example.com" {+ t.Fatalf("expected tcadpAPIBaseURL to honor TCADP_APISERVER_URL fallback, got %q", got)+ }+}++// TestParseWithTCADPPrefersExplicitArgumentOverEnvVar pins the order:+// an explicit tcadpAPIServer argument wins over both env vars.+func TestParseWithTCADPPrefersExplicitArgumentOverEnvVar(t *testing.T) {+ t.Setenv("TCADP_APISERVER", "https://env.tcadp.example.com")+ t.Setenv("TCADP_APISERVER_URL", "https://legacy.tcadp.example.com")++ if got := tcadpAPIBaseURL("https://explicit.tcadp.example.com"); got != "https://explicit.tcadp.example.com" {+ t.Fatalf("explicit arg must win over env vars, got %q", got)+ }+}++// TestParseWithTCADPFallsBackToCanonicalTCADPAPISERVEREnvVar verifies+// the canonical env var works when the legacy one is unset.+func TestParseWithTCADPFallsBackToCanonicalTCADPAPISERVEREnvVar(t *testing.T) {+ t.Setenv("TCADP_APISERVER", "https://canonical.tcadp.example.com")+ t.Setenv("TCADP_APISERVER_URL", "")++ if got := tcadpAPIBaseURL(""); got != "https://canonical.tcadp.example.com" {+ t.Fatalf("expected tcadpAPIBaseURL to honor TCADP_APISERVER fallback, got %q", got)+ }+}++// TestParseWithTCADPReturnsEmptyWhenNoEnvVarOrArgSet guards against a+// regression where the fallback ordering ever leaks the literal string+// "" into the parse flow (which would surface as a malformed-URL error+// later instead of the helpful "configure tcadp_apiserver" error).+func TestParseWithTCADPReturnsEmptyWhenNoEnvVarOrArgSet(t *testing.T) {+ os.Unsetenv("TCADP_APISERVER_URL")+ os.Unsetenv("TCADP_APISERVER")++ if got := tcadpAPIBaseURL(""); got != "" {+ t.Fatalf("expected empty result when no env var and no arg, got %q", got)+ }+}diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex 02a34f9bfa..84f9610291 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -81,8 +81,8 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [method := normalizeXLSXParseMethod(p.ParseMethod)switch method {case "tcadp":- return parseSpreadsheetWithTCADP(- filename, data, "XLS",+ return parseWithTCADP(+ ctx, filename, data, "XLS",p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,p.OutputFormat,diff --git a/internal/parser/parser/xls_tcadp.go b/internal/parser/parser/xls_tcadp.godeleted file mode 100644index 9468450161..0000000000--- a/internal/parser/parser/xls_tcadp.go+++ /dev/null@@ -1,86 +0,0 @@-package parser--import (- "context"- "encoding/base64"- "encoding/json"- "fmt"- "io"- "net/http"- "os"- "strings"-- models "ragflow/internal/entity/models"-)--func parseSpreadsheetWithTCADP(filename string, data []byte, fileType string, tcadpAPIServer, tcadpAPIKey, tableResultType, markdownImageResponseType string, outputFormat string) ParseResult {- if len(data) == 0 {- return emptyPDFResult(filename)- }- baseURL := strings.TrimSpace(tcadpAPIServer)- if baseURL == "" {- baseURL = strings.TrimSpace(os.Getenv("TCADP_APISERVER"))- }- if baseURL == "" {- return ParseResult{Err: fmt.Errorf("parser: TCADP requires tcadp_apiserver or TCADP_APISERVER")}- }- apiKey := strings.TrimSpace(tcadpAPIKey)- if apiKey == "" {- apiKey = strings.TrimSpace(os.Getenv("TCADP_API_KEY"))- }- requestBody := map[string]any{- "file_type": fileType,- "file_base64": base64.StdEncoding.EncodeToString(data),- "file_start_page_number": 1,- "file_end_page_number": 1000,- "config": map[string]any{- "TableResultType": tableResultType,- "MarkdownImageResponseType": markdownImageResponseType,- },- }- resp, err := models.PostJSONRequest(context.Background(), models.NewDriverHTTPClient(false), strings.TrimRight(baseURL, "/")+"/reconstruct_document", bearer(apiKey), requestBody)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP submit: %w", err)}- }- defer resp.Body.Close()- raw, err := io.ReadAll(resp.Body)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP read submit: %w", err)}- }- if resp.StatusCode >= 300 {- return ParseResult{Err: fmt.Errorf("parser: TCADP HTTP %d: %s", resp.StatusCode, string(raw))}- }- var payload struct {- DocumentRecognizeResultURL string `json:"DocumentRecognizeResultUrl"`- }- if err := json.Unmarshal(raw, &payload); err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP decode submit: %w", err)}- }- if payload.DocumentRecognizeResultURL == "" {- return ParseResult{Err: fmt.Errorf("parser: TCADP returned no DocumentRecognizeResultUrl")}- }- downloadReq, err := http.NewRequestWithContext(context.Background(), http.MethodGet, payload.DocumentRecognizeResultURL, nil)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP download request: %w", err)}- }- if auth := bearer(apiKey); auth != "" {- downloadReq.Header.Set("Authorization", auth)- }- downloadResp, err := models.NewDriverHTTPClient(false).Do(downloadReq)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP download: %w", err)}- }- defer downloadResp.Body.Close()- zipBytes, err := io.ReadAll(downloadResp.Body)- if err != nil {- return ParseResult{Err: fmt.Errorf("parser: TCADP read zip: %w", err)}- }- if downloadResp.StatusCode >= 300 {- return ParseResult{Err: fmt.Errorf("parser: TCADP download HTTP %d: %s", downloadResp.StatusCode, string(zipBytes))}- }- items, pageCount, err := tcadpItemsFromZip(zipBytes)- if err != nil {- return ParseResult{Err: err}- }- return pdfItemsToResult(filename, items, outputFormat, pageCount)-}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex 02f698941c..b896a1ed37 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -98,8 +98,8 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, datamethod := normalizeXLSXParseMethod(p.ParseMethod)switch method {case "tcadp":- return parseSpreadsheetWithTCADP(- filename, data, "XLSX",+ return parseWithTCADP(+ ctx, filename, data, "XLSX",p.TCADPAPIServer, p.TCADPAPIKey,p.TCADPTableResultType, p.TCADPMarkdownImageResponseType,p.OutputFormat,