RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): route PDF parse logs through zap and drive colspan e2e in-process (#19947)
xugangqiang11c56bafc435
diff --git a/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go b/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.goindex 64b11e0f9e..82572b6652 100644--- a/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go+++ b/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go@@ -9,13 +9,15 @@ import ("testing""ragflow/internal/common"+ "ragflow/internal/deepdoc/native"+ infnative "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"pdf "ragflow/internal/deepdoc/parser/pdf/type")// Test1PdfColspanEndToEnd parses real_pdfs/1.pdf through the PRODUCTION path-// (real DLA+TSR+OCR against DEEPDOC_URL) and asserts the table HTML carries-// colspan="6" on the merged header — matching Python's golden+// (real DLA+TSR+OCR via the in-process Go DeepDoc analyzer) and asserts the+// table HTML carries colspan="6" on the merged header — matching Python's golden// (output/py/ocr_real/tables/1.pdf.json: row0 collapses to 3 text cells// because the header spans 6 of the 8 columns).//@@ -28,9 +30,19 @@ import (// emits colspan=6. TSR input is identical on both sides, so the divergence// was purely a Go assembly bug, not a model issue.//-// Requires DEEPDOC_URL (OSS DeepDoc) reachable; skips otherwise.+// Requires DEEPDOC_MODEL_DIR (in-process DeepDoc model weights); skips otherwise.func Test1PdfColspanEndToEnd(t *testing.T) {- client := mustConnectInferenceClient(t)+ modelDir := os.Getenv("DEEPDOC_MODEL_DIR")+ if modelDir == "" {+ t.Skip("set DEEPDOC_MODEL_DIR to run the in-process colspan e2e test")+ }+ if err := native.InitORT(); err != nil {+ t.Fatalf("InitORT: %v", err)+ }+ analyzer, err := infnative.NewAnalyzer(modelDir, infnative.DefaultDropScore)+ if err != nil {+ t.Fatalf("NewAnalyzer: %v", err)+ }pdfDir := common.GetEnv("BATCH_PARITY_PDF_DIR")if pdfDir == "" {@@ -44,7 +56,7 @@ func Test1PdfColspanEndToEnd(t *testing.T) {cfg := pdf.DefaultParserConfig()p := NewParser(cfg)- result, err := p.Parse(t.Context(), data, client)+ result, err := p.Parse(t.Context(), data, analyzer)if err != nil {t.Fatalf("Parse 1.pdf: %v", err)}diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.goindex 7c505d67a8..432a5f4c3c 100644--- a/internal/deepdoc/parser/pdf/parser.go+++ b/internal/deepdoc/parser/pdf/parser.go@@ -4,10 +4,12 @@ import ("context""fmt""image"- "log/slog""math""sort"+ "go.uber.org/zap"++ "ragflow/internal/common"lyt "ragflow/internal/deepdoc/parser/pdf/layout"tbl "ragflow/internal/deepdoc/parser/pdf/table"pdf "ragflow/internal/deepdoc/parser/pdf/type"@@ -139,7 +141,8 @@ func resolvePagesToProcess(ranges [][]int, pageCount int) []int {func (p *Parser) extractOutlines(engine pdf.PDFEngine) []pdf.Outline {outlines, outlineErr := engine.Outlines()if outlineErr != nil {- slog.Warn("Failed to extract PDF outlines; continuing without them", "err", outlineErr)+ common.Warn("deepdoc pdf parse: extract outlines failed; continuing without them",+ zap.Error(outlineErr))outlines = nil}return outlines@@ -173,7 +176,8 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,ctx = context.WithValue(ctx, pageNumCtxKey, pg)chars, extractErr := engine.ExtractChars(pg)if extractErr != nil {- slog.Warn("processPage: ExtractChars failed", "page", pg, "err", extractErr)+ common.Warn("deepdoc pdf parse: processPage ExtractChars failed",+ zap.Int("page", pg), zap.Error(extractErr))chars = nil}medianH := util.MedianCharHeight(chars)@@ -194,7 +198,9 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,}// First pass: render at the default DLA DPI (216 DPI).+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "render start"))pageImg, renderErr := p.renderPageToImage(ctx, engine, pg)+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "render done"))pageZoom := pdf.DlaScalevar ocrBoxes []pdf.TextBoxvar updatedChars []pdf.TextChar@@ -204,13 +210,17 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,var dlaRegions []pdf.DLAPageRegionsif pageImg != nil && renderErr == nil {+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "ocr start"))ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, pageImg, chars, pg, renderErr, isScanNoise, docAnalyzer, pageZoom)+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "ocr done"))annotated, pageTables, dlaRegions = p.enrichOnePageWithDeepDoc(ctx, pageImg, ocrBoxes, pg, renderErr, docAnalyzer, tb, pageZoom)+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "dla_tsr done"))}if renderErr != nil {- slog.Warn("processPage: RenderPageToImage failed", "page", pg, "err", renderErr)+ common.Warn("deepdoc pdf parse: processPage RenderPageToImage failed",+ zap.Int("page", pg), zap.Error(renderErr))}// Per-page zoom retry: if no boxes were produced at the default zoom@@ -220,7 +230,8 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,// render to an unsafe DPI and spike memory on large pages.const maxRetryZoom = 9.0retryZoom := math.Min(p.Config.Zoom*pdf.DlaScale, maxRetryZoom)- slog.Debug("per-page zoom retry", "page", pg, "zoom", retryZoom)+ common.Debug("deepdoc pdf parse: per-page zoom retry",+ zap.Int("page", pg), zap.Float64("zoom", retryZoom))retryImg, retryRenderErr := p.renderAtDPI(ctx, engine, pg, retryZoom*72)if retryRenderErr == nil && retryImg != nil {ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, retryImg, chars, pg, retryRenderErr, isScanNoise, docAnalyzer, retryZoom)@@ -229,7 +240,8 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,pageImg = retryImgpageZoom = retryZoom} else if retryRenderErr != nil {- slog.Warn("processPage: retry-zoom render failed", "page", pg, "err", retryRenderErr)+ common.Warn("deepdoc pdf parse: processPage retry-zoom render failed",+ zap.Int("page", pg), zap.Error(retryRenderErr))}}@@ -405,6 +417,10 @@ func (p *Parser) runPageWorkers(ctx context.Context, engine pdf.PDFEngine,recordErr(r.Err)}resultMap[r.PageNumber] = &r+ common.Info("deepdoc pdf parse: page finished",+ zap.Int("page", r.PageNumber),+ zap.Int("done", i+1),+ zap.Int("total", submitted))}results := make([]*pageResult, 0, len(pages))@@ -440,7 +456,8 @@ func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResultscontinue}if r.Err != nil {- slog.Warn("page worker failed", "page", r.PageNumber, "err", r.Err)+ common.Warn("deepdoc pdf parse: page worker failed",+ zap.Int("page", r.PageNumber), zap.Error(r.Err))}// Store per-page PDF-point dimensions for buildLayout.if r.PageHeight > 0 {@@ -609,18 +626,22 @@ func (p *Parser) processPages(ctx context.Context, engine pdf.PDFEngine, docAnaltb := NewTableBuilderFor(docAnalyzer)pages := resolvePagesToProcess(p.Config.Pages, pageCount)+ common.Info("deepdoc pdf parse: total pages",+ zap.Int("page_count", pageCount),+ zap.Int("pages_to_parse", len(pages)))if len(p.Config.Pages) > 0 {- slog.Info("deepdoc pdf parse: page ranges applied",- "configured_ranges", p.Config.Pages,- "page_count", pageCount,- "pages_to_parse", pages)+ common.Info("deepdoc pdf parse: page ranges applied",+ zap.Any("configured_ranges", p.Config.Pages),+ zap.Int("page_count", pageCount),+ zap.Ints("pages_to_parse", pages))} else {- slog.Debug("deepdoc pdf parse: parsing all pages", "page_count", pageCount)+ common.Info("deepdoc pdf parse: parsing all pages", zap.Int("page_count", pageCount))}pageResults, pageErr := p.runPageWorkers(ctx, engine, pages, docAnalyzer, tb)if pageErr != nil {- slog.Warn("runPageWorkers: some pages failed", "err", pageErr)+ common.Warn("deepdoc pdf parse: runPageWorkers some pages failed",+ zap.Error(pageErr))}result, err := p.assembleDocument(ctx, pages, pageResults, outlines)