Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(deepdoc): route PDF parse logs through zap and drive colspan e2e in-process (#19947)

xugangqiang11c56bafc435
diff --git a/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go b/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go
index 64b11e0f9e..82572b6652 100644
--- a/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go
+++ b/internal/deepdoc/parser/pdf/one_pdf_colspan_e2e_test.go
@@ -9,13 +9,15 @@ import (
"testing"
"ragflow/internal/common"
+ "ragflow/internal/deepdoc/native"
+ infnative "ragflow/internal/deepdoc/parser/pdf/inference/native_analyzer"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
)
// Test1PdfColspanEndToEnd parses real_pdfs/1.pdf through the PRODUCTION path
-// (real DLA+TSR+OCR against DEEPDOC_URL) and asserts the table HTML carries
-// colspan="6" on the merged header — matching Python's golden
+// (real DLA+TSR+OCR via the in-process Go DeepDoc analyzer) and asserts the
+// table HTML carries colspan="6" on the merged header — matching Python's golden
// (output/py/ocr_real/tables/1.pdf.json: row0 collapses to 3 text cells
// because the header spans 6 of the 8 columns).
//
@@ -28,9 +30,19 @@ import (
// emits colspan=6. TSR input is identical on both sides, so the divergence
// was purely a Go assembly bug, not a model issue.
//
-// Requires DEEPDOC_URL (OSS DeepDoc) reachable; skips otherwise.
+// Requires DEEPDOC_MODEL_DIR (in-process DeepDoc model weights); skips otherwise.
func Test1PdfColspanEndToEnd(t *testing.T) {
- client := mustConnectInferenceClient(t)
+ modelDir := os.Getenv("DEEPDOC_MODEL_DIR")
+ if modelDir == "" {
+ t.Skip("set DEEPDOC_MODEL_DIR to run the in-process colspan e2e test")
+ }
+ if err := native.InitORT(); err != nil {
+ t.Fatalf("InitORT: %v", err)
+ }
+ analyzer, err := infnative.NewAnalyzer(modelDir, infnative.DefaultDropScore)
+ if err != nil {
+ t.Fatalf("NewAnalyzer: %v", err)
+ }
pdfDir := common.GetEnv("BATCH_PARITY_PDF_DIR")
if pdfDir == "" {
@@ -44,7 +56,7 @@ func Test1PdfColspanEndToEnd(t *testing.T) {
cfg := pdf.DefaultParserConfig()
p := NewParser(cfg)
- result, err := p.Parse(t.Context(), data, client)
+ result, err := p.Parse(t.Context(), data, analyzer)
if err != nil {
t.Fatalf("Parse 1.pdf: %v", err)
}
diff --git a/internal/deepdoc/parser/pdf/parser.go b/internal/deepdoc/parser/pdf/parser.go
index 7c505d67a8..432a5f4c3c 100644
--- a/internal/deepdoc/parser/pdf/parser.go
+++ b/internal/deepdoc/parser/pdf/parser.go
@@ -4,10 +4,12 @@ import (
"context"
"fmt"
"image"
- "log/slog"
"math"
"sort"
+ "go.uber.org/zap"
+
+ "ragflow/internal/common"
lyt "ragflow/internal/deepdoc/parser/pdf/layout"
tbl "ragflow/internal/deepdoc/parser/pdf/table"
pdf "ragflow/internal/deepdoc/parser/pdf/type"
@@ -139,7 +141,8 @@ func resolvePagesToProcess(ranges [][]int, pageCount int) []int {
func (p *Parser) extractOutlines(engine pdf.PDFEngine) []pdf.Outline {
outlines, outlineErr := engine.Outlines()
if outlineErr != nil {
- slog.Warn("Failed to extract PDF outlines; continuing without them", "err", outlineErr)
+ common.Warn("deepdoc pdf parse: extract outlines failed; continuing without them",
+ zap.Error(outlineErr))
outlines = nil
}
return outlines
@@ -173,7 +176,8 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
ctx = context.WithValue(ctx, pageNumCtxKey, pg)
chars, extractErr := engine.ExtractChars(pg)
if extractErr != nil {
- slog.Warn("processPage: ExtractChars failed", "page", pg, "err", extractErr)
+ common.Warn("deepdoc pdf parse: processPage ExtractChars failed",
+ zap.Int("page", pg), zap.Error(extractErr))
chars = nil
}
medianH := util.MedianCharHeight(chars)
@@ -194,7 +198,9 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
}
// First pass: render at the default DLA DPI (216 DPI).
+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "render start"))
pageImg, renderErr := p.renderPageToImage(ctx, engine, pg)
+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "render done"))
pageZoom := pdf.DlaScale
var ocrBoxes []pdf.TextBox
var updatedChars []pdf.TextChar
@@ -204,13 +210,17 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
var dlaRegions []pdf.DLAPageRegions
if pageImg != nil && renderErr == nil {
+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "ocr start"))
ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, pageImg, chars, pg, renderErr, isScanNoise, docAnalyzer, pageZoom)
+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "ocr done"))
annotated, pageTables, dlaRegions = p.enrichOnePageWithDeepDoc(
ctx, pageImg, ocrBoxes, pg, renderErr, docAnalyzer, tb, pageZoom)
+ common.Info("deepdoc pdf parse: stage", zap.Int("page", pg), zap.String("stage", "dla_tsr done"))
}
if renderErr != nil {
- slog.Warn("processPage: RenderPageToImage failed", "page", pg, "err", renderErr)
+ common.Warn("deepdoc pdf parse: processPage RenderPageToImage failed",
+ zap.Int("page", pg), zap.Error(renderErr))
}
// Per-page zoom retry: if no boxes were produced at the default zoom
@@ -220,7 +230,8 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
// render to an unsafe DPI and spike memory on large pages.
const maxRetryZoom = 9.0
retryZoom := math.Min(p.Config.Zoom*pdf.DlaScale, maxRetryZoom)
- slog.Debug("per-page zoom retry", "page", pg, "zoom", retryZoom)
+ common.Debug("deepdoc pdf parse: per-page zoom retry",
+ zap.Int("page", pg), zap.Float64("zoom", retryZoom))
retryImg, retryRenderErr := p.renderAtDPI(ctx, engine, pg, retryZoom*72)
if retryRenderErr == nil && retryImg != nil {
ocrBoxes, updatedChars, ocrUsed = p.processPageBoxes(ctx, retryImg, chars, pg, retryRenderErr, isScanNoise, docAnalyzer, retryZoom)
@@ -229,7 +240,8 @@ func (p *Parser) processPage(ctx context.Context, engine pdf.PDFEngine, pg int,
pageImg = retryImg
pageZoom = retryZoom
} else if retryRenderErr != nil {
- slog.Warn("processPage: retry-zoom render failed", "page", pg, "err", retryRenderErr)
+ common.Warn("deepdoc pdf parse: processPage retry-zoom render failed",
+ zap.Int("page", pg), zap.Error(retryRenderErr))
}
}
@@ -405,6 +417,10 @@ func (p *Parser) runPageWorkers(ctx context.Context, engine pdf.PDFEngine,
recordErr(r.Err)
}
resultMap[r.PageNumber] = &r
+ common.Info("deepdoc pdf parse: page finished",
+ zap.Int("page", r.PageNumber),
+ zap.Int("done", i+1),
+ zap.Int("total", submitted))
}
results := make([]*pageResult, 0, len(pages))
@@ -440,7 +456,8 @@ func (p *Parser) assembleDocument(ctx context.Context, pages []int, pageResults
continue
}
if r.Err != nil {
- slog.Warn("page worker failed", "page", r.PageNumber, "err", r.Err)
+ common.Warn("deepdoc pdf parse: page worker failed",
+ zap.Int("page", r.PageNumber), zap.Error(r.Err))
}
// Store per-page PDF-point dimensions for buildLayout.
if r.PageHeight > 0 {
@@ -609,18 +626,22 @@ func (p *Parser) processPages(ctx context.Context, engine pdf.PDFEngine, docAnal
tb := NewTableBuilderFor(docAnalyzer)
pages := resolvePagesToProcess(p.Config.Pages, pageCount)
+ common.Info("deepdoc pdf parse: total pages",
+ zap.Int("page_count", pageCount),
+ zap.Int("pages_to_parse", len(pages)))
if len(p.Config.Pages) > 0 {
- slog.Info("deepdoc pdf parse: page ranges applied",
- "configured_ranges", p.Config.Pages,
- "page_count", pageCount,
- "pages_to_parse", pages)
+ common.Info("deepdoc pdf parse: page ranges applied",
+ zap.Any("configured_ranges", p.Config.Pages),
+ zap.Int("page_count", pageCount),
+ zap.Ints("pages_to_parse", pages))
} else {
- slog.Debug("deepdoc pdf parse: parsing all pages", "page_count", pageCount)
+ common.Info("deepdoc pdf parse: parsing all pages", zap.Int("page_count", pageCount))
}
pageResults, pageErr := p.runPageWorkers(ctx, engine, pages, docAnalyzer, tb)
if pageErr != nil {
- slog.Warn("runPageWorkers: some pages failed", "err", pageErr)
+ common.Warn("deepdoc pdf parse: runPageWorkers some pages failed",
+ zap.Error(pageErr))
}
result, err := p.assembleDocument(ctx, pages, pageResults, outlines)