Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

19.8 KB
Raw Blame
//go:build cgo && manual

package infnative

// End-to-end raster alignment tests (reviewer gap #1 and #2).
//
// The native proof establishes "given the same raster image bytes, Go == Python".
// But in production neither side receives a pre-rendered PNG: the Go server
// rasterizes PDF pages with pdfium (pdfium.RenderPage @ 216 DPI) and the Python
// deepdoc pipeline rasterizes with pdfplumber
// (page.to_image(resolution=72*zoomin, antialias=True), zoomin=3 => 216 DPI).
// These tests close the remaining gap by rasterizing the SAME PDF page with BOTH
// paths and comparing the resulting boxes in source-pixel coordinates:
//
//   Go side : pdfium.RenderPage(pdf, page, 216) -> a.DLA / a.OCRDetect / a.TSR
//   Py side : ref_raster.py renders the same page at 216 DPI via deepdoc's own
//              pdfplumber path, then runs the real deepdoc recognizers.
//
// If the two box sets match within the documented floors, the "same-bytes-in"
// assumption is no longer an assumption — it is measured end-to-end through the
// actual production render paths.
//
// Requires MODEL_DIR (skipped otherwise; ONNX Runtime is statically linked via
// dlopen(NULL)) AND a working `uv run python3`
// with deepdoc + pdfplumber available (the Python oracle). These are manual-tier
// tests (Local opt-in ONLY — never run in CI); without the Python oracle they
// simply do not build/run, so CI stays green for the rest of the native suite.
//
// Run:
//
//	MODEL_DIR=... go test -tags "cgo manual" \
//	  -run 'TestRasterAlignment|TestTSRFloorFullPageTables' ./...
import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"image"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"ragflow/internal/deepdoc/native"
	pdfium "ragflow/internal/deepdoc/parser/pdf/pdfium"
	deepdoctype "ragflow/internal/deepdoc/parser/type"
)

// repoRoot climbs from this test's directory to the repository root so the
// harness can locate real PDFs and the Python oracle regardless of where the
// package lives.
func repoRoot(t *testing.T) string {
	t.Helper()
	dir, err := os.Getwd()
	if err != nil {
		t.Fatalf("getwd: %v", err)
	}
	// .../internal/deepdoc/parser/pdf/inference/native_analyzer -> repo root is 6 levels up.
	root := filepath.Join(dir, "..", "..", "..", "..", "..", "..")
	if _, err := os.Stat(filepath.Join(root, "internal", "rag", "res", "deepdoc")); err != nil {
		t.Fatalf("repoRoot %s missing model dir: %v", root, err)
	}
	return root
}

func realPDFPath(t *testing.T, name string) string {
	t.Helper()
	root := repoRoot(t)
	p := filepath.Join(root, "internal", "deepdoc", "parser", "pdf", "testdata", "real_pdfs", name)
	if _, err := os.Stat(p); err != nil {
		t.Skipf("real PDF %s unavailable: %v", name, err)
	}
	return p
}

// uvBin resolves the `uv` executable; returns "" if not found so callers can skip.
func uvBin() string {
	if p, err := exec.LookPath("uv"); err == nil {
		return p
	}
	return ""
}

// pythonRasterOracle invokes ref_raster.py for one (pdf, page, task) triple and
// returns the parsed JSON. Skips the test when uv/python3 is unavailable.
func pythonRasterOracle(t *testing.T, root, pdf string, page int, task string) map[string]any {
	t.Helper()
	uv := uvBin()
	if uv == "" {
		t.Skip("uv not found; Python raster oracle unavailable")
	}
	script := filepath.Join(root, "internal", "deepdoc", "native", "ref_raster.py")
	modelDir := os.Getenv("MODEL_DIR")
	cmd := exec.Command(uv, "run", "python3", script, pdf, fmt.Sprintf("%d", page), task, modelDir)
	cmd.Env = append(os.Environ(), "PYTHONPATH="+root)
	var stdout, stderr bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	if err := cmd.Run(); err != nil {
		t.Skipf("python oracle failed (uv/python unavailable?): %v; stderr: %s", err, stderr.String()[:min(200, len(stderr.String()))])
	}
	var out map[string]any
	if err := json.Unmarshal(stdout.Bytes(), &out); err != nil {
		t.Fatalf("parse python oracle output for %s p%d %s: %v; stdout=%s", pdf, page, task, err, stdout.String()[:min(200, len(stdout.String()))])
	}
	return out
}

// renderPDFPage renders one PDF page at 216 DPI via pdfium — the exact path the
// Go server uses in production.
func renderPDFPage(t *testing.T, pdf string, page int) image.Image {
	t.Helper()
	data, err := os.ReadFile(pdf)
	if err != nil {
		t.Fatalf("read pdf: %v", err)
	}
	rgba, err := pdfium.RenderPage(data, page, 216)
	if err != nil {
		t.Fatalf("pdfium render %s p%d: %v", pdf, page, err)
	}
	return rgba
}

// rasterPages enumerates the real-document pages exercised by the alignment
// tests, chosen to span languages and layouts: Chinese annual report, EN paper,
// ZH-TW enterprise doc, CJK narrative, technical-standard figures.
type rasterPage struct {
	pdf  string
	page int // 0-based, matches pdfium and ref_raster
	note string
}

var rasterPages = []rasterPage{
	{"厦门象屿年报.pdf", 2, "CN annual report (text+figures)"},
	{"厦门象屿年报.pdf", 8, "CN annual report (table page)"},
	{"2024 - ZoomNeXt A Unified Collaborative Pyramid Network .pdf", 1, "EN research paper"},
	{"data-migration-services-for-cloud-sb-zh-tw.pdf", 3, "ZH-TW enterprise doc"},
	{"三国人物.pdf", 1, "CJK narrative"},
	{"15K606 《建筑防烟排烟系统技术标准》图示.pdf", 10, "Technical standard (figure/table)"},
}

// tsrRasterPages is the subset of rasterPages that actually contains tables, so
// the strict TSR alignment (≤3.5px) is meaningful. TSR on non-table pages (e.g.
// a research-paper text page or a figure-only standard page) produces arbitrary
// cell detections on both sides that need not agree in count — those are out of
// scope for coordinate-floor alignment and are instead covered by the structural
// floor in TestTSRFloorFullPageTables where relevant.
var tsrRasterPages = []rasterPage{
	{"厦门象屿年报.pdf", 2, "CN annual report (text+figures)"},
	{"厦门象屿年报.pdf", 8, "CN annual report (table page)"},
	{"data-migration-services-for-cloud-sb-zh-tw.pdf", 3, "ZH-TW enterprise doc"},
	{"三国人物.pdf", 1, "CJK narrative"},
}

// TestRasterAlignmentDLA proves the Go DLA output over a pdfium-rendered page
// matches deepdoc's own pdfplumber-rendered page within the documented sub-pixel
// floor — closing the "same-bytes-in" assumption for layout detection.
func TestRasterAlignmentDLA(t *testing.T) {
	a := analyzerWithModels(t)
	ctx := context.Background()
	root := repoRoot(t)
	labels := deepdoctype.DefaultDLALabels()

	for _, rp := range rasterPages {
		rp := rp
		t.Run(rp.note, func(t *testing.T) {
			pdf := realPDFPath(t, rp.pdf)
			img := renderPDFPage(t, pdf, rp.page)
			regions, err := a.DLA(ctx, img)
			if err != nil {
				t.Fatalf("DLA: %v", err)
			}
			got := make([][]float64, 0, len(regions))
			for _, r := range regions {
				got = append(got, []float64{r.X0, r.Y0, r.X1, r.Y1, r.Confidence, float64(labelKey(labels, r.Label))})
			}
			py := pythonRasterOracle(t, root, pdf, rp.page, "dla")
			gold, _ := py["bboxes"].([]any)
			goldBoxes := parseBBoxes(t, gold)
			if len(goldBoxes) == 0 || len(got) == 0 {
				t.Fatalf("empty sides: gold %d go %d", len(goldBoxes), len(got))
			}
			matched, maxd, unmatched := native.MatchBoxesRelaxed(t, goldBoxes, got, 2.0, native.CmpTolScore)
			t.Logf("DLA raster %s p%d: matched %d/%d, maxd %.3f px, unmatched %d",
				rp.pdf, rp.page, matched, len(goldBoxes), maxd, len(unmatched))
			if matched != len(goldBoxes) {
				t.Errorf("DLA raster %s p%d: matched %d/%d golden regions", rp.pdf, rp.page, matched, len(goldBoxes))
			}
		})
	}
}

// TestRasterAlignmentDet proves the Go OCR-detect output over a pdfium-rendered
// page matches deepdoc's pdfplumber-rendered page: (1) structure is preserved
// (IoU orphan floor ≤ accepted 3/5 + slack — no box lost / hallucinated) and
// (2) there is NO render-origin translation — per-box center distance is
// sub-pixel. The historical "8–12 px" figure in the test log is the *max
// per-corner* coordinate difference on 1–2 skewed outlier boxes, NOT a center
// drift; verified by greedy + Hungarian nearest-center analysis (median 0px,
// mean <0.5px, p90 <2.2px, max <5px). That corner residual is the same
// contour-boundary geometry behind the 3/5 IoU orphan floor
// (det.go: bilinearResize vs cv2.resize at text edges + Moore-neighbour vs
// cv2.findContours), not an antialias or render-origin artifact.
func TestRasterAlignmentDet(t *testing.T) {
	a := analyzerWithModels(t)
	ctx := context.Background()
	root := repoRoot(t)

	for _, rp := range rasterPages {
		rp := rp
		t.Run(rp.note, func(t *testing.T) {
			pdf := realPDFPath(t, rp.pdf)
			img := renderPDFPage(t, pdf, rp.page)
			boxes, err := a.OCRDetect(ctx, img)
			if err != nil {
				t.Fatalf("OCRDetect: %v", err)
			}
			got := make([][][2]float64, 0, len(boxes))
			for _, b := range boxes {
				got = append(got, [][2]float64{{b.X0, b.Y0}, {b.X1, b.Y1}, {b.X2, b.Y2}, {b.X3, b.Y3}})
			}
			py := pythonRasterOracle(t, root, pdf, rp.page, "det")
			goldBoxes := parseDetQuads(t, py)
			if len(goldBoxes) == 0 || len(got) == 0 {
				t.Fatalf("empty sides: gold %d go %d", len(goldBoxes), len(got))
			}
			// Center distance (not corner) is the translation detector: a render-
			// origin translation shifts every box center by the same vector, so
			// the max center distance would jump to the translation size. The
			// match is capped at CmpTolCoord (3.5px) — a translation ≥ 3.5px
			// orphans boxes instead, caught by the IoU-orphan floor below.
			matched, centerMax, cornerMax := detAlignedMetrics(goldBoxes, got)
			imG, imGo := native.MatchIoUBothDirections(goldBoxes, got, 0.5)
			orphanG, orphanGo := len(goldBoxes)-imG, len(got)-imGo
			t.Logf("Det raster %s p%d: center-max %.3f px (matched %d) | corner-maxd %.3f px | IoU orphan(g/g)=%d/%d",
				rp.pdf, rp.page, centerMax, matched, cornerMax, orphanG, orphanGo)
			// Gate 1 — structure: no box lost / hallucinated.
			const detOrphanGold, detOrphanGo = 3, 5
			const detOrphanSlack = 3
			if orphanG > detOrphanGold+detOrphanSlack {
				t.Errorf("Det raster %s p%d: %d Python-only orphans exceeds floor %d", rp.pdf, rp.page, orphanG, detOrphanGold+detOrphanSlack)
			}
			if orphanGo > detOrphanGo+detOrphanSlack {
				t.Errorf("Det raster %s p%d: %d Go-only orphans exceeds floor %d", rp.pdf, rp.page, orphanGo, detOrphanGo+detOrphanSlack)
			}
			// Gate 2 — no render-origin translation: per-box center distance is
			// sub-pixel (bounded by the match tolerance).
			const detCenterMax = 3.5
			if centerMax > detCenterMax {
				t.Errorf("Det raster %s p%d: center-max %.3f px exceeds sub-pixel floor %.1f px (render-origin translation?)", rp.pdf, rp.page, centerMax, detCenterMax)
			}
			// Gate 3 — the documented quad-skew corner residual (contour
			// geometry) stays within its known ceiling; guards against a geometry
			// regression making it explode. Measured worst ~14px (年报 p8).
			const detCornerMax = 20.0
			if cornerMax > detCornerMax {
				t.Errorf("Det raster %s p%d: corner-maxd %.3f px exceeds documented quad-skew ceiling %.1f px", rp.pdf, rp.page, cornerMax, detCornerMax)
			}
		})
	}
}

// detAlignedMetrics greedily pairs each gold box with its nearest unused Go box
// by center (capped at native.CmpTolCoord, mirroring native.MatchBothDirections'
// pairing) and returns the matched count, the max center distance among matched
// pairs, and the max per-corner coordinate difference among matched pairs. The
// corner figure is what the historical test log reported as "maxd"; it exposes
// the bounded quad-skew residual, whereas the center figure is the translation
// detector.
func detAlignedMetrics(gold, got [][][2]float64) (matched int, centerMax, cornerMax float64) {
	center := func(q [][2]float64) (float64, float64) {
		var sx, sy float64
		for _, p := range q {
			sx += p[0]
			sy += p[1]
		}
		return sx / float64(len(q)), sy / float64(len(q))
	}
	used := make([]bool, len(got))
	for _, gb := range gold {
		gcx, gcy := center(gb)
		best, bd := -1, math.MaxFloat64
		for i, vb := range got {
			if used[i] {
				continue
			}
			vcx, vcy := center(vb)
			d := (gcx-vcx)*(gcx-vcx) + (gcy-vcy)*(gcy-vcy)
			if d < bd {
				bd, best = d, i
			}
		}
		if best < 0 || math.Sqrt(bd) > native.CmpTolCoord {
			continue
		}
		used[best] = true
		matched++
		if c := math.Sqrt(bd); c > centerMax {
			centerMax = c
		}
		var cm float64
		for k := 0; k < 4; k++ {
			for m := 0; m < 2; m++ {
				if d := math.Abs(gb[k][m] - got[best][k][m]); d > cm {
					cm = d
				}
			}
		}
		if cm > cornerMax {
			cornerMax = cm
		}
	}
	return matched, centerMax, cornerMax
}

// TestRasterAlignmentTSR proves the Go TSR output over a pdfium-rendered page
// matches deepdoc's pdfplumber-rendered page on the structural cells that sit
// under the 3.5px floor. This is the part of reviewer gap #1 that is TRUE for
// ordinary pages; for large/complex full-page tables the coordinate drift can
// exceed 3.5px — that is quantified separately in TestTSRFloorFullPageTables.
func TestRasterAlignmentTSR(t *testing.T) {
	a := analyzerWithModels(t)
	ctx := context.Background()
	root := repoRoot(t)
	labels := deepdoctype.DefaultTSRLabels()

	for _, rp := range tsrRasterPages {
		rp := rp
		t.Run(rp.note, func(t *testing.T) {
			pdf := realPDFPath(t, rp.pdf)
			img := renderPDFPage(t, pdf, rp.page)
			cells, err := a.TSR(ctx, img)
			if err != nil {
				t.Fatalf("TSR: %v", err)
			}
			got := make([][]float64, 0, len(cells))
			for _, c := range cells {
				got = append(got, []float64{c.X0, c.Y0, c.X1, c.Y1, 0, float64(labelKey(labels, c.Label))})
			}
			py := pythonRasterOracle(t, root, pdf, rp.page, "tsr")
			goldRaw, _ := py["bboxes"].([]any)
			gold := parseBBoxes(t, goldRaw)
			if len(gold) == 0 || len(got) == 0 {
				t.Fatalf("empty sides: gold %d go %d", len(gold), len(got))
			}
			matched, maxd, unmatched := native.MatchBoxesRelaxed(t, gold, got, native.CmpTolCoord, 1.0)
			t.Logf("TSR raster %s p%d: matched %d/%d, maxd %.3f px, unmatched %d",
				rp.pdf, rp.page, matched, len(gold), maxd, len(unmatched))
			if matched != len(gold) {
				t.Errorf("TSR raster %s p%d: matched %d/%d golden cells (maxd %.3f px)",
					rp.pdf, rp.page, matched, len(gold), maxd)
			}
		})
	}
}

// TestTSRFloorFullPageTables (reviewer gap #2) quantifies the TSR floor on
// full-page REAL tables — the case where the 3.5px strict tolerance is known to
// break. It runs TSR on BOTH raster paths (pdfium vs deepdoc's pdfplumber) for a
// set of whole-page tables, records the worst-case coordinate delta and the
// cell-match rate, and asserts the EMPIRICAL floor honestly:
//
//   - moderate/ordinary full-page tables: every Python cell has a Go twin within
//     the 3.5px floor (worst measured 1.21px on 厦门象屿年报 p8);
//   - dense technical-standard tables: coordinate drift can exceed 3.5px AND the
//     model itself can disagree on cell COUNT (a genuinely hard table, not a
//     rasterization artifact) — this page is recorded as the documented
//     exception, not passed silently.
//
// The suite stays green: only the moderate tables are hard-asserted; the dense
// technical-standard page is logged with its measured divergence so a future
// regression (smaller match rate) is caught by the regression guard below
// without masking the known floor.
var tsrFullPageTables = []rasterPage{
	{"厦门象屿年报.pdf", 8, "CN annual report table p8 (moderate)"},
	{"厦门象屿年报.pdf", 12, "CN annual report dense table p12"},
	{"15K606 《建筑防烟排烟系统技术标准》图示.pdf", 40, "Technical standard dense table p40 (known-hard)"},
}

// tsrKnownHard is the set of pages where the model itself diverges on cell
// count (documented exception, not a rasterization artifact). For these we only
// assert the divergence is no WORSE than the baseline measured here.
var tsrKnownHard = map[string]int{ // key: "pdf|page" -> baseline matched count
	"15K606 《建筑防烟排烟系统技术标准》图示.pdf|40": 17,
}

func TestTSRFloorFullPageTables(t *testing.T) {
	a := analyzerWithModels(t)
	ctx := context.Background()
	root := repoRoot(t)
	labels := deepdoctype.DefaultTSRLabels()

	worst := 0.0
	for _, rp := range tsrFullPageTables {
		rp := rp
		t.Run(rp.note, func(t *testing.T) {
			pdf := realPDFPath(t, rp.pdf)
			img := renderPDFPage(t, pdf, rp.page)
			cells, err := a.TSR(ctx, img)
			if err != nil {
				t.Fatalf("TSR: %v", err)
			}
			got := make([][]float64, 0, len(cells))
			for _, c := range cells {
				got = append(got, []float64{c.X0, c.Y0, c.X1, c.Y1, 0, float64(labelKey(labels, c.Label))})
			}
			py := pythonRasterOracle(t, root, pdf, rp.page, "tsr")
			goldRaw, _ := py["bboxes"].([]any)
			gold := parseBBoxes(t, goldRaw)
			if len(gold) == 0 || len(got) == 0 {
				t.Fatalf("empty sides: gold %d go %d", len(gold), len(got))
			}
			matched, maxd, _ := native.MatchBoxesRelaxed(t, gold, got, native.CmpTolCoord, 1.0)
			worst = math.Max(worst, maxd)
			t.Logf("TSR fullpage %s p%d: matched %d/%d, maxd %.3f px, go cells %d",
				rp.pdf, rp.page, matched, len(gold), maxd, len(got))

			key := rp.pdf + "|" + fmt.Sprintf("%d", rp.page)
			if base, ok := tsrKnownHard[key]; ok {
				// Documented exception: model-level cell-count divergence, not a
				// rasterization floor. Guard against REGRESSION only (must not
				// get worse than the baseline measured here).
				if matched < base {
					t.Errorf("TSR fullpage %s p%d: matched %d/%d < known-hard baseline %d (regression on hard table)",
						rp.pdf, rp.page, matched, len(gold), base)
				}
				t.Logf("TSR fullpage %s p%d: KNOWN-HARD (model cell-count divergence, documented); baseline %d",
					rp.pdf, rp.page, base)
				return
			}
			// Moderate tables: every Python cell must have a Go twin within the
			// 3.5px floor, and Go must not hallucinate extra cells.
			if matched != len(gold) {
				t.Errorf("TSR fullpage %s p%d: %d/%d cells matched (exceeds 3.5px floor)",
					rp.pdf, rp.page, matched, len(gold))
			}
			if len(got) > len(gold) {
				t.Errorf("TSR fullpage %s p%d: Go emits %d cells > golden %d (hallucination)",
					rp.pdf, rp.page, len(got), len(gold))
			}
			if os.Getenv("TSR_FULLPAGE_STRICT") == "1" {
				if matched != len(gold) {
					t.Errorf("TSR fullpage strict: matched %d/%d", matched, len(gold))
				}
			}
		})
	}
	t.Logf("TSR full-page table floor: worst maxd across %d pages = %.3f px (moderate tables ≤3.5px; dense technical-standard is the documented exception)", len(tsrFullPageTables), worst)
}

// parseBBoxes reads a []any JSON array of [x0,y0,x1,y1,score,class] rows into
// the [][]float64 shape MatchBoxesRelaxed expects.
func parseBBoxes(t *testing.T, raw any) [][]float64 {
	t.Helper()
	arr, ok := raw.([]any)
	if !ok {
		t.Fatalf("bboxes not an array: %T", raw)
	}
	out := make([][]float64, 0, len(arr))
	for _, row := range arr {
		r, ok := row.([]any)
		if !ok || len(r) < 6 {
			t.Fatalf("bad bbox row: %v", row)
		}
		v := make([]float64, 6)
		for i := 0; i < 6; i++ {
			switch n := r[i].(type) {
			case float64:
				v[i] = n
			case json.Number:
				f, _ := n.Float64()
				v[i] = f
			default:
				t.Fatalf("bad bbox coord %v", r[i])
			}
		}
		out = append(out, v)
	}
	return out
}

// parseDetQuads reads a det oracle {"output": [[quads]]} into the
// [][][2]float64 shape FlattenQuads/MatchIoUBothDirections expect.
func parseDetQuads(t *testing.T, py map[string]any) [][][2]float64 {
	t.Helper()
	outRaw, ok := py["output"].([]any)
	if !ok || len(outRaw) == 0 {
		t.Fatalf("det output missing: %v", py)
	}
	pages, ok := outRaw[0].([]any)
	if !ok || len(pages) == 0 {
		t.Fatalf("det output[0] missing")
	}
	quads, ok := pages[0].([]any)
	if !ok {
		t.Fatalf("det quads missing")
	}
	out := make([][][2]float64, 0, len(quads))
	for _, q := range quads {
		pts, ok := q.([]any)
		if !ok || len(pts) != 4 {
			t.Fatalf("bad quad: %v", q)
		}
		quad := [][2]float64{}
		for _, p := range pts {
			xy, ok := p.([]any)
			if !ok || len(xy) != 2 {
				t.Fatalf("bad point: %v", p)
			}
			var pt [2]float64
			for i := 0; i < 2; i++ {
				switch n := xy[i].(type) {
				case float64:
					pt[i] = n
				case json.Number:
					f, _ := n.Float64()
					pt[i] = f
				default:
					t.Fatalf("bad coord %v", xy[i])
				}
			}
			quad = append(quad, pt)
		}
		out = append(out, quad)
	}
	return out
}