Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

6.4 KB
Raw Blame
package layout

import (
	"testing"

	pdf "ragflow/internal/deepdoc/parser/pdf/type"
)

func TestCharsToBoxes(t *testing.T) {
	t.Run("empty chars", func(t *testing.T) {
		if boxes := CharsToBoxes(nil, 0, false); boxes != nil {
			t.Error("nil chars → nil boxes")
		}
		if boxes := CharsToBoxes([]pdf.TextChar{}, 0, false); boxes != nil {
			t.Error("empty chars → nil boxes")
		}
	})
	t.Run("single char", func(t *testing.T) {
		chars := []pdf.TextChar{
			{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "A", PageNumber: 0},
		}
		boxes := CharsToBoxes(chars, 0, false)
		if len(boxes) != 1 {
			t.Fatalf("expected 1 box, got %d", len(boxes))
		}
		if boxes[0].Text != "A" {
			t.Errorf("Text = %q, want 'A'", boxes[0].Text)
		}
	})
	t.Run("two lines", func(t *testing.T) {
		chars := []pdf.TextChar{
			{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "A", PageNumber: 0},
			{X0: 60, X1: 68, Top: 100, Bottom: 112, Text: "B", PageNumber: 0},
			{X0: 50, X1: 58, Top: 114, Bottom: 126, Text: "C", PageNumber: 0},
		}
		boxes := CharsToBoxes(chars, 0, false)
		if len(boxes) != 2 {
			t.Errorf("expected 2 lines, got %d", len(boxes))
		}
	})
	t.Run("preserves whitespace lines", func(t *testing.T) {
		chars := []pdf.TextChar{
			{Text: " ", X0: 10, Top: 100, X1: 15, Bottom: 112},
			{Text: "Hello", X0: 10, Top: 120, X1: 50, Bottom: 132},
		}
		boxes := CharsToBoxes(chars, 0, false)
		if len(boxes) != 2 {
			t.Errorf("expected 2 boxes (whitespace preserved), got %d", len(boxes))
		}
	})
}

func TestGroupCharsToLines(t *testing.T) {
	t.Run("empty", func(t *testing.T) {
		if lines := GroupCharsToLines(nil, false); lines != nil {
			t.Error("nil → nil")
		}
	})
	t.Run("single char", func(t *testing.T) {
		chars := []pdf.TextChar{{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "A"}}
		lines := GroupCharsToLines(chars, false)
		if len(lines) != 1 || len(lines[0]) != 1 {
			t.Error("single char → single line")
		}
	})
	t.Run("multi column same line", func(t *testing.T) {
		chars := []pdf.TextChar{
			{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "H"},
			{X0: 60, X1: 68, Top: 100, Bottom: 112, Text: "i"},
			{X0: 300, X1: 308, Top: 100, Bottom: 112, Text: "B"},
			{X0: 50, X1: 58, Top: 114, Bottom: 126, Text: "A"},
		}
		lines := GroupCharsToLines(chars, false)
		if len(lines) != 2 {
			t.Errorf("expected 2 lines, got %d", len(lines))
		}
	})
}

func TestLineToTextBox(t *testing.T) {
	t.Run("basic", func(t *testing.T) {
		chars := []pdf.TextChar{
			{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "H"},
			{X0: 60, X1: 68, Top: 100, Bottom: 112, Text: "i"},
		}
		box := LineToTextBox(chars)
		if box.Text != "Hi" {
			t.Errorf("Text = %q, want 'Hi'", box.Text)
		}
		if box.X0 != 50 || box.X1 != 68 {
			t.Errorf("bbox = [%f, %f], want [50, 68]", box.X0, box.X1)
		}
	})
	t.Run("empty chars", func(t *testing.T) {
		box := LineToTextBox(nil)
		if box.Text != "" {
			t.Errorf("nil chars → empty box, got %q", box.Text)
		}
	})
	t.Run("single char", func(t *testing.T) {
		chars := []pdf.TextChar{{X0: 10, X1: 18, Top: 100, Bottom: 112, Text: "X"}}
		box := LineToTextBox(chars)
		if box.Text != "X" {
			t.Errorf("Text = %q, want 'X'", box.Text)
		}
	})
	// Word boundaries must be recovered for non-Latin scripts too. Before the fix
	// the gap rule was gated on an ASCII-only regex, so Cyrillic chars never got a
	// space regardless of the gap between them (a whole page welded into one token).
	t.Run("cyrillic word gap", func(t *testing.T) {
		chars := []pdf.TextChar{
			// "Ок" — kern-sized gap (2, ~0.25 width): same word
			{X0: 50, X1: 58, Top: 100, Bottom: 112, Text: "О"},
			{X0: 60, X1: 68, Top: 100, Bottom: 112, Text: "к"},
			// large gap before "н": a word boundary
			{X0: 120, X1: 128, Top: 100, Bottom: 112, Text: "н"},
		}
		box := LineToTextBox(chars)
		if box.Text != "Ок н" {
			t.Errorf("Text = %q, want 'Ок н' (space inserted only at the word gap)", box.Text)
		}
	})
	// CJK does not separate words with spaces, so a gap between CJK glyphs is
	// ordinary tracking and must NOT become one, however wide.
	t.Run("cjk gap is not a word boundary", func(t *testing.T) {
		chars := []pdf.TextChar{
			{X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"},
			// gap of 4 on 6-wide glyphs — far over the 0.25*mean threshold
			{X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "丙"},
		}
		box := LineToTextBox(chars)
		if box.Text != "乙丙" {
			t.Errorf("Text = %q, want '乙丙' (no space between CJK glyphs)", box.Text)
		}
	})
	// A CJK/Latin boundary is likewise not a space the geometry may invent.
	t.Run("cjk adjacent to latin gets no invented space", func(t *testing.T) {
		chars := []pdf.TextChar{
			{X0: 12, X1: 18, Top: 100, Bottom: 112, Text: "乙"},
			{X0: 22, X1: 28, Top: 100, Bottom: 112, Text: "A"},
		}
		box := LineToTextBox(chars)
		if box.Text != "乙A" {
			t.Errorf("Text = %q, want '乙A'", box.Text)
		}
	})
}

// TestCharsToBoxesKeepsSuperscriptInPlace pins the X-first ordering inside a
// fuzzy Y band. GroupCharsToLines sorts by Bottom, so a raised superscript
// arrives before the base line; without the re-order it is emitted at the
// start of the line ("2I C" instead of "I2C").
func TestCharsToBoxesKeepsSuperscriptInPlace(t *testing.T) {
	chars := []pdf.TextChar{
		{X0: 8, X1: 13, Top: 0, Bottom: 10, Text: "C", PageNumber: 0},
		{X0: 5, X1: 8, Top: -3, Bottom: 2, Text: "2", PageNumber: 0}, // superscript
		{X0: 0, X1: 5, Top: 0, Bottom: 10, Text: "I", PageNumber: 0},
	}
	boxes := CharsToBoxes(chars, 0, false)
	if len(boxes) != 1 {
		t.Fatalf("boxes = %d, want 1", len(boxes))
	}
	if boxes[0].Text != "I2C" {
		t.Fatalf("Text = %q, want 'I2C' (superscript stays between I and C)", boxes[0].Text)
	}
}

// TestCharsToBoxesOrdersChainedBaselinesTransitively pins a strict weak
// ordering. A pairwise |Top difference| comparator cycles for Top 0/4/8 with
// band 5 (0<4, 4<8, but 8 is not within 5 of 0), which sort.SliceStable does
// not permit; bands are assigned before sorting instead.
func TestCharsToBoxesOrdersChainedBaselinesTransitively(t *testing.T) {
	chars := []pdf.TextChar{
		{X0: 10, X1: 15, Top: 0, Bottom: 10, Text: "A", PageNumber: 0},
		{X0: 5, X1: 10, Top: 4, Bottom: 14, Text: "B", PageNumber: 0},
		{X0: 0, X1: 5, Top: 8, Bottom: 18, Text: "C", PageNumber: 0},
	}
	boxes := CharsToBoxes(chars, 0, false)
	if len(boxes) != 1 {
		t.Fatalf("boxes = %d, want 1", len(boxes))
	}
	if boxes[0].Text != "CBA" {
		t.Fatalf("Text = %q, want 'CBA' (one chained band in X order)", boxes[0].Text)
	}
}