RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(deepdoc): sample English pages deterministically (#20689)
leilei3167a87168bf3e03
diff --git a/internal/deepdoc/parser/pdf/tool/char_adapter.go b/internal/deepdoc/parser/pdf/tool/char_adapter.goindex 6a92bed42e..b965322c6d 100644--- a/internal/deepdoc/parser/pdf/tool/char_adapter.go+++ b/internal/deepdoc/parser/pdf/tool/char_adapter.go@@ -43,9 +43,12 @@ func LoadPythonChars(jsonPath string) (*PythonCharEngine, error) {// pushing every box off-page — LoadPythonChars fails fast instead.Dims [][]int `json:"dims"`// IsEnglish is the document-level english verdict from the SAME Python- // run that produced the text golden. Capturing it (instead of- // re-deriving via random sampling) keeps the replay input identical to- // Python's. Optional: older dumps omit it.+ // run that produced the text golden. Capturing it keeps the replay+ // input identical to Python's. Re-deriving with the Go sampler is+ // reproducible, but it is not Python's random.choices distribution:+ // a contiguous English block of about 30% of a page votes English+ // here, and an unseeded draw of that density does not. Optional:+ // older dumps omit it.IsEnglish *bool `json:"is_english"`}if err := json.Unmarshal(data, &wrapper); err != nil {@@ -103,8 +106,9 @@ func (e *PythonCharEngine) PageChars() map[int][]pdf.TextChar { return e.chars }// IsEnglish returns the document-level english verdict captured from the same// Python run that produced the dump, or nil when the dump omits it (older-// charspy files). Prefer it over re-deriving via random sampling, which is-// unstable on documents with small english runs.+// charspy files). Prefer the captured verdict. The Go sampler is+// deterministic, and it still does not follow Python's random.choices+// distribution when the English text is one modest contiguous block.func (e *PythonCharEngine) IsEnglish() *bool { return e.isEnglish }// PageDims returns per-page image dimensions [w,h] in ZM/image pixels. Theydiff --git a/internal/deepdoc/parser/pdf/util/eng_detect.go b/internal/deepdoc/parser/pdf/util/eng_detect.goindex c72ebe637f..ac1d465341 100644--- a/internal/deepdoc/parser/pdf/util/eng_detect.go+++ b/internal/deepdoc/parser/pdf/util/eng_detect.go@@ -1,7 +1,8 @@package utilimport (- "math/rand/v2"+ "maps"+ "slices""strings"pdf "ragflow/internal/deepdoc/parser/pdf/type"@@ -32,34 +33,118 @@ func IsASCIIPrintable(r rune) bool {return false}-// DefaultSampleChars returns a random sample of up to n character texts,-// concatenated. Matches Python's random.choices([c["text"] for c in-// page_chars], k=min(100, len(page_chars))).+// DefaultSampleChars returns up to n character texts, concatenated. The page+// is split into that many equal spans in page order, and each span gives one+// character. A span is English when the page has a run of 30 consecutive+// ASCII characters and more than half of the span is English text: ASCII+// characters, or a gap between two such runs that is shorter than each of+// them. An English span gives its first ASCII character; any other span+// gives its first non-ASCII character when it has one.+//+// A page with no real 30-character run therefore never samples one. The+// page votes English when the sample contains 30 ASCII characters in a+// row, so a contiguous English block flips that vote at about 30% of the+// page. On pages of 1000 to 8000 characters, a block of 29% stays within+// 29 spans at every offset. A block of 30% covers 30 spans when it starts+// on a span boundary, and can cover only 29 when it starts in the middle+// of a span (300 characters at offset 5 on a 1000-character page). A block+// of 31% covers 30 spans at every offset. A 30-character heading on a long,+// mostly CJK page still covers only a few spans, so it does not vote+// English.+//+// Python's random.choices draws character texts independently, with+// replacement, in an unseeded order. A reproducible sample is not that+// distribution: at an ASCII share of about 30 to 60 percent, random.choices+// almost never draws 30 ASCII characters in a row, even when those+// characters form one contiguous block. This sampler walks the page in+// order, so the block is a run in the sample. Whether punctuation inside+// English text is bridged depends only on the text around it, not on page+// length or where it falls against the spans. The same page always yields+// the same sample.func DefaultSampleChars(chars []pdf.TextChar, n int) string {if n <= 0 || len(chars) == 0 {return ""}+ text := make([]bool, len(chars))+ var runs [][2]int+ run, runStart := 0, 0+ for i, c := range chars {+ text[i] = !breaksASCIIRun(c.Text)+ for _, r := range c.Text {+ if !IsASCIIPrintable(r) {+ run = 0+ continue+ }+ if run == 0 {+ runStart = i+ }+ run+++ if run == 30 {+ runs = append(runs, [2]int{runStart, i})+ } else if run > 30 {+ runs[len(runs)-1][1] = i+ }+ }+ }+ for k := 1; k < len(runs); k++ {+ prev, next := runs[k-1], runs[k]+ gap := next[0] - prev[1] - 1+ if gap < prev[1]-prev[0]+1 && gap < next[1]-next[0]+1 {+ for j := prev[1] + 1; j < next[0]; j++ {+ text[j] = true+ }+ }+ }+m := min(n, len(chars))- // Fisher-Yates shuffle on indices, then take first m.- indices := make([]int, len(chars))- for i := range indices {- indices[i] = i- }- rand.Shuffle(len(indices), func(i, j int) {- indices[i], indices[j] = indices[j], indices[i]- })var buf strings.Builder- for i := 0; i < m; i++ {- buf.WriteString(chars[indices[i]].Text)+ for i := range m {+ start := i * len(chars) / m+ end := (i + 1) * len(chars) / m+ english := false+ if len(runs) > 0 {+ k := 0+ for j := start; j < end; j++ {+ if text[j] {+ k+++ }+ }+ english = k*2 > end-start+ }+ // An English span that lies inside a bridged gap has no ASCII+ // character of its own; it gives the first one of the next run.+ limit := end+ if english {+ limit = len(chars)+ }+ pick := start+ for j := start; j < limit; j++ {+ if breaksASCIIRun(chars[j].Text) != english {+ pick = j+ break+ }+ }+ buf.WriteString(chars[pick].Text)}return buf.String()}-// FullTextFromChars concatenates all chars text across pages for scan noise detection.+// breaksASCIIRun reports whether s contains a rune outside an English run.+func breaksASCIIRun(s string) bool {+ for _, r := range s {+ if !IsASCIIPrintable(r) {+ return true+ }+ }+ return false+}++// FullTextFromChars concatenates character text in ascending page number.+// Map iteration order would otherwise change the string from call to call.func FullTextFromChars(pageChars map[int][]pdf.TextChar) string {var sb strings.Builder- for _, chars := range pageChars {- for _, c := range chars {+ for _, page := range slices.Sorted(maps.Keys(pageChars)) {+ for _, c := range pageChars[page] {sb.WriteString(c.Text)}}@@ -67,7 +152,7 @@ func FullTextFromChars(pageChars map[int][]pdf.TextChar) string {}// DetectEnglishPage reports whether one page contains a run of 30+-// consecutive ASCII-printable characters in a random sample of up to 100+// consecutive ASCII-printable characters in a sample of up to 100// extracted character texts.func DetectEnglishPage(chars []pdf.TextChar, sample pdf.SampleFunc) bool {if len(chars) == 0 {diff --git a/internal/deepdoc/parser/pdf/util/eng_detect_test.go b/internal/deepdoc/parser/pdf/util/eng_detect_test.goindex dba232121b..1ee17f452f 100644--- a/internal/deepdoc/parser/pdf/util/eng_detect_test.go+++ b/internal/deepdoc/parser/pdf/util/eng_detect_test.go@@ -52,18 +52,568 @@ func TestDefaultSampleChars(t *testing.T) {}}+func TestDefaultSampleCharsStable(t *testing.T) {+ chars := make([]pdf.TextChar, 40)+ for i := range chars {+ chars[i] = pdf.TextChar{Text: string(rune('a' + i%26))}+ }+ // All ASCII, so each span of four gives its first character.+ const want = "aeimquycgk"+ var first string+ for range 30 {+ got := DefaultSampleChars(chars, 10)+ if first == "" {+ first = got+ continue+ }+ if got != first {+ t.Fatalf("DefaultSampleChars changed across calls: %q vs %q", first, got)+ }+ }+ if first != want {+ t.Fatalf("DefaultSampleChars = %q, want %q", first, want)+ }+}++func TestDetectEnglishPageStable(t *testing.T) {+ // 29 ASCII, one CJK, 29 ASCII. In page order the longest ASCII run is 29,+ // so the page is not English. A shuffle of the same characters usually+ // builds a run of 30 and flips the vote.+ chars := make([]pdf.TextChar, 59)+ for i := range chars {+ if i == 29 {+ chars[i] = pdf.TextChar{Text: "你"}+ continue+ }+ chars[i] = pdf.TextChar{Text: "A"}+ }+ first := DetectEnglishPage(chars, nil)+ for range 20 {+ if got := DetectEnglishPage(chars, nil); got != first {+ t.Fatalf("DetectEnglishPage flipped: %v then %v", first, got)+ }+ }+ if first {+ t.Fatal("DetectEnglishPage = true, want false")+ }+}++func TestDetectEnglishPageAlternatingDoesNotAlias(t *testing.T) {+ // 200 glyphs repeating a short mix. A fixed stride of len/n == 2 samples+ // only A from A/你, and a cycling span offset samples only A from A你你A.+ // Neither page has a 30-character ASCII run, so the vote must stay false.+ cycles := []struct {+ name string+ cycle []string+ }{+ {name: "alternating", cycle: []string{"A", "你"}},+ {name: "period four", cycle: []string{"A", "你", "你", "A"}},+ }+ for _, tc := range cycles {+ t.Run(tc.name, func(t *testing.T) {+ chars := make([]pdf.TextChar, 200)+ for i := range chars {+ chars[i] = pdf.TextChar{Text: tc.cycle[i%len(tc.cycle)]}+ }+ sample := DefaultSampleChars(chars, 100)+ allA := true+ for _, r := range sample {+ if r != 'A' {+ allA = false+ break+ }+ }+ if allA {+ t.Fatalf("DefaultSampleChars = %q, want a sample that is not all A", sample)+ }+ if DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage = true, want false")+ }+ })+ }+}++func TestDetectEnglishPageNeedsRealRun(t *testing.T) {+ // One CJK glyph per period of 2-12 at every phase, across page lengths+ // that give span widths of one to five. No page has a 30-character ASCII+ // run, so no alignment of glyphs and spans may vote English.+ for period := 2; period <= 12; period++ {+ for phase := range period {+ for pageLen := 101; pageLen <= 513; pageLen++ {+ chars := make([]pdf.TextChar, pageLen)+ for i := range chars {+ chars[i].Text = "A"+ if i%period == phase {+ chars[i].Text = "你"+ }+ }+ if DetectEnglishPage(chars, nil) {+ t.Fatalf("period %d, phase %d, length %d: DetectEnglishPage = true, want false", period, phase, pageLen)+ }+ }+ }+ }+}++func TestDetectEnglishPageLongEnglish(t *testing.T) {+ // Longer than the 100-character sample, so detection takes the span path.+ // A letter, with a curly quote or em dash every 67th character (about 30+ // per 1000). Those marks are not ASCII-printable, and the page is still+ // English: the sample has to keep a 30-character letter run.+ const pageLen = 4000+ chars := make([]pdf.TextChar, pageLen)+ for i := range chars {+ switch (i + 1) % 67 {+ case 0:+ chars[i] = pdf.TextChar{Text: "—"}+ case 34:+ chars[i] = pdf.TextChar{Text: "“"}+ default:+ chars[i] = pdf.TextChar{Text: "e"}+ }+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage = false, want true")+ }+ again := DetectEnglishPage(chars, nil)+ if !again {+ t.Fatal("DetectEnglishPage flipped to false")+ }+}++func TestReviewLongEnglishOnePercentQuotes(t *testing.T) {+ // 99% ASCII, one curly quote every 100 characters, including index 0.+ // With 100 samples the span offset is i%10, so those quotes land on+ // every tenth sample and used to break the run before it reached 30.+ chars := make([]pdf.TextChar, 1000)+ for i := range chars {+ chars[i].Text = "e"+ if i%100 == 0 {+ chars[i].Text = "“"+ }+ }+ sample := DefaultSampleChars(chars, 100)+ run := 0+ for _, r := range sample {+ if IsASCIIPrintable(r) {+ run+++ if run >= 30 {+ break+ }+ continue+ }+ run = 0+ }+ if run < 30 {+ t.Fatalf("DefaultSampleChars = %q, want a 30-character ASCII run", sample)+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage = false, want true")+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage flipped to false")+ }+}++func TestDetectEnglishPageSixToTenPercentPunctuation(t *testing.T) {+ // A curly quote every 12 characters, about 8% of the page, so letters+ // between quotes run only 11 long. One clean 40-character stretch is the+ // page's only real 30-character run. Every span is still mostly ASCII,+ // and the page has to stay English.+ const pageLen = 1000+ chars := make([]pdf.TextChar, pageLen)+ placed := 0+ for i := range chars {+ chars[i].Text = "e"+ if i%12 == 0 && (i < 400 || i >= 440) {+ chars[i].Text = "“"+ placed+++ }+ }+ if placed*100 < 6*pageLen || placed*100 > 10*pageLen {+ t.Fatalf("non-ASCII punctuation = %d/%d, want 6-10%%", placed, pageLen)+ }+ if !asciiRunAtLeast(chars, 0, pageLen-1, 30) {+ t.Fatal("fixture has no 30-character ASCII run")+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage = false, want true")+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage flipped to false")+ }+}++func TestDetectEnglishPagePhaseAlignedPunctuation(t *testing.T) {+ // Quotes share the first character of a span and are denser than one in+ // twenty. A raw page-order sample is ten copies of a quote and nine+ // letters, so its longest ASCII run stays under 30. Each page still has+ // real runs of 30 ASCII characters, and the vote has to stay English.+ cases := []struct {+ name string+ fill func([]pdf.TextChar)+ }{+ {+ name: "quote clusters",+ // 92% ASCII: four curly quotes, then 46 letters.+ fill: func(chars []pdf.TextChar) {+ for i := range chars {+ chars[i].Text = "e"+ if i%50 < 4 {+ chars[i].Text = "“"+ }+ }+ },+ },+ {+ name: "quote clusters fill half a span",+ // 90% ASCII: five curly quotes, then 45 letters. Each affected+ // span of ten is exactly half quotes. The quotes sit between two+ // real runs and are shorter than each, so they read as text.+ fill: func(chars []pdf.TextChar) {+ for i := range chars {+ chars[i].Text = "e"+ if i%50 < 5 {+ chars[i].Text = "“"+ }+ }+ },+ },+ {+ name: "quotes then ascii",+ // A curly quote every 10 characters through index 790, then+ // ASCII. Sampled quotes sit at positions 0, 10, ..., 70.+ fill: func(chars []pdf.TextChar) {+ for i := range chars {+ chars[i].Text = "e"+ if i <= 790 && i%10 == 0 {+ chars[i].Text = "“"+ }+ }+ },+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ chars := make([]pdf.TextChar, 1000)+ tc.fill(chars)+ non := 0+ for i := range chars {+ if breaksASCIIRun(chars[i].Text) {+ non+++ }+ }+ if non*20 <= len(chars) {+ t.Fatalf("non-ASCII = %d/%d, want denser than 1 in 20", non, len(chars))+ }+ if !asciiRunAtLeast(chars, 0, len(chars)-1, 30) {+ t.Fatal("fixture has no 30-character ASCII run")+ }+ sample := DefaultSampleChars(chars, 100)+ run := 0+ for _, r := range sample {+ if IsASCIIPrintable(r) {+ run+++ if run >= 30 {+ break+ }+ continue+ }+ run = 0+ }+ if run < 30 {+ t.Fatalf("DefaultSampleChars = %q, want a 30-character ASCII run", sample)+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage = false, want true")+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage flipped to false")+ }+ })+ }+}++func TestDetectEnglishPageQuoteClustersIgnorePhaseAndLength(t *testing.T) {+ // k curly quotes every 50 characters, shifted through every phase, on+ // pages of at least three periods. Span width changes with page length,+ // so neither where the clusters fall nor how long the page is may change+ // the vote. Up to 20 quotes leave real runs of 30 that outlast each+ // cluster, so the page is English; beyond that no real run is left.+ for _, pageLen := range []int{150, 200, 333, 500, 777, 1000, 1500, 2000, 4000} {+ for k := 1; k <= 25; k++ {+ want := k <= 20+ for phase := range 50 {+ chars := make([]pdf.TextChar, pageLen)+ for i := range chars {+ chars[i].Text = "e"+ if (i+phase)%50 < k {+ chars[i].Text = "“"+ }+ }+ if got := DetectEnglishPage(chars, nil); got != want {+ t.Fatalf("length %d, %d quotes, phase %d: DetectEnglishPage = %v, want %v", pageLen, k, phase, got, want)+ }+ }+ }+ }+}++func TestDetectEnglishPageShortEnglishBlockOnCJK(t *testing.T) {+ // A 30-letter heading is the page's only real 30-character run. Letters+ // after it are isolated, so those spans are not English text and the+ // heading alone must not outvote the rest of the page.+ cases := []struct {+ name string+ wantASCII int+ letter func(i int) bool+ }{+ {+ // One letter every 10 characters, CJK elsewhere: 127 ASCII out+ // of 1000, about 87% CJK.+ name: "periodic letters",+ wantASCII: 127,+ letter: func(i int) bool { return i < 30 || i%10 == 5 },+ },+ {+ // Alternating letter and CJK: each later span is exactly half+ // ASCII.+ name: "alternating",+ wantASCII: 515,+ letter: func(i int) bool { return i < 30 || i%2 == 0 },+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ chars := make([]pdf.TextChar, 1000)+ ascii := 0+ for i := range chars {+ chars[i].Text = "你"+ if tc.letter(i) {+ chars[i].Text = "e"+ ascii+++ }+ }+ if ascii != tc.wantASCII {+ t.Fatalf("ASCII characters = %d, want %d", ascii, tc.wantASCII)+ }+ if !asciiRunAtLeast(chars, 0, len(chars)-1, 30) {+ t.Fatal("fixture has no 30-character ASCII run")+ }+ if DetectEnglishPage(chars, nil) {+ t.Fatalf("DetectEnglishPage = true, want false; sample %q", DefaultSampleChars(chars, 100))+ }+ if DetectEnglishPage(chars, nil) {+ t.Fatal("DetectEnglishPage flipped to true")+ }+ })+ }+}++// asciiRunAtLeast reports whether chars[lo] through chars[hi] contain n+// consecutive ASCII-printable runes.+func asciiRunAtLeast(chars []pdf.TextChar, lo, hi, n int) bool {+ run := 0+ for j := lo; j <= hi; j++ {+ for _, r := range chars[j].Text {+ if !IsASCIIPrintable(r) {+ run = 0+ continue+ }+ run+++ if run >= n {+ return true+ }+ }+ }+ return false+}+func TestFullTextFromChars(t *testing.T) {chars := map[int][]pdf.TextChar{0: {{Text: "Hello"}, {Text: " "}, {Text: "World"}},1: {{Text: "Page2"}},}- got := FullTextFromChars(chars)- if !strings.Contains(got, "Hello") || !strings.Contains(got, "World") || !strings.Contains(got, "Page2") {- t.Errorf("FullTextFromChars = %q, missing expected content", got)+ const want = "Hello WorldPage2"+ if got := FullTextFromChars(chars); got != want {+ t.Errorf("FullTextFromChars = %q, want %q", got, want)}if FullTextFromChars(nil) != "" {t.Error("nil should return empty")}++ // Keys are inserted out of page order. Concatenation follows ascending+ // page number, and a second call returns the same text.+ scrambled := map[int][]pdf.TextChar{+ 5: {{Text: "c"}},+ 1: {{Text: "a"}},+ 3: {{Text: "b"}},+ }+ for range 30 {+ if got := FullTextFromChars(scrambled); got != "abc" {+ t.Fatalf("FullTextFromChars = %q, want abc", got)+ }+ }+}++func TestDetectEnglishPageContiguousThreshold(t *testing.T) {+ // These pages vote English. Python's random.choices draws character+ // texts independently, so the same ASCII share almost never produces 30+ // ASCII characters in a row.+ t.Run("30 percent block on CJK", func(t *testing.T) {+ for _, n := range []int{1000, 2000, 4000, 8000} {+ for _, atEnd := range []bool{false, true} {+ block := n * 30 / 100+ lo := 0+ if atEnd {+ lo = n - block+ }+ chars := cjkPage(n)+ for i := lo; i < lo+block; i++ {+ chars[i].Text = "e"+ }+ if !DetectEnglishPage(chars, nil) || !DetectEnglishPage(chars, nil) {+ t.Fatalf("n=%d atEnd=%v: DetectEnglishPage = false, want true", n, atEnd)+ }+ }+ }+ })++ t.Run("50/50 bilingual", func(t *testing.T) {+ for _, n := range []int{1000, 4000, 8000} {+ half := n / 2+ for _, englishFirst := range []bool{true, false} {+ chars := cjkPage(n)+ for i := range chars {+ if (i < half) == englishFirst {+ chars[i].Text = "e"+ }+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatalf("n=%d englishFirst=%v: DetectEnglishPage = false, want true", n, englishFirst)+ }+ }+ }+ })++ t.Run("numeric table", func(t *testing.T) {+ // 60% digits in one block, the rest CJK labels.+ for _, n := range []int{1000, 4000, 8000} {+ block := n * 60 / 100+ for _, lo := range []int{0, n - block} {+ chars := cjkPage(n)+ digits := 0+ for i := lo; i < lo+block; i++ {+ chars[i].Text = string(rune('0' + i%10))+ digits+++ }+ if digits*100/n != 60 {+ t.Fatalf("digits = %d/%d, want 60%%", digits, n)+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatalf("n=%d lo=%d: DetectEnglishPage = false, want true", n, lo)+ }+ }+ }+ })++ t.Run("interleaved with one run", func(t *testing.T) {+ // Three ASCII characters and two CJK, so the page is 60% ASCII and+ // the longest run is 3. One 30-character run then makes each span+ // count as English text.+ for _, n := range []int{1000, 8000} {+ chars := make([]pdf.TextChar, n)+ ascii := 0+ for i := range chars {+ if i%5 < 3 {+ chars[i].Text = "e"+ ascii+++ } else {+ chars[i].Text = "你"+ }+ }+ if ascii*5 != n*3 {+ t.Fatalf("ASCII = %d/%d, want 60%%", ascii, n)+ }+ if DetectEnglishPage(chars, nil) {+ t.Fatal("interleaved page without a 30-character run voted English")+ }+ start := n / 5 * 2+ for i := start; i < start+30; i++ {+ if chars[i].Text != "e" {+ chars[i].Text = "e"+ ascii+++ }+ }+ if ascii*100/n > 62 {+ t.Fatalf("ASCII = %d/%d, want about 60%%", ascii, n)+ }+ if !asciiRunAtLeast(chars, start, start+29, 30) {+ t.Fatal("fixture has no 30-character ASCII run")+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatalf("n=%d: DetectEnglishPage = false, want true; sample %q", n, DefaultSampleChars(chars, 100))+ }+ }+ })++ t.Run("29 and 30 percent boundary", func(t *testing.T) {+ // Pages of 1000 to 8000 characters. 29% of the page never covers 30+ // spans. 31% does, at every offset. 30% does when the block starts+ // on a span boundary, and a 1000-character block of 300 characters+ // at offset 5 covers only 29.+ for _, n := range []int{1000, 1001, 2000, 3456, 4000, 8000} {+ if offset, bad := slidingBlockDisagrees(n, n*29/100, false); bad {+ t.Fatalf("29%% n=%d offset=%d: DetectEnglishPage = true, want false", n, offset)+ }+ if offset, bad := slidingBlockDisagrees(n, n*31/100, true); bad {+ t.Fatalf("31%% n=%d offset=%d: DetectEnglishPage = false, want true", n, offset)+ }+ chars := cjkPage(n)+ block := n * 30 / 100+ for i := range block {+ chars[i].Text = "e"+ }+ if !DetectEnglishPage(chars, nil) {+ t.Fatalf("30%% n=%d offset=0: DetectEnglishPage = false, want true", n)+ }+ }+ chars := cjkPage(1000)+ for i := 5; i < 305; i++ {+ chars[i].Text = "e"+ }+ if DetectEnglishPage(chars, nil) {+ t.Fatal("30% of 1000 characters at offset 5: DetectEnglishPage = true, want false")+ }+ })+}++// cjkPage returns n CJK characters.+func cjkPage(n int) []pdf.TextChar {+ chars := make([]pdf.TextChar, n)+ for i := range chars {+ chars[i].Text = "你"+ }+ return chars+}++// slidingBlockDisagrees slides one ASCII block across a CJK page and reports+// the first offset whose English vote is not want.+func slidingBlockDisagrees(n, block int, want bool) (int, bool) {+ chars := cjkPage(n)+ for i := range block {+ chars[i].Text = "e"+ }+ for offset := 0; ; offset++ {+ if DetectEnglishPage(chars, nil) != want {+ return offset, true+ }+ if offset == n-block {+ return 0, false+ }+ chars[offset].Text = "你"+ chars[offset+block].Text = "e"+ }}func TestDetectEnglish(t *testing.T) {