RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): bound the crop page cache by bytes and release pages when a crop task panics (#21880)
Xavierando16c81d3017f9
diff --git a/internal/ingestion/component/chunker/pdfcrop_cgo.go b/internal/ingestion/component/chunker/pdfcrop_cgo.goindex adb840babc..30d0e18437 100644--- a/internal/ingestion/component/chunker/pdfcrop_cgo.go+++ b/internal/ingestion/component/chunker/pdfcrop_cgo.go@@ -126,7 +126,7 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [// relied on in-order processing and is unsafe under fan-out. render is// called outside the lock; the warning on failure is logged here so the// cache stays render-agnostic and unit-testable.- cache := newLRUPageCache(pageCacheLimit())+ cache := newLRUPageCache(pageCacheLimit(), pageCacheBytes())render := func(pn int) (image.Image, bool) {img, rerr := deepdocpdf.RenderPageToImage(engine, pn)if rerr != nil || img == nil {@@ -202,39 +202,7 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [return}- single := make(map[int]image.Image, len(positions))- acquired := make([]int, 0, len(positions))- for _, pos := range positions {- for _, pn := range pos.PageNumbers {- if _, ok := single[pn]; ok {- continue- }- if img, ok := cache.Acquire(render, pn); ok {- single[pn] = img- acquired = append(acquired, pn)- }- }- }- // Proceed whenever at least one spanned page resolved to an- // image — freshly rendered or served from the page cache.- if len(single) == 0 {- return- }- var img string- if cropChunkType(ck) == "text" {- img = util.CropTextSectionPositions(positions, single, deepdoctype.DlaScale)- } else {- // Media previews are clean position-only crops (Python- // cropout): no 120pt context bands, no edge dimming. The- // text path above is the only one that keeps context.- img = util.CropMediaSectionPositions(positions, single, deepdoctype.DlaScale)- }- // Release the page bitmaps now that cropping is done. They are no- // longer needed and may be evicted under the LRU cap; the image- // below is encoded from single and does not depend on the pages.- for _, pn := range acquired {- cache.Release(pn)- }+ img := cropChunkPreview(cache, render, cropChunkType(ck), positions)if img == "" {return}@@ -291,6 +259,41 @@ func cropImageChunks(ctx context.Context, engine deepdoctype.PDFEngine, chunks [return out}+// cropChunkPreview crops the preview of one chunk from the pages its positions+// span and returns it PNG-encoded in base64, or "" when no page resolved.+// The pages are released when cropping ends, even if it panics: the crop pool+// recovers the panic, and a page left referenced would stay pinned in cache.+func cropChunkPreview(cache *lruPageCache, render func(int) (image.Image, bool), chunkType string, positions []deepdoctype.Position) string {+ pages := make(map[int]image.Image, len(positions))+ defer func() {+ for pn := range pages {+ cache.Release(pn)+ }+ }()+ for _, pos := range positions {+ for _, pn := range pos.PageNumbers {+ if _, ok := pages[pn]; ok {+ continue+ }+ if img, ok := cache.Acquire(render, pn); ok {+ pages[pn] = img+ }+ }+ }+ // Proceed whenever at least one spanned page resolved to an image —+ // freshly rendered or served from the page cache.+ if len(pages) == 0 {+ return ""+ }+ if chunkType == "text" {+ return util.CropTextSectionPositions(positions, pages, deepdoctype.DlaScale)+ }+ // Media previews are clean position-only crops (Python cropout): no 120pt+ // context bands, no edge dimming. The text path above is the only one that+ // keeps context.+ return util.CropMediaSectionPositions(positions, pages, deepdoctype.DlaScale)+}+// needsCrop reports whether a chunk should be cropped to a page-region// preview from its PDF positions. Image/table chunks get their media region// cropped; text chunks with positions get a rendered preview of the textdiff --git a/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go b/internal/ingestion/component/chunker/pdfcrop_concurrency_test.goindex b7924bd781..8aba4fbfd1 100644--- a/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_concurrency_test.go@@ -5,12 +5,15 @@ package chunkerimport ("context""image"+ "image/color"+ "math""strings""sync""testing""time"deepdocpdf "ragflow/internal/deepdoc/parser/pdf"+ deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/schema")@@ -192,3 +195,45 @@ func TestCropImageChunks_ProcessWidePoolLimitsConcurrency(t *testing.T) {release()wg.Wait()}++// unreadableImage is a page bitmap whose pixels cannot be read, so cropping+// from it panics inside the crop task.+type unreadableImage struct{ *image.RGBA }++func (unreadableImage) At(int, int) color.Color { panic("unreadable page bitmap") }++// TestCropChunkPreview_PanicReleasesPages runs a crop that panics through the+// shared crop pool and checks that the pool reports the panic and that every+// page the crop acquired is released, so the shared cache can still evict it.+func TestCropChunkPreview_PanicReleasesPages(t *testing.T) {+ resetCropWorkerPoolForTest(t, 1)+ cache := newLRUPageCache(64, math.MaxInt64)+ render := func(int) (image.Image, bool) {+ return unreadableImage{image.NewRGBA(image.Rect(0, 0, 64, 64))}, true+ }+ positions := []deepdoctype.Position{{PageNumbers: []int{0, 1}, Left: 1, Right: 10, Top: 1, Bottom: 10}}++ ctx := context.Background()+ fut, err := cropWorkerPool().Submit(ctx, func(context.Context) {+ cropChunkPreview(cache, render, "image", positions)+ })+ if err != nil {+ t.Fatalf("submit crop job: %v", err)+ }+ res, err := fut.Wait(ctx)+ if err != nil {+ t.Fatalf("wait crop job: %v", err)+ }+ if res.Err == nil || !strings.Contains(res.Err.Error(), "panic") {+ t.Fatalf("crop job error = %v, want the recovered panic", res.Err)+ }+ for _, pn := range []int{0, 1} {+ el, ok := cache.items[pn]+ if !ok {+ t.Fatalf("page %d was never acquired", pn)+ }+ if ref := el.Value.(pageCacheEntry).ref; ref != 0 {+ t.Errorf("page %d ref = %d after the crop panicked, want 0", pn, ref)+ }+ }+}diff --git a/internal/ingestion/component/chunker/pdfcrop_pagecache.go b/internal/ingestion/component/chunker/pdfcrop_pagecache.goindex ea78ade30f..b7a4729dc8 100644--- a/internal/ingestion/component/chunker/pdfcrop_pagecache.go+++ b/internal/ingestion/component/chunker/pdfcrop_pagecache.go@@ -12,13 +12,16 @@// worker is actively cropping from it. When the last user Release-s it// (ref → 0) it becomes eligible for eviction — so a page an in-flight// worker still needs is never dropped, and never re-rendered.-// - LRU cap: at most cap pages are kept; when over cap the least-recently-// used ref==0 pages are evicted, bounding peak memory on long PDFs.+// - LRU caps: at most cap pages and maxBytes of bitmap memory are kept; when+// over either cap the least-recently used ref==0 pages are evicted. The+// page cap bounds long PDFs; the byte cap bounds documents with few but+// large pages, where the page cap alone still allows tens of gigabytes.package chunkerimport ("container/list""image"+ "math""os""strconv""sync"@@ -27,9 +30,10 @@ import (// pageCacheEntry pairs a page number with its rendered bitmap and a reference// count for storage in the LRU list.type pageCacheEntry struct {- pn int- img image.Image- ref int+ pn int+ img image.Image+ bytes int64+ ref int}// lruPageCache is a concurrency-safe bounded LRU of rendered page bitmaps with@@ -37,26 +41,38 @@ type pageCacheEntry struct {// can never lose a page it still holds a reference to. Concurrent loads of the// same page are serialized so workers share the loaded bitmap.type lruPageCache struct {- loading [64]sync.Mutex- mu sync.Mutex- ll *list.List // front = most-recently-used; element value = pageCacheEntry- items map[int]*list.Element- cap int+ loading [64]sync.Mutex+ mu sync.Mutex+ ll *list.List // front = most-recently-used; element value = pageCacheEntry+ items map[int]*list.Element+ cap int+ maxBytes int64+ bytes int64 // bitmap bytes of every retained page, in use or not}-// newLRUPageCache returns a cache that retains at most capacity pages. A-// capacity below 1 is clamped to 1.-func newLRUPageCache(capacity int) *lruPageCache {+// newLRUPageCache returns a cache that retains at most capacity pages and+// maxBytes of page bitmaps. A capacity below 1 is clamped to 1. A page larger+// than maxBytes is still served to the workers that hold it, and dropped as+// soon as the last one releases it.+func newLRUPageCache(capacity int, maxBytes int64) *lruPageCache {if capacity < 1 {capacity = 1}return &lruPageCache{- ll: list.New(),- items: make(map[int]*list.Element),- cap: capacity,+ ll: list.New(),+ items: make(map[int]*list.Element),+ cap: capacity,+ maxBytes: maxBytes,}}+// imageBytes returns the memory held by a rendered page bitmap, counted at 4+// bytes per pixel (the RGBA layout the page renderers produce).+func imageBytes(img image.Image) int64 {+ b := img.Bounds()+ return int64(b.Dx()) * int64(b.Dy()) * 4+}+// Acquire returns the cached or freshly rendered image for pn and marks it// referenced. The caller must call Release(pn) once cropping from pn is done.// render is invoked outside the lock on a miss, so the CGO pdfium render is@@ -91,9 +107,11 @@ func (c *lruPageCache) Acquire(render func(int) (image.Image, bool), pn int) (imreturn nil, false}+ size := imageBytes(img)c.mu.Lock()- el := c.ll.PushFront(pageCacheEntry{pn: pn, img: img, ref: 1})+ el := c.ll.PushFront(pageCacheEntry{pn: pn, img: img, bytes: size, ref: 1})c.items[pn] = el+ c.bytes += sizec.evictLocked()c.mu.Unlock()return img, true@@ -130,11 +148,11 @@ func (c *lruPageCache) Release(pn int) {}// evictLocked drops least-recently-used pages whose ref is 0 until the cache-// is within capacity. Caller must hold c.mu. It never evicts a page still-// referenced by an in-flight worker, so eviction stops early (never-// over-shrinks) only when every retained page is in use.+// is within both its page and byte caps. Caller must hold c.mu. It never+// evicts a page still referenced by an in-flight worker, so eviction stops+// early (never over-shrinks) only when every retained page is in use.func (c *lruPageCache) evictLocked() {- for c.ll.Len() > c.cap {+ for c.ll.Len() > c.cap || c.bytes > c.maxBytes {var victim *list.Elementfor e := c.ll.Back(); e != nil; e = e.Prev() {if e.Value.(pageCacheEntry).ref == 0 {@@ -145,8 +163,10 @@ func (c *lruPageCache) evictLocked() {if victim == nil {return // all retained pages are in use; cannot shrink further}+ e := victim.Value.(pageCacheEntry)c.ll.Remove(victim)- delete(c.items, victim.Value.(pageCacheEntry).pn)+ delete(c.items, e.pn)+ c.bytes -= e.bytes}}@@ -162,3 +182,16 @@ func pageCacheLimit() int {}return 64}++// pageCacheBytes bounds the bitmap memory of the shared rendered-page cache.+// At 216 DPI an A4 page is ~18 MB but an A0 sheet is ~290 MB, so 64 A0 pages+// would retain ~18 GB under the page cap alone. 1 GiB still covers ~59 A4+// pages; override (in MiB) with RAGFLOW_PAGE_CACHE_MB.+func pageCacheBytes() int64 {+ if v := os.Getenv("RAGFLOW_PAGE_CACHE_MB"); v != "" {+ if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 && n <= math.MaxInt64>>20 {+ return n << 20+ }+ }+ return 1 << 30+}diff --git a/internal/ingestion/component/chunker/pdfcrop_pagecache_test.go b/internal/ingestion/component/chunker/pdfcrop_pagecache_test.goindex ef4602f5cf..d211ae7097 100644--- a/internal/ingestion/component/chunker/pdfcrop_pagecache_test.go+++ b/internal/ingestion/component/chunker/pdfcrop_pagecache_test.go@@ -2,6 +2,8 @@ package chunkerimport ("image"+ "math"+ "strconv""sync""testing""time"@@ -31,7 +33,7 @@ func renderCountingFake(rendered *map[int]int, mu *sync.Mutex) func(int) (image.}func TestLRUPageCache_BasicMissThenHit(t *testing.T) {- c := newLRUPageCache(2)+ c := newLRUPageCache(2, math.MaxInt64)rendered := make(map[int]int)var mu sync.Mutexr := renderCountingFake(&rendered, &mu)@@ -55,7 +57,7 @@ func TestLRUPageCache_BasicMissThenHit(t *testing.T) {}func TestLRUPageCache_EvictsLeastRecent(t *testing.T) {- c := newLRUPageCache(2)+ c := newLRUPageCache(2, math.MaxInt64)rendered := make(map[int]int)var mu sync.Mutexr := renderCountingFake(&rendered, &mu)@@ -86,7 +88,7 @@ func TestLRUPageCache_EvictsLeastRecent(t *testing.T) {}func TestLRUPageCache_GetRefreshesRecency(t *testing.T) {- c := newLRUPageCache(2)+ c := newLRUPageCache(2, math.MaxInt64)rendered := make(map[int]int)var mu sync.Mutexr := renderCountingFake(&rendered, &mu)@@ -105,7 +107,7 @@ func TestLRUPageCache_GetRefreshesRecency(t *testing.T) {}func TestLRUPageCache_CapacityClampedTo1(t *testing.T) {- c := newLRUPageCache(0)+ c := newLRUPageCache(0, math.MaxInt64)rendered := make(map[int]int)var mu sync.Mutexr := renderCountingFake(&rendered, &mu)@@ -125,7 +127,7 @@ func TestLRUPageCache_CapacityClampedTo1(t *testing.T) {// be evicted under the LRU cap, so the worker never loses the bitmap it is// still cropping from and never re-renders it.func TestLRUPageCache_InFlightPageNotEvicted(t *testing.T) {- c := newLRUPageCache(2)+ c := newLRUPageCache(2, math.MaxInt64)rendered := make(map[int]int)var mu sync.Mutexr := renderCountingFake(&rendered, &mu)@@ -148,7 +150,7 @@ func TestLRUPageCache_InFlightPageNotEvicted(t *testing.T) {}func TestLRUPageCache_ReleasedPageBecomesEvictable(t *testing.T) {- c := newLRUPageCache(2)+ c := newLRUPageCache(2, math.MaxInt64)rendered := make(map[int]int)var mu sync.Mutexr := renderCountingFake(&rendered, &mu)@@ -170,7 +172,7 @@ func TestLRUPageCache_ReleasedPageBecomesEvictable(t *testing.T) {}func TestLRUPageCache_ConcurrentSafe(t *testing.T) {- c := newLRUPageCache(8)+ c := newLRUPageCache(8, math.MaxInt64)var wg sync.WaitGroupfor g := 0; g < 32; g++ {wg.Add(1)@@ -208,7 +210,7 @@ func TestLRUPageCache_ConcurrentSamePageRendersOnce(t *testing.T) {workers = 8pn = 7)- c := newLRUPageCache(4)+ c := newLRUPageCache(4, math.MaxInt64)var (mu sync.Mutex@@ -295,7 +297,7 @@ func TestLRUPageCache_ConcurrentSamePageRendersOnce(t *testing.T) {// not take the page-load stripe. Pages 7 and 71 share a stripe (71%64 == 7);// rendering page 71 must not delay a hit on page 7.func TestLRUPageCache_HitDoesNotWaitForSameStripeRender(t *testing.T) {- c := newLRUPageCache(4)+ c := newLRUPageCache(4, math.MaxInt64)if _, ok := c.Acquire(func(int) (image.Image, bool) { return fakeImg(7), true }, 7); !ok {t.Fatal("prime page 7")}@@ -375,3 +377,90 @@ func TestPageCacheLimit_DefaultAndOverride(t *testing.T) {t.Fatalf("invalid override should fall back to 64, got %d", got)}}++// sizedRender returns a render func whose page pn is a w×w RGBA bitmap+// (4*w*w bytes) for w = widths[pn], counting renders per page.+func sizedRender(widths map[int]int, rendered map[int]int) func(int) (image.Image, bool) {+ return func(pn int) (image.Image, bool) {+ rendered[pn]+++ w := widths[pn]+ return image.NewRGBA(image.Rect(0, 0, w, w)), true+ }+}++// TestLRUPageCache_ByteBudgetEvictsLeastRecent proves the byte cap evicts idle+// pages even when the page cap has room: three 400-byte pages under an+// 800-byte budget keep only the two most recent.+func TestLRUPageCache_ByteBudgetEvictsLeastRecent(t *testing.T) {+ c := newLRUPageCache(64, 800)+ rendered := make(map[int]int)+ r := sizedRender(map[int]int{1: 10, 2: 10, 3: 10}, rendered)+ for _, pn := range []int{1, 2, 3} {+ if _, ok := c.Acquire(r, pn); !ok {+ t.Fatalf("acquire page %d failed", pn)+ }+ c.Release(pn)+ }+ if got := c.ll.Len(); got != 2 {+ t.Fatalf("retained pages = %d, want 2 (byte budget holds two 400-byte pages)", got)+ }+ if c.bytes != 800 {+ t.Fatalf("retained bytes = %d, want 800", c.bytes)+ }+ if _, ok := c.items[1]; ok {+ t.Fatal("page 1 is least recent and should have been evicted by the byte budget")+ }+ c.Acquire(r, 1)+ c.Release(1)+ if rendered[1] != 2 {+ t.Fatalf("page 1 rendered %d times, want 2 (re-render after byte eviction)", rendered[1])+ }+}++// TestLRUPageCache_OversizedPageServedThenDropped proves a page larger than the+// whole byte budget is still served to its holder, is not evicted while held,+// and is dropped as soon as it is released.+func TestLRUPageCache_OversizedPageServedThenDropped(t *testing.T) {+ c := newLRUPageCache(64, 800)+ rendered := make(map[int]int)+ r := sizedRender(map[int]int{1: 10, 9: 20}, rendered) // 400 and 1600 bytes+ c.Acquire(r, 1)+ c.Release(1)++ img, ok := c.Acquire(r, 9)+ if !ok || img.Bounds().Dx() != 20 {+ t.Fatalf("oversized page not served: ok=%v img=%v", ok, img)+ }+ if _, held := c.items[9]; !held {+ t.Fatal("oversized page evicted while still referenced")+ }+ if _, kept := c.items[1]; kept {+ t.Fatal("idle page 1 should have been evicted to make room")+ }+ c.Release(9)+ if c.ll.Len() != 0 || c.bytes != 0 {+ t.Fatalf("after release: %d pages / %d bytes retained, want 0 / 0", c.ll.Len(), c.bytes)+ }+}++func TestPageCacheBytes_DefaultAndOverride(t *testing.T) {+ const def = int64(1 << 30)+ maxMB := int64(math.MaxInt64 >> 20)+ for _, tc := range []struct {+ env string+ want int64+ }{+ {"", def},+ {"256", 256 << 20},+ {strconv.FormatInt(maxMB, 10), maxMB << 20},+ {"0", def},+ {"-1", def},+ {"lots", def},+ {strconv.FormatInt(maxMB+1, 10), def}, // overflows when shifted into bytes+ } {+ t.Setenv("RAGFLOW_PAGE_CACHE_MB", tc.env)+ if got := pageCacheBytes(); got != tc.want {+ t.Errorf("RAGFLOW_PAGE_CACHE_MB=%q: pageCacheBytes() = %d, want %d", tc.env, got, tc.want)+ }+ }+}