Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(parser): add opt-in VLM enhancement for images (#20166)

jay7772131fd92c2fa5e
diff --git a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go b/internal/deepdoc/parser/pdf/pdfoxide_bridge.go
index 340d634076..799aad4a8e 100644
--- a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go
+++ b/internal/deepdoc/parser/pdf/pdfoxide_bridge.go
@@ -45,6 +45,11 @@ func (e *PDFOxideEngine) RenderPageImage(pageNum int, dpi float64) (image.Image,
return e.Inner.RenderPageImage(pageNum, dpi)
}
+// PageSize returns the effective dimensions of a zero-based page in PDF points.
+func (e *PDFOxideEngine) PageSize(pageNum int) (float64, float64, error) {
+ return e.Inner.PageSize(pageNum)
+}
+
func (e *PDFOxideEngine) ExtractChars(pageNum int) ([]pdf.TextChar, error) {
chars, err := e.Inner.ExtractChars(pageNum)
if err != nil {
diff --git a/internal/deepdoc/parser/pdf/util/crop.go b/internal/deepdoc/parser/pdf/util/crop.go
index 18f030f638..9bca119466 100644
--- a/internal/deepdoc/parser/pdf/util/crop.go
+++ b/internal/deepdoc/parser/pdf/util/crop.go
@@ -6,7 +6,6 @@ import (
"image"
"image/color"
"math"
- "strings"
"go.uber.org/zap"
@@ -20,23 +19,63 @@ import (
//
// Python: pdf_parser.py:1802 RAGFlowPdfParser.crop()
func CropSectionImage(posTag string, decodedImages map[int]image.Image, zoom float64) string {
+ return encodeCroppedImage(cropSectionImageRaster(posTag, decodedImages, zoom))
+}
+
+func cropSectionImageRaster(posTag string, decodedImages map[int]image.Image, zoom float64) image.Image {
if len(decodedImages) == 0 {
common.Warn("cropSectionImage: no page images available, skipping image generation")
- return ""
+ return nil
}
positions := ExtractPositions(posTag)
if len(positions) == 0 {
common.Warn("cropSectionImage: empty position list in tag", zap.String("posTag", posTag[:min(80, len(posTag))]))
- return ""
+ return nil
}
+ return cropSectionPositionsRaster(positions, decodedImages, zoom)
+}
+
+type sectionRasterSegment struct {
+ page int
+ x0 int
+ y0 int
+ x1 int
+ y1 int
+ isEdge bool
+}
- // Filter valid positions (all pages available).
- var valid []pdf.Position
+type sectionRasterPlan struct {
+ segments []sectionRasterSegment
+ width int
+ height int
+}
+
+const (
+ sectionRasterContextPad = 120.0
+ sectionRasterGap = 6
+)
+
+func cropSectionPositionsRaster(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) image.Image {
+ plan, ok := buildSectionRasterPlan(positions, decodedImages, zoom)
+ if !ok {
+ return nil
+ }
+ return renderSectionRaster(plan, decodedImages)
+}
+
+func buildSectionRasterPlan(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) (*sectionRasterPlan, bool) {
+ if len(decodedImages) == 0 || len(positions) == 0 || zoom <= 0 || math.IsNaN(zoom) || math.IsInf(zoom, 0) {
+ return nil, false
+ }
+ valid := make([]pdf.Position, 0, len(positions))
for _, pos := range positions {
+ if len(pos.PageNumbers) == 0 {
+ continue
+ }
allValid := true
- for _, pn := range pos.PageNumbers {
- if _, ok := decodedImages[pn]; !ok {
+ for _, pageNum := range pos.PageNumbers {
+ if img, ok := decodedImages[pageNum]; !ok || img == nil {
allValid = false
break
}
@@ -46,196 +85,168 @@ func CropSectionImage(posTag string, decodedImages map[int]image.Image, zoom flo
}
}
if len(valid) == 0 {
- common.Warn("cropSectionImage: no valid positions after filtering, skipping crop")
- return ""
+ return nil, false
}
- // Context padding (Python: 120px above first, 120 below last, 6px gap)
- const contextPad = 120.0
- const gap = 6
-
- // Compute max width across original positions for full-width edge bands.
maxWidth := 6.0
for _, pos := range valid {
- w := pos.Right - pos.Left
- if w > maxWidth {
- maxWidth = w
+ if width := pos.Right - pos.Left; width > maxWidth {
+ maxWidth = width
}
}
- // Python-style: insert synthetic context bands at edges.
- // Original positions are all middle entries (narrow width).
- // Synthetic bands are edge entries (full width + semi-transparent overlay).
first := valid[0]
last := valid[len(valid)-1]
- firstPageIdx := first.PageNumbers[0]
- lastPageIdx := last.PageNumbers[len(last.PageNumbers)-1]
- lastPageH := float64(decodedImages[lastPageIdx].Bounds().Dy()) / zoom
-
- // topBand: 120px context above the first content position.
- topBandPos := pdf.Position{
- PageNumbers: []int{firstPageIdx},
+ firstPage := first.PageNumbers[0]
+ lastPage := last.PageNumbers[len(last.PageNumbers)-1]
+ lastPageHeight := float64(decodedImages[lastPage].Bounds().Dy()) / zoom
+ topBand := pdf.Position{
+ PageNumbers: []int{firstPage},
Left: first.Left,
Right: first.Right,
- Top: math.Max(0, first.Top-contextPad),
- Bottom: math.Max(first.Top-gap, 0),
+ Top: math.Max(0, first.Top-sectionRasterContextPad),
+ Bottom: math.Max(first.Top-sectionRasterGap, 0),
}
- // bottomBand: 120px context below the last content position.
- bottomBandPos := pdf.Position{
- PageNumbers: []int{lastPageIdx},
+ bottomBand := pdf.Position{
+ PageNumbers: []int{lastPage},
Left: last.Left,
Right: last.Right,
- Top: math.Min(lastPageH, last.Bottom+gap),
- Bottom: math.Min(lastPageH, last.Bottom+contextPad),
+ Top: math.Min(lastPageHeight, last.Bottom+sectionRasterGap),
+ Bottom: math.Min(lastPageHeight, last.Bottom+sectionRasterContextPad),
}
- // Build entry list: [topBand, original positions..., bottomBand].
- type segment struct {
- img image.Image
- isEdge bool
+ type positionEntry struct {
+ position pdf.Position
+ isEdge bool
}
- var segments []segment
-
- allPos := make([]struct {
- pos pdf.Position
- isEdge bool
- }, 0, len(valid)+2)
- allPos = append(allPos, struct {
- pos pdf.Position
- isEdge bool
- }{topBandPos, true})
+ entries := make([]positionEntry, 0, len(valid)+2)
+ entries = append(entries, positionEntry{position: topBand, isEdge: true})
for _, pos := range valid {
- allPos = append(allPos, struct {
- pos pdf.Position
- isEdge bool
- }{pos, false})
- }
- allPos = append(allPos, struct {
- pos pdf.Position
- isEdge bool
- }{bottomBandPos, true})
-
- for _, entry := range allPos {
- pos := entry.pos
- isEdge := entry.isEdge
-
- top := pos.Top
- bottom := pos.Bottom
- left := pos.Left
- right := pos.Right
-
- // Width: edge segments are full-width, middle are narrow.
- if !isEdge {
- right = math.Max(left+10, right)
- } else {
+ entries = append(entries, positionEntry{position: pos})
+ }
+ entries = append(entries, positionEntry{position: bottomBand, isEdge: true})
+
+ plan := &sectionRasterPlan{}
+ maxInt := int(^uint(0) >> 1)
+ for _, entry := range entries {
+ pos := entry.position
+ left, right := pos.Left, pos.Right
+ if entry.isEdge {
right = left + maxWidth
+ } else {
+ right = math.Max(left+10, right)
}
-
- pn0 := pos.PageNumbers[0]
-
- // Accumulate bottom for multi-page positions.
- accumBottom := bottom * zoom
- for _, pn := range pos.PageNumbers[1:] {
- if pn == pn0 {
- continue
- }
- if img, ok := decodedImages[pn]; ok {
- accumBottom += float64(img.Bounds().Dy())
+ firstPage := pos.PageNumbers[0]
+ accumBottom := pos.Bottom * zoom
+ for _, pageNum := range pos.PageNumbers[1:] {
+ if pageNum != firstPage {
+ accumBottom += float64(decodedImages[pageNum].Bounds().Dy())
}
}
- pageImg, ok := decodedImages[pn0]
- if !ok {
- common.Warn("cropSectionImage: page image not found", zap.Int("page", pn0))
- return ""
- }
- pageH := float64(pageImg.Bounds().Dy())
- bottomClamped := math.Min(accumBottom, pageH)
-
- // Crop first page of this position.
- cropped := FastCrop(pageImg,
- int(left*zoom), int(top*zoom),
- int(right*zoom), int(bottomClamped))
- if isEdge {
- cropped = applyEdgeOverlay(cropped)
- }
- segments = append(segments, segment{img: cropped, isEdge: isEdge})
+ pageImage := decodedImages[firstPage]
+ pageHeight := float64(pageImage.Bounds().Dy())
+ bottomClamped := math.Min(accumBottom, pageHeight)
+ plan.addCrop(firstPage, int(left*zoom), int(pos.Top*zoom), int(right*zoom), int(bottomClamped), entry.isEdge)
- // Subsequent pages (only those different from the first page).
- bottomRemaining := accumBottom - pageH
- for _, pn := range pos.PageNumbers[1:] {
- if pn == pn0 {
+ bottomRemaining := accumBottom - pageHeight
+ for _, pageNum := range pos.PageNumbers[1:] {
+ if pageNum == firstPage {
continue
}
- pageImg2, ok := decodedImages[pn]
- if !ok {
- common.Warn("cropSectionImage: page image not found for subsequent page", zap.Int("page", pn))
- return ""
+ if bottomRemaining <= 0 {
+ break
}
- pageH2 := float64(pageImg2.Bounds().Dy())
- bottomClamped2 := math.Min(bottomRemaining, pageH2)
- cropped2 := FastCrop(pageImg2,
- int(left*zoom), 0,
- int(right*zoom), int(bottomClamped2))
- if isEdge {
- cropped2 = applyEdgeOverlay(cropped2)
+ pageImage := decodedImages[pageNum]
+ bottomClamped := math.Min(bottomRemaining, float64(pageImage.Bounds().Dy()))
+ bottomPixels := int(bottomClamped)
+ if bottomPixels <= 0 {
+ break
}
- segments = append(segments, segment{img: cropped2, isEdge: isEdge})
- bottomRemaining -= bottomClamped2
+ plan.addCrop(pageNum, int(left*zoom), 0, int(right*zoom), bottomPixels, entry.isEdge)
+ bottomRemaining -= bottomClamped
}
}
-
- if len(segments) == 0 {
- return ""
+ if len(plan.segments) == 0 {
+ return nil, false
+ }
+ for _, segment := range plan.segments {
+ width, height := cropSize(decodedImages[segment.page], segment.x0, segment.y0, segment.x1, segment.y1)
+ plan.width = max(plan.width, width)
+ if height > maxInt-sectionRasterGap || plan.height > maxInt-(height+sectionRasterGap) {
+ return nil, false
+ }
+ plan.height += height + sectionRasterGap
}
+ return plan, plan.width > 0 && plan.height > 0
+}
- // Stitch vertically with gray background and 6px gaps.
- totalH := 0
- maxW := 0
- for _, seg := range segments {
- totalH += seg.img.Bounds().Dy() + gap
- maxW = max(maxW, seg.img.Bounds().Dx())
+func (p *sectionRasterPlan) addCrop(page, x0, y0, x1, y1 int, isEdge bool) {
+ p.segments = append(p.segments, sectionRasterSegment{
+ page: page, x0: x0, y0: y0, x1: x1, y1: y1, isEdge: isEdge,
+ })
+}
+
+func cropSize(img image.Image, x0, y0, x1, y1 int) (int, int) {
+ bounds := cropRectBounds(img, x0, y0, x1, y1)
+ if bounds.Empty() {
+ return 1, 1
}
- stitched := image.NewRGBA(image.Rect(0, 0, maxW, totalH))
+ return bounds.Dx(), bounds.Dy()
+}
- // Fill background using direct Pix slice write (matching fastCrop pattern).
- // Gray 245,245,245,255 as BGRA bytes.
- for y := 0; y < totalH; y++ {
- row := stitched.Pix[stitched.PixOffset(0, y):stitched.PixOffset(maxW, y)]
+func renderSectionRaster(plan *sectionRasterPlan, decodedImages map[int]image.Image) image.Image {
+ segments := make([]image.Image, 0, len(plan.segments))
+ for _, segment := range plan.segments {
+ cropped := FastCrop(decodedImages[segment.page], segment.x0, segment.y0, segment.x1, segment.y1)
+ if segment.isEdge {
+ cropped = applyEdgeOverlay(cropped)
+ }
+ segments = append(segments, cropped)
+ }
+ stitched := image.NewRGBA(image.Rect(0, 0, plan.width, plan.height))
+ for y := 0; y < plan.height; y++ {
+ row := stitched.Pix[stitched.PixOffset(0, y):stitched.PixOffset(plan.width, y)]
for i := 0; i < len(row); i += 4 {
- row[i] = 245 // B
- row[i+1] = 245 // G
- row[i+2] = 245 // R
- row[i+3] = 255 // A
+ row[i], row[i+1], row[i+2], row[i+3] = 245, 245, 245, 255
}
}
curY := 0
- for _, seg := range segments {
- srcW := seg.img.Bounds().Dx()
- srcH := seg.img.Bounds().Dy()
- if rgba, ok := seg.img.(*image.RGBA); ok {
- // Fast path: direct Pix slice copy (matching fastCrop in geometry.go).
- srcMinX := seg.img.Bounds().Min.X
- srcMinY := seg.img.Bounds().Min.Y
- for ry := 0; ry < srcH; ry++ {
- srcStart := rgba.PixOffset(srcMinX, srcMinY+ry)
+ for _, segment := range segments {
+ srcW, srcH := segment.Bounds().Dx(), segment.Bounds().Dy()
+ if rgba, ok := segment.(*image.RGBA); ok {
+ srcMinX, srcMinY := segment.Bounds().Min.X, segment.Bounds().Min.Y
+ for row := 0; row < srcH; row++ {
+ srcStart := rgba.PixOffset(srcMinX, srcMinY+row)
srcRow := rgba.Pix[srcStart : srcStart+srcW*4]
- dstStart := stitched.PixOffset(0, curY+ry)
+ dstStart := stitched.PixOffset(0, curY+row)
copy(stitched.Pix[dstStart:], srcRow)
}
} else {
- // Fallback: pixel-by-pixel for non-RGBA images (e.g. edge overlays).
for y := 0; y < srcH; y++ {
for x := 0; x < srcW; x++ {
- stitched.Set(x, curY+y, seg.img.At(x+seg.img.Bounds().Min.X, y+seg.img.Bounds().Min.Y))
+ stitched.Set(x, curY+y, segment.At(x+segment.Bounds().Min.X, y+segment.Bounds().Min.Y))
}
}
}
- curY += srcH + gap
+ curY += srcH + sectionRasterGap
}
+ return stitched
+}
- data, err := EncodePNG(stitched)
+func rasterPlanExceedsLimit(plan *sectionRasterPlan, maxPixels int64) bool {
+ if plan == nil || plan.width <= 0 || plan.height <= 0 || maxPixels <= 0 {
+ return true
+ }
+ return int64(plan.width) > maxPixels/int64(plan.height)
+}
+
+func encodeCroppedImage(img image.Image) string {
+ if img == nil {
+ return ""
+ }
+ data, err := EncodePNG(img)
if err != nil {
common.Warn("cropSectionImage: PNG encode failed", zap.Error(err))
return ""
@@ -598,25 +609,24 @@ func CropImageRegion(img image.Image, r pdf.DLARegion) (image.Image, error) {
//
// Python: pdf_parser.py:1802 RAGFlowPdfParser.crop()
func CropSectionPositions(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) string {
- if len(positions) == 0 {
- return ""
- }
- var tag strings.Builder
- for _, pos := range positions {
- if len(pos.PageNumbers) == 0 {
- continue
- }
- if len(pos.PageNumbers) == 1 {
- tag.WriteString(FormatPositionTag(pos.PageNumbers[0], pos.Left, pos.Right, pos.Top, pos.Bottom))
- } else {
- from, to := pos.PageNumbers[0], pos.PageNumbers[len(pos.PageNumbers)-1]
- tag.WriteString(FormatPositionTagRange(from, to, pos.Left, pos.Right, pos.Top, pos.Bottom))
- }
- }
- if tag.Len() == 0 {
- return ""
+ return encodeCroppedImage(CropSectionPositionsRaster(positions, decodedImages, zoom))
+}
+
+// CropSectionPositionsRaster crops typed PDF positions and returns the raster
+// without encoding it. Callers that need both OCR and a VLM payload can reuse
+// this image and encode it once after OCR.
+func CropSectionPositionsRaster(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) image.Image {
+ return cropSectionPositionsRaster(positions, decodedImages, zoom)
+}
+
+// CropSectionPositionsRasterLimited rejects a section before rendering when
+// the planned stitched raster exceeds maxPixels.
+func CropSectionPositionsRasterLimited(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64, maxPixels int64) image.Image {
+ plan, ok := buildSectionRasterPlan(positions, decodedImages, zoom)
+ if !ok || rasterPlanExceedsLimit(plan, maxPixels) {
+ return nil
}
- return CropSectionImage(tag.String(), decodedImages, zoom)
+ return renderSectionRaster(plan, decodedImages)
}
// PositionsFromMatrix converts the _pdf_positions / positions matrix form
diff --git a/internal/deepdoc/parser/pdf/util/crop_test.go b/internal/deepdoc/parser/pdf/util/crop_test.go
index dd24e15d19..056113c981 100644
--- a/internal/deepdoc/parser/pdf/util/crop_test.go
+++ b/internal/deepdoc/parser/pdf/util/crop_test.go
@@ -31,6 +31,19 @@ func decodePNG(t *testing.T, data []byte) image.Image {
return img
}
+func TestFastCropReturnsPlaceholderForInvertedBounds(t *testing.T) {
+ src := makeTestPageImage(100, 100, color.RGBA{255, 0, 0, 255})
+ for _, bounds := range [][4]int{
+ {80, 10, 20, 90},
+ {10, 80, 90, 20},
+ } {
+ got := FastCrop(src, bounds[0], bounds[1], bounds[2], bounds[3]).Bounds()
+ if got != image.Rect(0, 0, 1, 1) {
+ t.Errorf("FastCrop(%v) bounds = %v, want 1x1 placeholder", bounds, got)
+ }
+ }
+}
+
func TestCropSectionImage_SinglePage(t *testing.T) {
pageImages := map[int]image.Image{
0: makeTestPageImage(200, 300, color.RGBA{255, 0, 0, 255}),
@@ -147,6 +160,35 @@ func TestCropSectionImage_ZoomScaling(t *testing.T) {
}
}
+type accessCountingImage struct {
+ bounds image.Rectangle
+ reads int
+}
+
+func (i *accessCountingImage) ColorModel() color.Model { return color.RGBAModel }
+func (i *accessCountingImage) Bounds() image.Rectangle { return i.bounds }
+func (i *accessCountingImage) At(int, int) color.Color {
+ i.reads++
+ return color.RGBA{A: 255}
+}
+
+func TestCropSectionPositionsRasterLimitedRejectsOversizedStitchBeforeCopy(t *testing.T) {
+ page := &accessCountingImage{bounds: image.Rect(0, 0, 400, 400)}
+ positions := []pdf.Position{
+ {PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},
+ {PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},
+ {PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},
+ }
+
+ got := CropSectionPositionsRasterLimited(positions, map[int]image.Image{0: page}, 1, 5_000)
+ if got != nil {
+ t.Fatalf("limited crop = %v, want nil when stitched pixels exceed the limit", got.Bounds())
+ }
+ if page.reads != 0 {
+ t.Fatalf("source pixel reads = %d, want 0 before rejecting the oversized stitch", page.reads)
+ }
+}
+
func TestRotateImageCW(t *testing.T) {
// Create a 3x2 image with known colors: (0,0)=red, (1,0)=green, (2,0)=blue,
// (0,1)=white, (1,1)=black, (2,1)=gray
@@ -301,34 +343,29 @@ func colorEqual(a, b color.Color) bool {
return ar == br && ag == bg && ab == bb && aa == ba
}
-// TestCropSectionImage_MultiPage verifies the bottomRemaining fix for 3+ page
-// positions where page heights differ. Regression test for Bug #3.
+// TestCropSectionImage_MultiPage verifies that a non-positive trailing-page
+// remainder does not become an inverted crop region.
func TestCropSectionImage_MultiPage(t *testing.T) {
// Page 0: tall (2000px), Page 1: short (800px), Page 2: short (800px)
- // Content spans all 3 pages. The old bug subtracted full pageH2 from
- // bottomRemaining instead of the actual clamped value, causing negative
- // y1 on the last page → 1×1 placeholder crop.
+ // The range stores only its endpoint page numbers. With these page heights,
+ // the remaining crop height is exhausted before the trailing endpoint.
pageImages := map[int]image.Image{
0: makeTestPageImage(100, 2000, color.RGBA{200, 0, 0, 255}),
1: makeTestPageImage(100, 800, color.RGBA{0, 200, 0, 255}),
2: makeTestPageImage(100, 800, color.RGBA{0, 0, 200, 255}),
}
- // pdf.Position spans pages 0-2, bottom reaches into page 2.
+ // The position range covers pages 0-2.
posTag := "@@1-3\t0.0\t100.0\t0.0\t500.0##"
b64 := CropSectionImage(posTag, pageImages, 1)
if b64 == "" {
t.Fatal("expected non-empty result for multi-page position")
}
- // Decode and check height: content 500pt + bottom on page 1 clamped
- // to 800 → page 1 crop 0-800, page 2 crop 0-200. Total with 2x6px gaps
- // should be ~2000 + 200 + 12 = 2212.
+ // The first page and context bands should still yield a non-empty crop.
decoded, _ := base64.StdEncoding.DecodeString(b64)
img := decodePNG(t, decoded)
h := img.Bounds().Dy()
- // Without the fix, page 2 gets negative y1 → 1x1 output (~100 + gap).
- // With fix, proper crop from all 3 pages.
if h < 500 {
- t.Errorf("multi-page height too small: got %d, want >= 500 (bug: bottomRemaining over-subtraction)", h)
+ t.Errorf("multi-page height too small: got %d, want >= 500", h)
}
t.Logf("multi-page stitch height: %d", h)
}
diff --git a/internal/deepdoc/parser/pdf/util/geometry.go b/internal/deepdoc/parser/pdf/util/geometry.go
index 65d3f98875..cdd2df25f3 100644
--- a/internal/deepdoc/parser/pdf/util/geometry.go
+++ b/internal/deepdoc/parser/pdf/util/geometry.go
@@ -193,7 +193,29 @@ func RectOverlap(a, b Rect) float64 {
// Uses direct Pix slice copy for *image.RGBA sources (zero allocation per row);
// falls back to pixel-by-pixel for other image types.
func FastCrop(src image.Image, x0, y0, x1, y1 int) *image.RGBA {
- // Clamp to source bounds
+ b := cropRectBounds(src, x0, y0, x1, y1)
+ if b.Empty() {
+ return image.NewRGBA(image.Rect(0, 0, 1, 1))
+ }
+ w, h := b.Dx(), b.Dy()
+ dst := image.NewRGBA(image.Rect(0, 0, w, h))
+ if rgba, ok := src.(*image.RGBA); ok {
+ for y := b.Min.Y; y < b.Max.Y; y++ {
+ srcRow := rgba.Pix[rgba.PixOffset(b.Min.X, y):rgba.PixOffset(b.Max.X, y)]
+ dstRow := dst.Pix[dst.PixOffset(0, y-b.Min.Y):]
+ copy(dstRow, srcRow)
+ }
+ } else {
+ for y := b.Min.Y; y < b.Max.Y; y++ {
+ for x := b.Min.X; x < b.Max.X; x++ {
+ dst.Set(x-b.Min.X, y-b.Min.Y, src.At(x, y))
+ }
+ }
+ }
+ return dst
+}
+
+func cropRectBounds(src image.Image, x0, y0, x1, y1 int) image.Rectangle {
b := src.Bounds()
if x0 < b.Min.X {
x0 = b.Min.X
@@ -208,25 +230,9 @@ func FastCrop(src image.Image, x0, y0, x1, y1 int) *image.RGBA {
y1 = b.Max.Y
}
if x0 >= x1 || y0 >= y1 {
- return image.NewRGBA(image.Rect(0, 0, 1, 1))
- }
- w, h := x1-x0, y1-y0
- dst := image.NewRGBA(image.Rect(0, 0, w, h))
- if rgba, ok := src.(*image.RGBA); ok {
- for y := y0; y < y1; y++ {
- srcRow := rgba.Pix[rgba.PixOffset(x0, y):rgba.PixOffset(x1, y)]
- dstRow := dst.Pix[dst.PixOffset(0, y-y0):]
- copy(dstRow, srcRow)
- }
-
- } else {
- for y := y0; y < y1; y++ {
- for x := x0; x < x1; x++ {
- dst.Set(x-x0, y-y0, src.At(x, y))
- }
- }
+ return image.Rectangle{}
}
- return dst
+ return image.Rect(x0, y0, x1, y1)
}
// ── Geometry helpers (pure functions, moved from type/types.go) ─────────
diff --git a/internal/ingestion/component/chunker/python_parity_test.go b/internal/ingestion/component/chunker/python_parity_test.go
index cc30f24e70..a94210a1e8 100644
--- a/internal/ingestion/component/chunker/python_parity_test.go
+++ b/internal/ingestion/component/chunker/python_parity_test.go
@@ -111,10 +111,8 @@ func TestChunker_PreservesLayoutField(t *testing.T) {
}
// TestChunker_PassesPDFPositionsThrough pins the _pdf_positions
-// pass-through. The PDF parser path (DeepDOC) attaches bbox
-// coordinates under this key; the chunker must forward them
-// so downstream layout-aware components can rebuild the page
-// geometry.
+// pass-through. DeepDOC's PDF parser attaches bounding boxes under this key;
+// downstream layout-aware components use them to rebuild page geometry.
func TestChunker_PassesPDFPositionsThrough(t *testing.T) {
positions := [][]float64{{0.1, 0.2, 0.3, 0.4}, {0.5, 0.6, 0.7, 0.8}}
items := []map[string]any{
@@ -134,6 +132,28 @@ func TestChunker_PassesPDFPositionsThrough(t *testing.T) {
}
}
+func TestChunker_PreservesVisualParentMetadata(t *testing.T) {
+ items := []map[string]any{
+ {
+ "text": "OCR text from a table image",
+ "doc_type_kwd": "image",
+ "parent_table_id": "docx-table-1",
+ "media_order": 2,
+ },
+ }
+ out := invokeAsTokenChunker(t, items)
+ chunks := chunksFromOutput(t, out)
+ if len(chunks) == 0 {
+ t.Fatal("no chunks emitted")
+ }
+ if got, want := chunks[0]["parent_table_id"], "docx-table-1"; got != want {
+ t.Errorf("chunks[0].parent_table_id = %v, want %v", got, want)
+ }
+ if got, want := chunks[0]["media_order"], float64(2); got != want {
+ t.Errorf("chunks[0].media_order = %v, want %v", got, want)
+ }
+}
+
// TestChunker_DropsContentWithWeightOnlyItems enforces the pre-index wire
// contract: chunkers consume canonical "text" only.
func TestChunker_DropsContentWithWeightOnlyItems(t *testing.T) {
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index a15703104d..279813cd9b 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -830,6 +830,7 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)
Positions: it.Positions,
Image: it.Image,
PageNumber: it.PageNumber,
+ Extra: cloneExtra(it.Extra),
}
if ctxAbove != "" {
out.ContextAbove = ctxAbove
@@ -1642,11 +1643,17 @@ func cloneChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {
if in.Positions != nil {
out.Positions = append(json.RawMessage(nil), in.Positions...)
}
- if in.Extra != nil {
- out.Extra = make(map[string]json.RawMessage, len(in.Extra))
- for k, v := range in.Extra {
- out.Extra[k] = append(json.RawMessage(nil), v...)
- }
+ out.Extra = cloneExtra(in.Extra)
+ return out
+}
+
+func cloneExtra(in map[string]json.RawMessage) map[string]json.RawMessage {
+ if in == nil {
+ return nil
+ }
+ out := make(map[string]json.RawMessage, len(in))
+ for key, value := range in {
+ out[key] = append(json.RawMessage(nil), value...)
}
return out
}
diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.go
index 0ebe28b4a7..8aad0b2665 100644
--- a/internal/ingestion/component/dispatch_model.go
+++ b/internal/ingestion/component/dispatch_model.go
@@ -47,8 +47,9 @@ var resolveModelConfig = defaultResolveModelConfig
// configuredMediaModelID extracts a per-call model reference from a parser setup.
// Image parsing stores the VLM model reference in parse_method when it is not
-// "ocr" (mirroring Python rag/flow/parser/parser.py:_image). Other media
-// families use vlm.llm_id, matching the frontend parser form.
+// "ocr" (mirroring Python rag/flow/parser/parser.py:_image); "ocr" selects the
+// tenant default vision model, and Go does not run OCR for standalone images.
+// Other media families use vlm.llm_id, matching the frontend parser form.
func configuredMediaModelID(setup schema.ParserSetup, family string) string {
if family == "image" {
if ref := getStringOr(setup, "parse_method", ""); ref != "" && !strings.EqualFold(ref, "ocr") {
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index 2b2d068e22..2ba8240b38 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -14,36 +14,17 @@
// limitations under the License.
//
-// Media dispatch: image, audio, video parser branches that require
-// model access (OCR, IMAGE2TEXT, SPEECH2TEXT) at the component
-// layer. Mirrors Python's _image / _audio / _video methods in
-// rag/flow/parser/parser.py and rag/app/picture.py.
-//
-// These follow the maybeDispatchPDFVision pattern: they bypass
-// dispatchParse and call the model directly from the component
-// layer, returning a parser.ParseResult.
+// Media dispatch: image, audio, and video branches that require model access
+// (IMAGE2TEXT, SPEECH2TEXT) at the component layer.
package component
import (
- "bytes"
"context"
- "encoding/base64"
"fmt"
- "image"
-
- // Import image decoders for common formats.
- _ "golang.org/x/image/bmp"
- _ "golang.org/x/image/tiff"
- _ "golang.org/x/image/webp"
- _ "image/gif"
- _ "image/jpeg"
- _ "image/png"
"os"
"path/filepath"
- "sort"
"strings"
- "unicode/utf8"
"go.uber.org/zap"
@@ -90,13 +71,7 @@ func maybeDispatchVideo(
fmt.Errorf("Parser: video parsing is not yet supported; underlying video analysis capability is pending")
}
-// Image dispatch: OCR + IMAGE2TEXT vision describe ---
-// Mirrors Python's rag/app/picture.py:chunk() image branch:
-// 1. Try PaddleOCR if layout_recognize is "@PaddleOCR"
-// 2. Fallback to local ONNX OCR (DeepDoc /predict/ocr endpoint)
-// 3. If OCR text is short (≤32 chars or ≤32 English words),
-// also call IMAGE2TEXT VLM describe()
-// 4. Returns combined text
+// Image dispatch: IMAGE2TEXT vision describe ---
func maybeDispatchImage(
ctx context.Context,
@@ -106,6 +81,7 @@ func maybeDispatchImage(
binary []byte,
inputs map[string]any,
setups map[string]schema.ParserSetup,
+ enableVisionEnhancement bool,
) (parser.ParseResult, bool, error) {
if fileType != utility.FileTypeVISUAL {
return parser.ParseResult{}, false, nil
@@ -115,56 +91,52 @@ func maybeDispatchImage(
return parser.ParseResult{}, false, nil
}
tenantID := getStringOr(inputs, "tenant_id", "")
- if tenantID == "" {
- return parser.ParseResult{}, true,
- fmt.Errorf("parser: image requires tenant_id")
- }
-
- // --- Phase 1: OCR ---
- var ocrText string
-
- // Step 1a: Try PaddleOCR if layout_recognize is set to PaddleOCR.
- // Mirrors Python's picture.py:_try_paddleocr_image().
- layoutRecognize := getStringOr(setup, "layout_recognize", "")
- if layoutRecognize != "" {
- recognizer, _ := normalizeLayoutRecognizer(layoutRecognize)
- if recognizer == "PaddleOCR" {
- if txt, err := runPaddleOCRImage(binary, filename); err == nil && txt != "" {
- ocrText = txt
+ if !enableVisionEnhancement {
+ return parser.ParseResult{}, true, fmt.Errorf("parser: image has no searchable text because vision enhancement is disabled")
+ }
+ parsed := dispatchParse(ctx, fileType, filename, binary, setups)
+ if parsed.Err != nil {
+ return parsed, true, parsed.Err
+ }
+ if len(parsed.JSON) == 0 {
+ return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image item")
+ }
+ imageData, _ := parsed.JSON[0]["image"].(string)
+ if imageData == "" {
+ return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image payload")
+ }
+ result, handled, err := maybeDispatchImageVLM(ctx, db, imageData, tenantID, setup, inputs)
+ result.Warnings = append(result.Warnings, parsed.Warnings...)
+ result.File = parsed.File
+ if err == nil && len(result.JSON) > 0 {
+ text, _ := result.JSON[0]["text"].(string)
+ if strings.TrimSpace(text) == "" {
+ if len(result.Warnings) > 0 {
+ return result, true, fmt.Errorf("parser: image has no searchable text: %s", result.Warnings[0])
}
+ return result, true, fmt.Errorf("parser: vision enhancement returned no searchable text")
}
}
+ return result, handled, err
+}
- // Step 1b: Fallback to local ONNX OCR (DeepDoc /predict/ocr).
- // Mirrors Python's picture.py:ocr(np.array(img)) from deepdoc.vision.
- if ocrText == "" {
- if txt, err := runLocalImageOCR(binary); err == nil && txt != "" {
- ocrText = txt
- }
- }
-
- // The image family always emits a structured JSON item carrying the
- // image attachment (data URI) and doc_type_kwd, mirroring Python
- // rag/app/picture.py:71-72 (doc["image"]=img, doc["doc_type_kwd"]=
- // "image"). picture.py has no "text" output mode — it always returns
- // a structured doc — so output_format is hardcoded to "json" and any
- // setup override is ignored. The former behavior returned a bare Text
- // string, which dropped the image attachment, set doc_type to "text",
- // and on the default json path produced JSON=nil so downstream
- // Chunkers rejected the payload with errRequiredField{"json"}.
- imageB64 := base64.StdEncoding.EncodeToString(binary)
- dataURI := "data:" + imageMIME(filename) + ";base64," + imageB64
-
- // --- Phase 2: VLM description (when OCR text is short) ---
- // Mirrors Python's check: if (eng and len(txt.split()) > 32) or len(txt) > 32
- // then use OCR text only; otherwise call cv_mdl.describe().
+func maybeDispatchImageVLM(
+ ctx context.Context,
+ db *gorm.DB,
+ dataURI string,
+ tenantID string,
+ setup schema.ParserSetup,
+ inputs map[string]any,
+) (parser.ParseResult, bool, error) {
+ // --- Optional VLM description ---
lang := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
- if vlmGateShouldSkip(ocrText, lang) {
- // OCR returned substantial text — skip VLM.
- return imageDispatchResult(ocrText, dataURI), true, nil
+ if tenantID == "" {
+ result := imageDispatchResult("", dataURI)
+ result.Warnings = append(result.Warnings, "image VLM enhancement skipped: tenant ID is missing")
+ return result, true, nil
}
- // Short OCR text (or no text): supplement with VLM describe.
+ // Use the configured image VLM or the tenant default.
modelRef := configuredMediaModelID(setup, "image")
var driver modelModule.ModelDriver
var modelName string
@@ -180,13 +152,13 @@ func maybeDispatchImage(
} else {
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
}
+ if err == nil && driver == nil {
+ err = fmt.Errorf("no usable vision model")
+ }
if err != nil {
- // If VLM is unavailable, but we have OCR text, return it.
- if ocrText != "" {
- return imageDispatchResult(ocrText, dataURI), true, nil
- }
- return parser.ParseResult{}, true,
- fmt.Errorf("parser: picture image2text model: %w", err)
+ result := imageDispatchResult("", dataURI)
+ result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement skipped: model unavailable: %v", err))
+ return result, true, nil
}
prompt := defaultImageVisionPrompt(lang)
@@ -206,32 +178,20 @@ func maybeDispatchImage(
vision := true
resp, err := driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)
if err != nil {
- if ocrText != "" {
- return imageDispatchResult(ocrText, dataURI), true, nil
- }
- return parser.ParseResult{}, true,
- fmt.Errorf("parser: picture describe: %w", err)
+ result := imageDispatchResult("", dataURI)
+ result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement failed: %v", err))
+ return result, true, nil
}
vlmText := ""
if resp != nil && resp.Answer != nil {
vlmText = strings.TrimSpace(*resp.Answer)
}
- // Combine OCR + VLM text.
- // Mirrors Python: txt += "\n" + ans
- combined := ocrText
- if vlmText != "" {
- if combined != "" {
- combined += "\n" + vlmText
- } else {
- combined = vlmText
- }
- }
- return imageDispatchResult(combined, dataURI), true, nil
+ return imageDispatchResult(vlmText, dataURI), true, nil
}
// imageDispatchResult builds the structured JSON payload for the image
-// family: a single item carrying the combined text, the image attachment
+// family: a single item carrying the VLM description, the image attachment
// (data URI), and doc_type_kwd "image". Mirrors Python
// rag/app/picture.py:71-72.
func imageDispatchResult(text, dataURI string) parser.ParseResult {
@@ -336,59 +296,6 @@ func writeTempAudioFile(filename string, binary []byte) (string, error) {
return tmp.Name(), nil
}
-// normalizeLayoutRecognizer parses layout_recognize strings like
-// "model@PaddleOCR" → ("PaddleOCR", "model@PaddleOCR").
-// Mirrors Python's common/parser_config_utils.py:normalize_layout_recognizer().
-func normalizeLayoutRecognizer(raw string) (recognizer, modelName string) {
- lowered := strings.ToLower(raw)
- if strings.HasSuffix(lowered, "@paddleocr") {
- return "PaddleOCR", raw
- }
- if strings.HasSuffix(lowered, "@mineru") {
- return "MinerU", raw
- }
- if strings.HasSuffix(lowered, "@somark") {
- return "SoMark", raw
- }
- if strings.HasSuffix(lowered, "@opendataloader") {
- return "OpenDataLoader", raw
- }
- return raw, ""
-}
-
-// imageMIME maps common image filename extensions to MIME types
-// for constructing base64 data URIs.
-func imageMIME(filename string) string {
- dot := strings.LastIndex(filename, ".")
- if dot == -1 {
- return "image/png"
- }
- switch strings.ToLower(filename[dot+1:]) {
- case "jpg", "jpeg":
- return "image/jpeg"
- case "png":
- return "image/png"
- case "gif":
- return "image/gif"
- case "bmp":
- return "image/bmp"
- case "webp":
- return "image/webp"
- case "svg":
- return "image/svg+xml"
- case "tiff", "tif":
- return "image/tiff"
- case "ico":
- return "image/x-icon"
- case "avif":
- return "image/avif"
- case "heic":
- return "image/heic"
- default:
- return "image/png"
- }
-}
-
// videoMIME maps common video filename extensions to MIME types
// for constructing base64 data URIs. Retained as a reference for the
// future real video-parsing implementation (provider-specific frame
@@ -422,196 +329,3 @@ func videoMIME(filename string) string {
return "video/mp4"
}
}
-
-// --- OCR helpers for picture dispatch ---
-
-// vlmGateShouldSkip determines whether the OCR text is substantial enough
-// to skip the secondary VLM description call.
-// Mirrors Python picture.py:chunk():
-//
-// txt = txt.strip()
-// if (eng and len(txt.split()) > 32) or len(txt) > 32 -> skip VLM
-func vlmGateShouldSkip(ocrText, lang string) bool {
- ocrText = strings.TrimSpace(ocrText)
- if ocrText == "" {
- return false
- }
- eng := strings.EqualFold(lang, "english")
- wordCount := len(strings.Fields(ocrText))
- charCount := utf8.RuneCountInString(ocrText)
- return (eng && wordCount > 32) || charCount > 32
-}
-
-// runPaddleOCRImage tries PaddleOCR remote API for image text extraction.
-// Mirrors Python's picture.py:_try_paddleocr_image() which creates a
-// PaddleOCRParser and calls parse_image().
-func runPaddleOCRImage(binary []byte, filename string) (string, error) {
- client := parser.NewPaddleOCRClientFromEnv()
- if !client.Enabled() {
- return "", fmt.Errorf("paddleocr: not configured (set PADDLEOCR_ACCESS_TOKEN)")
- }
- return client.ParseImage(binary, filename)
-}
-
-// runLocalImageOCR detects and recognizes text in an image using the
-// in-process DeepDoc analyzer (ONNX models served locally via the native
-// backend). Mirrors Python's deepdoc.vision.OCR local ONNX pipeline; the
-// external HTTP service is no longer a backend (see parser.GetDocAnalyzer).
-//
-// Pipeline:
-// 1. Decode image bytes → image.Image
-// 2. OCRDetect → find text region boxes
-// 3. For each box: crop → OCRRecognize → text
-// 4. Sort boxes by Y, then X (reading order)
-// 5. Join all recognized text with newlines
-func runLocalImageOCR(binary []byte) (string, error) {
- analyzer, err := parser.GetDocAnalyzer()
- if err != nil {
- return "", fmt.Errorf("local OCR: %w", err)
- }
-
- img, _, err := image.Decode(bytes.NewReader(binary))
- if err != nil {
- return "", fmt.Errorf("local OCR: decode image: %w", err)
- }
-
- // Step 1: Detect text regions.
- ctx := context.Background()
- boxes, err := analyzer.OCRDetect(ctx, img)
- if err != nil {
- return "", fmt.Errorf("local OCR: detect: %w", err)
- }
- if len(boxes) == 0 {
- return "", nil
- }
-
- // Step 2: Sort boxes by Y (top to bottom), then X (left to right)
- // for reading-order text assembly.
- sort.Slice(boxes, func(i, j int) bool {
- yi := (boxes[i].Y0 + boxes[i].Y2) / 2
- yj := (boxes[j].Y0 + boxes[j].Y2) / 2
- if yi < yj {
- return true
- }
- if yi > yj {
- return false
- }
- return boxes[i].X0 < boxes[j].X0
- })
-
- // Step 3: Recognize text per box.
- var texts []string
- bounds := img.Bounds()
- for _, box := range boxes {
- // Convert quad box to axis-aligned crop rect.
- x0 := int(min4(box.X0, box.X1, box.X2, box.X3))
- y0 := int(min4(box.Y0, box.Y1, box.Y2, box.Y3))
- x1 := int(max4(box.X0, box.X1, box.X2, box.X3))
- y1 := int(max4(box.Y0, box.Y1, box.Y2, box.Y3))
-
- // Clamp to image bounds.
- if x0 < bounds.Min.X {
- x0 = bounds.Min.X
- }
- if y0 < bounds.Min.Y {
- y0 = bounds.Min.Y
- }
- if x1 > bounds.Max.X {
- x1 = bounds.Max.X
- }
- if y1 > bounds.Max.Y {
- y1 = bounds.Max.Y
- }
- if x1 <= x0 || y1 <= y0 {
- continue
- }
-
- // Crop the region. This requires an image type that supports
- // cropping; for simplicity we recode through a sub-image.
- crop := cropImage(img, x0, y0, x1, y1)
- if crop == nil {
- continue
- }
-
- recTexts, err := analyzer.OCRRecognize(ctx, crop)
- if err != nil {
- continue // skip boxes that fail recognition
- }
- for _, t := range recTexts {
- s := strings.TrimSpace(t.Text)
- if s != "" {
- texts = append(texts, s)
- }
- }
- }
-
- if len(texts) == 0 {
- return "", nil
- }
- return strings.Join(texts, "\n"), nil
-}
-
-// cropImage extracts a sub-rectangle from img. Works with any image.Image
-// by converting to RGBA if needed, then cropping.
-func cropImage(img image.Image, x0, y0, x1, y1 int) image.Image {
- bounds := img.Bounds()
- cropRect := image.Rect(
- bounds.Min.X+x0, bounds.Min.Y+y0,
- bounds.Min.X+x1, bounds.Min.Y+y1,
- )
- switch src := img.(type) {
- case *image.RGBA:
- return src.SubImage(cropRect)
- case *image.NRGBA:
- return src.SubImage(cropRect)
- case *image.RGBA64:
- return src.SubImage(cropRect)
- case *image.NRGBA64:
- return src.SubImage(cropRect)
- case *image.Gray:
- return src.SubImage(cropRect)
- case *image.Gray16:
- return src.SubImage(cropRect)
- case *image.YCbCr:
- return src.SubImage(cropRect)
- case *image.Paletted:
- return src.SubImage(cropRect)
- default:
- // Convert to RGBA for cropping.
- rgba := image.NewRGBA(cropRect)
- for y := cropRect.Min.Y; y < cropRect.Max.Y; y++ {
- for x := cropRect.Min.X; x < cropRect.Max.X; x++ {
- rgba.Set(x, y, img.At(x, y))
- }
- }
- return rgba
- }
-}
-
-func min4(a, b, c, d float64) float64 {
- m := a
- if b < m {
- m = b
- }
- if c < m {
- m = c
- }
- if d < m {
- m = d
- }
- return m
-}
-
-func max4(a, b, c, d float64) float64 {
- m := a
- if b > m {
- m = b
- }
- if c > m {
- m = c
- }
- if d > m {
- m = d
- }
- return m
-}
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index f07e675ce7..c94ac8e784 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -34,6 +34,17 @@ import (
"ragflow/internal/utility"
)
+func TestMaybeDispatchImageWithoutVisionReportsNoContent(t *testing.T) {
+ result, handled, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL,
+ "photo.png", []byte("image bytes"), nil, defaultSetups(), false)
+ if !handled {
+ t.Fatal("image was not handled")
+ }
+ if err == nil || !strings.Contains(err.Error(), "vision enhancement") {
+ t.Fatalf("error = %v, want explicit vision requirement; result = %+v", err, result)
+ }
+}
+
// imagePromptCaptureDriver embeds ModelDriver so it satisfies the interface
// without listing every method; only ChatWithMessages is overridden to record
// the messages the image branch selected.
@@ -41,11 +52,13 @@ type imagePromptCaptureDriver struct {
modelModule.ModelDriver
mu sync.Mutex
captured []modelModule.Message
+ ctxErr error
}
func (d *imagePromptCaptureDriver) ChatWithMessages(ctx context.Context, modelName string, messages []modelModule.Message, apiConfig *modelModule.APIConfig, chatModelConfig *modelModule.ChatConfig, usage *common.ModelUsage) (*modelModule.ChatResponse, error) {
d.mu.Lock()
d.captured = append(d.captured, messages...)
+ d.ctxErr = ctx.Err()
d.mu.Unlock()
ans := "captured"
return &modelModule.ChatResponse{Answer: &ans}, nil
@@ -107,6 +120,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
[]byte("not-a-real-image"),
map[string]any{"tenant_id": "t1"},
setups,
+ true,
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -158,6 +172,7 @@ func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {
[]byte("not-a-real-image"),
map[string]any{"tenant_id": "t1", "lang": "Japanese"},
setups,
+ true,
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -200,6 +215,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
[]byte("not-a-real-image"),
map[string]any{"tenant_id": "t1"},
setups,
+ true,
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -222,7 +238,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
t.Errorf("image = %q, want a data URI (data:<mime>;base64,<b64>)", img)
}
if txt, _ := item["text"].(string); txt == "" {
- t.Errorf("text field empty; want non-empty combined OCR+VLM text")
+ t.Errorf("text field empty; want non-empty VLM description")
}
}
@@ -251,6 +267,7 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {
[]byte("not-a-real-image"),
map[string]any{"tenant_id": "t1"},
setups,
+ true,
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -442,6 +459,7 @@ func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {
[]byte("not-a-real-image"),
map[string]any{"tenant_id": "t1"},
setups,
+ true,
)
if err != nil {
t.Fatalf("maybeDispatchImage: %v", err)
@@ -516,12 +534,12 @@ func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {
}
// TestImageDecoders_RegisteredFormats validates that image decoders for WebP, BMP,
-// TIFF, PNG, JPEG, and GIF are registered by media_dispatch.go and can decode
+// TIFF, PNG, JPEG, and GIF are registered by vision_image.go and can decode
// their respective binary payloads via image.Decode without importing the decoder
// packages directly in the test file.
func TestImageDecoders_RegisteredFormats(t *testing.T) {
// Fixed binary fixtures for image formats decoded via decoders registered in
- // media_dispatch.go (neither standard library nor x/image decoders are imported
+ // vision_image.go (neither standard library nor x/image decoders are imported
// in this test file).
const (
webpB64 = "UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAD8D+JaQAA3AA/ua1AAA="
@@ -563,93 +581,3 @@ func TestImageDecoders_RegisteredFormats(t *testing.T) {
})
}
}
-
-// TestVLMGateShouldSkip verifies the rune vs word count threshold
-// for skipping VLM description. Specifically:
-// - Zero-allocation rune counting via utf8.RuneCountInString correctly handles multi-byte UTF-8.
-// - Whitespace trimming matches Python txt.strip(): surrounding whitespace is trimmed before counting.
-// - CJK text is measured in unicode runes: 12 CJK characters occupy 36 bytes (>32 bytes)
-// but only 12 runes (<=32 runes), so VLM must NOT be skipped.
-// - CJK text >32 runes (e.g. 33 runes) skips VLM.
-// - CJK exact boundary text (32 runes) triggers VLM.
-// - English text >32 words skips VLM.
-// - English short text (<=32 words and <=32 chars) triggers VLM.
-// - English text with <=32 words but >32 chars skips VLM.
-func TestVLMGateShouldSkip(t *testing.T) {
- tests := []struct {
- name string
- lang string
- ocrText string
- wantSkip bool
- }{
- {
- name: "empty text does not skip",
- lang: "Chinese",
- ocrText: "",
- wantSkip: false,
- },
- {
- name: "whitespace only text does not skip",
- lang: "Chinese",
- ocrText: " \n\t ",
- wantSkip: false,
- },
- {
- name: "CJK substantial text (>32 runes, >32 bytes) skips VLM",
- lang: "Chinese",
- ocrText: strings.Repeat("中", 33), // 33 runes, 99 bytes
- wantSkip: true,
- },
- {
- name: "CJK short text with >32 bytes but <=32 runes triggers VLM",
- lang: "Chinese",
- ocrText: strings.Repeat("中", 12), // 12 runes, 36 bytes (>32 bytes)
- wantSkip: false,
- },
- {
- name: "CJK exact boundary text (32 runes, 96 bytes) triggers VLM",
- lang: "Chinese",
- ocrText: strings.Repeat("中", 32), // 32 runes, 96 bytes (32 is not > 32)
- wantSkip: false,
- },
- {
- name: "CJK exact boundary text with whitespace padding trims to <=32 runes and triggers VLM",
- lang: "Chinese",
- ocrText: " " + strings.Repeat("中", 32) + " ", // 32 runes after trim
- wantSkip: false,
- },
- {
- name: "CJK substantial text with whitespace padding trims to >32 runes and skips VLM",
- lang: "Chinese",
- ocrText: " " + strings.Repeat("中", 33) + " ", // 33 runes after trim
- wantSkip: true,
- },
- {
- name: "English substantial text (>32 words) skips VLM",
- lang: "English",
- ocrText: strings.Repeat("word ", 33), // 33 words
- wantSkip: true,
- },
- {
- name: "English short text (<=32 words and <=32 chars) triggers VLM",
- lang: "English",
- ocrText: "hello world", // 2 words, 11 chars
- wantSkip: false,
- },
- {
- name: "English text with <=32 words but >32 chars skips VLM",
- lang: "English",
- ocrText: "abcdefghijklmnopqrstuvwxyz01234567", // 1 word, 34 chars (>32 chars)
- wantSkip: true,
- },
- }
-
- for _, tc := range tests {
- t.Run(tc.name, func(t *testing.T) {
- got := vlmGateShouldSkip(tc.ocrText, tc.lang)
- if got != tc.wantSkip {
- t.Errorf("vlmGateShouldSkip(%q, %q) = %v, want %v", tc.ocrText, tc.lang, got, tc.wantSkip)
- }
- })
- }
-}
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index f44cd62f5e..72dd2cdd34 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -110,7 +110,8 @@ const pageFormFeed = '\f'
// the goroutine that returned from Invoke. The static Param is
// read-only after construction.
type ParserComponent struct {
- setups map[string]schema.ParserSetup
+ setups map[string]schema.ParserSetup
+ enableVisionEnhancement bool
}
// NewParserComponent constructs a Parser from a DSL param map.
@@ -123,6 +124,7 @@ type ParserComponent struct {
// Param map shape (all keys optional):
//
// {
+// "enable_vision_enhancement": bool,
// "pdf": map[string]any,
// "docx": map[string]any,
// ...
@@ -139,8 +141,16 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
// Canvases saved by the Python-era frontend nest the per-family setups
// under a "setups" key; lift them so every family lands at the top level.
params = schema.FlattenLegacyParserSetups(params)
+ var enableVisionEnhancement bool
+ if raw, exists := params["enable_vision_enhancement"]; exists {
+ var ok bool
+ enableVisionEnhancement, ok = raw.(bool)
+ if !ok {
+ return nil, errors.New("parser: enable_vision_enhancement must be a boolean")
+ }
+ }
for k, raw := range params {
- if k == "outputs" || k == "allowed_output_format" {
+ if k == "outputs" || k == "allowed_output_format" || k == "enable_vision_enhancement" {
continue
}
ftCfg, ok := raw.(map[string]any)
@@ -155,7 +165,7 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {
}
}
normalizeParserOutputFormats(s)
- pc := &ParserComponent{setups: s}
+ pc := &ParserComponent{setups: s, enableVisionEnhancement: enableVisionEnhancement}
if err := pc.Check(); err != nil {
return nil, fmt.Errorf("parser: %w", err)
}
@@ -206,11 +216,8 @@ func cloneParserSetupValue(value any) any {
}
}
-// Check mirrors the applicable subset of Python ParserParam.check()
-// (rag/flow/parser/parser.py:251-321). Runs at construction time so
-// a malformed DSL surfaces as a canvas compile failure rather than a
-// mid-run error. Returns the first validation error encountered
-// (Python raises ValueError on the first failure).
+// Check validates parser methods at construction time so a malformed DSL
+// surfaces as a canvas compile failure rather than a mid-run error.
//
// NOT covered here (intentional):
// - audio/video vlm.llm_id: Python's check() does not validate it
@@ -233,11 +240,13 @@ func (c *ParserComponent) Check() error {
}
}
}
- // image family (parser.py:283-287).
+ // image family (parser.py:283-287). The legacy "ocr" setup value selects
+ // the tenant default vision model; image OCR is not run by this path.
if img, ok := c.setups["image"]; ok {
pm, _ := img["parse_method"].(string)
- // OCR mode does not need a VLM language; any other value does.
- if pm != "ocr" {
+ // A model selected for optional image enhancement needs a language
+ // only when enhancement is enabled.
+ if c.enableVisionEnhancement && pm != "ocr" {
if lang, _ := img["lang"].(string); lang == "" {
return errors.New("image VLM language does not support empty value")
}
@@ -472,9 +481,9 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
}
var handledImage bool
if !handledVision && !handledMedia {
- // Image/Picture dispatch: OCR + IMAGE2TEXT vision describe.
+ // Image/Picture dispatch: optional IMAGE2TEXT vision description.
// Mirrors Python's rag/app/picture.py:chunk() image branch.
- dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups)
+ dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement)
if visionErr != nil {
return nil, visionErr
}
@@ -491,14 +500,11 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
if !handledVision && !handledMedia && !handledImage && !handledAudio {
dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups)
- // Vision figure enhancement: on the JSON output path,
- // append vision-model descriptions to embedded image and
- // table items. Mirrors Python's enhance_media_sections_with_vision
- // (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).
- // Errors (including context cancellation) are intentionally
- // discarded — enhancement is best-effort, matching Python's
- // try/except pass pattern.
- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)
+ if c.enableVisionEnhancement {
+ // Enhancement is optional; parser-provided text and image metadata
+ // remain available if a vision model cannot describe an image.
+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)
+ }
}
// Known/supported families must fail loudly when dispatch or
// parsing breaks. Only unknown families keep the raw-text fallback.
diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.go
index f3f4b8c433..afffa883c0 100644
--- a/internal/ingestion/component/parser_check_test.go
+++ b/internal/ingestion/component/parser_check_test.go
@@ -23,8 +23,8 @@ import (
"ragflow/internal/ingestion/component/schema"
)
-// TestParserComponent_Check covers the construction-time business
-// validation for parser methods that require language configuration.
+// TestParserComponent_Check covers construction-time validation for parser
+// methods that require language configuration.
//
// audio/video vlm.llm_id is not validated: Python's check() has no
// such branch, and audio resolves its model at dispatch time with a
@@ -33,6 +33,7 @@ func TestParserComponent_Check(t *testing.T) {
cases := []struct {
name string
setups map[string]schema.ParserSetup
+ enhance bool
wantErr string // non-empty substring expected in error; empty means no error
}{
// --- PDF family (parser.py:252-261) ---
@@ -96,6 +97,7 @@ func TestParserComponent_Check(t *testing.T) {
{
name: "image: non-ocr without lang → error",
setups: map[string]schema.ParserSetup{"image": {"parse_method": "vlm_xyz", "lang": ""}},
+ enhance: true,
wantErr: "image VLM language",
},
{
@@ -138,7 +140,7 @@ func TestParserComponent_Check(t *testing.T) {
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- c := &ParserComponent{setups: tc.setups}
+ c := &ParserComponent{setups: tc.setups, enableVisionEnhancement: tc.enhance}
err := c.Check()
if tc.wantErr != "" {
if err == nil {
@@ -183,3 +185,28 @@ func TestParserComponent_New_RunsCheck(t *testing.T) {
t.Errorf("want nil component on error, got %T", c)
}
}
+
+func TestNewParserComponentVisionEnhancementSetting(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ params map[string]any
+ enabled bool
+ }{
+ {name: "absent", params: nil},
+ {name: "explicit false", params: map[string]any{"enable_vision_enhancement": false}},
+ {name: "explicit true", params: map[string]any{"enable_vision_enhancement": true}, enabled: true},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ component, err := NewParserComponent(test.params)
+ if err != nil {
+ t.Fatalf("NewParserComponent: %v", err)
+ }
+ if got := component.(*ParserComponent).enableVisionEnhancement; got != test.enabled {
+ t.Fatalf("enableVisionEnhancement = %v, want %v", got, test.enabled)
+ }
+ })
+ }
+ if _, err := NewParserComponent(map[string]any{"enable_vision_enhancement": "true"}); err == nil || !strings.Contains(err.Error(), "must be a boolean") {
+ t.Fatalf("string switch error = %v, want boolean validation", err)
+ }
+}
diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.go
index 14e5088708..dd53b20ced 100644
--- a/internal/ingestion/component/schema/schema_test.go
+++ b/internal/ingestion/component/schema/schema_test.go
@@ -280,6 +280,33 @@ func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {
}
}
+func TestChunkDocVisualParentMetadataRoundTrip(t *testing.T) {
+ imageItem := map[string]any{
+ "text": "caption",
+ "doc_type_kwd": "image",
+ "image": "aGVsbG8=",
+ "parent_table_id": "docx-table-1",
+ "row_index": 2,
+ "column_index": 3,
+ "media_order": 4,
+ }
+ doc, err := ChunkDocFromMap(imageItem)
+ if err != nil {
+ t.Fatalf("ChunkDocFromMap: %v", err)
+ }
+ got := doc.ToMap()
+ for key, want := range map[string]any{
+ "parent_table_id": "docx-table-1",
+ "row_index": float64(2),
+ "column_index": float64(3),
+ "media_order": float64(4),
+ } {
+ if got[key] != want {
+ t.Errorf("metadata %q = %v, want %v", key, got[key], want)
+ }
+ }
+}
+
func TestChunkerOutputsJSONRoundTrip(t *testing.T) {
original := ChunkerOutputs{
OutputFormat: PayloadFormatChunks,
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index ba36f432a0..3f7011b83b 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -14,9 +14,9 @@
// limitations under the License.
//
-// Package component — vision figure enhancement: enriches parsed JSON items
-// (PDF / DOCX / Markdown, OutputFormat=="json") with vision-model descriptions
-// of embedded images/tables. Mirrors Python's enhance_media_sections_with_vision
+// Package component — vision media enhancement: enriches parsed JSON items
+// with vision-model descriptions of embedded images/tables.
+// Mirrors Python's enhance_media_sections_with_vision
// (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).
package component
@@ -26,6 +26,8 @@ import (
"encoding/base64"
"errors"
"fmt"
+ "image"
+ "io"
"os"
"path/filepath"
"strings"
@@ -48,6 +50,7 @@ import (
var (
figureVisionPromptBuilder func(language string) (string, error) = buildFigureVisionPrompt
visionChatInvoker = defaultVisionChatInvoker
+ visionImageCropperFactory = newVisionImageCropper
)
const (
@@ -56,7 +59,8 @@ const (
// visionChatTimeout bounds a single VLM call so a hung endpoint cannot
// occupy one of the concurrency slots indefinitely. Python wraps the
// per-image call in @timeout(30, 3) (deepdoc/parser/figure_parser.py).
- visionChatTimeout = 30 * time.Second
+ visionChatTimeout = 30 * time.Second
+ visionMediaItemBudget = 10 * time.Second
)
var (
@@ -98,60 +102,63 @@ func isUsableVisionImage(raw string) bool {
if raw == "" {
return false
}
+ if len(raw) > maxVLMEncodedBytes+256 {
+ return false
+ }
if strings.HasPrefix(raw, "data:image/") {
idx := strings.Index(raw, "base64,")
if idx < 0 {
return false
}
- return isValidBase64(raw[idx+len("base64,"):])
+ return isValidBase64(raw[idx+len("base64,"):], false)
}
if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
return true
}
- cleaned := strings.Map(func(r rune) rune {
- if r == '\r' || r == '\n' || r == ' ' || r == '\t' {
- return -1
- }
- return r
- }, raw)
- return isValidBase64(cleaned)
-}
-
-func isValidBase64(s string) bool {
- if s == "" {
+ if len(raw) > maxVLMEncodedBytes {
return false
}
- if _, err := base64.StdEncoding.DecodeString(s); err == nil {
- return true
- }
- _, err := base64.RawStdEncoding.DecodeString(s)
- return err == nil
+ return isValidBase64(raw, true)
}
-// isVisionEnhancementAllowed mirrors Python's 3 call sites in
-// rag/flow/parser/parser.py:772/978/1115 (PDF/DOCX/Markdown JSON branches).
-func isVisionEnhancementAllowed(fileType utility.FileType) bool {
- switch fileType {
- case utility.FileTypePDF, utility.FileTypeDOCX, utility.FileTypeMarkdown:
- return true
- default:
+func isValidBase64(s string, stripWhitespace bool) bool {
+ if s == "" || len(s) > maxVLMEncodedBytes {
return false
}
+ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
+ var source io.Reader = strings.NewReader(s)
+ if stripWhitespace {
+ source = &visionPayloadReader{source: s}
+ }
+ decoded := base64.NewDecoder(encoding, source)
+ n, err := io.CopyN(io.Discard, decoded, int64(maxVisionImageBytes)+1)
+ if n > int64(maxVisionImageBytes) {
+ return false
+ }
+ if err == io.EOF {
+ return n > 0
+ }
+ }
+ return false
}
-// visionImageCropper yields a vision-usable base64 image for a parsed item.
-// Under cgo it crops on demand from the source PDF when the item carries
-// positions but no inlined image; under !cgo it returns the inlined image
-// (the only form available without a native renderer). Close releases any
-// re-acquired engine so native handles are not leaked.
+// visionImageCropper materializes the VLM payload for a parsed item. Under
+// cgo it crops PDF sections on demand; inline images stay encoded. Close
+// releases any re-acquired native engine.
type visionImageCropper interface {
- Crop(item map[string]any) (string, error)
+ Crop(ctx context.Context, item map[string]any) (*visionImage, error)
Close() error
}
-// maybeDispatchVisionEnhancement enriches parsed JSON items with vision-model
-// descriptions of embedded images and tables (doc_type_kwd in {"image", "table"}
-// with non-empty image field).
+type visionImage struct {
+ Raster image.Image
+ VLMData string
+ VLMDataValidated bool
+}
+
+// maybeDispatchVisionEnhancement appends VLM descriptions to parsed image
+// resources and table regions. PDF OCR remains in the PDF parser; non-PDF
+// image text comes from this optional enhancement.
// Mirrors Python's enhance_media_sections_with_vision in rag/flow/parser/utils.py:162.
func maybeDispatchVisionEnhancement(
ctx context.Context,
@@ -161,159 +168,230 @@ func maybeDispatchVisionEnhancement(
inputs map[string]any,
setups map[string]schema.ParserSetup,
) (parser.ParseResult, bool, error) {
- // 0. FileType allowlist guard.
- if !isVisionEnhancementAllowed(fileType) {
- return dispatched, false, nil
- }
// Only enhance successful JSON output format containing items.
if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {
return dispatched, false, nil
}
tenantID := getStringOr(inputs, "tenant_id", "")
- if tenantID == "" {
- return dispatched, false, nil
- }
- // Language priority mirrors Python's enhance_media_sections_with_vision
- // (rag/flow/parser/parser.py:778): the run-level dataset language first
- // (the Parser pulls it into inputs from Globals), then the family setup's
- // lang, then English.
family := resolveParserFamily(fileType)
setup := setups[family]
language := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))
- // 1. Collect target items (images/tables that can be described). A target
- // may carry an inlined image (docx/markdown, or any pre-inlined source) or,
- // under cgo for PDF, only PDF positions — in which case the cropper
- // re-acquires the source PDF and crops on demand (the parser no longer
- // inlines PDF media). The cropper transparently handles both forms.
- type target struct {
- idx int
- }
- var targets []target
+ // Collect visual resources, including Markdown images whose type is text
+ // because flatten_media_to_text is enabled.
+ var items []int
for i, item := range dispatched.JSON {
- kd, _ := item["doc_type_kwd"].(string)
- if kd != "image" && kd != "table" {
- continue
- }
if img, _ := item["image"].(string); img != "" {
- targets = append(targets, target{idx: i})
+ items = append(items, i)
continue
}
- if _, ok := parser.ExtractPDFPositions(item); ok {
- targets = append(targets, target{idx: i})
+ kd, _ := item["doc_type_kwd"].(string)
+ if kd == "image" || kd == "table" {
+ if _, ok := parser.ExtractPDFPositions(item); ok {
+ items = append(items, i)
+ }
}
}
- if len(targets) == 0 {
+ if len(items) == 0 {
return dispatched, false, nil
}
-
- // Acquire the on-demand cropper (cgo: crops from storage; !cgo: returns
- // the inlined image). Best-effort: a failure here means no vision
- // enhancement, matching Python's try/except pass.
- cropper, cerr := newVisionImageCropper(ctx, db, inputs)
- if cerr != nil {
- return dispatched, false, nil
- }
- defer cropper.Close()
-
- // 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant
- // default. Mirror Python's vlm_conf["llm_id"] preference.
- modelRef := configuredMediaModelID(setup, family)
+ // Resolve VLM before materializing image payloads, so disabled or
+ // unavailable enhancement does not spend time loading media.
var driver modelModule.ModelDriver
var modelName string
var apiConfig *modelModule.APIConfig
- var err error
- if modelRef != "" {
- driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)
- if err != nil {
- common.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",
- zap.String("family", family), zap.String("modelRef", modelRef), zap.String("tenant", tenantID), zap.Error(err))
+ var prompt string
+ vlmReady := false
+ var resolveErr error
+ if tenantID != "" {
+ modelRef := configuredMediaModelID(setup, family)
+ var err error
+ if modelRef != "" {
+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)
+ if err != nil {
+ common.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",
+ zap.String("family", family), zap.String("modelRef", modelRef), zap.String("tenant", tenantID), zap.Error(err))
+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ }
+ } else {
driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
}
- } else {
- driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)
+ if err == nil {
+ prompt, err = figureVisionPromptBuilder(language)
+ vlmReady = err == nil && driver != nil
+ }
+ resolveErr = err
}
- if err != nil {
- // Model not available — skip vision enhancement silently, matching Python's try/except pass.
+ if !vlmReady {
+ warning := "vision enhancement skipped: tenant ID is missing"
+ if tenantID != "" {
+ warning = "vision enhancement skipped: no usable vision model"
+ if resolveErr != nil {
+ warning = fmt.Sprintf("vision enhancement skipped: %v", resolveErr)
+ }
+ }
+ dispatched.Warnings = append(dispatched.Warnings, warning)
return dispatched, false, nil
}
- // Hoist prompt once: language is invariant across items.
- prompt, perr := figureVisionPromptBuilder(language)
- if perr != nil {
- //nolint:nilerr // Vision enhancement is best-effort.
+ // Materialize one resource at a time. The VLM semaphore is acquired before
+ // materialization, so at most visionEnhancementConcurrency encoded payloads
+ // remain live while model requests run.
+ cropper, cerr := visionImageCropperFactory(ctx, db, inputs)
+ if cerr != nil {
+ dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement skipped: %v", cerr))
return dispatched, false, nil
}
-
- // 3. Concurrently invoke VLM — acquire semaphore before launching goroutine
- // so live goroutine count is bounded by visionEnhancementConcurrency.
- // Stop scheduling new VLM calls after cancellation; already-running calls
- // finish via wg.Wait() and cancellation propagates via ctx through
- // visionChatInvoker.
- descriptions := make([]string, len(targets))
+ defer cropper.Close()
+ modified := false
+ descriptions := make([]string, len(items))
+ failures := make([]string, len(items))
var wg sync.WaitGroup
sem := make(chan struct{}, visionEnhancementConcurrency)
-
-dispatch:
- for slot, tg := range targets {
- if ctx.Err() != nil {
- break dispatch
+ for slot, itemIdx := range items {
+ if err := ctx.Err(); err != nil {
+ break
}
+ vlmSlot := false
select {
case sem <- struct{}{}:
+ vlmSlot = true
case <-ctx.Done():
- break dispatch
+ // The loop exits below when no slot was acquired.
}
- wg.Add(1)
- go func(slot int, itemIdx int) {
- defer wg.Done()
- defer func() { <-sem }()
-
- img, ierr := cropper.Crop(dispatched.JSON[itemIdx])
- if ierr != nil || img == "" {
+ if !vlmSlot {
+ break
+ }
+ var releaseSlotOnce sync.Once
+ releaseSlot := func() {
+ releaseSlotOnce.Do(func() { <-sem })
+ }
+ var resource *visionImage
+ func() {
+ itemCtx, cancelItem := context.WithTimeout(ctx, visionMediaItemBudget)
+ defer cancelItem()
+ release, err := parser.AcquireImageMedia(itemCtx)
+ if err != nil {
+ if ctx.Err() == nil && vlmReady {
+ if payload, _ := dispatched.JSON[itemIdx]["image"].(string); payload != "" {
+ resource = &visionImage{VLMData: payload}
+ }
+ }
+ if resource == nil && ctx.Err() == nil {
+ failures[slot] = fmt.Sprintf("media admission failed: %v", err)
+ }
return
}
- if !isUsableVisionImage(img) {
- common.Warn("vision enhancement: invalid image data skipped",
- zap.Int("item", itemIdx))
+ defer release()
+ item := dispatched.JSON[itemIdx]
+ resource, err = cropper.Crop(itemCtx, item)
+ if err != nil || resource == nil {
+ cropErr := err
+ if ctx.Err() == nil && vlmReady {
+ if payload, _ := item["image"].(string); isUsableVisionImage(payload) {
+ resource = &visionImage{VLMData: payload, VLMDataValidated: true}
+ }
+ }
+ if resource == nil && ctx.Err() == nil {
+ if cropErr != nil {
+ failures[slot] = fmt.Sprintf("image materialization failed: %v", cropErr)
+ } else {
+ failures[slot] = "no usable image payload"
+ }
+ }
return
}
- messages := buildVisionMessages(prompt, img)
+ if ctx.Err() == nil && vlmReady && resource.VLMData == "" && resource.Raster != nil {
+ resource.VLMData, err = encodeVisionRaster(resource.Raster)
+ if err != nil {
+ resource.VLMData = ""
+ failures[slot] = fmt.Sprintf("image encoding failed: %v", err)
+ } else {
+ resource.VLMDataValidated = resource.VLMData != ""
+ }
+ }
+ }()
+ if err := ctx.Err(); err != nil {
+ releaseSlot()
+ break
+ }
+ if resource == nil {
+ releaseSlot()
+ continue
+ }
+ if !resource.VLMDataValidated && !isUsableVisionImage(resource.VLMData) {
+ if failures[slot] == "" {
+ failures[slot] = "image payload is invalid or exceeds configured limits"
+ }
+ releaseSlot()
+ continue
+ }
+ wg.Add(1)
+ go func(slot int, imageData string) {
+ defer wg.Done()
+ defer releaseSlot()
+
+ messages := buildVisionMessages(prompt, imageData)
if len(messages) == 0 {
+ failures[slot] = "could not build a VLM request"
return
}
resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)
if ierr != nil {
+ failures[slot] = fmt.Sprintf("VLM request failed: %v", ierr)
return
}
descriptions[slot] = extractVisionAnswer(resp)
- }(slot, tg.idx)
+ if strings.TrimSpace(descriptions[slot]) == "" {
+ failures[slot] = "VLM returned an empty description"
+ }
+ }(slot, resource.VLMData)
}
wg.Wait()
if err := ctx.Err(); err != nil {
- return dispatched, false, err
+ return dispatched, modified, err
+ }
+ failedItems := 0
+ firstFailure := ""
+ for _, failure := range failures {
+ if failure == "" {
+ continue
+ }
+ failedItems++
+ if firstFailure == "" {
+ firstFailure = failure
+ }
+ }
+ if failedItems > 0 {
+ warning := fmt.Sprintf("vision enhancement failed for %d item(s): %s", failedItems, firstFailure)
+ dispatched.Warnings = append(dispatched.Warnings, warning)
+ common.Warn("vision enhancement: one or more image descriptions failed",
+ zap.Int("failed_items", failedItems), zap.String("first_failure", firstFailure))
}
- // 4. Append descriptions to item text (single newline \n, matching Python).
- modified := false
- for slot, tg := range targets {
+ // Append descriptions after parser-provided text and captions.
+ for slot, itemIdx := range items {
desc := strings.TrimSpace(descriptions[slot])
if desc == "" {
continue
}
- existing, _ := dispatched.JSON[tg.idx]["text"].(string)
- if existing != "" {
- dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc
- } else {
- dispatched.JSON[tg.idx]["text"] = desc
- }
+ appendItemText(dispatched.JSON[itemIdx], desc)
modified = true
}
return dispatched, modified, nil
}
+func appendItemText(item map[string]any, text string) {
+ existing, _ := item["text"].(string)
+ if existing == "" {
+ item["text"] = text
+ return
+ }
+ item["text"] = existing + "\n" + text
+}
+
func buildFigureVisionPrompt(language string) (string, error) {
template, err := loadFigureVisionPromptFile(figureVisionPromptFile)
if err != nil {
diff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.go
index f38674bddb..c7551421c9 100644
--- a/internal/ingestion/component/vision_enhancement_cgo.go
+++ b/internal/ingestion/component/vision_enhancement_cgo.go
@@ -20,9 +20,14 @@ package component
import (
"context"
+ "errors"
"image"
+ "math"
"sync"
+ "time"
+ "go.uber.org/zap"
+ "ragflow/internal/common"
deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
"ragflow/internal/deepdoc/parser/pdf/util"
deepdoctype "ragflow/internal/deepdoc/parser/type"
@@ -40,6 +45,14 @@ var (
visionEngineOpener func(data []byte) (deepdoctype.PDFEngine, error)
)
+const visionPDFSourceBudget = 90 * time.Second
+
+type visionPageSizer interface {
+ PageSize(pageNum int) (float64, float64, error)
+}
+
+var _ visionPageSizer = (*deepdocpdf.PDFOxideEngine)(nil)
+
func init() {
visionSourceFetcher = FetchBinary
visionEngineOpener = deepdocpdf.NewEngine
@@ -53,43 +66,72 @@ func init() {
// chunker does at index time. The engine is opened lazily and at most once per
// vision-enhancement call, then released via Close.
type visionPDFCropper struct {
- ctx context.Context
- db *gorm.DB
- inputs map[string]any
+ invocationCtx context.Context
+ sourceCtx context.Context
+ cancelSource context.CancelFunc
+ db *gorm.DB
+ inputs map[string]any
- once sync.Once
- engine deepdoctype.PDFEngine
- engErr error
+ mu sync.Mutex
+ initialized bool
+ engine deepdoctype.PDFEngine
+ engErr error
+ sourceErr error
}
// newVisionImageCropper builds the on-demand cropper. It never touches storage
// here; the source PDF is re-acquired lazily on the first Crop call that needs
// it.
func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]any) (visionImageCropper, error) {
- return &visionPDFCropper{ctx: ctx, db: db, inputs: inputs}, nil
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ sourceCtx, cancelSource := context.WithTimeout(ctx, visionPDFSourceBudget)
+ return &visionPDFCropper{
+ invocationCtx: ctx,
+ sourceCtx: sourceCtx,
+ cancelSource: cancelSource,
+ db: db,
+ inputs: inputs,
+ }, nil
}
-func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {
+func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visionImage, error) {
+ if err := ctx.Err(); err != nil {
+ return nil, err
+ }
// Fast path: an inlined image (docx/markdown, or any pre-inlined source)
// is used directly — no storage access, no engine.
if img, _ := item["image"].(string); img != "" {
- return img, nil
+ return materializeInlineVisionImage(img)
}
matrix, ok := parser.ExtractPDFPositions(item)
if !ok {
- return "", nil
+ return nil, nil
}
positions := util.PositionsFromMatrix(matrix)
if len(positions) == 0 {
- return "", nil
+ return nil, nil
}
if err := c.ensureEngine(); err != nil {
// Best-effort: a missing/unreadable source PDF means no vision
// description for this item, not a hard failure.
- return "", nil
+ return nil, nil
+ }
+ if err := c.invocationCtx.Err(); err != nil {
+ return nil, err
+ }
+ // Source acquisition has its own invocation-level budget and can outlive
+ // this item's materialization deadline. Once the shared source is ready,
+ // give rendering a fresh per-item budget while still honoring invocation
+ // cancellation.
+ if err := ctx.Err(); err != nil && !errors.Is(err, context.DeadlineExceeded) {
+ return nil, err
}
+ renderCtx, cancelRender := context.WithTimeout(c.invocationCtx, visionMediaItemBudget)
+ defer cancelRender()
if c.engine == nil {
- return "", nil
+ return nil, nil
}
// Render each distinct page the positions span (1-based → 0-based is
// handled by PositionsFromMatrix). Reuse the chunker's sliding-window
@@ -100,8 +142,14 @@ func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {
pages[pn] = struct{}{}
}
}
+ if !pdfPagesRasterWithinVisionLimits(c.engine, pages) {
+ return nil, nil
+ }
single := make(map[int]image.Image, len(pages))
for pn := range pages {
+ if err := renderCtx.Err(); err != nil {
+ return nil, err
+ }
img, rerr := deepdocpdf.RenderPageToImage(c.engine, pn)
if rerr != nil || img == nil {
continue
@@ -109,50 +157,116 @@ func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {
single[pn] = img
}
if len(single) == 0 {
- return "", nil
+ return nil, nil
+ }
+ raster := util.CropSectionPositionsRasterLimited(positions, single, deepdoctype.DlaScale, maxVisionImagePixels)
+ if raster == nil {
+ return nil, nil
}
- return util.CropSectionPositions(positions, single, deepdoctype.DlaScale), nil
+ return &visionImage{Raster: raster}, nil
}
-func (c *visionPDFCropper) ensureEngine() error {
- c.once.Do(func() {
- data, err := c.acquireSource()
- if err != nil || len(data) == 0 {
- c.engErr = err
- return
+func pdfPagesRasterWithinVisionLimits(engine deepdoctype.PDFEngine, pageNums map[int]struct{}) bool {
+ sizer, ok := engine.(visionPageSizer)
+ if !ok {
+ return false
+ }
+ var totalPixels int64
+ for pageNum := range pageNums {
+ widthPoints, heightPoints, err := sizer.PageSize(pageNum)
+ if err != nil {
+ return false
}
- // Only PDFs can be cropped. Guard against other binary types so a
- // docx/markdown item that happens to reach here stays a no-op.
- if len(data) < 5 || string(data[:5]) != "%PDF-" {
- return
+ widthPixels := math.Ceil(widthPoints * deepdoctype.DlaScale)
+ heightPixels := math.Ceil(heightPoints * deepdoctype.DlaScale)
+ if math.IsNaN(widthPixels) || math.IsInf(widthPixels, 0) ||
+ math.IsNaN(heightPixels) || math.IsInf(heightPixels, 0) ||
+ widthPixels <= 0 || heightPixels <= 0 ||
+ widthPixels > maxVisionImageEdge || heightPixels > maxVisionImageEdge ||
+ widthPixels*heightPixels > float64(maxVisionImagePixels) {
+ return false
}
- eng, oerr := visionEngineOpener(data)
- if oerr != nil {
- c.engErr = oerr
- return
+ pagePixels := int64(widthPixels * heightPixels)
+ if pagePixels > maxVisionImagePixels-totalPixels {
+ return false
}
- c.engine = eng
- })
+ totalPixels += pagePixels
+ }
+ return true
+}
+
+func (c *visionPDFCropper) ensureEngine() error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.initialized {
+ return c.engErr
+ }
+ if err := c.sourceCtx.Err(); err != nil {
+ c.initialized = true
+ c.engErr = err
+ c.sourceErr = err
+ return err
+ }
+ data, err := c.acquireSource(c.sourceCtx)
+ if err != nil {
+ c.sourceErr = err
+ if c.sourceCtx.Err() != nil {
+ c.initialized = true
+ c.engErr = err
+ } else if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+ c.initialized = true
+ c.engErr = err
+ }
+ return err
+ }
+ if err := c.sourceCtx.Err(); err != nil {
+ c.initialized = true
+ c.engErr = err
+ c.sourceErr = err
+ return err
+ }
+ c.sourceErr = nil
+ // Empty and non-PDF sources are deterministic no-op results for this
+ // invocation. A transient fetch error can be retried by a later item while
+ // the shared source context remains active.
+ c.initialized = true
+ if len(data) == 0 || len(data) < 5 || string(data[:5]) != "%PDF-" {
+ return nil
+ }
+ eng, err := visionEngineOpener(data)
+ if err != nil {
+ c.engErr = err
+ c.sourceErr = err
+ return err
+ }
+ c.engine = eng
return c.engErr
}
-func (c *visionPDFCropper) acquireSource() ([]byte, error) {
+func (c *visionPDFCropper) acquireSource(ctx context.Context) ([]byte, error) {
if bucket, _ := getString(c.inputs, "bucket"); bucket != "" {
if path, _ := getString(c.inputs, "path"); path != "" {
- return visionSourceFetcher(c.ctx, bucket, path)
+ return visionSourceFetcher(ctx, bucket, path)
}
}
if docID, _ := getString(c.inputs, "doc_id"); docID != "" {
- ref, err := ResolveDocumentStorage(c.ctx, c.db, docID)
+ ref, err := ResolveDocumentStorage(ctx, c.db, docID)
if err != nil || ref == nil {
return nil, err
}
- return visionSourceFetcher(c.ctx, ref.Bucket, ref.Path)
+ return visionSourceFetcher(ctx, ref.Bucket, ref.Path)
}
return nil, nil
}
func (c *visionPDFCropper) Close() error {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.cancelSource()
+ if c.sourceErr != nil && c.invocationCtx.Err() == nil {
+ common.Warn("vision enhancement: source PDF unavailable for VLM crop", zap.Error(c.sourceErr))
+ c.sourceErr = nil
+ }
if c.engine != nil {
return c.engine.Close()
}
diff --git a/internal/ingestion/component/vision_enhancement_cgo_test.go b/internal/ingestion/component/vision_enhancement_cgo_test.go
index 6940f473f0..d1cc4d5cd5 100644
--- a/internal/ingestion/component/vision_enhancement_cgo_test.go
+++ b/internal/ingestion/component/vision_enhancement_cgo_test.go
@@ -6,6 +6,7 @@ import (
"context"
"image"
"testing"
+ "time"
deepdoctype "ragflow/internal/deepdoc/parser/type"
)
@@ -13,7 +14,10 @@ import (
// mockVisionEngine returns a fixed image for any rendered page so the
// on-demand cropper can exercise CropSectionPositions without a real PDF.
type mockVisionEngine struct {
- closed *bool
+ closed *bool
+ pageWidth float64
+ pageHeight float64
+ renderCalls *int
}
func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {
@@ -21,8 +25,21 @@ func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {
}
func (m mockVisionEngine) RenderPage(int, float64) ([]byte, error) { return nil, nil }
func (m mockVisionEngine) RenderPageImage(int, float64) (image.Image, error) {
+ if m.renderCalls != nil {
+ *m.renderCalls++
+ }
return image.NewRGBA(image.Rect(0, 0, 1000, 1000)), nil
}
+func (m mockVisionEngine) PageSize(int) (float64, float64, error) {
+ width, height := m.pageWidth, m.pageHeight
+ if width == 0 {
+ width = 333
+ }
+ if height == 0 {
+ height = 333
+ }
+ return width, height, nil
+}
func (m mockVisionEngine) RawData() []byte { return nil }
func (m mockVisionEngine) PageCount() (int, error) { return 1, nil }
func (m mockVisionEngine) Outlines() ([]deepdoctype.Outline, error) { return nil, nil }
@@ -64,18 +81,90 @@ func TestVisionCropImage_OnDemandFromPositions(t *testing.T) {
}
defer cropper.Close()
- img, err := cropper.Crop(cgoPositions())
+ img, err := cropper.Crop(context.Background(), cgoPositions())
if err != nil {
t.Fatalf("Crop: %v", err)
}
- if img == "" {
- t.Fatal("Crop returned empty; expected an on-demand cropped base64 image")
+ if img == nil || img.Raster == nil {
+ t.Fatal("Crop returned no raster; expected an on-demand cropped image")
+ }
+ if img.Raster.Bounds().Empty() {
+ t.Fatal("Crop returned an empty raster")
}
if closed {
t.Fatal("engine closed before use completed")
}
}
+func TestVisionCropImage_RejectsOversizedPageBeforeRendering(t *testing.T) {
+ renderCalls := 0
+ oldFetcher := visionSourceFetcher
+ oldOpener := visionEngineOpener
+ defer func() {
+ visionSourceFetcher = oldFetcher
+ visionEngineOpener = oldOpener
+ }()
+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
+ return []byte("%PDF-fake-engine-bytes"), nil
+ }
+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
+ return mockVisionEngine{pageWidth: 5000, pageHeight: 1000, renderCalls: &renderCalls}, nil
+ }
+
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ defer cropper.Close()
+
+ img, err := cropper.Crop(context.Background(), cgoPositions())
+ if err != nil {
+ t.Fatalf("Crop: %v", err)
+ }
+ if img != nil {
+ t.Fatalf("Crop = %#v, want nil for an oversized page", img)
+ }
+ if renderCalls != 0 {
+ t.Fatalf("render calls = %d, want page rejected before raster allocation", renderCalls)
+ }
+}
+
+func TestVisionCropImage_RejectsAggregatePagePixelsBeforeRendering(t *testing.T) {
+ renderCalls := 0
+ oldFetcher := visionSourceFetcher
+ oldOpener := visionEngineOpener
+ defer func() {
+ visionSourceFetcher = oldFetcher
+ visionEngineOpener = oldOpener
+ }()
+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
+ return []byte("%PDF-fake-engine-bytes"), nil
+ }
+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
+ return mockVisionEngine{pageWidth: 2000, pageHeight: 2000, renderCalls: &renderCalls}, nil
+ }
+
+ item := map[string]any{
+ "_pdf_positions": [][]any{{[]any{1, 2}, 10.0, 100.0, 10.0, 100.0}},
+ }
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ defer cropper.Close()
+
+ img, err := cropper.Crop(context.Background(), item)
+ if err != nil {
+ t.Fatalf("Crop: %v", err)
+ }
+ if img != nil {
+ t.Fatalf("Crop = %#v, want nil when combined page pixels exceed the vision budget", img)
+ }
+ if renderCalls != 0 {
+ t.Fatalf("render calls = %d, want pages rejected before raster allocation", renderCalls)
+ }
+}
+
// TestVisionCropImage_PassthroughInline proves that when the item already
// carries an inlined image (docx/markdown, or any pre-inlined source), the
// cropper returns it directly without touching storage.
@@ -95,12 +184,12 @@ func TestVisionCropImage_PassthroughInline(t *testing.T) {
defer cropper.Close()
const inline = "data:image/png;base64,preinlined"
- img, err := cropper.Crop(map[string]any{"image": inline})
+ img, err := cropper.Crop(context.Background(), map[string]any{"image": inline})
if err != nil {
t.Fatalf("Crop: %v", err)
}
- if img != inline {
- t.Fatalf("Crop = %q, want passthrough %q", img, inline)
+ if img == nil || img.VLMData != inline {
+ t.Fatalf("Crop = %#v, want VLM payload %q", img, inline)
}
if fetchCalled {
t.Fatal("storage fetch must not be called for an inlined image")
@@ -124,12 +213,12 @@ func TestVisionCropImage_NoImageNoPositions(t *testing.T) {
}
defer cropper.Close()
- img, err := cropper.Crop(map[string]any{"doc_type_kwd": "image"})
+ img, err := cropper.Crop(context.Background(), map[string]any{"doc_type_kwd": "image"})
if err != nil {
t.Fatalf("Crop: %v", err)
}
- if img != "" {
- t.Fatalf("Crop = %q, want empty (no image, no positions)", img)
+ if img != nil {
+ t.Fatalf("Crop = %#v, want nil (no image, no positions)", img)
}
if fetchCalled {
t.Fatal("storage fetch must not be called without positions")
@@ -161,12 +250,12 @@ func TestVisionCropImage_NonPDFBytesIsNoOp(t *testing.T) {
}
defer cropper.Close()
- img, err := cropper.Crop(cgoPositions())
+ img, err := cropper.Crop(context.Background(), cgoPositions())
if err != nil {
t.Fatalf("Crop: %v", err)
}
- if img != "" {
- t.Fatalf("Crop = %q, want empty for non-PDF bytes", img)
+ if img != nil {
+ t.Fatalf("Crop = %#v, want nil for non-PDF bytes", img)
}
if openerCalled {
t.Fatal("engine opener must not run on non-PDF bytes")
@@ -194,7 +283,7 @@ func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {
if err != nil {
t.Fatalf("newVisionImageCropper: %v", err)
}
- if _, err := cropper.Crop(cgoPositions()); err != nil {
+ if _, err := cropper.Crop(context.Background(), cgoPositions()); err != nil {
t.Fatalf("Crop: %v", err)
}
if err := cropper.Close(); err != nil {
@@ -204,3 +293,147 @@ func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {
t.Fatal("engine should be closed after use")
}
}
+
+func TestVisionCropImage_RetriesAfterTransientSourceFailure(t *testing.T) {
+ oldFetcher := visionSourceFetcher
+ oldOpener := visionEngineOpener
+ t.Cleanup(func() {
+ visionSourceFetcher = oldFetcher
+ visionEngineOpener = oldOpener
+ })
+
+ fetchCalls := 0
+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
+ fetchCalls++
+ if fetchCalls == 1 {
+ return nil, context.DeadlineExceeded
+ }
+ return []byte("%PDF-fake-engine-bytes"), nil
+ }
+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
+ return mockVisionEngine{}, nil
+ }
+
+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ t.Cleanup(func() { _ = cropper.Close() })
+
+ if _, err := cropper.Crop(context.Background(), cgoPositions()); err != nil {
+ t.Fatalf("first Crop: %v", err)
+ }
+ img, err := cropper.Crop(context.Background(), cgoPositions())
+ if err != nil {
+ t.Fatalf("second Crop: %v", err)
+ }
+ if img == nil || img.Raster == nil {
+ t.Fatal("second Crop returned no raster after the source fetch recovered")
+ }
+ if fetchCalls != 2 {
+ t.Fatalf("source fetch calls = %d, want 2 after transient failure", fetchCalls)
+ }
+}
+
+func TestVisionCropImage_ReusesSourceAfterItemContextExpires(t *testing.T) {
+ oldFetcher := visionSourceFetcher
+ oldOpener := visionEngineOpener
+ t.Cleanup(func() {
+ visionSourceFetcher = oldFetcher
+ visionEngineOpener = oldOpener
+ })
+
+ fetchStarted := make(chan struct{})
+ finishFetch := make(chan struct{})
+ fetchCalls := 0
+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {
+ fetchCalls++
+ select {
+ case fetchStarted <- struct{}{}:
+ default:
+ }
+ select {
+ case <-finishFetch:
+ return []byte("%PDF-fake-engine-bytes"), nil
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+ }
+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {
+ return mockVisionEngine{}, nil
+ }
+
+ invocationCtx, cancelInvocation := context.WithCancel(context.Background())
+ defer cancelInvocation()
+ cropper, err := newVisionImageCropper(invocationCtx, nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ t.Cleanup(func() { _ = cropper.Close() })
+
+ firstItemCtx, cancelFirstItem := context.WithTimeout(invocationCtx, 20*time.Millisecond)
+ defer cancelFirstItem()
+ firstResult := make(chan struct {
+ img *visionImage
+ err error
+ }, 1)
+ go func() {
+ img, err := cropper.Crop(firstItemCtx, cgoPositions())
+ firstResult <- struct {
+ img *visionImage
+ err error
+ }{img: img, err: err}
+ }()
+ <-fetchStarted
+ <-firstItemCtx.Done()
+ close(finishFetch)
+ first := <-firstResult
+ if first.err != nil {
+ t.Fatalf("first Crop: %v", first.err)
+ }
+ if first.img == nil || first.img.Raster == nil {
+ t.Fatal("first Crop returned no raster after the shared source fetch completed")
+ }
+
+ secondItemCtx, cancelSecondItem := context.WithCancel(invocationCtx)
+ defer cancelSecondItem()
+ second, err := cropper.Crop(secondItemCtx, cgoPositions())
+ if err != nil {
+ t.Fatalf("second Crop: %v", err)
+ }
+ if second == nil || second.Raster == nil {
+ t.Fatal("second Crop returned no raster after the shared source fetch completed")
+ }
+ if fetchCalls != 1 {
+ t.Fatalf("source fetch calls = %d, want 1 across item contexts", fetchCalls)
+ }
+}
+
+func TestVisionCropImage_CanceledParentDoesNotRetrySource(t *testing.T) {
+ oldFetcher := visionSourceFetcher
+ t.Cleanup(func() { visionSourceFetcher = oldFetcher })
+
+ ctx, cancel := context.WithCancel(context.Background())
+ fetchCalls := 0
+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {
+ fetchCalls++
+ cancel()
+ return nil, context.Canceled
+ }
+
+ cropper, err := newVisionImageCropper(ctx, nil, map[string]any{"bucket": "b", "path": "p"})
+ if err != nil {
+ t.Fatalf("newVisionImageCropper: %v", err)
+ }
+ t.Cleanup(func() { _ = cropper.Close() })
+
+ if img, err := cropper.Crop(ctx, cgoPositions()); err != nil || img != nil {
+ t.Fatalf("first Crop = (%#v, %v), want best-effort no result after cancellation", img, err)
+ }
+ if _, err := cropper.Crop(ctx, cgoPositions()); err == nil {
+ t.Fatal("subsequent Crop with a canceled parent context should return its cancellation")
+ }
+ if fetchCalls != 1 {
+ t.Fatalf("source fetch calls = %d, want one; canceled parent must prevent retries", fetchCalls)
+ }
+}
diff --git a/internal/ingestion/component/vision_enhancement_nocgo.go b/internal/ingestion/component/vision_enhancement_nocgo.go
index df3522199d..07d51a5641 100644
--- a/internal/ingestion/component/vision_enhancement_nocgo.go
+++ b/internal/ingestion/component/vision_enhancement_nocgo.go
@@ -35,11 +35,11 @@ func newVisionImageCropper(_ context.Context, _ *gorm.DB, _ map[string]any) (vis
return visionPDFCropperNoop{}, nil
}
-func (visionPDFCropperNoop) Crop(item map[string]any) (string, error) {
+func (visionPDFCropperNoop) Crop(_ context.Context, item map[string]any) (*visionImage, error) {
if img, _ := item["image"].(string); img != "" {
- return img, nil
+ return materializeInlineVisionImage(img)
}
- return "", nil
+ return nil, nil
}
func (visionPDFCropperNoop) Close() error { return nil }
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index cb68846f30..ee8ae28aa6 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -17,17 +17,24 @@
package component
import (
+ "bytes"
"context"
+ "encoding/base64"
"errors"
"fmt"
+ "image"
+ "image/png"
"os"
"path/filepath"
+ "strings"
"sync"
"testing"
"time"
"ragflow/internal/common"
"ragflow/internal/dao"
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
"ragflow/internal/ingestion/component/schema"
@@ -41,6 +48,14 @@ type visionEnhanceFakeDriver struct {
modelModule.ModelDriver
}
+type failingVisionImageCropper struct{ err error }
+
+func (c failingVisionImageCropper) Crop(context.Context, map[string]any) (*visionImage, error) {
+ return nil, c.err
+}
+
+func (failingVisionImageCropper) Close() error { return nil }
+
type visionEnhanceCaptureInvoker struct {
mu sync.Mutex
images []string
@@ -105,6 +120,265 @@ func fakePrompt(language string) (string, error) {
return "describe the figure in " + language, nil
}
+func visionTestPNGBase64(t *testing.T) string {
+ t.Helper()
+ var encoded bytes.Buffer
+ if err := png.Encode(&encoded, image.NewRGBA(image.Rect(0, 0, 20, 20))); err != nil {
+ t.Fatalf("encode image: %v", err)
+ }
+ return base64.StdEncoding.EncodeToString(encoded.Bytes())
+}
+
+func TestParserComponent_VisionEnhancementSwitchGatesEmbeddedImage(t *testing.T) {
+ previousAnalyzer := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previousAnalyzer })
+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
+ return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true
+ })
+
+ resolverCalls := 0
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ resolverCalls++
+ return fakeResolver(ctx, db, tenantID, modelType)
+ }, invoker.invoke, fakePrompt)
+ inputs := func() map[string]any {
+ return map[string]any{
+ "binary": []byte(`<p><img src="data:image/png;base64,` + visionTestPNGBase64(t) + `" alt="figure"></p>`),
+ "file_type": "html",
+ "name": "figure.html",
+ "tenant_id": "tenant-1",
+ }
+ }
+
+ defaultComponent, err := NewParserComponent(nil)
+ if err != nil {
+ t.Fatalf("NewParserComponent(nil): %v", err)
+ }
+ withoutVision, err := defaultComponent.Invoke(t.Context(), nil, inputs())
+ if err != nil {
+ t.Fatalf("default Invoke: %v", err)
+ }
+ if resolverCalls != 0 || len(invoker.images) != 0 {
+ t.Fatalf("default-off enhancement used model: %d resolves, %d VLM calls", resolverCalls, len(invoker.images))
+ }
+ withoutItems, ok := withoutVision["json"].([]map[string]any)
+ if !ok || len(withoutItems) == 0 {
+ t.Fatalf("default JSON = %T/%v", withoutVision["json"], withoutVision["json"])
+ }
+
+ enabledComponent, err := NewParserComponent(map[string]any{"enable_vision_enhancement": true})
+ if err != nil {
+ t.Fatalf("NewParserComponent(enabled): %v", err)
+ }
+ withVision, err := enabledComponent.Invoke(t.Context(), nil, inputs())
+ if err != nil {
+ t.Fatalf("enabled Invoke: %v", err)
+ }
+ if resolverCalls != 1 || len(invoker.images) != 1 {
+ t.Fatalf("enabled enhancement used model %d/%d times, want 1/1", resolverCalls, len(invoker.images))
+ }
+ withItems, ok := withVision["json"].([]map[string]any)
+ if !ok || len(withItems) == 0 {
+ t.Fatalf("enabled JSON = %T/%v", withVision["json"], withVision["json"])
+ }
+ withoutText, _ := withoutItems[0]["text"].(string)
+ withText, _ := withItems[0]["text"].(string)
+ if strings.Contains(withoutText, "a diagram of a pipeline") || !strings.Contains(withText, "a diagram of a pipeline") {
+ t.Fatalf("image texts without/with enhancement = %q / %q", withoutText, withText)
+ }
+}
+
+func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{{
+ "text": "Existing caption",
+ "image": visionTestPNGBase64(t),
+ "doc_type_kwd": "image",
+ }},
+ }
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(), dao.DB, utility.FileTypeXLSX, dispatched,
+ map[string]any{"tenant_id": "t1"}, nil)
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want true")
+ }
+ want := "Existing caption\na diagram of a pipeline"
+ if got := res.JSON[0]["text"]; got != want {
+ t.Errorf("enhanced text = %q, want %q", got, want)
+ }
+ if len(invoker.images) != 1 {
+ t.Errorf("VLM calls = %d, want 1", len(invoker.images))
+ }
+}
+
+func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {
+ swapVisionGlobals(t, fakeResolver, func(context.Context, modelModule.ModelDriver, string, []modelModule.Message, *modelModule.APIConfig) (*modelModule.ChatResponse, error) {
+ return nil, errors.New("VLM unavailable")
+ }, fakePrompt)
+
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{{
+ "image": visionTestPNGBase64(t),
+ "doc_type_kwd": "image",
+ }},
+ }
+ result, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeDOCX, dispatched,
+ map[string]any{"tenant_id": "t1"}, nil)
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "VLM unavailable") {
+ t.Fatalf("warnings = %v, want the VLM failure", result.Warnings)
+ }
+}
+
+func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+
+ originalFactory := visionImageCropperFactory
+ visionImageCropperFactory = func(context.Context, *gorm.DB, map[string]any) (visionImageCropper, error) {
+ return failingVisionImageCropper{err: errors.New("crop failed")}, nil
+ }
+ t.Cleanup(func() { visionImageCropperFactory = originalFactory })
+
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{{
+ "image": visionTestPNGBase64(t),
+ "doc_type_kwd": "image",
+ }},
+ }
+ result, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,
+ map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}},
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want inline VLM fallback")
+ }
+ if len(invoker.images) != 1 {
+ t.Fatalf("VLM calls = %d, want 1", len(invoker.images))
+ }
+ if !strings.HasPrefix(invoker.images[0], "data:image/png;base64,") {
+ t.Errorf("VLM image URL = %q, want inline PNG data URI", invoker.images[0])
+ }
+ if got, want := result.JSON[0]["text"], "a diagram of a pipeline"; got != want {
+ t.Errorf("enhanced text = %q, want %q", got, want)
+ }
+}
+
+func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.T) {
+ for _, parseMethod := range []string{"deepdoc", "mineru"} {
+ t.Run(parseMethod, func(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{{
+ "text": "Existing table",
+ "image": visionTestPNGBase64(t),
+ "doc_type_kwd": "table",
+ }},
+ }
+ res, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,
+ map[string]any{"tenant_id": "t1"},
+ map[string]schema.ParserSetup{"pdf": {"parse_method": parseMethod}},
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want VLM enhancement")
+ }
+ if got, want := res.JSON[0]["text"], "Existing table\na diagram of a pipeline"; got != want {
+ t.Errorf("enhanced text = %q, want %q", got, want)
+ }
+ })
+ }
+}
+
+func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+ imagePayload := visionTestPNGBase64(t)
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{
+ {"text": "<table><tr><td>figure</td></tr></table>", "image": imagePayload, "doc_type_kwd": "table"},
+ {
+ "text": "cell alt",
+ "image": imagePayload,
+ "doc_type_kwd": "image",
+ "parent_table_id": "html-table-1",
+ "row_index": 1,
+ "column_index": 1,
+ "media_order": 1,
+ },
+ },
+ }
+
+ result, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,
+ map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "deepdoc"}},
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want table and cell image VLM processing")
+ }
+ if got, want := result.JSON[0]["text"], "<table><tr><td>figure</td></tr></table>\na diagram of a pipeline"; got != want {
+ t.Errorf("table text = %q, want %q", got, want)
+ }
+ cellWant := "cell alt\na diagram of a pipeline"
+ if got := result.JSON[1]["text"]; got != cellWant {
+ t.Errorf("cell image text = %q, want %q", got, cellWant)
+ }
+ if len(invoker.images) != 2 {
+ t.Errorf("VLM calls = %d, want table plus cell image", len(invoker.images))
+ }
+}
+
+func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
+ dispatched := parser.ParseResult{
+ OutputFormat: "json",
+ JSON: []map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}},
+ }
+
+ result, handled, err := maybeDispatchVisionEnhancement(
+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,
+ map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}},
+ )
+ if err != nil {
+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)
+ }
+ if !handled {
+ t.Fatal("handled = false, want VLM enhancement")
+ }
+ if got, want := result.JSON[0]["text"], "existing\na diagram of a pipeline"; got != want {
+ t.Errorf("image text = %q, want %q", got, want)
+ }
+ if len(invoker.images) != 1 {
+ t.Errorf("VLM calls = %d, want 1", len(invoker.images))
+ }
+}
+
func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
testCases := []struct {
name string
@@ -274,11 +548,13 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {
}
}
-func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
+func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
+ invoker := &visionEnhanceCaptureInvoker{}
+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
dispatched := parser.ParseResult{
OutputFormat: "json",
JSON: []map[string]any{
- {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
+ {"text": "caption", "image": "aGVsbG8=", "doc_type_kwd": "image"},
},
}
@@ -291,11 +567,14 @@ func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
- if handled {
- t.Error("handled = true, want false for FileTypeOTHER")
+ if !handled {
+ t.Fatal("handled = false, want true for a visual payload in FileTypeOTHER")
}
- if res.JSON[0]["text"] != "" {
- t.Errorf("text = %q, want empty", res.JSON[0]["text"])
+ if got, want := res.JSON[0]["text"], "caption\na diagram of a pipeline"; got != want {
+ t.Errorf("text = %q, want %q", got, want)
+ }
+ if len(invoker.images) != 1 {
+ t.Fatalf("VLM calls = %d, want 1", len(invoker.images))
}
}
diff --git a/internal/ingestion/component/vision_image.go b/internal/ingestion/component/vision_image.go
new file mode 100644
index 0000000000..6a09a45cdd
--- /dev/null
+++ b/internal/ingestion/component/vision_image.go
@@ -0,0 +1,205 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "bytes"
+ "encoding/base64"
+ "fmt"
+ "image"
+ "image/png"
+ "io"
+ "strings"
+
+ _ "golang.org/x/image/bmp"
+ _ "golang.org/x/image/tiff"
+ _ "golang.org/x/image/webp"
+ _ "image/gif"
+ _ "image/jpeg"
+
+ parserlib "ragflow/internal/parser/parser"
+)
+
+const (
+ maxVisionImageBytes = parserlib.MaxImagePayloadBytes
+ maxVisionImagePixels = parserlib.MaxImagePixels
+ maxVisionImageEdge = parserlib.MaxImageEdge
+ maxVLMEncodedBytes = (maxVisionImageBytes+2)/3*4 + 256
+)
+
+func materializeInlineVisionImage(raw string) (*visionImage, error) {
+ raw = strings.TrimSpace(raw)
+ if raw == "" {
+ return nil, nil
+ }
+ materialized := &visionImage{VLMData: raw}
+ if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {
+ materialized.VLMDataValidated = isUsableVisionImage(raw)
+ return materialized, nil
+ }
+
+ isDataURI := strings.HasPrefix(strings.ToLower(raw), "data:image/")
+ payload := raw
+ if isDataURI {
+ comma := strings.IndexByte(raw, ',')
+ if comma <= 0 || !strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {
+ return materialized, nil
+ }
+ payload = raw[comma+1:]
+ } else if strings.HasPrefix(strings.ToLower(raw), "data:") {
+ return materialized, nil
+ }
+ format, err := inspectVisionImagePayload(payload)
+ if err != nil {
+ materialized.VLMDataValidated = false
+ return materialized, nil
+ }
+ materialized.VLMDataValidated = true
+ if strings.ContainsAny(payload, "\r\n \t") {
+ if data, decodeErr := decodeVisionPayload(payload); decodeErr == nil {
+ payload = base64.StdEncoding.EncodeToString(data)
+ }
+ }
+ if isDataURI {
+ if payload != raw[strings.IndexByte(raw, ',')+1:] {
+ materialized.VLMData = raw[:strings.IndexByte(raw, ',')+1] + payload
+ }
+ return materialized, nil
+ }
+ if mimeType := imageMIMEForFormat(format); mimeType != "" {
+ materialized.VLMData = "data:" + mimeType + ";base64," + payload
+ }
+ return materialized, nil
+}
+
+func inspectVisionImagePayload(payload string) (string, error) {
+ if payload == "" || len(payload) > maxVLMEncodedBytes {
+ return "", fmt.Errorf("vision image: encoded payload exceeds limit")
+ }
+ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
+ decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: payload})
+ config, format, err := image.DecodeConfig(io.LimitReader(decoder, maxVisionImageBytes+1))
+ if err != nil {
+ continue
+ }
+ if config.Width <= 0 || config.Height <= 0 || config.Width > maxVisionImageEdge || config.Height > maxVisionImageEdge || int64(config.Width)*int64(config.Height) > maxVisionImagePixels {
+ return "", fmt.Errorf("vision image: image dimensions %dx%d exceed limits", config.Width, config.Height)
+ }
+ return format, nil
+ }
+ return "", fmt.Errorf("vision image: invalid image payload")
+}
+
+func decodeVisionPayload(raw string) ([]byte, error) {
+ encoded := strings.TrimSpace(raw)
+ if strings.HasPrefix(strings.ToLower(encoded), "data:") {
+ comma := strings.IndexByte(encoded, ',')
+ if comma < 0 || !strings.Contains(strings.ToLower(encoded[:comma]), ";base64") {
+ return nil, fmt.Errorf("vision image: unsupported data URI")
+ }
+ encoded = encoded[comma+1:]
+ }
+ maxEncodedBytes := (maxVisionImageBytes+2)/3*4 + 256
+ if len(encoded) == 0 || len(encoded) > maxEncodedBytes {
+ return nil, fmt.Errorf("vision image: encoded payload exceeds limit")
+ }
+ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
+ decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: encoded})
+ data, err := io.ReadAll(io.LimitReader(decoder, maxVisionImageBytes+1))
+ if err != nil {
+ continue
+ }
+ if len(data) > maxVisionImageBytes {
+ return nil, fmt.Errorf("vision image: payload exceeds %d bytes", maxVisionImageBytes)
+ }
+ if len(data) == 0 {
+ return nil, fmt.Errorf("vision image: empty base64 payload")
+ }
+ return data, nil
+ }
+ return nil, fmt.Errorf("vision image: decode base64 failed")
+}
+
+type visionPayloadReader struct {
+ source string
+ offset int
+}
+
+func (r *visionPayloadReader) Read(dst []byte) (int, error) {
+ written := 0
+ for written < len(dst) && r.offset < len(r.source) {
+ b := r.source[r.offset]
+ r.offset++
+ if b == '\r' || b == '\n' || b == ' ' || b == '\t' {
+ continue
+ }
+ dst[written] = b
+ written++
+ }
+ if written > 0 {
+ return written, nil
+ }
+ if r.offset == len(r.source) {
+ return 0, io.EOF
+ }
+ return 0, nil
+}
+
+func imageWithinVisionLimits(img image.Image) bool {
+ if img == nil {
+ return false
+ }
+ bounds := img.Bounds()
+ width, height := bounds.Dx(), bounds.Dy()
+ return width > 0 && height > 0 && width <= maxVisionImageEdge && height <= maxVisionImageEdge && int64(width)*int64(height) <= maxVisionImagePixels
+}
+
+func imageMIMEForFormat(format string) string {
+ switch strings.ToLower(format) {
+ case "jpeg":
+ return "image/jpeg"
+ case "png":
+ return "image/png"
+ case "gif":
+ return "image/gif"
+ case "bmp":
+ return "image/bmp"
+ case "tiff":
+ return "image/tiff"
+ case "webp":
+ return "image/webp"
+ default:
+ return ""
+ }
+}
+
+func encodeVisionRaster(img image.Image) (string, error) {
+ if img == nil {
+ return "", nil
+ }
+ if !imageWithinVisionLimits(img) {
+ return "", fmt.Errorf("vision image: raster dimensions exceed limits")
+ }
+ var encoded bytes.Buffer
+ if err := png.Encode(&encoded, img); err != nil {
+ return "", fmt.Errorf("vision image: encode raster: %w", err)
+ }
+ if encoded.Len() > maxVisionImageBytes {
+ return "", fmt.Errorf("vision image: encoded raster exceeds %d bytes", maxVisionImageBytes)
+ }
+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), nil
+}
diff --git a/internal/ingestion/component/vision_image_test.go b/internal/ingestion/component/vision_image_test.go
new file mode 100644
index 0000000000..450ac8ac0b
--- /dev/null
+++ b/internal/ingestion/component/vision_image_test.go
@@ -0,0 +1,54 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package component
+
+import (
+ "bytes"
+ "encoding/base64"
+ "image"
+ "image/png"
+ "strings"
+ "testing"
+)
+
+func TestMaterializeInlineVisionImageKeepsVLMForImageDimensionLimit(t *testing.T) {
+ var encoded bytes.Buffer
+ large := image.NewRGBA(image.Rect(0, 0, maxVisionImageEdge+1, 1))
+ if err := png.Encode(&encoded, large); err != nil {
+ t.Fatalf("encode oversized image: %v", err)
+ }
+ dataURI := "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes())
+
+ materialized, err := materializeInlineVisionImage(dataURI)
+ if err != nil {
+ t.Fatalf("materializeInlineVisionImage: %v", err)
+ }
+ if materialized == nil || materialized.Raster != nil {
+ t.Fatalf("raster = %#v, want OCR to skip the oversized image", materialized)
+ }
+ if materialized.VLMData != dataURI || !isUsableVisionImage(materialized.VLMData) {
+ t.Fatal("oversized OCR input should preserve its valid VLM payload")
+ }
+}
+
+func TestIsUsableVisionImageRejectsPayloadOverVLMByteBudget(t *testing.T) {
+ encodedLength := base64.StdEncoding.EncodedLen(maxVisionImageBytes + 3)
+ payload := strings.Repeat("A", encodedLength)
+ if isUsableVisionImage(payload) {
+ t.Fatal("base64 payload above the VLM byte budget must be rejected")
+ }
+}
diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.go
index 8ba42ad7d6..f6a2f8569c 100644
--- a/internal/parser/parser/docx_ir.go
+++ b/internal/parser/parser/docx_ir.go
@@ -27,6 +27,7 @@ package parser
import (
"encoding/base64"
"encoding/json"
+ "fmt"
"html"
"strings"
)
@@ -204,6 +205,81 @@ func docxIRTableToHTML(el docxIRElement) string {
return sb.String()
}
+type docxTableImage struct {
+ data []byte
+ row int
+ column int
+ included bool
+}
+
+func forEachDOCXTableImage(el docxIRElement, visit func(docxTableImage) bool) bool {
+ for rowIndex, row := range el.Rows {
+ for columnIndex, cell := range row.Cells {
+ completed := forEachDOCXIRImage(cell.Content, func(data []byte) bool {
+ return visit(docxTableImage{data: data, row: rowIndex + 1, column: columnIndex + 1})
+ })
+ if !completed {
+ return false
+ }
+ }
+ }
+ return true
+}
+
+// forEachDOCXIRImage visits image payloads without first collecting a second
+// document-sized slice of image references. Returning false from visit stops
+// traversal as soon as the caller's media budget is exhausted.
+func forEachDOCXIRImage(elements []docxIRElement, visit func([]byte) bool) bool {
+ for _, el := range elements {
+ switch el.Type {
+ case "image":
+ if len(el.Data) > 0 && !visit(el.Data) {
+ return false
+ }
+ case "paragraph", "heading":
+ for _, run := range el.contentRuns() {
+ if run.Type == "image" && len(run.Data) > 0 && !visit(run.Data) {
+ return false
+ }
+ if !forEachDOCXIRImage(run.Content, visit) {
+ return false
+ }
+ }
+ case "table":
+ for _, row := range el.Rows {
+ for _, cell := range row.Cells {
+ if !forEachDOCXIRImage(cell.Content, visit) {
+ return false
+ }
+ }
+ }
+ case "list":
+ if !forEachDOCXIRListImage(docxIRList{Items: el.Items}, visit) {
+ return false
+ }
+ case "text_box":
+ if !forEachDOCXIRImage(el.contentBlocks(), visit) {
+ return false
+ }
+ }
+ }
+ return true
+}
+
+func forEachDOCXIRListImage(list docxIRList, visit func([]byte) bool) bool {
+ for _, item := range list.Items {
+ if !forEachDOCXIRImage(item.Content, visit) {
+ return false
+ }
+ if item.Nested != nil {
+ if !forEachDOCXIRListImage(*item.Nested, visit) {
+ return false
+ }
+ }
+ }
+ return true
+}
+
// docxElementText returns the plain-text rendering of any supported
// IR element type. Used by buildPPTXJSONSections to flatten one slide
// section, and by extractDOCXFiguresFromIR so that tables, lists, and
@@ -253,60 +329,86 @@ func docxElementText(el docxIRElement, cellSep string) string {
// buildDOCXJSONSections converts an office_oxide IR JSON string into a
// slice of structured items compatible with the chunker's JSON input
// contract. Each item carries at least text and doc_type_kwd.
-func buildDOCXJSONSections(irJSON string) []map[string]any {
+func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[string]any {
var ir docxIRDocument
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
return nil
}
+ if budget == nil {
+ budget = newEmbeddedMediaBudget()
+ }
var sections []map[string]any
+ tableSequence := 0
for _, sec := range ir.Sections {
for _, el := range sec.Elements {
switch el.Type {
case "paragraph", "heading":
- sections = appendDOCXParagraphSections(sections, el, el.Type == "heading")
+ sections = appendDOCXParagraphSections(sections, el, el.Type == "heading", budget)
case "image":
- b64 := base64.StdEncoding.EncodeToString(el.Data)
- sections = append(sections, map[string]any{
- "text": "",
- "image": b64,
- "doc_type_kwd": "image",
- })
+ sections, _ = appendDOCXImageSection(sections, el.Data, budget, nil)
case "table":
+ tableSequence++
html := docxIRTableToHTML(el)
if html == "<table></table>" {
continue
}
- sections = append(sections, map[string]any{
+ table := map[string]any{
"text": html,
"image": nil,
"doc_type_kwd": "table",
+ }
+ tableID := ""
+ foundImage := false
+ var tableImages []docxTableImage
+ forEachDOCXTableImage(el, func(tableImage docxTableImage) bool {
+ foundImage = true
+ included, keepWalking := budget.include(tableImage.data)
+ if included || keepWalking {
+ tableImage.included = included
+ tableImages = append(tableImages, tableImage)
+ }
+ return keepWalking
})
+ if foundImage {
+ tableID = fmt.Sprintf("docx-table-%d", tableSequence)
+ table["source_table_id"] = tableID
+ }
+ sections = append(sections, table)
+ for mediaOrder, tableImage := range tableImages {
+ metadata := map[string]any{
+ "parent_table_id": tableID,
+ "row_index": tableImage.row,
+ "column_index": tableImage.column,
+ "media_order": mediaOrder + 1,
+ }
+ sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata)
+ }
case "list":
for _, item := range el.Items {
text := extractTextFromListItem(item)
- if text == "" {
- continue
+ if text != "" {
+ sections = append(sections, map[string]any{
+ "text": text,
+ "image": nil,
+ "doc_type_kwd": "text",
+ })
}
+ sections = appendDOCXListImageSections(sections, docxIRList{Items: []docxIRListItem{item}}, budget)
+ }
+
+ case "text_box":
+ text := extractTextFromBlockElements(el.contentBlocks())
+ if text != "" {
sections = append(sections, map[string]any{
"text": text,
"image": nil,
"doc_type_kwd": "text",
})
}
-
- case "text_box":
- text := extractTextFromBlockElements(el.contentBlocks())
- if text == "" {
- continue
- }
- sections = append(sections, map[string]any{
- "text": text,
- "image": nil,
- "doc_type_kwd": "text",
- })
+ sections = appendDOCXIRImageSections(sections, el.contentBlocks(), budget)
}
}
}
@@ -317,7 +419,7 @@ func buildDOCXJSONSections(irJSON string) []map[string]any {
// order. A paragraph is not necessarily a single text unit: office_oxide can
// place an image between two text runs, and dropping that run would make the
// downstream DOCX chunker lose both media order and adjacency context.
-func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool) []map[string]any {
+func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *embeddedMediaBudget) []map[string]any {
var text strings.Builder
flushText := func() {
value := strings.TrimSpace(text.String())
@@ -336,14 +438,7 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he
sections = append(sections, item)
}
appendImage := func(data []byte) {
- if len(data) == 0 {
- return
- }
- sections = append(sections, map[string]any{
- "text": "",
- "image": base64.StdEncoding.EncodeToString(data),
- "doc_type_kwd": "image",
- })
+ sections, _ = appendDOCXImageSection(sections, data, budget, nil)
}
for _, run := range el.contentRuns() {
switch run.Type {
@@ -365,28 +460,77 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, he
return sections
}
+func appendDOCXImageSection(sections []map[string]any, data []byte, budget *embeddedMediaBudget, metadata map[string]any) ([]map[string]any, bool) {
+ if len(data) == 0 {
+ return sections, true
+ }
+ included, keepWalking := budget.include(data)
+ if !included && !keepWalking {
+ return sections, false
+ }
+ return appendDOCXImagePayload(sections, data, included, metadata), keepWalking
+}
+
+func appendDOCXListImageSections(sections []map[string]any, list docxIRList, budget *embeddedMediaBudget) []map[string]any {
+ for _, item := range list.Items {
+ sections = appendDOCXIRImageSections(sections, item.Content, budget)
+ if item.Nested != nil {
+ sections = appendDOCXListImageSections(sections, *item.Nested, budget)
+ }
+ }
+ return sections
+}
+
+func appendDOCXIRImageSections(sections []map[string]any, elements []docxIRElement, budget *embeddedMediaBudget) []map[string]any {
+ forEachDOCXIRImage(elements, func(data []byte) bool {
+ var keepWalking bool
+ sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil)
+ return keepWalking
+ })
+ return sections
+}
+
+func appendDOCXImagePayload(sections []map[string]any, data []byte, included bool, metadata map[string]any) []map[string]any {
+ item := map[string]any{
+ "text": "",
+ "image": nil,
+ "doc_type_kwd": "image",
+ }
+ if included {
+ item["image"] = base64.StdEncoding.EncodeToString(data)
+ } else {
+ item["media_omitted"] = true
+ }
+ for key, value := range metadata {
+ item[key] = value
+ }
+ return append(sections, item)
+}
+
// --- figure extraction (used by the cgo parser path) ---
// extractDOCXFiguresFromIR parses the office_oxide IR JSON and
-// returns every embedded image block together with the plain text
-// immediately surrounding it. The context matches what Python's
+// returns the images admitted by budget together with the plain text
+// immediately surrounding each image. The context matches what Python's
// naive_merge_docx attaches as context_above / context_below on
// each chunk that carries an image.
//
// Reuses the IR already obtained from the doc handle in
// ParseWithResult so the binary is not opened twice.
-func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
+func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXFigure {
var ir docxIRDocument
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
return nil
}
+ if budget == nil {
+ budget = newEmbeddedMediaBudget()
+ }
var flat []flatBlock
for _, sec := range ir.Sections {
for _, el := range sec.Elements {
if el.Type == "image" {
- b64 := base64.StdEncoding.EncodeToString(el.Data)
- flat = append(flat, flatBlock{image: b64})
+ flat = append(flat, flatBlock{imageData: el.Data})
continue
}
if el.Type == "paragraph" || el.Type == "heading" {
@@ -400,10 +544,17 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
var figures []DOCXFigure
for i, block := range flat {
- if block.image == "" {
+ if len(block.imageData) == 0 {
continue
}
- fig := DOCXFigure{Image: block.image}
+ included, keepWalking := budget.include(block.imageData)
+ if !included && !keepWalking {
+ break
+ }
+ fig := DOCXFigure{}
+ if included {
+ fig.Image = base64.StdEncoding.EncodeToString(block.imageData)
+ }
// Collect text above (backward scan up to docxContextWindow
// chars, or until another image is hit).
@@ -426,6 +577,9 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {
}
figures = append(figures, fig)
+ if !keepWalking {
+ break
+ }
}
return figures
}
@@ -447,7 +601,7 @@ func docxInlineFlatBlocks(el docxIRElement) []flatBlock {
if len(data) == 0 {
return
}
- blocks = append(blocks, flatBlock{image: base64.StdEncoding.EncodeToString(data)})
+ blocks = append(blocks, flatBlock{imageData: data})
}
for _, run := range el.contentRuns() {
switch run.Type {
@@ -474,8 +628,8 @@ func docxInlineFlatBlocks(el docxIRElement) []flatBlock {
// flatBlock is a flattened IR element used internally to collect
// text / image context around embedded figures.
type flatBlock struct {
- text string
- image string // base64-encoded image data (empty for non-image)
+ text string
+ imageData []byte
}
const docxContextWindow = 512
@@ -484,7 +638,7 @@ func collectDOCXPrevText(flat []flatBlock, idx, maxLen int) string {
var parts []string
remaining := maxLen
for i := idx - 1; i >= 0 && remaining > 0; i-- {
- if flat[i].image != "" {
+ if len(flat[i].imageData) > 0 {
break // stop at previous image
}
if flat[i].text == "" {
@@ -512,7 +666,7 @@ func collectDOCXNextText(flat []flatBlock, idx, maxLen int) string {
var parts []string
remaining := maxLen
for i := idx + 1; i < len(flat) && remaining > 0; i++ {
- if flat[i].image != "" {
+ if len(flat[i].imageData) > 0 {
break // stop at next image
}
if flat[i].text == "" {
diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.go
index 1a33eeeb8b..03c98ffe2c 100644
--- a/internal/parser/parser/docx_ir_test.go
+++ b/internal/parser/parser/docx_ir_test.go
@@ -1,8 +1,11 @@
package parser
import (
+ "encoding/base64"
"encoding/json"
+ "fmt"
"reflect"
+ "strings"
"testing"
)
@@ -22,7 +25,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {
{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"item1"}]}]}]}
]}]}`
- got := buildDOCXJSONSections(irJSON)
+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
want := []map[string]any{
{"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},
{"text": "Hello", "image": nil, "doc_type_kwd": "text"},
@@ -43,7 +46,7 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {
{"type":"paragraph","content":[{"type":"text","text":"keep"}]},
{"type":"table","rows":[]}
]}]}`
- got := buildDOCXJSONSections(irJSON)
+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
if len(got) != 1 || got[0]["text"] != "keep" {
t.Fatalf("expected only the paragraph to survive, got %+v", got)
}
@@ -57,7 +60,7 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {
{"type":"text","text":"after"}
]}
]}]}`
- got := buildDOCXJSONSections(irJSON)
+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
if len(got) != 3 {
t.Fatalf("sections = %+v, want text/image/text sequence", got)
}
@@ -72,6 +75,189 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {
}
}
+func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[
+ {"type":"table","rows":[{"cells":[
+ {"content":[{"type":"paragraph","content":[
+ {"type":"text","text":"before"},
+ {"type":"image","data":"aGVsbG8="},
+ {"type":"text","text":"after"}
+ ]}]},
+ {"content":[{"type":"image","data":"aW1hZ2U="}]}
+ ]}]}
+ ]}]}`
+
+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ if len(got) != 3 {
+ t.Fatalf("sections = %+v, want table plus two cell image items", got)
+ }
+ if got[0]["doc_type_kwd"] != "table" || got[0]["text"] != "<table><tr><td>beforeafter</td><td></td></tr></table>" || got[0]["source_table_id"] != "docx-table-1" {
+ t.Fatalf("table item = %+v", got[0])
+ }
+ for i, want := range []string{"aGVsbG8=", "aW1hZ2U="} {
+ item := got[i+1]
+ if item["doc_type_kwd"] != "image" || item["image"] != want {
+ t.Errorf("image item %d = %+v, want payload %q", i, item, want)
+ }
+ if item["parent_table_id"] != "docx-table-1" || item["row_index"] != 1 || item["column_index"] != i+1 || item["media_order"] != i+1 {
+ t.Errorf("image item %d metadata = %+v", i, item)
+ }
+ }
+}
+
+func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[
+ {"type":"text","text":"before"},
+ {"type":"image","data":"YWJjZA=="},
+ {"type":"text","text":"after"}
+ ]}]}]}`
+ budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
+
+ items := buildDOCXJSONSections(irJSON, budget)
+ if len(items) != 3 {
+ t.Fatalf("items = %+v, want text/image/text with omitted image metadata", items)
+ }
+ if items[0]["text"] != "before" || items[2]["text"] != "after" {
+ t.Fatalf("inline text was lost around omitted image: %+v", items)
+ }
+ if items[1]["doc_type_kwd"] != "image" || items[1]["image"] != nil || items[1]["media_omitted"] != true {
+ t.Fatalf("omitted inline image item = %+v, want metadata without payload", items[1])
+ }
+ if len(budget.warnings()) == 0 {
+ t.Fatal("omitted image should produce a parser warning")
+ }
+}
+
+func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[
+ {"content":[{"type":"image","data":"YWJj"}]},
+ {"content":[{"type":"image","data":"ZGVm"}]}
+ ]}]}]}]}`
+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 10}
+
+ items := buildDOCXJSONSections(irJSON, budget)
+ if len(items) != 3 {
+ t.Fatalf("items = %+v, want table and two image metadata items", items)
+ }
+ if items[1]["image"] != "YWJj" || items[1]["media_omitted"] == true {
+ t.Fatalf("first table image = %+v, want accepted payload", items[1])
+ }
+ omitted := items[2]
+ if omitted["image"] != nil || omitted["media_omitted"] != true {
+ t.Fatalf("second table image = %+v, want omitted payload", omitted)
+ }
+ if omitted["parent_table_id"] != items[0]["source_table_id"] || omitted["row_index"] != 1 || omitted["column_index"] != 2 || omitted["media_order"] != 2 {
+ t.Fatalf("omitted table image lost location metadata: %+v", omitted)
+ }
+}
+
+func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {
+ var ir strings.Builder
+ ir.WriteString(`{"sections":[{"elements":[`)
+ for i := 0; i < 257; i++ {
+ if i > 0 {
+ ir.WriteByte(',')
+ }
+ ir.WriteString(`{"type":"image","data":"YQ=="}`)
+ }
+ ir.WriteString(`]}]}`)
+
+ budget := newEmbeddedMediaBudget()
+ items := buildDOCXJSONSections(ir.String(), budget)
+ imageCount := 0
+ for _, item := range items {
+ if item["doc_type_kwd"] == "image" {
+ imageCount++
+ }
+ }
+ if imageCount != budget.maxItems {
+ t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)
+ }
+ if len(budget.warnings()) == 0 {
+ t.Fatal("truncated media count should produce a parser warning")
+ }
+}
+
+func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[
+ {"type":"paragraph","content":[{"type":"text","text":"slide text"}]},
+ {"type":"image","data":"YWJjZA=="}
+ ]}]}`
+ budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
+
+ items, err := buildPPTXJSONSections(irJSON, budget)
+ if err != nil {
+ t.Fatalf("buildPPTXJSONSections: %v", err)
+ }
+ if len(items) != 2 || items[0]["text"] != "slide text" {
+ t.Fatalf("items = %+v, want text and omitted image items", items)
+ }
+ omitted := items[1]
+ if omitted["image"] != nil || omitted["media_omitted"] != true || omitted["slide_number"] != 1 || omitted["media_order"] != 1 {
+ t.Fatalf("omitted slide image lost metadata: %+v", omitted)
+ }
+ if itemsAllEmpty([]map[string]any{omitted}) {
+ t.Fatal("an image-only slide with a budget-omitted image must not be replaced by whole-deck fallback")
+ }
+}
+
+func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {
+ var ir strings.Builder
+ ir.WriteString(`{"sections":[{"elements":[`)
+ for i := 0; i < 257; i++ {
+ if i > 0 {
+ ir.WriteByte(',')
+ }
+ ir.WriteString(`{"type":"image","data":"YQ=="}`)
+ }
+ ir.WriteString(`]}]}`)
+
+ budget := newEmbeddedMediaBudget()
+ items, err := buildPPTXJSONSections(ir.String(), budget)
+ if err != nil {
+ t.Fatalf("buildPPTXJSONSections: %v", err)
+ }
+ imageCount := 0
+ for _, item := range items {
+ if item["doc_type_kwd"] == "image" {
+ imageCount++
+ }
+ }
+ if imageCount != budget.maxItems {
+ t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)
+ }
+ if len(budget.warnings()) == 0 {
+ t.Fatal("truncated media count should produce a parser warning")
+ }
+}
+
+func TestEmbeddedMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {
+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 4}
+ if included, keepGoing := budget.include([]byte("abc")); !included || !keepGoing {
+ t.Fatalf("first image admission = (%v, %v), want (true, true)", included, keepGoing)
+ }
+ if included, keepGoing := budget.include([]byte("def")); included || !keepGoing {
+ t.Fatalf("aggregate overflow admission = (%v, %v), want (false, true)", included, keepGoing)
+ }
+ if included, keepGoing := budget.include([]byte("12345")); included || !keepGoing {
+ t.Fatalf("per-image overflow admission = (%v, %v), want (false, true)", included, keepGoing)
+ }
+ if len(budget.warnings()) == 0 {
+ t.Fatal("omitted byte payloads should produce a parser warning")
+ }
+}
+
+func TestBuildDOCXJSONSectionsEncodesOnlyAcceptedImageData(t *testing.T) {
+ data := []byte("text is not a valid image, but the parser preserves it")
+ encoded := base64.StdEncoding.EncodeToString(data)
+ irJSON := fmt.Sprintf(`{"sections":[{"elements":[{"type":"image","data":%q}]}]}`, encoded)
+ budget := &embeddedMediaBudget{maxImageBytes: 8, maxTotalBytes: 8, maxItems: 4}
+ items := buildDOCXJSONSections(irJSON, budget)
+ if len(items) != 1 || items[0]["image"] != nil || items[0]["media_omitted"] != true {
+ t.Fatalf("oversized image items = %+v, want metadata without payload", items)
+ }
+}
+
// TestJoinDOCXIRRuns pins that only text-type runs are concatenated;
// non-text runs (e.g. nested image runs) are skipped.
func TestJoinDOCXIRRuns(t *testing.T) {
@@ -111,7 +297,7 @@ func TestExtractDOCXFiguresFromIR(t *testing.T) {
{"type":"paragraph","content":[{"type":"text","text":"after"}]}
]}]}`
- figs := extractDOCXFiguresFromIR(irJSON)
+ figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
if len(figs) != 1 {
t.Fatalf("expected 1 figure, got %d", len(figs))
}
@@ -135,7 +321,7 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {
{"type":"text","text":"after"}
]}
]}]}`
- figs := extractDOCXFiguresFromIR(irJSON)
+ figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
if len(figs) != 1 {
t.Fatalf("expected 1 inline figure, got %d", len(figs))
}
@@ -144,13 +330,30 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {
}
}
+func TestExtractDOCXFiguresFromIR_BudgetKeepsContextWithoutPayload(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[
+ {"type":"paragraph","content":[{"type":"text","text":"before"}]},
+ {"type":"image","data":"YWJjZA=="},
+ {"type":"paragraph","content":[{"type":"text","text":"after"}]}
+ ]}]}`
+ budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}
+
+ figures := extractDOCXFiguresFromIR(irJSON, budget)
+ if len(figures) != 1 {
+ t.Fatalf("figures = %+v, want one metadata record", figures)
+ }
+ if figures[0].Image != "" || figures[0].ContextAbove != "before" || figures[0].ContextBelow != "after" || figures[0].Marker != "before" {
+ t.Fatalf("budgeted figure = %+v, want context metadata without payload", figures[0])
+ }
+}
+
// TestExtractDOCXFiguresFromIR_NoImage returns nil when the IR has no
// image blocks.
func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {
irJSON := `{"sections":[{"elements":[
{"type":"paragraph","content":[{"type":"text","text":"only text"}]}
]}]}`
- if figs := extractDOCXFiguresFromIR(irJSON); figs != nil {
+ if figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget()); figs != nil {
t.Fatalf("expected nil, got %+v", figs)
}
}
@@ -158,7 +361,7 @@ func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {
// TestExtractDOCXFiguresFromIR_BadJSON returns nil on unparseable IR,
// mirroring the json.Unmarshal error guard.
func TestExtractDOCXFiguresFromIR_BadJSON(t *testing.T) {
- if figs := extractDOCXFiguresFromIR("{not json"); figs != nil {
+ if figs := extractDOCXFiguresFromIR("{not json", newEmbeddedMediaBudget()); figs != nil {
t.Fatalf("expected nil for bad JSON, got %+v", figs)
}
}
@@ -176,7 +379,7 @@ func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) {
{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"list item"}]}]}]},
{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"box text"}]}]}
]}]}`
- figs := extractDOCXFiguresFromIR(irJSON)
+ figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())
if len(figs) != 1 {
t.Fatalf("expected 1 figure, got %d", len(figs))
}
@@ -203,7 +406,7 @@ func TestCollectDOCXText_BoundedByMaxLen(t *testing.T) {
flat = append(flat, flatBlock{text: "x"})
}
imgIdx := len(flat)
- flat = append(flat, flatBlock{image: "img"})
+ flat = append(flat, flatBlock{imageData: []byte("img")})
for i := 0; i < 600; i++ {
flat = append(flat, flatBlock{text: "x"})
}
@@ -278,7 +481,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {
{"content":[{"type":"paragraph","content":[{"type":"text","text":"only item"}]}],"nested":null}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON)
+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
if len(sections) != 1 || sections[0]["text"] != "only item" {
t.Fatalf("expected single item \"only item\", got %+v", sections)
}
@@ -301,7 +504,7 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) {
]}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON)
+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)
}
@@ -352,7 +555,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {
]}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON)
+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)
}
@@ -373,7 +576,7 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {
]}}
]}
]}]}`
- sections := buildDOCXJSONSections(irJSON)
+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)
}
@@ -381,3 +584,48 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {
t.Errorf("nested list text = %q, want %q", got, "L1\nL2a\nL2b")
}
}
+
+func TestBuildDOCXJSONSections_EmitsListAndTextBoxImages(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[
+ {"type":"list","items":[
+ {"content":[{"type":"paragraph","content":[
+ {"type":"text","text":"list text"},
+ {"type":"image","data":"aGVsbG8="}
+ ]}],"nested":{"items":[
+ {"content":[{"type":"paragraph","content":[
+ {"type":"image","data":"d29ybGQ="}
+ ]}]}
+ ]}}
+ ]},
+ {"type":"text_box","content":[{"type":"paragraph","content":[
+ {"type":"text","text":"text box text"},
+ {"type":"image","data":"cGljdHVyZQ=="}
+ ]}]}
+ ]}]}`
+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())
+ want := []struct {
+ docType string
+ text string
+ image string
+ }{
+ {docType: "text", text: "list text"},
+ {docType: "image", image: "aGVsbG8="},
+ {docType: "image", image: "d29ybGQ="},
+ {docType: "text", text: "text box text"},
+ {docType: "image", image: "cGljdHVyZQ=="},
+ }
+ if len(sections) != len(want) {
+ t.Fatalf("sections = %+v, want %d items", sections, len(want))
+ }
+ for i, expected := range want {
+ if got := sections[i]["doc_type_kwd"]; got != expected.docType {
+ t.Errorf("section %d type = %v, want %q", i, got, expected.docType)
+ }
+ if expected.text != "" && sections[i]["text"] != expected.text {
+ t.Errorf("section %d text = %v, want %q", i, sections[i]["text"], expected.text)
+ }
+ if expected.image != "" && sections[i]["image"] != expected.image {
+ t.Errorf("section %d image = %v, want %q", i, sections[i]["image"], expected.image)
+ }
+ }
+}
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go
index 54833ccef4..77288fabb4 100644
--- a/internal/parser/parser/docx_parser.go
+++ b/internal/parser/parser/docx_parser.go
@@ -68,8 +68,8 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {
// ParseWithResult produces structured JSON items (when
// p.outputFormat == "json") or markdown (default) from a
-// docx document. Embedded images are extracted in both paths
-// for downstream vision-figure dispatch.
+// docx document. The JSON path emits bounded image items; the
+// Markdown path attaches bounded figure metadata to the file result.
//
// JSON path mirrors python parser.py:_docx() output_format == "json".
// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().
@@ -106,22 +106,18 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
}
// Extract IR JSON for section building (JSON path) and
- // embedded-image extraction (both paths).
+ // embedded-image extraction on the Markdown path.
irJSON, irErr := doc.ToIRJSON()
- var figures []DOCXFigure
- if irErr == nil {
- figures = extractDOCXFiguresFromIR(irJSON)
- }
- if len(figures) > 0 {
- fileMeta["figures"] = buildFiguresMap(figures)
- }
if p.outputFormat == "json" {
if irErr != nil {
return ParseResult{Err: fmt.Errorf("docx to-ir-json: %w", irErr)}
}
- var sections []map[string]any
- sections = buildDOCXJSONSections(irJSON)
+ budget := newEmbeddedMediaBudget()
+ sections := buildDOCXJSONSections(irJSON, budget)
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err}
+ }
// remove_header_footer: drop sections whose normalized text
// matches a docx header/footer entry (mirrors Python
// parser.py:889-891 extract_docx_header_footer_texts +
@@ -143,6 +139,15 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
OutputFormat: "json",
File: fileMeta,
JSON: sections,
+ Warnings: budget.warnings(),
+ }
+ }
+
+ budget := newEmbeddedMediaBudget()
+ if irErr == nil {
+ figures := extractDOCXFiguresFromIR(irJSON, budget)
+ if len(figures) > 0 {
+ fileMeta["figures"] = buildFiguresMap(figures)
}
}
@@ -187,6 +192,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
OutputFormat: "markdown",
File: fileMeta,
Markdown: md,
+ Warnings: budget.warnings(),
}
}
diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.go
index 2d82ebec92..42e32d47d8 100644
--- a/internal/parser/parser/docx_parser_test.go
+++ b/internal/parser/parser/docx_parser_test.go
@@ -29,7 +29,7 @@ func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -46,7 +46,7 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -67,7 +67,7 @@ func TestBuildDOCXJSONSections_Images(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"image","data":"` + b64 + `"}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -90,7 +90,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {
{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]}
]}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1", len(sections))
}
@@ -120,7 +120,7 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {
{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]},
{"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 4 {
t.Fatalf("got %d sections, want 4", len(sections))
}
@@ -153,7 +153,7 @@ func TestBuildDOCXJSONSections_List(t *testing.T) {
{"content":[{"type":"paragraph","content":[{"type":"text","text":"Item 3"}]}]}
],"level":0}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 3 {
t.Fatalf("got %d sections, want 3 (each list item should be a section)", len(sections))
}
@@ -171,7 +171,7 @@ func TestBuildDOCXJSONSections_TextBox(t *testing.T) {
ir := `{"sections":[{"title":"","elements":[
{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"Boxed paragraph"}]}],"width_emu":null}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1 (text_box content should become a section)", len(sections))
}
@@ -192,7 +192,7 @@ func TestBuildDOCXJSONSections_MixedWithList(t *testing.T) {
],"level":0},
{"type":"paragraph","content":[{"type":"text","text":"Trailer"}],"style":"Normal"}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 4 {
t.Fatalf("got %d sections, want 4 (para + 2 list items + para)", len(sections))
}
@@ -210,7 +210,7 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {
{"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]},
{"type":"table","rows":[]}
]}]}`
- sections := buildDOCXJSONSections(ir)
+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())
if len(sections) != 1 {
t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections))
}
diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.go
new file mode 100644
index 0000000000..5df4e73238
--- /dev/null
+++ b/internal/parser/parser/html_media_test.go
@@ -0,0 +1,82 @@
+package parser
+
+import (
+ "context"
+ "encoding/base64"
+ "strings"
+ "testing"
+)
+
+const htmlMediaDataURI = "data:image/png;base64,aGVsbG8="
+
+func TestHTMLParser_EmitsInlineImagesInDocumentOrder(t *testing.T) {
+ input := `<p>before <img alt="figure alt" src="` + htmlMediaDataURI + `"> after</p>`
+ result := NewHTMLParser().ParseWithResult(context.Background(), "inline.html", []byte(input))
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ if len(result.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none for a valid inline image", result.Warnings)
+ }
+ if len(result.JSON) != 3 {
+ t.Fatalf("items = %+v, want text/image/text", result.JSON)
+ }
+ if result.JSON[0]["text"] != "before" || result.JSON[0]["doc_type_kwd"] != "text" {
+ t.Errorf("before item = %+v", result.JSON[0])
+ }
+ imageItem := result.JSON[1]
+ if imageItem["doc_type_kwd"] != "image" || imageItem["image"] != htmlMediaDataURI || imageItem["text"] != "figure alt" || imageItem["media_order"] != 1 {
+ t.Errorf("image item = %+v", imageItem)
+ }
+ if result.JSON[2]["text"] != "after" || result.JSON[2]["doc_type_kwd"] != "text" {
+ t.Errorf("after item = %+v", result.JSON[2])
+ }
+}
+
+func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {
+ input := `<table><tr><td>A<img alt="first" src="` + htmlMediaDataURI + `"></td><td><img src="https://example.com/second.png"></td></tr></table>`
+ result := NewHTMLParser().ParseWithResult(context.Background(), "table.html", []byte(input))
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ if len(result.JSON) != 2 {
+ t.Fatalf("items = %+v, want table and one inline image item", result.JSON)
+ }
+ if result.JSON[0]["doc_type_kwd"] != "table" || result.JSON[0]["source_table_id"] != "html-table-1" {
+ t.Errorf("table item = %+v", result.JSON[0])
+ }
+ tableText, _ := result.JSON[0]["text"].(string)
+ if strings.Contains(tableText, "data:image/") {
+ t.Error("table text duplicates the separately emitted inline image payload")
+ }
+ if !strings.Contains(tableText, `alt="first"`) {
+ t.Errorf("table text lost the image alt text: %q", tableText)
+ }
+ if result.JSON[1]["image"] != htmlMediaDataURI {
+ t.Errorf("inline image payload = %v, want data URI", result.JSON[1]["image"])
+ }
+ item := result.JSON[1]
+ if item["doc_type_kwd"] != "image" || item["parent_table_id"] != "html-table-1" {
+ t.Errorf("inline image item = %+v", item)
+ }
+ if item["row_index"] != 1 || item["column_index"] != 1 || item["media_order"] != 1 {
+ t.Errorf("inline image position = %+v", item)
+ }
+}
+
+func TestHTMLParser_RejectsOversizedInlineImagePayload(t *testing.T) {
+ payload := base64.StdEncoding.EncodeToString(make([]byte, maxEmbeddedImageBytes+1))
+ input := "<p><img alt=\"large\" src=\"data:image/png;base64," + payload + "\"></p>"
+ result := NewHTMLParser().ParseWithResult(context.Background(), "large.html", []byte(input))
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ for _, item := range result.JSON {
+ if item["doc_type_kwd"] == "image" {
+ t.Fatal("oversized data URI emitted as an image item")
+ }
+ }
+ if len(result.Warnings) == 0 {
+ t.Fatal("oversized inline image should produce a parser warning")
+ }
+}
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index ae34b6e29f..1a8a94f3d1 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -19,7 +19,9 @@ package parser
import (
"bytes"
"context"
+ "encoding/base64"
"fmt"
+ "io"
"strings"
"golang.org/x/net/html"
@@ -53,19 +55,8 @@ func (p *HTMLParser) ConfigureFromSetup(setup map[string]any) {
}
}
-// ParseWithResult emits one item per block-level HTML element
-// (headings, paragraphs, lists, pre blocks). The walker is a
-// pure-Go replacement for the previous `fmt.Printf` debug output:
-// it descends the html.Parse tree, joins the leaf text of each
-// block-level element, and emits the python-compatible
-// `{text, doc_type_kwd:"text"}` shape.
-//
-// Phase 2.5 (Slice 1) of port-rag-flow-pipeline-to-go.md makes
-// HTMLParser a ParseResultProducer so the dispatch seam routes
-// the html family through the structured path. Inline formatting
-// (bold / links / images) is intentionally NOT surfaced as a
-// separate ck_type — the python HtmlParser collapses inline
-// formatting into the parent block's text.
+// ParseWithResult emits normalized text items for block-level HTML elements
+// and independent image items for valid inline image payloads.
func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
// x/net/html assumes UTF-8 input, so a GBK/Big5/Shift-JIS page would
// otherwise surface as U+FFFD mojibake. Decode first, mirroring the
@@ -87,7 +78,11 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{Err: fmt.Errorf("html parse: %w", err)}
}
var items []map[string]any
- walkHTMLBlocks(doc, &items)
+ state := &htmlWalkState{}
+ walkHTMLBlocksWithState(doc, &items, state)
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err}
+ }
// remove_toc: post-parse text heuristic (mirrors Python
// parser.py:1087-1088 remove_toc → remove_contents_table).
if p.RemoveTOC {
@@ -102,7 +97,8 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
"name": filename,
"encoding": encName,
},
- JSON: items,
+ JSON: items,
+ Warnings: state.warnings(),
}
}
@@ -112,12 +108,23 @@ func decodeHTMLToUTF8(data []byte) ([]byte, string) {
return DecodeToUTF8(data, "text/html")
}
-// walkHTMLBlocks emits one normalized item per block-level
-// descendant of root. Inline elements (b, i, a, span, …) are
-// collapsed into the parent's text via leafText. <script>,
-// <style>, and <noscript> blocks are skipped entirely so they
-// don't pollute the downstream chunker input.
-func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
+type htmlWalkState struct {
+ tableSequence int
+ mediaOrder int
+ skippedInlineImages int
+}
+
+func (s *htmlWalkState) warnings() []string {
+ if s.skippedInlineImages == 0 {
+ return nil
+ }
+ return []string{fmt.Sprintf("HTML parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages)}
+}
+
+// walkHTMLBlocksWithState emits normalized text and image items in document
+// order. Inline formatting is collapsed into its parent text; script, style,
+// and noscript subtrees are skipped.
+func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *htmlWalkState) {
for child := root.FirstChild; child != nil; child = child.NextSibling {
if child.Type == html.TextNode {
if emitsLooseHTMLText(root) {
@@ -138,7 +145,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
continue
case "html", "body":
// Wrapper elements: descend into their children.
- walkHTMLBlocks(child, out)
+ walkHTMLBlocksWithState(child, out, state)
continue
case "table":
// Emit the <table> as a SINGLE structured doc_type_kwd:"table"
@@ -149,19 +156,12 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {
// table context). We emit ONLY this item — no duplicate
// doc_type_kwd:"text" copy — so the table is embedded once and
// its markup does not pollute neighbouring prose chunks.
- markup := renderTableHTML(child)
- if strings.TrimSpace(markup) != "" {
- *out = append(*out, map[string]any{
- "text": markup,
- "doc_type_kwd": "table",
- "ck_type": "table",
- })
- }
+ emitHTMLTable(child, out, state)
continue
}
ckType := htmlTagToCkType(tag)
trim := tag != "pre" && tag != "textarea"
- htmlLeafText(child, out, ckType, trim)
+ htmlLeafTextWithState(child, out, ckType, trim, state)
}
}
@@ -285,7 +285,7 @@ func (w *leafWriter) hardBreak() {
w.endsNL = true
}
-// htmlLeafText joins the visible text of an HTML node and its
+// htmlLeafTextWithState joins the visible text of an HTML node and its
// descendants and emits items directly into out. <script>/<style>/<noscript>
// subtrees are skipped. Whitespace is folded per CSS rules (so
// "<h1>Hello world</h1>" becomes "Hello world" and "<br>" survives as a
@@ -297,10 +297,10 @@ func (w *leafWriter) hardBreak() {
// doc_type_kwd:"table" item at its document position (see walkHTMLLeaf's
// "table" case) — the prose around it is flushed as ordinary text items, so
// the table is never relocated to the end and never duplicated.
-func htmlLeafText(n *html.Node, out *[]map[string]any, ckType string, trim bool) {
+func htmlLeafTextWithState(n *html.Node, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {
var b bytes.Buffer
w := &leafWriter{b: &b}
- walkHTMLLeaf(n, w, out)
+ walkHTMLLeaf(n, w, out, ckType, trim, state)
flushLeafText(w, out, ckType, trim)
}
@@ -325,7 +325,7 @@ func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim boo
w.endsNL = false
}
-func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
+func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {
switch n.Type {
case html.TextNode:
w.writeText(n.Data)
@@ -337,11 +337,23 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
w.hardBreak()
return
}
+ if n.Data == "img" {
+ src := htmlAttribute(n, "src")
+ if !usableHTMLImageSource(src) {
+ if isHTMLDataImageSource(src) {
+ state.skippedInlineImages++
+ }
+ return
+ }
+ flushLeafText(w, out, ckType, trim)
+ appendHTMLImageItem(out, state, src, htmlAttribute(n, "alt"), "", 0, 0)
+ return
+ }
if n.Data == "pre" || n.Data == "textarea" {
// Verbatim: no folding, no injected block breaks.
w.pre = true
for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLLeaf(child, w, out)
+ walkHTMLLeaf(child, w, out, ckType, trim, state)
}
w.pre = false
return
@@ -355,15 +367,8 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
// row/column structure survives; we do NOT also inline the markup
// into the parent's text, which would duplicate the table and
// pollute the prose chunk with raw tags.
- markup := renderTableHTML(n)
- if strings.TrimSpace(markup) != "" {
- flushLeafText(w, out, "text", true)
- *out = append(*out, map[string]any{
- "text": markup,
- "doc_type_kwd": "table",
- "ck_type": "table",
- })
- }
+ flushLeafText(w, out, "text", true)
+ emitHTMLTable(n, out, state)
return
}
// Add a line break between block children so headings, paragraphs,
@@ -378,7 +383,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
}
}
for child := n.FirstChild; child != nil; child = child.NextSibling {
- walkHTMLLeaf(child, w, out)
+ walkHTMLLeaf(child, w, out, ckType, trim, state)
}
if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {
w.hardBreak()
@@ -386,6 +391,201 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {
}
}
+type htmlTableImage struct {
+ src string
+ alt string
+ row int
+ column int
+}
+
+func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {
+ images, skippedInlineImages := htmlTableImages(n)
+ state.skippedInlineImages += skippedInlineImages
+ markup := renderTableHTML(n)
+ if strings.TrimSpace(markup) == "" {
+ return
+ }
+ table := map[string]any{
+ "text": markup,
+ "doc_type_kwd": "table",
+ "ck_type": "table",
+ }
+ tableID := ""
+ if len(images) > 0 {
+ state.tableSequence++
+ tableID = fmt.Sprintf("html-table-%d", state.tableSequence)
+ table["source_table_id"] = tableID
+ }
+ *out = append(*out, table)
+ for _, media := range images {
+ appendHTMLImageItem(out, state, media.src, media.alt, tableID, media.row, media.column)
+ }
+}
+
+func htmlTableImages(table *html.Node) ([]htmlTableImage, int) {
+ var images []htmlTableImage
+ skippedInlineImages := 0
+ rowCount := 0
+ columnCounts := make(map[int]int)
+ var walk func(node *html.Node, row, column int)
+ walk = func(node *html.Node, row, column int) {
+ if node.Type == html.ElementNode {
+ switch node.Data {
+ case "tr":
+ rowCount++
+ row = rowCount
+ column = 0
+ case "td", "th":
+ columnCounts[row]++
+ column = columnCounts[row]
+ case "img":
+ src := htmlAttribute(node, "src")
+ if !usableHTMLImageSource(src) {
+ if isHTMLDataImageSource(src) {
+ skippedInlineImages++
+ removeHTMLImageSource(node)
+ }
+ } else {
+ images = append(images, htmlTableImage{
+ src: src,
+ alt: htmlAttribute(node, "alt"),
+ row: row,
+ column: column,
+ })
+ // The image is emitted as its own item below, so leave its
+ // alt text and table cell in the markup without duplicating
+ // the (potentially large) data URI.
+ removeHTMLImageSource(node)
+ }
+ }
+ }
+ for child := node.FirstChild; child != nil; child = child.NextSibling {
+ walk(child, row, column)
+ }
+ }
+ walk(table, 0, 0)
+ return images, skippedInlineImages
+}
+
+func removeHTMLImageSource(n *html.Node) {
+ attrs := n.Attr[:0]
+ for _, attr := range n.Attr {
+ if !strings.EqualFold(attr.Key, "src") {
+ attrs = append(attrs, attr)
+ }
+ }
+ n.Attr = attrs
+}
+
+func appendHTMLImageItem(out *[]map[string]any, state *htmlWalkState, src, alt, parentTableID string, row, column int) {
+ if comma := strings.IndexByte(src, ','); comma > 0 && len(src) >= len("data:image/") && strings.EqualFold(src[:len("data:image/")], "data:image/") {
+ src = strings.ToLower(src[:comma]) + src[comma:]
+ }
+ state.mediaOrder++
+ item := map[string]any{
+ "text": strings.TrimSpace(alt),
+ "doc_type_kwd": "image",
+ "ck_type": "image",
+ "media_order": state.mediaOrder,
+ }
+ item["image"] = src
+ if parentTableID != "" {
+ item["parent_table_id"] = parentTableID
+ item["row_index"] = row
+ item["column_index"] = column
+ }
+ *out = append(*out, item)
+}
+
+func usableHTMLImageSource(src string) bool {
+ src = strings.TrimSpace(src)
+ if !isHTMLDataImageSource(src) {
+ return false
+ }
+ separator := strings.IndexByte(src, ',')
+ if separator <= len("data:image/") || separator > 128 || !strings.EqualFold(src[separator-len(";base64"):separator], ";base64") {
+ return false
+ }
+ return validBase64ImagePayload(src[separator+1:], maxEmbeddedImageBytes)
+}
+
+func isHTMLDataImageSource(src string) bool {
+ const dataPrefix = "data:image/"
+ src = strings.TrimSpace(src)
+ return len(src) >= len(dataPrefix) && strings.EqualFold(src[:len(dataPrefix)], dataPrefix)
+}
+
+func validBase64ImagePayload(payload string, maxDecodedBytes int) bool {
+ if maxDecodedBytes < 0 {
+ return false
+ }
+ maxEncodedBytes := base64.StdEncoding.EncodedLen(maxDecodedBytes)
+ encodedBytes := 0
+ for i := 0; i < len(payload); i++ {
+ if payload[i] == '\r' || payload[i] == '\n' {
+ continue
+ }
+ encodedBytes++
+ if encodedBytes > maxEncodedBytes {
+ return false
+ }
+ }
+ if encodedBytes == 0 {
+ return false
+ }
+ decodedSize, ok := base64DecodedSize(payload, encodedBytes)
+ if !ok || decodedSize > int64(maxDecodedBytes) {
+ return false
+ }
+
+ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {
+ decoder := base64.NewDecoder(encoding, strings.NewReader(payload))
+ decodedBytes, err := io.CopyN(io.Discard, decoder, int64(maxDecodedBytes)+1)
+ if decodedBytes > int64(maxDecodedBytes) {
+ return false
+ }
+ if err == io.EOF {
+ return decodedBytes > 0
+ }
+ }
+ return false
+}
+
+func base64DecodedSize(payload string, encodedBytes int) (int64, bool) {
+ padding := 0
+ sawPadding := false
+ for i := 0; i < len(payload); i++ {
+ if payload[i] == '\r' || payload[i] == '\n' {
+ continue
+ }
+ if payload[i] == '=' {
+ sawPadding = true
+ padding++
+ if padding > 2 {
+ return 0, false
+ }
+ continue
+ }
+ if sawPadding {
+ return 0, false
+ }
+ }
+ if encodedBytes%4 == 1 || (padding > 0 && encodedBytes%4 != 0) {
+ return 0, false
+ }
+ decodedSize := int64(encodedBytes/4)*3 + int64(encodedBytes%4)*3/4 - int64(padding)
+ return decodedSize, decodedSize >= 0
+}
+
+func htmlAttribute(n *html.Node, key string) string {
+ for _, attr := range n.Attr {
+ if strings.EqualFold(attr.Key, key) {
+ return attr.Val
+ }
+ }
+ return ""
+}
+
func isBlockTag(tag string) bool {
switch tag {
case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",
diff --git a/internal/parser/parser/image_media.go b/internal/parser/parser/image_media.go
new file mode 100644
index 0000000000..cce226f896
--- /dev/null
+++ b/internal/parser/parser/image_media.go
@@ -0,0 +1,73 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package parser
+
+import (
+ "context"
+ "sync"
+
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+)
+
+const (
+ // MaxImagePayloadBytes bounds compressed image data accepted by media paths.
+ MaxImagePayloadBytes = 32 << 20
+ // MaxImagePixels bounds decoded image dimensions before allocating a raster.
+ MaxImagePixels = 40_000_000
+ // MaxImageEdge bounds either decoded image dimension before raster allocation.
+ MaxImageEdge = 12_000
+)
+
+// AcquireImageMedia limits concurrent image materialization across Parser
+// payload encoding and ingestion VLM crop paths. The native analyzer keeps
+// its own inference limit.
+func AcquireImageMedia(ctx context.Context) (func(), error) {
+ return sharedImageMediaAdmission().acquire(ctx)
+}
+
+type imageMediaAdmission struct {
+ slots chan struct{}
+}
+
+func newImageMediaAdmission(limit int) *imageMediaAdmission {
+ if limit < 1 {
+ limit = 1
+ }
+ return &imageMediaAdmission{slots: make(chan struct{}, limit)}
+}
+
+func (a *imageMediaAdmission) acquire(ctx context.Context) (func(), error) {
+ select {
+ case a.slots <- struct{}{}:
+ var once sync.Once
+ return func() { once.Do(func() { <-a.slots }) }, nil
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ }
+}
+
+var (
+ imageMediaAdmissionOnce sync.Once
+ imageMediaAdmissionPool *imageMediaAdmission
+)
+
+func sharedImageMediaAdmission() *imageMediaAdmission {
+ imageMediaAdmissionOnce.Do(func() {
+ imageMediaAdmissionPool = newImageMediaAdmission(deepdocpdf.DeepDocConcurrency())
+ })
+ return imageMediaAdmissionPool
+}
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 5e35e8e173..49e4431993 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -108,16 +108,30 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, d
doc := markdownNew().Parse([]byte(rendered))
var items []map[string]any
- walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText, p.FetchRemoteImages)
+ mediaBudget := newEmbeddedMediaBudget()
+ imageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)
+ defer cancel()
+ unresolvedImages := walkMarkdownBlocksWithImages(imageCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, mediaBudget)
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err}
+ }
if items == nil {
items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}
}
+ warnings := mediaBudget.warnings()
+ if mediaBudget.items > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {
+ warnings = append(warnings, "Markdown image processing reached the document time budget")
+ }
+ if unresolvedImages > 0 {
+ warnings = append(warnings, fmt.Sprintf("Markdown parser could not resolve %d image source(s)", unresolvedImages))
+ }
return ParseResult{
OutputFormat: "json",
File: map[string]any{
"name": filename,
},
- JSON: items,
+ JSON: items,
+ Warnings: warnings,
}
}
@@ -376,7 +390,8 @@ func markdownTableCells(line string) []string {
// doc_type_kwd:"table" to keep the table whole and attach table context
// to neighbouring chunks (chunker/token.go). Non-table HTML blocks
// (<div>, <style>, …) are emitted as ordinary text with no ck_type.
-func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool) {
+func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, budget *embeddedMediaBudget) int {
+ unresolvedImages := 0
for _, child := range doc.GetChildren() {
var ckType string
var docTypeKwd string
@@ -457,8 +472,36 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten,
if imgURL, ok := findBlockImage(child); ok {
isRemote := strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")
if fetchRemoteImages || !isRemote {
- if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {
- item["image"] = imgData
+ switch {
+ case isRemote && ctx.Err() != nil:
+ item["media_omitted"] = true
+ case !budget.reserveItem():
+ item["media_omitted"] = true
+ default:
+ encoded, raw, resolved := resolveImageURL(ctx, imgURL)
+ if !resolved {
+ unresolvedImages++
+ break
+ }
+ size := len(raw)
+ if raw == nil {
+ encodedBytes := len(encoded) - strings.Count(encoded, "\r") - strings.Count(encoded, "\n")
+ decodedSize, ok := base64DecodedSize(encoded, encodedBytes)
+ if !ok {
+ unresolvedImages++
+ break
+ }
+ size = int(decodedSize)
+ }
+ if !budget.includeReservedSize(size) {
+ item["media_omitted"] = true
+ break
+ }
+ if raw != nil {
+ item["image"] = base64.StdEncoding.EncodeToString(raw)
+ } else {
+ item["image"] = encoded
+ }
if !flatten {
item["doc_type_kwd"] = "image"
}
@@ -468,6 +511,7 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten,
*out = append(*out, item)
}
+ return unresolvedImages
}
// isTableHTML reports whether block text is an outer <table> element (the
@@ -502,44 +546,53 @@ func findBlockImage(n ast.Node) (string, bool) {
return url, found
}
-// resolveImageURL resolves a Markdown image URL to its base64-encoded data.
+// resolveImageURL resolves a Markdown image URL to an encoded payload or raw
+// bytes.
// Supports:
-// - data:image/... URIs → decoded directly
+// - data:image/... URIs → validated encoded payload
// - http:// / https:// URLs → fetched (with basic SSRF filtering)
//
// Local / relative paths are not fetched (security). Returns
-// (base64String, true) on success, ("", false) when resolution fails.
-func resolveImageURL(imageURL string) (string, bool) {
- if strings.HasPrefix(imageURL, dataURIPrefix) {
+// exactly one of encoded and raw populated on success.
+func resolveImageURL(ctx context.Context, imageURL string) (encoded string, raw []byte, ok bool) {
+ if strings.HasPrefix(strings.ToLower(imageURL), dataURIPrefix) {
// data:image/png;base64,xxxx
idx := strings.Index(imageURL, "base64,")
if idx < 0 {
- return "", false
+ idx = strings.Index(strings.ToLower(imageURL), "base64,")
}
- return imageURL[idx+len("base64,"):], true
+ if idx < 0 {
+ return "", nil, false
+ }
+ encoded = imageURL[idx+len("base64,"):]
+ if !validBase64ImagePayload(encoded, MaxImagePayloadBytes) {
+ return "", nil, false
+ }
+ return encoded, nil, true
}
if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {
- b64, err := fetchImageAsBase64(imageURL)
+ raw, err := fetchImage(ctx, imageURL)
if err != nil {
- return "", false
+ return "", nil, false
}
- return b64, true
+ return "", raw, true
}
// Local / relative paths — not fetched for security.
- return "", false
+ return "", nil, false
}
-// fetchImageAsBase64 fetches an HTTP(S) image URL and returns its
-// content as a base64-encoded string. Local/private addresses and
-// redirects to them are rejected (SSRF guard). Hostnames are resolved
-// once and the validated IP is pinned in a custom DialContext to
-// prevent DNS-rebinding TOCTOU attacks.
-func fetchImageAsBase64(rawURL string) (string, error) {
+// fetchImage fetches a bounded HTTP(S) image payload. Local/private addresses
+// and redirects to them are rejected (SSRF guard). Hostnames are resolved
+// once and the validated IP is pinned in a custom DialContext to prevent
+// DNS-rebinding TOCTOU attacks.
+func fetchImage(ctx context.Context, rawURL string) ([]byte, error) {
rawURL = strings.TrimSpace(rawURL)
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
parsed, err := url.Parse(rawURL)
if err != nil {
- return "", fmt.Errorf("markdown: invalid image URL: %w", err)
+ return nil, fmt.Errorf("markdown: invalid image URL: %w", err)
}
// pinned maps hostname (without port) → validated IP. The hostname
@@ -549,7 +602,7 @@ func fetchImageAsBase64(rawURL string) (string, error) {
pinned := make(map[string]net.IP)
pinHost := func(host string) error {
- ip, err := resolveAndValidateHost(host)
+ ip, err := resolveAndValidateHost(ctx, host)
if err != nil {
return err
}
@@ -564,7 +617,7 @@ func fetchImageAsBase64(rawURL string) (string, error) {
}
if err := pinHost(parsed.Host); err != nil {
- return "", err
+ return nil, err
}
transport := &http.Transport{
@@ -594,32 +647,36 @@ func fetchImageAsBase64(rawURL string) (string, error) {
},
}
- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
- defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)
if err != nil {
- return "", fmt.Errorf("markdown: create image request: %w", err)
+ return nil, fmt.Errorf("markdown: create image request: %w", err)
}
resp, err := client.Do(req)
if err != nil {
- return "", fmt.Errorf("markdown: fetch image %s: %w", rawURL, err)
+ return nil, fmt.Errorf("markdown: fetch image %s: %w", rawURL, err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
- return "", fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)
+ return nil, fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)
}
- body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024)) // 32 MiB cap
+ body, err := io.ReadAll(io.LimitReader(resp.Body, MaxImagePayloadBytes+1))
if err != nil {
- return "", fmt.Errorf("markdown: read image %s: %w", rawURL, err)
+ return nil, fmt.Errorf("markdown: read image %s: %w", rawURL, err)
+ }
+ if len(body) > MaxImagePayloadBytes {
+ return nil, fmt.Errorf("markdown: image %s exceeds %d-byte limit", rawURL, MaxImagePayloadBytes)
+ }
+ if len(body) == 0 {
+ return nil, fmt.Errorf("markdown: image %s is empty", rawURL)
}
- return base64.StdEncoding.EncodeToString(body), nil
+ return body, nil
}
// resolveAndValidateHost resolves a host (which may include a port),
// validates none of its IPs are internal/private, and returns the
// first public IP for connection pinning.
-func resolveAndValidateHost(host string) (net.IP, error) {
+func resolveAndValidateHost(ctx context.Context, host string) (net.IP, error) {
hostname := host
if h, _, err := net.SplitHostPort(host); err == nil {
hostname = h
@@ -634,7 +691,7 @@ func resolveAndValidateHost(host string) (net.IP, error) {
return ip, nil
}
- addrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), hostname)
+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)
if err != nil {
return nil, fmt.Errorf("markdown: cannot resolve image host: %s", hostname)
}
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index 610c312c79..07e475c720 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -1,11 +1,13 @@
package parser
import (
+ "context"
"encoding/base64"
"net/http"
"net/http/httptest"
"os"
"strings"
+ "sync/atomic"
"testing"
)
@@ -438,18 +440,18 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {
func TestResolveImageURL_DataURI(t *testing.T) {
b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))
- result, found := resolveImageURL("data:image/png;base64," + b64)
+ result, raw, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64)
if !found {
t.Fatal("expected image found for data URI")
}
- if result != b64 {
+ if result != b64 || raw != nil {
t.Fatalf("got %q, want %q", result, b64)
}
}
func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {
// Local / relative paths are not fetched (security); resolution fails.
- if _, found := resolveImageURL("./local/image.png"); found {
+ if _, _, found := resolveImageURL(context.Background(), "./local/image.png"); found {
t.Fatal("expected no image resolved for a local path")
}
}
@@ -468,13 +470,52 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {
}))
defer ts.Close()
- result, found := resolveImageURL(ts.URL + "/image.png")
+ encoded, result, found := resolveImageURL(context.Background(), ts.URL+"/image.png")
if !found {
t.Fatal("expected image found for HTTP URL")
}
- expectedB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-bytes"))
- if result != expectedB64 {
- t.Fatalf("got %q, want %q", result, expectedB64)
+ if encoded != "" || string(result) != "fake-png-bytes" {
+ t.Fatalf("got encoded=%q raw=%q, want raw image bytes", encoded, result)
+ }
+}
+
+func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {
+ previous := ssrfAllowLoopback
+ ssrfAllowLoopback = true
+ t.Cleanup(func() { ssrfAllowLoopback = previous })
+
+ var fetches atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ fetches.Add(1)
+ w.Header().Set("Content-Type", "image/png")
+ _, _ = w.Write([]byte("abc"))
+ }))
+ defer server.Close()
+
+ markdown := "![first](" + server.URL + "/1)\n\n" +
+ "![second](" + server.URL + "/2)\n\n" +
+ "![third](" + server.URL + "/3)\n"
+ doc := markdownNew().Parse([]byte(markdown))
+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}
+ var items []map[string]any
+ if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, budget); unresolved != 0 {
+ t.Fatalf("unresolved images = %d, want 0", unresolved)
+ }
+ if got := fetches.Load(); got != 2 {
+ t.Fatalf("remote fetches = %d, want 2 before the item limit", got)
+ }
+ if len(items) != 3 || items[0]["image"] != base64.StdEncoding.EncodeToString([]byte("abc")) {
+ t.Fatalf("first image was not retained: %+v", items)
+ }
+ for _, index := range []int{1, 2} {
+ if items[index]["image"] != nil || items[index]["media_omitted"] != true {
+ t.Fatalf("image %d = %+v, want omitted payload", index, items[index])
+ }
+ }
+ warnings := strings.Join(budget.warnings(), "\n")
+ if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte document image budget") ||
+ !strings.Contains(warnings, "stopped extracting embedded images after the 2-item document limit") {
+ t.Fatalf("warnings = %q, want document byte and item limit warnings", warnings)
}
}
@@ -517,21 +558,21 @@ func TestFindBlockImage(t *testing.T) {
}
}
-func TestFetchImageAsBase64_RejectsCredentials(t *testing.T) {
- _, err := fetchImageAsBase64("https://user:[email protected]/img.png")
+func TestFetchImage_RejectsCredentials(t *testing.T) {
+ _, err := fetchImage(context.Background(), "https://user:[email protected]/img.png")
if err == nil {
t.Fatal("expected error for URL with credentials")
}
}
-func TestFetchImageAsBase64_InvalidURL(t *testing.T) {
+func TestFetchImage_InvalidURL(t *testing.T) {
withSSRFBypass(t)
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer ts.Close()
- _, err := fetchImageAsBase64(ts.URL + "/nonexistent.png")
+ _, err := fetchImage(context.Background(), ts.URL+"/nonexistent.png")
if err == nil {
t.Fatal("expected error for 404 response")
}
diff --git a/internal/parser/parser/media_budget.go b/internal/parser/parser/media_budget.go
new file mode 100644
index 0000000000..5e25ea1af3
--- /dev/null
+++ b/internal/parser/parser/media_budget.go
@@ -0,0 +1,114 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package parser
+
+import (
+ "fmt"
+)
+
+const (
+ maxEmbeddedImageBytes = MaxImagePayloadBytes
+ maxEmbeddedDocumentImageBytes = 64 << 20
+ maxEmbeddedMediaItems = 256
+)
+
+type embeddedMediaBudget struct {
+ maxImageBytes int
+ maxTotalBytes int
+ maxItems int
+
+ items int
+ totalBytes int
+ oversizedImages int
+ documentOverflows int
+ countExhausted bool
+}
+
+func newEmbeddedMediaBudget() *embeddedMediaBudget {
+ return &embeddedMediaBudget{
+ maxImageBytes: maxEmbeddedImageBytes,
+ maxTotalBytes: maxEmbeddedDocumentImageBytes,
+ maxItems: maxEmbeddedMediaItems,
+ }
+}
+
+// include reserves budget for one image payload. The second result is false
+// when callers should stop walking images because the document item limit has
+// been reached. Byte-limit omissions can keep walking so later text and media
+// metadata remain available.
+func (b *embeddedMediaBudget) include(data []byte) (bool, bool) {
+ if !b.reserveItem() {
+ return false, false
+ }
+ return b.includeReservedSize(len(data)), true
+}
+
+// reserveItem lets a parser cap image source attempts before fetching bytes.
+func (b *embeddedMediaBudget) reserveItem() bool {
+ if b.countExhausted {
+ return false
+ }
+ if b.items >= b.maxItems {
+ b.countExhausted = true
+ return false
+ }
+ b.items++
+ return true
+}
+
+// includeReservedSize charges a previously reserved image without retaining
+// or decoding its payload just to measure its size.
+func (b *embeddedMediaBudget) includeReservedSize(size int) bool {
+ if size > b.maxImageBytes {
+ b.oversizedImages++
+ return false
+ }
+ if size > b.maxTotalBytes-b.totalBytes {
+ b.documentOverflows++
+ return false
+ }
+ b.totalBytes += size
+ return true
+}
+
+func (b *embeddedMediaBudget) warnings() []string {
+ if b == nil {
+ return nil
+ }
+ var warnings []string
+ if b.oversizedImages > 0 {
+ warnings = append(warnings, fmt.Sprintf(
+ "omitted %d embedded image payload(s) larger than the %d-byte per-image limit",
+ b.oversizedImages,
+ b.maxImageBytes,
+ ))
+ }
+ if b.documentOverflows > 0 {
+ warnings = append(warnings, fmt.Sprintf(
+ "omitted %d embedded image payload(s) after reaching the %d-byte document image budget",
+ b.documentOverflows,
+ b.maxTotalBytes,
+ ))
+ }
+ if b.countExhausted {
+ warnings = append(warnings, fmt.Sprintf(
+ "stopped extracting embedded images after the %d-item document limit",
+ b.maxItems,
+ ))
+ }
+ return warnings
+}
diff --git a/internal/parser/parser/media_budget_test.go b/internal/parser/parser/media_budget_test.go
new file mode 100644
index 0000000000..df8f13ff6a
--- /dev/null
+++ b/internal/parser/parser/media_budget_test.go
@@ -0,0 +1,50 @@
+package parser
+
+import (
+ "reflect"
+ "testing"
+)
+
+func TestEmbeddedMediaBudgetReservedItemsAndWarnings(t *testing.T) {
+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 3}
+
+ if !budget.reserveItem() || !budget.includeReservedSize(3) {
+ t.Fatal("first image should fit")
+ }
+ if !budget.reserveItem() || budget.includeReservedSize(3) {
+ t.Fatal("second image should exceed the document byte limit")
+ }
+ if !budget.reserveItem() || budget.includeReservedSize(5) {
+ t.Fatal("third image should exceed the per-image byte limit")
+ }
+ if budget.reserveItem() {
+ t.Fatal("fourth image should be rejected before its source is read")
+ }
+ if included, keepWalking := budget.include([]byte("x")); included || keepWalking {
+ t.Fatalf("include after count exhaustion = (%v, %v), want (false, false)", included, keepWalking)
+ }
+ if budget.items != 3 || budget.totalBytes != 3 {
+ t.Fatalf("budget counters = (%d items, %d bytes), want (3, 3)", budget.items, budget.totalBytes)
+ }
+ want := []string{
+ "omitted 1 embedded image payload(s) larger than the 4-byte per-image limit",
+ "omitted 1 embedded image payload(s) after reaching the 5-byte document image budget",
+ "stopped extracting embedded images after the 3-item document limit",
+ }
+ if got := budget.warnings(); !reflect.DeepEqual(got, want) {
+ t.Fatalf("warnings = %q, want %q", got, want)
+ }
+}
+
+func TestEmbeddedMediaBudgetIncludesExactByteLimits(t *testing.T) {
+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}
+ if included, keepWalking := budget.include([]byte("abcd")); !included || !keepWalking {
+ t.Fatalf("exact-limit image = (%v, %v), want (true, true)", included, keepWalking)
+ }
+ if included, keepWalking := budget.include([]byte("e")); included || !keepWalking {
+ t.Fatalf("image after exact document limit = (%v, %v), want (false, true)", included, keepWalking)
+ }
+ if budget.totalBytes != 4 {
+ t.Fatalf("totalBytes = %d, want 4", budget.totalBytes)
+ }
+}
diff --git a/internal/parser/parser/office_image_fixture_cgo_test.go b/internal/parser/parser/office_image_fixture_cgo_test.go
new file mode 100644
index 0000000000..c7be30b61c
--- /dev/null
+++ b/internal/parser/parser/office_image_fixture_cgo_test.go
@@ -0,0 +1,120 @@
+//go:build cgo
+
+package parser
+
+import (
+ "bytes"
+ "encoding/base64"
+ "errors"
+ "image"
+ "image/color"
+ "image/png"
+ "os"
+ "path/filepath"
+ "testing"
+
+ officeOxide "github.com/yfedoseev/office_oxide/go"
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+func useFailingOCRAnalyzer(t *testing.T) {
+ t.Helper()
+ previous := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })
+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
+ return &deepdocpdf.MockDocAnalyzer{Healthy: true, OCRDetectErr: errors.New("image parser must not invoke OCR")}, true
+ })
+}
+
+func TestDOCXParser_RealTableImage(t *testing.T) {
+ useFailingOCRAnalyzer(t)
+ data, err := os.ReadFile(filepath.Join("testdata", "docx_table_image.docx"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ p := NewDOCXParser()
+ p.ConfigureFromSetup(map[string]any{"output_format": "json"})
+ result := p.ParseWithResult(t.Context(), "docx_table_image.docx", data)
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ if len(result.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
+ }
+ var tableID string
+ for _, item := range result.JSON {
+ if item["doc_type_kwd"] == "table" {
+ tableID, _ = item["source_table_id"].(string)
+ }
+ }
+ if tableID == "" {
+ t.Fatalf("table with image not found: %+v", result.JSON)
+ }
+ for _, item := range result.JSON {
+ if item["doc_type_kwd"] != "image" || item["parent_table_id"] != tableID {
+ continue
+ }
+ encoded, _ := item["image"].(string)
+ decoded, err := base64.StdEncoding.DecodeString(encoded)
+ if err != nil {
+ t.Fatalf("table image payload: %v", err)
+ }
+ if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {
+ t.Fatalf("table image is not PNG: %v", err)
+ }
+ if item["row_index"] != 1 || item["column_index"] != 1 || item["media_order"] != 1 {
+ t.Fatalf("table image location = %+v", item)
+ }
+ return
+ }
+ t.Fatalf("no image linked to table %q: %+v", tableID, result.JSON)
+}
+
+func TestPPTXParser_RealSlideImage(t *testing.T) {
+ useFailingOCRAnalyzer(t)
+ raster := image.NewRGBA(image.Rect(0, 0, 32, 16))
+ for y := 0; y < 16; y++ {
+ for x := 0; x < 32; x++ {
+ raster.Set(x, y, color.RGBA{R: 255, A: 255})
+ }
+ }
+ var imageData bytes.Buffer
+ if err := png.Encode(&imageData, raster); err != nil {
+ t.Fatal(err)
+ }
+ w := officeOxide.NewPptxWriter()
+ defer w.Close()
+ slide := w.AddSlide()
+ w.AddSlideText(slide, "slide with image")
+ w.AddSlideImage(slide, imageData.Bytes(), "png", 914400, 914400, 914400, 457200)
+ data, err := w.ToBytes()
+ if err != nil {
+ t.Fatal(err)
+ }
+ result := NewPPTXParser().ParseWithResult(t.Context(), "slide_image.pptx", data)
+ if result.Err != nil {
+ t.Fatal(result.Err)
+ }
+ if len(result.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
+ }
+ for _, item := range result.JSON {
+ if item["doc_type_kwd"] != "image" {
+ continue
+ }
+ encoded, _ := item["image"].(string)
+ decoded, err := base64.StdEncoding.DecodeString(encoded)
+ if err != nil {
+ t.Fatalf("slide image payload: %v", err)
+ }
+ if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {
+ t.Fatalf("slide image is not PNG: %v", err)
+ }
+ if item["slide_number"] != 1 || item["media_order"] != 1 {
+ t.Fatalf("slide image location = %+v", item)
+ }
+ return
+ }
+ t.Fatalf("slide image not emitted: %+v", result.JSON)
+}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index f8c3db1df5..0c22e5267b 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -297,7 +297,7 @@ func deprecatedHTML4Excel(setup map[string]any, parserName string) {
// worksheet as structured parser items. Excelize exposes both kinds through
// GetPictureCells/GetPictures; walking the reported anchor cells avoids
// scanning the worksheet's entire coordinate space.
-func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []string) {
+func extractXLSXImages(f *excelize.File, sheet string, budget *embeddedMediaBudget) ([]map[string]any, []string) {
cells, err := f.GetPictureCells(sheet)
if err != nil {
return nil, []string{fmt.Sprintf("XLSX image discovery failed for sheet %q: %v", sheet, err)}
@@ -308,6 +308,7 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []stri
items := make([]map[string]any, 0, len(cells))
warnings := make([]string, 0)
+cellLoop:
for _, cell := range cells {
pictures, err := f.GetPictures(sheet, cell)
if err != nil {
@@ -323,24 +324,37 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []stri
warnings = append(warnings, fmt.Sprintf("XLSX image skipped for sheet %q cell %s: unsupported extension %q", sheet, cell, picture.Extension))
continue
}
- encoded := base64.StdEncoding.EncodeToString(picture.File)
+ included, keepWalking := true, true
+ if budget != nil {
+ included, keepWalking = budget.include(picture.File)
+ }
+ if !keepWalking {
+ break cellLoop
+ }
alt := cell
if picture.Format != nil && picture.Format.AltText != "" {
alt = picture.Format.AltText
}
row, col := axisToRC(cell)
- items = append(items, map[string]any{
+ item := map[string]any{
"text": alt,
"doc_type_kwd": "image",
"ck_type": "image",
- "image": "data:" + mimeType + ";base64," + encoded,
"sheet": sheet,
"cell": cell,
"row_start": row,
"row_end": row,
"col_start": col,
"col_end": col,
- })
+ }
+ if included {
+ encoded := base64.StdEncoding.EncodeToString(picture.File)
+ item["image"] = "data:" + mimeType + ";base64," + encoded
+ } else {
+ item["image"] = nil
+ item["media_omitted"] = true
+ }
+ items = append(items, item)
}
}
return items, warnings
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index ddbfd3c1f1..6bb517a548 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -367,7 +367,7 @@ func TestXLSXImageMIMEType(t *testing.T) {
func TestExtractXLSXImagesWarningForInvalidSheet(t *testing.T) {
f := excelize.NewFile()
defer f.Close()
- _, warnings := extractXLSXImages(f, "MissingSheet")
+ _, warnings := extractXLSXImages(f, "MissingSheet", nil)
if len(warnings) != 1 || !strings.Contains(warnings[0], "image discovery failed") {
t.Fatalf("warnings = %v, want image discovery warning", warnings)
}
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 4904151a1c..4d7752deb7 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -62,8 +62,8 @@ type ParseResult struct {
// `[]map[string]any` with `text` + `doc_type_kwd` keys (and
// optional `image` / `layout` / `positions` fields);
// Markdown / HTML / text emit normalized
- // `{text, doc_type_kwd}` items; image emits OCR/VLM result
- // items.
+ // `{text, doc_type_kwd}` items; image emits an image payload and, after
+ // ingestion enhancement, its VLM description.
JSON []map[string]any
// Markdown is a backend response awaiting normalization.
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index a7646a7b87..d60782848b 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -241,12 +241,13 @@ Tail text`)
}
}
-func TestWalkHTMLBlocks_SkipsHeadLooseText(t *testing.T) {
+func TestWalkHTMLBlocksWithState_SkipsHeadLooseText(t *testing.T) {
head := &html.Node{Type: html.ElementNode, Data: "head"}
head.AppendChild(&html.Node{Type: html.TextNode, Data: "Head metadata"})
var items []map[string]any
- walkHTMLBlocks(head, &items)
+ state := &htmlWalkState{}
+ walkHTMLBlocksWithState(head, &items, state)
if len(items) != 0 {
t.Fatalf("JSON len = %d, want 0: %#v", len(items), items)
}
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index 8534d72f7f..f421c04888 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -380,8 +380,8 @@ func resolveDocAnalyzer(factory func() (deepdoctype.DocAnalyzer, bool)) (deepdoc
// GetDocAnalyzer returns the configured in-process DeepDoc analyzer. It is the
// single production entry point now that the external HTTP service is no longer
-// a backend. Callers outside the parser package (e.g. standalone image OCR in
-// the ingestion component) use this instead of constructing a client.
+// a backend. PDF and non-PDF image parsers use it instead of constructing a
+// client.
func GetDocAnalyzer() (deepdoctype.DocAnalyzer, error) {
return deepDocAnalyzerFromEnv()
}
diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go
index 9148080038..cb97a28118 100644
--- a/internal/parser/parser/picture_parser.go
+++ b/internal/parser/parser/picture_parser.go
@@ -14,23 +14,14 @@
// limitations under the License.
//
-// PictureParser validates and stores configuration for image files.
-// The actual OCR and VLM description is performed by the
-// component-layer maybeDispatchImage, which mirrors Python's
-// rag/app/picture.py:chunk() image branch.
-//
-// Python reference:
-// - Image is opened with PIL, converted to RGB
-// - PaddleOCR tried first (if layout_recognize == "@PaddleOCR")
-// - Falls back to local deepdoc.vision.OCR (ONNX text detection)
-// - If OCR text is short (≤32 chars / ≤32 words for English),
-// calls IMAGE2TEXT VLM describe() for a natural-language description
-// - Returns tokenized text with media context attached
+// PictureParser validates image files and returns their image payload for
+// optional VLM enhancement in the ingestion component.
package parser
import (
"context"
+ "encoding/base64"
"fmt"
"path/filepath"
"strings"
@@ -50,7 +41,7 @@ var imageExtensions = map[string]bool{
"ai": true, "raw": true, "wmf": true,
}
-// PictureParser handles image files for OCR and VLM description.
+// PictureParser emits image files for optional vision enhancement.
type PictureParser struct {
OutputFormat string
}
@@ -71,10 +62,7 @@ func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {
}
}
-// ParseWithResult implements ParseResultProducer. It validates the
-// file extension against the image extension whitelist. The actual
-// OCR and VLM description happens via maybeDispatchImage at the
-// component layer (mirrors Python's picture.py:chunk()).
+// ParseWithResult validates the image extension and returns its image payload.
func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
ext := strings.ToLower(filepath.Ext(filename))
if len(ext) > 1 && ext[0] == '.' {
@@ -94,6 +82,9 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
Err: fmt.Errorf("picture: unsupported extension %q (filename: %s); accepted: .jpg/.jpeg/.png/.gif/.bmp/.tiff/.tif/.webp/.svg/.ico/.avif/.heic/...", ext, filename),
}
}
+ if len(data) == 0 || len(data) > MaxImagePayloadBytes {
+ return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), MaxImagePayloadBytes)}
+ }
// Parser output is normalized to JSON at the component boundary, so an
// absent backend format defaults to JSON here as well.
@@ -102,13 +93,55 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
outFmt = "json"
}
+ release, err := AcquireImageMedia(ctx)
+ if err != nil {
+ return ParseResult{Err: err}
+ }
+ imagePayload := "data:" + pictureMIME(ext) + ";base64," + base64.StdEncoding.EncodeToString(data)
+ release()
+ item := map[string]any{
+ "text": "",
+ "image": imagePayload,
+ "doc_type_kwd": DocTypeImage,
+ }
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err}
+ }
return ParseResult{
OutputFormat: outFmt,
File: map[string]any{
"name": filename,
"size": len(data),
- "doc_type_kwd": "image",
+ "doc_type_kwd": DocTypeImage,
},
+ JSON: []map[string]any{item},
+ }
+}
+
+func pictureMIME(ext string) string {
+ switch ext {
+ case "jpg", "jpeg":
+ return "image/jpeg"
+ case "png":
+ return "image/png"
+ case "gif":
+ return "image/gif"
+ case "bmp":
+ return "image/bmp"
+ case "webp":
+ return "image/webp"
+ case "svg":
+ return "image/svg+xml"
+ case "tiff", "tif":
+ return "image/tiff"
+ case "ico":
+ return "image/x-icon"
+ case "avif":
+ return "image/avif"
+ case "heic":
+ return "image/heic"
+ default:
+ return "image/png"
}
}
diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.go
index c53360984d..6190b410ea 100644
--- a/internal/parser/parser/picture_parser_test.go
+++ b/internal/parser/parser/picture_parser_test.go
@@ -17,10 +17,29 @@
package parser
import (
+ "encoding/base64"
"strings"
"testing"
)
+func TestPictureParserKeepsImageWithoutOCR(t *testing.T) {
+ data := []byte("picture bytes")
+ res := NewPictureParser().ParseWithResult(t.Context(), "photo.png", data)
+ if res.Err != nil {
+ t.Fatal(res.Err)
+ }
+ if len(res.JSON) != 1 || res.JSON[0]["text"] != "" {
+ t.Fatalf("picture text = %+v, want empty text", res.JSON)
+ }
+ wantImage := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)
+ if res.JSON[0]["image"] != wantImage {
+ t.Fatalf("picture image = %v, want %q", res.JSON[0]["image"], wantImage)
+ }
+ if len(res.Warnings) != 0 {
+ t.Fatalf("picture warnings = %v, want none", res.Warnings)
+ }
+}
+
func TestNewPictureParser(t *testing.T) {
p := NewPictureParser()
if p == nil {
diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.go
index e0507e53d0..35351c9bdf 100644
--- a/internal/parser/parser/pptx_ir.go
+++ b/internal/parser/parser/pptx_ir.go
@@ -17,16 +17,17 @@
package parser
import (
+ "encoding/base64"
"encoding/json"
"fmt"
"strings"
)
-// buildPPTXJSONSections converts office_oxide presentation IR JSON into
-// one JSON item per slide. The IR carries exactly one section per slide,
-// so section index + 1 is the slide number. Per-slide text is flattened
-// with the shared IR walker (docxElementText), which covers paragraphs,
-// headings, text boxes, tables, and (nested) lists.
+// buildPPTXJSONSections converts office_oxide presentation IR JSON into one
+// text item per slide followed by that slide's image items. The IR carries
+// exactly one section per slide, so section index + 1 is the slide number.
+// Per-slide text is flattened with the shared IR walker (docxElementText),
+// which covers paragraphs, headings, text boxes, tables, and (nested) lists.
//
// A slide without extractable text (blank layout, image-only) still
// yields an item with an empty text field, so slide_number stays
@@ -34,11 +35,14 @@ import (
//
// Pure Go (no cgo) so the IR → items transform is unit-testable without
// the office_oxide native library.
-func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {
+func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[string]any, error) {
var ir docxIRDocument
if err := json.Unmarshal([]byte(irJSON), &ir); err != nil {
return nil, fmt.Errorf("presentation ir-json decode: %w", err)
}
+ if budget == nil {
+ budget = newEmbeddedMediaBudget()
+ }
items := make([]map[string]any, 0, len(ir.Sections))
for i, sec := range ir.Sections {
var lines []string
@@ -58,19 +62,47 @@ func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {
"doc_type_kwd": "text",
"slide_number": i + 1,
})
+ mediaOrder := 0
+ forEachDOCXIRImage(sec.Elements, func(data []byte) bool {
+ mediaOrder++
+ imageItem := map[string]any{
+ "text": "",
+ "image": nil,
+ "doc_type_kwd": "image",
+ "ck_type": "image",
+ "slide_number": i + 1,
+ "media_order": mediaOrder,
+ }
+ included, keepWalking := budget.include(data)
+ if included {
+ imageItem["image"] = base64.StdEncoding.EncodeToString(data)
+ } else if keepWalking {
+ imageItem["media_omitted"] = true
+ } else {
+ return false
+ }
+ items = append(items, imageItem)
+ return keepWalking
+ })
}
return items, nil
}
-// itemsAllEmpty reports whether every item carries no extractable text.
-// A deck whose IR sections all flatten to "" (e.g. text the IR walker does
-// not yet cover) must fall back to PlainText instead of emitting empty
-// chunks that the Tokenizer later filters, silently yielding 0 chunks.
+// itemsAllEmpty reports whether every item lacks both text and image payloads.
+// A deck whose IR sections all flatten to "" and contain no images must fall
+// back to PlainText instead of emitting empty chunks that the Tokenizer later
+// filters, silently yielding 0 chunks.
func itemsAllEmpty(items []map[string]any) bool {
for _, it := range items {
if text, _ := it["text"].(string); strings.TrimSpace(text) != "" {
return false
}
+ if image, _ := it["image"].(string); strings.TrimSpace(image) != "" {
+ return false
+ }
+ if omitted, _ := it["media_omitted"].(bool); omitted {
+ return false
+ }
}
return true
}
diff --git a/internal/parser/parser/pptx_ir_test.go b/internal/parser/parser/pptx_ir_test.go
index 7e5d708df4..dfa37c01a4 100644
--- a/internal/parser/parser/pptx_ir_test.go
+++ b/internal/parser/parser/pptx_ir_test.go
@@ -103,11 +103,6 @@ func TestBuildPPTXJSONSections(t *testing.T) {
irJSON: `{"sections":[{"elements":[{"type":"text","text":"loose text"}]}]}`,
wantTexts: []string{"loose text"},
},
- {
- name: "image-only slide yields empty text",
- irJSON: `{"sections":[{"elements":[{"type":"image","data":"aGVsbG8="}]}]}`,
- wantTexts: []string{""},
- },
{
// bwbd.pptx regression: office_oxide wraps grouped slide shapes
// (graphicFrame tables) in a text_box, so the slide's text_box
@@ -176,7 +171,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- items, err := buildPPTXJSONSections(tt.irJSON)
+ items, err := buildPPTXJSONSections(tt.irJSON, newEmbeddedMediaBudget())
if tt.wantErr {
if err == nil {
t.Fatalf("buildPPTXJSONSections: want error, got items %+v", items)
@@ -204,6 +199,47 @@ func TestBuildPPTXJSONSections(t *testing.T) {
}
}
+func TestBuildPPTXJSONSections_ExtractsSlideImagesInOrder(t *testing.T) {
+ irJSON := `{"sections":[{"elements":[
+ {"type":"paragraph","content":[{"type":"text","text":"slide text"}]},
+ {"type":"image","data":"aGVsbG8="},
+ {"type":"table","rows":[{"cells":[{"content":[{"type":"image","data":"aW1hZ2U="}]}]}]}
+ ]},{"elements":[{"type":"image","data":"c2Vjb25k"}]}]}`
+
+ items, err := buildPPTXJSONSections(irJSON, newEmbeddedMediaBudget())
+ if err != nil {
+ t.Fatalf("buildPPTXJSONSections: %v", err)
+ }
+ want := []struct {
+ docType string
+ text string
+ image string
+ slide int
+ order int
+ }{
+ {docType: "text", text: "slide text", slide: 1},
+ {docType: "image", image: "aGVsbG8=", slide: 1, order: 1},
+ {docType: "image", image: "aW1hZ2U=", slide: 1, order: 2},
+ {docType: "text", slide: 2},
+ {docType: "image", image: "c2Vjb25k", slide: 2, order: 1},
+ }
+ if len(items) != len(want) {
+ t.Fatalf("items = %+v, want %d slide/text and image items", items, len(want))
+ }
+ for i, w := range want {
+ item := items[i]
+ if item["doc_type_kwd"] != w.docType || item["slide_number"] != w.slide {
+ t.Errorf("item %d identity = %+v, want type %q slide %d", i, item, w.docType, w.slide)
+ }
+ if w.docType == "image" && (item["image"] != w.image || item["media_order"] != w.order) {
+ t.Errorf("item %d image = %+v, want payload %q order %d", i, item, w.image, w.order)
+ }
+ if w.docType == "text" && item["text"] != w.text {
+ t.Errorf("item %d text = %v, want %q", i, item["text"], w.text)
+ }
+ }
+}
+
// TestItemsAllEmpty pins the all-empty fallback gate: empty and whitespace
// items trigger the PlainText salvage, any non-empty item does not.
func TestItemsAllEmpty(t *testing.T) {
@@ -224,6 +260,11 @@ func TestItemsAllEmpty(t *testing.T) {
items: []map[string]any{{"text": ""}, {"text": "hello"}},
want: false,
},
+ {
+ name: "image payload",
+ items: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"}},
+ want: false,
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.go
index 059cd2f97b..8b7a6c9bb4 100644
--- a/internal/parser/parser/pptx_parser.go
+++ b/internal/parser/parser/pptx_parser.go
@@ -164,10 +164,14 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
JSON: itemsFromPlainText(text),
}
}
- items, err := buildPPTXJSONSections(irJSON)
+ budget := newEmbeddedMediaBudget()
+ items, err := buildPPTXJSONSections(irJSON, budget)
if err != nil {
return ParseResult{Err: err}
}
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err}
+ }
if len(items) == 0 || itemsAllEmpty(items) {
// A deck whose IR carries no sections at all, or whose sections
// all flatten to empty text (text the IR walker does not cover):
@@ -186,5 +190,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
OutputFormat: "json",
File: map[string]any{"name": filename, "format": effFormat},
JSON: items,
+ Warnings: budget.warnings(),
}
}
diff --git a/internal/parser/parser/testdata/docx_table_image.docx b/internal/parser/parser/testdata/docx_table_image.docx
new file mode 100644
index 0000000000..6ecdaf602d
Binary files /dev/null and b/internal/parser/parser/testdata/docx_table_image.docx differ
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index 7dc42a4269..23cde8c1a1 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -85,10 +85,15 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
}
}
- items, warnings, sheetsCount, err := parseXLSXBytes(data)
+ mediaBudget := newEmbeddedMediaBudget()
+ items, warnings, sheetsCount, err := parseXLSXBytes(data, mediaBudget)
+ if err := ctx.Err(); err != nil {
+ return ParseResult{Err: err}
+ }
if err != nil {
return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}
}
+ warnings = append(warnings, mediaBudget.warnings()...)
return ParseResult{
OutputFormat: spreadsheetOutputFormat,
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index 6f5d8dbf54..bf71d359cb 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -3,10 +3,16 @@ package parser
import (
"bytes"
"context"
+ "encoding/base64"
+ "errors"
+ "image"
+ "image/png"
"strings"
"testing"
"github.com/xuri/excelize/v2"
+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
)
func createTestExcelBytes(t *testing.T) []byte {
@@ -112,3 +118,100 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
t.Errorf("resHTML.JSON should have items")
}
}
+
+func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
+ var imageData bytes.Buffer
+ if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
+ t.Fatalf("encode fixture: %v", err)
+ }
+ imageBytes := imageData.Bytes()
+ f := excelize.NewFile()
+ defer f.Close()
+ if err := f.AddPictureFromBytes("Sheet1", "C3", &excelize.Picture{
+ Extension: ".png",
+ File: imageBytes,
+ Format: &excelize.GraphicOptions{AltText: "sheet image"},
+ }); err != nil {
+ t.Fatalf("AddPictureFromBytes: %v", err)
+ }
+ var workbook bytes.Buffer
+ if err := f.Write(&workbook); err != nil {
+ t.Fatalf("write workbook: %v", err)
+ }
+
+ previous := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })
+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
+ return &deepdocpdf.MockDocAnalyzer{Healthy: true, OCRDetectErr: errors.New("xls OCR sentinel")}, true
+ })
+
+ p, err := NewXLSParser("excelize")
+ if err != nil {
+ t.Fatalf("NewXLSParser: %v", err)
+ }
+ result := p.ParseWithResult(t.Context(), "with-image.xls", workbook.Bytes())
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ var imageItem map[string]any
+ for _, item := range result.JSON {
+ if item["doc_type_kwd"] == "image" {
+ imageItem = item
+ break
+ }
+ }
+ if imageItem == nil || imageItem["image"] != "data:image/png;base64,"+base64.StdEncoding.EncodeToString(imageBytes) {
+ t.Fatalf("image item = %+v, want retained picture payload", imageItem)
+ }
+ if len(result.Warnings) != 0 {
+ t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
+ }
+}
+
+func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {
+ var imageData bytes.Buffer
+ if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {
+ t.Fatalf("encode fixture: %v", err)
+ }
+ f := excelize.NewFile()
+ defer f.Close()
+ for i := 0; i <= maxEmbeddedMediaItems; i++ {
+ cell, err := excelize.CoordinatesToCellName(3, i+1)
+ if err != nil {
+ t.Fatalf("cell %d: %v", i, err)
+ }
+ if err := f.AddPictureFromBytes("Sheet1", cell, &excelize.Picture{Extension: ".png", File: imageData.Bytes()}); err != nil {
+ t.Fatalf("AddPictureFromBytes(%s): %v", cell, err)
+ }
+ }
+ var workbook bytes.Buffer
+ if err := f.Write(&workbook); err != nil {
+ t.Fatalf("write workbook: %v", err)
+ }
+
+ previous := deepdoctype.NativeDocAnalyzerFactory
+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })
+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {
+ return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true
+ })
+ p, err := NewXLSParser("excelize")
+ if err != nil {
+ t.Fatalf("NewXLSParser: %v", err)
+ }
+ result := p.ParseWithResult(t.Context(), "many-images.xls", workbook.Bytes())
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ imageCount := 0
+ for _, item := range result.JSON {
+ if item["doc_type_kwd"] == "image" {
+ imageCount++
+ }
+ }
+ if imageCount != maxEmbeddedMediaItems {
+ t.Fatalf("image items = %d, want %d", imageCount, maxEmbeddedMediaItems)
+ }
+ if warnings := strings.Join(result.Warnings, "\n"); !strings.Contains(warnings, "stopped extracting embedded images after the 256-item document limit") {
+ t.Fatalf("warnings = %q, want image-count limit warning", warnings)
+ }
+}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index d71ca9c170..e8cc2ee7ed 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -107,10 +107,18 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
// for spreadsheet processing.
}
- items, warnings, sheets, err := parseXLSXBytes(data)
+ mediaBudget := newEmbeddedMediaBudget()
+ items, warnings, sheets, err := parseXLSXBytes(data, mediaBudget)
if err == nil {
+ if ctx.Err() != nil {
+ return ParseResult{Err: ctx.Err()}
+ }
+ warnings = append(warnings, mediaBudget.warnings()...)
return xlsxParseResult(filename, items, warnings, sheets)
}
+ if ctx.Err() != nil {
+ return ParseResult{Err: ctx.Err()}
+ }
normalized, normalizeWarnings, changed, normalizeErr := normalizeXLSXForRead(data)
if normalizeErr != nil {
@@ -119,15 +127,22 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data
if !changed {
return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}
}
- items, warnings, sheets, retryErr := parseXLSXBytes(normalized)
+ // The first parse attempt is discarded, so its media reservations must not
+ // affect the normalized retry's output.
+ mediaBudget = newEmbeddedMediaBudget()
+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized, mediaBudget)
if retryErr != nil {
return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}
}
+ if ctx.Err() != nil {
+ return ParseResult{Err: ctx.Err()}
+ }
warnings = append(normalizeWarnings, warnings...)
+ warnings = append(warnings, mediaBudget.warnings()...)
return xlsxParseResult(filename, items, warnings, sheets)
}
-func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {
+func parseXLSXBytes(data []byte, mediaBudget *embeddedMediaBudget) ([]map[string]any, []string, int, error) {
f, err := excelize.OpenReader(bytes.NewReader(data))
if err != nil {
return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)
@@ -143,7 +158,7 @@ func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {
return nil, warnings, len(sheets), err
}
warnings = append(warnings, sheetWarnings...)
- images, imageWarnings := extractXLSXImages(f, sheet)
+ images, imageWarnings := extractXLSXImages(f, sheet, mediaBudget)
for _, image := range images {
row, _ := numericItemInt(image["row_start"])
col, _ := numericItemInt(image["col_start"])