RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(parser): add opt-in VLM enhancement for images (#20166)
jay7772131fd92c2fa5e
diff --git a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go b/internal/deepdoc/parser/pdf/pdfoxide_bridge.goindex 340d634076..799aad4a8e 100644--- a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go+++ b/internal/deepdoc/parser/pdf/pdfoxide_bridge.go@@ -45,6 +45,11 @@ func (e *PDFOxideEngine) RenderPageImage(pageNum int, dpi float64) (image.Image,return e.Inner.RenderPageImage(pageNum, dpi)}+// PageSize returns the effective dimensions of a zero-based page in PDF points.+func (e *PDFOxideEngine) PageSize(pageNum int) (float64, float64, error) {+ return e.Inner.PageSize(pageNum)+}+func (e *PDFOxideEngine) ExtractChars(pageNum int) ([]pdf.TextChar, error) {chars, err := e.Inner.ExtractChars(pageNum)if err != nil {diff --git a/internal/deepdoc/parser/pdf/util/crop.go b/internal/deepdoc/parser/pdf/util/crop.goindex 18f030f638..9bca119466 100644--- a/internal/deepdoc/parser/pdf/util/crop.go+++ b/internal/deepdoc/parser/pdf/util/crop.go@@ -6,7 +6,6 @@ import ("image""image/color""math"- "strings""go.uber.org/zap"@@ -20,23 +19,63 @@ import (//// Python: pdf_parser.py:1802 RAGFlowPdfParser.crop()func CropSectionImage(posTag string, decodedImages map[int]image.Image, zoom float64) string {+ return encodeCroppedImage(cropSectionImageRaster(posTag, decodedImages, zoom))+}++func cropSectionImageRaster(posTag string, decodedImages map[int]image.Image, zoom float64) image.Image {if len(decodedImages) == 0 {common.Warn("cropSectionImage: no page images available, skipping image generation")- return ""+ return nil}positions := ExtractPositions(posTag)if len(positions) == 0 {common.Warn("cropSectionImage: empty position list in tag", zap.String("posTag", posTag[:min(80, len(posTag))]))- return ""+ return nil}+ return cropSectionPositionsRaster(positions, decodedImages, zoom)+}++type sectionRasterSegment struct {+ page int+ x0 int+ y0 int+ x1 int+ y1 int+ isEdge bool+}- // Filter valid positions (all pages available).- var valid []pdf.Position+type sectionRasterPlan struct {+ segments []sectionRasterSegment+ width int+ height int+}++const (+ sectionRasterContextPad = 120.0+ sectionRasterGap = 6+)++func cropSectionPositionsRaster(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) image.Image {+ plan, ok := buildSectionRasterPlan(positions, decodedImages, zoom)+ if !ok {+ return nil+ }+ return renderSectionRaster(plan, decodedImages)+}++func buildSectionRasterPlan(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) (*sectionRasterPlan, bool) {+ if len(decodedImages) == 0 || len(positions) == 0 || zoom <= 0 || math.IsNaN(zoom) || math.IsInf(zoom, 0) {+ return nil, false+ }+ valid := make([]pdf.Position, 0, len(positions))for _, pos := range positions {+ if len(pos.PageNumbers) == 0 {+ continue+ }allValid := true- for _, pn := range pos.PageNumbers {- if _, ok := decodedImages[pn]; !ok {+ for _, pageNum := range pos.PageNumbers {+ if img, ok := decodedImages[pageNum]; !ok || img == nil {allValid = falsebreak}@@ -46,196 +85,168 @@ func CropSectionImage(posTag string, decodedImages map[int]image.Image, zoom flo}}if len(valid) == 0 {- common.Warn("cropSectionImage: no valid positions after filtering, skipping crop")- return ""+ return nil, false}- // Context padding (Python: 120px above first, 120 below last, 6px gap)- const contextPad = 120.0- const gap = 6-- // Compute max width across original positions for full-width edge bands.maxWidth := 6.0for _, pos := range valid {- w := pos.Right - pos.Left- if w > maxWidth {- maxWidth = w+ if width := pos.Right - pos.Left; width > maxWidth {+ maxWidth = width}}- // Python-style: insert synthetic context bands at edges.- // Original positions are all middle entries (narrow width).- // Synthetic bands are edge entries (full width + semi-transparent overlay).first := valid[0]last := valid[len(valid)-1]- firstPageIdx := first.PageNumbers[0]- lastPageIdx := last.PageNumbers[len(last.PageNumbers)-1]- lastPageH := float64(decodedImages[lastPageIdx].Bounds().Dy()) / zoom-- // topBand: 120px context above the first content position.- topBandPos := pdf.Position{- PageNumbers: []int{firstPageIdx},+ firstPage := first.PageNumbers[0]+ lastPage := last.PageNumbers[len(last.PageNumbers)-1]+ lastPageHeight := float64(decodedImages[lastPage].Bounds().Dy()) / zoom+ topBand := pdf.Position{+ PageNumbers: []int{firstPage},Left: first.Left,Right: first.Right,- Top: math.Max(0, first.Top-contextPad),- Bottom: math.Max(first.Top-gap, 0),+ Top: math.Max(0, first.Top-sectionRasterContextPad),+ Bottom: math.Max(first.Top-sectionRasterGap, 0),}- // bottomBand: 120px context below the last content position.- bottomBandPos := pdf.Position{- PageNumbers: []int{lastPageIdx},+ bottomBand := pdf.Position{+ PageNumbers: []int{lastPage},Left: last.Left,Right: last.Right,- Top: math.Min(lastPageH, last.Bottom+gap),- Bottom: math.Min(lastPageH, last.Bottom+contextPad),+ Top: math.Min(lastPageHeight, last.Bottom+sectionRasterGap),+ Bottom: math.Min(lastPageHeight, last.Bottom+sectionRasterContextPad),}- // Build entry list: [topBand, original positions..., bottomBand].- type segment struct {- img image.Image- isEdge bool+ type positionEntry struct {+ position pdf.Position+ isEdge bool}- var segments []segment-- allPos := make([]struct {- pos pdf.Position- isEdge bool- }, 0, len(valid)+2)- allPos = append(allPos, struct {- pos pdf.Position- isEdge bool- }{topBandPos, true})+ entries := make([]positionEntry, 0, len(valid)+2)+ entries = append(entries, positionEntry{position: topBand, isEdge: true})for _, pos := range valid {- allPos = append(allPos, struct {- pos pdf.Position- isEdge bool- }{pos, false})- }- allPos = append(allPos, struct {- pos pdf.Position- isEdge bool- }{bottomBandPos, true})-- for _, entry := range allPos {- pos := entry.pos- isEdge := entry.isEdge-- top := pos.Top- bottom := pos.Bottom- left := pos.Left- right := pos.Right-- // Width: edge segments are full-width, middle are narrow.- if !isEdge {- right = math.Max(left+10, right)- } else {+ entries = append(entries, positionEntry{position: pos})+ }+ entries = append(entries, positionEntry{position: bottomBand, isEdge: true})++ plan := §ionRasterPlan{}+ maxInt := int(^uint(0) >> 1)+ for _, entry := range entries {+ pos := entry.position+ left, right := pos.Left, pos.Right+ if entry.isEdge {right = left + maxWidth+ } else {+ right = math.Max(left+10, right)}-- pn0 := pos.PageNumbers[0]-- // Accumulate bottom for multi-page positions.- accumBottom := bottom * zoom- for _, pn := range pos.PageNumbers[1:] {- if pn == pn0 {- continue- }- if img, ok := decodedImages[pn]; ok {- accumBottom += float64(img.Bounds().Dy())+ firstPage := pos.PageNumbers[0]+ accumBottom := pos.Bottom * zoom+ for _, pageNum := range pos.PageNumbers[1:] {+ if pageNum != firstPage {+ accumBottom += float64(decodedImages[pageNum].Bounds().Dy())}}- pageImg, ok := decodedImages[pn0]- if !ok {- common.Warn("cropSectionImage: page image not found", zap.Int("page", pn0))- return ""- }- pageH := float64(pageImg.Bounds().Dy())- bottomClamped := math.Min(accumBottom, pageH)-- // Crop first page of this position.- cropped := FastCrop(pageImg,- int(left*zoom), int(top*zoom),- int(right*zoom), int(bottomClamped))- if isEdge {- cropped = applyEdgeOverlay(cropped)- }- segments = append(segments, segment{img: cropped, isEdge: isEdge})+ pageImage := decodedImages[firstPage]+ pageHeight := float64(pageImage.Bounds().Dy())+ bottomClamped := math.Min(accumBottom, pageHeight)+ plan.addCrop(firstPage, int(left*zoom), int(pos.Top*zoom), int(right*zoom), int(bottomClamped), entry.isEdge)- // Subsequent pages (only those different from the first page).- bottomRemaining := accumBottom - pageH- for _, pn := range pos.PageNumbers[1:] {- if pn == pn0 {+ bottomRemaining := accumBottom - pageHeight+ for _, pageNum := range pos.PageNumbers[1:] {+ if pageNum == firstPage {continue}- pageImg2, ok := decodedImages[pn]- if !ok {- common.Warn("cropSectionImage: page image not found for subsequent page", zap.Int("page", pn))- return ""+ if bottomRemaining <= 0 {+ break}- pageH2 := float64(pageImg2.Bounds().Dy())- bottomClamped2 := math.Min(bottomRemaining, pageH2)- cropped2 := FastCrop(pageImg2,- int(left*zoom), 0,- int(right*zoom), int(bottomClamped2))- if isEdge {- cropped2 = applyEdgeOverlay(cropped2)+ pageImage := decodedImages[pageNum]+ bottomClamped := math.Min(bottomRemaining, float64(pageImage.Bounds().Dy()))+ bottomPixels := int(bottomClamped)+ if bottomPixels <= 0 {+ break}- segments = append(segments, segment{img: cropped2, isEdge: isEdge})- bottomRemaining -= bottomClamped2+ plan.addCrop(pageNum, int(left*zoom), 0, int(right*zoom), bottomPixels, entry.isEdge)+ bottomRemaining -= bottomClamped}}-- if len(segments) == 0 {- return ""+ if len(plan.segments) == 0 {+ return nil, false+ }+ for _, segment := range plan.segments {+ width, height := cropSize(decodedImages[segment.page], segment.x0, segment.y0, segment.x1, segment.y1)+ plan.width = max(plan.width, width)+ if height > maxInt-sectionRasterGap || plan.height > maxInt-(height+sectionRasterGap) {+ return nil, false+ }+ plan.height += height + sectionRasterGap}+ return plan, plan.width > 0 && plan.height > 0+}- // Stitch vertically with gray background and 6px gaps.- totalH := 0- maxW := 0- for _, seg := range segments {- totalH += seg.img.Bounds().Dy() + gap- maxW = max(maxW, seg.img.Bounds().Dx())+func (p *sectionRasterPlan) addCrop(page, x0, y0, x1, y1 int, isEdge bool) {+ p.segments = append(p.segments, sectionRasterSegment{+ page: page, x0: x0, y0: y0, x1: x1, y1: y1, isEdge: isEdge,+ })+}++func cropSize(img image.Image, x0, y0, x1, y1 int) (int, int) {+ bounds := cropRectBounds(img, x0, y0, x1, y1)+ if bounds.Empty() {+ return 1, 1}- stitched := image.NewRGBA(image.Rect(0, 0, maxW, totalH))+ return bounds.Dx(), bounds.Dy()+}- // Fill background using direct Pix slice write (matching fastCrop pattern).- // Gray 245,245,245,255 as BGRA bytes.- for y := 0; y < totalH; y++ {- row := stitched.Pix[stitched.PixOffset(0, y):stitched.PixOffset(maxW, y)]+func renderSectionRaster(plan *sectionRasterPlan, decodedImages map[int]image.Image) image.Image {+ segments := make([]image.Image, 0, len(plan.segments))+ for _, segment := range plan.segments {+ cropped := FastCrop(decodedImages[segment.page], segment.x0, segment.y0, segment.x1, segment.y1)+ if segment.isEdge {+ cropped = applyEdgeOverlay(cropped)+ }+ segments = append(segments, cropped)+ }+ stitched := image.NewRGBA(image.Rect(0, 0, plan.width, plan.height))+ for y := 0; y < plan.height; y++ {+ row := stitched.Pix[stitched.PixOffset(0, y):stitched.PixOffset(plan.width, y)]for i := 0; i < len(row); i += 4 {- row[i] = 245 // B- row[i+1] = 245 // G- row[i+2] = 245 // R- row[i+3] = 255 // A+ row[i], row[i+1], row[i+2], row[i+3] = 245, 245, 245, 255}}curY := 0- for _, seg := range segments {- srcW := seg.img.Bounds().Dx()- srcH := seg.img.Bounds().Dy()- if rgba, ok := seg.img.(*image.RGBA); ok {- // Fast path: direct Pix slice copy (matching fastCrop in geometry.go).- srcMinX := seg.img.Bounds().Min.X- srcMinY := seg.img.Bounds().Min.Y- for ry := 0; ry < srcH; ry++ {- srcStart := rgba.PixOffset(srcMinX, srcMinY+ry)+ for _, segment := range segments {+ srcW, srcH := segment.Bounds().Dx(), segment.Bounds().Dy()+ if rgba, ok := segment.(*image.RGBA); ok {+ srcMinX, srcMinY := segment.Bounds().Min.X, segment.Bounds().Min.Y+ for row := 0; row < srcH; row++ {+ srcStart := rgba.PixOffset(srcMinX, srcMinY+row)srcRow := rgba.Pix[srcStart : srcStart+srcW*4]- dstStart := stitched.PixOffset(0, curY+ry)+ dstStart := stitched.PixOffset(0, curY+row)copy(stitched.Pix[dstStart:], srcRow)}} else {- // Fallback: pixel-by-pixel for non-RGBA images (e.g. edge overlays).for y := 0; y < srcH; y++ {for x := 0; x < srcW; x++ {- stitched.Set(x, curY+y, seg.img.At(x+seg.img.Bounds().Min.X, y+seg.img.Bounds().Min.Y))+ stitched.Set(x, curY+y, segment.At(x+segment.Bounds().Min.X, y+segment.Bounds().Min.Y))}}}- curY += srcH + gap+ curY += srcH + sectionRasterGap}+ return stitched+}- data, err := EncodePNG(stitched)+func rasterPlanExceedsLimit(plan *sectionRasterPlan, maxPixels int64) bool {+ if plan == nil || plan.width <= 0 || plan.height <= 0 || maxPixels <= 0 {+ return true+ }+ return int64(plan.width) > maxPixels/int64(plan.height)+}++func encodeCroppedImage(img image.Image) string {+ if img == nil {+ return ""+ }+ data, err := EncodePNG(img)if err != nil {common.Warn("cropSectionImage: PNG encode failed", zap.Error(err))return ""@@ -598,25 +609,24 @@ func CropImageRegion(img image.Image, r pdf.DLARegion) (image.Image, error) {//// Python: pdf_parser.py:1802 RAGFlowPdfParser.crop()func CropSectionPositions(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) string {- if len(positions) == 0 {- return ""- }- var tag strings.Builder- for _, pos := range positions {- if len(pos.PageNumbers) == 0 {- continue- }- if len(pos.PageNumbers) == 1 {- tag.WriteString(FormatPositionTag(pos.PageNumbers[0], pos.Left, pos.Right, pos.Top, pos.Bottom))- } else {- from, to := pos.PageNumbers[0], pos.PageNumbers[len(pos.PageNumbers)-1]- tag.WriteString(FormatPositionTagRange(from, to, pos.Left, pos.Right, pos.Top, pos.Bottom))- }- }- if tag.Len() == 0 {- return ""+ return encodeCroppedImage(CropSectionPositionsRaster(positions, decodedImages, zoom))+}++// CropSectionPositionsRaster crops typed PDF positions and returns the raster+// without encoding it. Callers that need both OCR and a VLM payload can reuse+// this image and encode it once after OCR.+func CropSectionPositionsRaster(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64) image.Image {+ return cropSectionPositionsRaster(positions, decodedImages, zoom)+}++// CropSectionPositionsRasterLimited rejects a section before rendering when+// the planned stitched raster exceeds maxPixels.+func CropSectionPositionsRasterLimited(positions []pdf.Position, decodedImages map[int]image.Image, zoom float64, maxPixels int64) image.Image {+ plan, ok := buildSectionRasterPlan(positions, decodedImages, zoom)+ if !ok || rasterPlanExceedsLimit(plan, maxPixels) {+ return nil}- return CropSectionImage(tag.String(), decodedImages, zoom)+ return renderSectionRaster(plan, decodedImages)}// PositionsFromMatrix converts the _pdf_positions / positions matrix formdiff --git a/internal/deepdoc/parser/pdf/util/crop_test.go b/internal/deepdoc/parser/pdf/util/crop_test.goindex dd24e15d19..056113c981 100644--- a/internal/deepdoc/parser/pdf/util/crop_test.go+++ b/internal/deepdoc/parser/pdf/util/crop_test.go@@ -31,6 +31,19 @@ func decodePNG(t *testing.T, data []byte) image.Image {return img}+func TestFastCropReturnsPlaceholderForInvertedBounds(t *testing.T) {+ src := makeTestPageImage(100, 100, color.RGBA{255, 0, 0, 255})+ for _, bounds := range [][4]int{+ {80, 10, 20, 90},+ {10, 80, 90, 20},+ } {+ got := FastCrop(src, bounds[0], bounds[1], bounds[2], bounds[3]).Bounds()+ if got != image.Rect(0, 0, 1, 1) {+ t.Errorf("FastCrop(%v) bounds = %v, want 1x1 placeholder", bounds, got)+ }+ }+}+func TestCropSectionImage_SinglePage(t *testing.T) {pageImages := map[int]image.Image{0: makeTestPageImage(200, 300, color.RGBA{255, 0, 0, 255}),@@ -147,6 +160,35 @@ func TestCropSectionImage_ZoomScaling(t *testing.T) {}}+type accessCountingImage struct {+ bounds image.Rectangle+ reads int+}++func (i *accessCountingImage) ColorModel() color.Model { return color.RGBAModel }+func (i *accessCountingImage) Bounds() image.Rectangle { return i.bounds }+func (i *accessCountingImage) At(int, int) color.Color {+ i.reads+++ return color.RGBA{A: 255}+}++func TestCropSectionPositionsRasterLimitedRejectsOversizedStitchBeforeCopy(t *testing.T) {+ page := &accessCountingImage{bounds: image.Rect(0, 0, 400, 400)}+ positions := []pdf.Position{+ {PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},+ {PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},+ {PageNumbers: []int{0}, Left: 10, Right: 30, Top: 150, Bottom: 170},+ }++ got := CropSectionPositionsRasterLimited(positions, map[int]image.Image{0: page}, 1, 5_000)+ if got != nil {+ t.Fatalf("limited crop = %v, want nil when stitched pixels exceed the limit", got.Bounds())+ }+ if page.reads != 0 {+ t.Fatalf("source pixel reads = %d, want 0 before rejecting the oversized stitch", page.reads)+ }+}+func TestRotateImageCW(t *testing.T) {// Create a 3x2 image with known colors: (0,0)=red, (1,0)=green, (2,0)=blue,// (0,1)=white, (1,1)=black, (2,1)=gray@@ -301,34 +343,29 @@ func colorEqual(a, b color.Color) bool {return ar == br && ag == bg && ab == bb && aa == ba}-// TestCropSectionImage_MultiPage verifies the bottomRemaining fix for 3+ page-// positions where page heights differ. Regression test for Bug #3.+// TestCropSectionImage_MultiPage verifies that a non-positive trailing-page+// remainder does not become an inverted crop region.func TestCropSectionImage_MultiPage(t *testing.T) {// Page 0: tall (2000px), Page 1: short (800px), Page 2: short (800px)- // Content spans all 3 pages. The old bug subtracted full pageH2 from- // bottomRemaining instead of the actual clamped value, causing negative- // y1 on the last page → 1×1 placeholder crop.+ // The range stores only its endpoint page numbers. With these page heights,+ // the remaining crop height is exhausted before the trailing endpoint.pageImages := map[int]image.Image{0: makeTestPageImage(100, 2000, color.RGBA{200, 0, 0, 255}),1: makeTestPageImage(100, 800, color.RGBA{0, 200, 0, 255}),2: makeTestPageImage(100, 800, color.RGBA{0, 0, 200, 255}),}- // pdf.Position spans pages 0-2, bottom reaches into page 2.+ // The position range covers pages 0-2.posTag := "@@1-3\t0.0\t100.0\t0.0\t500.0##"b64 := CropSectionImage(posTag, pageImages, 1)if b64 == "" {t.Fatal("expected non-empty result for multi-page position")}- // Decode and check height: content 500pt + bottom on page 1 clamped- // to 800 → page 1 crop 0-800, page 2 crop 0-200. Total with 2x6px gaps- // should be ~2000 + 200 + 12 = 2212.+ // The first page and context bands should still yield a non-empty crop.decoded, _ := base64.StdEncoding.DecodeString(b64)img := decodePNG(t, decoded)h := img.Bounds().Dy()- // Without the fix, page 2 gets negative y1 → 1x1 output (~100 + gap).- // With fix, proper crop from all 3 pages.if h < 500 {- t.Errorf("multi-page height too small: got %d, want >= 500 (bug: bottomRemaining over-subtraction)", h)+ t.Errorf("multi-page height too small: got %d, want >= 500", h)}t.Logf("multi-page stitch height: %d", h)}diff --git a/internal/deepdoc/parser/pdf/util/geometry.go b/internal/deepdoc/parser/pdf/util/geometry.goindex 65d3f98875..cdd2df25f3 100644--- a/internal/deepdoc/parser/pdf/util/geometry.go+++ b/internal/deepdoc/parser/pdf/util/geometry.go@@ -193,7 +193,29 @@ func RectOverlap(a, b Rect) float64 {// Uses direct Pix slice copy for *image.RGBA sources (zero allocation per row);// falls back to pixel-by-pixel for other image types.func FastCrop(src image.Image, x0, y0, x1, y1 int) *image.RGBA {- // Clamp to source bounds+ b := cropRectBounds(src, x0, y0, x1, y1)+ if b.Empty() {+ return image.NewRGBA(image.Rect(0, 0, 1, 1))+ }+ w, h := b.Dx(), b.Dy()+ dst := image.NewRGBA(image.Rect(0, 0, w, h))+ if rgba, ok := src.(*image.RGBA); ok {+ for y := b.Min.Y; y < b.Max.Y; y++ {+ srcRow := rgba.Pix[rgba.PixOffset(b.Min.X, y):rgba.PixOffset(b.Max.X, y)]+ dstRow := dst.Pix[dst.PixOffset(0, y-b.Min.Y):]+ copy(dstRow, srcRow)+ }+ } else {+ for y := b.Min.Y; y < b.Max.Y; y++ {+ for x := b.Min.X; x < b.Max.X; x++ {+ dst.Set(x-b.Min.X, y-b.Min.Y, src.At(x, y))+ }+ }+ }+ return dst+}++func cropRectBounds(src image.Image, x0, y0, x1, y1 int) image.Rectangle {b := src.Bounds()if x0 < b.Min.X {x0 = b.Min.X@@ -208,25 +230,9 @@ func FastCrop(src image.Image, x0, y0, x1, y1 int) *image.RGBA {y1 = b.Max.Y}if x0 >= x1 || y0 >= y1 {- return image.NewRGBA(image.Rect(0, 0, 1, 1))- }- w, h := x1-x0, y1-y0- dst := image.NewRGBA(image.Rect(0, 0, w, h))- if rgba, ok := src.(*image.RGBA); ok {- for y := y0; y < y1; y++ {- srcRow := rgba.Pix[rgba.PixOffset(x0, y):rgba.PixOffset(x1, y)]- dstRow := dst.Pix[dst.PixOffset(0, y-y0):]- copy(dstRow, srcRow)- }-- } else {- for y := y0; y < y1; y++ {- for x := x0; x < x1; x++ {- dst.Set(x-x0, y-y0, src.At(x, y))- }- }+ return image.Rectangle{}}- return dst+ return image.Rect(x0, y0, x1, y1)}// ── Geometry helpers (pure functions, moved from type/types.go) ─────────diff --git a/internal/ingestion/component/chunker/python_parity_test.go b/internal/ingestion/component/chunker/python_parity_test.goindex cc30f24e70..a94210a1e8 100644--- a/internal/ingestion/component/chunker/python_parity_test.go+++ b/internal/ingestion/component/chunker/python_parity_test.go@@ -111,10 +111,8 @@ func TestChunker_PreservesLayoutField(t *testing.T) {}// TestChunker_PassesPDFPositionsThrough pins the _pdf_positions-// pass-through. The PDF parser path (DeepDOC) attaches bbox-// coordinates under this key; the chunker must forward them-// so downstream layout-aware components can rebuild the page-// geometry.+// pass-through. DeepDOC's PDF parser attaches bounding boxes under this key;+// downstream layout-aware components use them to rebuild page geometry.func TestChunker_PassesPDFPositionsThrough(t *testing.T) {positions := [][]float64{{0.1, 0.2, 0.3, 0.4}, {0.5, 0.6, 0.7, 0.8}}items := []map[string]any{@@ -134,6 +132,28 @@ func TestChunker_PassesPDFPositionsThrough(t *testing.T) {}}+func TestChunker_PreservesVisualParentMetadata(t *testing.T) {+ items := []map[string]any{+ {+ "text": "OCR text from a table image",+ "doc_type_kwd": "image",+ "parent_table_id": "docx-table-1",+ "media_order": 2,+ },+ }+ out := invokeAsTokenChunker(t, items)+ chunks := chunksFromOutput(t, out)+ if len(chunks) == 0 {+ t.Fatal("no chunks emitted")+ }+ if got, want := chunks[0]["parent_table_id"], "docx-table-1"; got != want {+ t.Errorf("chunks[0].parent_table_id = %v, want %v", got, want)+ }+ if got, want := chunks[0]["media_order"], float64(2); got != want {+ t.Errorf("chunks[0].media_order = %v, want %v", got, want)+ }+}+// TestChunker_DropsContentWithWeightOnlyItems enforces the pre-index wire// contract: chunkers consume canonical "text" only.func TestChunker_DropsContentWithWeightOnlyItems(t *testing.T) {diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex a15703104d..279813cd9b 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -830,6 +830,7 @@ func buildChunkDoc(it schema.ChunkDoc, ckType, text, ctxAbove, ctxBelow string)Positions: it.Positions,Image: it.Image,PageNumber: it.PageNumber,+ Extra: cloneExtra(it.Extra),}if ctxAbove != "" {out.ContextAbove = ctxAbove@@ -1642,11 +1643,17 @@ func cloneChunkDoc(in schema.ChunkDoc) schema.ChunkDoc {if in.Positions != nil {out.Positions = append(json.RawMessage(nil), in.Positions...)}- if in.Extra != nil {- out.Extra = make(map[string]json.RawMessage, len(in.Extra))- for k, v := range in.Extra {- out.Extra[k] = append(json.RawMessage(nil), v...)- }+ out.Extra = cloneExtra(in.Extra)+ return out+}++func cloneExtra(in map[string]json.RawMessage) map[string]json.RawMessage {+ if in == nil {+ return nil+ }+ out := make(map[string]json.RawMessage, len(in))+ for key, value := range in {+ out[key] = append(json.RawMessage(nil), value...)}return out}diff --git a/internal/ingestion/component/dispatch_model.go b/internal/ingestion/component/dispatch_model.goindex 0ebe28b4a7..8aad0b2665 100644--- a/internal/ingestion/component/dispatch_model.go+++ b/internal/ingestion/component/dispatch_model.go@@ -47,8 +47,9 @@ var resolveModelConfig = defaultResolveModelConfig// configuredMediaModelID extracts a per-call model reference from a parser setup.// Image parsing stores the VLM model reference in parse_method when it is not-// "ocr" (mirroring Python rag/flow/parser/parser.py:_image). Other media-// families use vlm.llm_id, matching the frontend parser form.+// "ocr" (mirroring Python rag/flow/parser/parser.py:_image); "ocr" selects the+// tenant default vision model, and Go does not run OCR for standalone images.+// Other media families use vlm.llm_id, matching the frontend parser form.func configuredMediaModelID(setup schema.ParserSetup, family string) string {if family == "image" {if ref := getStringOr(setup, "parse_method", ""); ref != "" && !strings.EqualFold(ref, "ocr") {diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex 2b2d068e22..2ba8240b38 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -14,36 +14,17 @@// limitations under the License.//-// Media dispatch: image, audio, video parser branches that require-// model access (OCR, IMAGE2TEXT, SPEECH2TEXT) at the component-// layer. Mirrors Python's _image / _audio / _video methods in-// rag/flow/parser/parser.py and rag/app/picture.py.-//-// These follow the maybeDispatchPDFVision pattern: they bypass-// dispatchParse and call the model directly from the component-// layer, returning a parser.ParseResult.+// Media dispatch: image, audio, and video branches that require model access+// (IMAGE2TEXT, SPEECH2TEXT) at the component layer.package componentimport (- "bytes""context"- "encoding/base64""fmt"- "image"-- // Import image decoders for common formats.- _ "golang.org/x/image/bmp"- _ "golang.org/x/image/tiff"- _ "golang.org/x/image/webp"- _ "image/gif"- _ "image/jpeg"- _ "image/png""os""path/filepath"- "sort""strings"- "unicode/utf8""go.uber.org/zap"@@ -90,13 +71,7 @@ func maybeDispatchVideo(fmt.Errorf("Parser: video parsing is not yet supported; underlying video analysis capability is pending")}-// Image dispatch: OCR + IMAGE2TEXT vision describe ----// Mirrors Python's rag/app/picture.py:chunk() image branch:-// 1. Try PaddleOCR if layout_recognize is "@PaddleOCR"-// 2. Fallback to local ONNX OCR (DeepDoc /predict/ocr endpoint)-// 3. If OCR text is short (≤32 chars or ≤32 English words),-// also call IMAGE2TEXT VLM describe()-// 4. Returns combined text+// Image dispatch: IMAGE2TEXT vision describe ---func maybeDispatchImage(ctx context.Context,@@ -106,6 +81,7 @@ func maybeDispatchImage(binary []byte,inputs map[string]any,setups map[string]schema.ParserSetup,+ enableVisionEnhancement bool,) (parser.ParseResult, bool, error) {if fileType != utility.FileTypeVISUAL {return parser.ParseResult{}, false, nil@@ -115,56 +91,52 @@ func maybeDispatchImage(return parser.ParseResult{}, false, nil}tenantID := getStringOr(inputs, "tenant_id", "")- if tenantID == "" {- return parser.ParseResult{}, true,- fmt.Errorf("parser: image requires tenant_id")- }-- // --- Phase 1: OCR ---- var ocrText string-- // Step 1a: Try PaddleOCR if layout_recognize is set to PaddleOCR.- // Mirrors Python's picture.py:_try_paddleocr_image().- layoutRecognize := getStringOr(setup, "layout_recognize", "")- if layoutRecognize != "" {- recognizer, _ := normalizeLayoutRecognizer(layoutRecognize)- if recognizer == "PaddleOCR" {- if txt, err := runPaddleOCRImage(binary, filename); err == nil && txt != "" {- ocrText = txt+ if !enableVisionEnhancement {+ return parser.ParseResult{}, true, fmt.Errorf("parser: image has no searchable text because vision enhancement is disabled")+ }+ parsed := dispatchParse(ctx, fileType, filename, binary, setups)+ if parsed.Err != nil {+ return parsed, true, parsed.Err+ }+ if len(parsed.JSON) == 0 {+ return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image item")+ }+ imageData, _ := parsed.JSON[0]["image"].(string)+ if imageData == "" {+ return parser.ParseResult{}, true, fmt.Errorf("parser: image parser returned no image payload")+ }+ result, handled, err := maybeDispatchImageVLM(ctx, db, imageData, tenantID, setup, inputs)+ result.Warnings = append(result.Warnings, parsed.Warnings...)+ result.File = parsed.File+ if err == nil && len(result.JSON) > 0 {+ text, _ := result.JSON[0]["text"].(string)+ if strings.TrimSpace(text) == "" {+ if len(result.Warnings) > 0 {+ return result, true, fmt.Errorf("parser: image has no searchable text: %s", result.Warnings[0])}+ return result, true, fmt.Errorf("parser: vision enhancement returned no searchable text")}}+ return result, handled, err+}- // Step 1b: Fallback to local ONNX OCR (DeepDoc /predict/ocr).- // Mirrors Python's picture.py:ocr(np.array(img)) from deepdoc.vision.- if ocrText == "" {- if txt, err := runLocalImageOCR(binary); err == nil && txt != "" {- ocrText = txt- }- }-- // The image family always emits a structured JSON item carrying the- // image attachment (data URI) and doc_type_kwd, mirroring Python- // rag/app/picture.py:71-72 (doc["image"]=img, doc["doc_type_kwd"]=- // "image"). picture.py has no "text" output mode — it always returns- // a structured doc — so output_format is hardcoded to "json" and any- // setup override is ignored. The former behavior returned a bare Text- // string, which dropped the image attachment, set doc_type to "text",- // and on the default json path produced JSON=nil so downstream- // Chunkers rejected the payload with errRequiredField{"json"}.- imageB64 := base64.StdEncoding.EncodeToString(binary)- dataURI := "data:" + imageMIME(filename) + ";base64," + imageB64-- // --- Phase 2: VLM description (when OCR text is short) ---- // Mirrors Python's check: if (eng and len(txt.split()) > 32) or len(txt) > 32- // then use OCR text only; otherwise call cv_mdl.describe().+func maybeDispatchImageVLM(+ ctx context.Context,+ db *gorm.DB,+ dataURI string,+ tenantID string,+ setup schema.ParserSetup,+ inputs map[string]any,+) (parser.ParseResult, bool, error) {+ // --- Optional VLM description ---lang := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))- if vlmGateShouldSkip(ocrText, lang) {- // OCR returned substantial text — skip VLM.- return imageDispatchResult(ocrText, dataURI), true, nil+ if tenantID == "" {+ result := imageDispatchResult("", dataURI)+ result.Warnings = append(result.Warnings, "image VLM enhancement skipped: tenant ID is missing")+ return result, true, nil}- // Short OCR text (or no text): supplement with VLM describe.+ // Use the configured image VLM or the tenant default.modelRef := configuredMediaModelID(setup, "image")var driver modelModule.ModelDrivervar modelName string@@ -180,13 +152,13 @@ func maybeDispatchImage(} else {driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)}+ if err == nil && driver == nil {+ err = fmt.Errorf("no usable vision model")+ }if err != nil {- // If VLM is unavailable, but we have OCR text, return it.- if ocrText != "" {- return imageDispatchResult(ocrText, dataURI), true, nil- }- return parser.ParseResult{}, true,- fmt.Errorf("parser: picture image2text model: %w", err)+ result := imageDispatchResult("", dataURI)+ result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement skipped: model unavailable: %v", err))+ return result, true, nil}prompt := defaultImageVisionPrompt(lang)@@ -206,32 +178,20 @@ func maybeDispatchImage(vision := trueresp, err := driver.ChatWithMessages(ctx, modelName, messages, apiConfig, &modelModule.ChatConfig{Vision: &vision}, nil)if err != nil {- if ocrText != "" {- return imageDispatchResult(ocrText, dataURI), true, nil- }- return parser.ParseResult{}, true,- fmt.Errorf("parser: picture describe: %w", err)+ result := imageDispatchResult("", dataURI)+ result.Warnings = append(result.Warnings, fmt.Sprintf("image VLM enhancement failed: %v", err))+ return result, true, nil}vlmText := ""if resp != nil && resp.Answer != nil {vlmText = strings.TrimSpace(*resp.Answer)}- // Combine OCR + VLM text.- // Mirrors Python: txt += "\n" + ans- combined := ocrText- if vlmText != "" {- if combined != "" {- combined += "\n" + vlmText- } else {- combined = vlmText- }- }- return imageDispatchResult(combined, dataURI), true, nil+ return imageDispatchResult(vlmText, dataURI), true, nil}// imageDispatchResult builds the structured JSON payload for the image-// family: a single item carrying the combined text, the image attachment+// family: a single item carrying the VLM description, the image attachment// (data URI), and doc_type_kwd "image". Mirrors Python// rag/app/picture.py:71-72.func imageDispatchResult(text, dataURI string) parser.ParseResult {@@ -336,59 +296,6 @@ func writeTempAudioFile(filename string, binary []byte) (string, error) {return tmp.Name(), nil}-// normalizeLayoutRecognizer parses layout_recognize strings like-// "model@PaddleOCR" → ("PaddleOCR", "model@PaddleOCR").-// Mirrors Python's common/parser_config_utils.py:normalize_layout_recognizer().-func normalizeLayoutRecognizer(raw string) (recognizer, modelName string) {- lowered := strings.ToLower(raw)- if strings.HasSuffix(lowered, "@paddleocr") {- return "PaddleOCR", raw- }- if strings.HasSuffix(lowered, "@mineru") {- return "MinerU", raw- }- if strings.HasSuffix(lowered, "@somark") {- return "SoMark", raw- }- if strings.HasSuffix(lowered, "@opendataloader") {- return "OpenDataLoader", raw- }- return raw, ""-}--// imageMIME maps common image filename extensions to MIME types-// for constructing base64 data URIs.-func imageMIME(filename string) string {- dot := strings.LastIndex(filename, ".")- if dot == -1 {- return "image/png"- }- switch strings.ToLower(filename[dot+1:]) {- case "jpg", "jpeg":- return "image/jpeg"- case "png":- return "image/png"- case "gif":- return "image/gif"- case "bmp":- return "image/bmp"- case "webp":- return "image/webp"- case "svg":- return "image/svg+xml"- case "tiff", "tif":- return "image/tiff"- case "ico":- return "image/x-icon"- case "avif":- return "image/avif"- case "heic":- return "image/heic"- default:- return "image/png"- }-}-// videoMIME maps common video filename extensions to MIME types// for constructing base64 data URIs. Retained as a reference for the// future real video-parsing implementation (provider-specific frame@@ -422,196 +329,3 @@ func videoMIME(filename string) string {return "video/mp4"}}--// --- OCR helpers for picture dispatch -----// vlmGateShouldSkip determines whether the OCR text is substantial enough-// to skip the secondary VLM description call.-// Mirrors Python picture.py:chunk():-//-// txt = txt.strip()-// if (eng and len(txt.split()) > 32) or len(txt) > 32 -> skip VLM-func vlmGateShouldSkip(ocrText, lang string) bool {- ocrText = strings.TrimSpace(ocrText)- if ocrText == "" {- return false- }- eng := strings.EqualFold(lang, "english")- wordCount := len(strings.Fields(ocrText))- charCount := utf8.RuneCountInString(ocrText)- return (eng && wordCount > 32) || charCount > 32-}--// runPaddleOCRImage tries PaddleOCR remote API for image text extraction.-// Mirrors Python's picture.py:_try_paddleocr_image() which creates a-// PaddleOCRParser and calls parse_image().-func runPaddleOCRImage(binary []byte, filename string) (string, error) {- client := parser.NewPaddleOCRClientFromEnv()- if !client.Enabled() {- return "", fmt.Errorf("paddleocr: not configured (set PADDLEOCR_ACCESS_TOKEN)")- }- return client.ParseImage(binary, filename)-}--// runLocalImageOCR detects and recognizes text in an image using the-// in-process DeepDoc analyzer (ONNX models served locally via the native-// backend). Mirrors Python's deepdoc.vision.OCR local ONNX pipeline; the-// external HTTP service is no longer a backend (see parser.GetDocAnalyzer).-//-// Pipeline:-// 1. Decode image bytes → image.Image-// 2. OCRDetect → find text region boxes-// 3. For each box: crop → OCRRecognize → text-// 4. Sort boxes by Y, then X (reading order)-// 5. Join all recognized text with newlines-func runLocalImageOCR(binary []byte) (string, error) {- analyzer, err := parser.GetDocAnalyzer()- if err != nil {- return "", fmt.Errorf("local OCR: %w", err)- }-- img, _, err := image.Decode(bytes.NewReader(binary))- if err != nil {- return "", fmt.Errorf("local OCR: decode image: %w", err)- }-- // Step 1: Detect text regions.- ctx := context.Background()- boxes, err := analyzer.OCRDetect(ctx, img)- if err != nil {- return "", fmt.Errorf("local OCR: detect: %w", err)- }- if len(boxes) == 0 {- return "", nil- }-- // Step 2: Sort boxes by Y (top to bottom), then X (left to right)- // for reading-order text assembly.- sort.Slice(boxes, func(i, j int) bool {- yi := (boxes[i].Y0 + boxes[i].Y2) / 2- yj := (boxes[j].Y0 + boxes[j].Y2) / 2- if yi < yj {- return true- }- if yi > yj {- return false- }- return boxes[i].X0 < boxes[j].X0- })-- // Step 3: Recognize text per box.- var texts []string- bounds := img.Bounds()- for _, box := range boxes {- // Convert quad box to axis-aligned crop rect.- x0 := int(min4(box.X0, box.X1, box.X2, box.X3))- y0 := int(min4(box.Y0, box.Y1, box.Y2, box.Y3))- x1 := int(max4(box.X0, box.X1, box.X2, box.X3))- y1 := int(max4(box.Y0, box.Y1, box.Y2, box.Y3))-- // Clamp to image bounds.- if x0 < bounds.Min.X {- x0 = bounds.Min.X- }- if y0 < bounds.Min.Y {- y0 = bounds.Min.Y- }- if x1 > bounds.Max.X {- x1 = bounds.Max.X- }- if y1 > bounds.Max.Y {- y1 = bounds.Max.Y- }- if x1 <= x0 || y1 <= y0 {- continue- }-- // Crop the region. This requires an image type that supports- // cropping; for simplicity we recode through a sub-image.- crop := cropImage(img, x0, y0, x1, y1)- if crop == nil {- continue- }-- recTexts, err := analyzer.OCRRecognize(ctx, crop)- if err != nil {- continue // skip boxes that fail recognition- }- for _, t := range recTexts {- s := strings.TrimSpace(t.Text)- if s != "" {- texts = append(texts, s)- }- }- }-- if len(texts) == 0 {- return "", nil- }- return strings.Join(texts, "\n"), nil-}--// cropImage extracts a sub-rectangle from img. Works with any image.Image-// by converting to RGBA if needed, then cropping.-func cropImage(img image.Image, x0, y0, x1, y1 int) image.Image {- bounds := img.Bounds()- cropRect := image.Rect(- bounds.Min.X+x0, bounds.Min.Y+y0,- bounds.Min.X+x1, bounds.Min.Y+y1,- )- switch src := img.(type) {- case *image.RGBA:- return src.SubImage(cropRect)- case *image.NRGBA:- return src.SubImage(cropRect)- case *image.RGBA64:- return src.SubImage(cropRect)- case *image.NRGBA64:- return src.SubImage(cropRect)- case *image.Gray:- return src.SubImage(cropRect)- case *image.Gray16:- return src.SubImage(cropRect)- case *image.YCbCr:- return src.SubImage(cropRect)- case *image.Paletted:- return src.SubImage(cropRect)- default:- // Convert to RGBA for cropping.- rgba := image.NewRGBA(cropRect)- for y := cropRect.Min.Y; y < cropRect.Max.Y; y++ {- for x := cropRect.Min.X; x < cropRect.Max.X; x++ {- rgba.Set(x, y, img.At(x, y))- }- }- return rgba- }-}--func min4(a, b, c, d float64) float64 {- m := a- if b < m {- m = b- }- if c < m {- m = c- }- if d < m {- m = d- }- return m-}--func max4(a, b, c, d float64) float64 {- m := a- if b > m {- m = b- }- if c > m {- m = c- }- if d > m {- m = d- }- return m-}diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex f07e675ce7..c94ac8e784 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -34,6 +34,17 @@ import ("ragflow/internal/utility")+func TestMaybeDispatchImageWithoutVisionReportsNoContent(t *testing.T) {+ result, handled, err := maybeDispatchImage(t.Context(), dao.DB, utility.FileTypeVISUAL,+ "photo.png", []byte("image bytes"), nil, defaultSetups(), false)+ if !handled {+ t.Fatal("image was not handled")+ }+ if err == nil || !strings.Contains(err.Error(), "vision enhancement") {+ t.Fatalf("error = %v, want explicit vision requirement; result = %+v", err, result)+ }+}+// imagePromptCaptureDriver embeds ModelDriver so it satisfies the interface// without listing every method; only ChatWithMessages is overridden to record// the messages the image branch selected.@@ -41,11 +52,13 @@ type imagePromptCaptureDriver struct {modelModule.ModelDrivermu sync.Mutexcaptured []modelModule.Message+ ctxErr error}func (d *imagePromptCaptureDriver) ChatWithMessages(ctx context.Context, modelName string, messages []modelModule.Message, apiConfig *modelModule.APIConfig, chatModelConfig *modelModule.ChatConfig, usage *common.ModelUsage) (*modelModule.ChatResponse, error) {d.mu.Lock()d.captured = append(d.captured, messages...)+ d.ctxErr = ctx.Err()d.mu.Unlock()ans := "captured"return &modelModule.ChatResponse{Answer: &ans}, nil@@ -107,6 +120,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {[]byte("not-a-real-image"),map[string]any{"tenant_id": "t1"},setups,+ true,)if err != nil {t.Fatalf("maybeDispatchImage: %v", err)@@ -158,6 +172,7 @@ func TestMaybeDispatchImage_DefaultPromptUsesDatasetLanguage(t *testing.T) {[]byte("not-a-real-image"),map[string]any{"tenant_id": "t1", "lang": "Japanese"},setups,+ true,)if err != nil {t.Fatalf("maybeDispatchImage: %v", err)@@ -200,6 +215,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {[]byte("not-a-real-image"),map[string]any{"tenant_id": "t1"},setups,+ true,)if err != nil {t.Fatalf("maybeDispatchImage: %v", err)@@ -222,7 +238,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {t.Errorf("image = %q, want a data URI (data:<mime>;base64,<b64>)", img)}if txt, _ := item["text"].(string); txt == "" {- t.Errorf("text field empty; want non-empty combined OCR+VLM text")+ t.Errorf("text field empty; want non-empty VLM description")}}@@ -251,6 +267,7 @@ func TestMaybeDispatchImage_HardcodesJSONOutput(t *testing.T) {[]byte("not-a-real-image"),map[string]any{"tenant_id": "t1"},setups,+ true,)if err != nil {t.Fatalf("maybeDispatchImage: %v", err)@@ -442,6 +459,7 @@ func TestMaybeDispatchImage_UsesConfiguredVLMModel(t *testing.T) {[]byte("not-a-real-image"),map[string]any{"tenant_id": "t1"},setups,+ true,)if err != nil {t.Fatalf("maybeDispatchImage: %v", err)@@ -516,12 +534,12 @@ func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {}// TestImageDecoders_RegisteredFormats validates that image decoders for WebP, BMP,-// TIFF, PNG, JPEG, and GIF are registered by media_dispatch.go and can decode+// TIFF, PNG, JPEG, and GIF are registered by vision_image.go and can decode// their respective binary payloads via image.Decode without importing the decoder// packages directly in the test file.func TestImageDecoders_RegisteredFormats(t *testing.T) {// Fixed binary fixtures for image formats decoded via decoders registered in- // media_dispatch.go (neither standard library nor x/image decoders are imported+ // vision_image.go (neither standard library nor x/image decoders are imported// in this test file).const (webpB64 = "UklGRiQAAABXRUJQVlA4IBgAAAAwAQCdASoBAAEAD8D+JaQAA3AA/ua1AAA="@@ -563,93 +581,3 @@ func TestImageDecoders_RegisteredFormats(t *testing.T) {})}}--// TestVLMGateShouldSkip verifies the rune vs word count threshold-// for skipping VLM description. Specifically:-// - Zero-allocation rune counting via utf8.RuneCountInString correctly handles multi-byte UTF-8.-// - Whitespace trimming matches Python txt.strip(): surrounding whitespace is trimmed before counting.-// - CJK text is measured in unicode runes: 12 CJK characters occupy 36 bytes (>32 bytes)-// but only 12 runes (<=32 runes), so VLM must NOT be skipped.-// - CJK text >32 runes (e.g. 33 runes) skips VLM.-// - CJK exact boundary text (32 runes) triggers VLM.-// - English text >32 words skips VLM.-// - English short text (<=32 words and <=32 chars) triggers VLM.-// - English text with <=32 words but >32 chars skips VLM.-func TestVLMGateShouldSkip(t *testing.T) {- tests := []struct {- name string- lang string- ocrText string- wantSkip bool- }{- {- name: "empty text does not skip",- lang: "Chinese",- ocrText: "",- wantSkip: false,- },- {- name: "whitespace only text does not skip",- lang: "Chinese",- ocrText: " \n\t ",- wantSkip: false,- },- {- name: "CJK substantial text (>32 runes, >32 bytes) skips VLM",- lang: "Chinese",- ocrText: strings.Repeat("中", 33), // 33 runes, 99 bytes- wantSkip: true,- },- {- name: "CJK short text with >32 bytes but <=32 runes triggers VLM",- lang: "Chinese",- ocrText: strings.Repeat("中", 12), // 12 runes, 36 bytes (>32 bytes)- wantSkip: false,- },- {- name: "CJK exact boundary text (32 runes, 96 bytes) triggers VLM",- lang: "Chinese",- ocrText: strings.Repeat("中", 32), // 32 runes, 96 bytes (32 is not > 32)- wantSkip: false,- },- {- name: "CJK exact boundary text with whitespace padding trims to <=32 runes and triggers VLM",- lang: "Chinese",- ocrText: " " + strings.Repeat("中", 32) + " ", // 32 runes after trim- wantSkip: false,- },- {- name: "CJK substantial text with whitespace padding trims to >32 runes and skips VLM",- lang: "Chinese",- ocrText: " " + strings.Repeat("中", 33) + " ", // 33 runes after trim- wantSkip: true,- },- {- name: "English substantial text (>32 words) skips VLM",- lang: "English",- ocrText: strings.Repeat("word ", 33), // 33 words- wantSkip: true,- },- {- name: "English short text (<=32 words and <=32 chars) triggers VLM",- lang: "English",- ocrText: "hello world", // 2 words, 11 chars- wantSkip: false,- },- {- name: "English text with <=32 words but >32 chars skips VLM",- lang: "English",- ocrText: "abcdefghijklmnopqrstuvwxyz01234567", // 1 word, 34 chars (>32 chars)- wantSkip: true,- },- }-- for _, tc := range tests {- t.Run(tc.name, func(t *testing.T) {- got := vlmGateShouldSkip(tc.ocrText, tc.lang)- if got != tc.wantSkip {- t.Errorf("vlmGateShouldSkip(%q, %q) = %v, want %v", tc.ocrText, tc.lang, got, tc.wantSkip)- }- })- }-}diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex f44cd62f5e..72dd2cdd34 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -110,7 +110,8 @@ const pageFormFeed = '\f'// the goroutine that returned from Invoke. The static Param is// read-only after construction.type ParserComponent struct {- setups map[string]schema.ParserSetup+ setups map[string]schema.ParserSetup+ enableVisionEnhancement bool}// NewParserComponent constructs a Parser from a DSL param map.@@ -123,6 +124,7 @@ type ParserComponent struct {// Param map shape (all keys optional)://// {+// "enable_vision_enhancement": bool,// "pdf": map[string]any,// "docx": map[string]any,// ...@@ -139,8 +141,16 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {// Canvases saved by the Python-era frontend nest the per-family setups// under a "setups" key; lift them so every family lands at the top level.params = schema.FlattenLegacyParserSetups(params)+ var enableVisionEnhancement bool+ if raw, exists := params["enable_vision_enhancement"]; exists {+ var ok bool+ enableVisionEnhancement, ok = raw.(bool)+ if !ok {+ return nil, errors.New("parser: enable_vision_enhancement must be a boolean")+ }+ }for k, raw := range params {- if k == "outputs" || k == "allowed_output_format" {+ if k == "outputs" || k == "allowed_output_format" || k == "enable_vision_enhancement" {continue}ftCfg, ok := raw.(map[string]any)@@ -155,7 +165,7 @@ func NewParserComponent(params map[string]any) (runtime.Component, error) {}}normalizeParserOutputFormats(s)- pc := &ParserComponent{setups: s}+ pc := &ParserComponent{setups: s, enableVisionEnhancement: enableVisionEnhancement}if err := pc.Check(); err != nil {return nil, fmt.Errorf("parser: %w", err)}@@ -206,11 +216,8 @@ func cloneParserSetupValue(value any) any {}}-// Check mirrors the applicable subset of Python ParserParam.check()-// (rag/flow/parser/parser.py:251-321). Runs at construction time so-// a malformed DSL surfaces as a canvas compile failure rather than a-// mid-run error. Returns the first validation error encountered-// (Python raises ValueError on the first failure).+// Check validates parser methods at construction time so a malformed DSL+// surfaces as a canvas compile failure rather than a mid-run error.//// NOT covered here (intentional):// - audio/video vlm.llm_id: Python's check() does not validate it@@ -233,11 +240,13 @@ func (c *ParserComponent) Check() error {}}}- // image family (parser.py:283-287).+ // image family (parser.py:283-287). The legacy "ocr" setup value selects+ // the tenant default vision model; image OCR is not run by this path.if img, ok := c.setups["image"]; ok {pm, _ := img["parse_method"].(string)- // OCR mode does not need a VLM language; any other value does.- if pm != "ocr" {+ // A model selected for optional image enhancement needs a language+ // only when enhancement is enabled.+ if c.enableVisionEnhancement && pm != "ocr" {if lang, _ := img["lang"].(string); lang == "" {return errors.New("image VLM language does not support empty value")}@@ -472,9 +481,9 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st}var handledImage boolif !handledVision && !handledMedia {- // Image/Picture dispatch: OCR + IMAGE2TEXT vision describe.+ // Image/Picture dispatch: optional IMAGE2TEXT vision description.// Mirrors Python's rag/app/picture.py:chunk() image branch.- dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups)+ dispatched, handledImage, visionErr = maybeDispatchImage(ctx, db, fileTypeExt, filename, binary, inputs, setups, c.enableVisionEnhancement)if visionErr != nil {return nil, visionErr}@@ -491,14 +500,11 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[stif !handledVision && !handledMedia && !handledImage && !handledAudio {dispatched = dispatchParse(ctx, fileTypeExt, filename, binary, setups)- // Vision figure enhancement: on the JSON output path,- // append vision-model descriptions to embedded image and- // table items. Mirrors Python's enhance_media_sections_with_vision- // (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).- // Errors (including context cancellation) are intentionally- // discarded — enhancement is best-effort, matching Python's- // try/except pass pattern.- dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)+ if c.enableVisionEnhancement {+ // Enhancement is optional; parser-provided text and image metadata+ // remain available if a vision model cannot describe an image.+ dispatched, _, _ = maybeDispatchVisionEnhancement(ctx, db, fileTypeExt, dispatched, inputs, setups)+ }}// Known/supported families must fail loudly when dispatch or// parsing breaks. Only unknown families keep the raw-text fallback.diff --git a/internal/ingestion/component/parser_check_test.go b/internal/ingestion/component/parser_check_test.goindex f3f4b8c433..afffa883c0 100644--- a/internal/ingestion/component/parser_check_test.go+++ b/internal/ingestion/component/parser_check_test.go@@ -23,8 +23,8 @@ import ("ragflow/internal/ingestion/component/schema")-// TestParserComponent_Check covers the construction-time business-// validation for parser methods that require language configuration.+// TestParserComponent_Check covers construction-time validation for parser+// methods that require language configuration.//// audio/video vlm.llm_id is not validated: Python's check() has no// such branch, and audio resolves its model at dispatch time with a@@ -33,6 +33,7 @@ func TestParserComponent_Check(t *testing.T) {cases := []struct {name stringsetups map[string]schema.ParserSetup+ enhance boolwantErr string // non-empty substring expected in error; empty means no error}{// --- PDF family (parser.py:252-261) ---@@ -96,6 +97,7 @@ func TestParserComponent_Check(t *testing.T) {{name: "image: non-ocr without lang → error",setups: map[string]schema.ParserSetup{"image": {"parse_method": "vlm_xyz", "lang": ""}},+ enhance: true,wantErr: "image VLM language",},{@@ -138,7 +140,7 @@ func TestParserComponent_Check(t *testing.T) {for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- c := &ParserComponent{setups: tc.setups}+ c := &ParserComponent{setups: tc.setups, enableVisionEnhancement: tc.enhance}err := c.Check()if tc.wantErr != "" {if err == nil {@@ -183,3 +185,28 @@ func TestParserComponent_New_RunsCheck(t *testing.T) {t.Errorf("want nil component on error, got %T", c)}}++func TestNewParserComponentVisionEnhancementSetting(t *testing.T) {+ for _, test := range []struct {+ name string+ params map[string]any+ enabled bool+ }{+ {name: "absent", params: nil},+ {name: "explicit false", params: map[string]any{"enable_vision_enhancement": false}},+ {name: "explicit true", params: map[string]any{"enable_vision_enhancement": true}, enabled: true},+ } {+ t.Run(test.name, func(t *testing.T) {+ component, err := NewParserComponent(test.params)+ if err != nil {+ t.Fatalf("NewParserComponent: %v", err)+ }+ if got := component.(*ParserComponent).enableVisionEnhancement; got != test.enabled {+ t.Fatalf("enableVisionEnhancement = %v, want %v", got, test.enabled)+ }+ })+ }+ if _, err := NewParserComponent(map[string]any{"enable_vision_enhancement": "true"}); err == nil || !strings.Contains(err.Error(), "must be a boolean") {+ t.Fatalf("string switch error = %v, want boolean validation", err)+ }+}diff --git a/internal/ingestion/component/schema/schema_test.go b/internal/ingestion/component/schema/schema_test.goindex 14e5088708..dd53b20ced 100644--- a/internal/ingestion/component/schema/schema_test.go+++ b/internal/ingestion/component/schema/schema_test.go@@ -280,6 +280,33 @@ func TestChunkDocLegacyRowIRKeysPassThrough(t *testing.T) {}}+func TestChunkDocVisualParentMetadataRoundTrip(t *testing.T) {+ imageItem := map[string]any{+ "text": "caption",+ "doc_type_kwd": "image",+ "image": "aGVsbG8=",+ "parent_table_id": "docx-table-1",+ "row_index": 2,+ "column_index": 3,+ "media_order": 4,+ }+ doc, err := ChunkDocFromMap(imageItem)+ if err != nil {+ t.Fatalf("ChunkDocFromMap: %v", err)+ }+ got := doc.ToMap()+ for key, want := range map[string]any{+ "parent_table_id": "docx-table-1",+ "row_index": float64(2),+ "column_index": float64(3),+ "media_order": float64(4),+ } {+ if got[key] != want {+ t.Errorf("metadata %q = %v, want %v", key, got[key], want)+ }+ }+}+func TestChunkerOutputsJSONRoundTrip(t *testing.T) {original := ChunkerOutputs{OutputFormat: PayloadFormatChunks,diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex ba36f432a0..3f7011b83b 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -14,9 +14,9 @@// limitations under the License.//-// Package component — vision figure enhancement: enriches parsed JSON items-// (PDF / DOCX / Markdown, OutputFormat=="json") with vision-model descriptions-// of embedded images/tables. Mirrors Python's enhance_media_sections_with_vision+// Package component — vision media enhancement: enriches parsed JSON items+// with vision-model descriptions of embedded images/tables.+// Mirrors Python's enhance_media_sections_with_vision// (rag/flow/parser/utils.py:162, called at parser.py:772/978/1115).package component@@ -26,6 +26,8 @@ import ("encoding/base64""errors""fmt"+ "image"+ "io""os""path/filepath""strings"@@ -48,6 +50,7 @@ import (var (figureVisionPromptBuilder func(language string) (string, error) = buildFigureVisionPromptvisionChatInvoker = defaultVisionChatInvoker+ visionImageCropperFactory = newVisionImageCropper)const (@@ -56,7 +59,8 @@ const (// visionChatTimeout bounds a single VLM call so a hung endpoint cannot// occupy one of the concurrency slots indefinitely. Python wraps the// per-image call in @timeout(30, 3) (deepdoc/parser/figure_parser.py).- visionChatTimeout = 30 * time.Second+ visionChatTimeout = 30 * time.Second+ visionMediaItemBudget = 10 * time.Second)var (@@ -98,60 +102,63 @@ func isUsableVisionImage(raw string) bool {if raw == "" {return false}+ if len(raw) > maxVLMEncodedBytes+256 {+ return false+ }if strings.HasPrefix(raw, "data:image/") {idx := strings.Index(raw, "base64,")if idx < 0 {return false}- return isValidBase64(raw[idx+len("base64,"):])+ return isValidBase64(raw[idx+len("base64,"):], false)}if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {return true}- cleaned := strings.Map(func(r rune) rune {- if r == '\r' || r == '\n' || r == ' ' || r == '\t' {- return -1- }- return r- }, raw)- return isValidBase64(cleaned)-}--func isValidBase64(s string) bool {- if s == "" {+ if len(raw) > maxVLMEncodedBytes {return false}- if _, err := base64.StdEncoding.DecodeString(s); err == nil {- return true- }- _, err := base64.RawStdEncoding.DecodeString(s)- return err == nil+ return isValidBase64(raw, true)}-// isVisionEnhancementAllowed mirrors Python's 3 call sites in-// rag/flow/parser/parser.py:772/978/1115 (PDF/DOCX/Markdown JSON branches).-func isVisionEnhancementAllowed(fileType utility.FileType) bool {- switch fileType {- case utility.FileTypePDF, utility.FileTypeDOCX, utility.FileTypeMarkdown:- return true- default:+func isValidBase64(s string, stripWhitespace bool) bool {+ if s == "" || len(s) > maxVLMEncodedBytes {return false}+ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {+ var source io.Reader = strings.NewReader(s)+ if stripWhitespace {+ source = &visionPayloadReader{source: s}+ }+ decoded := base64.NewDecoder(encoding, source)+ n, err := io.CopyN(io.Discard, decoded, int64(maxVisionImageBytes)+1)+ if n > int64(maxVisionImageBytes) {+ return false+ }+ if err == io.EOF {+ return n > 0+ }+ }+ return false}-// visionImageCropper yields a vision-usable base64 image for a parsed item.-// Under cgo it crops on demand from the source PDF when the item carries-// positions but no inlined image; under !cgo it returns the inlined image-// (the only form available without a native renderer). Close releases any-// re-acquired engine so native handles are not leaked.+// visionImageCropper materializes the VLM payload for a parsed item. Under+// cgo it crops PDF sections on demand; inline images stay encoded. Close+// releases any re-acquired native engine.type visionImageCropper interface {- Crop(item map[string]any) (string, error)+ Crop(ctx context.Context, item map[string]any) (*visionImage, error)Close() error}-// maybeDispatchVisionEnhancement enriches parsed JSON items with vision-model-// descriptions of embedded images and tables (doc_type_kwd in {"image", "table"}-// with non-empty image field).+type visionImage struct {+ Raster image.Image+ VLMData string+ VLMDataValidated bool+}++// maybeDispatchVisionEnhancement appends VLM descriptions to parsed image+// resources and table regions. PDF OCR remains in the PDF parser; non-PDF+// image text comes from this optional enhancement.// Mirrors Python's enhance_media_sections_with_vision in rag/flow/parser/utils.py:162.func maybeDispatchVisionEnhancement(ctx context.Context,@@ -161,159 +168,230 @@ func maybeDispatchVisionEnhancement(inputs map[string]any,setups map[string]schema.ParserSetup,) (parser.ParseResult, bool, error) {- // 0. FileType allowlist guard.- if !isVisionEnhancementAllowed(fileType) {- return dispatched, false, nil- }// Only enhance successful JSON output format containing items.if dispatched.Err != nil || dispatched.OutputFormat != "json" || len(dispatched.JSON) == 0 {return dispatched, false, nil}tenantID := getStringOr(inputs, "tenant_id", "")- if tenantID == "" {- return dispatched, false, nil- }- // Language priority mirrors Python's enhance_media_sections_with_vision- // (rag/flow/parser/parser.py:778): the run-level dataset language first- // (the Parser pulls it into inputs from Globals), then the family setup's- // lang, then English.family := resolveParserFamily(fileType)setup := setups[family]language := resolveVisionLanguage(inputs, getStringOr(setup, "lang", ""))- // 1. Collect target items (images/tables that can be described). A target- // may carry an inlined image (docx/markdown, or any pre-inlined source) or,- // under cgo for PDF, only PDF positions — in which case the cropper- // re-acquires the source PDF and crops on demand (the parser no longer- // inlines PDF media). The cropper transparently handles both forms.- type target struct {- idx int- }- var targets []target+ // Collect visual resources, including Markdown images whose type is text+ // because flatten_media_to_text is enabled.+ var items []intfor i, item := range dispatched.JSON {- kd, _ := item["doc_type_kwd"].(string)- if kd != "image" && kd != "table" {- continue- }if img, _ := item["image"].(string); img != "" {- targets = append(targets, target{idx: i})+ items = append(items, i)continue}- if _, ok := parser.ExtractPDFPositions(item); ok {- targets = append(targets, target{idx: i})+ kd, _ := item["doc_type_kwd"].(string)+ if kd == "image" || kd == "table" {+ if _, ok := parser.ExtractPDFPositions(item); ok {+ items = append(items, i)+ }}}- if len(targets) == 0 {+ if len(items) == 0 {return dispatched, false, nil}-- // Acquire the on-demand cropper (cgo: crops from storage; !cgo: returns- // the inlined image). Best-effort: a failure here means no vision- // enhancement, matching Python's try/except pass.- cropper, cerr := newVisionImageCropper(ctx, db, inputs)- if cerr != nil {- return dispatched, false, nil- }- defer cropper.Close()-- // 2. Resolve the per-call IMAGE2TEXT model, then fall back to the tenant- // default. Mirror Python's vlm_conf["llm_id"] preference.- modelRef := configuredMediaModelID(setup, family)+ // Resolve VLM before materializing image payloads, so disabled or+ // unavailable enhancement does not spend time loading media.var driver modelModule.ModelDrivervar modelName stringvar apiConfig *modelModule.APIConfig- var err error- if modelRef != "" {- driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)- if err != nil {- common.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",- zap.String("family", family), zap.String("modelRef", modelRef), zap.String("tenant", tenantID), zap.Error(err))+ var prompt string+ vlmReady := false+ var resolveErr error+ if tenantID != "" {+ modelRef := configuredMediaModelID(setup, family)+ var err error+ if modelRef != "" {+ driver, modelName, apiConfig, _, err = resolveModelConfig(ctx, db, tenantID, entity.ModelTypeImage2Text, modelRef)+ if err != nil {+ common.Warn("vision enhancement: per-call VLM resolve failed, falling back to tenant default",+ zap.String("family", family), zap.String("modelRef", modelRef), zap.String("tenant", tenantID), zap.Error(err))+ driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ }+ } else {driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)}- } else {- driver, modelName, apiConfig, _, err = resolveTenantModelByType(ctx, db, tenantID, entity.ModelTypeImage2Text)+ if err == nil {+ prompt, err = figureVisionPromptBuilder(language)+ vlmReady = err == nil && driver != nil+ }+ resolveErr = err}- if err != nil {- // Model not available — skip vision enhancement silently, matching Python's try/except pass.+ if !vlmReady {+ warning := "vision enhancement skipped: tenant ID is missing"+ if tenantID != "" {+ warning = "vision enhancement skipped: no usable vision model"+ if resolveErr != nil {+ warning = fmt.Sprintf("vision enhancement skipped: %v", resolveErr)+ }+ }+ dispatched.Warnings = append(dispatched.Warnings, warning)return dispatched, false, nil}- // Hoist prompt once: language is invariant across items.- prompt, perr := figureVisionPromptBuilder(language)- if perr != nil {- //nolint:nilerr // Vision enhancement is best-effort.+ // Materialize one resource at a time. The VLM semaphore is acquired before+ // materialization, so at most visionEnhancementConcurrency encoded payloads+ // remain live while model requests run.+ cropper, cerr := visionImageCropperFactory(ctx, db, inputs)+ if cerr != nil {+ dispatched.Warnings = append(dispatched.Warnings, fmt.Sprintf("vision enhancement skipped: %v", cerr))return dispatched, false, nil}-- // 3. Concurrently invoke VLM — acquire semaphore before launching goroutine- // so live goroutine count is bounded by visionEnhancementConcurrency.- // Stop scheduling new VLM calls after cancellation; already-running calls- // finish via wg.Wait() and cancellation propagates via ctx through- // visionChatInvoker.- descriptions := make([]string, len(targets))+ defer cropper.Close()+ modified := false+ descriptions := make([]string, len(items))+ failures := make([]string, len(items))var wg sync.WaitGroupsem := make(chan struct{}, visionEnhancementConcurrency)--dispatch:- for slot, tg := range targets {- if ctx.Err() != nil {- break dispatch+ for slot, itemIdx := range items {+ if err := ctx.Err(); err != nil {+ break}+ vlmSlot := falseselect {case sem <- struct{}{}:+ vlmSlot = truecase <-ctx.Done():- break dispatch+ // The loop exits below when no slot was acquired.}- wg.Add(1)- go func(slot int, itemIdx int) {- defer wg.Done()- defer func() { <-sem }()-- img, ierr := cropper.Crop(dispatched.JSON[itemIdx])- if ierr != nil || img == "" {+ if !vlmSlot {+ break+ }+ var releaseSlotOnce sync.Once+ releaseSlot := func() {+ releaseSlotOnce.Do(func() { <-sem })+ }+ var resource *visionImage+ func() {+ itemCtx, cancelItem := context.WithTimeout(ctx, visionMediaItemBudget)+ defer cancelItem()+ release, err := parser.AcquireImageMedia(itemCtx)+ if err != nil {+ if ctx.Err() == nil && vlmReady {+ if payload, _ := dispatched.JSON[itemIdx]["image"].(string); payload != "" {+ resource = &visionImage{VLMData: payload}+ }+ }+ if resource == nil && ctx.Err() == nil {+ failures[slot] = fmt.Sprintf("media admission failed: %v", err)+ }return}- if !isUsableVisionImage(img) {- common.Warn("vision enhancement: invalid image data skipped",- zap.Int("item", itemIdx))+ defer release()+ item := dispatched.JSON[itemIdx]+ resource, err = cropper.Crop(itemCtx, item)+ if err != nil || resource == nil {+ cropErr := err+ if ctx.Err() == nil && vlmReady {+ if payload, _ := item["image"].(string); isUsableVisionImage(payload) {+ resource = &visionImage{VLMData: payload, VLMDataValidated: true}+ }+ }+ if resource == nil && ctx.Err() == nil {+ if cropErr != nil {+ failures[slot] = fmt.Sprintf("image materialization failed: %v", cropErr)+ } else {+ failures[slot] = "no usable image payload"+ }+ }return}- messages := buildVisionMessages(prompt, img)+ if ctx.Err() == nil && vlmReady && resource.VLMData == "" && resource.Raster != nil {+ resource.VLMData, err = encodeVisionRaster(resource.Raster)+ if err != nil {+ resource.VLMData = ""+ failures[slot] = fmt.Sprintf("image encoding failed: %v", err)+ } else {+ resource.VLMDataValidated = resource.VLMData != ""+ }+ }+ }()+ if err := ctx.Err(); err != nil {+ releaseSlot()+ break+ }+ if resource == nil {+ releaseSlot()+ continue+ }+ if !resource.VLMDataValidated && !isUsableVisionImage(resource.VLMData) {+ if failures[slot] == "" {+ failures[slot] = "image payload is invalid or exceeds configured limits"+ }+ releaseSlot()+ continue+ }+ wg.Add(1)+ go func(slot int, imageData string) {+ defer wg.Done()+ defer releaseSlot()++ messages := buildVisionMessages(prompt, imageData)if len(messages) == 0 {+ failures[slot] = "could not build a VLM request"return}resp, ierr := visionChatInvoker(ctx, driver, modelName, messages, apiConfig)if ierr != nil {+ failures[slot] = fmt.Sprintf("VLM request failed: %v", ierr)return}descriptions[slot] = extractVisionAnswer(resp)- }(slot, tg.idx)+ if strings.TrimSpace(descriptions[slot]) == "" {+ failures[slot] = "VLM returned an empty description"+ }+ }(slot, resource.VLMData)}wg.Wait()if err := ctx.Err(); err != nil {- return dispatched, false, err+ return dispatched, modified, err+ }+ failedItems := 0+ firstFailure := ""+ for _, failure := range failures {+ if failure == "" {+ continue+ }+ failedItems+++ if firstFailure == "" {+ firstFailure = failure+ }+ }+ if failedItems > 0 {+ warning := fmt.Sprintf("vision enhancement failed for %d item(s): %s", failedItems, firstFailure)+ dispatched.Warnings = append(dispatched.Warnings, warning)+ common.Warn("vision enhancement: one or more image descriptions failed",+ zap.Int("failed_items", failedItems), zap.String("first_failure", firstFailure))}- // 4. Append descriptions to item text (single newline \n, matching Python).- modified := false- for slot, tg := range targets {+ // Append descriptions after parser-provided text and captions.+ for slot, itemIdx := range items {desc := strings.TrimSpace(descriptions[slot])if desc == "" {continue}- existing, _ := dispatched.JSON[tg.idx]["text"].(string)- if existing != "" {- dispatched.JSON[tg.idx]["text"] = existing + "\n" + desc- } else {- dispatched.JSON[tg.idx]["text"] = desc- }+ appendItemText(dispatched.JSON[itemIdx], desc)modified = true}return dispatched, modified, nil}+func appendItemText(item map[string]any, text string) {+ existing, _ := item["text"].(string)+ if existing == "" {+ item["text"] = text+ return+ }+ item["text"] = existing + "\n" + text+}+func buildFigureVisionPrompt(language string) (string, error) {template, err := loadFigureVisionPromptFile(figureVisionPromptFile)if err != nil {diff --git a/internal/ingestion/component/vision_enhancement_cgo.go b/internal/ingestion/component/vision_enhancement_cgo.goindex f38674bddb..c7551421c9 100644--- a/internal/ingestion/component/vision_enhancement_cgo.go+++ b/internal/ingestion/component/vision_enhancement_cgo.go@@ -20,9 +20,14 @@ package componentimport ("context"+ "errors""image"+ "math""sync"+ "time"+ "go.uber.org/zap"+ "ragflow/internal/common"deepdocpdf "ragflow/internal/deepdoc/parser/pdf""ragflow/internal/deepdoc/parser/pdf/util"deepdoctype "ragflow/internal/deepdoc/parser/type"@@ -40,6 +45,14 @@ var (visionEngineOpener func(data []byte) (deepdoctype.PDFEngine, error))+const visionPDFSourceBudget = 90 * time.Second++type visionPageSizer interface {+ PageSize(pageNum int) (float64, float64, error)+}++var _ visionPageSizer = (*deepdocpdf.PDFOxideEngine)(nil)+func init() {visionSourceFetcher = FetchBinaryvisionEngineOpener = deepdocpdf.NewEngine@@ -53,43 +66,72 @@ func init() {// chunker does at index time. The engine is opened lazily and at most once per// vision-enhancement call, then released via Close.type visionPDFCropper struct {- ctx context.Context- db *gorm.DB- inputs map[string]any+ invocationCtx context.Context+ sourceCtx context.Context+ cancelSource context.CancelFunc+ db *gorm.DB+ inputs map[string]any- once sync.Once- engine deepdoctype.PDFEngine- engErr error+ mu sync.Mutex+ initialized bool+ engine deepdoctype.PDFEngine+ engErr error+ sourceErr error}// newVisionImageCropper builds the on-demand cropper. It never touches storage// here; the source PDF is re-acquired lazily on the first Crop call that needs// it.func newVisionImageCropper(ctx context.Context, db *gorm.DB, inputs map[string]any) (visionImageCropper, error) {- return &visionPDFCropper{ctx: ctx, db: db, inputs: inputs}, nil+ if ctx == nil {+ ctx = context.Background()+ }+ sourceCtx, cancelSource := context.WithTimeout(ctx, visionPDFSourceBudget)+ return &visionPDFCropper{+ invocationCtx: ctx,+ sourceCtx: sourceCtx,+ cancelSource: cancelSource,+ db: db,+ inputs: inputs,+ }, nil}-func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {+func (c *visionPDFCropper) Crop(ctx context.Context, item map[string]any) (*visionImage, error) {+ if err := ctx.Err(); err != nil {+ return nil, err+ }// Fast path: an inlined image (docx/markdown, or any pre-inlined source)// is used directly — no storage access, no engine.if img, _ := item["image"].(string); img != "" {- return img, nil+ return materializeInlineVisionImage(img)}matrix, ok := parser.ExtractPDFPositions(item)if !ok {- return "", nil+ return nil, nil}positions := util.PositionsFromMatrix(matrix)if len(positions) == 0 {- return "", nil+ return nil, nil}if err := c.ensureEngine(); err != nil {// Best-effort: a missing/unreadable source PDF means no vision// description for this item, not a hard failure.- return "", nil+ return nil, nil+ }+ if err := c.invocationCtx.Err(); err != nil {+ return nil, err+ }+ // Source acquisition has its own invocation-level budget and can outlive+ // this item's materialization deadline. Once the shared source is ready,+ // give rendering a fresh per-item budget while still honoring invocation+ // cancellation.+ if err := ctx.Err(); err != nil && !errors.Is(err, context.DeadlineExceeded) {+ return nil, err}+ renderCtx, cancelRender := context.WithTimeout(c.invocationCtx, visionMediaItemBudget)+ defer cancelRender()if c.engine == nil {- return "", nil+ return nil, nil}// Render each distinct page the positions span (1-based → 0-based is// handled by PositionsFromMatrix). Reuse the chunker's sliding-window@@ -100,8 +142,14 @@ func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {pages[pn] = struct{}{}}}+ if !pdfPagesRasterWithinVisionLimits(c.engine, pages) {+ return nil, nil+ }single := make(map[int]image.Image, len(pages))for pn := range pages {+ if err := renderCtx.Err(); err != nil {+ return nil, err+ }img, rerr := deepdocpdf.RenderPageToImage(c.engine, pn)if rerr != nil || img == nil {continue@@ -109,50 +157,116 @@ func (c *visionPDFCropper) Crop(item map[string]any) (string, error) {single[pn] = img}if len(single) == 0 {- return "", nil+ return nil, nil+ }+ raster := util.CropSectionPositionsRasterLimited(positions, single, deepdoctype.DlaScale, maxVisionImagePixels)+ if raster == nil {+ return nil, nil}- return util.CropSectionPositions(positions, single, deepdoctype.DlaScale), nil+ return &visionImage{Raster: raster}, nil}-func (c *visionPDFCropper) ensureEngine() error {- c.once.Do(func() {- data, err := c.acquireSource()- if err != nil || len(data) == 0 {- c.engErr = err- return+func pdfPagesRasterWithinVisionLimits(engine deepdoctype.PDFEngine, pageNums map[int]struct{}) bool {+ sizer, ok := engine.(visionPageSizer)+ if !ok {+ return false+ }+ var totalPixels int64+ for pageNum := range pageNums {+ widthPoints, heightPoints, err := sizer.PageSize(pageNum)+ if err != nil {+ return false}- // Only PDFs can be cropped. Guard against other binary types so a- // docx/markdown item that happens to reach here stays a no-op.- if len(data) < 5 || string(data[:5]) != "%PDF-" {- return+ widthPixels := math.Ceil(widthPoints * deepdoctype.DlaScale)+ heightPixels := math.Ceil(heightPoints * deepdoctype.DlaScale)+ if math.IsNaN(widthPixels) || math.IsInf(widthPixels, 0) ||+ math.IsNaN(heightPixels) || math.IsInf(heightPixels, 0) ||+ widthPixels <= 0 || heightPixels <= 0 ||+ widthPixels > maxVisionImageEdge || heightPixels > maxVisionImageEdge ||+ widthPixels*heightPixels > float64(maxVisionImagePixels) {+ return false}- eng, oerr := visionEngineOpener(data)- if oerr != nil {- c.engErr = oerr- return+ pagePixels := int64(widthPixels * heightPixels)+ if pagePixels > maxVisionImagePixels-totalPixels {+ return false}- c.engine = eng- })+ totalPixels += pagePixels+ }+ return true+}++func (c *visionPDFCropper) ensureEngine() error {+ c.mu.Lock()+ defer c.mu.Unlock()+ if c.initialized {+ return c.engErr+ }+ if err := c.sourceCtx.Err(); err != nil {+ c.initialized = true+ c.engErr = err+ c.sourceErr = err+ return err+ }+ data, err := c.acquireSource(c.sourceCtx)+ if err != nil {+ c.sourceErr = err+ if c.sourceCtx.Err() != nil {+ c.initialized = true+ c.engErr = err+ } else if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {+ c.initialized = true+ c.engErr = err+ }+ return err+ }+ if err := c.sourceCtx.Err(); err != nil {+ c.initialized = true+ c.engErr = err+ c.sourceErr = err+ return err+ }+ c.sourceErr = nil+ // Empty and non-PDF sources are deterministic no-op results for this+ // invocation. A transient fetch error can be retried by a later item while+ // the shared source context remains active.+ c.initialized = true+ if len(data) == 0 || len(data) < 5 || string(data[:5]) != "%PDF-" {+ return nil+ }+ eng, err := visionEngineOpener(data)+ if err != nil {+ c.engErr = err+ c.sourceErr = err+ return err+ }+ c.engine = engreturn c.engErr}-func (c *visionPDFCropper) acquireSource() ([]byte, error) {+func (c *visionPDFCropper) acquireSource(ctx context.Context) ([]byte, error) {if bucket, _ := getString(c.inputs, "bucket"); bucket != "" {if path, _ := getString(c.inputs, "path"); path != "" {- return visionSourceFetcher(c.ctx, bucket, path)+ return visionSourceFetcher(ctx, bucket, path)}}if docID, _ := getString(c.inputs, "doc_id"); docID != "" {- ref, err := ResolveDocumentStorage(c.ctx, c.db, docID)+ ref, err := ResolveDocumentStorage(ctx, c.db, docID)if err != nil || ref == nil {return nil, err}- return visionSourceFetcher(c.ctx, ref.Bucket, ref.Path)+ return visionSourceFetcher(ctx, ref.Bucket, ref.Path)}return nil, nil}func (c *visionPDFCropper) Close() error {+ c.mu.Lock()+ defer c.mu.Unlock()+ c.cancelSource()+ if c.sourceErr != nil && c.invocationCtx.Err() == nil {+ common.Warn("vision enhancement: source PDF unavailable for VLM crop", zap.Error(c.sourceErr))+ c.sourceErr = nil+ }if c.engine != nil {return c.engine.Close()}diff --git a/internal/ingestion/component/vision_enhancement_cgo_test.go b/internal/ingestion/component/vision_enhancement_cgo_test.goindex 6940f473f0..d1cc4d5cd5 100644--- a/internal/ingestion/component/vision_enhancement_cgo_test.go+++ b/internal/ingestion/component/vision_enhancement_cgo_test.go@@ -6,6 +6,7 @@ import ("context""image""testing"+ "time"deepdoctype "ragflow/internal/deepdoc/parser/type")@@ -13,7 +14,10 @@ import (// mockVisionEngine returns a fixed image for any rendered page so the// on-demand cropper can exercise CropSectionPositions without a real PDF.type mockVisionEngine struct {- closed *bool+ closed *bool+ pageWidth float64+ pageHeight float64+ renderCalls *int}func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {@@ -21,8 +25,21 @@ func (m mockVisionEngine) ExtractChars(int) ([]deepdoctype.TextChar, error) {}func (m mockVisionEngine) RenderPage(int, float64) ([]byte, error) { return nil, nil }func (m mockVisionEngine) RenderPageImage(int, float64) (image.Image, error) {+ if m.renderCalls != nil {+ *m.renderCalls+++ }return image.NewRGBA(image.Rect(0, 0, 1000, 1000)), nil}+func (m mockVisionEngine) PageSize(int) (float64, float64, error) {+ width, height := m.pageWidth, m.pageHeight+ if width == 0 {+ width = 333+ }+ if height == 0 {+ height = 333+ }+ return width, height, nil+}func (m mockVisionEngine) RawData() []byte { return nil }func (m mockVisionEngine) PageCount() (int, error) { return 1, nil }func (m mockVisionEngine) Outlines() ([]deepdoctype.Outline, error) { return nil, nil }@@ -64,18 +81,90 @@ func TestVisionCropImage_OnDemandFromPositions(t *testing.T) {}defer cropper.Close()- img, err := cropper.Crop(cgoPositions())+ img, err := cropper.Crop(context.Background(), cgoPositions())if err != nil {t.Fatalf("Crop: %v", err)}- if img == "" {- t.Fatal("Crop returned empty; expected an on-demand cropped base64 image")+ if img == nil || img.Raster == nil {+ t.Fatal("Crop returned no raster; expected an on-demand cropped image")+ }+ if img.Raster.Bounds().Empty() {+ t.Fatal("Crop returned an empty raster")}if closed {t.Fatal("engine closed before use completed")}}+func TestVisionCropImage_RejectsOversizedPageBeforeRendering(t *testing.T) {+ renderCalls := 0+ oldFetcher := visionSourceFetcher+ oldOpener := visionEngineOpener+ defer func() {+ visionSourceFetcher = oldFetcher+ visionEngineOpener = oldOpener+ }()+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {+ return []byte("%PDF-fake-engine-bytes"), nil+ }+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {+ return mockVisionEngine{pageWidth: 5000, pageHeight: 1000, renderCalls: &renderCalls}, nil+ }++ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ defer cropper.Close()++ img, err := cropper.Crop(context.Background(), cgoPositions())+ if err != nil {+ t.Fatalf("Crop: %v", err)+ }+ if img != nil {+ t.Fatalf("Crop = %#v, want nil for an oversized page", img)+ }+ if renderCalls != 0 {+ t.Fatalf("render calls = %d, want page rejected before raster allocation", renderCalls)+ }+}++func TestVisionCropImage_RejectsAggregatePagePixelsBeforeRendering(t *testing.T) {+ renderCalls := 0+ oldFetcher := visionSourceFetcher+ oldOpener := visionEngineOpener+ defer func() {+ visionSourceFetcher = oldFetcher+ visionEngineOpener = oldOpener+ }()+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {+ return []byte("%PDF-fake-engine-bytes"), nil+ }+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {+ return mockVisionEngine{pageWidth: 2000, pageHeight: 2000, renderCalls: &renderCalls}, nil+ }++ item := map[string]any{+ "_pdf_positions": [][]any{{[]any{1, 2}, 10.0, 100.0, 10.0, 100.0}},+ }+ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ defer cropper.Close()++ img, err := cropper.Crop(context.Background(), item)+ if err != nil {+ t.Fatalf("Crop: %v", err)+ }+ if img != nil {+ t.Fatalf("Crop = %#v, want nil when combined page pixels exceed the vision budget", img)+ }+ if renderCalls != 0 {+ t.Fatalf("render calls = %d, want pages rejected before raster allocation", renderCalls)+ }+}+// TestVisionCropImage_PassthroughInline proves that when the item already// carries an inlined image (docx/markdown, or any pre-inlined source), the// cropper returns it directly without touching storage.@@ -95,12 +184,12 @@ func TestVisionCropImage_PassthroughInline(t *testing.T) {defer cropper.Close()const inline = "data:image/png;base64,preinlined"- img, err := cropper.Crop(map[string]any{"image": inline})+ img, err := cropper.Crop(context.Background(), map[string]any{"image": inline})if err != nil {t.Fatalf("Crop: %v", err)}- if img != inline {- t.Fatalf("Crop = %q, want passthrough %q", img, inline)+ if img == nil || img.VLMData != inline {+ t.Fatalf("Crop = %#v, want VLM payload %q", img, inline)}if fetchCalled {t.Fatal("storage fetch must not be called for an inlined image")@@ -124,12 +213,12 @@ func TestVisionCropImage_NoImageNoPositions(t *testing.T) {}defer cropper.Close()- img, err := cropper.Crop(map[string]any{"doc_type_kwd": "image"})+ img, err := cropper.Crop(context.Background(), map[string]any{"doc_type_kwd": "image"})if err != nil {t.Fatalf("Crop: %v", err)}- if img != "" {- t.Fatalf("Crop = %q, want empty (no image, no positions)", img)+ if img != nil {+ t.Fatalf("Crop = %#v, want nil (no image, no positions)", img)}if fetchCalled {t.Fatal("storage fetch must not be called without positions")@@ -161,12 +250,12 @@ func TestVisionCropImage_NonPDFBytesIsNoOp(t *testing.T) {}defer cropper.Close()- img, err := cropper.Crop(cgoPositions())+ img, err := cropper.Crop(context.Background(), cgoPositions())if err != nil {t.Fatalf("Crop: %v", err)}- if img != "" {- t.Fatalf("Crop = %q, want empty for non-PDF bytes", img)+ if img != nil {+ t.Fatalf("Crop = %#v, want nil for non-PDF bytes", img)}if openerCalled {t.Fatal("engine opener must not run on non-PDF bytes")@@ -194,7 +283,7 @@ func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {if err != nil {t.Fatalf("newVisionImageCropper: %v", err)}- if _, err := cropper.Crop(cgoPositions()); err != nil {+ if _, err := cropper.Crop(context.Background(), cgoPositions()); err != nil {t.Fatalf("Crop: %v", err)}if err := cropper.Close(); err != nil {@@ -204,3 +293,147 @@ func TestVisionCropImage_EngineClosedAfterUse(t *testing.T) {t.Fatal("engine should be closed after use")}}++func TestVisionCropImage_RetriesAfterTransientSourceFailure(t *testing.T) {+ oldFetcher := visionSourceFetcher+ oldOpener := visionEngineOpener+ t.Cleanup(func() {+ visionSourceFetcher = oldFetcher+ visionEngineOpener = oldOpener+ })++ fetchCalls := 0+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {+ fetchCalls+++ if fetchCalls == 1 {+ return nil, context.DeadlineExceeded+ }+ return []byte("%PDF-fake-engine-bytes"), nil+ }+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {+ return mockVisionEngine{}, nil+ }++ cropper, err := newVisionImageCropper(context.Background(), nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ t.Cleanup(func() { _ = cropper.Close() })++ if _, err := cropper.Crop(context.Background(), cgoPositions()); err != nil {+ t.Fatalf("first Crop: %v", err)+ }+ img, err := cropper.Crop(context.Background(), cgoPositions())+ if err != nil {+ t.Fatalf("second Crop: %v", err)+ }+ if img == nil || img.Raster == nil {+ t.Fatal("second Crop returned no raster after the source fetch recovered")+ }+ if fetchCalls != 2 {+ t.Fatalf("source fetch calls = %d, want 2 after transient failure", fetchCalls)+ }+}++func TestVisionCropImage_ReusesSourceAfterItemContextExpires(t *testing.T) {+ oldFetcher := visionSourceFetcher+ oldOpener := visionEngineOpener+ t.Cleanup(func() {+ visionSourceFetcher = oldFetcher+ visionEngineOpener = oldOpener+ })++ fetchStarted := make(chan struct{})+ finishFetch := make(chan struct{})+ fetchCalls := 0+ visionSourceFetcher = func(ctx context.Context, bucket, path string) ([]byte, error) {+ fetchCalls+++ select {+ case fetchStarted <- struct{}{}:+ default:+ }+ select {+ case <-finishFetch:+ return []byte("%PDF-fake-engine-bytes"), nil+ case <-ctx.Done():+ return nil, ctx.Err()+ }+ }+ visionEngineOpener = func([]byte) (deepdoctype.PDFEngine, error) {+ return mockVisionEngine{}, nil+ }++ invocationCtx, cancelInvocation := context.WithCancel(context.Background())+ defer cancelInvocation()+ cropper, err := newVisionImageCropper(invocationCtx, nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ t.Cleanup(func() { _ = cropper.Close() })++ firstItemCtx, cancelFirstItem := context.WithTimeout(invocationCtx, 20*time.Millisecond)+ defer cancelFirstItem()+ firstResult := make(chan struct {+ img *visionImage+ err error+ }, 1)+ go func() {+ img, err := cropper.Crop(firstItemCtx, cgoPositions())+ firstResult <- struct {+ img *visionImage+ err error+ }{img: img, err: err}+ }()+ <-fetchStarted+ <-firstItemCtx.Done()+ close(finishFetch)+ first := <-firstResult+ if first.err != nil {+ t.Fatalf("first Crop: %v", first.err)+ }+ if first.img == nil || first.img.Raster == nil {+ t.Fatal("first Crop returned no raster after the shared source fetch completed")+ }++ secondItemCtx, cancelSecondItem := context.WithCancel(invocationCtx)+ defer cancelSecondItem()+ second, err := cropper.Crop(secondItemCtx, cgoPositions())+ if err != nil {+ t.Fatalf("second Crop: %v", err)+ }+ if second == nil || second.Raster == nil {+ t.Fatal("second Crop returned no raster after the shared source fetch completed")+ }+ if fetchCalls != 1 {+ t.Fatalf("source fetch calls = %d, want 1 across item contexts", fetchCalls)+ }+}++func TestVisionCropImage_CanceledParentDoesNotRetrySource(t *testing.T) {+ oldFetcher := visionSourceFetcher+ t.Cleanup(func() { visionSourceFetcher = oldFetcher })++ ctx, cancel := context.WithCancel(context.Background())+ fetchCalls := 0+ visionSourceFetcher = func(context.Context, string, string) ([]byte, error) {+ fetchCalls+++ cancel()+ return nil, context.Canceled+ }++ cropper, err := newVisionImageCropper(ctx, nil, map[string]any{"bucket": "b", "path": "p"})+ if err != nil {+ t.Fatalf("newVisionImageCropper: %v", err)+ }+ t.Cleanup(func() { _ = cropper.Close() })++ if img, err := cropper.Crop(ctx, cgoPositions()); err != nil || img != nil {+ t.Fatalf("first Crop = (%#v, %v), want best-effort no result after cancellation", img, err)+ }+ if _, err := cropper.Crop(ctx, cgoPositions()); err == nil {+ t.Fatal("subsequent Crop with a canceled parent context should return its cancellation")+ }+ if fetchCalls != 1 {+ t.Fatalf("source fetch calls = %d, want one; canceled parent must prevent retries", fetchCalls)+ }+}diff --git a/internal/ingestion/component/vision_enhancement_nocgo.go b/internal/ingestion/component/vision_enhancement_nocgo.goindex df3522199d..07d51a5641 100644--- a/internal/ingestion/component/vision_enhancement_nocgo.go+++ b/internal/ingestion/component/vision_enhancement_nocgo.go@@ -35,11 +35,11 @@ func newVisionImageCropper(_ context.Context, _ *gorm.DB, _ map[string]any) (visreturn visionPDFCropperNoop{}, nil}-func (visionPDFCropperNoop) Crop(item map[string]any) (string, error) {+func (visionPDFCropperNoop) Crop(_ context.Context, item map[string]any) (*visionImage, error) {if img, _ := item["image"].(string); img != "" {- return img, nil+ return materializeInlineVisionImage(img)}- return "", nil+ return nil, nil}func (visionPDFCropperNoop) Close() error { return nil }diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex cb68846f30..ee8ae28aa6 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -17,17 +17,24 @@package componentimport (+ "bytes""context"+ "encoding/base64""errors""fmt"+ "image"+ "image/png""os""path/filepath"+ "strings""sync""testing""time""ragflow/internal/common""ragflow/internal/dao"+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"+ deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/entity"modelModule "ragflow/internal/entity/models""ragflow/internal/ingestion/component/schema"@@ -41,6 +48,14 @@ type visionEnhanceFakeDriver struct {modelModule.ModelDriver}+type failingVisionImageCropper struct{ err error }++func (c failingVisionImageCropper) Crop(context.Context, map[string]any) (*visionImage, error) {+ return nil, c.err+}++func (failingVisionImageCropper) Close() error { return nil }+type visionEnhanceCaptureInvoker struct {mu sync.Muteximages []string@@ -105,6 +120,265 @@ func fakePrompt(language string) (string, error) {return "describe the figure in " + language, nil}+func visionTestPNGBase64(t *testing.T) string {+ t.Helper()+ var encoded bytes.Buffer+ if err := png.Encode(&encoded, image.NewRGBA(image.Rect(0, 0, 20, 20))); err != nil {+ t.Fatalf("encode image: %v", err)+ }+ return base64.StdEncoding.EncodeToString(encoded.Bytes())+}++func TestParserComponent_VisionEnhancementSwitchGatesEmbeddedImage(t *testing.T) {+ previousAnalyzer := deepdoctype.NativeDocAnalyzerFactory+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previousAnalyzer })+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {+ return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true+ })++ resolverCalls := 0+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, func(ctx context.Context, db *gorm.DB, tenantID string, modelType entity.ModelType) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ resolverCalls+++ return fakeResolver(ctx, db, tenantID, modelType)+ }, invoker.invoke, fakePrompt)+ inputs := func() map[string]any {+ return map[string]any{+ "binary": []byte(`<p><img src="data:image/png;base64,` + visionTestPNGBase64(t) + `" alt="figure"></p>`),+ "file_type": "html",+ "name": "figure.html",+ "tenant_id": "tenant-1",+ }+ }++ defaultComponent, err := NewParserComponent(nil)+ if err != nil {+ t.Fatalf("NewParserComponent(nil): %v", err)+ }+ withoutVision, err := defaultComponent.Invoke(t.Context(), nil, inputs())+ if err != nil {+ t.Fatalf("default Invoke: %v", err)+ }+ if resolverCalls != 0 || len(invoker.images) != 0 {+ t.Fatalf("default-off enhancement used model: %d resolves, %d VLM calls", resolverCalls, len(invoker.images))+ }+ withoutItems, ok := withoutVision["json"].([]map[string]any)+ if !ok || len(withoutItems) == 0 {+ t.Fatalf("default JSON = %T/%v", withoutVision["json"], withoutVision["json"])+ }++ enabledComponent, err := NewParserComponent(map[string]any{"enable_vision_enhancement": true})+ if err != nil {+ t.Fatalf("NewParserComponent(enabled): %v", err)+ }+ withVision, err := enabledComponent.Invoke(t.Context(), nil, inputs())+ if err != nil {+ t.Fatalf("enabled Invoke: %v", err)+ }+ if resolverCalls != 1 || len(invoker.images) != 1 {+ t.Fatalf("enabled enhancement used model %d/%d times, want 1/1", resolverCalls, len(invoker.images))+ }+ withItems, ok := withVision["json"].([]map[string]any)+ if !ok || len(withItems) == 0 {+ t.Fatalf("enabled JSON = %T/%v", withVision["json"], withVision["json"])+ }+ withoutText, _ := withoutItems[0]["text"].(string)+ withText, _ := withItems[0]["text"].(string)+ if strings.Contains(withoutText, "a diagram of a pipeline") || !strings.Contains(withText, "a diagram of a pipeline") {+ t.Fatalf("image texts without/with enhancement = %q / %q", withoutText, withText)+ }+}++func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)++ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{{+ "text": "Existing caption",+ "image": visionTestPNGBase64(t),+ "doc_type_kwd": "image",+ }},+ }+ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(), dao.DB, utility.FileTypeXLSX, dispatched,+ map[string]any{"tenant_id": "t1"}, nil)+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want true")+ }+ want := "Existing caption\na diagram of a pipeline"+ if got := res.JSON[0]["text"]; got != want {+ t.Errorf("enhanced text = %q, want %q", got, want)+ }+ if len(invoker.images) != 1 {+ t.Errorf("VLM calls = %d, want 1", len(invoker.images))+ }+}++func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {+ swapVisionGlobals(t, fakeResolver, func(context.Context, modelModule.ModelDriver, string, []modelModule.Message, *modelModule.APIConfig) (*modelModule.ChatResponse, error) {+ return nil, errors.New("VLM unavailable")+ }, fakePrompt)++ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{{+ "image": visionTestPNGBase64(t),+ "doc_type_kwd": "image",+ }},+ }+ result, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeDOCX, dispatched,+ map[string]any{"tenant_id": "t1"}, nil)+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !strings.Contains(strings.Join(result.Warnings, "\n"), "VLM unavailable") {+ t.Fatalf("warnings = %v, want the VLM failure", result.Warnings)+ }+}++func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)++ originalFactory := visionImageCropperFactory+ visionImageCropperFactory = func(context.Context, *gorm.DB, map[string]any) (visionImageCropper, error) {+ return failingVisionImageCropper{err: errors.New("crop failed")}, nil+ }+ t.Cleanup(func() { visionImageCropperFactory = originalFactory })++ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{{+ "image": visionTestPNGBase64(t),+ "doc_type_kwd": "image",+ }},+ }+ result, handled, err := maybeDispatchVisionEnhancement(+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,+ map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}},+ )+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want inline VLM fallback")+ }+ if len(invoker.images) != 1 {+ t.Fatalf("VLM calls = %d, want 1", len(invoker.images))+ }+ if !strings.HasPrefix(invoker.images[0], "data:image/png;base64,") {+ t.Errorf("VLM image URL = %q, want inline PNG data URI", invoker.images[0])+ }+ if got, want := result.JSON[0]["text"], "a diagram of a pipeline"; got != want {+ t.Errorf("enhanced text = %q, want %q", got, want)+ }+}++func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.T) {+ for _, parseMethod := range []string{"deepdoc", "mineru"} {+ t.Run(parseMethod, func(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)++ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{{+ "text": "Existing table",+ "image": visionTestPNGBase64(t),+ "doc_type_kwd": "table",+ }},+ }+ res, handled, err := maybeDispatchVisionEnhancement(+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,+ map[string]any{"tenant_id": "t1"},+ map[string]schema.ParserSetup{"pdf": {"parse_method": parseMethod}},+ )+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want VLM enhancement")+ }+ if got, want := res.JSON[0]["text"], "Existing table\na diagram of a pipeline"; got != want {+ t.Errorf("enhanced text = %q, want %q", got, want)+ }+ })+ }+}++func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)+ imagePayload := visionTestPNGBase64(t)+ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{+ {"text": "<table><tr><td>figure</td></tr></table>", "image": imagePayload, "doc_type_kwd": "table"},+ {+ "text": "cell alt",+ "image": imagePayload,+ "doc_type_kwd": "image",+ "parent_table_id": "html-table-1",+ "row_index": 1,+ "column_index": 1,+ "media_order": 1,+ },+ },+ }++ result, handled, err := maybeDispatchVisionEnhancement(+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,+ map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "deepdoc"}},+ )+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want table and cell image VLM processing")+ }+ if got, want := result.JSON[0]["text"], "<table><tr><td>figure</td></tr></table>\na diagram of a pipeline"; got != want {+ t.Errorf("table text = %q, want %q", got, want)+ }+ cellWant := "cell alt\na diagram of a pipeline"+ if got := result.JSON[1]["text"]; got != cellWant {+ t.Errorf("cell image text = %q, want %q", got, cellWant)+ }+ if len(invoker.images) != 2 {+ t.Errorf("VLM calls = %d, want table plus cell image", len(invoker.images))+ }+}++func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)+ dispatched := parser.ParseResult{+ OutputFormat: "json",+ JSON: []map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}},+ }++ result, handled, err := maybeDispatchVisionEnhancement(+ t.Context(), dao.DB, utility.FileTypePDF, dispatched,+ map[string]any{"tenant_id": "t1"}, map[string]schema.ParserSetup{"pdf": {"parse_method": "mineru"}},+ )+ if err != nil {+ t.Fatalf("maybeDispatchVisionEnhancement: %v", err)+ }+ if !handled {+ t.Fatal("handled = false, want VLM enhancement")+ }+ if got, want := result.JSON[0]["text"], "existing\na diagram of a pipeline"; got != want {+ t.Errorf("image text = %q, want %q", got, want)+ }+ if len(invoker.images) != 1 {+ t.Errorf("VLM calls = %d, want 1", len(invoker.images))+ }+}+func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {testCases := []struct {name string@@ -274,11 +548,13 @@ func TestVisionEnhancement_MarkdownOutputUntouched(t *testing.T) {}}-func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {+func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {+ invoker := &visionEnhanceCaptureInvoker{}+ swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)dispatched := parser.ParseResult{OutputFormat: "json",JSON: []map[string]any{- {"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},+ {"text": "caption", "image": "aGVsbG8=", "doc_type_kwd": "image"},},}@@ -291,11 +567,14 @@ func TestVisionEnhancement_NonAllowedFileTypeSkipped(t *testing.T) {if err != nil {t.Fatalf("unexpected error: %v", err)}- if handled {- t.Error("handled = true, want false for FileTypeOTHER")+ if !handled {+ t.Fatal("handled = false, want true for a visual payload in FileTypeOTHER")}- if res.JSON[0]["text"] != "" {- t.Errorf("text = %q, want empty", res.JSON[0]["text"])+ if got, want := res.JSON[0]["text"], "caption\na diagram of a pipeline"; got != want {+ t.Errorf("text = %q, want %q", got, want)+ }+ if len(invoker.images) != 1 {+ t.Fatalf("VLM calls = %d, want 1", len(invoker.images))}}diff --git a/internal/ingestion/component/vision_image.go b/internal/ingestion/component/vision_image.gonew file mode 100644index 0000000000..6a09a45cdd--- /dev/null+++ b/internal/ingestion/component/vision_image.go@@ -0,0 +1,205 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "bytes"+ "encoding/base64"+ "fmt"+ "image"+ "image/png"+ "io"+ "strings"++ _ "golang.org/x/image/bmp"+ _ "golang.org/x/image/tiff"+ _ "golang.org/x/image/webp"+ _ "image/gif"+ _ "image/jpeg"++ parserlib "ragflow/internal/parser/parser"+)++const (+ maxVisionImageBytes = parserlib.MaxImagePayloadBytes+ maxVisionImagePixels = parserlib.MaxImagePixels+ maxVisionImageEdge = parserlib.MaxImageEdge+ maxVLMEncodedBytes = (maxVisionImageBytes+2)/3*4 + 256+)++func materializeInlineVisionImage(raw string) (*visionImage, error) {+ raw = strings.TrimSpace(raw)+ if raw == "" {+ return nil, nil+ }+ materialized := &visionImage{VLMData: raw}+ if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") {+ materialized.VLMDataValidated = isUsableVisionImage(raw)+ return materialized, nil+ }++ isDataURI := strings.HasPrefix(strings.ToLower(raw), "data:image/")+ payload := raw+ if isDataURI {+ comma := strings.IndexByte(raw, ',')+ if comma <= 0 || !strings.HasSuffix(strings.ToLower(raw[:comma]), ";base64") {+ return materialized, nil+ }+ payload = raw[comma+1:]+ } else if strings.HasPrefix(strings.ToLower(raw), "data:") {+ return materialized, nil+ }+ format, err := inspectVisionImagePayload(payload)+ if err != nil {+ materialized.VLMDataValidated = false+ return materialized, nil+ }+ materialized.VLMDataValidated = true+ if strings.ContainsAny(payload, "\r\n \t") {+ if data, decodeErr := decodeVisionPayload(payload); decodeErr == nil {+ payload = base64.StdEncoding.EncodeToString(data)+ }+ }+ if isDataURI {+ if payload != raw[strings.IndexByte(raw, ',')+1:] {+ materialized.VLMData = raw[:strings.IndexByte(raw, ',')+1] + payload+ }+ return materialized, nil+ }+ if mimeType := imageMIMEForFormat(format); mimeType != "" {+ materialized.VLMData = "data:" + mimeType + ";base64," + payload+ }+ return materialized, nil+}++func inspectVisionImagePayload(payload string) (string, error) {+ if payload == "" || len(payload) > maxVLMEncodedBytes {+ return "", fmt.Errorf("vision image: encoded payload exceeds limit")+ }+ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {+ decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: payload})+ config, format, err := image.DecodeConfig(io.LimitReader(decoder, maxVisionImageBytes+1))+ if err != nil {+ continue+ }+ if config.Width <= 0 || config.Height <= 0 || config.Width > maxVisionImageEdge || config.Height > maxVisionImageEdge || int64(config.Width)*int64(config.Height) > maxVisionImagePixels {+ return "", fmt.Errorf("vision image: image dimensions %dx%d exceed limits", config.Width, config.Height)+ }+ return format, nil+ }+ return "", fmt.Errorf("vision image: invalid image payload")+}++func decodeVisionPayload(raw string) ([]byte, error) {+ encoded := strings.TrimSpace(raw)+ if strings.HasPrefix(strings.ToLower(encoded), "data:") {+ comma := strings.IndexByte(encoded, ',')+ if comma < 0 || !strings.Contains(strings.ToLower(encoded[:comma]), ";base64") {+ return nil, fmt.Errorf("vision image: unsupported data URI")+ }+ encoded = encoded[comma+1:]+ }+ maxEncodedBytes := (maxVisionImageBytes+2)/3*4 + 256+ if len(encoded) == 0 || len(encoded) > maxEncodedBytes {+ return nil, fmt.Errorf("vision image: encoded payload exceeds limit")+ }+ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {+ decoder := base64.NewDecoder(encoding, &visionPayloadReader{source: encoded})+ data, err := io.ReadAll(io.LimitReader(decoder, maxVisionImageBytes+1))+ if err != nil {+ continue+ }+ if len(data) > maxVisionImageBytes {+ return nil, fmt.Errorf("vision image: payload exceeds %d bytes", maxVisionImageBytes)+ }+ if len(data) == 0 {+ return nil, fmt.Errorf("vision image: empty base64 payload")+ }+ return data, nil+ }+ return nil, fmt.Errorf("vision image: decode base64 failed")+}++type visionPayloadReader struct {+ source string+ offset int+}++func (r *visionPayloadReader) Read(dst []byte) (int, error) {+ written := 0+ for written < len(dst) && r.offset < len(r.source) {+ b := r.source[r.offset]+ r.offset+++ if b == '\r' || b == '\n' || b == ' ' || b == '\t' {+ continue+ }+ dst[written] = b+ written+++ }+ if written > 0 {+ return written, nil+ }+ if r.offset == len(r.source) {+ return 0, io.EOF+ }+ return 0, nil+}++func imageWithinVisionLimits(img image.Image) bool {+ if img == nil {+ return false+ }+ bounds := img.Bounds()+ width, height := bounds.Dx(), bounds.Dy()+ return width > 0 && height > 0 && width <= maxVisionImageEdge && height <= maxVisionImageEdge && int64(width)*int64(height) <= maxVisionImagePixels+}++func imageMIMEForFormat(format string) string {+ switch strings.ToLower(format) {+ case "jpeg":+ return "image/jpeg"+ case "png":+ return "image/png"+ case "gif":+ return "image/gif"+ case "bmp":+ return "image/bmp"+ case "tiff":+ return "image/tiff"+ case "webp":+ return "image/webp"+ default:+ return ""+ }+}++func encodeVisionRaster(img image.Image) (string, error) {+ if img == nil {+ return "", nil+ }+ if !imageWithinVisionLimits(img) {+ return "", fmt.Errorf("vision image: raster dimensions exceed limits")+ }+ var encoded bytes.Buffer+ if err := png.Encode(&encoded, img); err != nil {+ return "", fmt.Errorf("vision image: encode raster: %w", err)+ }+ if encoded.Len() > maxVisionImageBytes {+ return "", fmt.Errorf("vision image: encoded raster exceeds %d bytes", maxVisionImageBytes)+ }+ return "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes()), nil+}diff --git a/internal/ingestion/component/vision_image_test.go b/internal/ingestion/component/vision_image_test.gonew file mode 100644index 0000000000..450ac8ac0b--- /dev/null+++ b/internal/ingestion/component/vision_image_test.go@@ -0,0 +1,54 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package component++import (+ "bytes"+ "encoding/base64"+ "image"+ "image/png"+ "strings"+ "testing"+)++func TestMaterializeInlineVisionImageKeepsVLMForImageDimensionLimit(t *testing.T) {+ var encoded bytes.Buffer+ large := image.NewRGBA(image.Rect(0, 0, maxVisionImageEdge+1, 1))+ if err := png.Encode(&encoded, large); err != nil {+ t.Fatalf("encode oversized image: %v", err)+ }+ dataURI := "data:image/png;base64," + base64.StdEncoding.EncodeToString(encoded.Bytes())++ materialized, err := materializeInlineVisionImage(dataURI)+ if err != nil {+ t.Fatalf("materializeInlineVisionImage: %v", err)+ }+ if materialized == nil || materialized.Raster != nil {+ t.Fatalf("raster = %#v, want OCR to skip the oversized image", materialized)+ }+ if materialized.VLMData != dataURI || !isUsableVisionImage(materialized.VLMData) {+ t.Fatal("oversized OCR input should preserve its valid VLM payload")+ }+}++func TestIsUsableVisionImageRejectsPayloadOverVLMByteBudget(t *testing.T) {+ encodedLength := base64.StdEncoding.EncodedLen(maxVisionImageBytes + 3)+ payload := strings.Repeat("A", encodedLength)+ if isUsableVisionImage(payload) {+ t.Fatal("base64 payload above the VLM byte budget must be rejected")+ }+}diff --git a/internal/parser/parser/docx_ir.go b/internal/parser/parser/docx_ir.goindex 8ba42ad7d6..f6a2f8569c 100644--- a/internal/parser/parser/docx_ir.go+++ b/internal/parser/parser/docx_ir.go@@ -27,6 +27,7 @@ package parserimport ("encoding/base64""encoding/json"+ "fmt""html""strings")@@ -204,6 +205,81 @@ func docxIRTableToHTML(el docxIRElement) string {return sb.String()}+type docxTableImage struct {+ data []byte+ row int+ column int+ included bool+}++func forEachDOCXTableImage(el docxIRElement, visit func(docxTableImage) bool) bool {+ for rowIndex, row := range el.Rows {+ for columnIndex, cell := range row.Cells {+ completed := forEachDOCXIRImage(cell.Content, func(data []byte) bool {+ return visit(docxTableImage{data: data, row: rowIndex + 1, column: columnIndex + 1})+ })+ if !completed {+ return false+ }+ }+ }+ return true+}++// forEachDOCXIRImage visits image payloads without first collecting a second+// document-sized slice of image references. Returning false from visit stops+// traversal as soon as the caller's media budget is exhausted.+func forEachDOCXIRImage(elements []docxIRElement, visit func([]byte) bool) bool {+ for _, el := range elements {+ switch el.Type {+ case "image":+ if len(el.Data) > 0 && !visit(el.Data) {+ return false+ }+ case "paragraph", "heading":+ for _, run := range el.contentRuns() {+ if run.Type == "image" && len(run.Data) > 0 && !visit(run.Data) {+ return false+ }+ if !forEachDOCXIRImage(run.Content, visit) {+ return false+ }+ }+ case "table":+ for _, row := range el.Rows {+ for _, cell := range row.Cells {+ if !forEachDOCXIRImage(cell.Content, visit) {+ return false+ }+ }+ }+ case "list":+ if !forEachDOCXIRListImage(docxIRList{Items: el.Items}, visit) {+ return false+ }+ case "text_box":+ if !forEachDOCXIRImage(el.contentBlocks(), visit) {+ return false+ }+ }+ }+ return true+}++func forEachDOCXIRListImage(list docxIRList, visit func([]byte) bool) bool {+ for _, item := range list.Items {+ if !forEachDOCXIRImage(item.Content, visit) {+ return false+ }+ if item.Nested != nil {+ if !forEachDOCXIRListImage(*item.Nested, visit) {+ return false+ }+ }+ }+ return true+}+// docxElementText returns the plain-text rendering of any supported// IR element type. Used by buildPPTXJSONSections to flatten one slide// section, and by extractDOCXFiguresFromIR so that tables, lists, and@@ -253,60 +329,86 @@ func docxElementText(el docxIRElement, cellSep string) string {// buildDOCXJSONSections converts an office_oxide IR JSON string into a// slice of structured items compatible with the chunker's JSON input// contract. Each item carries at least text and doc_type_kwd.-func buildDOCXJSONSections(irJSON string) []map[string]any {+func buildDOCXJSONSections(irJSON string, budget *embeddedMediaBudget) []map[string]any {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil}+ if budget == nil {+ budget = newEmbeddedMediaBudget()+ }var sections []map[string]any+ tableSequence := 0for _, sec := range ir.Sections {for _, el := range sec.Elements {switch el.Type {case "paragraph", "heading":- sections = appendDOCXParagraphSections(sections, el, el.Type == "heading")+ sections = appendDOCXParagraphSections(sections, el, el.Type == "heading", budget)case "image":- b64 := base64.StdEncoding.EncodeToString(el.Data)- sections = append(sections, map[string]any{- "text": "",- "image": b64,- "doc_type_kwd": "image",- })+ sections, _ = appendDOCXImageSection(sections, el.Data, budget, nil)case "table":+ tableSequence++html := docxIRTableToHTML(el)if html == "<table></table>" {continue}- sections = append(sections, map[string]any{+ table := map[string]any{"text": html,"image": nil,"doc_type_kwd": "table",+ }+ tableID := ""+ foundImage := false+ var tableImages []docxTableImage+ forEachDOCXTableImage(el, func(tableImage docxTableImage) bool {+ foundImage = true+ included, keepWalking := budget.include(tableImage.data)+ if included || keepWalking {+ tableImage.included = included+ tableImages = append(tableImages, tableImage)+ }+ return keepWalking})+ if foundImage {+ tableID = fmt.Sprintf("docx-table-%d", tableSequence)+ table["source_table_id"] = tableID+ }+ sections = append(sections, table)+ for mediaOrder, tableImage := range tableImages {+ metadata := map[string]any{+ "parent_table_id": tableID,+ "row_index": tableImage.row,+ "column_index": tableImage.column,+ "media_order": mediaOrder + 1,+ }+ sections = appendDOCXImagePayload(sections, tableImage.data, tableImage.included, metadata)+ }case "list":for _, item := range el.Items {text := extractTextFromListItem(item)- if text == "" {- continue+ if text != "" {+ sections = append(sections, map[string]any{+ "text": text,+ "image": nil,+ "doc_type_kwd": "text",+ })}+ sections = appendDOCXListImageSections(sections, docxIRList{Items: []docxIRListItem{item}}, budget)+ }++ case "text_box":+ text := extractTextFromBlockElements(el.contentBlocks())+ if text != "" {sections = append(sections, map[string]any{"text": text,"image": nil,"doc_type_kwd": "text",})}-- case "text_box":- text := extractTextFromBlockElements(el.contentBlocks())- if text == "" {- continue- }- sections = append(sections, map[string]any{- "text": text,- "image": nil,- "doc_type_kwd": "text",- })+ sections = appendDOCXIRImageSections(sections, el.contentBlocks(), budget)}}}@@ -317,7 +419,7 @@ func buildDOCXJSONSections(irJSON string) []map[string]any {// order. A paragraph is not necessarily a single text unit: office_oxide can// place an image between two text runs, and dropping that run would make the// downstream DOCX chunker lose both media order and adjacency context.-func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool) []map[string]any {+func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, heading bool, budget *embeddedMediaBudget) []map[string]any {var text strings.BuilderflushText := func() {value := strings.TrimSpace(text.String())@@ -336,14 +438,7 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, hesections = append(sections, item)}appendImage := func(data []byte) {- if len(data) == 0 {- return- }- sections = append(sections, map[string]any{- "text": "",- "image": base64.StdEncoding.EncodeToString(data),- "doc_type_kwd": "image",- })+ sections, _ = appendDOCXImageSection(sections, data, budget, nil)}for _, run := range el.contentRuns() {switch run.Type {@@ -365,28 +460,77 @@ func appendDOCXParagraphSections(sections []map[string]any, el docxIRElement, hereturn sections}+func appendDOCXImageSection(sections []map[string]any, data []byte, budget *embeddedMediaBudget, metadata map[string]any) ([]map[string]any, bool) {+ if len(data) == 0 {+ return sections, true+ }+ included, keepWalking := budget.include(data)+ if !included && !keepWalking {+ return sections, false+ }+ return appendDOCXImagePayload(sections, data, included, metadata), keepWalking+}++func appendDOCXListImageSections(sections []map[string]any, list docxIRList, budget *embeddedMediaBudget) []map[string]any {+ for _, item := range list.Items {+ sections = appendDOCXIRImageSections(sections, item.Content, budget)+ if item.Nested != nil {+ sections = appendDOCXListImageSections(sections, *item.Nested, budget)+ }+ }+ return sections+}++func appendDOCXIRImageSections(sections []map[string]any, elements []docxIRElement, budget *embeddedMediaBudget) []map[string]any {+ forEachDOCXIRImage(elements, func(data []byte) bool {+ var keepWalking bool+ sections, keepWalking = appendDOCXImageSection(sections, data, budget, nil)+ return keepWalking+ })+ return sections+}++func appendDOCXImagePayload(sections []map[string]any, data []byte, included bool, metadata map[string]any) []map[string]any {+ item := map[string]any{+ "text": "",+ "image": nil,+ "doc_type_kwd": "image",+ }+ if included {+ item["image"] = base64.StdEncoding.EncodeToString(data)+ } else {+ item["media_omitted"] = true+ }+ for key, value := range metadata {+ item[key] = value+ }+ return append(sections, item)+}+// --- figure extraction (used by the cgo parser path) ---// extractDOCXFiguresFromIR parses the office_oxide IR JSON and-// returns every embedded image block together with the plain text-// immediately surrounding it. The context matches what Python's+// returns the images admitted by budget together with the plain text+// immediately surrounding each image. The context matches what Python's// naive_merge_docx attaches as context_above / context_below on// each chunk that carries an image.//// Reuses the IR already obtained from the doc handle in// ParseWithResult so the binary is not opened twice.-func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {+func extractDOCXFiguresFromIR(irJSON string, budget *embeddedMediaBudget) []DOCXFigure {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil}+ if budget == nil {+ budget = newEmbeddedMediaBudget()+ }var flat []flatBlockfor _, sec := range ir.Sections {for _, el := range sec.Elements {if el.Type == "image" {- b64 := base64.StdEncoding.EncodeToString(el.Data)- flat = append(flat, flatBlock{image: b64})+ flat = append(flat, flatBlock{imageData: el.Data})continue}if el.Type == "paragraph" || el.Type == "heading" {@@ -400,10 +544,17 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {var figures []DOCXFigurefor i, block := range flat {- if block.image == "" {+ if len(block.imageData) == 0 {continue}- fig := DOCXFigure{Image: block.image}+ included, keepWalking := budget.include(block.imageData)+ if !included && !keepWalking {+ break+ }+ fig := DOCXFigure{}+ if included {+ fig.Image = base64.StdEncoding.EncodeToString(block.imageData)+ }// Collect text above (backward scan up to docxContextWindow// chars, or until another image is hit).@@ -426,6 +577,9 @@ func extractDOCXFiguresFromIR(irJSON string) []DOCXFigure {}figures = append(figures, fig)+ if !keepWalking {+ break+ }}return figures}@@ -447,7 +601,7 @@ func docxInlineFlatBlocks(el docxIRElement) []flatBlock {if len(data) == 0 {return}- blocks = append(blocks, flatBlock{image: base64.StdEncoding.EncodeToString(data)})+ blocks = append(blocks, flatBlock{imageData: data})}for _, run := range el.contentRuns() {switch run.Type {@@ -474,8 +628,8 @@ func docxInlineFlatBlocks(el docxIRElement) []flatBlock {// flatBlock is a flattened IR element used internally to collect// text / image context around embedded figures.type flatBlock struct {- text string- image string // base64-encoded image data (empty for non-image)+ text string+ imageData []byte}const docxContextWindow = 512@@ -484,7 +638,7 @@ func collectDOCXPrevText(flat []flatBlock, idx, maxLen int) string {var parts []stringremaining := maxLenfor i := idx - 1; i >= 0 && remaining > 0; i-- {- if flat[i].image != "" {+ if len(flat[i].imageData) > 0 {break // stop at previous image}if flat[i].text == "" {@@ -512,7 +666,7 @@ func collectDOCXNextText(flat []flatBlock, idx, maxLen int) string {var parts []stringremaining := maxLenfor i := idx + 1; i < len(flat) && remaining > 0; i++ {- if flat[i].image != "" {+ if len(flat[i].imageData) > 0 {break // stop at next image}if flat[i].text == "" {diff --git a/internal/parser/parser/docx_ir_test.go b/internal/parser/parser/docx_ir_test.goindex 1a33eeeb8b..03c98ffe2c 100644--- a/internal/parser/parser/docx_ir_test.go+++ b/internal/parser/parser/docx_ir_test.go@@ -1,8 +1,11 @@package parserimport (+ "encoding/base64""encoding/json"+ "fmt""reflect"+ "strings""testing")@@ -22,7 +25,7 @@ func TestBuildDOCXJSONSections_FromJSON(t *testing.T) {{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"item1"}]}]}]}]}]}`- got := buildDOCXJSONSections(irJSON)+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())want := []map[string]any{{"text": "Title", "image": nil, "doc_type_kwd": "text", "ck_type": "heading"},{"text": "Hello", "image": nil, "doc_type_kwd": "text"},@@ -43,7 +46,7 @@ func TestBuildDOCXJSONSections_EmptyTableSkipped(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"keep"}]},{"type":"table","rows":[]}]}]}`- got := buildDOCXJSONSections(irJSON)+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())if len(got) != 1 || got[0]["text"] != "keep" {t.Fatalf("expected only the paragraph to survive, got %+v", got)}@@ -57,7 +60,7 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {{"type":"text","text":"after"}]}]}]}`- got := buildDOCXJSONSections(irJSON)+ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())if len(got) != 3 {t.Fatalf("sections = %+v, want text/image/text sequence", got)}@@ -72,6 +75,189 @@ func TestBuildDOCXJSONSections_PreservesInlineImageOrder(t *testing.T) {}}+func TestBuildDOCXJSONSections_ExtractsTableCellImages(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"table","rows":[{"cells":[+ {"content":[{"type":"paragraph","content":[+ {"type":"text","text":"before"},+ {"type":"image","data":"aGVsbG8="},+ {"type":"text","text":"after"}+ ]}]},+ {"content":[{"type":"image","data":"aW1hZ2U="}]}+ ]}]}+ ]}]}`++ got := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ if len(got) != 3 {+ t.Fatalf("sections = %+v, want table plus two cell image items", got)+ }+ if got[0]["doc_type_kwd"] != "table" || got[0]["text"] != "<table><tr><td>beforeafter</td><td></td></tr></table>" || got[0]["source_table_id"] != "docx-table-1" {+ t.Fatalf("table item = %+v", got[0])+ }+ for i, want := range []string{"aGVsbG8=", "aW1hZ2U="} {+ item := got[i+1]+ if item["doc_type_kwd"] != "image" || item["image"] != want {+ t.Errorf("image item %d = %+v, want payload %q", i, item, want)+ }+ if item["parent_table_id"] != "docx-table-1" || item["row_index"] != 1 || item["column_index"] != i+1 || item["media_order"] != i+1 {+ t.Errorf("image item %d metadata = %+v", i, item)+ }+ }+}++func TestBuildDOCXJSONSections_BudgetKeepsInlineTextAroundOmittedImage(t *testing.T) {+ irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[+ {"type":"text","text":"before"},+ {"type":"image","data":"YWJjZA=="},+ {"type":"text","text":"after"}+ ]}]}]}`+ budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}++ items := buildDOCXJSONSections(irJSON, budget)+ if len(items) != 3 {+ t.Fatalf("items = %+v, want text/image/text with omitted image metadata", items)+ }+ if items[0]["text"] != "before" || items[2]["text"] != "after" {+ t.Fatalf("inline text was lost around omitted image: %+v", items)+ }+ if items[1]["doc_type_kwd"] != "image" || items[1]["image"] != nil || items[1]["media_omitted"] != true {+ t.Fatalf("omitted inline image item = %+v, want metadata without payload", items[1])+ }+ if len(budget.warnings()) == 0 {+ t.Fatal("omitted image should produce a parser warning")+ }+}++func TestBuildDOCXJSONSections_BudgetKeepsTableImageMetadata(t *testing.T) {+ irJSON := `{"sections":[{"elements":[{"type":"table","rows":[{"cells":[+ {"content":[{"type":"image","data":"YWJj"}]},+ {"content":[{"type":"image","data":"ZGVm"}]}+ ]}]}]}]}`+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 10}++ items := buildDOCXJSONSections(irJSON, budget)+ if len(items) != 3 {+ t.Fatalf("items = %+v, want table and two image metadata items", items)+ }+ if items[1]["image"] != "YWJj" || items[1]["media_omitted"] == true {+ t.Fatalf("first table image = %+v, want accepted payload", items[1])+ }+ omitted := items[2]+ if omitted["image"] != nil || omitted["media_omitted"] != true {+ t.Fatalf("second table image = %+v, want omitted payload", omitted)+ }+ if omitted["parent_table_id"] != items[0]["source_table_id"] || omitted["row_index"] != 1 || omitted["column_index"] != 2 || omitted["media_order"] != 2 {+ t.Fatalf("omitted table image lost location metadata: %+v", omitted)+ }+}++func TestBuildDOCXJSONSections_BudgetCapsMediaCount(t *testing.T) {+ var ir strings.Builder+ ir.WriteString(`{"sections":[{"elements":[`)+ for i := 0; i < 257; i++ {+ if i > 0 {+ ir.WriteByte(',')+ }+ ir.WriteString(`{"type":"image","data":"YQ=="}`)+ }+ ir.WriteString(`]}]}`)++ budget := newEmbeddedMediaBudget()+ items := buildDOCXJSONSections(ir.String(), budget)+ imageCount := 0+ for _, item := range items {+ if item["doc_type_kwd"] == "image" {+ imageCount+++ }+ }+ if imageCount != budget.maxItems {+ t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)+ }+ if len(budget.warnings()) == 0 {+ t.Fatal("truncated media count should produce a parser warning")+ }+}++func TestBuildPPTXJSONSections_BudgetKeepsSlideMetadata(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"slide text"}]},+ {"type":"image","data":"YWJjZA=="}+ ]}]}`+ budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}++ items, err := buildPPTXJSONSections(irJSON, budget)+ if err != nil {+ t.Fatalf("buildPPTXJSONSections: %v", err)+ }+ if len(items) != 2 || items[0]["text"] != "slide text" {+ t.Fatalf("items = %+v, want text and omitted image items", items)+ }+ omitted := items[1]+ if omitted["image"] != nil || omitted["media_omitted"] != true || omitted["slide_number"] != 1 || omitted["media_order"] != 1 {+ t.Fatalf("omitted slide image lost metadata: %+v", omitted)+ }+ if itemsAllEmpty([]map[string]any{omitted}) {+ t.Fatal("an image-only slide with a budget-omitted image must not be replaced by whole-deck fallback")+ }+}++func TestBuildPPTXJSONSections_BudgetCapsMediaCount(t *testing.T) {+ var ir strings.Builder+ ir.WriteString(`{"sections":[{"elements":[`)+ for i := 0; i < 257; i++ {+ if i > 0 {+ ir.WriteByte(',')+ }+ ir.WriteString(`{"type":"image","data":"YQ=="}`)+ }+ ir.WriteString(`]}]}`)++ budget := newEmbeddedMediaBudget()+ items, err := buildPPTXJSONSections(ir.String(), budget)+ if err != nil {+ t.Fatalf("buildPPTXJSONSections: %v", err)+ }+ imageCount := 0+ for _, item := range items {+ if item["doc_type_kwd"] == "image" {+ imageCount+++ }+ }+ if imageCount != budget.maxItems {+ t.Fatalf("image items = %d, want count limit %d", imageCount, budget.maxItems)+ }+ if len(budget.warnings()) == 0 {+ t.Fatal("truncated media count should produce a parser warning")+ }+}++func TestEmbeddedMediaBudgetRejectsPerImageAndDocumentByteLimits(t *testing.T) {+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 4}+ if included, keepGoing := budget.include([]byte("abc")); !included || !keepGoing {+ t.Fatalf("first image admission = (%v, %v), want (true, true)", included, keepGoing)+ }+ if included, keepGoing := budget.include([]byte("def")); included || !keepGoing {+ t.Fatalf("aggregate overflow admission = (%v, %v), want (false, true)", included, keepGoing)+ }+ if included, keepGoing := budget.include([]byte("12345")); included || !keepGoing {+ t.Fatalf("per-image overflow admission = (%v, %v), want (false, true)", included, keepGoing)+ }+ if len(budget.warnings()) == 0 {+ t.Fatal("omitted byte payloads should produce a parser warning")+ }+}++func TestBuildDOCXJSONSectionsEncodesOnlyAcceptedImageData(t *testing.T) {+ data := []byte("text is not a valid image, but the parser preserves it")+ encoded := base64.StdEncoding.EncodeToString(data)+ irJSON := fmt.Sprintf(`{"sections":[{"elements":[{"type":"image","data":%q}]}]}`, encoded)+ budget := &embeddedMediaBudget{maxImageBytes: 8, maxTotalBytes: 8, maxItems: 4}+ items := buildDOCXJSONSections(irJSON, budget)+ if len(items) != 1 || items[0]["image"] != nil || items[0]["media_omitted"] != true {+ t.Fatalf("oversized image items = %+v, want metadata without payload", items)+ }+}+// TestJoinDOCXIRRuns pins that only text-type runs are concatenated;// non-text runs (e.g. nested image runs) are skipped.func TestJoinDOCXIRRuns(t *testing.T) {@@ -111,7 +297,7 @@ func TestExtractDOCXFiguresFromIR(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"after"}]}]}]}`- figs := extractDOCXFiguresFromIR(irJSON)+ figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())if len(figs) != 1 {t.Fatalf("expected 1 figure, got %d", len(figs))}@@ -135,7 +321,7 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {{"type":"text","text":"after"}]}]}]}`- figs := extractDOCXFiguresFromIR(irJSON)+ figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())if len(figs) != 1 {t.Fatalf("expected 1 inline figure, got %d", len(figs))}@@ -144,13 +330,30 @@ func TestExtractDOCXFiguresFromIR_InlineImage(t *testing.T) {}}+func TestExtractDOCXFiguresFromIR_BudgetKeepsContextWithoutPayload(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"before"}]},+ {"type":"image","data":"YWJjZA=="},+ {"type":"paragraph","content":[{"type":"text","text":"after"}]}+ ]}]}`+ budget := &embeddedMediaBudget{maxImageBytes: 3, maxTotalBytes: 5, maxItems: 10}++ figures := extractDOCXFiguresFromIR(irJSON, budget)+ if len(figures) != 1 {+ t.Fatalf("figures = %+v, want one metadata record", figures)+ }+ if figures[0].Image != "" || figures[0].ContextAbove != "before" || figures[0].ContextBelow != "after" || figures[0].Marker != "before" {+ t.Fatalf("budgeted figure = %+v, want context metadata without payload", figures[0])+ }+}+// TestExtractDOCXFiguresFromIR_NoImage returns nil when the IR has no// image blocks.func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {irJSON := `{"sections":[{"elements":[{"type":"paragraph","content":[{"type":"text","text":"only text"}]}]}]}`- if figs := extractDOCXFiguresFromIR(irJSON); figs != nil {+ if figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget()); figs != nil {t.Fatalf("expected nil, got %+v", figs)}}@@ -158,7 +361,7 @@ func TestExtractDOCXFiguresFromIR_NoImage(t *testing.T) {// TestExtractDOCXFiguresFromIR_BadJSON returns nil on unparseable IR,// mirroring the json.Unmarshal error guard.func TestExtractDOCXFiguresFromIR_BadJSON(t *testing.T) {- if figs := extractDOCXFiguresFromIR("{not json"); figs != nil {+ if figs := extractDOCXFiguresFromIR("{not json", newEmbeddedMediaBudget()); figs != nil {t.Fatalf("expected nil for bad JSON, got %+v", figs)}}@@ -176,7 +379,7 @@ func TestExtractDOCXFiguresFromIR_NonParagraphContext(t *testing.T) {{"type":"list","items":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"list item"}]}]}]},{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"box text"}]}]}]}]}`- figs := extractDOCXFiguresFromIR(irJSON)+ figs := extractDOCXFiguresFromIR(irJSON, newEmbeddedMediaBudget())if len(figs) != 1 {t.Fatalf("expected 1 figure, got %d", len(figs))}@@ -203,7 +406,7 @@ func TestCollectDOCXText_BoundedByMaxLen(t *testing.T) {flat = append(flat, flatBlock{text: "x"})}imgIdx := len(flat)- flat = append(flat, flatBlock{image: "img"})+ flat = append(flat, flatBlock{imageData: []byte("img")})for i := 0; i < 600; i++ {flat = append(flat, flatBlock{text: "x"})}@@ -278,7 +481,7 @@ func TestExtractTextFromListItem_NestedNull(t *testing.T) {{"content":[{"type":"paragraph","content":[{"type":"text","text":"only item"}]}],"nested":null}]}]}]}`- sections := buildDOCXJSONSections(irJSON)+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())if len(sections) != 1 || sections[0]["text"] != "only item" {t.Fatalf("expected single item \"only item\", got %+v", sections)}@@ -301,7 +504,7 @@ func TestExtractTextFromListItem_NestedTable(t *testing.T) {]}]}]}]}`- sections := buildDOCXJSONSections(irJSON)+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)}@@ -352,7 +555,7 @@ func TestBuildDOCXJSONSections_TextBoxTable(t *testing.T) {]}]}]}]}`- sections := buildDOCXJSONSections(irJSON)+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)}@@ -373,7 +576,7 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {]}}]}]}]}`- sections := buildDOCXJSONSections(irJSON)+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("expected 1 section, got %d: %+v", len(sections), sections)}@@ -381,3 +584,48 @@ func TestBuildDOCXJSONSections_NestedList(t *testing.T) {t.Errorf("nested list text = %q, want %q", got, "L1\nL2a\nL2b")}}++func TestBuildDOCXJSONSections_EmitsListAndTextBoxImages(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"list","items":[+ {"content":[{"type":"paragraph","content":[+ {"type":"text","text":"list text"},+ {"type":"image","data":"aGVsbG8="}+ ]}],"nested":{"items":[+ {"content":[{"type":"paragraph","content":[+ {"type":"image","data":"d29ybGQ="}+ ]}]}+ ]}}+ ]},+ {"type":"text_box","content":[{"type":"paragraph","content":[+ {"type":"text","text":"text box text"},+ {"type":"image","data":"cGljdHVyZQ=="}+ ]}]}+ ]}]}`+ sections := buildDOCXJSONSections(irJSON, newEmbeddedMediaBudget())+ want := []struct {+ docType string+ text string+ image string+ }{+ {docType: "text", text: "list text"},+ {docType: "image", image: "aGVsbG8="},+ {docType: "image", image: "d29ybGQ="},+ {docType: "text", text: "text box text"},+ {docType: "image", image: "cGljdHVyZQ=="},+ }+ if len(sections) != len(want) {+ t.Fatalf("sections = %+v, want %d items", sections, len(want))+ }+ for i, expected := range want {+ if got := sections[i]["doc_type_kwd"]; got != expected.docType {+ t.Errorf("section %d type = %v, want %q", i, got, expected.docType)+ }+ if expected.text != "" && sections[i]["text"] != expected.text {+ t.Errorf("section %d text = %v, want %q", i, sections[i]["text"], expected.text)+ }+ if expected.image != "" && sections[i]["image"] != expected.image {+ t.Errorf("section %d image = %v, want %q", i, sections[i]["image"], expected.image)+ }+ }+}diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.goindex 54833ccef4..77288fabb4 100644--- a/internal/parser/parser/docx_parser.go+++ b/internal/parser/parser/docx_parser.go@@ -68,8 +68,8 @@ func (p *DOCXParser) ConfigureFromSetup(setup map[string]any) {// ParseWithResult produces structured JSON items (when// p.outputFormat == "json") or markdown (default) from a-// docx document. Embedded images are extracted in both paths-// for downstream vision-figure dispatch.+// docx document. The JSON path emits bounded image items; the+// Markdown path attaches bounded figure metadata to the file result.//// JSON path mirrors python parser.py:_docx() output_format == "json".// Markdown path mirrors python naive.py: Docx() → naive_merge_docx().@@ -106,22 +106,18 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data}// Extract IR JSON for section building (JSON path) and- // embedded-image extraction (both paths).+ // embedded-image extraction on the Markdown path.irJSON, irErr := doc.ToIRJSON()- var figures []DOCXFigure- if irErr == nil {- figures = extractDOCXFiguresFromIR(irJSON)- }- if len(figures) > 0 {- fileMeta["figures"] = buildFiguresMap(figures)- }if p.outputFormat == "json" {if irErr != nil {return ParseResult{Err: fmt.Errorf("docx to-ir-json: %w", irErr)}}- var sections []map[string]any- sections = buildDOCXJSONSections(irJSON)+ budget := newEmbeddedMediaBudget()+ sections := buildDOCXJSONSections(irJSON, budget)+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }// remove_header_footer: drop sections whose normalized text// matches a docx header/footer entry (mirrors Python// parser.py:889-891 extract_docx_header_footer_texts +@@ -143,6 +139,15 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, dataOutputFormat: "json",File: fileMeta,JSON: sections,+ Warnings: budget.warnings(),+ }+ }++ budget := newEmbeddedMediaBudget()+ if irErr == nil {+ figures := extractDOCXFiguresFromIR(irJSON, budget)+ if len(figures) > 0 {+ fileMeta["figures"] = buildFiguresMap(figures)}}@@ -187,6 +192,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, dataOutputFormat: "markdown",File: fileMeta,Markdown: md,+ Warnings: budget.warnings(),}}diff --git a/internal/parser/parser/docx_parser_test.go b/internal/parser/parser/docx_parser_test.goindex 2d82ebec92..42e32d47d8 100644--- a/internal/parser/parser/docx_parser_test.go+++ b/internal/parser/parser/docx_parser_test.go@@ -29,7 +29,7 @@ func TestBuildDOCXJSONSections_Paragraphs(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"paragraph","content":[{"type":"text","text":"Hello world"}],"style":"Normal"}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -46,7 +46,7 @@ func TestBuildDOCXJSONSections_Headings(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"heading","level":1,"content":[{"type":"text","text":"Chapter 1"}]}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -67,7 +67,7 @@ func TestBuildDOCXJSONSections_Images(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"image","data":"` + b64 + `"}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -90,7 +90,7 @@ func TestBuildDOCXJSONSections_Tables(t *testing.T) {{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"A2"}]}]},{"content":[{"type":"paragraph","content":[{"type":"text","text":"B2"}]}]}]}]}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("got %d sections, want 1", len(sections))}@@ -120,7 +120,7 @@ func TestBuildDOCXJSONSections_MixedContent(t *testing.T) {{"type":"table","rows":[{"cells":[{"content":[{"type":"paragraph","content":[{"type":"text","text":"cell"}]}]}]}]},{"type":"heading","level":2,"content":[{"type":"text","text":"Sub title"}]}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 4 {t.Fatalf("got %d sections, want 4", len(sections))}@@ -153,7 +153,7 @@ func TestBuildDOCXJSONSections_List(t *testing.T) {{"content":[{"type":"paragraph","content":[{"type":"text","text":"Item 3"}]}]}],"level":0}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 3 {t.Fatalf("got %d sections, want 3 (each list item should be a section)", len(sections))}@@ -171,7 +171,7 @@ func TestBuildDOCXJSONSections_TextBox(t *testing.T) {ir := `{"sections":[{"title":"","elements":[{"type":"text_box","content":[{"type":"paragraph","content":[{"type":"text","text":"Boxed paragraph"}]}],"width_emu":null}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("got %d sections, want 1 (text_box content should become a section)", len(sections))}@@ -192,7 +192,7 @@ func TestBuildDOCXJSONSections_MixedWithList(t *testing.T) {],"level":0},{"type":"paragraph","content":[{"type":"text","text":"Trailer"}],"style":"Normal"}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 4 {t.Fatalf("got %d sections, want 4 (para + 2 list items + para)", len(sections))}@@ -210,7 +210,7 @@ func TestBuildDOCXJSONSections_EmptySkipped(t *testing.T) {{"type":"paragraph","content":[{"type":"text","text":"Only this matters"}]},{"type":"table","rows":[]}]}]}`- sections := buildDOCXJSONSections(ir)+ sections := buildDOCXJSONSections(ir, newEmbeddedMediaBudget())if len(sections) != 1 {t.Fatalf("got %d sections, want 1 (only non-empty paragraph)", len(sections))}diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.gonew file mode 100644index 0000000000..5df4e73238--- /dev/null+++ b/internal/parser/parser/html_media_test.go@@ -0,0 +1,82 @@+package parser++import (+ "context"+ "encoding/base64"+ "strings"+ "testing"+)++const htmlMediaDataURI = "data:image/png;base64,aGVsbG8="++func TestHTMLParser_EmitsInlineImagesInDocumentOrder(t *testing.T) {+ input := `<p>before <img alt="figure alt" src="` + htmlMediaDataURI + `"> after</p>`+ result := NewHTMLParser().ParseWithResult(context.Background(), "inline.html", []byte(input))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ if len(result.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none for a valid inline image", result.Warnings)+ }+ if len(result.JSON) != 3 {+ t.Fatalf("items = %+v, want text/image/text", result.JSON)+ }+ if result.JSON[0]["text"] != "before" || result.JSON[0]["doc_type_kwd"] != "text" {+ t.Errorf("before item = %+v", result.JSON[0])+ }+ imageItem := result.JSON[1]+ if imageItem["doc_type_kwd"] != "image" || imageItem["image"] != htmlMediaDataURI || imageItem["text"] != "figure alt" || imageItem["media_order"] != 1 {+ t.Errorf("image item = %+v", imageItem)+ }+ if result.JSON[2]["text"] != "after" || result.JSON[2]["doc_type_kwd"] != "text" {+ t.Errorf("after item = %+v", result.JSON[2])+ }+}++func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {+ input := `<table><tr><td>A<img alt="first" src="` + htmlMediaDataURI + `"></td><td><img src="https://example.com/second.png"></td></tr></table>`+ result := NewHTMLParser().ParseWithResult(context.Background(), "table.html", []byte(input))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ if len(result.JSON) != 2 {+ t.Fatalf("items = %+v, want table and one inline image item", result.JSON)+ }+ if result.JSON[0]["doc_type_kwd"] != "table" || result.JSON[0]["source_table_id"] != "html-table-1" {+ t.Errorf("table item = %+v", result.JSON[0])+ }+ tableText, _ := result.JSON[0]["text"].(string)+ if strings.Contains(tableText, "data:image/") {+ t.Error("table text duplicates the separately emitted inline image payload")+ }+ if !strings.Contains(tableText, `alt="first"`) {+ t.Errorf("table text lost the image alt text: %q", tableText)+ }+ if result.JSON[1]["image"] != htmlMediaDataURI {+ t.Errorf("inline image payload = %v, want data URI", result.JSON[1]["image"])+ }+ item := result.JSON[1]+ if item["doc_type_kwd"] != "image" || item["parent_table_id"] != "html-table-1" {+ t.Errorf("inline image item = %+v", item)+ }+ if item["row_index"] != 1 || item["column_index"] != 1 || item["media_order"] != 1 {+ t.Errorf("inline image position = %+v", item)+ }+}++func TestHTMLParser_RejectsOversizedInlineImagePayload(t *testing.T) {+ payload := base64.StdEncoding.EncodeToString(make([]byte, maxEmbeddedImageBytes+1))+ input := "<p><img alt=\"large\" src=\"data:image/png;base64," + payload + "\"></p>"+ result := NewHTMLParser().ParseWithResult(context.Background(), "large.html", []byte(input))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ for _, item := range result.JSON {+ if item["doc_type_kwd"] == "image" {+ t.Fatal("oversized data URI emitted as an image item")+ }+ }+ if len(result.Warnings) == 0 {+ t.Fatal("oversized inline image should produce a parser warning")+ }+}diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex ae34b6e29f..1a8a94f3d1 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -19,7 +19,9 @@ package parserimport ("bytes""context"+ "encoding/base64""fmt"+ "io""strings""golang.org/x/net/html"@@ -53,19 +55,8 @@ func (p *HTMLParser) ConfigureFromSetup(setup map[string]any) {}}-// ParseWithResult emits one item per block-level HTML element-// (headings, paragraphs, lists, pre blocks). The walker is a-// pure-Go replacement for the previous `fmt.Printf` debug output:-// it descends the html.Parse tree, joins the leaf text of each-// block-level element, and emits the python-compatible-// `{text, doc_type_kwd:"text"}` shape.-//-// Phase 2.5 (Slice 1) of port-rag-flow-pipeline-to-go.md makes-// HTMLParser a ParseResultProducer so the dispatch seam routes-// the html family through the structured path. Inline formatting-// (bold / links / images) is intentionally NOT surfaced as a-// separate ck_type — the python HtmlParser collapses inline-// formatting into the parent block's text.+// ParseWithResult emits normalized text items for block-level HTML elements+// and independent image items for valid inline image payloads.func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {// x/net/html assumes UTF-8 input, so a GBK/Big5/Shift-JIS page would// otherwise surface as U+FFFD mojibake. Decode first, mirroring the@@ -87,7 +78,11 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{Err: fmt.Errorf("html parse: %w", err)}}var items []map[string]any- walkHTMLBlocks(doc, &items)+ state := &htmlWalkState{}+ walkHTMLBlocksWithState(doc, &items, state)+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }// remove_toc: post-parse text heuristic (mirrors Python// parser.py:1087-1088 remove_toc → remove_contents_table).if p.RemoveTOC {@@ -102,7 +97,8 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data"name": filename,"encoding": encName,},- JSON: items,+ JSON: items,+ Warnings: state.warnings(),}}@@ -112,12 +108,23 @@ func decodeHTMLToUTF8(data []byte) ([]byte, string) {return DecodeToUTF8(data, "text/html")}-// walkHTMLBlocks emits one normalized item per block-level-// descendant of root. Inline elements (b, i, a, span, …) are-// collapsed into the parent's text via leafText. <script>,-// <style>, and <noscript> blocks are skipped entirely so they-// don't pollute the downstream chunker input.-func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {+type htmlWalkState struct {+ tableSequence int+ mediaOrder int+ skippedInlineImages int+}++func (s *htmlWalkState) warnings() []string {+ if s.skippedInlineImages == 0 {+ return nil+ }+ return []string{fmt.Sprintf("HTML parser skipped %d invalid or oversized inline image(s)", s.skippedInlineImages)}+}++// walkHTMLBlocksWithState emits normalized text and image items in document+// order. Inline formatting is collapsed into its parent text; script, style,+// and noscript subtrees are skipped.+func walkHTMLBlocksWithState(root *html.Node, out *[]map[string]any, state *htmlWalkState) {for child := root.FirstChild; child != nil; child = child.NextSibling {if child.Type == html.TextNode {if emitsLooseHTMLText(root) {@@ -138,7 +145,7 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {continuecase "html", "body":// Wrapper elements: descend into their children.- walkHTMLBlocks(child, out)+ walkHTMLBlocksWithState(child, out, state)continuecase "table":// Emit the <table> as a SINGLE structured doc_type_kwd:"table"@@ -149,19 +156,12 @@ func walkHTMLBlocks(root *html.Node, out *[]map[string]any) {// table context). We emit ONLY this item — no duplicate// doc_type_kwd:"text" copy — so the table is embedded once and// its markup does not pollute neighbouring prose chunks.- markup := renderTableHTML(child)- if strings.TrimSpace(markup) != "" {- *out = append(*out, map[string]any{- "text": markup,- "doc_type_kwd": "table",- "ck_type": "table",- })- }+ emitHTMLTable(child, out, state)continue}ckType := htmlTagToCkType(tag)trim := tag != "pre" && tag != "textarea"- htmlLeafText(child, out, ckType, trim)+ htmlLeafTextWithState(child, out, ckType, trim, state)}}@@ -285,7 +285,7 @@ func (w *leafWriter) hardBreak() {w.endsNL = true}-// htmlLeafText joins the visible text of an HTML node and its+// htmlLeafTextWithState joins the visible text of an HTML node and its// descendants and emits items directly into out. <script>/<style>/<noscript>// subtrees are skipped. Whitespace is folded per CSS rules (so// "<h1>Hello world</h1>" becomes "Hello world" and "<br>" survives as a@@ -297,10 +297,10 @@ func (w *leafWriter) hardBreak() {// doc_type_kwd:"table" item at its document position (see walkHTMLLeaf's// "table" case) — the prose around it is flushed as ordinary text items, so// the table is never relocated to the end and never duplicated.-func htmlLeafText(n *html.Node, out *[]map[string]any, ckType string, trim bool) {+func htmlLeafTextWithState(n *html.Node, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {var b bytes.Bufferw := &leafWriter{b: &b}- walkHTMLLeaf(n, w, out)+ walkHTMLLeaf(n, w, out, ckType, trim, state)flushLeafText(w, out, ckType, trim)}@@ -325,7 +325,7 @@ func flushLeafText(w *leafWriter, out *[]map[string]any, ckType string, trim boow.endsNL = false}-func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {+func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any, ckType string, trim bool, state *htmlWalkState) {switch n.Type {case html.TextNode:w.writeText(n.Data)@@ -337,11 +337,23 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {w.hardBreak()return}+ if n.Data == "img" {+ src := htmlAttribute(n, "src")+ if !usableHTMLImageSource(src) {+ if isHTMLDataImageSource(src) {+ state.skippedInlineImages+++ }+ return+ }+ flushLeafText(w, out, ckType, trim)+ appendHTMLImageItem(out, state, src, htmlAttribute(n, "alt"), "", 0, 0)+ return+ }if n.Data == "pre" || n.Data == "textarea" {// Verbatim: no folding, no injected block breaks.w.pre = truefor child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLLeaf(child, w, out)+ walkHTMLLeaf(child, w, out, ckType, trim, state)}w.pre = falsereturn@@ -355,15 +367,8 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {// row/column structure survives; we do NOT also inline the markup// into the parent's text, which would duplicate the table and// pollute the prose chunk with raw tags.- markup := renderTableHTML(n)- if strings.TrimSpace(markup) != "" {- flushLeafText(w, out, "text", true)- *out = append(*out, map[string]any{- "text": markup,- "doc_type_kwd": "table",- "ck_type": "table",- })- }+ flushLeafText(w, out, "text", true)+ emitHTMLTable(n, out, state)return}// Add a line break between block children so headings, paragraphs,@@ -378,7 +383,7 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {}}for child := n.FirstChild; child != nil; child = child.NextSibling {- walkHTMLLeaf(child, w, out)+ walkHTMLLeaf(child, w, out, ckType, trim, state)}if !w.pre && isBlockTag(n.Data) && w.b.Len() > 0 && !w.endsNL {w.hardBreak()@@ -386,6 +391,201 @@ func walkHTMLLeaf(n *html.Node, w *leafWriter, out *[]map[string]any) {}}+type htmlTableImage struct {+ src string+ alt string+ row int+ column int+}++func emitHTMLTable(n *html.Node, out *[]map[string]any, state *htmlWalkState) {+ images, skippedInlineImages := htmlTableImages(n)+ state.skippedInlineImages += skippedInlineImages+ markup := renderTableHTML(n)+ if strings.TrimSpace(markup) == "" {+ return+ }+ table := map[string]any{+ "text": markup,+ "doc_type_kwd": "table",+ "ck_type": "table",+ }+ tableID := ""+ if len(images) > 0 {+ state.tableSequence+++ tableID = fmt.Sprintf("html-table-%d", state.tableSequence)+ table["source_table_id"] = tableID+ }+ *out = append(*out, table)+ for _, media := range images {+ appendHTMLImageItem(out, state, media.src, media.alt, tableID, media.row, media.column)+ }+}++func htmlTableImages(table *html.Node) ([]htmlTableImage, int) {+ var images []htmlTableImage+ skippedInlineImages := 0+ rowCount := 0+ columnCounts := make(map[int]int)+ var walk func(node *html.Node, row, column int)+ walk = func(node *html.Node, row, column int) {+ if node.Type == html.ElementNode {+ switch node.Data {+ case "tr":+ rowCount+++ row = rowCount+ column = 0+ case "td", "th":+ columnCounts[row]+++ column = columnCounts[row]+ case "img":+ src := htmlAttribute(node, "src")+ if !usableHTMLImageSource(src) {+ if isHTMLDataImageSource(src) {+ skippedInlineImages+++ removeHTMLImageSource(node)+ }+ } else {+ images = append(images, htmlTableImage{+ src: src,+ alt: htmlAttribute(node, "alt"),+ row: row,+ column: column,+ })+ // The image is emitted as its own item below, so leave its+ // alt text and table cell in the markup without duplicating+ // the (potentially large) data URI.+ removeHTMLImageSource(node)+ }+ }+ }+ for child := node.FirstChild; child != nil; child = child.NextSibling {+ walk(child, row, column)+ }+ }+ walk(table, 0, 0)+ return images, skippedInlineImages+}++func removeHTMLImageSource(n *html.Node) {+ attrs := n.Attr[:0]+ for _, attr := range n.Attr {+ if !strings.EqualFold(attr.Key, "src") {+ attrs = append(attrs, attr)+ }+ }+ n.Attr = attrs+}++func appendHTMLImageItem(out *[]map[string]any, state *htmlWalkState, src, alt, parentTableID string, row, column int) {+ if comma := strings.IndexByte(src, ','); comma > 0 && len(src) >= len("data:image/") && strings.EqualFold(src[:len("data:image/")], "data:image/") {+ src = strings.ToLower(src[:comma]) + src[comma:]+ }+ state.mediaOrder+++ item := map[string]any{+ "text": strings.TrimSpace(alt),+ "doc_type_kwd": "image",+ "ck_type": "image",+ "media_order": state.mediaOrder,+ }+ item["image"] = src+ if parentTableID != "" {+ item["parent_table_id"] = parentTableID+ item["row_index"] = row+ item["column_index"] = column+ }+ *out = append(*out, item)+}++func usableHTMLImageSource(src string) bool {+ src = strings.TrimSpace(src)+ if !isHTMLDataImageSource(src) {+ return false+ }+ separator := strings.IndexByte(src, ',')+ if separator <= len("data:image/") || separator > 128 || !strings.EqualFold(src[separator-len(";base64"):separator], ";base64") {+ return false+ }+ return validBase64ImagePayload(src[separator+1:], maxEmbeddedImageBytes)+}++func isHTMLDataImageSource(src string) bool {+ const dataPrefix = "data:image/"+ src = strings.TrimSpace(src)+ return len(src) >= len(dataPrefix) && strings.EqualFold(src[:len(dataPrefix)], dataPrefix)+}++func validBase64ImagePayload(payload string, maxDecodedBytes int) bool {+ if maxDecodedBytes < 0 {+ return false+ }+ maxEncodedBytes := base64.StdEncoding.EncodedLen(maxDecodedBytes)+ encodedBytes := 0+ for i := 0; i < len(payload); i++ {+ if payload[i] == '\r' || payload[i] == '\n' {+ continue+ }+ encodedBytes+++ if encodedBytes > maxEncodedBytes {+ return false+ }+ }+ if encodedBytes == 0 {+ return false+ }+ decodedSize, ok := base64DecodedSize(payload, encodedBytes)+ if !ok || decodedSize > int64(maxDecodedBytes) {+ return false+ }++ for _, encoding := range []*base64.Encoding{base64.StdEncoding, base64.RawStdEncoding} {+ decoder := base64.NewDecoder(encoding, strings.NewReader(payload))+ decodedBytes, err := io.CopyN(io.Discard, decoder, int64(maxDecodedBytes)+1)+ if decodedBytes > int64(maxDecodedBytes) {+ return false+ }+ if err == io.EOF {+ return decodedBytes > 0+ }+ }+ return false+}++func base64DecodedSize(payload string, encodedBytes int) (int64, bool) {+ padding := 0+ sawPadding := false+ for i := 0; i < len(payload); i++ {+ if payload[i] == '\r' || payload[i] == '\n' {+ continue+ }+ if payload[i] == '=' {+ sawPadding = true+ padding+++ if padding > 2 {+ return 0, false+ }+ continue+ }+ if sawPadding {+ return 0, false+ }+ }+ if encodedBytes%4 == 1 || (padding > 0 && encodedBytes%4 != 0) {+ return 0, false+ }+ decodedSize := int64(encodedBytes/4)*3 + int64(encodedBytes%4)*3/4 - int64(padding)+ return decodedSize, decodedSize >= 0+}++func htmlAttribute(n *html.Node, key string) string {+ for _, attr := range n.Attr {+ if strings.EqualFold(attr.Key, key) {+ return attr.Val+ }+ }+ return ""+}+func isBlockTag(tag string) bool {switch tag {case "h1", "h2", "h3", "h4", "h5", "h6", "p", "li", "pre",diff --git a/internal/parser/parser/image_media.go b/internal/parser/parser/image_media.gonew file mode 100644index 0000000000..cce226f896--- /dev/null+++ b/internal/parser/parser/image_media.go@@ -0,0 +1,73 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package parser++import (+ "context"+ "sync"++ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"+)++const (+ // MaxImagePayloadBytes bounds compressed image data accepted by media paths.+ MaxImagePayloadBytes = 32 << 20+ // MaxImagePixels bounds decoded image dimensions before allocating a raster.+ MaxImagePixels = 40_000_000+ // MaxImageEdge bounds either decoded image dimension before raster allocation.+ MaxImageEdge = 12_000+)++// AcquireImageMedia limits concurrent image materialization across Parser+// payload encoding and ingestion VLM crop paths. The native analyzer keeps+// its own inference limit.+func AcquireImageMedia(ctx context.Context) (func(), error) {+ return sharedImageMediaAdmission().acquire(ctx)+}++type imageMediaAdmission struct {+ slots chan struct{}+}++func newImageMediaAdmission(limit int) *imageMediaAdmission {+ if limit < 1 {+ limit = 1+ }+ return &imageMediaAdmission{slots: make(chan struct{}, limit)}+}++func (a *imageMediaAdmission) acquire(ctx context.Context) (func(), error) {+ select {+ case a.slots <- struct{}{}:+ var once sync.Once+ return func() { once.Do(func() { <-a.slots }) }, nil+ case <-ctx.Done():+ return nil, ctx.Err()+ }+}++var (+ imageMediaAdmissionOnce sync.Once+ imageMediaAdmissionPool *imageMediaAdmission+)++func sharedImageMediaAdmission() *imageMediaAdmission {+ imageMediaAdmissionOnce.Do(func() {+ imageMediaAdmissionPool = newImageMediaAdmission(deepdocpdf.DeepDocConcurrency())+ })+ return imageMediaAdmissionPool+}diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 5e35e8e173..49e4431993 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -108,16 +108,30 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, ddoc := markdownNew().Parse([]byte(rendered))var items []map[string]any- walkMarkdownBlocksWithImages(doc, &items, p.FlattenMediaToText, p.FetchRemoteImages)+ mediaBudget := newEmbeddedMediaBudget()+ imageCtx, cancel := context.WithTimeout(ctx, 60*time.Second)+ defer cancel()+ unresolvedImages := walkMarkdownBlocksWithImages(imageCtx, doc, &items, p.FlattenMediaToText, p.FetchRemoteImages, mediaBudget)+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }if items == nil {items = []map[string]any{{"text": "", "doc_type_kwd": "text"}}}+ warnings := mediaBudget.warnings()+ if mediaBudget.items > 0 && imageCtx.Err() == context.DeadlineExceeded && ctx.Err() == nil {+ warnings = append(warnings, "Markdown image processing reached the document time budget")+ }+ if unresolvedImages > 0 {+ warnings = append(warnings, fmt.Sprintf("Markdown parser could not resolve %d image source(s)", unresolvedImages))+ }return ParseResult{OutputFormat: "json",File: map[string]any{"name": filename,},- JSON: items,+ JSON: items,+ Warnings: warnings,}}@@ -376,7 +390,8 @@ func markdownTableCells(line string) []string {// doc_type_kwd:"table" to keep the table whole and attach table context// to neighbouring chunks (chunker/token.go). Non-table HTML blocks// (<div>, <style>, …) are emitted as ordinary text with no ck_type.-func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool) {+func walkMarkdownBlocksWithImages(ctx context.Context, doc ast.Node, out *[]map[string]any, flatten, fetchRemoteImages bool, budget *embeddedMediaBudget) int {+ unresolvedImages := 0for _, child := range doc.GetChildren() {var ckType stringvar docTypeKwd string@@ -457,8 +472,36 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten,if imgURL, ok := findBlockImage(child); ok {isRemote := strings.HasPrefix(imgURL, "http://") || strings.HasPrefix(imgURL, "https://")if fetchRemoteImages || !isRemote {- if imgData, resolved := resolveImageURL(imgURL); resolved && imgData != "" {- item["image"] = imgData+ switch {+ case isRemote && ctx.Err() != nil:+ item["media_omitted"] = true+ case !budget.reserveItem():+ item["media_omitted"] = true+ default:+ encoded, raw, resolved := resolveImageURL(ctx, imgURL)+ if !resolved {+ unresolvedImages+++ break+ }+ size := len(raw)+ if raw == nil {+ encodedBytes := len(encoded) - strings.Count(encoded, "\r") - strings.Count(encoded, "\n")+ decodedSize, ok := base64DecodedSize(encoded, encodedBytes)+ if !ok {+ unresolvedImages+++ break+ }+ size = int(decodedSize)+ }+ if !budget.includeReservedSize(size) {+ item["media_omitted"] = true+ break+ }+ if raw != nil {+ item["image"] = base64.StdEncoding.EncodeToString(raw)+ } else {+ item["image"] = encoded+ }if !flatten {item["doc_type_kwd"] = "image"}@@ -468,6 +511,7 @@ func walkMarkdownBlocksWithImages(doc ast.Node, out *[]map[string]any, flatten,*out = append(*out, item)}+ return unresolvedImages}// isTableHTML reports whether block text is an outer <table> element (the@@ -502,44 +546,53 @@ func findBlockImage(n ast.Node) (string, bool) {return url, found}-// resolveImageURL resolves a Markdown image URL to its base64-encoded data.+// resolveImageURL resolves a Markdown image URL to an encoded payload or raw+// bytes.// Supports:-// - data:image/... URIs → decoded directly+// - data:image/... URIs → validated encoded payload// - http:// / https:// URLs → fetched (with basic SSRF filtering)//// Local / relative paths are not fetched (security). Returns-// (base64String, true) on success, ("", false) when resolution fails.-func resolveImageURL(imageURL string) (string, bool) {- if strings.HasPrefix(imageURL, dataURIPrefix) {+// exactly one of encoded and raw populated on success.+func resolveImageURL(ctx context.Context, imageURL string) (encoded string, raw []byte, ok bool) {+ if strings.HasPrefix(strings.ToLower(imageURL), dataURIPrefix) {// data:image/png;base64,xxxxidx := strings.Index(imageURL, "base64,")if idx < 0 {- return "", false+ idx = strings.Index(strings.ToLower(imageURL), "base64,")}- return imageURL[idx+len("base64,"):], true+ if idx < 0 {+ return "", nil, false+ }+ encoded = imageURL[idx+len("base64,"):]+ if !validBase64ImagePayload(encoded, MaxImagePayloadBytes) {+ return "", nil, false+ }+ return encoded, nil, true}if strings.HasPrefix(imageURL, "http://") || strings.HasPrefix(imageURL, "https://") {- b64, err := fetchImageAsBase64(imageURL)+ raw, err := fetchImage(ctx, imageURL)if err != nil {- return "", false+ return "", nil, false}- return b64, true+ return "", raw, true}// Local / relative paths — not fetched for security.- return "", false+ return "", nil, false}-// fetchImageAsBase64 fetches an HTTP(S) image URL and returns its-// content as a base64-encoded string. Local/private addresses and-// redirects to them are rejected (SSRF guard). Hostnames are resolved-// once and the validated IP is pinned in a custom DialContext to-// prevent DNS-rebinding TOCTOU attacks.-func fetchImageAsBase64(rawURL string) (string, error) {+// fetchImage fetches a bounded HTTP(S) image payload. Local/private addresses+// and redirects to them are rejected (SSRF guard). Hostnames are resolved+// once and the validated IP is pinned in a custom DialContext to prevent+// DNS-rebinding TOCTOU attacks.+func fetchImage(ctx context.Context, rawURL string) ([]byte, error) {rawURL = strings.TrimSpace(rawURL)+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)+ defer cancel()parsed, err := url.Parse(rawURL)if err != nil {- return "", fmt.Errorf("markdown: invalid image URL: %w", err)+ return nil, fmt.Errorf("markdown: invalid image URL: %w", err)}// pinned maps hostname (without port) → validated IP. The hostname@@ -549,7 +602,7 @@ func fetchImageAsBase64(rawURL string) (string, error) {pinned := make(map[string]net.IP)pinHost := func(host string) error {- ip, err := resolveAndValidateHost(host)+ ip, err := resolveAndValidateHost(ctx, host)if err != nil {return err}@@ -564,7 +617,7 @@ func fetchImageAsBase64(rawURL string) (string, error) {}if err := pinHost(parsed.Host); err != nil {- return "", err+ return nil, err}transport := &http.Transport{@@ -594,32 +647,36 @@ func fetchImageAsBase64(rawURL string) (string, error) {},}- ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)- defer cancel()req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil)if err != nil {- return "", fmt.Errorf("markdown: create image request: %w", err)+ return nil, fmt.Errorf("markdown: create image request: %w", err)}resp, err := client.Do(req)if err != nil {- return "", fmt.Errorf("markdown: fetch image %s: %w", rawURL, err)+ return nil, fmt.Errorf("markdown: fetch image %s: %w", rawURL, err)}defer resp.Body.Close()if resp.StatusCode < 200 || resp.StatusCode >= 300 {- return "", fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)+ return nil, fmt.Errorf("markdown: fetch image %s: HTTP %d", rawURL, resp.StatusCode)}- body, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024)) // 32 MiB cap+ body, err := io.ReadAll(io.LimitReader(resp.Body, MaxImagePayloadBytes+1))if err != nil {- return "", fmt.Errorf("markdown: read image %s: %w", rawURL, err)+ return nil, fmt.Errorf("markdown: read image %s: %w", rawURL, err)+ }+ if len(body) > MaxImagePayloadBytes {+ return nil, fmt.Errorf("markdown: image %s exceeds %d-byte limit", rawURL, MaxImagePayloadBytes)+ }+ if len(body) == 0 {+ return nil, fmt.Errorf("markdown: image %s is empty", rawURL)}- return base64.StdEncoding.EncodeToString(body), nil+ return body, nil}// resolveAndValidateHost resolves a host (which may include a port),// validates none of its IPs are internal/private, and returns the// first public IP for connection pinning.-func resolveAndValidateHost(host string) (net.IP, error) {+func resolveAndValidateHost(ctx context.Context, host string) (net.IP, error) {hostname := hostif h, _, err := net.SplitHostPort(host); err == nil {hostname = h@@ -634,7 +691,7 @@ func resolveAndValidateHost(host string) (net.IP, error) {return ip, nil}- addrs, err := net.DefaultResolver.LookupIPAddr(context.Background(), hostname)+ addrs, err := net.DefaultResolver.LookupIPAddr(ctx, hostname)if err != nil {return nil, fmt.Errorf("markdown: cannot resolve image host: %s", hostname)}diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex 610c312c79..07e475c720 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -1,11 +1,13 @@package parserimport (+ "context""encoding/base64""net/http""net/http/httptest""os""strings"+ "sync/atomic""testing")@@ -438,18 +440,18 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {func TestResolveImageURL_DataURI(t *testing.T) {b64 := base64.StdEncoding.EncodeToString([]byte("fakeimage"))- result, found := resolveImageURL("data:image/png;base64," + b64)+ result, raw, found := resolveImageURL(context.Background(), "data:image/png;base64,"+b64)if !found {t.Fatal("expected image found for data URI")}- if result != b64 {+ if result != b64 || raw != nil {t.Fatalf("got %q, want %q", result, b64)}}func TestResolveImageURL_LocalPathNotFetched(t *testing.T) {// Local / relative paths are not fetched (security); resolution fails.- if _, found := resolveImageURL("./local/image.png"); found {+ if _, _, found := resolveImageURL(context.Background(), "./local/image.png"); found {t.Fatal("expected no image resolved for a local path")}}@@ -468,13 +470,52 @@ func TestResolveImageURL_HTTPImage(t *testing.T) {}))defer ts.Close()- result, found := resolveImageURL(ts.URL + "/image.png")+ encoded, result, found := resolveImageURL(context.Background(), ts.URL+"/image.png")if !found {t.Fatal("expected image found for HTTP URL")}- expectedB64 := base64.StdEncoding.EncodeToString([]byte("fake-png-bytes"))- if result != expectedB64 {- t.Fatalf("got %q, want %q", result, expectedB64)+ if encoded != "" || string(result) != "fake-png-bytes" {+ t.Fatalf("got encoded=%q raw=%q, want raw image bytes", encoded, result)+ }+}++func TestMarkdownImageBudgetSkipsFetchAfterCountLimit(t *testing.T) {+ previous := ssrfAllowLoopback+ ssrfAllowLoopback = true+ t.Cleanup(func() { ssrfAllowLoopback = previous })++ var fetches atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ fetches.Add(1)+ w.Header().Set("Content-Type", "image/png")+ _, _ = w.Write([]byte("abc"))+ }))+ defer server.Close()++ markdown := "\n\n" ++ "\n\n" ++ "\n"+ doc := markdownNew().Parse([]byte(markdown))+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}+ var items []map[string]any+ if unresolved := walkMarkdownBlocksWithImages(t.Context(), doc, &items, false, true, budget); unresolved != 0 {+ t.Fatalf("unresolved images = %d, want 0", unresolved)+ }+ if got := fetches.Load(); got != 2 {+ t.Fatalf("remote fetches = %d, want 2 before the item limit", got)+ }+ if len(items) != 3 || items[0]["image"] != base64.StdEncoding.EncodeToString([]byte("abc")) {+ t.Fatalf("first image was not retained: %+v", items)+ }+ for _, index := range []int{1, 2} {+ if items[index]["image"] != nil || items[index]["media_omitted"] != true {+ t.Fatalf("image %d = %+v, want omitted payload", index, items[index])+ }+ }+ warnings := strings.Join(budget.warnings(), "\n")+ if !strings.Contains(warnings, "omitted 1 embedded image payload(s) after reaching the 4-byte document image budget") ||+ !strings.Contains(warnings, "stopped extracting embedded images after the 2-item document limit") {+ t.Fatalf("warnings = %q, want document byte and item limit warnings", warnings)}}@@ -517,21 +558,21 @@ func TestFindBlockImage(t *testing.T) {}}-func TestFetchImageAsBase64_RejectsCredentials(t *testing.T) {- _, err := fetchImageAsBase64("https://user:[email protected]/img.png")+func TestFetchImage_RejectsCredentials(t *testing.T) {+ _, err := fetchImage(context.Background(), "https://user:[email protected]/img.png")if err == nil {t.Fatal("expected error for URL with credentials")}}-func TestFetchImageAsBase64_InvalidURL(t *testing.T) {+func TestFetchImage_InvalidURL(t *testing.T) {withSSRFBypass(t)ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {w.WriteHeader(http.StatusNotFound)}))defer ts.Close()- _, err := fetchImageAsBase64(ts.URL + "/nonexistent.png")+ _, err := fetchImage(context.Background(), ts.URL+"/nonexistent.png")if err == nil {t.Fatal("expected error for 404 response")}diff --git a/internal/parser/parser/media_budget.go b/internal/parser/parser/media_budget.gonew file mode 100644index 0000000000..5e25ea1af3--- /dev/null+++ b/internal/parser/parser/media_budget.go@@ -0,0 +1,114 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package parser++import (+ "fmt"+)++const (+ maxEmbeddedImageBytes = MaxImagePayloadBytes+ maxEmbeddedDocumentImageBytes = 64 << 20+ maxEmbeddedMediaItems = 256+)++type embeddedMediaBudget struct {+ maxImageBytes int+ maxTotalBytes int+ maxItems int++ items int+ totalBytes int+ oversizedImages int+ documentOverflows int+ countExhausted bool+}++func newEmbeddedMediaBudget() *embeddedMediaBudget {+ return &embeddedMediaBudget{+ maxImageBytes: maxEmbeddedImageBytes,+ maxTotalBytes: maxEmbeddedDocumentImageBytes,+ maxItems: maxEmbeddedMediaItems,+ }+}++// include reserves budget for one image payload. The second result is false+// when callers should stop walking images because the document item limit has+// been reached. Byte-limit omissions can keep walking so later text and media+// metadata remain available.+func (b *embeddedMediaBudget) include(data []byte) (bool, bool) {+ if !b.reserveItem() {+ return false, false+ }+ return b.includeReservedSize(len(data)), true+}++// reserveItem lets a parser cap image source attempts before fetching bytes.+func (b *embeddedMediaBudget) reserveItem() bool {+ if b.countExhausted {+ return false+ }+ if b.items >= b.maxItems {+ b.countExhausted = true+ return false+ }+ b.items+++ return true+}++// includeReservedSize charges a previously reserved image without retaining+// or decoding its payload just to measure its size.+func (b *embeddedMediaBudget) includeReservedSize(size int) bool {+ if size > b.maxImageBytes {+ b.oversizedImages+++ return false+ }+ if size > b.maxTotalBytes-b.totalBytes {+ b.documentOverflows+++ return false+ }+ b.totalBytes += size+ return true+}++func (b *embeddedMediaBudget) warnings() []string {+ if b == nil {+ return nil+ }+ var warnings []string+ if b.oversizedImages > 0 {+ warnings = append(warnings, fmt.Sprintf(+ "omitted %d embedded image payload(s) larger than the %d-byte per-image limit",+ b.oversizedImages,+ b.maxImageBytes,+ ))+ }+ if b.documentOverflows > 0 {+ warnings = append(warnings, fmt.Sprintf(+ "omitted %d embedded image payload(s) after reaching the %d-byte document image budget",+ b.documentOverflows,+ b.maxTotalBytes,+ ))+ }+ if b.countExhausted {+ warnings = append(warnings, fmt.Sprintf(+ "stopped extracting embedded images after the %d-item document limit",+ b.maxItems,+ ))+ }+ return warnings+}diff --git a/internal/parser/parser/media_budget_test.go b/internal/parser/parser/media_budget_test.gonew file mode 100644index 0000000000..df8f13ff6a--- /dev/null+++ b/internal/parser/parser/media_budget_test.go@@ -0,0 +1,50 @@+package parser++import (+ "reflect"+ "testing"+)++func TestEmbeddedMediaBudgetReservedItemsAndWarnings(t *testing.T) {+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 5, maxItems: 3}++ if !budget.reserveItem() || !budget.includeReservedSize(3) {+ t.Fatal("first image should fit")+ }+ if !budget.reserveItem() || budget.includeReservedSize(3) {+ t.Fatal("second image should exceed the document byte limit")+ }+ if !budget.reserveItem() || budget.includeReservedSize(5) {+ t.Fatal("third image should exceed the per-image byte limit")+ }+ if budget.reserveItem() {+ t.Fatal("fourth image should be rejected before its source is read")+ }+ if included, keepWalking := budget.include([]byte("x")); included || keepWalking {+ t.Fatalf("include after count exhaustion = (%v, %v), want (false, false)", included, keepWalking)+ }+ if budget.items != 3 || budget.totalBytes != 3 {+ t.Fatalf("budget counters = (%d items, %d bytes), want (3, 3)", budget.items, budget.totalBytes)+ }+ want := []string{+ "omitted 1 embedded image payload(s) larger than the 4-byte per-image limit",+ "omitted 1 embedded image payload(s) after reaching the 5-byte document image budget",+ "stopped extracting embedded images after the 3-item document limit",+ }+ if got := budget.warnings(); !reflect.DeepEqual(got, want) {+ t.Fatalf("warnings = %q, want %q", got, want)+ }+}++func TestEmbeddedMediaBudgetIncludesExactByteLimits(t *testing.T) {+ budget := &embeddedMediaBudget{maxImageBytes: 4, maxTotalBytes: 4, maxItems: 2}+ if included, keepWalking := budget.include([]byte("abcd")); !included || !keepWalking {+ t.Fatalf("exact-limit image = (%v, %v), want (true, true)", included, keepWalking)+ }+ if included, keepWalking := budget.include([]byte("e")); included || !keepWalking {+ t.Fatalf("image after exact document limit = (%v, %v), want (false, true)", included, keepWalking)+ }+ if budget.totalBytes != 4 {+ t.Fatalf("totalBytes = %d, want 4", budget.totalBytes)+ }+}diff --git a/internal/parser/parser/office_image_fixture_cgo_test.go b/internal/parser/parser/office_image_fixture_cgo_test.gonew file mode 100644index 0000000000..c7be30b61c--- /dev/null+++ b/internal/parser/parser/office_image_fixture_cgo_test.go@@ -0,0 +1,120 @@+//go:build cgo++package parser++import (+ "bytes"+ "encoding/base64"+ "errors"+ "image"+ "image/color"+ "image/png"+ "os"+ "path/filepath"+ "testing"++ officeOxide "github.com/yfedoseev/office_oxide/go"+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"+ deepdoctype "ragflow/internal/deepdoc/parser/type"+)++func useFailingOCRAnalyzer(t *testing.T) {+ t.Helper()+ previous := deepdoctype.NativeDocAnalyzerFactory+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {+ return &deepdocpdf.MockDocAnalyzer{Healthy: true, OCRDetectErr: errors.New("image parser must not invoke OCR")}, true+ })+}++func TestDOCXParser_RealTableImage(t *testing.T) {+ useFailingOCRAnalyzer(t)+ data, err := os.ReadFile(filepath.Join("testdata", "docx_table_image.docx"))+ if err != nil {+ t.Fatal(err)+ }+ p := NewDOCXParser()+ p.ConfigureFromSetup(map[string]any{"output_format": "json"})+ result := p.ParseWithResult(t.Context(), "docx_table_image.docx", data)+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ if len(result.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)+ }+ var tableID string+ for _, item := range result.JSON {+ if item["doc_type_kwd"] == "table" {+ tableID, _ = item["source_table_id"].(string)+ }+ }+ if tableID == "" {+ t.Fatalf("table with image not found: %+v", result.JSON)+ }+ for _, item := range result.JSON {+ if item["doc_type_kwd"] != "image" || item["parent_table_id"] != tableID {+ continue+ }+ encoded, _ := item["image"].(string)+ decoded, err := base64.StdEncoding.DecodeString(encoded)+ if err != nil {+ t.Fatalf("table image payload: %v", err)+ }+ if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {+ t.Fatalf("table image is not PNG: %v", err)+ }+ if item["row_index"] != 1 || item["column_index"] != 1 || item["media_order"] != 1 {+ t.Fatalf("table image location = %+v", item)+ }+ return+ }+ t.Fatalf("no image linked to table %q: %+v", tableID, result.JSON)+}++func TestPPTXParser_RealSlideImage(t *testing.T) {+ useFailingOCRAnalyzer(t)+ raster := image.NewRGBA(image.Rect(0, 0, 32, 16))+ for y := 0; y < 16; y++ {+ for x := 0; x < 32; x++ {+ raster.Set(x, y, color.RGBA{R: 255, A: 255})+ }+ }+ var imageData bytes.Buffer+ if err := png.Encode(&imageData, raster); err != nil {+ t.Fatal(err)+ }+ w := officeOxide.NewPptxWriter()+ defer w.Close()+ slide := w.AddSlide()+ w.AddSlideText(slide, "slide with image")+ w.AddSlideImage(slide, imageData.Bytes(), "png", 914400, 914400, 914400, 457200)+ data, err := w.ToBytes()+ if err != nil {+ t.Fatal(err)+ }+ result := NewPPTXParser().ParseWithResult(t.Context(), "slide_image.pptx", data)+ if result.Err != nil {+ t.Fatal(result.Err)+ }+ if len(result.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)+ }+ for _, item := range result.JSON {+ if item["doc_type_kwd"] != "image" {+ continue+ }+ encoded, _ := item["image"].(string)+ decoded, err := base64.StdEncoding.DecodeString(encoded)+ if err != nil {+ t.Fatalf("slide image payload: %v", err)+ }+ if _, err := png.Decode(bytes.NewReader(decoded)); err != nil {+ t.Fatalf("slide image is not PNG: %v", err)+ }+ if item["slide_number"] != 1 || item["media_order"] != 1 {+ t.Fatalf("slide image location = %+v", item)+ }+ return+ }+ t.Fatalf("slide image not emitted: %+v", result.JSON)+}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex f8c3db1df5..0c22e5267b 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -297,7 +297,7 @@ func deprecatedHTML4Excel(setup map[string]any, parserName string) {// worksheet as structured parser items. Excelize exposes both kinds through// GetPictureCells/GetPictures; walking the reported anchor cells avoids// scanning the worksheet's entire coordinate space.-func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []string) {+func extractXLSXImages(f *excelize.File, sheet string, budget *embeddedMediaBudget) ([]map[string]any, []string) {cells, err := f.GetPictureCells(sheet)if err != nil {return nil, []string{fmt.Sprintf("XLSX image discovery failed for sheet %q: %v", sheet, err)}@@ -308,6 +308,7 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []striitems := make([]map[string]any, 0, len(cells))warnings := make([]string, 0)+cellLoop:for _, cell := range cells {pictures, err := f.GetPictures(sheet, cell)if err != nil {@@ -323,24 +324,37 @@ func extractXLSXImages(f *excelize.File, sheet string) ([]map[string]any, []striwarnings = append(warnings, fmt.Sprintf("XLSX image skipped for sheet %q cell %s: unsupported extension %q", sheet, cell, picture.Extension))continue}- encoded := base64.StdEncoding.EncodeToString(picture.File)+ included, keepWalking := true, true+ if budget != nil {+ included, keepWalking = budget.include(picture.File)+ }+ if !keepWalking {+ break cellLoop+ }alt := cellif picture.Format != nil && picture.Format.AltText != "" {alt = picture.Format.AltText}row, col := axisToRC(cell)- items = append(items, map[string]any{+ item := map[string]any{"text": alt,"doc_type_kwd": "image","ck_type": "image",- "image": "data:" + mimeType + ";base64," + encoded,"sheet": sheet,"cell": cell,"row_start": row,"row_end": row,"col_start": col,"col_end": col,- })+ }+ if included {+ encoded := base64.StdEncoding.EncodeToString(picture.File)+ item["image"] = "data:" + mimeType + ";base64," + encoded+ } else {+ item["image"] = nil+ item["media_omitted"] = true+ }+ items = append(items, item)}}return items, warningsdiff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex ddbfd3c1f1..6bb517a548 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -367,7 +367,7 @@ func TestXLSXImageMIMEType(t *testing.T) {func TestExtractXLSXImagesWarningForInvalidSheet(t *testing.T) {f := excelize.NewFile()defer f.Close()- _, warnings := extractXLSXImages(f, "MissingSheet")+ _, warnings := extractXLSXImages(f, "MissingSheet", nil)if len(warnings) != 1 || !strings.Contains(warnings[0], "image discovery failed") {t.Fatalf("warnings = %v, want image discovery warning", warnings)}diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex 4904151a1c..4d7752deb7 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -62,8 +62,8 @@ type ParseResult struct {// `[]map[string]any` with `text` + `doc_type_kwd` keys (and// optional `image` / `layout` / `positions` fields);// Markdown / HTML / text emit normalized- // `{text, doc_type_kwd}` items; image emits OCR/VLM result- // items.+ // `{text, doc_type_kwd}` items; image emits an image payload and, after+ // ingestion enhancement, its VLM description.JSON []map[string]any// Markdown is a backend response awaiting normalization.diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.goindex a7646a7b87..d60782848b 100644--- a/internal/parser/parser/parse_with_result_test.go+++ b/internal/parser/parser/parse_with_result_test.go@@ -241,12 +241,13 @@ Tail text`)}}-func TestWalkHTMLBlocks_SkipsHeadLooseText(t *testing.T) {+func TestWalkHTMLBlocksWithState_SkipsHeadLooseText(t *testing.T) {head := &html.Node{Type: html.ElementNode, Data: "head"}head.AppendChild(&html.Node{Type: html.TextNode, Data: "Head metadata"})var items []map[string]any- walkHTMLBlocks(head, &items)+ state := &htmlWalkState{}+ walkHTMLBlocksWithState(head, &items, state)if len(items) != 0 {t.Fatalf("JSON len = %d, want 0: %#v", len(items), items)}diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 8534d72f7f..f421c04888 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -380,8 +380,8 @@ func resolveDocAnalyzer(factory func() (deepdoctype.DocAnalyzer, bool)) (deepdoc// GetDocAnalyzer returns the configured in-process DeepDoc analyzer. It is the// single production entry point now that the external HTTP service is no longer-// a backend. Callers outside the parser package (e.g. standalone image OCR in-// the ingestion component) use this instead of constructing a client.+// a backend. PDF and non-PDF image parsers use it instead of constructing a+// client.func GetDocAnalyzer() (deepdoctype.DocAnalyzer, error) {return deepDocAnalyzerFromEnv()}diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.goindex 9148080038..cb97a28118 100644--- a/internal/parser/parser/picture_parser.go+++ b/internal/parser/parser/picture_parser.go@@ -14,23 +14,14 @@// limitations under the License.//-// PictureParser validates and stores configuration for image files.-// The actual OCR and VLM description is performed by the-// component-layer maybeDispatchImage, which mirrors Python's-// rag/app/picture.py:chunk() image branch.-//-// Python reference:-// - Image is opened with PIL, converted to RGB-// - PaddleOCR tried first (if layout_recognize == "@PaddleOCR")-// - Falls back to local deepdoc.vision.OCR (ONNX text detection)-// - If OCR text is short (≤32 chars / ≤32 words for English),-// calls IMAGE2TEXT VLM describe() for a natural-language description-// - Returns tokenized text with media context attached+// PictureParser validates image files and returns their image payload for+// optional VLM enhancement in the ingestion component.package parserimport ("context"+ "encoding/base64""fmt""path/filepath""strings"@@ -50,7 +41,7 @@ var imageExtensions = map[string]bool{"ai": true, "raw": true, "wmf": true,}-// PictureParser handles image files for OCR and VLM description.+// PictureParser emits image files for optional vision enhancement.type PictureParser struct {OutputFormat string}@@ -71,10 +62,7 @@ func (p *PictureParser) ConfigureFromSetup(setup map[string]any) {}}-// ParseWithResult implements ParseResultProducer. It validates the-// file extension against the image extension whitelist. The actual-// OCR and VLM description happens via maybeDispatchImage at the-// component layer (mirrors Python's picture.py:chunk()).+// ParseWithResult validates the image extension and returns its image payload.func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {ext := strings.ToLower(filepath.Ext(filename))if len(ext) > 1 && ext[0] == '.' {@@ -94,6 +82,9 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, daErr: fmt.Errorf("picture: unsupported extension %q (filename: %s); accepted: .jpg/.jpeg/.png/.gif/.bmp/.tiff/.tif/.webp/.svg/.ico/.avif/.heic/...", ext, filename),}}+ if len(data) == 0 || len(data) > MaxImagePayloadBytes {+ return ParseResult{Err: fmt.Errorf("picture: image payload size %d is outside the %d-byte limit", len(data), MaxImagePayloadBytes)}+ }// Parser output is normalized to JSON at the component boundary, so an// absent backend format defaults to JSON here as well.@@ -102,13 +93,55 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, daoutFmt = "json"}+ release, err := AcquireImageMedia(ctx)+ if err != nil {+ return ParseResult{Err: err}+ }+ imagePayload := "data:" + pictureMIME(ext) + ";base64," + base64.StdEncoding.EncodeToString(data)+ release()+ item := map[string]any{+ "text": "",+ "image": imagePayload,+ "doc_type_kwd": DocTypeImage,+ }+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }return ParseResult{OutputFormat: outFmt,File: map[string]any{"name": filename,"size": len(data),- "doc_type_kwd": "image",+ "doc_type_kwd": DocTypeImage,},+ JSON: []map[string]any{item},+ }+}++func pictureMIME(ext string) string {+ switch ext {+ case "jpg", "jpeg":+ return "image/jpeg"+ case "png":+ return "image/png"+ case "gif":+ return "image/gif"+ case "bmp":+ return "image/bmp"+ case "webp":+ return "image/webp"+ case "svg":+ return "image/svg+xml"+ case "tiff", "tif":+ return "image/tiff"+ case "ico":+ return "image/x-icon"+ case "avif":+ return "image/avif"+ case "heic":+ return "image/heic"+ default:+ return "image/png"}}diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.goindex c53360984d..6190b410ea 100644--- a/internal/parser/parser/picture_parser_test.go+++ b/internal/parser/parser/picture_parser_test.go@@ -17,10 +17,29 @@package parserimport (+ "encoding/base64""strings""testing")+func TestPictureParserKeepsImageWithoutOCR(t *testing.T) {+ data := []byte("picture bytes")+ res := NewPictureParser().ParseWithResult(t.Context(), "photo.png", data)+ if res.Err != nil {+ t.Fatal(res.Err)+ }+ if len(res.JSON) != 1 || res.JSON[0]["text"] != "" {+ t.Fatalf("picture text = %+v, want empty text", res.JSON)+ }+ wantImage := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)+ if res.JSON[0]["image"] != wantImage {+ t.Fatalf("picture image = %v, want %q", res.JSON[0]["image"], wantImage)+ }+ if len(res.Warnings) != 0 {+ t.Fatalf("picture warnings = %v, want none", res.Warnings)+ }+}+func TestNewPictureParser(t *testing.T) {p := NewPictureParser()if p == nil {diff --git a/internal/parser/parser/pptx_ir.go b/internal/parser/parser/pptx_ir.goindex e0507e53d0..35351c9bdf 100644--- a/internal/parser/parser/pptx_ir.go+++ b/internal/parser/parser/pptx_ir.go@@ -17,16 +17,17 @@package parserimport (+ "encoding/base64""encoding/json""fmt""strings")-// buildPPTXJSONSections converts office_oxide presentation IR JSON into-// one JSON item per slide. The IR carries exactly one section per slide,-// so section index + 1 is the slide number. Per-slide text is flattened-// with the shared IR walker (docxElementText), which covers paragraphs,-// headings, text boxes, tables, and (nested) lists.+// buildPPTXJSONSections converts office_oxide presentation IR JSON into one+// text item per slide followed by that slide's image items. The IR carries+// exactly one section per slide, so section index + 1 is the slide number.+// Per-slide text is flattened with the shared IR walker (docxElementText),+// which covers paragraphs, headings, text boxes, tables, and (nested) lists.//// A slide without extractable text (blank layout, image-only) still// yields an item with an empty text field, so slide_number stays@@ -34,11 +35,14 @@ import (//// Pure Go (no cgo) so the IR → items transform is unit-testable without// the office_oxide native library.-func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {+func buildPPTXJSONSections(irJSON string, budget *embeddedMediaBudget) ([]map[string]any, error) {var ir docxIRDocumentif err := json.Unmarshal([]byte(irJSON), &ir); err != nil {return nil, fmt.Errorf("presentation ir-json decode: %w", err)}+ if budget == nil {+ budget = newEmbeddedMediaBudget()+ }items := make([]map[string]any, 0, len(ir.Sections))for i, sec := range ir.Sections {var lines []string@@ -58,19 +62,47 @@ func buildPPTXJSONSections(irJSON string) ([]map[string]any, error) {"doc_type_kwd": "text","slide_number": i + 1,})+ mediaOrder := 0+ forEachDOCXIRImage(sec.Elements, func(data []byte) bool {+ mediaOrder+++ imageItem := map[string]any{+ "text": "",+ "image": nil,+ "doc_type_kwd": "image",+ "ck_type": "image",+ "slide_number": i + 1,+ "media_order": mediaOrder,+ }+ included, keepWalking := budget.include(data)+ if included {+ imageItem["image"] = base64.StdEncoding.EncodeToString(data)+ } else if keepWalking {+ imageItem["media_omitted"] = true+ } else {+ return false+ }+ items = append(items, imageItem)+ return keepWalking+ })}return items, nil}-// itemsAllEmpty reports whether every item carries no extractable text.-// A deck whose IR sections all flatten to "" (e.g. text the IR walker does-// not yet cover) must fall back to PlainText instead of emitting empty-// chunks that the Tokenizer later filters, silently yielding 0 chunks.+// itemsAllEmpty reports whether every item lacks both text and image payloads.+// A deck whose IR sections all flatten to "" and contain no images must fall+// back to PlainText instead of emitting empty chunks that the Tokenizer later+// filters, silently yielding 0 chunks.func itemsAllEmpty(items []map[string]any) bool {for _, it := range items {if text, _ := it["text"].(string); strings.TrimSpace(text) != "" {return false}+ if image, _ := it["image"].(string); strings.TrimSpace(image) != "" {+ return false+ }+ if omitted, _ := it["media_omitted"].(bool); omitted {+ return false+ }}return true}diff --git a/internal/parser/parser/pptx_ir_test.go b/internal/parser/parser/pptx_ir_test.goindex 7e5d708df4..dfa37c01a4 100644--- a/internal/parser/parser/pptx_ir_test.go+++ b/internal/parser/parser/pptx_ir_test.go@@ -103,11 +103,6 @@ func TestBuildPPTXJSONSections(t *testing.T) {irJSON: `{"sections":[{"elements":[{"type":"text","text":"loose text"}]}]}`,wantTexts: []string{"loose text"},},- {- name: "image-only slide yields empty text",- irJSON: `{"sections":[{"elements":[{"type":"image","data":"aGVsbG8="}]}]}`,- wantTexts: []string{""},- },{// bwbd.pptx regression: office_oxide wraps grouped slide shapes// (graphicFrame tables) in a text_box, so the slide's text_box@@ -176,7 +171,7 @@ func TestBuildPPTXJSONSections(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- items, err := buildPPTXJSONSections(tt.irJSON)+ items, err := buildPPTXJSONSections(tt.irJSON, newEmbeddedMediaBudget())if tt.wantErr {if err == nil {t.Fatalf("buildPPTXJSONSections: want error, got items %+v", items)@@ -204,6 +199,47 @@ func TestBuildPPTXJSONSections(t *testing.T) {}}+func TestBuildPPTXJSONSections_ExtractsSlideImagesInOrder(t *testing.T) {+ irJSON := `{"sections":[{"elements":[+ {"type":"paragraph","content":[{"type":"text","text":"slide text"}]},+ {"type":"image","data":"aGVsbG8="},+ {"type":"table","rows":[{"cells":[{"content":[{"type":"image","data":"aW1hZ2U="}]}]}]}+ ]},{"elements":[{"type":"image","data":"c2Vjb25k"}]}]}`++ items, err := buildPPTXJSONSections(irJSON, newEmbeddedMediaBudget())+ if err != nil {+ t.Fatalf("buildPPTXJSONSections: %v", err)+ }+ want := []struct {+ docType string+ text string+ image string+ slide int+ order int+ }{+ {docType: "text", text: "slide text", slide: 1},+ {docType: "image", image: "aGVsbG8=", slide: 1, order: 1},+ {docType: "image", image: "aW1hZ2U=", slide: 1, order: 2},+ {docType: "text", slide: 2},+ {docType: "image", image: "c2Vjb25k", slide: 2, order: 1},+ }+ if len(items) != len(want) {+ t.Fatalf("items = %+v, want %d slide/text and image items", items, len(want))+ }+ for i, w := range want {+ item := items[i]+ if item["doc_type_kwd"] != w.docType || item["slide_number"] != w.slide {+ t.Errorf("item %d identity = %+v, want type %q slide %d", i, item, w.docType, w.slide)+ }+ if w.docType == "image" && (item["image"] != w.image || item["media_order"] != w.order) {+ t.Errorf("item %d image = %+v, want payload %q order %d", i, item, w.image, w.order)+ }+ if w.docType == "text" && item["text"] != w.text {+ t.Errorf("item %d text = %v, want %q", i, item["text"], w.text)+ }+ }+}+// TestItemsAllEmpty pins the all-empty fallback gate: empty and whitespace// items trigger the PlainText salvage, any non-empty item does not.func TestItemsAllEmpty(t *testing.T) {@@ -224,6 +260,11 @@ func TestItemsAllEmpty(t *testing.T) {items: []map[string]any{{"text": ""}, {"text": "hello"}},want: false,},+ {+ name: "image payload",+ items: []map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"}},+ want: false,+ },}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex 059cd2f97b..8b7a6c9bb4 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -164,10 +164,14 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, dataJSON: itemsFromPlainText(text),}}- items, err := buildPPTXJSONSections(irJSON)+ budget := newEmbeddedMediaBudget()+ items, err := buildPPTXJSONSections(irJSON, budget)if err != nil {return ParseResult{Err: err}}+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }if len(items) == 0 || itemsAllEmpty(items) {// A deck whose IR carries no sections at all, or whose sections// all flatten to empty text (text the IR walker does not cover):@@ -186,5 +190,6 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, dataOutputFormat: "json",File: map[string]any{"name": filename, "format": effFormat},JSON: items,+ Warnings: budget.warnings(),}}diff --git a/internal/parser/parser/testdata/docx_table_image.docx b/internal/parser/parser/testdata/docx_table_image.docxnew file mode 100644index 0000000000..6ecdaf602dBinary files /dev/null and b/internal/parser/parser/testdata/docx_table_image.docx differdiff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex 7dc42a4269..23cde8c1a1 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -85,10 +85,15 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [}}- items, warnings, sheetsCount, err := parseXLSXBytes(data)+ mediaBudget := newEmbeddedMediaBudget()+ items, warnings, sheetsCount, err := parseXLSXBytes(data, mediaBudget)+ if err := ctx.Err(); err != nil {+ return ParseResult{Err: err}+ }if err != nil {return ParseResult{Err: fmt.Errorf("xls parse: %w", err)}}+ warnings = append(warnings, mediaBudget.warnings()...)return ParseResult{OutputFormat: spreadsheetOutputFormat,diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex 6f5d8dbf54..bf71d359cb 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -3,10 +3,16 @@ package parserimport ("bytes""context"+ "encoding/base64"+ "errors"+ "image"+ "image/png""strings""testing""github.com/xuri/excelize/v2"+ deepdocpdf "ragflow/internal/deepdoc/parser/pdf"+ deepdoctype "ragflow/internal/deepdoc/parser/type")func createTestExcelBytes(t *testing.T) []byte {@@ -112,3 +118,100 @@ func TestXLSXParser_JSONOutput(t *testing.T) {t.Errorf("resHTML.JSON should have items")}}++func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {+ var imageData bytes.Buffer+ if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {+ t.Fatalf("encode fixture: %v", err)+ }+ imageBytes := imageData.Bytes()+ f := excelize.NewFile()+ defer f.Close()+ if err := f.AddPictureFromBytes("Sheet1", "C3", &excelize.Picture{+ Extension: ".png",+ File: imageBytes,+ Format: &excelize.GraphicOptions{AltText: "sheet image"},+ }); err != nil {+ t.Fatalf("AddPictureFromBytes: %v", err)+ }+ var workbook bytes.Buffer+ if err := f.Write(&workbook); err != nil {+ t.Fatalf("write workbook: %v", err)+ }++ previous := deepdoctype.NativeDocAnalyzerFactory+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {+ return &deepdocpdf.MockDocAnalyzer{Healthy: true, OCRDetectErr: errors.New("xls OCR sentinel")}, true+ })++ p, err := NewXLSParser("excelize")+ if err != nil {+ t.Fatalf("NewXLSParser: %v", err)+ }+ result := p.ParseWithResult(t.Context(), "with-image.xls", workbook.Bytes())+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ var imageItem map[string]any+ for _, item := range result.JSON {+ if item["doc_type_kwd"] == "image" {+ imageItem = item+ break+ }+ }+ if imageItem == nil || imageItem["image"] != "data:image/png;base64,"+base64.StdEncoding.EncodeToString(imageBytes) {+ t.Fatalf("image item = %+v, want retained picture payload", imageItem)+ }+ if len(result.Warnings) != 0 {+ t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)+ }+}++func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {+ var imageData bytes.Buffer+ if err := png.Encode(&imageData, image.NewRGBA(image.Rect(0, 0, 2, 2))); err != nil {+ t.Fatalf("encode fixture: %v", err)+ }+ f := excelize.NewFile()+ defer f.Close()+ for i := 0; i <= maxEmbeddedMediaItems; i++ {+ cell, err := excelize.CoordinatesToCellName(3, i+1)+ if err != nil {+ t.Fatalf("cell %d: %v", i, err)+ }+ if err := f.AddPictureFromBytes("Sheet1", cell, &excelize.Picture{Extension: ".png", File: imageData.Bytes()}); err != nil {+ t.Fatalf("AddPictureFromBytes(%s): %v", cell, err)+ }+ }+ var workbook bytes.Buffer+ if err := f.Write(&workbook); err != nil {+ t.Fatalf("write workbook: %v", err)+ }++ previous := deepdoctype.NativeDocAnalyzerFactory+ t.Cleanup(func() { deepdoctype.NativeDocAnalyzerFactory = previous })+ deepdoctype.SetNativeDocAnalyzerFactory(func() (deepdoctype.DocAnalyzer, bool) {+ return &deepdocpdf.MockDocAnalyzer{Healthy: true}, true+ })+ p, err := NewXLSParser("excelize")+ if err != nil {+ t.Fatalf("NewXLSParser: %v", err)+ }+ result := p.ParseWithResult(t.Context(), "many-images.xls", workbook.Bytes())+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ imageCount := 0+ for _, item := range result.JSON {+ if item["doc_type_kwd"] == "image" {+ imageCount+++ }+ }+ if imageCount != maxEmbeddedMediaItems {+ t.Fatalf("image items = %d, want %d", imageCount, maxEmbeddedMediaItems)+ }+ if warnings := strings.Join(result.Warnings, "\n"); !strings.Contains(warnings, "stopped extracting embedded images after the 256-item document limit") {+ t.Fatalf("warnings = %q, want image-count limit warning", warnings)+ }+}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex d71ca9c170..e8cc2ee7ed 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -107,10 +107,18 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, data// for spreadsheet processing.}- items, warnings, sheets, err := parseXLSXBytes(data)+ mediaBudget := newEmbeddedMediaBudget()+ items, warnings, sheets, err := parseXLSXBytes(data, mediaBudget)if err == nil {+ if ctx.Err() != nil {+ return ParseResult{Err: ctx.Err()}+ }+ warnings = append(warnings, mediaBudget.warnings()...)return xlsxParseResult(filename, items, warnings, sheets)}+ if ctx.Err() != nil {+ return ParseResult{Err: ctx.Err()}+ }normalized, normalizeWarnings, changed, normalizeErr := normalizeXLSXForRead(data)if normalizeErr != nil {@@ -119,15 +127,22 @@ func (p *XLSXParser) ParseWithResult(ctx context.Context, filename string, dataif !changed {return ParseResult{Err: fmt.Errorf("xlsx parse: %w", err)}}- items, warnings, sheets, retryErr := parseXLSXBytes(normalized)+ // The first parse attempt is discarded, so its media reservations must not+ // affect the normalized retry's output.+ mediaBudget = newEmbeddedMediaBudget()+ items, warnings, sheets, retryErr := parseXLSXBytes(normalized, mediaBudget)if retryErr != nil {return ParseResult{Err: fmt.Errorf("xlsx parse: %w; retry after normalization: %v", err, retryErr)}}+ if ctx.Err() != nil {+ return ParseResult{Err: ctx.Err()}+ }warnings = append(normalizeWarnings, warnings...)+ warnings = append(warnings, mediaBudget.warnings()...)return xlsxParseResult(filename, items, warnings, sheets)}-func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {+func parseXLSXBytes(data []byte, mediaBudget *embeddedMediaBudget) ([]map[string]any, []string, int, error) {f, err := excelize.OpenReader(bytes.NewReader(data))if err != nil {return nil, nil, 0, fmt.Errorf("open XLSX: %w", err)@@ -143,7 +158,7 @@ func parseXLSXBytes(data []byte) ([]map[string]any, []string, int, error) {return nil, warnings, len(sheets), err}warnings = append(warnings, sheetWarnings...)- images, imageWarnings := extractXLSXImages(f, sheet)+ images, imageWarnings := extractXLSXImages(f, sheet, mediaBudget)for _, image := range images {row, _ := numericItemInt(image["row_start"])col, _ := numericItemInt(image["col_start"])