Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

63.6 KB
Raw Blame
package docparser

import (
	"bytes"
	"context"
	"encoding/base64"
	"encoding/binary"
	"encoding/xml"
	"errors"
	"fmt"
	"html"
	"image"
	"image/gif"
	_ "image/jpeg"
	_ "image/png"
	"io"
	"log"
	"mime"
	"net/http"
	"net/url"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"

	_ "golang.org/x/image/bmp"  // Register BMP decoding for image.Decode and DecodeConfig.
	_ "golang.org/x/image/webp" // Register WebP decoding for image.Decode and DecodeConfig.

	"github.com/Tencent/WeKnora/internal/searchutil"
	secutils "github.com/Tencent/WeKnora/internal/utils"

	"github.com/Tencent/WeKnora/internal/types"
	"github.com/Tencent/WeKnora/internal/types/interfaces"
	"github.com/google/uuid"
)

const (
	// minImageDimension is the minimum width/height in pixels; images smaller
	// than this on either axis are treated as icons and filtered out.
	minImageDimension = 64
	// minImageBytes is the minimum file size in bytes; very small images are
	// almost certainly icons or decorative elements.
	minImageBytes = 512 // 512 bytes
	// inlineImageRollbackTimeout bounds cleanup even if the request context was
	// cancelled while one of the inline image saves was failing.
	inlineImageRollbackTimeout = 5 * time.Second
)

// isIconImage returns true if the image data looks like a small icon or
// decorative element that should be filtered out. It checks pixel dimensions
// when decodable, and falls back to raw byte size otherwise.
func isIconImage(data []byte) bool {
	cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
	if err != nil {
		// Cannot decode dimensions — fall back to size-only heuristic.
		return len(data) < minImageBytes
	}
	if cfg.Width < minImageDimension && cfg.Height < minImageDimension {
		return true
	}
	return false
}

// StoredImage describes an image that has been saved to storage.
type StoredImage struct {
	OriginalRef string // reference in the original markdown
	ServingURL  string // provider:// URL (e.g. local://images/xxx.png, minio://bucket/key)
	MimeType    string
	// Inline marks Base64 images stored from inline document content, so only
	// resources created by this resolver are eligible for partial-save cleanup.
	Inline bool
	// NewlyStored marks a resource created by this resolver invocation, including
	// document-parser ImageRefs, so guard failures can clean up unbound uploads.
	NewlyStored bool
	// SkipMultimodal keeps a saved inline image out of OCR/caption fan-out once
	// its syntax-specific legacy budget is exhausted. The image still remains in
	// the document and resource bindings.
	SkipMultimodal bool
	// SourceLocators place the image in the original file (e.g. the scanned
	// page it was rendered from). Filled by ingestion after resolution.
	SourceLocators types.SourceLocators
}

// ImageResolver reads images from a DocReader ReadResult (inline bytes only)
// and saves them via FileService, replacing markdown references with unified URLs.
type ImageResolver struct {
	// TenantID for storage path namespacing
	TenantID uint64
}

// NewImageResolver creates a resolver.
func NewImageResolver() *ImageResolver {
	return &ImageResolver{}
}

// ResolveAndStore reads images from the convert result, persists them via fileSvc,
// and replaces markdown references with provider:// URLs.
// It returns the updated markdown and a list of stored images.
func (r *ImageResolver) ResolveAndStore(
	ctx context.Context,
	result *types.ReadResult,
	fileSvc interfaces.FileService,
	tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
	var inlineImages []StoredImage
	defer func() {
		if err == nil {
			return
		}
		updatedMarkdown = result.MarkdownContent
		if fileSvc == nil {
			return
		}
		rollbackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), inlineImageRollbackTimeout)
		defer cancel()
		for _, image := range inlineImages {
			if deleteErr := fileSvc.DeleteFile(rollbackCtx, image.ServingURL); deleteErr != nil {
				log.Printf("WARN: failed to rollback inline image %s: %v", image.ServingURL, deleteErr)
			}
		}
	}()

	ctx = context.WithValue(ctx, inlineQuotaKey{}, &inlineQuota{})
	markdown := UnwrapLinkedImages(result.MarkdownContent)
	// When the document-wide storage cap is reached, this resolver order gives
	// Markdown, HTML, then bare Base64 references deterministic priority.
	md2, imgDataURIs, err := r.ResolveDataURIImages(ctx, markdown, fileSvc, tenantID)
	inlineImages = append(inlineImages, imgDataURIs...)
	images = append(images, imgDataURIs...)
	if err != nil {
		return markdown, images, err
	}
	markdown = md2

	md3, imgHTML, err := r.ResolveHTMLDataURIImages(ctx, markdown, fileSvc, tenantID)
	inlineImages = append(inlineImages, imgHTML...)
	images = append(images, imgHTML...)
	if err != nil {
		return markdown, images, err
	}
	markdown = md3

	md4, imgBare, err := r.ResolveBareBase64Content(ctx, markdown, fileSvc, tenantID)
	inlineImages = append(inlineImages, imgBare...)
	images = append(images, imgBare...)
	if err != nil {
		return markdown, images, err
	}
	markdown = md4

	if len(result.ImageRefs) == 0 {
		return markdown, sortImagesByPosition(markdown, images), nil
	}

	// Build a map of original_ref -> image ref for fast lookup
	refMap := make(map[string]types.ImageRef)
	for _, ref := range result.ImageRefs {
		refMap[ref.OriginalRef] = ref
	}
	savedRefs := make(map[string]StoredImage)

	matches := scanMarkdownImageTargets(markdown)

	// Process in reverse order to preserve positions when replacing
	for i := len(matches) - 1; i >= 0; i-- {
		match := matches[i]
		rawTarget := markdown[match.TargetStart:match.TargetEnd]
		refPath, pathStart, pathEnd, ok := splitMarkdownImageTarget(rawTarget, refMap)
		if !ok {
			continue
		}

		// Skip already-resolved URLs (http/https, unified /files/, or provider:// scheme)
		if strings.HasPrefix(refPath, "http://") || strings.HasPrefix(refPath, "https://") ||
			isProviderScheme(refPath) {
			continue
		}

		// Find inline image bytes from the result
		stored, ok := r.saveReferencedImage(ctx, fileSvc, tenantID, refPath, refMap, savedRefs)
		if !ok {
			continue
		}
		images = appendStoredImage(images, stored)

		// Replace in markdown
		absolutePathStart := match.TargetStart + pathStart
		absolutePathEnd := match.TargetStart + pathEnd
		markdown = markdown[:absolutePathStart] + stored.ServingURL + markdown[absolutePathEnd:]
	}

	md5, imgRelativeHTML, _ := r.ResolveRelativeHTMLImages(ctx, markdown, fileSvc, tenantID, refMap, savedRefs)
	markdown = md5
	images = append(images, imgRelativeHTML...)

	return markdown, sortImagesByPosition(markdown, images), nil
}

// sortImagesByPosition orders images by where they first appear in markdown,
// since each resolution pass collects its own kind and references are
// rewritten back to front. Callers number images by this order. Images not
// found in markdown keep their relative order at the end.
func sortImagesByPosition(markdown string, images []StoredImage) []StoredImage {
	position := func(image StoredImage) int {
		if i := strings.Index(markdown, image.ServingURL); image.ServingURL != "" && i >= 0 {
			return i
		}
		return len(markdown)
	}
	sort.SliceStable(images, func(a, b int) bool {
		return position(images[a]) < position(images[b])
	})
	return images
}

func appendStoredImage(images []StoredImage, stored StoredImage) []StoredImage {
	for _, existing := range images {
		if existing.OriginalRef == stored.OriginalRef && existing.ServingURL == stored.ServingURL {
			return images
		}
	}
	return append(images, stored)
}

func (r *ImageResolver) saveReferencedImage(
	ctx context.Context,
	fileSvc interfaces.FileService,
	tenantID uint64,
	refPath string,
	refMap map[string]types.ImageRef,
	savedRefs map[string]StoredImage,
) (StoredImage, bool) {
	if stored, ok := savedRefs[refPath]; ok {
		return stored, true
	}

	ref, found := refMap[refPath]
	if !found || len(ref.ImageData) == 0 {
		return StoredImage{}, false
	}

	if !ref.IsOriginal && isIconImage(ref.ImageData) {
		return StoredImage{}, false
	}

	// Reuse a previously saved upload when the same source image (identified by
	// ref.Filename) has already been persisted under a different markdown ref
	// path (e.g. "images/foo.png" vs "./images/foo.png"). This avoids writing
	// the same bytes to object storage multiple times.
	if ref.Filename != "" {
		if cached, ok := savedRefs["__filename__:"+ref.Filename]; ok {
			stored := StoredImage{
				OriginalRef: refPath,
				ServingURL:  cached.ServingURL,
				MimeType:    cached.MimeType,
			}
			savedRefs[refPath] = stored
			return stored, true
		}
	}

	ext := extFromMime(ref.MimeType)
	if ext == "" {
		ext = filepath.Ext(ref.Filename)
	}
	if ext == "" {
		ext = ".png"
	}

	fileName := uuid.New().String() + ext
	servingURL, saveErr := fileSvc.SaveBytes(ctx, ref.ImageData, tenantID, fileName, false)
	if saveErr != nil {
		log.Printf("WARN: failed to save image %s: %v", refPath, saveErr)
		return StoredImage{}, false
	}

	stored := StoredImage{
		OriginalRef: refPath,
		ServingURL:  servingURL,
		MimeType:    ref.MimeType,
		NewlyStored: true,
	}
	savedRefs[refPath] = stored
	if ref.Filename != "" {
		savedRefs["__filename__:"+ref.Filename] = stored
	}
	return stored, true
}

func extFromMime(mime string) string {
	switch mime {
	case "image/png":
		return ".png"
	case "image/jpeg":
		return ".jpg"
	case "image/gif":
		return ".gif"
	case "image/webp":
		return ".webp"
	case "image/bmp":
		return ".bmp"
	case "image/svg+xml":
		return ".svg"
	default:
		return ""
	}
}

// isProviderScheme checks if the path uses a provider:// scheme (local://, minio://, cos://, tos://).
func isProviderScheme(p string) bool {
	for _, prefix := range []string{"local://", "minio://", "cos://", "tos://", "s3://", "obs://"} {
		if strings.HasPrefix(p, prefix) {
			return true
		}
	}
	return false
}

// isWhitelistedImageHost checks if the image URL's host is in the whitelist.
// Whitelisted hosts are trusted (e.g. internal MinerU service) — images are
// still downloaded for validation and OCR/caption analysis, but not uploaded
// to object storage. The markdown keeps the original URL.
// Configure via IMAGE_HOST_KEEP_URL env var (comma-separated hosts).
func isWhitelistedImageHost(rawURL string) bool {
	whitelist := strings.TrimSpace(os.Getenv("IMAGE_HOST_KEEP_URL"))
	if whitelist == "" {
		return false
	}
	u, err := url.Parse(rawURL)
	if err != nil || u.Host == "" {
		return false
	}
	host := strings.ToLower(u.Host)
	hostname := strings.ToLower(u.Hostname())
	for _, h := range strings.Split(whitelist, ",") {
		h = strings.ToLower(strings.TrimSpace(h))
		if h == "" {
			continue
		}
		// Exact host match (includes port) or hostname match (any port)
		if host == h || hostname == h {
			return true
		}
	}
	return false
}

// ---------------------------------------------------------------------------
// Helper functions for base64 image handling
// ---------------------------------------------------------------------------

// cleanBase64Payload removes whitespace characters from a base64 payload string.
func cleanBase64Payload(payload string) string {
	payload = strings.ReplaceAll(payload, "\n", "")
	payload = strings.ReplaceAll(payload, "\r", "")
	payload = strings.ReplaceAll(payload, "\t", "")
	payload = strings.ReplaceAll(payload, " ", "")
	return payload
}

// decodeBase64Flexible tries standard, raw, URL-safe, and raw-URL-safe base64 decodings.
func decodeBase64Flexible(payload string) ([]byte, error) {
	if data, err := base64.StdEncoding.DecodeString(payload); err == nil {
		return data, nil
	}
	if data, err := base64.RawStdEncoding.DecodeString(payload); err == nil {
		return data, nil
	}
	if data, err := base64.URLEncoding.DecodeString(payload); err == nil {
		return data, nil
	}
	return base64.RawURLEncoding.DecodeString(payload)
}

// sniffImageMime detects the MIME type by examining the magic bytes of image data.
func sniffImageMime(data []byte) string {
	if len(data) < 4 {
		return ""
	}
	if data[0] == 0x89 && data[1] == 'P' && data[2] == 'N' && data[3] == 'G' {
		return "image/png"
	}
	if data[0] == 0xFF && data[1] == 0xD8 && data[2] == 0xFF {
		return "image/jpeg"
	}
	if data[0] == 'G' && data[1] == 'I' && data[2] == 'F' {
		return "image/gif"
	}
	if len(data) >= 12 &&
		data[0] == 'R' && data[1] == 'I' && data[2] == 'F' && data[3] == 'F' &&
		data[8] == 'W' && data[9] == 'E' && data[10] == 'B' && data[11] == 'P' {
		return "image/webp"
	}
	if data[0] == 'B' && data[1] == 'M' {
		return "image/bmp"
	}
	return ""
}

func sniffInlineImageMime(data []byte) string {
	if mimeType := sniffImageMime(data); mimeType != "" {
		return mimeType
	}
	// Read through declarations and comments without imposing a prefix length.
	decoder := xml.NewDecoder(bytes.NewReader(data))
	for {
		token, err := decoder.Token()
		if err != nil {
			break
		}
		if text, ok := token.(xml.CharData); ok && len(bytes.TrimSpace(text)) != 0 {
			break
		}
		if root, ok := token.(xml.StartElement); ok {
			if root.Name.Local == "svg" {
				return "image/svg+xml"
			}
			break
		}
	}

	return ""
}

// ---------------------------------------------------------------------------
// HTML <img> tag data URI resolution
// ---------------------------------------------------------------------------

// imgHTMLDataURI matches HTML <img> tags with inline data:image/*;base64,... in the src attribute.
// A missing quote must not extend the payload into subsequent tags or prose.
var imgHTMLDataURI = regexp.MustCompile(
	`(?i)<img\s[^>]*?src\s*=\s*["'](data:image/[^;<>]+;base64,[^"'<>]+)["'][^>]*?/?\s*>`,
)

// imgHTMLSrc matches an HTML <img> tag carrying a quoted src attribute. It does
// not care what the src points at — callers select the references they handle by
// inspecting the scheme, which is how the relative and the remote paths divide
// the same tags between them.
//
// Shared with the search layer so that an image stored here can always be
// matched back to the tag it came from. See searchutil.HTMLImageSrcRegex for
// the submatch layout and the known limits.
var imgHTMLSrc = searchutil.HTMLImageSrcRegex

// Inline resource caps are independent from the legacy per-syntax 30-image AI budget.
// Excess images are omitted instead of leaving Base64 in searchable text.
const (
	maxInlineImagesPerDocument = 1000
	maxInlineBytesPerDocument  = 100 * 1024 * 1024
	inlineImageOmitted         = "[image omitted]"
)

type inlineQuotaKey struct{}

type inlineQuota struct {
	count int
	bytes int
}

// Check decoded-byte limits before allocating a potentially huge image.
func oversizedInlinePayload(payload string) bool {
	return len(payload) > ((maxRemoteImageSize+2)/3)*4
}

func quotaForInlineImages(ctx context.Context) *inlineQuota {
	if q, ok := ctx.Value(inlineQuotaKey{}).(*inlineQuota); ok {
		return q
	}
	return &inlineQuota{}
}

func (q *inlineQuota) permits(size int) bool {
	if q.count >= maxInlineImagesPerDocument || size > maxInlineBytesPerDocument-q.bytes {
		return false
	}
	q.count++
	q.bytes += size
	return true
}

type inlineRewrite struct {
	start int
	end   int
	value string
}

// The regex match offsets refer to the original Markdown. Build the output
// in one pass instead of copying the entire document on each substitution.
func applyInlineRewrites(markdown string, edits []inlineRewrite) string {
	if len(edits) == 0 {
		return markdown
	}
	var b strings.Builder
	b.Grow(len(markdown))
	cursor := 0
	for _, e := range edits {
		if e.start < cursor {
			continue
		}
		b.WriteString(markdown[cursor:e.start])
		b.WriteString(e.value)
		cursor = e.end
	}
	b.WriteString(markdown[cursor:])
	return b.String()
}

// ResolveHTMLDataURIImages finds <img src="data:image/*;base64,..."> tags in markdown,
// decodes the images, stores them via fileSvc, and replaces each tag with a markdown
// image reference using the storage URL.
func (r *ImageResolver) ResolveHTMLDataURIImages(
	ctx context.Context,
	markdown string,
	fileSvc interfaces.FileService,
	tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
	matches := imgHTMLDataURI.FindAllStringSubmatchIndex(markdown, -1)
	if len(matches) == 0 {
		return markdown, nil, nil
	}
	codeRanges := markdownCodeLiteralRanges(markdown)
	var rewrites []inlineRewrite
	quota := quotaForInlineImages(ctx)

	for i := range matches {
		m := matches[i]
		if _, ok := codeLiteralRangeAt(codeRanges, m[0]); ok {
			continue
		}
		dataURI := markdown[m[2]:m[3]]
		mimeType, payload, ok := parseImageDataURI(dataURI)
		if !ok {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		payload = cleanBase64Payload(payload)
		if payload == "" {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if oversizedInlinePayload(payload) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		data, decErr := decodeBase64Flexible(payload)
		if decErr != nil {
			log.Printf("WARN: HTML img data URI base64 decode failed: %v", decErr)
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if len(data) > maxRemoteImageSize {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if !isCompleteInlineImage(data, mimeType) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if isIconImage(data) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], ""})
			continue
		}
		if !quota.permits(len(data)) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		ext := extFromMime(mimeType)
		if ext == "" {
			ext = ".png"
		}
		fileName := uuid.New().String() + ext
		servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
		if saveErr != nil {
			return markdown, images, fmt.Errorf("save HTML data URI image: %w", saveErr)
		}
		images = append(images, StoredImage{
			OriginalRef: "html-img-data-uri",
			ServingURL:  servingURL,
			MimeType:    mimeType,
			Inline:      true,
		})
		rewrites = append(rewrites, inlineRewrite{m[0], m[1], fmt.Sprintf("![image](%s)", servingURL)})
	}
	applyMultimodalBudget(images)
	return applyInlineRewrites(markdown, rewrites), images, nil
}

// ResolveRelativeHTMLImages finds HTML <img> tags whose src points at a
// relative document image reference, stores the corresponding bytes via
// fileSvc, and replaces only the src attribute value with the storage URL.
func (r *ImageResolver) ResolveRelativeHTMLImages(
	ctx context.Context,
	markdown string,
	fileSvc interfaces.FileService,
	tenantID uint64,
	refMap map[string]types.ImageRef,
	savedRefs map[string]StoredImage,
) (updatedMarkdown string, images []StoredImage, err error) {
	matches := imgHTMLSrc.FindAllStringSubmatchIndex(markdown, -1)
	if len(matches) == 0 {
		return markdown, nil, nil
	}

	for i := len(matches) - 1; i >= 0; i-- {
		m := matches[i]
		src := strings.TrimSpace(markdown[m[4]:m[5]])
		if src == "" || strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") ||
			isProviderScheme(src) || strings.HasPrefix(strings.ToLower(src), "data:image/") {
			continue
		}

		stored, ok := r.saveReferencedImage(ctx, fileSvc, tenantID, src, refMap, savedRefs)
		if !ok {
			continue
		}
		images = appendStoredImage(images, stored)
		markdown = markdown[:m[4]] + stored.ServingURL + markdown[m[5]:]
	}

	return markdown, images, nil
}

// ---------------------------------------------------------------------------
// Bare base64/data URI resolution (catch-all)
// ---------------------------------------------------------------------------

// bareDataURIPattern matches standalone data:image/*;base64,... strings.
var bareDataURIPattern = regexp.MustCompile(
	`(?i)data:image/([^;\s]+);base64,[ \t\r\n]*([^\r\n\s"'<>)]{1,})`,
)

// bareBase64CommaPrefixed matches base64,DATA patterns (partial data URIs missing the mime prefix).
var bareBase64CommaPrefixed = regexp.MustCompile(
	`base64,[ \t\r\n]*([^\r\n\s"'<>)]{32,})`,
)

const wrappedBase64LineLength = 76

// wrappedBase64PayloadEnd extends a payload over Base64 fragments separated by newlines.
// Width is not a boundary: encoding syntax and a complete image determine the end.
// On failure the returned end stays on the first line. Callers must reject an
// ambiguous continuation rather than erase its prefix and allow the tail through.
// supported is true only after full validation within the decode budget.
// Decoded data never grows beyond the existing single-image cap.
func wrappedBase64PayloadEnd(markdown string, payloadStart, payloadEnd int) (end int, complete, supported bool) {
	// Sentence punctuation belongs to the document, including after damaged tokens.
	firstLine := strings.TrimRight(markdown[payloadStart:payloadEnd], ".,;")
	payloadEnd = payloadStart + len(firstLine)
	if len(firstLine) < 32 && !looksLikeImageBase64(firstLine) {
		return payloadEnd, true, false
	}
	if !isStrictBase64Line(firstLine) || oversizedInlinePayload(firstLine) {
		return payloadEnd, false, false
	}
	// Keep partial encoding groups across line breaks, which need not align
	// with Base64's four-character groups.
	pending := firstLine[len(firstLine)/4*4:]
	imageData, err := decodeBase64Flexible(firstLine[:len(firstLine)/4*4])
	if err != nil {
		return payloadEnd, false, false
	}
	imageValidity := func() (complete, supported bool) {
		data := imageData
		if pending != "" {
			tail, err := decodeBase64Flexible(pending)
			if err != nil || len(tail) > maxRemoteImageSize-len(data) {
				return false, false
			}
			data = append(data, tail...)
		}
		return validateInlineImageData(data)
	}
	if complete, supported := imageValidity(); complete {
		return payloadEnd, true, supported
	}

	currentEnd := payloadEnd
	binaryImage := sniffImageMime(imageData) != ""
	// Cache the opening element boundary. A self-closing root can end on a
	// full encoded line; child /> tags must not trigger repeated XML validation.
	svgOpeningEnd := -1
	for {
		lineStart := currentEnd
		if lineStart < len(markdown) && markdown[lineStart] == '\r' {
			lineStart++
		}
		if lineStart >= len(markdown) || markdown[lineStart] != '\n' {
			return payloadEnd, false, false
		}
		lineStart++
		lineEnd := strings.IndexByte(markdown[lineStart:], '\n')
		if lineEnd < 0 {
			lineEnd = len(markdown)
		} else {
			lineEnd += lineStart
		}
		candidateEnd := lineEnd
		if candidateEnd > lineStart && markdown[candidateEnd-1] == '\r' {
			candidateEnd--
		}
		linePayloadEnd := candidateEnd
		fragment := markdown[lineStart:candidateEnd]
		if boundary := strings.IndexAny(fragment, " \t\"'<>)]"); boundary >= 0 {
			fragment = fragment[:boundary]
		}
		fragment = strings.TrimRight(fragment, ".,;")
		candidateEnd = lineStart + len(fragment)
		line := fragment
		if len(line) == 0 || !isStrictBase64Line(line) {
			return payloadEnd, false, false
		}
		if oversizedInlinePayload(line) {
			return payloadEnd, false, false
		}
		encoded := pending + line
		end := len(encoded) / 4 * 4
		pending = encoded[end:]
		lineData, err := decodeBase64Flexible(encoded[:end])
		if err != nil {
			return payloadEnd, false, false
		}
		if len(lineData) > maxRemoteImageSize-len(imageData) {
			return payloadEnd, false, false
		}
		needed := len(imageData) + len(lineData)
		if needed > cap(imageData) {
			grown := make([]byte, len(imageData), min(maxRemoteImageSize, max(needed, 2*cap(imageData))))
			copy(grown, imageData)
			imageData = grown
		}
		imageData = append(imageData, lineData...)
		tail := bytes.TrimSpace(imageData[max(0, len(imageData)-64):])
		if !binaryImage && svgOpeningEnd < 0 && bytes.HasSuffix(tail, []byte("/>")) {
			decoder := xml.NewDecoder(bytes.NewReader(imageData))
			for {
				token, err := decoder.Token()
				if err != nil {
					break
				}
				if _, ok := token.(xml.StartElement); ok {
					svgOpeningEnd = int(decoder.InputOffset())
					break
				}
			}
		}
		selfClosingRoot := svgOpeningEnd >= 2 &&
			bytes.Equal(imageData[svgOpeningEnd-2:svgOpeningEnd], []byte("/>")) &&
			len(bytes.TrimSpace(imageData[svgOpeningEnd:])) == 0
		if binaryImage || candidateEnd < linePayloadEnd || strings.Contains(line, "=") || len(line)%4 != 0 ||
			bytes.HasSuffix(tail, []byte("</svg>")) || selfClosingRoot {
			if complete, supported := imageValidity(); complete {
				return candidateEnd, true, supported
			}
		}
		if candidateEnd < linePayloadEnd || (strings.Contains(line, "=") && pending == "") {
			return payloadEnd, false, false
		}
		currentEnd = candidateEnd
	}
}

// ErrAmbiguousInlineImage means the image cannot be separated from following
// lines without guessing. Callers should keep the source and stop ingestion.
var ErrAmbiguousInlineImage = errors.New(
	"cannot determine the end of a malformed multiline inline image; repair or remove the image")

func hasInlineImageContinuation(markdown string, end int) bool {
	rest := strings.TrimLeft(markdown[end:], " \t\r")
	if !strings.HasPrefix(rest, "\n") {
		return false
	}
	line, _, _ := strings.Cut(rest[1:], "\n")
	line = strings.TrimSpace(line)
	if line == "" || markdownATXHeading.MatchString(line) {
		return false
	}
	// A non-ASCII letter establishes a prose start. ASCII text and corrupt
	// prefixes such as !AAAA remain ambiguous; do not leave their tails behind.
	first, _ := utf8.DecodeRuneInString(line)
	return first < utf8.RuneSelf || !unicode.IsLetter(first)
}

func isStrictBase64Line(line string) bool {
	for i := 0; i < len(line); i++ {
		c := line[i]
		if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
			c == '+' || c == '/' || c == '_' || c == '-' || c == '=' {
			continue
		}
		return false
	}
	return true
}

const (
	// Fixed support limits for inline raster images bound decoding memory.
	// Even valid images over 16,777,216 pixels are omitted; this is not corruption.
	// GIFs also have aggregate pixel and frame limits.
	maxInlineImagePixels = 16 * 1024 * 1024
	maxInlineGIFPixels   = 32 * 1024 * 1024
	maxInlineGIFFrames   = 256
)

func isCompleteInlineImage(data []byte, mimeType string) bool {
	// A declared image MIME type alone cannot validate unrecognized bytes.
	if extFromMime(mimeType) == "" && sniffInlineImageMime(data) == "" {
		return false
	}
	_, supported := validateInlineImageData(data)
	return supported
}

// validateInlineImageData distinguishes a complete boundary from permission to store.
// Over-budget containers with an end boundary may be omitted without decoding
// their pixels. Only images that pass full validation are supported for storage.
func validateInlineImageData(data []byte) (complete, supported bool) {
	mimeType := sniffImageMime(data)
	// Header checks allocate decoder buffers too: run them only once an image
	// boundary is present, rather than for every fragment of a truncated payload.
	switch mimeType {
	case "image/png":
		complete = bytes.HasSuffix(data, []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xAE, 0x42, 0x60, 0x82})
	case "image/jpeg":
		complete = bytes.HasSuffix(data, []byte{0xFF, 0xD9})
	case "image/gif":
		complete = len(data) > 0 && data[len(data)-1] == ';'
	case "image/webp":
		complete = len(data) >= 12 && uint64(binary.LittleEndian.Uint32(data[4:8]))+8 == uint64(len(data))
	case "image/bmp":
		complete = len(data) >= 6 && uint64(binary.LittleEndian.Uint32(data[2:6])) == uint64(len(data))
	}
	if mimeType != "" {
		if !complete {
			return false, false
		}
		cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
		if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
			return false, false
		}
		withinBudget := uint64(cfg.Width) <= maxInlineImagePixels/uint64(cfg.Height)
		if mimeType == "image/gif" {
			valid, framesWithinBudget := gifDecodeBudget(data)
			if !valid {
				return false, false
			}
			withinBudget = withinBudget && framesWithinBudget
		}
		if !withinBudget {
			log.Printf("WARN: inline %s image %dx%d exceeds supported decode budget; omitted",
				mimeType, cfg.Width, cfg.Height)
			return true, false
		}
		if mimeType == "image/gif" {
			_, err = gif.DecodeAll(bytes.NewReader(data))
		} else {
			_, _, err = image.Decode(bytes.NewReader(data))
		}
		return err == nil, err == nil
	}
	decoder := xml.NewDecoder(bytes.NewReader(data))
	depth, roots := 0, 0
	for {
		token, err := decoder.Token()
		if err == io.EOF {
			valid := roots == 1 && depth == 0
			return valid, valid
		}
		if err != nil {
			return false, false
		}
		switch token := token.(type) {
		case xml.StartElement:
			if depth == 0 {
				roots++
				if roots != 1 || token.Name.Local != "svg" {
					return false, false
				}
			}
			depth++
		case xml.EndElement:
			depth--
		case xml.CharData:
			if depth == 0 && len(bytes.TrimSpace(token)) != 0 {
				return false, false
			}
		}
	}
}

// gifDecodeBudget checks the frame envelope before bounded LZW validation.
func gifDecodeBudget(data []byte) (valid, withinBudget bool) {
	if len(data) < 13 {
		return false, false
	}
	offset := 13
	if data[10]&0x80 != 0 {
		offset += 3 << (1 + (data[10] & 7))
	}
	frames, pixels := 0, uint64(0)
	skipBlocks := func() bool {
		for offset < len(data) {
			size := int(data[offset])
			offset++
			if size == 0 {
				return true
			}
			if size > len(data)-offset {
				return false
			}
			offset += size
		}
		return false
	}
	for offset < len(data) {
		block := data[offset]
		offset++
		switch block {
		case 0x3B:
			return frames > 0 && offset == len(data), frames <= maxInlineGIFFrames && pixels <= maxInlineGIFPixels
		case 0x21:
			if offset >= len(data) {
				return false, false
			}
			offset++ // Extension label; its data uses the same sub-block envelope.
			if !skipBlocks() {
				return false, false
			}
		case 0x2C:
			if len(data)-offset < 9 {
				return false, false
			}
			descriptor := data[offset : offset+9]
			width := uint64(binary.LittleEndian.Uint16(descriptor[4:6]))
			height := uint64(binary.LittleEndian.Uint16(descriptor[6:8]))
			pixels += width * height
			frames++
			if width == 0 || height == 0 {
				return false, false
			}
			offset += 9
			if descriptor[8]&0x80 != 0 {
				offset += 3 << (1 + (descriptor[8] & 7))
			}
			if offset >= len(data) {
				return false, false
			}
			offset++ // LZW minimum code size.
			if !skipBlocks() {
				return false, false
			}
		default:
			return false, false
		}
	}
	return false, false
}

// HasUnresolvedInlineImagePayload guards the chunking boundary against data
// URIs that a parser could not recognize or resolve. It deliberately ignores
// ordinary prose mentioning base64 and non-image base64 examples.
func HasUnresolvedInlineImagePayload(markdown string) bool {
	if !containsBase64Marker(markdown) {
		return false
	}
	markdown = withoutMarkdownCodeLiterals(markdown)
	for _, match := range unresolvedImageDataURI.FindAllString(markdown, -1) {
		if idx := strings.Index(strings.ToLower(match), "base64,"); idx >= 0 &&
			looksLikeImageBase64(strings.TrimSpace(match[idx+len("base64,"):])) {
			return true
		}
	}
	for _, match := range unresolvedBareBase64Image.FindAllStringSubmatch(markdown, -1) {
		if looksLikeImageBase64(match[1]) {
			return true
		}
	}
	return false
}

func containsBase64Marker(markdown string) bool {
	const marker = "base64,"
	for i := 0; i+len(marker) <= len(markdown); i++ {
		if markdown[i] != 'b' && markdown[i] != 'B' {
			continue
		}
		if strings.EqualFold(markdown[i:i+len(marker)], marker) {
			return true
		}
	}
	return false
}

type markdownCodeLiteralRange struct {
	start int
	end   int
}

// withoutMarkdownCodeLiterals masks fenced and inline code while preserving
// byte offsets. The image resolver uses the same ranges before replacing a
// match, so documentation snippets cannot be either uploaded or rejected.
func withoutMarkdownCodeLiterals(markdown string) string {
	ranges := markdownCodeLiteralRanges(markdown)
	if len(ranges) == 0 {
		return markdown
	}
	masked := []byte(markdown)
	for _, r := range ranges {
		for i := r.start; i < r.end; i++ {
			if masked[i] != '\n' {
				masked[i] = ' '
			}
		}
	}
	return string(masked)
}

// Literal code containers must not be interpreted as image references.
var (
	htmlCodeLiteralPattern = regexp.MustCompile(`(?is)<pre\b[^>]*>.*?</pre\s*>|<code\b[^>]*>.*?</code\s*>`)
	markdownListItemPrefix = regexp.MustCompile(`^(?:[-*+]|[0-9]+[.)])[ \t]`)
	markdownATXHeading     = regexp.MustCompile(`^#{1,6}(?:[ \t]|$)`)
)

// indentedCodeLiteralRanges identifies indented code and list content in one
// forward scan. A list child may have four leading spaces and still contain a
// normal image, while four more spaces beyond its content indent introduce code.
func indentedCodeLiteralRanges(markdown string, blocks []markdownCodeLiteralRange) []markdownCodeLiteralRange {
	var ranges []markdownCodeLiteralRange
	var listContentIndent int
	hasListItem := false
	blankLines := 0
	inParagraph := false

	for lineStart := 0; lineStart < len(markdown); {
		lineEnd := strings.IndexByte(markdown[lineStart:], '\n')
		if lineEnd < 0 {
			lineEnd = len(markdown)
		} else {
			lineEnd += lineStart
		}
		line := markdown[lineStart:lineEnd]
		indent := len(line) - len(strings.TrimLeft(line, " \t"))
		trimmed := strings.TrimSpace(line)

		// An indented code block cannot interrupt an existing paragraph.
		indentedCode := strings.HasPrefix(line, "\t") || indent >= 4
		if _, inBlock := codeLiteralRangeAt(blocks, lineStart); inBlock {
			inParagraph = false
		} else if indentedCode && !inParagraph && (!hasListItem || blankLines >= 2 || indent >= listContentIndent) {
			ranges = append(ranges, markdownCodeLiteralRange{start: lineStart, end: lineEnd})
		} else {
			inParagraph = trimmed != "" && !markdownATXHeading.MatchString(trimmed)
		}

		if trimmed == "" {
			blankLines++
			if blankLines >= 2 {
				hasListItem = false
			}
		} else {
			if marker := markdownListItemPrefix.FindString(trimmed); marker != "" {
				blankLines = 0
				listContentIndent = indent + len(marker) + 4
				hasListItem = true
			} else if indent == 0 {
				hasListItem = false
			}
		}

		if lineEnd == len(markdown) {
			break
		}
		lineStart = lineEnd + 1
	}
	return ranges
}

func mergeCodeLiteralRanges(ranges []markdownCodeLiteralRange) []markdownCodeLiteralRange {
	sort.Slice(ranges, func(i, j int) bool { return ranges[i].start < ranges[j].start })
	merged := make([]markdownCodeLiteralRange, 0, len(ranges))
	for _, span := range ranges {
		if len(merged) > 0 && span.start <= merged[len(merged)-1].end {
			if span.end > merged[len(merged)-1].end {
				merged[len(merged)-1].end = span.end
			}
			continue
		}
		merged = append(merged, span)
	}
	return merged
}

func markdownCodeLiteralRanges(markdown string) []markdownCodeLiteralRange {
	protected := fencedCodeLiteralRanges(markdown)
	for _, match := range htmlCodeLiteralPattern.FindAllStringIndex(markdown, -1) {
		protected = append(protected, markdownCodeLiteralRange{start: match[0], end: match[1]})
	}
	protected = mergeCodeLiteralRanges(protected)
	protected = append(protected, indentedCodeLiteralRanges(markdown, protected)...)
	protected = mergeCodeLiteralRanges(protected)
	var inline []markdownCodeLiteralRange
	for i := 0; i < len(markdown); {
		if end, ok := codeLiteralRangeAt(protected, i); ok {
			i = end
			continue
		}
		if markdown[i] != '`' || isEscaped(markdown, i) {
			i++
			continue
		}
		runLength := 1
		for i+runLength < len(markdown) && markdown[i+runLength] == '`' {
			runLength++
		}
		closePos := findInlineCodeLiteralClose(markdown, protected, i+runLength, runLength)
		if closePos < 0 {
			i += runLength
			continue
		}
		inline = append(inline, markdownCodeLiteralRange{start: i, end: closePos + runLength})
		i = closePos + runLength
	}
	return mergeCodeLiteralRanges(append(protected, inline...))
}

func fencedCodeLiteralRanges(markdown string) []markdownCodeLiteralRange {
	var ranges []markdownCodeLiteralRange
	var fence byte
	var fenceLength, fenceStart, fenceQuoteDepth int

	for lineStart := 0; lineStart < len(markdown); {
		lineEnd := strings.IndexByte(markdown[lineStart:], '\n')
		if lineEnd < 0 {
			lineEnd = len(markdown)
		} else {
			lineEnd += lineStart
		}
		line := markdown[lineStart:lineEnd]
		trimmed := strings.TrimLeft(line, " \t")
		quoteDepth := 0
		for strings.HasPrefix(trimmed, ">") {
			quoteDepth++
			trimmed = strings.TrimLeft(trimmed[1:], " \t")
		}
		// A quoted fence ends with its container, even if it is unclosed.
		if fence != 0 && quoteDepth < fenceQuoteDepth {
			ranges = append(ranges, markdownCodeLiteralRange{start: fenceStart, end: lineStart})
			fence = 0
		}
		if fence == 0 {
			if marker, length := codeFence(trimmed); length > 0 {
				fence, fenceLength, fenceStart, fenceQuoteDepth = marker, length, lineStart, quoteDepth
			}
		} else if marker, length := codeFence(trimmed); quoteDepth == fenceQuoteDepth &&
			marker == fence && length >= fenceLength &&
			strings.TrimSpace(trimmed[length:]) == "" {
			end := lineEnd
			if end < len(markdown) {
				end++
			}
			ranges = append(ranges, markdownCodeLiteralRange{start: fenceStart, end: end})
			fence, fenceLength = 0, 0
		}
		if lineEnd == len(markdown) {
			break
		}
		lineStart = lineEnd + 1
	}
	if fence != 0 {
		ranges = append(ranges, markdownCodeLiteralRange{start: fenceStart, end: len(markdown)})
	}
	return ranges
}

func codeLiteralRangeAt(ranges []markdownCodeLiteralRange, offset int) (int, bool) {
	index := sort.Search(len(ranges), func(i int) bool { return ranges[i].end > offset })
	if index < len(ranges) && ranges[index].start <= offset {
		return ranges[index].end, true
	}
	return 0, false
}

func findInlineCodeLiteralClose(markdown string, ranges []markdownCodeLiteralRange, start, runLength int) int {
	for i := start; i < len(markdown); {
		if end, ok := codeLiteralRangeAt(ranges, i); ok {
			// Inline HTML does not terminate the surrounding Markdown code span.
			if strings.EqualFold(markdown[i:min(i+5, len(markdown))], "<code") {
				i = end
				continue
			}
			return -1
		}
		if markdown[i] == '\n' {
			next := i + 1
			for next < len(markdown) && (markdown[next] == ' ' || markdown[next] == '\t' || markdown[next] == '\r') {
				next++
			}
			if next < len(markdown) && markdown[next] == '\n' {
				return -1
			}
		}
		if markdown[i] != '`' {
			i++
			continue
		}
		end := i
		for end < len(markdown) && markdown[end] == '`' {
			end++
		}
		if end-i == runLength {
			return i
		}
		i = end
	}
	return -1
}

func codeFence(line string) (byte, int) {
	if len(line) < 3 || (line[0] != '`' && line[0] != '~') {
		return 0, 0
	}
	marker := line[0]
	length := 0
	for length < len(line) && line[length] == marker {
		length++
	}
	if length < 3 {
		return 0, 0
	}
	return marker, length
}

func looksLikeImageBase64(payload string) bool {
	// A short prefix is sufficient for binary magic, but not for an SVG root.
	header, err := decodeBase64Flexible(payload[:min(len(payload), 64)])
	if err == nil && sniffImageMime(header) != "" {
		return true
	}
	// Even the guard must not allocate more than the single-image limit.
	limit := (maxRemoteImageSize / 3) * 4
	data, err := decodeBase64Flexible(payload[:min(len(payload), limit)])
	if err != nil {
		return false
	}
	if sniffInlineImageMime(data) != "" {
		return true
	}
	// The guard can also recognize a root start split across wrapped lines.
	trimmed := bytes.TrimSpace(data)
	return bytes.HasPrefix(trimmed, []byte("<svg ")) || bytes.HasPrefix(trimmed, []byte("<svg>"))
}

var (
	unresolvedImageDataURI    = regexp.MustCompile(`(?i)data:image/[^;\s"'<>)]*;base64,\s*[^\s"'<>)]{1,}`)
	unresolvedBareBase64Image = regexp.MustCompile(`(?i)base64,([A-Za-z0-9+/_=-]{64,})`)
)

// ResolveBareBase64Content finds remaining bare data URIs and base64 image content
// in the markdown text, decodes and stores them, and replaces with image references.
// This acts as a catch-all after the standard markdown and HTML resolvers.
func (r *ImageResolver) ResolveBareBase64Content(
	ctx context.Context,
	markdown string,
	fileSvc interfaces.FileService,
	tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
	originalMarkdown := markdown
	if _, ok := ctx.Value(inlineQuotaKey{}).(*inlineQuota); !ok {
		ctx = context.WithValue(ctx, inlineQuotaKey{}, &inlineQuota{})
	}
	md, imgs1, err := r.resolveBareDataURIs(ctx, markdown, fileSvc, tenantID, markdownCodeLiteralRanges(markdown))
	images = append(images, imgs1...)
	if err != nil {
		return originalMarkdown, images, err
	}
	markdown = md

	md2, imgs2, err := r.resolveBareBase64Prefix(ctx, markdown, fileSvc, tenantID, markdownCodeLiteralRanges(markdown))
	images = append(images, imgs2...)
	if err != nil {
		return originalMarkdown, images, err
	}
	markdown = md2

	return markdown, images, nil
}

func (r *ImageResolver) resolveBareDataURIs(
	ctx context.Context,
	markdown string,
	fileSvc interfaces.FileService,
	tenantID uint64,
	codeRanges []markdownCodeLiteralRange,
) (string, []StoredImage, error) {
	matches := bareDataURIPattern.FindAllStringSubmatchIndex(markdown, -1)
	if len(matches) == 0 {
		return markdown, nil, nil
	}

	var images []StoredImage
	var rewrites []inlineRewrite
	quota := quotaForInlineImages(ctx)
	for i := range matches {
		m := matches[i]
		if _, ok := codeLiteralRangeAt(codeRanges, m[0]); ok {
			continue
		}
		// Well-formed HTML images were handled above. Quoted leftovers may
		// be malformed attributes and still need local resolution or omission.
		insideWrapper := false
		if m[0] > 0 {
			prev := markdown[m[0]-1]
			insideWrapper = prev == '"' || prev == '\''
			if prev == '(' || (prev == '<' && m[0] >= 2 && markdown[m[0]-2] == '(') {
				insideWrapper = true // likely inside a broken ![...](...) ref
			}
		}
		mimeType := "image/" + strings.ToLower(markdown[m[2]:m[3]])
		payloadEnd, complete, supported := m[5], true, false
		if extFromMime(mimeType) != "" || looksLikeImageBase64(markdown[m[4]:m[5]]) {
			payloadEnd, complete, supported = wrappedBase64PayloadEnd(markdown, m[4], m[5])
		}
		if !complete {
			if hasInlineImageContinuation(markdown, payloadEnd) {
				return markdown, images, ErrAmbiguousInlineImage
			}
			rewrites = append(rewrites, inlineRewrite{m[0], payloadEnd, inlineImageOmitted})
			continue
		}
		m[1], m[5] = payloadEnd, payloadEnd
		payload := markdown[m[4]:m[5]]

		payload = cleanBase64Payload(payload)
		if payload == "" {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if oversizedInlinePayload(payload) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		data, decErr := decodeBase64Flexible(payload)
		if decErr != nil {
			// Preserve short explanatory data-URI examples instead of treating
			// them as actual broken images.
			if len(payload) >= 64 {
				log.Printf("WARN: bare data URI base64 decode failed: %v", decErr)
				rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			}
			continue
		}
		if len(data) > maxRemoteImageSize {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if sniffInlineImageMime(data) == "" && mimeType != "image/svg+xml" && len(payload) < 100 {
			continue // not a real image, e.g. a short unquoted tutorial example
		}
		if !supported {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if isIconImage(data) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], ""})
			continue
		}
		if !quota.permits(len(data)) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		ext := extFromMime(mimeType)
		if ext == "" {
			ext = ".png"
		}
		fileName := uuid.New().String() + ext
		servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
		if saveErr != nil {
			return markdown, images, fmt.Errorf("save bare data URI image: %w", saveErr)
		}
		images = append(images, StoredImage{
			OriginalRef: "bare-data-uri",
			ServingURL:  servingURL,
			MimeType:    mimeType,
			Inline:      true,
		})
		if insideWrapper {
			// Inside a broken markdown ref like ![weird]alt](data:...) — replace data URI only
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], servingURL})
		} else {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], fmt.Sprintf("![image](%s)", servingURL)})
		}
	}
	applyMultimodalBudget(images)
	return applyInlineRewrites(markdown, rewrites), images, nil
}

func (r *ImageResolver) resolveBareBase64Prefix(
	ctx context.Context,
	markdown string,
	fileSvc interfaces.FileService,
	tenantID uint64,
	codeRanges []markdownCodeLiteralRange,
) (string, []StoredImage, error) {
	matches := bareBase64CommaPrefixed.FindAllStringSubmatchIndex(markdown, -1)
	if len(matches) == 0 {
		return markdown, nil, nil
	}

	var images []StoredImage
	var rewrites []inlineRewrite
	quota := quotaForInlineImages(ctx)
	for i := range matches {
		m := matches[i]
		if _, ok := codeLiteralRangeAt(codeRanges, m[0]); ok {
			continue
		}
		// Skip if preceded by ';' — this is part of a data URI handled above
		if m[0] > 0 && markdown[m[0]-1] == ';' {
			continue
		}
		payloadEnd, complete, supported := wrappedBase64PayloadEnd(markdown, m[2], m[3])
		if !complete && hasInlineImageContinuation(markdown, payloadEnd) &&
			looksLikeImageBase64(markdown[m[2]:m[3]]) {
			return markdown, images, ErrAmbiguousInlineImage
		}
		m[1], m[3] = payloadEnd, payloadEnd
		payload := markdown[m[2]:m[3]]
		payload = cleanBase64Payload(payload)
		if payload == "" {
			continue
		}
		if oversizedInlinePayload(payload) {
			if !looksLikeImageBase64(payload) {
				continue
			}
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		data, decErr := decodeBase64Flexible(payload)
		if decErr != nil {
			if !looksLikeImageBase64(payload) {
				continue
			}
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if len(data) > maxRemoteImageSize {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		mimeType := sniffInlineImageMime(data)
		if mimeType == "" {
			if looksLikeImageBase64(payload) {
				rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			}
			continue
		}
		if !supported {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		if isIconImage(data) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], ""})
			continue
		}
		if !quota.permits(len(data)) {
			rewrites = append(rewrites, inlineRewrite{m[0], m[1], inlineImageOmitted})
			continue
		}
		ext := extFromMime(mimeType)
		if ext == "" {
			ext = ".png"
		}
		fileName := uuid.New().String() + ext
		servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
		if saveErr != nil {
			return markdown, images, fmt.Errorf("save bare base64 image: %w", saveErr)
		}
		images = append(images, StoredImage{
			OriginalRef: "bare-base64",
			ServingURL:  servingURL,
			MimeType:    mimeType,
			Inline:      true,
		})
		rewrites = append(rewrites, inlineRewrite{m[0], m[1], fmt.Sprintf("![image](%s)", servingURL)})
	}
	applyMultimodalBudget(images)
	return applyInlineRewrites(markdown, rewrites), images, nil
}

// ---------------------------------------------------------------------------
// Remote image resolution (for manual / web-clipped markdown content)
// ---------------------------------------------------------------------------

const (
	// maxRemoteImageSize is the maximum allowed size for a single remote image download.
	maxRemoteImageSize = 10 * 1024 * 1024 // 10 MB
	// maxRemoteImages is the maximum number of remote images to process per document.
	maxRemoteImages = 30
	// maxInlineImagesForMultimodal is the legacy per-syntax number of inline
	// images that reach image understanding. Additional inline images are saved
	// and bound, but do not fan out OCR/caption work.
	maxInlineImagesForMultimodal = 30
	// remoteImageFetchTimeout is the per-image HTTP request timeout.
	remoteImageFetchTimeout = 15 * time.Second
)

// applyMultimodalBudget flags images beyond the per-syntax AI quota.
// Inline resolvers visit matches in document order and construct text in one pass,
// the document's leading maxInlineImagesForMultimodal go to AI understanding,
// the rest stay stored and bound but skip OCR/caption fan-out.
func applyMultimodalBudget(images []StoredImage) {
	for i := range images {
		images[i].SkipMultimodal = i >= maxInlineImagesForMultimodal
	}
}

// reLinkedImage matches the nested [![alt](img_url)](link_url) pattern where
// an image is wrapped inside a Markdown link. We unwrap it to just ![alt](img_url)
// so that downstream image-processing regexes only have to handle the flat form.
// The URL groups support one level of balanced parentheses.
var reLinkedImage = regexp.MustCompile(
	`\[!\[([^\]]*)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)\]` + // [![alt](img_url)]
		`\([^()\s]*(?:\([^)]*\)[^()\s]*)*\)`, // (link_url) — captured but discarded
)

// UnwrapLinkedImages replaces all [![alt](img_url)](link_url) occurrences in
// the markdown with just ![alt](img_url), stripping the outer link wrapper.
// This should be called before any image-extraction regex so that only the
// flat ![alt](url) form needs to be handled.
func UnwrapLinkedImages(markdown string) string {
	matches := reLinkedImage.FindAllStringSubmatchIndex(markdown, -1)
	if len(matches) == 0 {
		return markdown
	}
	codeRanges := markdownCodeLiteralRanges(markdown)
	var rewrites []inlineRewrite
	for _, match := range matches {
		if _, ok := codeLiteralRangeAt(codeRanges, match[0]); !ok {
			rewrites = append(rewrites, inlineRewrite{
				match[0], match[1], markdown[match[0]+1 : match[5]+1],
			})
		}
	}
	return applyInlineRewrites(markdown, rewrites)
}

// imgMarkdownPattern matches Markdown image syntax: ![alt](url).
// The alt-text group uses .*? (non-greedy) to allow literal ] in alt text.
// The URL group supports one level of balanced parentheses so that URLs
// like https://example.com/item_(abc)/123 are captured in full.
var imgMarkdownPattern = regexp.MustCompile(`!\[(.*?)\]\(([^()\s]*(?:\([^)]*\)[^()\s]*)*)\)`)

// parseImageDataURI splits a data URI into image MIME type and base64 payload.
func parseImageDataURI(dataURI string) (mimeType string, b64Payload string, ok bool) {
	const sep = ";base64,"
	idx := strings.Index(strings.ToLower(dataURI), sep)
	if idx < 0 {
		return "", "", false
	}
	meta := strings.TrimSpace(dataURI[:idx])
	const prefix = "data:image/"
	if len(meta) < len(prefix) || !strings.EqualFold(meta[:len(prefix)], prefix) {
		return "", "", false
	}
	sub := strings.TrimSpace(meta[len(prefix):])
	mimeType = "image/" + strings.ToLower(sub)
	b64Payload = strings.TrimSpace(dataURI[idx+len(sep):])
	if b64Payload == "" {
		return "", "", false
	}
	return mimeType, b64Payload, true
}

// ResolveDataURIImages finds embedded data:image/*;base64 images in markdown,
// decodes and stores each valid image, and replaces each reference with the
// returned provider URL. A per-image decoded-size limit still applies.
func (r *ImageResolver) ResolveDataURIImages(
	ctx context.Context,
	markdown string,
	fileSvc interfaces.FileService,
	tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
	markdown = UnwrapLinkedImages(markdown)
	matches := scanMarkdownImageTargets(markdown)
	if len(matches) == 0 {
		return markdown, nil, nil
	}
	codeRanges := markdownCodeLiteralRanges(markdown)
	var rewrites []inlineRewrite
	quota := quotaForInlineImages(ctx)

	for _, match := range matches {
		if _, ok := codeLiteralRangeAt(codeRanges, match.ImageStart); ok {
			continue
		}
		start, end := trimMarkdownSpaceBounds(markdown, match.TargetStart, match.TargetEnd)
		if titleStart, ok := parseMarkdownImageTitleSuffix(markdown[start:end]); ok {
			start, end = trimMarkdownSpaceBounds(markdown, start, start+titleStart)
		}
		if end-start >= 2 && markdown[start] == '<' && markdown[end-1] == '>' {
			start++
			end--
		}
		dataURI := markdown[start:end]
		if !strings.HasPrefix(strings.ToLower(dataURI), "data:image/") || !containsBase64Marker(dataURI) {
			continue
		}
		// A missing ')' can make the scanner consume the next paragraph.
		// Let the bare resolver find a wrapped payload's actual boundary.
		if strings.ContainsAny(dataURI, "\r\n") {
			continue
		}
		mimeType, payload, ok := parseImageDataURI(dataURI)
		if !ok {
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, inlineImageOmitted})
			continue
		}
		payload = cleanBase64Payload(payload)
		if payload == "" {
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, inlineImageOmitted})
			continue
		}
		if oversizedInlinePayload(payload) {
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, inlineImageOmitted})
			continue
		}
		data, decErr := decodeBase64Flexible(payload)
		if decErr != nil {
			log.Printf("WARN: data URI base64 decode failed: %v", decErr)
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, inlineImageOmitted})
			continue
		}
		if len(data) > maxRemoteImageSize {
			log.Printf("WARN: data URI image exceeds size limit (%d bytes)", maxRemoteImageSize)
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, inlineImageOmitted})
			continue
		}
		if !isCompleteInlineImage(data, mimeType) {
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, inlineImageOmitted})
			continue
		}
		if isIconImage(data) {
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, ""})
			continue
		}
		if !quota.permits(len(data)) {
			rewrites = append(rewrites, inlineRewrite{match.ImageStart, match.TargetEnd + 1, inlineImageOmitted})
			continue
		}
		ext := extFromMime(mimeType)
		if ext == "" {
			ext = ".png"
		}
		fileName := uuid.New().String() + ext
		servingURL, saveErr := fileSvc.SaveBytes(ctx, data, tenantID, fileName, false)
		if saveErr != nil {
			return markdown, images, fmt.Errorf("save data URI image: %w", saveErr)
		}
		images = append(images, StoredImage{
			OriginalRef: dataURI,
			ServingURL:  servingURL,
			MimeType:    mimeType,
			Inline:      true,
		})
		rewrites = append(rewrites, inlineRewrite{start, end, servingURL})
	}
	applyMultimodalBudget(images)
	return applyInlineRewrites(markdown, rewrites), images, nil
}

// errRemoteImageIsIcon marks the one skip reason worth aggregating: a page can
// carry hundreds of tracking pixels and spacer GIFs, and a line for each would
// drown the reasons an operator actually needs to see.
var errRemoteImageIsIcon = errors.New("filtered out as icon")

// remoteImageResult describes one image that was fetched successfully.
type remoteImageResult struct {
	// ServingURL is empty when KeepOriginalURL is set. The caller fills it in
	// with the normalized request URL — NOT the raw document bytes. Later stages
	// both search the document for this string and fetch it, so a padded or
	// entity-encoded value satisfies neither.
	ServingURL string
	MimeType   string
	// KeepOriginalURL marks a whitelisted host. Its bytes are downloaded so that
	// OCR/caption analysis can run, but it is not uploaded: the image keeps being
	// served from its original host, reached through the normalized URL.
	KeepOriginalURL bool
}

// fetchAndStoreRemoteImage applies the SSRF policy, downloads the image,
// rejects icons and uploads the bytes to storage.
//
// Both the Markdown and the HTML scan go through here so that the SSRF check,
// the icon filter and the whitelist behaviour cannot drift apart between the
// two syntaxes.
func fetchAndStoreRemoteImage(
	ctx context.Context,
	client *http.Client,
	fileSvc interfaces.FileService,
	tenantID uint64,
	imgURL string,
) (*remoteImageResult, error) {
	whitelisted := isWhitelistedImageHost(imgURL)

	if !whitelisted {
		if err := secutils.ValidateURLForSSRF(imgURL); err != nil {
			return nil, fmt.Errorf("blocked by SSRF policy: %w", err)
		}
	}

	data, mimeType, err := downloadImage(ctx, client, imgURL)
	if err != nil {
		return nil, fmt.Errorf("download: %w", err)
	}

	if isIconImage(data) {
		return nil, errRemoteImageIsIcon
	}

	if whitelisted {
		return &remoteImageResult{MimeType: mimeType, KeepOriginalURL: true}, nil
	}

	ext := extFromMime(mimeType)
	if ext == "" {
		ext = extFromURLPath(imgURL)
	}
	if ext == "" {
		ext = ".png" // safe default
	}
	servingURL, err := fileSvc.SaveBytes(ctx, data, tenantID, uuid.New().String()+ext, false)
	if err != nil {
		return nil, fmt.Errorf("save: %w", err)
	}
	return &remoteImageResult{ServingURL: servingURL, MimeType: mimeType}, nil
}

// isRemoteHTTPURL reports whether raw is an absolute http(s) URL.
//
// The comparison is deliberately byte-exact. Downstream fetchers compare the
// scheme the same way, so anything accepted here has to be spelled the way they
// expect; per-syntax normalization belongs in the SrcOf of the scan that needs
// it, not in this predicate.
func isRemoteHTTPURL(raw string) bool {
	return strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://")
}

// remotePassSpec describes one scan of a document: which references it matches,
// where the URL sits inside a match, and how to turn the raw document bytes into
// the URL to request.
type remotePassSpec struct {
	Pattern  *regexp.Regexp
	URLGroup int
	Syntax   string
	SrcOf    func(string) string
}

// remotePassStats accumulates the reasons a scan passed a reference over, so
// that a partially resolved document can be explained afterwards.
type remotePassStats struct {
	resolved   int
	iconSkips  int
	overBudget int
}

func (s *remotePassStats) report(syntax string) {
	if s.iconSkips > 0 {
		log.Printf("INFO: skipped %d %s remote image(s) as icons", s.iconSkips, syntax)
	}
	if s.overBudget > 0 {
		log.Printf(
			"WARN: remote image limit of %d reached for %s images; %d reference(s) left unresolved",
			maxRemoteImages, syntax, s.overBudget,
		)
	}
}

// resolveRemoteImagePass rewrites every remote http(s) reference matched by
// spec.Pattern, replacing the span of its URL capture group with the URL the
// image will be served from: the storage URL, or the normalized original for a
// whitelisted host. Matches are walked in reverse so the spans still to be
// processed stay valid after each replacement.
func resolveRemoteImagePass(
	ctx context.Context,
	markdown string,
	spec remotePassSpec,
	client func() *http.Client,
	fileSvc interfaces.FileService,
	tenantID uint64,
) (string, []StoredImage) {
	matches := spec.Pattern.FindAllStringSubmatchIndex(markdown, -1)
	if len(matches) == 0 {
		return markdown, nil
	}

	var (
		images []StoredImage
		stats  remotePassStats
	)

	for i := len(matches) - 1; i >= 0; i-- {
		m := matches[i]
		start, end := m[2*spec.URLGroup], m[2*spec.URLGroup+1]
		if start < 0 {
			continue
		}
		imgURL := spec.SrcOf(markdown[start:end])
		if !isRemoteHTTPURL(imgURL) {
			continue
		}

		if ctx.Err() != nil {
			log.Printf("WARN: remote image resolution cancelled: %v", ctx.Err())
			break
		}

		if stats.resolved >= maxRemoteImages {
			stats.overBudget++
			continue
		}

		res, err := fetchAndStoreRemoteImage(ctx, client(), fileSvc, tenantID, imgURL)
		if err != nil {
			if errors.Is(err, errRemoteImageIsIcon) {
				stats.iconSkips++
			} else {
				log.Printf("WARN: skipped %s remote image %s: %v", spec.Syntax, imgURL, err)
			}
			continue
		}
		stats.resolved++

		// A whitelisted host is served from where it already lives, so the
		// document keeps pointing there. The span is still rewritten with the
		// normalized URL: later stages both locate the image by searching the
		// document for its ServingURL and fetch that same string, so the two have
		// to agree, and an entity-encoded or padded attribute value satisfies
		// neither.
		servingURL := res.ServingURL
		if res.KeepOriginalURL {
			servingURL = imgURL
		}

		images = append(images, StoredImage{
			OriginalRef: imgURL,
			ServingURL:  servingURL,
			MimeType:    res.MimeType,
		})
		markdown = markdown[:start] + servingURL + markdown[end:]
	}

	stats.report(spec.Syntax)
	return markdown, images
}

// htmlAttrSrc normalizes an HTML src attribute value into the URL to request.
//
// Three things are HTML-specific. An attribute value may be padded, so it is
// trimmed — on both sides of the decode, because an entity can itself expand to
// whitespace. It may carry entities, and `&amp;` is the correct spelling of a
// query separator, so it is unescaped. And the scheme may be written in any
// case, while every fetcher downstream compares it byte-for-byte, so it is
// lowercased.
//
// Markdown targets get none of this: they are handed through unchanged so that
// existing documents resolve exactly the references they resolve today.
func htmlAttrSrc(raw string) string {
	src := strings.TrimSpace(html.UnescapeString(strings.TrimSpace(raw)))
	if i := strings.Index(src, "://"); i > 0 {
		src = strings.ToLower(src[:i]) + src[i:]
	}
	return src
}

// markdownSrc hands a Markdown image target through untouched. Markdown has no
// attribute quoting or entity encoding to undo, and leaving it alone is what
// keeps existing documents resolving exactly what they resolve today.
func markdownSrc(raw string) string { return raw }

// ResolveRemoteImages downloads remote http(s) images into storage and points
// the document at the stored copies. Both Markdown image syntax and HTML <img>
// tags with a quoted src are covered.
func (r *ImageResolver) ResolveRemoteImages(
	ctx context.Context,
	markdown string,
	fileSvc interfaces.FileService,
	tenantID uint64,
) (updatedMarkdown string, images []StoredImage, err error) {
	markdown = UnwrapLinkedImages(markdown)

	// The client owns a dedicated transport, so it is built only once a scan
	// actually has something to fetch.
	var httpClient *http.Client
	client := func() *http.Client {
		if httpClient == nil {
			httpClient = secutils.NewSSRFSafeHTTPClient(secutils.SSRFSafeHTTPClientConfig{
				Timeout:      remoteImageFetchTimeout,
				MaxRedirects: 5,
			})
		}
		return httpClient
	}

	// Two scans, each with its own budget. Sharing one would let HTML images eat
	// into the number of Markdown images a document already gets resolved today.
	markdown, mdImages := resolveRemoteImagePass(ctx, markdown, remotePassSpec{
		Pattern: imgMarkdownPattern,
		// Group 2 of imgMarkdownPattern is the target; group 1 is the alt text.
		URLGroup: 2,
		Syntax:   "markdown",
		SrcOf:    markdownSrc,
	}, client, fileSvc, tenantID)

	markdown, htmlImages := resolveRemoteImagePass(ctx, markdown, remotePassSpec{
		Pattern:  imgHTMLSrc,
		URLGroup: searchutil.HTMLImageSrcURLGroup,
		Syntax:   "html",
		SrcOf:    htmlAttrSrc,
	}, client, fileSvc, tenantID)

	images = append(images, mdImages...)
	images = append(images, htmlImages...)
	return markdown, images, nil
}

// downloadImage fetches an image from remoteURL using the provided SSRF-safe
// client. It validates Content-Type and enforces maxRemoteImageSize.
func downloadImage(ctx context.Context, client *http.Client, remoteURL string) (data []byte, mimeType string, err error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, remoteURL, nil)
	if err != nil {
		return nil, "", fmt.Errorf("create request: %w", err)
	}
	// Some CDNs require a browser-like User-Agent.
	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; WeKnora/1.0)")

	resp, err := client.Do(req)
	if err != nil {
		return nil, "", fmt.Errorf("HTTP GET: %w", err)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return nil, "", fmt.Errorf("unexpected status %d", resp.StatusCode)
	}

	// Determine MIME type from Content-Type header.
	ct := resp.Header.Get("Content-Type")
	mimeType, _, _ = mime.ParseMediaType(ct)
	if mimeType == "" {
		mimeType = "application/octet-stream"
	}

	// Only allow image content types (or octet-stream which we sniff later).
	if !strings.HasPrefix(mimeType, "image/") && mimeType != "application/octet-stream" {
		return nil, "", fmt.Errorf("non-image content type: %s", mimeType)
	}

	// Read body with size limit.
	limited := io.LimitReader(resp.Body, maxRemoteImageSize+1)
	body, err := io.ReadAll(limited)
	if err != nil {
		return nil, "", fmt.Errorf("read body: %w", err)
	}
	if len(body) > maxRemoteImageSize {
		return nil, "", fmt.Errorf("image exceeds %d bytes limit", maxRemoteImageSize)
	}

	// If MIME was octet-stream, sniff the real type from body.
	if mimeType == "application/octet-stream" {
		detected := http.DetectContentType(body)
		if strings.HasPrefix(detected, "image/") {
			mimeType = detected
		} else {
			return nil, "", fmt.Errorf("downloaded data is not an image (sniffed: %s)", detected)
		}
	}

	return body, mimeType, nil
}

// extFromURLPath extracts the image file extension from the URL path segment.
func extFromURLPath(rawURL string) string {
	p := path.Ext(path.Base(rawURL))
	switch strings.ToLower(p) {
	case ".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".svg":
		return strings.ToLower(p)
	default:
		return ""
	}
}