Skip to content

CanvasAnvil is an open source AI diagram maker and flowchart generator - an infinite-canvas diagram editor and self-hosted drawio alternative that writes editable .drawio files. Text to diagram, then fix nodes, layout and labels in place. Also AI presentations, interior design CAD/BOM, posters and infographics.

35.8 KB
Raw Blame
import { getDocumentProxy } from "unpdf"
import JSZip from "jszip"
import { bytesToBase64, decodeXmlText, getMimeByExt } from "@/shared/files/binary"
import type { ExtractedVisualAsset, VisualCategory } from "@/shared/files/types"

/**
 * Pulls figures and tables out of PDFs and .docx files so a diagram request
 * can reference the pictures in an uploaded paper, not just its prose.
 */

type PdfTextBox = { text: string; x: number; y: number; w: number; h: number }
type RegionBox = { x: number; y: number; w: number; h: number }
type PdfTextLine = { text: string; x: number; y: number; w: number; h: number }
const USEFUL_VISUAL_PATTERNS = [
    /(framework|pipeline|workflow|architecture|system diagram|method overview|flowchart|block diagram|methods?|approach|methodology|方法框架|流程图|系统结构|模型架构)/i,
    /(table\s*\d+|main results|comparison|baseline|sota|significance|accuracy|f1|auc|results table|主实验|结果对比|对比表|显著性|表\s*\d+)/i,
    /(ablation|sensitivity|parameter analysis|influence|impact|mechanism|变量分析|消融|参数敏感性|影响因素|机制分析)/i,
    /(qualitative|visualization|case study|example|heatmap|confusion matrix|roc|pr curve|结果可视化|案例图|关键结果图|可视化)/i,
    /(dataset|data distribution|sample structure|demographics|statistics|数据分布|样本构成|数据结构|样本结构)/i,
]

const EXCLUDED_VISUAL_HINT_RE =
    /(equation|theorem|proof|appendix|related work|literature review|survey|公式|定理|证明|附录|相关工作|文献综述|参数设置)/i

const CAPTION_ANCHOR_RE = /(\bfig(?:ure)?\.?\s*\d+\b|\btable\s*\d+\b|图\s*\d+|表\s*\d+)/i
const CAPTION_LINE_START_RE = /^\s*(fig(?:ure)?\.?|table|图|表)\s*\d+/i

const CATEGORY_PATTERNS: Array<{ category: VisualCategory; re: RegExp[] }> = [
    {
        category: "framework",
        re: [
            /(framework|pipeline|workflow|architecture|flowchart|block diagram|method overview|模型架构|方法框架|流程图|系统结构)/i,
        ],
    },
    {
        category: "result_table",
        re: [
            /(table\s*\d+|main results|comparison|baseline|sota|significance|results table|结果对比|对比表|显著性|主实验|表\s*\d+)/i,
        ],
    },
    {
        category: "mechanism",
        re: [
            /(ablation|sensitivity|parameter analysis|influence|impact|mechanism|消融|参数敏感性|影响因素|机制分析)/i,
        ],
    },
    {
        category: "key_visual",
        re: [
            /(qualitative|visualization|case study|example|heatmap|confusion matrix|roc|pr curve|可视化|案例图|关键结果图)/i,
        ],
    },
    {
        category: "data_structure",
        re: [
            /(dataset|data distribution|sample structure|demographics|statistics|数据分布|样本构成|数据结构|样本结构)/i,
        ],
    },
]
const clamp = (v: number, min: number, max: number) => Math.max(min, Math.min(max, v))

const renderPdfPageCanvas = async (page: any, targetWidth = 980) => {
    const viewportAtScale1 = page.getViewport({ scale: 1 })
    const scale = targetWidth / Math.max(1, viewportAtScale1.width)
    const viewport = page.getViewport({ scale })

    const canvas = document.createElement("canvas")
    canvas.width = Math.max(1, Math.floor(viewport.width))
    canvas.height = Math.max(1, Math.floor(viewport.height))
    const ctx = canvas.getContext("2d")
    if (!ctx) return null

    await page.render({ canvasContext: ctx, viewport }).promise

    let pageText = ""
    const textBoxes: PdfTextBox[] = []
    try {
        const text = await page.getTextContent()
        const items = Array.isArray(text?.items) ? text.items : []
        for (const it of items) {
            const str = String((it as any)?.str || "").trim()
            if (!str) continue
            pageText += `${str} `

            const transform = Array.isArray((it as any)?.transform) ? (it as any).transform : [1, 0, 0, 1, 0, 0]
            const tx = Number(transform[4] || 0)
            const ty = Number(transform[5] || 0)
            const vw = Math.abs(Number((it as any)?.width || 0) * scale)
            const vh = Math.max(8, Math.abs(Number((it as any)?.height || 0) * scale))
            const [vx, vy] = viewport.convertToViewportPoint(tx, ty)
            const x = clamp(Math.floor(vx), 0, canvas.width - 1)
            const y = clamp(Math.floor(vy - vh), 0, canvas.height - 1)
            const w = clamp(Math.ceil(vw || str.length * 6), 6, canvas.width - x)
            const h = clamp(Math.ceil(vh), 8, canvas.height - y)
            textBoxes.push({ text: str, x, y, w, h })
        }
    } catch {
      // A page whose text layer will not decode still yields its image.
    }

    return { canvas, textBoxes, pageText: pageText.trim() }
}

const buildTextMask = (width: number, height: number, textBoxes: PdfTextBox[]) => {
    const mask = new Uint8Array(width * height)
    for (const box of textBoxes) {
        const x0 = clamp(box.x - 2, 0, width - 1)
        const y0 = clamp(box.y - 2, 0, height - 1)
        const x1 = clamp(box.x + box.w + 2, 0, width - 1)
        const y1 = clamp(box.y + box.h + 2, 0, height - 1)
        for (let y = y0; y <= y1; y += 1) {
            const row = y * width
            for (let x = x0; x <= x1; x += 1) mask[row + x] = 1
        }
    }
    return mask
}

const detectVisualRegions = (canvas: HTMLCanvasElement, textMask: Uint8Array): RegionBox[] => {
    const ctx = canvas.getContext("2d")
    if (!ctx) return []
    const width = canvas.width
    const height = canvas.height
    const imageData = ctx.getImageData(0, 0, width, height).data

    const step = 2
    const sw = Math.max(1, Math.floor(width / step))
    const sh = Math.max(1, Math.floor(height / step))
    const small = new Uint8Array(sw * sh)

    for (let sy = 0; sy < sh; sy += 1) {
        for (let sx = 0; sx < sw; sx += 1) {
            const x = sx * step
            const y = sy * step
            const idx = (y * width + x) * 4
            const r = imageData[idx]
            const g = imageData[idx + 1]
            const b = imageData[idx + 2]
            const a = imageData[idx + 3]
            if (a < 10) continue
            const bright = (r + g + b) / 3
            if (bright > 245) continue
            const pixelIdx = y * width + x
            if (textMask[pixelIdx]) continue
            small[sy * sw + sx] = 1
        }
    }

    const visited = new Uint8Array(sw * sh)
    const boxes: RegionBox[] = []
    const queueX = new Int32Array(sw * sh)
    const queueY = new Int32Array(sw * sh)
    const minArea = Math.max(140, Math.floor(sw * sh * 0.0012))

    for (let y = 0; y < sh; y += 1) {
        for (let x = 0; x < sw; x += 1) {
            const start = y * sw + x
            if (!small[start] || visited[start]) continue

            let qh = 0
            let qt = 0
            visited[start] = 1
            queueX[qt] = x
            queueY[qt] = y
            qt += 1

            let minX = x
            let minY = y
            let maxX = x
            let maxY = y
            let area = 0

            while (qh < qt) {
                const cx = queueX[qh]
                const cy = queueY[qh]
                qh += 1
                area += 1
                minX = Math.min(minX, cx)
                minY = Math.min(minY, cy)
                maxX = Math.max(maxX, cx)
                maxY = Math.max(maxY, cy)

                for (let ny = cy - 1; ny <= cy + 1; ny += 1) {
                    for (let nx = cx - 1; nx <= cx + 1; nx += 1) {
                        if (nx < 0 || ny < 0 || nx >= sw || ny >= sh) continue
                        const ni = ny * sw + nx
                        if (!small[ni] || visited[ni]) continue
                        visited[ni] = 1
                        queueX[qt] = nx
                        queueY[qt] = ny
                        qt += 1
                    }
                }
            }

            if (area < minArea) continue
            const bw = (maxX - minX + 1) * step
            const bh = (maxY - minY + 1) * step
            const ratio = (bw * bh) / Math.max(1, width * height)
            if (ratio > 0.75) continue
            if (bw < 80 || bh < 60) continue
            boxes.push({
                x: clamp(minX * step - 8, 0, width - 1),
                y: clamp(minY * step - 8, 0, height - 1),
                w: clamp(bw + 16, 1, width),
                h: clamp(bh + 16, 1, height),
            })
        }
    }

    if (boxes.length <= 1) return boxes

    const merged: RegionBox[] = []
    for (const box of boxes.sort((a, b) => a.y - b.y || a.x - b.x)) {
        let hit = false
        for (const m of merged) {
            const overlapX = Math.max(0, Math.min(box.x + box.w, m.x + m.w) - Math.max(box.x, m.x))
            const overlapY = Math.max(0, Math.min(box.y + box.h, m.y + m.h) - Math.max(box.y, m.y))
            const overlap = overlapX * overlapY
            const minArea2 = Math.min(box.w * box.h, m.w * m.h)
            const near =
                Math.abs(box.x + box.w / 2 - (m.x + m.w / 2)) < 80 &&
                Math.abs(box.y + box.h / 2 - (m.y + m.h / 2)) < 80
            if (overlap > minArea2 * 0.2 || near) {
                const nx = Math.min(m.x, box.x)
                const ny = Math.min(m.y, box.y)
                const nw = Math.max(m.x + m.w, box.x + box.w) - nx
                const nh = Math.max(m.y + m.h, box.y + box.h) - ny
                m.x = nx
                m.y = ny
                m.w = nw
                m.h = nh
                hit = true
                break
            }
        }
        if (!hit) merged.push({ ...box })
    }

    return merged
}

const cropRegionToDataUrl = (canvas: HTMLCanvasElement, box: RegionBox) => {
    const c = document.createElement("canvas")
    c.width = clamp(Math.floor(box.w), 1, canvas.width)
    c.height = clamp(Math.floor(box.h), 1, canvas.height)
    const ctx = c.getContext("2d")
    if (!ctx) return ""
    ctx.drawImage(canvas, box.x, box.y, box.w, box.h, 0, 0, c.width, c.height)
    // Use PNG for sharper tables/diagrams; we keep maxAssets small to control payload size.
    return c.toDataURL("image/png")
}

const buildRegionTextHint = (box: RegionBox, textBoxes: PdfTextBox[]) => {
    const centerY = box.y + box.h / 2
    const candidates = textBoxes
        .filter((t) => {
            const horizontalOverlap = Math.max(0, Math.min(box.x + box.w, t.x + t.w) - Math.max(box.x, t.x))
            const overlapRatio = horizontalOverlap / Math.max(1, Math.min(box.w, t.w))
            const verticalDistance = Math.abs(t.y + t.h / 2 - centerY)
            return overlapRatio > 0.2 && verticalDistance < 240
        })
        .sort((a, b) => Math.abs(a.y + a.h / 2 - centerY) - Math.abs(b.y + b.h / 2 - centerY))
        .slice(0, 10)
    return candidates.map((x) => x.text).join(" ").trim().slice(0, 600)
}

const expandRegion = (box: RegionBox, width: number, height: number, padX: number, padY: number): RegionBox => {
    const x = clamp(Math.floor(box.x - padX), 0, width - 1)
    const y = clamp(Math.floor(box.y - padY), 0, height - 1)
    const right = clamp(Math.ceil(box.x + box.w + padX), x + 1, width)
    const bottom = clamp(Math.ceil(box.y + box.h + padY), y + 1, height)
    return { x, y, w: right - x, h: bottom - y }
}

const tightenRegionByPixels = (canvas: HTMLCanvasElement, box: RegionBox): RegionBox => {
    const ctx = canvas.getContext("2d")
    if (!ctx) return box
    const sx = clamp(Math.floor(box.x), 0, canvas.width - 1)
    const sy = clamp(Math.floor(box.y), 0, canvas.height - 1)
    const sw = clamp(Math.floor(box.w), 1, canvas.width - sx)
    const sh = clamp(Math.floor(box.h), 1, canvas.height - sy)
    const data = ctx.getImageData(sx, sy, sw, sh).data

    let minX = sw
    let minY = sh
    let maxX = -1
    let maxY = -1

    for (let y = 0; y < sh; y += 1) {
        for (let x = 0; x < sw; x += 1) {
            const idx = (y * sw + x) * 4
            const r = data[idx]
            const g = data[idx + 1]
            const b = data[idx + 2]
            const a = data[idx + 3]
            if (a < 20) continue
            const bright = (r + g + b) / 3
            if (bright > 244) continue
            minX = Math.min(minX, x)
            minY = Math.min(minY, y)
            maxX = Math.max(maxX, x)
            maxY = Math.max(maxY, y)
        }
    }

    if (maxX < 0 || maxY < 0) return box
    const pad = 16
    const nx = clamp(sx + minX - pad, 0, canvas.width - 1)
    const ny = clamp(sy + minY - pad, 0, canvas.height - 1)
    const nr = clamp(sx + maxX + pad + 1, nx + 1, canvas.width)
    const nb = clamp(sy + maxY + pad + 1, ny + 1, canvas.height)
    return { x: nx, y: ny, w: nr - nx, h: nb - ny }
}

const buildTextLines = (textBoxes: PdfTextBox[]): PdfTextLine[] => {
    const sorted = [...textBoxes].sort((a, b) => a.y - b.y || a.x - b.x)
    const lines: Array<{ boxes: PdfTextBox[]; yCenter: number }> = []

    for (const box of sorted) {
        const center = box.y + box.h / 2
        let target = lines.find((line) => Math.abs(line.yCenter - center) <= 10)
        if (!target) {
            target = { boxes: [], yCenter: center }
            lines.push(target)
        }
        target.boxes.push(box)
        target.yCenter = (target.yCenter * (target.boxes.length - 1) + center) / target.boxes.length
    }

    return lines
        .map((line) => {
            const boxes = line.boxes.sort((a, b) => a.x - b.x)
            const x = Math.min(...boxes.map((b) => b.x))
            const y = Math.min(...boxes.map((b) => b.y))
            const right = Math.max(...boxes.map((b) => b.x + b.w))
            const bottom = Math.max(...boxes.map((b) => b.y + b.h))
            const text = boxes.map((b) => b.text).join(" ").replace(/\s+/g, " ").trim()
            return { text, x, y, w: right - x, h: bottom - y }
        })
        .filter((line) => line.text.length > 0)
}

const collectCaptionAnchorRegions = (
    canvas: HTMLCanvasElement,
    textBoxes: PdfTextBox[]
): Array<{ box: RegionBox; hint: string }> => {
    const width = canvas.width
    const height = canvas.height
    const lines = buildTextLines(textBoxes)
    const anchors = lines.filter((line) => {
        if (EXCLUDED_VISUAL_HINT_RE.test(line.text)) return false
        return CAPTION_ANCHOR_RE.test(line.text) || USEFUL_VISUAL_PATTERNS.some((re) => re.test(line.text))
    })

    const results: Array<{ box: RegionBox; hint: string }> = []
    for (const line of anchors) {
        const isTable = /(table|表)/i.test(line.text)
        const isFigure = /(fig|figure|图)/i.test(line.text)
        const yCenter = line.y + line.h / 2

        let top = yCenter - height * 0.35
        let bottom = yCenter + height * 0.35
        if (isTable) {
            top = yCenter - height * 0.08
            bottom = yCenter + height * 0.48
        } else if (isFigure) {
            top = yCenter - height * 0.58
            bottom = yCenter + height * 0.12
        }

        const base: RegionBox = {
            x: Math.floor(width * 0.05),
            y: clamp(Math.floor(top), 0, height - 1),
            w: Math.floor(width * 0.9),
            h: clamp(Math.floor(bottom - top), 1, height),
        }
        const expanded = expandRegion(base, width, height, 18, 20)
        const tightened = tightenRegionByPixels(canvas, expanded)
        const areaRatio = (tightened.w * tightened.h) / Math.max(1, width * height)
        if (areaRatio < 0.05 || areaRatio > 0.9) continue
        results.push({ box: tightened, hint: line.text })
    }

    return results
}

const unionBoxes = (a: RegionBox, b: RegionBox): RegionBox => {
    const x = Math.min(a.x, b.x)
    const y = Math.min(a.y, b.y)
    const right = Math.max(a.x + a.w, b.x + b.w)
    const bottom = Math.max(a.y + a.h, b.y + b.h)
    return { x, y, w: right - x, h: bottom - y }
}

const detectCaptionKind = (text: string): "figure" | "table" => {
    const t = String(text || "").toLowerCase()
    if (t.startsWith("table") || t.includes(" table ") || /表\s*\d+/.test(text)) return "table"
    return "figure"
}

const collectCaptionBasedCrops = (args: {
    canvas: HTMLCanvasElement
    textBoxes: PdfTextBox[]
    detectedRegions: RegionBox[]
}): Array<{ box: RegionBox; hint: string; kind: "figure" | "table" }> => {
    const { canvas, textBoxes, detectedRegions } = args
    const width = canvas.width
    const height = canvas.height
    const lines = buildTextLines(textBoxes)

    const candidates = lines
        .filter((line) => {
            const text = String(line.text || "")
            if (!CAPTION_LINE_START_RE.test(text)) return false
            // Avoid catching in-paragraph references like "see Figure 1"
            const trimmed = text.trim()
            const looksLikeCaption = /^(\s*(fig(?:ure)?\.?|table|图|表)\s*\d+)\s*[:.]/i.test(trimmed)
            return looksLikeCaption || trimmed.length <= 80
        })
        .slice(0, 40)

    const out: Array<{ box: RegionBox; hint: string; kind: "figure" | "table" }> = []

    for (const line of candidates) {
        const kind = detectCaptionKind(line.text)
        const captionBox: RegionBox = expandRegion(
            { x: line.x, y: line.y, w: line.w, h: line.h },
            width,
            height,
            14,
            10
        )

        const captionTop = captionBox.y
        const captionBottom = captionBox.y + captionBox.h

        const window: RegionBox = (() => {
            if (kind === "table") {
                // Table captions are often above the table; capture below by default.
                const top = clamp(Math.floor(captionTop - height * 0.06), 0, height - 1)
                const bottom = clamp(Math.floor(captionBottom + height * 0.62), top + 1, height)
                return { x: Math.floor(width * 0.04), y: top, w: Math.floor(width * 0.92), h: bottom - top }
            }
            // Figure captions are often below the figure; capture above by default.
            const top = clamp(Math.floor(captionTop - height * 0.68), 0, height - 1)
            const bottom = clamp(Math.floor(captionBottom + height * 0.10), top + 1, height)
            return { x: Math.floor(width * 0.04), y: top, w: Math.floor(width * 0.92), h: bottom - top }
        })()

        const windowBottom = window.y + window.h
        const windowTop = window.y

        const inWindow = detectedRegions
            .map((r) => ({ r, cy: r.y + r.h / 2, cx: r.x + r.w / 2 }))
            .filter(({ r, cy }) => {
                const within = cy >= windowTop && cy <= windowBottom
                if (!within) return false
                if (kind === "figure") return r.y + r.h <= captionTop + 36
                return r.y >= captionBottom - 36
            })

        const pick = (() => {
            if (inWindow.length === 0) return null
            const scored = inWindow
                .map(({ r }) => {
                    const area = r.w * r.h
                    const dist = kind === "figure"
                        ? Math.abs((r.y + r.h) - captionTop)
                        : Math.abs(r.y - captionBottom)
                    // Prefer big + close regions.
                    const score = area - dist * 120
                    return { r, score }
                })
                .sort((a, b) => b.score - a.score)
            return scored[0]?.r || null
        })()

        let cropBox: RegionBox
        if (pick) {
            cropBox = unionBoxes(pick, captionBox)
            // Include any nearby regions that likely belong to the same figure/table.
            for (const { r } of inWindow) {
                const overlapX = Math.max(0, Math.min(cropBox.x + cropBox.w, r.x + r.w) - Math.max(cropBox.x, r.x))
                const overlapY = Math.max(0, Math.min(cropBox.y + cropBox.h, r.y + r.h) - Math.max(cropBox.y, r.y))
                const overlap = overlapX * overlapY
                const minArea = Math.min(cropBox.w * cropBox.h, r.w * r.h)
                const near = Math.abs((r.y + r.h / 2) - (cropBox.y + cropBox.h / 2)) < 120
                if (overlap > minArea * 0.08 || near) cropBox = unionBoxes(cropBox, r)
            }
            cropBox = expandRegion(cropBox, width, height, 24, 22)
            cropBox = tightenRegionByPixels(canvas, cropBox)
        } else {
            // No clear visual region detected; fall back to a reasonable window around the caption.
            cropBox = tightenRegionByPixels(canvas, expandRegion(window, width, height, 8, 10))
        }

        const ratio = (cropBox.w * cropBox.h) / Math.max(1, width * height)
        if (ratio < 0.04) {
            // Too small => half page fallback.
            const half: RegionBox =
                kind === "table"
                    ? { x: 0, y: Math.floor(height * 0.45), w: width, h: height - Math.floor(height * 0.45) }
                    : { x: 0, y: 0, w: width, h: Math.floor(height * 0.60) }
            cropBox = tightenRegionByPixels(canvas, expandRegion(half, width, height, 0, 0))
        }
        const ratio2 = (cropBox.w * cropBox.h) / Math.max(1, width * height)
        if (ratio2 < 0.06) {
            // Still abnormal => full page fallback.
            cropBox = { x: 0, y: 0, w: width, h: height }
        }

        out.push({ box: cropBox, hint: String(line.text || "").slice(0, 200), kind })
    }

    return out
}

const detectVisualCategory = (hintText: string): VisualCategory => {
    const text = String(hintText || "")
    for (const item of CATEGORY_PATTERNS) {
        if (item.re.some((re) => re.test(text))) return item.category
    }
    return "other"
}

const scoreUsefulVisualHint = (text: string) => {
    const hint = String(text || "")
    let score = 0
    let matched = 0
    for (const re of USEFUL_VISUAL_PATTERNS) {
        if (!re.test(hint)) continue
        matched += 1
        score += 2
    }
    if (/(figure|fig\.?\s*\d+|图\s*\d+)/i.test(hint)) score += 1
    if (/(table\s*\d+|表\s*\d+)/i.test(hint)) score += 1
    if (EXCLUDED_VISUAL_HINT_RE.test(hint)) score -= 3
    return { score, matched }
}

export async function extractPdfVisualAssets(
    file: File,
    options?: { maxPages?: number; maxAssets?: number; targetWidth?: number }
): Promise<ExtractedVisualAsset[]> {
    const maxPages = Math.max(1, options?.maxPages ?? 12)
    const maxAssets = Math.max(1, options?.maxAssets ?? 10)
    // Higher default resolution to preserve figure/table readability.
    const targetWidth = Math.max(980, options?.targetWidth ?? 1400)

    const buffer = await file.arrayBuffer()
    const pdf = await getDocumentProxy(new Uint8Array(buffer))
    const pageCount = Number((pdf as any)?.numPages || 0)
    if (!pageCount) return []

    type CandidateAsset = {
        page: number
        score: number
        textHint: string
        dataUrl: string
        areaRatio: number
        category: VisualCategory
    }

    const accepted: CandidateAsset[] = []
    const fallback: CandidateAsset[] = []

    for (let pageNo = 1; pageNo <= Math.min(pageCount, maxPages); pageNo += 1) {
        try {
            const page = await (pdf as any).getPage(pageNo)
            const rendered = await renderPdfPageCanvas(page, targetWidth)
            if (!rendered) continue

            const pageText = String(rendered.pageText || "").slice(0, 1400)
            const textMask = buildTextMask(rendered.canvas.width, rendered.canvas.height, rendered.textBoxes)
            const detectedRegions = detectVisualRegions(rendered.canvas, textMask)
            const captionCrops = collectCaptionBasedCrops({
                canvas: rendered.canvas,
                textBoxes: rendered.textBoxes,
                detectedRegions,
            })
            const anchorRegions = collectCaptionAnchorRegions(rendered.canvas, rendered.textBoxes)
            const candidateRegions = [
                ...captionCrops.map((item) => ({ box: item.box, seedHint: item.hint, source: "caption" as const })),
                ...anchorRegions.map((item) => ({ box: item.box, seedHint: item.hint, source: "anchor" as const })),
                ...detectedRegions.map((box) => ({ box, seedHint: "", source: "detected" as const })),
            ]

            for (const candidate of candidateRegions) {
                if (accepted.length >= maxAssets * 8) break
                const expanded =
                    candidate.source === "caption"
                        ? expandRegion(candidate.box, rendered.canvas.width, rendered.canvas.height, 6, 8)
                        : expandRegion(candidate.box, rendered.canvas.width, rendered.canvas.height, 28, 28)
                const region = tightenRegionByPixels(rendered.canvas, expanded)

                const dataUrl = cropRegionToDataUrl(rendered.canvas, region)
                if (!dataUrl.startsWith("data:image")) continue

                const areaRatio = (region.w * region.h) / Math.max(1, rendered.canvas.width * rendered.canvas.height)
                const localHint = buildRegionTextHint(region, rendered.textBoxes)
                const textHint = `${candidate.seedHint} ${localHint} ${pageText}`.trim().slice(0, 1200)
                const scored = scoreUsefulVisualHint(`${textHint} ${pageText}`)
                const category = detectVisualCategory(textHint)
                const fullRegionBonus = areaRatio > 0.08 && areaRatio < 0.88 ? 1 : 0
                const anchorBonus = candidate.source === "caption" ? 4 : candidate.source === "anchor" ? 2 : 0
                const categoryBonus = category !== "other" ? 1 : 0
                const finalScore = scored.score + fullRegionBonus + anchorBonus + categoryBonus
                const item: CandidateAsset = { page: pageNo, score: finalScore, textHint, dataUrl, areaRatio, category }

                if ((scored.matched > 0 || candidate.source === "caption") && finalScore > 0) {
                    accepted.push(item)
                } else if (areaRatio > 0.08 && areaRatio < 0.9) {
                    fallback.push(item)
                }
            }
        } catch (e) {
            console.error("Failed to extract visual asset from PDF page", pageNo, e)
        }
    }

    const dedupeByData = (items: CandidateAsset[]) => {
        const seen = new Set<string>()
        const out: CandidateAsset[] = []
        for (const item of items.sort((a, b) => b.score - a.score || a.page - b.page)) {
            const key = `${item.page}|${item.dataUrl.slice(0, 120)}`
            if (seen.has(key)) continue
            seen.add(key)
            out.push(item)
        }
        return out
    }

    const pickByPriority = (items: CandidateAsset[]) => {
        const priority: VisualCategory[] = [
            "framework",
            "result_table",
            "mechanism",
            "key_visual",
            "data_structure",
        ]
        const sorted = dedupeByData(items)
        const picked: CandidateAsset[] = []
        const used = new Set<string>()

        for (const category of priority) {
            const hit = sorted.find((item) => item.category === category && !used.has(item.dataUrl))
            if (!hit) continue
            picked.push(hit)
            used.add(hit.dataUrl)
            if (picked.length >= maxAssets) return picked
        }

        for (const item of sorted) {
            if (used.has(item.dataUrl)) continue
            picked.push(item)
            used.add(item.dataUrl)
            if (picked.length >= maxAssets) break
        }

        return picked
    }

    const selected = pickByPriority(accepted.length > 0 ? accepted : fallback).sort((a, b) => a.page - b.page)

    return selected.map((x, idx) => ({
        id: `pdf-${file.name}-${x.page}-${idx + 1}`,
        sourceFileName: file.name,
        sourceType: "pdf",
        page: x.page,
        order: idx + 1,
        dataUrl: x.dataUrl,
        textHint: x.textHint,
    }))
}

export async function extractWordVisualAssets(
    file: File,
    options?: { maxAssets?: number }
): Promise<ExtractedVisualAsset[]> {
    const maxAssets = Math.max(1, options?.maxAssets ?? 10)
    const buffer = await file.arrayBuffer()
    const zip = await JSZip.loadAsync(buffer)
    const mediaEntries = Object.values(zip.files).filter(
        (f) => !f.dir && /^word\/media\//i.test(String(f.name || ""))
    )
    const assets: ExtractedVisualAsset[] = []

    const relsXml = zip.file("word/_rels/document.xml.rels")
        ? await zip.file("word/_rels/document.xml.rels")!.async("string")
        : ""
    const docXml = zip.file("word/document.xml")
        ? await zip.file("word/document.xml")!.async("string")
        : ""

    const ridToTarget = new Map<string, string>()
    if (relsXml) {
        const relRe = /<Relationship\b[^>]*\bId="([^"]+)"[^>]*\bTarget="([^"]+)"[^>]*\/>/g
        let rm: RegExpExecArray | null
        while ((rm = relRe.exec(relsXml))) {
            const id = String(rm[1] || "").trim()
            const target = String(rm[2] || "").trim()
            if (!id || !target) continue
            ridToTarget.set(id, target.replace(/\\/g, "/"))
        }
    }

    const paragraphs: Array<{ text: string; embeds: string[]; metrics: Record<string, { w: number; h: number; area: number }> }> = []
    if (docXml) {
        const pRe = /<w:p\b[\s\S]*?<\/w:p>/g
        const embedsRe = /\br:embed="(rId\d+)"/g
        const tRe = /<w:t\b[^>]*>([\s\S]*?)<\/w:t>/g
        const extentRe = /<(?:wp:extent|a:ext)\b[^>]*\bcx="(\d+)"[^>]*\bcy="(\d+)"/g
        const emuToPx = (v: number) => Math.max(0, Math.round(v / 9525))
        let pm: RegExpExecArray | null
        while ((pm = pRe.exec(docXml))) {
            const p = pm[0]
            const embeds: string[] = []
            let em: RegExpExecArray | null
            while ((em = embedsRe.exec(p))) embeds.push(String(em[1] || ""))
            const parts: string[] = []
            let tm: RegExpExecArray | null
            while ((tm = tRe.exec(p))) parts.push(decodeXmlText(tm[1] || ""))
            const text = parts.join("").replace(/\s+/g, " ").trim()
            const metricList: Array<{ w: number; h: number; area: number }> = []
            let xm: RegExpExecArray | null
            while ((xm = extentRe.exec(p))) {
                const cx = Number(xm[1] || 0)
                const cy = Number(xm[2] || 0)
                if (!Number.isFinite(cx) || !Number.isFinite(cy)) continue
                const w = emuToPx(cx)
                const h = emuToPx(cy)
                metricList.push({ w, h, area: w * h })
            }
            const metrics: Record<string, { w: number; h: number; area: number }> = {}
            for (let i = 0; i < embeds.length; i += 1) {
                metrics[embeds[i]] = metricList[i] || { w: 0, h: 0, area: 0 }
            }
            paragraphs.push({ text, embeds, metrics })
        }
    }

    const captionRe = /^(fig(?:ure)?\.?|table|图|表)\s*\d+\s*[:.]/i
    const mediaHintByName = new Map<string, string>()
    const mediaMetricByName = new Map<string, { w: number; h: number; area: number }>()
    for (let i = 0; i < paragraphs.length; i += 1) {
        const p = paragraphs[i]
        if (!p.embeds || p.embeds.length === 0) continue
        for (const rid of p.embeds) {
            const target = ridToTarget.get(rid) || ""
            const media = target ? `word/${target.replace(/^\.?\//, "")}` : ""
            if (!media) continue
            const prev = paragraphs[i - 1]?.text || ""
            const next = paragraphs[i + 1]?.text || ""
            const here = p.text || ""
            const cap = [prev, next, here].find((t) => captionRe.test(String(t || "").trim()))
            const hint = [cap || "", prev, here, next].filter(Boolean).join(" ").replace(/\s+/g, " ").trim().slice(0, 800)
            if (hint) mediaHintByName.set(media.toLowerCase(), hint)
            const m = p.metrics?.[rid]
            if (m && m.area > 0) mediaMetricByName.set(media.toLowerCase(), m)
        }
    }

    type Candidate = { entry: any; hint: string; metric: { w: number; h: number; area: number }; score: number; category: VisualCategory }
    const excludedHintRe =
        /(equation|formula|eq\.?\s*\(?\d+\)?|proof|appendix|related work|supplement|公式|推导|证明|附录|文献综述|参数设置)/i
    const candidates: Candidate[] = []

    for (const entry of mediaEntries) {
        const key = String(entry.name || "").toLowerCase()
        const hint = mediaHintByName.get(key) || ""
        const metric = mediaMetricByName.get(key) || { w: 0, h: 0, area: 0 }
        const scored = scoreUsefulVisualHint(hint)
        const isCaptionLike = captionRe.test(String(hint || "").trim())
        const isExcluded = excludedHintRe.test(hint) || EXCLUDED_VISUAL_HINT_RE.test(hint)
        const sizeBonus = metric.area >= 180000 ? 2 : metric.area >= 70000 ? 1 : 0
        const tinyPenalty = metric.area > 0 && metric.area < 14000 ? 4 : 0
        const score = scored.score + (isCaptionLike ? 2 : 0) + sizeBonus - tinyPenalty - (isExcluded ? 6 : 0)
        const category = detectVisualCategory(hint)
        candidates.push({ entry, hint, metric, score, category })
    }

    const priority: VisualCategory[] = [
        "framework",
        "result_table",
        "mechanism",
        "key_visual",
        "data_structure",
        "other",
    ]

    const keep = candidates
        .filter((c) => c.score > 0 || c.metric.area >= 260000)
        .sort((a, b) => {
            const pa = priority.indexOf(a.category)
            const pb = priority.indexOf(b.category)
            if (pa !== pb) return pa - pb
            if (b.score !== a.score) return b.score - a.score
            return (b.metric.area || 0) - (a.metric.area || 0)
        })
        .slice(0, maxAssets)

    for (let i = 0; i < keep.length; i += 1) {
        const { entry, hint } = keep[i]
        try {
            const bytes = await entry.async("uint8array")
            const mime = getMimeByExt(entry.name)
            const dataUrl = `data:${mime};base64,${bytesToBase64(bytes)}`
            assets.push({
                id: `word-${file.name}-${i + 1}`,
                sourceFileName: file.name,
                sourceType: "word",
                order: i + 1,
                dataUrl,
                textHint: hint,
            })
        } catch (e) {
            console.error("Failed to extract visual asset from Word", entry.name, e)
        }
    }

    return assets
}

export async function extractThirdPartyVisualAssets(
    file: File,
    options: { apiToken: string; apiBase?: string; maxAssets?: number; maxWaitMs?: number }
): Promise<ExtractedVisualAsset[]> {
    const token = String(options.apiToken || "").trim()
    if (!token) throw new Error("Third-party parser token is empty")
    const apiBase = String(options.apiBase || "https://mineru.net").trim().replace(/\/+$/, "")
    const maxAssets = Math.max(1, options.maxAssets ?? 10)
    const maxWaitMs = Math.max(5000, options.maxWaitMs ?? 120000)
    const fileBuf = new Uint8Array(await file.arrayBuffer())
    const resp = await fetch("/api/third-party-parser", {
        method: "POST",
        headers: { "Content-Type": "application/json" },
        body: JSON.stringify({
            fileName: file.name,
            fileType: file.type || "",
            fileBase64: bytesToBase64(fileBuf),
            apiToken: token,
            apiBase,
            maxAssets,
            maxWaitMs,
        }),
    })
    if (!resp.ok) {
        let detail = ""
        try {
            const errPayload = await resp.json()
            detail = String(errPayload?.error || "")
        } catch {
          // The proxy did not send a JSON body; the status code below is enough.
        }
        throw new Error(
            `Third-party parser proxy failed: ${resp.status}${detail ? ` - ${detail}` : ""}`,
        )
    }
    const payload = await resp.json()
    if (!payload?.success) {
        throw new Error(String(payload?.error || "Third-party parser proxy error"))
    }
    const items = Array.isArray(payload?.assets) ? payload.assets : []
    return items
        .filter((x: any) => x && typeof x.dataUrl === "string" && x.dataUrl.startsWith("data:image"))
        .slice(0, maxAssets)
        .map((x: any, idx: number) => ({
            id: `third-party-${file.name}-${idx + 1}`,
            sourceFileName: file.name,
            sourceType: "third_party",
            order: idx + 1,
            dataUrl: String(x.dataUrl),
            textHint: typeof x.textHint === "string" ? x.textHint : "",
        }))
}