Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
# Copyright (c) Opendatalab. All rights reserved.
"""PDF 公式识别输入、结果回填和公式编号规范化。"""
from __future__ import annotations
import math
from typing import TYPE_CHECKING, Any
import numpy as np
from docvortex.assets import image_size as _normalize_page_size
from docvortex.content.text import build_tagged_formula_content
from docvortex.geometry import bbox_to_quad as _medium_bbox_to_quad
from docvortex.geometry import normalize_to_int_bbox
from PIL import Image
from ....model.ocr.image import get_rotate_crop_image_for_text_rec, rgb_to_bgr
from ....types import RAW_FORMULA_NUMBER, BlockType
from .model_inputs import _bbox_to_pixel_bbox, _normalize_layout_bbox_to_unit, _normalize_medium_content
if TYPE_CHECKING:
from ....model.runtime.hybrid import HybridLocalModelContext
def _is_hybrid_equation_block(block: dict[str, Any]) -> bool:
"""判断 raw Hybrid/VLM block 是否表示可合并编号的行间公式。"""
return str(block.get("type") or "").lower() in {
BlockType.EQUATION,
"display_formula",
}
def _is_hybrid_formula_number_block(block: dict[str, Any]) -> bool:
"""判断 raw Hybrid/VLM block 是否表示公式编号。"""
return str(block.get("type") or "").lower() == RAW_FORMULA_NUMBER
def _normalize_hybrid_formula_bbox(bbox: Any) -> list[float] | None:
"""将 Hybrid/VLM 公式框规范为合法浮点四元组,非法或退化框返回 None。"""
if not isinstance(bbox, (list, tuple)) or len(bbox) != 4:
return None
if any(isinstance(value, bool) for value in bbox):
return None
try:
normalized_bbox = [float(value) for value in bbox]
except (TypeError, ValueError):
return None
if any(not math.isfinite(value) for value in normalized_bbox):
return None
x0, y0, x1, y1 = normalized_bbox
if x1 <= x0 or y1 <= y0:
return None
return normalized_bbox
def _merge_hybrid_formula_number_block(equation_block: dict[str, Any], number_block: dict[str, Any]) -> None:
"""把公式编号的 bbox 和非空内容合并到相邻 Hybrid/VLM 公式 block。"""
equation_bbox = _normalize_hybrid_formula_bbox(equation_block.get("bbox"))
number_bbox = _normalize_hybrid_formula_bbox(number_block.get("bbox"))
if equation_bbox is not None and number_bbox is not None:
equation_block["bbox"] = [
min(equation_bbox[0], number_bbox[0]),
min(equation_bbox[1], number_bbox[1]),
max(equation_bbox[2], number_bbox[2]),
max(equation_bbox[3], number_bbox[3]),
]
target_key = "latex" if equation_block.get("latex") else "content"
tagged_content = build_tagged_formula_content(
str(equation_block.get(target_key) or ""),
str(number_block.get("content") or number_block.get("text") or ""),
)
if tagged_content:
equation_block[target_key] = tagged_content
def optimize_hybrid_formula_number_blocks(page_model_list: list[dict[str, Any]]) -> list[dict[str, Any]]:
"""合并 Hybrid raw model list 中的 formula_number,并返回新的 block 列表。"""
optimized_blocks: list[dict[str, Any]] = []
blocks = list(page_model_list)
for index, block in enumerate(blocks):
if not _is_hybrid_formula_number_block(block):
optimized_blocks.append(block)
continue
prev_block = blocks[index - 1] if index > 0 else None
if prev_block and _is_hybrid_equation_block(prev_block):
_merge_hybrid_formula_number_block(prev_block, block)
continue
next_block = blocks[index + 1] if index + 1 < len(blocks) else None
next_next_block = blocks[index + 2] if index + 2 < len(blocks) else None
if (
next_block
and _is_hybrid_equation_block(next_block)
and (next_next_block is None or not _is_hybrid_formula_number_block(next_next_block))
):
_merge_hybrid_formula_number_block(next_block, block)
continue
fallback_block = dict(block)
fallback_block["type"] = BlockType.TEXT
optimized_blocks.append(fallback_block)
return optimized_blocks
def _build_formula_inputs(images_layout_res: list[list[dict[str, Any]]]) -> list[list[dict[str, Any]]]:
"""构造完整 MFD/MFR 输入,保留全部行内和行间公式框。"""
formula_inputs = []
for layout_res in images_layout_res:
page_formula_inputs = []
for res in layout_res:
label = res.get("label")
if label not in ["inline_formula", "display_formula"]:
continue
bbox = res.get("bbox")
if bbox is None or len(bbox) != 4:
continue
page_formula_inputs.append(
{
"label": label,
"bbox": list(bbox),
"score": float(res.get("score", 0.0)),
# layout 只提供公式位置;未运行 MFR 的 high/xhigh OCR 必须保留空 LaTeX。
"latex": "",
}
)
formula_inputs.append(page_formula_inputs)
return formula_inputs
def _split_formula_results(
images_formula_list: list[list[dict[str, Any]]],
) -> tuple[list[list[dict[str, Any]]], list[list[dict[str, Any]]]]:
"""按原始标签拆分 MFR 结果,避免行间公式进入 inline sidecar。"""
inline_formula_list = []
display_formula_list = []
for page_formula_list in images_formula_list:
inline_formula_list.append([formula for formula in page_formula_list if formula.get("label") == "inline_formula"])
display_formula_list.append([formula for formula in page_formula_list if formula.get("label") == "display_formula"])
return inline_formula_list, display_formula_list
def _apply_medium_display_formula_results(
model_list: list[list[dict[str, Any]]],
display_formula_list: list[list[dict[str, Any]]],
images_pil_list: list[Image.Image],
) -> None:
"""将 medium 行间公式 LaTeX 按页和 bbox 回填到对应 equation 块。"""
for page_idx, (page_model_list, page_display_formula_list, page_image) in enumerate(
zip(model_list, display_formula_list, images_pil_list)
):
page_size = _normalize_page_size(page_image)
equation_blocks_by_bbox: dict[tuple[float, ...], list[dict[str, Any]]] = {}
for block in page_model_list:
if block.get("type") != BlockType.EQUATION:
continue
block_bbox = block.get("bbox")
if block_bbox is None or len(block_bbox) != 4:
continue
equation_blocks_by_bbox.setdefault(tuple(float(value) for value in block_bbox), []).append(block)
for formula in page_display_formula_list:
normalized_bbox = _normalize_layout_bbox_to_unit(formula.get("bbox"), page_size)
if normalized_bbox is None:
continue
matched_blocks = equation_blocks_by_bbox.get(tuple(normalized_bbox), [])
if len(matched_blocks) != 1:
raise ValueError(
"Hybrid medium display formula must match exactly one equation block: "
f"page_idx={page_idx}, bbox={normalized_bbox}, matches={len(matched_blocks)}"
)
matched_blocks[0]["content"] = formula.get("latex", "")
def _formula_item_to_pixel_bbox(item: dict[str, Any]) -> list[int] | None:
"""读取公式项的四点 bbox,并转换为后续裁图使用的整数坐标。"""
bbox = item.get("bbox")
if bbox is not None and len(bbox) == 4:
return [int(float(v)) for v in bbox]
return None
def _apply_medium_formula_number_ocr(
local_context: HybridLocalModelContext,
model_list: list[list[dict[str, Any]]],
np_images: list[np.ndarray],
) -> None:
"""对 medium formula_number 裁剪图执行 OCR-rec,并把编号文本回填到原始 layout 项。"""
need_rec_items: list[dict[str, Any]] = []
formula_number_crops: list[np.ndarray] = []
for block_list, np_img in zip(model_list, np_images):
image_h, image_w = np_img.shape[:2]
bgr_image = rgb_to_bgr(np_img)
for block_item in block_list:
if block_item.get("type") != RAW_FORMULA_NUMBER:
continue
formula_number_bbox = normalize_to_int_bbox(
_bbox_to_pixel_bbox(block_item.get("bbox"), (image_w, image_h)),
image_size=(image_h, image_w),
)
if formula_number_bbox is None:
continue
# 使用 OCR rec 的标准旋转裁剪逻辑,保证 medium 编号裁图与正文 OCR-rec 输入一致。
formula_number_crops.append(
get_rotate_crop_image_for_text_rec(
bgr_image,
_medium_bbox_to_quad(formula_number_bbox).copy(),
)
)
need_rec_items.append(block_item)
if not formula_number_crops:
return
ocr_result_list = local_context.ocr_model.ocr(
formula_number_crops,
det=False,
tqdm_enable=True,
tqdm_desc="OCR-rec",
)[0]
if len(ocr_result_list) != len(need_rec_items):
raise ValueError(
"Hybrid medium formula number OCR rec result count mismatch: "
f"ocr_result_list={len(ocr_result_list)}, need_rec_items={len(need_rec_items)}"
)
for block_item, ocr_result in zip(need_rec_items, ocr_result_list):
if not ocr_result or len(ocr_result) < 2:
continue
ocr_text, _ = ocr_result
normalized_text = _normalize_medium_content(ocr_text)
if normalized_text:
block_item["content"] = normalized_text