Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

8.7 KB
Raw Blame
# Copyright (c) Opendatalab. All rights reserved.
"""
This code is refered from:
https://github.com/WenmuZhou/DBNet.pytorch/blob/master/post_processing/seg_detector_representer.py
"""

from __future__ import annotations
from docvortex.image import (
    morphology,
    rasterize_polygons,
    trace_contours,
    contour_length,
    minimum_rectangle,
    rectangle_corners,
    simplify_contour,
)


from typing import Any, Literal

import numpy as np
from shapely.geometry import Polygon
import pyclipper


class DBPostProcess(object):
    """
    The post process for Differentiable Binarization (DB).
    """

    def __init__(
        self,
        thresh: float = 0.3,
        box_thresh: float = 0.7,
        max_candidates: int = 1000,
        unclip_ratio: float = 2.0,
        use_dilation: bool = False,
        score_mode: Literal["fast", "slow"] = "fast",
        box_type: Literal["quad", "poly"] = "quad",
    ) -> None:
        """配置 DB 阈值、膨胀和多边形模式,输入统一为 NumPy 概率图。"""
        self.thresh = thresh
        self.box_thresh = box_thresh
        self.max_candidates = max_candidates
        self.unclip_ratio = unclip_ratio
        self.min_size = 3
        self.score_mode = score_mode
        self.box_type = box_type
        assert score_mode in ["slow", "fast"], "Score mode must be in [slow, fast] but got: {}".format(score_mode)

        self.dilation_kernel = None if not use_dilation else np.array([[1, 1], [1, 1]])

    def polygons_from_bitmap(
        self, pred: np.ndarray, _bitmap: np.ndarray, dest_width: int, dest_height: int
    ) -> tuple[list[list[list[float]]], list[float]]:
        """从概率图轮廓恢复印章等弯曲文字的多边形。"""

        bitmap = _bitmap
        height, width = bitmap.shape

        boxes = []
        scores = []

        contours = trace_contours((bitmap * 255).astype(np.uint8))

        for contour in contours[: self.max_candidates]:
            epsilon = 0.002 * contour_length(contour, closed=True)
            approx = simplify_contour(contour, epsilon, closed=True)
            points = approx.reshape((-1, 2))
            if points.shape[0] < 4:
                continue

            score = self.box_score_fast(pred, points.reshape(-1, 2))
            if self.box_thresh > score:
                continue

            if points.shape[0] > 2:
                box = self.unclip(points, self.unclip_ratio)
                if len(box) > 1:
                    continue
            else:
                continue

            box = np.array(box).reshape(-1, 2)
            if len(box) == 0:
                continue

            _, sside = self.get_mini_boxes(box.reshape((-1, 1, 2)))
            if sside < self.min_size + 2:
                continue

            box = np.array(box)
            box[:, 0] = np.clip(np.round(box[:, 0] / width * dest_width), 0, dest_width)
            box[:, 1] = np.clip(np.round(box[:, 1] / height * dest_height), 0, dest_height)
            boxes.append(box.tolist())
            scores.append(score)
        return boxes, scores

    def boxes_from_bitmap(
        self, pred: np.ndarray, _bitmap: np.ndarray, dest_width: int, dest_height: int
    ) -> tuple[np.ndarray, list[float]]:
        """从概率图恢复普通文字的四边形。"""

        bitmap = _bitmap
        height, width = bitmap.shape

        contours = trace_contours((bitmap * 255).astype(np.uint8))

        num_contours = min(len(contours), self.max_candidates)

        boxes = []
        scores = []
        for index in range(num_contours):
            contour = contours[index]
            points, sside = self.get_mini_boxes(contour)
            if sside < self.min_size:
                continue
            points = np.array(points)
            if self.score_mode == "fast":
                score = self.box_score_fast(pred, points.reshape(-1, 2))
            else:
                score = self.box_score_slow(pred, contour)
            if self.box_thresh > score:
                continue

            box = self.unclip(points, self.unclip_ratio)
            if len(box) > 1:
                continue
            box = np.array(box).reshape(-1, 1, 2)
            box, sside = self.get_mini_boxes(box)
            if sside < self.min_size + 2:
                continue
            box = np.array(box)

            box[:, 0] = np.clip(np.round(box[:, 0] / width * dest_width), 0, dest_width)
            box[:, 1] = np.clip(np.round(box[:, 1] / height * dest_height), 0, dest_height)
            boxes.append(box.astype(np.int16))
            scores.append(score)
        return np.array(boxes, dtype=np.int16), scores

    def unclip(self, box: np.ndarray, unclip_ratio: float) -> list[list[list[float]]]:
        """按面积与周长扩张检测轮廓。"""
        poly = Polygon(box)
        distance = poly.area * unclip_ratio / poly.length
        offset = pyclipper.PyclipperOffset()
        offset.AddPath(box, pyclipper.JT_ROUND, pyclipper.ET_CLOSEDPOLYGON)
        expanded = offset.Execute(distance)
        return expanded

    def get_mini_boxes(self, contour: np.ndarray) -> tuple[list[np.ndarray], float]:
        """返回轮廓的有序最小外接矩形与短边长度。"""

        bounding_box = minimum_rectangle(contour)
        points = sorted(rectangle_corners(bounding_box), key=lambda x: x[0])

        index_1, index_2, index_3, index_4 = 0, 1, 2, 3
        if points[1][1] > points[0][1]:
            index_1 = 0
            index_4 = 1
        else:
            index_1 = 1
            index_4 = 0
        if points[3][1] > points[2][1]:
            index_2 = 2
            index_3 = 3
        else:
            index_2 = 3
            index_3 = 2

        box = [points[index_1], points[index_2], points[index_3], points[index_4]]
        return box, min(bounding_box[1])

    def box_score_fast(self, bitmap: np.ndarray, _box: np.ndarray) -> float:
        """计算外接框覆盖区域的平均检测分数。"""

        h, w = bitmap.shape[:2]
        box = _box.copy()
        xmin = np.clip(np.floor(box[:, 0].min()).astype(np.int32), 0, w - 1)
        xmax = np.clip(np.ceil(box[:, 0].max()).astype(np.int32), 0, w - 1)
        ymin = np.clip(np.floor(box[:, 1].min()).astype(np.int32), 0, h - 1)
        ymax = np.clip(np.ceil(box[:, 1].max()).astype(np.int32), 0, h - 1)

        mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)
        box[:, 0] = box[:, 0] - xmin
        box[:, 1] = box[:, 1] - ymin
        mask = rasterize_polygons(mask.shape, [box.astype(np.int32)])
        return self._masked_mean(bitmap[ymin : ymax + 1, xmin : xmax + 1], mask)

    def box_score_slow(self, bitmap: np.ndarray, contour: np.ndarray) -> float:
        """计算原始轮廓覆盖区域的平均检测分数。"""

        h, w = bitmap.shape[:2]
        contour = contour.copy()
        contour = np.reshape(contour, (-1, 2))

        xmin = np.clip(np.min(contour[:, 0]), 0, w - 1)
        xmax = np.clip(np.max(contour[:, 0]), 0, w - 1)
        ymin = np.clip(np.min(contour[:, 1]), 0, h - 1)
        ymax = np.clip(np.max(contour[:, 1]), 0, h - 1)

        mask = np.zeros((ymax - ymin + 1, xmax - xmin + 1), dtype=np.uint8)

        contour[:, 0] = contour[:, 0] - xmin
        contour[:, 1] = contour[:, 1] - ymin

        mask = rasterize_polygons(mask.shape, [contour.astype(np.int32)])
        return self._masked_mean(bitmap[ymin : ymax + 1, xmin : xmax + 1], mask)

    @staticmethod
    def _masked_mean(bitmap: np.ndarray, mask: np.ndarray) -> float:
        """按非零掩码取平均,double 累计保持检测阈值原有数值语义。"""
        pixels = bitmap[mask != 0]
        return float(pixels.sum(dtype=np.float64) / pixels.size) if pixels.size else 0.0

    def __call__(self, outs_dict: dict[str, np.ndarray], shape_list: np.ndarray) -> list[dict[str, Any]]:
        """逐项恢复原图坐标,保留普通文字与印章各自的点序和轮廓。"""

        pred = outs_dict["maps"]
        pred = pred[:, 0, :, :]
        segmentation = pred > self.thresh

        boxes_batch = []
        for batch_index in range(pred.shape[0]):
            src_h, src_w, ratio_h, ratio_w = shape_list[batch_index]
            if self.dilation_kernel is not None:
                mask = morphology(np.array(segmentation[batch_index]).astype(np.uint8), self.dilation_kernel.shape[::-1])
            else:
                mask = segmentation[batch_index]
            if self.box_type == "poly":
                boxes, scores = self.polygons_from_bitmap(pred[batch_index], mask, src_w, src_h)
            elif self.box_type == "quad":
                boxes, scores = self.boxes_from_bitmap(pred[batch_index], mask, src_w, src_h)
            else:
                raise ValueError("box_type can only be one of ['quad', 'poly']")

            boxes_batch.append({"points": boxes})
        return boxes_batch


__all__ = ["DBPostProcess"]