Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

10.5 KB
Raw Blame
"""Layered chapter rebuild from MinerU ``layout.json``.

Layers (all deterministic, no LLM):
  0. column blacklist   — feature-column names are never chapters (closed vocabulary)
  1. regex              — ``第X课/章/节/单元`` headings (works regardless of height)
  2. position filter    — heading must sit in the page-top band (y0 < top)
  3. adjacent merge     — same-page title blocks split by line-wrap rejoin

Input is the MinerU layout dict (``layout["pdf_info"]``), one page object per
page with ``page_idx`` / ``para_blocks``; blocks carry ``type`` / ``bbox`` /
``lines[].spans[].content``.
"""

from __future__ import annotations

from collections.abc import Sequence
from dataclasses import dataclass, field
import re

from .column_blacklist import COLUMN_BLACKLIST

CHAPTER_RE = re.compile(r"^第\s*[一二三四五六七八九十百\d]+\s*[课章节单元]")

# Title blocks whose text is shorter than this are layout noise (figure
# captions etc.) — never chapters even if they regex-match.
MIN_TITLE_CHARS = 4


@dataclass
class Chapter:
    """One rebuilt chapter: title + start page (0-based) + bbox on that page."""

    title: str
    page_idx: int
    bbox: list[float]
    end_page_idx: int | None = None  # filled by assign_page_ranges
    level: int = 1  # reserved: 1=lesson/chapter (课/章), 2=section (框/节)
    meta: dict = field(default_factory=dict)


def block_text(block: dict) -> str:
    return "".join(
        span.get("content", "") for line in block.get("lines", []) for span in line.get("spans", [])
    ).strip()


def merge_adjacent(blocks: list[dict], gap: float = 40.0) -> list[dict]:
    """Rejoin same-page title blocks split by line-wrap.

    Sort by y0; vertical gap < ``gap`` with x-overlap merges into the first
    block (bbox union, text concat). Mutates nothing — returns new dicts.
    """
    merged: list[dict] = []
    for block in sorted(blocks, key=lambda b: b["bbox"][1]):
        if merged:
            prev = merged[-1]
            vgap = block["bbox"][1] - prev["bbox"][3]
            x_overlap = min(block["bbox"][2], prev["bbox"][2]) - max(
                block["bbox"][0], prev["bbox"][0]
            )
            if vgap < gap and x_overlap > 0:
                prev["bbox"] = [
                    min(prev["bbox"][0], block["bbox"][0]),
                    min(prev["bbox"][1], block["bbox"][1]),
                    max(prev["bbox"][2], block["bbox"][2]),
                    max(prev["bbox"][3], block["bbox"][3]),
                ]
                prev["text"] = (prev.get("text", "") + block_text(block)).strip()
                continue
        item = dict(block)
        item["text"] = block_text(block)
        merged.append(item)
    return merged


def layout_page_count(layout: dict) -> int:
    """Exclusive upper page bound, including sparse physical page indices."""
    pages = layout.get("pdf_info", [])
    return max([len(pages), *(page["page_idx"] + 1 for page in pages)])


def rebuild(
    layout: dict,
    *,
    top_band: float = 120.0,
    merge_gap: float = 40.0,
) -> list[Chapter]:
    """Run the layered pipeline over ``layout`` and return chapter starts."""
    chapters: list[Chapter] = []
    for page in layout.get("pdf_info", []):
        title_blocks = [b for b in page.get("para_blocks", []) if b.get("type") == "title"]
        for block in merge_adjacent(title_blocks, gap=merge_gap):
            text = block.get("text", "")
            if text in COLUMN_BLACKLIST:  # layer 0 — closed column names
                continue
            if not CHAPTER_RE.match(text):  # layer 1 — regex first
                continue
            if len(text.replace(" ", "")) < MIN_TITLE_CHARS:
                continue
            if block["bbox"][1] >= top_band:  # layer 2 — page-top band
                continue
            chapters.append(
                Chapter(
                    title=text,
                    page_idx=page["page_idx"],
                    bbox=list(block["bbox"]),
                )
            )
    chapters = _dedupe_keep_last(chapters)  # TOC page duplicates body hits — keep body
    return assign_page_ranges(chapters, page_count=layout_page_count(layout))


def _dedupe_keep_last(chapters: list[Chapter]) -> list[Chapter]:
    """Same title appearing on TOC page AND in the body: keep the body hit."""
    by_title: dict[str, Chapter] = {}
    order: list[str] = []
    for chapter in chapters:
        if chapter.title in by_title:
            by_title[chapter.title] = chapter  # later (higher page) wins
        else:
            by_title[chapter.title] = chapter
            order.append(chapter.title)
    return [by_title[t] for t in order]


def assign_page_ranges(chapters: list[Chapter], *, page_count: int) -> list[Chapter]:
    """Return half-open page ranges, including the final physical page."""
    chapters.sort(key=lambda chapter: chapter.page_idx)
    for i, chapter in enumerate(chapters):
        chapter.end_page_idx = (
            max(chapters[i + 1].page_idx, chapter.page_idx + 1)
            if i + 1 < len(chapters)
            else max(page_count, chapter.page_idx + 1)
        )
    return chapters


# ── Mid-page section-frame detection (no page-top band constraint) ─────────
# Measured heading heights: frame ~22-24, exploration ~27, body lines ~21,
# lesson-title wrap residue ~28. Column names go to the blacklist; stray
# publisher header lines go to PUBLISHER_NOISE.

PUBLISHER_NOISE = frozenset({"人民教育出版社", "出版社", "思想政治", "目录", "后记"})


def detect_frames(
    layout: dict,
    *,
    height_range: tuple[float, float] = (20.0, 25.0),
    extras_range: tuple[float, float] = (26.0, 29.0),
    first_lesson_page_idx: int | None = None,
    lesson_titles: Sequence[str] = (),
) -> tuple[list[dict], list[dict]]:
    """Detect section-frame headings (and chapter-level extras like 综合探究).

    Returns ``(frames, extras)`` — each item ``{title, page_idx, bbox, height}``.
    Frames are mid-page title blocks in the frame height band; extras
    (exploration sections / afterword) sit in the slightly taller band.
    Blacklist + publisher noise + lesson-level regex filtered out.
    """
    frames: list[dict] = []
    extras: list[dict] = []
    for page in layout.get("pdf_info", []):
        for block in page.get("para_blocks", []):
            if block.get("type") != "title":
                continue
            text = block_text(block)
            if not text or text in COLUMN_BLACKLIST or text in PUBLISHER_NOISE:
                continue
            if CHAPTER_RE.match(text):
                continue  # lesson-level handled by the footer path
            height = block["bbox"][3] - block["bbox"][1]
            item = {
                "title": text,
                "page_idx": page["page_idx"],
                "bbox": list(block["bbox"]),
                "height": round(height, 1),
            }
            if height_range[0] <= height <= height_range[1]:
                if len(text) >= 6:
                    # front-matter noise + lesson-title wrap residue
                    # (substring of a lesson title)
                    if (
                        first_lesson_page_idx is not None
                        and item["page_idx"] < first_lesson_page_idx
                    ):
                        continue
                    if any(text in lt for lt in lesson_titles):
                        continue
                    frames.append(item)
            elif extras_range[0] <= height <= extras_range[1] and len(text) >= 3:
                extras.append(item)
    return frames, extras


# ── Level-aware boundaries + printed-offset consistency check ──────────────

UNIT_RE_MAP = {
    "课": re.compile(r"^第\s*[一二三四五六七八九十百\d]+\s*课"),
    "章": re.compile(r"^第\s*[一二三四五六七八九十百\d]+\s*章"),
    "节": re.compile(r"^第\s*[一二三四五六七八九十百\d]+\s*节"),
    "单元": re.compile(r"^第\s*[一二三四五六七八九十百\d]+\s*单元"),
}

PARENT_UNITS = {
    "课": ("单元", "章"),
    "章": ("单元",),
    "节": ("单元", "章"),
    "单元": (),
}


def rebuild_from_headers_level(layout: dict, unit: str = "课") -> list[Chapter]:
    """Level-aware rebuild: only ``unit``-level running-header changes mark boundaries.

    In textbooks with two-level running headers (e.g. rotating
    "第X章/第Y节" pairs), a change at the other level must not start a
    chapter — otherwise pseudo-chapters get split out.
    """
    # Imported here to avoid a circular import (page_headers imports this
    # module's CHAPTER_RE / Chapter / assign_page_ranges at top level).
    from .page_headers import page_facts

    unit_re = UNIT_RE_MAP.get(unit)
    if unit_re is None:
        raise ValueError(f"unknown unit: {unit}")
    chapters: list[Chapter] = []
    parent_titles: dict[str, str] = {}
    last_boundary: tuple[str, ...] | None = None
    page_count = layout_page_count(layout)
    for page in sorted(layout.get("pdf_info", []), key=lambda item: item["page_idx"]):
        footers, printed = page_facts(page)
        for title in footers:
            for parent_unit in PARENT_UNITS[unit]:
                if UNIT_RE_MAP[parent_unit].match(title):
                    parent_titles[parent_unit] = title
                    if parent_unit == "单元":
                        parent_titles.pop("章", None)
        for title in footers:
            if not unit_re.match(title):
                continue
            boundary = (
                *(parent_titles.get(parent_unit, "") for parent_unit in PARENT_UNITS[unit]),
                title,
            )
            if boundary == last_boundary:
                continue
            last_boundary = boundary
            chapters.append(
                Chapter(
                    title=title,
                    page_idx=page["page_idx"],
                    bbox=[],
                    meta={"printed_page": int(printed) if printed.isdigit() else None},
                )
            )
    return assign_page_ranges(chapters, page_count=page_count)


def verify_offset(chapters: list[Chapter]) -> dict:
    """Printed-offset consistency: physical (1-based) − printed page number.

    An offset that varies across chapters means a chapter boundary was
    mis-detected.
    """
    offsets = sorted(
        {c.page_idx + 1 - c.meta["printed_page"] for c in chapters if c.meta.get("printed_page")}
    )
    return {
        "offsets": offsets,
        "consistent": len(offsets) <= 1,
        "ok": bool(offsets) and len(offsets) == 1,
    }