Skip to content

armgit / OXR

Public Imported from GitHub
0

An open-source document parsing model with SOTA accuracy and speed, using an RSI loop during post-training.

15.3 KB
Raw Blame
import re

from oxr.pipeline.models import Block, BlockType, ContentFormat

FORMULA_NUMBERING_ROLE = "formula_numbering"
_TEXT_COMMANDS = {
    "text",
    "textrm",
    "textnormal",
    "textbf",
    "textit",
    "mbox",
    "hbox",
    "verb",
}
_LATEX_COMMAND = re.compile(r"\\(?:[A-Za-z]+|.)", re.DOTALL)
_MARKDOWN_CODE = re.compile(r"(?P<ticks>`+)[\s\S]*?(?P=ticks)|~~~[\s\S]*?~~~")
_HTML_COMMENT = re.compile(r"<!--[\s\S]*?(?:-->|\Z)")
_HTML_TAG = re.compile(
    r"</?[A-Za-z][A-Za-z0-9-]*(?:\s[^>]*)?/?>"
    r"|<![A-Z][^>]*(?:>|\Z)|<\?[^>]*(?:>|\Z)"
)
_HTML_ENTITY_TAG = re.compile(
    r"&(?:lt|#0*60|#x0*3c);/?[A-Za-z][\s\S]*?"
    r"&(?:gt|#0*62|#x0*3e);",
    re.IGNORECASE,
)


def _protected_markup(text: str, index: int) -> re.Match[str] | None:
    return (
        _HTML_COMMENT.match(text, index)
        or _HTML_TAG.match(text, index)
        or _HTML_ENTITY_TAG.match(text, index)
    )


def _valid_math_open(text: str, index: int, width: int) -> bool:
    if width == 2:
        return not text.startswith("$$$", index)
    end = index + 1
    return end < len(text) and not text[end].isspace() and text[end] != "$"


def _valid_math_close(text: str, index: int, width: int) -> bool:
    if width == 2:
        return index > 0 and not text.startswith("$$$", index)
    return index > 0 and not text[index - 1].isspace() and not text.startswith("$$", index)


def space_latex_row_breaks(latex: str) -> str:
    """Separate LaTeX row breaks from an adjacent parenthesis or group brace."""
    result: list[str] = []
    index = 0
    while index < len(latex):
        if latex[index] == "%":
            end = latex.find("\n", index)
            end = len(latex) if end < 0 else end
            result.append(latex[index:end])
            index = end
            continue

        match = _LATEX_COMMAND.match(latex, index)
        if not match:
            result.append(latex[index])
            index += 1
            continue

        command = match.group()
        end = match.end()
        if command[1:] in _TEXT_COMMANDS:
            cursor = end
            while cursor < len(latex) and latex[cursor].isspace():
                cursor += 1
            if cursor < len(latex) and latex[cursor] == "{":
                depth = 1
                cursor += 1
                while cursor < len(latex) and depth:
                    escaped = _LATEX_COMMAND.match(latex, cursor)
                    if escaped:
                        cursor = escaped.end()
                        continue
                    if latex[cursor] == "{":
                        depth += 1
                    elif latex[cursor] == "}":
                        depth -= 1
                    cursor += 1
                if depth:
                    return latex
                end = cursor

        result.append(latex[index:end])
        if command == r"\\" and end < len(latex) and latex[end] in "(){}":
            result.append(" ")
        index = end

    return "".join(result)


def space_markdown_math(markdown: str) -> str:
    """Apply row-break spacing only inside complete dollar-delimited math."""
    result: list[str] = []
    index = 0
    while index < len(markdown):
        code = _MARKDOWN_CODE.match(markdown, index)
        if code:
            result.append(code.group())
            index = code.end()
            continue
        html = _protected_markup(markdown, index)
        if html:
            result.append(html.group())
            index = html.end()
            continue
        if markdown[index] != "$" or _is_escaped(markdown, index):
            result.append(markdown[index])
            index += 1
            continue

        width = 2 if markdown.startswith("$$", index) else 1
        if not _valid_math_open(markdown, index, width):
            result.append(markdown[index : index + width])
            index += width
            continue

        cursor = index + width
        end = None
        while cursor < len(markdown):
            if _MARKDOWN_CODE.match(markdown, cursor) or _protected_markup(markdown, cursor):
                break
            if markdown[cursor] == "$" and not _is_escaped(markdown, cursor):
                found_width = 2 if markdown.startswith("$$", cursor) else 1
                if (
                    found_width == width
                    and _valid_math_close(markdown, cursor, width)
                ):
                    end = cursor
                break
            cursor += 1

        if end is None:
            result.append(markdown[index : index + width])
            index += width
            continue

        delimiter = "$" * width
        result.append(
            delimiter
            + space_latex_row_breaks(markdown[index + width : end])
            + delimiter
        )
        index = end + width

    return "".join(result)


def _parse_braced_group(text: str, open_index: int) -> tuple[str, int] | None:
    if open_index >= len(text) or text[open_index] != "{":
        return None

    depth = 1
    index = open_index + 1
    while index < len(text):
        char = text[index]
        if char == "\\":
            index += 2
            continue
        if char == "{":
            depth += 1
        elif char == "}":
            depth -= 1
            if depth == 0:
                return text[open_index + 1:index], index + 1
        index += 1

    return None


def _has_command_boundary(text: str, index: int) -> bool:
    return index >= len(text) or not text[index].isalpha()


def _numbering_command_at(text: str, index: int) -> str | None:
    if text.startswith(r"\tag*", index) and _has_command_boundary(text, index + len(r"\tag*")):
        return "tag*"
    if text.startswith(r"\tag", index) and _has_command_boundary(text, index + len(r"\tag")):
        return "tag"
    if text.startswith(r"\leqno", index) and _has_command_boundary(text, index + len(r"\leqno")):
        return "leqno"
    if text.startswith(r"\eqno", index) and _has_command_boundary(text, index + len(r"\eqno")):
        return "eqno"
    return None


def _find_next_numbering_command(text: str, start: int) -> int:
    index = start
    while index < len(text):
        if _numbering_command_at(text, index):
            return index
        index += 1
    return len(text)


def _append_numbering(target: list[str], value: str) -> None:
    value = value.strip()
    if value:
        target.append(value)


def _extract_formula_numbering(text: str) -> tuple[str, str, str]:
    formula_parts: list[str] = []
    before_numbers: list[str] = []
    after_numbers: list[str] = []
    index = 0

    while index < len(text):
        command = _numbering_command_at(text, index)
        if command in {"tag", "tag*"}:
            command_end = index + len(r"\tag*") if command == "tag*" else index + len(r"\tag")
            group_start = command_end
            while group_start < len(text) and text[group_start].isspace():
                group_start += 1

            parsed = _parse_braced_group(text, group_start)
            if parsed is None:
                formula_parts.append(text[index])
                index += 1
                continue

            number_text, index = parsed
            number_text = number_text.strip()
            if command == "tag":
                _append_numbering(after_numbers, f"({number_text})")
            else:
                _append_numbering(after_numbers, number_text)
            continue

        if command in {"eqno", "leqno"}:
            command_end = index + len(r"\leqno") if command == "leqno" else index + len(r"\eqno")
            number_start = command_end
            while number_start < len(text) and text[number_start].isspace():
                number_start += 1
            number_end = _find_next_numbering_command(text, number_start)
            number_text = text[number_start:number_end].strip()
            if command == "leqno":
                _append_numbering(before_numbers, number_text)
            else:
                _append_numbering(after_numbers, number_text)
            index = number_end
            continue

        formula_parts.append(text[index])
        index += 1

    return "".join(formula_parts).strip(), " ".join(before_numbers), " ".join(after_numbers)


def _is_escaped(text: str, index: int) -> bool:
    slash_count = 0
    cursor = index - 1
    while cursor >= 0 and text[cursor] == "\\":
        slash_count += 1
        cursor -= 1
    return slash_count % 2 == 1


def clean_formula_markdown_delimiters(latex: str) -> str:
    """Normalize clear wrapper/seam dollars in a Markdown-only formula copy.

    Leave mixed prose, incomplete math, text commands, escaped currency, and
    comments intact. Never modify recognized blocks or infer formula numbers.
    """
    dollars = []
    visible = []
    depth = 0
    index = 0
    while index < len(latex):
        if latex[index] == "%":
            end = latex.find("\n", index)
            index = len(latex) if end < 0 else end
            continue
        command = _LATEX_COMMAND.match(latex, index)
        if command:
            index = command.end()
            if command.group()[1:] in _TEXT_COMMANDS:
                if command.group()[1:] == "verb":
                    return latex
                cursor = index
                while cursor < len(latex) and latex[cursor].isspace():
                    cursor += 1
                group = _parse_braced_group(latex, cursor)
                if group is None:
                    return latex
                index = group[1]
            visible.append(" ")
            continue
        char = latex[index]
        if char == "{":
            depth += 1
        elif char == "}":
            depth -= 1
            if depth < 0:
                return latex
        elif char == "$":
            if depth or latex.startswith("$$", index) or (index and latex[index - 1] == "$"):
                return latex
            dollars.append(index)
        visible.append(char)
        index += 1

    if depth or not dollars:
        return latex
    prose = "".join(visible)
    if "`" in prose or re.search(r"[\u3400-\u9fff]|\b[A-Za-z]{2,}\s+[A-Za-z]{2,}\b", prose):
        return latex
    wrapper = len(dollars) == 2 and dollars == [0, len(latex) - 1]
    seams = len(dollars) % 2 == 0 and all(
        latex[start + 1:end].isspace()
        for start, end in zip(dollars[::2], dollars[1::2])
    )
    if not wrapper and not seams:
        return latex
    if wrapper:
        return latex[1:-1]
    operators = ("=", "+", "-", "*", "/", "<", ">")
    # Only join explicit continuations; preserve independent or unclear rows.
    if not all(
        latex[:start].rstrip().endswith(operators)
        or latex[end + 1:].lstrip().startswith(operators)
        for start, end in zip(dollars[::2], dollars[1::2])
    ):
        return latex
    parts = []
    cursor = 0
    for start, end in zip(dollars[::2], dollars[1::2]):
        parts.extend((latex[cursor:start], latex[start + 1:end]))
        cursor = end + 1
    parts.append(latex[cursor:])
    return "".join(parts)


def _find_unescaped(text: str, marker: str, start: int) -> int:
    index = start
    while True:
        index = text.find(marker, index)
        if index == -1:
            return -1
        if not _is_escaped(text, index):
            return index
        index += len(marker)


def _format_numbered_math(
    open_marker: str,
    formula: str,
    close_marker: str,
    before: str,
    after: str,
) -> str:
    parts = []
    if before:
        parts.append(before)
    parts.append(f"{open_marker}{formula}{close_marker}")
    if after:
        parts.append(after)
    return " ".join(parts)


def _rewrite_numbered_math_range(
    open_marker: str,
    formula: str,
    close_marker: str,
) -> str:
    cleaned_formula, before, after = _extract_formula_numbering(formula)
    if not before and not after:
        return f"{open_marker}{formula}{close_marker}"
    return _format_numbered_math(open_marker, cleaned_formula, close_marker, before, after)


def move_formula_numbering_outside_math(text: str) -> str:
    result = []
    index = 0

    while index < len(text):
        if text.startswith(r"\[", index):
            close_index = text.find(r"\]", index + 2)
            if close_index != -1:
                formula = text[index + 2 : close_index]
                result.append(_rewrite_numbered_math_range(r"\[", formula, r"\]"))
                index = close_index + 2
                continue

        if text.startswith(r"\(", index):
            close_index = text.find(r"\)", index + 2)
            if close_index != -1:
                formula = text[index + 2 : close_index]
                result.append(_rewrite_numbered_math_range(r"\(", formula, r"\)"))
                index = close_index + 2
                continue

        if text.startswith("$$", index) and not _is_escaped(text, index):
            close_index = _find_unescaped(text, "$$", index + 2)
            if close_index != -1:
                formula = text[index + 2 : close_index]
                result.append(_rewrite_numbered_math_range("$$", formula, "$$"))
                index = close_index + 2
                continue

        if text[index] == "$" and not _is_escaped(text, index):
            close_index = _find_unescaped(text, "$", index + 1)
            if close_index != -1:
                formula = text[index + 1 : close_index]
                result.append(_rewrite_numbered_math_range("$", formula, "$"))
                index = close_index + 1
                continue

        result.append(text[index])
        index += 1

    return "".join(result)


def _numbering_text_block(source: Block, content: str) -> Block:
    return Block(
        idx=source.idx,
        type=BlockType.TEXT,
        bbox=list(source.bbox),
        content=content,
        format=ContentFormat.TEXT,
        generated_role=FORMULA_NUMBERING_ROLE,
    )


def expand_formula_numbering_blocks(blocks: list[Block]) -> list[Block]:
    expanded: list[Block] = []
    for block in blocks:
        if block.type != BlockType.FORMULA:
            expanded.append(block)
            continue

        text_before = getattr(block, "text_before", "").strip()
        text_after = getattr(block, "text_after", "").strip()
        if text_before:
            expanded.append(_numbering_text_block(block, text_before))
        expanded.append(block)
        if text_after:
            expanded.append(_numbering_text_block(block, text_after))

    for idx, block in enumerate(expanded):
        block.idx = idx
    return expanded


def postprocess_formula(block: Block, raw_text: str) -> Block:
    """Clean up formula block output to valid LaTeX."""
    text = raw_text.strip()
    match = re.search(r"```latex\n(.*?)\n```", text, re.DOTALL)
    if match:
        text = match.group(1)
    elif "```" in text:
        match = re.search(r"```(?:.*?)\n(.*?)\n```", text, re.DOTALL)
        if match:
            text = match.group(1)

    # Formula blocks store bare KaTeX/LaTeX; assembly owns display-math delimiters.
    wrappers = [
        r"^\$\$(.*?)\$\$$",
        r"^\$(.*?)\$$",
        r"^\\\[(.*?)\\\]$",
        r"^\\\((.*?)\\\)$",
    ]
    for pattern in wrappers:
        match = re.match(pattern, text, re.DOTALL)
        if match:
            text = match.group(1).strip()
            break

    text, text_before, text_after = _extract_formula_numbering(text)
    text = space_latex_row_breaks(text)

    block.content = text
    block.format = ContentFormat.LATEX
    block.text_before = text_before
    block.text_after = text_after
    block.generated_role = ""
    return block