Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

17.4 KB
Raw Blame
"""mineru parse — document parsing command."""

from __future__ import annotations

import shlex
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Literal, TypeAlias

import typer

from ...doclib.client import DoclibClient
from ...doclib.types import (
    ContentNextRequest,
    DocContentExportRequest,
    DocContentResponse,
    ParseRequest,
    ParseResponse,
    TelemetryObservation,
    TelemetryObservationsRequest,
)
from ...errors import MineruError, error_response
from ...filetypes import is_page_range_parse_extension
from ...parser.page_range import format_page_range, normalize_page_range_input, parse_page_range_set
from ...types import Tier
from ...utils.i18n import t
from ..contracts import CliContext, CliResult
from ..guidance import api_key_guidance_for_error
from ..path_utils import normalize_cli_path
from ..runtime import cli_ok, emit_result, run_cli


@dataclass(frozen=True)
class ParseTextOutput:
    text: str


@dataclass(frozen=True)
class ParseSummaryOutput:
    text: str


ParseCommandResult: TypeAlias = (
    CliResult[dict[str, Any]] | CliResult[ParseTextOutput] | CliResult[ParseSummaryOutput] | CliResult[None]
)


def _normalize_output_path(output: str | None) -> str | None:
    if output is None or output == "-":
        return output
    return normalize_cli_path(output)


def _ensure_output_parent(output: str | None) -> str | None:
    normalized = _normalize_output_path(output)
    if normalized in (None, "-"):
        return normalized
    Path(normalized).parent.mkdir(parents=True, exist_ok=True)
    return normalized


def _output_info(path: str) -> dict[str, str]:
    return {"status": "written", "path": path}


def _emit_notice(message: str, *, json_mode: bool) -> None:
    emit_result(CliContext(json_mode=json_mode), cli_ok(notices=[message]))


def _validate_page_range_input(page_range: str | None) -> None:
    """复用共享语法校验,并把错误参数定位到 CLI 的 pages 选项。"""
    try:
        normalize_page_range_input(page_range)
    except MineruError as exc:
        raise MineruError(exc.code, str(exc), "pages") from None


def parse_cmd(
    path: str = typer.Argument(..., help=t("Path to the document file")),
    tier: Tier | None = typer.Option(
        None,
        "--tier",
        help=t("Parse tier: flash, basic, standard, advanced (default: server decides)"),
    ),
    pages: str = typer.Option(
        None,
        "-p",
        "--pages",
        help=t("PDF pages: '1-5,8', 'r3-r1' (counted from the end), or 'all'; default: first 10 pages"),
    ),
    after: str = typer.Option(None, "--after", help=t("Continue reading after a content cursor from earlier output")),
    limit: int | None = typer.Option(None, "--limit", help=t("Soft character limit for printed content")),
    format: Literal["markdown"] = typer.Option("markdown", "-f", "--format", help=t("Output format: markdown")),
    force: bool = typer.Option(False, "--force", help=t("Force re-parse, ignore cache")),
    remote: bool = typer.Option(False, "--remote", help=t("Use the mineru.net official remote parse service")),
    wait: int = typer.Option(60, "--wait", help=t("Max seconds to wait for parse to complete")),
    no_wait: bool = typer.Option(False, "--no-wait", help=t("Don't wait — return immediately")),
    output: str = typer.Option(None, "-o", "--output", help=t("Output path; creates parent directories")),
    no_marker: bool = typer.Option(False, "--no-marker", help=t("Omit document structure markers from output")),
    json_mode: bool = typer.Option(False, "--json", help=t("JSON output")),
    verbose: bool = typer.Option(False, "-v", "--verbose", help=t("Verbose output")),
) -> None:
    """Parse a document file."""
    ctx = CliContext(json_mode=json_mode, verbose=verbose)
    effective_limit = 30000 if limit is None else limit
    run_cli(
        ctx,
        lambda: _parse(
            path,
            tier=tier,
            pages=pages,
            after=after,
            limit=effective_limit,
            format=format,
            force=force,
            remote=remote,
            wait=wait,
            no_wait=no_wait,
            output=output,
            no_marker=no_marker,
            json_mode=json_mode,
            verbose=verbose,
            explicit_tier=tier is not None,
            explicit_limit=limit is not None,
            explicit_remote=remote,
        ),
        error_guidance=api_key_guidance_for_error if remote else None,
    )


def _parse(
    path: str,
    *,
    tier: Tier | None,
    pages: str | None,
    after: str | None,
    limit: int,
    format: Literal["markdown"],
    force: bool,
    remote: bool,
    wait: int,
    no_wait: bool,
    output: str | None,
    no_marker: bool,
    json_mode: bool,
    verbose: bool,
    explicit_tier: bool,
    explicit_limit: bool,
    explicit_remote: bool,
) -> ParseCommandResult:
    file_path = normalize_cli_path(path)

    if not Path(file_path).exists():
        raise MineruError("file_not_found", t("File not found: {path}", path=file_path), "path")

    _validate_page_range_input(pages)
    pages = normalize_page_range_input(pages) or None

    client = DoclibClient(timeout=wait + 30)

    # send parse request
    result = client.ensure_parse(
        ParseRequest(
            path=file_path,
            tier=tier,
            page_range=pages,
            force=force,
            remote=remote,
        )
    )

    req_tier = result.tier
    status = result.status
    wait_parse_ids = result.wait_parse_ids
    next_marker_tier = tier if explicit_tier else None
    next_marker_limit = limit if explicit_limit else None
    next_marker_remote = remote if explicit_remote else False

    # cached
    if status == "done":
        if verbose:
            _emit_notice(t("Cache hit — returning cached result."), json_mode=json_mode)
        return _prepare_parse_result(
            client,
            result,
            json_mode=json_mode,
            output=output,
            after=after,
            limit=limit,
            format=format,
            no_marker=no_marker,
            source_path=file_path,
            next_marker_tier=next_marker_tier,
            next_marker_limit=next_marker_limit,
            next_marker_remote=next_marker_remote,
        )

    # no-wait
    if no_wait or not wait_parse_ids:
        if json_mode:
            return cli_ok(_parse_json_payload(result, None))
        return _prepare_parse_summary_output(result, exit_code=0)

    # poll until done or timeout
    if verbose:
        _emit_notice(t("Parse queued (tier={tier}). Waiting up to {wait}s...", tier=req_tier, wait=wait), json_mode=json_mode)

    wait_started_at = time.time()
    deadline = time.time() + wait
    interval = 0.5
    latest_wait_status = status
    while time.time() < deadline:
        time.sleep(interval)
        interval = min(interval * 1.2, 3.0)
        try:
            s = client.list_parses(ids=wait_parse_ids)
        except Exception:
            continue

        wait_id_set = set(wait_parse_ids)
        parse_rows = [row for row in s.parses if row.id in wait_id_set]
        statuses = {row.status for row in parse_rows}
        if parse_rows and statuses == {"done"}:
            st = "done"
        elif "failed" in statuses:
            st = "failed"
        elif "parsing" in statuses:
            st = "parsing"
        else:
            st = "pending"
        if st in ("pending", "parsing"):
            latest_wait_status = st
        if verbose and not json_mode:
            _emit_notice(t("  Parse status: {status}", status=st), json_mode=json_mode)

        if st == "done":
            _record_parse_wait(client, wait_parse_ids, "succeeded", wait_started_at)
            done_result = result.model_copy(update={"status": "done"})
            if not is_page_range_parse_extension(file_path):
                # 非 PDF 的初始页数只是估计,完成后以整本任务的实际逻辑页范围为准。
                done_result.page_range = format_page_range(
                    page_no for row in parse_rows for page_no in parse_page_range_set(row.page_range)
                )
            return _prepare_parse_result(
                client,
                done_result,
                json_mode=json_mode,
                output=output,
                after=after,
                limit=limit,
                format=format,
                no_marker=no_marker,
                source_path=file_path,
                next_marker_tier=next_marker_tier,
                next_marker_limit=next_marker_limit,
                next_marker_remote=next_marker_remote,
            )
        if st == "failed":
            failed = next((row for row in parse_rows if row.status == "failed"), None)
            _record_parse_wait(client, wait_parse_ids, "failed", wait_started_at)
            error_code = (failed.error_code if failed else None) or "parse_failed"
            error_msg = (failed.error_msg if failed else None) or t("Parse failed.")
            error_param = "parse_server.remote.api_key" if error_code == "invalid_api_key" else None
            raise MineruError(error_code, error_msg, error_param)

    _record_parse_wait(client, wait_parse_ids, "timeout", wait_started_at)
    timeout_error = MineruError("parse_wait_timeout", t("Parse did not finish within {seconds} seconds.", seconds=wait), "wait")
    if json_mode:
        return cli_ok(
            _parse_json_payload(
                result,
                None,
                error=timeout_error,
                parse_overrides={"status": latest_wait_status, "tip": t("Re-run the same command to continue waiting.")},
            ),
            exit_code=1,
        )
    return _prepare_parse_summary_output(
        result.model_copy(update={"status": latest_wait_status}),
        waited_seconds=wait,
        exit_code=1,
    )


def _record_parse_wait(client: DoclibClient, parse_ids: list[int], status: str, started_at: float) -> None:
    duration_ms = max(0, int((time.time() - started_at) * 1000))
    try:
        client.record_observations(
            TelemetryObservationsRequest(
                observations=[
                    TelemetryObservation(
                        metric_name="parse.wait",
                        parse_ids=parse_ids,
                        duration_ms=duration_ms,
                        dimensions={"status": status},
                    )
                ]
            )
        )
    except Exception:
        pass


def _prepare_parse_result(
    client: DoclibClient,
    parse_result: ParseResponse,
    json_mode: bool,
    output: str | None = None,
    after: str | None = None,
    limit: int = 30000,
    format: str = "markdown",
    no_marker: bool = False,
    source_path: str | None = None,
    next_marker_tier: Tier | None = None,
    next_marker_limit: int | None = None,
    next_marker_remote: bool = False,
) -> ParseCommandResult:
    """Fetch and prepare parsed content for CLI output."""
    sha256 = parse_result.sha256
    tier = parse_result.tier
    # 非 PDF 始终读取整本,避免把调度时的预估页数当成输出过滤条件。
    page_range = parse_result.page_range if source_path is None or is_page_range_parse_extension(source_path) else None
    output_path = _ensure_output_parent(output)
    if output_path and output_path != "-":
        exported = client.export_doc_content(
            sha256,
            DocContentExportRequest(
                tier=tier,
                page_range=page_range,
                format=format,
                output=output_path,
                no_marker=no_marker,
            ),
        )
        if json_mode:
            return cli_ok(_parse_json_payload(parse_result, None, output=exported.output))
        return cli_ok(ParseTextOutput(t("Written to {path}", path=exported.output)), render=_render_parse_text)

    content = _fetch_doc_content(
        client,
        sha256,
        tier=tier,
        page_range=page_range,
        after=after,
        limit=limit,
        format=format,
        no_marker=no_marker,
    )
    if not content.content and content.format == "markdown" and not content.content_ranges:
        raise MineruError("parse_empty", t("No content returned from parse."))
    if json_mode:
        return cli_ok(_parse_json_payload(parse_result, content))
    return _prepare_parse_content_output(
        content,
        source_path=source_path,
        no_marker=no_marker,
        next_marker_tier=next_marker_tier,
        next_marker_limit=next_marker_limit,
        next_marker_remote=next_marker_remote,
    )


def _fetch_doc_content(
    client: DoclibClient,
    sha256: str,
    *,
    tier: Tier,
    page_range: str | None,
    after: str | None,
    limit: int,
    format: str,
    no_marker: bool,
) -> DocContentResponse:
    return client.get_doc_content(
        sha256,
        tier=tier,
        page_range=page_range,
        after=after,
        limit=limit,
        format=format,
        no_marker=no_marker,
    )


def _parse_json_payload(
    parse_result: ParseResponse,
    content: DocContentResponse | None,
    *,
    output: str | None = None,
    error: MineruError | None = None,
    parse_overrides: dict[str, str | None] | None = None,
) -> dict[str, Any]:
    parse_payload = parse_result.model_dump(mode="json")
    if parse_overrides:
        parse_payload.update(parse_overrides)
    payload: dict[str, Any] = {
        "parse": parse_payload,
        "content": content.model_dump(mode="json") if content is not None else None,
    }
    if output is not None:
        payload["output"] = _output_info(output)
    if error is not None:
        payload["error"] = error_response(error)["error"]
    return payload


def _prepare_parse_summary_output(
    parse_result: ParseResponse,
    *,
    waited_seconds: int | None = None,
    exit_code: int = 0,
) -> CliResult[ParseSummaryOutput] | CliResult[None]:
    status = parse_result.status
    tip = parse_result.tip or ""
    if status in ("pending", "parsing"):
        return cli_ok(
            ParseSummaryOutput(_format_parse_in_progress(parse_result, waited_seconds=waited_seconds, tip=tip)),
            render=_render_parse_summary,
            exit_code=exit_code,
        )
    return cli_ok(
        ParseSummaryOutput(t("Parse complete (tier={tier}) {tip}", tier=parse_result.tier, tip=tip)),
        render=_render_parse_summary,
    )


def _format_parse_in_progress(parse_result: ParseResponse, *, waited_seconds: int | None, tip: str = "") -> str:
    elapsed = f" {t('after {seconds}s', seconds=waited_seconds)}" if waited_seconds is not None else ""
    lines = [t("Parse still in progress{elapsed} (tier={tier}).", elapsed=elapsed, tier=parse_result.tier)]
    parse_ids = parse_result.wait_parse_ids
    if parse_ids:
        label = t("Parse ID") if len(parse_ids) == 1 else t("Parse IDs")
        lines.append(t("{label}: {ids}", label=label, ids=", ".join(str(parse_id) for parse_id in parse_ids)))
        if len(parse_ids) == 1:
            lines.append(f"Check status: mineru show parse {parse_ids[0]}")
        else:
            lines.append("Check status:")
            lines.extend(f"  mineru show parse {parse_id}" for parse_id in parse_ids)
    if tip:
        lines.append(tip)
    return "\n".join(lines)


def _render_parse_summary(data: ParseSummaryOutput) -> str:
    return data.text


def _prepare_parse_content_output(
    content: DocContentResponse,
    *,
    source_path: str | None,
    no_marker: bool,
    next_marker_tier: Tier | None = None,
    next_marker_limit: int | None = None,
    next_marker_remote: bool = False,
) -> CliResult[ParseTextOutput] | CliResult[None]:
    if not content.content and content.content_ranges:
        message = t("No renderable content in requested pages.")
        if content.next_request and not no_marker:
            marker = _parse_next_marker(
                source_path,
                content.next_request,
                tier=next_marker_tier,
                limit=next_marker_limit,
                remote=next_marker_remote,
            )
            if marker:
                return cli_ok(ParseTextOutput(_append_next_marker(message, marker)), render=_render_parse_text)
        return cli_ok(None, notices=[message])
    if content.next_request and not no_marker:
        marker = _parse_next_marker(
            source_path,
            content.next_request,
            tier=next_marker_tier,
            limit=next_marker_limit,
            remote=next_marker_remote,
        )
        if marker:
            return cli_ok(ParseTextOutput(_append_next_marker(content.content, marker)), render=_render_parse_text)
    return cli_ok(ParseTextOutput(content.content), render=_render_parse_text)


def _render_parse_text(data: ParseTextOutput) -> str:
    return data.text


def _append_next_marker(content: str, marker: str) -> str:
    newline_count = len(content) - len(content.rstrip("\n"))
    separator = "\n" * max(2 - newline_count, 0)
    return f"{content}{separator}{marker}"


def _parse_next_marker(
    path: str | None,
    next_request: ContentNextRequest,
    *,
    tier: Tier | None = None,
    limit: int | None = None,
    remote: bool = False,
) -> str | None:
    if not path:
        return None
    parts = ["mineru", "parse", path]
    if tier:
        parts.extend(["--tier", tier])
    if remote:
        parts.append("--remote")
    if limit is not None:
        parts.extend(["--limit", str(limit)])
    if next_request.page_range:
        parts.extend(["--pages", next_request.page_range])
    if next_request.after:
        parts.extend(["--after", next_request.after])
    return f"<!-- Next: {shlex.join(parts)} -->"