Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

8.9 KB
Raw Blame
from __future__ import annotations

from doclib_test_utils import create_parse_services

from task_test_utils import execution_request
from task_test_utils import stored_source, execution_output

import asyncio
import json
from io import BytesIO
from pathlib import Path
from typing import Any

import pytest
from _odf_test_utils import build_odf_package, build_odp_fixture, build_ods_fixture, build_odt_fixture
from docvortex.analyzers.native import OdpModel, OdsModel, OdtModel
from docvortex.analyzers.native.office.odf.metadata import MAX_ODT_METADATA_PAGE_COUNT, extract_odf_metadata
from docvortex.document.detection import guess_suffix_by_path

from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.doclib.core.db import DatabaseManager
from mineru.doclib.core.file_io import extract_metadata
from mineru.doclib.core.fts import FTSManager
from mineru.errors import InvalidRequestError
from mineru.parser import api_server, parse, parse_async
from mineru.parser.api_server import FileStore


@pytest.mark.parametrize(
    ("suffix", "model_class", "payload", "page_count"),
    [
        ("odt", OdtModel, build_odt_fixture(), 1),
        ("ods", OdsModel, build_ods_fixture(), 2),
        ("odp", OdpModel, build_odp_fixture(), 3),
    ],
    ids=["odt", "ods", "odp"],
)
def test_odf_models_and_analyze_keep_flash_contract(
    suffix: str,
    model_class: type[Any],
    payload: bytes,
    page_count: int,
) -> None:
    """验证三个 ODF 模型、同步/异步入口及输入流所有权。"""
    stream = BytesIO(payload)
    model_pages = model_class().predict(stream)
    assert not stream.closed
    assert len(model_pages) == page_count

    middle, model = doc_analyze(payload, effort="xhigh", parse_mode="ocr", file_suffix=suffix)  # type: ignore[arg-type]
    async_middle, async_model = asyncio.run(
        aio_doc_analyze(payload, effort="medium", parse_mode="auto", file_suffix=suffix)  # type: ignore[arg-type]
    )
    assert model.pages == async_model.pages == model_pages
    assert middle.model_dump() == async_middle.model_dump()
    assert model.metadata.file_suffix == middle.metadata.file_suffix == suffix
    assert model.extensions["mineru"]["tier"] == middle.extensions["mineru"]["tier"] == "flash"
    assert model.extensions["mineru"]["parse_mode"] == middle.extensions["mineru"]["parse_mode"] == "txt"


def test_plain_text_renamed_to_odf_is_not_accepted(tmp_path: Path) -> None:
    """验证 ODF 扩展名本身不能把普通文本升级为结构化文档。"""
    source = tmp_path / "fake.odt"
    source.write_text("a,b\n1,2\n", encoding="utf-8")
    assert guess_suffix_by_path(source) not in {"odt", "ods", "odp"}
    with pytest.raises(InvalidRequestError, match="Unsupported file type") as caught:
        parse(source)
    assert caught.value.code == "file_type_unsupported"
    assert caught.value.param == "input"


@pytest.mark.parametrize(
    ("suffix", "payload", "expected"),
    [
        ("odt", build_odt_fixture(), {"page_count": 3, "title": "ODT Meta", "author": "Alice", "keywords": "one"}),
        ("ods", build_ods_fixture(), {"page_count": 2}),
        ("odp", build_odp_fixture(), {"page_count": 3}),
    ],
    ids=["odt", "ods", "odp"],
)
def test_doclib_extracts_odf_metadata(
    tmp_path: Path,
    suffix: str,
    payload: bytes,
    expected: dict[str, object],
) -> None:
    """验证 doclib ODF 元数据分支不复用 CSV 或 RTF 逻辑。"""
    source = tmp_path / f"sample.{suffix}"
    source.write_bytes(payload)
    metadata = asyncio.run(extract_metadata(str(source)))
    for key, value in expected.items():
        assert metadata[key] == value


def test_odt_metadata_page_count_is_bounded_before_doclib_range_expansion() -> None:
    """验证 producer page-count 不会把微小 ODT 扩张为无界任务范围。"""
    content = """<office:document-content
 xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0">
 <office:body><office:text/></office:body>
</office:document-content>"""
    meta = """<office:document-meta
 xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0"
 xmlns:meta="urn:oasis:names:tc:opendocument:xmlns:meta:1.0">
 <office:meta><meta:document-statistic meta:page-count="999999999999"/></office:meta>
</office:document-meta>"""

    metadata = extract_odf_metadata(BytesIO(build_odf_package("odt", content, meta_xml=meta)), "odt")

    assert MAX_ODT_METADATA_PAGE_COUNT == 10_000
    assert metadata["page_count"] == MAX_ODT_METADATA_PAGE_COUNT


@pytest.mark.parametrize(
    ("suffix", "payload"),
    [("odt", build_odt_fixture()), ("ods", build_ods_fixture()), ("odp", build_odp_fixture())],
    ids=["odt", "ods", "odp"],
)
def test_public_parser_handles_odf_sync_and_async(
    tmp_path: Path,
    suffix: str,
    payload: bytes,
) -> None:
    """验证路径解析器依靠内容识别进入 ODF,并保留原始后缀元数据。"""
    source = tmp_path / f"sample.{suffix}"
    source.write_bytes(payload)
    result = parse(source)
    async_result = asyncio.run(parse_async(source))
    assert result.middle_json.metadata.file_suffix == async_result.middle_json.metadata.file_suffix == suffix
    assert result.middle_json.model_dump() == async_result.middle_json.model_dump()


def test_odf_parse_server_job_emits_flash_outputs(tmp_path: Path) -> None:
    """验证 local parse server 接受 ODF 并输出严格 Middle JSON 与结构化内容。"""
    source = tmp_path / "sample.odt"
    source.write_bytes(build_odt_fixture())
    file_store = FileStore(tmp_path / "api-files")
    request = execution_request(file_store, stored_source(file_store, source), tier="standard", output_formats=["default"])
    record = api_server.ExecutionStore().create(request, file_store)
    asyncio.run(api_server._run_execution(record, file_store, image_analysis=True))
    parsed_file = record
    assert parsed_file.status == "completed"
    middle_data = execution_output(file_store, record, "middle_json.json")
    payload = json.loads(middle_data)
    assert payload["metadata"]["file_suffix"] == "odt"
    assert payload["extensions"]["mineru"]["tier"] == "flash"
    assert payload["extensions"]["mineru"]["parse_mode"] == "txt"


@pytest.mark.parametrize(
    ("suffix", "payload", "page_count"),
    [("odt", build_odt_fixture(), 3), ("ods", build_ods_fixture(), 2), ("odp", build_odp_fixture(), 3)],
    ids=["odt", "ods", "odp"],
)
def test_doclib_ingests_odf_as_local_flash(
    tmp_path: Path,
    suffix: str,
    payload: bytes,
    page_count: int,
) -> None:
    """验证 doclib 为 ODF 建立本地 flash parse row 和正确页数。"""

    class _NoRulesConfig:
        async def match_rules(self, path: str, rule_type: str) -> list[dict[str, object]]:
            """关闭 parsing rules,让测试只观察默认 ODF 行为。"""
            return []

    async def run() -> None:
        """执行隔离 SQLite 入库并检查文档与解析任务。"""
        db = DatabaseManager(str(tmp_path / "doclib.db"))
        await db.initialize()
        service = create_parse_services(
            db=db,
            fts=FTSManager(db),
            config_svc=_NoRulesConfig(),  # type: ignore[arg-type]
            data_dir=str(tmp_path / "data"),
            parse_lock_timeout_sec=1800,
        )
        source = tmp_path / f"sample.{suffix}"
        source.write_bytes(payload)
        response = await service.queue.request_parse(str(source), tier="flash")
        doc = await db.fetchone(
            "SELECT file_type, page_count FROM docs WHERE sha256=?",
            (response.sha256,),
        )
        parses = await db.fetchall(
            "SELECT tier, status, privacy FROM parses WHERE sha256=?",
            (response.sha256,),
        )
        assert response.tier == "flash"
        assert doc == {"file_type": suffix, "page_count": page_count}
        assert parses == [{"tier": "flash", "status": "pending", "privacy": "local"}]

    asyncio.run(run())


def test_doclib_rejects_odf_remote_parse(tmp_path: Path) -> None:
    """验证 ODF 继承非 PDF/image 的严格 remote 拒绝语义。"""

    class _NoRulesConfig:
        async def match_rules(self, path: str, rule_type: str) -> list[dict[str, object]]:
            """关闭 parsing rules,让测试只观察主动请求校验。"""
            return []

    async def run() -> None:
        """创建隔离 doclib 并断言稳定错误码。"""
        db = DatabaseManager(str(tmp_path / "doclib.db"))
        await db.initialize()
        service = create_parse_services(
            db=db,
            fts=FTSManager(db),
            config_svc=_NoRulesConfig(),  # type: ignore[arg-type]
            data_dir=str(tmp_path / "data"),
            parse_lock_timeout_sec=1800,
        )
        source = tmp_path / "sample.odt"
        source.write_bytes(build_odt_fixture())
        with pytest.raises(InvalidRequestError) as exc_info:
            await service.queue.request_parse(str(source), tier="flash", remote=True)
        assert exc_info.value.code == "remote_unsupported_for_file_type"
        assert exc_info.value.param == "remote"

    asyncio.run(run())