Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

7.0 KB
Raw Blame
from __future__ import annotations

from task_test_utils import execution_request
from task_test_utils import stored_source, execution_output

import asyncio
import json
from pathlib import Path

import pytest
from docvortex.document.detection import guess_suffix_by_bytes, guess_suffix_by_path

from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.errors import InvalidRequestError
from mineru.parser import api_server, parse, parse_async
from mineru.parser.api_server import FileStore
from mineru.render.html import render_html
from mineru.render.markdown import render_markdown
from mineru.types import TableBlock, TableBodyBlock


def test_csv_doc_analyze_sync_async_and_render_contracts_match() -> None:
    """验证同步异步 Analyze 元数据和既有表格渲染链保持一致。"""
    payload = "姓名,年龄\n张三,30\n".encode()
    middle, model = doc_analyze(payload, file_suffix="csv")
    async_middle, async_model = asyncio.run(aio_doc_analyze(payload, file_suffix="csv"))

    assert model.metadata.file_suffix == async_model.metadata.file_suffix == "csv"
    assert model.extensions["mineru"]["tier"] == async_model.extensions["mineru"]["tier"] == "flash"
    assert model.extensions["mineru"]["parse_mode"] == async_model.extensions["mineru"]["parse_mode"] == "txt"
    assert middle.model_dump() == async_middle.model_dump()
    assert len(middle.pages) == 1
    table = middle.pages[0].blocks[0]
    assert isinstance(table, TableBlock)
    assert isinstance(table.content[0], TableBodyBlock)
    assert render_markdown(middle) == "| 姓名 | 年龄 |\n| --- | --- |\n| 张三 | 30 |"
    assert "<table>" in render_html(middle)


def test_csv_path_parsing_and_signatureless_detection(tmp_path: Path) -> None:
    """验证 .csv 扩展名兜底、无路径字节不猜 CSV,并阻止 .txt 自动升级。"""
    csv_path = tmp_path / "sample.csv"
    csv_path.write_text("name,age\nAlice,30\n", encoding="utf-8")
    text_path = tmp_path / "sample.txt"
    text_path.write_text("name,age\nAlice,30\n", encoding="utf-8")
    fake_pdf_path = tmp_path / "fake.csv"
    fake_pdf_path.write_bytes(b"%PDF-1.7\n")

    assert guess_suffix_by_bytes(csv_path.read_bytes()) == "txt"
    assert guess_suffix_by_bytes(csv_path.read_bytes(), str(csv_path)) == "csv"
    assert guess_suffix_by_path(csv_path) == "csv"
    assert guess_suffix_by_path(text_path) == "txt"
    assert guess_suffix_by_path(fake_pdf_path) == "pdf"

    result = parse(csv_path)
    async_result = asyncio.run(parse_async(csv_path))
    assert result.middle_json.metadata.file_suffix == async_result.middle_json.metadata.file_suffix == "csv"
    assert result.markdown() == async_result.markdown()
    with pytest.raises(InvalidRequestError, match="Unsupported file type: txt") as caught:
        parse(text_path)
    assert caught.value.code == "file_type_unsupported"
    assert caught.value.param == "input"


def test_tsv_doc_analyze_records_independent_suffix_and_reuses_csv_engine() -> None:
    """验证 tsv 复用 CSV 引擎、自动嗅探 Tab 分隔符,并同步异步一致记录独立后缀。"""
    payload = "姓名\t年龄\n张三\t30\n".encode()
    middle, model = doc_analyze(payload, file_suffix="tsv")
    async_middle, async_model = asyncio.run(aio_doc_analyze(payload, file_suffix="tsv"))

    assert model.metadata.file_suffix == async_model.metadata.file_suffix == "tsv"
    assert model.extensions["mineru"]["tier"] == async_model.extensions["mineru"]["tier"] == "flash"
    assert model.extensions["mineru"]["parse_mode"] == async_model.extensions["mineru"]["parse_mode"] == "txt"
    assert middle.model_dump() == async_middle.model_dump()
    assert len(middle.pages) == 1
    table = middle.pages[0].blocks[0]
    assert isinstance(table, TableBlock)
    assert isinstance(table.content[0], TableBodyBlock)
    assert render_markdown(middle) == "| 姓名 | 年龄 |\n| --- | --- |\n| 张三 | 30 |"
    assert "<table>" in render_html(middle)


def test_tsv_path_parsing_records_independent_suffix(tmp_path: Path) -> None:
    """验证 .tsv 扩展名兜底返回独立后缀,解析产物如实记录 tsv。"""
    tsv_path = tmp_path / "sample.tsv"
    tsv_path.write_text("name\tage\nAlice\t30\n", encoding="utf-8")

    assert guess_suffix_by_path(tsv_path) == "tsv"
    result = parse(tsv_path)
    async_result = asyncio.run(parse_async(tsv_path))
    assert result.middle_json.metadata.file_suffix == async_result.middle_json.metadata.file_suffix == "tsv"
    assert "| name | age |" in result.markdown()


def test_tsv_parse_server_job_emits_structured_outputs_with_independent_suffix(tmp_path: Path) -> None:
    """验证 parse-server 实际解析 .tsv,输出结构化表格并记录独立 file_suffix。"""
    source = tmp_path / "sample.tsv"
    source.write_text("name\tage\nAlice\t30\n", encoding="utf-8")
    file_store = FileStore(tmp_path / "api-files")
    request = execution_request(file_store, stored_source(file_store, source), tier="standard", output_formats=["default"])
    record = api_server.ExecutionStore().create(request, file_store)

    asyncio.run(api_server._run_execution(record, file_store, image_analysis=True))

    parsed_file = record
    assert parsed_file.status == "completed"

    markdown_data = execution_output(file_store, record, "markdown.md")
    markdown = markdown_data.decode()
    assert "| name | age |" in markdown

    middle_data = execution_output(file_store, record, "middle_json.json")
    middle_payload = json.loads(middle_data)
    assert middle_payload["metadata"]["file_suffix"] == "tsv"
    assert middle_payload["extensions"]["mineru"]["tier"] == "flash"
    assert middle_payload["extensions"]["mineru"]["parse_mode"] == "txt"


def test_csv_parse_server_job_emits_structured_outputs_with_flash_metadata(tmp_path: Path) -> None:
    """验证 parse-server 实际解析 CSV,并输出 Markdown、Middle JSON 与结构化内容。"""
    source = tmp_path / "sample.csv"
    source.write_text("name,age\nAlice,30\n", encoding="utf-8")
    file_store = FileStore(tmp_path / "api-files")
    request = execution_request(file_store, stored_source(file_store, source), tier="standard", output_formats=["default"])
    record = api_server.ExecutionStore().create(request, file_store)

    asyncio.run(api_server._run_execution(record, file_store, image_analysis=True))

    parsed_file = record
    assert parsed_file.status == "completed"

    markdown_data = execution_output(file_store, record, "markdown.md")
    markdown = markdown_data.decode()
    assert "| name | age |" in markdown

    middle_data = execution_output(file_store, record, "middle_json.json")
    middle_payload = json.loads(middle_data)
    assert middle_payload["metadata"]["file_suffix"] == "csv"
    assert middle_payload["extensions"]["mineru"]["tier"] == "flash"
    assert middle_payload["extensions"]["mineru"]["parse_mode"] == "txt"

    structured_data = execution_output(file_store, record, "structured_content.json")
    structured_payload = json.loads(structured_data)
    assert structured_payload["metadata"]["file_suffix"] == "csv"
    assert structured_payload["pages"][0]["blocks"][0]["type"] == "table"