Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
from __future__ import annotations
from task_test_utils import execution_request
from task_test_utils import stored_source, execution_output
import asyncio
import json
from pathlib import Path
import pytest
from docvortex.document.detection import guess_suffix_by_bytes, guess_suffix_by_path
from mineru.backend.analyze import aio_doc_analyze, doc_analyze
from mineru.errors import InvalidRequestError
from mineru.parser import api_server, parse, parse_async
from mineru.parser.api_server import FileStore
from mineru.render.html import render_html
from mineru.render.markdown import render_markdown
from mineru.types import TableBlock, TableBodyBlock
def test_csv_doc_analyze_sync_async_and_render_contracts_match() -> None:
"""验证同步异步 Analyze 元数据和既有表格渲染链保持一致。"""
payload = "姓名,年龄\n张三,30\n".encode()
middle, model = doc_analyze(payload, file_suffix="csv")
async_middle, async_model = asyncio.run(aio_doc_analyze(payload, file_suffix="csv"))
assert model.metadata.file_suffix == async_model.metadata.file_suffix == "csv"
assert model.extensions["mineru"]["tier"] == async_model.extensions["mineru"]["tier"] == "flash"
assert model.extensions["mineru"]["parse_mode"] == async_model.extensions["mineru"]["parse_mode"] == "txt"
assert middle.model_dump() == async_middle.model_dump()
assert len(middle.pages) == 1
table = middle.pages[0].blocks[0]
assert isinstance(table, TableBlock)
assert isinstance(table.content[0], TableBodyBlock)
assert render_markdown(middle) == "| 姓名 | 年龄 |\n| --- | --- |\n| 张三 | 30 |"
assert "<table>" in render_html(middle)
def test_csv_path_parsing_and_signatureless_detection(tmp_path: Path) -> None:
"""验证 .csv 扩展名兜底、无路径字节不猜 CSV,并阻止 .txt 自动升级。"""
csv_path = tmp_path / "sample.csv"
csv_path.write_text("name,age\nAlice,30\n", encoding="utf-8")
text_path = tmp_path / "sample.txt"
text_path.write_text("name,age\nAlice,30\n", encoding="utf-8")
fake_pdf_path = tmp_path / "fake.csv"
fake_pdf_path.write_bytes(b"%PDF-1.7\n")
assert guess_suffix_by_bytes(csv_path.read_bytes()) == "txt"
assert guess_suffix_by_bytes(csv_path.read_bytes(), str(csv_path)) == "csv"
assert guess_suffix_by_path(csv_path) == "csv"
assert guess_suffix_by_path(text_path) == "txt"
assert guess_suffix_by_path(fake_pdf_path) == "pdf"
result = parse(csv_path)
async_result = asyncio.run(parse_async(csv_path))
assert result.middle_json.metadata.file_suffix == async_result.middle_json.metadata.file_suffix == "csv"
assert result.markdown() == async_result.markdown()
with pytest.raises(InvalidRequestError, match="Unsupported file type: txt") as caught:
parse(text_path)
assert caught.value.code == "file_type_unsupported"
assert caught.value.param == "input"
def test_tsv_doc_analyze_records_independent_suffix_and_reuses_csv_engine() -> None:
"""验证 tsv 复用 CSV 引擎、自动嗅探 Tab 分隔符,并同步异步一致记录独立后缀。"""
payload = "姓名\t年龄\n张三\t30\n".encode()
middle, model = doc_analyze(payload, file_suffix="tsv")
async_middle, async_model = asyncio.run(aio_doc_analyze(payload, file_suffix="tsv"))
assert model.metadata.file_suffix == async_model.metadata.file_suffix == "tsv"
assert model.extensions["mineru"]["tier"] == async_model.extensions["mineru"]["tier"] == "flash"
assert model.extensions["mineru"]["parse_mode"] == async_model.extensions["mineru"]["parse_mode"] == "txt"
assert middle.model_dump() == async_middle.model_dump()
assert len(middle.pages) == 1
table = middle.pages[0].blocks[0]
assert isinstance(table, TableBlock)
assert isinstance(table.content[0], TableBodyBlock)
assert render_markdown(middle) == "| 姓名 | 年龄 |\n| --- | --- |\n| 张三 | 30 |"
assert "<table>" in render_html(middle)
def test_tsv_path_parsing_records_independent_suffix(tmp_path: Path) -> None:
"""验证 .tsv 扩展名兜底返回独立后缀,解析产物如实记录 tsv。"""
tsv_path = tmp_path / "sample.tsv"
tsv_path.write_text("name\tage\nAlice\t30\n", encoding="utf-8")
assert guess_suffix_by_path(tsv_path) == "tsv"
result = parse(tsv_path)
async_result = asyncio.run(parse_async(tsv_path))
assert result.middle_json.metadata.file_suffix == async_result.middle_json.metadata.file_suffix == "tsv"
assert "| name | age |" in result.markdown()
def test_tsv_parse_server_job_emits_structured_outputs_with_independent_suffix(tmp_path: Path) -> None:
"""验证 parse-server 实际解析 .tsv,输出结构化表格并记录独立 file_suffix。"""
source = tmp_path / "sample.tsv"
source.write_text("name\tage\nAlice\t30\n", encoding="utf-8")
file_store = FileStore(tmp_path / "api-files")
request = execution_request(file_store, stored_source(file_store, source), tier="standard", output_formats=["default"])
record = api_server.ExecutionStore().create(request, file_store)
asyncio.run(api_server._run_execution(record, file_store, image_analysis=True))
parsed_file = record
assert parsed_file.status == "completed"
markdown_data = execution_output(file_store, record, "markdown.md")
markdown = markdown_data.decode()
assert "| name | age |" in markdown
middle_data = execution_output(file_store, record, "middle_json.json")
middle_payload = json.loads(middle_data)
assert middle_payload["metadata"]["file_suffix"] == "tsv"
assert middle_payload["extensions"]["mineru"]["tier"] == "flash"
assert middle_payload["extensions"]["mineru"]["parse_mode"] == "txt"
def test_csv_parse_server_job_emits_structured_outputs_with_flash_metadata(tmp_path: Path) -> None:
"""验证 parse-server 实际解析 CSV,并输出 Markdown、Middle JSON 与结构化内容。"""
source = tmp_path / "sample.csv"
source.write_text("name,age\nAlice,30\n", encoding="utf-8")
file_store = FileStore(tmp_path / "api-files")
request = execution_request(file_store, stored_source(file_store, source), tier="standard", output_formats=["default"])
record = api_server.ExecutionStore().create(request, file_store)
asyncio.run(api_server._run_execution(record, file_store, image_analysis=True))
parsed_file = record
assert parsed_file.status == "completed"
markdown_data = execution_output(file_store, record, "markdown.md")
markdown = markdown_data.decode()
assert "| name | age |" in markdown
middle_data = execution_output(file_store, record, "middle_json.json")
middle_payload = json.loads(middle_data)
assert middle_payload["metadata"]["file_suffix"] == "csv"
assert middle_payload["extensions"]["mineru"]["tier"] == "flash"
assert middle_payload["extensions"]["mineru"]["parse_mode"] == "txt"
structured_data = execution_output(file_store, record, "structured_content.json")
structured_payload = json.loads(structured_data)
assert structured_payload["metadata"]["file_suffix"] == "csv"
assert structured_payload["pages"][0]["blocks"][0]["type"] == "table"