Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
refactor: extract shared legacy middle JSON reader to backend postprocess
jinzhenj6f0a27c09111
diff --git a/mineru/doclib/core/middle_json.py b/mineru/backend/postprocess/legacy_middle_json.pysimilarity index 87%rename from mineru/doclib/core/middle_json.pyrename to mineru/backend/postprocess/legacy_middle_json.pyindex c0fdc723..5074f42f 100644--- a/mineru/doclib/core/middle_json.py+++ b/mineru/backend/postprocess/legacy_middle_json.py@@ -1,7 +1,7 @@-"""仅供 Doclib 持久化结果使用的 Middle JSON 读取与历史转换边界。+# Copyright (c) Opendatalab. All rights reserved.+"""历史/旧协议 Middle JSON 读取与转换边界。-两个历史分支集中在此,未来可一并移除;通用 ParseResult 与 DocVortex codec-不调用本模块。读取只构造内存对象,不重写文件、不重新推理。+两个历史分支集中在此,未来可一并移除。读取只构造内存对象,不重写文件、不重新推理。"""from __future__ import annotations@@ -64,7 +64,7 @@ def _legacy_extensions(payload: dict[str, Any]) -> dict[str, Any]:def _read_legacy_345(payload: dict[str, Any], *, page_field: str) -> MiddleJson:"""把可识别的 3.4.5 页面转为当前 raw ModelJson,再做确定性后处理。"""- from ...backend.postprocess.legacy_schema_adapter import legacy_page_to_model_list+ from .legacy_schema_adapter import legacy_page_to_model_listfrom docvortex.postprocess.document import model_json_to_middle_jsonpages = payload[page_field]@@ -102,15 +102,15 @@ def _read_legacy_v2(payload: dict[str, Any]) -> MiddleJson:)-def read_cached_middle_json(payload: dict[str, Any]) -> MiddleJson:- """先识别显式协议,再转换两个历史缓存族;失败时绝不跨协议兜底。"""+def read_legacy_middle_json(payload: dict[str, Any]) -> MiddleJson:+ """先识别显式协议,再转换两个历史分支;失败时绝不跨协议兜底。"""if not isinstance(payload, dict):- raise ValueError("Cached Middle JSON must be an object; source reparse required")+ raise ValueError("Middle JSON must be an object; source reparse required")if "schema" in payload:document = MiddleJson.from_dict(payload)else:if {"metadata", "producer", "page_index_map"}.intersection(payload):- raise ValueError("Mixed cached document envelope; source reparse required")+ raise ValueError("Mixed document envelope; source reparse required")version = payload.get("schema_version")source = deepcopy(payload)if "pdf_info" in source:@@ -122,9 +122,9 @@ def read_cached_middle_json(payload: dict[str, Any]) -> MiddleJson:elif version == "2.0" and "pages" in source:document = _read_legacy_v2(source)else:- raise ValueError("Unsupported cached Middle JSON format; source reparse required")+ raise ValueError("Unsupported Middle JSON format; source reparse required")validate_mineru_metadata(document)return document-__all__ = ["read_cached_middle_json"]+__all__ = ["read_legacy_middle_json"]\ No newline at end of filediff --git a/mineru/doclib/background/compaction.py b/mineru/doclib/background/compaction.pyindex aa079163..cf6c0c0d 100644--- a/mineru/doclib/background/compaction.py+++ b/mineru/doclib/background/compaction.py@@ -14,7 +14,7 @@ from typing import Any, castfrom ...parser.page_range import format_page_rangefrom ...types import Tierfrom ..core.db import DatabaseManager-from ..core.middle_json import read_cached_middle_json+from ...backend.postprocess.legacy_middle_json import read_legacy_middle_jsonfrom ..rows import ParseBatchRow, ParseGroupRow, ParseRowfrom ..services.parse_svc import parse_batch_json_path, parse_page_range_setfrom ..types import PARSE_STATUS_DONE, PARSE_STATUS_SUPERSEDED@@ -24,7 +24,7 @@ logger = logging.getLogger("mineru.compaction")def _normalize_batch_pages(batch_payload: dict[str, Any]) -> list[dict[str, Any]]:"""读取当前和受支持的历史批次,仅在内存中统一页面结构。"""- return [page.to_dict() for page in read_cached_middle_json(batch_payload).pages]+ return [page.to_dict() for page in read_legacy_middle_json(batch_payload).pages]class Compaction:@@ -150,7 +150,7 @@ class Compaction:try:with open(fpath, encoding="utf-8") as f:batch_payload = json.load(f)- middle_json = read_cached_middle_json(batch_payload)+ middle_json = read_legacy_middle_json(batch_payload)batch_pages = [page.to_dict() for page in middle_json.pages]except Exception:return Nonediff --git a/mineru/doclib/services/parse_svc.py b/mineru/doclib/services/parse_svc.pyindex ef21791c..05ff4684 100644--- a/mineru/doclib/services/parse_svc.py+++ b/mineru/doclib/services/parse_svc.py@@ -28,7 +28,7 @@ from ...filetypes import ()from ...parser.api_client import _APITransportError, _V1APIErrorfrom ...parser.base import ParseResult-from ..core.middle_json import read_cached_middle_json+from ...backend.postprocess.legacy_middle_json import read_legacy_middle_jsonfrom ...parser.page_range import (expand_page_range,format_page_range as _page_numbers_to_range_str,@@ -1538,7 +1538,7 @@ def _json_batch_is_readable(data_dir: str, sha256: str, tier: Tier, batch: Parsejson_path = parse_batch_json_path(data_dir, sha256, tier, batch["page_range"], batch["done_at"])try:with open(json_path, encoding="utf-8") as stream:- result = read_cached_middle_json(json.load(stream))+ result = read_legacy_middle_json(json.load(stream))available = {page.page_idx + 1 for page in result.pages}return bool(available) and parse_page_range_set(batch["page_range"]) <= availableexcept (OSError, ValueError, TypeError):@@ -1569,7 +1569,7 @@ def load_pages_from_done_batches(try:with open(fpath, encoding="utf-8") as f:data = json.load(f)- parse_result = read_cached_middle_json(data)+ parse_result = read_legacy_middle_json(data)for page in parse_result.pages:pages_by_page_idx[page.page_idx] = pageexcept (OSError, ValueError, TypeError):diff --git a/tests/unittest/test_doclib_middle_json.py b/tests/unittest/test_doclib_middle_json.pyindex 367d9598..b098d8a4 100644--- a/tests/unittest/test_doclib_middle_json.py+++ b/tests/unittest/test_doclib_middle_json.py@@ -7,7 +7,7 @@ import jsonimport pytestfrom docvortex.schema import MiddleJson-from mineru.doclib.core.middle_json import read_cached_middle_json+from mineru.backend.postprocess.legacy_middle_json import read_legacy_middle_jsonfrom mineru.parser import ParseResult@@ -98,7 +98,7 @@ def test_legacy_345_roundtrip_preserves_origin_and_pages(wrapped: bool) -> None:"""两个旧页面封装均可在缓存边界转换,输入和来源不受污染。"""payload = _legacy_345(wrapped=wrapped)before = deepcopy(payload)- result = read_cached_middle_json(payload)+ result = read_legacy_middle_json(payload)assert payload == beforeassert [page.page_idx for page in result.pages] == [3, 5]assert result.is_full_document is False@@ -106,7 +106,7 @@ def test_legacy_345_roundtrip_preserves_origin_and_pages(wrapped: bool) -> None:assert result.extensions["mineru"] == {"tier": "standard", "parse_mode": "ocr"}assert result.pages[0].blocks[0].content[0].content == "正文 "assert result.pages[0].blocks[0].content[1].content == "x^2"- assert read_cached_middle_json(result.to_dict()) == result+ assert read_legacy_middle_json(result.to_dict()) == resultassert MiddleJson.from_json(result.to_json()) == result@@ -124,14 +124,14 @@ def test_v2_changes_only_envelope_without_postprocess(monkeypatch: pytest.Monkeymonkeypatch.setattr(host_document, "build_middle_json", unexpected)payload = _legacy_v2()before = deepcopy(payload)- result = read_cached_middle_json(payload)+ result = read_legacy_middle_json(payload)assert payload == beforeassert result.to_dict()["pages"] == payload["pages"]assert result.metadata.file_suffix == "pdf"assert result.metadata.producer.version == "3.4.4"assert result.extensions["application"] == payload["extensions"]["application"]assert result.extensions["mineru"] == {"tier": "standard", "parse_mode": "ocr"}- assert read_cached_middle_json(result.to_dict()).to_dict() == result.to_dict()+ assert read_legacy_middle_json(result.to_dict()).to_dict() == result.to_dict()result.extensions["application"]["items"].append("changed")assert payload == before@@ -150,7 +150,7 @@ def test_v2_changes_only_envelope_without_postprocess(monkeypatch: pytest.Monkeydef test_missing_metadata_stays_unknown(updates: dict, expected: dict | None) -> None:"""缺失或不可识别的记录保持未知,不制造当前版本或默认解析档位。"""payload = {"pdf_info": [], **updates}- result = read_cached_middle_json(payload)+ result = read_legacy_middle_json(payload)assert result.metadata.producer.model_dump() == {"name": "mineru", "version": "unknown"}assert result.metadata.file_suffix == "pdf"assert result.extensions.get("mineru") == expected@@ -161,7 +161,7 @@ def test_v2_missing_product_records_preserve_application_extensions() -> None:payload = _legacy_v2()for key in ("mineru_version", "effort", "parse_mode"):del payload[key]- document = read_cached_middle_json(payload)+ document = read_legacy_middle_json(payload)assert document.metadata.producer.version == "unknown"assert document.extensions == payload["extensions"]@@ -170,7 +170,7 @@ def test_consistent_existing_product_extension_is_preserved() -> None:"""已有可靠产品扩展与旧字段一致时直接保留,不产生重复记录。"""payload = _legacy_v2()payload["extensions"]["mineru"] = {"tier": "standard", "parse_mode": "ocr"}- assert read_cached_middle_json(payload).extensions == payload["extensions"]+ assert read_legacy_middle_json(payload).extensions == payload["extensions"]def test_345_conflicting_full_document_flag_fails() -> None:@@ -178,7 +178,7 @@ def test_345_conflicting_full_document_flag_fails() -> None:payload = _legacy_345()payload["is_full_document"] = Truewith pytest.raises(ValueError, match="Conflicting legacy is_full_document"):- read_cached_middle_json(payload)+ read_legacy_middle_json(payload)@pytest.mark.parametrize(@@ -194,7 +194,7 @@ def test_conflicting_legacy_metadata_aliases_fail(updates: dict) -> None:payload = _legacy_345()payload.update(updates)with pytest.raises(ValueError, match="Conflicting legacy"):- read_cached_middle_json(payload)+ read_legacy_middle_json(payload)@pytest.mark.parametrize(@@ -215,7 +215,7 @@ def test_ambiguous_or_conflicting_v2_is_rejected(mutate: Callable[[dict], objectpayload = _legacy_v2()mutate(payload)with pytest.raises(ValueError):- read_cached_middle_json(payload)+ read_legacy_middle_json(payload)@pytest.mark.parametrize(@@ -231,7 +231,7 @@ def test_ambiguous_or_conflicting_v2_is_rejected(mutate: Callable[[dict], objectdef test_mixed_pages_are_not_silently_emptied(payload: dict) -> None:"""拒绝把当前 blocks 误当作旧页面后悄悄丢弃正文。"""with pytest.raises(ValueError):- read_cached_middle_json(payload)+ read_legacy_middle_json(payload)@pytest.mark.parametrize("indices", [[3, 3], [5, 3], [-1, 3], [True, 3]])@@ -241,14 +241,14 @@ def test_invalid_legacy_page_indices_fail(indices: list) -> None:for page, index in zip(payload["pdf_info"], indices):page["page_idx"] = indexwith pytest.raises(ValueError):- read_cached_middle_json(payload)+ read_legacy_middle_json(payload)@pytest.mark.parametrize("family", ["345", "v1", "v2"])def test_compatibility_is_not_exposed_by_generic_readers(family: str) -> None:"""Doclib 兼容成功不改变通用 ParseResult 或 DocVortex 的严格边界。"""payload = _legacy_v2() if family == "v2" else _legacy_345(wrapped=family == "v1")- assert read_cached_middle_json(payload).pages+ assert read_legacy_middle_json(payload).pagesfor reader in [ParseResult.from_dict, MiddleJson.from_dict]:with pytest.raises(ValueError):reader(payload)