Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

refactor: extract shared legacy middle JSON reader to backend postprocess

jinzhenj6f0a27c09111
diff --git a/mineru/doclib/core/middle_json.py b/mineru/backend/postprocess/legacy_middle_json.py
similarity index 87%
rename from mineru/doclib/core/middle_json.py
rename to mineru/backend/postprocess/legacy_middle_json.py
index c0fdc723..5074f42f 100644
--- a/mineru/doclib/core/middle_json.py
+++ b/mineru/backend/postprocess/legacy_middle_json.py
@@ -1,7 +1,7 @@
-"""仅供 Doclib 持久化结果使用的 Middle JSON 读取与历史转换边界。
+# Copyright (c) Opendatalab. All rights reserved.
+"""历史/旧协议 Middle JSON 读取与转换边界。
-两个历史分支集中在此,未来可一并移除;通用 ParseResult 与 DocVortex codec
-不调用本模块。读取只构造内存对象,不重写文件、不重新推理。
+两个历史分支集中在此,未来可一并移除。读取只构造内存对象,不重写文件、不重新推理。
"""
from __future__ import annotations
@@ -64,7 +64,7 @@ def _legacy_extensions(payload: dict[str, Any]) -> dict[str, Any]:
def _read_legacy_345(payload: dict[str, Any], *, page_field: str) -> MiddleJson:
"""把可识别的 3.4.5 页面转为当前 raw ModelJson,再做确定性后处理。"""
- from ...backend.postprocess.legacy_schema_adapter import legacy_page_to_model_list
+ from .legacy_schema_adapter import legacy_page_to_model_list
from docvortex.postprocess.document import model_json_to_middle_json
pages = payload[page_field]
@@ -102,15 +102,15 @@ def _read_legacy_v2(payload: dict[str, Any]) -> MiddleJson:
)
-def read_cached_middle_json(payload: dict[str, Any]) -> MiddleJson:
- """先识别显式协议,再转换两个历史缓存族;失败时绝不跨协议兜底。"""
+def read_legacy_middle_json(payload: dict[str, Any]) -> MiddleJson:
+ """先识别显式协议,再转换两个历史分支;失败时绝不跨协议兜底。"""
if not isinstance(payload, dict):
- raise ValueError("Cached Middle JSON must be an object; source reparse required")
+ raise ValueError("Middle JSON must be an object; source reparse required")
if "schema" in payload:
document = MiddleJson.from_dict(payload)
else:
if {"metadata", "producer", "page_index_map"}.intersection(payload):
- raise ValueError("Mixed cached document envelope; source reparse required")
+ raise ValueError("Mixed document envelope; source reparse required")
version = payload.get("schema_version")
source = deepcopy(payload)
if "pdf_info" in source:
@@ -122,9 +122,9 @@ def read_cached_middle_json(payload: dict[str, Any]) -> MiddleJson:
elif version == "2.0" and "pages" in source:
document = _read_legacy_v2(source)
else:
- raise ValueError("Unsupported cached Middle JSON format; source reparse required")
+ raise ValueError("Unsupported Middle JSON format; source reparse required")
validate_mineru_metadata(document)
return document
-__all__ = ["read_cached_middle_json"]
+__all__ = ["read_legacy_middle_json"]
\ No newline at end of file
diff --git a/mineru/doclib/background/compaction.py b/mineru/doclib/background/compaction.py
index aa079163..cf6c0c0d 100644
--- a/mineru/doclib/background/compaction.py
+++ b/mineru/doclib/background/compaction.py
@@ -14,7 +14,7 @@ from typing import Any, cast
from ...parser.page_range import format_page_range
from ...types import Tier
from ..core.db import DatabaseManager
-from ..core.middle_json import read_cached_middle_json
+from ...backend.postprocess.legacy_middle_json import read_legacy_middle_json
from ..rows import ParseBatchRow, ParseGroupRow, ParseRow
from ..services.parse_svc import parse_batch_json_path, parse_page_range_set
from ..types import PARSE_STATUS_DONE, PARSE_STATUS_SUPERSEDED
@@ -24,7 +24,7 @@ logger = logging.getLogger("mineru.compaction")
def _normalize_batch_pages(batch_payload: dict[str, Any]) -> list[dict[str, Any]]:
"""读取当前和受支持的历史批次,仅在内存中统一页面结构。"""
- return [page.to_dict() for page in read_cached_middle_json(batch_payload).pages]
+ return [page.to_dict() for page in read_legacy_middle_json(batch_payload).pages]
class Compaction:
@@ -150,7 +150,7 @@ class Compaction:
try:
with open(fpath, encoding="utf-8") as f:
batch_payload = json.load(f)
- middle_json = read_cached_middle_json(batch_payload)
+ middle_json = read_legacy_middle_json(batch_payload)
batch_pages = [page.to_dict() for page in middle_json.pages]
except Exception:
return None
diff --git a/mineru/doclib/services/parse_svc.py b/mineru/doclib/services/parse_svc.py
index ef21791c..05ff4684 100644
--- a/mineru/doclib/services/parse_svc.py
+++ b/mineru/doclib/services/parse_svc.py
@@ -28,7 +28,7 @@ from ...filetypes import (
)
from ...parser.api_client import _APITransportError, _V1APIError
from ...parser.base import ParseResult
-from ..core.middle_json import read_cached_middle_json
+from ...backend.postprocess.legacy_middle_json import read_legacy_middle_json
from ...parser.page_range import (
expand_page_range,
format_page_range as _page_numbers_to_range_str,
@@ -1538,7 +1538,7 @@ def _json_batch_is_readable(data_dir: str, sha256: str, tier: Tier, batch: Parse
json_path = parse_batch_json_path(data_dir, sha256, tier, batch["page_range"], batch["done_at"])
try:
with open(json_path, encoding="utf-8") as stream:
- result = read_cached_middle_json(json.load(stream))
+ result = read_legacy_middle_json(json.load(stream))
available = {page.page_idx + 1 for page in result.pages}
return bool(available) and parse_page_range_set(batch["page_range"]) <= available
except (OSError, ValueError, TypeError):
@@ -1569,7 +1569,7 @@ def load_pages_from_done_batches(
try:
with open(fpath, encoding="utf-8") as f:
data = json.load(f)
- parse_result = read_cached_middle_json(data)
+ parse_result = read_legacy_middle_json(data)
for page in parse_result.pages:
pages_by_page_idx[page.page_idx] = page
except (OSError, ValueError, TypeError):
diff --git a/tests/unittest/test_doclib_middle_json.py b/tests/unittest/test_doclib_middle_json.py
index 367d9598..b098d8a4 100644
--- a/tests/unittest/test_doclib_middle_json.py
+++ b/tests/unittest/test_doclib_middle_json.py
@@ -7,7 +7,7 @@ import json
import pytest
from docvortex.schema import MiddleJson
-from mineru.doclib.core.middle_json import read_cached_middle_json
+from mineru.backend.postprocess.legacy_middle_json import read_legacy_middle_json
from mineru.parser import ParseResult
@@ -98,7 +98,7 @@ def test_legacy_345_roundtrip_preserves_origin_and_pages(wrapped: bool) -> None:
"""两个旧页面封装均可在缓存边界转换,输入和来源不受污染。"""
payload = _legacy_345(wrapped=wrapped)
before = deepcopy(payload)
- result = read_cached_middle_json(payload)
+ result = read_legacy_middle_json(payload)
assert payload == before
assert [page.page_idx for page in result.pages] == [3, 5]
assert result.is_full_document is False
@@ -106,7 +106,7 @@ def test_legacy_345_roundtrip_preserves_origin_and_pages(wrapped: bool) -> None:
assert result.extensions["mineru"] == {"tier": "standard", "parse_mode": "ocr"}
assert result.pages[0].blocks[0].content[0].content == "正文 "
assert result.pages[0].blocks[0].content[1].content == "x^2"
- assert read_cached_middle_json(result.to_dict()) == result
+ assert read_legacy_middle_json(result.to_dict()) == result
assert MiddleJson.from_json(result.to_json()) == result
@@ -124,14 +124,14 @@ def test_v2_changes_only_envelope_without_postprocess(monkeypatch: pytest.Monkey
monkeypatch.setattr(host_document, "build_middle_json", unexpected)
payload = _legacy_v2()
before = deepcopy(payload)
- result = read_cached_middle_json(payload)
+ result = read_legacy_middle_json(payload)
assert payload == before
assert result.to_dict()["pages"] == payload["pages"]
assert result.metadata.file_suffix == "pdf"
assert result.metadata.producer.version == "3.4.4"
assert result.extensions["application"] == payload["extensions"]["application"]
assert result.extensions["mineru"] == {"tier": "standard", "parse_mode": "ocr"}
- assert read_cached_middle_json(result.to_dict()).to_dict() == result.to_dict()
+ assert read_legacy_middle_json(result.to_dict()).to_dict() == result.to_dict()
result.extensions["application"]["items"].append("changed")
assert payload == before
@@ -150,7 +150,7 @@ def test_v2_changes_only_envelope_without_postprocess(monkeypatch: pytest.Monkey
def test_missing_metadata_stays_unknown(updates: dict, expected: dict | None) -> None:
"""缺失或不可识别的记录保持未知,不制造当前版本或默认解析档位。"""
payload = {"pdf_info": [], **updates}
- result = read_cached_middle_json(payload)
+ result = read_legacy_middle_json(payload)
assert result.metadata.producer.model_dump() == {"name": "mineru", "version": "unknown"}
assert result.metadata.file_suffix == "pdf"
assert result.extensions.get("mineru") == expected
@@ -161,7 +161,7 @@ def test_v2_missing_product_records_preserve_application_extensions() -> None:
payload = _legacy_v2()
for key in ("mineru_version", "effort", "parse_mode"):
del payload[key]
- document = read_cached_middle_json(payload)
+ document = read_legacy_middle_json(payload)
assert document.metadata.producer.version == "unknown"
assert document.extensions == payload["extensions"]
@@ -170,7 +170,7 @@ def test_consistent_existing_product_extension_is_preserved() -> None:
"""已有可靠产品扩展与旧字段一致时直接保留,不产生重复记录。"""
payload = _legacy_v2()
payload["extensions"]["mineru"] = {"tier": "standard", "parse_mode": "ocr"}
- assert read_cached_middle_json(payload).extensions == payload["extensions"]
+ assert read_legacy_middle_json(payload).extensions == payload["extensions"]
def test_345_conflicting_full_document_flag_fails() -> None:
@@ -178,7 +178,7 @@ def test_345_conflicting_full_document_flag_fails() -> None:
payload = _legacy_345()
payload["is_full_document"] = True
with pytest.raises(ValueError, match="Conflicting legacy is_full_document"):
- read_cached_middle_json(payload)
+ read_legacy_middle_json(payload)
@pytest.mark.parametrize(
@@ -194,7 +194,7 @@ def test_conflicting_legacy_metadata_aliases_fail(updates: dict) -> None:
payload = _legacy_345()
payload.update(updates)
with pytest.raises(ValueError, match="Conflicting legacy"):
- read_cached_middle_json(payload)
+ read_legacy_middle_json(payload)
@pytest.mark.parametrize(
@@ -215,7 +215,7 @@ def test_ambiguous_or_conflicting_v2_is_rejected(mutate: Callable[[dict], object
payload = _legacy_v2()
mutate(payload)
with pytest.raises(ValueError):
- read_cached_middle_json(payload)
+ read_legacy_middle_json(payload)
@pytest.mark.parametrize(
@@ -231,7 +231,7 @@ def test_ambiguous_or_conflicting_v2_is_rejected(mutate: Callable[[dict], object
def test_mixed_pages_are_not_silently_emptied(payload: dict) -> None:
"""拒绝把当前 blocks 误当作旧页面后悄悄丢弃正文。"""
with pytest.raises(ValueError):
- read_cached_middle_json(payload)
+ read_legacy_middle_json(payload)
@pytest.mark.parametrize("indices", [[3, 3], [5, 3], [-1, 3], [True, 3]])
@@ -241,14 +241,14 @@ def test_invalid_legacy_page_indices_fail(indices: list) -> None:
for page, index in zip(payload["pdf_info"], indices):
page["page_idx"] = index
with pytest.raises(ValueError):
- read_cached_middle_json(payload)
+ read_legacy_middle_json(payload)
@pytest.mark.parametrize("family", ["345", "v1", "v2"])
def test_compatibility_is_not_exposed_by_generic_readers(family: str) -> None:
"""Doclib 兼容成功不改变通用 ParseResult 或 DocVortex 的严格边界。"""
payload = _legacy_v2() if family == "v2" else _legacy_345(wrapped=family == "v1")
- assert read_cached_middle_json(payload).pages
+ assert read_legacy_middle_json(payload).pages
for reader in [ParseResult.from_dict, MiddleJson.from_dict]:
with pytest.raises(ValueError):
reader(payload)