Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
Verify Hybrid unit and citation styles through shared font references
myhloli7ddc17fc745d
diff --git a/docs/pdf-script-local-font-validation.md b/docs/pdf-script-local-font-validation.mdnew file mode 100644index 00000000..84496562--- /dev/null+++ b/docs/pdf-script-local-font-validation.md@@ -0,0 +1,23 @@+# PDF 同字体局部正文复核验证++DocVortex 的共享上下标分类现在会以同字体连续英数字串中的可信正文样本复核+弱候选。MinerU 的 Hybrid 原生文字回填继续调用同一实现,没有增加产品参数或分支。++新增最小真实字符 fixture 验证中文论文4的 `12gpm` 全部正文、`[8]` 整体上标,+并通过生产 `chars_to_content()` 路径验证输出。原始字符索引、几何和字体策略不变。++本地文本/样式组 **206 项通过**;完整回归 **3958 passed、4 skipped、4 deselected**。+四项排除仍为既有基线:++- `test_doclib_app_startup.py::test_basic_extra_includes_preflight_runtime_dependencies`+- `test_doclib_app_startup.py::test_standard_is_the_highest_model_runtime_extra`+- `test_legacy_schema_adapter.py::test_doclib_compaction_rejects_unknown_legacy_schema`+- `test_mfr_latex_utils.py::test_pp_formulanet_fix_latex_uses_shared_mathring_repair`++引擎 [三平台 CI](https://github.com/myhloli/DocVortex/actions/runs/34052061906) 通过,+源代码为 `be494c615dfa597457e950f43cc1dcc6b1cfc970`。31 份语料仅目标片段四个字符的+上下标样式改变;真实引用、数学角标、表格角标和英文作者名保持原有正确结果。+原有少线表诊断单列,不改金标或放宽阈值。++升级后已保存的 Doclib 结果不自动重写,重启服务并用 `--force` 重新解析以获取修复。+0.1.0 安装包及发行草稿更新,不发布到 PyPI。diff --git a/tests/fixtures/pdf_mixed_font_script_line.json b/tests/fixtures/pdf_mixed_font_script_line.jsonnew file mode 100644index 00000000..1450a065--- /dev/null+++ b/tests/fixtures/pdf_mixed_font_script_line.json@@ -0,0 +1,777 @@+{+ "source": "demo/pdfs/中文论文4.pdf",+ "source_sha256": "697f37feb86cb49c905ceac0074e2569969e25043c15768cf6c0a86973fdc899",+ "page_index": 0,+ "coordinates": "upright PDF points, top-left origin",+ "page_size": [+ 595,+ 842+ ],+ "extraction": {+ "pypdfium2": "5.10.1",+ "font_policy": "droid-cjk-v1",+ "source_revision": "b4afb58e415e45a3317ef908b4ac3628d20fa181",+ "note": "Original char_idx and optional source_indices; layout loose boxes and upright tight/origin maps from the traced line."+ },+ "chars": [+ {+ "char": "势",+ "char_idx": 1346,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 308.74395751953125,+ 468.2242126464844,+ 318.44744873046875,+ 477.5395812988281+ ],+ "tight": [+ 309.35528564453125,+ 468.6317443847656,+ 318.03021240234375,+ 477.5395812988281+ ],+ "origin": [+ 308.74395751953125,+ 476.59832763671875+ ],+ "source_indices": [+ 1346+ ]+ },+ {+ "char": "高",+ "char_idx": 1347,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 319.11700439453125,+ 468.2242126464844,+ 328.82049560546875,+ 477.5007629394531+ ],+ "tight": [+ 319.65069580078125,+ 468.5638122558594,+ 328.28680419921875,+ 477.5007629394531+ ],+ "origin": [+ 319.11700439453125,+ 476.59832763671875+ ],+ "source_indices": [+ 1347+ ]+ },+ {+ "char": "度",+ "char_idx": 1348,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 329.5870666503906,+ 468.2242126464844,+ 339.29058837890625,+ 477.5007629394531+ ],+ "tight": [+ 329.96551513671875,+ 468.5929260253906,+ 338.83453369140625,+ 477.5007629394531+ ],+ "origin": [+ 329.58709716796875,+ 476.59832763671875+ ],+ "source_indices": [+ 1348+ ]+ },+ {+ "char": "差",+ "char_idx": 1349,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 340.05712890625,+ 468.2242126464844,+ 349.7606506347656,+ 477.1611328125+ ],+ "tight": [+ 340.5520324707031,+ 468.5250244140625,+ 349.0814208984375,+ 477.1611328125+ ],+ "origin": [+ 340.0571594238281,+ 476.59832763671875+ ],+ "source_indices": [+ 1349+ ]+ },+ {+ "char": "约",+ "char_idx": 1350,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 350.5272521972656,+ 468.2242126464844,+ 360.2307434082031,+ 477.4619445800781+ ],+ "tight": [+ 351.0221252441406,+ 468.6317443847656,+ 359.473876953125,+ 477.4619445800781+ ],+ "origin": [+ 350.5272521972656,+ 476.59832763671875+ ],+ "source_indices": [+ 1350+ ]+ },+ {+ "char": " ",+ "char_idx": 1351,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 361.4825134277344,+ 476.59832763671875,+ 361.4825134277344,+ 476.59832763671875+ ],+ "tight": null,+ "origin": [+ 361.4825134277344,+ 476.59832763671875+ ],+ "source_indices": [+ 1351+ ]+ },+ {+ "char": "为",+ "char_idx": 1352,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 361.4825134277344,+ 468.2242126464844,+ 371.18603515625,+ 477.4619445800781+ ],+ "tight": [+ 362.3170166015625,+ 468.6317443847656,+ 370.1283264160156,+ 477.4619445800781+ ],+ "origin": [+ 361.4825134277344,+ 476.59832763671875+ ],+ "source_indices": [+ 1352+ ]+ },+ {+ "char": "1",+ "char_idx": 1353,+ "font": {+ "name": "DY5+ZLOBQy-5",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 372.593017578125,+ 469.4662780761719,+ 382.28680419921875,+ 476.25872802734375+ ],+ "tight": [+ 375.7272644042969,+ 469.58270263671875,+ 379.06524658203125,+ 476.25872802734375+ ],+ "origin": [+ 372.593017578125,+ 477.5686950683594+ ],+ "source_indices": [+ 1353+ ]+ },+ {+ "char": "2",+ "char_idx": 1354,+ "font": {+ "name": "DY5+ZLOBQy-5",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 377.8426208496094,+ 469.4662780761719,+ 387.5364074707031,+ 476.25872802734375+ ],+ "tight": [+ 380.7730712890625,+ 469.58270263671875,+ 384.5089111328125,+ 476.25872802734375+ ],+ "origin": [+ 377.8426208496094,+ 477.5686950683594+ ],+ "source_indices": [+ 1354+ ]+ },+ {+ "char": "g",+ "char_idx": 1355,+ "font": {+ "name": "DY5+ZLOBQy-5",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 387.53643798828125,+ 470.0484924316406,+ 397.230224609375,+ 478.0732727050781+ ],+ "tight": [+ 390.0593566894531,+ 472.00860595703125,+ 394.61029052734375,+ 478.0732727050781+ ],+ "origin": [+ 387.53643798828125,+ 478.1509094238281+ ],+ "source_indices": [+ 1355+ ]+ },+ {+ "char": "p",+ "char_idx": 1356,+ "font": {+ "name": "DY5+ZLOBQy-5",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 392.9801025390625,+ 470.0484924316406,+ 402.6739196777344,+ 478.0732727050781+ ],+ "tight": [+ 395.5224304199219,+ 471.90185546875,+ 399.976318359375,+ 478.0732727050781+ ],+ "origin": [+ 392.9801025390625,+ 478.1509094238281+ ],+ "source_indices": [+ 1356+ ]+ },+ {+ "char": "m",+ "char_idx": 1357,+ "font": {+ "name": "DY5+ZLOBQy-5",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 399.9569091796875,+ 469.4662780761719,+ 409.65069580078125,+ 476.25872802734375+ ],+ "tight": [+ 400.966064453125,+ 471.91156005859375,+ 408.5445251464844,+ 476.25872802734375+ ],+ "origin": [+ 399.9569091796875,+ 477.5686950683594+ ],+ "source_indices": [+ 1357+ ]+ },+ {+ "char": "[",+ "char_idx": 1358,+ "font": {+ "name": "DY132+ZLOBRb-134",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 409.2886962890625,+ 468.0956726074219,+ 411.1726379394531,+ 473.1444091796875+ ],+ "tight": [+ 409.2886962890625,+ 468.0956726074219,+ 411.1726379394531,+ 473.1444091796875+ ],+ "origin": [+ 408.7870178222656,+ 472.2317810058594+ ],+ "source_indices": [+ 1358+ ]+ },+ {+ "char": "8",+ "char_idx": 1359,+ "font": {+ "name": "DY5+ZLOBQy-5",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 410.53369140625,+ 468.57598876953125,+ 415.86529541015625,+ 472.3758850097656+ ],+ "tight": [+ 412.14544677734375,+ 468.65069580078125,+ 414.2588806152344,+ 472.3758850097656+ ],+ "origin": [+ 410.53369140625,+ 473.0323181152344+ ],+ "source_indices": [+ 1359+ ]+ },+ {+ "char": "]",+ "char_idx": 1360,+ "font": {+ "name": "DY132+ZLOBRb-134",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 415.08612060546875,+ 468.0956726074219,+ 416.97003173828125,+ 473.1444091796875+ ],+ "tight": [+ 415.08612060546875,+ 468.0956726074219,+ 416.97003173828125,+ 473.1444091796875+ ],+ "origin": [+ 414.8032531738281,+ 472.2317810058594+ ],+ "source_indices": [+ 1360+ ]+ },+ {+ "char": "。",+ "char_idx": 1361,+ "font": {+ "name": "DY132+ZLOBRb-134",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 420.5962829589844,+ 475.1622314453125,+ 422.6825256347656,+ 477.2388000488281+ ],+ "tight": [+ 420.5962829589844,+ 475.1622314453125,+ 422.6825256347656,+ 477.2388000488281+ ],+ "origin": [+ 419.26690673828125,+ 476.1131896972656+ ],+ "source_indices": [+ 1361+ ]+ },+ {+ "char": "本",+ "char_idx": 1362,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 430.1348571777344,+ 468.2242126464844,+ 439.8383483886719,+ 477.4619445800781+ ],+ "tight": [+ 430.5520935058594,+ 468.5929260253906,+ 439.3822937011719,+ 477.4619445800781+ ],+ "origin": [+ 430.1348571777344,+ 476.59832763671875+ ],+ "source_indices": [+ 1362+ ]+ },+ {+ "char": "文",+ "char_idx": 1363,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 441.0998229980469,+ 468.2242126464844,+ 450.8033447265625,+ 477.5395812988281+ ],+ "tight": [+ 441.4782409667969,+ 468.6705627441406,+ 450.2308044433594,+ 477.5395812988281+ ],+ "origin": [+ 441.0998229980469,+ 476.59832763671875+ ],+ "source_indices": [+ 1363+ ]+ },+ {+ "char": "选",+ "char_idx": 1364,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 452.0647888183594,+ 468.2242126464844,+ 461.768310546875,+ 477.3843078613281+ ],+ "tight": [+ 452.5208435058594,+ 468.6317443847656,+ 461.3122253417969,+ 477.3843078613281+ ],+ "origin": [+ 452.0647888183594,+ 476.59832763671875+ ],+ "source_indices": [+ 1364+ ]+ },+ {+ "char": "取",+ "char_idx": 1365,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 463.02001953125,+ 468.2242126464844,+ 472.7235412597656,+ 477.4619445800781+ ],+ "tight": [+ 463.5149230957031,+ 468.9422607421875,+ 472.3062744140625,+ 477.4619445800781+ ],+ "origin": [+ 463.02001953125,+ 476.59832763671875+ ],+ "source_indices": [+ 1365+ ]+ },+ {+ "char": "的",+ "char_idx": 1366,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 473.97528076171875,+ 468.2242126464844,+ 483.67877197265625,+ 477.4619445800781+ ],+ "tight": [+ 474.7709655761719,+ 468.6317443847656,+ 482.92193603515625,+ 477.4619445800781+ ],+ "origin": [+ 473.97528076171875,+ 476.59832763671875+ ],+ "source_indices": [+ 1366+ ]+ },+ {+ "char": "资",+ "char_idx": 1367,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 484.9305419921875,+ 468.2242126464844,+ 494.634033203125,+ 477.578369140625+ ],+ "tight": [+ 485.3866271972656,+ 468.6317443847656,+ 494.1003723144531,+ 477.578369140625+ ],+ "origin": [+ 484.9305419921875,+ 476.59832763671875+ ],+ "source_indices": [+ 1367+ ]+ },+ {+ "char": "料",+ "char_idx": 1368,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 495.88580322265625,+ 468.2242126464844,+ 505.5893249511719,+ 477.5007629394531+ ],+ "tight": [+ 496.3030700683594,+ 468.6317443847656,+ 505.28851318359375,+ 477.5007629394531+ ],+ "origin": [+ 495.88580322265625,+ 476.59832763671875+ ],+ "source_indices": [+ 1368+ ]+ },+ {+ "char": "为",+ "char_idx": 1369,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 506.841064453125,+ 468.2242126464844,+ 516.5445556640625,+ 477.4619445800781+ ],+ "tight": [+ 507.6755676269531,+ 468.6317443847656,+ 515.4868774414062,+ 477.4619445800781+ ],+ "origin": [+ 506.841064453125,+ 476.59832763671875+ ],+ "source_indices": [+ 1369+ ]+ },+ {+ "char": "重",+ "char_idx": 1370,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 517.7963256835938,+ 468.2242126464844,+ 527.4998168945312,+ 477.199951171875+ ],+ "tight": [+ 518.3300170898438,+ 468.7870178222656,+ 526.9273071289062,+ 477.199951171875+ ],+ "origin": [+ 517.7963256835938,+ 476.59832763671875+ ],+ "source_indices": [+ 1370+ ]+ },+ {+ "char": "庆",+ "char_idx": 1371,+ "font": {+ "name": "��������_GBK",+ "flags": 524292,+ "size": 1.0,+ "weight": 455+ },+ "bbox": [+ 528.7515869140625,+ 468.2242126464844,+ 538.455078125,+ 477.4619445800781+ ],+ "tight": [+ 529.1300048828125,+ 468.5929260253906,+ 537.8146362304688,+ 477.4619445800781+ ],+ "origin": [+ 528.7515869140625,+ 476.59832763671875+ ],+ "source_indices": [+ 1371+ ]+ }+ ],+ "expected": {+ "1353": "body",+ "1354": "body",+ "1355": "body",+ "1356": "body",+ "1357": "body",+ "1358": "sup",+ "1359": "sup",+ "1360": "sup"+ }+}diff --git a/tests/unittest/test_pdf_native_script_detection.py b/tests/unittest/test_pdf_native_script_detection.pyindex 357e80f3..2af98563 100644--- a/tests/unittest/test_pdf_native_script_detection.py+++ b/tests/unittest/test_pdf_native_script_detection.py@@ -1042,3 +1042,29 @@ def test_rotated_powerpoint_page_keeps_all_plain_text_on_baseline() -> None:assert len(rendered_lines) == 33assert rendered_lines[0] == "Energy Metabolism"assert all("<sup>" not in content and "<sub>" not in content for content in rendered_lines)+++def test_mixed_font_unit_and_citation_use_shared_local_body_reference() -> None:+ """最小真实字符夹具通过 Hybrid 生产路径恢复正文单位和完整引用上标。"""+ from copy import deepcopy++ payload = json.loads((_PROJECT_ROOT / "tests/fixtures/pdf_mixed_font_script_line.json").read_text(encoding="utf-8"))+ chars = []+ tight_bboxes = {}+ origins = {}+ for row in payload["chars"]:+ char = dict(row)+ tight, origin = char.pop("tight"), char.pop("origin")+ char["bbox"] = Bbox(char["bbox"])+ chars.append(char)+ if tight is not None:+ tight_bboxes[char["char_idx"]] = tuple(tight)+ if origin is not None:+ origins[char["char_idx"]] = tuple(origin)+ before = deepcopy(payload)+ roles = native._classify_char_script_roles(chars, tight_bboxes=tight_bboxes, origins=origins)+ assert {str(c["char_idx"]): role for c, role in zip(chars, roles) if 1353 <= c["char_idx"] <= 1360} == payload["expected"]+ text = "".join(_render_chars(_ScriptFixture(chars, tight_bboxes, origins)).split())+ assert "12gpm<sup>[8]</sup>" in text+ assert "<sub>" not in text+ assert payload == before