Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix(pdf): restore native drop caps and word boundaries

myhloli11ea32032f8c
diff --git a/mineru/backend/analysis/pdf/text/content.py b/mineru/backend/analysis/pdf/text/content.py
index f26d312a..97053c5c 100644
--- a/mineru/backend/analysis/pdf/text/content.py
+++ b/mineru/backend/analysis/pdf/text/content.py
@@ -4,7 +4,7 @@
from __future__ import annotations
-
+import math
from typing import TYPE_CHECKING, Any, Literal
import numpy as np
@@ -253,6 +253,99 @@ def _group_page_spans_by_block(
return block_lines
+def _recover_native_drop_caps(
+ page_model_list: list[dict[str, Any]],
+ block_lines: dict[int, list[_AnalyzeLine]],
+ page_size: tuple[float, float],
+ geometry: PDFPageTextGeometry | None,
+ excluded_regions: list[BBox],
+) -> None:
+ """仅补回正文首行紧邻且未被认领的大号首字,组行完成后更新几何避免串行。"""
+ if geometry is None:
+ return
+ claimed: set[int] = set()
+ all_spans = [span for lines in block_lines.values() for line in lines for span in line.spans]
+ # 先按实际字形高度筛选少量候选,普通页面不为每个正文块重扫全部字符。
+ candidate_positions: dict[str, list[int]] = {}
+ for position in range(1, len(geometry.chars)):
+ cap, following = geometry.chars[position - 1 : position + 1]
+ cap_box = geometry.tight_bboxes.get(cap.get("char_idx"))
+ next_box = geometry.tight_bboxes.get(following.get("char_idx"))
+ if cap_box is not None and next_box is not None and next_box[3] > next_box[1]:
+ if cap_box[3] - cap_box[1] >= 1.8 * (next_box[3] - next_box[1]):
+ candidate_positions.setdefault(str(following.get("char", "")), []).append(position)
+ for block_index, lines in block_lines.items():
+ if not lines or page_model_list[block_index].get("type") != BlockType.TEXT:
+ continue
+ first_line = lines[0]
+ if not first_line.spans or first_line.spans[0].type != ContentType.TEXT:
+ continue
+ first_span = first_line.spans[0]
+ if not first_span.content or first_span.image is not None or not first_span.metadata.get("_native_tight_spacing", True):
+ continue
+ block_bbox = _bbox_to_pixel_bbox(page_model_list[block_index].get("bbox"), page_size)
+ if block_bbox is None:
+ continue
+ for position in candidate_positions.get(first_span.content[0], []):
+ following = geometry.chars[position]
+ cap = geometry.chars[position - 1]
+ cap_text = str(cap.get("char", ""))
+ cap_index, next_index = cap.get("char_idx"), following.get("char_idx")
+ if (
+ len(cap_text) != 1
+ or not cap_text.isascii()
+ or not cap_text.isalpha()
+ or cap_index is None
+ or next_index != cap_index + 1
+ or cap_index in claimed
+ or cap.get("visible") is False
+ or cap.get("text_render_mode") == 3
+ ):
+ continue
+ cap_bbox = geometry.tight_bboxes.get(cap_index)
+ next_bbox = geometry.tight_bboxes.get(next_index)
+ if cap_bbox is None or next_bbox is None or not all(math.isfinite(value) for value in (*cap_bbox, *next_bbox)):
+ continue
+ angles = (float(cap.get("writing_angle", 0)), float(following.get("writing_angle", 0)))
+ if any(not math.isfinite(angle) or abs(math.degrees(angle)) > 0.1 for angle in angles):
+ continue
+ height = next_bbox[3] - next_bbox[1]
+ if (
+ height <= 0
+ or cap_bbox[3] - cap_bbox[1] < 1.8 * height
+ or abs(cap_bbox[1] - next_bbox[1]) > 0.25 * height
+ or not 0 <= next_bbox[0] - cap_bbox[2] <= 0.5 * height
+ or not 0 <= next_bbox[0] - first_span.bbox[0] <= 0.5 * height
+ or not first_span.bbox[1] <= (next_bbox[1] + next_bbox[3]) / 2 <= first_span.bbox[3]
+ or not (
+ block_bbox[0] <= cap_bbox[0] < cap_bbox[2] <= block_bbox[2]
+ and block_bbox[1] <= cap_bbox[1] < cap_bbox[3] <= block_bbox[3]
+ )
+ ):
+ continue
+ center_x, center_y = (cap_bbox[0] + cap_bbox[2]) / 2, (cap_bbox[1] + cap_bbox[3]) / 2
+ if any(box[0] <= center_x <= box[2] and box[1] <= center_y <= box[3] for box in excluded_regions):
+ continue
+ # 已由其他行、公式或相邻栏框完整覆盖的首字不能再补入当前正文。
+ if any(span.bbox[0] <= center_x <= span.bbox[2] and span.bbox[1] <= center_y <= span.bbox[3] for span in all_spans):
+ continue
+ first_span.content = cap_text + first_span.content
+ first_span.bbox = (
+ min(first_span.bbox[0], cap_bbox[0]),
+ min(first_span.bbox[1], cap_bbox[1]),
+ max(first_span.bbox[2], cap_bbox[2]),
+ max(first_span.bbox[3], cap_bbox[3]),
+ )
+ first_line.bbox = (
+ min(first_line.bbox[0], cap_bbox[0]),
+ min(first_line.bbox[1], cap_bbox[1]),
+ max(first_line.bbox[2], cap_bbox[2]),
+ max(first_line.bbox[3], cap_bbox[3]),
+ )
+ claimed.add(cap_index)
+ break
+
+
def _apply_window_post_ocr(
local_model_context: HybridLocalModelContext,
page_block_lines_list: list[dict[int, list[_AnalyzeLine]]],
@@ -510,6 +603,10 @@ def _fill_window_block_content_and_lines(
page_size,
target_block_types,
)
+ if parse_mode == "txt":
+ _recover_native_drop_caps(
+ page_model_list, block_lines, page_size, evidence.geometry, inline_math_regions + table_regions
+ )
page_block_line_results.append(
(
page_model_list,
diff --git a/mineru/backend/analysis/pdf/text/native.py b/mineru/backend/analysis/pdf/text/native.py
index dbbd5cf3..be1ba8e8 100644
--- a/mineru/backend/analysis/pdf/text/native.py
+++ b/mineru/backend/analysis/pdf/text/native.py
@@ -12,7 +12,7 @@ from typing import Any, Callable, cast
import numpy as np
from docvortex.analyzers.pdf import PDF_NATIVE_SCRIPT_MARKUP_KEY, ScriptRole, classify_char_script_roles
-from docvortex.analyzers.pdf import is_generated_cjk_space, join_tight_text, needs_tight_space
+from docvortex.analyzers.pdf import is_generated_cjk_space, is_ligature_expansion_space, join_tight_text, needs_tight_space
from docvortex.assets import calculate_contrast
from docvortex.document.pdf import Char, PDFPage, get_lines_from_chars
from docvortex.geometry import calculate_overlap_area_in_bbox1_area_ratio
@@ -567,6 +567,7 @@ def _owned_span_texts(owner: Any, spans: list[_AnalyzeSpan], median_height: floa
functions = (
needs_tight_space,
is_generated_cjk_space,
+ is_ligature_expansion_space,
chars_to_content,
__replace_unicode,
__replace_ligatures,
@@ -761,6 +762,7 @@ def fill_char_in_spans(
LINE_STOP_FLAG = (
+ "\u0002", # PDFium 字符接口中的短横占位符必须先归属行尾,再由既有清洗规则转为连字符。
".",
"!",
"?",
@@ -1100,7 +1102,10 @@ def chars_to_content(
if (
span.metadata.get("_native_tight_spacing", True)
and 0 < idx < len(chars) - 1
- and is_generated_cjk_space(chars[idx - 1], char1, chars[idx + 1], origins=origins)
+ and (
+ is_generated_cjk_space(chars[idx - 1], char1, chars[idx + 1], origins=origins)
+ or is_ligature_expansion_space(chars, idx, tight_bboxes=tight_bboxes, origins=origins)
+ )
):
# 原始相邻项仍是空格,旧字距分支也不会重新插入被忽略的生成空格。
continue
@@ -1140,6 +1145,7 @@ def chars_to_content(
_STANDARD_CONTENT_FUNCTIONS = (
needs_tight_space,
is_generated_cjk_space,
+ is_ligature_expansion_space,
chars_to_content,
__replace_unicode,
__replace_ligatures,
diff --git a/tests/fixtures/pdf_text_5635_5636/before.md b/tests/fixtures/pdf_text_5635_5636/before.md
new file mode 100644
index 00000000..5964c9fc
--- /dev/null
+++ b/tests/fixtures/pdf_text_5635_5636/before.md
@@ -0,0 +1,19 @@
+his book is a practical guide to algorithmic trading strategies that can be readily implemented by both retail and institutional traders. It is not an academic treatise on fi nancial theory. Rather, I hope to make accessible to the reader some of the most useful fi nancial research done in the past few decades, mixing them with insights I gained from actually exploiting some of those theories in live trading.
+
+Because strategies take a central place in this book, we will cover a wide array of them, broadly divided into the mean-reverting and momentum camps, and we will lay out standard techniques for trading each category of strategies, and equally important, the fundamental reasons why a strategy should work. The emphasis throughout is on simple and linear strategies, as an antidote to the overfi tting and data-snooping biases that often plague complex strategies.
+
+In the mean-reverting camp, we will discuss the multiple statistical tech niques (augmented Dickey-Fuller [ADF] test, Hurst exponent, Variance Ratio test, half-life) for detecting “time series” mean reversion or stationarity, and for detecting cointegration of a portfolio of instruments (cointegrated augmented Dickey Fuller [CADF] test, Johansen test). Beyond the mechanical application of these statistical tests to time series, we strive to convey an intuitive understanding of what they are really testing and the simple mathematical equations behind them.
+
+We will explain the simplest techniques and strategies for trading meanreverting portfolios (linear, Bollinger band, Kalman fi lter), and whether us ing raw prices, log prices, or ratios make the most sense as inputs to these tests and strategies. In particular, we show that the Kalman fi lter is useful to traders in multiple ways and in multiple strategies. Distinction between time series versus cross-sectional mean reversion will be made. We will debate the pros and cons of “scaling-in” and highlight the danger of data errors in mean-reverting strategies, especially those that deal with spreads.
+
+Examples of mean-reverting strategies will be drawn from interday and intraday stocks models, exchange-traded fund (ETF) pairs and triplets, ETFs versus their component stocks, currency pairs, and futures calendar and intermarket spreads. We will explain what makes trading some of these strategies quite challenging in recent years due to the rise of dark pools and high-frequency trading. We will also illustrate how certain fundamental considerations can explain the temporary unhinging of a hitherto very profi table ETF pair and how the same considerations can lead one to construct an improved version of the strategy. When discussing currency trading, we take care to explain why even the calculation of returns may seem foreign to an equity trader, and where such concepts as rollover interest may sometimes be important. Much emphasis will be devoted to the study of spot returns versus roll returns in futures, and several futures trading strategies can be derived or understood from a simple mathematical model of futures prices. The concepts of backwardation and contango will be illustrated graphically as well as mathematically. The chapter on mean reversion of currencies and futures cumulates in the study of a very special future: the volatility (VX) future, and how it can form the basis of some quite lucrative strategies.
+
+In the momentum camp, we start by explaining a few statistical tests for times series momentum. The main theme, though, is to explore the four main drivers of momentum in stocks and futures and to propose strategies that can extract time series and cross-sectional momentum. Roll returns in futures is one of those drivers, but it turns out that forced asset sales and purchases is the main driver of stock and ETF momentum in many diverse circumstances. Some of the newer momentum strategies based on news events, news sentiment, leveraged ETFs, order fl ow, and high-frequency trading will be covered. Finally, we will look at the pros and cons of mo mentum versus mean-reverting strategies and discover their diametrically diff erent risk-return characteristics under diff erent market regimes in recent fi nancial history.
+
+I have always maintained that it is easy to fi nd published, supposedly profi table, strategies in the many books, magazines, or blogs out there, but much harder to see why they may be fl awed and perhaps ultimately doomed. So, despite the emphasis on suggesting prototype strategies, we the use of in-sample data to fi nd the half-life and therefore the look back. Furthermore, an unlimited amount of capital may be needed to generate the P&L because there was no maximum imposed on the market value of the portfolio. So I certainly don’t recommend it as a practical trading strategy. (There is a more practical version of this mean-reverting strategy in Chapter 5.) But it does illustrate that a nonstationary price series need not discourage us from trading a mean reversion strategy, and that we don’t need very complicated strategies or technical indicators to extract profi ts from a meanreverting series.
+
+Since the goal for traders is ultimately to determine whether the expected return or Sharpe ratio of a mean-reverting trading strategy is good enough, why do we bother to go through the stationarity tests (ADF or Variance Ratio) and the calculation of half-life at all? Can’t we just run a backtest on the trading strategy directly and be done with it? The reason why we went through all these preliminary tests is that their statistical signifi cance is usually higher than a direct backtest of a trading strategy. These preliminary tests make use of every day’s (or, more generally, every bar’s) price data for the test, while a backtest usually generates a signifi cantly smaller number of round trip trades for us to collect performance statistics. Furthermore, the outcome of a backtest is dependent on the specifi cs of a trading strategy, with a specifi c set of trading parameters. However, given a price series that passed the stationarity statistical tests, or at least one with a short enough half-life, we can be assured that we can eventually fi nd a profi table trading strategy, maybe just not the one that we have backtested.
+
+## ■ Cointegration
+
+As we stated in the introduction of this chapter, most fi nancial price series are not stationary or mean reverting. But, fortunately, we are not confi ned to trading those “prefabricated” fi nancial price series: We can proactively cre ate a portfolio of individual price series so that the market value (or price) series of this portfolio is stationary. This is the notion of cointegration: If we
\ No newline at end of file
diff --git a/tests/fixtures/pdf_text_5635_5636/before_middle.json b/tests/fixtures/pdf_text_5635_5636/before_middle.json
new file mode 100644
index 00000000..ff068dd2
--- /dev/null
+++ b/tests/fixtures/pdf_text_5635_5636/before_middle.json
@@ -0,0 +1,335 @@
+{
+ "metadata": {
+ "file_suffix": "pdf",
+ "producer": {
+ "name": "mineru",
+ "version": "4.0.12"
+ },
+ "document": {
+ "producer_application": "pypdf",
+ "page_count": 3,
+ "page_count_kind": "physical"
+ }
+ },
+ "extensions": {
+ "mineru": {
+ "tier": "standard",
+ "parse_mode": "txt"
+ },
+ "docvortex_layout": {
+ "version": 1,
+ "pages": [
+ {
+ "page_idx": 0,
+ "width_pt": 432.0,
+ "height_pt": 648.0
+ },
+ {
+ "page_idx": 1,
+ "width_pt": 432.0,
+ "height_pt": 648.0
+ },
+ {
+ "page_idx": 2,
+ "width_pt": 432.0,
+ "height_pt": 648.0
+ }
+ ]
+ }
+ },
+ "pages": [
+ {
+ "page_idx": 0,
+ "blocks": [
+ {
+ "type": "header",
+ "index": 0,
+ "bbox": [
+ 0.424,
+ 0.166,
+ 0.577,
+ 0.186
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "P R E FA C E"
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 1,
+ "bbox": [
+ 0.116,
+ 0.367,
+ 0.881,
+ 0.498
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "his book is a practical guide to algorithmic trading strategies that can be readily implemented by both retail and institutional traders. It is not an academic treatise on fi nancial theory. Rather, I hope to make accessible to the reader some of the most useful fi nancial research done in the past few decades, mixing them with insights I gained from actually exploiting some of those theories in live trading."
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 2,
+ "bbox": [
+ 0.117,
+ 0.498,
+ 0.881,
+ 0.65
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Because strategies take a central place in this book, we will cover a wide array of them, broadly divided into the mean-reverting and momentum camps, and we will lay out standard techniques for trading each category of strategies, and equally important, the fundamental reasons why a strategy should work. The emphasis throughout is on simple and linear strategies, as an antidote to the overfi tting and data-snooping biases that often plague complex strategies."
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 3,
+ "bbox": [
+ 0.117,
+ 0.649,
+ 0.88,
+ 0.822
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "In the mean-reverting camp, we will discuss the multiple statistical tech niques (augmented Dickey-Fuller [ADF] test, Hurst exponent, Variance Ratio test, half-life) for detecting “time series” mean reversion or stationarity, and for detecting cointegration of a portfolio of instruments (cointegrated augmented Dickey Fuller [CADF] test, Johansen test). Beyond the mechanical application of these statistical tests to time series, we strive to convey an intuitive understanding of what they are really testing and the simple mathematical equations behind them."
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 4,
+ "bbox": [
+ 0.117,
+ 0.822,
+ 0.881,
+ 0.909
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "We will explain the simplest techniques and strategies for trading meanreverting portfolios (linear, Bollinger band, Kalman fi lter), and whether us ing raw prices, log prices, or ratios make the most sense as inputs to these tests and strategies. In particular, we show that the Kalman fi lter is useful"
+ }
+ ]
+ },
+ {
+ "type": "aside_text",
+ "index": 5,
+ "bbox": [
+ 0.928,
+ 0.486,
+ 0.962,
+ 0.504
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "ix"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "page_idx": 1,
+ "blocks": [
+ {
+ "type": "aside_text",
+ "index": 0,
+ "bbox": [
+ 0.043,
+ 0.488,
+ 0.068,
+ 0.571
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "PREFACE | X"
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 1,
+ "bbox": [
+ 0.117,
+ 0.067,
+ 0.88,
+ 0.153
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "to traders in multiple ways and in multiple strategies. Distinction between time series versus cross-sectional mean reversion will be made. We will debate the pros and cons of “scaling-in” and highlight the danger of data errors in mean-reverting strategies, especially those that deal with spreads."
+ }
+ ],
+ "continues_prev": true
+ },
+ {
+ "type": "text",
+ "index": 2,
+ "bbox": [
+ 0.117,
+ 0.153,
+ 0.882,
+ 0.564
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Examples of mean-reverting strategies will be drawn from interday and intraday stocks models, exchange-traded fund (ETF) pairs and triplets, ETFs versus their component stocks, currency pairs, and futures calendar and intermarket spreads. We will explain what makes trading some of these strategies quite challenging in recent years due to the rise of dark pools and high-frequency trading. We will also illustrate how certain fundamental considerations can explain the temporary unhinging of a hitherto very profi table ETF pair and how the same considerations can lead one to construct an improved version of the strategy. When discussing currency trading, we take care to explain why even the calculation of returns may seem foreign to an equity trader, and where such concepts as rollover interest may sometimes be important. Much emphasis will be devoted to the study of spot returns versus roll returns in futures, and several futures trading strategies can be derived or understood from a simple mathematical model of futures prices. The concepts of backwardation and contango will be illustrated graphically as well as mathematically. The chapter on mean reversion of currencies and futures cumulates in the study of a very special future: the volatility (VX) future, and how it can form the basis of some quite lucrative strategies."
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 3,
+ "bbox": [
+ 0.117,
+ 0.563,
+ 0.882,
+ 0.822
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "In the momentum camp, we start by explaining a few statistical tests for times series momentum. The main theme, though, is to explore the four main drivers of momentum in stocks and futures and to propose strategies that can extract time series and cross-sectional momentum. Roll returns in futures is one of those drivers, but it turns out that forced asset sales and purchases is the main driver of stock and ETF momentum in many diverse circumstances. Some of the newer momentum strategies based on news events, news sentiment, leveraged ETFs, order fl ow, and high-frequency trading will be covered. Finally, we will look at the pros and cons of mo mentum versus mean-reverting strategies and discover their diametrically diff erent risk-return characteristics under diff erent market regimes in recent fi nancial history."
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 4,
+ "bbox": [
+ 0.117,
+ 0.822,
+ 0.882,
+ 0.911
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "I have always maintained that it is easy to fi nd published, supposedly profi table, strategies in the many books, magazines, or blogs out there, but much harder to see why they may be fl awed and perhaps ultimately doomed. So, despite the emphasis on suggesting prototype strategies, we"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "page_idx": 2,
+ "blocks": [
+ {
+ "type": "aside_text",
+ "index": 0,
+ "bbox": [
+ 0.042,
+ 0.485,
+ 0.07,
+ 0.671
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "50 ALGORITHMIC TRADING"
+ }
+ ]
+ },
+ {
+ "type": "header",
+ "index": 1,
+ "bbox": [
+ 0.371,
+ 0.073,
+ 0.655,
+ 0.094
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Example 2.5 (Continued)"
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "index": 2,
+ "bbox": [
+ 0.145,
+ 0.116,
+ 0.843,
+ 0.334
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "the use of in-sample data to fi nd the half-life and therefore the look back. Furthermore, an unlimited amount of capital may be needed to generate the P&L because there was no maximum imposed on the market value of the portfolio. So I certainly don’t recommend it as a practical trading strategy. (There is a more practical version of this mean-reverting strategy in Chapter 5.) But it does illustrate that a nonstationary price series need not discourage us from trading a mean reversion strategy, and that we don’t need very complicated strategies or technical indicators to extract profi ts from a meanreverting series."
+ }
+ ],
+ "continues_prev": true
+ },
+ {
+ "type": "text",
+ "index": 3,
+ "bbox": [
+ 0.117,
+ 0.398,
+ 0.882,
+ 0.725
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Since the goal for traders is ultimately to determine whether the expected return or Sharpe ratio of a mean-reverting trading strategy is good enough, why do we bother to go through the stationarity tests (ADF or Variance Ratio) and the calculation of half-life at all? Can’t we just run a backtest on the trading strategy directly and be done with it? The reason why we went through all these preliminary tests is that their statistical signifi cance is usually higher than a direct backtest of a trading strategy. These preliminary tests make use of every day’s (or, more generally, every bar’s) price data for the test, while a backtest usually generates a signifi cantly smaller number of round trip trades for us to collect performance statistics. Furthermore, the outcome of a backtest is dependent on the specifi cs of a trading strategy, with a specifi c set of trading parameters. However, given a price series that passed the stationarity statistical tests, or at least one with a short enough half-life, we can be assured that we can eventually fi nd a profi table trading strategy, maybe just not the one that we have backtested."
+ }
+ ]
+ },
+ {
+ "type": "paragraph_title",
+ "index": 4,
+ "bbox": [
+ 0.12,
+ 0.757,
+ 0.387,
+ 0.78
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "■ Cointegration"
+ }
+ ],
+ "level": 2
+ },
+ {
+ "type": "text",
+ "index": 5,
+ "bbox": [
+ 0.117,
+ 0.799,
+ 0.882,
+ 0.909
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "As we stated in the introduction of this chapter, most fi nancial price series are not stationary or mean reverting. But, fortunately, we are not confi ned to trading those “prefabricated” fi nancial price series: We can proactively cre ate a portfolio of individual price series so that the market value (or price) series of this portfolio is stationary. This is the notion of cointegration: If we"
+ }
+ ]
+ }
+ ]
+ }
+ ],
+ "is_full_document": true,
+ "schema": "docvortex.middle",
+ "schema_version": "2.0"
+}
\ No newline at end of file
diff --git a/tests/fixtures/pdf_text_5635_5636/before_model.json b/tests/fixtures/pdf_text_5635_5636/before_model.json
new file mode 100644
index 00000000..a68e8298
--- /dev/null
+++ b/tests/fixtures/pdf_text_5635_5636/before_model.json
@@ -0,0 +1,1090 @@
+{
+ "metadata": {
+ "file_suffix": "pdf",
+ "producer": {
+ "name": "mineru",
+ "version": "4.0.12"
+ },
+ "document": {
+ "producer_application": "pypdf",
+ "page_count": 3,
+ "page_count_kind": "physical"
+ }
+ },
+ "extensions": {
+ "mineru": {
+ "tier": "standard",
+ "parse_mode": "txt"
+ },
+ "docvortex_layout": {
+ "version": 1,
+ "pages": [
+ {
+ "page_idx": 0,
+ "width_pt": 432.0,
+ "height_pt": 648.0
+ },
+ {
+ "page_idx": 1,
+ "width_pt": 432.0,
+ "height_pt": 648.0
+ },
+ {
+ "page_idx": 2,
+ "width_pt": 432.0,
+ "height_pt": 648.0
+ }
+ ]
+ }
+ },
+ "pages": [
+ [
+ {
+ "type": "header",
+ "bbox": [
+ 0.424,
+ 0.166,
+ 0.577,
+ 0.186
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "P R E FA C E"
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.116,
+ 0.367,
+ 0.881,
+ 0.498
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "his book is a practical guide to algorithmic trading strategies that can be readily implemented by both retail and institutional traders. It is not an academic treatise on fi nancial theory. Rather, I hope to make accessible to the reader some of the most useful fi nancial research done in the past few decades, mixing them with insights I gained from actually exploiting some of those theories in live trading."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.164,
+ 0.37,
+ 0.874,
+ 0.388
+ ]
+ },
+ {
+ "bbox": [
+ 0.165,
+ 0.391,
+ 0.876,
+ 0.41
+ ]
+ },
+ {
+ "bbox": [
+ 0.126,
+ 0.414,
+ 0.873,
+ 0.43
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.436,
+ 0.873,
+ 0.452
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.457,
+ 0.874,
+ 0.475
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.478,
+ 0.438,
+ 0.496
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.498,
+ 0.881,
+ 0.65
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Because strategies take a central place in this book, we will cover a wide array of them, broadly divided into the mean-reverting and momentum camps, and we will lay out standard techniques for trading each category of strategies, and equally important, the fundamental reasons why a strategy should work. The emphasis throughout is on simple and linear strategies, as an antidote to the overfi tting and data-snooping biases that often plague complex strategies."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.15,
+ 0.499,
+ 0.874,
+ 0.516
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.523,
+ 0.875,
+ 0.538
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.544,
+ 0.874,
+ 0.561
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.564,
+ 0.873,
+ 0.584
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.587,
+ 0.873,
+ 0.604
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.61,
+ 0.873,
+ 0.626
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.632,
+ 0.318,
+ 0.648
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.649,
+ 0.88,
+ 0.822
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "In the mean-reverting camp, we will discuss the multiple statistical tech niques (augmented Dickey-Fuller [ADF] test, Hurst exponent, Variance Ratio test, half-life) for detecting “time series” mean reversion or stationarity, and for detecting cointegration of a portfolio of instruments (cointegrated augmented Dickey Fuller [CADF] test, Johansen test). Beyond the mechanical application of these statistical tests to time series, we strive to convey an intuitive understanding of what they are really testing and the simple mathematical equations behind them."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.152,
+ 0.652,
+ 0.87,
+ 0.668
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.673,
+ 0.873,
+ 0.689
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.694,
+ 0.873,
+ 0.712
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.716,
+ 0.874,
+ 0.733
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.739,
+ 0.871,
+ 0.754
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.76,
+ 0.874,
+ 0.776
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.782,
+ 0.873,
+ 0.798
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.805,
+ 0.445,
+ 0.818
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.822,
+ 0.881,
+ 0.909
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "We will explain the simplest techniques and strategies for trading meanreverting portfolios (linear, Bollinger band, Kalman fi lter), and whether us ing raw prices, log prices, or ratios make the most sense as inputs to these tests and strategies. In particular, we show that the Kalman fi lter is useful"
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.152,
+ 0.824,
+ 0.873,
+ 0.842
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.848,
+ 0.87,
+ 0.861
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.869,
+ 0.873,
+ 0.884
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.89,
+ 0.875,
+ 0.906
+ ]
+ }
+ ]
+ },
+ {
+ "type": "aside_text",
+ "bbox": [
+ 0.928,
+ 0.486,
+ 0.962,
+ 0.504
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "ix"
+ }
+ ]
+ }
+ ],
+ [
+ {
+ "type": "aside_text",
+ "bbox": [
+ 0.043,
+ 0.488,
+ 0.068,
+ 0.571
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "PREFACE | X"
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.067,
+ 0.88,
+ 0.153
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "to traders in multiple ways and in multiple strategies. Distinction between time series versus cross-sectional mean reversion will be made. We will debate the pros and cons of “scaling-in” and highlight the danger of data errors in mean-reverting strategies, especially those that deal with spreads."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.125,
+ 0.069,
+ 0.874,
+ 0.084
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.092,
+ 0.871,
+ 0.105
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.113,
+ 0.875,
+ 0.128
+ ]
+ },
+ {
+ "bbox": [
+ 0.121,
+ 0.134,
+ 0.799,
+ 0.15
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.153,
+ 0.882,
+ 0.564
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Examples of mean-reverting strategies will be drawn from interday and intraday stocks models, exchange-traded fund (ETF) pairs and triplets, ETFs versus their component stocks, currency pairs, and futures calendar and intermarket spreads. We will explain what makes trading some of these strategies quite challenging in recent years due to the rise of dark pools and high-frequency trading. We will also illustrate how certain fundamental considerations can explain the temporary unhinging of a hitherto very profi table ETF pair and how the same considerations can lead one to construct an improved version of the strategy. When discussing currency trading, we take care to explain why even the calculation of returns may seem foreign to an equity trader, and where such concepts as rollover interest may sometimes be important. Much emphasis will be devoted to the study of spot returns versus roll returns in futures, and several futures trading strategies can be derived or understood from a simple mathematical model of futures prices. The concepts of backwardation and contango will be illustrated graphically as well as mathematically. The chapter on mean reversion of currencies and futures cumulates in the study of a very special future: the volatility (VX) future, and how it can form the basis of some quite lucrative strategies."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.151,
+ 0.154,
+ 0.876,
+ 0.172
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.177,
+ 0.871,
+ 0.193
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.198,
+ 0.873,
+ 0.215
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.219,
+ 0.876,
+ 0.236
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.241,
+ 0.876,
+ 0.258
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.264,
+ 0.871,
+ 0.278
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.286,
+ 0.873,
+ 0.301
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.307,
+ 0.875,
+ 0.323
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.328,
+ 0.874,
+ 0.345
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.35,
+ 0.874,
+ 0.367
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.373,
+ 0.871,
+ 0.387
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.394,
+ 0.871,
+ 0.407
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.415,
+ 0.872,
+ 0.429
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.437,
+ 0.875,
+ 0.452
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.458,
+ 0.875,
+ 0.474
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.478,
+ 0.875,
+ 0.497
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.502,
+ 0.872,
+ 0.516
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.524,
+ 0.873,
+ 0.538
+ ]
+ },
+ {
+ "bbox": [
+ 0.127,
+ 0.547,
+ 0.378,
+ 0.561
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.563,
+ 0.882,
+ 0.822
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "In the momentum camp, we start by explaining a few statistical tests for times series momentum. The main theme, though, is to explore the four main drivers of momentum in stocks and futures and to propose strategies that can extract time series and cross-sectional momentum. Roll returns in futures is one of those drivers, but it turns out that forced asset sales and purchases is the main driver of stock and ETF momentum in many diverse circumstances. Some of the newer momentum strategies based on news events, news sentiment, leveraged ETFs, order fl ow, and high-frequency trading will be covered. Finally, we will look at the pros and cons of mo mentum versus mean-reverting strategies and discover their diametrically diff erent risk-return characteristics under diff erent market regimes in recent fi nancial history."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.151,
+ 0.566,
+ 0.874,
+ 0.582
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.588,
+ 0.873,
+ 0.604
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.608,
+ 0.874,
+ 0.626
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.632,
+ 0.874,
+ 0.646
+ ]
+ },
+ {
+ "bbox": [
+ 0.127,
+ 0.653,
+ 0.875,
+ 0.667
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.674,
+ 0.875,
+ 0.691
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.697,
+ 0.875,
+ 0.711
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.717,
+ 0.873,
+ 0.734
+ ]
+ },
+ {
+ "bbox": [
+ 0.127,
+ 0.739,
+ 0.869,
+ 0.753
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.761,
+ 0.871,
+ 0.777
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.781,
+ 0.873,
+ 0.798
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.806,
+ 0.333,
+ 0.819
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.822,
+ 0.882,
+ 0.911
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "I have always maintained that it is easy to fi nd published, supposedly profi table, strategies in the many books, magazines, or blogs out there, but much harder to see why they may be fl awed and perhaps ultimately doomed. So, despite the emphasis on suggesting prototype strategies, we"
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.15,
+ 0.823,
+ 0.871,
+ 0.842
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.847,
+ 0.872,
+ 0.863
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.868,
+ 0.872,
+ 0.886
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.889,
+ 0.875,
+ 0.908
+ ]
+ }
+ ]
+ }
+ ],
+ [
+ {
+ "type": "aside_text",
+ "bbox": [
+ 0.042,
+ 0.485,
+ 0.07,
+ 0.671
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "50 ALGORITHMIC TRADING"
+ }
+ ]
+ },
+ {
+ "type": "header",
+ "bbox": [
+ 0.371,
+ 0.073,
+ 0.655,
+ 0.094
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Example 2.5 (Continued)"
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.145,
+ 0.116,
+ 0.843,
+ 0.334
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "the use of in-sample data to fi nd the half-life and therefore the look back. Furthermore, an unlimited amount of capital may be needed to generate the P&L because there was no maximum imposed on the market value of the portfolio. So I certainly don’t recommend it as a practical trading strategy. (There is a more practical version of this mean-reverting strategy in Chapter 5.) But it does illustrate that a nonstationary price series need not discourage us from trading a mean reversion strategy, and that we don’t need very complicated strategies or technical indicators to extract profi ts from a meanreverting series."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.152,
+ 0.118,
+ 0.823,
+ 0.134
+ ]
+ },
+ {
+ "bbox": [
+ 0.152,
+ 0.141,
+ 0.816,
+ 0.156
+ ]
+ },
+ {
+ "bbox": [
+ 0.151,
+ 0.162,
+ 0.802,
+ 0.178
+ ]
+ },
+ {
+ "bbox": [
+ 0.153,
+ 0.184,
+ 0.807,
+ 0.197
+ ]
+ },
+ {
+ "bbox": [
+ 0.151,
+ 0.206,
+ 0.808,
+ 0.221
+ ]
+ },
+ {
+ "bbox": [
+ 0.152,
+ 0.227,
+ 0.815,
+ 0.243
+ ]
+ },
+ {
+ "bbox": [
+ 0.151,
+ 0.247,
+ 0.838,
+ 0.266
+ ]
+ },
+ {
+ "bbox": [
+ 0.152,
+ 0.271,
+ 0.824,
+ 0.285
+ ]
+ },
+ {
+ "bbox": [
+ 0.151,
+ 0.293,
+ 0.79,
+ 0.307
+ ]
+ },
+ {
+ "bbox": [
+ 0.152,
+ 0.316,
+ 0.312,
+ 0.329
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.398,
+ 0.882,
+ 0.725
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "Since the goal for traders is ultimately to determine whether the expected return or Sharpe ratio of a mean-reverting trading strategy is good enough, why do we bother to go through the stationarity tests (ADF or Variance Ratio) and the calculation of half-life at all? Can’t we just run a backtest on the trading strategy directly and be done with it? The reason why we went through all these preliminary tests is that their statistical signifi cance is usually higher than a direct backtest of a trading strategy. These preliminary tests make use of every day’s (or, more generally, every bar’s) price data for the test, while a backtest usually generates a signifi cantly smaller number of round trip trades for us to collect performance statistics. Furthermore, the outcome of a backtest is dependent on the specifi cs of a trading strategy, with a specifi c set of trading parameters. However, given a price series that passed the stationarity statistical tests, or at least one with a short enough half-life, we can be assured that we can eventually fi nd a profi table trading strategy, maybe just not the one that we have backtested."
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.152,
+ 0.401,
+ 0.871,
+ 0.417
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.423,
+ 0.875,
+ 0.439
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.445,
+ 0.872,
+ 0.459
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.466,
+ 0.873,
+ 0.481
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.488,
+ 0.874,
+ 0.503
+ ]
+ },
+ {
+ "bbox": [
+ 0.127,
+ 0.511,
+ 0.874,
+ 0.525
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.532,
+ 0.873,
+ 0.548
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.553,
+ 0.874,
+ 0.568
+ ]
+ },
+ {
+ "bbox": [
+ 0.125,
+ 0.573,
+ 0.873,
+ 0.589
+ ]
+ },
+ {
+ "bbox": [
+ 0.127,
+ 0.596,
+ 0.872,
+ 0.611
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.616,
+ 0.874,
+ 0.635
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.638,
+ 0.875,
+ 0.656
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.66,
+ 0.874,
+ 0.677
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.681,
+ 0.875,
+ 0.7
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.706,
+ 0.689,
+ 0.719
+ ]
+ }
+ ]
+ },
+ {
+ "type": "paragraph_title",
+ "bbox": [
+ 0.12,
+ 0.757,
+ 0.387,
+ 0.78
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "■ Cointegration"
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.127,
+ 0.756,
+ 0.384,
+ 0.78
+ ]
+ }
+ ]
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.799,
+ 0.882,
+ 0.909
+ ],
+ "content": [
+ {
+ "type": "text",
+ "content": "As we stated in the introduction of this chapter, most fi nancial price series are not stationary or mean reverting. But, fortunately, we are not confi ned to trading those “prefabricated” fi nancial price series: We can proactively cre ate a portfolio of individual price series so that the market value (or price) series of this portfolio is stationary. This is the notion of cointegration: If we"
+ }
+ ],
+ "lines": [
+ {
+ "bbox": [
+ 0.125,
+ 0.802,
+ 0.875,
+ 0.818
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.826,
+ 0.874,
+ 0.839
+ ]
+ },
+ {
+ "bbox": [
+ 0.124,
+ 0.845,
+ 0.87,
+ 0.862
+ ]
+ },
+ {
+ "bbox": [
+ 0.122,
+ 0.867,
+ 0.873,
+ 0.884
+ ]
+ },
+ {
+ "bbox": [
+ 0.123,
+ 0.889,
+ 0.874,
+ 0.905
+ ]
+ }
+ ]
+ }
+ ]
+ ],
+ "page_index_map": [],
+ "schema": "docvortex.model",
+ "schema_version": "2.0"
+}
\ No newline at end of file
diff --git a/tests/fixtures/pdf_text_5635_5636/layout.json b/tests/fixtures/pdf_text_5635_5636/layout.json
new file mode 100644
index 00000000..72cb1c2f
--- /dev/null
+++ b/tests/fixtures/pdf_text_5635_5636/layout.json
@@ -0,0 +1,195 @@
+[
+ [
+ {
+ "type": "header",
+ "bbox": [
+ 0.424,
+ 0.166,
+ 0.577,
+ 0.186
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.116,
+ 0.367,
+ 0.881,
+ 0.498
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.498,
+ 0.881,
+ 0.65
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.649,
+ 0.88,
+ 0.822
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.822,
+ 0.881,
+ 0.909
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "aside_text",
+ "bbox": [
+ 0.928,
+ 0.486,
+ 0.962,
+ 0.504
+ ],
+ "angle": 0,
+ "content": "ix"
+ }
+ ],
+ [
+ {
+ "type": "aside_text",
+ "bbox": [
+ 0.043,
+ 0.488,
+ 0.068,
+ 0.571
+ ],
+ "angle": 0,
+ "content": "PREFACE | X"
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.067,
+ 0.88,
+ 0.153
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.153,
+ 0.882,
+ 0.564
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.563,
+ 0.882,
+ 0.822
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.822,
+ 0.882,
+ 0.911
+ ],
+ "angle": 0,
+ "content": null
+ }
+ ],
+ [
+ {
+ "type": "aside_text",
+ "bbox": [
+ 0.042,
+ 0.485,
+ 0.07,
+ 0.671
+ ],
+ "angle": 0,
+ "content": "50 ALGORITHMIC TRADING"
+ },
+ {
+ "type": "header",
+ "bbox": [
+ 0.371,
+ 0.073,
+ 0.655,
+ 0.094
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.145,
+ 0.116,
+ 0.843,
+ 0.334
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.398,
+ 0.882,
+ 0.725
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "paragraph_title",
+ "bbox": [
+ 0.12,
+ 0.757,
+ 0.387,
+ 0.78
+ ],
+ "angle": 0,
+ "content": null
+ },
+ {
+ "type": "text",
+ "bbox": [
+ 0.117,
+ 0.799,
+ 0.882,
+ 0.909
+ ],
+ "angle": 0,
+ "content": null
+ }
+ ]
+]
\ No newline at end of file
diff --git a/tests/fixtures/pdf_text_5635_5636/ocr_detection.json b/tests/fixtures/pdf_text_5635_5636/ocr_detection.json
new file mode 100644
index 00000000..fca6a61c
--- /dev/null
+++ b/tests/fixtures/pdf_text_5635_5636/ocr_detection.json
@@ -0,0 +1,1086 @@
+[
+ [
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 508,
+ 300,
+ 691,
+ 333
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 197,
+ 666,
+ 1049,
+ 698
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 198,
+ 703,
+ 1051,
+ 738
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 151,
+ 745,
+ 1047,
+ 774
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 784,
+ 1047,
+ 814
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 822,
+ 1049,
+ 855
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 861,
+ 525,
+ 893
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 180,
+ 899,
+ 1049,
+ 929
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 941,
+ 1050,
+ 969
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 980,
+ 1049,
+ 1009
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 146,
+ 1016,
+ 1047,
+ 1051
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1056,
+ 1048,
+ 1087
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1098,
+ 1048,
+ 1126
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1137,
+ 382,
+ 1166
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 182,
+ 1174,
+ 1044,
+ 1203
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1212,
+ 1047,
+ 1240
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1250,
+ 1048,
+ 1282
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1289,
+ 1049,
+ 1319
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1331,
+ 1045,
+ 1357
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1368,
+ 1049,
+ 1397
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1408,
+ 1047,
+ 1437
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1449,
+ 534,
+ 1472
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 182,
+ 1483,
+ 1047,
+ 1515
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1526,
+ 1044,
+ 1550
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 1565,
+ 1048,
+ 1592
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 146,
+ 1602,
+ 1050,
+ 1630
+ ],
+ "score": 1.0,
+ "text": ""
+ }
+ ],
+ [
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 125,
+ 1049,
+ 152
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 165,
+ 1045,
+ 189
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 203,
+ 1050,
+ 230
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 145,
+ 242,
+ 959,
+ 270
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 181,
+ 277,
+ 1051,
+ 309
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 318,
+ 1045,
+ 347
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 356,
+ 1047,
+ 387
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 395,
+ 1051,
+ 425
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 434,
+ 1051,
+ 464
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 476,
+ 1045,
+ 501
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 514,
+ 1048,
+ 542
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 552,
+ 1050,
+ 581
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 591,
+ 1049,
+ 621
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 630,
+ 1049,
+ 660
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 671,
+ 1045,
+ 696
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 710,
+ 1045,
+ 733
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 747,
+ 1046,
+ 773
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 786,
+ 1050,
+ 813
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 825,
+ 1050,
+ 853
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 861,
+ 1050,
+ 894
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 903,
+ 1046,
+ 928
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 943,
+ 1048,
+ 968
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 152,
+ 984,
+ 454,
+ 1009
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 181,
+ 1019,
+ 1049,
+ 1047
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1059,
+ 1048,
+ 1087
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 1094,
+ 1049,
+ 1126
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1137,
+ 1049,
+ 1162
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 152,
+ 1175,
+ 1050,
+ 1200
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 1213,
+ 1050,
+ 1243
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1254,
+ 1050,
+ 1279
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 146,
+ 1290,
+ 1048,
+ 1322
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 152,
+ 1331,
+ 1043,
+ 1356
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1370,
+ 1045,
+ 1398
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1406,
+ 1047,
+ 1437
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1450,
+ 400,
+ 1475
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 180,
+ 1482,
+ 1045,
+ 1515
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1525,
+ 1046,
+ 1553
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1563,
+ 1046,
+ 1594
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1601,
+ 1050,
+ 1635
+ ],
+ "score": 1.0,
+ "text": ""
+ }
+ ],
+ [
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 446,
+ 132,
+ 782,
+ 168
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 183,
+ 213,
+ 988,
+ 241
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 183,
+ 254,
+ 979,
+ 281
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 181,
+ 291,
+ 963,
+ 320
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 184,
+ 331,
+ 968,
+ 355
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 181,
+ 371,
+ 970,
+ 398
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 182,
+ 408,
+ 978,
+ 438
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 181,
+ 445,
+ 1006,
+ 478
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 182,
+ 488,
+ 989,
+ 513
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 181,
+ 527,
+ 948,
+ 552
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 182,
+ 568,
+ 375,
+ 592
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 182,
+ 721,
+ 1045,
+ 750
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 762,
+ 1050,
+ 791
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 801,
+ 1046,
+ 827
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 839,
+ 1048,
+ 865
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 878,
+ 1049,
+ 906
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 152,
+ 919,
+ 1049,
+ 945
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 957,
+ 1048,
+ 986
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 995,
+ 1049,
+ 1023
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1032,
+ 1048,
+ 1061
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 152,
+ 1073,
+ 1046,
+ 1099
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1108,
+ 1049,
+ 1143
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1149,
+ 1050,
+ 1180
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 1188,
+ 1049,
+ 1219
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 146,
+ 1225,
+ 1050,
+ 1260
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 1270,
+ 827,
+ 1294
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 152,
+ 1370,
+ 176,
+ 1394
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 191,
+ 1360,
+ 461,
+ 1404
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 150,
+ 1444,
+ 1050,
+ 1472
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 1486,
+ 1049,
+ 1510
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 149,
+ 1521,
+ 1044,
+ 1551
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 147,
+ 1560,
+ 1048,
+ 1592
+ ],
+ "score": 1.0,
+ "text": ""
+ },
+ {
+ "label": "ocr_text",
+ "bbox": [
+ 148,
+ 1601,
+ 1049,
+ 1629
+ ],
+ "score": 1.0,
+ "text": ""
+ }
+ ]
+]
\ No newline at end of file
diff --git a/tests/fixtures/pdf_text_5635_5636/source.json b/tests/fixtures/pdf_text_5635_5636/source.json
new file mode 100644
index 00000000..82570511
--- /dev/null
+++ b/tests/fixtures/pdf_text_5635_5636/source.json
@@ -0,0 +1,27 @@
+{
+ "issues": [
+ "https://github.com/opendatalab/MinerU/issues/5635",
+ "https://github.com/opendatalab/MinerU/issues/5636"
+ ],
+ "attachment_url": "https://github.com/user-attachments/files/33325966/bug_repro_sample_p7_p8_p63.pdf",
+ "sha256": "1a61d515a91fdda973a10c08e3a99083920235adafaa2659fc4ef42bad756620",
+ "original_pages": [
+ 7,
+ 8,
+ 63
+ ],
+ "render_scale": 2.7777777777777777,
+ "baseline": {
+ "mineru": "4.0.12",
+ "tier": "standard",
+ "ocr_mode": "auto",
+ "resolved_parse_mode": "txt",
+ "platform": "macOS arm64"
+ },
+ "failure_counts": {
+ "missing_initial_T": 1,
+ "fi nancial": 5,
+ "mo mentum": 1
+ },
+ "description": "三页原件、原始布局和 OCR 检测结果用于固定推理输入;before 文件记录修改前实际输出。"
+}
diff --git a/tests/fixtures/pdf_text_5635_5636/source.pdf b/tests/fixtures/pdf_text_5635_5636/source.pdf
new file mode 100644
index 00000000..5f50f769
Binary files /dev/null and b/tests/fixtures/pdf_text_5635_5636/source.pdf differ
diff --git a/tests/unittest/test_pdf_native_text_5635_5636.py b/tests/unittest/test_pdf_native_text_5635_5636.py
new file mode 100644
index 00000000..577c013e
--- /dev/null
+++ b/tests/unittest/test_pdf_native_text_5635_5636.py
@@ -0,0 +1,160 @@
+"""复放真实 layout/OCR 检测结果,隔离推理随机性验证原生文本修复。"""
+
+from copy import deepcopy
+import json
+from pathlib import Path
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+from docvortex.document.pdf import PDFDocument
+from docvortex.postprocess.document import model_json_to_middle_json
+from docvortex.schema import ModelJson
+
+from mineru.backend.analysis.pdf.normalization import _normalize_pdf_model_list
+from mineru.backend.analysis.pdf.text.content import _fill_window_block_content_and_lines
+from mineru.backend.analysis.pdf.constants import VLM_TXT_DET_TYPE
+from mineru.backend.analysis.pdf.snapshots import create_page_snapshot_cache
+from mineru.parser.base import ParseResult
+
+FIXTURE = Path(__file__).parents[1] / "fixtures/pdf_text_5635_5636"
+
+
[email protected]("effort", ["medium", "high", "xhigh"])
[email protected]("owned", [False, True])
+def test_frozen_native_text_pipeline_restores_source_words(effort: str, owned: bool) -> None:
+ """同一三页输入在回填、模型协议、中间协议和 Markdown 均保留完整词语。"""
+ layout = json.loads((FIXTURE / "layout.json").read_text())
+ detection = json.loads((FIXTURE / "ocr_detection.json").read_text())
+ images = []
+ context = MagicMock()
+ with PDFDocument((FIXTURE / "source.pdf").read_bytes()) as document:
+ try:
+ for index in range(document.page_count):
+ rendered = document.render_page(index, scale=200 / 72)
+ images.append({"img_pil": rendered.pil_image, "scale": 200 / 72})
+ pages = _fill_window_block_content_and_lines(
+ images,
+ [document[index] for index in range(document.page_count)],
+ layout,
+ [[], [], []],
+ detection,
+ "txt",
+ effort,
+ VLM_TXT_DET_TYPE,
+ context,
+ page_snapshots=create_page_snapshot_cache(
+ [document[index] for index in range(document.page_count)], enabled=owned
+ ),
+ )
+ _normalize_pdf_model_list(pages)
+ finally:
+ for image in images:
+ image["img_pil"].close()
+ context.ocr_model.ocr.assert_not_called()
+ model_dict = json.loads((FIXTURE / "before_model.json").read_text())
+ model_dict["pages"] = pages
+ model = ModelJson.from_dict(model_dict)
+ middle = model_json_to_middle_json(model)
+ markdown = ParseResult(middle, model).markdown()
+ for value in [json.dumps(model.to_dict(), ensure_ascii=False), json.dumps(middle.to_dict(), ensure_ascii=False), markdown]:
+ assert "This book is a practical" in value
+ assert value.count("financial") == 5
+ assert "fi nancial" not in value
+ assert "mo mentum" not in value
+ assert "tech niques" not in value
+ first = next(block for block in pages[0] if block["type"] == "text")
+ assert first["lines"][0]["bbox"][0] <= 54 / 432
+ assert first["lines"][0]["bbox"][3] >= 262.36 / 648
+
+
+def test_drop_cap_recovery_guards() -> None:
+ """不把正常字号、标题、别栏字符或已认领首字重复补入段落。"""
+ from mineru.backend.analysis.pdf.text.content import _recover_native_drop_caps
+ from mineru.backend.analysis.pdf.text.models import _AnalyzeLine, _AnalyzeSpan
+ from docvortex.document.pdf import PDFPageTextGeometry
+
+ chars: list[dict[str, Any]] = [
+ {"char": "T", "char_idx": 0, "bbox": (10.0, 10.0, 20.0, 32.0), "writing_angle": 0.0},
+ {"char": "h", "char_idx": 1, "bbox": (20.5, 10.0, 25.0, 18.0), "writing_angle": 0.0},
+ {"char": "i", "char_idx": 2, "bbox": (25.0, 10.0, 28.0, 18.0), "writing_angle": 0.0},
+ ]
+ for case in ["drop_cap", "ordinary", "title", "other_column", "already_present", "formula", "code", "owned_elsewhere"]:
+ current = deepcopy(chars)
+ content = "This" if case == "already_present" else "his"
+ bbox = (10.0, 10.0, 80.0, 60.0)
+ if case == "ordinary":
+ current[0]["bbox"] = (10.0, 10.0, 20.0, 18.0)
+ if case == "other_column":
+ current[0]["bbox"] = (0.0, 10.0, 5.0, 32.0)
+ geometry = PDFPageTextGeometry(chars=current, tight_bboxes={i: c["bbox"] for i, c in enumerate(current)}, origins={})
+ span = _AnalyzeSpan("text", (20.5, 10.0, 75.0, 18.0), content=content)
+ lines = {0: [_AnalyzeLine(span.bbox, [span])]}
+ kind = "paragraph_title" if case == "title" else "code" if case == "code" else "text"
+ blocks = [{"type": kind, "bbox": [v / 100 for v in bbox]}]
+ if case == "owned_elsewhere":
+ owned = _AnalyzeSpan("text", current[0]["bbox"], content="T")
+ lines[1] = [_AnalyzeLine(owned.bbox, [owned])]
+ blocks.append({"type": "paragraph_title", "bbox": [v / 100 for v in bbox]})
+ _recover_native_drop_caps(blocks, lines, (100.0, 100.0), geometry, [current[0]["bbox"]] if case == "formula" else [])
+ assert span.content == ("This" if case in {"drop_cap", "already_present"} else "his")
+ if case == "drop_cap":
+ _recover_native_drop_caps(blocks, lines, (100.0, 100.0), geometry, [])
+ assert span.content == "This"
+
+
[email protected]("tight_spacing", [False, True])
+def test_owned_ligature_text_matches_reference_and_preserves_code(tight_spacing: bool) -> None:
+ """真实快照的 Rust/Python 组装结果一致,代码区域保留源空格。"""
+ from dataclasses import asdict
+ from docvortex import _native
+ from mineru.backend.analysis.pdf.text.native import fill_char_in_spans
+ from mineru.backend.analysis.pdf.text.models import _AnalyzeSpan
+
+ with PDFDocument((FIXTURE / "source.pdf").read_bytes()) as document:
+ owner = document[0].get_text_snapshot()
+ if owner is None:
+ pytest.skip("Rust snapshot unavailable")
+ results = []
+ before = _native.text_snapshot_stats()[2]
+ for current_owner in [None, owner]:
+ geometry = owner.materialize_geometry()
+ span = _AnalyzeSpan(
+ "text",
+ (0.0, 0.0, 432.0, 648.0),
+ metadata={
+ "chars": [],
+ "height": 648.0,
+ "width": 432.0,
+ "_native_tight_spacing": tight_spacing,
+ },
+ )
+ pending = fill_char_in_spans(
+ [span],
+ geometry.chars,
+ 12.0,
+ tight_bboxes=geometry.tight_bboxes,
+ origins=geometry.origins,
+ detect_scripts=False,
+ _native_text=current_owner,
+ )
+ assert not pending
+ assert span.content.count("financial" if tight_spacing else "fi nancial") == 2
+ results.append(asdict(span))
+ assert results[0] == results[1]
+ assert _native.text_snapshot_stats()[2] == before + 1
+
+
[email protected]("text", ["a\ufffeb", "a\uffffb"])
+def test_noncharacters_are_not_unconditionally_rewritten(text: str) -> None:
+ """有原生字符归属的非字符保持原值,不把任意 U+FFFE/U+FFFF 当作断词符。"""
+ from mineru.backend.analysis.pdf.text.native import chars_to_content
+ from mineru.backend.analysis.pdf.text.models import _AnalyzeSpan
+
+ chars = [
+ {"char": value, "char_idx": index, "bbox": (index * 5.0, 0.0, index * 5.0 + 5.0, 10.0)}
+ for index, value in enumerate(text)
+ ]
+ span = _AnalyzeSpan("text", (0.0, 0.0, 20.0, 10.0), metadata={"chars": chars})
+ chars_to_content(span, detect_scripts=False)
+ assert span.content == text