Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

Merge dev into PR #1672 and preserve integrated behavior

Bingxi Zhao (Frank)bb70aa2253f4
diff --cc deeptutor/api/routers/settings.py
index 93f192389,0b08409ca..c15f3850a
--- a/deeptutor/api/routers/settings.py
+++ b/deeptutor/api/routers/settings.py
@@@ -1242,7 -1246,7 +1246,8 @@@ async def update_mineru_settings(payloa
"enable_formula": payload.enable_formula,
"enable_table": payload.enable_table,
"is_ocr": payload.is_ocr,
+ "max_pages_per_part": current.get("max_pages_per_part", 180),
+ "normalize_tiny_scans": current.get("normalize_tiny_scans", False),
"allow_local_model_download": payload.allow_local_model_download,
}
)
diff --cc deeptutor/services/config/runtime_settings.py
index 9cb46224c,6f4d3f26a..4e001bac6
--- a/deeptutor/services/config/runtime_settings.py
+++ b/deeptutor/services/config/runtime_settings.py
@@@ -185,7 -186,7 +186,8 @@@ _DEFAULT_MINERU_ENGINE: dict[str, Any]
"enable_formula": True,
"enable_table": True,
"is_ocr": False,
+ "max_pages_per_part": 180,
+ "normalize_tiny_scans": False,
"allow_local_model_download": False,
}
@@@ -1155,9 -1164,7 +1165,10 @@@ class RuntimeSettingsService
"enable_formula": _coerce_bool(settings.get("enable_formula"), True),
"enable_table": _coerce_bool(settings.get("enable_table"), True),
"is_ocr": _coerce_bool(settings.get("is_ocr"), False),
+ "max_pages_per_part": _coerce_clamped_int(
+ settings.get("max_pages_per_part"), 180, 1, 200
+ ),
+ "normalize_tiny_scans": _coerce_bool(settings.get("normalize_tiny_scans"), False),
"allow_local_model_download": _coerce_bool(
settings.get("allow_local_model_download"), False
),
diff --cc deeptutor/services/parsing/engines/mineru/config.py
index 389adc967,2c77b5e20..779e56272
--- a/deeptutor/services/parsing/engines/mineru/config.py
+++ b/deeptutor/services/parsing/engines/mineru/config.py
@@@ -111,12 -105,8 +122,13 @@@ def resolve_mineru_config() -> MinerUCo
enable_formula=bool(settings.get("enable_formula", True)),
enable_table=bool(settings.get("enable_table", True)),
is_ocr=bool(settings.get("is_ocr", False)),
+ normalize_tiny_scans=bool(settings.get("normalize_tiny_scans", False)),
allow_local_model_download=bool(settings.get("allow_local_model_download", False)),
+ max_pages_per_part=(
+ DEFAULT_MAX_PAGES_PER_PART
+ if settings.get("max_pages_per_part") is None
+ else int(settings.get("max_pages_per_part")) # type: ignore[arg-type]
+ ),
)
diff --cc deeptutor/services/parsing/engines/mineru/engine.py
index d6cd6f040,624618453..477aca2e4
--- a/deeptutor/services/parsing/engines/mineru/engine.py
+++ b/deeptutor/services/parsing/engines/mineru/engine.py
@@@ -48,14 -50,11 +50,19 @@@ class MinerUParser
"enable_formula": config.enable_formula,
"enable_table": config.enable_table,
"is_ocr": config.is_ocr,
+ **(
+ {
+ "segmentation": "source-pages-v1",
+ "max_pages_per_part": config.max_pages_per_part,
+ }
+ if config.is_cloud
+ else {}
+ ),
+ **(
+ {"normalization": POLICY}
+ if config.is_cloud and config.normalize_tiny_scans
+ else {}
+ ),
},
)
diff --cc docs-for-user/SOURCE_VISUALS.md
index 17482ed4d,10c2553a0..d74276874
--- a/docs-for-user/SOURCE_VISUALS.md
+++ b/docs-for-user/SOURCE_VISUALS.md
@@@ -27,31 -27,62 +27,91 @@@ that require OCR when no usable OCR eng
Other RAG providers do not yet index this visual manifest. Interactive
exercises, visual annotations, and mastery updates are separate future work.
+## Resuming MinerU cloud PDF slices
+
+Large cloud PDFs are sliced using `engines.mineru.max_pages_per_part` in
+Document Parsing settings (default 180; clamped to 1–200). This advanced setting
+is preserved when saving the legacy MinerU settings form. Local parsing and
+small/non-PDF inputs keep their existing behavior.
+
+Merged content-list `page_idx` values refer to the original PDF, including
+nested blocks. For example, page index 0 of the second 180-page slice becomes
+180. Invalid local indices fail the parse instead of silently mislabeling
+source pages. Other per-part diagnostic artifacts retain their original local
+numbering and `partNN_` filename prefix.
+
+Completed slice archives are saved under the active workspace's parse cache in
+`.mineru-segments/`, outside disposable failed-parse directories. Retrying the
+same source and parser settings reuses completed slices; a changed document,
+endpoint, model, language, OCR/formula/table setting, or slice size starts a new
+checkpoint set. API tokens and signed URLs are never written to checkpoints.
+Each archive is SHA-256 checked before reuse; an incomplete or corrupt checkpoint
+is downloaded again. Concurrent jobs may still duplicate a cloud request, but
+checkpoint writes are atomic. Cache write failures do not fail a successful parse.
+
+These archives contain parsed document content, images, and any source copies
+returned by MinerU, just like the normal parse cache. They are retained until the workspace cache is cleared; deleting
+`.mineru-segments/` while no parse is running only discards resumable progress.
+This change versions the cloud parser signature so older merged page indices
+are not reused from the normal parse cache. Existing knowledge-base indexes
+need an explicit rebuild to consume corrected pages.
++
+ ## Tiny scanned PDF pages with MinerU
+
+ Some scanned PDFs encode a full-resolution page in an unusually small physical
+ page box. MinerU's official cloud backend can optionally enlarge these pages in
+ a temporary upload copy. The original file, page order, compressed image bytes,
+ and soft masks are preserved; language, OCR, model, formula, and table settings
+ remain as configured. This is a geometry workaround, not a guarantee of better
+ OCR or preservation of arbitrary interactive PDF semantics.
+
+ The option defaults to off. An administrator can enable it through the existing
+ `PUT /api/settings/document-parsing` endpoint with this partial payload:
+
+ ```json
+ {"engines": {"mineru": {"normalize_tiny_scans": true}}}
+ ```
+
+ Use `false` to disable it. The option also lives at
+ `engines.mineru.normalize_tiny_scans` in `document_parsing.json`. Enabling it
+ changes the cloud parse-cache signature; old parses remain intact. Existing
+ indexes are not rebuilt automatically. The optional
+ `deeptutor[parse-pymupdf4llm]` extra provides the required PyMuPDF dependency.
+
+ Only text-free pages below 144 points on their longest side, with a
+ high-resolution image covering at least 80% of the page at an apparent density
+ of at least 1200 DPI, qualify. The longest side is scaled to 768 points.
+ Rotated, cropped, annotated, vector-bearing, or non-default UserUnit pages are
+ left unchanged. Local MinerU, custom cloud endpoints, normal PDFs, and non-PDF
+ inputs keep their existing behavior. Temporary copies are removed after success
+ or failure, and upload is refused if compressed image streams change.
+
+ ## Optional image-description batches
+
+ The existing LlamaIndex image-description pass can send multiple images per
+ vision request. Set `image_description_batch_size` through
+ `PUT /api/knowledge-bases/rag-pipelines/llamaindex/config`, for example:
+
+ ```json
+ {"image_description_batch_size": 4}
+ ```
+
+ The default is `1`, preserving individual requests; accepted values are clamped
+ to 1–8. Concurrency limits count batches when enabled, and the existing timeout
+ covers the whole batch including any split attempts. Returned captions are
+ matched by explicit IDs, never by response order. Malformed JSON/ID maps and
+ explicit context overflow split into smaller groups, eventually using the
+ existing single-image prompt. A group of N images makes at most 2N−1 completion
+ calls, with provider retries disabled for this mode. Authentication and rate
+ limits stop queued groups in the job; other API/transport errors do not split.
+
+ This only affects subsequently processed images in the existing LlamaIndex
+ description pass. It does not enable descriptions for structured source visuals
+ or alter reading-material captions. Batches use a different structured prompt
+ and cache complete, successful groups separately from independent single-image
+ captions. The digest includes ordered image contents and metadata, prompts,
+ model identity, and retry policy. Failed, incomplete, or canceled groups are
+ not cached; successful split groups can be reused. Setting the size
+ back to `1` restores the normal single-image path. The model must support
+ multiple image blocks; unsupported API responses are reported without a burst
+ of fallback requests.
diff --cc tests/services/config/test_settings_profile.py
index 6647a51d0,df68f726f..d04f5bbcc
--- a/tests/services/config/test_settings_profile.py
+++ b/tests/services/config/test_settings_profile.py
@@@ -116,7 -116,7 +116,8 @@@ def test_export_is_value_free_and_prese
"enable_formula": True,
"enable_table": True,
"is_ocr": False,
+ "max_pages_per_part": 180,
+ "normalize_tiny_scans": False,
"allow_local_model_download": False,
}
assert profile["settings"]["document_parsing"]["engines"]["tika"] == {