Skip to content

Agent Zero AI framework

Tune LiteParse OCR defaults

Alessandro9e4b2f1843b2
diff --git a/plugins/_document_query/README.md b/plugins/_document_query/README.md
index 62655189c..3defbd250 100644
--- a/plugins/_document_query/README.md
+++ b/plugins/_document_query/README.md
@@ -8,6 +8,7 @@ timeouts and thread-safe parsers.
- **Strategy-pattern parsers** - MIME-type routing to dedicated parser classes
- **Centralized fetching** - local and HTTP(S) resources are fetched once, size-checked, then passed to parsers
- **LiteParse first path** - fast local parsing for PDFs and supported document/image formats, with legacy fallbacks
+- **Adaptive OCR** - large text-rich PDFs skip OCR automatically to avoid pathological parse times
- **Bounded parser execution** - sync parsers are offloaded to asyncio.to_thread and globally capped across chats
- **Configurable timeouts** - per-document and gather-level timeouts
- **Expanded format support** - PDF, HTML, text, YAML, XML, TOML, JS, TS, images, and catch-all Unstructured
@@ -29,14 +30,17 @@ See default_config.yaml for all options. Key settings:
| chunk_overlap | 100 | Text splitter overlap |
| search_threshold | 0.5 | Similarity search threshold |
| liteparse_enabled | true | Prefer LiteParse before legacy parser fallbacks |
-| liteparse_num_workers | 1 | Max LiteParse OCR workers per parser job |
-| liteparse_subprocess | true | Run LiteParse in a child process so native crashes fall back safely |
+| liteparse_num_workers | 2 | Max LiteParse OCR workers per parser job |
+| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for text-rich PDFs at or above this effective page count |
| thread_offload | true | Offload sync parsers to thread pool |
LiteParse is installed into the Agent Zero framework runtime from hooks.py during
plugin install/startup. If installation fails, the plugin logs the error and
continues with the legacy parser fallbacks.
+LiteParse always runs in a child process so native parser and OCR failures stay
+isolated from the Web UI process.
+
## Parsers
| Parser | MIME Types | Backend |
diff --git a/plugins/_document_query/default_config.yaml b/plugins/_document_query/default_config.yaml
index a850016cc..ddef41b90 100644
--- a/plugins/_document_query/default_config.yaml
+++ b/plugins/_document_query/default_config.yaml
@@ -28,7 +28,10 @@ liteparse_target_pages:
liteparse_dpi: 150
liteparse_preserve_very_small_text: false
liteparse_output_format: text
-liteparse_num_workers: 1 # LiteParse defaults to CPU cores - 1; cap it for web runtime stability
-liteparse_subprocess: true # isolate LiteParse native runtime crashes from the Web UI process
+liteparse_num_workers: 2 # balanced default for OCR speed without overloading shared Web UI runtime
+liteparse_ocr_auto_disable: true # disable OCR automatically for large text-rich PDFs
+liteparse_ocr_auto_disable_pages: 30 # OCR-on runtime climbs sharply around this page count
+liteparse_ocr_auto_min_chars_per_page: 80
+liteparse_ocr_auto_sample_pages: 5
pdf_ocr_fallback: true # enable legacy Tesseract fallback after PyMuPDF
thread_offload: true # offload sync parsers to thread pool
diff --git a/plugins/_document_query/helpers/parsers/liteparse.py b/plugins/_document_query/helpers/parsers/liteparse.py
index 110b10f4d..ce6f06523 100644
--- a/plugins/_document_query/helpers/parsers/liteparse.py
+++ b/plugins/_document_query/helpers/parsers/liteparse.py
@@ -6,16 +6,29 @@ import json
import os
import subprocess
import sys
+from dataclasses import dataclass
from pathlib import Path
from plugins._document_query.helpers.fetch import FetchedDocument
from .base import BaseParser
+DEFAULT_OCR_AUTO_DISABLE_PAGES = 30
+DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE = 80
+DEFAULT_OCR_AUTO_SAMPLE_PAGES = 5
+
+
+@dataclass(frozen=True)
+class _PdfTextProfile:
+ page_count: int
+ sampled_pages: int
+ text_chars: int
+
+
class LiteParseParser(BaseParser):
"""Fast parser powered by run-llama/liteparse when available."""
- DEFAULT_NUM_WORKERS = 1
+ DEFAULT_NUM_WORKERS = 2
mimetypes = [
"application/pdf",
@@ -35,9 +48,8 @@ class LiteParseParser(BaseParser):
return bool(config.get("liteparse_enabled", True))
def _parse_sync(self, document: FetchedDocument, config: dict) -> str:
- if config.get("liteparse_subprocess", True):
- return self._parse_subprocess(document, config)
- return self._parse_in_process(document, config)
+ # Keep LiteParse native/OCR failures isolated from the Web UI process.
+ return self._parse_subprocess(document, config)
def _parse_in_process(self, document: FetchedDocument, config: dict) -> str:
try:
@@ -58,7 +70,7 @@ class LiteParseParser(BaseParser):
with document.local_file() as file_path:
payload = {
"file_path": file_path,
- "kwargs": self._liteparse_kwargs(config),
+ "kwargs": self._liteparse_kwargs(config, document, file_path),
}
env = os.environ.copy()
project_root = str(Path(__file__).resolve().parents[4])
@@ -104,9 +116,18 @@ class LiteParseParser(BaseParser):
raise ValueError("LiteParse returned no text")
return text
- def _liteparse_kwargs(self, config: dict) -> dict:
+ def _liteparse_kwargs(
+ self,
+ config: dict,
+ document: FetchedDocument | None = None,
+ file_path: str | None = None,
+ ) -> dict:
+ ocr_enabled = bool(config.get("liteparse_ocr_enabled", True))
+ if ocr_enabled and self._should_disable_ocr(config, document, file_path):
+ ocr_enabled = False
+
kwargs = {
- "ocr_enabled": bool(config.get("liteparse_ocr_enabled", True)),
+ "ocr_enabled": ocr_enabled,
"ocr_language": config.get("liteparse_ocr_language", "eng"),
"max_pages": int(config.get("liteparse_max_pages", 1000)),
"dpi": float(config.get("liteparse_dpi", 150)),
@@ -138,6 +159,149 @@ class LiteParseParser(BaseParser):
return kwargs
+ def _should_disable_ocr(
+ self,
+ config: dict,
+ document: FetchedDocument | None,
+ file_path: str | None,
+ ) -> bool:
+ if not bool(config.get("liteparse_ocr_auto_disable", True)):
+ return False
+ if not document or document.mimetype != "application/pdf" or not file_path:
+ return False
+
+ profile = _pdf_text_profile(file_path, config)
+ if not profile or profile.sampled_pages <= 0:
+ return False
+
+ effective_pages = _effective_page_budget(config, profile.page_count)
+ auto_disable_pages = _positive_int(
+ config.get("liteparse_ocr_auto_disable_pages"),
+ DEFAULT_OCR_AUTO_DISABLE_PAGES,
+ )
+ if effective_pages < auto_disable_pages:
+ return False
+
+ min_chars_per_page = _positive_int(
+ config.get("liteparse_ocr_auto_min_chars_per_page"),
+ DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE,
+ )
+ return (profile.text_chars / profile.sampled_pages) >= min_chars_per_page
+
+
+def _pdf_text_profile(file_path: str, config: dict) -> _PdfTextProfile | None:
+ try:
+ import fitz
+ except Exception:
+ return None
+
+ try:
+ with fitz.open(file_path) as doc:
+ page_count = doc.page_count
+ if page_count <= 0:
+ return _PdfTextProfile(0, 0, 0)
+ sample_indexes = _sample_page_indexes(
+ page_count=page_count,
+ sample_pages=_positive_int(
+ config.get("liteparse_ocr_auto_sample_pages"),
+ DEFAULT_OCR_AUTO_SAMPLE_PAGES,
+ ),
+ target_pages=config.get("liteparse_target_pages"),
+ )
+ text_chars = 0
+ for page_index in sample_indexes:
+ text = doc[page_index].get_text("text") or ""
+ text_chars += len("".join(text.split()))
+ return _PdfTextProfile(
+ page_count=page_count,
+ sampled_pages=len(sample_indexes),
+ text_chars=text_chars,
+ )
+ except Exception:
+ return None
+
+
+def _effective_page_budget(config: dict, page_count: int) -> int:
+ target_pages = config.get("liteparse_target_pages")
+ if target_pages not in (None, ""):
+ parsed_target_count = _target_page_count(str(target_pages), page_count)
+ if parsed_target_count:
+ return parsed_target_count
+
+ max_pages = _positive_int(config.get("liteparse_max_pages"), 1000)
+ return min(max_pages, page_count)
+
+
+def _target_page_count(value: str, page_count: int) -> int:
+ pages = _target_page_numbers(value, page_count)
+ if pages is None:
+ return 0
+ return len(pages)
+
+
+def _target_page_numbers(value: str, page_count: int) -> set[int] | None:
+ pages: set[int] = set()
+ for raw_part in value.split(","):
+ part = raw_part.strip()
+ if not part:
+ continue
+ if "-" in part:
+ start_raw, end_raw = part.split("-", 1)
+ try:
+ start = int(start_raw.strip())
+ end = int(end_raw.strip())
+ except ValueError:
+ return None
+ if start <= 0 or end <= 0:
+ return None
+ if start > end:
+ start, end = end, start
+ pages.update(range(start, min(end, page_count) + 1))
+ else:
+ try:
+ page = int(part)
+ except ValueError:
+ return None
+ if page <= 0:
+ return None
+ if page <= page_count:
+ pages.add(page)
+ return pages
+
+
+def _sample_page_indexes(
+ page_count: int,
+ sample_pages: int,
+ target_pages: str | None = None,
+) -> list[int]:
+ if page_count <= 0 or sample_pages <= 0:
+ return []
+
+ candidate_indexes: list[int]
+ if target_pages not in (None, ""):
+ target_page_numbers = _target_page_numbers(str(target_pages), page_count)
+ if target_page_numbers:
+ candidate_indexes = sorted(page - 1 for page in target_page_numbers)
+ else:
+ candidate_indexes = list(range(page_count))
+ else:
+ candidate_indexes = list(range(page_count))
+
+ if len(candidate_indexes) <= sample_pages:
+ return candidate_indexes
+
+ anchors = [0, 1, 2, len(candidate_indexes) // 2, len(candidate_indexes) - 1]
+ indexes = []
+ seen = set()
+ for anchor in anchors:
+ index = candidate_indexes[min(max(anchor, 0), len(candidate_indexes) - 1)]
+ if index not in seen:
+ seen.add(index)
+ indexes.append(index)
+ if len(indexes) >= sample_pages:
+ break
+ return indexes
+
def _detect_tessdata_path() -> str:
env_path = os.getenv("TESSDATA_PREFIX", "")
diff --git a/plugins/_document_query/webui/config.html b/plugins/_document_query/webui/config.html
index 9dde55ef4..e2a0bbb38 100644
--- a/plugins/_document_query/webui/config.html
+++ b/plugins/_document_query/webui/config.html
@@ -28,8 +28,7 @@
liteparse_dpi: 150,
liteparse_preserve_very_small_text: false,
liteparse_output_format: 'text',
- liteparse_num_workers: 1,
- liteparse_subprocess: true,
+ liteparse_num_workers: 2,
pdf_ocr_fallback: true,
thread_offload: true,
},
@@ -51,7 +50,7 @@
this.ensureNumber('gather_timeout', 120, 1);
this.ensureInt('liteparse_max_pages', 1000, 1);
this.ensureNumber('liteparse_dpi', 150, 72);
- this.ensureInt('liteparse_num_workers', 1, 1);
+ this.ensureInt('liteparse_num_workers', 2, 1);
this.syncChunkOverlap();
},
ensureInt(key, fallback, min = null, max = null) {
@@ -293,7 +292,7 @@
</div>
<div class="field-control">
<input type="number" min="1" step="1"
- @change="ensureInt('liteparse_num_workers', 1, 1)"
+ @change="ensureInt('liteparse_num_workers', 2, 1)"
x-model.number="config.liteparse_num_workers" />
</div>
</div>
@@ -430,20 +429,6 @@
</div>
</div>
- <div class="field">
- <div class="field-label">
- <div class="field-title">Run LiteParse in subprocess</div>
- <div class="field-description">
- Isolate LiteParse native runtime failures from the Web UI process.
- </div>
- </div>
- <div class="field-control">
- <label class="toggle">
- <input type="checkbox" x-model="config.liteparse_subprocess" />
- <span class="toggler"></span>
- </label>
- </div>
- </div>
</div>
</template>
diff --git a/tests/test_document_query_plugin.py b/tests/test_document_query_plugin.py
index c1d3e9484..c03475b47 100644
--- a/tests/test_document_query_plugin.py
+++ b/tests/test_document_query_plugin.py
@@ -10,6 +10,7 @@ from plugins._document_query.helpers.fetch import FetchedDocument, fetch_public_
from plugins._document_query.helpers.document_query import DocumentQueryHelper
from plugins._document_query.helpers.parsers.base import BaseParser
from plugins._document_query.helpers.parsers import get_parsers_for_mimetype
+from plugins._document_query.helpers.parsers import liteparse as liteparse_module
from plugins._document_query.helpers.parsers.liteparse import LiteParseParser
from plugins._document_query.helpers.parsers.text import TextParser
@@ -119,8 +120,9 @@ def test_default_config_bounds_liteparse_runtime_concurrency():
assert "parser_concurrency: 1" in default_config
assert "context_intro_chunks: 2" in default_config
- assert "liteparse_num_workers: 1" in default_config
- assert "liteparse_subprocess: true" in default_config
+ assert "liteparse_num_workers: 2" in default_config
+ assert "liteparse_ocr_auto_disable_pages: 30" in default_config
+ assert "liteparse_subprocess" not in default_config
def test_config_panel_exposes_document_query_settings():
@@ -153,11 +155,11 @@ def test_config_panel_exposes_document_query_settings():
"liteparse_preserve_very_small_text",
"liteparse_output_format",
"liteparse_num_workers",
- "liteparse_subprocess",
"pdf_ocr_fallback",
"thread_offload",
]:
assert f"config.{setting}" in config_html
+ assert "liteparse_subprocess" not in config_html
def test_document_query_thumbnail_matches_plugin_hub_limits():
@@ -173,9 +175,169 @@ def test_document_query_thumbnail_matches_plugin_hub_limits():
def test_liteparse_parser_caps_workers_by_default():
parser = LiteParseParser()
- assert parser._liteparse_kwargs({})["num_workers"] == 1
+ assert parser._liteparse_kwargs({})["num_workers"] == 2
assert parser._liteparse_kwargs({"liteparse_num_workers": "3"})["num_workers"] == 3
- assert parser._liteparse_kwargs({"liteparse_num_workers": ""})["num_workers"] == 1
+ assert parser._liteparse_kwargs({"liteparse_num_workers": ""})["num_workers"] == 2
+
+
+def test_liteparse_parser_always_uses_subprocess(monkeypatch):
+ fetched = FetchedDocument(
+ uri="/tmp/report.pdf",
+ source_uri="/tmp/report.pdf",
+ scheme="file",
+ mimetype="application/pdf",
+ content=b"",
+ local_path="/tmp/report.pdf",
+ )
+ parser = LiteParseParser()
+
+ monkeypatch.setattr(parser, "_parse_subprocess", lambda _document, _config: "ok")
+
+ def fail_in_process(_document, _config):
+ raise AssertionError("LiteParse must stay isolated from the Web UI process")
+
+ monkeypatch.setattr(parser, "_parse_in_process", fail_in_process)
+
+ assert parser._parse_sync(fetched, {"liteparse_subprocess": False}) == "ok"
+
+
+def test_liteparse_auto_disables_ocr_for_large_text_pdf(monkeypatch):
+ parser = LiteParseParser()
+ fetched = FetchedDocument(
+ uri="/tmp/report.pdf",
+ source_uri="/tmp/report.pdf",
+ scheme="file",
+ mimetype="application/pdf",
+ content=b"",
+ local_path="/tmp/report.pdf",
+ )
+ monkeypatch.setattr(
+ liteparse_module,
+ "_pdf_text_profile",
+ lambda _file_path, _config: liteparse_module._PdfTextProfile(
+ page_count=277,
+ sampled_pages=5,
+ text_chars=2500,
+ ),
+ )
+
+ kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/report.pdf")
+
+ assert kwargs["ocr_enabled"] is False
+
+
+def test_liteparse_keeps_ocr_for_small_pdf(monkeypatch):
+ parser = LiteParseParser()
+ fetched = FetchedDocument(
+ uri="/tmp/bill.pdf",
+ source_uri="/tmp/bill.pdf",
+ scheme="file",
+ mimetype="application/pdf",
+ content=b"",
+ local_path="/tmp/bill.pdf",
+ )
+ monkeypatch.setattr(
+ liteparse_module,
+ "_pdf_text_profile",
+ lambda _file_path, _config: liteparse_module._PdfTextProfile(
+ page_count=10,
+ sampled_pages=5,
+ text_chars=2500,
+ ),
+ )
+
+ kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/bill.pdf")
+
+ assert kwargs["ocr_enabled"] is True
+
+
+def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):
+ parser = LiteParseParser()
+ fetched = FetchedDocument(
+ uri="/tmp/scan.pdf",
+ source_uri="/tmp/scan.pdf",
+ scheme="file",
+ mimetype="application/pdf",
+ content=b"",
+ local_path="/tmp/scan.pdf",
+ )
+ monkeypatch.setattr(
+ liteparse_module,
+ "_pdf_text_profile",
+ lambda _file_path, _config: liteparse_module._PdfTextProfile(
+ page_count=277,
+ sampled_pages=5,
+ text_chars=20,
+ ),
+ )
+
+ kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/scan.pdf")
+
+ assert kwargs["ocr_enabled"] is True
+
+
+def test_liteparse_respects_explicit_ocr_disabled(monkeypatch):
+ parser = LiteParseParser()
+ fetched = FetchedDocument(
+ uri="/tmp/bill.pdf",
+ source_uri="/tmp/bill.pdf",
+ scheme="file",
+ mimetype="application/pdf",
+ content=b"",
+ local_path="/tmp/bill.pdf",
+ )
+ monkeypatch.setattr(
+ liteparse_module,
+ "_pdf_text_profile",
+ lambda _file_path, _config: liteparse_module._PdfTextProfile(
+ page_count=10,
+ sampled_pages=5,
+ text_chars=0,
+ ),
+ )
+
+ kwargs = parser._liteparse_kwargs(
+ {"liteparse_ocr_enabled": False},
+ fetched,
+ "/tmp/bill.pdf",
+ )
+
+ assert kwargs["ocr_enabled"] is False
+
+
+def test_liteparse_target_pages_can_keep_ocr_enabled_for_large_pdf(monkeypatch):
+ parser = LiteParseParser()
+ fetched = FetchedDocument(
+ uri="/tmp/report.pdf",
+ source_uri="/tmp/report.pdf",
+ scheme="file",
+ mimetype="application/pdf",
+ content=b"",
+ local_path="/tmp/report.pdf",
+ )
+ monkeypatch.setattr(
+ liteparse_module,
+ "_pdf_text_profile",
+ lambda _file_path, _config: liteparse_module._PdfTextProfile(
+ page_count=277,
+ sampled_pages=5,
+ text_chars=2500,
+ ),
+ )
+
+ small_range = parser._liteparse_kwargs(
+ {"liteparse_target_pages": "1-10"},
+ fetched,
+ "/tmp/report.pdf",
+ )
+ large_range = parser._liteparse_kwargs(
+ {"liteparse_target_pages": "1-40"},
+ fetched,
+ "/tmp/report.pdf",
+ )
+
+ assert small_range["ocr_enabled"] is True
+ assert large_range["ocr_enabled"] is False
def test_query_optimize_prompt_filename_is_spelled_correctly():