Agent Zero AI framework
Tune LiteParse OCR defaults
Alessandro9e4b2f1843b2
diff --git a/plugins/_document_query/README.md b/plugins/_document_query/README.mdindex 62655189c..3defbd250 100644--- a/plugins/_document_query/README.md+++ b/plugins/_document_query/README.md@@ -8,6 +8,7 @@ timeouts and thread-safe parsers.- **Strategy-pattern parsers** - MIME-type routing to dedicated parser classes- **Centralized fetching** - local and HTTP(S) resources are fetched once, size-checked, then passed to parsers- **LiteParse first path** - fast local parsing for PDFs and supported document/image formats, with legacy fallbacks+- **Adaptive OCR** - large text-rich PDFs skip OCR automatically to avoid pathological parse times- **Bounded parser execution** - sync parsers are offloaded to asyncio.to_thread and globally capped across chats- **Configurable timeouts** - per-document and gather-level timeouts- **Expanded format support** - PDF, HTML, text, YAML, XML, TOML, JS, TS, images, and catch-all Unstructured@@ -29,14 +30,17 @@ See default_config.yaml for all options. Key settings:| chunk_overlap | 100 | Text splitter overlap || search_threshold | 0.5 | Similarity search threshold || liteparse_enabled | true | Prefer LiteParse before legacy parser fallbacks |-| liteparse_num_workers | 1 | Max LiteParse OCR workers per parser job |-| liteparse_subprocess | true | Run LiteParse in a child process so native crashes fall back safely |+| liteparse_num_workers | 2 | Max LiteParse OCR workers per parser job |+| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for text-rich PDFs at or above this effective page count || thread_offload | true | Offload sync parsers to thread pool |LiteParse is installed into the Agent Zero framework runtime from hooks.py duringplugin install/startup. If installation fails, the plugin logs the error andcontinues with the legacy parser fallbacks.+LiteParse always runs in a child process so native parser and OCR failures stay+isolated from the Web UI process.+## Parsers| Parser | MIME Types | Backend |diff --git a/plugins/_document_query/default_config.yaml b/plugins/_document_query/default_config.yamlindex a850016cc..ddef41b90 100644--- a/plugins/_document_query/default_config.yaml+++ b/plugins/_document_query/default_config.yaml@@ -28,7 +28,10 @@ liteparse_target_pages:liteparse_dpi: 150liteparse_preserve_very_small_text: falseliteparse_output_format: text-liteparse_num_workers: 1 # LiteParse defaults to CPU cores - 1; cap it for web runtime stability-liteparse_subprocess: true # isolate LiteParse native runtime crashes from the Web UI process+liteparse_num_workers: 2 # balanced default for OCR speed without overloading shared Web UI runtime+liteparse_ocr_auto_disable: true # disable OCR automatically for large text-rich PDFs+liteparse_ocr_auto_disable_pages: 30 # OCR-on runtime climbs sharply around this page count+liteparse_ocr_auto_min_chars_per_page: 80+liteparse_ocr_auto_sample_pages: 5pdf_ocr_fallback: true # enable legacy Tesseract fallback after PyMuPDFthread_offload: true # offload sync parsers to thread pooldiff --git a/plugins/_document_query/helpers/parsers/liteparse.py b/plugins/_document_query/helpers/parsers/liteparse.pyindex 110b10f4d..ce6f06523 100644--- a/plugins/_document_query/helpers/parsers/liteparse.py+++ b/plugins/_document_query/helpers/parsers/liteparse.py@@ -6,16 +6,29 @@ import jsonimport osimport subprocessimport sys+from dataclasses import dataclassfrom pathlib import Pathfrom plugins._document_query.helpers.fetch import FetchedDocumentfrom .base import BaseParser+DEFAULT_OCR_AUTO_DISABLE_PAGES = 30+DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE = 80+DEFAULT_OCR_AUTO_SAMPLE_PAGES = 5+++@dataclass(frozen=True)+class _PdfTextProfile:+ page_count: int+ sampled_pages: int+ text_chars: int++class LiteParseParser(BaseParser):"""Fast parser powered by run-llama/liteparse when available."""- DEFAULT_NUM_WORKERS = 1+ DEFAULT_NUM_WORKERS = 2mimetypes = ["application/pdf",@@ -35,9 +48,8 @@ class LiteParseParser(BaseParser):return bool(config.get("liteparse_enabled", True))def _parse_sync(self, document: FetchedDocument, config: dict) -> str:- if config.get("liteparse_subprocess", True):- return self._parse_subprocess(document, config)- return self._parse_in_process(document, config)+ # Keep LiteParse native/OCR failures isolated from the Web UI process.+ return self._parse_subprocess(document, config)def _parse_in_process(self, document: FetchedDocument, config: dict) -> str:try:@@ -58,7 +70,7 @@ class LiteParseParser(BaseParser):with document.local_file() as file_path:payload = {"file_path": file_path,- "kwargs": self._liteparse_kwargs(config),+ "kwargs": self._liteparse_kwargs(config, document, file_path),}env = os.environ.copy()project_root = str(Path(__file__).resolve().parents[4])@@ -104,9 +116,18 @@ class LiteParseParser(BaseParser):raise ValueError("LiteParse returned no text")return text- def _liteparse_kwargs(self, config: dict) -> dict:+ def _liteparse_kwargs(+ self,+ config: dict,+ document: FetchedDocument | None = None,+ file_path: str | None = None,+ ) -> dict:+ ocr_enabled = bool(config.get("liteparse_ocr_enabled", True))+ if ocr_enabled and self._should_disable_ocr(config, document, file_path):+ ocr_enabled = False+kwargs = {- "ocr_enabled": bool(config.get("liteparse_ocr_enabled", True)),+ "ocr_enabled": ocr_enabled,"ocr_language": config.get("liteparse_ocr_language", "eng"),"max_pages": int(config.get("liteparse_max_pages", 1000)),"dpi": float(config.get("liteparse_dpi", 150)),@@ -138,6 +159,149 @@ class LiteParseParser(BaseParser):return kwargs+ def _should_disable_ocr(+ self,+ config: dict,+ document: FetchedDocument | None,+ file_path: str | None,+ ) -> bool:+ if not bool(config.get("liteparse_ocr_auto_disable", True)):+ return False+ if not document or document.mimetype != "application/pdf" or not file_path:+ return False++ profile = _pdf_text_profile(file_path, config)+ if not profile or profile.sampled_pages <= 0:+ return False++ effective_pages = _effective_page_budget(config, profile.page_count)+ auto_disable_pages = _positive_int(+ config.get("liteparse_ocr_auto_disable_pages"),+ DEFAULT_OCR_AUTO_DISABLE_PAGES,+ )+ if effective_pages < auto_disable_pages:+ return False++ min_chars_per_page = _positive_int(+ config.get("liteparse_ocr_auto_min_chars_per_page"),+ DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE,+ )+ return (profile.text_chars / profile.sampled_pages) >= min_chars_per_page+++def _pdf_text_profile(file_path: str, config: dict) -> _PdfTextProfile | None:+ try:+ import fitz+ except Exception:+ return None++ try:+ with fitz.open(file_path) as doc:+ page_count = doc.page_count+ if page_count <= 0:+ return _PdfTextProfile(0, 0, 0)+ sample_indexes = _sample_page_indexes(+ page_count=page_count,+ sample_pages=_positive_int(+ config.get("liteparse_ocr_auto_sample_pages"),+ DEFAULT_OCR_AUTO_SAMPLE_PAGES,+ ),+ target_pages=config.get("liteparse_target_pages"),+ )+ text_chars = 0+ for page_index in sample_indexes:+ text = doc[page_index].get_text("text") or ""+ text_chars += len("".join(text.split()))+ return _PdfTextProfile(+ page_count=page_count,+ sampled_pages=len(sample_indexes),+ text_chars=text_chars,+ )+ except Exception:+ return None+++def _effective_page_budget(config: dict, page_count: int) -> int:+ target_pages = config.get("liteparse_target_pages")+ if target_pages not in (None, ""):+ parsed_target_count = _target_page_count(str(target_pages), page_count)+ if parsed_target_count:+ return parsed_target_count++ max_pages = _positive_int(config.get("liteparse_max_pages"), 1000)+ return min(max_pages, page_count)+++def _target_page_count(value: str, page_count: int) -> int:+ pages = _target_page_numbers(value, page_count)+ if pages is None:+ return 0+ return len(pages)+++def _target_page_numbers(value: str, page_count: int) -> set[int] | None:+ pages: set[int] = set()+ for raw_part in value.split(","):+ part = raw_part.strip()+ if not part:+ continue+ if "-" in part:+ start_raw, end_raw = part.split("-", 1)+ try:+ start = int(start_raw.strip())+ end = int(end_raw.strip())+ except ValueError:+ return None+ if start <= 0 or end <= 0:+ return None+ if start > end:+ start, end = end, start+ pages.update(range(start, min(end, page_count) + 1))+ else:+ try:+ page = int(part)+ except ValueError:+ return None+ if page <= 0:+ return None+ if page <= page_count:+ pages.add(page)+ return pages+++def _sample_page_indexes(+ page_count: int,+ sample_pages: int,+ target_pages: str | None = None,+) -> list[int]:+ if page_count <= 0 or sample_pages <= 0:+ return []++ candidate_indexes: list[int]+ if target_pages not in (None, ""):+ target_page_numbers = _target_page_numbers(str(target_pages), page_count)+ if target_page_numbers:+ candidate_indexes = sorted(page - 1 for page in target_page_numbers)+ else:+ candidate_indexes = list(range(page_count))+ else:+ candidate_indexes = list(range(page_count))++ if len(candidate_indexes) <= sample_pages:+ return candidate_indexes++ anchors = [0, 1, 2, len(candidate_indexes) // 2, len(candidate_indexes) - 1]+ indexes = []+ seen = set()+ for anchor in anchors:+ index = candidate_indexes[min(max(anchor, 0), len(candidate_indexes) - 1)]+ if index not in seen:+ seen.add(index)+ indexes.append(index)+ if len(indexes) >= sample_pages:+ break+ return indexes+def _detect_tessdata_path() -> str:env_path = os.getenv("TESSDATA_PREFIX", "")diff --git a/plugins/_document_query/webui/config.html b/plugins/_document_query/webui/config.htmlindex 9dde55ef4..e2a0bbb38 100644--- a/plugins/_document_query/webui/config.html+++ b/plugins/_document_query/webui/config.html@@ -28,8 +28,7 @@liteparse_dpi: 150,liteparse_preserve_very_small_text: false,liteparse_output_format: 'text',- liteparse_num_workers: 1,- liteparse_subprocess: true,+ liteparse_num_workers: 2,pdf_ocr_fallback: true,thread_offload: true,},@@ -51,7 +50,7 @@this.ensureNumber('gather_timeout', 120, 1);this.ensureInt('liteparse_max_pages', 1000, 1);this.ensureNumber('liteparse_dpi', 150, 72);- this.ensureInt('liteparse_num_workers', 1, 1);+ this.ensureInt('liteparse_num_workers', 2, 1);this.syncChunkOverlap();},ensureInt(key, fallback, min = null, max = null) {@@ -293,7 +292,7 @@</div><div class="field-control"><input type="number" min="1" step="1"- @change="ensureInt('liteparse_num_workers', 1, 1)"+ @change="ensureInt('liteparse_num_workers', 2, 1)"x-model.number="config.liteparse_num_workers" /></div></div>@@ -430,20 +429,6 @@</div></div>- <div class="field">- <div class="field-label">- <div class="field-title">Run LiteParse in subprocess</div>- <div class="field-description">- Isolate LiteParse native runtime failures from the Web UI process.- </div>- </div>- <div class="field-control">- <label class="toggle">- <input type="checkbox" x-model="config.liteparse_subprocess" />- <span class="toggler"></span>- </label>- </div>- </div></div></template>diff --git a/tests/test_document_query_plugin.py b/tests/test_document_query_plugin.pyindex c1d3e9484..c03475b47 100644--- a/tests/test_document_query_plugin.py+++ b/tests/test_document_query_plugin.py@@ -10,6 +10,7 @@ from plugins._document_query.helpers.fetch import FetchedDocument, fetch_public_from plugins._document_query.helpers.document_query import DocumentQueryHelperfrom plugins._document_query.helpers.parsers.base import BaseParserfrom plugins._document_query.helpers.parsers import get_parsers_for_mimetype+from plugins._document_query.helpers.parsers import liteparse as liteparse_modulefrom plugins._document_query.helpers.parsers.liteparse import LiteParseParserfrom plugins._document_query.helpers.parsers.text import TextParser@@ -119,8 +120,9 @@ def test_default_config_bounds_liteparse_runtime_concurrency():assert "parser_concurrency: 1" in default_configassert "context_intro_chunks: 2" in default_config- assert "liteparse_num_workers: 1" in default_config- assert "liteparse_subprocess: true" in default_config+ assert "liteparse_num_workers: 2" in default_config+ assert "liteparse_ocr_auto_disable_pages: 30" in default_config+ assert "liteparse_subprocess" not in default_configdef test_config_panel_exposes_document_query_settings():@@ -153,11 +155,11 @@ def test_config_panel_exposes_document_query_settings():"liteparse_preserve_very_small_text","liteparse_output_format","liteparse_num_workers",- "liteparse_subprocess","pdf_ocr_fallback","thread_offload",]:assert f"config.{setting}" in config_html+ assert "liteparse_subprocess" not in config_htmldef test_document_query_thumbnail_matches_plugin_hub_limits():@@ -173,9 +175,169 @@ def test_document_query_thumbnail_matches_plugin_hub_limits():def test_liteparse_parser_caps_workers_by_default():parser = LiteParseParser()- assert parser._liteparse_kwargs({})["num_workers"] == 1+ assert parser._liteparse_kwargs({})["num_workers"] == 2assert parser._liteparse_kwargs({"liteparse_num_workers": "3"})["num_workers"] == 3- assert parser._liteparse_kwargs({"liteparse_num_workers": ""})["num_workers"] == 1+ assert parser._liteparse_kwargs({"liteparse_num_workers": ""})["num_workers"] == 2+++def test_liteparse_parser_always_uses_subprocess(monkeypatch):+ fetched = FetchedDocument(+ uri="/tmp/report.pdf",+ source_uri="/tmp/report.pdf",+ scheme="file",+ mimetype="application/pdf",+ content=b"",+ local_path="/tmp/report.pdf",+ )+ parser = LiteParseParser()++ monkeypatch.setattr(parser, "_parse_subprocess", lambda _document, _config: "ok")++ def fail_in_process(_document, _config):+ raise AssertionError("LiteParse must stay isolated from the Web UI process")++ monkeypatch.setattr(parser, "_parse_in_process", fail_in_process)++ assert parser._parse_sync(fetched, {"liteparse_subprocess": False}) == "ok"+++def test_liteparse_auto_disables_ocr_for_large_text_pdf(monkeypatch):+ parser = LiteParseParser()+ fetched = FetchedDocument(+ uri="/tmp/report.pdf",+ source_uri="/tmp/report.pdf",+ scheme="file",+ mimetype="application/pdf",+ content=b"",+ local_path="/tmp/report.pdf",+ )+ monkeypatch.setattr(+ liteparse_module,+ "_pdf_text_profile",+ lambda _file_path, _config: liteparse_module._PdfTextProfile(+ page_count=277,+ sampled_pages=5,+ text_chars=2500,+ ),+ )++ kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/report.pdf")++ assert kwargs["ocr_enabled"] is False+++def test_liteparse_keeps_ocr_for_small_pdf(monkeypatch):+ parser = LiteParseParser()+ fetched = FetchedDocument(+ uri="/tmp/bill.pdf",+ source_uri="/tmp/bill.pdf",+ scheme="file",+ mimetype="application/pdf",+ content=b"",+ local_path="/tmp/bill.pdf",+ )+ monkeypatch.setattr(+ liteparse_module,+ "_pdf_text_profile",+ lambda _file_path, _config: liteparse_module._PdfTextProfile(+ page_count=10,+ sampled_pages=5,+ text_chars=2500,+ ),+ )++ kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/bill.pdf")++ assert kwargs["ocr_enabled"] is True+++def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):+ parser = LiteParseParser()+ fetched = FetchedDocument(+ uri="/tmp/scan.pdf",+ source_uri="/tmp/scan.pdf",+ scheme="file",+ mimetype="application/pdf",+ content=b"",+ local_path="/tmp/scan.pdf",+ )+ monkeypatch.setattr(+ liteparse_module,+ "_pdf_text_profile",+ lambda _file_path, _config: liteparse_module._PdfTextProfile(+ page_count=277,+ sampled_pages=5,+ text_chars=20,+ ),+ )++ kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/scan.pdf")++ assert kwargs["ocr_enabled"] is True+++def test_liteparse_respects_explicit_ocr_disabled(monkeypatch):+ parser = LiteParseParser()+ fetched = FetchedDocument(+ uri="/tmp/bill.pdf",+ source_uri="/tmp/bill.pdf",+ scheme="file",+ mimetype="application/pdf",+ content=b"",+ local_path="/tmp/bill.pdf",+ )+ monkeypatch.setattr(+ liteparse_module,+ "_pdf_text_profile",+ lambda _file_path, _config: liteparse_module._PdfTextProfile(+ page_count=10,+ sampled_pages=5,+ text_chars=0,+ ),+ )++ kwargs = parser._liteparse_kwargs(+ {"liteparse_ocr_enabled": False},+ fetched,+ "/tmp/bill.pdf",+ )++ assert kwargs["ocr_enabled"] is False+++def test_liteparse_target_pages_can_keep_ocr_enabled_for_large_pdf(monkeypatch):+ parser = LiteParseParser()+ fetched = FetchedDocument(+ uri="/tmp/report.pdf",+ source_uri="/tmp/report.pdf",+ scheme="file",+ mimetype="application/pdf",+ content=b"",+ local_path="/tmp/report.pdf",+ )+ monkeypatch.setattr(+ liteparse_module,+ "_pdf_text_profile",+ lambda _file_path, _config: liteparse_module._PdfTextProfile(+ page_count=277,+ sampled_pages=5,+ text_chars=2500,+ ),+ )++ small_range = parser._liteparse_kwargs(+ {"liteparse_target_pages": "1-10"},+ fetched,+ "/tmp/report.pdf",+ )+ large_range = parser._liteparse_kwargs(+ {"liteparse_target_pages": "1-40"},+ fetched,+ "/tmp/report.pdf",+ )++ assert small_range["ocr_enabled"] is True+ assert large_range["ocr_enabled"] is Falsedef test_query_optimize_prompt_filename_is_spelled_correctly():