Agent Zero AI framework
fix(document-query): bound long PDF processing
Alessandrobc2d447dddf4
diff --git a/plugins/_document_query/README.md b/plugins/_document_query/README.mdindex 3defbd250..dfa306b73 100644--- a/plugins/_document_query/README.md+++ b/plugins/_document_query/README.md@@ -8,7 +8,8 @@ timeouts and thread-safe parsers.- **Strategy-pattern parsers** - MIME-type routing to dedicated parser classes- **Centralized fetching** - local and HTTP(S) resources are fetched once, size-checked, then passed to parsers- **LiteParse first path** - fast local parsing for PDFs and supported document/image formats, with legacy fallbacks-- **Adaptive OCR** - large text-rich PDFs skip OCR automatically to avoid pathological parse times+- **Adaptive OCR** - long PDFs skip OCR automatically to avoid pathological parse times+- **Adaptive indexing** - very large extracted documents increase chunk size to keep embedding work bounded- **Bounded parser execution** - sync parsers are offloaded to asyncio.to_thread and globally capped across chats- **Configurable timeouts** - per-document and gather-level timeouts- **Expanded format support** - PDF, HTML, text, YAML, XML, TOML, JS, TS, images, and catch-all Unstructured@@ -28,10 +29,11 @@ See default_config.yaml for all options. Key settings:| context_intro_chunks | 2 | Leading chunks included per document for title/abstract grounding || chunk_size | 1000 | Text splitter chunk size || chunk_overlap | 100 | Text splitter overlap |+| max_index_chunks | 1200 | Maximum indexed chunks before adaptive chunk sizing, or 0 for no cap || search_threshold | 0.5 | Similarity search threshold || liteparse_enabled | true | Prefer LiteParse before legacy parser fallbacks || liteparse_num_workers | 2 | Max LiteParse OCR workers per parser job |-| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for text-rich PDFs at or above this effective page count |+| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for PDFs at or above this effective page count || thread_offload | true | Offload sync parsers to thread pool |LiteParse is installed into the Agent Zero framework runtime from hooks.py duringdiff --git a/plugins/_document_query/default_config.yaml b/plugins/_document_query/default_config.yamlindex ddef41b90..43adb8c67 100644--- a/plugins/_document_query/default_config.yaml+++ b/plugins/_document_query/default_config.yaml@@ -13,6 +13,7 @@ parser_concurrency: 1 # max parser jobs running across all chats in thiscontext_intro_chunks: 2 # always include leading chunks per document for title/abstract groundingchunk_size: 1000chunk_overlap: 100+max_index_chunks: 1200 # adapt chunk size above this many indexed chunkssearch_threshold: 0.5search_limit: 100max_remote_bytes: 52428800 # 50 MB@@ -29,9 +30,8 @@ liteparse_dpi: 150liteparse_preserve_very_small_text: falseliteparse_output_format: textliteparse_num_workers: 2 # balanced default for OCR speed without overloading shared Web UI runtime-liteparse_ocr_auto_disable: true # disable OCR automatically for large text-rich PDFs+liteparse_ocr_auto_disable: true # disable OCR automatically for long PDFsliteparse_ocr_auto_disable_pages: 30 # OCR-on runtime climbs sharply around this page count-liteparse_ocr_auto_min_chars_per_page: 80liteparse_ocr_auto_sample_pages: 5pdf_ocr_fallback: true # enable legacy Tesseract fallback after PyMuPDFthread_offload: true # offload sync parsers to thread pooldiff --git a/plugins/_document_query/helpers/document_query.py b/plugins/_document_query/helpers/document_query.pyindex e1553c872..4a7fa897a 100644--- a/plugins/_document_query/helpers/document_query.py+++ b/plugins/_document_query/helpers/document_query.py@@ -40,6 +40,14 @@ def _positive_int(value: Any, default: int) -> int:return parsed if parsed > 0 else default+def _nonnegative_int(value: Any, default: int) -> int:+ try:+ parsed = int(value)+ except (TypeError, ValueError):+ return default+ return parsed if parsed >= 0 else default++def _parser_semaphore(config: dict) -> asyncio.Semaphore:concurrency = _positive_int(config.get("parser_concurrency"),@@ -66,6 +74,7 @@ class DocumentQueryStore:DEFAULT_CHUNK_SIZE = 1000DEFAULT_CHUNK_OVERLAP = 100+ DEFAULT_MAX_INDEX_CHUNKS = 1200@staticmethoddef get(agent: Agent):@@ -103,12 +112,7 @@ class DocumentQueryStore:doc_metadata = metadata or {}doc_metadata["document_uri"] = document_uridoc_metadata["timestamp"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")- chunk_size = self.config.get("chunk_size", self.DEFAULT_CHUNK_SIZE)- chunk_overlap = self.config.get("chunk_overlap", self.DEFAULT_CHUNK_OVERLAP)- text_splitter = RecursiveCharacterTextSplitter(- chunk_size=chunk_size, chunk_overlap=chunk_overlap- )- chunks = text_splitter.split_text(text)+ chunks = self._split_text_for_index(text)docs = []for i, chunk in enumerate(chunks):chunk_metadata = doc_metadata.copy()@@ -129,6 +133,51 @@ class DocumentQueryStore:PrintStyle.error(f"Error adding document '{document_uri}': {err_text}")return False, []+ def _split_text_for_index(self, text: str) -> list[str]:+ chunk_size = _positive_int(+ self.config.get("chunk_size"),+ self.DEFAULT_CHUNK_SIZE,+ )+ chunk_overlap = min(+ _nonnegative_int(+ self.config.get("chunk_overlap"),+ self.DEFAULT_CHUNK_OVERLAP,+ ),+ max(0, chunk_size - 1),+ )+ chunks = self._split_text(text, chunk_size, chunk_overlap)++ max_chunks = _nonnegative_int(+ self.config.get("max_index_chunks"),+ self.DEFAULT_MAX_INDEX_CHUNKS,+ )+ if not max_chunks or len(chunks) <= max_chunks:+ return chunks++ overlap_ratio = chunk_overlap / chunk_size if chunk_size else 0+ overlap_ratio = max(0, min(overlap_ratio, 0.5))+ target_size = max(+ chunk_size + 1,+ int(len(text) / max(1, max_chunks * (1 - overlap_ratio))) + 1,+ )++ for _ in range(8):+ target_overlap = min(int(target_size * overlap_ratio), target_size - 1)+ chunks = self._split_text(text, target_size, target_overlap)+ if len(chunks) <= max_chunks or target_size >= len(text):+ return chunks+ target_size = min(len(text), int(target_size * 1.25) + 1)++ return chunks++ @staticmethod+ def _split_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:+ text_splitter = RecursiveCharacterTextSplitter(+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ )+ return text_splitter.split_text(text)+async def get_document(self, document_uri: str) -> Optional[Document]:if not self.vector_db:return Nonediff --git a/plugins/_document_query/helpers/parsers/liteparse.py b/plugins/_document_query/helpers/parsers/liteparse.pyindex ce6f06523..0ff327571 100644--- a/plugins/_document_query/helpers/parsers/liteparse.py+++ b/plugins/_document_query/helpers/parsers/liteparse.py@@ -14,7 +14,6 @@ from .base import BaseParserDEFAULT_OCR_AUTO_DISABLE_PAGES = 30-DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE = 80DEFAULT_OCR_AUTO_SAMPLE_PAGES = 5@@ -182,11 +181,7 @@ class LiteParseParser(BaseParser):if effective_pages < auto_disable_pages:return False- min_chars_per_page = _positive_int(- config.get("liteparse_ocr_auto_min_chars_per_page"),- DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE,- )- return (profile.text_chars / profile.sampled_pages) >= min_chars_per_page+ return Truedef _pdf_text_profile(file_path: str, config: dict) -> _PdfTextProfile | None:diff --git a/plugins/_document_query/hooks.py b/plugins/_document_query/hooks.pyindex 45eed56b8..04cb609c8 100644--- a/plugins/_document_query/hooks.py+++ b/plugins/_document_query/hooks.py@@ -15,7 +15,7 @@ from helpers.print_style import PrintStyle_LOCK = threading.Lock()_CHECKED = False_PLUGIN_DIR = Path(__file__).resolve().parent-_REQUIREMENTS_FILE = _PLUGIN_DIR / "requirements.txt"+_ROOT_REQUIREMENTS_FILE = _PLUGIN_DIR.parents[1] / "requirements.txt"def has_liteparse() -> bool:@@ -66,9 +66,10 @@ def _install_requirements() -> None:raise RuntimeError("Document Query plugin requires 'uv' to install liteparse automatically")- if not _REQUIREMENTS_FILE.is_file():+ requirement = _liteparse_requirement()+ if not requirement:raise RuntimeError(- f"Document Query requirements file not found: {_REQUIREMENTS_FILE}"+ f"Document Query LiteParse requirement not found in {_ROOT_REQUIREMENTS_FILE}")cmd = [@@ -77,9 +78,18 @@ def _install_requirements() -> None:"install","--python",sys.executable,- "-r",- str(_REQUIREMENTS_FILE),+ requirement,]PrintStyle.info("Document Query: liteparse not found, installing plugin dependency")subprocess.check_call(cmd, cwd=str(_PLUGIN_DIR))+++def _liteparse_requirement() -> str:+ if not _ROOT_REQUIREMENTS_FILE.is_file():+ return ""+ for line in _ROOT_REQUIREMENTS_FILE.read_text(encoding="utf-8").splitlines():+ requirement = line.strip()+ if requirement.startswith("liteparse"):+ return requirement+ return ""diff --git a/plugins/_document_query/webui/config.html b/plugins/_document_query/webui/config.htmlindex e2a0bbb38..fc889efef 100644--- a/plugins/_document_query/webui/config.html+++ b/plugins/_document_query/webui/config.html@@ -15,6 +15,7 @@context_intro_chunks: 2,chunk_size: 1000,chunk_overlap: 100,+ max_index_chunks: 1200,search_threshold: 0.5,search_limit: 100,max_remote_bytes: 52428800,@@ -40,6 +41,7 @@this.ensureInt('context_intro_chunks', 2, 0);this.ensureInt('chunk_size', 1000, 100);this.ensureInt('chunk_overlap', 100, 0);+ this.ensureInt('max_index_chunks', 1200, 0);this.ensureInt('search_limit', 100, 1);this.ensureInt('max_remote_bytes', 52428800, 1);this.ensureNumber('search_threshold', 0.5, 0, 1);@@ -172,6 +174,20 @@</div></div>+ <div class="field">+ <div class="field-label">+ <div class="field-title">Max index chunks</div>+ <div class="field-description">+ Adapt chunk size when a parsed document would exceed this many indexed chunks. Use 0 for no cap.+ </div>+ </div>+ <div class="field-control">+ <input type="number" min="0" step="50"+ @change="ensureInt('max_index_chunks', 1200, 0)"+ x-model.number="config.max_index_chunks" />+ </div>+ </div>+<div class="field"><div class="field-label"><div class="field-title">Search limit</div>diff --git a/tests/test_document_query_plugin.py b/tests/test_document_query_plugin.pyindex c03475b47..e3ece0ebd 100644--- a/tests/test_document_query_plugin.py+++ b/tests/test_document_query_plugin.py@@ -6,8 +6,12 @@ from pathlib import Pathimport pytestfrom PIL import Image+from plugins._document_query import hooks as document_query_hooksfrom plugins._document_query.helpers.fetch import FetchedDocument, fetch_public_resource-from plugins._document_query.helpers.document_query import DocumentQueryHelper+from plugins._document_query.helpers.document_query import (+ DocumentQueryHelper,+ DocumentQueryStore,+)from plugins._document_query.helpers.parsers.base import BaseParserfrom plugins._document_query.helpers.parsers import get_parsers_for_mimetypefrom plugins._document_query.helpers.parsers import liteparse as liteparse_module@@ -105,12 +109,14 @@ def test_compatibility_imports_point_to_plugin_classes():def test_liteparse_is_installed_by_docker_and_plugin_hook_requirements():root_requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8")- plugin_requirements = (- ROOT / "plugins" / "_document_query" / "requirements.txt"+ hooks_source = (+ ROOT / "plugins" / "_document_query" / "hooks.py").read_text(encoding="utf-8")assert "liteparse==2.0.3" in root_requirements- assert plugin_requirements.strip().splitlines() == ["liteparse==2.0.3"]+ assert "_ROOT_REQUIREMENTS_FILE" in hooks_source+ assert document_query_hooks._liteparse_requirement() == "liteparse==2.0.3"+ assert not (ROOT / "plugins" / "_document_query" / "requirements.txt").exists()def test_default_config_bounds_liteparse_runtime_concurrency():@@ -120,6 +126,7 @@ def test_default_config_bounds_liteparse_runtime_concurrency():assert "parser_concurrency: 1" in default_configassert "context_intro_chunks: 2" in default_config+ assert "max_index_chunks: 1200" in default_configassert "liteparse_num_workers: 2" in default_configassert "liteparse_ocr_auto_disable_pages: 30" in default_configassert "liteparse_subprocess" not in default_config@@ -137,6 +144,7 @@ def test_config_panel_exposes_document_query_settings():"gather_timeout","chunk_size","chunk_overlap",+ "max_index_chunks","search_threshold","search_limit","context_intro_chunks",@@ -162,6 +170,32 @@ def test_config_panel_exposes_document_query_settings():assert "liteparse_subprocess" not in config_html+def test_document_query_adapts_chunk_size_for_large_documents():+ store = object.__new__(DocumentQueryStore)+ store.config = {+ "chunk_size": 100,+ "chunk_overlap": 10,+ "max_index_chunks": 10,+ }++ chunks = store._split_text_for_index(("alpha beta gamma delta " * 400).strip())++ assert 1 < len(chunks) <= 10+++def test_document_query_allows_uncapped_index_chunks():+ store = object.__new__(DocumentQueryStore)+ store.config = {+ "chunk_size": 100,+ "chunk_overlap": 10,+ "max_index_chunks": 0,+ }++ chunks = store._split_text_for_index(("alpha beta gamma delta " * 400).strip())++ assert len(chunks) > 10++def test_document_query_thumbnail_matches_plugin_hub_limits():thumbnail = ROOT / "plugins" / "_document_query" / "webui" / "thumbnail.jpg"@@ -251,7 +285,7 @@ def test_liteparse_keeps_ocr_for_small_pdf(monkeypatch):assert kwargs["ocr_enabled"] is True-def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):+def test_liteparse_disables_ocr_for_large_text_sparse_pdf(monkeypatch):parser = LiteParseParser()fetched = FetchedDocument(uri="/tmp/scan.pdf",@@ -273,7 +307,7 @@ def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/scan.pdf")- assert kwargs["ocr_enabled"] is True+ assert kwargs["ocr_enabled"] is Falsedef test_liteparse_respects_explicit_ocr_disabled(monkeypatch):