Skip to content

Agent Zero AI framework

fix(document-query): bound long PDF processing

Alessandrobc2d447dddf4
diff --git a/plugins/_document_query/README.md b/plugins/_document_query/README.md
index 3defbd250..dfa306b73 100644
--- a/plugins/_document_query/README.md
+++ b/plugins/_document_query/README.md
@@ -8,7 +8,8 @@ timeouts and thread-safe parsers.
- **Strategy-pattern parsers** - MIME-type routing to dedicated parser classes
- **Centralized fetching** - local and HTTP(S) resources are fetched once, size-checked, then passed to parsers
- **LiteParse first path** - fast local parsing for PDFs and supported document/image formats, with legacy fallbacks
-- **Adaptive OCR** - large text-rich PDFs skip OCR automatically to avoid pathological parse times
+- **Adaptive OCR** - long PDFs skip OCR automatically to avoid pathological parse times
+- **Adaptive indexing** - very large extracted documents increase chunk size to keep embedding work bounded
- **Bounded parser execution** - sync parsers are offloaded to asyncio.to_thread and globally capped across chats
- **Configurable timeouts** - per-document and gather-level timeouts
- **Expanded format support** - PDF, HTML, text, YAML, XML, TOML, JS, TS, images, and catch-all Unstructured
@@ -28,10 +29,11 @@ See default_config.yaml for all options. Key settings:
| context_intro_chunks | 2 | Leading chunks included per document for title/abstract grounding |
| chunk_size | 1000 | Text splitter chunk size |
| chunk_overlap | 100 | Text splitter overlap |
+| max_index_chunks | 1200 | Maximum indexed chunks before adaptive chunk sizing, or 0 for no cap |
| search_threshold | 0.5 | Similarity search threshold |
| liteparse_enabled | true | Prefer LiteParse before legacy parser fallbacks |
| liteparse_num_workers | 2 | Max LiteParse OCR workers per parser job |
-| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for text-rich PDFs at or above this effective page count |
+| liteparse_ocr_auto_disable_pages | 30 | Disable OCR for PDFs at or above this effective page count |
| thread_offload | true | Offload sync parsers to thread pool |
LiteParse is installed into the Agent Zero framework runtime from hooks.py during
diff --git a/plugins/_document_query/default_config.yaml b/plugins/_document_query/default_config.yaml
index ddef41b90..43adb8c67 100644
--- a/plugins/_document_query/default_config.yaml
+++ b/plugins/_document_query/default_config.yaml
@@ -13,6 +13,7 @@ parser_concurrency: 1 # max parser jobs running across all chats in this
context_intro_chunks: 2 # always include leading chunks per document for title/abstract grounding
chunk_size: 1000
chunk_overlap: 100
+max_index_chunks: 1200 # adapt chunk size above this many indexed chunks
search_threshold: 0.5
search_limit: 100
max_remote_bytes: 52428800 # 50 MB
@@ -29,9 +30,8 @@ liteparse_dpi: 150
liteparse_preserve_very_small_text: false
liteparse_output_format: text
liteparse_num_workers: 2 # balanced default for OCR speed without overloading shared Web UI runtime
-liteparse_ocr_auto_disable: true # disable OCR automatically for large text-rich PDFs
+liteparse_ocr_auto_disable: true # disable OCR automatically for long PDFs
liteparse_ocr_auto_disable_pages: 30 # OCR-on runtime climbs sharply around this page count
-liteparse_ocr_auto_min_chars_per_page: 80
liteparse_ocr_auto_sample_pages: 5
pdf_ocr_fallback: true # enable legacy Tesseract fallback after PyMuPDF
thread_offload: true # offload sync parsers to thread pool
diff --git a/plugins/_document_query/helpers/document_query.py b/plugins/_document_query/helpers/document_query.py
index e1553c872..4a7fa897a 100644
--- a/plugins/_document_query/helpers/document_query.py
+++ b/plugins/_document_query/helpers/document_query.py
@@ -40,6 +40,14 @@ def _positive_int(value: Any, default: int) -> int:
return parsed if parsed > 0 else default
+def _nonnegative_int(value: Any, default: int) -> int:
+ try:
+ parsed = int(value)
+ except (TypeError, ValueError):
+ return default
+ return parsed if parsed >= 0 else default
+
+
def _parser_semaphore(config: dict) -> asyncio.Semaphore:
concurrency = _positive_int(
config.get("parser_concurrency"),
@@ -66,6 +74,7 @@ class DocumentQueryStore:
DEFAULT_CHUNK_SIZE = 1000
DEFAULT_CHUNK_OVERLAP = 100
+ DEFAULT_MAX_INDEX_CHUNKS = 1200
@staticmethod
def get(agent: Agent):
@@ -103,12 +112,7 @@ class DocumentQueryStore:
doc_metadata = metadata or {}
doc_metadata["document_uri"] = document_uri
doc_metadata["timestamp"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
- chunk_size = self.config.get("chunk_size", self.DEFAULT_CHUNK_SIZE)
- chunk_overlap = self.config.get("chunk_overlap", self.DEFAULT_CHUNK_OVERLAP)
- text_splitter = RecursiveCharacterTextSplitter(
- chunk_size=chunk_size, chunk_overlap=chunk_overlap
- )
- chunks = text_splitter.split_text(text)
+ chunks = self._split_text_for_index(text)
docs = []
for i, chunk in enumerate(chunks):
chunk_metadata = doc_metadata.copy()
@@ -129,6 +133,51 @@ class DocumentQueryStore:
PrintStyle.error(f"Error adding document '{document_uri}': {err_text}")
return False, []
+ def _split_text_for_index(self, text: str) -> list[str]:
+ chunk_size = _positive_int(
+ self.config.get("chunk_size"),
+ self.DEFAULT_CHUNK_SIZE,
+ )
+ chunk_overlap = min(
+ _nonnegative_int(
+ self.config.get("chunk_overlap"),
+ self.DEFAULT_CHUNK_OVERLAP,
+ ),
+ max(0, chunk_size - 1),
+ )
+ chunks = self._split_text(text, chunk_size, chunk_overlap)
+
+ max_chunks = _nonnegative_int(
+ self.config.get("max_index_chunks"),
+ self.DEFAULT_MAX_INDEX_CHUNKS,
+ )
+ if not max_chunks or len(chunks) <= max_chunks:
+ return chunks
+
+ overlap_ratio = chunk_overlap / chunk_size if chunk_size else 0
+ overlap_ratio = max(0, min(overlap_ratio, 0.5))
+ target_size = max(
+ chunk_size + 1,
+ int(len(text) / max(1, max_chunks * (1 - overlap_ratio))) + 1,
+ )
+
+ for _ in range(8):
+ target_overlap = min(int(target_size * overlap_ratio), target_size - 1)
+ chunks = self._split_text(text, target_size, target_overlap)
+ if len(chunks) <= max_chunks or target_size >= len(text):
+ return chunks
+ target_size = min(len(text), int(target_size * 1.25) + 1)
+
+ return chunks
+
+ @staticmethod
+ def _split_text(text: str, chunk_size: int, chunk_overlap: int) -> list[str]:
+ text_splitter = RecursiveCharacterTextSplitter(
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ )
+ return text_splitter.split_text(text)
+
async def get_document(self, document_uri: str) -> Optional[Document]:
if not self.vector_db:
return None
diff --git a/plugins/_document_query/helpers/parsers/liteparse.py b/plugins/_document_query/helpers/parsers/liteparse.py
index ce6f06523..0ff327571 100644
--- a/plugins/_document_query/helpers/parsers/liteparse.py
+++ b/plugins/_document_query/helpers/parsers/liteparse.py
@@ -14,7 +14,6 @@ from .base import BaseParser
DEFAULT_OCR_AUTO_DISABLE_PAGES = 30
-DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE = 80
DEFAULT_OCR_AUTO_SAMPLE_PAGES = 5
@@ -182,11 +181,7 @@ class LiteParseParser(BaseParser):
if effective_pages < auto_disable_pages:
return False
- min_chars_per_page = _positive_int(
- config.get("liteparse_ocr_auto_min_chars_per_page"),
- DEFAULT_OCR_AUTO_MIN_CHARS_PER_PAGE,
- )
- return (profile.text_chars / profile.sampled_pages) >= min_chars_per_page
+ return True
def _pdf_text_profile(file_path: str, config: dict) -> _PdfTextProfile | None:
diff --git a/plugins/_document_query/hooks.py b/plugins/_document_query/hooks.py
index 45eed56b8..04cb609c8 100644
--- a/plugins/_document_query/hooks.py
+++ b/plugins/_document_query/hooks.py
@@ -15,7 +15,7 @@ from helpers.print_style import PrintStyle
_LOCK = threading.Lock()
_CHECKED = False
_PLUGIN_DIR = Path(__file__).resolve().parent
-_REQUIREMENTS_FILE = _PLUGIN_DIR / "requirements.txt"
+_ROOT_REQUIREMENTS_FILE = _PLUGIN_DIR.parents[1] / "requirements.txt"
def has_liteparse() -> bool:
@@ -66,9 +66,10 @@ def _install_requirements() -> None:
raise RuntimeError(
"Document Query plugin requires 'uv' to install liteparse automatically"
)
- if not _REQUIREMENTS_FILE.is_file():
+ requirement = _liteparse_requirement()
+ if not requirement:
raise RuntimeError(
- f"Document Query requirements file not found: {_REQUIREMENTS_FILE}"
+ f"Document Query LiteParse requirement not found in {_ROOT_REQUIREMENTS_FILE}"
)
cmd = [
@@ -77,9 +78,18 @@ def _install_requirements() -> None:
"install",
"--python",
sys.executable,
- "-r",
- str(_REQUIREMENTS_FILE),
+ requirement,
]
PrintStyle.info("Document Query: liteparse not found, installing plugin dependency")
subprocess.check_call(cmd, cwd=str(_PLUGIN_DIR))
+
+
+def _liteparse_requirement() -> str:
+ if not _ROOT_REQUIREMENTS_FILE.is_file():
+ return ""
+ for line in _ROOT_REQUIREMENTS_FILE.read_text(encoding="utf-8").splitlines():
+ requirement = line.strip()
+ if requirement.startswith("liteparse"):
+ return requirement
+ return ""
diff --git a/plugins/_document_query/webui/config.html b/plugins/_document_query/webui/config.html
index e2a0bbb38..fc889efef 100644
--- a/plugins/_document_query/webui/config.html
+++ b/plugins/_document_query/webui/config.html
@@ -15,6 +15,7 @@
context_intro_chunks: 2,
chunk_size: 1000,
chunk_overlap: 100,
+ max_index_chunks: 1200,
search_threshold: 0.5,
search_limit: 100,
max_remote_bytes: 52428800,
@@ -40,6 +41,7 @@
this.ensureInt('context_intro_chunks', 2, 0);
this.ensureInt('chunk_size', 1000, 100);
this.ensureInt('chunk_overlap', 100, 0);
+ this.ensureInt('max_index_chunks', 1200, 0);
this.ensureInt('search_limit', 100, 1);
this.ensureInt('max_remote_bytes', 52428800, 1);
this.ensureNumber('search_threshold', 0.5, 0, 1);
@@ -172,6 +174,20 @@
</div>
</div>
+ <div class="field">
+ <div class="field-label">
+ <div class="field-title">Max index chunks</div>
+ <div class="field-description">
+ Adapt chunk size when a parsed document would exceed this many indexed chunks. Use 0 for no cap.
+ </div>
+ </div>
+ <div class="field-control">
+ <input type="number" min="0" step="50"
+ @change="ensureInt('max_index_chunks', 1200, 0)"
+ x-model.number="config.max_index_chunks" />
+ </div>
+ </div>
+
<div class="field">
<div class="field-label">
<div class="field-title">Search limit</div>
diff --git a/tests/test_document_query_plugin.py b/tests/test_document_query_plugin.py
index c03475b47..e3ece0ebd 100644
--- a/tests/test_document_query_plugin.py
+++ b/tests/test_document_query_plugin.py
@@ -6,8 +6,12 @@ from pathlib import Path
import pytest
from PIL import Image
+from plugins._document_query import hooks as document_query_hooks
from plugins._document_query.helpers.fetch import FetchedDocument, fetch_public_resource
-from plugins._document_query.helpers.document_query import DocumentQueryHelper
+from plugins._document_query.helpers.document_query import (
+ DocumentQueryHelper,
+ DocumentQueryStore,
+)
from plugins._document_query.helpers.parsers.base import BaseParser
from plugins._document_query.helpers.parsers import get_parsers_for_mimetype
from plugins._document_query.helpers.parsers import liteparse as liteparse_module
@@ -105,12 +109,14 @@ def test_compatibility_imports_point_to_plugin_classes():
def test_liteparse_is_installed_by_docker_and_plugin_hook_requirements():
root_requirements = (ROOT / "requirements.txt").read_text(encoding="utf-8")
- plugin_requirements = (
- ROOT / "plugins" / "_document_query" / "requirements.txt"
+ hooks_source = (
+ ROOT / "plugins" / "_document_query" / "hooks.py"
).read_text(encoding="utf-8")
assert "liteparse==2.0.3" in root_requirements
- assert plugin_requirements.strip().splitlines() == ["liteparse==2.0.3"]
+ assert "_ROOT_REQUIREMENTS_FILE" in hooks_source
+ assert document_query_hooks._liteparse_requirement() == "liteparse==2.0.3"
+ assert not (ROOT / "plugins" / "_document_query" / "requirements.txt").exists()
def test_default_config_bounds_liteparse_runtime_concurrency():
@@ -120,6 +126,7 @@ def test_default_config_bounds_liteparse_runtime_concurrency():
assert "parser_concurrency: 1" in default_config
assert "context_intro_chunks: 2" in default_config
+ assert "max_index_chunks: 1200" in default_config
assert "liteparse_num_workers: 2" in default_config
assert "liteparse_ocr_auto_disable_pages: 30" in default_config
assert "liteparse_subprocess" not in default_config
@@ -137,6 +144,7 @@ def test_config_panel_exposes_document_query_settings():
"gather_timeout",
"chunk_size",
"chunk_overlap",
+ "max_index_chunks",
"search_threshold",
"search_limit",
"context_intro_chunks",
@@ -162,6 +170,32 @@ def test_config_panel_exposes_document_query_settings():
assert "liteparse_subprocess" not in config_html
+def test_document_query_adapts_chunk_size_for_large_documents():
+ store = object.__new__(DocumentQueryStore)
+ store.config = {
+ "chunk_size": 100,
+ "chunk_overlap": 10,
+ "max_index_chunks": 10,
+ }
+
+ chunks = store._split_text_for_index(("alpha beta gamma delta " * 400).strip())
+
+ assert 1 < len(chunks) <= 10
+
+
+def test_document_query_allows_uncapped_index_chunks():
+ store = object.__new__(DocumentQueryStore)
+ store.config = {
+ "chunk_size": 100,
+ "chunk_overlap": 10,
+ "max_index_chunks": 0,
+ }
+
+ chunks = store._split_text_for_index(("alpha beta gamma delta " * 400).strip())
+
+ assert len(chunks) > 10
+
+
def test_document_query_thumbnail_matches_plugin_hub_limits():
thumbnail = ROOT / "plugins" / "_document_query" / "webui" / "thumbnail.jpg"
@@ -251,7 +285,7 @@ def test_liteparse_keeps_ocr_for_small_pdf(monkeypatch):
assert kwargs["ocr_enabled"] is True
-def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):
+def test_liteparse_disables_ocr_for_large_text_sparse_pdf(monkeypatch):
parser = LiteParseParser()
fetched = FetchedDocument(
uri="/tmp/scan.pdf",
@@ -273,7 +307,7 @@ def test_liteparse_keeps_ocr_for_large_text_sparse_pdf(monkeypatch):
kwargs = parser._liteparse_kwargs({}, fetched, "/tmp/scan.pdf")
- assert kwargs["ocr_enabled"] is True
+ assert kwargs["ocr_enabled"] is False
def test_liteparse_respects_explicit_ocr_disabled(monkeypatch):