Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

fix: reject template options where chunk_overlap is not smaller than chunk_size

Frankie-Xu28990afbdc2c
diff --git a/hyperextract/utils/template_engine/parsers/options.py b/hyperextract/utils/template_engine/parsers/options.py
index 5525f31..58e55e7 100644
--- a/hyperextract/utils/template_engine/parsers/options.py
+++ b/hyperextract/utils/template_engine/parsers/options.py
@@ -1,7 +1,7 @@
"""Options models and parser."""
from ontomem.merger import MergeStrategy
-from pydantic import BaseModel
+from pydantic import BaseModel, ValidationError, model_validator
from .schemas import (
VALID_AUTOTYPES,
@@ -41,6 +41,34 @@ class Options(BaseModel):
observation_time: str | None = None
observation_location: str | None = None
+ @model_validator(mode="after")
+ def _validate_chunk_options(self) -> "Options":
+ """Reject YAML options that would make character chunking collapse."""
+ if self.chunk_size is not None and self.chunk_size < 1:
+ raise ValueError(
+ f"options.chunk_size must be >= 1 when set (got {self.chunk_size})"
+ )
+ if self.chunk_overlap is not None and self.chunk_overlap < 0:
+ raise ValueError(
+ "options.chunk_overlap must be >= 0 when set "
+ f"(got {self.chunk_overlap})"
+ )
+ if (
+ self.chunk_size is not None
+ and self.chunk_overlap is not None
+ and self.chunk_overlap >= self.chunk_size
+ ):
+ raise ValueError(
+ "options.chunk_overlap must be smaller than options.chunk_size "
+ f"(got chunk_overlap={self.chunk_overlap}, "
+ f"chunk_size={self.chunk_size})"
+ )
+ if self.max_workers is not None and self.max_workers < 1:
+ raise ValueError(
+ f"options.max_workers must be >= 1 when set (got {self.max_workers})"
+ )
+ return self
+
COMMON_PARAMS = ("chunk_size", "chunk_overlap", "max_workers", "verbose")
@@ -89,7 +117,10 @@ def parse_option(
}
if override:
options_dict.update(override)
- options = Options(**options_dict)
+ try:
+ options = Options(**options_dict)
+ except ValidationError as exc:
+ raise ValueError(_author_options_error(exc)) from None
if autotype in ("model",):
return _build_kwargs(options, MODEL_PARAMS, COMMON_PARAMS)
@@ -121,6 +152,17 @@ def parse_option(
)
+def _author_options_error(exc: ValidationError) -> str:
+ """Flatten Pydantic errors into a template-author-facing sentence."""
+ messages: list[str] = []
+ for err in exc.errors():
+ msg = err.get("msg", "").removeprefix("Value error, ")
+ if msg:
+ messages.append(msg)
+ detail = "; ".join(messages) if messages else str(exc)
+ return f"Invalid template options: {detail}"
+
+
def _build_kwargs(
options: Options, specific_params: tuple, common_params: tuple
) -> dict:
diff --git a/tests/template_engine/test_parser_options.py b/tests/template_engine/test_parser_options.py
index 8a6788f..71dd487 100644
--- a/tests/template_engine/test_parser_options.py
+++ b/tests/template_engine/test_parser_options.py
@@ -1,13 +1,19 @@
"""Unit tests for template parsers - options (merge strategy resolution)."""
+from pathlib import Path
from typing import get_args
import pytest
+import hyperextract
+from hyperextract.utils.template_engine.parsers import load_template, parse_option
from hyperextract.utils.template_engine.parsers.options import resolve_merge_strategy
from hyperextract.utils.template_engine.parsers.schemas.base import (
VALID_MERGE_STRATEGIES,
)
+from hyperextract.utils.template_engine.parsers.schemas.naive import NaiveOptionsSchema
+
+PRESETS_DIR = Path(hyperextract.__file__).parent / "templates" / "presets"
class TestResolveMergeStrategy:
@@ -31,3 +37,66 @@ class TestResolveMergeStrategy:
expected = getattr(MergeStrategy.LLM, strategy[len("llm_") :].upper())
assert resolve_merge_strategy(strategy) == expected
+
+
+class TestParseOptionChunkValidation:
+ """YAML ``options.chunk_size`` / ``chunk_overlap`` cross-checks."""
+
+ def test_valid_chunk_size_and_overlap_enter_kwargs(self):
+ options = NaiveOptionsSchema(chunk_size=2048, chunk_overlap=256)
+ kwargs = parse_option(options, "model")
+ assert kwargs["chunk_size"] == 2048
+ assert kwargs["chunk_overlap"] == 256
+
+ def test_overlap_equal_to_size_fails(self):
+ options = NaiveOptionsSchema(chunk_size=2048, chunk_overlap=2048)
+ with pytest.raises(ValueError, match="chunk_overlap"):
+ parse_option(options, "model")
+
+ def test_overlap_greater_than_size_fails(self):
+ options = NaiveOptionsSchema(chunk_size=256, chunk_overlap=4096)
+ with pytest.raises(ValueError, match="chunk_overlap"):
+ parse_option(options, "model")
+
+ @pytest.mark.parametrize("chunk_size", [0, -1])
+ def test_chunk_size_not_positive_fails(self, chunk_size):
+ options = NaiveOptionsSchema(chunk_size=chunk_size)
+ with pytest.raises(ValueError, match="chunk_size"):
+ parse_option(options, "model")
+
+ def test_negative_overlap_fails(self):
+ options = NaiveOptionsSchema(chunk_overlap=-1)
+ with pytest.raises(ValueError, match="chunk_overlap"):
+ parse_option(options, "model")
+
+ def test_size_only_with_overlap_none_passes(self):
+ options = NaiveOptionsSchema(chunk_size=2048, chunk_overlap=None)
+ kwargs = parse_option(options, "model")
+ assert kwargs["chunk_size"] == 2048
+ assert "chunk_overlap" not in kwargs
+
+ def test_both_chunk_fields_default_pass(self):
+ kwargs = parse_option(NaiveOptionsSchema(), "model")
+ assert "chunk_size" not in kwargs
+ assert "chunk_overlap" not in kwargs
+
+ def test_options_none_returns_empty_dict(self):
+ assert parse_option(None, "model") == {}
+
+
+class TestParseOptionPresets:
+ """Bundled presets, including education, must still parse after load_template."""
+
+ @pytest.mark.parametrize(
+ "relpath",
+ [
+ "education/course_concept_graph.yaml",
+ "education/curriculum_structure.yaml",
+ "general/workflow_graph.yaml",
+ "general/base_graph.yaml",
+ "general/base_model.yaml",
+ ],
+ )
+ def test_preset_load_and_parse_option(self, relpath):
+ cfg = load_template(PRESETS_DIR / relpath)
+ parse_option(cfg.options, cfg.type)