Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

refactor(template_engine): 简化选项解析逻辑并移除冗余代码

yifanfeng97f4914185dcf7
diff --git a/hyperextract/utils/template_engine/factory.py b/hyperextract/utils/template_engine/factory.py
index b636d6c..7d194fe 100644
--- a/hyperextract/utils/template_engine/factory.py
+++ b/hyperextract/utils/template_engine/factory.py
@@ -10,12 +10,12 @@ from langchain_core.embeddings import Embeddings
from .parsers import (
TemplateCfg,
- OptionsBuilder,
Options,
parse_output,
IdentifierResolver,
parse_guideline,
localize_template,
+ parse_option,
)
@@ -89,12 +89,6 @@ class TemplateFactory:
data_schema=data_schema,
llm_client=llm_client,
embedder=embedder,
- strategy_or_merger=OptionsBuilder.resolve_merge_strategy(
- options.merge_strategy,
- lambda x: str(x),
- llm_client,
- data_schema,
- ),
prompt=prompt,
chunk_size=options.chunk_size,
chunk_overlap=options.chunk_overlap,
diff --git a/hyperextract/utils/template_engine/parsers/__init__.py b/hyperextract/utils/template_engine/parsers/__init__.py
index c615f90..2920fdd 100644
--- a/hyperextract/utils/template_engine/parsers/__init__.py
+++ b/hyperextract/utils/template_engine/parsers/__init__.py
@@ -8,10 +8,7 @@ from .loader import (
from .output import parse_output
from .guideline import parse_guideline
from .identifiers import IdentifierResolver
-from .options import (
- Options,
- OptionsBuilder,
-)
+from .options import parse_option
__all__ = [
@@ -21,6 +18,5 @@ __all__ = [
"parse_output",
"parse_guideline",
"IdentifierResolver",
- "Options",
- "OptionsBuilder",
+ "parse_option",
]
diff --git a/hyperextract/utils/template_engine/parsers/identifiers.py b/hyperextract/utils/template_engine/parsers/identifiers.py
index eedc93f..2302515 100644
--- a/hyperextract/utils/template_engine/parsers/identifiers.py
+++ b/hyperextract/utils/template_engine/parsers/identifiers.py
@@ -1,6 +1,6 @@
"""Identifier models and resolver."""
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import Callable, Dict, List, Optional, Tuple, Union
from pydantic import BaseModel
diff --git a/hyperextract/utils/template_engine/parsers/options.py b/hyperextract/utils/template_engine/parsers/options.py
index 3ce4066..605c20d 100644
--- a/hyperextract/utils/template_engine/parsers/options.py
+++ b/hyperextract/utils/template_engine/parsers/options.py
@@ -1,447 +1,142 @@
-"""Options models and builder."""
+"""Options models and parser."""
-from typing import Callable, List, Optional, Type
+from typing import List
from pydantic import BaseModel
from ontomem.merger import MergeStrategy
+from .schemas import (
+ VALID_AUTOTYPES,
+ VALID_MERGE_STRATEGIES,
+ NaiveOptionsSchema,
+ GraphOptionsSchema,
+)
-from .output import parse_output
-from .identifiers import IdentifierResolver
+
+def resolve_merge_strategy(strategy_name: VALID_MERGE_STRATEGIES) -> MergeStrategy:
+ """Resolve merge strategy from string to MergeStrategy enum."""
+ if "llm" in strategy_name:
+ parts = strategy_name.split("_")
+ if len(parts) == 2 and parts[0] in ["llm"]:
+ return getattr(getattr(MergeStrategy, parts[0].upper()), parts[1].upper())
+ else:
+ return MergeStrategy[strategy_name.upper()]
class Options(BaseModel):
- """Runtime options configuration - excludes LLM, Embedder, Schema, Prompt."""
-
- merge_strategy: Optional[str] = None
- entity_merge_strategy: Optional[str] = None
- relation_merge_strategy: Optional[str] = None
- extraction_mode: Optional[str] = None
- observation_time: Optional[str] = None
- observation_location: Optional[str] = None
- chunk_size: int = 2048
- chunk_overlap: int = 256
- max_workers: int = 10
- verbose: bool = False
- fields_for_search: Optional[List[str]] = None
- entity_fields_for_search: Optional[List[str]] = None
- relation_fields_for_search: Optional[List[str]] = None
-
- def get_merge_strategy(self) -> Optional[str]:
- """Get merge strategy."""
- return self.merge_strategy or self.entity_merge_strategy
-
-
-class ModelOptions:
- """Options for AutoModel."""
-
- def __init__(
- self,
- merge_strategy: str = "llm_balanced",
- chunk_size: int = 2048,
- chunk_overlap: int = 256,
- max_workers: int = 10,
- verbose: bool = False,
- ):
- self.merge_strategy = merge_strategy
- self.chunk_size = chunk_size
- self.chunk_overlap = chunk_overlap
- self.max_workers = max_workers
- self.verbose = verbose
-
-
-class ListOptions:
- """Options for AutoList."""
-
- def __init__(
- self,
- chunk_size: int = 2048,
- chunk_overlap: int = 256,
- max_workers: int = 10,
- verbose: bool = False,
- ):
- self.chunk_size = chunk_size
- self.chunk_overlap = chunk_overlap
- self.max_workers = max_workers
- self.verbose = verbose
-
-
-class SetOptions:
- """Options for AutoSet."""
-
- def __init__(
- self,
- merge_strategy: str = "llm_balanced",
- chunk_size: int = 2048,
- chunk_overlap: int = 256,
- max_workers: int = 10,
- verbose: bool = False,
- ):
- self.merge_strategy = merge_strategy
- self.chunk_size = chunk_size
- self.chunk_overlap = chunk_overlap
- self.max_workers = max_workers
- self.verbose = verbose
-
-
-class GraphOptions:
- """Options for AutoGraph/AutoHypergraph."""
-
- def __init__(
- self,
- extraction_mode: str = "two_stage",
- entity_merge_strategy: str = "llm_balanced",
- relation_merge_strategy: str = "llm_balanced",
- chunk_size: int = 2048,
- chunk_overlap: int = 256,
- max_workers: int = 10,
- verbose: bool = False,
- entity_fields_for_search: Optional[List[str]] = None,
- relation_fields_for_search: Optional[List[str]] = None,
- ):
- self.extraction_mode = extraction_mode
- self.entity_merge_strategy = entity_merge_strategy
- self.relation_merge_strategy = relation_merge_strategy
- self.chunk_size = chunk_size
- self.chunk_overlap = chunk_overlap
- self.max_workers = max_workers
- self.verbose = verbose
- self.entity_fields_for_search = entity_fields_for_search
- self.relation_fields_for_search = relation_fields_for_search
-
-
-class TemporalGraphOptions(GraphOptions):
- """Options for AutoTemporalGraph."""
-
- def __init__(
- self,
- extraction_mode: str = "two_stage",
- entity_merge_strategy: str = "llm_balanced",
- relation_merge_strategy: str = "llm_balanced",
- observation_time: Optional[str] = None,
- chunk_size: int = 2048,
- chunk_overlap: int = 256,
- max_workers: int = 10,
- verbose: bool = False,
- entity_fields_for_search: Optional[List[str]] = None,
- relation_fields_for_search: Optional[List[str]] = None,
- ):
- super().__init__(
- extraction_mode=extraction_mode,
- entity_merge_strategy=entity_merge_strategy,
- relation_merge_strategy=relation_merge_strategy,
- chunk_size=chunk_size,
- chunk_overlap=chunk_overlap,
- max_workers=max_workers,
- verbose=verbose,
- entity_fields_for_search=entity_fields_for_search,
- relation_fields_for_search=relation_fields_for_search,
- )
- self.observation_time = observation_time
-
-
-class SpatialGraphOptions(GraphOptions):
- """Options for AutoSpatialGraph."""
-
- def __init__(
- self,
- extraction_mode: str = "two_stage",
- entity_merge_strategy: str = "llm_balanced",
- relation_merge_strategy: str = "llm_balanced",
- observation_location: Optional[str] = None,
- chunk_size: int = 2048,
- chunk_overlap: int = 256,
- max_workers: int = 10,
- verbose: bool = False,
- entity_fields_for_search: Optional[List[str]] = None,
- relation_fields_for_search: Optional[List[str]] = None,
- ):
- super().__init__(
- extraction_mode=extraction_mode,
- entity_merge_strategy=entity_merge_strategy,
- relation_merge_strategy=relation_merge_strategy,
- chunk_size=chunk_size,
- chunk_overlap=chunk_overlap,
- max_workers=max_workers,
- verbose=verbose,
- entity_fields_for_search=entity_fields_for_search,
- relation_fields_for_search=relation_fields_for_search,
- )
- self.observation_location = observation_location
-
-
-class SpatioTemporalGraphOptions(TemporalGraphOptions):
- """Options for AutoSpatioTemporalGraph."""
-
- def __init__(
- self,
- extraction_mode: str = "two_stage",
- entity_merge_strategy: str = "llm_balanced",
- relation_merge_strategy: str = "llm_balanced",
- observation_time: Optional[str] = None,
- observation_location: Optional[str] = None,
- chunk_size: int = 2048,
- chunk_overlap: int = 256,
- max_workers: int = 10,
- verbose: bool = False,
- entity_fields_for_search: Optional[List[str]] = None,
- relation_fields_for_search: Optional[List[str]] = None,
- ):
- super().__init__(
- extraction_mode=extraction_mode,
- entity_merge_strategy=entity_merge_strategy,
- relation_merge_strategy=relation_merge_strategy,
- observation_time=observation_time,
- chunk_size=chunk_size,
- chunk_overlap=chunk_overlap,
- max_workers=max_workers,
- verbose=verbose,
- entity_fields_for_search=entity_fields_for_search,
- relation_fields_for_search=relation_fields_for_search,
- )
- self.observation_location = observation_location
-
-
-class OptionsBuilder:
- """Options Builder - handles option parsing, validation, merge strategy resolution."""
-
- @staticmethod
- def get_language(config) -> str:
- """Get language setting."""
- lang = config.language
- if isinstance(lang, list):
- return lang[0] if lang else "zh"
- return lang or "zh"
-
- @staticmethod
- def get_extraction_mode(config) -> str:
- """Get extraction mode."""
- return config.options.extraction_mode if config.options else "two_stage"
-
- VALID_MERGE_STRATEGIES = {
- "merge_field",
- "keep_incoming",
- "keep_existing",
- "llm_balanced",
- "llm_prefer_incoming",
- "llm_prefer_existing",
+ """Unified Options - covers all possible configuration for all autotypes."""
+
+ chunk_size: int | None = None
+ chunk_overlap: int | None = None
+ max_workers: int | None = None
+ verbose: bool | None = None
+
+ strategy_or_merger: str | None = None
+ fields_for_index: List[str] | None = None
+
+ extraction_mode: str | None = None
+ node_strategy_or_merger: str | None = None
+ edge_strategy_or_merger: str | None = None
+ node_fields_for_index: List[str] | None = None
+ edge_fields_for_index: List[str] | None = None
+
+ observation_time: str | None = None
+ observation_location: str | None = None
+
+
+COMMON_PARAMS = ("chunk_size", "chunk_overlap", "max_workers", "verbose")
+
+MODEL_PARAMS = ("strategy_or_merger",)
+LIST_PARAMS = ("fields_for_index",)
+SET_PARAMS = ("strategy_or_merger", "fields_for_index")
+
+GRAPH_PARAMS = (
+ "extraction_mode",
+ "node_strategy_or_merger",
+ "edge_strategy_or_merger",
+ "node_fields_for_index",
+ "edge_fields_for_index",
+)
+
+YAML_TO_AUTOTYPE_MAPPING = {
+ "merge_strategy": "strategy_or_merger",
+ "entity_merge_strategy": "node_strategy_or_merger",
+ "relation_merge_strategy": "edge_strategy_or_merger",
+ "entity_fields_for_search": "node_fields_for_index",
+ "relation_fields_for_search": "edge_fields_for_index",
+}
+
+
+def parse_option(
+ options: NaiveOptionsSchema | GraphOptionsSchema,
+ autotype: VALID_AUTOTYPES,
+ override: dict | None = None,
+) -> dict:
+ """Parse options and return kwargs for specific autotype.
+
+ Args:
+ options: Unified Options configuration
+ autotype: auto type (model, list, set, graph, hypergraph, temporal_graph, spatial_graph, spatio_temporal_graph)
+ override: Optional override for specific parameters
+
+ Returns:
+ dict: Only contains parameters needed for the corresponding autotype
+ """
+ # Map YAML parameter names to autotype parameter names
+ options_dict = options.model_dump()
+ options_dict = {
+ YAML_TO_AUTOTYPE_MAPPING.get(k, k): v for k, v in options_dict.items()
}
+ if override:
+ options_dict.update(override)
+ options = Options(**options_dict)
- @staticmethod
- def resolve_merge_strategy(
- strategy_name: Optional[str],
- key_extractor: Callable,
- llm_client,
- item_schema: Type,
- ):
- """Resolve merge strategy."""
- if strategy_name:
- strategy_name = strategy_name.lower()
- if strategy_name not in OptionsBuilder.VALID_MERGE_STRATEGIES:
- raise ValueError(
- f"Invalid merge strategy '{strategy_name}'. "
- f"Valid options: {OptionsBuilder.VALID_MERGE_STRATEGIES}"
- )
- # 处理下划线格式,转换为点格式
- if '_' in strategy_name and '.' not in strategy_name:
- parts = strategy_name.split('_')
- if len(parts) == 2 and parts[0] in ['llm']:
- return getattr(getattr(MergeStrategy, parts[0].upper()), parts[1].upper())
- elif '.' in strategy_name:
- parts = strategy_name.split('.')
- return getattr(getattr(MergeStrategy, parts[0].upper()), parts[1].upper())
- else:
- return MergeStrategy[strategy_name.upper()]
- return MergeStrategy.LLM.BALANCED
-
- @classmethod
- def build_model_options(cls, config):
- """Build options for AutoModel."""
- options = config.options or Options()
- return ModelOptions(
- merge_strategy=options.merge_strategy or "llm_balanced",
- chunk_size=options.chunk_size,
- chunk_overlap=options.chunk_overlap,
- max_workers=options.max_workers,
- verbose=options.verbose,
- )
+ if autotype in ("model",):
+ return _build_kwargs(options, MODEL_PARAMS, COMMON_PARAMS)
- @classmethod
- def build_list_options(cls, config):
- """Build options for AutoList."""
- options = config.options or Options()
- return ListOptions(
- chunk_size=options.chunk_size,
- chunk_overlap=options.chunk_overlap,
- max_workers=options.max_workers,
- verbose=options.verbose,
- )
+ if autotype in ("list",):
+ return _build_kwargs(options, LIST_PARAMS, COMMON_PARAMS)
- @classmethod
- def build_set_options(cls, config):
- """Build options for AutoSet."""
- options = config.options or Options()
- return SetOptions(
- merge_strategy=options.merge_strategy or "llm_balanced",
- chunk_size=options.chunk_size,
- chunk_overlap=options.chunk_overlap,
- max_workers=options.max_workers,
- verbose=options.verbose,
- )
+ if autotype in ("set",):
+ return _build_kwargs(options, SET_PARAMS, COMMON_PARAMS)
- @classmethod
- def build_graph_options(cls, config):
- """Build options for AutoGraph/AutoHypergraph."""
- options = config.options or Options()
- extraction_mode = cls.get_extraction_mode(config)
- return GraphOptions(
- extraction_mode=extraction_mode,
- entity_merge_strategy=options.entity_merge_strategy or "llm_balanced",
- relation_merge_strategy=options.relation_merge_strategy or "llm_balanced",
- chunk_size=options.chunk_size,
- chunk_overlap=options.chunk_overlap,
- max_workers=options.max_workers,
- verbose=options.verbose,
- entity_fields_for_search=options.entity_fields_for_search,
- relation_fields_for_search=options.relation_fields_for_search,
- )
+ if autotype in ("graph", "hypergraph"):
+ return _build_kwargs(options, GRAPH_PARAMS, COMMON_PARAMS)
- @classmethod
- def build_temporal_graph_options(cls, config):
- """Build options for AutoTemporalGraph."""
- options = config.options or Options()
- extraction_mode = cls.get_extraction_mode(config)
- return TemporalGraphOptions(
- extraction_mode=extraction_mode,
- entity_merge_strategy=options.entity_merge_strategy or "llm_balanced",
- relation_merge_strategy=options.relation_merge_strategy or "llm_balanced",
- observation_time=options.observation_time,
- chunk_size=options.chunk_size,
- chunk_overlap=options.chunk_overlap,
- max_workers=options.max_workers,
- verbose=options.verbose,
- entity_fields_for_search=options.entity_fields_for_search,
- relation_fields_for_search=options.relation_fields_for_search,
+ if autotype in ("temporal_graph",):
+ return _build_kwargs(
+ options, ("observation_time",) + GRAPH_PARAMS, COMMON_PARAMS
)
- @classmethod
- def build_spatial_graph_options(cls, config):
- """Build options for AutoSpatialGraph."""
- options = config.options or Options()
- extraction_mode = cls.get_extraction_mode(config)
- return SpatialGraphOptions(
- extraction_mode=extraction_mode,
- entity_merge_strategy=options.entity_merge_strategy or "llm_balanced",
- relation_merge_strategy=options.relation_merge_strategy or "llm_balanced",
- observation_location=options.observation_location,
- chunk_size=options.chunk_size,
- chunk_overlap=options.chunk_overlap,
- max_workers=options.max_workers,
- verbose=options.verbose,
- entity_fields_for_search=options.entity_fields_for_search,
- relation_fields_for_search=options.relation_fields_for_search,
+ if autotype in ("spatial_graph",):
+ return _build_kwargs(
+ options, ("observation_location",) + GRAPH_PARAMS, COMMON_PARAMS
)
- @classmethod
- def build_spatio_temporal_graph_options(cls, config):
- """Build options for AutoSpatioTemporalGraph."""
- options = config.options or Options()
- extraction_mode = cls.get_extraction_mode(config)
- return SpatioTemporalGraphOptions(
- extraction_mode=extraction_mode,
- entity_merge_strategy=options.entity_merge_strategy or "llm_balanced",
- relation_merge_strategy=options.relation_merge_strategy or "llm_balanced",
- observation_time=options.observation_time,
- observation_location=options.observation_location,
- chunk_size=options.chunk_size,
- chunk_overlap=options.chunk_overlap,
- max_workers=options.max_workers,
- verbose=options.verbose,
- entity_fields_for_search=options.entity_fields_for_search,
- relation_fields_for_search=options.relation_fields_for_search,
+ if autotype in ("spatio_temporal_graph",):
+ return _build_kwargs(
+ options,
+ ("observation_time", "observation_location") + GRAPH_PARAMS,
+ COMMON_PARAMS,
)
- @classmethod
- def validate_model_config(cls, config):
- """Validate AutoModel configuration."""
- schema_dict = parse_output(config)
- if schema_dict.get("schema") is None:
- raise ValueError(f"AutoModel requires schema definition: {config.name}")
- return schema_dict
-
- @classmethod
- def validate_list_config(cls, config):
- """Validate AutoList configuration."""
- schema_dict = parse_output(config)
- if schema_dict.get("item_schema") is None:
- raise ValueError(f"AutoList requires item_schema definition: {config.name}")
- return schema_dict
-
- @classmethod
- def validate_set_config(cls, config):
- """Validate AutoSet configuration."""
- schema_dict = parse_output(config)
- if schema_dict.get("item_schema") is None:
- raise ValueError(f"AutoSet requires item_schema definition: {config.name}")
-
- identifiers = IdentifierResolver.resolve_all(config)
- if identifiers.get("item_id_extractor") is None:
- raise ValueError(f"AutoSet requires item_id configuration: {config.name}")
-
- return schema_dict, identifiers
-
- @classmethod
- def validate_graph_config(cls, config):
- """Validate AutoGraph/AutoHypergraph configuration."""
- schema_dict = parse_output(config)
- if schema_dict.get("entity_schema") is None or schema_dict.get("relation_schema") is None:
- raise ValueError(f"AutoGraph requires entity_schema and relation_schema definition: {config.name}")
-
- identifiers = IdentifierResolver.resolve_all(config)
- required = ["entity_key_extractor", "relation_key_extractor", "entities_in_relation_extractor"]
- if not all(identifiers.get(k) for k in required):
- raise ValueError(f"AutoGraph requires complete identifiers configuration: {config.name}")
-
- return schema_dict, identifiers
-
- @classmethod
- def validate_temporal_graph_config(cls, config):
- """Validate AutoTemporalGraph configuration."""
- schema_dict, identifiers = cls.validate_graph_config(config)
-
- if identifiers.get("time_in_relation_extractor") is None:
- raise ValueError(f"AutoTemporalGraph requires time_field in identifiers: {config.name}")
-
- return schema_dict, identifiers
-
- @classmethod
- def validate_spatial_graph_config(cls, config):
- """Validate AutoSpatialGraph configuration."""
- schema_dict, identifiers = cls.validate_graph_config(config)
-
- if identifiers.get("location_in_relation_extractor") is None:
- raise ValueError(f"AutoSpatialGraph requires location_field in identifiers: {config.name}")
-
- return schema_dict, identifiers
- @classmethod
- def validate_spatio_temporal_graph_config(cls, config):
- """Validate AutoSpatioTemporalGraph configuration."""
- schema_dict, identifiers = cls.validate_graph_config(config)
+def _build_kwargs(
+ options: Options, specific_params: tuple, common_params: tuple
+) -> dict:
+ """Build kwargs, only returning non-None parameters."""
+ kwargs = {}
+ all_params = specific_params + common_params
- if not identifiers.get("time_in_relation_extractor") or not identifiers.get("location_in_relation_extractor"):
- raise ValueError(f"AutoSpatioTemporalGraph requires time_field and location_field in identifiers: {config.name}")
+ for param in all_params:
+ value = getattr(options, param, None)
+ if value is not None:
+ if "strategy_or_merger" in param:
+ value = resolve_merge_strategy(value)
+ kwargs[param] = value
- return schema_dict, identifiers
+ return kwargs
__all__ = [
- "Options",
- "ModelOptions",
- "ListOptions",
- "SetOptions",
- "GraphOptions",
- "TemporalGraphOptions",
- "SpatialGraphOptions",
- "SpatioTemporalGraphOptions",
- "OptionsBuilder",
+ "parse_option",
]