Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

feat: 新增 AutoSet、AutoGraph 和 AutoHypergraph 的索引字段配置,优化向量索引功能

yifanfeng971c4e49444f82
diff --git a/hyperextract/core/set.py b/hyperextract/core/set.py
index 3ac90f1..92ca987 100644
--- a/hyperextract/core/set.py
+++ b/hyperextract/core/set.py
@@ -96,6 +96,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
chunk_overlap: int = 200,
max_workers: int = 10,
show_progress: bool = True,
+ fields_for_index: List[str] | None = None,
**kwargs: Any,
):
"""Initialize AutoSet with key extractor and merge strategy.
@@ -113,12 +114,26 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
chunk_overlap: Overlapping characters between adjacent chunks.
max_workers: Maximum concurrent extraction tasks.
show_progress: Whether to log progress information.
+ fields_for_index: Optional list of field names in item_schema to include in vector index.
+ If None, all text fields are indexed by default.
+ Useful for optimizing search on complex schemas.
+ Example: ['name', 'summary'] (only index these fields)
**kwargs: Additional arguments passed to create_merger() when strategy_or_merger is
a MergeStrategy enum. Ignored if strategy_or_merger is a BaseMerger instance.
"""
- # Store item_schema
+ # Store item_schema and index config
self.item_schema = item_schema
+ self.fields_for_index = fields_for_index
+
+ # Validate fields_for_index if provided
+ if self.fields_for_index:
+ for field_name in self.fields_for_index:
+ if field_name not in item_schema.model_fields:
+ raise ValueError(
+ f"Field '{field_name}' not found in item schema '{item_schema.__name__}'. "
+ f"Available fields: {list(item_schema.model_fields.keys())}"
+ )
# Create AutoSetSchema container dynamically (similar to AutoList's AutoListSchema)
container_name = f"{item_schema.__name__}Set"
@@ -161,6 +176,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
embedder=embedder,
strategy_or_merger=self._merger,
verbose=show_progress,
+ fields_for_index=fields_for_index, # Pass field selection to OMem
)
super().__init__(
@@ -182,7 +198,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
Overrides parent method to include AutoSet-specific parameters.
Returns:
- New AutoSet instance.
+ New AutoSet instance with identical configuration.
"""
return self.__class__(
item_schema=self.item_schema,
@@ -195,6 +211,7 @@ class AutoSet(BaseAutoType[AutoSetSchema[Item]], Generic[Item]):
chunk_overlap=self.chunk_overlap,
max_workers=self.max_workers,
show_progress=self.show_progress,
+ fields_for_index=self.fields_for_index, # Persist index field configuration
)
def _default_prompt(self) -> str:
diff --git a/hyperextract/graphs/base.py b/hyperextract/graphs/base.py
index aa1148e..4f93116 100644
--- a/hyperextract/graphs/base.py
+++ b/hyperextract/graphs/base.py
@@ -122,6 +122,8 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
chunk_overlap: int = 200,
max_workers: int = 10,
show_progress: bool = True,
+ node_fields_for_index: List[str] | None = None,
+ edge_fields_for_index: List[str] | None = None,
**kwargs: Any,
):
"""Initialize AutoGraph with node/edge schemas and configuration.
@@ -144,6 +146,12 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
chunk_overlap: Overlapping characters between chunks.
max_workers: Maximum concurrent extraction tasks.
show_progress: Whether to log progress.
+ node_fields_for_index: Optional list of field names in node_schema to include in vector index.
+ If None, all text fields are indexed by default.
+ Example: ['name', 'description'] (only index these node fields)
+ edge_fields_for_index: Optional list of field names in edge_schema to include in vector index.
+ If None, all text fields are indexed by default.
+ Example: ['relation_type', 'description'] (only index these edge fields)
**kwargs: Additional arguments passed to create_merger() when strategy_or_merger is
a MergeStrategy enum. Ignored if strategy_or_merger is a BaseMerger instance.
"""
@@ -154,6 +162,26 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
self.edge_key_extractor = edge_key_extractor
self.nodes_in_edge_extractor = nodes_in_edge_extractor
self.extraction_mode = extraction_mode
+ self.node_fields_for_index = node_fields_for_index
+ self.edge_fields_for_index = edge_fields_for_index
+
+ # Validate node fields for index
+ if self.node_fields_for_index:
+ for field_name in self.node_fields_for_index:
+ if field_name not in node_schema.model_fields:
+ raise ValueError(
+ f"Field '{field_name}' not found in node schema '{node_schema.__name__}'. "
+ f"Available fields: {list(node_schema.model_fields.keys())}"
+ )
+
+ # Validate edge fields for index
+ if self.edge_fields_for_index:
+ for field_name in self.edge_fields_for_index:
+ if field_name not in edge_schema.model_fields:
+ raise ValueError(
+ f"Field '{field_name}' not found in edge schema '{edge_schema.__name__}'. "
+ f"Available fields: {list(edge_schema.model_fields.keys())}"
+ )
# Initialize prompts (use custom if provided, otherwise use defaults)
self.node_prompt = prompt_for_node_extraction or self._default_node_prompt()
@@ -215,6 +243,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
embedder=embedder,
strategy_or_merger=self.node_merger,
verbose=show_progress,
+ fields_for_index=node_fields_for_index, # Pass node field selection to OMem
)
self._edge_memory = OMem(
@@ -224,6 +253,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
embedder=embedder,
strategy_or_merger=self.edge_merger,
verbose=show_progress,
+ fields_for_index=edge_fields_for_index, # Pass edge field selection to OMem
)
# Call parent init
@@ -268,6 +298,8 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
chunk_overlap=self.chunk_overlap,
max_workers=self.max_workers,
show_progress=self.show_progress,
+ node_fields_for_index=self.node_fields_for_index, # Persist node index field configuration
+ edge_fields_for_index=self.edge_fields_for_index, # Persist edge index field configuration
)
@property
diff --git a/hyperextract/hypergraphs/base.py b/hyperextract/hypergraphs/base.py
index 291a8cd..3047367 100644
--- a/hyperextract/hypergraphs/base.py
+++ b/hyperextract/hypergraphs/base.py
@@ -118,6 +118,8 @@ class AutoHypergraph(
chunk_overlap: int = 200,
max_workers: int = 10,
show_progress: bool = True,
+ node_fields_for_index: List[str] | None = None,
+ edge_fields_for_index: List[str] | None = None,
**kwargs: Any,
):
"""Initialize AutoHypergraph with node/edge schemas and configuration.
@@ -147,6 +149,12 @@ class AutoHypergraph(
chunk_overlap: Overlapping characters between chunks.
max_workers: Maximum concurrent extraction tasks.
show_progress: Whether to log progress.
+ node_fields_for_index: Optional list of field names in node_schema to include in vector index.
+ If None, all text fields are indexed by default.
+ Example: ['name', 'properties'] (only index these node fields)
+ edge_fields_for_index: Optional list of field names in edge_schema to include in vector index.
+ If None, all text fields are indexed by default.
+ Example: ['summary', 'type'] (only index these edge fields)
**kwargs: Additional arguments for merger creation.
"""
# Store schemas and extractors
@@ -160,6 +168,28 @@ class AutoHypergraph(
self.chunk_overlap = chunk_overlap
self.max_workers = max_workers
self.show_progress = show_progress
+ self.node_fields_for_index = node_fields_for_index
+ self.edge_fields_for_index = edge_fields_for_index
+
+ # Validate node fields for index
+ if self.node_fields_for_index:
+ for field_name in self.node_fields_for_index:
+ if field_name not in node_schema.model_fields:
+ raise ValueError(
+ f"Field '{field_name}' not found in node schema '{node_schema.__name__}'. "
+ f"Available fields: {list(node_schema.model_fields.keys())}"
+ )
+
+ # Validate edge fields for index
+ if self.edge_fields_for_index:
+ for field_name in self.edge_fields_for_index:
+ if field_name not in edge_schema.model_fields:
+ raise ValueError(
+ f"Field '{field_name}' not found in edge schema '{edge_schema.__name__}'. "
+ f"Available fields: {list(edge_schema.model_fields.keys())}"
+ )
+ self.max_workers = max_workers
+ self.show_progress = show_progress
# Initialize prompts
self.node_prompt = prompt_for_node_extraction or self._default_node_prompt()
@@ -215,6 +245,7 @@ class AutoHypergraph(
embedder=embedder,
strategy_or_merger=self.node_merger,
verbose=show_progress,
+ fields_for_index=node_fields_for_index, # Pass node field selection to OMem
)
self._edge_memory = OMem(
@@ -224,6 +255,7 @@ class AutoHypergraph(
embedder=embedder,
strategy_or_merger=self.edge_merger,
verbose=show_progress,
+ fields_for_index=edge_fields_for_index, # Pass edge field selection to OMem
)
# Initialize Base Class
@@ -312,6 +344,8 @@ class AutoHypergraph(
chunk_overlap=self.chunk_overlap,
max_workers=self.max_workers,
show_progress=self.show_progress,
+ node_fields_for_index=self.node_fields_for_index, # Persist node index field configuration
+ edge_fields_for_index=self.edge_fields_for_index, # Persist edge index field configuration
)
# ==================== State Management Lifecycle Hooks ====================