Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
重构 AutoGraph 类,优化节点和边的提取逻辑,支持批量处理和动态模型创建;更新相关类型注解和方法名称以提高可读性
yifanfeng978ac0f8319a3b
diff --git a/hyperextract/graphs/base.py b/hyperextract/graphs/base.pyindex 00467a0..41b6166 100644--- a/hyperextract/graphs/base.py+++ b/hyperextract/graphs/base.py@@ -4,15 +4,12 @@ Provides automatic deduplication for both nodes and edges using OMem.Supports single-stage and two-stage extraction strategies with consistency validation."""-from typing import Any, List, Type, Tuple, Callable, TypeVar, Generic, TYPE_CHECKING+from typing import Any, List, Type, Tuple, Callable, TypeVar, Generic, TYPE_CHECKING, Unionfrom pathlib import Path-from datetime import datetimefrom pydantic import BaseModel, Field, create_modelfrom langchain_core.embeddings import Embeddingsfrom langchain_core.language_models.chat_models import BaseChatModelfrom langchain_core.prompts import ChatPromptTemplate-from langchain_core.documents import Document-from langchain_community.vectorstores import FAISSfrom ontomem import OMemfrom ontomem.merger import MergeStrategy, create_merger, BaseMerger@@ -20,18 +17,40 @@ from ..core.base import BaseAutoTypefrom ..utils.logging import logger-NodeItem = TypeVar("NodeItem", bound=BaseModel)-EdgeItem = TypeVar("EdgeItem", bound=BaseModel)+Node = TypeVar("Node", bound=BaseModel)+Edge = TypeVar("Edge", bound=BaseModel)-class AutoGraphSchema(BaseModel, Generic[NodeItem, EdgeItem]):+class AutoGraphSchema(BaseModel, Generic[Node, Edge]):"""Generic schema container for graph-based knowledge patterns."""- nodes: List[NodeItem] = Field(default_factory=list, description="Graph nodes/entities")- edges: List[EdgeItem] = Field(default_factory=list, description="Graph edges/relationships")+ nodes: List[Node] = Field(+ default_factory=list, description="Graph nodes/entities"+ )+ edges: List[Edge] = Field(+ default_factory=list, description="Graph edges/relationships"+ )-class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeItem, EdgeItem]):+class NodeListSchema(BaseModel, Generic[Node]):+ """Intermediate schema for batch node extraction."""+ items: List[Node] = Field(+ default_factory=list,+ description="List of identified entities or nodes found in the text."+ )+++class EdgeListSchema(BaseModel, Generic[Edge]):+ """Intermediate schema for batch edge extraction."""+ items: List[Edge] = Field(+ default_factory=list,+ description="List of identified relationships or edges found in the text."+ )+++class AutoGraph(+ BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]+):"""AutoGraph - extracts knowledge graphs with nodes and edges from text.This pattern extracts structured knowledge graphs consisting of entities (nodes) and@@ -72,20 +91,21 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeI"""if TYPE_CHECKING:- graph_schema: Type[AutoGraphSchema[NodeItem, EdgeItem]]+ graph_schema: Type[AutoGraphSchema[Node, Edge]]def __init__(self,- node_schema: Type[NodeItem],- edge_schema: Type[EdgeItem],- node_key_extractor: Callable[[NodeItem], str],- edge_key_extractor: Callable[[EdgeItem], str],- edge_to_nodes_extractor: Callable[[EdgeItem], Tuple[str, str]],+ node_schema: Type[Node],+ edge_schema: Type[Edge],+ node_key_extractor: Callable[[Node], str],+ edge_key_extractor: Callable[[Edge], str],+ edge_to_nodes_extractor: Callable[[Edge], Tuple[str, str]],llm_client: BaseChatModel,embedder: Embeddings,*,extraction_mode: str = "one_stage",- node_strategy_or_merger: MergeStrategy | BaseMerger = MergeStrategy.LLM.BALANCED,+ node_strategy_or_merger: MergeStrategy+ | BaseMerger = MergeStrategy.LLM.BALANCED,edge_strategy_or_merger: MergeStrategy | BaseMerger = MergeStrategy.MERGE_FIELD,repair_strategy: str = "drop", # "drop" or "create_ghost"prompt: str = "",@@ -93,6 +113,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIchunk_overlap: int = 200,max_workers: int = 10,show_progress: bool = True,+ **kwargs: Any,):"""Initialize AutoGraph with node/edge schemas and configuration.@@ -113,6 +134,8 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIchunk_overlap: Overlapping characters between chunks.max_workers: Maximum concurrent extraction tasks.show_progress: Whether to log progress.+ **kwargs: Additional arguments passed to create_merger() when strategy_or_merger is+ a MergeStrategy enum. Ignored if strategy_or_merger is a BaseMerger instance."""# Store schemas and extractorsself.node_schema = node_schema@@ -123,7 +146,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIself.extraction_mode = extraction_modeself.repair_strategy = repair_strategy- # Create dynamic GraphSchema container+ # Create dynamic GraphSchema containersgraph_schema_name = f"{node_schema.__name__}{edge_schema.__name__}Graph"self.graph_schema = create_model(graph_schema_name,@@ -131,6 +154,14 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIedges=(List[edge_schema], Field(default_factory=list)),)+ # Create schema for list extraction (two-stage mode) with 'items' field+ self.node_list_schema = create_model(+ "NodeList", items=(List[node_schema], Field(default_factory=list, description="Extracted nodes"))+ )+ self.edge_list_schema = create_model(+ "EdgeList", items=(List[edge_schema], Field(default_factory=list, description="Extracted edges"))+ )+# Initialize Node Mergerif isinstance(node_strategy_or_merger, BaseMerger):self.node_merger = node_strategy_or_merger@@ -140,6 +171,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIkey_extractor=node_key_extractor,llm_client=llm_client,item_schema=node_schema,+ **kwargs, # Pass additional arguments to create_merger)# Initialize Edge Merger@@ -151,6 +183,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIkey_extractor=edge_key_extractor,llm_client=llm_client,item_schema=edge_schema,+ **kwargs, # Pass additional arguments to create_merger)# Initialize OMem instances (Before super().__init__)@@ -184,7 +217,6 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIshow_progress=show_progress,)-def _default_prompt(self) -> str:"""Returns the default extraction prompt based on extraction mode."""if self.extraction_mode == "one_stage":@@ -203,37 +235,53 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeI)@property- def data(self) -> AutoGraphSchema[NodeItem, EdgeItem]:+ def data(self) -> AutoGraphSchema[Node, Edge]:"""Returns the current graph state (nodes and edges).Returns:AutoGraphSchema containing all nodes and edges."""return self.graph_schema(- nodes=self._node_memory.items,- edges=self._edge_memory.items+ nodes=self._node_memory.items, edges=self._edge_memory.items)+ @property+ def nodes(self) -> List[Node]:+ """Returns the current node collection.++ Returns:+ List of nodes.+ """+ return self._node_memory.items++ @property+ def edges(self) -> List[Edge]:+ """Returns the current edge collection.++ Returns:+ List of edges.+ """+ return self._edge_memory.items+def empty(self) -> bool:- """Checks if the graph is empty (no nodes and no edges).+ """Checks if the graph is empty (no nodes).Returns:- True if both node and edge collections are empty, False otherwise.+ True if node collections are empty, False otherwise."""- return len(self._node_memory.items) == 0 and len(self._edge_memory.items) == 0+ return self._node_memory.empty()# ==================== State Management Lifecycle Hooks ====================def _init_data_state(self) -> None:"""Initialize or reset graph data structures."""- if hasattr(self, '_node_memory'):- self._node_memory.clear()- if hasattr(self, '_edge_memory'):- self._edge_memory.clear()+ self._node_memory.clear()+ self._edge_memory.clear()def _init_index_state(self) -> None:"""Initialize vector index to empty state."""- self._index = None+ self._node_memory.clear_index()+ self._edge_memory.clear_index()def _set_data_state(self, data: AutoGraphSchema) -> None:"""Replace graph data with new data (full reset).@@ -243,12 +291,12 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeI"""self._node_memory.clear()self._edge_memory.clear()-+if data.nodes:self._node_memory.add(data.nodes)if data.edges:self._edge_memory.add(data.edges)-+self.clear_index()def _update_data_state(self, incoming_data: AutoGraphSchema) -> None:@@ -278,16 +326,16 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIExtracted and validated graph."""if self.extraction_mode == "one_stage":- raw_graph = self._extract_one_stage(text)+ raw_graph = self._extract_data_by_one_stage(text)elif self.extraction_mode == "two_stage":- raw_graph = self._extract_two_stage(text)+ raw_graph = self._extract_data_by_two_stage(text)else:raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")# Validate and repair graph consistencyreturn self._validate_and_repair(raw_graph)- def _extract_one_stage(self, text: str) -> AutoGraphSchema:+ def _extract_data_by_one_stage(self, text: str) -> AutoGraphSchema:"""Extract nodes and edges simultaneously using single LLM call.Args:@@ -317,121 +365,99 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeI# Merge multiple graphsreturn self.merge_batch_data(graph_list)- def _extract_two_stage(self, text: str) -> AutoGraphSchema:- """Extract nodes first, then edges with node context.+ def _extract_data_by_two_stage(self, text: str) -> AutoGraphSchema:+ """Extract nodes first, then edges with node context (batch processing).++ Process:+ 1. Split text into chunks.+ 2. Batch extract nodes for all chunks.+ 3. Batch extract edges for all chunks (using chunk-specific nodes as context).+ 4. Construct partial graphs (tuples of nodes/edges).+ 5. Merge all partial graphs into one global graph.Args:text: Input text.Returns:- Extracted graph with validated consistency.+ Extracted and validated graph."""- # Stage 1: Extract nodes- nodes = self._extract_nodes(text)+ # 1. Prepare chunks+ if len(text) <= self.chunk_size:+ chunks = [text]+ else:+ chunks = self.text_splitter.split_text(text)++ # 2. Batch Extract Nodes (returns List[NodeListSchema])+ chunk_node_lists = self._extract_nodes_batch(chunks)- # Stage 2: Extract edges with node context- edges = self._extract_edges(text, nodes)+ # 3. Batch Extract Edges (Context-aware, returns List[EdgeListSchema])+ chunk_edge_lists = self._extract_edges_batch(chunks, chunk_node_lists)- return self.graph_schema(nodes=nodes, edges=edges)+ # 4. Construct Partial Graphs (Tuple format for merge optimization)+ partial_graphs = []+ for node_list_obj, edge_list_obj in zip(chunk_node_lists, chunk_edge_lists):+ nodes = node_list_obj.items if node_list_obj else []+ edges = edge_list_obj.items if edge_list_obj else []+ partial_graphs.append((nodes, edges))- def _extract_nodes(self, text: str) -> List[NodeItem]:- """Extract only nodes from text.+ # 5. Global Merge (passes tuples to merge_batch_data)+ return self.merge_batch_data(partial_graphs)++ def _extract_nodes_batch(self, chunks: List[str]) -> List[Any]:+ """Batch extract nodes from multiple text chunks.Args:- text: Input text.+ chunks: List of text chunks.Returns:- List of extracted nodes.+ List of NodeListSchema objects with extracted nodes."""- # Create a temporary schema containing only nodes- node_list_schema = create_model(- "NodeList",- nodes=(List[self.node_schema], Field(default_factory=list))- )-prompt_template = ChatPromptTemplate.from_template(- "Extract all entities/nodes from the following text:\n\n{chunk_text}"+ "Extract all relevant entities/nodes from the following text:\n\n{chunk_text}")- llm_chain = prompt_template | self.llm_client.with_structured_output(node_list_schema)-- if len(text) <= self.chunk_size:- result = llm_chain.invoke({"chunk_text": text})- node_lists = [result.nodes]- else:- chunks = self.text_splitter.split_text(text)- inputs = [{"chunk_text": chunk} for chunk in chunks]- results = llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})- node_lists = [r.nodes for r in results]-- # Merge nodes from all chunks- all_nodes = []- for node_list in node_lists:- all_nodes.extend(node_list)-- # Deduplicate using temporary OMem (Reuse self.node_merger)- temp_mem = OMem(- memory_schema=self.node_schema,- key_extractor=self.node_key_extractor,- llm_client=self.llm_client,- embedder=self.embedder,- strategy_or_merger=self.node_merger,- verbose=False,+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self.node_list_schema)- temp_mem.add(all_nodes)- return temp_mem.items- def _extract_edges(self, text: str, nodes: List[NodeItem]) -> List[EdgeItem]:- """Extract edges given node context.+ inputs = [{"chunk_text": chunk} for chunk in chunks]+ return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})++ def _extract_edges_batch(self, chunks: List[str], node_lists: List[Any]) -> List[Any]:+ """Batch extract edges using corresponding node lists as context.Args:- text: Input text.- nodes: List of known nodes to constrain edge extraction.+ chunks: List of text chunks.+ node_lists: List of NodeListSchema objects (one per chunk).Returns:- List of extracted edges.+ List of EdgeListSchema objects with extracted edges."""- # Format node context- node_keys = [self.node_key_extractor(n) for n in nodes]- node_context = "Known entities: " + ", ".join(node_keys)-- # Create edge list schema- edge_list_schema = create_model(- "EdgeList",- edges=(List[self.edge_schema], Field(default_factory=list))- )+ inputs = []+ for chunk, node_list_obj in zip(chunks, node_lists):+ nodes = node_list_obj.items if node_list_obj else []+ if not nodes:+ node_context = "No specific entities identified in this chunk."+ else:+ node_keys = [self.node_key_extractor(n) for n in nodes]+ node_context = "Known entities: " + ", ".join(node_keys)++ inputs.append({+ "chunk_text": chunk,+ "node_context": node_context+ })prompt_template = ChatPromptTemplate.from_template(- f"Extract relationships/edges from the text.\n{node_context}\n"- "CONSTRAINT: Only extract edges between the entities listed above.\n\n"- "Text:\n{{chunk_text}}"+ "Extract relationships/edges from the text based on the provided entities.\n"+ "CONSTRAINT: Only extract edges connecting the provided entities. "+ "Do not invent new entities.\n\n"+ "{node_context}\n\n"+ "Text:\n{chunk_text}")- llm_chain = prompt_template | self.llm_client.with_structured_output(edge_list_schema)-- if len(text) <= self.chunk_size:- result = llm_chain.invoke({"chunk_text": text})- edge_lists = [result.edges]- else:- chunks = self.text_splitter.split_text(text)- inputs = [{"chunk_text": chunk} for chunk in chunks]- results = llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})- edge_lists = [r.edges for r in results]-- # Merge edges- all_edges = []- for edge_list in edge_lists:- all_edges.extend(edge_list)-- # Deduplicate (Reuse self.edge_merger)- temp_mem = OMem(- memory_schema=self.edge_schema,- key_extractor=self.edge_key_extractor,- llm_client=self.llm_client,- embedder=self.embedder,- strategy_or_merger=self.edge_merger,- verbose=False,+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self.edge_list_schema)- temp_mem.add(all_edges)- return temp_mem.items++ return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})def _validate_and_repair(self, graph: AutoGraphSchema) -> AutoGraphSchema:"""Validate graph consistency and repair dangling edges.@@ -459,12 +485,16 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIrefined_edges.append(edge)elif self.repair_strategy == "create_ghost":# Create ghost nodes for missing endpoints- if not src_exists and src_key not in [self.node_key_extractor(g) for g in ghost_nodes]:+ if not src_exists and src_key not in [+ self.node_key_extractor(g) for g in ghost_nodes+ ]:ghost_node = self._create_ghost_node(src_key)ghost_nodes.append(ghost_node)valid_node_keys.add(src_key)- if not dst_exists and dst_key not in [self.node_key_extractor(g) for g in ghost_nodes]:+ if not dst_exists and dst_key not in [+ self.node_key_extractor(g) for g in ghost_nodes+ ]:ghost_node = self._create_ghost_node(dst_key)ghost_nodes.append(ghost_node)valid_node_keys.add(dst_key)@@ -481,7 +511,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIreturn self.graph_schema(nodes=all_nodes, edges=refined_edges)- def _create_ghost_node(self, node_key: str) -> NodeItem:+ def _create_ghost_node(self, node_key: str) -> Node:"""Create a minimal ghost node with only the key field populated.Args:@@ -495,60 +525,53 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIfor field_name, field_info in self.node_schema.model_fields.items():if field_info.is_required():# Try to populate with key if field name matches common patterns- if field_name in ['id', 'name', 'key', 'entity', 'node_id']:+ if field_name in ["id", "name", "key", "entity", "node_id"]:node_dict[field_name] = node_keyelse:# Use default or empty valuenode_dict[field_name] = ""-+return self.node_schema(**node_dict)# ==================== Merge Logic ====================- def merge_batch_data(self, data_list: List[AutoGraphSchema]) -> AutoGraphSchema:- """Merge multiple graphs into one.+ def merge_batch_data(+ self,+ data_list_or_tuple: List[Union[AutoGraphSchema, Tuple[List[Node], List[Edge]]]]+ ) -> AutoGraphSchema:+ """Merge multiple graphs or node/edge tuples into one.++ Supports two input formats:+ - AutoGraphSchema objects (standard format)+ - Tuples of (List[Node], List[Edge]) (optimization for batch processing)Args:- data_list: List of graphs to merge.+ data_list_or_tuple: List of AutoGraphSchema or (nodes, edges) tuples.Returns:Merged graph."""- if not data_list:- return self.graph_schema()-- if len(data_list) == 1:- return data_list[0]-- # Collect all nodes and edges- all_nodes = []- all_edges = []- for graph in data_list:- all_nodes.extend(graph.nodes)- all_edges.extend(graph.edges)-- # Deduplicate using temporary OMem instances (Reuse mergers)- temp_node_mem = OMem(- memory_schema=self.node_schema,- key_extractor=self.node_key_extractor,- llm_client=self.llm_client,- embedder=self.embedder,- strategy_or_merger=self.node_merger,- verbose=False,- )- temp_node_mem.add(all_nodes)+ if isinstance(data_list_or_tuple[0], AutoGraphSchema):+ all_nodes, all_edges = [], []- temp_edge_mem = OMem(- memory_schema=self.edge_schema,- key_extractor=self.edge_key_extractor,- llm_client=self.llm_client,- embedder=self.embedder,- strategy_or_merger=self.edge_merger,- verbose=False,- )- temp_edge_mem.add(all_edges)+ for graph in data_list_or_tuple:+ all_nodes.extend(graph.nodes)+ all_edges.extend(graph.edges)++ else:+ assert len(data_list_or_tuple) == 2, "Invalid input format for batch merging"+ nodes_list, edges_list = data_list_or_tuple[0], data_list_or_tuple[1]+ assert isinstance(nodes_list[0], NodeListSchema), "Expected NodeListSchema in first element"+ assert isinstance(edges_list[0], EdgeListSchema), "Expected EdgeListSchema in second element"++ all_nodes, all_edges = [], []+ for nodes, edges in zip(nodes_list, edges_list):+ all_nodes.extend(nodes.items)+ all_edges.extend(edges.items)- return self.graph_schema(nodes=temp_node_mem.items, edges=temp_edge_mem.items)+ merged_nodes = self.node_merger.merge(all_nodes) if all_nodes else []+ merged_edges = self.edge_merger.merge(all_edges) if all_edges else []+ return self.graph_schema(nodes=merged_nodes, edges=merged_edges)# ==================== Indexing & Search ====================@@ -565,11 +588,13 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeI# Typically we index nodes; edges are optionalif index_nodes:self._node_memory.build_index()-+if index_edges:self._edge_memory.build_index()- def search(self, query: str, top_k: int = 3, search_type: str = "nodes") -> List[Any]:+ def search(+ self, query: str, top_k: int = 3, search_type: str = "nodes"+ ) -> List[Any]:"""Search the graph using semantic similarity.Args:@@ -593,7 +618,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeI"""Save indices to disk."""folder = Path(folder_path)folder.mkdir(parents=True, exist_ok=True)-+# Save node indexnode_index_path = folder / "node_index"node_index_path.mkdir(exist_ok=True)@@ -613,7 +638,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeIdef load_index(self, folder_path: str | Path) -> None:"""Load indices from disk."""folder = Path(folder_path)-+# Load node indexnode_index_path = folder / "node_index"if node_index_path.exists():