Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
feat: 新增 AutoHypergraph 类,支持复杂关系提取和批量处理,优化数据结构和提取逻辑
yifanfeng97393dd9a36dd1
diff --git a/hyperextract/hypergraphs/base.py b/hyperextract/hypergraphs/base.pyindex e69de29..90f321e 100644--- a/hyperextract/hypergraphs/base.py+++ b/hyperextract/hypergraphs/base.py@@ -0,0 +1,608 @@+"""Hypergraph Knowledge Pattern - extracts hypergraphs where edges connect multiple nodes.++Key Difference from AutoGraph:+- Edges are "Hyperedges" that connect arbitrary numbers of nodes (not just source/target).+- Consistency validation checks if *ALL* participants in a hyperedge exist in the node registry.+"""++from typing import (+ Any,+ List,+ Type,+ Tuple,+ Callable,+ TypeVar,+ Generic,+ TYPE_CHECKING,+)+from pathlib import Path+from pydantic import BaseModel, Field, create_model+from langchain_core.embeddings import Embeddings+from langchain_core.language_models.chat_models import BaseChatModel+from langchain_core.prompts import ChatPromptTemplate+from ontomem import OMem+from ontomem.merger import MergeStrategy, create_merger, BaseMerger++from ..core.base import BaseAutoType+from ..utils.logging import logger+++Node = TypeVar("Node", bound=BaseModel)+Edge = TypeVar("Edge", bound=BaseModel) # Represents a Hyperedge+++class AutoHypergraphSchema(BaseModel, Generic[Node, Edge]):+ """Generic schema container for hypergraph data."""++ nodes: List[Node] = Field(default_factory=list, description="Graph nodes/entities")+ edges: List[Edge] = Field(+ default_factory=list, description="Hyperedges connecting multiple entities"+ )+++class NodeListSchema(BaseModel, Generic[Node]):+ """Intermediate schema for batch node extraction."""++ items: List[Node] = Field(+ default_factory=list,+ description="List of identified entities or nodes found in the text.",+ )+++class EdgeListSchema(BaseModel, Generic[Edge]):+ """Intermediate schema for batch hyperedge extraction."""++ items: List[Edge] = Field(+ default_factory=list,+ description="List of identified hyperedges found in the text.",+ )+++class AutoHypergraph(BaseAutoType[AutoHypergraphSchema[Node, Edge]], Generic[Node, Edge]):+ """AutoHypergraph - extracts complex relationships involving multiple entities.++ Suitable for:+ - Events with multiple participants (e.g., meetings, transactions)+ - Group memberships+ - Complex dependencies++ The `nodes_in_edge_extractor` must return a Tuple of ALL node keys involved in the edge.+ Validation uses 'Strict Mode': checks if ALL nodes in that tuple exist.++ Example:+ >>> class Person(BaseModel):+ ... name: str+ ...+ >>> class Event(BaseModel):+ ... label: str # e.g., "meeting", "collaboration"+ ... participants: List[str] # IDs of people involved+ ...+ >>> hypergraph = AutoHypergraph(+ ... node_schema=Person,+ ... edge_schema=Event,+ ... node_key_extractor=lambda x: x.name,+ ... # CRITICAL: Sort participants to ensure {A, B} and {B, A} map to same key+ ... edge_key_extractor=lambda x: f"{x.label}_{sorted(x.participants)}",+ ... nodes_in_edge_extractor=lambda x: tuple(x.participants),+ ... llm_client=llm,+ ... embedder=embedder,+ ... extraction_mode="two_stage"+ ... )+ """++ if TYPE_CHECKING:+ graph_schema: Type[AutoHypergraphSchema[Node, Edge]]++ def __init__(+ self,+ node_schema: Type[Node],+ edge_schema: Type[Edge],+ node_key_extractor: Callable[[Node], str],+ edge_key_extractor: Callable[[Edge], str],+ # Returns a tuple of ALL node keys involved in this hyperedge+ nodes_in_edge_extractor: Callable[[Edge], Tuple[str, ...]],+ llm_client: BaseChatModel,+ embedder: Embeddings,+ *,+ extraction_mode: str = "two_stage", # Recommended for complex relations+ node_strategy_or_merger: MergeStrategy+ | BaseMerger = MergeStrategy.LLM.BALANCED,+ edge_strategy_or_merger: MergeStrategy+ | BaseMerger = MergeStrategy.LLM.BALANCED,+ prompt: str = "",+ prompt_for_node_extraction: str = "",+ prompt_for_edge_extraction: str = "",+ chunk_size: int = 2000,+ chunk_overlap: int = 200,+ max_workers: int = 10,+ show_progress: bool = True,+ **kwargs: Any,+ ):+ """Initialize AutoHypergraph with node/edge schemas and configuration.++ Args:+ node_schema: Pydantic BaseModel for nodes/entities.+ edge_schema: Pydantic BaseModel for hyperedges/relationships.+ node_key_extractor: Function to extract unique key from node.+ edge_key_extractor: Function to extract unique key from hyperedge.+ CRITICAL: Since hyperedges are unordered sets of nodes (i.e., {A, B} == {B, A}),+ you MUST sort the participating node keys in this function to ensure consistent+ deduplication. For example:+ lambda x: f"{x.relation_type}_{sorted(x.participants)}"+ Without sorting, two hyperedges with the same nodes in different order will+ be treated as different edges, breaking the deduplication mechanism.+ nodes_in_edge_extractor: Function to extract ALL node keys from a hyperedge+ as a tuple (e.g., lambda x: tuple(x.participants)).+ llm_client: Language model client for extraction.+ embedder: Embedding model for vector indexing.+ extraction_mode: "two_stage" (recommended) or "one_stage".+ node_strategy_or_merger: Merge strategy for duplicate nodes.+ edge_strategy_or_merger: Merge strategy for duplicate hyperedges.+ prompt: Custom extraction prompt for one-stage mode.+ prompt_for_node_extraction: Custom extraction prompt for node extraction.+ prompt_for_edge_extraction: Custom extraction prompt for hyperedge extraction.+ chunk_size: Maximum characters per chunk.+ chunk_overlap: Overlapping characters between chunks.+ max_workers: Maximum concurrent extraction tasks.+ show_progress: Whether to log progress.+ **kwargs: Additional arguments for merger creation.+ """+ # Store schemas and extractors+ self.node_schema = node_schema+ self.edge_schema = edge_schema+ self.node_key_extractor = node_key_extractor+ self.edge_key_extractor = edge_key_extractor+ self.nodes_in_edge_extractor = nodes_in_edge_extractor+ self.extraction_mode = extraction_mode+ self.chunk_size = chunk_size+ self.chunk_overlap = chunk_overlap+ self.max_workers = max_workers+ self.show_progress = show_progress++ # Initialize prompts+ self.node_prompt = prompt_for_node_extraction or self._default_node_prompt()+ self.edge_prompt = prompt_for_edge_extraction or self._default_edge_prompt()++ # Create dynamic HypergraphSchema+ graph_schema_name = f"{node_schema.__name__}{edge_schema.__name__}Hypergraph"+ self.graph_schema = create_model(+ graph_schema_name,+ nodes=(List[node_schema], Field(default_factory=list)),+ hyperedges=(List[edge_schema], Field(default_factory=list)),+ )++ # Helper Schemas for Batch Extraction+ self.node_list_schema = create_model(+ f"{node_schema.__name__}List",+ items=(List[node_schema], Field(default_factory=list)),+ )+ self.edge_list_schema = create_model(+ f"{edge_schema.__name__}List",+ items=(List[edge_schema], Field(default_factory=list)),+ )++ # Initialize Node Merger+ if isinstance(node_strategy_or_merger, BaseMerger):+ self.node_merger = node_strategy_or_merger+ else:+ self.node_merger = create_merger(+ strategy=node_strategy_or_merger,+ key_extractor=node_key_extractor,+ llm_client=llm_client,+ item_schema=node_schema,+ **kwargs,+ )++ # Initialize Edge Merger+ if isinstance(edge_strategy_or_merger, BaseMerger):+ self.edge_merger = edge_strategy_or_merger+ else:+ self.edge_merger = create_merger(+ strategy=edge_strategy_or_merger,+ key_extractor=edge_key_extractor,+ llm_client=llm_client,+ item_schema=edge_schema,+ **kwargs,+ )++ # Initialize OMem instances+ self._node_memory = OMem(+ memory_schema=node_schema,+ key_extractor=node_key_extractor,+ llm_client=llm_client,+ embedder=embedder,+ strategy_or_merger=self.node_merger,+ verbose=show_progress,+ )++ self._edge_memory = OMem(+ memory_schema=edge_schema,+ key_extractor=edge_key_extractor,+ llm_client=llm_client,+ embedder=embedder,+ strategy_or_merger=self.edge_merger,+ verbose=show_progress,+ )++ # Initialize Base Class+ super().__init__(+ data_schema=self.graph_schema,+ llm_client=llm_client,+ embedder=embedder,+ prompt=prompt or self._default_edge_prompt(),+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ show_progress=show_progress,+ )++ # ==================== Prompts ====================++ def _default_node_prompt(self) -> str:+ """Default prompt for node extraction."""+ return (+ "Extract all distinct entities from the text. "+ "Entities will serve as participants in complex events later.\n\n"+ "### Source Text:\n"+ )++ def _default_edge_prompt(self) -> str:+ """Default prompt for hyperedge extraction (multi-entity relationships)."""+ return (+ "You are an expert hypergraph extraction assistant. "+ "Extract complex relationships (hyperedges) that involve MULTIPLE entities simultaneously.\n\n"+ "CRITICAL RULES:\n"+ "1. A hyperedge can connect 2, 3, or more entities (e.g., a meeting with multiple people).\n"+ "2. Identify ALL participants for each relationship.\n"+ "3. ONLY use entities from the provided 'Known Entities' list.\n"+ "4. If an entity is not in the list, exclude it from the hyperedge.\n\n"+ )++ # ==================== State Management Lifecycle Hooks ====================++ def _init_data_state(self) -> None:+ """Initialize or reset hypergraph data structures."""+ self._node_memory.clear()+ self._edge_memory.clear()++ def _init_index_state(self) -> None:+ """Initialize vector index to empty state."""+ self._node_memory.clear_index()+ self._edge_memory.clear_index()++ def _set_data_state(self, data: AutoHypergraphSchema) -> None:+ """Replace hypergraph data with new data (full reset)."""+ self._node_memory.clear()+ self._edge_memory.clear()++ if data.nodes:+ self._node_memory.add(data.nodes)+ if data.edges:+ self._edge_memory.add(data.edges)++ self.clear_index()++ def _update_data_state(self, incoming_data: AutoHypergraphSchema) -> None:+ """Merge incoming hypergraph data into current state."""+ if self.empty():+ self._set_data_state(incoming_data)+ else:+ if incoming_data.nodes:+ self._node_memory.add(incoming_data.nodes)+ if incoming_data.edges:+ self._edge_memory.add(incoming_data.edges)+ self.clear_index()++ @property+ def data(self) -> AutoHypergraphSchema:+ """Returns the current hypergraph state."""+ return self.graph_schema(+ nodes=self._node_memory.items, edges=self._edge_memory.items+ )++ @property+ def nodes(self) -> List[Node]:+ """Returns the current node collection."""+ return self._node_memory.items++ @property+ def edges(self) -> List[Edge]:+ """Returns the current hyperedge collection."""+ return self._edge_memory.items++ def empty(self) -> bool:+ """Checks if the hypergraph is empty."""+ return self._node_memory.empty()++ # ==================== Extraction Pipeline ====================++ def _extract_data(self, text: str) -> AutoHypergraphSchema:+ """Main extraction logic dispatcher."""+ if self.extraction_mode == "two_stage":+ raw_graph = self._extract_data_by_two_stage(text)+ elif self.extraction_mode == "one_stage":+ raise NotImplementedError("Single-stage extraction not yet supported for AutoHypergraph.")+ else:+ raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")++ # Prune dangling hyperedges to ensure consistency+ return self._prune_dangling_edges(raw_graph)++ def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:+ """Extract nodes first, then hyperedges with node context (batch processing).++ Process:+ 1. Split text into chunks.+ 2. Batch extract nodes for all chunks.+ 3. Batch extract hyperedges for all chunks (using chunk-specific nodes as context).+ 4. Merge all partial results into one global hypergraph.+ """+ # 1. Prepare chunks+ if len(text) <= self.chunk_size:+ chunks = [text]+ else:+ chunks = self.text_splitter.split_text(text)++ if self.show_progress:+ logger.info(f"Extracting from {len(chunks)} chunks...")++ # 2. Batch Extract Nodes+ chunk_node_lists = self._extract_nodes_batch(chunks)++ # 3. Batch Extract Hyperedges (Context-aware)+ chunk_edge_lists = self._extract_edges_batch(chunks, chunk_node_lists)++ # 4. Construct Partial Graphs (Tuple format for merge optimization)+ partial_hypergraphs = (+ [node_list.items for node_list in chunk_node_lists],+ [edge_list.items for edge_list in chunk_edge_lists],+ )++ # 5. Global Merge+ return self.merge_batch_data(partial_hypergraphs)++ def _extract_nodes_batch(self, chunks: List[str]) -> List[NodeListSchema[Node]]:+ """Batch extract nodes from multiple text chunks."""+ prompt_template = ChatPromptTemplate.from_template(+ f"{self.node_prompt}{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self.node_list_schema+ )++ inputs = [{"chunk_text": chunk} for chunk in chunks]+ return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})++ def _extract_edges_batch(+ self, chunks: List[str], node_lists: List[NodeListSchema[Node]]+ ) -> List[EdgeListSchema[Edge]]:+ """Batch extract hyperedges using corresponding node lists as context."""+ inputs = []+ for chunk, node_list in zip(chunks, node_lists):+ nodes = node_list.items if node_list else []+ if not nodes:+ node_context = "No entities identified in this chunk."+ else:+ node_keys = [f"- {self.node_key_extractor(n)}" for n in nodes]+ node_context = "Allowed Entities:\n" + "\n".join(node_keys)++ inputs.append({"chunk_text": chunk, "node_context": node_context})++ prompt_template = ChatPromptTemplate.from_template(+ f"{self.edge_prompt}{{node_context}}\n\n### Text Chunk:\n{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self.edge_list_schema+ )++ return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})++ def _prune_dangling_edges(self, graph: AutoHypergraphSchema) -> AutoHypergraphSchema:+ """Prune hyperedges where ANY participating node does not exist (Strict Consistency).++ Args:+ graph: Raw hypergraph that may contain dangling hyperedges.++ Returns:+ Hypergraph with only valid hyperedges (ALL participants must exist in nodes).+ """+ valid_nodes = graph.nodes+ valid_node_keys = {self.node_key_extractor(n) for n in valid_nodes}++ # Also check long-term memory+ if self._node_memory:+ valid_node_keys.update(self._node_memory.keys)++ refined_edges = []+ dropped_count = 0++ for edge in graph.edges:+ # Get tuple of all participant node keys+ participant_keys = self.nodes_in_edge_extractor(edge)++ # Strict mode: ALL participants must exist+ is_valid = True+ for pk in participant_keys:+ if pk not in valid_node_keys:+ is_valid = False+ break++ # Keep edge only if all participants exist and there's at least one participant+ if is_valid and len(participant_keys) > 0:+ refined_edges.append(edge)+ else:+ dropped_count += 1+ logger.debug(+ f"Pruning dangling hyperedge with participants: {participant_keys}"+ )++ if dropped_count > 0:+ logger.info(+ f"Pruned {dropped_count} dangling hyperedges to ensure consistency."+ )++ return self.graph_schema(nodes=valid_nodes, edges=refined_edges)++ # ==================== Merge Logic ====================++ def merge_batch_data(+ self,+ data_list_or_tuple: List[AutoHypergraphSchema] | Tuple[List[List[Node]], List[List[Edge]]],+ ) -> AutoHypergraphSchema:+ """Merge multiple hypergraphs or node/edge tuples into one.++ Supports two input formats:+ - List of AutoHypergraphSchema objects (standard format)+ - Tuple of (List[List[Node]], List[List[Edge]]) (optimization for batch processing)++ Args:+ data_list_or_tuple: Either a list of AutoHypergraphSchema objects or a tuple of+ (nodes_lists, edges_lists) where each list contains items from multiple chunks.++ Returns:+ Merged hypergraph.+ """+ if isinstance(data_list_or_tuple, list) and isinstance(+ data_list_or_tuple[0], self.graph_schema+ ):+ # Format: List of AutoHypergraphSchema+ all_nodes, all_edges = [], []+ for graph in data_list_or_tuple:+ all_nodes.extend(graph.nodes)+ all_edges.extend(graph.edges)++ else:+ # Format: Tuple of (nodes_lists, edges_lists)+ assert len(data_list_or_tuple) == 2, (+ "Invalid input format for batch merging"+ )+ nodes_lists, edges_lists = data_list_or_tuple[0], data_list_or_tuple[1]++ all_nodes, all_edges = [], []+ for node_list, edge_list in zip(nodes_lists, edges_lists):+ all_nodes.extend(node_list)+ all_edges.extend(edge_list)++ merged_nodes = self.node_merger.merge(all_nodes) if all_nodes else []+ merged_edges = self.edge_merger.merge(all_edges) if all_edges else []+ return self.graph_schema(nodes=merged_nodes, edges=merged_edges)++ # ==================== Indexing & Search ====================++ def build_index(self, index_nodes: bool = True, index_edges: bool = False):+ """Build vector index for hypergraph search.++ Args:+ index_nodes: Whether to index nodes (default: True).+ index_edges: Whether to index hyperedges (default: False).+ """+ if index_nodes:+ self.build_node_index()++ if index_edges:+ self.build_edge_index()++ def build_node_index(self) -> None:+ """Build vector index specifically for nodes."""+ if not self.empty():+ self._node_memory.build_index()++ def build_edge_index(self) -> None:+ """Build vector index specifically for hyperedges."""+ if not self.empty():+ self._edge_memory.build_index()++ def search(+ self,+ query: str,+ top_k: int = 3,+ search_nodes: bool = True,+ search_edges: bool = False,+ ) -> List[Node] | List[Edge] | Tuple[List[Node], List[Edge]]:+ """Unified hypergraph search interface.++ Args:+ query: Search query string.+ top_k: Number of results to return per type.+ search_nodes: Whether to search nodes.+ search_edges: Whether to search hyperedges.++ Returns:+ - List[Node] if only search_nodes is True.+ - List[Edge] if only search_edges is True.+ - Tuple[List[Node], List[Edge]] if both are True.+ """+ if not search_nodes and not search_edges:+ raise ValueError(+ "At least one of search_nodes or search_edges must be True."+ )++ if search_nodes and search_edges:+ if not self._node_memory.has_index():+ raise ValueError("Node index not built. Call build_index() first.")+ if not self._edge_memory.has_index():+ raise ValueError("Edge index not built. Call build_index() first.")+ nodes = self.search_nodes(query, top_k)+ edges = self.search_edges(query, top_k)+ return (nodes, edges)++ if search_nodes:+ if not self._node_memory.has_index():+ raise ValueError("Node index not built. Call build_index() first.")+ return self.search_nodes(query, top_k)++ if search_edges:+ if not self._edge_memory.has_index():+ raise ValueError("Edge index not built. Call build_index() first.")+ return self.search_edges(query, top_k)++ def search_nodes(self, query: str, top_k: int = 3) -> List[Node]:+ """Semantic search for nodes/entities only."""+ return self._node_memory.search(query=query, top_k=top_k)++ def search_edges(self, query: str, top_k: int = 3) -> List[Edge]:+ """Semantic search for hyperedges/relationships only."""+ return self._edge_memory.search(query=query, top_k=top_k)++ # ==================== Serialization ====================++ def dump_index(self, folder_path: str | Path) -> None:+ """Save indices to disk."""+ folder = Path(folder_path)+ folder.mkdir(parents=True, exist_ok=True)++ # Save node index+ node_index_path = folder / "node_index"+ node_index_path.mkdir(exist_ok=True)+ try:+ self._node_memory.dump_index(str(node_index_path))+ except Exception as e:+ logger.warning(f"Failed to save node index: {e}")++ # Save edge index+ edge_index_path = folder / "edge_index"+ edge_index_path.mkdir(exist_ok=True)+ try:+ self._edge_memory.dump_index(str(edge_index_path))+ except Exception as e:+ logger.warning(f"Failed to save edge index: {e}")++ def load_index(self, folder_path: str | Path) -> None:+ """Load indices from disk."""+ folder = Path(folder_path)++ # Load node index+ node_index_path = folder / "node_index"+ if node_index_path.exists():+ try:+ self._node_memory.load_index(str(node_index_path))+ except Exception as e:+ logger.warning(f"Failed to load node index: {e}")++ # Load edge index+ edge_index_path = folder / "edge_index"+ if edge_index_path.exists():+ try:+ self._edge_memory.load_index(str(edge_index_path))+ except Exception as e:+ logger.warning(f"Failed to load edge index: {e}")