Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
新增 AutoGraph 类及其相关功能,支持从文本提取知识图谱,包括节点和边的自动去重、提取策略和一致性验证
yifanfeng97b89e305a0afb
diff --git a/hyperextract/graphs/base.py b/hyperextract/graphs/base.pyindex e69de29..00467a0 100644--- a/hyperextract/graphs/base.py+++ b/hyperextract/graphs/base.py@@ -0,0 +1,631 @@+"""Graph Knowledge Pattern - extracts knowledge graphs with nodes and edges from text.++Provides automatic deduplication for both nodes and edges using OMem.+Supports single-stage and two-stage extraction strategies with consistency validation.+"""++from typing import Any, List, Type, Tuple, Callable, TypeVar, Generic, TYPE_CHECKING+from pathlib import Path+from datetime import datetime+from pydantic import BaseModel, Field, create_model+from langchain_core.embeddings import Embeddings+from langchain_core.language_models.chat_models import BaseChatModel+from langchain_core.prompts import ChatPromptTemplate+from langchain_core.documents import Document+from langchain_community.vectorstores import FAISS+from ontomem import OMem+from ontomem.merger import MergeStrategy, create_merger, BaseMerger++from ..core.base import BaseAutoType+from ..utils.logging import logger+++NodeItem = TypeVar("NodeItem", bound=BaseModel)+EdgeItem = TypeVar("EdgeItem", bound=BaseModel)+++class AutoGraphSchema(BaseModel, Generic[NodeItem, EdgeItem]):+ """Generic schema container for graph-based knowledge patterns."""++ nodes: List[NodeItem] = Field(default_factory=list, description="Graph nodes/entities")+ edges: List[EdgeItem] = Field(default_factory=list, description="Graph edges/relationships")+++class AutoGraph(BaseAutoType[AutoGraphSchema[NodeItem, EdgeItem]], Generic[NodeItem, EdgeItem]):+ """AutoGraph - extracts knowledge graphs with nodes and edges from text.++ This pattern extracts structured knowledge graphs consisting of entities (nodes) and+ their relationships (edges). Suitable for entity relationship extraction, knowledge+ graph construction, and semantic network building.++ Key characteristics:+ - Extraction target: Graph structure with nodes and edges+ - Deduplication: Automatic deduplication for both nodes and edges using OMem+ - Node merge strategy: Configurable (LLM-powered intelligent merging by default)+ - Edge merge strategy: Configurable (simple merge by default)+ - Extraction modes:+ * one_stage: Extract nodes and edges simultaneously (faster, simpler prompt)+ * two_stage: Extract nodes first, then edges with node context (more accurate)+ - Consistency validation: Ensures edges only connect existing nodes++ Example:+ >>> class Entity(BaseModel):+ ... name: str+ ... type: str+ ... properties: dict = {}+ >>>+ >>> class Relation(BaseModel):+ ... source: str+ ... target: str+ ... relation_type: str+ >>>+ >>> graph = AutoGraph(+ ... node_schema=Entity,+ ... edge_schema=Relation,+ ... node_key_extractor=lambda x: x.name,+ ... edge_key_extractor=lambda x: f"{x.source}-{x.relation_type}-{x.target}",+ ... edge_to_nodes_extractor=lambda x: (x.source, x.target),+ ... llm_client=llm,+ ... embedder=embedder,+ ... extraction_mode="two_stage"+ ... )+ """++ if TYPE_CHECKING:+ graph_schema: Type[AutoGraphSchema[NodeItem, EdgeItem]]++ def __init__(+ self,+ node_schema: Type[NodeItem],+ edge_schema: Type[EdgeItem],+ node_key_extractor: Callable[[NodeItem], str],+ edge_key_extractor: Callable[[EdgeItem], str],+ edge_to_nodes_extractor: Callable[[EdgeItem], Tuple[str, str]],+ llm_client: BaseChatModel,+ embedder: Embeddings,+ *,+ extraction_mode: str = "one_stage",+ node_strategy_or_merger: MergeStrategy | BaseMerger = MergeStrategy.LLM.BALANCED,+ edge_strategy_or_merger: MergeStrategy | BaseMerger = MergeStrategy.MERGE_FIELD,+ repair_strategy: str = "drop", # "drop" or "create_ghost"+ prompt: str = "",+ chunk_size: int = 2000,+ chunk_overlap: int = 200,+ max_workers: int = 10,+ show_progress: bool = True,+ ):+ """Initialize AutoGraph with node/edge schemas and configuration.++ Args:+ node_schema: Pydantic BaseModel for nodes/entities.+ edge_schema: Pydantic BaseModel for edges/relationships.+ node_key_extractor: Function to extract unique key from node (e.g., lambda x: x.id).+ edge_key_extractor: Function to extract unique key from edge (e.g., lambda x: f"{x.src}-{x.rel}-{x.dst}").+ edge_to_nodes_extractor: Function to extract (source_key, target_key) from edge for validation.+ llm_client: Language model client for extraction.+ embedder: Embedding model for vector indexing.+ extraction_mode: "one_stage" (extract nodes+edges together) or "two_stage" (nodes first, then edges).+ node_strategy_or_merger: Merge strategy for duplicate nodes (default: LLM.BALANCED).+ edge_strategy_or_merger: Merge strategy for duplicate edges (default: MERGE_FIELD).+ repair_strategy: How to handle dangling edges - "drop" (remove) or "create_ghost" (create minimal nodes).+ prompt: Custom extraction prompt.+ chunk_size: Maximum characters per chunk.+ chunk_overlap: Overlapping characters between chunks.+ max_workers: Maximum concurrent extraction tasks.+ show_progress: Whether to log progress.+ """+ # Store schemas and extractors+ self.node_schema = node_schema+ self.edge_schema = edge_schema+ self.node_key_extractor = node_key_extractor+ self.edge_key_extractor = edge_key_extractor+ self.edge_to_nodes_extractor = edge_to_nodes_extractor+ self.extraction_mode = extraction_mode+ self.repair_strategy = repair_strategy++ # Create dynamic GraphSchema container+ graph_schema_name = f"{node_schema.__name__}{edge_schema.__name__}Graph"+ self.graph_schema = create_model(+ graph_schema_name,+ nodes=(List[node_schema], Field(default_factory=list)),+ edges=(List[edge_schema], Field(default_factory=list)),+ )++ # Initialize Node Merger+ if isinstance(node_strategy_or_merger, BaseMerger):+ self.node_merger = node_strategy_or_merger+ else:+ self.node_merger = create_merger(+ strategy=node_strategy_or_merger,+ key_extractor=node_key_extractor,+ llm_client=llm_client,+ item_schema=node_schema,+ )++ # Initialize Edge Merger+ if isinstance(edge_strategy_or_merger, BaseMerger):+ self.edge_merger = edge_strategy_or_merger+ else:+ self.edge_merger = create_merger(+ strategy=edge_strategy_or_merger,+ key_extractor=edge_key_extractor,+ llm_client=llm_client,+ item_schema=edge_schema,+ )++ # Initialize OMem instances (Before super().__init__)+ self._node_memory = OMem(+ memory_schema=node_schema,+ key_extractor=node_key_extractor,+ llm_client=llm_client,+ embedder=embedder,+ strategy_or_merger=self.node_merger,+ verbose=show_progress,+ )++ self._edge_memory = OMem(+ memory_schema=edge_schema,+ key_extractor=edge_key_extractor,+ llm_client=llm_client,+ embedder=embedder,+ strategy_or_merger=self.edge_merger,+ verbose=show_progress,+ )++ # Call parent init+ super().__init__(+ data_schema=self.graph_schema,+ llm_client=llm_client,+ embedder=embedder,+ prompt=prompt or self._default_prompt(),+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ show_progress=show_progress,+ )+++ def _default_prompt(self) -> str:+ """Returns the default extraction prompt based on extraction mode."""+ if self.extraction_mode == "one_stage":+ return (+ "You are an expert knowledge graph extraction assistant. "+ "Extract entities (nodes) and their relationships (edges) from the following text. "+ "IMPORTANT CONSTRAINT: Every edge must connect two nodes that are present in the nodes list. "+ "Do not create edges between entities that are not explicitly listed as nodes.\n\n"+ "### Source Text:\n"+ )+ else: # two_stage+ return (+ "You are an expert knowledge extraction assistant. "+ "Extract structured information from the following text according to the specified schema.\n\n"+ "### Source Text:\n"+ )++ @property+ def data(self) -> AutoGraphSchema[NodeItem, EdgeItem]:+ """Returns the current graph state (nodes and edges).++ Returns:+ AutoGraphSchema containing all nodes and edges.+ """+ return self.graph_schema(+ nodes=self._node_memory.items,+ edges=self._edge_memory.items+ )++ def empty(self) -> bool:+ """Checks if the graph is empty (no nodes and no edges).++ Returns:+ True if both node and edge collections are empty, False otherwise.+ """+ return len(self._node_memory.items) == 0 and len(self._edge_memory.items) == 0++ # ==================== State Management Lifecycle Hooks ====================++ def _init_data_state(self) -> None:+ """Initialize or reset graph data structures."""+ if hasattr(self, '_node_memory'):+ self._node_memory.clear()+ if hasattr(self, '_edge_memory'):+ self._edge_memory.clear()++ def _init_index_state(self) -> None:+ """Initialize vector index to empty state."""+ self._index = None++ def _set_data_state(self, data: AutoGraphSchema) -> None:+ """Replace graph data with new data (full reset).++ Args:+ data: New graph data to set.+ """+ self._node_memory.clear()+ self._edge_memory.clear()++ if data.nodes:+ self._node_memory.add(data.nodes)+ if data.edges:+ self._edge_memory.add(data.edges)++ self.clear_index()++ def _update_data_state(self, incoming_data: AutoGraphSchema) -> None:+ """Merge incoming graph data into current state.++ Args:+ incoming_data: Incremental graph data to merge.+ """+ if self.empty():+ self._set_data_state(incoming_data)+ else:+ if incoming_data.nodes:+ self._node_memory.add(incoming_data.nodes)+ if incoming_data.edges:+ self._edge_memory.add(incoming_data.edges)+ self.clear_index()++ # ==================== Extraction Pipeline ====================++ def _extract_data(self, text: str) -> AutoGraphSchema:+ """Main extraction logic dispatcher.++ Args:+ text: Input text to extract graph from.++ Returns:+ Extracted and validated graph.+ """+ if self.extraction_mode == "one_stage":+ raw_graph = self._extract_one_stage(text)+ elif self.extraction_mode == "two_stage":+ raw_graph = self._extract_two_stage(text)+ else:+ raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")++ # Validate and repair graph consistency+ return self._validate_and_repair(raw_graph)++ def _extract_one_stage(self, text: str) -> AutoGraphSchema:+ """Extract nodes and edges simultaneously using single LLM call.++ Args:+ text: Input text.++ Returns:+ Raw extracted graph (may contain inconsistencies).+ """+ prompt_template = ChatPromptTemplate.from_template(+ f"{self.prompt}{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self.graph_schema+ )++ # Extract from single chunk or multiple chunks+ if len(text) <= self.chunk_size:+ graph = llm_chain.invoke({"chunk_text": text})+ graph_list = [graph]+ else:+ chunks = self.text_splitter.split_text(text)+ inputs = [{"chunk_text": chunk} for chunk in chunks]+ graph_list = llm_chain.batch(+ inputs, config={"max_concurrency": self.max_workers}+ )++ # Merge multiple graphs+ return self.merge_batch_data(graph_list)++ def _extract_two_stage(self, text: str) -> AutoGraphSchema:+ """Extract nodes first, then edges with node context.++ Args:+ text: Input text.++ Returns:+ Extracted graph with validated consistency.+ """+ # Stage 1: Extract nodes+ nodes = self._extract_nodes(text)++ # Stage 2: Extract edges with node context+ edges = self._extract_edges(text, nodes)++ return self.graph_schema(nodes=nodes, edges=edges)++ def _extract_nodes(self, text: str) -> List[NodeItem]:+ """Extract only nodes from text.++ Args:+ text: Input text.++ Returns:+ List of extracted nodes.+ """+ # Create a temporary schema containing only nodes+ node_list_schema = create_model(+ "NodeList",+ nodes=(List[self.node_schema], Field(default_factory=list))+ )++ prompt_template = ChatPromptTemplate.from_template(+ "Extract all entities/nodes from the following text:\n\n{chunk_text}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(node_list_schema)++ if len(text) <= self.chunk_size:+ result = llm_chain.invoke({"chunk_text": text})+ node_lists = [result.nodes]+ else:+ chunks = self.text_splitter.split_text(text)+ inputs = [{"chunk_text": chunk} for chunk in chunks]+ results = llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})+ node_lists = [r.nodes for r in results]++ # Merge nodes from all chunks+ all_nodes = []+ for node_list in node_lists:+ all_nodes.extend(node_list)++ # Deduplicate using temporary OMem (Reuse self.node_merger)+ temp_mem = OMem(+ memory_schema=self.node_schema,+ key_extractor=self.node_key_extractor,+ llm_client=self.llm_client,+ embedder=self.embedder,+ strategy_or_merger=self.node_merger,+ verbose=False,+ )+ temp_mem.add(all_nodes)+ return temp_mem.items++ def _extract_edges(self, text: str, nodes: List[NodeItem]) -> List[EdgeItem]:+ """Extract edges given node context.++ Args:+ text: Input text.+ nodes: List of known nodes to constrain edge extraction.++ Returns:+ List of extracted edges.+ """+ # Format node context+ node_keys = [self.node_key_extractor(n) for n in nodes]+ node_context = "Known entities: " + ", ".join(node_keys)++ # Create edge list schema+ edge_list_schema = create_model(+ "EdgeList",+ edges=(List[self.edge_schema], Field(default_factory=list))+ )++ prompt_template = ChatPromptTemplate.from_template(+ f"Extract relationships/edges from the text.\n{node_context}\n"+ "CONSTRAINT: Only extract edges between the entities listed above.\n\n"+ "Text:\n{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(edge_list_schema)++ if len(text) <= self.chunk_size:+ result = llm_chain.invoke({"chunk_text": text})+ edge_lists = [result.edges]+ else:+ chunks = self.text_splitter.split_text(text)+ inputs = [{"chunk_text": chunk} for chunk in chunks]+ results = llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})+ edge_lists = [r.edges for r in results]++ # Merge edges+ all_edges = []+ for edge_list in edge_lists:+ all_edges.extend(edge_list)++ # Deduplicate (Reuse self.edge_merger)+ temp_mem = OMem(+ memory_schema=self.edge_schema,+ key_extractor=self.edge_key_extractor,+ llm_client=self.llm_client,+ embedder=self.embedder,+ strategy_or_merger=self.edge_merger,+ verbose=False,+ )+ temp_mem.add(all_edges)+ return temp_mem.items++ def _validate_and_repair(self, graph: AutoGraphSchema) -> AutoGraphSchema:+ """Validate graph consistency and repair dangling edges.++ Args:+ graph: Raw graph that may contain dangling edges.++ Returns:+ Validated and repaired graph.+ """+ valid_nodes = graph.nodes+ valid_node_keys = {self.node_key_extractor(n) for n in valid_nodes}++ refined_edges = []+ ghost_nodes = []++ for edge in graph.edges:+ src_key, dst_key = self.edge_to_nodes_extractor(edge)++ # Check if both endpoints exist+ src_exists = src_key in valid_node_keys+ dst_exists = dst_key in valid_node_keys++ if src_exists and dst_exists:+ refined_edges.append(edge)+ elif self.repair_strategy == "create_ghost":+ # Create ghost nodes for missing endpoints+ if not src_exists and src_key not in [self.node_key_extractor(g) for g in ghost_nodes]:+ ghost_node = self._create_ghost_node(src_key)+ ghost_nodes.append(ghost_node)+ valid_node_keys.add(src_key)++ if not dst_exists and dst_key not in [self.node_key_extractor(g) for g in ghost_nodes]:+ ghost_node = self._create_ghost_node(dst_key)+ ghost_nodes.append(ghost_node)+ valid_node_keys.add(dst_key)++ refined_edges.append(edge)+ else: # repair_strategy == "drop"+ logger.warning(+ f"Dropping dangling edge: {src_key} -> {dst_key} "+ f"(src_exists={src_exists}, dst_exists={dst_exists})"+ )++ # Combine original nodes with ghost nodes+ all_nodes = valid_nodes + ghost_nodes++ return self.graph_schema(nodes=all_nodes, edges=refined_edges)++ def _create_ghost_node(self, node_key: str) -> NodeItem:+ """Create a minimal ghost node with only the key field populated.++ Args:+ node_key: The key value for the ghost node.++ Returns:+ Minimal node instance.+ """+ # Attempt to create minimal node with only required fields+ node_dict = {}+ for field_name, field_info in self.node_schema.model_fields.items():+ if field_info.is_required():+ # Try to populate with key if field name matches common patterns+ if field_name in ['id', 'name', 'key', 'entity', 'node_id']:+ node_dict[field_name] = node_key+ else:+ # Use default or empty value+ node_dict[field_name] = ""++ return self.node_schema(**node_dict)++ # ==================== Merge Logic ====================++ def merge_batch_data(self, data_list: List[AutoGraphSchema]) -> AutoGraphSchema:+ """Merge multiple graphs into one.++ Args:+ data_list: List of graphs to merge.++ Returns:+ Merged graph.+ """+ if not data_list:+ return self.graph_schema()++ if len(data_list) == 1:+ return data_list[0]++ # Collect all nodes and edges+ all_nodes = []+ all_edges = []+ for graph in data_list:+ all_nodes.extend(graph.nodes)+ all_edges.extend(graph.edges)++ # Deduplicate using temporary OMem instances (Reuse mergers)+ temp_node_mem = OMem(+ memory_schema=self.node_schema,+ key_extractor=self.node_key_extractor,+ llm_client=self.llm_client,+ embedder=self.embedder,+ strategy_or_merger=self.node_merger,+ verbose=False,+ )+ temp_node_mem.add(all_nodes)++ temp_edge_mem = OMem(+ memory_schema=self.edge_schema,+ key_extractor=self.edge_key_extractor,+ llm_client=self.llm_client,+ embedder=self.embedder,+ strategy_or_merger=self.edge_merger,+ verbose=False,+ )+ temp_edge_mem.add(all_edges)++ return self.graph_schema(nodes=temp_node_mem.items, edges=temp_edge_mem.items)++ # ==================== Indexing & Search ====================++ def build_index(self, index_nodes: bool = True, index_edges: bool = False):+ """Build vector index for graph search.++ Args:+ index_nodes: Whether to index nodes (default: True).+ index_edges: Whether to index edges (default: False).+ """+ if self.empty():+ return++ # Typically we index nodes; edges are optional+ if index_nodes:+ self._node_memory.build_index()++ if index_edges:+ self._edge_memory.build_index()++ def search(self, query: str, top_k: int = 3, search_type: str = "nodes") -> List[Any]:+ """Search the graph using semantic similarity.++ Args:+ query: Search query.+ top_k: Number of results.+ search_type: "nodes" or "edges".++ Returns:+ List of relevant items.+ """+ if search_type == "nodes":+ return self._node_memory.retrieve(query=query, top_k=top_k)+ elif search_type == "edges":+ return self._edge_memory.retrieve(query=query, top_k=top_k)+ else:+ raise ValueError(f"Invalid search_type: {search_type}")++ # ==================== Serialization ====================++ def dump_index(self, folder_path: str | Path) -> None:+ """Save indices to disk."""+ folder = Path(folder_path)+ folder.mkdir(parents=True, exist_ok=True)++ # Save node index+ node_index_path = folder / "node_index"+ node_index_path.mkdir(exist_ok=True)+ try:+ self._node_memory.save_to_disk(str(node_index_path))+ except Exception as e:+ logger.warning(f"Failed to save node index: {e}")++ # Save edge index+ edge_index_path = folder / "edge_index"+ edge_index_path.mkdir(exist_ok=True)+ try:+ self._edge_memory.save_to_disk(str(edge_index_path))+ except Exception as e:+ logger.warning(f"Failed to save edge index: {e}")++ def load_index(self, folder_path: str | Path) -> None:+ """Load indices from disk."""+ folder = Path(folder_path)++ # Load node index+ node_index_path = folder / "node_index"+ if node_index_path.exists():+ try:+ self._node_memory.load_from_disk(str(node_index_path))+ except Exception as e:+ logger.warning(f"Failed to load node index: {e}")++ # Load edge index+ edge_index_path = folder / "edge_index"+ if edge_index_path.exists():+ try:+ self._edge_memory.load_from_disk(str(edge_index_path))+ except Exception as e:+ logger.warning(f"Failed to load edge index: {e}")