Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
Refactor hypergraph extraction framework: - Update AutoHypergraph and related schemas to use NodeSchema and EdgeSchema for improved clarity and type safety. - Implement EdgeAutoHypergraph for edge-first extraction strategy, enhancing narrative-driven knowledge graph capabilities. - Revise HyperGraph_RAG to utilize knowledge segments as hyperedges, improving contextual relationship representation. - Introduce new extraction prompts and schemas for better entity and edge extraction processes. - Clean up imports and remove unused classes to streamline the codebase.
yifanfeng977cda10b6bbaa
diff --git a/examples/hypergraph_rag_demo.py b/examples/hypergraph_rag_demo.pyindex 10cecd3..2076488 100644--- a/examples/hypergraph_rag_demo.py+++ b/examples/hypergraph_rag_demo.py@@ -9,7 +9,6 @@ This demo demonstrates the HyperGraph_RAG System:import osimport sys-# 添加项目根目录到路径sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))from dotenv import load_dotenv@@ -113,28 +112,22 @@ if __name__ == "__main__":print("=" * 80)# 分离低阶边(2个参与者)和高阶边(3个以上参与者)- low_order_edges = [e for e in rag.edges if len(e.participants) == 2]- high_order_edges = [e for e in rag.edges if len(e.participants) > 2]+ low_order_edges = [e for e in rag.edges if len(e.related_entities) == 2]+ high_order_edges = [e for e in rag.edges if len(e.related_entities) > 2]print(f"\n--- Low-Order Edges (Pairwise) ---")for i, edge in enumerate(low_order_edges, 1):- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")- print(f" Description: {edge.description}")- keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"- print(f" Keywords: {keywords_str}")- print(f" Strength: {edge.strength}/10")+ print(f"\n{i}. Participants: {' <-> '.join(edge.related_entities)}")+ print(f" Knowledge Segment: {edge.knowledge_segment}")+ print(f" Completeness Score: {edge.completeness_score}/10")print(f"\n--- High-Order Edges (Hyperedges) ---")for i, edge in enumerate(high_order_edges, 1):- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")- print(f" Description: {edge.description}")- keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"- print(f" Keywords: {keywords_str}")- print(f" Strength: {edge.strength}/10")-- print(- f"\n✓ Total edges extracted: {len(rag.edges)} ({len(low_order_edges)} low-order, {len(high_order_edges)} high-order)\n"- )+ print(f"\n{i}. Participants: {' <-> '.join(edge.related_entities)}")+ print(f" Knowledge Segment: {edge.knowledge_segment}")+ print(f" Completeness Score: {edge.completeness_score}/10")++ print(f"\n✓ Total edges extracted: {len(rag.edges)} ({len(low_order_edges)} low-order, {len(high_order_edges)} high-order)\n")# ============================================================================# Display Extracted Entities@@ -165,16 +158,16 @@ if __name__ == "__main__":for entity_type, count in sorted(type_counts.items()):print(f" {entity_type}: {count}")- # Average edge strength+ # Average edge completenessif rag.edges:- avg_strength = sum(e.strength for e in rag.edges) / len(rag.edges)- print(f"\nAverage Edge Strength: {avg_strength:.2f}/10")+ avg_completeness = sum(e.completeness_score for e in rag.edges) / len(rag.edges)+ print(f"\nAverage Edge Completeness: {avg_completeness:.2f}/10")# Centrality (number of relationships per entity)print("\nEntity Centrality (number of relationships):")centrality = {}for edge in rag.edges:- for participant in edge.participants:+ for participant in edge.related_entities:centrality[participant] = centrality.get(participant, 0) + 1for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[@@ -215,19 +208,15 @@ if __name__ == "__main__":if results:for j, result in enumerate(results, 1):- # EdgeAutoHypergraph (and base AutoHypergraph) default search returns (doc, score) tuple+ # AutoHypergraph search returns (doc, score) tuple# The 'doc' is the EdgeSchema objectedge = result[0] if isinstance(result, tuple) else resultscore = result[1] if isinstance(result, tuple) else 1.0- # Note: EdgeSchema might not have strength if extracted via generic AutoHypergraph, but HyperGraph_RAG defines it.- # We handle potential missing attrs safely- strength = getattr(edge, "strength", "N/A")-print(f"\n {j}. [Score: {score:.3f}]")- print(f" Participants: {' <-> '.join(edge.participants)}")- print(f" Description: {edge.description}")- print(f" Strength: {strength}/10")+ print(f" Participants: {' <-> '.join(edge.related_entities)}")+ print(f" Knowledge Segment: {edge.knowledge_segment}")+ print(f" Completeness Score: {edge.completeness_score}/10")else:print(" (No relevant relationships found)")except Exception as e:diff --git a/hyperextract/graphs/atom.py b/hyperextract/graphs/atom.pyindex 15f2326..c7c3044 100644--- a/hyperextract/graphs/atom.py+++ b/hyperextract/graphs/atom.py@@ -370,8 +370,6 @@ class Atom(AutoGraph[NodeSchema, EdgeSchema]):nodes_in_edge_extractor=nodes_in_edge_fn,llm_client=llm_client,embedder=embedder,- # Enforce one-stage extraction (first extract edges, then derive nodes)- extraction_mode="one_stage",# Inject customized promptsprompt_for_edge_extraction=Atom_EDGE_EXTRACTION_PROMPT,# Configure deduplication strategydiff --git a/hyperextract/graphs/base.py b/hyperextract/graphs/base.pyindex 2af3720..61ae6e4 100644--- a/hyperextract/graphs/base.py+++ b/hyperextract/graphs/base.py@@ -431,7 +431,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeSchema, EdgeSchema]], Generic[Ntext: Input text.Returns:- Raw extracted graph (may contain inconsistencies).+ Raw extracted graph data."""prompt_template = ChatPromptTemplate.from_template(f"{self.prompt}{{chunk_text}}"diff --git a/hyperextract/graphs/itext2kg_star.py b/hyperextract/graphs/itext2kg_star.pyindex b8d2d77..7fed016 100644--- a/hyperextract/graphs/itext2kg_star.py+++ b/hyperextract/graphs/itext2kg_star.py@@ -174,8 +174,6 @@ class iText2KG_Star(AutoGraph[NodeSchema, EdgeSchema]):nodes_in_edge_extractor=nodes_in_edge_fn,llm_client=llm_client,embedder=embedder,- # Enforce one-stage extraction (first extract edges, then derive nodes)- extraction_mode="one_stage",# Inject customized promptsprompt_for_edge_extraction=iText2KG_Star_EDGE_EXTRACTION_PROMPT,# Configure deduplication strategydiff --git a/hyperextract/hypergraphs/__init__.py b/hyperextract/hypergraphs/__init__.pyindex 559ed23..c801ad0 100644--- a/hyperextract/hypergraphs/__init__.py+++ b/hyperextract/hypergraphs/__init__.py@@ -1,13 +1,11 @@from .base import AutoHypergraphfrom .hyper_rag import Hyper_RAGfrom .hypergraph_rag import HyperGraph_RAG-from .cog_rag import Cog_RAG, Cog_RAG_ThemeLayer, Cog_RAG_DetailLayer+# from .cog_rag import Cog_RAG__all__ = ["AutoHypergraph","Hyper_RAG","HyperGraph_RAG",- "Cog_RAG",- "Cog_RAG_ThemeLayer",- "Cog_RAG_DetailLayer",+ # "Cog_RAG",]diff --git a/hyperextract/hypergraphs/base.py b/hyperextract/hypergraphs/base.pyindex a9d4721..5e4261d 100644--- a/hyperextract/hypergraphs/base.py+++ b/hyperextract/hypergraphs/base.py@@ -28,8 +28,8 @@ from ..core.base import BaseAutoTypefrom ..utils.logging import logger-Node = TypeVar("Node", bound=BaseModel)-Edge = TypeVar("Edge", bound=BaseModel) # Represents a Hyperedge+NodeSchema = TypeVar("NodeSchema", bound=BaseModel)+EdgeSchema = TypeVar("EdgeSchema", bound=BaseModel) # Represents a Hyperedge# ============================================================================@@ -66,52 +66,39 @@ CRITICAL RULES:"""-DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal--You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.-Your task is to Populate these concepts with their specific Participant Entities.-CRITICAL RULES:-1. Extract ALL key participants for each concept.-2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).-"""---class AutoHypergraphSchema(BaseModel, Generic[Node, Edge]):+class AutoHypergraphSchema(BaseModel, Generic[NodeSchema, EdgeSchema]):"""Generic schema container for hypergraph data."""- nodes: List[Node] = Field(default_factory=list, description="Graph nodes/entities")- edges: List[Edge] = Field(+ nodes: List[NodeSchema] = Field(+ default_factory=list, description="Graph nodes/entities"+ )+ edges: List[EdgeSchema] = Field(default_factory=list, description="Hyperedges connecting multiple entities")-class NodeListSchema(BaseModel, Generic[Node]):+class NodeListSchema(BaseModel, Generic[NodeSchema]):"""Intermediate schema for batch node extraction."""- items: List[Node] = Field(+ items: List[NodeSchema] = Field(default_factory=list,description="List of identified entities or nodes found in the text.",)-class EdgeDescriptionList(BaseModel):- """Intermediate schema for identifying edge concepts first."""- items: List[str] = Field(description="List of primary edge concepts/themes identified")-----class EdgeListSchema(BaseModel, Generic[Edge]):+class EdgeListSchema(BaseModel, Generic[EdgeSchema]):"""Intermediate schema for batch hyperedge extraction."""- items: List[Edge] = Field(+ items: List[EdgeSchema] = Field(default_factory=list,description="List of identified hyperedges found in the text.",)class AutoHypergraph(- BaseAutoType[AutoHypergraphSchema[Node, Edge]], Generic[Node, Edge]+ BaseAutoType[AutoHypergraphSchema[NodeSchema, EdgeSchema]],+ Generic[NodeSchema, EdgeSchema],):"""AutoHypergraph - Node-First extraction: Extract entities first, then relationships.@@ -149,16 +136,16 @@ class AutoHypergraph("""if TYPE_CHECKING:- graph_schema: Type[AutoHypergraphSchema[Node, Edge]]+ graph_schema: Type[AutoHypergraphSchema[NodeSchema, EdgeSchema]]def __init__(self,- node_schema: Type[Node],- edge_schema: Type[Edge],- node_key_extractor: Callable[[Node], str],- edge_key_extractor: Callable[[Edge], str],+ node_schema: Type[NodeSchema],+ edge_schema: Type[EdgeSchema],+ node_key_extractor: Callable[[NodeSchema], str],+ edge_key_extractor: Callable[[EdgeSchema], str],# Returns a tuple of ALL node keys involved in this hyperedge- nodes_in_edge_extractor: Callable[[Edge], Tuple[str, ...]],+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, ...]],llm_client: BaseChatModel,embedder: Embeddings,*,@@ -318,7 +305,7 @@ class AutoHypergraph("""return DEFAULT_HYPERGRAPH_PROMPT- def _create_empty_instance(self) -> "AutoHypergraph[Node, Edge]":+ def _create_empty_instance(self) -> "AutoHypergraph[NodeSchema, EdgeSchema]":"""Creates a new empty AutoHypergraph instance with the same configuration.Overrides parent method to handle AutoHypergraph-specific parameters.@@ -391,12 +378,12 @@ class AutoHypergraph()@property- def nodes(self) -> List[Node]:+ def nodes(self) -> List[NodeSchema]:"""Returns the current node collection."""return self._node_memory.items@property- def edges(self) -> List[Edge]:+ def edges(self) -> List[EdgeSchema]:"""Returns the current hyperedge collection."""return self._edge_memory.items@@ -411,15 +398,43 @@ class AutoHypergraph(if self.extraction_mode == "two_stage":raw_graph = self._extract_data_by_two_stage(text)elif self.extraction_mode == "one_stage":- raise NotImplementedError(- "Single-stage extraction not yet supported for AutoHypergraph."- )+ raw_graph = self._extract_data_by_one_stage(text)else:raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")# Prune dangling hyperedges to ensure consistencyreturn self._prune_dangling_edges(raw_graph)+ def _extract_data_by_one_stage(self, text: str) -> AutoHypergraphSchema:+ """Extract nodes and edges simultaneously using single LLM call.++ Args:+ text: Input text.++ Returns:+ Raw extracted hypergraph data.+ """+ prompt_template = ChatPromptTemplate.from_template(+ f"{self.prompt}{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ self.graph_schema+ )++ # Extract from single chunk or multiple chunks+ if len(text) <= self.chunk_size:+ graph = llm_chain.invoke({"chunk_text": text})+ graph_list = [graph]+ else:+ chunks = self.text_splitter.split_text(text)+ inputs = [{"chunk_text": chunk} for chunk in chunks]+ graph_list = llm_chain.batch(+ inputs, config={"max_concurrency": self.max_workers}+ )++ # Merge multiple hypergraphs+ return self.merge_batch_data(graph_list)+def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:"""Extract nodes first, then hyperedges with node context (batch processing).@@ -453,7 +468,9 @@ class AutoHypergraph(# 5. Global Mergereturn self.merge_batch_data(partial_hypergraphs)- def _extract_nodes_batch(self, chunks: List[str]) -> List[NodeListSchema[Node]]:+ def _extract_nodes_batch(+ self, chunks: List[str]+ ) -> List[NodeListSchema[NodeSchema]]:"""Batch extract nodes from multiple text chunks."""prompt_template = ChatPromptTemplate.from_template(f"{self.node_prompt}{{chunk_text}}"@@ -466,8 +483,8 @@ class AutoHypergraph(return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})def _extract_edges_batch(- self, chunks: List[str], node_lists: List[NodeListSchema[Node]]- ) -> List[EdgeListSchema[Edge]]:+ self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]+ ) -> List[EdgeListSchema[EdgeSchema]]:"""Batch extract hyperedges using corresponding node lists as context."""inputs = []for chunk, node_list in zip(chunks, node_lists):@@ -542,7 +559,7 @@ class AutoHypergraph(def merge_batch_data(self,data_list_or_tuple: List[AutoHypergraphSchema]- | Tuple[List[List[Node]], List[List[Edge]]],+ | Tuple[List[List[NodeSchema]], List[List[EdgeSchema]]],) -> AutoHypergraphSchema:"""Merge multiple hypergraphs or node/edge tuples into one.@@ -620,7 +637,7 @@ class AutoHypergraph(query: str,top_k_nodes: int = 3,top_k_edges: int = 3,- ) -> Tuple[List[Node], List[Edge]]:+ ) -> Tuple[List[NodeSchema], List[EdgeSchema]]:"""Unified hypergraph search interface.Retrieves nodes and hyperedges semantically related to the query.@@ -645,8 +662,8 @@ class AutoHypergraph("At least one of top_k_nodes or top_k_edges must be positive.")- nodes: List[Node] = []- edges: List[Edge] = []+ nodes: List[NodeSchema] = []+ edges: List[EdgeSchema] = []if top_k_nodes > 0:if not self._node_memory.has_index():@@ -660,11 +677,11 @@ class AutoHypergraph(return nodes, edges- def search_nodes(self, query: str, top_k: int = 3) -> List[Node]:+ def search_nodes(self, query: str, top_k: int = 3) -> List[NodeSchema]:"""Semantic search for nodes/entities only."""return self._node_memory.search(query=query, top_k=top_k)- def search_edges(self, query: str, top_k: int = 3) -> List[Edge]:+ def search_edges(self, query: str, top_k: int = 3) -> List[EdgeSchema]:"""Semantic search for hyperedges/relationships only."""return self._edge_memory.search(query=query, top_k=top_k)@@ -782,193 +799,3 @@ class AutoHypergraph(self._edge_memory.load_index(str(edge_index_path))except Exception as e:logger.warning(f"Failed to load edge index: {e}")--class EdgeAutoHypergraph(AutoHypergraph[Node, Edge], Generic[Node, Edge]):- """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.-- Extraction Strategy: Two-Stage (Edge-First)- 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).- 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.-- Suitable for:- - Narrative-driven knowledge graphs (Theme -> Character)- - Event-driven extraction (Event -> Participant)- """-- def __init__(- self,- node_schema: Type[Node],- edge_schema: Type[Edge],- node_key_extractor: Callable[[Node], str],- edge_key_extractor: Callable[[Edge], str],- nodes_in_edge_extractor: Callable[[Edge], Tuple[str, ...]],- llm_client: BaseChatModel,- embedder: Embeddings,- *,- prompt_for_node_extraction: str = "",- **kwargs: Any,- ):- """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.-- Args:- prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".- (Extract Nodes given Edges).- """- super().__init__(- node_schema=node_schema,- edge_schema=edge_schema,- node_key_extractor=node_key_extractor,- edge_key_extractor=edge_key_extractor,- nodes_in_edge_extractor=nodes_in_edge_extractor,- llm_client=llm_client,- embedder=embedder,- prompt_for_node_extraction=prompt_for_node_extraction,- **kwargs,- )-- # Build Stage 2 Schema (Edges containing nested Nodes)- # We assume the edge schema has a 'participants' field (List[str]).- # We create a new model where 'participants' is List[NodeSchema].-- # 1. Base fields from EdgeSchema (excluding participants)- valid_fields = {- k: (v.annotation, v)- for k, v in edge_schema.model_fields.items()- if k != "participants"- }-- # 2. Add Nested Participants field- valid_fields["participants"] = (- List[node_schema],- Field(description="List of entities participating in this concept/edge.")- )-- # 3. Create Contextual Edge Model- self.contextual_edge_schema = create_model(- f"Contextual{edge_schema.__name__}",- **valid_fields- )-- # 4. Create Result List Model- self.stage2_result_schema = create_model(- f"{edge_schema.__name__}FirstResult",- items=(List[self.contextual_edge_schema], Field(default_factory=list))- )-- # Set default prompt if not provided- if not prompt_for_node_extraction:- self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT-- def _extract_data(self, text: str) -> AutoHypergraphSchema:- """Main extraction logic dispatcher (Edge-First)."""- if self.extraction_mode == "two_stage":- raw_graph = self._extract_data_by_two_stage(text)- elif self.extraction_mode == "one_stage":- raise NotImplementedError(- "Single-stage extraction not yet supported for EdgeAutoHypergraph."- )- else:- raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")-- # Note: Dangle pruning might be less critical here since nodes are derived from edges,- # but we keep it for consistency if nodes are merged globally.- return self._prune_dangling_edges(raw_graph)-- def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:- """Extract edges/themes first, then entities within those contexts (batch processing).-- Process:- 1. Split text into chunks.- 2. Batch extract Edge Concepts (Theme Strings) for all chunks.- 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).- 4. Merge all partial results into one global hypergraph.- """- # 1. Prepare chunks- if len(text) <= self.chunk_size:- chunks = [text]- else:- chunks = self.text_splitter.split_text(text)-- if self.verbose:- logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")-- # 2. Stage 1: Identify Edge Concepts (Strings)- edge_concepts_lists = self._extract_edge_concepts_batch(chunks)-- # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)- chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)-- # 4. Global Merge- # chunk_results is List[AutoHypergraphSchema]- return self.merge_batch_data(chunk_results)-- def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:- """Stage 1: Batch extract edge concept strings."""- # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts- # We assume the prompt is geared towards finding a LIST of themes/concepts.-- prompt_template = ChatPromptTemplate.from_template(- f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"- )- chain = prompt_template | self.llm_client.with_structured_output(EdgeDescriptionList)-- inputs = [{"chunk_text": chunk} for chunk in chunks]- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})-- return [res.items if res else [] for res in results]-- def _extract_details_batch(self, chunks: List[str], edge_concepts_list: List[List[str]]) -> List[AutoHypergraphSchema]:- """Generic Stage 2: Extract entities and details given identified edge concepts."""-- # 1. Prepare Inputs- inputs = []- valid_map = {}- for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):- if concepts:- concepts_str = "- " + "\n- ".join(concepts)- inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})- valid_map[len(inputs) - 1] = i-- if not inputs:- return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]-- # 2. Chain Execution- # Uses self.node_prompt as the instruction for contextual node extraction- prompt = ChatPromptTemplate.from_template(- f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"- )- chain = prompt | self.llm_client.with_structured_output(self.stage2_result_schema)-- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})-- # 3. Transform Nested Result -> Flat AutoHypergraphSchema- final_schemas = [self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))]-- for i, result in enumerate(results):- if not result or not result.items:- continue-- original_idx = valid_map[i]- current_nodes = []- current_edges = []-- for item in result.items:- # item is a ContextualEdge (has nested participants=List[Node])-- # Extract Nodes- nested_nodes = item.participants- node_names = []- for n in nested_nodes:- current_nodes.append(n)- node_names.append(self.node_key_extractor(n))-- # Reconstruct Flat Edge- edge_data = item.model_dump()- edge_data["participants"] = sorted(node_names) # Ensure sorted for consistency-- new_edge = self.edge_schema(**edge_data)- current_edges.append(new_edge)-- final_schemas[original_idx] = self.graph_schema(nodes=current_nodes, edges=current_edges)-- return final_schemasdiff --git a/hyperextract/hypergraphs/cog_rag.py b/hyperextract/hypergraphs/cog_rag.pyindex f363ecc..525ef1c 100644--- a/hyperextract/hypergraphs/cog_rag.py+++ b/hyperextract/hypergraphs/cog_rag.py@@ -3,15 +3,16 @@ COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System PatternExtracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities."""-from typing import List-from pydantic import BaseModel, Field-from hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchema, EdgeAutoHypergraph+from typing import List, Type, Callable, Tuple, Any+from pydantic import BaseModel, Field, create_model+from hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchemafrom hyperextract.utils.logging import loggerfrom langchain_core.prompts import ChatPromptTemplatefrom langchain_core.embeddings import Embeddingsfrom langchain_core.language_models import BaseChatModelfrom ontomem.merger import CustomRuleMerger+# ============================================================================# Node Schema# ============================================================================@@ -68,10 +69,29 @@ class EdgeSchema(BaseModel):description="Numerical score indicating relationship strength (1-10)",)++class EdgeDescriptionList(BaseModel):+ """Intermediate schema for identifying edge concepts first."""++ items: List[str] = Field(+ description="List of primary edge concepts/themes identified"+ )++# ============================================================================# Extraction Prompts# ============================================================================++DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal-+You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.+Your task is to Populate these concepts with their specific Participant Entities.++CRITICAL RULES:+1. Extract ALL key participants for each concept.+2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).+"""+COG_RAG_THEME_PROMPT = """-Goal-Analyze the text to identify the primary themes (narrative arcs).@@ -149,6 +169,211 @@ Merge strategy:# ============================================================================+class EdgeAutoHypergraph(AutoHypergraph[NodeSchema, EdgeSchema]):+ """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.++ Extraction Strategy: Two-Stage (Edge-First)+ 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).+ 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.++ Suitable for:+ - Narrative-driven knowledge graphs (Theme -> Character)+ - Event-driven extraction (Event -> Participant)+ """++ def __init__(+ self,+ node_schema: Type[NodeSchema],+ edge_schema: Type[EdgeSchema],+ node_key_extractor: Callable[[NodeSchema], str],+ edge_key_extractor: Callable[[EdgeSchema], str],+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, ...]],+ llm_client: BaseChatModel,+ embedder: Embeddings,+ *,+ prompt_for_node_extraction: str = "",+ **kwargs: Any,+ ):+ """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.++ Args:+ prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".+ (Extract Nodes given Edges).+ """+ super().__init__(+ node_schema=node_schema,+ edge_schema=edge_schema,+ node_key_extractor=node_key_extractor,+ edge_key_extractor=edge_key_extractor,+ nodes_in_edge_extractor=nodes_in_edge_extractor,+ llm_client=llm_client,+ embedder=embedder,+ prompt_for_node_extraction=prompt_for_node_extraction,+ **kwargs,+ )++ # Build Stage 2 Schema (Edges containing nested Nodes)+ # We assume the edge schema has a 'participants' field (List[str]).+ # We create a new model where 'participants' is List[NodeSchema].++ # 1. Base fields from EdgeSchema (excluding participants)+ valid_fields = {+ k: (v.annotation, v)+ for k, v in edge_schema.model_fields.items()+ if k != "participants"+ }++ # 2. Add Nested Participants field+ valid_fields["participants"] = (+ List[node_schema],+ Field(description="List of entities participating in this concept/edge."),+ )++ # 3. Create Contextual Edge Model+ self.contextual_edge_schema = create_model(+ f"Contextual{edge_schema.__name__}", **valid_fields+ )++ # 4. Create Result List Model+ self.stage2_result_schema = create_model(+ f"{edge_schema.__name__}FirstResult",+ items=(List[self.contextual_edge_schema], Field(default_factory=list)),+ )++ # Set default prompt if not provided+ if not prompt_for_node_extraction:+ self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT++ def _extract_data(self, text: str) -> AutoHypergraphSchema:+ """Main extraction logic dispatcher (Edge-First)."""+ if self.extraction_mode == "two_stage":+ raw_graph = self._extract_data_by_two_stage(text)+ elif self.extraction_mode == "one_stage":+ raise NotImplementedError(+ "Single-stage extraction not yet supported for EdgeAutoHypergraph."+ )+ else:+ raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")++ # Note: Dangle pruning might be less critical here since nodes are derived from edges,+ # but we keep it for consistency if nodes are merged globally.+ return self._prune_dangling_edges(raw_graph)++ def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:+ """Extract edges/themes first, then entities within those contexts (batch processing).++ Process:+ 1. Split text into chunks.+ 2. Batch extract Edge Concepts (Theme Strings) for all chunks.+ 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).+ 4. Merge all partial results into one global hypergraph.+ """+ # 1. Prepare chunks+ if len(text) <= self.chunk_size:+ chunks = [text]+ else:+ chunks = self.text_splitter.split_text(text)++ if self.verbose:+ logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")++ # 2. Stage 1: Identify Edge Concepts (Strings)+ edge_concepts_lists = self._extract_edge_concepts_batch(chunks)++ # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)+ chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)++ # 4. Global Merge+ # chunk_results is List[AutoHypergraphSchema]+ return self.merge_batch_data(chunk_results)++ def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:+ """Stage 1: Batch extract edge concept strings."""+ # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts+ # We assume the prompt is geared towards finding a LIST of themes/concepts.++ prompt_template = ChatPromptTemplate.from_template(+ f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"+ )+ chain = prompt_template | self.llm_client.with_structured_output(+ EdgeDescriptionList+ )++ inputs = [{"chunk_text": chunk} for chunk in chunks]+ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})++ return [res.items if res else [] for res in results]++ def _extract_details_batch(+ self, chunks: List[str], edge_concepts_list: List[List[str]]+ ) -> List[AutoHypergraphSchema]:+ """Generic Stage 2: Extract entities and details given identified edge concepts."""++ # 1. Prepare Inputs+ inputs = []+ valid_map = {}+ for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):+ if concepts:+ concepts_str = "- " + "\n- ".join(concepts)+ inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})+ valid_map[len(inputs) - 1] = i++ if not inputs:+ return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]++ # 2. Chain Execution+ # Uses self.node_prompt as the instruction for contextual node extraction+ prompt = ChatPromptTemplate.from_template(+ f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"+ )+ chain = prompt | self.llm_client.with_structured_output(+ self.stage2_result_schema+ )++ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})++ # 3. Transform Nested Result -> Flat AutoHypergraphSchema+ final_schemas = [+ self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))+ ]++ for i, result in enumerate(results):+ if not result or not result.items:+ continue++ original_idx = valid_map[i]+ current_nodes = []+ current_edges = []++ for item in result.items:+ # item is a ContextualEdge (has nested participants=List[Node])++ # Extract Nodes+ nested_nodes = item.participants+ node_names = []+ for n in nested_nodes:+ current_nodes.append(n)+ node_names.append(self.node_key_extractor(n))++ # Reconstruct Flat Edge+ edge_data = item.model_dump()+ edge_data["participants"] = sorted(+ node_names+ ) # Ensure sorted for consistency++ new_edge = self.edge_schema(**edge_data)+ current_edges.append(new_edge)++ final_schemas[original_idx] = self.graph_schema(+ nodes=current_nodes, edges=current_edges+ )++ return final_schemas+++# ============================================================================++class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):"""Layer 1 of Cog_RAG: Theme-First Extraction.@@ -228,6 +453,7 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):verbose=verbose,)+# ============================================================================# Detail Hypergraph (Layer 2: Micro Relations)# ============================================================================@@ -295,15 +521,16 @@ class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):# Main System Class (Wrapper)# ============================================================================+class Cog_RAG:"""Cognitive-Inspired Dual-Hypergraph RAG System.-+Combines two Hypergraph Layers:1. Theme Layer (Cog_RAG_ThemeLayer): Captures macro narratives and themes.2. Detail Layer (Cog_RAG_DetailLayer): Captures micro entity relationships."""-+def __init__(self,llm_client: BaseChatModel,@@ -319,7 +546,7 @@ class Cog_RAG:chunk_size=chunk_size,chunk_overlap=chunk_overlap,max_workers=max_workers,- verbose=verbose+ verbose=verbose,)self.detail_layer = Cog_RAG_DetailLayer(llm_client=llm_client,@@ -327,7 +554,7 @@ class Cog_RAG:chunk_size=chunk_size,chunk_overlap=chunk_overlap,max_workers=max_workers,- verbose=verbose+ verbose=verbose,)self.llm = llm_clientself.verbose = verbose@@ -337,7 +564,7 @@ class Cog_RAG:if self.verbose:logger.info("Cog_RAG: Feeding text to Theme Layer...")self.theme_layer.feed_text(text)-+if self.verbose:logger.info("Cog_RAG: Feeding text to Detail Layer...")self.detail_layer.feed_text(text)@@ -358,36 +585,35 @@ class Cog_RAG:"""# 1. Macro Context: Themesthemes = self.theme_layer.search_edges(query, top_k=top_k_themes)-+# 2. Micro Context: Entities (Nodes)# Note: The user requested searching for "Specific Entities" in the second step.entities = self.detail_layer.search_nodes(query, top_k=top_k_entities)-- return {- "themes": themes,- "entities": entities- }++ return {"themes": themes, "entities": entities}def chat(self, query: str, top_k_themes: int = 3, top_k_entities: int = 3):"""Generate an answer using context from both layers."""results = self.search(query, top_k_themes, top_k_entities)-+context_parts = []if results["themes"]:context_parts.append("=== MACRO THEMES (Narrative Context) ===")for t in results["themes"]:- context_parts.append(f"- {t.description} (Involved: {', '.join(t.participants)})")-+ context_parts.append(+ f"- {t.description} (Involved: {', '.join(t.participants)})"+ )+if results["entities"]:context_parts.append("\n=== MICRO ENTITIES (Specific Details) ===")for e in results["entities"]:context_parts.append(f"- [{e.type}] {e.name}: {e.description}")-+if not context_parts:context = "No relevant information found."else:context = "\n".join(context_parts)-+prompt = ChatPromptTemplate.from_template("You are an intelligent assistant using a Dual-Layer Hypergraph Knowledge Base.\n""Use the provided Themes (Macro) and Key Entities (Micro) to answer the user's question comprehensively.\n\n"@@ -395,13 +621,14 @@ class Cog_RAG:"Question: {question}\n\n""Answer:")-+chain = prompt | self.llmreturn chain.invoke({"context": context, "question": query})-+def dump(self, folder_path: str):"""Save both extracted graphs."""import os+self.theme_layer.dump(os.path.join(folder_path, "theme_layer"))self.detail_layer.dump(os.path.join(folder_path, "detail_layer"))@@ -422,4 +649,3 @@ class Cog_RAG:def edges(self):"""Return all edges (Themes + Relations)."""return self.theme_layer.edges + self.detail_layer.edges-diff --git a/hyperextract/hypergraphs/hypergraph_rag.py b/hyperextract/hypergraphs/hypergraph_rag.pyindex 7903039..ac4cd39 100644--- a/hyperextract/hypergraphs/hypergraph_rag.py+++ b/hyperextract/hypergraphs/hypergraph_rag.py@@ -1,14 +1,24 @@-"""-HyperGraph_RAG: Integration of EdgeAutoHypergraph (Edge-First Extraction).-Matches the structure of Hyper_RAG regarding Schemas and Mergers.+"""HyperGraph_RAG implementation using AutoHypergraph as the core engine.++This module provides a specialized implementation of the HyperGraphRAG algorithm+using the AutoHypergraph framework, designed for extracting semantic knowledge+represented as hypergraphs where hyperedges ("knowledge segments") connect+multiple entities within specific contexts or events.++The HyperGraphRAG approach represents knowledge as contextual relationships where+each hyperedge encodes a complete semantic unit (sentence or event), maintaining+the natural information flow and contextual dependencies of the source material."""from typing import List+from hashlib import md5from pydantic import BaseModel, Field-from hyperextract.hypergraphs.base import EdgeAutoHypergraphfrom langchain_core.embeddings import Embeddingsfrom langchain_core.language_models import BaseChatModel-from ontomem.merger import CustomRuleMerger+from langchain_core.prompts.chat import ChatPromptTemplate+from ontomem.merger import MergeStrategy++from .base import AutoHypergraph, AutoHypergraphSchema# ============================================================================# Node Schema@@ -16,14 +26,19 @@ from ontomem.merger import CustomRuleMergerclass NodeSchema(BaseModel):- """Represents an entity extracted from the source text."""+ """Represents an entity extracted from the source text within a specific context."""- name: str = Field(description="Name of the entity")+ name: str = Field(+ description="Name of the entity. Use the same language as the input text. If English, capitalize the name."+ )type: str = Field(- description="Entity type (person, organization, geo, event, role, concept, etc.)",+ description="Type of the entity (e.g., organization, person, geo, event, category, concept, object).")description: str = Field(- description="Comprehensive description of the entity's attributes and activities",+ description="Comprehensive description of the entity's attributes and activities strictly within the current knowledge segment."+ )+ key_score: int = Field(+ description="A score from 0 to 100 indicating the importance of the entity within this context.")@@ -33,73 +48,121 @@ class NodeSchema(BaseModel):class EdgeSchema(BaseModel):- """Represents a relationship or event connecting multiple entities (low-order or high-order)."""+ """Represents a knowledge segment (hyper-edge) that connects multiple entities through a specific context or event."""- participants: List[str] = Field(- description="Names of entities involved in this relationship"+ knowledge_segment: str = Field(+ description="A complete sentence or phrase from the text that captures a specific relationship, event, or context connecting the entities.")- description: str = Field(- description="Detailed explanation of the relationship or event"- )- keywords: List[str] = Field(- description="List of keywords summarizing the relationship themes (e.g., ['conflict', 'trade', 'alliance'])",+ completeness_score: int = Field(+ description="A score from 0 to 10 indicating how complete and meaningful this knowledge segment is on its own.")- strength: int = Field(- ge=1,- le=10,- description="Numerical score indicating relationship strength (1-10)",+ related_entities: List[str] = Field(+ description="A list of all entity names involved in or referenced by this knowledge segment.")-# ============================================================================-# Extraction Prompts (Edge-First)-# ============================================================================+# Below Schema is used for Information Extraction with LLM+class TEMP_EdgeSchema(BaseModel):+ """Represents a knowledge segment (hyper-edge) that connects multiple entities through a specific context or event."""-HyperGraph_RAG_THEME_EXTRACTION_PROMPT = """--Goal--Identify the key "Themes", "Narratives", or "Complex Events" present in the text.-A Theme/Event should represent a significant interaction or grouping involving multiple participants.+ knowledge_segment: str = Field(+ description="A complete sentence or phrase from the text that captures a specific relationship, event, or context connecting the entities."+ )+ completeness_score: int = Field(+ description="A score from 0 to 10 indicating how complete and meaningful this knowledge segment is on its own."+ )+ related_entities: List[NodeSchema] = Field(+ description="A list of all entities involved in or referenced by this knowledge segment."+ )--Output--Return a list of these Themes/Events as descriptive strings.-Do not list participants data formats yet, just the essence of the event (e.g., "The secret meeting at Titan Station", "The ambush in the Aurora Sector").-"""-HyperGraph_RAG_CONTEXT_NODE_PROMPT = """--Goal--You are an expert knowledge extractor.-You have been given a list of "Themes/Events" (Hyperedges) that occurred in the text.-Your job is to identify the specific ENTITIES (participants) involved in each theme.+class TEMP_EdgeListSchema(BaseModel):+ """Intermediate schema for batch hyperedge extraction."""--Output--For each Theme, list the entities that participated.-"""+ items: List[TEMP_EdgeSchema] = Field(+ description="List of extracted hyperedges with their knowledge segments and related entities."+ )# ============================================================================-# Merge Templates for LLM.CUSTOM_RULE Strategy+# Extraction Prompts# ============================================================================-Hyper_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.-You will receive a list of objects representing the same Entity.--Your task is to merge them into a SINGLE object exactly matching the schema.--Merge strategy:-1. **name/type**: Keep the most frequent or precise value.-2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.-"""--Hyper_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.-You will receive a list of objects representing the same Hyperedge (Relationship/Event).--Your task is to merge them into a SINGLE object exactly matching the schema.--Merge strategy:-1. **participants**: Keep the list (they should be identical for the same hyperedge key).-2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.-3. **keywords**: Combine keyword lists from all inputs, removing duplicates.-4. **strength**: Calculate the average of the input strengths (round to nearest integer).+HyperGraph_RAG_EDGE_EXTRACTION_PROMPT = """+You are an expert information extraction system specializing in Hypergraph RAG.+Your goal is to extract "Hyper-edges" from the provided text.++A **Hyper-edge** represents a specific "knowledge segment" (a sentence or event context) that connects multiple entities together.++# Instructions+1. **Analyze the Text**: Break the text down into complete, meaningful knowledge segments (sentences or clauses describing an event/relationship).+2. **Extract Entities**: For *each* segment, identify all relevant entities involved.+3. **Structure the Output**: For each segment, output an `EdgeSchema` containing:+ - The `knowledge_segment` text.+ - A `completeness_score` (0-10).+ - A list of `related_entities` found strictly within that segment.++# Extraction Rules+- **Entity Name**: Keep original language. Capitalize if English.+- **Entity Description**: Describe what the entity is doing or how it represents within *that specific segment*.+- **Entity Type**: Common types include organization, person, geo, event, category, concept, object.++# Few-Shot Examples++## Example 1+**Input Text**:+"Alex clenched his jaw, the buzz of frustration dull against the backdrop of Taylor's authoritarian certainty."++**Output**:+[+ {+ "knowledge_segment": "Alex clenched his jaw, the buzz of frustration dull against the backdrop of Taylor's authoritarian certainty.",+ "completeness_score": 8,+ "related_entities": [+ {+ "name": "Alex",+ "type": "person",+ "description": "Alex is a person displaying frustration by clenching his jaw.",+ "key_score": 90+ },+ {+ "name": "Taylor",+ "type": "person",+ "description": "Taylor is a person exhibiting authoritarian certainty causing Alex's frustration.",+ "key_score": 85+ }+ ]+ }+]++## Example 2+**Input Text**:+"The device could change the game for us," Taylor said, observing the machine with reverence.++**Output**:+[+ {+ "knowledge_segment": "\"The device could change the game for us,\" Taylor said, observing the machine with reverence.",+ "completeness_score": 9,+ "related_entities": [+ {+ "name": "Taylor",+ "type": "person",+ "description": "Taylor is a person who believes the device is game-changing and observes it with reverence.",+ "key_score": 95+ },+ {+ "name": "device",+ "type": "object",+ "description": "An object that Taylor believes could change the game.",+ "key_score": 90+ }+ ]+ }+]++\n\n+### Source Text:"""# ============================================================================@@ -107,13 +170,34 @@ Merge strategy:# ============================================================================-class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):- """- HyperGraph_RAG: Hypergraph-based RAG with Edge-First Extraction Strategy.+class HyperGraph_RAG(AutoHypergraph[NodeSchema, EdgeSchema]):+ """HyperGraphRAG extractor using semantic knowledge segments as hyperedges.++ This class implements the HyperGraphRAG algorithm which models knowledge as a hypergraph+ where each hyperedge represents a complete "knowledge segment" (atomic semantic unit)+ that connects multiple related entities. Unlike traditional knowledge graphs that represent+ pairwise relationships, hypergraphs can naturally express n-ary relationships and maintain+ the original context and semantic integrity of information.++ The extraction process simultaneously identifies both knowledge segments (hyperedges) and+ the entities involved within each segment, followed by deduplication and merging across+ multiple text chunks.- Difference from Hyper_RAG:- - Uses EdgeAutoHypergraph (extracts Themes first, then Entities).- - Suitable for high-level narrative understanding where Themes map to specific Actors.+ **Schema Components:**++ - **NodeSchema**: Represents an entity (node) extracted from the source text within a specific context.+ - `name` (str): The entity name, maintaining original language and capitalization.+ - `type` (str): Entity type such as 'person', 'organization', 'event', 'location', 'concept', etc.+ - `description` (str): Comprehensive description of the entity's attributes and activities within the knowledge segment.+ - `key_score` (int): Importance score (0-100) indicating the entity's significance in context.++ - **EdgeSchema**: Represents a knowledge segment (hyper-edge) that connects multiple entities through+ a specific context or event.+ - `knowledge_segment` (str): The actual text span (sentence or phrase) from source material capturing+ the relationship, event, or context connecting entities.+ - `completeness_score` (int): Quality score (0-10) indicating how complete and meaningful the segment+ is as a standalone knowledge unit.+ - `related_entities` (List[str]): Names of all entities involved in or referenced by this knowledge segment."""def __init__(@@ -125,38 +209,25 @@ class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):max_workers: int = 10,verbose: bool = True,):- """- Initialize HyperGraph_RAG engine.+ """Initialize HyperGraph_RAG extraction engine.++ Configures the hypergraph extraction pipeline with LLM-based knowledge segment+ and entity identification, along with semantic embeddings for vector-based retrieval.Args:- llm_client (BaseChatModel): LLM client for text generation.- embedder (Embeddings): Embedding model for text embeddings.- chunk_size (int): Size of text chunks for extraction.- chunk_overlap (int): Overlap between text chunks.- max_workers (int): Maximum number of workers for extraction.- verbose (bool): Display detailed logs.+ llm_client (BaseChatModel): Language model client for structured extraction.+ embedder (Embeddings): Embedding model for semantic vector representation.+ chunk_size (int): Text chunk size for processing (default: 2048 tokens).+ chunk_overlap (int): Overlap between consecutive chunks to preserve context (default: 256 tokens).+ max_workers (int): Maximum parallel workers for batch extraction (default: 10).+ verbose (bool): Whether to display detailed extraction logs (default: True)."""# 1. Define Key Extractorsnode_key_fn = lambda x: x.name- edge_key_fn = lambda x: tuple(sorted(x.participants))- nodes_in_edge_fn = lambda x: x.participants-- # 2. Setup Mergers- node_merger = CustomRuleMerger(- key_extractor=node_key_fn,- llm_client=llm_client,- item_schema=NodeSchema,- rule=Hyper_RAG_NODE_MERGE_RULE,- )-- edge_merger = CustomRuleMerger(- key_extractor=edge_key_fn,- llm_client=llm_client,- item_schema=EdgeSchema,- rule=Hyper_RAG_EDGE_MERGE_RULE,- )+ edge_key_fn = lambda x: md5(x.knowledge_segment.encode()).hexdigest()+ nodes_in_edge_fn = lambda x: x.related_entities- # 3. Initialize EdgeAutoHypergraph (Parent)+ # 4. Call parent class initializationsuper().__init__(node_schema=NodeSchema,edge_schema=EdgeSchema,@@ -165,21 +236,82 @@ class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):nodes_in_edge_extractor=nodes_in_edge_fn,llm_client=llm_client,embedder=embedder,- # Prompts- # Stage 1: Extract Themes- prompt_for_edge_extraction=HyperGraph_RAG_THEME_EXTRACTION_PROMPT,- # Stage 2: Extract Particles given Themes- prompt_for_node_extraction=HyperGraph_RAG_CONTEXT_NODE_PROMPT,- # Merger strategies- node_strategy_or_merger=node_merger,- edge_strategy_or_merger=edge_merger,- # Indexing optimization+ # Inject customized prompts for extraction+ prompt_for_edge_extraction=HyperGraph_RAG_EDGE_EXTRACTION_PROMPT,+ # Pass custom merger instances+ node_strategy_or_merger=MergeStrategy.LLM.BALANCED,+ edge_strategy_or_merger=MergeStrategy.LLM.BALANCED,+ # Optimize indexingnode_fields_for_index=["name", "type", "description"],- edge_fields_for_index=["keywords", "description"],- # BaseAutoType Params+ edge_fields_for_index=["knowledge_segment"],+ # Other parameterschunk_size=chunk_size,chunk_overlap=chunk_overlap,max_workers=max_workers,verbose=verbose,)+ def _extract_data(self, text: str) -> AutoHypergraphSchema[NodeSchema, EdgeSchema]:+ """Extract hypergraph knowledge from text via knowledge segment identification.++ Performs simultaneous extraction of knowledge segments (hyperedges) and associated entities+ from the input text. The extraction process:+ 1. Segments the text into knowledge-bearing units (sentences/clauses)+ 2. Identifies all entities referenced within each knowledge segment+ 3. Merges and deduplicates extracted components across multiple chunks+ 4. Prunes dangling edges (hyperedges without valid entity references)++ Args:+ text (str): Source text for hypergraph extraction.++ Returns:+ AutoHypergraphSchema[NodeSchema, EdgeSchema]: Extracted hypergraph containing+ deduplicated and merged entities (nodes) and knowledge segments (hyperedges).+ """+ # 1. Extract Hyperedges++ prompt_template = ChatPromptTemplate.from_template(+ f"{self.prompt}{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ TEMP_EdgeListSchema+ )++ # Extract from single chunk or multiple chunks+ if len(text) <= self.chunk_size:+ raw_hyperedges = llm_chain.invoke({"chunk_text": text})+ raw_hyperedges_list = [raw_hyperedges]+ else:+ chunks = self.text_splitter.split_text(text)+ inputs = [{"chunk_text": chunk} for chunk in chunks]+ raw_hyperedges_list = llm_chain.batch(+ inputs, config={"max_concurrency": self.max_workers}+ )++ # Extract Nodes and Edges from raw_hyperedges_list+ all_nodes_list, all_edges_list = [], []++ for raw_hyperedges in raw_hyperedges_list:+ cur_nodes, cur_edges = [], []+ data_list = raw_hyperedges.items+ for data in data_list:+ data: TEMP_EdgeSchema+ cur_nodes.extend(data.related_entities)+ cur_edges.append(+ EdgeSchema(+ knowledge_segment=data.knowledge_segment,+ completeness_score=data.completeness_score,+ related_entities=[+ self.node_key_extractor(node)+ for node in data.related_entities+ ],+ )+ )+ all_nodes_list.append(cur_nodes)+ all_edges_list.append(cur_edges)++ # Merge multiple hypergraphs+ partial_hypergraphs = (all_nodes_list, all_edges_list)+ raw_hypergraph = self.merge_batch_data(partial_hypergraphs)++ return self._prune_dangling_edges(raw_hypergraph)