Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
add cog_rag and hypergraph_rag
haoohu39c73f4a29d6
diff --git a/examples/cog_rag_demo.py b/examples/cog_rag_demo.pynew file mode 100644index 0000000..79952f8--- /dev/null+++ b/examples/cog_rag_demo.py@@ -0,0 +1,282 @@+"""+Cog_RAG Demo: Theme-based Knowledge Extraction + Entity-Relation Detail Retrieval++This demo demonstrates the unified Cog_RAG System Wrapper:+1. **Layer 1 (Theme Layer)**: Extracts Key Entities and Themes (Narrative Arcs).+2. **Layer 2 (Detail Layer)**: Extracts Entities and Semantic Relationships.+3. **Process**:+ - Feeds the same story to the Unified System.+ - Performs a "Theme-First" retrieval process automatically.+"""++import os+import sys++# 添加项目根目录到路径+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from langchain_openai import ChatOpenAI, OpenAIEmbeddings++# Try to load dotenv, ignoring if not installed+try:+ from dotenv import load_dotenv+ load_dotenv()+except ImportError:+ pass++from hyperextract.hypergraphs import Cog_RAG++# ============================================================================+# Sample Story for Extraction (Shared with AutoHypergraph Demo)+# ============================================================================++STORY = """+【银河纪元 3042年 - 赤红星云事件报告】++危机在这一周达到了顶峰。整个银河系似乎都陷入了暗流涌动的政治漩涡。++【第一阶段:秘密接触】+臭名昭著的星际海盗黑蛇驾驶着他的旗舰复仇号,在泰坦空间站秘密会见了银河联邦的背叛者议员 Valerius。+目击者称,两人在午夜时分登上了空间站的顶层禁区。据情报部门推测,这是一次肮脏的交易——+双方达成了针对皇室的同盟关系。参与此会议的还有 Valerius 手下的特务头目 Shadow,+他负责处理所有不可告人的细节。这个同盟的形成标志着大麻烦即将到来。++【第二阶段:叛变与夺权】+在这次会议的两天后,事态急剧恶化。黑蛇这种疯狂的家伙竟然在极光星区伏击了皇家护卫队。+这场突然的冲突中,皇家护卫队的队长 Leona 与副队长 Marcus 奋力抵抗,+但他们陷入了黑蛇手下的精锐战士的包围。这场激战导致队长 Leona 身受重伤。+而黑蛇的目标不是为了杀戮,而是为了夺取一件神圣的物品——传说中的虚空水晶。+这枚水晶散发着幽蓝的光芒,据说能够打开先祖之门的第一道密封。+黑蛇在战斗中成功夺取了水晶,但 Leona 在临终前用最后的力量将其重新夺回。++【第三阶段:流亡与隐藏】+Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9。+在这个被遗忘的角落,她向仅有的几个信任者求救:还有副队长 Marcus 以及隐秘组织"地下抵抗军"的领导人 Cipher。+同时,议员 Valerius 非法持有了开启先祖之门的钥匙——这把钥匙本应被安全保管在皇家金库中。+他现在藏匿在浮空都市中,与黑蛇遥相呼应,计划着下一步的政变。++【第四阶段:势力重组】+银河联邦的最高委员会得知了这一系列事件,决定采取行动。+高级顾问 Artemis 被任命为特使,她与 Leona 在废土行星 Z-9 进行了秘密会晤,+讨论如何制止这场即将到来的危机。同时,地下抵抗军的 Cipher 也加入了这次会议,+三人共同制定了一个大胆的计划。++【第五阶段:最后的筹码】+虽然黑蛇未能获得虚空水晶,但他已经从 Valerius 那里获得了先祖之门钥匙的复制品。+根据情报,这个复制品目前被秘密运往到黑蛇的秘密基地——位于辐射荒漠中的要塞。+参与此次运送的还有 Shadow 和黑蛇手下的精锐战士群体。+与此同时,Leona 和 Marcus 正在准备一场反击,他们计划在虚空水晶的力量下重新夺回局势的控制权。++目前局势:+- 黑蛇:手持复制品钥匙,掌控旗舰复仇号,计划进行最终的攻击+- 议员 Valerius:躲在浮空都市中,与黑蛇保持联系+- Leona:守护虚空水晶,在废土行星 Z-9 养伤,准备反击+- 银河联邦:通过特使 Artemis 支持 Leona,正在调集力量+- 地下抵抗军:Cipher 领导,与 Leona 和 Marcus 联手,准备对抗黑蛇和 Valerius 的同盟++这场战争的走向仍不确定。虚空水晶、先祖之门、以及隐藏在这一切背后的秘密,+都将在接下来的日子里浮出水面。银河的命运,可能就在这些关键人物和物品的碰撞中决定。+"""+++if __name__ == "__main__":+ # ============================================================================+ # Initialize the CogRAG system with LLM and Embeddings+ # ============================================================================+ print("=" * 80)+ print("Initializing Unified CogRAG System...")+ print("=" * 80)++ llm_client = ChatOpenAI(+ model="gpt-4o-mini",+ extra_body={+ "system": "You are a helpful assistant. Always respond in Chinese."+ },+ temperature=0,+ )+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")++ # Initialize the Unified System Wrapper+ rag = Cog_RAG(+ llm_client=llm_client,+ embedder=embedder,+ verbose=True,+ )++ # ============================================================================+ # Feed text to the system+ # ============================================================================+ print("\nFeeding story to the system...")+ rag.feed_text(STORY)+ print("✓ Story processed successfully\n")++ # ============================================================================+ # Display Extracted Entities+ # ============================================================================+ print("=" * 80)+ print("EXTRACTED ENTITIES (Both Layers)")+ print("=" * 80)++ # Note: Cog_RAG.nodes property aggregates nodes from both layers+ nodes = rag.nodes+ sorted_nodes = sorted(nodes, key=lambda x: x.name)++ for node in sorted_nodes:+ print(f"\n【{node.name}】(Type: {node.type})")+ print(f" Description: {node.description}")++ print(f"\n✓ Total unique entities extracted: {len(nodes)}\n")++ # ============================================================================+ # Display Extracted Hyperedges (Separate Layers)+ # ============================================================================+ print("=" * 80)+ print("EXTRACTED HYPEREDGES (Dual Layer)")+ print("=" * 80)++ print(f"\n--- Layer 1: Theme Hyperedges (Macro) ---")+ for i, edge in enumerate(rag.theme_layer.edges, 1):+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")+ print(f" Theme: {edge.description}")++ print(f"\n--- Layer 2: Relation Hyperedges (Micro) ---")+ for i, edge in enumerate(rag.detail_layer.edges, 1):+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")+ print(f" Description: {edge.description}")+ keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"+ print(f" Keywords: {keywords_str}")+ print(f" Strength: {edge.strength}/10")++ print(+ f"\n✓ Total edges extracted: {len(rag.edges)} ({len(rag.theme_layer.edges)} themes, {len(rag.detail_layer.edges)} relations)\n"+ )++ # ============================================================================+ # Statistics+ # ============================================================================+ print("=" * 80)+ print("STATISTICS")+ print("=" * 80)++ # Entity type distribution+ type_counts = {}+ for node in rag.nodes:+ type_counts[node.type] = type_counts.get(node.type, 0) + 1++ print("\nEntity Type Distribution:")+ for entity_type, count in sorted(type_counts.items()):+ print(f" {entity_type}: {count}")++ # Centrality (number of relationships/themes per entity)+ print("\nEntity Participation (Centrality):")+ centrality = {}+ for edge in rag.edges:+ for participant in edge.participants:+ centrality[participant] = centrality.get(participant, 0) + 1++ for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[+ :10+ ]:+ print(f" {entity}: {count}")++ # ============================================================================+ # Build Semantic Index for Q&A+ # ============================================================================+ print("\n" + "=" * 80)+ print("Building Semantic Index...")+ print("=" * 80)++ rag.build_index()+ print("✓ Semantic index built successfully\n")++ # ============================================================================+ # Q&A: Semantic Search over Hypergraph+ # ============================================================================+ print("=" * 80)+ print("Q&A: DUAL-LAYER SEMANTIC SEARCH")+ print("=" * 80)++ queries = [+ "虚空水晶现在在哪里?",+ "黑蛇和议员Valerius有什么阴谋?",+ "谁参与了泰坦空间站的秘密会议?",+ "Leona目前的状况如何?",+ ]++ for i, query in enumerate(queries, 1):+ print(f"\n【问题 {i}】{query}")+ print("-" * 60)++ try:+ # Cog_RAG returns a dict with 'themes' and 'entities'+ results = rag.search(query, top_k_themes=3, top_k_entities=3)++ print(" [Layer 1: Macro Themes]")+ if results['themes']:+ for j, theme in enumerate(results['themes'], 1):+ print(f" {j}. {theme.description}")+ else:+ print(" (No relevant themes found)")++ print("\n [Layer 2: Micro Entities]")+ if results['entities']:+ for j, entity in enumerate(results['entities'], 1):+ print(f" {j}. [{entity.name}] {entity.description[:100]}...")+ else:+ print(" (No relevant entities found)")++ except Exception as e:+ print(f" Search error: {str(e)}")++ # ============================================================================+ # Chat with CogRAG+ # ============================================================================+ print("\n" + "=" * 80)+ print("💬 Interactive Chat with CogRAG")+ print("=" * 80)+ print("Engaging in multi-turn dialogue based on extracted knowledge...\n")++ chat_queries = [+ "虚空水晶现在在哪里?",+ "黑蛇和议员Valerius有什么阴谋?",+ "谁参与了泰坦空间站的秘密会议?",+ "Leona目前的状况如何?",+ ]++ for q in chat_queries:+ print(f"❓ User: {q}")+ try:+ response = rag.chat(q)+ print(f"🤖 CogRAG: {response.content}\n")+ except Exception as e:+ print(f"⚠️ Chat error: {e}\n")++ # ============================================================================+ # Save Results+ # ============================================================================+ print("\n" + "=" * 80)+ print("Saving Results...")+ print("=" * 80)++ try:+ output_dir = "temp/cog_rag_demo"+ rag.dump(output_dir)+ print(f"✓ Results saved to {output_dir}/")++ # List saved files+ saved_files = []+ if os.path.exists(output_dir):+ for root, dirs, files in os.walk(output_dir):+ for file in files:+ rel_path = os.path.relpath(os.path.join(root, file), output_dir)+ saved_files.append(rel_path)++ if saved_files:+ print("\n Saved files:")+ for f in saved_files:+ print(f" - {f}")+ except Exception as e:+ print(f"⚠ Warning: Could not save results: {str(e)}")++ print("\n" + "=" * 80 + "\n")+diff --git a/examples/hypergraph_rag_demo.py b/examples/hypergraph_rag_demo.pynew file mode 100644index 0000000..10cecd3--- /dev/null+++ b/examples/hypergraph_rag_demo.py@@ -0,0 +1,287 @@+"""+HyperGraph_RAG Demo: Edge-First Extraction Strategy++This demo demonstrates the HyperGraph_RAG System:+1. **Edge-First Strategy**: Extracts Themes/Events (Hyperedges) first, then contextually extracts participating entities.+2. **Narrative Understanding**: Focuses on identifying high-level themes before drilling down to actors.+"""++import os+import sys++# 添加项目根目录到路径+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from dotenv import load_dotenv+from langchain_openai import ChatOpenAI, OpenAIEmbeddings++from hyperextract.hypergraphs import HyperGraph_RAG++load_dotenv()++# ============================================================================+# Sample Story for Extraction (Shared with AutoHypergraph Demo)+# ============================================================================++STORY = """+【银河纪元 3042年 - 赤红星云事件报告】++危机在这一周达到了顶峰。整个银河系似乎都陷入了暗流涌动的政治漩涡。++【第一阶段:秘密接触】+臭名昭著的星际海盗黑蛇驾驶着他的旗舰复仇号,在泰坦空间站秘密会见了银河联邦的背叛者议员 Valerius。+目击者称,两人在午夜时分登上了空间站的顶层禁区。据情报部门推测,这是一次肮脏的交易——+双方达成了针对皇室的同盟关系。参与此会议的还有 Valerius 手下的特务头目 Shadow,+他负责处理所有不可告人的细节。这个同盟的形成标志着大麻烦即将到来。++【第二阶段:叛变与夺权】+在这次会议的两天后,事态急剧恶化。黑蛇这种疯狂的家伙竟然在极光星区伏击了皇家护卫队。+这场突然的冲突中,皇家护卫队的队长 Leona 与副队长 Marcus 奋力抵抗,+但他们陷入了黑蛇手下的精锐战士的包围。这场激战导致队长 Leona 身受重伤。+而黑蛇的目标不是为了杀戮,而是为了夺取一件神圣的物品——传说中的虚空水晶。+这枚水晶散发着幽蓝的光芒,据说能够打开先祖之门的第一道密封。+黑蛇在战斗中成功夺取了水晶,但 Leona 在临终前用最后的力量将其重新夺回。++【第三阶段:流亡与隐藏】+Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9。+在这个被遗忘的角落,她向仅有的几个信任者求救:还有副队长 Marcus 以及隐秘组织"地下抵抗军"的领导人 Cipher。+同时,议员 Valerius 非法持有了开启先祖之门的钥匙——这把钥匙本应被安全保管在皇家金库中。+他现在藏匿在浮空都市中,与黑蛇遥相呼应,计划着下一步的政变。++【第四阶段:势力重组】+银河联邦的最高委员会得知了这一系列事件,决定采取行动。+高级顾问 Artemis 被任命为特使,她与 Leona 在废土行星 Z-9 进行了秘密会晤,+讨论如何制止这场即将到来的危机。同时,地下抵抗军的 Cipher 也加入了这次会议,+三人共同制定了一个大胆的计划。++【第五阶段:最后的筹码】+虽然黑蛇未能获得虚空水晶,但他已经从 Valerius 那里获得了先祖之门钥匙的复制品。+根据情报,这个复制品目前被秘密运往到黑蛇的秘密基地——位于辐射荒漠中的要塞。+参与此次运送的还有 Shadow 和黑蛇手下的精锐战士群体。+与此同时,Leona 和 Marcus 正在准备一场反击,他们计划在虚空水晶的力量下重新夺回局势的控制权。++目前局势:+- 黑蛇:手持复制品钥匙,掌控旗舰复仇号,计划进行最终的攻击+- 议员 Valerius:躲在浮空都市中,与黑蛇保持联系+- Leona:守护虚空水晶,在废土行星 Z-9 养伤,准备反击+- 银河联邦:通过特使 Artemis 支持 Leona,正在调集力量+- 地下抵抗军:Cipher 领导,与 Leona 和 Marcus 联手,准备对抗黑蛇和 Valerius 的同盟++这场战争的走向仍不确定。虚空水晶、先祖之门、以及隐藏在这一切背后的秘密,+都将在接下来的日子里浮出水面。银河的命运,可能就在这些关键人物和物品的碰撞中决定。+"""+++if __name__ == "__main__":+ # ============================================================================+ # Initialize the HyperGraph_RAG system+ # ============================================================================+ print("=" * 80)+ print("Initializing HyperGraph_RAG System (Edge-First)...")+ print("=" * 80)++ llm_client = ChatOpenAI(+ model="gpt-4o-mini",+ extra_body={+ "system": "You are a helpful assistant. Always respond in Chinese."+ },+ temperature=0,+ )+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")++ # Initialize HyperGraph_RAG+ # Note: enable_llm_cache=False for demo purposes to force new extractions if needed+ rag = HyperGraph_RAG(+ llm_client=llm_client,+ embedder=embedder,+ verbose=True,+ chunk_size=1000,+ )++ # ============================================================================+ # Feed text to the system+ # ============================================================================+ print("\nFeeding story to the system...")+ rag.feed_text(STORY)+ print("✓ Story processed successfully\n")++ # ============================================================================+ # Display Extracted Hyperedges+ # ============================================================================+ print("=" * 80)+ print("EXTRACTED HYPEREDGES")+ print("=" * 80)++ # 分离低阶边(2个参与者)和高阶边(3个以上参与者)+ low_order_edges = [e for e in rag.edges if len(e.participants) == 2]+ high_order_edges = [e for e in rag.edges if len(e.participants) > 2]++ print(f"\n--- Low-Order Edges (Pairwise) ---")+ for i, edge in enumerate(low_order_edges, 1):+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")+ print(f" Description: {edge.description}")+ keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"+ print(f" Keywords: {keywords_str}")+ print(f" Strength: {edge.strength}/10")++ print(f"\n--- High-Order Edges (Hyperedges) ---")+ for i, edge in enumerate(high_order_edges, 1):+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")+ print(f" Description: {edge.description}")+ keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"+ print(f" Keywords: {keywords_str}")+ print(f" Strength: {edge.strength}/10")++ print(+ f"\n✓ Total edges extracted: {len(rag.edges)} ({len(low_order_edges)} low-order, {len(high_order_edges)} high-order)\n"+ )++ # ============================================================================+ # Display Extracted Entities+ # ============================================================================+ print("=" * 80)+ print("EXTRACTED ENTITIES")+ print("=" * 80)++ for node in rag.nodes:+ print(f"\n【{node.name}】(Type: {node.type})")+ print(f" Description: {node.description}")++ print(f"\n✓ Total entities extracted: {len(rag.nodes)}\n")++ # ============================================================================+ # Statistics+ # ============================================================================+ print("=" * 80)+ print("STATISTICS")+ print("=" * 80)++ # Entity type distribution+ type_counts = {}+ for node in rag.nodes:+ type_counts[node.type] = type_counts.get(node.type, 0) + 1++ print("\nEntity Type Distribution:")+ for entity_type, count in sorted(type_counts.items()):+ print(f" {entity_type}: {count}")++ # Average edge strength+ if rag.edges:+ avg_strength = sum(e.strength for e in rag.edges) / len(rag.edges)+ print(f"\nAverage Edge Strength: {avg_strength:.2f}/10")++ # Centrality (number of relationships per entity)+ print("\nEntity Centrality (number of relationships):")+ centrality = {}+ for edge in rag.edges:+ for participant in edge.participants:+ centrality[participant] = centrality.get(participant, 0) + 1++ for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[+ :10+ ]:+ print(f" {entity}: {count}")++ # ============================================================================+ # Build Semantic Index for Q&A+ # ============================================================================+ print("\n" + "=" * 80)+ print("Building Semantic Index...")+ print("=" * 80)++ rag.build_index()+ print("✓ Semantic index built successfully\n")++ # ============================================================================+ # Q&A: Semantic Search over Hypergraph+ # ============================================================================+ print("=" * 80)+ print("Q&A: SEMANTIC SEARCH")+ print("=" * 80)++ queries = [+ "虚空水晶现在在哪里?",+ "黑蛇和议员Valerius有什么阴谋?",+ "谁参与了泰坦空间站的秘密会议?",+ "Leona目前的状况如何?",+ ]++ for i, query in enumerate(queries, 1):+ print(f"\n【问题 {i}】{query}")+ print("-" * 60)++ try:+ results = rag.search_edges(query, top_k=3)++ if results:+ for j, result in enumerate(results, 1):+ # EdgeAutoHypergraph (and base AutoHypergraph) default search returns (doc, score) tuple+ # The 'doc' is the EdgeSchema object+ edge = result[0] if isinstance(result, tuple) else result+ score = result[1] if isinstance(result, tuple) else 1.0++ # Note: EdgeSchema might not have strength if extracted via generic AutoHypergraph, but HyperGraph_RAG defines it.+ # We handle potential missing attrs safely+ strength = getattr(edge, "strength", "N/A")++ print(f"\n {j}. [Score: {score:.3f}]")+ print(f" Participants: {' <-> '.join(edge.participants)}")+ print(f" Description: {edge.description}")+ print(f" Strength: {strength}/10")+ else:+ print(" (No relevant relationships found)")+ except Exception as e:+ print(f" Search error: {str(e)}")++ # ============================================================================+ # Chat with HyperGraph_RAG+ # ============================================================================+ print("\n" + "=" * 80)+ print("💬 Interactive Chat with HyperGraph_RAG")+ print("=" * 80)+ print("Engaging in multi-turn dialogue based on extracted knowledge...\n")++ chat_queries = [+ "虚空水晶现在在哪里?",+ "黑蛇和议员Valerius有什么阴谋?",+ "谁参与了泰坦空间站的秘密会议?",+ "Leona目前的状况如何?",+ ]++ for q in chat_queries:+ print(f"❓ User: {q}")+ try:+ response = rag.chat(q)+ print(f"🤖 HyperGraph_RAG: {response.content}\n")+ except Exception as e:+ print(f"⚠️ Chat error: {e}\n")++ # ============================================================================+ # Save Results+ # ============================================================================+ print("\n" + "=" * 80)+ print("Saving Results...")+ print("=" * 80)++ try:+ output_dir = "temp/hypergraph_rag_demo"+ rag.dump(output_dir)+ print(f"✓ Results saved to {output_dir}/")++ # List saved files+ saved_files = []+ if os.path.exists(output_dir):+ for root, dirs, files in os.walk(output_dir):+ for file in files:+ rel_path = os.path.relpath(os.path.join(root, file), output_dir)+ saved_files.append(rel_path)++ if saved_files:+ print("\n Saved files:")+ for f in saved_files:+ print(f" - {f}")+ except Exception as e:+ print(f"⚠ Warning: Could not save results: {str(e)}")++ print("\n" + "=" * 80 + "\n")+diff --git a/hyperextract/hypergraphs/__init__.py b/hyperextract/hypergraphs/__init__.pyindex 29ee874..559ed23 100644--- a/hyperextract/hypergraphs/__init__.py+++ b/hyperextract/hypergraphs/__init__.py@@ -1,7 +1,13 @@from .base import AutoHypergraphfrom .hyper_rag import Hyper_RAG+from .hypergraph_rag import HyperGraph_RAG+from .cog_rag import Cog_RAG, Cog_RAG_ThemeLayer, Cog_RAG_DetailLayer__all__ = ["AutoHypergraph","Hyper_RAG",+ "HyperGraph_RAG",+ "Cog_RAG",+ "Cog_RAG_ThemeLayer",+ "Cog_RAG_DetailLayer",]diff --git a/hyperextract/hypergraphs/base.py b/hyperextract/hypergraphs/base.pyindex 67bde4c..a9d4721 100644--- a/hyperextract/hypergraphs/base.py+++ b/hyperextract/hypergraphs/base.py@@ -66,6 +66,15 @@ CRITICAL RULES:"""+DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal-+You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.+Your task is to Populate these concepts with their specific Participant Entities.++CRITICAL RULES:+1. Extract ALL key participants for each concept.+2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).+"""+class AutoHypergraphSchema(BaseModel, Generic[Node, Edge]):"""Generic schema container for hypergraph data."""@@ -85,6 +94,13 @@ class NodeListSchema(BaseModel, Generic[Node]):)+class EdgeDescriptionList(BaseModel):+ """Intermediate schema for identifying edge concepts first."""+ items: List[str] = Field(description="List of primary edge concepts/themes identified")++++class EdgeListSchema(BaseModel, Generic[Edge]):"""Intermediate schema for batch hyperedge extraction."""@@ -97,7 +113,11 @@ class EdgeListSchema(BaseModel, Generic[Edge]):class AutoHypergraph(BaseAutoType[AutoHypergraphSchema[Node, Edge]], Generic[Node, Edge]):- """AutoHypergraph - extracts complex relationships involving multiple entities.+ """AutoHypergraph - Node-First extraction: Extract entities first, then relationships.++ Extraction Strategy: Two-Stage (Node-First)+ 1. Stage 1: Extract all Nodes (entities) from text.+ 2. Stage 2: Extract Edges (relationships) using identified Nodes as context.Suitable for:- Events with multiple participants (e.g., meetings, transactions)@@ -762,3 +782,193 @@ class AutoHypergraph(self._edge_memory.load_index(str(edge_index_path))except Exception as e:logger.warning(f"Failed to load edge index: {e}")++class EdgeAutoHypergraph(AutoHypergraph[Node, Edge], Generic[Node, Edge]):+ """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.++ Extraction Strategy: Two-Stage (Edge-First)+ 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).+ 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.++ Suitable for:+ - Narrative-driven knowledge graphs (Theme -> Character)+ - Event-driven extraction (Event -> Participant)+ """++ def __init__(+ self,+ node_schema: Type[Node],+ edge_schema: Type[Edge],+ node_key_extractor: Callable[[Node], str],+ edge_key_extractor: Callable[[Edge], str],+ nodes_in_edge_extractor: Callable[[Edge], Tuple[str, ...]],+ llm_client: BaseChatModel,+ embedder: Embeddings,+ *,+ prompt_for_node_extraction: str = "",+ **kwargs: Any,+ ):+ """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.++ Args:+ prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".+ (Extract Nodes given Edges).+ """+ super().__init__(+ node_schema=node_schema,+ edge_schema=edge_schema,+ node_key_extractor=node_key_extractor,+ edge_key_extractor=edge_key_extractor,+ nodes_in_edge_extractor=nodes_in_edge_extractor,+ llm_client=llm_client,+ embedder=embedder,+ prompt_for_node_extraction=prompt_for_node_extraction,+ **kwargs,+ )++ # Build Stage 2 Schema (Edges containing nested Nodes)+ # We assume the edge schema has a 'participants' field (List[str]).+ # We create a new model where 'participants' is List[NodeSchema].++ # 1. Base fields from EdgeSchema (excluding participants)+ valid_fields = {+ k: (v.annotation, v)+ for k, v in edge_schema.model_fields.items()+ if k != "participants"+ }++ # 2. Add Nested Participants field+ valid_fields["participants"] = (+ List[node_schema],+ Field(description="List of entities participating in this concept/edge.")+ )++ # 3. Create Contextual Edge Model+ self.contextual_edge_schema = create_model(+ f"Contextual{edge_schema.__name__}",+ **valid_fields+ )++ # 4. Create Result List Model+ self.stage2_result_schema = create_model(+ f"{edge_schema.__name__}FirstResult",+ items=(List[self.contextual_edge_schema], Field(default_factory=list))+ )++ # Set default prompt if not provided+ if not prompt_for_node_extraction:+ self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT++ def _extract_data(self, text: str) -> AutoHypergraphSchema:+ """Main extraction logic dispatcher (Edge-First)."""+ if self.extraction_mode == "two_stage":+ raw_graph = self._extract_data_by_two_stage(text)+ elif self.extraction_mode == "one_stage":+ raise NotImplementedError(+ "Single-stage extraction not yet supported for EdgeAutoHypergraph."+ )+ else:+ raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")++ # Note: Dangle pruning might be less critical here since nodes are derived from edges,+ # but we keep it for consistency if nodes are merged globally.+ return self._prune_dangling_edges(raw_graph)++ def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:+ """Extract edges/themes first, then entities within those contexts (batch processing).++ Process:+ 1. Split text into chunks.+ 2. Batch extract Edge Concepts (Theme Strings) for all chunks.+ 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).+ 4. Merge all partial results into one global hypergraph.+ """+ # 1. Prepare chunks+ if len(text) <= self.chunk_size:+ chunks = [text]+ else:+ chunks = self.text_splitter.split_text(text)++ if self.verbose:+ logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")++ # 2. Stage 1: Identify Edge Concepts (Strings)+ edge_concepts_lists = self._extract_edge_concepts_batch(chunks)++ # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)+ chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)++ # 4. Global Merge+ # chunk_results is List[AutoHypergraphSchema]+ return self.merge_batch_data(chunk_results)++ def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:+ """Stage 1: Batch extract edge concept strings."""+ # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts+ # We assume the prompt is geared towards finding a LIST of themes/concepts.++ prompt_template = ChatPromptTemplate.from_template(+ f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"+ )+ chain = prompt_template | self.llm_client.with_structured_output(EdgeDescriptionList)++ inputs = [{"chunk_text": chunk} for chunk in chunks]+ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})++ return [res.items if res else [] for res in results]++ def _extract_details_batch(self, chunks: List[str], edge_concepts_list: List[List[str]]) -> List[AutoHypergraphSchema]:+ """Generic Stage 2: Extract entities and details given identified edge concepts."""++ # 1. Prepare Inputs+ inputs = []+ valid_map = {}+ for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):+ if concepts:+ concepts_str = "- " + "\n- ".join(concepts)+ inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})+ valid_map[len(inputs) - 1] = i++ if not inputs:+ return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]++ # 2. Chain Execution+ # Uses self.node_prompt as the instruction for contextual node extraction+ prompt = ChatPromptTemplate.from_template(+ f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"+ )+ chain = prompt | self.llm_client.with_structured_output(self.stage2_result_schema)++ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})++ # 3. Transform Nested Result -> Flat AutoHypergraphSchema+ final_schemas = [self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))]++ for i, result in enumerate(results):+ if not result or not result.items:+ continue++ original_idx = valid_map[i]+ current_nodes = []+ current_edges = []++ for item in result.items:+ # item is a ContextualEdge (has nested participants=List[Node])++ # Extract Nodes+ nested_nodes = item.participants+ node_names = []+ for n in nested_nodes:+ current_nodes.append(n)+ node_names.append(self.node_key_extractor(n))++ # Reconstruct Flat Edge+ edge_data = item.model_dump()+ edge_data["participants"] = sorted(node_names) # Ensure sorted for consistency++ new_edge = self.edge_schema(**edge_data)+ current_edges.append(new_edge)++ final_schemas[original_idx] = self.graph_schema(nodes=current_nodes, edges=current_edges)++ return final_schemasdiff --git a/hyperextract/hypergraphs/cog_rag.py b/hyperextract/hypergraphs/cog_rag.pyindex e69de29..f363ecc 100644--- a/hyperextract/hypergraphs/cog_rag.py+++ b/hyperextract/hypergraphs/cog_rag.py@@ -0,0 +1,425 @@+"""+COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System Pattern+Extracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities.+"""++from typing import List+from pydantic import BaseModel, Field+from hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchema, EdgeAutoHypergraph+from hyperextract.utils.logging import logger+from langchain_core.prompts import ChatPromptTemplate+from langchain_core.embeddings import Embeddings+from langchain_core.language_models import BaseChatModel+from ontomem.merger import CustomRuleMerger++# ============================================================================+# Node Schema+# ============================================================================+++class NodeSchema(BaseModel):+ """Represents an entity extracted from the source text."""++ name: str = Field(description="Name of the entity")+ type: str = Field(+ description="Entity type (person, organization, geo, event, role, concept, etc.)",+ )+ description: str = Field(+ description="Comprehensive description of the entity's attributes and activities",+ )+++# ============================================================================+# Edge Schema (Theme)+# ============================================================================+++class ThemeSchema(BaseModel):+ """Represents a Theme or Narrative Arc connecting multiple entities (Hyperedge)."""++ participants: List[str] = Field(+ description="Names of key entities involved in this theme"+ )+ description: str = Field(+ description="A sentence that describes the primary theme, reflecting the main conflict, resolution, or key message"+ )+++# ============================================================================+# Edge Schema (Entity Relation)+# ============================================================================+++class EdgeSchema(BaseModel):+ """Represents a relationship or event connecting multiple entities (low-order or high-order)."""++ participants: List[str] = Field(+ description="Names of entities involved in this relationship"+ )+ description: str = Field(+ description="Detailed explanation of the relationship or event"+ )+ keywords: List[str] = Field(+ description="List of keywords summarizing the relationship themes (e.g., ['conflict', 'trade', 'alliance'])",+ )+ strength: int = Field(+ ge=1,+ le=10,+ description="Numerical score indicating relationship strength (1-10)",+ )++# ============================================================================+# Extraction Prompts+# ============================================================================++COG_RAG_THEME_PROMPT = """+-Goal-+Analyze the text to identify the primary themes (narrative arcs).+A Theme is a high-level concept, conflict, or main idea acting as a Hyperedge.+"""++COG_RAG_KEY_ENTITY_PROMPT = """+-Goal-+You are given a list of "Primary Themes" identified from the text.+Your task is to populate these themes with key entities (Participants).+"""++COG_RAG_NODE_EXTRACTION_PROMPT = """+-Goal-+Identify relevant entities from the text.+Entities will serve as participants in complex events later.+### Source Text:+"""++COG_RAG_EDGE_EXTRACTION_PROMPT = """+-Goal-+You are an expert hypergraph knowledge extraction assistant.+Extract "Hyperedges" that represent relationships, events, or thematic groupings involving MULTIPLE entities (2 or more) simultaneously.++-Definition-+A Hyperedge is a unified concept for any connection.+It treats all involved entities as **participants** in a shared context+(e.g., a conversation, a joint mission, a conflict, or a shared concept).++-Constraints-+1. You MUST ONLY use names from the "Allowed Entities" list provided below.+2. Do NOT create edges for entities that represent strictly different concepts with no interaction.+3. Ensure every edge has at least 2 participants.++"""++# ============================================================================+# Merge Templates for LLM.CUSTOM_RULE Strategy+# ============================================================================++COG_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.+You will receive a list of objects representing the same Entity.++Your task is to merge them into a SINGLE object exactly matching the schema.++Merge strategy:+1. **name/type**: Keep the most frequent or precise value.+2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.+"""++COG_RAG_THEME_MERGE_RULE = """You are an intelligent data merging assistant.+You will receive a list of objects representing the same Theme.++Your task is to merge them into a SINGLE object exactly matching the schema.++Merge strategy:+1. **participants**: Keep the list (they should be identical for the same theme key).+2. **description**: Synthesize a global theme description that encompasses the narrative details from all input themes. Write it in the third person.+"""++COG_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.+You will receive a list of objects representing the same Relationship.++Your task is to merge them into a SINGLE object exactly matching the schema.++Merge strategy:+1. **participants**: Keep the list (they should be identical for the same relationship key).+2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.+3. **keywords**: Combine keyword lists from all inputs, removing duplicates.+4. **strength**: Calculate the average of the input strengths (round to nearest integer).+"""++# ============================================================================+# Theme Hypergraph (Layer 1: Macro Narrative)+# ============================================================================+++class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):+ """+ Layer 1 of Cog_RAG: Theme-First Extraction.+ Extracts Themes (Hyperedges) and Key Entities (Nodes).+ """++ def __init__(+ self,+ llm_client: BaseChatModel,+ embedder: Embeddings,+ chunk_size: int = 2048,+ chunk_overlap: int = 256,+ max_workers: int = 10,+ verbose: bool = True,+ ):+ """+ Initialize COG_RAG engine.++ Args:+ llm_client (BaseChatModel): LLM client for text generation.+ embedder (Embeddings): Embedding model for text embeddings.+ chunk_size (int): Size of text chunks for indexing.+ chunk_overlap (int): Overlap between text chunks for indexing.+ max_workers (int): Maximum number of workers for indexing.+ verbose (bool): Display detailed execution logs and progress information.+ """+ # 1. Define Key Extractors+ # Node key: exact match by name+ node_key_fn = lambda x: x.name++ # Edge key: sorted tuple of participants+ edge_key_fn = lambda x: tuple(sorted(x.participants))++ # 2. Edge consistency check: tell AutoHypergraph which nodes this edge connects+ nodes_in_edge_fn = lambda x: x.participants++ # 3. Create custom Mergers with LLM.CUSTOM_RULE strategy+ node_merger = CustomRuleMerger(+ key_extractor=node_key_fn,+ llm_client=llm_client,+ item_schema=NodeSchema,+ rule=COG_RAG_NODE_MERGE_RULE,+ )++ edge_merger = CustomRuleMerger(+ key_extractor=edge_key_fn,+ llm_client=llm_client,+ item_schema=ThemeSchema,+ rule=COG_RAG_THEME_MERGE_RULE,+ )++ # 4. Call parent class initialization+ super().__init__(+ node_schema=NodeSchema,+ edge_schema=ThemeSchema,+ node_key_extractor=node_key_fn,+ edge_key_extractor=edge_key_fn,+ nodes_in_edge_extractor=nodes_in_edge_fn,+ llm_client=llm_client,+ embedder=embedder,+ # Enforce two-stage extraction+ extraction_mode="two_stage",+ # Inject customized prompts for extraction+ # prompt_for_node_extraction acts as the Stage 2 (Edge -> Node) prompt+ prompt_for_node_extraction=COG_RAG_KEY_ENTITY_PROMPT,+ prompt_for_edge_extraction=COG_RAG_THEME_PROMPT,+ # Pass custom merger instances+ node_strategy_or_merger=node_merger,+ edge_strategy_or_merger=edge_merger,+ # Optimize indexing+ node_fields_for_index=["name", "type", "description"],+ edge_fields_for_index=["description"],+ # Other parameters+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ verbose=verbose,+ )++# ============================================================================+# Detail Hypergraph (Layer 2: Micro Relations)+# ============================================================================+++class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):+ """+ Layer 2 of Cog_RAG: Entity-First Extraction.+ Extracts Entities (Nodes) and Semantic Relationships (Hyperedges).+ """++ def __init__(+ self,+ llm_client: BaseChatModel,+ embedder: Embeddings,+ chunk_size: int = 2048,+ chunk_overlap: int = 256,+ max_workers: int = 10,+ verbose: bool = True,+ ):+ """+ Initialize COG_RAG Entity-Relation engine.+ """+ node_key_fn = lambda x: x.name+ edge_key_fn = lambda x: tuple(sorted(x.participants))+ nodes_in_edge_fn = lambda x: x.participants++ node_merger = CustomRuleMerger(+ key_extractor=node_key_fn,+ llm_client=llm_client,+ item_schema=NodeSchema,+ rule=COG_RAG_NODE_MERGE_RULE,+ )++ edge_merger = CustomRuleMerger(+ key_extractor=edge_key_fn,+ llm_client=llm_client,+ item_schema=EdgeSchema,+ rule=COG_RAG_EDGE_MERGE_RULE,+ )++ super().__init__(+ node_schema=NodeSchema,+ edge_schema=EdgeSchema,+ node_key_extractor=node_key_fn,+ edge_key_extractor=edge_key_fn,+ nodes_in_edge_extractor=nodes_in_edge_fn,+ llm_client=llm_client,+ embedder=embedder,+ extraction_mode="two_stage",+ prompt_for_node_extraction=COG_RAG_NODE_EXTRACTION_PROMPT,+ prompt_for_edge_extraction=COG_RAG_EDGE_EXTRACTION_PROMPT,+ node_strategy_or_merger=node_merger,+ edge_strategy_or_merger=edge_merger,+ node_fields_for_index=["name", "type", "description"],+ edge_fields_for_index=["description", "keywords"],+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ verbose=verbose,+ )+++# ============================================================================+# Main System Class (Wrapper)+# ============================================================================++class Cog_RAG:+ """+ Cognitive-Inspired Dual-Hypergraph RAG System.++ Combines two Hypergraph Layers:+ 1. Theme Layer (Cog_RAG_ThemeLayer): Captures macro narratives and themes.+ 2. Detail Layer (Cog_RAG_DetailLayer): Captures micro entity relationships.+ """++ def __init__(+ self,+ llm_client: BaseChatModel,+ embedder: Embeddings,+ chunk_size: int = 2048,+ chunk_overlap: int = 256,+ max_workers: int = 10,+ verbose: bool = True,+ ):+ self.theme_layer = Cog_RAG_ThemeLayer(+ llm_client=llm_client,+ embedder=embedder,+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ verbose=verbose+ )+ self.detail_layer = Cog_RAG_DetailLayer(+ llm_client=llm_client,+ embedder=embedder,+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ verbose=verbose+ )+ self.llm = llm_client+ self.verbose = verbose++ def feed_text(self, text: str):+ """Feed text to both hypergraph layers."""+ if self.verbose:+ logger.info("Cog_RAG: Feeding text to Theme Layer...")+ self.theme_layer.feed_text(text)++ if self.verbose:+ logger.info("Cog_RAG: Feeding text to Detail Layer...")+ self.detail_layer.feed_text(text)++ def build_index(self):+ """Build indices for both layers."""+ if self.verbose:+ logger.info("Cog_RAG: Building indices...")+ self.theme_layer.build_index()+ self.detail_layer.build_index()++ def search(self, query: str, top_k_themes: int = 3, top_k_entities: int = 3):+ """+ Dual-Layer Search.+ Strategy:+ 1. Macro: Search for Themes in ThemeLayer (Edge-First).+ 2. Micro: Search for Entities in DetailLayer (Node-First).+ """+ # 1. Macro Context: Themes+ themes = self.theme_layer.search_edges(query, top_k=top_k_themes)++ # 2. Micro Context: Entities (Nodes)+ # Note: The user requested searching for "Specific Entities" in the second step.+ entities = self.detail_layer.search_nodes(query, top_k=top_k_entities)++ return {+ "themes": themes,+ "entities": entities+ }++ def chat(self, query: str, top_k_themes: int = 3, top_k_entities: int = 3):+ """Generate an answer using context from both layers."""+ results = self.search(query, top_k_themes, top_k_entities)++ context_parts = []+ if results["themes"]:+ context_parts.append("=== MACRO THEMES (Narrative Context) ===")+ for t in results["themes"]:+ context_parts.append(f"- {t.description} (Involved: {', '.join(t.participants)})")++ if results["entities"]:+ context_parts.append("\n=== MICRO ENTITIES (Specific Details) ===")+ for e in results["entities"]:+ context_parts.append(f"- [{e.type}] {e.name}: {e.description}")++ if not context_parts:+ context = "No relevant information found."+ else:+ context = "\n".join(context_parts)++ prompt = ChatPromptTemplate.from_template(+ "You are an intelligent assistant using a Dual-Layer Hypergraph Knowledge Base.\n"+ "Use the provided Themes (Macro) and Key Entities (Micro) to answer the user's question comprehensively.\n\n"+ "{context}\n\n"+ "Question: {question}\n\n"+ "Answer:"+ )++ chain = prompt | self.llm+ return chain.invoke({"context": context, "question": query})++ def dump(self, folder_path: str):+ """Save both extracted graphs."""+ import os+ self.theme_layer.dump(os.path.join(folder_path, "theme_layer"))+ self.detail_layer.dump(os.path.join(folder_path, "detail_layer"))++ @property+ def nodes(self):+ """Return combined unique nodes from both layers (simple aggregation)."""+ # Note: This is a property for valid introspection, but nodes are managed separately.+ # We perform a simple name-based deduction for display.+ seen = set()+ unique = []+ for n in self.theme_layer.nodes + self.detail_layer.nodes:+ if n.name not in seen:+ seen.add(n.name)+ unique.append(n)+ return unique++ @property+ def edges(self):+ """Return all edges (Themes + Relations)."""+ return self.theme_layer.edges + self.detail_layer.edges+diff --git a/hyperextract/hypergraphs/hypergraph_rag.py b/hyperextract/hypergraphs/hypergraph_rag.pyindex e69de29..7903039 100644--- a/hyperextract/hypergraphs/hypergraph_rag.py+++ b/hyperextract/hypergraphs/hypergraph_rag.py@@ -0,0 +1,185 @@+"""+HyperGraph_RAG: Integration of EdgeAutoHypergraph (Edge-First Extraction).+Matches the structure of Hyper_RAG regarding Schemas and Mergers.+"""++from typing import List+from pydantic import BaseModel, Field+from hyperextract.hypergraphs.base import EdgeAutoHypergraph+from langchain_core.embeddings import Embeddings+from langchain_core.language_models import BaseChatModel+from ontomem.merger import CustomRuleMerger++# ============================================================================+# Node Schema+# ============================================================================+++class NodeSchema(BaseModel):+ """Represents an entity extracted from the source text."""++ name: str = Field(description="Name of the entity")+ type: str = Field(+ description="Entity type (person, organization, geo, event, role, concept, etc.)",+ )+ description: str = Field(+ description="Comprehensive description of the entity's attributes and activities",+ )+++# ============================================================================+# Edge Schema+# ============================================================================+++class EdgeSchema(BaseModel):+ """Represents a relationship or event connecting multiple entities (low-order or high-order)."""++ participants: List[str] = Field(+ description="Names of entities involved in this relationship"+ )+ description: str = Field(+ description="Detailed explanation of the relationship or event"+ )+ keywords: List[str] = Field(+ description="List of keywords summarizing the relationship themes (e.g., ['conflict', 'trade', 'alliance'])",+ )+ strength: int = Field(+ ge=1,+ le=10,+ description="Numerical score indicating relationship strength (1-10)",+ )+++# ============================================================================+# Extraction Prompts (Edge-First)+# ============================================================================++HyperGraph_RAG_THEME_EXTRACTION_PROMPT = """+-Goal-+Identify the key "Themes", "Narratives", or "Complex Events" present in the text.+A Theme/Event should represent a significant interaction or grouping involving multiple participants.++-Output-+Return a list of these Themes/Events as descriptive strings.+Do not list participants data formats yet, just the essence of the event (e.g., "The secret meeting at Titan Station", "The ambush in the Aurora Sector").+"""++HyperGraph_RAG_CONTEXT_NODE_PROMPT = """+-Goal-+You are an expert knowledge extractor.+You have been given a list of "Themes/Events" (Hyperedges) that occurred in the text.+Your job is to identify the specific ENTITIES (participants) involved in each theme.++-Output-+For each Theme, list the entities that participated.+"""+++# ============================================================================+# Merge Templates for LLM.CUSTOM_RULE Strategy+# ============================================================================++Hyper_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.+You will receive a list of objects representing the same Entity.++Your task is to merge them into a SINGLE object exactly matching the schema.++Merge strategy:+1. **name/type**: Keep the most frequent or precise value.+2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.+"""++Hyper_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.+You will receive a list of objects representing the same Hyperedge (Relationship/Event).++Your task is to merge them into a SINGLE object exactly matching the schema.++Merge strategy:+1. **participants**: Keep the list (they should be identical for the same hyperedge key).+2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.+3. **keywords**: Combine keyword lists from all inputs, removing duplicates.+4. **strength**: Calculate the average of the input strengths (round to nearest integer).+"""++# ============================================================================+# HyperGraph_RAG Class+# ============================================================================+++class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):+ """+ HyperGraph_RAG: Hypergraph-based RAG with Edge-First Extraction Strategy.++ Difference from Hyper_RAG:+ - Uses EdgeAutoHypergraph (extracts Themes first, then Entities).+ - Suitable for high-level narrative understanding where Themes map to specific Actors.+ """++ def __init__(+ self,+ llm_client: BaseChatModel,+ embedder: Embeddings,+ chunk_size: int = 2048,+ chunk_overlap: int = 256,+ max_workers: int = 10,+ verbose: bool = True,+ ):+ """+ Initialize HyperGraph_RAG engine.++ Args:+ llm_client (BaseChatModel): LLM client for text generation.+ embedder (Embeddings): Embedding model for text embeddings.+ chunk_size (int): Size of text chunks for extraction.+ chunk_overlap (int): Overlap between text chunks.+ max_workers (int): Maximum number of workers for extraction.+ verbose (bool): Display detailed logs.+ """+ # 1. Define Key Extractors+ node_key_fn = lambda x: x.name+ edge_key_fn = lambda x: tuple(sorted(x.participants))+ nodes_in_edge_fn = lambda x: x.participants++ # 2. Setup Mergers+ node_merger = CustomRuleMerger(+ key_extractor=node_key_fn,+ llm_client=llm_client,+ item_schema=NodeSchema,+ rule=Hyper_RAG_NODE_MERGE_RULE,+ )++ edge_merger = CustomRuleMerger(+ key_extractor=edge_key_fn,+ llm_client=llm_client,+ item_schema=EdgeSchema,+ rule=Hyper_RAG_EDGE_MERGE_RULE,+ )++ # 3. Initialize EdgeAutoHypergraph (Parent)+ super().__init__(+ node_schema=NodeSchema,+ edge_schema=EdgeSchema,+ node_key_extractor=node_key_fn,+ edge_key_extractor=edge_key_fn,+ nodes_in_edge_extractor=nodes_in_edge_fn,+ llm_client=llm_client,+ embedder=embedder,+ # Prompts+ # Stage 1: Extract Themes+ prompt_for_edge_extraction=HyperGraph_RAG_THEME_EXTRACTION_PROMPT,+ # Stage 2: Extract Particles given Themes+ prompt_for_node_extraction=HyperGraph_RAG_CONTEXT_NODE_PROMPT,+ # Merger strategies+ node_strategy_or_merger=node_merger,+ edge_strategy_or_merger=edge_merger,+ # Indexing optimization+ node_fields_for_index=["name", "type", "description"],+ edge_fields_for_index=["keywords", "description"],+ # BaseAutoType Params+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ verbose=verbose,+ )+