Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

add cog_rag and hypergraph_rag

haoohu39c73f4a29d6
diff --git a/examples/cog_rag_demo.py b/examples/cog_rag_demo.py
new file mode 100644
index 0000000..79952f8
--- /dev/null
+++ b/examples/cog_rag_demo.py
@@ -0,0 +1,282 @@
+"""
+Cog_RAG Demo: Theme-based Knowledge Extraction + Entity-Relation Detail Retrieval
+
+This demo demonstrates the unified Cog_RAG System Wrapper:
+1. **Layer 1 (Theme Layer)**: Extracts Key Entities and Themes (Narrative Arcs).
+2. **Layer 2 (Detail Layer)**: Extracts Entities and Semantic Relationships.
+3. **Process**:
+ - Feeds the same story to the Unified System.
+ - Performs a "Theme-First" retrieval process automatically.
+"""
+
+import os
+import sys
+
+# 添加项目根目录到路径
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+# Try to load dotenv, ignoring if not installed
+try:
+ from dotenv import load_dotenv
+ load_dotenv()
+except ImportError:
+ pass
+
+from hyperextract.hypergraphs import Cog_RAG
+
+# ============================================================================
+# Sample Story for Extraction (Shared with AutoHypergraph Demo)
+# ============================================================================
+
+STORY = """
+【银河纪元 3042年 - 赤红星云事件报告】
+
+危机在这一周达到了顶峰。整个银河系似乎都陷入了暗流涌动的政治漩涡。
+
+【第一阶段:秘密接触】
+臭名昭著的星际海盗黑蛇驾驶着他的旗舰复仇号,在泰坦空间站秘密会见了银河联邦的背叛者议员 Valerius。
+目击者称,两人在午夜时分登上了空间站的顶层禁区。据情报部门推测,这是一次肮脏的交易——
+双方达成了针对皇室的同盟关系。参与此会议的还有 Valerius 手下的特务头目 Shadow,
+他负责处理所有不可告人的细节。这个同盟的形成标志着大麻烦即将到来。
+
+【第二阶段:叛变与夺权】
+在这次会议的两天后,事态急剧恶化。黑蛇这种疯狂的家伙竟然在极光星区伏击了皇家护卫队。
+这场突然的冲突中,皇家护卫队的队长 Leona 与副队长 Marcus 奋力抵抗,
+但他们陷入了黑蛇手下的精锐战士的包围。这场激战导致队长 Leona 身受重伤。
+而黑蛇的目标不是为了杀戮,而是为了夺取一件神圣的物品——传说中的虚空水晶。
+这枚水晶散发着幽蓝的光芒,据说能够打开先祖之门的第一道密封。
+黑蛇在战斗中成功夺取了水晶,但 Leona 在临终前用最后的力量将其重新夺回。
+
+【第三阶段:流亡与隐藏】
+Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9。
+在这个被遗忘的角落,她向仅有的几个信任者求救:还有副队长 Marcus 以及隐秘组织"地下抵抗军"的领导人 Cipher。
+同时,议员 Valerius 非法持有了开启先祖之门的钥匙——这把钥匙本应被安全保管在皇家金库中。
+他现在藏匿在浮空都市中,与黑蛇遥相呼应,计划着下一步的政变。
+
+【第四阶段:势力重组】
+银河联邦的最高委员会得知了这一系列事件,决定采取行动。
+高级顾问 Artemis 被任命为特使,她与 Leona 在废土行星 Z-9 进行了秘密会晤,
+讨论如何制止这场即将到来的危机。同时,地下抵抗军的 Cipher 也加入了这次会议,
+三人共同制定了一个大胆的计划。
+
+【第五阶段:最后的筹码】
+虽然黑蛇未能获得虚空水晶,但他已经从 Valerius 那里获得了先祖之门钥匙的复制品。
+根据情报,这个复制品目前被秘密运往到黑蛇的秘密基地——位于辐射荒漠中的要塞。
+参与此次运送的还有 Shadow 和黑蛇手下的精锐战士群体。
+与此同时,Leona 和 Marcus 正在准备一场反击,他们计划在虚空水晶的力量下重新夺回局势的控制权。
+
+目前局势:
+- 黑蛇:手持复制品钥匙,掌控旗舰复仇号,计划进行最终的攻击
+- 议员 Valerius:躲在浮空都市中,与黑蛇保持联系
+- Leona:守护虚空水晶,在废土行星 Z-9 养伤,准备反击
+- 银河联邦:通过特使 Artemis 支持 Leona,正在调集力量
+- 地下抵抗军:Cipher 领导,与 Leona 和 Marcus 联手,准备对抗黑蛇和 Valerius 的同盟
+
+这场战争的走向仍不确定。虚空水晶、先祖之门、以及隐藏在这一切背后的秘密,
+都将在接下来的日子里浮出水面。银河的命运,可能就在这些关键人物和物品的碰撞中决定。
+"""
+
+
+if __name__ == "__main__":
+ # ============================================================================
+ # Initialize the CogRAG system with LLM and Embeddings
+ # ============================================================================
+ print("=" * 80)
+ print("Initializing Unified CogRAG System...")
+ print("=" * 80)
+
+ llm_client = ChatOpenAI(
+ model="gpt-4o-mini",
+ extra_body={
+ "system": "You are a helpful assistant. Always respond in Chinese."
+ },
+ temperature=0,
+ )
+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")
+
+ # Initialize the Unified System Wrapper
+ rag = Cog_RAG(
+ llm_client=llm_client,
+ embedder=embedder,
+ verbose=True,
+ )
+
+ # ============================================================================
+ # Feed text to the system
+ # ============================================================================
+ print("\nFeeding story to the system...")
+ rag.feed_text(STORY)
+ print("✓ Story processed successfully\n")
+
+ # ============================================================================
+ # Display Extracted Entities
+ # ============================================================================
+ print("=" * 80)
+ print("EXTRACTED ENTITIES (Both Layers)")
+ print("=" * 80)
+
+ # Note: Cog_RAG.nodes property aggregates nodes from both layers
+ nodes = rag.nodes
+ sorted_nodes = sorted(nodes, key=lambda x: x.name)
+
+ for node in sorted_nodes:
+ print(f"\n【{node.name}】(Type: {node.type})")
+ print(f" Description: {node.description}")
+
+ print(f"\n✓ Total unique entities extracted: {len(nodes)}\n")
+
+ # ============================================================================
+ # Display Extracted Hyperedges (Separate Layers)
+ # ============================================================================
+ print("=" * 80)
+ print("EXTRACTED HYPEREDGES (Dual Layer)")
+ print("=" * 80)
+
+ print(f"\n--- Layer 1: Theme Hyperedges (Macro) ---")
+ for i, edge in enumerate(rag.theme_layer.edges, 1):
+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
+ print(f" Theme: {edge.description}")
+
+ print(f"\n--- Layer 2: Relation Hyperedges (Micro) ---")
+ for i, edge in enumerate(rag.detail_layer.edges, 1):
+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
+ print(f" Description: {edge.description}")
+ keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"
+ print(f" Keywords: {keywords_str}")
+ print(f" Strength: {edge.strength}/10")
+
+ print(
+ f"\n✓ Total edges extracted: {len(rag.edges)} ({len(rag.theme_layer.edges)} themes, {len(rag.detail_layer.edges)} relations)\n"
+ )
+
+ # ============================================================================
+ # Statistics
+ # ============================================================================
+ print("=" * 80)
+ print("STATISTICS")
+ print("=" * 80)
+
+ # Entity type distribution
+ type_counts = {}
+ for node in rag.nodes:
+ type_counts[node.type] = type_counts.get(node.type, 0) + 1
+
+ print("\nEntity Type Distribution:")
+ for entity_type, count in sorted(type_counts.items()):
+ print(f" {entity_type}: {count}")
+
+ # Centrality (number of relationships/themes per entity)
+ print("\nEntity Participation (Centrality):")
+ centrality = {}
+ for edge in rag.edges:
+ for participant in edge.participants:
+ centrality[participant] = centrality.get(participant, 0) + 1
+
+ for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[
+ :10
+ ]:
+ print(f" {entity}: {count}")
+
+ # ============================================================================
+ # Build Semantic Index for Q&A
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("Building Semantic Index...")
+ print("=" * 80)
+
+ rag.build_index()
+ print("✓ Semantic index built successfully\n")
+
+ # ============================================================================
+ # Q&A: Semantic Search over Hypergraph
+ # ============================================================================
+ print("=" * 80)
+ print("Q&A: DUAL-LAYER SEMANTIC SEARCH")
+ print("=" * 80)
+
+ queries = [
+ "虚空水晶现在在哪里?",
+ "黑蛇和议员Valerius有什么阴谋?",
+ "谁参与了泰坦空间站的秘密会议?",
+ "Leona目前的状况如何?",
+ ]
+
+ for i, query in enumerate(queries, 1):
+ print(f"\n【问题 {i}】{query}")
+ print("-" * 60)
+
+ try:
+ # Cog_RAG returns a dict with 'themes' and 'entities'
+ results = rag.search(query, top_k_themes=3, top_k_entities=3)
+
+ print(" [Layer 1: Macro Themes]")
+ if results['themes']:
+ for j, theme in enumerate(results['themes'], 1):
+ print(f" {j}. {theme.description}")
+ else:
+ print(" (No relevant themes found)")
+
+ print("\n [Layer 2: Micro Entities]")
+ if results['entities']:
+ for j, entity in enumerate(results['entities'], 1):
+ print(f" {j}. [{entity.name}] {entity.description[:100]}...")
+ else:
+ print(" (No relevant entities found)")
+
+ except Exception as e:
+ print(f" Search error: {str(e)}")
+
+ # ============================================================================
+ # Chat with CogRAG
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("💬 Interactive Chat with CogRAG")
+ print("=" * 80)
+ print("Engaging in multi-turn dialogue based on extracted knowledge...\n")
+
+ chat_queries = [
+ "虚空水晶现在在哪里?",
+ "黑蛇和议员Valerius有什么阴谋?",
+ "谁参与了泰坦空间站的秘密会议?",
+ "Leona目前的状况如何?",
+ ]
+
+ for q in chat_queries:
+ print(f"❓ User: {q}")
+ try:
+ response = rag.chat(q)
+ print(f"🤖 CogRAG: {response.content}\n")
+ except Exception as e:
+ print(f"⚠️ Chat error: {e}\n")
+
+ # ============================================================================
+ # Save Results
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("Saving Results...")
+ print("=" * 80)
+
+ try:
+ output_dir = "temp/cog_rag_demo"
+ rag.dump(output_dir)
+ print(f"✓ Results saved to {output_dir}/")
+
+ # List saved files
+ saved_files = []
+ if os.path.exists(output_dir):
+ for root, dirs, files in os.walk(output_dir):
+ for file in files:
+ rel_path = os.path.relpath(os.path.join(root, file), output_dir)
+ saved_files.append(rel_path)
+
+ if saved_files:
+ print("\n Saved files:")
+ for f in saved_files:
+ print(f" - {f}")
+ except Exception as e:
+ print(f"⚠ Warning: Could not save results: {str(e)}")
+
+ print("\n" + "=" * 80 + "\n")
+
diff --git a/examples/hypergraph_rag_demo.py b/examples/hypergraph_rag_demo.py
new file mode 100644
index 0000000..10cecd3
--- /dev/null
+++ b/examples/hypergraph_rag_demo.py
@@ -0,0 +1,287 @@
+"""
+HyperGraph_RAG Demo: Edge-First Extraction Strategy
+
+This demo demonstrates the HyperGraph_RAG System:
+1. **Edge-First Strategy**: Extracts Themes/Events (Hyperedges) first, then contextually extracts participating entities.
+2. **Narrative Understanding**: Focuses on identifying high-level themes before drilling down to actors.
+"""
+
+import os
+import sys
+
+# 添加项目根目录到路径
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+from hyperextract.hypergraphs import HyperGraph_RAG
+
+load_dotenv()
+
+# ============================================================================
+# Sample Story for Extraction (Shared with AutoHypergraph Demo)
+# ============================================================================
+
+STORY = """
+【银河纪元 3042年 - 赤红星云事件报告】
+
+危机在这一周达到了顶峰。整个银河系似乎都陷入了暗流涌动的政治漩涡。
+
+【第一阶段:秘密接触】
+臭名昭著的星际海盗黑蛇驾驶着他的旗舰复仇号,在泰坦空间站秘密会见了银河联邦的背叛者议员 Valerius。
+目击者称,两人在午夜时分登上了空间站的顶层禁区。据情报部门推测,这是一次肮脏的交易——
+双方达成了针对皇室的同盟关系。参与此会议的还有 Valerius 手下的特务头目 Shadow,
+他负责处理所有不可告人的细节。这个同盟的形成标志着大麻烦即将到来。
+
+【第二阶段:叛变与夺权】
+在这次会议的两天后,事态急剧恶化。黑蛇这种疯狂的家伙竟然在极光星区伏击了皇家护卫队。
+这场突然的冲突中,皇家护卫队的队长 Leona 与副队长 Marcus 奋力抵抗,
+但他们陷入了黑蛇手下的精锐战士的包围。这场激战导致队长 Leona 身受重伤。
+而黑蛇的目标不是为了杀戮,而是为了夺取一件神圣的物品——传说中的虚空水晶。
+这枚水晶散发着幽蓝的光芒,据说能够打开先祖之门的第一道密封。
+黑蛇在战斗中成功夺取了水晶,但 Leona 在临终前用最后的力量将其重新夺回。
+
+【第三阶段:流亡与隐藏】
+Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9。
+在这个被遗忘的角落,她向仅有的几个信任者求救:还有副队长 Marcus 以及隐秘组织"地下抵抗军"的领导人 Cipher。
+同时,议员 Valerius 非法持有了开启先祖之门的钥匙——这把钥匙本应被安全保管在皇家金库中。
+他现在藏匿在浮空都市中,与黑蛇遥相呼应,计划着下一步的政变。
+
+【第四阶段:势力重组】
+银河联邦的最高委员会得知了这一系列事件,决定采取行动。
+高级顾问 Artemis 被任命为特使,她与 Leona 在废土行星 Z-9 进行了秘密会晤,
+讨论如何制止这场即将到来的危机。同时,地下抵抗军的 Cipher 也加入了这次会议,
+三人共同制定了一个大胆的计划。
+
+【第五阶段:最后的筹码】
+虽然黑蛇未能获得虚空水晶,但他已经从 Valerius 那里获得了先祖之门钥匙的复制品。
+根据情报,这个复制品目前被秘密运往到黑蛇的秘密基地——位于辐射荒漠中的要塞。
+参与此次运送的还有 Shadow 和黑蛇手下的精锐战士群体。
+与此同时,Leona 和 Marcus 正在准备一场反击,他们计划在虚空水晶的力量下重新夺回局势的控制权。
+
+目前局势:
+- 黑蛇:手持复制品钥匙,掌控旗舰复仇号,计划进行最终的攻击
+- 议员 Valerius:躲在浮空都市中,与黑蛇保持联系
+- Leona:守护虚空水晶,在废土行星 Z-9 养伤,准备反击
+- 银河联邦:通过特使 Artemis 支持 Leona,正在调集力量
+- 地下抵抗军:Cipher 领导,与 Leona 和 Marcus 联手,准备对抗黑蛇和 Valerius 的同盟
+
+这场战争的走向仍不确定。虚空水晶、先祖之门、以及隐藏在这一切背后的秘密,
+都将在接下来的日子里浮出水面。银河的命运,可能就在这些关键人物和物品的碰撞中决定。
+"""
+
+
+if __name__ == "__main__":
+ # ============================================================================
+ # Initialize the HyperGraph_RAG system
+ # ============================================================================
+ print("=" * 80)
+ print("Initializing HyperGraph_RAG System (Edge-First)...")
+ print("=" * 80)
+
+ llm_client = ChatOpenAI(
+ model="gpt-4o-mini",
+ extra_body={
+ "system": "You are a helpful assistant. Always respond in Chinese."
+ },
+ temperature=0,
+ )
+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")
+
+ # Initialize HyperGraph_RAG
+ # Note: enable_llm_cache=False for demo purposes to force new extractions if needed
+ rag = HyperGraph_RAG(
+ llm_client=llm_client,
+ embedder=embedder,
+ verbose=True,
+ chunk_size=1000,
+ )
+
+ # ============================================================================
+ # Feed text to the system
+ # ============================================================================
+ print("\nFeeding story to the system...")
+ rag.feed_text(STORY)
+ print("✓ Story processed successfully\n")
+
+ # ============================================================================
+ # Display Extracted Hyperedges
+ # ============================================================================
+ print("=" * 80)
+ print("EXTRACTED HYPEREDGES")
+ print("=" * 80)
+
+ # 分离低阶边(2个参与者)和高阶边(3个以上参与者)
+ low_order_edges = [e for e in rag.edges if len(e.participants) == 2]
+ high_order_edges = [e for e in rag.edges if len(e.participants) > 2]
+
+ print(f"\n--- Low-Order Edges (Pairwise) ---")
+ for i, edge in enumerate(low_order_edges, 1):
+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
+ print(f" Description: {edge.description}")
+ keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"
+ print(f" Keywords: {keywords_str}")
+ print(f" Strength: {edge.strength}/10")
+
+ print(f"\n--- High-Order Edges (Hyperedges) ---")
+ for i, edge in enumerate(high_order_edges, 1):
+ print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
+ print(f" Description: {edge.description}")
+ keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"
+ print(f" Keywords: {keywords_str}")
+ print(f" Strength: {edge.strength}/10")
+
+ print(
+ f"\n✓ Total edges extracted: {len(rag.edges)} ({len(low_order_edges)} low-order, {len(high_order_edges)} high-order)\n"
+ )
+
+ # ============================================================================
+ # Display Extracted Entities
+ # ============================================================================
+ print("=" * 80)
+ print("EXTRACTED ENTITIES")
+ print("=" * 80)
+
+ for node in rag.nodes:
+ print(f"\n【{node.name}】(Type: {node.type})")
+ print(f" Description: {node.description}")
+
+ print(f"\n✓ Total entities extracted: {len(rag.nodes)}\n")
+
+ # ============================================================================
+ # Statistics
+ # ============================================================================
+ print("=" * 80)
+ print("STATISTICS")
+ print("=" * 80)
+
+ # Entity type distribution
+ type_counts = {}
+ for node in rag.nodes:
+ type_counts[node.type] = type_counts.get(node.type, 0) + 1
+
+ print("\nEntity Type Distribution:")
+ for entity_type, count in sorted(type_counts.items()):
+ print(f" {entity_type}: {count}")
+
+ # Average edge strength
+ if rag.edges:
+ avg_strength = sum(e.strength for e in rag.edges) / len(rag.edges)
+ print(f"\nAverage Edge Strength: {avg_strength:.2f}/10")
+
+ # Centrality (number of relationships per entity)
+ print("\nEntity Centrality (number of relationships):")
+ centrality = {}
+ for edge in rag.edges:
+ for participant in edge.participants:
+ centrality[participant] = centrality.get(participant, 0) + 1
+
+ for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[
+ :10
+ ]:
+ print(f" {entity}: {count}")
+
+ # ============================================================================
+ # Build Semantic Index for Q&A
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("Building Semantic Index...")
+ print("=" * 80)
+
+ rag.build_index()
+ print("✓ Semantic index built successfully\n")
+
+ # ============================================================================
+ # Q&A: Semantic Search over Hypergraph
+ # ============================================================================
+ print("=" * 80)
+ print("Q&A: SEMANTIC SEARCH")
+ print("=" * 80)
+
+ queries = [
+ "虚空水晶现在在哪里?",
+ "黑蛇和议员Valerius有什么阴谋?",
+ "谁参与了泰坦空间站的秘密会议?",
+ "Leona目前的状况如何?",
+ ]
+
+ for i, query in enumerate(queries, 1):
+ print(f"\n【问题 {i}】{query}")
+ print("-" * 60)
+
+ try:
+ results = rag.search_edges(query, top_k=3)
+
+ if results:
+ for j, result in enumerate(results, 1):
+ # EdgeAutoHypergraph (and base AutoHypergraph) default search returns (doc, score) tuple
+ # The 'doc' is the EdgeSchema object
+ edge = result[0] if isinstance(result, tuple) else result
+ score = result[1] if isinstance(result, tuple) else 1.0
+
+ # Note: EdgeSchema might not have strength if extracted via generic AutoHypergraph, but HyperGraph_RAG defines it.
+ # We handle potential missing attrs safely
+ strength = getattr(edge, "strength", "N/A")
+
+ print(f"\n {j}. [Score: {score:.3f}]")
+ print(f" Participants: {' <-> '.join(edge.participants)}")
+ print(f" Description: {edge.description}")
+ print(f" Strength: {strength}/10")
+ else:
+ print(" (No relevant relationships found)")
+ except Exception as e:
+ print(f" Search error: {str(e)}")
+
+ # ============================================================================
+ # Chat with HyperGraph_RAG
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("💬 Interactive Chat with HyperGraph_RAG")
+ print("=" * 80)
+ print("Engaging in multi-turn dialogue based on extracted knowledge...\n")
+
+ chat_queries = [
+ "虚空水晶现在在哪里?",
+ "黑蛇和议员Valerius有什么阴谋?",
+ "谁参与了泰坦空间站的秘密会议?",
+ "Leona目前的状况如何?",
+ ]
+
+ for q in chat_queries:
+ print(f"❓ User: {q}")
+ try:
+ response = rag.chat(q)
+ print(f"🤖 HyperGraph_RAG: {response.content}\n")
+ except Exception as e:
+ print(f"⚠️ Chat error: {e}\n")
+
+ # ============================================================================
+ # Save Results
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("Saving Results...")
+ print("=" * 80)
+
+ try:
+ output_dir = "temp/hypergraph_rag_demo"
+ rag.dump(output_dir)
+ print(f"✓ Results saved to {output_dir}/")
+
+ # List saved files
+ saved_files = []
+ if os.path.exists(output_dir):
+ for root, dirs, files in os.walk(output_dir):
+ for file in files:
+ rel_path = os.path.relpath(os.path.join(root, file), output_dir)
+ saved_files.append(rel_path)
+
+ if saved_files:
+ print("\n Saved files:")
+ for f in saved_files:
+ print(f" - {f}")
+ except Exception as e:
+ print(f"⚠ Warning: Could not save results: {str(e)}")
+
+ print("\n" + "=" * 80 + "\n")
+
diff --git a/hyperextract/hypergraphs/__init__.py b/hyperextract/hypergraphs/__init__.py
index 29ee874..559ed23 100644
--- a/hyperextract/hypergraphs/__init__.py
+++ b/hyperextract/hypergraphs/__init__.py
@@ -1,7 +1,13 @@
from .base import AutoHypergraph
from .hyper_rag import Hyper_RAG
+from .hypergraph_rag import HyperGraph_RAG
+from .cog_rag import Cog_RAG, Cog_RAG_ThemeLayer, Cog_RAG_DetailLayer
__all__ = [
"AutoHypergraph",
"Hyper_RAG",
+ "HyperGraph_RAG",
+ "Cog_RAG",
+ "Cog_RAG_ThemeLayer",
+ "Cog_RAG_DetailLayer",
]
diff --git a/hyperextract/hypergraphs/base.py b/hyperextract/hypergraphs/base.py
index 67bde4c..a9d4721 100644
--- a/hyperextract/hypergraphs/base.py
+++ b/hyperextract/hypergraphs/base.py
@@ -66,6 +66,15 @@ CRITICAL RULES:
"""
+DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal-
+You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.
+Your task is to Populate these concepts with their specific Participant Entities.
+
+CRITICAL RULES:
+1. Extract ALL key participants for each concept.
+2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).
+"""
+
class AutoHypergraphSchema(BaseModel, Generic[Node, Edge]):
"""Generic schema container for hypergraph data."""
@@ -85,6 +94,13 @@ class NodeListSchema(BaseModel, Generic[Node]):
)
+class EdgeDescriptionList(BaseModel):
+ """Intermediate schema for identifying edge concepts first."""
+ items: List[str] = Field(description="List of primary edge concepts/themes identified")
+
+
+
+
class EdgeListSchema(BaseModel, Generic[Edge]):
"""Intermediate schema for batch hyperedge extraction."""
@@ -97,7 +113,11 @@ class EdgeListSchema(BaseModel, Generic[Edge]):
class AutoHypergraph(
BaseAutoType[AutoHypergraphSchema[Node, Edge]], Generic[Node, Edge]
):
- """AutoHypergraph - extracts complex relationships involving multiple entities.
+ """AutoHypergraph - Node-First extraction: Extract entities first, then relationships.
+
+ Extraction Strategy: Two-Stage (Node-First)
+ 1. Stage 1: Extract all Nodes (entities) from text.
+ 2. Stage 2: Extract Edges (relationships) using identified Nodes as context.
Suitable for:
- Events with multiple participants (e.g., meetings, transactions)
@@ -762,3 +782,193 @@ class AutoHypergraph(
self._edge_memory.load_index(str(edge_index_path))
except Exception as e:
logger.warning(f"Failed to load edge index: {e}")
+
+class EdgeAutoHypergraph(AutoHypergraph[Node, Edge], Generic[Node, Edge]):
+ """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.
+
+ Extraction Strategy: Two-Stage (Edge-First)
+ 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).
+ 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.
+
+ Suitable for:
+ - Narrative-driven knowledge graphs (Theme -> Character)
+ - Event-driven extraction (Event -> Participant)
+ """
+
+ def __init__(
+ self,
+ node_schema: Type[Node],
+ edge_schema: Type[Edge],
+ node_key_extractor: Callable[[Node], str],
+ edge_key_extractor: Callable[[Edge], str],
+ nodes_in_edge_extractor: Callable[[Edge], Tuple[str, ...]],
+ llm_client: BaseChatModel,
+ embedder: Embeddings,
+ *,
+ prompt_for_node_extraction: str = "",
+ **kwargs: Any,
+ ):
+ """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.
+
+ Args:
+ prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".
+ (Extract Nodes given Edges).
+ """
+ super().__init__(
+ node_schema=node_schema,
+ edge_schema=edge_schema,
+ node_key_extractor=node_key_extractor,
+ edge_key_extractor=edge_key_extractor,
+ nodes_in_edge_extractor=nodes_in_edge_extractor,
+ llm_client=llm_client,
+ embedder=embedder,
+ prompt_for_node_extraction=prompt_for_node_extraction,
+ **kwargs,
+ )
+
+ # Build Stage 2 Schema (Edges containing nested Nodes)
+ # We assume the edge schema has a 'participants' field (List[str]).
+ # We create a new model where 'participants' is List[NodeSchema].
+
+ # 1. Base fields from EdgeSchema (excluding participants)
+ valid_fields = {
+ k: (v.annotation, v)
+ for k, v in edge_schema.model_fields.items()
+ if k != "participants"
+ }
+
+ # 2. Add Nested Participants field
+ valid_fields["participants"] = (
+ List[node_schema],
+ Field(description="List of entities participating in this concept/edge.")
+ )
+
+ # 3. Create Contextual Edge Model
+ self.contextual_edge_schema = create_model(
+ f"Contextual{edge_schema.__name__}",
+ **valid_fields
+ )
+
+ # 4. Create Result List Model
+ self.stage2_result_schema = create_model(
+ f"{edge_schema.__name__}FirstResult",
+ items=(List[self.contextual_edge_schema], Field(default_factory=list))
+ )
+
+ # Set default prompt if not provided
+ if not prompt_for_node_extraction:
+ self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT
+
+ def _extract_data(self, text: str) -> AutoHypergraphSchema:
+ """Main extraction logic dispatcher (Edge-First)."""
+ if self.extraction_mode == "two_stage":
+ raw_graph = self._extract_data_by_two_stage(text)
+ elif self.extraction_mode == "one_stage":
+ raise NotImplementedError(
+ "Single-stage extraction not yet supported for EdgeAutoHypergraph."
+ )
+ else:
+ raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")
+
+ # Note: Dangle pruning might be less critical here since nodes are derived from edges,
+ # but we keep it for consistency if nodes are merged globally.
+ return self._prune_dangling_edges(raw_graph)
+
+ def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:
+ """Extract edges/themes first, then entities within those contexts (batch processing).
+
+ Process:
+ 1. Split text into chunks.
+ 2. Batch extract Edge Concepts (Theme Strings) for all chunks.
+ 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).
+ 4. Merge all partial results into one global hypergraph.
+ """
+ # 1. Prepare chunks
+ if len(text) <= self.chunk_size:
+ chunks = [text]
+ else:
+ chunks = self.text_splitter.split_text(text)
+
+ if self.verbose:
+ logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")
+
+ # 2. Stage 1: Identify Edge Concepts (Strings)
+ edge_concepts_lists = self._extract_edge_concepts_batch(chunks)
+
+ # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)
+ chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)
+
+ # 4. Global Merge
+ # chunk_results is List[AutoHypergraphSchema]
+ return self.merge_batch_data(chunk_results)
+
+ def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:
+ """Stage 1: Batch extract edge concept strings."""
+ # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts
+ # We assume the prompt is geared towards finding a LIST of themes/concepts.
+
+ prompt_template = ChatPromptTemplate.from_template(
+ f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"
+ )
+ chain = prompt_template | self.llm_client.with_structured_output(EdgeDescriptionList)
+
+ inputs = [{"chunk_text": chunk} for chunk in chunks]
+ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
+
+ return [res.items if res else [] for res in results]
+
+ def _extract_details_batch(self, chunks: List[str], edge_concepts_list: List[List[str]]) -> List[AutoHypergraphSchema]:
+ """Generic Stage 2: Extract entities and details given identified edge concepts."""
+
+ # 1. Prepare Inputs
+ inputs = []
+ valid_map = {}
+ for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):
+ if concepts:
+ concepts_str = "- " + "\n- ".join(concepts)
+ inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})
+ valid_map[len(inputs) - 1] = i
+
+ if not inputs:
+ return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]
+
+ # 2. Chain Execution
+ # Uses self.node_prompt as the instruction for contextual node extraction
+ prompt = ChatPromptTemplate.from_template(
+ f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"
+ )
+ chain = prompt | self.llm_client.with_structured_output(self.stage2_result_schema)
+
+ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
+
+ # 3. Transform Nested Result -> Flat AutoHypergraphSchema
+ final_schemas = [self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))]
+
+ for i, result in enumerate(results):
+ if not result or not result.items:
+ continue
+
+ original_idx = valid_map[i]
+ current_nodes = []
+ current_edges = []
+
+ for item in result.items:
+ # item is a ContextualEdge (has nested participants=List[Node])
+
+ # Extract Nodes
+ nested_nodes = item.participants
+ node_names = []
+ for n in nested_nodes:
+ current_nodes.append(n)
+ node_names.append(self.node_key_extractor(n))
+
+ # Reconstruct Flat Edge
+ edge_data = item.model_dump()
+ edge_data["participants"] = sorted(node_names) # Ensure sorted for consistency
+
+ new_edge = self.edge_schema(**edge_data)
+ current_edges.append(new_edge)
+
+ final_schemas[original_idx] = self.graph_schema(nodes=current_nodes, edges=current_edges)
+
+ return final_schemas
diff --git a/hyperextract/hypergraphs/cog_rag.py b/hyperextract/hypergraphs/cog_rag.py
index e69de29..f363ecc 100644
--- a/hyperextract/hypergraphs/cog_rag.py
+++ b/hyperextract/hypergraphs/cog_rag.py
@@ -0,0 +1,425 @@
+"""
+COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System Pattern
+Extracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities.
+"""
+
+from typing import List
+from pydantic import BaseModel, Field
+from hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchema, EdgeAutoHypergraph
+from hyperextract.utils.logging import logger
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.embeddings import Embeddings
+from langchain_core.language_models import BaseChatModel
+from ontomem.merger import CustomRuleMerger
+
+# ============================================================================
+# Node Schema
+# ============================================================================
+
+
+class NodeSchema(BaseModel):
+ """Represents an entity extracted from the source text."""
+
+ name: str = Field(description="Name of the entity")
+ type: str = Field(
+ description="Entity type (person, organization, geo, event, role, concept, etc.)",
+ )
+ description: str = Field(
+ description="Comprehensive description of the entity's attributes and activities",
+ )
+
+
+# ============================================================================
+# Edge Schema (Theme)
+# ============================================================================
+
+
+class ThemeSchema(BaseModel):
+ """Represents a Theme or Narrative Arc connecting multiple entities (Hyperedge)."""
+
+ participants: List[str] = Field(
+ description="Names of key entities involved in this theme"
+ )
+ description: str = Field(
+ description="A sentence that describes the primary theme, reflecting the main conflict, resolution, or key message"
+ )
+
+
+# ============================================================================
+# Edge Schema (Entity Relation)
+# ============================================================================
+
+
+class EdgeSchema(BaseModel):
+ """Represents a relationship or event connecting multiple entities (low-order or high-order)."""
+
+ participants: List[str] = Field(
+ description="Names of entities involved in this relationship"
+ )
+ description: str = Field(
+ description="Detailed explanation of the relationship or event"
+ )
+ keywords: List[str] = Field(
+ description="List of keywords summarizing the relationship themes (e.g., ['conflict', 'trade', 'alliance'])",
+ )
+ strength: int = Field(
+ ge=1,
+ le=10,
+ description="Numerical score indicating relationship strength (1-10)",
+ )
+
+# ============================================================================
+# Extraction Prompts
+# ============================================================================
+
+COG_RAG_THEME_PROMPT = """
+-Goal-
+Analyze the text to identify the primary themes (narrative arcs).
+A Theme is a high-level concept, conflict, or main idea acting as a Hyperedge.
+"""
+
+COG_RAG_KEY_ENTITY_PROMPT = """
+-Goal-
+You are given a list of "Primary Themes" identified from the text.
+Your task is to populate these themes with key entities (Participants).
+"""
+
+COG_RAG_NODE_EXTRACTION_PROMPT = """
+-Goal-
+Identify relevant entities from the text.
+Entities will serve as participants in complex events later.
+### Source Text:
+"""
+
+COG_RAG_EDGE_EXTRACTION_PROMPT = """
+-Goal-
+You are an expert hypergraph knowledge extraction assistant.
+Extract "Hyperedges" that represent relationships, events, or thematic groupings involving MULTIPLE entities (2 or more) simultaneously.
+
+-Definition-
+A Hyperedge is a unified concept for any connection.
+It treats all involved entities as **participants** in a shared context
+(e.g., a conversation, a joint mission, a conflict, or a shared concept).
+
+-Constraints-
+1. You MUST ONLY use names from the "Allowed Entities" list provided below.
+2. Do NOT create edges for entities that represent strictly different concepts with no interaction.
+3. Ensure every edge has at least 2 participants.
+
+"""
+
+# ============================================================================
+# Merge Templates for LLM.CUSTOM_RULE Strategy
+# ============================================================================
+
+COG_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.
+You will receive a list of objects representing the same Entity.
+
+Your task is to merge them into a SINGLE object exactly matching the schema.
+
+Merge strategy:
+1. **name/type**: Keep the most frequent or precise value.
+2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.
+"""
+
+COG_RAG_THEME_MERGE_RULE = """You are an intelligent data merging assistant.
+You will receive a list of objects representing the same Theme.
+
+Your task is to merge them into a SINGLE object exactly matching the schema.
+
+Merge strategy:
+1. **participants**: Keep the list (they should be identical for the same theme key).
+2. **description**: Synthesize a global theme description that encompasses the narrative details from all input themes. Write it in the third person.
+"""
+
+COG_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.
+You will receive a list of objects representing the same Relationship.
+
+Your task is to merge them into a SINGLE object exactly matching the schema.
+
+Merge strategy:
+1. **participants**: Keep the list (they should be identical for the same relationship key).
+2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.
+3. **keywords**: Combine keyword lists from all inputs, removing duplicates.
+4. **strength**: Calculate the average of the input strengths (round to nearest integer).
+"""
+
+# ============================================================================
+# Theme Hypergraph (Layer 1: Macro Narrative)
+# ============================================================================
+
+
+class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
+ """
+ Layer 1 of Cog_RAG: Theme-First Extraction.
+ Extracts Themes (Hyperedges) and Key Entities (Nodes).
+ """
+
+ def __init__(
+ self,
+ llm_client: BaseChatModel,
+ embedder: Embeddings,
+ chunk_size: int = 2048,
+ chunk_overlap: int = 256,
+ max_workers: int = 10,
+ verbose: bool = True,
+ ):
+ """
+ Initialize COG_RAG engine.
+
+ Args:
+ llm_client (BaseChatModel): LLM client for text generation.
+ embedder (Embeddings): Embedding model for text embeddings.
+ chunk_size (int): Size of text chunks for indexing.
+ chunk_overlap (int): Overlap between text chunks for indexing.
+ max_workers (int): Maximum number of workers for indexing.
+ verbose (bool): Display detailed execution logs and progress information.
+ """
+ # 1. Define Key Extractors
+ # Node key: exact match by name
+ node_key_fn = lambda x: x.name
+
+ # Edge key: sorted tuple of participants
+ edge_key_fn = lambda x: tuple(sorted(x.participants))
+
+ # 2. Edge consistency check: tell AutoHypergraph which nodes this edge connects
+ nodes_in_edge_fn = lambda x: x.participants
+
+ # 3. Create custom Mergers with LLM.CUSTOM_RULE strategy
+ node_merger = CustomRuleMerger(
+ key_extractor=node_key_fn,
+ llm_client=llm_client,
+ item_schema=NodeSchema,
+ rule=COG_RAG_NODE_MERGE_RULE,
+ )
+
+ edge_merger = CustomRuleMerger(
+ key_extractor=edge_key_fn,
+ llm_client=llm_client,
+ item_schema=ThemeSchema,
+ rule=COG_RAG_THEME_MERGE_RULE,
+ )
+
+ # 4. Call parent class initialization
+ super().__init__(
+ node_schema=NodeSchema,
+ edge_schema=ThemeSchema,
+ node_key_extractor=node_key_fn,
+ edge_key_extractor=edge_key_fn,
+ nodes_in_edge_extractor=nodes_in_edge_fn,
+ llm_client=llm_client,
+ embedder=embedder,
+ # Enforce two-stage extraction
+ extraction_mode="two_stage",
+ # Inject customized prompts for extraction
+ # prompt_for_node_extraction acts as the Stage 2 (Edge -> Node) prompt
+ prompt_for_node_extraction=COG_RAG_KEY_ENTITY_PROMPT,
+ prompt_for_edge_extraction=COG_RAG_THEME_PROMPT,
+ # Pass custom merger instances
+ node_strategy_or_merger=node_merger,
+ edge_strategy_or_merger=edge_merger,
+ # Optimize indexing
+ node_fields_for_index=["name", "type", "description"],
+ edge_fields_for_index=["description"],
+ # Other parameters
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ max_workers=max_workers,
+ verbose=verbose,
+ )
+
+# ============================================================================
+# Detail Hypergraph (Layer 2: Micro Relations)
+# ============================================================================
+
+
+class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):
+ """
+ Layer 2 of Cog_RAG: Entity-First Extraction.
+ Extracts Entities (Nodes) and Semantic Relationships (Hyperedges).
+ """
+
+ def __init__(
+ self,
+ llm_client: BaseChatModel,
+ embedder: Embeddings,
+ chunk_size: int = 2048,
+ chunk_overlap: int = 256,
+ max_workers: int = 10,
+ verbose: bool = True,
+ ):
+ """
+ Initialize COG_RAG Entity-Relation engine.
+ """
+ node_key_fn = lambda x: x.name
+ edge_key_fn = lambda x: tuple(sorted(x.participants))
+ nodes_in_edge_fn = lambda x: x.participants
+
+ node_merger = CustomRuleMerger(
+ key_extractor=node_key_fn,
+ llm_client=llm_client,
+ item_schema=NodeSchema,
+ rule=COG_RAG_NODE_MERGE_RULE,
+ )
+
+ edge_merger = CustomRuleMerger(
+ key_extractor=edge_key_fn,
+ llm_client=llm_client,
+ item_schema=EdgeSchema,
+ rule=COG_RAG_EDGE_MERGE_RULE,
+ )
+
+ super().__init__(
+ node_schema=NodeSchema,
+ edge_schema=EdgeSchema,
+ node_key_extractor=node_key_fn,
+ edge_key_extractor=edge_key_fn,
+ nodes_in_edge_extractor=nodes_in_edge_fn,
+ llm_client=llm_client,
+ embedder=embedder,
+ extraction_mode="two_stage",
+ prompt_for_node_extraction=COG_RAG_NODE_EXTRACTION_PROMPT,
+ prompt_for_edge_extraction=COG_RAG_EDGE_EXTRACTION_PROMPT,
+ node_strategy_or_merger=node_merger,
+ edge_strategy_or_merger=edge_merger,
+ node_fields_for_index=["name", "type", "description"],
+ edge_fields_for_index=["description", "keywords"],
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ max_workers=max_workers,
+ verbose=verbose,
+ )
+
+
+# ============================================================================
+# Main System Class (Wrapper)
+# ============================================================================
+
+class Cog_RAG:
+ """
+ Cognitive-Inspired Dual-Hypergraph RAG System.
+
+ Combines two Hypergraph Layers:
+ 1. Theme Layer (Cog_RAG_ThemeLayer): Captures macro narratives and themes.
+ 2. Detail Layer (Cog_RAG_DetailLayer): Captures micro entity relationships.
+ """
+
+ def __init__(
+ self,
+ llm_client: BaseChatModel,
+ embedder: Embeddings,
+ chunk_size: int = 2048,
+ chunk_overlap: int = 256,
+ max_workers: int = 10,
+ verbose: bool = True,
+ ):
+ self.theme_layer = Cog_RAG_ThemeLayer(
+ llm_client=llm_client,
+ embedder=embedder,
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ max_workers=max_workers,
+ verbose=verbose
+ )
+ self.detail_layer = Cog_RAG_DetailLayer(
+ llm_client=llm_client,
+ embedder=embedder,
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ max_workers=max_workers,
+ verbose=verbose
+ )
+ self.llm = llm_client
+ self.verbose = verbose
+
+ def feed_text(self, text: str):
+ """Feed text to both hypergraph layers."""
+ if self.verbose:
+ logger.info("Cog_RAG: Feeding text to Theme Layer...")
+ self.theme_layer.feed_text(text)
+
+ if self.verbose:
+ logger.info("Cog_RAG: Feeding text to Detail Layer...")
+ self.detail_layer.feed_text(text)
+
+ def build_index(self):
+ """Build indices for both layers."""
+ if self.verbose:
+ logger.info("Cog_RAG: Building indices...")
+ self.theme_layer.build_index()
+ self.detail_layer.build_index()
+
+ def search(self, query: str, top_k_themes: int = 3, top_k_entities: int = 3):
+ """
+ Dual-Layer Search.
+ Strategy:
+ 1. Macro: Search for Themes in ThemeLayer (Edge-First).
+ 2. Micro: Search for Entities in DetailLayer (Node-First).
+ """
+ # 1. Macro Context: Themes
+ themes = self.theme_layer.search_edges(query, top_k=top_k_themes)
+
+ # 2. Micro Context: Entities (Nodes)
+ # Note: The user requested searching for "Specific Entities" in the second step.
+ entities = self.detail_layer.search_nodes(query, top_k=top_k_entities)
+
+ return {
+ "themes": themes,
+ "entities": entities
+ }
+
+ def chat(self, query: str, top_k_themes: int = 3, top_k_entities: int = 3):
+ """Generate an answer using context from both layers."""
+ results = self.search(query, top_k_themes, top_k_entities)
+
+ context_parts = []
+ if results["themes"]:
+ context_parts.append("=== MACRO THEMES (Narrative Context) ===")
+ for t in results["themes"]:
+ context_parts.append(f"- {t.description} (Involved: {', '.join(t.participants)})")
+
+ if results["entities"]:
+ context_parts.append("\n=== MICRO ENTITIES (Specific Details) ===")
+ for e in results["entities"]:
+ context_parts.append(f"- [{e.type}] {e.name}: {e.description}")
+
+ if not context_parts:
+ context = "No relevant information found."
+ else:
+ context = "\n".join(context_parts)
+
+ prompt = ChatPromptTemplate.from_template(
+ "You are an intelligent assistant using a Dual-Layer Hypergraph Knowledge Base.\n"
+ "Use the provided Themes (Macro) and Key Entities (Micro) to answer the user's question comprehensively.\n\n"
+ "{context}\n\n"
+ "Question: {question}\n\n"
+ "Answer:"
+ )
+
+ chain = prompt | self.llm
+ return chain.invoke({"context": context, "question": query})
+
+ def dump(self, folder_path: str):
+ """Save both extracted graphs."""
+ import os
+ self.theme_layer.dump(os.path.join(folder_path, "theme_layer"))
+ self.detail_layer.dump(os.path.join(folder_path, "detail_layer"))
+
+ @property
+ def nodes(self):
+ """Return combined unique nodes from both layers (simple aggregation)."""
+ # Note: This is a property for valid introspection, but nodes are managed separately.
+ # We perform a simple name-based deduction for display.
+ seen = set()
+ unique = []
+ for n in self.theme_layer.nodes + self.detail_layer.nodes:
+ if n.name not in seen:
+ seen.add(n.name)
+ unique.append(n)
+ return unique
+
+ @property
+ def edges(self):
+ """Return all edges (Themes + Relations)."""
+ return self.theme_layer.edges + self.detail_layer.edges
+
diff --git a/hyperextract/hypergraphs/hypergraph_rag.py b/hyperextract/hypergraphs/hypergraph_rag.py
index e69de29..7903039 100644
--- a/hyperextract/hypergraphs/hypergraph_rag.py
+++ b/hyperextract/hypergraphs/hypergraph_rag.py
@@ -0,0 +1,185 @@
+"""
+HyperGraph_RAG: Integration of EdgeAutoHypergraph (Edge-First Extraction).
+Matches the structure of Hyper_RAG regarding Schemas and Mergers.
+"""
+
+from typing import List
+from pydantic import BaseModel, Field
+from hyperextract.hypergraphs.base import EdgeAutoHypergraph
+from langchain_core.embeddings import Embeddings
+from langchain_core.language_models import BaseChatModel
+from ontomem.merger import CustomRuleMerger
+
+# ============================================================================
+# Node Schema
+# ============================================================================
+
+
+class NodeSchema(BaseModel):
+ """Represents an entity extracted from the source text."""
+
+ name: str = Field(description="Name of the entity")
+ type: str = Field(
+ description="Entity type (person, organization, geo, event, role, concept, etc.)",
+ )
+ description: str = Field(
+ description="Comprehensive description of the entity's attributes and activities",
+ )
+
+
+# ============================================================================
+# Edge Schema
+# ============================================================================
+
+
+class EdgeSchema(BaseModel):
+ """Represents a relationship or event connecting multiple entities (low-order or high-order)."""
+
+ participants: List[str] = Field(
+ description="Names of entities involved in this relationship"
+ )
+ description: str = Field(
+ description="Detailed explanation of the relationship or event"
+ )
+ keywords: List[str] = Field(
+ description="List of keywords summarizing the relationship themes (e.g., ['conflict', 'trade', 'alliance'])",
+ )
+ strength: int = Field(
+ ge=1,
+ le=10,
+ description="Numerical score indicating relationship strength (1-10)",
+ )
+
+
+# ============================================================================
+# Extraction Prompts (Edge-First)
+# ============================================================================
+
+HyperGraph_RAG_THEME_EXTRACTION_PROMPT = """
+-Goal-
+Identify the key "Themes", "Narratives", or "Complex Events" present in the text.
+A Theme/Event should represent a significant interaction or grouping involving multiple participants.
+
+-Output-
+Return a list of these Themes/Events as descriptive strings.
+Do not list participants data formats yet, just the essence of the event (e.g., "The secret meeting at Titan Station", "The ambush in the Aurora Sector").
+"""
+
+HyperGraph_RAG_CONTEXT_NODE_PROMPT = """
+-Goal-
+You are an expert knowledge extractor.
+You have been given a list of "Themes/Events" (Hyperedges) that occurred in the text.
+Your job is to identify the specific ENTITIES (participants) involved in each theme.
+
+-Output-
+For each Theme, list the entities that participated.
+"""
+
+
+# ============================================================================
+# Merge Templates for LLM.CUSTOM_RULE Strategy
+# ============================================================================
+
+Hyper_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.
+You will receive a list of objects representing the same Entity.
+
+Your task is to merge them into a SINGLE object exactly matching the schema.
+
+Merge strategy:
+1. **name/type**: Keep the most frequent or precise value.
+2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.
+"""
+
+Hyper_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.
+You will receive a list of objects representing the same Hyperedge (Relationship/Event).
+
+Your task is to merge them into a SINGLE object exactly matching the schema.
+
+Merge strategy:
+1. **participants**: Keep the list (they should be identical for the same hyperedge key).
+2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.
+3. **keywords**: Combine keyword lists from all inputs, removing duplicates.
+4. **strength**: Calculate the average of the input strengths (round to nearest integer).
+"""
+
+# ============================================================================
+# HyperGraph_RAG Class
+# ============================================================================
+
+
+class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):
+ """
+ HyperGraph_RAG: Hypergraph-based RAG with Edge-First Extraction Strategy.
+
+ Difference from Hyper_RAG:
+ - Uses EdgeAutoHypergraph (extracts Themes first, then Entities).
+ - Suitable for high-level narrative understanding where Themes map to specific Actors.
+ """
+
+ def __init__(
+ self,
+ llm_client: BaseChatModel,
+ embedder: Embeddings,
+ chunk_size: int = 2048,
+ chunk_overlap: int = 256,
+ max_workers: int = 10,
+ verbose: bool = True,
+ ):
+ """
+ Initialize HyperGraph_RAG engine.
+
+ Args:
+ llm_client (BaseChatModel): LLM client for text generation.
+ embedder (Embeddings): Embedding model for text embeddings.
+ chunk_size (int): Size of text chunks for extraction.
+ chunk_overlap (int): Overlap between text chunks.
+ max_workers (int): Maximum number of workers for extraction.
+ verbose (bool): Display detailed logs.
+ """
+ # 1. Define Key Extractors
+ node_key_fn = lambda x: x.name
+ edge_key_fn = lambda x: tuple(sorted(x.participants))
+ nodes_in_edge_fn = lambda x: x.participants
+
+ # 2. Setup Mergers
+ node_merger = CustomRuleMerger(
+ key_extractor=node_key_fn,
+ llm_client=llm_client,
+ item_schema=NodeSchema,
+ rule=Hyper_RAG_NODE_MERGE_RULE,
+ )
+
+ edge_merger = CustomRuleMerger(
+ key_extractor=edge_key_fn,
+ llm_client=llm_client,
+ item_schema=EdgeSchema,
+ rule=Hyper_RAG_EDGE_MERGE_RULE,
+ )
+
+ # 3. Initialize EdgeAutoHypergraph (Parent)
+ super().__init__(
+ node_schema=NodeSchema,
+ edge_schema=EdgeSchema,
+ node_key_extractor=node_key_fn,
+ edge_key_extractor=edge_key_fn,
+ nodes_in_edge_extractor=nodes_in_edge_fn,
+ llm_client=llm_client,
+ embedder=embedder,
+ # Prompts
+ # Stage 1: Extract Themes
+ prompt_for_edge_extraction=HyperGraph_RAG_THEME_EXTRACTION_PROMPT,
+ # Stage 2: Extract Particles given Themes
+ prompt_for_node_extraction=HyperGraph_RAG_CONTEXT_NODE_PROMPT,
+ # Merger strategies
+ node_strategy_or_merger=node_merger,
+ edge_strategy_or_merger=edge_merger,
+ # Indexing optimization
+ node_fields_for_index=["name", "type", "description"],
+ edge_fields_for_index=["keywords", "description"],
+ # BaseAutoType Params
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ max_workers=max_workers,
+ verbose=verbose,
+ )
+