Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

undate cog_rag and graph_rag

haoohu64c3c0d02fdd
diff --git a/examples/method_graph_rag_demo.py b/examples/method_graph_rag_demo.py
new file mode 100644
index 0000000..96a70e3
--- /dev/null
+++ b/examples/method_graph_rag_demo.py
@@ -0,0 +1,260 @@
+"""
+Graph-RAG Demo: Entity-Relationship Extraction with Standard Binary Graphs
+
+This demo demonstrates:
+1. Extracting entities and binary relationships from a complex sci-fi story.
+2. Analysis of standard graph relationships (Source -> Target).
+3. Semantic Search over the constructed Knowledge Graph (Q&A).
+"""
+
+import os
+import sys
+
+# 添加项目根目录到路径
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from dotenv import load_dotenv
+load_dotenv()
+
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from hyperextract.methods.rag import Graph_RAG
+
+# ============================================================================
+# Sample Story for Extraction (Shared with AutoHypergraph Demo)
+# ============================================================================
+
+STORY = """
+【银河纪元 3042年 - 赤红星云事件报告】
+
+危机在这一周达到了顶峰。整个银河系似乎都陷入了暗流涌动的政治漩涡。
+
+【第一阶段:秘密接触】
+臭名昭著的星际海盗黑蛇驾驶着他的旗舰复仇号,在泰坦空间站秘密会见了银河联邦的背叛者议员 Valerius。
+目击者称,两人在午夜时分登上了空间站的顶层禁区。据情报部门推测,这是一次肮脏的交易——
+双方达成了针对皇室的同盟关系。参与此会议的还有 Valerius 手下的特务头目 Shadow,
+他负责处理所有不可告人的细节。这个同盟的形成标志着大麻烦即将到来。
+
+【第二阶段:叛变与夺权】
+在这次会议的两天后,事态急剧恶化。黑蛇这种疯狂的家伙竟然在极光星区伏击了皇家护卫队。
+这场突然的冲突中,皇家护卫队的队长 Leona 与副队长 Marcus 奋力抵抗,
+但他们陷入了黑蛇手下的精锐战士的包围。这场激战导致队长 Leona 身受重伤。
+而黑蛇的目标不是为了杀戮,而是为了夺取一件神圣的物品——传说中的虚空水晶。
+这枚水晶散发着幽蓝的光芒,据说能够打开先祖之门的第一道密封。
+黑蛇在战斗中成功夺取了水晶,但 Leona 在临终前用最后的力量将其重新夺回。
+
+【第三阶段:流亡与隐藏】
+Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9。
+在这个被遗忘的角落,她向仅有的几个信任者求救:还有副队长 Marcus 以及隐秘组织"地下抵抗军"的领导人 Cipher。
+同时,议员 Valerius 非法持有了开启先祖之门的钥匙——这把钥匙本应被安全保管在皇家金库中。
+他现在藏匿在浮空都市中,与黑蛇遥相呼应,计划着下一步的政变。
+
+【第四阶段:势力重组】
+银河联邦的最高委员会得知了这一系列事件,决定采取行动。
+高级顾问 Artemis 被任命为特使,她与 Leona 在废土行星 Z-9 进行了秘密会晤,
+讨论如何制止这场即将到来的危机。同时,地下抵抗军的 Cipher 也加入了这次会议,
+三人共同制定了一个大胆的计划。
+
+【第五阶段:最后的筹码】
+虽然黑蛇未能获得虚空水晶,但他已经从 Valerius 那里获得了先祖之门钥匙的复制品。
+根据情报,这个复制品目前被秘密运往到黑蛇的秘密基地——位于辐射荒漠中的要塞。
+参与此次运送的还有 Shadow 和黑蛇手下的精锐战士群体。
+与此同时,Leona 和 Marcus 正在准备一场反击,他们计划在虚空水晶的力量下重新夺回局势的控制权。
+
+目前局势:
+- 黑蛇:手持复制品钥匙,掌控旗舰复仇号,计划进行最终的攻击
+- 议员 Valerius:躲在浮空都市中,与黑蛇保持联系
+- Leona:守护虚空水晶,在废土行星 Z-9 养伤,准备反击
+- 银河联邦:通过特使 Artemis 支持 Leona,正在调集力量
+- 地下抵抗军:Cipher 领导,与 Leona 和 Marcus 联手,准备对抗黑蛇和 Valerius 的同盟
+
+这场战争的走向仍不确定。虚空水晶、先祖之门、以及隐藏在这一切背后的秘密,
+都将在接下来的日子里浮出水面。银河的命运,可能就在这些关键人物和物品的碰撞中决定。
+"""
+
+
+if __name__ == "__main__":
+ # ============================================================================
+ # Initialize the Graph-RAG system with LLM and Embeddings
+ # ============================================================================
+ print("=" * 80)
+ print("Initializing Graph-RAG System...")
+ print("=" * 80)
+
+ # Use a capable model for extraction
+ llm_client = ChatOpenAI(
+ model="gpt-4o-mini",
+ extra_body={
+ "system": "You are a helpful assistant. Always respond in Chinese."
+ },
+ temperature=0,
+ )
+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")
+
+ rag = Graph_RAG(
+ llm_client=llm_client,
+ embedder=embedder,
+ verbose=True,
+ chunk_size=1000,
+ )
+
+ # Feed text to the system
+ # ============================================================================
+ print("\nFeeding story to the system...")
+ rag.feed_text(STORY)
+ print("✓ Story processed successfully\n")
+
+ # ============================================================================
+ # Display Extracted Entities
+ # ============================================================================
+ print("=" * 80)
+ print("EXTRACTED ENTITIES")
+ print("=" * 80)
+
+ for node in rag.nodes:
+ print(f"\n【{node.name}】(Type: {node.type})")
+ print(f" Description: {node.description}")
+
+ print(f"\n✓ Total entities extracted: {len(rag.nodes)}\n")
+
+ # ============================================================================
+ # Display Extracted Edges
+ # ============================================================================
+ print("=" * 80)
+ print("EXTRACTED EDGES (Binary Relationships)")
+ print("=" * 80)
+
+ for i, edge in enumerate(rag.edges, 1):
+ print(f"\n{i}. {edge.source} -> {edge.target}")
+ print(f" Description: {edge.description}")
+ print(f" Keywords: {edge.keywords}")
+ print(f" Strength: {edge.strength}/10")
+
+ print(f"\n✓ Total edges extracted: {len(rag.edges)}\n")
+
+ # ============================================================================
+ # Statistics
+ # ============================================================================
+ print("=" * 80)
+ print("STATISTICS")
+ print("=" * 80)
+
+ # Entity type distribution
+ type_counts = {}
+ for node in rag.nodes:
+ type_counts[node.type] = type_counts.get(node.type, 0) + 1
+
+ print("\nEntity Type Distribution:")
+ for entity_type, count in sorted(type_counts.items()):
+ print(f" {entity_type}: {count}")
+
+ # Average edge strength
+ if rag.edges:
+ avg_strength = sum(e.strength for e in rag.edges) / len(rag.edges)
+ print(f"\nAverage Edge Strength: {avg_strength:.2f}/10")
+
+ # Centrality (Out-degree + In-degree)
+ print("\nEntity Centrality (Degree):")
+ centrality = {}
+ for edge in rag.edges:
+ centrality[edge.source] = centrality.get(edge.source, 0) + 1
+ centrality[edge.target] = centrality.get(edge.target, 0) + 1
+
+ for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[
+ :10
+ ]:
+ print(f" {entity}: {count}")
+
+ # ============================================================================
+ # Build Semantic Index & Communities
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("Building Semantic Index & Communities...")
+ print("=" * 80)
+
+ rag.build_index()
+ print("✓ Semantic index built successfully")
+
+ rag.build_communities()
+ print("✓ Communities built successfully\n")
+
+ # ============================================================================
+ # Display Community Reports
+ # ============================================================================
+ print("=" * 80)
+ print("COMMUNITY REPORTS")
+ print("=" * 80)
+
+ if not rag.community_reports:
+ print("No community reports generated.")
+ else:
+ for cid, report in rag.community_reports.items():
+ print(f"\n【{report.title}】 (ID: {report.id}, Rating: {report.rating}/10)")
+ print(f" Summary: {report.summary}")
+ print(f" Findings:")
+ for finding in report.findings:
+ print(f" - {finding.summary}")
+
+ # ============================================================================
+ # Q&A: Semantic Search over Graph
+ # ============================================================================
+ print("=" * 80)
+ print("Q&A: COMPARISON (Global vs Local vs Standard)")
+ print("=" * 80)
+
+ queries = [
+ "虚空水晶现在在哪里?",
+ "黑蛇和议员Valerius有什么阴谋?",
+ "谁参与了泰坦空间站的秘密会议?",
+ "Leona目前的状况如何?",
+ ]
+
+ # 1. Global Search Example
+ print("\n" + "-" * 60)
+ print("[1. Global Search] (Broad Understanding / Map-Reduce)")
+ print("-" * 60)
+ for q in queries:
+ print(f"\nQuery: {q}")
+ print("Response:")
+ try:
+ print(rag.global_search(q))
+ except Exception as e:
+ print(f"Error: {e}")
+
+ # 2. Local Search Example
+ print("\n" + "-" * 60)
+ print("[2. Local Search] (Targeted Subgraph + Community Context)")
+ print("-" * 60)
+ for q in queries:
+ print(f"\nQuery: {q}")
+ print("Response:")
+ try:
+ print(rag.local_search(q))
+ except Exception as e:
+ print(f"Error: {e}")
+
+ # 3. Standard Edge Search
+ print("\n" + "-" * 60)
+ print("[3. Standard Edge Search] (Vector similarity only)")
+ print("-" * 60)
+
+ for i, query in enumerate(queries, 1):
+ print(f"\nQuery: {query}")
+ try:
+ results = rag.search_edges(query, top_k=3)
+
+ if results:
+ for j, result in enumerate(results, 1):
+ print(f" [{j}] {result.source} -> {result.target}: {result.description[:100]}...")
+ else:
+ print(" No relevant edges found.")
+ except Exception as e:
+ print(f" Search failed: {e}")
+
+ # ============================================================================
+ # Save Results
+ # ============================================================================
+ print("\n" + "=" * 80)
+ print("SAVING RESULTS")
+ print("=" * 80)
+
+ rag.dump("temp/graph_rag_demo")
diff --git a/hyperextract/methods/rag/__init__.py b/hyperextract/methods/rag/__init__.py
index d55f995..ad0a501 100644
--- a/hyperextract/methods/rag/__init__.py
+++ b/hyperextract/methods/rag/__init__.py
@@ -8,11 +8,13 @@ from .hyper_rag import Hyper_RAG
from .hypergraph_rag import HyperGraph_RAG
from .cog_rag import Cog_RAG
from .light_rag import Light_RAG
+from .graph_rag import Graph_RAG
__all__ = [
"Hyper_RAG",
"HyperGraph_RAG",
"Cog_RAG",
"Light_RAG",
+ "Graph_RAG",
]
diff --git a/hyperextract/methods/rag/cog_rag.py b/hyperextract/methods/rag/cog_rag.py
index 9c29d7e..89c6a3c 100644
--- a/hyperextract/methods/rag/cog_rag.py
+++ b/hyperextract/methods/rag/cog_rag.py
@@ -3,6 +3,9 @@ COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System Pattern
Extracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities.
"""
+import os
+import json
+from datetime import datetime
from typing import List
from hashlib import md5
from pydantic import BaseModel, Field
@@ -354,6 +357,11 @@ class Cog_RAG:
max_workers: int = 10,
verbose: bool = False,
):
+ self.metadata = {
+ "created_at": datetime.now(),
+ "updated_at": datetime.now(),
+ }
+
self.theme_layer = Cog_RAG_ThemeLayer(
llm_client=llm_client,
embedder=embedder,
@@ -440,12 +448,75 @@ class Cog_RAG:
return chain.invoke({"context": context, "question": query})
def dump(self, folder_path: str):
- """Save both extracted graphs."""
- import os
+ """Save both extracted graphs and metadata."""
+ if not os.path.exists(folder_path):
+ os.makedirs(folder_path)
+
+ metadata_path = os.path.join(folder_path, "metadata.json")
+ with open(metadata_path, "w", encoding="utf-8") as f:
+ json.dump(self.metadata, f, indent=2, ensure_ascii=False, default=str)
+
+ # 导出根目录 data.json 以供 CLI 合法性校验和 info 命令统计
+ root_data = {
+ "nodes": [n.model_dump() for n in self.nodes],
+ "edges": [e.model_dump() for e in self.edges]
+ }
+ with open(os.path.join(folder_path, "data.json"), "w", encoding="utf-8") as f:
+ json.dump(root_data, f, ensure_ascii=False, indent=2)
self.theme_layer.dump(os.path.join(folder_path, "theme_layer"))
self.detail_layer.dump(os.path.join(folder_path, "detail_layer"))
+ def load(self, folder_path: str):
+ """Load extracted graphs and metadata."""
+ metadata_path = os.path.join(folder_path, "metadata.json")
+ if os.path.exists(metadata_path):
+ with open(metadata_path, "r", encoding="utf-8") as f:
+ params = json.load(f)
+ if isinstance(self.metadata, dict):
+ self.metadata.update(params)
+ else:
+ self.metadata = params
+
+ theme_path = os.path.join(folder_path, "theme_layer")
+ if os.path.exists(theme_path) and os.path.exists(os.path.join(theme_path, "data.json")):
+ self.theme_layer.load(theme_path)
+
+ detail_path = os.path.join(folder_path, "detail_layer")
+ if os.path.exists(detail_path) and os.path.exists(os.path.join(detail_path, "data.json")):
+ self.detail_layer.load(detail_path)
+
+ def show(
+ self,
+ node_label_extractor=None,
+ edge_label_extractor=None,
+ ):
+ """Visualize the specified layer interactively."""
+ print("\n[Cog_RAG] 这是一个双层图谱系统。请选择你想可视化的层级:")
+ print("1. Detail Layer (细节层:具体实体及其关系) - [默认]")
+ print("2. Theme Layer (主题层:宏观主题及其参与者)")
+
+ try:
+ choice = input(f"请输入选项 [1/2]: ").strip()
+ except EOFError:
+ choice = "1"
+
+ layer_map = {
+ "2": ("Theme Layer", self.theme_layer),
+ "1": ("Detail Layer", self.detail_layer),
+ }
+
+ layer_name, layer_obj = layer_map.get(choice, layer_map["1"])
+
+ print(f"Visualizing Cog_RAG ({layer_name})...")
+ try:
+ layer_obj.show(
+ node_label_extractor=node_label_extractor,
+ edge_label_extractor=edge_label_extractor,
+ )
+ except Exception as e:
+ print(f"Error visualizing {layer_name}: {e}")
+
@property
def nodes(self):
"""Return combined unique nodes from both layers (simple aggregation)."""
diff --git a/hyperextract/methods/rag/graph_rag.py b/hyperextract/methods/rag/graph_rag.py
new file mode 100644
index 0000000..75029e3
--- /dev/null
+++ b/hyperextract/methods/rag/graph_rag.py
@@ -0,0 +1,626 @@
+"""
+Graph-RAG: Graph-based Retrieval-Augmented Generation
+Extracts and manages entity-relationship knowledge graphs with standard binary edges.
+"""
+
+import json
+from pathlib import Path
+from typing import List, Dict, Optional, Any, Set
+import networkx as nx
+from pydantic import BaseModel, Field
+from langchain_core.embeddings import Embeddings
+from langchain_core.language_models import BaseChatModel
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from hyperextract.types import AutoGraph
+from ontomem.merger import CustomRuleMerger
+
+try:
+ from graspologic.partition import hierarchical_leiden
+ HAS_GRASPOLOGIC = True
+except ImportError:
+ HAS_GRASPOLOGIC = False
+
+# ============================================================================
+# Node Schema
+# ============================================================================
+
+
+class NodeSchema(BaseModel):
+ """Represents an entity extracted from the source text."""
+
+ name: str = Field(description="Name of the entity")
+ type: str = Field(
+ description="Entity type (person, organization, geo, event, role, concept, etc.)",
+ )
+ description: str = Field(
+ description="Comprehensive description of the entity's attributes and activities",
+ )
+
+
+# ============================================================================
+# Edge Schema
+# ============================================================================
+
+
+class EdgeSchema(BaseModel):
+ """Represents a directed relationship between two entities."""
+
+ source: str = Field(description="Name of the source entity")
+ target: str = Field(description="Name of the target entity")
+ description: str = Field(
+ description="Detailed explanation of the relationship or event connection"
+ )
+ keywords: str = Field(
+ description="Keywords summarizing the relationship (e.g., 'conflict, trade')",
+ )
+ strength: int = Field(
+ ge=1,
+ le=10,
+ description="Numerical score indicating relationship strength (1-10)",
+ )
+
+
+# ============================================================================
+# Community Report Schema
+# ============================================================================
+
+class CommunityFinding(BaseModel):
+ summary: str = Field(description="Summary of the insight")
+ explanation: str = Field(description="Detailed explanation of the insight")
+
+class CommunityReport(BaseModel):
+ """Represents a summary of a graph community."""
+
+ title: str = Field(description="Short title for the community")
+ summary: str = Field(description="Detailed executive summary of the community's structure and key entities")
+ rating: float = Field(description="Impact severity rating (0-10)")
+ findings: List[CommunityFinding] = Field(description="List of key findings")
+ # Optional fields (filled programmatically)
+ id: Optional[str] = Field(default=None, description="Community ID")
+ key_entities: Optional[List[str]] = Field(default_factory=list, description="List of key entities in this community")
+
+
+# ============================================================================
+# Extraction Prompts
+# ============================================================================
+
+Graph_RAG_NODE_EXTRACTION_PROMPT = """
+-Goal-
+Identify relevant entities from the text.
+Entities will serve as nodes in the knowledge graph.
+
+### Source Text:
+{source_text}
+"""
+
+Graph_RAG_EDGE_EXTRACTION_PROMPT = """
+-Goal-
+You are an expert knowledge graph extraction assistant.
+Extract binary "Edges" that represent relationships between exactly TWO entities.
+
+-Definition-
+An Edge represents a specific connection (action, relation, ownership, etc.) from a Source entity to a Target entity.
+
+-Constraints-
+1. You MUST ONLY use names from the "Allowed Entities" list provided below.
+2. Ensure every edge has exactly one Source and one Target.
+3. Provide a clear, comprehensive description for each edge.
+
+# Provided Entities
+{known_nodes}
+
+### Source Text:
+{source_text}
+"""
+
+COMMUNITY_REPORT_PROMPT = """
+You are an expert analyst.
+Given the following list of entities and relationships within a specific community, generate a comprehensive report.
+
+### Community Data:
+Entities: {entities}
+Relationships: {relationships}
+
+### Output Requirements:
+1. Title: A short name for the community.
+2. Summary: An executive summary of the community's structure and dynamics.
+3. Impact Rating: Score 0-10 based on importance/severity.
+4. Key Findings: 3-5 main insights. Each finding must include a "summary" and an "explanation".
+
+Output must strictly follow the JSON schema.
+"""
+
+GLOBAL_SEARCH_PROMPT = """
+You are an intelligent assistant capable of answering questions based *solely* on the provided community summaries.
+
+### Question:
+{query}
+
+### Community Summaries:
+{context}
+
+### Instructions:
+- Synthesize the information from the community summaries to answer the question.
+- Do not use outside knowledge.
+- If the answer includes multiple points, structure them clearly.
+- If the context is insufficient, state that you cannot answer.
+
+Answer:
+"""
+
+# ============================================================================
+# Merge Templates for LLM.CUSTOM_RULE Strategy
+# ============================================================================
+
+Graph_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.
+You will receive a list of objects representing the same Entity.
+
+Your task is to merge them into a SINGLE object exactly matching the schema.
+
+Merge strategy:
+1. **name/type**: Keep the most frequent or precise value.
+2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.
+"""
+
+Graph_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.
+You will receive a list of objects representing the same Relationship (Edge).
+
+Your task is to merge them into a SINGLE object exactly matching the schema.
+
+Merge strategy:
+1. **source/target**: Keep them identical to the inputs.
+2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.
+3. **keywords**: Combine keyword lists from all inputs, removing duplicates.
+4. **strength**: Calculate the average of the input strengths (round to nearest integer).
+"""
+
+# ============================================================================
+# Graph_RAG Class
+# ============================================================================
+
+
+class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):
+ """
+ Graph-RAG: Graph-based Retrieval-Augmented Generation
+
+ Extracts entity-relationships (binary edges) and supports advanced GraphRAG features:
+ - Community Detection (Leiden/Modularity)
+ - Community Reports (Summarization)
+ - Global Search (Map-Reduce over summaries)
+ - Local Search (Neighbor-based retrieval with community context)
+
+ Implements the architecture of Microsoft GraphRAG / Nano-GraphRAG.
+ """
+
+ def __init__(
+ self,
+ llm_client: BaseChatModel,
+ embedder: Embeddings,
+ chunk_size: int = 2048,
+ chunk_overlap: int = 256,
+ max_workers: int = 10,
+ verbose: bool = False,
+ ):
+ """
+ Initialize Graph_RAG engine.
+
+ Args:
+ llm_client (BaseChatModel): LLM client for text generation.
+ embedder (Embeddings): Embedding model for text embeddings.
+ chunk_size (int): Size of text chunks for indexing.
+ chunk_overlap (int): Overlap between text chunks for indexing.
+ max_workers (int): Maximum number of workers for indexing.
+ verbose (bool): Display detailed execution logs and progress information.
+ """
+ # 1. Define Key Extractors (critical for deduplication)
+ # Node key: exact match by name
+ node_key_fn = lambda x: x.name
+
+ # Edge key: source + target tuple
+ edge_key_fn = lambda x: (x.source, x.target)
+
+ # 2. Edge consistency check: tell AutoGraph which nodes this edge connects
+ nodes_in_edge_fn = lambda x: (x.source, x.target)
+
+ # 3. Create custom Mergers with LLM.CUSTOM_RULE strategy
+ # Node merger: merges entity descriptions using NODE_MERGE_RULE
+ node_merger = CustomRuleMerger(
+ key_extractor=node_key_fn,
+ llm_client=llm_client,
+ item_schema=NodeSchema,
+ rule=Graph_RAG_NODE_MERGE_RULE,
+ )
+
+ # Edge merger: merges relationship descriptions using EDGE_MERGE_RULE
+ edge_merger = CustomRuleMerger(
+ key_extractor=edge_key_fn,
+ llm_client=llm_client,
+ item_schema=EdgeSchema,
+ rule=Graph_RAG_EDGE_MERGE_RULE,
+ )
+
+ # 4. Call parent class initialization
+ super().__init__(
+ node_schema=NodeSchema,
+ edge_schema=EdgeSchema,
+ node_key_extractor=node_key_fn,
+ edge_key_extractor=edge_key_fn,
+ nodes_in_edge_extractor=nodes_in_edge_fn,
+ llm_client=llm_client,
+ embedder=embedder,
+ # Enforce two-stage extraction (nodes first, then edges)
+ extraction_mode="two_stage",
+ # Inject customized prompts for extraction
+ prompt_for_node_extraction=Graph_RAG_NODE_EXTRACTION_PROMPT,
+ prompt_for_edge_extraction=Graph_RAG_EDGE_EXTRACTION_PROMPT,
+ # Pass custom merger instances
+ node_strategy_or_merger=node_merger,
+ edge_strategy_or_merger=edge_merger,
+ # Optimize indexing
+ node_fields_for_index=["name", "type", "description"],
+ edge_fields_for_index=["keywords", "description"],
+ # Display labels
+ node_label_extractor=lambda x: x.name,
+ edge_label_extractor=lambda x: f"{x.description[:5]}...",
+ # Other parameters
+ chunk_size=chunk_size,
+ chunk_overlap=chunk_overlap,
+ max_workers=max_workers,
+ verbose=verbose,
+ )
+
+ # Community Management
+ self.community_reports: Dict[str, CommunityReport] = {}
+ self._community_graph: Optional[nx.Graph] = None
+ self._community_hierarchy: Dict[int, Dict[str, List[str]]] = {} # level -> {community_id: [node_names]}
+ self._node_to_community: Dict[str, Dict[str, Any]] = {} # node_name -> {level: community_id}
+
+ def dump(self, folder_path: str | Path) -> None:
+ """Saves graph state: internal data, community reports, and GraphML for visualization."""
+ root = Path(folder_path)
+
+ if self.verbose:
+ print("Saving Results...")
+ print("=" * 80)
+
+ # 1. Save standard graph data (nodes, edges, index)
+ super().dump(folder_path)
+
+ # 2. Save Community Data
+ community_data = {
+ "reports": {k: v.model_dump() for k, v in self.community_reports.items()},
+ "hierarchy": {str(k): v for k, v in self._community_hierarchy.items()},
+ "node_map": self._node_to_community
+ }
+
+ comm_path = root / "community_data.json"
+ with open(comm_path, "w", encoding="utf-8") as f:
+ json.dump(community_data, f, indent=2, ensure_ascii=False)
+
+ # 3. Export GraphML for visualization (e.g., Gephi)
+ try:
+ graphml_path = root / "graph.graphml"
+ G_export = nx.Graph()
+ for n in self.nodes:
+ G_export.add_node(n.name, label=n.name, type=n.type, description=n.description)
+ for e in self.edges:
+ G_export.add_edge(e.source, e.target, weight=float(e.strength), description=e.description)
+
+ nx.write_graphml(G_export, graphml_path)
+ except Exception as e:
+ if self.verbose:
+ print(f"Warning: Failed to save GraphML: {e}")
+
+ if self.verbose:
+ print(f"✓ Results saved to {root}/")
+
+ def load(self, folder_path: str | Path) -> None:
+ """Loads graph state from directory."""
+ root = Path(folder_path)
+
+ # 1. Load standard graph data
+ super().load(folder_path)
+
+ # 2. Rebuild NetworkX graph
+ self._community_graph = nx.Graph()
+ for n in self.nodes:
+ self._community_graph.add_node(n.name, **n.model_dump())
+ for e in self.edges:
+ self._community_graph.add_edge(e.source, e.target, weight=float(e.strength), **e.model_dump())
+
+ # 3. Load Community Data
+ comm_path = root / "community_data.json"
+ if comm_path.exists():
+ with open(comm_path, "r", encoding="utf-8") as f:
+ data = json.load(f)
+
+ self.community_reports = {
+ k: CommunityReport.model_validate(v) for k, v in data.get("reports", {}).items()
+ }
+ self._community_hierarchy = {
+ int(k) if k.isdigit() else k: v for k, v in data.get("hierarchy", {}).items()
+ }
+ self._node_to_community = data.get("node_map", {})
+
+ if self.verbose:
+ print(f"Loaded {len(self.community_reports)} community reports.")
+
+
+ # ============================================================================
+ # Community Detection & Summarization
+ # ============================================================================
+
+ def build_communities(self, level: int = 0):
+ """
+ Detects communities in the graph and generates reports for them.
+ Uses Leiden algorithm (via graspologic).
+
+ Args:
+ level: The hierarchical level for Leiden algorithm (default 0).
+ """
+ if self.empty():
+ if self.verbose:
+ print("Graph is empty, cannot build communities.")
+ return
+
+ if self.verbose:
+ print("Building graph topology...")
+
+ # Construct NetworkX graph
+ G = nx.Graph()
+ for node in self.nodes:
+ G.add_node(node.name, **node.model_dump())
+
+ for edge in self.edges:
+ # Use 'weight' if available, mapped from 'strength'
+ G.add_edge(edge.source, edge.target, weight=float(edge.strength), **edge.model_dump())
+
+ self._community_graph = G
+
+ if not HAS_GRASPOLOGIC:
+ if self.verbose:
+ print("Error: `graspologic` is not installed. Please install it to use Leiden community detection.")
+ return
+
+ # Community Detection Strategy
+ communities = []
+ community_algo = "Leiden"
+
+ if self.verbose:
+ print("Using Hierarchical Leiden algorithm (graspologic)...")
+
+ try:
+ # Create a subgraph without isolated nodes for Leiden to avoid warnings
+ G_leiden = G.copy()
+ isolates = list(nx.isolates(G_leiden))
+ G_leiden.remove_nodes_from(isolates)
+
+ if G_leiden.number_of_nodes() == 0:
+ if self.verbose:
+ print("No connected nodes found. Skipping community detection.")
+ return
+
+ # Hierarchical Leiden returns a list of CommunityAssignment objects
+ # Each assignment has (node, cluster, level)
+ hierarchical_clusters = hierarchical_leiden(G_leiden, max_cluster_size=100)
+
+ # Transform to our internal structure: level -> community_id -> [nodes]
+ self._community_hierarchy = {}
+ self._node_to_community = {}
+
+ # Group by level first
+ clusters_by_level = {}
+ for assignment in hierarchical_clusters:
+ lvl = assignment.level
+ cluster_id = f"LEVEL_{lvl}_CID_{assignment.cluster}"
+ node = assignment.node
+
+ if lvl not in clusters_by_level:
+ clusters_by_level[lvl] = {}
+ if cluster_id not in clusters_by_level[lvl]:
+ clusters_by_level[lvl][cluster_id] = []
+
+ clusters_by_level[lvl][cluster_id].append(node)
+
+ # Track node mapping
+ if node not in self._node_to_community:
+ self._node_to_community[node] = {}
+ self._node_to_community[node][f"level_{lvl}"] = cluster_id
+
+ self._community_hierarchy = clusters_by_level
+
+ # Select the requested level for reports
+ # In graspologic, level 0 is usually the top-most (coarsest)
+ if level in self._community_hierarchy:
+ communities = list(self._community_hierarchy[level].values())
+ community_algo = f"Leiden (Level {level})"
+ elif self._community_hierarchy:
+ # Fallback to the first available level if requested level doesn't exist
+ fallback_lvl = next(iter(self._community_hierarchy))
+ communities = list(self._community_hierarchy[fallback_lvl].values())
+ community_algo = f"Leiden (Fallback Level {fallback_lvl})"
+
+ except Exception as e:
+ if self.verbose:
+ print(f"Leiden algorithm failed: {e}.")
+ return
+
+ if self.verbose:
+ print(f"Detected {len(communities)} communities using {community_algo}.")
+
+ # Generate Reports
+ if self.verbose:
+ print("Generating community reports...")
+ self.community_reports = {}
+
+ chain = (
+ ChatPromptTemplate.from_template(COMMUNITY_REPORT_PROMPT)
+ | self.llm_client.with_structured_output(CommunityReport, method="function_calling")
+ )
+
+ for i, community_nodes in enumerate(communities):
+ cid = f"COMMUNITY_{i}"
+ # Gather context
+ subgraph_nodes = [n for n in self.nodes if n.name in community_nodes]
+ # Edges where *both* source and target are in the community (strict)
+ # Or edges where *at least one* is in the community (relaxed) -> tailored for coverage
+ subgraph_edges = [e for e in self.edges if e.source in community_nodes and e.target in community_nodes]
+
+ if not subgraph_nodes:
+ continue
+
+ # Format minimal context for LLM to save tokens
+ entities_txt = "\\n".join([f"- {n.name} ({n.type}): {n.description}" for n in subgraph_nodes])
+ relationships_txt = "\\n".join([f"- {e.source}->{e.target}: {e.description}" for e in subgraph_edges])
+
+ try:
+ report = chain.invoke({
+ "entities": entities_txt,
+ "relationships": relationships_txt
+ })
+ # Post-processing
+ report.id = cid
+ report.key_entities = list(community_nodes)[:10] # Track top entities
+ self.community_reports[cid] = report
+
+ # Check mapping for manual fallback methods
+ for node_name in community_nodes:
+ if node_name not in self._node_to_community:
+ self._node_to_community[node_name] = {}
+ self._node_to_community[node_name]["manual"] = cid
+
+ except Exception as e:
+ if self.verbose:
+ print(f"Failed to generate report for community {cid}: {e}")
+
+ if self.verbose:
+ print(f"Successfully generated {len(self.community_reports)} community reports.")
+
+ # ============================================================================
+ # Global Search
+ # ============================================================================
+
+ def global_search(self, query: str) -> str:
+ """
+ Global Search: Answering questions by synthesizing community summaries.
+ Best for: "What are the main themes?", "How does X affect Y generally?".
+ """
+ if not self.community_reports:
+ print("No community reports found. Building communities first...")
+ self.build_communities()
+
+ # Gather all community summaries
+ # For a production system, you would start with a "Map" phase (rate communities by relevance)
+ # Here we do a simple aggregation (suitable for small-medium graphs)
+
+ context_parts = []
+ for r in self.community_reports.values():
+ context_parts.append(f"## Community: {r.title} (Rating: {r.rating})\\nSummary: {r.summary}\\n")
+
+ full_context = "\\n".join(context_parts)
+
+ prompt = ChatPromptTemplate.from_template(GLOBAL_SEARCH_PROMPT)
+ chain = prompt | self.llm_client | StrOutputParser()
+
+ if self.verbose:
+ print("Executing Global Search...")
+
+ response = chain.invoke({"query": query, "context": full_context})
+ return response
+
+ # ============================================================================
+ # Local Search
+ # ============================================================================
+
+ def local_search(self, query: str, hops: int = 1) -> str:
+ """
+ Local Search: Focuses on specific entities mentioned in the query.
+ Best for: "What did X do?", "Who is related to Y?".
+
+ Using: Entity Vector Search -> Graph Traversal (BFS) -> Context window ingestion including Community Summaries.
+ """
+ # 1. Identify entities in query (Using Vector Search on Nodes)
+ self.build_index() # Ensure index exists
+ relevant_nodes = self.search_nodes(query, top_k=5)
+
+ if not relevant_nodes:
+ return "No relevant entities found in the graph for this query."
+
+ start_node_names = {n.name for n in relevant_nodes}
+ visited_nodes = set()
+
+ # 2. Traverse Graph (Manual BFS for 'hops')
+ if not self._community_graph:
+ G = nx.Graph()
+ for node in self.nodes:
+ G.add_node(node.name)
+ for edge in self.edges:
+ G.add_edge(edge.source, edge.target)
+ self._community_graph = G
+ else:
+ G = self._community_graph
+
+ current_layer = start_node_names
+
+ for _ in range(hops):
+ next_layer = set()
+ for node_name in current_layer:
+ if node_name in G:
+ neighbors = list(G.neighbors(node_name))
+ next_layer.update(neighbors)
+
+ visited_nodes.update(current_layer)
+ current_layer = next_layer
+
+ # 3. Retrieve detailed context for visited nodes and their edges
+ final_context_edges = [
+ e for e in self.edges
+ if e.source in visited_nodes or e.target in visited_nodes
+ ]
+
+ # Limit to reasonable context size
+ final_context_edges = final_context_edges[:50]
+ edges_txt = "\\n".join([f"- {e.source}->{e.target}: {e.description}" for e in final_context_edges])
+
+ # 4. (Enhanced) Retrieve Community Context for key nodes
+ community_context = []
+ seen_community_ids: Set[str] = set()
+
+ # Try to find which communities the start_node_names belong to
+ if self.community_reports:
+ # Method A: Use exact node-to-community mapping if available from build_communities
+ for node_name in start_node_names:
+ if node_name in self._node_to_community:
+ mapping = self._node_to_community[node_name]
+ # We might have multiple levels, pick any or all
+ for cid in mapping.values():
+ if cid in self.community_reports and cid not in seen_community_ids:
+ report = self.community_reports[cid]
+ community_context.append(f"Community: {report.title}\\nSummary: {report.summary}")
+ seen_community_ids.add(cid)
+
+ # Method B: Fallback if mapping missed (or inconsistent), check key_entities presence
+ for report in self.community_reports.values():
+ if report.id not in seen_community_ids and report.key_entities:
+ if set(report.key_entities) & start_node_names:
+ community_context.append(f"Community: {report.title}\\nSummary: {report.summary}")
+ seen_community_ids.add(report.id)
+
+ community_txt = "\\n\\n".join(community_context) if community_context else "No specific community context available."
+
+ # Generate Answer
+ prompt_content = f"""
+ Answer the query based on the local graph context provided.
+ Query: {query}
+
+ ### Community Context (High-level themes):
+ {community_txt}
+
+ ### Direct Relationships (Fact-based details):
+ {edges_txt}
+
+ Answer:
+ """
+
+ return self.llm_client.invoke(prompt_content).content
diff --git a/hyperextract/methods/registry.py b/hyperextract/methods/registry.py
index d1b2784..5df5257 100644
--- a/hyperextract/methods/registry.py
+++ b/hyperextract/methods/registry.py
@@ -98,9 +98,16 @@ def list_method_cfgs() -> Dict[str, MethodCfg]:
def _init_registry():
"""Initialize the registry with built-in methods."""
- from hyperextract.methods.rag import Light_RAG, Hyper_RAG, HyperGraph_RAG, Cog_RAG
+ from hyperextract.methods.rag import Light_RAG, Hyper_RAG, HyperGraph_RAG, Cog_RAG, Graph_RAG
from hyperextract.methods.typical import iText2KG, iText2KG_Star, KG_Gen, Atom
+ register_method(
+ name="graph_rag",
+ method_class=Graph_RAG,
+ autotype="graph",
+ description="Graph-RAG: Graph-based Retrieval-Augmented Generation with Community detection",
+ )
+
register_method(
name="light_rag",
method_class=Light_RAG,