Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
undate cog_rag and graph_rag
haoohu64c3c0d02fdd
diff --git a/examples/method_graph_rag_demo.py b/examples/method_graph_rag_demo.pynew file mode 100644index 0000000..96a70e3--- /dev/null+++ b/examples/method_graph_rag_demo.py@@ -0,0 +1,260 @@+"""+Graph-RAG Demo: Entity-Relationship Extraction with Standard Binary Graphs++This demo demonstrates:+1. Extracting entities and binary relationships from a complex sci-fi story.+2. Analysis of standard graph relationships (Source -> Target).+3. Semantic Search over the constructed Knowledge Graph (Q&A).+"""++import os+import sys++# 添加项目根目录到路径+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))++from dotenv import load_dotenv+load_dotenv()++from langchain_openai import ChatOpenAI, OpenAIEmbeddings+from hyperextract.methods.rag import Graph_RAG++# ============================================================================+# Sample Story for Extraction (Shared with AutoHypergraph Demo)+# ============================================================================++STORY = """+【银河纪元 3042年 - 赤红星云事件报告】++危机在这一周达到了顶峰。整个银河系似乎都陷入了暗流涌动的政治漩涡。++【第一阶段:秘密接触】+臭名昭著的星际海盗黑蛇驾驶着他的旗舰复仇号,在泰坦空间站秘密会见了银河联邦的背叛者议员 Valerius。+目击者称,两人在午夜时分登上了空间站的顶层禁区。据情报部门推测,这是一次肮脏的交易——+双方达成了针对皇室的同盟关系。参与此会议的还有 Valerius 手下的特务头目 Shadow,+他负责处理所有不可告人的细节。这个同盟的形成标志着大麻烦即将到来。++【第二阶段:叛变与夺权】+在这次会议的两天后,事态急剧恶化。黑蛇这种疯狂的家伙竟然在极光星区伏击了皇家护卫队。+这场突然的冲突中,皇家护卫队的队长 Leona 与副队长 Marcus 奋力抵抗,+但他们陷入了黑蛇手下的精锐战士的包围。这场激战导致队长 Leona 身受重伤。+而黑蛇的目标不是为了杀戮,而是为了夺取一件神圣的物品——传说中的虚空水晶。+这枚水晶散发着幽蓝的光芒,据说能够打开先祖之门的第一道密封。+黑蛇在战斗中成功夺取了水晶,但 Leona 在临终前用最后的力量将其重新夺回。++【第三阶段:流亡与隐藏】+Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9。+在这个被遗忘的角落,她向仅有的几个信任者求救:还有副队长 Marcus 以及隐秘组织"地下抵抗军"的领导人 Cipher。+同时,议员 Valerius 非法持有了开启先祖之门的钥匙——这把钥匙本应被安全保管在皇家金库中。+他现在藏匿在浮空都市中,与黑蛇遥相呼应,计划着下一步的政变。++【第四阶段:势力重组】+银河联邦的最高委员会得知了这一系列事件,决定采取行动。+高级顾问 Artemis 被任命为特使,她与 Leona 在废土行星 Z-9 进行了秘密会晤,+讨论如何制止这场即将到来的危机。同时,地下抵抗军的 Cipher 也加入了这次会议,+三人共同制定了一个大胆的计划。++【第五阶段:最后的筹码】+虽然黑蛇未能获得虚空水晶,但他已经从 Valerius 那里获得了先祖之门钥匙的复制品。+根据情报,这个复制品目前被秘密运往到黑蛇的秘密基地——位于辐射荒漠中的要塞。+参与此次运送的还有 Shadow 和黑蛇手下的精锐战士群体。+与此同时,Leona 和 Marcus 正在准备一场反击,他们计划在虚空水晶的力量下重新夺回局势的控制权。++目前局势:+- 黑蛇:手持复制品钥匙,掌控旗舰复仇号,计划进行最终的攻击+- 议员 Valerius:躲在浮空都市中,与黑蛇保持联系+- Leona:守护虚空水晶,在废土行星 Z-9 养伤,准备反击+- 银河联邦:通过特使 Artemis 支持 Leona,正在调集力量+- 地下抵抗军:Cipher 领导,与 Leona 和 Marcus 联手,准备对抗黑蛇和 Valerius 的同盟++这场战争的走向仍不确定。虚空水晶、先祖之门、以及隐藏在这一切背后的秘密,+都将在接下来的日子里浮出水面。银河的命运,可能就在这些关键人物和物品的碰撞中决定。+"""+++if __name__ == "__main__":+ # ============================================================================+ # Initialize the Graph-RAG system with LLM and Embeddings+ # ============================================================================+ print("=" * 80)+ print("Initializing Graph-RAG System...")+ print("=" * 80)++ # Use a capable model for extraction+ llm_client = ChatOpenAI(+ model="gpt-4o-mini",+ extra_body={+ "system": "You are a helpful assistant. Always respond in Chinese."+ },+ temperature=0,+ )+ embedder = OpenAIEmbeddings(model="text-embedding-3-small")++ rag = Graph_RAG(+ llm_client=llm_client,+ embedder=embedder,+ verbose=True,+ chunk_size=1000,+ )++ # Feed text to the system+ # ============================================================================+ print("\nFeeding story to the system...")+ rag.feed_text(STORY)+ print("✓ Story processed successfully\n")++ # ============================================================================+ # Display Extracted Entities+ # ============================================================================+ print("=" * 80)+ print("EXTRACTED ENTITIES")+ print("=" * 80)++ for node in rag.nodes:+ print(f"\n【{node.name}】(Type: {node.type})")+ print(f" Description: {node.description}")++ print(f"\n✓ Total entities extracted: {len(rag.nodes)}\n")++ # ============================================================================+ # Display Extracted Edges+ # ============================================================================+ print("=" * 80)+ print("EXTRACTED EDGES (Binary Relationships)")+ print("=" * 80)++ for i, edge in enumerate(rag.edges, 1):+ print(f"\n{i}. {edge.source} -> {edge.target}")+ print(f" Description: {edge.description}")+ print(f" Keywords: {edge.keywords}")+ print(f" Strength: {edge.strength}/10")++ print(f"\n✓ Total edges extracted: {len(rag.edges)}\n")++ # ============================================================================+ # Statistics+ # ============================================================================+ print("=" * 80)+ print("STATISTICS")+ print("=" * 80)++ # Entity type distribution+ type_counts = {}+ for node in rag.nodes:+ type_counts[node.type] = type_counts.get(node.type, 0) + 1++ print("\nEntity Type Distribution:")+ for entity_type, count in sorted(type_counts.items()):+ print(f" {entity_type}: {count}")++ # Average edge strength+ if rag.edges:+ avg_strength = sum(e.strength for e in rag.edges) / len(rag.edges)+ print(f"\nAverage Edge Strength: {avg_strength:.2f}/10")++ # Centrality (Out-degree + In-degree)+ print("\nEntity Centrality (Degree):")+ centrality = {}+ for edge in rag.edges:+ centrality[edge.source] = centrality.get(edge.source, 0) + 1+ centrality[edge.target] = centrality.get(edge.target, 0) + 1++ for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[+ :10+ ]:+ print(f" {entity}: {count}")++ # ============================================================================+ # Build Semantic Index & Communities+ # ============================================================================+ print("\n" + "=" * 80)+ print("Building Semantic Index & Communities...")+ print("=" * 80)++ rag.build_index()+ print("✓ Semantic index built successfully")++ rag.build_communities()+ print("✓ Communities built successfully\n")++ # ============================================================================+ # Display Community Reports+ # ============================================================================+ print("=" * 80)+ print("COMMUNITY REPORTS")+ print("=" * 80)++ if not rag.community_reports:+ print("No community reports generated.")+ else:+ for cid, report in rag.community_reports.items():+ print(f"\n【{report.title}】 (ID: {report.id}, Rating: {report.rating}/10)")+ print(f" Summary: {report.summary}")+ print(f" Findings:")+ for finding in report.findings:+ print(f" - {finding.summary}")++ # ============================================================================+ # Q&A: Semantic Search over Graph+ # ============================================================================+ print("=" * 80)+ print("Q&A: COMPARISON (Global vs Local vs Standard)")+ print("=" * 80)++ queries = [+ "虚空水晶现在在哪里?",+ "黑蛇和议员Valerius有什么阴谋?",+ "谁参与了泰坦空间站的秘密会议?",+ "Leona目前的状况如何?",+ ]++ # 1. Global Search Example+ print("\n" + "-" * 60)+ print("[1. Global Search] (Broad Understanding / Map-Reduce)")+ print("-" * 60)+ for q in queries:+ print(f"\nQuery: {q}")+ print("Response:")+ try:+ print(rag.global_search(q))+ except Exception as e:+ print(f"Error: {e}")++ # 2. Local Search Example+ print("\n" + "-" * 60)+ print("[2. Local Search] (Targeted Subgraph + Community Context)")+ print("-" * 60)+ for q in queries:+ print(f"\nQuery: {q}")+ print("Response:")+ try:+ print(rag.local_search(q))+ except Exception as e:+ print(f"Error: {e}")++ # 3. Standard Edge Search+ print("\n" + "-" * 60)+ print("[3. Standard Edge Search] (Vector similarity only)")+ print("-" * 60)++ for i, query in enumerate(queries, 1):+ print(f"\nQuery: {query}")+ try:+ results = rag.search_edges(query, top_k=3)++ if results:+ for j, result in enumerate(results, 1):+ print(f" [{j}] {result.source} -> {result.target}: {result.description[:100]}...")+ else:+ print(" No relevant edges found.")+ except Exception as e:+ print(f" Search failed: {e}")++ # ============================================================================+ # Save Results+ # ============================================================================+ print("\n" + "=" * 80)+ print("SAVING RESULTS")+ print("=" * 80)++ rag.dump("temp/graph_rag_demo")diff --git a/hyperextract/methods/rag/__init__.py b/hyperextract/methods/rag/__init__.pyindex d55f995..ad0a501 100644--- a/hyperextract/methods/rag/__init__.py+++ b/hyperextract/methods/rag/__init__.py@@ -8,11 +8,13 @@ from .hyper_rag import Hyper_RAGfrom .hypergraph_rag import HyperGraph_RAGfrom .cog_rag import Cog_RAGfrom .light_rag import Light_RAG+from .graph_rag import Graph_RAG__all__ = ["Hyper_RAG","HyperGraph_RAG","Cog_RAG","Light_RAG",+ "Graph_RAG",]diff --git a/hyperextract/methods/rag/cog_rag.py b/hyperextract/methods/rag/cog_rag.pyindex 9c29d7e..89c6a3c 100644--- a/hyperextract/methods/rag/cog_rag.py+++ b/hyperextract/methods/rag/cog_rag.py@@ -3,6 +3,9 @@ COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System PatternExtracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities."""+import os+import json+from datetime import datetimefrom typing import Listfrom hashlib import md5from pydantic import BaseModel, Field@@ -354,6 +357,11 @@ class Cog_RAG:max_workers: int = 10,verbose: bool = False,):+ self.metadata = {+ "created_at": datetime.now(),+ "updated_at": datetime.now(),+ }+self.theme_layer = Cog_RAG_ThemeLayer(llm_client=llm_client,embedder=embedder,@@ -440,12 +448,75 @@ class Cog_RAG:return chain.invoke({"context": context, "question": query})def dump(self, folder_path: str):- """Save both extracted graphs."""- import os+ """Save both extracted graphs and metadata."""+ if not os.path.exists(folder_path):+ os.makedirs(folder_path)++ metadata_path = os.path.join(folder_path, "metadata.json")+ with open(metadata_path, "w", encoding="utf-8") as f:+ json.dump(self.metadata, f, indent=2, ensure_ascii=False, default=str)++ # 导出根目录 data.json 以供 CLI 合法性校验和 info 命令统计+ root_data = {+ "nodes": [n.model_dump() for n in self.nodes],+ "edges": [e.model_dump() for e in self.edges]+ }+ with open(os.path.join(folder_path, "data.json"), "w", encoding="utf-8") as f:+ json.dump(root_data, f, ensure_ascii=False, indent=2)self.theme_layer.dump(os.path.join(folder_path, "theme_layer"))self.detail_layer.dump(os.path.join(folder_path, "detail_layer"))+ def load(self, folder_path: str):+ """Load extracted graphs and metadata."""+ metadata_path = os.path.join(folder_path, "metadata.json")+ if os.path.exists(metadata_path):+ with open(metadata_path, "r", encoding="utf-8") as f:+ params = json.load(f)+ if isinstance(self.metadata, dict):+ self.metadata.update(params)+ else:+ self.metadata = params++ theme_path = os.path.join(folder_path, "theme_layer")+ if os.path.exists(theme_path) and os.path.exists(os.path.join(theme_path, "data.json")):+ self.theme_layer.load(theme_path)++ detail_path = os.path.join(folder_path, "detail_layer")+ if os.path.exists(detail_path) and os.path.exists(os.path.join(detail_path, "data.json")):+ self.detail_layer.load(detail_path)++ def show(+ self,+ node_label_extractor=None,+ edge_label_extractor=None,+ ):+ """Visualize the specified layer interactively."""+ print("\n[Cog_RAG] 这是一个双层图谱系统。请选择你想可视化的层级:")+ print("1. Detail Layer (细节层:具体实体及其关系) - [默认]")+ print("2. Theme Layer (主题层:宏观主题及其参与者)")++ try:+ choice = input(f"请输入选项 [1/2]: ").strip()+ except EOFError:+ choice = "1"++ layer_map = {+ "2": ("Theme Layer", self.theme_layer),+ "1": ("Detail Layer", self.detail_layer),+ }++ layer_name, layer_obj = layer_map.get(choice, layer_map["1"])++ print(f"Visualizing Cog_RAG ({layer_name})...")+ try:+ layer_obj.show(+ node_label_extractor=node_label_extractor,+ edge_label_extractor=edge_label_extractor,+ )+ except Exception as e:+ print(f"Error visualizing {layer_name}: {e}")+@propertydef nodes(self):"""Return combined unique nodes from both layers (simple aggregation)."""diff --git a/hyperextract/methods/rag/graph_rag.py b/hyperextract/methods/rag/graph_rag.pynew file mode 100644index 0000000..75029e3--- /dev/null+++ b/hyperextract/methods/rag/graph_rag.py@@ -0,0 +1,626 @@+"""+Graph-RAG: Graph-based Retrieval-Augmented Generation+Extracts and manages entity-relationship knowledge graphs with standard binary edges.+"""++import json+from pathlib import Path+from typing import List, Dict, Optional, Any, Set+import networkx as nx+from pydantic import BaseModel, Field+from langchain_core.embeddings import Embeddings+from langchain_core.language_models import BaseChatModel+from langchain_core.prompts import ChatPromptTemplate+from langchain_core.output_parsers import StrOutputParser+from hyperextract.types import AutoGraph+from ontomem.merger import CustomRuleMerger++try:+ from graspologic.partition import hierarchical_leiden+ HAS_GRASPOLOGIC = True+except ImportError:+ HAS_GRASPOLOGIC = False++# ============================================================================+# Node Schema+# ============================================================================+++class NodeSchema(BaseModel):+ """Represents an entity extracted from the source text."""++ name: str = Field(description="Name of the entity")+ type: str = Field(+ description="Entity type (person, organization, geo, event, role, concept, etc.)",+ )+ description: str = Field(+ description="Comprehensive description of the entity's attributes and activities",+ )+++# ============================================================================+# Edge Schema+# ============================================================================+++class EdgeSchema(BaseModel):+ """Represents a directed relationship between two entities."""++ source: str = Field(description="Name of the source entity")+ target: str = Field(description="Name of the target entity")+ description: str = Field(+ description="Detailed explanation of the relationship or event connection"+ )+ keywords: str = Field(+ description="Keywords summarizing the relationship (e.g., 'conflict, trade')",+ )+ strength: int = Field(+ ge=1,+ le=10,+ description="Numerical score indicating relationship strength (1-10)",+ )+++# ============================================================================+# Community Report Schema+# ============================================================================++class CommunityFinding(BaseModel):+ summary: str = Field(description="Summary of the insight")+ explanation: str = Field(description="Detailed explanation of the insight")++class CommunityReport(BaseModel):+ """Represents a summary of a graph community."""++ title: str = Field(description="Short title for the community")+ summary: str = Field(description="Detailed executive summary of the community's structure and key entities")+ rating: float = Field(description="Impact severity rating (0-10)")+ findings: List[CommunityFinding] = Field(description="List of key findings")+ # Optional fields (filled programmatically)+ id: Optional[str] = Field(default=None, description="Community ID")+ key_entities: Optional[List[str]] = Field(default_factory=list, description="List of key entities in this community")+++# ============================================================================+# Extraction Prompts+# ============================================================================++Graph_RAG_NODE_EXTRACTION_PROMPT = """+-Goal-+Identify relevant entities from the text.+Entities will serve as nodes in the knowledge graph.++### Source Text:+{source_text}+"""++Graph_RAG_EDGE_EXTRACTION_PROMPT = """+-Goal-+You are an expert knowledge graph extraction assistant.+Extract binary "Edges" that represent relationships between exactly TWO entities.++-Definition-+An Edge represents a specific connection (action, relation, ownership, etc.) from a Source entity to a Target entity.++-Constraints-+1. You MUST ONLY use names from the "Allowed Entities" list provided below.+2. Ensure every edge has exactly one Source and one Target.+3. Provide a clear, comprehensive description for each edge.++# Provided Entities+{known_nodes}++### Source Text:+{source_text}+"""++COMMUNITY_REPORT_PROMPT = """+You are an expert analyst.+Given the following list of entities and relationships within a specific community, generate a comprehensive report.++### Community Data:+Entities: {entities}+Relationships: {relationships}++### Output Requirements:+1. Title: A short name for the community.+2. Summary: An executive summary of the community's structure and dynamics.+3. Impact Rating: Score 0-10 based on importance/severity.+4. Key Findings: 3-5 main insights. Each finding must include a "summary" and an "explanation".++Output must strictly follow the JSON schema.+"""++GLOBAL_SEARCH_PROMPT = """+You are an intelligent assistant capable of answering questions based *solely* on the provided community summaries.++### Question:+{query}++### Community Summaries:+{context}++### Instructions:+- Synthesize the information from the community summaries to answer the question.+- Do not use outside knowledge.+- If the answer includes multiple points, structure them clearly.+- If the context is insufficient, state that you cannot answer.++Answer:+"""++# ============================================================================+# Merge Templates for LLM.CUSTOM_RULE Strategy+# ============================================================================++Graph_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.+You will receive a list of objects representing the same Entity.++Your task is to merge them into a SINGLE object exactly matching the schema.++Merge strategy:+1. **name/type**: Keep the most frequent or precise value.+2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.+"""++Graph_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.+You will receive a list of objects representing the same Relationship (Edge).++Your task is to merge them into a SINGLE object exactly matching the schema.++Merge strategy:+1. **source/target**: Keep them identical to the inputs.+2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.+3. **keywords**: Combine keyword lists from all inputs, removing duplicates.+4. **strength**: Calculate the average of the input strengths (round to nearest integer).+"""++# ============================================================================+# Graph_RAG Class+# ============================================================================+++class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):+ """+ Graph-RAG: Graph-based Retrieval-Augmented Generation++ Extracts entity-relationships (binary edges) and supports advanced GraphRAG features:+ - Community Detection (Leiden/Modularity)+ - Community Reports (Summarization)+ - Global Search (Map-Reduce over summaries)+ - Local Search (Neighbor-based retrieval with community context)++ Implements the architecture of Microsoft GraphRAG / Nano-GraphRAG.+ """++ def __init__(+ self,+ llm_client: BaseChatModel,+ embedder: Embeddings,+ chunk_size: int = 2048,+ chunk_overlap: int = 256,+ max_workers: int = 10,+ verbose: bool = False,+ ):+ """+ Initialize Graph_RAG engine.++ Args:+ llm_client (BaseChatModel): LLM client for text generation.+ embedder (Embeddings): Embedding model for text embeddings.+ chunk_size (int): Size of text chunks for indexing.+ chunk_overlap (int): Overlap between text chunks for indexing.+ max_workers (int): Maximum number of workers for indexing.+ verbose (bool): Display detailed execution logs and progress information.+ """+ # 1. Define Key Extractors (critical for deduplication)+ # Node key: exact match by name+ node_key_fn = lambda x: x.name++ # Edge key: source + target tuple+ edge_key_fn = lambda x: (x.source, x.target)++ # 2. Edge consistency check: tell AutoGraph which nodes this edge connects+ nodes_in_edge_fn = lambda x: (x.source, x.target)++ # 3. Create custom Mergers with LLM.CUSTOM_RULE strategy+ # Node merger: merges entity descriptions using NODE_MERGE_RULE+ node_merger = CustomRuleMerger(+ key_extractor=node_key_fn,+ llm_client=llm_client,+ item_schema=NodeSchema,+ rule=Graph_RAG_NODE_MERGE_RULE,+ )++ # Edge merger: merges relationship descriptions using EDGE_MERGE_RULE+ edge_merger = CustomRuleMerger(+ key_extractor=edge_key_fn,+ llm_client=llm_client,+ item_schema=EdgeSchema,+ rule=Graph_RAG_EDGE_MERGE_RULE,+ )++ # 4. Call parent class initialization+ super().__init__(+ node_schema=NodeSchema,+ edge_schema=EdgeSchema,+ node_key_extractor=node_key_fn,+ edge_key_extractor=edge_key_fn,+ nodes_in_edge_extractor=nodes_in_edge_fn,+ llm_client=llm_client,+ embedder=embedder,+ # Enforce two-stage extraction (nodes first, then edges)+ extraction_mode="two_stage",+ # Inject customized prompts for extraction+ prompt_for_node_extraction=Graph_RAG_NODE_EXTRACTION_PROMPT,+ prompt_for_edge_extraction=Graph_RAG_EDGE_EXTRACTION_PROMPT,+ # Pass custom merger instances+ node_strategy_or_merger=node_merger,+ edge_strategy_or_merger=edge_merger,+ # Optimize indexing+ node_fields_for_index=["name", "type", "description"],+ edge_fields_for_index=["keywords", "description"],+ # Display labels+ node_label_extractor=lambda x: x.name,+ edge_label_extractor=lambda x: f"{x.description[:5]}...",+ # Other parameters+ chunk_size=chunk_size,+ chunk_overlap=chunk_overlap,+ max_workers=max_workers,+ verbose=verbose,+ )++ # Community Management+ self.community_reports: Dict[str, CommunityReport] = {}+ self._community_graph: Optional[nx.Graph] = None+ self._community_hierarchy: Dict[int, Dict[str, List[str]]] = {} # level -> {community_id: [node_names]}+ self._node_to_community: Dict[str, Dict[str, Any]] = {} # node_name -> {level: community_id}++ def dump(self, folder_path: str | Path) -> None:+ """Saves graph state: internal data, community reports, and GraphML for visualization."""+ root = Path(folder_path)++ if self.verbose:+ print("Saving Results...")+ print("=" * 80)++ # 1. Save standard graph data (nodes, edges, index)+ super().dump(folder_path)++ # 2. Save Community Data+ community_data = {+ "reports": {k: v.model_dump() for k, v in self.community_reports.items()},+ "hierarchy": {str(k): v for k, v in self._community_hierarchy.items()},+ "node_map": self._node_to_community+ }++ comm_path = root / "community_data.json"+ with open(comm_path, "w", encoding="utf-8") as f:+ json.dump(community_data, f, indent=2, ensure_ascii=False)++ # 3. Export GraphML for visualization (e.g., Gephi)+ try:+ graphml_path = root / "graph.graphml"+ G_export = nx.Graph()+ for n in self.nodes:+ G_export.add_node(n.name, label=n.name, type=n.type, description=n.description)+ for e in self.edges:+ G_export.add_edge(e.source, e.target, weight=float(e.strength), description=e.description)++ nx.write_graphml(G_export, graphml_path)+ except Exception as e:+ if self.verbose:+ print(f"Warning: Failed to save GraphML: {e}")++ if self.verbose:+ print(f"✓ Results saved to {root}/")++ def load(self, folder_path: str | Path) -> None:+ """Loads graph state from directory."""+ root = Path(folder_path)++ # 1. Load standard graph data+ super().load(folder_path)++ # 2. Rebuild NetworkX graph+ self._community_graph = nx.Graph()+ for n in self.nodes:+ self._community_graph.add_node(n.name, **n.model_dump())+ for e in self.edges:+ self._community_graph.add_edge(e.source, e.target, weight=float(e.strength), **e.model_dump())++ # 3. Load Community Data+ comm_path = root / "community_data.json"+ if comm_path.exists():+ with open(comm_path, "r", encoding="utf-8") as f:+ data = json.load(f)++ self.community_reports = {+ k: CommunityReport.model_validate(v) for k, v in data.get("reports", {}).items()+ }+ self._community_hierarchy = {+ int(k) if k.isdigit() else k: v for k, v in data.get("hierarchy", {}).items()+ }+ self._node_to_community = data.get("node_map", {})++ if self.verbose:+ print(f"Loaded {len(self.community_reports)} community reports.")+++ # ============================================================================+ # Community Detection & Summarization+ # ============================================================================++ def build_communities(self, level: int = 0):+ """+ Detects communities in the graph and generates reports for them.+ Uses Leiden algorithm (via graspologic).++ Args:+ level: The hierarchical level for Leiden algorithm (default 0).+ """+ if self.empty():+ if self.verbose:+ print("Graph is empty, cannot build communities.")+ return++ if self.verbose:+ print("Building graph topology...")++ # Construct NetworkX graph+ G = nx.Graph()+ for node in self.nodes:+ G.add_node(node.name, **node.model_dump())++ for edge in self.edges:+ # Use 'weight' if available, mapped from 'strength'+ G.add_edge(edge.source, edge.target, weight=float(edge.strength), **edge.model_dump())++ self._community_graph = G++ if not HAS_GRASPOLOGIC:+ if self.verbose:+ print("Error: `graspologic` is not installed. Please install it to use Leiden community detection.")+ return++ # Community Detection Strategy+ communities = []+ community_algo = "Leiden"++ if self.verbose:+ print("Using Hierarchical Leiden algorithm (graspologic)...")++ try:+ # Create a subgraph without isolated nodes for Leiden to avoid warnings+ G_leiden = G.copy()+ isolates = list(nx.isolates(G_leiden))+ G_leiden.remove_nodes_from(isolates)++ if G_leiden.number_of_nodes() == 0:+ if self.verbose:+ print("No connected nodes found. Skipping community detection.")+ return++ # Hierarchical Leiden returns a list of CommunityAssignment objects+ # Each assignment has (node, cluster, level)+ hierarchical_clusters = hierarchical_leiden(G_leiden, max_cluster_size=100)++ # Transform to our internal structure: level -> community_id -> [nodes]+ self._community_hierarchy = {}+ self._node_to_community = {}++ # Group by level first+ clusters_by_level = {}+ for assignment in hierarchical_clusters:+ lvl = assignment.level+ cluster_id = f"LEVEL_{lvl}_CID_{assignment.cluster}"+ node = assignment.node++ if lvl not in clusters_by_level:+ clusters_by_level[lvl] = {}+ if cluster_id not in clusters_by_level[lvl]:+ clusters_by_level[lvl][cluster_id] = []++ clusters_by_level[lvl][cluster_id].append(node)++ # Track node mapping+ if node not in self._node_to_community:+ self._node_to_community[node] = {}+ self._node_to_community[node][f"level_{lvl}"] = cluster_id++ self._community_hierarchy = clusters_by_level++ # Select the requested level for reports+ # In graspologic, level 0 is usually the top-most (coarsest)+ if level in self._community_hierarchy:+ communities = list(self._community_hierarchy[level].values())+ community_algo = f"Leiden (Level {level})"+ elif self._community_hierarchy:+ # Fallback to the first available level if requested level doesn't exist+ fallback_lvl = next(iter(self._community_hierarchy))+ communities = list(self._community_hierarchy[fallback_lvl].values())+ community_algo = f"Leiden (Fallback Level {fallback_lvl})"++ except Exception as e:+ if self.verbose:+ print(f"Leiden algorithm failed: {e}.")+ return++ if self.verbose:+ print(f"Detected {len(communities)} communities using {community_algo}.")++ # Generate Reports+ if self.verbose:+ print("Generating community reports...")+ self.community_reports = {}++ chain = (+ ChatPromptTemplate.from_template(COMMUNITY_REPORT_PROMPT)+ | self.llm_client.with_structured_output(CommunityReport, method="function_calling")+ )++ for i, community_nodes in enumerate(communities):+ cid = f"COMMUNITY_{i}"+ # Gather context+ subgraph_nodes = [n for n in self.nodes if n.name in community_nodes]+ # Edges where *both* source and target are in the community (strict)+ # Or edges where *at least one* is in the community (relaxed) -> tailored for coverage+ subgraph_edges = [e for e in self.edges if e.source in community_nodes and e.target in community_nodes]++ if not subgraph_nodes:+ continue++ # Format minimal context for LLM to save tokens+ entities_txt = "\\n".join([f"- {n.name} ({n.type}): {n.description}" for n in subgraph_nodes])+ relationships_txt = "\\n".join([f"- {e.source}->{e.target}: {e.description}" for e in subgraph_edges])++ try:+ report = chain.invoke({+ "entities": entities_txt,+ "relationships": relationships_txt+ })+ # Post-processing+ report.id = cid+ report.key_entities = list(community_nodes)[:10] # Track top entities+ self.community_reports[cid] = report++ # Check mapping for manual fallback methods+ for node_name in community_nodes:+ if node_name not in self._node_to_community:+ self._node_to_community[node_name] = {}+ self._node_to_community[node_name]["manual"] = cid++ except Exception as e:+ if self.verbose:+ print(f"Failed to generate report for community {cid}: {e}")++ if self.verbose:+ print(f"Successfully generated {len(self.community_reports)} community reports.")++ # ============================================================================+ # Global Search+ # ============================================================================++ def global_search(self, query: str) -> str:+ """+ Global Search: Answering questions by synthesizing community summaries.+ Best for: "What are the main themes?", "How does X affect Y generally?".+ """+ if not self.community_reports:+ print("No community reports found. Building communities first...")+ self.build_communities()++ # Gather all community summaries+ # For a production system, you would start with a "Map" phase (rate communities by relevance)+ # Here we do a simple aggregation (suitable for small-medium graphs)++ context_parts = []+ for r in self.community_reports.values():+ context_parts.append(f"## Community: {r.title} (Rating: {r.rating})\\nSummary: {r.summary}\\n")++ full_context = "\\n".join(context_parts)++ prompt = ChatPromptTemplate.from_template(GLOBAL_SEARCH_PROMPT)+ chain = prompt | self.llm_client | StrOutputParser()++ if self.verbose:+ print("Executing Global Search...")++ response = chain.invoke({"query": query, "context": full_context})+ return response++ # ============================================================================+ # Local Search+ # ============================================================================++ def local_search(self, query: str, hops: int = 1) -> str:+ """+ Local Search: Focuses on specific entities mentioned in the query.+ Best for: "What did X do?", "Who is related to Y?".++ Using: Entity Vector Search -> Graph Traversal (BFS) -> Context window ingestion including Community Summaries.+ """+ # 1. Identify entities in query (Using Vector Search on Nodes)+ self.build_index() # Ensure index exists+ relevant_nodes = self.search_nodes(query, top_k=5)++ if not relevant_nodes:+ return "No relevant entities found in the graph for this query."++ start_node_names = {n.name for n in relevant_nodes}+ visited_nodes = set()++ # 2. Traverse Graph (Manual BFS for 'hops')+ if not self._community_graph:+ G = nx.Graph()+ for node in self.nodes:+ G.add_node(node.name)+ for edge in self.edges:+ G.add_edge(edge.source, edge.target)+ self._community_graph = G+ else:+ G = self._community_graph++ current_layer = start_node_names++ for _ in range(hops):+ next_layer = set()+ for node_name in current_layer:+ if node_name in G:+ neighbors = list(G.neighbors(node_name))+ next_layer.update(neighbors)++ visited_nodes.update(current_layer)+ current_layer = next_layer++ # 3. Retrieve detailed context for visited nodes and their edges+ final_context_edges = [+ e for e in self.edges+ if e.source in visited_nodes or e.target in visited_nodes+ ]++ # Limit to reasonable context size+ final_context_edges = final_context_edges[:50]+ edges_txt = "\\n".join([f"- {e.source}->{e.target}: {e.description}" for e in final_context_edges])++ # 4. (Enhanced) Retrieve Community Context for key nodes+ community_context = []+ seen_community_ids: Set[str] = set()++ # Try to find which communities the start_node_names belong to+ if self.community_reports:+ # Method A: Use exact node-to-community mapping if available from build_communities+ for node_name in start_node_names:+ if node_name in self._node_to_community:+ mapping = self._node_to_community[node_name]+ # We might have multiple levels, pick any or all+ for cid in mapping.values():+ if cid in self.community_reports and cid not in seen_community_ids:+ report = self.community_reports[cid]+ community_context.append(f"Community: {report.title}\\nSummary: {report.summary}")+ seen_community_ids.add(cid)++ # Method B: Fallback if mapping missed (or inconsistent), check key_entities presence+ for report in self.community_reports.values():+ if report.id not in seen_community_ids and report.key_entities:+ if set(report.key_entities) & start_node_names:+ community_context.append(f"Community: {report.title}\\nSummary: {report.summary}")+ seen_community_ids.add(report.id)++ community_txt = "\\n\\n".join(community_context) if community_context else "No specific community context available."++ # Generate Answer+ prompt_content = f"""+ Answer the query based on the local graph context provided.+ Query: {query}++ ### Community Context (High-level themes):+ {community_txt}++ ### Direct Relationships (Fact-based details):+ {edges_txt}++ Answer:+ """++ return self.llm_client.invoke(prompt_content).contentdiff --git a/hyperextract/methods/registry.py b/hyperextract/methods/registry.pyindex d1b2784..5df5257 100644--- a/hyperextract/methods/registry.py+++ b/hyperextract/methods/registry.py@@ -98,9 +98,16 @@ def list_method_cfgs() -> Dict[str, MethodCfg]:def _init_registry():"""Initialize the registry with built-in methods."""- from hyperextract.methods.rag import Light_RAG, Hyper_RAG, HyperGraph_RAG, Cog_RAG+ from hyperextract.methods.rag import Light_RAG, Hyper_RAG, HyperGraph_RAG, Cog_RAG, Graph_RAGfrom hyperextract.methods.typical import iText2KG, iText2KG_Star, KG_Gen, Atom+ register_method(+ name="graph_rag",+ method_class=Graph_RAG,+ autotype="graph",+ description="Graph-RAG: Graph-based Retrieval-Augmented Generation with Community detection",+ )+register_method(name="light_rag",method_class=Light_RAG,