Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
refactor(graph_rag): 重构社区检测和搜索功能
yifanfeng971cdbc74b360a
diff --git a/examples/method_graph_rag_demo.py b/examples/method_graph_rag_demo.pyindex 96a70e3..0527c5f 100644--- a/examples/method_graph_rag_demo.py+++ b/examples/method_graph_rag_demo.py@@ -208,29 +208,32 @@ if __name__ == "__main__":"Leona目前的状况如何?",]- # 1. Global Search Example+ # 1. Global Search Example (Community-Enhanced Search)print("\n" + "-" * 60)- print("[1. Global Search] (Broad Understanding / Map-Reduce)")+ print("[1. Community-Enhanced Search] (use_community=True)")print("-" * 60)for q in queries:print(f"\nQuery: {q}")print("Response:")try:- print(rag.global_search(q))+ nodes, edges, community_context = rag.search(q, use_community=True)+ if community_context:+ print("Community Summaries:")+ for summary in community_context.get("summaries", []):+ print(f" - [{summary['title']}] (rating: {summary['rating']})")+ print(f" {summary['summary'][:100]}...")+ if nodes:+ print(f"Top nodes: {[n.name for n in nodes[:3]]}")+ if edges:+ print(f"Top edges: {[e.description[:50] for e in edges[:3]]}")except Exception as e:print(f"Error: {e}")- # 2. Local Search Example+ # 2. Local Search Example (Not implemented yet)print("\n" + "-" * 60)- print("[2. Local Search] (Targeted Subgraph + Community Context)")+ print("[2. Local Search] (Targeted Subgraph + Community Context) - Not Implemented")print("-" * 60)- for q in queries:- print(f"\nQuery: {q}")- print("Response:")- try:- print(rag.local_search(q))- except Exception as e:- print(f"Error: {e}")+ print("Local search is not yet available. Use chat() for conversational queries.")# 3. Standard Edge Searchprint("\n" + "-" * 60)diff --git a/hyperextract/cli/cli.py b/hyperextract/cli/cli.pyindex 9709806..baa54ec 100644--- a/hyperextract/cli/cli.py+++ b/hyperextract/cli/cli.py@@ -322,10 +322,7 @@ def show(kb_path: str = typer.Argument(..., help="Knowledge base directory")):console.print("[bold blue]Visualizing with OntoSight...[/bold blue]")try:- kb.show(- node_label_extractor=lambda n: getattr(n, "name", str(n)),- edge_label_extractor=lambda e: getattr(e, "relation", str(e)),- )+ kb.show()except Exception as e:console.print(f"[red]Error during visualization:[/red] {e}")raise typer.Exit(1)diff --git a/hyperextract/methods/rag/graph_rag.py b/hyperextract/methods/rag/graph_rag.pyindex 75029e3..718ad8a 100644--- a/hyperextract/methods/rag/graph_rag.py+++ b/hyperextract/methods/rag/graph_rag.py@@ -5,8 +5,7 @@ Extracts and manages entity-relationship knowledge graphs with standard binary eimport jsonfrom pathlib import Path-from typing import List, Dict, Optional, Any, Set-import networkx as nx+from typing import List, Dict, Optional, Any, Tuplefrom pydantic import BaseModel, Fieldfrom langchain_core.embeddings import Embeddingsfrom langchain_core.language_models import BaseChatModel@@ -21,6 +20,18 @@ try:except ImportError:HAS_GRASPOLOGIC = False++def _ensure_networkx():+ """Lazy import networkx, raise helpful error if not installed."""+ try:+ import networkx as nx+ return nx+ except ImportError:+ raise ImportError(+ "networkx is required for community detection. "+ "Install with: pip install networkx"+ )+# ============================================================================# Node Schema# ============================================================================@@ -188,7 +199,6 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):- Community Detection (Leiden/Modularity)- Community Reports (Summarization)- Global Search (Map-Reduce over summaries)- - Local Search (Neighbor-based retrieval with community context)Implements the architecture of Microsoft GraphRAG / Nano-GraphRAG."""@@ -272,7 +282,7 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):# Community Managementself.community_reports: Dict[str, CommunityReport] = {}- self._community_graph: Optional[nx.Graph] = None+ self._community_graph: Optional[Any] = Noneself._community_hierarchy: Dict[int, Dict[str, List[str]]] = {} # level -> {community_id: [node_names]}self._node_to_community: Dict[str, Dict[str, Any]] = {} # node_name -> {level: community_id}@@ -287,54 +297,34 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):# 1. Save standard graph data (nodes, edges, index)super().dump(folder_path)- # 2. Save Community Data- community_data = {- "reports": {k: v.model_dump() for k, v in self.community_reports.items()},- "hierarchy": {str(k): v for k, v in self._community_hierarchy.items()},- "node_map": self._node_to_community- }-- comm_path = root / "community_data.json"- with open(comm_path, "w", encoding="utf-8") as f:- json.dump(community_data, f, indent=2, ensure_ascii=False)+ # 2. Save Community Data (only if exists)+ if self.community_reports:+ community_data = {+ "reports": {k: v.model_dump() for k, v in self.community_reports.items()},+ "hierarchy": {str(k): v for k, v in self._community_hierarchy.items()},+ "node_map": self._node_to_community+ }++ comm_path = root / "community_data.json"+ with open(comm_path, "w", encoding="utf-8") as f:+ json.dump(community_data, f, indent=2, ensure_ascii=False)- # 3. Export GraphML for visualization (e.g., Gephi)- try:- graphml_path = root / "graph.graphml"- G_export = nx.Graph()- for n in self.nodes:- G_export.add_node(n.name, label=n.name, type=n.type, description=n.description)- for e in self.edges:- G_export.add_edge(e.source, e.target, weight=float(e.strength), description=e.description)-- nx.write_graphml(G_export, graphml_path)- except Exception as e:- if self.verbose:- print(f"Warning: Failed to save GraphML: {e}")-if self.verbose:print(f"✓ Results saved to {root}/")def load(self, folder_path: str | Path) -> None:"""Loads graph state from directory."""root = Path(folder_path)-+# 1. Load standard graph datasuper().load(folder_path)- # 2. Rebuild NetworkX graph- self._community_graph = nx.Graph()- for n in self.nodes:- self._community_graph.add_node(n.name, **n.model_dump())- for e in self.edges:- self._community_graph.add_edge(e.source, e.target, weight=float(e.strength), **e.model_dump())-- # 3. Load Community Data+ # 2. Load Community Data (if exists)comm_path = root / "community_data.json"if comm_path.exists():with open(comm_path, "r", encoding="utf-8") as f:data = json.load(f)-+self.community_reports = {k: CommunityReport.model_validate(v) for k, v in data.get("reports", {}).items()}@@ -342,15 +332,25 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):int(k) if k.isdigit() else k: v for k, v in data.get("hierarchy", {}).items()}self._node_to_community = data.get("node_map", {})-+if self.verbose:print(f"Loaded {len(self.community_reports)} community reports.")-# ============================================================================# Community Detection & Summarization# ============================================================================+ def _ensure_community_graph(self):+ """Lazily build _community_graph from nodes and edges when needed."""+ if self._community_graph is None:+ nx = _ensure_networkx()+ self._community_graph = nx.Graph()+ for n in self.nodes:+ self._community_graph.add_node(n.name, **n.model_dump())+ for e in self.edges:+ self._community_graph.add_edge(e.source, e.target, weight=float(e.strength), **e.model_dump())+ return self._community_graph+def build_communities(self, level: int = 0):"""Detects communities in the graph and generates reports for them.@@ -364,6 +364,8 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):print("Graph is empty, cannot build communities.")return+ nx = _ensure_networkx()+if self.verbose:print("Building graph topology...")@@ -498,129 +500,71 @@ class Graph_RAG(AutoGraph[NodeSchema, EdgeSchema]):print(f"Successfully generated {len(self.community_reports)} community reports.")# ============================================================================- # Global Search+ # Search Override with Community Support# ============================================================================- def global_search(self, query: str) -> str:- """- Global Search: Answering questions by synthesizing community summaries.- Best for: "What are the main themes?", "How does X affect Y generally?".+ def search(+ self,+ query: str,+ top_k_nodes: int = 3,+ top_k_edges: int = 3,+ top_k: int | None = None,+ use_community: bool = False,+ ) -> Tuple[List, List, Optional[Dict]]:+ """Unified graph search interface with optional community enhancement.++ Args:+ query: Search query string.+ top_k_nodes: Number of node results to return (default: 3).+ top_k_edges: Number of edge results to return (default: 3).+ top_k: If provided, sets both top_k_nodes and top_k_edges to this value.+ use_community: If True, enables community-aware search.+ Requires networkx and graspologic. Default: False.++ Returns:+ Tuple of (nodes, edges, community_context).+ community_context is None when use_community=False."""+ if use_community:+ return self._global_search_impl(query, top_k_nodes, top_k_edges)+ return super().search(query, top_k_nodes, top_k_edges, top_k)++ def _global_search_impl(+ self,+ query: str,+ top_k_nodes: int = 3,+ top_k_edges: int = 3,+ ) -> Tuple[List, List, Dict]:+ """Internal implementation for community-enhanced search."""if not self.community_reports:- print("No community reports found. Building communities first...")+ if self.verbose:+ print("No community reports found. Building communities first...")self.build_communities()- # Gather all community summaries- # For a production system, you would start with a "Map" phase (rate communities by relevance)- # Here we do a simple aggregation (suitable for small-medium graphs)-- context_parts = []- for r in self.community_reports.values():- context_parts.append(f"## Community: {r.title} (Rating: {r.rating})\\nSummary: {r.summary}\\n")-- full_context = "\\n".join(context_parts)-- prompt = ChatPromptTemplate.from_template(GLOBAL_SEARCH_PROMPT)- chain = prompt | self.llm_client | StrOutputParser()-- if self.verbose:- print("Executing Global Search...")-- response = chain.invoke({"query": query, "context": full_context})- return response-- # ============================================================================- # Local Search- # ============================================================================-- def local_search(self, query: str, hops: int = 1) -> str:- """- Local Search: Focuses on specific entities mentioned in the query.- Best for: "What did X do?", "Who is related to Y?".-- Using: Entity Vector Search -> Graph Traversal (BFS) -> Context window ingestion including Community Summaries.- """- # 1. Identify entities in query (Using Vector Search on Nodes)- self.build_index() # Ensure index exists- relevant_nodes = self.search_nodes(query, top_k=5)-- if not relevant_nodes:- return "No relevant entities found in the graph for this query."-- start_node_names = {n.name for n in relevant_nodes}- visited_nodes = set()-- # 2. Traverse Graph (Manual BFS for 'hops')- if not self._community_graph:- G = nx.Graph()- for node in self.nodes:- G.add_node(node.name)- for edge in self.edges:- G.add_edge(edge.source, edge.target)- self._community_graph = G- else:- G = self._community_graph-- current_layer = start_node_names-- for _ in range(hops):- next_layer = set()- for node_name in current_layer:- if node_name in G:- neighbors = list(G.neighbors(node_name))- next_layer.update(neighbors)-- visited_nodes.update(current_layer)- current_layer = next_layer-- # 3. Retrieve detailed context for visited nodes and their edges- final_context_edges = [- e for e in self.edges- if e.source in visited_nodes or e.target in visited_nodes- ]-- # Limit to reasonable context size- final_context_edges = final_context_edges[:50]- edges_txt = "\\n".join([f"- {e.source}->{e.target}: {e.description}" for e in final_context_edges])+ community_context = self._get_community_context_for_query(query)+ nodes, edges = super().search(query, top_k_nodes, top_k_edges)- # 4. (Enhanced) Retrieve Community Context for key nodes- community_context = []- seen_community_ids: Set[str] = set()+ return nodes, edges, community_context- # Try to find which communities the start_node_names belong to- if self.community_reports:- # Method A: Use exact node-to-community mapping if available from build_communities- for node_name in start_node_names:- if node_name in self._node_to_community:- mapping = self._node_to_community[node_name]- # We might have multiple levels, pick any or all- for cid in mapping.values():- if cid in self.community_reports and cid not in seen_community_ids:- report = self.community_reports[cid]- community_context.append(f"Community: {report.title}\\nSummary: {report.summary}")- seen_community_ids.add(cid)-- # Method B: Fallback if mapping missed (or inconsistent), check key_entities presence- for report in self.community_reports.values():- if report.id not in seen_community_ids and report.key_entities:- if set(report.key_entities) & start_node_names:- community_context.append(f"Community: {report.title}\\nSummary: {report.summary}")- seen_community_ids.add(report.id)-- community_txt = "\\n\\n".join(community_context) if community_context else "No specific community context available."-- # Generate Answer- prompt_content = f"""- Answer the query based on the local graph context provided.- Query: {query}-- ### Community Context (High-level themes):- {community_txt}+ def _get_community_context_for_query(self, query: str) -> Dict:+ """Get community context related to the query."""+ if not self.community_reports:+ return {}++ sorted_reports = sorted(+ self.community_reports.values(),+ key=lambda r: r.rating,+ reverse=True+ )[:3]++ return {+ "summaries": [+ {"title": r.title, "summary": r.summary, "rating": r.rating}+ for r in sorted_reports+ ]+ }- ### Direct Relationships (Fact-based details):- {edges_txt}-- Answer:- """-- return self.llm_client.invoke(prompt_content).content+ # ============================================================================+ # Deprecated: Global Search functionality is now integrated into search(use_community=True)+ # Use kb.search(query, use_community=True) instead+ # ============================================================================