Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.
update cog-rag
haoohuc44dc3187e15
diff --git a/examples/cog_rag_demo.py b/examples/cog_rag_demo.pyindex 79952f8..5c0ef0f 100644--- a/examples/cog_rag_demo.py+++ b/examples/cog_rag_demo.py@@ -2,32 +2,30 @@Cog_RAG Demo: Theme-based Knowledge Extraction + Entity-Relation Detail RetrievalThis demo demonstrates the unified Cog_RAG System Wrapper:-1. **Layer 1 (Theme Layer)**: Extracts Key Entities and Themes (Narrative Arcs).-2. **Layer 2 (Detail Layer)**: Extracts Entities and Semantic Relationships.+1. **Macro Layer (Theme Layer)**: Extracts Narrative Arcs (Themes) and their Participant Entities.+2. **Micro Layer (Detail Layer)**: Extracts Entities and their Semantic Relationships.3. **Process**:- Feeds the same story to the Unified System.- - Performs a "Theme-First" retrieval process automatically.+ - The system automatically distributes extraction tasks to both layers.+ - Performs a Dual-Layer Chat/Search."""import osimport sys-# 添加项目根目录到路径+# Add project root to pathsys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))-from langchain_openai import ChatOpenAI, OpenAIEmbeddings-# Try to load dotenv, ignoring if not installed-try:- from dotenv import load_dotenv- load_dotenv()-except ImportError:- pass-from hyperextract.hypergraphs import Cog_RAG+from dotenv import load_dotenv+load_dotenv()++from langchain_openai import ChatOpenAI, OpenAIEmbeddings+from hyperextract.hypergraphs.cog_rag import Cog_RAG# ============================================================================-# Sample Story for Extraction (Shared with AutoHypergraph Demo)+# Sample Story for Extraction# ============================================================================STORY = """@@ -81,12 +79,13 @@ Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9if __name__ == "__main__":# ============================================================================- # Initialize the CogRAG system with LLM and Embeddings+ # Initialize the Cog_RAG system# ============================================================================print("=" * 80)- print("Initializing Unified CogRAG System...")+ print("Initializing Unified Cog_RAG System...")print("=" * 80)+ # Use a capable model for extractionllm_client = ChatOpenAI(model="gpt-4o-mini",extra_body={@@ -100,97 +99,74 @@ if __name__ == "__main__":rag = Cog_RAG(llm_client=llm_client,embedder=embedder,- verbose=True,+ chunk_size=1000,+ verbose=True)# ============================================================================# Feed text to the system# ============================================================================- print("\nFeeding story to the system...")+ print("\nFeeding story to the Cog_RAG System (Dual-Layer Extraction)...")rag.feed_text(STORY)print("✓ Story processed successfully\n")# ============================================================================- # Display Extracted Entities+ # Display Extracted Data (Unified View)# ============================================================================print("=" * 80)- print("EXTRACTED ENTITIES (Both Layers)")+ print("EXTRACTED KNOWLEDGE")print("=" * 80)- # Note: Cog_RAG.nodes property aggregates nodes from both layers- nodes = rag.nodes- sorted_nodes = sorted(nodes, key=lambda x: x.name)-- for node in sorted_nodes:- print(f"\n【{node.name}】(Type: {node.type})")- print(f" Description: {node.description}")-- print(f"\n✓ Total unique entities extracted: {len(nodes)}\n")-- # ============================================================================- # Display Extracted Hyperedges (Separate Layers)- # ============================================================================- print("=" * 80)- print("EXTRACTED HYPEREDGES (Dual Layer)")- print("=" * 80)-- print(f"\n--- Layer 1: Theme Hyperedges (Macro) ---")+ # 1. Macro Themes+ print("\n--- MACRO LAYER: Themes (Narrative Arcs) ---")for i, edge in enumerate(rag.theme_layer.edges, 1):- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")- print(f" Theme: {edge.description}")-- print(f"\n--- Layer 2: Relation Hyperedges (Micro) ---")- for i, edge in enumerate(rag.detail_layer.edges, 1):- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")- print(f" Description: {edge.description}")- keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"- print(f" Keywords: {keywords_str}")- print(f" Strength: {edge.strength}/10")-- print(- f"\n✓ Total edges extracted: {len(rag.edges)} ({len(rag.theme_layer.edges)} themes, {len(rag.detail_layer.edges)} relations)\n"- )+ print(f"\nTheme {i}: {edge.description}")+ print(f" Participants: {', '.join(edge.participants)}")- # ============================================================================- # Statistics- # ============================================================================- print("=" * 80)- print("STATISTICS")- print("=" * 80)+ print(f"\n✓ Total Themes extracted: {len(rag.theme_layer.edges)}")- # Entity type distribution- type_counts = {}- for node in rag.nodes:- type_counts[node.type] = type_counts.get(node.type, 0) + 1+ # 2. Micro Relations+ print(f"\n--- MICRO LAYER: Entity Relations ---")++ low_order_edges = [e for e in rag.detail_layer.edges if len(e.participants) == 2]+ high_order_edges = [e for e in rag.detail_layer.edges if len(e.participants) > 2]- print("\nEntity Type Distribution:")- for entity_type, count in sorted(type_counts.items()):- print(f" {entity_type}: {count}")+ print(f"\nMicro Relations (Pairwise):")+ for i, edge in enumerate(low_order_edges[:5], 1): # Show top 5+ print(f"{i}. {edge.description} (Entities: {', '.join(edge.participants)})")+ if len(low_order_edges) > 5:+ print(f"... and {len(low_order_edges) - 5} more.")- # Centrality (number of relationships/themes per entity)- print("\nEntity Participation (Centrality):")- centrality = {}- for edge in rag.edges:- for participant in edge.participants:- centrality[participant] = centrality.get(participant, 0) + 1+ print(f"\nMicro Relations (Hyperedges):")+ for i, edge in enumerate(high_order_edges[:5], 1): # Show top 5+ print(f"{i}. {edge.description} (Entities: {', '.join(edge.participants)})")+ if len(high_order_edges) > 5:+ print(f"... and {len(high_order_edges) - 5} more.")- for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[- :10- ]:- print(f" {entity}: {count}")+ print(f"\n✓ Total Micro Relations extracted: {len(rag.detail_layer.edges)}")++ # 3. Entities+ print(f"\n--- UNIQUE ENTITIES (Combined) ---")+ print(f"Total Unique Entities: {len(rag.nodes)}")++ # Display top 5 entities+ for i, node in enumerate(rag.nodes[:5], 1):+ print(f"{i}. 【{node.name}】 ({node.type})")+ if len(rag.nodes) > 5:+ print(f"... and {len(rag.nodes) - 5} more.")# ============================================================================- # Build Semantic Index for Q&A+ # Build Semantic Index# ============================================================================print("\n" + "=" * 80)- print("Building Semantic Index...")+ print("Building Semantic Indexes...")print("=" * 80)rag.build_index()- print("✓ Semantic index built successfully\n")+ print("✓ Semantic indexes built successfully\n")# ============================================================================- # Q&A: Semantic Search over Hypergraph+ # Q&A: Dual-Layer Search# ============================================================================print("=" * 80)print("Q&A: DUAL-LAYER SEMANTIC SEARCH")@@ -206,48 +182,38 @@ if __name__ == "__main__":for i, query in enumerate(queries, 1):print(f"\n【问题 {i}】{query}")print("-" * 60)++ # Search BOTH layers+ results = rag.search(query, top_k_themes=2, top_k_entities=2)++ print(" [Macro Themes Found]:")+ if results['themes']:+ for t in results['themes']:+ print(f" - {t.description}")+ else:+ print(" (None)")- try:- # Cog_RAG returns a dict with 'themes' and 'entities'- results = rag.search(query, top_k_themes=3, top_k_entities=3)-- print(" [Layer 1: Macro Themes]")- if results['themes']:- for j, theme in enumerate(results['themes'], 1):- print(f" {j}. {theme.description}")- else:- print(" (No relevant themes found)")-- print("\n [Layer 2: Micro Entities]")- if results['entities']:- for j, entity in enumerate(results['entities'], 1):- print(f" {j}. [{entity.name}] {entity.description[:100]}...")- else:- print(" (No relevant entities found)")+ print(" [Micro Entities Found]:")+ if results['entities']:+ for e in results['entities']:+ print(f" - {e.name}: {e.description[:50]}...")+ else:+ print(" (None)")- except Exception as e:- print(f" Search error: {str(e)}")# ============================================================================- # Chat with CogRAG+ # Chat with Cog_RAG# ============================================================================print("\n" + "=" * 80)- print("💬 Interactive Chat with CogRAG")+ print("💬 Interactive Chat with Cog_RAG")print("=" * 80)- print("Engaging in multi-turn dialogue based on extracted knowledge...\n")-- chat_queries = [- "虚空水晶现在在哪里?",- "黑蛇和议员Valerius有什么阴谋?",- "谁参与了泰坦空间站的秘密会议?",- "Leona目前的状况如何?",- ]+ print("Engaging in multi-turn dialogue based on Dual-Layer Knowledge...\n")- for q in chat_queries:+ for q in queries:print(f"❓ User: {q}")try:response = rag.chat(q)- print(f"🤖 CogRAG: {response.content}\n")+ print(f"🤖 Cog_RAG: {response.content}\n")except Exception as e:print(f"⚠️ Chat error: {e}\n")@@ -260,6 +226,9 @@ if __name__ == "__main__":try:output_dir = "temp/cog_rag_demo"+ if not os.path.exists(output_dir):+ os.makedirs(output_dir)+rag.dump(output_dir)print(f"✓ Results saved to {output_dir}/")diff --git a/hyperextract/hypergraphs/__init__.py b/hyperextract/hypergraphs/__init__.pyindex c801ad0..1c0ff6d 100644--- a/hyperextract/hypergraphs/__init__.py+++ b/hyperextract/hypergraphs/__init__.py@@ -1,11 +1,11 @@from .base import AutoHypergraphfrom .hyper_rag import Hyper_RAGfrom .hypergraph_rag import HyperGraph_RAG-# from .cog_rag import Cog_RAG+from .cog_rag import Cog_RAG__all__ = ["AutoHypergraph","Hyper_RAG","HyperGraph_RAG",- # "Cog_RAG",+ "Cog_RAG",]diff --git a/hyperextract/hypergraphs/cog_rag.py b/hyperextract/hypergraphs/cog_rag.pyindex 525ef1c..4836114 100644--- a/hyperextract/hypergraphs/cog_rag.py+++ b/hyperextract/hypergraphs/cog_rag.py@@ -3,10 +3,10 @@ COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System PatternExtracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities."""-from typing import List, Type, Callable, Tuple, Any-from pydantic import BaseModel, Field, create_model+from typing import List, Tuple+from hashlib import md5+from pydantic import BaseModel, Fieldfrom hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchema-from hyperextract.utils.logging import loggerfrom langchain_core.prompts import ChatPromptTemplatefrom langchain_core.embeddings import Embeddingsfrom langchain_core.language_models import BaseChatModel@@ -31,7 +31,7 @@ class NodeSchema(BaseModel):# ============================================================================-# Edge Schema (Theme)+# Theme Schemas (Layer 1: Macro Narrative)# ============================================================================@@ -46,8 +46,27 @@ class ThemeSchema(BaseModel):)+class TEMP_ThemeSchema(BaseModel):+ """Temporary schema for extracting Themes with nested Entities."""++ description: str = Field(+ description="A sentence that describes the primary theme, reflecting the main conflict, resolution, or key message"+ )+ participants: List[NodeSchema] = Field(+ description="List of key entities involved in this theme, with their details."+ )+++class TEMP_ThemeListSchema(BaseModel):+ """List of extracted themes."""++ items: List[TEMP_ThemeSchema] = Field(+ description="List of extracted themes and their participants."+ )++# ============================================================================-# Edge Schema (Entity Relation)+# Edge Schema (Layer 2: Micro Entity Relation)# ============================================================================@@ -70,44 +89,28 @@ class EdgeSchema(BaseModel):)-class EdgeDescriptionList(BaseModel):- """Intermediate schema for identifying edge concepts first."""-- items: List[str] = Field(- description="List of primary edge concepts/themes identified"- )--# ============================================================================# Extraction Prompts# ============================================================================--DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal--You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.-Your task is to Populate these concepts with their specific Participant Entities.--CRITICAL RULES:-1. Extract ALL key participants for each concept.-2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).-"""-COG_RAG_THEME_PROMPT = """--Goal--Analyze the text to identify the primary themes (narrative arcs).-A Theme is a high-level concept, conflict, or main idea acting as a Hyperedge.-"""+You are an expert narrative analyst.+Your goal is to extract "Themes" (Narrative Arcs) from the text.-COG_RAG_KEY_ENTITY_PROMPT = """--Goal--You are given a list of "Primary Themes" identified from the text.-Your task is to populate these themes with key entities (Participants).+A **Theme** is a high-level concept, conflict, or main idea that connects multiple entities.+For each Theme, you must also identify the key **Participants** (Entities) involved.++# Instructions+1. Analyze the text to identify primary themes/events.+2. For each theme, identify the entities involved.+3. Output the Theme description and the list of Participant Entities."""COG_RAG_NODE_EXTRACTION_PROMPT = """-Goal-Identify relevant entities from the text.Entities will serve as participants in complex events later.+### Source Text:"""@@ -169,215 +172,11 @@ Merge strategy:# ============================================================================-class EdgeAutoHypergraph(AutoHypergraph[NodeSchema, EdgeSchema]):- """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.-- Extraction Strategy: Two-Stage (Edge-First)- 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).- 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.-- Suitable for:- - Narrative-driven knowledge graphs (Theme -> Character)- - Event-driven extraction (Event -> Participant)- """-- def __init__(- self,- node_schema: Type[NodeSchema],- edge_schema: Type[EdgeSchema],- node_key_extractor: Callable[[NodeSchema], str],- edge_key_extractor: Callable[[EdgeSchema], str],- nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, ...]],- llm_client: BaseChatModel,- embedder: Embeddings,- *,- prompt_for_node_extraction: str = "",- **kwargs: Any,- ):- """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.-- Args:- prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".- (Extract Nodes given Edges).- """- super().__init__(- node_schema=node_schema,- edge_schema=edge_schema,- node_key_extractor=node_key_extractor,- edge_key_extractor=edge_key_extractor,- nodes_in_edge_extractor=nodes_in_edge_extractor,- llm_client=llm_client,- embedder=embedder,- prompt_for_node_extraction=prompt_for_node_extraction,- **kwargs,- )-- # Build Stage 2 Schema (Edges containing nested Nodes)- # We assume the edge schema has a 'participants' field (List[str]).- # We create a new model where 'participants' is List[NodeSchema].-- # 1. Base fields from EdgeSchema (excluding participants)- valid_fields = {- k: (v.annotation, v)- for k, v in edge_schema.model_fields.items()- if k != "participants"- }-- # 2. Add Nested Participants field- valid_fields["participants"] = (- List[node_schema],- Field(description="List of entities participating in this concept/edge."),- )-- # 3. Create Contextual Edge Model- self.contextual_edge_schema = create_model(- f"Contextual{edge_schema.__name__}", **valid_fields- )-- # 4. Create Result List Model- self.stage2_result_schema = create_model(- f"{edge_schema.__name__}FirstResult",- items=(List[self.contextual_edge_schema], Field(default_factory=list)),- )-- # Set default prompt if not provided- if not prompt_for_node_extraction:- self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT-- def _extract_data(self, text: str) -> AutoHypergraphSchema:- """Main extraction logic dispatcher (Edge-First)."""- if self.extraction_mode == "two_stage":- raw_graph = self._extract_data_by_two_stage(text)- elif self.extraction_mode == "one_stage":- raise NotImplementedError(- "Single-stage extraction not yet supported for EdgeAutoHypergraph."- )- else:- raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")-- # Note: Dangle pruning might be less critical here since nodes are derived from edges,- # but we keep it for consistency if nodes are merged globally.- return self._prune_dangling_edges(raw_graph)-- def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:- """Extract edges/themes first, then entities within those contexts (batch processing).-- Process:- 1. Split text into chunks.- 2. Batch extract Edge Concepts (Theme Strings) for all chunks.- 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).- 4. Merge all partial results into one global hypergraph.- """- # 1. Prepare chunks- if len(text) <= self.chunk_size:- chunks = [text]- else:- chunks = self.text_splitter.split_text(text)-- if self.verbose:- logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")-- # 2. Stage 1: Identify Edge Concepts (Strings)- edge_concepts_lists = self._extract_edge_concepts_batch(chunks)-- # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)- chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)-- # 4. Global Merge- # chunk_results is List[AutoHypergraphSchema]- return self.merge_batch_data(chunk_results)-- def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:- """Stage 1: Batch extract edge concept strings."""- # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts- # We assume the prompt is geared towards finding a LIST of themes/concepts.-- prompt_template = ChatPromptTemplate.from_template(- f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"- )- chain = prompt_template | self.llm_client.with_structured_output(- EdgeDescriptionList- )-- inputs = [{"chunk_text": chunk} for chunk in chunks]- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})-- return [res.items if res else [] for res in results]-- def _extract_details_batch(- self, chunks: List[str], edge_concepts_list: List[List[str]]- ) -> List[AutoHypergraphSchema]:- """Generic Stage 2: Extract entities and details given identified edge concepts."""-- # 1. Prepare Inputs- inputs = []- valid_map = {}- for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):- if concepts:- concepts_str = "- " + "\n- ".join(concepts)- inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})- valid_map[len(inputs) - 1] = i-- if not inputs:- return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]-- # 2. Chain Execution- # Uses self.node_prompt as the instruction for contextual node extraction- prompt = ChatPromptTemplate.from_template(- f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"- )- chain = prompt | self.llm_client.with_structured_output(- self.stage2_result_schema- )-- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})-- # 3. Transform Nested Result -> Flat AutoHypergraphSchema- final_schemas = [- self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))- ]-- for i, result in enumerate(results):- if not result or not result.items:- continue-- original_idx = valid_map[i]- current_nodes = []- current_edges = []-- for item in result.items:- # item is a ContextualEdge (has nested participants=List[Node])-- # Extract Nodes- nested_nodes = item.participants- node_names = []- for n in nested_nodes:- current_nodes.append(n)- node_names.append(self.node_key_extractor(n))-- # Reconstruct Flat Edge- edge_data = item.model_dump()- edge_data["participants"] = sorted(- node_names- ) # Ensure sorted for consistency-- new_edge = self.edge_schema(**edge_data)- current_edges.append(new_edge)-- final_schemas[original_idx] = self.graph_schema(- nodes=current_nodes, edges=current_edges- )-- return final_schemas---# ============================================================================---class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):+class Cog_RAG_ThemeLayer(AutoHypergraph[NodeSchema, ThemeSchema]):"""Layer 1 of Cog_RAG: Theme-First Extraction.- Extracts Themes (Hyperedges) and Key Entities (Nodes).+ Extracts Themes (Hyperedges) and Key Entities (Nodes) simultaneously.+ Follows the HyperGraph_RAG pattern (Edge-First)."""def __init__(@@ -389,28 +188,17 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):max_workers: int = 10,verbose: bool = True,):- """- Initialize COG_RAG engine.-- Args:- llm_client (BaseChatModel): LLM client for text generation.- embedder (Embeddings): Embedding model for text embeddings.- chunk_size (int): Size of text chunks for indexing.- chunk_overlap (int): Overlap between text chunks for indexing.- max_workers (int): Maximum number of workers for indexing.- verbose (bool): Display detailed execution logs and progress information.- """# 1. Define Key Extractors# Node key: exact match by namenode_key_fn = lambda x: x.name- # Edge key: sorted tuple of participants- edge_key_fn = lambda x: tuple(sorted(x.participants))+ # Theme key: md5 hash of description (Semantic uniqueness)+ edge_key_fn = lambda x: md5(x.description.encode()).hexdigest()- # 2. Edge consistency check: tell AutoHypergraph which nodes this edge connects+ # Nodes in themenodes_in_edge_fn = lambda x: x.participants- # 3. Create custom Mergers with LLM.CUSTOM_RULE strategy+ # 2. Create custom Mergersnode_merger = CustomRuleMerger(key_extractor=node_key_fn,llm_client=llm_client,@@ -425,7 +213,6 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):rule=COG_RAG_THEME_MERGE_RULE,)- # 4. Call parent class initializationsuper().__init__(node_schema=NodeSchema,edge_schema=ThemeSchema,@@ -434,11 +221,7 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):nodes_in_edge_extractor=nodes_in_edge_fn,llm_client=llm_client,embedder=embedder,- # Enforce two-stage extraction- extraction_mode="two_stage",- # Inject customized prompts for extraction- # prompt_for_node_extraction acts as the Stage 2 (Edge -> Node) prompt- prompt_for_node_extraction=COG_RAG_KEY_ENTITY_PROMPT,+ # Prompt is utilized in _extract_data overrideprompt_for_edge_extraction=COG_RAG_THEME_PROMPT,# Pass custom merger instancesnode_strategy_or_merger=node_merger,@@ -446,13 +229,66 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):# Optimize indexingnode_fields_for_index=["name", "type", "description"],edge_fields_for_index=["description"],- # Other parameterschunk_size=chunk_size,chunk_overlap=chunk_overlap,max_workers=max_workers,verbose=verbose,)+ def _extract_data(self, text: str) -> AutoHypergraphSchema:+ """+ Extract Themes and nested Participants (One-Stage).+ Implements the HyperGraph_RAG extraction pattern.+ """+ prompt_template = ChatPromptTemplate.from_template(+ f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"+ )+ llm_chain = prompt_template | self.llm_client.with_structured_output(+ TEMP_ThemeListSchema+ )++ # Batch Processing+ if len(text) <= self.chunk_size:+ raw_results_list = [llm_chain.invoke({"chunk_text": text})]+ else:+ chunks = self.text_splitter.split_text(text)+ inputs = [{"chunk_text": chunk} for chunk in chunks]+ raw_results_list = llm_chain.batch(+ inputs, config={"max_concurrency": self.max_workers}+ )++ # Transform to Flat Schema+ all_nodes_list = []+ all_edges_list = []++ for result in raw_results_list:+ if not result or not result.items:+ continue++ cur_nodes = []+ cur_edges = []++ for item in result.items:+ item: TEMP_ThemeSchema+ extracted_nodes = item.participants+ cur_nodes.extend(extracted_nodes)++ cur_edges.append(+ ThemeSchema(+ description=item.description,+ participants=[+ self.node_key_extractor(n) for n in extracted_nodes+ ],+ )+ )++ all_nodes_list.append(cur_nodes)+ all_edges_list.append(cur_edges)++ # Merge+ raw_graph = self.merge_batch_data((all_nodes_list, all_edges_list))+ return self._prune_dangling_edges(raw_graph)+# ============================================================================# Detail Hypergraph (Layer 2: Micro Relations)@@ -463,6 +299,7 @@ class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):"""Layer 2 of Cog_RAG: Entity-First Extraction.Extracts Entities (Nodes) and Semantic Relationships (Hyperedges).+ Follows the Hyper_RAG pattern (Two-Stage)."""def __init__(@@ -474,10 +311,8 @@ class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):max_workers: int = 10,verbose: bool = True,):- """- Initialize COG_RAG Entity-Relation engine.- """node_key_fn = lambda x: x.name+ # Relations: Unique by Sorted Participantsedge_key_fn = lambda x: tuple(sorted(x.participants))nodes_in_edge_fn = lambda x: x.participants@@ -562,17 +397,17 @@ class Cog_RAG:def feed_text(self, text: str):"""Feed text to both hypergraph layers."""if self.verbose:- logger.info("Cog_RAG: Feeding text to Theme Layer...")+ print("Cog_RAG: Feeding text to Theme Layer...")self.theme_layer.feed_text(text)if self.verbose:- logger.info("Cog_RAG: Feeding text to Detail Layer...")+ print("Cog_RAG: Feeding text to Detail Layer...")self.detail_layer.feed_text(text)def build_index(self):"""Build indices for both layers."""if self.verbose:- logger.info("Cog_RAG: Building indices...")+ print("Cog_RAG: Building indices...")self.theme_layer.build_index()self.detail_layer.build_index()@@ -635,8 +470,6 @@ class Cog_RAG:@propertydef nodes(self):"""Return combined unique nodes from both layers (simple aggregation)."""- # Note: This is a property for valid introspection, but nodes are managed separately.- # We perform a simple name-based deduction for display.seen = set()unique = []for n in self.theme_layer.nodes + self.detail_layer.nodes: