Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

update cog-rag

haoohuc44dc3187e15
diff --git a/examples/cog_rag_demo.py b/examples/cog_rag_demo.py
index 79952f8..5c0ef0f 100644
--- a/examples/cog_rag_demo.py
+++ b/examples/cog_rag_demo.py
@@ -2,32 +2,30 @@
Cog_RAG Demo: Theme-based Knowledge Extraction + Entity-Relation Detail Retrieval
This demo demonstrates the unified Cog_RAG System Wrapper:
-1. **Layer 1 (Theme Layer)**: Extracts Key Entities and Themes (Narrative Arcs).
-2. **Layer 2 (Detail Layer)**: Extracts Entities and Semantic Relationships.
+1. **Macro Layer (Theme Layer)**: Extracts Narrative Arcs (Themes) and their Participant Entities.
+2. **Micro Layer (Detail Layer)**: Extracts Entities and their Semantic Relationships.
3. **Process**:
- Feeds the same story to the Unified System.
- - Performs a "Theme-First" retrieval process automatically.
+ - The system automatically distributes extraction tasks to both layers.
+ - Performs a Dual-Layer Chat/Search.
"""
import os
import sys
-# 添加项目根目录到路径
+# Add project root to path
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-
# Try to load dotenv, ignoring if not installed
-try:
- from dotenv import load_dotenv
- load_dotenv()
-except ImportError:
- pass
-from hyperextract.hypergraphs import Cog_RAG
+from dotenv import load_dotenv
+load_dotenv()
+
+from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+from hyperextract.hypergraphs.cog_rag import Cog_RAG
# ============================================================================
-# Sample Story for Extraction (Shared with AutoHypergraph Demo)
+# Sample Story for Extraction
# ============================================================================
STORY = """
@@ -81,12 +79,13 @@ Leona 被迫带着虚空水晶,从极光星区逃往荒凉的废土行星 Z-9
if __name__ == "__main__":
# ============================================================================
- # Initialize the CogRAG system with LLM and Embeddings
+ # Initialize the Cog_RAG system
# ============================================================================
print("=" * 80)
- print("Initializing Unified CogRAG System...")
+ print("Initializing Unified Cog_RAG System...")
print("=" * 80)
+ # Use a capable model for extraction
llm_client = ChatOpenAI(
model="gpt-4o-mini",
extra_body={
@@ -100,97 +99,74 @@ if __name__ == "__main__":
rag = Cog_RAG(
llm_client=llm_client,
embedder=embedder,
- verbose=True,
+ chunk_size=1000,
+ verbose=True
)
# ============================================================================
# Feed text to the system
# ============================================================================
- print("\nFeeding story to the system...")
+ print("\nFeeding story to the Cog_RAG System (Dual-Layer Extraction)...")
rag.feed_text(STORY)
print("✓ Story processed successfully\n")
# ============================================================================
- # Display Extracted Entities
+ # Display Extracted Data (Unified View)
# ============================================================================
print("=" * 80)
- print("EXTRACTED ENTITIES (Both Layers)")
+ print("EXTRACTED KNOWLEDGE")
print("=" * 80)
- # Note: Cog_RAG.nodes property aggregates nodes from both layers
- nodes = rag.nodes
- sorted_nodes = sorted(nodes, key=lambda x: x.name)
-
- for node in sorted_nodes:
- print(f"\n【{node.name}】(Type: {node.type})")
- print(f" Description: {node.description}")
-
- print(f"\n✓ Total unique entities extracted: {len(nodes)}\n")
-
- # ============================================================================
- # Display Extracted Hyperedges (Separate Layers)
- # ============================================================================
- print("=" * 80)
- print("EXTRACTED HYPEREDGES (Dual Layer)")
- print("=" * 80)
-
- print(f"\n--- Layer 1: Theme Hyperedges (Macro) ---")
+ # 1. Macro Themes
+ print("\n--- MACRO LAYER: Themes (Narrative Arcs) ---")
for i, edge in enumerate(rag.theme_layer.edges, 1):
- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
- print(f" Theme: {edge.description}")
-
- print(f"\n--- Layer 2: Relation Hyperedges (Micro) ---")
- for i, edge in enumerate(rag.detail_layer.edges, 1):
- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
- print(f" Description: {edge.description}")
- keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"
- print(f" Keywords: {keywords_str}")
- print(f" Strength: {edge.strength}/10")
-
- print(
- f"\n✓ Total edges extracted: {len(rag.edges)} ({len(rag.theme_layer.edges)} themes, {len(rag.detail_layer.edges)} relations)\n"
- )
+ print(f"\nTheme {i}: {edge.description}")
+ print(f" Participants: {', '.join(edge.participants)}")
- # ============================================================================
- # Statistics
- # ============================================================================
- print("=" * 80)
- print("STATISTICS")
- print("=" * 80)
+ print(f"\n✓ Total Themes extracted: {len(rag.theme_layer.edges)}")
- # Entity type distribution
- type_counts = {}
- for node in rag.nodes:
- type_counts[node.type] = type_counts.get(node.type, 0) + 1
+ # 2. Micro Relations
+ print(f"\n--- MICRO LAYER: Entity Relations ---")
+
+ low_order_edges = [e for e in rag.detail_layer.edges if len(e.participants) == 2]
+ high_order_edges = [e for e in rag.detail_layer.edges if len(e.participants) > 2]
- print("\nEntity Type Distribution:")
- for entity_type, count in sorted(type_counts.items()):
- print(f" {entity_type}: {count}")
+ print(f"\nMicro Relations (Pairwise):")
+ for i, edge in enumerate(low_order_edges[:5], 1): # Show top 5
+ print(f"{i}. {edge.description} (Entities: {', '.join(edge.participants)})")
+ if len(low_order_edges) > 5:
+ print(f"... and {len(low_order_edges) - 5} more.")
- # Centrality (number of relationships/themes per entity)
- print("\nEntity Participation (Centrality):")
- centrality = {}
- for edge in rag.edges:
- for participant in edge.participants:
- centrality[participant] = centrality.get(participant, 0) + 1
+ print(f"\nMicro Relations (Hyperedges):")
+ for i, edge in enumerate(high_order_edges[:5], 1): # Show top 5
+ print(f"{i}. {edge.description} (Entities: {', '.join(edge.participants)})")
+ if len(high_order_edges) > 5:
+ print(f"... and {len(high_order_edges) - 5} more.")
- for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[
- :10
- ]:
- print(f" {entity}: {count}")
+ print(f"\n✓ Total Micro Relations extracted: {len(rag.detail_layer.edges)}")
+
+ # 3. Entities
+ print(f"\n--- UNIQUE ENTITIES (Combined) ---")
+ print(f"Total Unique Entities: {len(rag.nodes)}")
+
+ # Display top 5 entities
+ for i, node in enumerate(rag.nodes[:5], 1):
+ print(f"{i}. 【{node.name}】 ({node.type})")
+ if len(rag.nodes) > 5:
+ print(f"... and {len(rag.nodes) - 5} more.")
# ============================================================================
- # Build Semantic Index for Q&A
+ # Build Semantic Index
# ============================================================================
print("\n" + "=" * 80)
- print("Building Semantic Index...")
+ print("Building Semantic Indexes...")
print("=" * 80)
rag.build_index()
- print("✓ Semantic index built successfully\n")
+ print("✓ Semantic indexes built successfully\n")
# ============================================================================
- # Q&A: Semantic Search over Hypergraph
+ # Q&A: Dual-Layer Search
# ============================================================================
print("=" * 80)
print("Q&A: DUAL-LAYER SEMANTIC SEARCH")
@@ -206,48 +182,38 @@ if __name__ == "__main__":
for i, query in enumerate(queries, 1):
print(f"\n【问题 {i}】{query}")
print("-" * 60)
+
+ # Search BOTH layers
+ results = rag.search(query, top_k_themes=2, top_k_entities=2)
+
+ print(" [Macro Themes Found]:")
+ if results['themes']:
+ for t in results['themes']:
+ print(f" - {t.description}")
+ else:
+ print(" (None)")
- try:
- # Cog_RAG returns a dict with 'themes' and 'entities'
- results = rag.search(query, top_k_themes=3, top_k_entities=3)
-
- print(" [Layer 1: Macro Themes]")
- if results['themes']:
- for j, theme in enumerate(results['themes'], 1):
- print(f" {j}. {theme.description}")
- else:
- print(" (No relevant themes found)")
-
- print("\n [Layer 2: Micro Entities]")
- if results['entities']:
- for j, entity in enumerate(results['entities'], 1):
- print(f" {j}. [{entity.name}] {entity.description[:100]}...")
- else:
- print(" (No relevant entities found)")
+ print(" [Micro Entities Found]:")
+ if results['entities']:
+ for e in results['entities']:
+ print(f" - {e.name}: {e.description[:50]}...")
+ else:
+ print(" (None)")
- except Exception as e:
- print(f" Search error: {str(e)}")
# ============================================================================
- # Chat with CogRAG
+ # Chat with Cog_RAG
# ============================================================================
print("\n" + "=" * 80)
- print("💬 Interactive Chat with CogRAG")
+ print("💬 Interactive Chat with Cog_RAG")
print("=" * 80)
- print("Engaging in multi-turn dialogue based on extracted knowledge...\n")
-
- chat_queries = [
- "虚空水晶现在在哪里?",
- "黑蛇和议员Valerius有什么阴谋?",
- "谁参与了泰坦空间站的秘密会议?",
- "Leona目前的状况如何?",
- ]
+ print("Engaging in multi-turn dialogue based on Dual-Layer Knowledge...\n")
- for q in chat_queries:
+ for q in queries:
print(f"❓ User: {q}")
try:
response = rag.chat(q)
- print(f"🤖 CogRAG: {response.content}\n")
+ print(f"🤖 Cog_RAG: {response.content}\n")
except Exception as e:
print(f"⚠️ Chat error: {e}\n")
@@ -260,6 +226,9 @@ if __name__ == "__main__":
try:
output_dir = "temp/cog_rag_demo"
+ if not os.path.exists(output_dir):
+ os.makedirs(output_dir)
+
rag.dump(output_dir)
print(f"✓ Results saved to {output_dir}/")
diff --git a/hyperextract/hypergraphs/__init__.py b/hyperextract/hypergraphs/__init__.py
index c801ad0..1c0ff6d 100644
--- a/hyperextract/hypergraphs/__init__.py
+++ b/hyperextract/hypergraphs/__init__.py
@@ -1,11 +1,11 @@
from .base import AutoHypergraph
from .hyper_rag import Hyper_RAG
from .hypergraph_rag import HyperGraph_RAG
-# from .cog_rag import Cog_RAG
+from .cog_rag import Cog_RAG
__all__ = [
"AutoHypergraph",
"Hyper_RAG",
"HyperGraph_RAG",
- # "Cog_RAG",
+ "Cog_RAG",
]
diff --git a/hyperextract/hypergraphs/cog_rag.py b/hyperextract/hypergraphs/cog_rag.py
index 525ef1c..4836114 100644
--- a/hyperextract/hypergraphs/cog_rag.py
+++ b/hyperextract/hypergraphs/cog_rag.py
@@ -3,10 +3,10 @@ COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System Pattern
Extracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities.
"""
-from typing import List, Type, Callable, Tuple, Any
-from pydantic import BaseModel, Field, create_model
+from typing import List, Tuple
+from hashlib import md5
+from pydantic import BaseModel, Field
from hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchema
-from hyperextract.utils.logging import logger
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.embeddings import Embeddings
from langchain_core.language_models import BaseChatModel
@@ -31,7 +31,7 @@ class NodeSchema(BaseModel):
# ============================================================================
-# Edge Schema (Theme)
+# Theme Schemas (Layer 1: Macro Narrative)
# ============================================================================
@@ -46,8 +46,27 @@ class ThemeSchema(BaseModel):
)
+class TEMP_ThemeSchema(BaseModel):
+ """Temporary schema for extracting Themes with nested Entities."""
+
+ description: str = Field(
+ description="A sentence that describes the primary theme, reflecting the main conflict, resolution, or key message"
+ )
+ participants: List[NodeSchema] = Field(
+ description="List of key entities involved in this theme, with their details."
+ )
+
+
+class TEMP_ThemeListSchema(BaseModel):
+ """List of extracted themes."""
+
+ items: List[TEMP_ThemeSchema] = Field(
+ description="List of extracted themes and their participants."
+ )
+
+
# ============================================================================
-# Edge Schema (Entity Relation)
+# Edge Schema (Layer 2: Micro Entity Relation)
# ============================================================================
@@ -70,44 +89,28 @@ class EdgeSchema(BaseModel):
)
-class EdgeDescriptionList(BaseModel):
- """Intermediate schema for identifying edge concepts first."""
-
- items: List[str] = Field(
- description="List of primary edge concepts/themes identified"
- )
-
-
# ============================================================================
# Extraction Prompts
# ============================================================================
-
-DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal-
-You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.
-Your task is to Populate these concepts with their specific Participant Entities.
-
-CRITICAL RULES:
-1. Extract ALL key participants for each concept.
-2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).
-"""
-
COG_RAG_THEME_PROMPT = """
--Goal-
-Analyze the text to identify the primary themes (narrative arcs).
-A Theme is a high-level concept, conflict, or main idea acting as a Hyperedge.
-"""
+You are an expert narrative analyst.
+Your goal is to extract "Themes" (Narrative Arcs) from the text.
-COG_RAG_KEY_ENTITY_PROMPT = """
--Goal-
-You are given a list of "Primary Themes" identified from the text.
-Your task is to populate these themes with key entities (Participants).
+A **Theme** is a high-level concept, conflict, or main idea that connects multiple entities.
+For each Theme, you must also identify the key **Participants** (Entities) involved.
+
+# Instructions
+1. Analyze the text to identify primary themes/events.
+2. For each theme, identify the entities involved.
+3. Output the Theme description and the list of Participant Entities.
"""
COG_RAG_NODE_EXTRACTION_PROMPT = """
-Goal-
Identify relevant entities from the text.
Entities will serve as participants in complex events later.
+
### Source Text:
"""
@@ -169,215 +172,11 @@ Merge strategy:
# ============================================================================
-class EdgeAutoHypergraph(AutoHypergraph[NodeSchema, EdgeSchema]):
- """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.
-
- Extraction Strategy: Two-Stage (Edge-First)
- 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).
- 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.
-
- Suitable for:
- - Narrative-driven knowledge graphs (Theme -> Character)
- - Event-driven extraction (Event -> Participant)
- """
-
- def __init__(
- self,
- node_schema: Type[NodeSchema],
- edge_schema: Type[EdgeSchema],
- node_key_extractor: Callable[[NodeSchema], str],
- edge_key_extractor: Callable[[EdgeSchema], str],
- nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, ...]],
- llm_client: BaseChatModel,
- embedder: Embeddings,
- *,
- prompt_for_node_extraction: str = "",
- **kwargs: Any,
- ):
- """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.
-
- Args:
- prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".
- (Extract Nodes given Edges).
- """
- super().__init__(
- node_schema=node_schema,
- edge_schema=edge_schema,
- node_key_extractor=node_key_extractor,
- edge_key_extractor=edge_key_extractor,
- nodes_in_edge_extractor=nodes_in_edge_extractor,
- llm_client=llm_client,
- embedder=embedder,
- prompt_for_node_extraction=prompt_for_node_extraction,
- **kwargs,
- )
-
- # Build Stage 2 Schema (Edges containing nested Nodes)
- # We assume the edge schema has a 'participants' field (List[str]).
- # We create a new model where 'participants' is List[NodeSchema].
-
- # 1. Base fields from EdgeSchema (excluding participants)
- valid_fields = {
- k: (v.annotation, v)
- for k, v in edge_schema.model_fields.items()
- if k != "participants"
- }
-
- # 2. Add Nested Participants field
- valid_fields["participants"] = (
- List[node_schema],
- Field(description="List of entities participating in this concept/edge."),
- )
-
- # 3. Create Contextual Edge Model
- self.contextual_edge_schema = create_model(
- f"Contextual{edge_schema.__name__}", **valid_fields
- )
-
- # 4. Create Result List Model
- self.stage2_result_schema = create_model(
- f"{edge_schema.__name__}FirstResult",
- items=(List[self.contextual_edge_schema], Field(default_factory=list)),
- )
-
- # Set default prompt if not provided
- if not prompt_for_node_extraction:
- self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT
-
- def _extract_data(self, text: str) -> AutoHypergraphSchema:
- """Main extraction logic dispatcher (Edge-First)."""
- if self.extraction_mode == "two_stage":
- raw_graph = self._extract_data_by_two_stage(text)
- elif self.extraction_mode == "one_stage":
- raise NotImplementedError(
- "Single-stage extraction not yet supported for EdgeAutoHypergraph."
- )
- else:
- raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")
-
- # Note: Dangle pruning might be less critical here since nodes are derived from edges,
- # but we keep it for consistency if nodes are merged globally.
- return self._prune_dangling_edges(raw_graph)
-
- def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:
- """Extract edges/themes first, then entities within those contexts (batch processing).
-
- Process:
- 1. Split text into chunks.
- 2. Batch extract Edge Concepts (Theme Strings) for all chunks.
- 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).
- 4. Merge all partial results into one global hypergraph.
- """
- # 1. Prepare chunks
- if len(text) <= self.chunk_size:
- chunks = [text]
- else:
- chunks = self.text_splitter.split_text(text)
-
- if self.verbose:
- logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")
-
- # 2. Stage 1: Identify Edge Concepts (Strings)
- edge_concepts_lists = self._extract_edge_concepts_batch(chunks)
-
- # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)
- chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)
-
- # 4. Global Merge
- # chunk_results is List[AutoHypergraphSchema]
- return self.merge_batch_data(chunk_results)
-
- def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:
- """Stage 1: Batch extract edge concept strings."""
- # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts
- # We assume the prompt is geared towards finding a LIST of themes/concepts.
-
- prompt_template = ChatPromptTemplate.from_template(
- f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"
- )
- chain = prompt_template | self.llm_client.with_structured_output(
- EdgeDescriptionList
- )
-
- inputs = [{"chunk_text": chunk} for chunk in chunks]
- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
-
- return [res.items if res else [] for res in results]
-
- def _extract_details_batch(
- self, chunks: List[str], edge_concepts_list: List[List[str]]
- ) -> List[AutoHypergraphSchema]:
- """Generic Stage 2: Extract entities and details given identified edge concepts."""
-
- # 1. Prepare Inputs
- inputs = []
- valid_map = {}
- for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):
- if concepts:
- concepts_str = "- " + "\n- ".join(concepts)
- inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})
- valid_map[len(inputs) - 1] = i
-
- if not inputs:
- return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]
-
- # 2. Chain Execution
- # Uses self.node_prompt as the instruction for contextual node extraction
- prompt = ChatPromptTemplate.from_template(
- f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"
- )
- chain = prompt | self.llm_client.with_structured_output(
- self.stage2_result_schema
- )
-
- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
-
- # 3. Transform Nested Result -> Flat AutoHypergraphSchema
- final_schemas = [
- self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))
- ]
-
- for i, result in enumerate(results):
- if not result or not result.items:
- continue
-
- original_idx = valid_map[i]
- current_nodes = []
- current_edges = []
-
- for item in result.items:
- # item is a ContextualEdge (has nested participants=List[Node])
-
- # Extract Nodes
- nested_nodes = item.participants
- node_names = []
- for n in nested_nodes:
- current_nodes.append(n)
- node_names.append(self.node_key_extractor(n))
-
- # Reconstruct Flat Edge
- edge_data = item.model_dump()
- edge_data["participants"] = sorted(
- node_names
- ) # Ensure sorted for consistency
-
- new_edge = self.edge_schema(**edge_data)
- current_edges.append(new_edge)
-
- final_schemas[original_idx] = self.graph_schema(
- nodes=current_nodes, edges=current_edges
- )
-
- return final_schemas
-
-
-# ============================================================================
-
-
-class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
+class Cog_RAG_ThemeLayer(AutoHypergraph[NodeSchema, ThemeSchema]):
"""
Layer 1 of Cog_RAG: Theme-First Extraction.
- Extracts Themes (Hyperedges) and Key Entities (Nodes).
+ Extracts Themes (Hyperedges) and Key Entities (Nodes) simultaneously.
+ Follows the HyperGraph_RAG pattern (Edge-First).
"""
def __init__(
@@ -389,28 +188,17 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
max_workers: int = 10,
verbose: bool = True,
):
- """
- Initialize COG_RAG engine.
-
- Args:
- llm_client (BaseChatModel): LLM client for text generation.
- embedder (Embeddings): Embedding model for text embeddings.
- chunk_size (int): Size of text chunks for indexing.
- chunk_overlap (int): Overlap between text chunks for indexing.
- max_workers (int): Maximum number of workers for indexing.
- verbose (bool): Display detailed execution logs and progress information.
- """
# 1. Define Key Extractors
# Node key: exact match by name
node_key_fn = lambda x: x.name
- # Edge key: sorted tuple of participants
- edge_key_fn = lambda x: tuple(sorted(x.participants))
+ # Theme key: md5 hash of description (Semantic uniqueness)
+ edge_key_fn = lambda x: md5(x.description.encode()).hexdigest()
- # 2. Edge consistency check: tell AutoHypergraph which nodes this edge connects
+ # Nodes in theme
nodes_in_edge_fn = lambda x: x.participants
- # 3. Create custom Mergers with LLM.CUSTOM_RULE strategy
+ # 2. Create custom Mergers
node_merger = CustomRuleMerger(
key_extractor=node_key_fn,
llm_client=llm_client,
@@ -425,7 +213,6 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
rule=COG_RAG_THEME_MERGE_RULE,
)
- # 4. Call parent class initialization
super().__init__(
node_schema=NodeSchema,
edge_schema=ThemeSchema,
@@ -434,11 +221,7 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
nodes_in_edge_extractor=nodes_in_edge_fn,
llm_client=llm_client,
embedder=embedder,
- # Enforce two-stage extraction
- extraction_mode="two_stage",
- # Inject customized prompts for extraction
- # prompt_for_node_extraction acts as the Stage 2 (Edge -> Node) prompt
- prompt_for_node_extraction=COG_RAG_KEY_ENTITY_PROMPT,
+ # Prompt is utilized in _extract_data override
prompt_for_edge_extraction=COG_RAG_THEME_PROMPT,
# Pass custom merger instances
node_strategy_or_merger=node_merger,
@@ -446,13 +229,66 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
# Optimize indexing
node_fields_for_index=["name", "type", "description"],
edge_fields_for_index=["description"],
- # Other parameters
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
max_workers=max_workers,
verbose=verbose,
)
+ def _extract_data(self, text: str) -> AutoHypergraphSchema:
+ """
+ Extract Themes and nested Participants (One-Stage).
+ Implements the HyperGraph_RAG extraction pattern.
+ """
+ prompt_template = ChatPromptTemplate.from_template(
+ f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"
+ )
+ llm_chain = prompt_template | self.llm_client.with_structured_output(
+ TEMP_ThemeListSchema
+ )
+
+ # Batch Processing
+ if len(text) <= self.chunk_size:
+ raw_results_list = [llm_chain.invoke({"chunk_text": text})]
+ else:
+ chunks = self.text_splitter.split_text(text)
+ inputs = [{"chunk_text": chunk} for chunk in chunks]
+ raw_results_list = llm_chain.batch(
+ inputs, config={"max_concurrency": self.max_workers}
+ )
+
+ # Transform to Flat Schema
+ all_nodes_list = []
+ all_edges_list = []
+
+ for result in raw_results_list:
+ if not result or not result.items:
+ continue
+
+ cur_nodes = []
+ cur_edges = []
+
+ for item in result.items:
+ item: TEMP_ThemeSchema
+ extracted_nodes = item.participants
+ cur_nodes.extend(extracted_nodes)
+
+ cur_edges.append(
+ ThemeSchema(
+ description=item.description,
+ participants=[
+ self.node_key_extractor(n) for n in extracted_nodes
+ ],
+ )
+ )
+
+ all_nodes_list.append(cur_nodes)
+ all_edges_list.append(cur_edges)
+
+ # Merge
+ raw_graph = self.merge_batch_data((all_nodes_list, all_edges_list))
+ return self._prune_dangling_edges(raw_graph)
+
# ============================================================================
# Detail Hypergraph (Layer 2: Micro Relations)
@@ -463,6 +299,7 @@ class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):
"""
Layer 2 of Cog_RAG: Entity-First Extraction.
Extracts Entities (Nodes) and Semantic Relationships (Hyperedges).
+ Follows the Hyper_RAG pattern (Two-Stage).
"""
def __init__(
@@ -474,10 +311,8 @@ class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):
max_workers: int = 10,
verbose: bool = True,
):
- """
- Initialize COG_RAG Entity-Relation engine.
- """
node_key_fn = lambda x: x.name
+ # Relations: Unique by Sorted Participants
edge_key_fn = lambda x: tuple(sorted(x.participants))
nodes_in_edge_fn = lambda x: x.participants
@@ -562,17 +397,17 @@ class Cog_RAG:
def feed_text(self, text: str):
"""Feed text to both hypergraph layers."""
if self.verbose:
- logger.info("Cog_RAG: Feeding text to Theme Layer...")
+ print("Cog_RAG: Feeding text to Theme Layer...")
self.theme_layer.feed_text(text)
if self.verbose:
- logger.info("Cog_RAG: Feeding text to Detail Layer...")
+ print("Cog_RAG: Feeding text to Detail Layer...")
self.detail_layer.feed_text(text)
def build_index(self):
"""Build indices for both layers."""
if self.verbose:
- logger.info("Cog_RAG: Building indices...")
+ print("Cog_RAG: Building indices...")
self.theme_layer.build_index()
self.detail_layer.build_index()
@@ -635,8 +470,6 @@ class Cog_RAG:
@property
def nodes(self):
"""Return combined unique nodes from both layers (simple aggregation)."""
- # Note: This is a property for valid introspection, but nodes are managed separately.
- # We perform a simple name-based deduction for display.
seen = set()
unique = []
for n in self.theme_layer.nodes + self.detail_layer.nodes: