Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

Refactor hypergraph extraction framework: - Update AutoHypergraph and related schemas to use NodeSchema and EdgeSchema for improved clarity and type safety. - Implement EdgeAutoHypergraph for edge-first extraction strategy, enhancing narrative-driven knowledge graph capabilities. - Revise HyperGraph_RAG to utilize knowledge segments as hyperedges, improving contextual relationship representation. - Introduce new extraction prompts and schemas for better entity and edge extraction processes. - Clean up imports and remove unused classes to streamline the codebase.

yifanfeng977cda10b6bbaa
diff --git a/examples/hypergraph_rag_demo.py b/examples/hypergraph_rag_demo.py
index 10cecd3..2076488 100644
--- a/examples/hypergraph_rag_demo.py
+++ b/examples/hypergraph_rag_demo.py
@@ -9,7 +9,6 @@ This demo demonstrates the HyperGraph_RAG System:
import os
import sys
-# 添加项目根目录到路径
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from dotenv import load_dotenv
@@ -113,28 +112,22 @@ if __name__ == "__main__":
print("=" * 80)
# 分离低阶边(2个参与者)和高阶边(3个以上参与者)
- low_order_edges = [e for e in rag.edges if len(e.participants) == 2]
- high_order_edges = [e for e in rag.edges if len(e.participants) > 2]
+ low_order_edges = [e for e in rag.edges if len(e.related_entities) == 2]
+ high_order_edges = [e for e in rag.edges if len(e.related_entities) > 2]
print(f"\n--- Low-Order Edges (Pairwise) ---")
for i, edge in enumerate(low_order_edges, 1):
- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
- print(f" Description: {edge.description}")
- keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"
- print(f" Keywords: {keywords_str}")
- print(f" Strength: {edge.strength}/10")
+ print(f"\n{i}. Participants: {' <-> '.join(edge.related_entities)}")
+ print(f" Knowledge Segment: {edge.knowledge_segment}")
+ print(f" Completeness Score: {edge.completeness_score}/10")
print(f"\n--- High-Order Edges (Hyperedges) ---")
for i, edge in enumerate(high_order_edges, 1):
- print(f"\n{i}. Participants: {' <-> '.join(edge.participants)}")
- print(f" Description: {edge.description}")
- keywords_str = ", ".join(edge.keywords) if edge.keywords else "N/A"
- print(f" Keywords: {keywords_str}")
- print(f" Strength: {edge.strength}/10")
-
- print(
- f"\n✓ Total edges extracted: {len(rag.edges)} ({len(low_order_edges)} low-order, {len(high_order_edges)} high-order)\n"
- )
+ print(f"\n{i}. Participants: {' <-> '.join(edge.related_entities)}")
+ print(f" Knowledge Segment: {edge.knowledge_segment}")
+ print(f" Completeness Score: {edge.completeness_score}/10")
+
+ print(f"\n✓ Total edges extracted: {len(rag.edges)} ({len(low_order_edges)} low-order, {len(high_order_edges)} high-order)\n")
# ============================================================================
# Display Extracted Entities
@@ -165,16 +158,16 @@ if __name__ == "__main__":
for entity_type, count in sorted(type_counts.items()):
print(f" {entity_type}: {count}")
- # Average edge strength
+ # Average edge completeness
if rag.edges:
- avg_strength = sum(e.strength for e in rag.edges) / len(rag.edges)
- print(f"\nAverage Edge Strength: {avg_strength:.2f}/10")
+ avg_completeness = sum(e.completeness_score for e in rag.edges) / len(rag.edges)
+ print(f"\nAverage Edge Completeness: {avg_completeness:.2f}/10")
# Centrality (number of relationships per entity)
print("\nEntity Centrality (number of relationships):")
centrality = {}
for edge in rag.edges:
- for participant in edge.participants:
+ for participant in edge.related_entities:
centrality[participant] = centrality.get(participant, 0) + 1
for entity, count in sorted(centrality.items(), key=lambda x: x[1], reverse=True)[
@@ -215,19 +208,15 @@ if __name__ == "__main__":
if results:
for j, result in enumerate(results, 1):
- # EdgeAutoHypergraph (and base AutoHypergraph) default search returns (doc, score) tuple
+ # AutoHypergraph search returns (doc, score) tuple
# The 'doc' is the EdgeSchema object
edge = result[0] if isinstance(result, tuple) else result
score = result[1] if isinstance(result, tuple) else 1.0
- # Note: EdgeSchema might not have strength if extracted via generic AutoHypergraph, but HyperGraph_RAG defines it.
- # We handle potential missing attrs safely
- strength = getattr(edge, "strength", "N/A")
-
print(f"\n {j}. [Score: {score:.3f}]")
- print(f" Participants: {' <-> '.join(edge.participants)}")
- print(f" Description: {edge.description}")
- print(f" Strength: {strength}/10")
+ print(f" Participants: {' <-> '.join(edge.related_entities)}")
+ print(f" Knowledge Segment: {edge.knowledge_segment}")
+ print(f" Completeness Score: {edge.completeness_score}/10")
else:
print(" (No relevant relationships found)")
except Exception as e:
diff --git a/hyperextract/graphs/atom.py b/hyperextract/graphs/atom.py
index 15f2326..c7c3044 100644
--- a/hyperextract/graphs/atom.py
+++ b/hyperextract/graphs/atom.py
@@ -370,8 +370,6 @@ class Atom(AutoGraph[NodeSchema, EdgeSchema]):
nodes_in_edge_extractor=nodes_in_edge_fn,
llm_client=llm_client,
embedder=embedder,
- # Enforce one-stage extraction (first extract edges, then derive nodes)
- extraction_mode="one_stage",
# Inject customized prompts
prompt_for_edge_extraction=Atom_EDGE_EXTRACTION_PROMPT,
# Configure deduplication strategy
diff --git a/hyperextract/graphs/base.py b/hyperextract/graphs/base.py
index 2af3720..61ae6e4 100644
--- a/hyperextract/graphs/base.py
+++ b/hyperextract/graphs/base.py
@@ -431,7 +431,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[NodeSchema, EdgeSchema]], Generic[N
text: Input text.
Returns:
- Raw extracted graph (may contain inconsistencies).
+ Raw extracted graph data.
"""
prompt_template = ChatPromptTemplate.from_template(
f"{self.prompt}{{chunk_text}}"
diff --git a/hyperextract/graphs/itext2kg_star.py b/hyperextract/graphs/itext2kg_star.py
index b8d2d77..7fed016 100644
--- a/hyperextract/graphs/itext2kg_star.py
+++ b/hyperextract/graphs/itext2kg_star.py
@@ -174,8 +174,6 @@ class iText2KG_Star(AutoGraph[NodeSchema, EdgeSchema]):
nodes_in_edge_extractor=nodes_in_edge_fn,
llm_client=llm_client,
embedder=embedder,
- # Enforce one-stage extraction (first extract edges, then derive nodes)
- extraction_mode="one_stage",
# Inject customized prompts
prompt_for_edge_extraction=iText2KG_Star_EDGE_EXTRACTION_PROMPT,
# Configure deduplication strategy
diff --git a/hyperextract/hypergraphs/__init__.py b/hyperextract/hypergraphs/__init__.py
index 559ed23..c801ad0 100644
--- a/hyperextract/hypergraphs/__init__.py
+++ b/hyperextract/hypergraphs/__init__.py
@@ -1,13 +1,11 @@
from .base import AutoHypergraph
from .hyper_rag import Hyper_RAG
from .hypergraph_rag import HyperGraph_RAG
-from .cog_rag import Cog_RAG, Cog_RAG_ThemeLayer, Cog_RAG_DetailLayer
+# from .cog_rag import Cog_RAG
__all__ = [
"AutoHypergraph",
"Hyper_RAG",
"HyperGraph_RAG",
- "Cog_RAG",
- "Cog_RAG_ThemeLayer",
- "Cog_RAG_DetailLayer",
+ # "Cog_RAG",
]
diff --git a/hyperextract/hypergraphs/base.py b/hyperextract/hypergraphs/base.py
index a9d4721..5e4261d 100644
--- a/hyperextract/hypergraphs/base.py
+++ b/hyperextract/hypergraphs/base.py
@@ -28,8 +28,8 @@ from ..core.base import BaseAutoType
from ..utils.logging import logger
-Node = TypeVar("Node", bound=BaseModel)
-Edge = TypeVar("Edge", bound=BaseModel) # Represents a Hyperedge
+NodeSchema = TypeVar("NodeSchema", bound=BaseModel)
+EdgeSchema = TypeVar("EdgeSchema", bound=BaseModel) # Represents a Hyperedge
# ============================================================================
@@ -66,52 +66,39 @@ CRITICAL RULES:
"""
-DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal-
-You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.
-Your task is to Populate these concepts with their specific Participant Entities.
-CRITICAL RULES:
-1. Extract ALL key participants for each concept.
-2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).
-"""
-
-
-class AutoHypergraphSchema(BaseModel, Generic[Node, Edge]):
+class AutoHypergraphSchema(BaseModel, Generic[NodeSchema, EdgeSchema]):
"""Generic schema container for hypergraph data."""
- nodes: List[Node] = Field(default_factory=list, description="Graph nodes/entities")
- edges: List[Edge] = Field(
+ nodes: List[NodeSchema] = Field(
+ default_factory=list, description="Graph nodes/entities"
+ )
+ edges: List[EdgeSchema] = Field(
default_factory=list, description="Hyperedges connecting multiple entities"
)
-class NodeListSchema(BaseModel, Generic[Node]):
+class NodeListSchema(BaseModel, Generic[NodeSchema]):
"""Intermediate schema for batch node extraction."""
- items: List[Node] = Field(
+ items: List[NodeSchema] = Field(
default_factory=list,
description="List of identified entities or nodes found in the text.",
)
-class EdgeDescriptionList(BaseModel):
- """Intermediate schema for identifying edge concepts first."""
- items: List[str] = Field(description="List of primary edge concepts/themes identified")
-
-
-
-
-class EdgeListSchema(BaseModel, Generic[Edge]):
+class EdgeListSchema(BaseModel, Generic[EdgeSchema]):
"""Intermediate schema for batch hyperedge extraction."""
- items: List[Edge] = Field(
+ items: List[EdgeSchema] = Field(
default_factory=list,
description="List of identified hyperedges found in the text.",
)
class AutoHypergraph(
- BaseAutoType[AutoHypergraphSchema[Node, Edge]], Generic[Node, Edge]
+ BaseAutoType[AutoHypergraphSchema[NodeSchema, EdgeSchema]],
+ Generic[NodeSchema, EdgeSchema],
):
"""AutoHypergraph - Node-First extraction: Extract entities first, then relationships.
@@ -149,16 +136,16 @@ class AutoHypergraph(
"""
if TYPE_CHECKING:
- graph_schema: Type[AutoHypergraphSchema[Node, Edge]]
+ graph_schema: Type[AutoHypergraphSchema[NodeSchema, EdgeSchema]]
def __init__(
self,
- node_schema: Type[Node],
- edge_schema: Type[Edge],
- node_key_extractor: Callable[[Node], str],
- edge_key_extractor: Callable[[Edge], str],
+ node_schema: Type[NodeSchema],
+ edge_schema: Type[EdgeSchema],
+ node_key_extractor: Callable[[NodeSchema], str],
+ edge_key_extractor: Callable[[EdgeSchema], str],
# Returns a tuple of ALL node keys involved in this hyperedge
- nodes_in_edge_extractor: Callable[[Edge], Tuple[str, ...]],
+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, ...]],
llm_client: BaseChatModel,
embedder: Embeddings,
*,
@@ -318,7 +305,7 @@ class AutoHypergraph(
"""
return DEFAULT_HYPERGRAPH_PROMPT
- def _create_empty_instance(self) -> "AutoHypergraph[Node, Edge]":
+ def _create_empty_instance(self) -> "AutoHypergraph[NodeSchema, EdgeSchema]":
"""Creates a new empty AutoHypergraph instance with the same configuration.
Overrides parent method to handle AutoHypergraph-specific parameters.
@@ -391,12 +378,12 @@ class AutoHypergraph(
)
@property
- def nodes(self) -> List[Node]:
+ def nodes(self) -> List[NodeSchema]:
"""Returns the current node collection."""
return self._node_memory.items
@property
- def edges(self) -> List[Edge]:
+ def edges(self) -> List[EdgeSchema]:
"""Returns the current hyperedge collection."""
return self._edge_memory.items
@@ -411,15 +398,43 @@ class AutoHypergraph(
if self.extraction_mode == "two_stage":
raw_graph = self._extract_data_by_two_stage(text)
elif self.extraction_mode == "one_stage":
- raise NotImplementedError(
- "Single-stage extraction not yet supported for AutoHypergraph."
- )
+ raw_graph = self._extract_data_by_one_stage(text)
else:
raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")
# Prune dangling hyperedges to ensure consistency
return self._prune_dangling_edges(raw_graph)
+ def _extract_data_by_one_stage(self, text: str) -> AutoHypergraphSchema:
+ """Extract nodes and edges simultaneously using single LLM call.
+
+ Args:
+ text: Input text.
+
+ Returns:
+ Raw extracted hypergraph data.
+ """
+ prompt_template = ChatPromptTemplate.from_template(
+ f"{self.prompt}{{chunk_text}}"
+ )
+ llm_chain = prompt_template | self.llm_client.with_structured_output(
+ self.graph_schema
+ )
+
+ # Extract from single chunk or multiple chunks
+ if len(text) <= self.chunk_size:
+ graph = llm_chain.invoke({"chunk_text": text})
+ graph_list = [graph]
+ else:
+ chunks = self.text_splitter.split_text(text)
+ inputs = [{"chunk_text": chunk} for chunk in chunks]
+ graph_list = llm_chain.batch(
+ inputs, config={"max_concurrency": self.max_workers}
+ )
+
+ # Merge multiple hypergraphs
+ return self.merge_batch_data(graph_list)
+
def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:
"""Extract nodes first, then hyperedges with node context (batch processing).
@@ -453,7 +468,9 @@ class AutoHypergraph(
# 5. Global Merge
return self.merge_batch_data(partial_hypergraphs)
- def _extract_nodes_batch(self, chunks: List[str]) -> List[NodeListSchema[Node]]:
+ def _extract_nodes_batch(
+ self, chunks: List[str]
+ ) -> List[NodeListSchema[NodeSchema]]:
"""Batch extract nodes from multiple text chunks."""
prompt_template = ChatPromptTemplate.from_template(
f"{self.node_prompt}{{chunk_text}}"
@@ -466,8 +483,8 @@ class AutoHypergraph(
return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})
def _extract_edges_batch(
- self, chunks: List[str], node_lists: List[NodeListSchema[Node]]
- ) -> List[EdgeListSchema[Edge]]:
+ self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]
+ ) -> List[EdgeListSchema[EdgeSchema]]:
"""Batch extract hyperedges using corresponding node lists as context."""
inputs = []
for chunk, node_list in zip(chunks, node_lists):
@@ -542,7 +559,7 @@ class AutoHypergraph(
def merge_batch_data(
self,
data_list_or_tuple: List[AutoHypergraphSchema]
- | Tuple[List[List[Node]], List[List[Edge]]],
+ | Tuple[List[List[NodeSchema]], List[List[EdgeSchema]]],
) -> AutoHypergraphSchema:
"""Merge multiple hypergraphs or node/edge tuples into one.
@@ -620,7 +637,7 @@ class AutoHypergraph(
query: str,
top_k_nodes: int = 3,
top_k_edges: int = 3,
- ) -> Tuple[List[Node], List[Edge]]:
+ ) -> Tuple[List[NodeSchema], List[EdgeSchema]]:
"""Unified hypergraph search interface.
Retrieves nodes and hyperedges semantically related to the query.
@@ -645,8 +662,8 @@ class AutoHypergraph(
"At least one of top_k_nodes or top_k_edges must be positive."
)
- nodes: List[Node] = []
- edges: List[Edge] = []
+ nodes: List[NodeSchema] = []
+ edges: List[EdgeSchema] = []
if top_k_nodes > 0:
if not self._node_memory.has_index():
@@ -660,11 +677,11 @@ class AutoHypergraph(
return nodes, edges
- def search_nodes(self, query: str, top_k: int = 3) -> List[Node]:
+ def search_nodes(self, query: str, top_k: int = 3) -> List[NodeSchema]:
"""Semantic search for nodes/entities only."""
return self._node_memory.search(query=query, top_k=top_k)
- def search_edges(self, query: str, top_k: int = 3) -> List[Edge]:
+ def search_edges(self, query: str, top_k: int = 3) -> List[EdgeSchema]:
"""Semantic search for hyperedges/relationships only."""
return self._edge_memory.search(query=query, top_k=top_k)
@@ -782,193 +799,3 @@ class AutoHypergraph(
self._edge_memory.load_index(str(edge_index_path))
except Exception as e:
logger.warning(f"Failed to load edge index: {e}")
-
-class EdgeAutoHypergraph(AutoHypergraph[Node, Edge], Generic[Node, Edge]):
- """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.
-
- Extraction Strategy: Two-Stage (Edge-First)
- 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).
- 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.
-
- Suitable for:
- - Narrative-driven knowledge graphs (Theme -> Character)
- - Event-driven extraction (Event -> Participant)
- """
-
- def __init__(
- self,
- node_schema: Type[Node],
- edge_schema: Type[Edge],
- node_key_extractor: Callable[[Node], str],
- edge_key_extractor: Callable[[Edge], str],
- nodes_in_edge_extractor: Callable[[Edge], Tuple[str, ...]],
- llm_client: BaseChatModel,
- embedder: Embeddings,
- *,
- prompt_for_node_extraction: str = "",
- **kwargs: Any,
- ):
- """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.
-
- Args:
- prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".
- (Extract Nodes given Edges).
- """
- super().__init__(
- node_schema=node_schema,
- edge_schema=edge_schema,
- node_key_extractor=node_key_extractor,
- edge_key_extractor=edge_key_extractor,
- nodes_in_edge_extractor=nodes_in_edge_extractor,
- llm_client=llm_client,
- embedder=embedder,
- prompt_for_node_extraction=prompt_for_node_extraction,
- **kwargs,
- )
-
- # Build Stage 2 Schema (Edges containing nested Nodes)
- # We assume the edge schema has a 'participants' field (List[str]).
- # We create a new model where 'participants' is List[NodeSchema].
-
- # 1. Base fields from EdgeSchema (excluding participants)
- valid_fields = {
- k: (v.annotation, v)
- for k, v in edge_schema.model_fields.items()
- if k != "participants"
- }
-
- # 2. Add Nested Participants field
- valid_fields["participants"] = (
- List[node_schema],
- Field(description="List of entities participating in this concept/edge.")
- )
-
- # 3. Create Contextual Edge Model
- self.contextual_edge_schema = create_model(
- f"Contextual{edge_schema.__name__}",
- **valid_fields
- )
-
- # 4. Create Result List Model
- self.stage2_result_schema = create_model(
- f"{edge_schema.__name__}FirstResult",
- items=(List[self.contextual_edge_schema], Field(default_factory=list))
- )
-
- # Set default prompt if not provided
- if not prompt_for_node_extraction:
- self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT
-
- def _extract_data(self, text: str) -> AutoHypergraphSchema:
- """Main extraction logic dispatcher (Edge-First)."""
- if self.extraction_mode == "two_stage":
- raw_graph = self._extract_data_by_two_stage(text)
- elif self.extraction_mode == "one_stage":
- raise NotImplementedError(
- "Single-stage extraction not yet supported for EdgeAutoHypergraph."
- )
- else:
- raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")
-
- # Note: Dangle pruning might be less critical here since nodes are derived from edges,
- # but we keep it for consistency if nodes are merged globally.
- return self._prune_dangling_edges(raw_graph)
-
- def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:
- """Extract edges/themes first, then entities within those contexts (batch processing).
-
- Process:
- 1. Split text into chunks.
- 2. Batch extract Edge Concepts (Theme Strings) for all chunks.
- 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).
- 4. Merge all partial results into one global hypergraph.
- """
- # 1. Prepare chunks
- if len(text) <= self.chunk_size:
- chunks = [text]
- else:
- chunks = self.text_splitter.split_text(text)
-
- if self.verbose:
- logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")
-
- # 2. Stage 1: Identify Edge Concepts (Strings)
- edge_concepts_lists = self._extract_edge_concepts_batch(chunks)
-
- # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)
- chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)
-
- # 4. Global Merge
- # chunk_results is List[AutoHypergraphSchema]
- return self.merge_batch_data(chunk_results)
-
- def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:
- """Stage 1: Batch extract edge concept strings."""
- # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts
- # We assume the prompt is geared towards finding a LIST of themes/concepts.
-
- prompt_template = ChatPromptTemplate.from_template(
- f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"
- )
- chain = prompt_template | self.llm_client.with_structured_output(EdgeDescriptionList)
-
- inputs = [{"chunk_text": chunk} for chunk in chunks]
- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
-
- return [res.items if res else [] for res in results]
-
- def _extract_details_batch(self, chunks: List[str], edge_concepts_list: List[List[str]]) -> List[AutoHypergraphSchema]:
- """Generic Stage 2: Extract entities and details given identified edge concepts."""
-
- # 1. Prepare Inputs
- inputs = []
- valid_map = {}
- for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):
- if concepts:
- concepts_str = "- " + "\n- ".join(concepts)
- inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})
- valid_map[len(inputs) - 1] = i
-
- if not inputs:
- return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]
-
- # 2. Chain Execution
- # Uses self.node_prompt as the instruction for contextual node extraction
- prompt = ChatPromptTemplate.from_template(
- f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"
- )
- chain = prompt | self.llm_client.with_structured_output(self.stage2_result_schema)
-
- results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
-
- # 3. Transform Nested Result -> Flat AutoHypergraphSchema
- final_schemas = [self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))]
-
- for i, result in enumerate(results):
- if not result or not result.items:
- continue
-
- original_idx = valid_map[i]
- current_nodes = []
- current_edges = []
-
- for item in result.items:
- # item is a ContextualEdge (has nested participants=List[Node])
-
- # Extract Nodes
- nested_nodes = item.participants
- node_names = []
- for n in nested_nodes:
- current_nodes.append(n)
- node_names.append(self.node_key_extractor(n))
-
- # Reconstruct Flat Edge
- edge_data = item.model_dump()
- edge_data["participants"] = sorted(node_names) # Ensure sorted for consistency
-
- new_edge = self.edge_schema(**edge_data)
- current_edges.append(new_edge)
-
- final_schemas[original_idx] = self.graph_schema(nodes=current_nodes, edges=current_edges)
-
- return final_schemas
diff --git a/hyperextract/hypergraphs/cog_rag.py b/hyperextract/hypergraphs/cog_rag.py
index f363ecc..525ef1c 100644
--- a/hyperextract/hypergraphs/cog_rag.py
+++ b/hyperextract/hypergraphs/cog_rag.py
@@ -3,15 +3,16 @@ COG_RAG: Cognitive-Inspired Dual-Hypergraph RAG System Pattern
Extracts and manages Theme-Entity relationships where Themes act as Hyperedges connecting multiple Entities.
"""
-from typing import List
-from pydantic import BaseModel, Field
-from hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchema, EdgeAutoHypergraph
+from typing import List, Type, Callable, Tuple, Any
+from pydantic import BaseModel, Field, create_model
+from hyperextract.hypergraphs.base import AutoHypergraph, AutoHypergraphSchema
from hyperextract.utils.logging import logger
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.embeddings import Embeddings
from langchain_core.language_models import BaseChatModel
from ontomem.merger import CustomRuleMerger
+
# ============================================================================
# Node Schema
# ============================================================================
@@ -68,10 +69,29 @@ class EdgeSchema(BaseModel):
description="Numerical score indicating relationship strength (1-10)",
)
+
+class EdgeDescriptionList(BaseModel):
+ """Intermediate schema for identifying edge concepts first."""
+
+ items: List[str] = Field(
+ description="List of primary edge concepts/themes identified"
+ )
+
+
# ============================================================================
# Extraction Prompts
# ============================================================================
+
+DEFAULT_CONTEXTUAL_NODE_PROMPT = """-Goal-
+You are given a list of "Primary Concepts" (Themes/Events/Relationships) identified from the text.
+Your task is to Populate these concepts with their specific Participant Entities.
+
+CRITICAL RULES:
+1. Extract ALL key participants for each concept.
+2. Nodes must be specific entities (People, Organizations, Locations, Concepts, etc.).
+"""
+
COG_RAG_THEME_PROMPT = """
-Goal-
Analyze the text to identify the primary themes (narrative arcs).
@@ -149,6 +169,211 @@ Merge strategy:
# ============================================================================
+class EdgeAutoHypergraph(AutoHypergraph[NodeSchema, EdgeSchema]):
+ """EdgeAutoHypergraph - Edge-First extraction: Extract relationships/themes first, then entities.
+
+ Extraction Strategy: Two-Stage (Edge-First)
+ 1. Stage 1: Extract all Edges/Concepts (high-level themes, events, narratives).
+ 2. Stage 2: Extract Nodes (entities) that participate in those specific Edges.
+
+ Suitable for:
+ - Narrative-driven knowledge graphs (Theme -> Character)
+ - Event-driven extraction (Event -> Participant)
+ """
+
+ def __init__(
+ self,
+ node_schema: Type[NodeSchema],
+ edge_schema: Type[EdgeSchema],
+ node_key_extractor: Callable[[NodeSchema], str],
+ edge_key_extractor: Callable[[EdgeSchema], str],
+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, ...]],
+ llm_client: BaseChatModel,
+ embedder: Embeddings,
+ *,
+ prompt_for_node_extraction: str = "",
+ **kwargs: Any,
+ ):
+ """Initialize EdgeAutoHypergraph and build dynamic Stage 2 schemas.
+
+ Args:
+ prompt_for_node_extraction: In pure Edge-First context, this is the "Stage 2 Prompt".
+ (Extract Nodes given Edges).
+ """
+ super().__init__(
+ node_schema=node_schema,
+ edge_schema=edge_schema,
+ node_key_extractor=node_key_extractor,
+ edge_key_extractor=edge_key_extractor,
+ nodes_in_edge_extractor=nodes_in_edge_extractor,
+ llm_client=llm_client,
+ embedder=embedder,
+ prompt_for_node_extraction=prompt_for_node_extraction,
+ **kwargs,
+ )
+
+ # Build Stage 2 Schema (Edges containing nested Nodes)
+ # We assume the edge schema has a 'participants' field (List[str]).
+ # We create a new model where 'participants' is List[NodeSchema].
+
+ # 1. Base fields from EdgeSchema (excluding participants)
+ valid_fields = {
+ k: (v.annotation, v)
+ for k, v in edge_schema.model_fields.items()
+ if k != "participants"
+ }
+
+ # 2. Add Nested Participants field
+ valid_fields["participants"] = (
+ List[node_schema],
+ Field(description="List of entities participating in this concept/edge."),
+ )
+
+ # 3. Create Contextual Edge Model
+ self.contextual_edge_schema = create_model(
+ f"Contextual{edge_schema.__name__}", **valid_fields
+ )
+
+ # 4. Create Result List Model
+ self.stage2_result_schema = create_model(
+ f"{edge_schema.__name__}FirstResult",
+ items=(List[self.contextual_edge_schema], Field(default_factory=list)),
+ )
+
+ # Set default prompt if not provided
+ if not prompt_for_node_extraction:
+ self.node_prompt = DEFAULT_CONTEXTUAL_NODE_PROMPT
+
+ def _extract_data(self, text: str) -> AutoHypergraphSchema:
+ """Main extraction logic dispatcher (Edge-First)."""
+ if self.extraction_mode == "two_stage":
+ raw_graph = self._extract_data_by_two_stage(text)
+ elif self.extraction_mode == "one_stage":
+ raise NotImplementedError(
+ "Single-stage extraction not yet supported for EdgeAutoHypergraph."
+ )
+ else:
+ raise ValueError(f"Invalid extraction_mode: {self.extraction_mode}")
+
+ # Note: Dangle pruning might be less critical here since nodes are derived from edges,
+ # but we keep it for consistency if nodes are merged globally.
+ return self._prune_dangling_edges(raw_graph)
+
+ def _extract_data_by_two_stage(self, text: str) -> AutoHypergraphSchema:
+ """Extract edges/themes first, then entities within those contexts (batch processing).
+
+ Process:
+ 1. Split text into chunks.
+ 2. Batch extract Edge Concepts (Theme Strings) for all chunks.
+ 3. Batch extract Nodes and Full Edges for all chunks (using Edge Concepts as context).
+ 4. Merge all partial results into one global hypergraph.
+ """
+ # 1. Prepare chunks
+ if len(text) <= self.chunk_size:
+ chunks = [text]
+ else:
+ chunks = self.text_splitter.split_text(text)
+
+ if self.verbose:
+ logger.info(f"Edge-First Extraction: Processing {len(chunks)} chunks...")
+
+ # 2. Stage 1: Identify Edge Concepts (Strings)
+ edge_concepts_lists = self._extract_edge_concepts_batch(chunks)
+
+ # 3. Stage 2: Extract Nodes and Full Edges (Context-aware)
+ chunk_results = self._extract_details_batch(chunks, edge_concepts_lists)
+
+ # 4. Global Merge
+ # chunk_results is List[AutoHypergraphSchema]
+ return self.merge_batch_data(chunk_results)
+
+ def _extract_edge_concepts_batch(self, chunks: List[str]) -> List[List[str]]:
+ """Stage 1: Batch extract edge concept strings."""
+ # This uses the 'prompt_for_edge_extraction' (self.edge_prompt) to find concepts
+ # We assume the prompt is geared towards finding a LIST of themes/concepts.
+
+ prompt_template = ChatPromptTemplate.from_template(
+ f"{self.edge_prompt}\n\n### Source Text:\n{{chunk_text}}"
+ )
+ chain = prompt_template | self.llm_client.with_structured_output(
+ EdgeDescriptionList
+ )
+
+ inputs = [{"chunk_text": chunk} for chunk in chunks]
+ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
+
+ return [res.items if res else [] for res in results]
+
+ def _extract_details_batch(
+ self, chunks: List[str], edge_concepts_list: List[List[str]]
+ ) -> List[AutoHypergraphSchema]:
+ """Generic Stage 2: Extract entities and details given identified edge concepts."""
+
+ # 1. Prepare Inputs
+ inputs = []
+ valid_map = {}
+ for i, (chunk, concepts) in enumerate(zip(chunks, edge_concepts_list)):
+ if concepts:
+ concepts_str = "- " + "\n- ".join(concepts)
+ inputs.append({"chunk_text": chunk, "edge_concepts": concepts_str})
+ valid_map[len(inputs) - 1] = i
+
+ if not inputs:
+ return [self.graph_schema(nodes=[], edges=[]) for _ in chunks]
+
+ # 2. Chain Execution
+ # Uses self.node_prompt as the instruction for contextual node extraction
+ prompt = ChatPromptTemplate.from_template(
+ f"{self.node_prompt}\n\n### Primary Concepts/Themes:\n{{edge_concepts}}\n\n### Source Text:\n{{chunk_text}}"
+ )
+ chain = prompt | self.llm_client.with_structured_output(
+ self.stage2_result_schema
+ )
+
+ results = chain.batch(inputs, config={"max_concurrency": self.max_workers})
+
+ # 3. Transform Nested Result -> Flat AutoHypergraphSchema
+ final_schemas = [
+ self.graph_schema(nodes=[], edges=[]) for _ in range(len(chunks))
+ ]
+
+ for i, result in enumerate(results):
+ if not result or not result.items:
+ continue
+
+ original_idx = valid_map[i]
+ current_nodes = []
+ current_edges = []
+
+ for item in result.items:
+ # item is a ContextualEdge (has nested participants=List[Node])
+
+ # Extract Nodes
+ nested_nodes = item.participants
+ node_names = []
+ for n in nested_nodes:
+ current_nodes.append(n)
+ node_names.append(self.node_key_extractor(n))
+
+ # Reconstruct Flat Edge
+ edge_data = item.model_dump()
+ edge_data["participants"] = sorted(
+ node_names
+ ) # Ensure sorted for consistency
+
+ new_edge = self.edge_schema(**edge_data)
+ current_edges.append(new_edge)
+
+ final_schemas[original_idx] = self.graph_schema(
+ nodes=current_nodes, edges=current_edges
+ )
+
+ return final_schemas
+
+
+# ============================================================================
+
+
class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
"""
Layer 1 of Cog_RAG: Theme-First Extraction.
@@ -228,6 +453,7 @@ class Cog_RAG_ThemeLayer(EdgeAutoHypergraph[NodeSchema, ThemeSchema]):
verbose=verbose,
)
+
# ============================================================================
# Detail Hypergraph (Layer 2: Micro Relations)
# ============================================================================
@@ -295,15 +521,16 @@ class Cog_RAG_DetailLayer(AutoHypergraph[NodeSchema, EdgeSchema]):
# Main System Class (Wrapper)
# ============================================================================
+
class Cog_RAG:
"""
Cognitive-Inspired Dual-Hypergraph RAG System.
-
+
Combines two Hypergraph Layers:
1. Theme Layer (Cog_RAG_ThemeLayer): Captures macro narratives and themes.
2. Detail Layer (Cog_RAG_DetailLayer): Captures micro entity relationships.
"""
-
+
def __init__(
self,
llm_client: BaseChatModel,
@@ -319,7 +546,7 @@ class Cog_RAG:
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
max_workers=max_workers,
- verbose=verbose
+ verbose=verbose,
)
self.detail_layer = Cog_RAG_DetailLayer(
llm_client=llm_client,
@@ -327,7 +554,7 @@ class Cog_RAG:
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
max_workers=max_workers,
- verbose=verbose
+ verbose=verbose,
)
self.llm = llm_client
self.verbose = verbose
@@ -337,7 +564,7 @@ class Cog_RAG:
if self.verbose:
logger.info("Cog_RAG: Feeding text to Theme Layer...")
self.theme_layer.feed_text(text)
-
+
if self.verbose:
logger.info("Cog_RAG: Feeding text to Detail Layer...")
self.detail_layer.feed_text(text)
@@ -358,36 +585,35 @@ class Cog_RAG:
"""
# 1. Macro Context: Themes
themes = self.theme_layer.search_edges(query, top_k=top_k_themes)
-
+
# 2. Micro Context: Entities (Nodes)
# Note: The user requested searching for "Specific Entities" in the second step.
entities = self.detail_layer.search_nodes(query, top_k=top_k_entities)
-
- return {
- "themes": themes,
- "entities": entities
- }
+
+ return {"themes": themes, "entities": entities}
def chat(self, query: str, top_k_themes: int = 3, top_k_entities: int = 3):
"""Generate an answer using context from both layers."""
results = self.search(query, top_k_themes, top_k_entities)
-
+
context_parts = []
if results["themes"]:
context_parts.append("=== MACRO THEMES (Narrative Context) ===")
for t in results["themes"]:
- context_parts.append(f"- {t.description} (Involved: {', '.join(t.participants)})")
-
+ context_parts.append(
+ f"- {t.description} (Involved: {', '.join(t.participants)})"
+ )
+
if results["entities"]:
context_parts.append("\n=== MICRO ENTITIES (Specific Details) ===")
for e in results["entities"]:
context_parts.append(f"- [{e.type}] {e.name}: {e.description}")
-
+
if not context_parts:
context = "No relevant information found."
else:
context = "\n".join(context_parts)
-
+
prompt = ChatPromptTemplate.from_template(
"You are an intelligent assistant using a Dual-Layer Hypergraph Knowledge Base.\n"
"Use the provided Themes (Macro) and Key Entities (Micro) to answer the user's question comprehensively.\n\n"
@@ -395,13 +621,14 @@ class Cog_RAG:
"Question: {question}\n\n"
"Answer:"
)
-
+
chain = prompt | self.llm
return chain.invoke({"context": context, "question": query})
-
+
def dump(self, folder_path: str):
"""Save both extracted graphs."""
import os
+
self.theme_layer.dump(os.path.join(folder_path, "theme_layer"))
self.detail_layer.dump(os.path.join(folder_path, "detail_layer"))
@@ -422,4 +649,3 @@ class Cog_RAG:
def edges(self):
"""Return all edges (Themes + Relations)."""
return self.theme_layer.edges + self.detail_layer.edges
-
diff --git a/hyperextract/hypergraphs/hypergraph_rag.py b/hyperextract/hypergraphs/hypergraph_rag.py
index 7903039..ac4cd39 100644
--- a/hyperextract/hypergraphs/hypergraph_rag.py
+++ b/hyperextract/hypergraphs/hypergraph_rag.py
@@ -1,14 +1,24 @@
-"""
-HyperGraph_RAG: Integration of EdgeAutoHypergraph (Edge-First Extraction).
-Matches the structure of Hyper_RAG regarding Schemas and Mergers.
+"""HyperGraph_RAG implementation using AutoHypergraph as the core engine.
+
+This module provides a specialized implementation of the HyperGraphRAG algorithm
+using the AutoHypergraph framework, designed for extracting semantic knowledge
+represented as hypergraphs where hyperedges ("knowledge segments") connect
+multiple entities within specific contexts or events.
+
+The HyperGraphRAG approach represents knowledge as contextual relationships where
+each hyperedge encodes a complete semantic unit (sentence or event), maintaining
+the natural information flow and contextual dependencies of the source material.
"""
from typing import List
+from hashlib import md5
from pydantic import BaseModel, Field
-from hyperextract.hypergraphs.base import EdgeAutoHypergraph
from langchain_core.embeddings import Embeddings
from langchain_core.language_models import BaseChatModel
-from ontomem.merger import CustomRuleMerger
+from langchain_core.prompts.chat import ChatPromptTemplate
+from ontomem.merger import MergeStrategy
+
+from .base import AutoHypergraph, AutoHypergraphSchema
# ============================================================================
# Node Schema
@@ -16,14 +26,19 @@ from ontomem.merger import CustomRuleMerger
class NodeSchema(BaseModel):
- """Represents an entity extracted from the source text."""
+ """Represents an entity extracted from the source text within a specific context."""
- name: str = Field(description="Name of the entity")
+ name: str = Field(
+ description="Name of the entity. Use the same language as the input text. If English, capitalize the name."
+ )
type: str = Field(
- description="Entity type (person, organization, geo, event, role, concept, etc.)",
+ description="Type of the entity (e.g., organization, person, geo, event, category, concept, object)."
)
description: str = Field(
- description="Comprehensive description of the entity's attributes and activities",
+ description="Comprehensive description of the entity's attributes and activities strictly within the current knowledge segment."
+ )
+ key_score: int = Field(
+ description="A score from 0 to 100 indicating the importance of the entity within this context."
)
@@ -33,73 +48,121 @@ class NodeSchema(BaseModel):
class EdgeSchema(BaseModel):
- """Represents a relationship or event connecting multiple entities (low-order or high-order)."""
+ """Represents a knowledge segment (hyper-edge) that connects multiple entities through a specific context or event."""
- participants: List[str] = Field(
- description="Names of entities involved in this relationship"
+ knowledge_segment: str = Field(
+ description="A complete sentence or phrase from the text that captures a specific relationship, event, or context connecting the entities."
)
- description: str = Field(
- description="Detailed explanation of the relationship or event"
- )
- keywords: List[str] = Field(
- description="List of keywords summarizing the relationship themes (e.g., ['conflict', 'trade', 'alliance'])",
+ completeness_score: int = Field(
+ description="A score from 0 to 10 indicating how complete and meaningful this knowledge segment is on its own."
)
- strength: int = Field(
- ge=1,
- le=10,
- description="Numerical score indicating relationship strength (1-10)",
+ related_entities: List[str] = Field(
+ description="A list of all entity names involved in or referenced by this knowledge segment."
)
-# ============================================================================
-# Extraction Prompts (Edge-First)
-# ============================================================================
+# Below Schema is used for Information Extraction with LLM
+class TEMP_EdgeSchema(BaseModel):
+ """Represents a knowledge segment (hyper-edge) that connects multiple entities through a specific context or event."""
-HyperGraph_RAG_THEME_EXTRACTION_PROMPT = """
--Goal-
-Identify the key "Themes", "Narratives", or "Complex Events" present in the text.
-A Theme/Event should represent a significant interaction or grouping involving multiple participants.
+ knowledge_segment: str = Field(
+ description="A complete sentence or phrase from the text that captures a specific relationship, event, or context connecting the entities."
+ )
+ completeness_score: int = Field(
+ description="A score from 0 to 10 indicating how complete and meaningful this knowledge segment is on its own."
+ )
+ related_entities: List[NodeSchema] = Field(
+ description="A list of all entities involved in or referenced by this knowledge segment."
+ )
--Output-
-Return a list of these Themes/Events as descriptive strings.
-Do not list participants data formats yet, just the essence of the event (e.g., "The secret meeting at Titan Station", "The ambush in the Aurora Sector").
-"""
-HyperGraph_RAG_CONTEXT_NODE_PROMPT = """
--Goal-
-You are an expert knowledge extractor.
-You have been given a list of "Themes/Events" (Hyperedges) that occurred in the text.
-Your job is to identify the specific ENTITIES (participants) involved in each theme.
+class TEMP_EdgeListSchema(BaseModel):
+ """Intermediate schema for batch hyperedge extraction."""
--Output-
-For each Theme, list the entities that participated.
-"""
+ items: List[TEMP_EdgeSchema] = Field(
+ description="List of extracted hyperedges with their knowledge segments and related entities."
+ )
# ============================================================================
-# Merge Templates for LLM.CUSTOM_RULE Strategy
+# Extraction Prompts
# ============================================================================
-Hyper_RAG_NODE_MERGE_RULE = """You are an intelligent data merging assistant.
-You will receive a list of objects representing the same Entity.
-
-Your task is to merge them into a SINGLE object exactly matching the schema.
-
-Merge strategy:
-1. **name/type**: Keep the most frequent or precise value.
-2. **description**: Synthesize a single, comprehensive description containing all unique details from the input descriptions. Write it in the third person. Resolve any contradictions coherently.
-"""
-
-Hyper_RAG_EDGE_MERGE_RULE = """You are an intelligent data merging assistant.
-You will receive a list of objects representing the same Hyperedge (Relationship/Event).
-
-Your task is to merge them into a SINGLE object exactly matching the schema.
-
-Merge strategy:
-1. **participants**: Keep the list (they should be identical for the same hyperedge key).
-2. **description**: Synthesize a single, comprehensive description covering the relationship dynamics from all inputs. Write it in the third person.
-3. **keywords**: Combine keyword lists from all inputs, removing duplicates.
-4. **strength**: Calculate the average of the input strengths (round to nearest integer).
+HyperGraph_RAG_EDGE_EXTRACTION_PROMPT = """
+You are an expert information extraction system specializing in Hypergraph RAG.
+Your goal is to extract "Hyper-edges" from the provided text.
+
+A **Hyper-edge** represents a specific "knowledge segment" (a sentence or event context) that connects multiple entities together.
+
+# Instructions
+1. **Analyze the Text**: Break the text down into complete, meaningful knowledge segments (sentences or clauses describing an event/relationship).
+2. **Extract Entities**: For *each* segment, identify all relevant entities involved.
+3. **Structure the Output**: For each segment, output an `EdgeSchema` containing:
+ - The `knowledge_segment` text.
+ - A `completeness_score` (0-10).
+ - A list of `related_entities` found strictly within that segment.
+
+# Extraction Rules
+- **Entity Name**: Keep original language. Capitalize if English.
+- **Entity Description**: Describe what the entity is doing or how it represents within *that specific segment*.
+- **Entity Type**: Common types include organization, person, geo, event, category, concept, object.
+
+# Few-Shot Examples
+
+## Example 1
+**Input Text**:
+"Alex clenched his jaw, the buzz of frustration dull against the backdrop of Taylor's authoritarian certainty."
+
+**Output**:
+[
+ {
+ "knowledge_segment": "Alex clenched his jaw, the buzz of frustration dull against the backdrop of Taylor's authoritarian certainty.",
+ "completeness_score": 8,
+ "related_entities": [
+ {
+ "name": "Alex",
+ "type": "person",
+ "description": "Alex is a person displaying frustration by clenching his jaw.",
+ "key_score": 90
+ },
+ {
+ "name": "Taylor",
+ "type": "person",
+ "description": "Taylor is a person exhibiting authoritarian certainty causing Alex's frustration.",
+ "key_score": 85
+ }
+ ]
+ }
+]
+
+## Example 2
+**Input Text**:
+"The device could change the game for us," Taylor said, observing the machine with reverence.
+
+**Output**:
+[
+ {
+ "knowledge_segment": "\"The device could change the game for us,\" Taylor said, observing the machine with reverence.",
+ "completeness_score": 9,
+ "related_entities": [
+ {
+ "name": "Taylor",
+ "type": "person",
+ "description": "Taylor is a person who believes the device is game-changing and observes it with reverence.",
+ "key_score": 95
+ },
+ {
+ "name": "device",
+ "type": "object",
+ "description": "An object that Taylor believes could change the game.",
+ "key_score": 90
+ }
+ ]
+ }
+]
+
+\n\n
+### Source Text:
"""
# ============================================================================
@@ -107,13 +170,34 @@ Merge strategy:
# ============================================================================
-class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):
- """
- HyperGraph_RAG: Hypergraph-based RAG with Edge-First Extraction Strategy.
+class HyperGraph_RAG(AutoHypergraph[NodeSchema, EdgeSchema]):
+ """HyperGraphRAG extractor using semantic knowledge segments as hyperedges.
+
+ This class implements the HyperGraphRAG algorithm which models knowledge as a hypergraph
+ where each hyperedge represents a complete "knowledge segment" (atomic semantic unit)
+ that connects multiple related entities. Unlike traditional knowledge graphs that represent
+ pairwise relationships, hypergraphs can naturally express n-ary relationships and maintain
+ the original context and semantic integrity of information.
+
+ The extraction process simultaneously identifies both knowledge segments (hyperedges) and
+ the entities involved within each segment, followed by deduplication and merging across
+ multiple text chunks.
- Difference from Hyper_RAG:
- - Uses EdgeAutoHypergraph (extracts Themes first, then Entities).
- - Suitable for high-level narrative understanding where Themes map to specific Actors.
+ **Schema Components:**
+
+ - **NodeSchema**: Represents an entity (node) extracted from the source text within a specific context.
+ - `name` (str): The entity name, maintaining original language and capitalization.
+ - `type` (str): Entity type such as 'person', 'organization', 'event', 'location', 'concept', etc.
+ - `description` (str): Comprehensive description of the entity's attributes and activities within the knowledge segment.
+ - `key_score` (int): Importance score (0-100) indicating the entity's significance in context.
+
+ - **EdgeSchema**: Represents a knowledge segment (hyper-edge) that connects multiple entities through
+ a specific context or event.
+ - `knowledge_segment` (str): The actual text span (sentence or phrase) from source material capturing
+ the relationship, event, or context connecting entities.
+ - `completeness_score` (int): Quality score (0-10) indicating how complete and meaningful the segment
+ is as a standalone knowledge unit.
+ - `related_entities` (List[str]): Names of all entities involved in or referenced by this knowledge segment.
"""
def __init__(
@@ -125,38 +209,25 @@ class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):
max_workers: int = 10,
verbose: bool = True,
):
- """
- Initialize HyperGraph_RAG engine.
+ """Initialize HyperGraph_RAG extraction engine.
+
+ Configures the hypergraph extraction pipeline with LLM-based knowledge segment
+ and entity identification, along with semantic embeddings for vector-based retrieval.
Args:
- llm_client (BaseChatModel): LLM client for text generation.
- embedder (Embeddings): Embedding model for text embeddings.
- chunk_size (int): Size of text chunks for extraction.
- chunk_overlap (int): Overlap between text chunks.
- max_workers (int): Maximum number of workers for extraction.
- verbose (bool): Display detailed logs.
+ llm_client (BaseChatModel): Language model client for structured extraction.
+ embedder (Embeddings): Embedding model for semantic vector representation.
+ chunk_size (int): Text chunk size for processing (default: 2048 tokens).
+ chunk_overlap (int): Overlap between consecutive chunks to preserve context (default: 256 tokens).
+ max_workers (int): Maximum parallel workers for batch extraction (default: 10).
+ verbose (bool): Whether to display detailed extraction logs (default: True).
"""
# 1. Define Key Extractors
node_key_fn = lambda x: x.name
- edge_key_fn = lambda x: tuple(sorted(x.participants))
- nodes_in_edge_fn = lambda x: x.participants
-
- # 2. Setup Mergers
- node_merger = CustomRuleMerger(
- key_extractor=node_key_fn,
- llm_client=llm_client,
- item_schema=NodeSchema,
- rule=Hyper_RAG_NODE_MERGE_RULE,
- )
-
- edge_merger = CustomRuleMerger(
- key_extractor=edge_key_fn,
- llm_client=llm_client,
- item_schema=EdgeSchema,
- rule=Hyper_RAG_EDGE_MERGE_RULE,
- )
+ edge_key_fn = lambda x: md5(x.knowledge_segment.encode()).hexdigest()
+ nodes_in_edge_fn = lambda x: x.related_entities
- # 3. Initialize EdgeAutoHypergraph (Parent)
+ # 4. Call parent class initialization
super().__init__(
node_schema=NodeSchema,
edge_schema=EdgeSchema,
@@ -165,21 +236,82 @@ class HyperGraph_RAG(EdgeAutoHypergraph[NodeSchema, EdgeSchema]):
nodes_in_edge_extractor=nodes_in_edge_fn,
llm_client=llm_client,
embedder=embedder,
- # Prompts
- # Stage 1: Extract Themes
- prompt_for_edge_extraction=HyperGraph_RAG_THEME_EXTRACTION_PROMPT,
- # Stage 2: Extract Particles given Themes
- prompt_for_node_extraction=HyperGraph_RAG_CONTEXT_NODE_PROMPT,
- # Merger strategies
- node_strategy_or_merger=node_merger,
- edge_strategy_or_merger=edge_merger,
- # Indexing optimization
+ # Inject customized prompts for extraction
+ prompt_for_edge_extraction=HyperGraph_RAG_EDGE_EXTRACTION_PROMPT,
+ # Pass custom merger instances
+ node_strategy_or_merger=MergeStrategy.LLM.BALANCED,
+ edge_strategy_or_merger=MergeStrategy.LLM.BALANCED,
+ # Optimize indexing
node_fields_for_index=["name", "type", "description"],
- edge_fields_for_index=["keywords", "description"],
- # BaseAutoType Params
+ edge_fields_for_index=["knowledge_segment"],
+ # Other parameters
chunk_size=chunk_size,
chunk_overlap=chunk_overlap,
max_workers=max_workers,
verbose=verbose,
)
+ def _extract_data(self, text: str) -> AutoHypergraphSchema[NodeSchema, EdgeSchema]:
+ """Extract hypergraph knowledge from text via knowledge segment identification.
+
+ Performs simultaneous extraction of knowledge segments (hyperedges) and associated entities
+ from the input text. The extraction process:
+ 1. Segments the text into knowledge-bearing units (sentences/clauses)
+ 2. Identifies all entities referenced within each knowledge segment
+ 3. Merges and deduplicates extracted components across multiple chunks
+ 4. Prunes dangling edges (hyperedges without valid entity references)
+
+ Args:
+ text (str): Source text for hypergraph extraction.
+
+ Returns:
+ AutoHypergraphSchema[NodeSchema, EdgeSchema]: Extracted hypergraph containing
+ deduplicated and merged entities (nodes) and knowledge segments (hyperedges).
+ """
+ # 1. Extract Hyperedges
+
+ prompt_template = ChatPromptTemplate.from_template(
+ f"{self.prompt}{{chunk_text}}"
+ )
+ llm_chain = prompt_template | self.llm_client.with_structured_output(
+ TEMP_EdgeListSchema
+ )
+
+ # Extract from single chunk or multiple chunks
+ if len(text) <= self.chunk_size:
+ raw_hyperedges = llm_chain.invoke({"chunk_text": text})
+ raw_hyperedges_list = [raw_hyperedges]
+ else:
+ chunks = self.text_splitter.split_text(text)
+ inputs = [{"chunk_text": chunk} for chunk in chunks]
+ raw_hyperedges_list = llm_chain.batch(
+ inputs, config={"max_concurrency": self.max_workers}
+ )
+
+ # Extract Nodes and Edges from raw_hyperedges_list
+ all_nodes_list, all_edges_list = [], []
+
+ for raw_hyperedges in raw_hyperedges_list:
+ cur_nodes, cur_edges = [], []
+ data_list = raw_hyperedges.items
+ for data in data_list:
+ data: TEMP_EdgeSchema
+ cur_nodes.extend(data.related_entities)
+ cur_edges.append(
+ EdgeSchema(
+ knowledge_segment=data.knowledge_segment,
+ completeness_score=data.completeness_score,
+ related_entities=[
+ self.node_key_extractor(node)
+ for node in data.related_entities
+ ],
+ )
+ )
+ all_nodes_list.append(cur_nodes)
+ all_edges_list.append(cur_edges)
+
+ # Merge multiple hypergraphs
+ partial_hypergraphs = (all_nodes_list, all_edges_list)
+ raw_hypergraph = self.merge_batch_data(partial_hypergraphs)
+
+ return self._prune_dangling_edges(raw_hypergraph)