Skip to content

Hypergraph is more powerful. Transform unstructured text into structured knowledge with LLMs. Graphs, hypergraphs, and spatio-temporal extractions — with one command.

Refactor AutoGraph and AutoTemporalGraph to use NodeSchema and EdgeSchema for improved type safety and clarity

yifanfeng97191cd744a41a
diff --git a/hyperextract/graphs/base.py b/hyperextract/graphs/base.py
index eaf6822..7c174fd 100644
--- a/hyperextract/graphs/base.py
+++ b/hyperextract/graphs/base.py
@@ -27,8 +27,8 @@ from ..core.base import BaseAutoType
from ..utils.logging import logger
-Node = TypeVar("Node", bound=BaseModel)
-Edge = TypeVar("Edge", bound=BaseModel)
+NodeSchema = TypeVar("NodeSchema", bound=BaseModel)
+EdgeSchema = TypeVar("EdgeSchema", bound=BaseModel)
# ==============================================================================
@@ -66,34 +66,34 @@ DEFAULT_EDGE_PROMPT = (
)
-class AutoGraphSchema(BaseModel, Generic[Node, Edge]):
+class AutoGraphSchema(BaseModel, Generic[NodeSchema, EdgeSchema]):
"""Generic schema container for graph-based knowledge patterns."""
- nodes: List[Node] = Field(default_factory=list, description="Graph nodes/entities")
- edges: List[Edge] = Field(
+ nodes: List[NodeSchema] = Field(default_factory=list, description="Graph nodes/entities")
+ edges: List[EdgeSchema] = Field(
default_factory=list, description="Graph edges/relationships"
)
-class NodeListSchema(BaseModel, Generic[Node]):
+class NodeListSchema(BaseModel, Generic[NodeSchema]):
"""Intermediate schema for batch node extraction."""
- items: List[Node] = Field(
+ items: List[NodeSchema] = Field(
default_factory=list,
description="List of identified entities or nodes found in the text.",
)
-class EdgeListSchema(BaseModel, Generic[Edge]):
+class EdgeListSchema(BaseModel, Generic[EdgeSchema]):
"""Intermediate schema for batch edge extraction."""
- items: List[Edge] = Field(
+ items: List[EdgeSchema] = Field(
default_factory=list,
description="List of identified relationships or edges found in the text.",
)
-class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
+class AutoGraph(BaseAutoType[AutoGraphSchema[NodeSchema, EdgeSchema]], Generic[NodeSchema, EdgeSchema]):
"""AutoGraph - extracts knowledge graphs with nodes and edges from text.
This pattern extracts structured knowledge graphs consisting of entities (nodes) and
@@ -134,15 +134,15 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
"""
if TYPE_CHECKING:
- graph_schema: Type[AutoGraphSchema[Node, Edge]]
+ graph_schema: Type[AutoGraphSchema[NodeSchema, EdgeSchema]]
def __init__(
self,
- node_schema: Type[Node],
- edge_schema: Type[Edge],
- node_key_extractor: Callable[[Node], str],
- edge_key_extractor: Callable[[Edge], str],
- nodes_in_edge_extractor: Callable[[Edge], Tuple[str, str]],
+ node_schema: Type[NodeSchema],
+ edge_schema: Type[EdgeSchema],
+ node_key_extractor: Callable[[NodeSchema], str],
+ edge_key_extractor: Callable[[EdgeSchema], str],
+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, str]],
llm_client: BaseChatModel,
embedder: Embeddings,
*,
@@ -290,7 +290,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
"""Returns the default prompt for one-stage graph extraction."""
return DEFAULT_GRAPH_PROMPT
- def _create_empty_instance(self) -> "AutoGraph[Node, Edge]":
+ def _create_empty_instance(self) -> "AutoGraph[NodeSchema, EdgeSchema]":
"""Creates a new empty AutoGraph instance with the same configuration as this one.
Overrides parent method to handle AutoGraph-specific parameters.
@@ -321,7 +321,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
)
@property
- def data(self) -> AutoGraphSchema[Node, Edge]:
+ def data(self) -> AutoGraphSchema[NodeSchema, EdgeSchema]:
"""Returns the current graph state (nodes and edges).
Returns:
@@ -332,7 +332,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
)
@property
- def nodes(self) -> List[Node]:
+ def nodes(self) -> List[NodeSchema]:
"""Returns the current node collection.
Returns:
@@ -341,7 +341,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
return self._node_memory.items
@property
- def edges(self) -> List[Edge]:
+ def edges(self) -> List[EdgeSchema]:
"""Returns the current edge collection.
Returns:
@@ -369,7 +369,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
self._node_memory.clear_index()
self._edge_memory.clear_index()
- def _set_data_state(self, data: AutoGraphSchema) -> None:
+ def _set_data_state(self, data: AutoGraphSchema[NodeSchema, EdgeSchema]) -> None:
"""Replace graph data with new data (full reset).
Args:
@@ -385,7 +385,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
self.clear_index()
- def _update_data_state(self, incoming_data: AutoGraphSchema) -> None:
+ def _update_data_state(self, incoming_data: AutoGraphSchema[NodeSchema, EdgeSchema]) -> None:
"""Merge incoming graph data into current state.
Args:
@@ -402,7 +402,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
# ==================== Extraction Pipeline ====================
- def _extract_data(self, text: str) -> AutoGraphSchema:
+ def _extract_data(self, text: str) -> AutoGraphSchema[NodeSchema, EdgeSchema]:
"""Main extraction logic dispatcher.
Args:
@@ -421,7 +421,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
# Prune dangling edges to ensure graph consistency
return self._prune_dangling_edges(raw_graph)
- def _extract_data_by_one_stage(self, text: str) -> AutoGraphSchema:
+ def _extract_data_by_one_stage(self, text: str) -> AutoGraphSchema[NodeSchema, EdgeSchema]:
"""Extract nodes and edges simultaneously using single LLM call.
Args:
@@ -451,7 +451,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
# Merge multiple graphs
return self.merge_batch_data(graph_list)
- def _extract_data_by_two_stage(self, text: str) -> AutoGraphSchema:
+ def _extract_data_by_two_stage(self, text: str) -> AutoGraphSchema[NodeSchema, EdgeSchema]:
"""Extract nodes first, then edges with node context (batch processing).
Process:
@@ -488,7 +488,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
# 5. Global Merge (passes tuples to merge_batch_data)
return self.merge_batch_data(partial_graphs)
- def _extract_nodes_batch(self, chunks: List[str]) -> List[NodeListSchema[Node]]:
+ def _extract_nodes_batch(self, chunks: List[str]) -> List[NodeListSchema[NodeSchema]]:
"""Batch extract nodes from multiple text chunks.
Args:
@@ -508,8 +508,8 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})
def _extract_edges_batch(
- self, chunks: List[str], node_lists: List[NodeListSchema[Node]]
- ) -> List[EdgeListSchema[Edge]]:
+ self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]
+ ) -> List[EdgeListSchema[EdgeSchema]]:
"""Batch extract edges using corresponding node lists as context.
Args:
@@ -539,7 +539,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})
- def _prune_dangling_edges(self, graph: AutoGraphSchema) -> AutoGraphSchema:
+ def _prune_dangling_edges(self, graph: AutoGraphSchema[NodeSchema, EdgeSchema]) -> AutoGraphSchema[NodeSchema, EdgeSchema]:
"""Prune edges that connect to non-existent nodes (Consistency Check).
Ensures graph consistency by removing any edges where either endpoint
@@ -584,14 +584,14 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
def merge_batch_data(
self,
- data_list_or_tuple: List[AutoGraphSchema]
- | Tuple[List[List[Node]], List[List[Edge]]],
- ) -> AutoGraphSchema:
+ data_list_or_tuple: List[AutoGraphSchema[NodeSchema, EdgeSchema]]
+ | Tuple[List[List[NodeSchema]], List[List[EdgeSchema]]],
+ ) -> AutoGraphSchema[NodeSchema, EdgeSchema]:
"""Merge multiple graphs or node/edge tuples into one.
Supports two input formats:
- List of AutoGraphSchema objects (standard format)
- - Tuple of (List[List[Node]], List[List[Edge]]) (optimization for batch processing)
+ - Tuple of (List[List[NodeSchema]], List[List[EdgeSchema]]) (optimization for batch processing)
Args:
data_list_or_tuple: Either a list of AutoGraphSchema objects or a tuple of
@@ -661,7 +661,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
top_k: int = 3,
search_nodes: bool = True,
search_edges: bool = True,
- ) -> List[Node] | List[Edge] | Tuple[List[Node], List[Edge]]:
+ ) -> List[NodeSchema] | List[EdgeSchema] | Tuple[List[NodeSchema], List[EdgeSchema]]:
"""Unified graph search interface.
By default, searches both nodes and edges simultaneously to provide comprehensive results.
@@ -675,9 +675,9 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
search_edges: Whether to search edges (default: True).
Returns:
- - Tuple[List[Node], List[Edge]] if both search_nodes and search_edges are True (default).
- - List[Node] if only search_nodes is True.
- - List[Edge] if only search_edges is True.
+ - Tuple[List[NodeSchema], List[EdgeSchema]] if both search_nodes and search_edges are True (default).
+ - List[NodeSchema] if only search_nodes is True.
+ - List[EdgeSchema] if only search_edges is True.
Raises:
ValueError: If neither search_nodes nor search_edges is True.
@@ -708,7 +708,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
edges = self.search_edges(query, top_k)
return edges
- def search_nodes(self, query: str, top_k: int = 3) -> List[Node]:
+ def search_nodes(self, query: str, top_k: int = 3) -> List[NodeSchema]:
"""Semantic search for nodes/entities only.
Args:
@@ -720,7 +720,7 @@ class AutoGraph(BaseAutoType[AutoGraphSchema[Node, Edge]], Generic[Node, Edge]):
"""
return self._node_memory.search(query=query, top_k=top_k)
- def search_edges(self, query: str, top_k: int = 3) -> List[Edge]:
+ def search_edges(self, query: str, top_k: int = 3) -> List[EdgeSchema]:
"""Semantic search for edges/relationships only.
Args:
diff --git a/hyperextract/graphs/spatial_graph.py b/hyperextract/graphs/spatial_graph.py
new file mode 100644
index 0000000..e69de29
diff --git a/hyperextract/graphs/temporal_graph.py b/hyperextract/graphs/temporal_graph.py
index bd5bcb6..eb126be 100644
--- a/hyperextract/graphs/temporal_graph.py
+++ b/hyperextract/graphs/temporal_graph.py
@@ -1,571 +1,292 @@
-"""A simple temporal graph implementation using AutoGraph as the core engine."""
+"""Generic temporal graph implementation supporting custom schemas with time-aware deduplication."""
-from typing import Optional, List
+from typing import Type, Callable, Tuple, Any, List, Optional
from datetime import datetime
-from pydantic import BaseModel, Field
-from semhash import SemHash
from langchain_core.language_models import BaseChatModel
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.embeddings import Embeddings
-from ontomem.merger import MergeStrategy, CustomRuleMerger
+from ontomem.merger import MergeStrategy, BaseMerger
-from .base import AutoGraph, AutoGraphSchema
-from ..utils.logging import logger
+from .base import AutoGraph, NodeSchema, EdgeSchema, NodeListSchema, EdgeListSchema
# ==============================================================================
-# 1. Schema Definition - Consistent with original AutoTemporalGraph implementation
+# Prompt Definition
# ==============================================================================
+GENERIC_TEMPORAL_NODE_PROMPT = """
+You are a professional entity extraction specialist.
+Your task is to extract all important entities (Nodes) from the text.
-class NodeSchema(BaseModel):
- """Schema for representing an entity."""
+# Core Principles
+1. **Comprehensiveness**: Extract persons, organizations, locations, events, concepts, and other noun-based entities.
+2. **Accuracy**: Keep entity names consistent with the source text.
+3. **Exclude Time Expressions**: **NEVER** extract dates, years, or time periods (e.g., "2023", "last year", "today") as entity nodes!
+ Time is an attribute of relationships, not a node.
+4. **Exclude Pure Numbers**: Do not extract standalone amounts or numeric values as independent nodes.
- label: str = Field(
- description=(
- "The semantic category of the entity (e.g., 'Person', 'Event', 'Location', 'Methodology', 'Position') as you understand it from the text. "
- "Use 'Relationship' objects if the concept is inherently relational or verbal (e.g., 'plans'). "
- "Prefer consistent, single-word categories where possible (e.g., 'Person', not 'Person_Entity'). "
- "Do not extract Date entities as they will be integrated in the relation."
- )
- )
- name: str = Field(
- description=(
- "The unique name or title identifying this entity, representing exactly one concept. "
- "For example, 'Yassir', 'CEO', or 'X'. Avoid combining multiple concepts (e.g., 'CEO of X'), "
- "since linking them should be done via Relationship objects. "
- "Verbs or multi-concept phrases (e.g., 'plans an escape') typically belong in Relationship objects. "
- "Do not extract Date entities as they will be integrated in the relation."
- )
- )
-
-
-class EdgeSchema(BaseModel):
- """Schema for representing a relationship between two entities."""
-
- startNode: NodeSchema = Field(
- description=(
- "The 'subject' or source entity of this relationship, which must appear in the EntitiesExtractor."
- )
- )
- endNode: NodeSchema = Field(
- description=(
- "The 'object' or target entity of this relationship, which must also appear in the EntitiesExtractor."
- )
- )
- name: str = Field(
- description=(
- "A single, canonical predicate in PRESENT TENSE capturing how the startNode and endNode relate "
- "(e.g., 'is_CEO', 'holds_position', 'is_located_in', 'works_at'). "
- "ALWAYS use present tense verbs regardless of the temporal context in the text. "
- "Avoid compound verbs (e.g., 'plans_and_executes'). "
- "If the text implies negation (e.g., 'no longer CEO'), still use the affirmative present form (e.g., 'is_CEO') "
- "and rely on 't_end' for the end date. "
- "AVOID preposition-only relation names like 'of', 'in', 'at' - use descriptive present-tense verbs instead."
- )
- )
- t_start: Optional[list[str]] = Field(
- default_factory=list,
- description=(
- "A time or interval indicating when this relationship begins or is active. "
- "Resolve relative temporal expressions based on the observation_date:\n"
- " - 'today' → exact observation_date\n"
- " - 'yesterday' → observation_date minus 1 day\n"
- " - 'this week' → Monday of observation_date's week\n"
- " - 'last week' → Monday of the week before observation_date\n"
- " - 'this month' → first day of observation_date's month\n"
- " - 'last month' → first day of the month before observation_date\n"
- " - 'this year' → January 1st of observation_date's year\n"
- " - 'last year' → January 1st of the year before observation_date\n"
- "Keep explicit dates as-is (e.g., '18-06-2024'). "
- "For example, if 'Yassir became CEO from 2023', then t_start=['01-01-2023']. "
- "This can be a single year, a date, or a resolved relative reference. "
- "Leave it [] if not specified."
- ),
- )
- t_end: Optional[list[str]] = Field(
- default_factory=list,
- description=(
- "A time or interval indicating when this relationship ceases to hold. "
- "Resolve relative temporal expressions based on the observation_date using the same rules as t_start:\n"
- " - 'today' → exact observation_date\n"
- " - 'yesterday' → observation_date minus 1 day\n"
- " - 'this week' → Monday of observation_date's week\n"
- " - etc. (same resolution rules as t_start)\n"
- "Keep explicit dates as-is. "
- "For example, if 'Yassir left his position in 2025', then t_end=['01-01-2025']. "
- "Use this field to capture any 'end action' (e.g., leaving a job, ending a marriage), "
- "while keeping the relationship name in a canonical present tense form (e.g., 'is_CEO' not 'was_CEO'). "
- "Leave it [] if no end date/time is given."
- ),
- )
- t_obs: Optional[list[str]] = Field(
- default=None,
- description=(
- "DO NOT EXTRACT OR INFER THIS FIELD. Leave it null/None. "
- "This field represents the extraction timestamp and will be populated manually by the user post-processing."
- ),
- )
- AutoTemporalGraphic_facts: Optional[list[str]] = Field(
- default_factory=list,
- description=(
- "A list of exact string copies of the AutoTemporalGraphic facts or sentences from the source text that provide evidence for this relationship. "
- "Select the specific facts from the input context that justify this edge. "
- "Do NOT paraphrase or modify the text. Copy it exactly as it appears in the source."
- ),
- )
-
-
-# ==============================================================================
-# 2. Prompt Definition - Adapted from original AutoTemporalGraph implementation
-# ==============================================================================
-
-
-AutoTemporalGraph_FACTOID_EXTRACTION_PROMPT = """
-Observation Date: {observation_date}
-
-You are an expert factoid extraction engine. Your primary function is to read a news paragraph and its associated observation date, and then decompose the text into a comprehensive list of AutoTemporalGraphic, self-contained, and temporally-grounded facts.
-
-## Task
-Given an input paragraph and an `observation_date`, generate a list of all distinct factoids present in the text.
-
-## Guidelines for Generating Temporal Factoids
-
-### 1. AutoTemporalGraphic Factoids
-- Convert compound or complex sentences into short, single-fact statements
-- Each factoid must contain exactly one piece of information or relationship
-- Ensure that each factoid is expressed directly and concisely, without redundancies or duplicating the same information across multiple statements
-- **Example:** "Unsupervised learning is dedicated to discovering intrinsic patterns in unlabeled datasets" becomes "Unsupervised learning discovers patterns in unlabeled data"
-
-### 2. Decontextualization
-- Replace pronouns (e.g., "it," "he," "they") with the full entity name or a clarifying noun phrase
-- Include any necessary modifiers so that each factoid is understandable in isolation
-
-### 3. Temporal Context
-- Convert ALL time references to absolute dates/times using the observation_date
-
-#### Conversion Rules:
-- "today" → exact observation_date
-- "yesterday" → observation_date minus 1 day
-- "this week" → Monday of observation_date's week
-- "last week" → Monday of the week before observation_date
-- "this month" → first day of observation_date's month
-- "last month" → first day of the month before observation_date
-- "this year" → January 1st of observation_date's year
-- "last year" → January 1st of the year before observation_date
-- Keep explicit dates as-is (e.g., "June 18, 2024")
-
-#### Additional Temporal Guidelines:
-- Position time references naturally within factoids
-- Split sentences with multiple time references into separate factoids
-- **NEVER include relative terms like "today," "yesterday," "last week" in the final factoids**
-
-### 4. Accuracy & Completeness
-- Preserve the original meaning without combining multiple facts into a single statement
-- Avoid adding details not present in the source text
-
-### 5. End Actions
-- If the text indicates the end of a role or an action (for example, someone leaving a position), be explicit about the role/action and the time it ended
-
-### 6. Redundancies
-- Eliminate redundancies by simplifying phrases
-- **Example:** Convert "the method is crucial for maintaining X" into "the method maintains X"
-
-## Example
-
-**Input:** "On June 18, 2024, Real Madrid won the Champions League final with a 2-1 victory. Following the triumph, fans of Real Madrid celebrated the Champions League victory across the city."
-
-**Output:**
-- Real Madrid won the Champions League final on June 18, 2024
-- The Champions League final ended with a 2-1 victory for Real Madrid on June 18, 2024
-- Fans of Real Madrid celebrated the Champions League victory across the city on June 18, 2024
+### Source Text:
"""
-AutoTemporalGraph_EDGE_EXTRACTION_PROMPT = """
-Observation Date: {observation_date}
-
-You are a precise knowledge extraction engine designed to distill unstructured text into a structured Knowledge Graph.
-Your goal is to extract all meaningful relationships (edges) between entities, while rigorously capturing their temporal bounds and grounding evidence.
-
-### 🎯 Objective
-Extract a list of relationships where each relationship consists of:
-1. **Start Node**: The subject entity (Name + Label).
-2. **End Node**: The object entity (Name + Label).
-3. **Relation Name**: A canonical, present-tense predicate (e.g., 'works_at', 'is_CEO_of').
-4. **Time Start (`t_start`)**: Strategies for when the relation began.
-5. **Time End (`t_end`)**: Strategies for when the relation ended.
-6. **Evidence (`AutoTemporalGraphic_facts`)**: The exact source sentences/facts supporting this edge.
-
-### 📝 Guidelines
-
-#### 1. Entity & Relation Standards
-- **Entities**: Extract precise names (e.g., "Apple Inc.", "Steve Jobs"). Assign accurate semantic labels (e.g., "Organization", "Person").
-- **Relations**: use **SIMPLE PRESENT TENSE** verbs only (e.g., use 'manages', NOT 'managed', 'was_managing', or 'will_manage').
-- **Forbidden**: Do NOT use vague prepositions like "in", "of", "at" as relation names. Use descriptive verbs instead (e.g., "located_in", "part_of").
-
-#### 2. Temporal Extraction (Critical)
-- **Format**: All dates must be in `YYYY-MM-DD` or `YYYY` format.
-- **Lists**: `t_start` and `t_end` must always be lists of strings. If no date is found, use an empty list `[]`.
-- **Relative Time Resolution**: Calculate absolute dates based on the **Observation Date** ({observation_date}).
- - "last year" -> {observation_date} year - 1 (Jan 1st)
- - "a few months ago" -> Estimate conservatively based on context.
- - "currently" -> implies the relationship is active (no `t_end`).
-- **End Actions**: If the text says someone "left" or "stopped", capture this in `t_end` while keeping the relation positive (e.g., Relation: "works_at", t_end: ["2023-01-01"]).
-- **t_obs**: Always set `t_obs` to null (or empty). Do not try to guess it.
-
-#### 3. Evidence Attribution (AutoTemporalGraphic_facts)
-- For every relationship extracted, identify the specific sentence(s) or AutoTemporalGraphic fact(s) in the source text that provide the evidence.
-- **COPY EXACTLY**: Populate the `AutoTemporalGraphic_facts` list with the exact strings from the input text.
-- **NO Paraphrasing**: Do not summarize or rewrite the source text in this field.
-- If a relationship is derived from multiple facts, include all distinct facts in the list.
-
-### 🧠 Few-Shot Examples
-
-**Input**: "Michel served as CFO at Acme Corp from 2019 to 2021. He was hired by Beta Inc in 2021, but left that role in 2023."
-**Output**:
-- (Michel, is_CFO_of, Acme Corp, t_start=["2019-01-01"], t_end=["2021-01-01"], AutoTemporalGraphic_facts=["Michel served as CFO at Acme Corp from 2019 to 2021."])
-- (Michel, works_at, Beta Inc, t_start=["2021-01-01"], t_end=["2023-01-01"], AutoTemporalGraphic_facts=["He was hired by Beta Inc in 2021, but left that role in 2023."])
-
-**Input**: "Sarah was a board member of GreenFuture until 2019."
-**Output**:
-- (Sarah, is_board_member_of, GreenFuture, t_start=[], t_end=["2019-01-01"], AutoTemporalGraphic_facts=["Sarah was a board member of GreenFuture until 2019."])
-
-**Input**: "(Observation Date: 2024-06-15) John Doe is no longer the CEO of GreenIT since a few months ago."
-**Output**:
-- (John Doe, is_CEO_of, GreenIT, t_start=[], t_end=["2024-03-15"], AutoTemporalGraphic_facts=["John Doe is no longer the CEO of GreenIT since a few months ago."])
- *(Reasoning: "a few months ago" from June is approx. March)*
-
-**Input**: "John Doe's marriage is happening on 26-02-2026."
-**Output**:
-- (John Doe, has_status, Married, t_start=["2026-02-26"], t_end=[], AutoTemporalGraphic_facts=["John Doe's marriage is happening on 26-02-2026."])
-
-### 🚀 Directives
-- Extract strictly what is in the text. Do not hallucinate external facts.
-- Use the **Observation Date** as the anchor for all relative temporal expressions.
-- Maintain consistency: The same entity name should be used across multiple relations.
-- Ensure `AutoTemporalGraphic_facts` contains exact copies of the source text strings.
-"""
+# Note: This prompt contains {observation_date} placeholder for dynamic injection during extraction
+GENERIC_TEMPORAL_EDGE_PROMPT = """
+You are an expert temporal knowledge extraction specialist.
+Extract meaningful relationships (edges) between the provided entities, paying STRICT attention to temporal correctness.
+### Temporal Extraction Rules
+Current Observation Date: {observation_date}
-# ============================================================================
-# Merge Templates for LLM.CUSTOM_RULE Strategy
-# ============================================================================
+1. **Relative Time Resolution**: You MUST resolve relative time expressions based on the Observation Date.
+ - "last year" -> Calculate the year before {observation_date}
+ - "yesterday" -> Calculate the date before {observation_date}
+ - "currently" -> The relationship is active (implies no end date)
+ - "this month" -> First day of the month in {observation_date}
+ - "last month" -> First day of the month before {observation_date}
+2. **Explicit Dates**: Keep explicit dates (e.g., "2023", "2024-01-01") exactly as written.
-EDGE_MERGE_RULE = """You are an intelligent data merging assistant.
-You will receive a list of edges representing the exact same relationship (Same Subject -> Same Predicate -> Same Object).
+3. **Missing Time**: If no time information is present, leave time fields empty. DO NOT hallucinate dates.
-Your task is to merge them into a SINGLE edge object exactly matching the schema.
+### General Constraints
+1. ONLY extract edges connecting entities from the known entity list provided below.
+2. DO NOT create edges involving entities that are not listed.
+3. Use the defined schema fields for time (e.g., t_start, t_end, timestamp) as specified in your output format.
-Merge strategy:
-1. **startNode/endNode**: Keep the node definitions consistent.
-2. **name**: Keep the canonical predicate name (since they are already grouped by this key).
-3. **t_start**: Collect ALL distinct start dates/times from all inputs into a single list. Remove exact duplicates.
-4. **t_end**: Collect ALL distinct end dates/times from all inputs into a single list. Remove exact duplicates.
-5. **t_obs**: Collect ALL distinct observation dates from all inputs into a single list.
-6. **AutoTemporalGraph_facts**: Merge all evidence strings into a single list. Remove exact duplicate strings.
"""
-# ==============================================================================
-# 3. AutoTemporalGraph Implementation - Inherits from AutoGraph
-# ==============================================================================
-
class AutoTemporalGraph(AutoGraph[NodeSchema, EdgeSchema]):
"""
- AutoTemporalGraph: A specialized AutoGraph for extracting high-quality triple-based KGs.
+ Generic Temporal Graph Extractor (AutoTemporalGraph).
- Features:
- - Two-stage extraction: Extracts AutoTemporalGraphic facts first, then derives edges and nodes from facts
- - Customized prompts from original AutoTemporalGraph implementation
- - Semantic Deduplication: Includes `match_nodes_and_update_edges` using SemHash with embeddings
- - Temporal tracking: t_start, t_end, and t_obs fields capture relationship timing and extraction metadata
- - Evidence attribution: AutoTemporalGraphic_facts field traces each extracted edge to source facts
- - Nested node schema: Maintains rich semantic information (name + label)
+ A flexible implementation supporting user-defined Node and Edge schemas with:
+ - **Schema Agnosticism**: Support any user-defined Node and Edge Pydantic models.
+ - **Temporal-Aware Deduplication**: Time information is integrated into edge deduplication logic.
+ - **Dynamic Time Injection**: Observation Date is injected during extraction for relative time resolution.
+
+ Key Design:
+ - `edge_timestamp_extractor`: Function to extract time info from edges (e.g., lambda x: x.year).
+ This ensures (A, rel, B) @ 2020 and (A, rel, B) @ 2021 are treated as different edges.
+ - Wrapper around `edge_key_extractor`: Combines base relation key with timestamp key.
+ - Dynamic Prompt Injection: Observation Date is injected into edge extraction prompts at runtime.
Example:
- >>> from langchain_openai import ChatOpenAI, OpenAIEmbeddings
- >>> llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
- >>> embedder = OpenAIEmbeddings()
- >>> kg = AutoTemporalGraph(llm_client=llm, embedder=embedder)
-
- >>> # 1. Extract: Facts -> Edges -> Nodes
- >>> kg.feed_text("Elon Musk is the CEO of SpaceX. Musk leads Tesla.")
- >>> print(f"Before dedup - Nodes: {len(kg.nodes)}, Edges: {len(kg.edges)}")
-
- >>> # 2. Deduplicate: Merges 'Elon Musk' and 'Musk'
- >>> kg.match_nodes_and_update_edges(threshold=0.85)
- >>> print(f"After dedup - Nodes: {len(kg.nodes)}, Edges: {len(kg.edges)}")
+ >>> from pydantic import BaseModel, Field
+ >>>
+ >>> class MyEntity(BaseModel):
+ ... name: str
+ ... category: str = "Unknown"
+ >>>
+ >>> class MyTemporalEdge(BaseModel):
+ ... src: str
+ ... dst: str
+ ... relation: str
+ ... year: Optional[str] = None
+ >>>
+ >>> kg = AutoTemporalGraph(
+ ... node_schema=MyEntity,
+ ... edge_schema=MyTemporalEdge,
+ ... node_key_extractor=lambda x: x.name,
+ ... edge_key_extractor=lambda x: f"{x.src}|{x.relation}|{x.dst}",
+ ... edge_timestamp_extractor=lambda x: x.year or "",
+ ... nodes_in_edge_extractor=lambda x: (x.src, x.dst),
+ ... llm_client=llm,
+ ... embedder=embedder,
+ ... observation_date="2024-01-15"
+ ... )
"""
def __init__(
self,
+ node_schema: Type[NodeSchema],
+ edge_schema: Type[EdgeSchema],
+ node_key_extractor: Callable[[NodeSchema], str],
+ edge_key_extractor: Callable[[EdgeSchema], str],
+ edge_timestamp_extractor: Callable[[EdgeSchema], str],
+ nodes_in_edge_extractor: Callable[[EdgeSchema], Tuple[str, str]],
llm_client: BaseChatModel,
embedder: Embeddings,
observation_date: str | None = None,
- chunk_size: int = 2000,
- chunk_overlap: int = 200,
- facts_per_chunk: int = 10,
- max_workers: int = 10,
- verbose: bool = False,
+ extraction_mode: str = "two_stage",
+ **kwargs: Any,
):
- """Initialize AutoTemporalGraph.
+ """
+ Initialize AutoTemporalGraph.
Args:
- llm_client: Language model for extraction
- embedder: Embedding model for vector indexing
- observation_date: Date when the extraction was performed, like '1997-10-10' or '1997-10-10 23:59:59'.
- If None, uses current date and time.
- chunk_size: Characters per chunk
- chunk_overlap: Overlapping characters between chunks
- facts_per_chunk: Max number of AutoTemporalGraphic facts to group into a single extraction batch (default: 20)
- max_workers: Max concurrent extraction workers
- verbose: Display detailed execution logs and progress information
+ node_schema: User-defined Node Pydantic model.
+ edge_schema: User-defined Edge Pydantic model with time fields.
+ node_key_extractor: Function to extract unique key from node (e.g., lambda x: x.name).
+ edge_key_extractor: Function to extract base relationship key (e.g., lambda x: f"{x.src}|{x.relation}|{x.dst}").
+ edge_timestamp_extractor: Function to extract time fingerprint from edge (e.g., lambda x: x.year).
+ This is critical for differentiating edges at different times.
+ nodes_in_edge_extractor: Function to extract (source_key, target_key) from edge.
+ llm_client: LangChain BaseChatModel for extraction.
+ embedder: LangChain Embeddings for semantic operations.
+ observation_date: Date context for relative time resolution (default: today in YYYY-MM-DD format).
+ extraction_mode: "one_stage" or "two_stage" (default: "two_stage").
+ **kwargs: Additional arguments passed to parent AutoGraph (e.g., prompt_for_node_extraction,
+ node_strategy_or_merger, edge_strategy_or_merger, etc.).
"""
-
- self.facts_per_chunk = facts_per_chunk
- self.observation_date = observation_date
-
- # 1. Define Key Extractors (critical for deduplication)
- # Node deduplication: exact match by name
- node_key_fn = lambda x: x.name
-
- # Edge deduplication: combination of subject-predicate-object triple
- edge_key_fn = lambda x: f"{x.startNode.name}|{x.name}|{x.endNode.name}"
-
- # 2. Edge consistency check: tell AutoGraph which nodes this edge connects
- nodes_in_edge_fn = lambda x: (x.startNode.name, x.endNode.name)
-
- # Edge merger: merges relationship descriptions using EDGE_MERGE_RULE
- edge_merger = CustomRuleMerger(
- key_extractor=edge_key_fn,
- llm_client=llm_client,
- item_schema=EdgeSchema,
- rule=EDGE_MERGE_RULE,
- )
-
- # 3. Call parent class initialization
- logger.info("🚀 Initializing AutoTemporalGraph")
+ # Set observation date (default: today)
+ self.observation_date = observation_date or datetime.now().strftime("%Y-%m-%d")
+
+ # Store raw extractors for instance recreation (_create_empty_instance)
+ self.raw_timestamp_extractor = edge_timestamp_extractor
+ self.raw_edge_key_extractor = edge_key_extractor
+
+ # -----------------------------------------------------------
+ # Core Logic 1: Decorate Edge Key Extractor
+ # -----------------------------------------------------------
+ # Combine "base relation fingerprint" (e.g., A|rel|B) with "time fingerprint" (e.g., 2023)
+ # This ensures OMem treats (A, rel, B) @ t1 and (A, rel, B) @ t2 as different records
+ def temporal_edge_key_wrapper(edge: EdgeSchema) -> str:
+ base_key = edge_key_extractor(edge)
+ time_key = edge_timestamp_extractor(edge)
+ # Only append timestamp if valid time info exists
+ if time_key:
+ return f"{base_key}|{time_key}"
+ return base_key
+
+ # -----------------------------------------------------------
+ # Core Logic 2: Prepare Edge Extraction Prompt
+ # -----------------------------------------------------------
+ # Extract user-provided edge prompt or use default
+ edge_prompt = kwargs.pop("prompt_for_edge_extraction", GENERIC_TEMPORAL_EDGE_PROMPT)
+
+ # Store the template (with {observation_date} placeholder) for later injection
+ self.temporal_edge_prompt_template = edge_prompt
+
+ # Initialize parent AutoGraph
super().__init__(
- node_schema=NodeSchema,
- edge_schema=EdgeSchema,
- node_key_extractor=node_key_fn,
- edge_key_extractor=edge_key_fn,
- nodes_in_edge_extractor=nodes_in_edge_fn,
+ node_schema=node_schema,
+ edge_schema=edge_schema,
+ node_key_extractor=node_key_extractor,
+ edge_key_extractor=temporal_edge_key_wrapper, # Inject wrapped extractor
+ nodes_in_edge_extractor=nodes_in_edge_extractor,
llm_client=llm_client,
embedder=embedder,
- # Enforce one-stage extraction (first extract edges, then derive nodes)
- extraction_mode="one_stage",
- # Inject customized prompts
- prompt_for_edge_extraction=AutoTemporalGraph_EDGE_EXTRACTION_PROMPT,
- # Configure deduplication strategy
- node_strategy_or_merger=MergeStrategy.KEEP_EXISTING,
- edge_strategy_or_merger=edge_merger,
- # Optimize indexing: only index name field
- node_fields_for_index=["name", "label"],
- edge_fields_for_index=["startNode", "name", "endNode"],
- # Other parameters
- chunk_size=chunk_size,
- chunk_overlap=chunk_overlap,
- max_workers=max_workers,
- verbose=verbose,
+ extraction_mode=extraction_mode,
+ prompt_for_edge_extraction=edge_prompt, # Pass template with placeholder
+ # Use default node prompt if not provided
+ prompt_for_node_extraction=kwargs.pop(
+ "prompt_for_node_extraction", GENERIC_TEMPORAL_NODE_PROMPT
+ ),
+ **kwargs,
)
- # ==================== Extraction Pipeline ====================
+ # ==============================================================================
+ # Override Extraction Methods to Dynamically Inject Observation Date
+ # ==============================================================================
- def _extract_data(self, text: str) -> AutoGraphSchema:
- """Extract AutoTemporalGraphic facts first, then extract edges (Two-stage extraction).
+ def _extract_edges_batch(
+ self, chunks: List[str], node_lists: List[NodeListSchema[NodeSchema]]
+ ) -> List[EdgeListSchema[EdgeSchema]]:
+ """
+ Override: Inject observation_date into edge extraction chain during two-stage extraction.
- Process:
- 1. Split text into chunks.
- 2. Batch extract AutoTemporalGraphic facts from chunks.
- 3. Consolidate facts into a unified context.
- 4. Split consolidated facts into chunks.
- 5. Batch extract edges from fact chunks.
- 6. Post-process: Set t_obs timestamp & Derive nodes.
- 7. Merge all partial graphs into one global graph.
+ This method is called during the second stage (edge extraction from nodes + text).
+ It dynamically injects the observation_date into the LLM chain inputs.
Args:
- text: Input text.
+ chunks: List of text chunks.
+ node_lists: List of extracted node schemas for each chunk.
Returns:
- Extracted and validated graph.
+ List of extracted edge schemas.
"""
- obs_date_str = self.observation_date or datetime.now().strftime("%Y-%m-%d")
-
- # ==================== Step 1: Extract AutoTemporalGraphic Facts ====================
- logger.info("🔍 [Phase 1] Extracting AutoTemporalGraphic Facts...")
-
- # 1. Prepare raw chunks
- if len(text) <= self.chunk_size:
- raw_chunks = [text]
- else:
- raw_chunks = self.text_splitter.split_text(text)
-
- # ==================== Step 2: Extract Edges from Facts ====================
- logger.info("🔍 [Phase 2] Extracting Edges from AutoTemporalGraphic Facts...")
-
- # 5. Group Facts into Chunks (Controlled by count AND size)
- # This ensures each fact remains intact and fact IDs reset per chunk
- fact_chunks = []
- current_batch = []
- current_char_count = 0
-
- # Estimate overhead: "[Fact 99]: " + newline is approx 12 chars
- PREFIX_OVERHEAD = 12
-
- for fact in raw_chunks:
- fact_len = len(fact) + PREFIX_OVERHEAD
-
- # Check constraints:
- # 1. Size limit (fuzzy check to keep under chunk_size)
- # 2. Count limit (facts_per_chunk)
- size_limit_reached = (current_char_count + fact_len) > self.chunk_size
- count_limit_reached = len(current_batch) >= self.facts_per_chunk
-
- if (size_limit_reached or count_limit_reached) and current_batch:
- # Flush current batch
- formatted_chunk = "\n".join(
- [f"[Fact {i + 1}]: {f}" for i, f in enumerate(current_batch)]
- )
- fact_chunks.append(formatted_chunk)
-
- # Reset
- current_batch = []
- current_char_count = 0
-
- current_batch.append(fact)
- current_char_count += fact_len
-
- # Flush remaining items
- if current_batch:
- formatted_chunk = "\n".join(
- [f"[Fact {i + 1}]: {f}" for i, f in enumerate(current_batch)]
+ inputs = []
+ for chunk, node_list in zip(chunks, node_lists):
+ nodes = node_list.items if node_list else []
+ if not nodes:
+ node_context = "No specific entities identified in this chunk."
+ else:
+ node_keys = [self.node_key_extractor(n) for n in nodes]
+ node_context = "Known entities: " + ", ".join(node_keys)
+
+ # Build input dictionary with dynamic observation_date injection
+ inputs.append(
+ {
+ "chunk_text": chunk,
+ "node_context": node_context,
+ "observation_date": self.observation_date, # <--- DYNAMIC INJECTION
+ }
)
- fact_chunks.append(formatted_chunk)
- # 6. Batch Extract Edges directly from Fact Chunks
- # Format the edge prompt with observation date
- edge_prompt_with_date = self.edge_prompt.format(observation_date=obs_date_str)
- edge_prompt_template = ChatPromptTemplate.from_template(
- f"{edge_prompt_with_date}\n\n### Source Text:\n{{chunk_text}}"
+ # Create prompt template with placeholder for observation_date
+ # The self.temporal_edge_prompt_template contains {observation_date} which will be filled
+ prompt_template = ChatPromptTemplate.from_template(
+ f"{self.temporal_edge_prompt_template}{{node_context}}\n\n### Source Text:\n{{chunk_text}}"
)
- edge_chain = edge_prompt_template | self.llm_client.with_structured_output(
+ llm_chain = prompt_template | self.llm_client.with_structured_output(
self.edge_list_schema
)
- edge_inputs = [{"chunk_text": chunk} for chunk in fact_chunks]
- chunk_edge_lists = edge_chain.batch(
- edge_inputs, config={"max_concurrency": self.max_workers}
- )
-
- # ==================== Step 3: Post-process & Merge ====================
-
- # 8. Post-process: Derive Nodes & Set t_obs
- all_nodes_lists, all_edges_lists = [], []
-
- for edge_list in chunk_edge_lists:
- derived_nodes, processed_edges = [], []
+ return llm_chain.batch(inputs, config={"max_concurrency": self.max_workers})
- if edge_list and edge_list.items:
- for edge in edge_list.items:
- edge: EdgeSchema
- # Set t_obs to extraction timestamp
- edge.t_obs = [obs_date_str]
-
- # Derive nodes from edges
- derived_nodes.append(edge.startNode)
- derived_nodes.append(edge.endNode)
- processed_edges.append(edge)
-
- all_nodes_lists.append(derived_nodes)
- all_edges_lists.append(processed_edges)
-
- # 9. Construct Partial Graphs
- partial_graphs = (all_nodes_lists, all_edges_lists)
-
- # 10. Global Merge
- raw_graph = self.merge_batch_data(partial_graphs)
-
- return self._prune_dangling_edges(raw_graph)
-
- # ==================== Node Deduplication ====================
-
- def match_nodes_and_update_edges(
- self, threshold: float = 0.8
- ) -> "AutoTemporalGraph":
- """Match nodes in the graph and update edges accordingly using SemHash with embeddings.
-
- This method identifies and merges similar nodes based on semantic similarity
- using embeddings from the instance's embedder. It updates edges to reflect
- any changes in node identities.
+ def _extract_data_by_one_stage(self, text: str) -> Any:
+ """
+ Override: Inject observation_date into one-stage extraction if the prompt contains the placeholder.
Args:
- threshold: Similarity threshold for matching nodes (0.0 to 1.0). Defaults to 0.8.
+ text: Input text to extract.
Returns:
- The updated AutoTemporalGraph instance with matched nodes and updated edges.
+ Extracted graph.
"""
- logger.info(
- f"🔄 Starting node matching and edge update (threshold={threshold})..."
- )
+ # Check if prompt template contains observation_date placeholder
+ needs_date = "{observation_date}" in self.prompt
- nodes, edges = self.nodes, self.edges
+ template_str = f"{self.prompt}{{chunk_text}}"
+ prompt_template = ChatPromptTemplate.from_template(template_str)
+ llm_chain = prompt_template | self.llm_client.with_structured_output(
+ self.graph_schema
+ )
- if not nodes:
- logger.warning("⚠️ No nodes to match; skipping matching.")
- return self
+ # Prepare base input
+ base_input = {}
+ if needs_date:
+ base_input["observation_date"] = self.observation_date
- # 1. Generate Embeddings using self.embedder.embed_documents
- node_names = [n.name for n in nodes]
- logger.info(f"📊 Generating embeddings for {len(node_names)} nodes...")
+ # Process text (chunked if necessary)
+ if len(text) <= self.chunk_size:
+ inp = {"chunk_text": text, **base_input}
+ graph = llm_chain.invoke(inp)
+ graph_list = [graph]
+ else:
+ chunks = self.text_splitter.split_text(text)
+ inputs = [{"chunk_text": chunk, **base_input} for chunk in chunks]
+ graph_list = llm_chain.batch(
+ inputs, config={"max_concurrency": self.max_workers}
+ )
- try:
- embeddings = self.embedder.embed_documents(node_names)
- except Exception as e:
- logger.error(f"❌ Failed to generate embeddings: {e}")
- return self
+ return self.merge_batch_data(graph_list)
- # 2. Initialize SemHash from embeddings
- semhash = SemHash.from_embeddings(
- embeddings=embeddings,
- records=node_names,
- model=None, # Embeddings already provided
- )
+ def _create_empty_instance(self) -> "AutoTemporalGraph[NodeSchema, EdgeSchema]":
+ """
+ Override: Recreate instance with all temporal-specific attributes.
- # 3. Run Deduplication
- result = semhash.self_deduplicate(threshold=threshold)
-
- # 4. Build Mapping from duplicates result
- mapping = {} # old_name -> new_name
- for record in result.filtered:
- if record.duplicates and len(record.duplicates) > 0:
- # Map the record to the first duplicate (representative)
- mapping[record.record] = record.duplicates[0][0]
-
- if not mapping:
- logger.info("✅ No similar nodes found above threshold.")
- return self
-
- logger.info(f"🔗 Found {len(mapping)} nodes to merge.")
-
- # 5. Apply mapping to graph data
- # Update Node names
- for node in nodes:
- if node.name in mapping:
- node.name = mapping[node.name]
-
- # Update Edge references (startNode and endNode are nested NodeSchema objects)
- for edge in edges:
- if edge.startNode.name in mapping:
- edge.startNode.name = mapping[edge.startNode.name]
- if edge.endNode.name in mapping:
- edge.endNode.name = mapping[edge.endNode.name]
-
- # 6. Update internal state
- new_data = AutoGraphSchema(nodes=nodes, edges=edges)
- self._set_data_state(new_data)
-
- logger.info(
- f"✅ Node matching complete: Nodes {len(nodes)} -> {len(self.nodes)}, Edges: {len(edges)}"
+ This is called by the parent class to create new instances during certain operations.
+ We need to preserve raw extractors and temporal configuration.
+ """
+ return self.__class__(
+ node_schema=self.node_schema,
+ edge_schema=self.edge_schema,
+ node_key_extractor=self.node_key_extractor,
+ edge_key_extractor=self.raw_edge_key_extractor, # Pass original (unwrapped)
+ edge_timestamp_extractor=self.raw_timestamp_extractor, # Pass original
+ nodes_in_edge_extractor=self.nodes_in_edge_extractor,
+ llm_client=self.llm_client,
+ embedder=self.embedder,
+ observation_date=self.observation_date,
+ extraction_mode=self.extraction_mode,
+ node_strategy_or_merger=self.node_merger,
+ edge_strategy_or_merger=self.edge_merger,
+ prompt_for_node_extraction=self.node_prompt,
+ prompt_for_edge_extraction=self.temporal_edge_prompt_template,
+ chunk_size=self.chunk_size,
+ chunk_overlap=self.chunk_overlap,
+ max_workers=self.max_workers,
+ verbose=self.verbose,
+ node_fields_for_index=self.node_fields_for_index,
+ edge_fields_for_index=self.edge_fields_for_index,
)
- return self